Develop Reference

Why my Python code is extracting the same data for all the elements in my list?

My project consists of making a competitive watch table for hotel rates for an agency. It is a painful action that I wanted to automate, the code extract correctly the name of hotels and the prices I want to extract but it's working correctly only for the first hotel and I don't know where is the problem. I provide you with the code and the output, if any of you can help me and thank you in advance.
NB : the code 2 works correctly but when i've added more operations the problem appeared
code 1
#!/usr/bin/env python
# coding: utf-8
import time
from time import sleep
import ast
import pandas as pd
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.common.exceptions import StaleElementReferenceException, NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome("C:\\Users\\marketing2\\Documents\\chromedriver.exe")
driver.get('https://tn.tunisiebooking.com/')
# params to select
params = {
'destination': 'Tozeur',
'date_from': '11/09/2021',
'date_to': '12/09/2021',
'bedroom': '1'
}
# select destination
destination_select = Select(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, 'ville_des'))))
destination_select.select_by_value(params['destination'])
# select bedroom
bedroom_select = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'select_ch'))))
bedroom_select.select_by_value(params['bedroom'])
# select dates
script = f"document.getElementById('checkin').value ='{params['date_from']}';"
script += f"document.getElementById('checkout').value ='{params['date_to']}';"
script += f"document.getElementById('depart').value ='{params['date_from']}';"
script += f"document.getElementById('arrivee').value ='{params['date_to']}';"
driver.execute_script(script)
# submit form
btn_rechercher = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="boutonr"]')))
btn_rechercher.click()
urls = []
hotels = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//div[starts-with(#id,'produit_affair')]")))
for hotel in hotels:
link = hotel.find_element_by_xpath(".//span[#class='tittre_hotel']/a").get_attribute("href")
urls.append(link)
for url in urls:
driver.get(url)
def existsElement(xpath):
try:
driver.find_element_by_id(xpath);
except NoSuchElementException:
return "false"
else:
return "true"
if (existsElement('result_par_arrangement')=="false"):
btn_t = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="moteur_rech"]/form/div/div[3]/div')))
btn_t.click()
sleep(10)
else :
pass
try:
name = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[#class='bloc_titre_hotels']/h2"))).text
arropt = driver.find_element_by_xpath("//div[contains(#class,'line_result')][1]")
opt = arropt.find_element_by_tag_name("b").text
num = len(arropt.find_elements_by_tag_name("option"))
optiondata = {}
achats = {}
marges= {}
selection = Select(driver.find_element_by_id("arrangement"))
for i in range(num):
try:
selection = Select(driver.find_element_by_id("arrangement"))
selection.select_by_index(i)
time.sleep(2)
arr = driver.find_element_by_xpath("//select[#id='arrangement']/option[#selected='selected']").text
prize = driver.find_element_by_id("prix_total").text
optiondata[arr] = (int(prize))
btn_passe = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="resultat"]/div/form/div/div[2]/div[1]/div[2]/div[2]/div')))
btn_passe.click()
# params to select
params = {
'civilite_acheteur': 'Mlle',
'prenom_acheteur': 'test',
'nom_acheteur': 'test',
'e_mail_acheteur': 'test#gmail.com',
'portable_acheteur': '22222222',
'ville_acheteur': 'Test',
}
# select civilite
civilite_acheteur = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.NAME, 'civilite_acheteur'))))
civilite_acheteur.select_by_value(params['civilite_acheteur'])
# saisir prenom
script = f"document.getElementsByName('prenom_acheteur')[0].value ='{params['prenom_acheteur']}';"
script += f"document.getElementsByName('nom_acheteur')[0].value ='{params['nom_acheteur']}';"
script += f"document.getElementsByName('e_mail_acheteur')[0].value ='{params['e_mail_acheteur']}';"
script += f"document.getElementsByName('portable_acheteur')[0].value ='{params['portable_acheteur']}';"
script += f"document.getElementsByName('ville_acheteur')[0].value ='{params['ville_acheteur']}';"
driver.execute_script(script)
# submit form
btn_agence = driver.find_element_by_id('titre_Nabeul')
btn_agence.click()
btn_continuez = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'boutonr')))
btn_continuez.click()
achat = int(driver.find_element_by_xpath('/html/body/header/div[2]/div[1]/div[1]/div[4]/div[2]/div[2]').text.replace(' TND', ''))
achats[arr]=achat
marge =int(((float(prize) - float(achat)) / float(achat)) * 100);
marges[arr]=marge
optiondata[arr]=prize,achat,marge
driver.get(url)
btn_display = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="moteur_rech"]/form/div/div[3]/div')))
btn_display.click()
sleep(10)
except StaleElementReferenceException:
pass
except NoSuchElementException:
pass
s="- {} | {} : {}".format(name, opt, optiondata)
print(s)
ds = []
for l in s.splitlines():
d = l.split('-')
if len(d) > 1:
df = pd.DataFrame(ast.literal_eval(d[1].strip()))
ds.append(df)
for df in ds:
df.reset_index(drop=True, inplace=True)
df = pd.concat(ds, axis= 1)
cols = df.columns
cols = [((col.split('.')[0], col)) for col in df.columns]
df.columns=pd.MultiIndex.from_tuples(cols)
print(df.T)
#print("{} : {} - {}".format(name, opt, optiondata))
code 2
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import StaleElementReferenceException,NoSuchElementException
urls = []
hotels = driver.find_elements_by_xpath("//div[starts-with(#id,'produit_affair')]")
for hotel in hotels:
link = hotel.find_element_by_xpath(".//span[#class='tittre_hotel']/a").get_attribute("href")
urls.append(link)
for url in urls:
driver.get(url)
try:
name = driver.find_element_by_xpath("//div[#class='bloc_titre_hotels']/h2").text
arropt = driver.find_element_by_xpath("//div[contains(#class,'line_result')][1]")
opt = arropt.find_element_by_tag_name("b").text
num = len(arropt.find_elements_by_tag_name("option"))
optiondata = {}
selection = Select(driver.find_element_by_id("arrangement"))
for i in range(num):
try:
selection = Select(driver.find_element_by_id("arrangement"))
selection.select_by_index(i)
time.sleep(2)
arr = driver.find_element_by_xpath("//select[#id='arrangement']/option[#selected='selected']").text
prize = driver.find_element_by_id("prix_total").text
optiondata[arr]=prize
except StaleElementReferenceException:
pass
except NoSuchElementException:
pass
print("{} : {} - {} - {}".format(name,opt,num,optiondata))

Your code is outdated. The HTML has been changed/updated and elements such as the one with identity boutonr doesn't exist on the page anymore.
Your loop and order of execution is wrong so this makes the code evaluating still the same fields.
You should not use or at least minimise the usage of time.sleep() to an absolute minimum as it is a waste of time for your code execution. Use WebDriverWait(...) instead
I don't speak French so I could not understand what you are after in your code, but this minimised example below should help you to understand the principle.
#!/usr/bin/env python
# coding: utf-8
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.common.exceptions import StaleElementReferenceException, NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome("C:\chromedriver.exe")
driver.get('https://tn.tunisiebooking.com/')
# params to select
params = { 'destination': 'Nabeul',
'date_from': '25/08/2021',
'date_to': '26/08/2021',
'bedroom': '1' }
# select destination
destination_select = Select(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, 'ville_des'))))
destination_select.select_by_value(params['destination'])
# select bedroom
bedroom_select = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'select_ch'))))
bedroom_select.select_by_value(params['bedroom'])
# select dates
script = f"document.getElementById('checkin').value ='{params['date_from']}';"
script += f"document.getElementById('checkout').value ='{params['date_to']}';"
script += f"document.getElementById('depart').value ='{params['date_from']}';"
script += f"document.getElementById('arrivee').value ='{params['date_to']}';"
driver.execute_script(script)
# submit form
btn_rechercher = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//div[#onclick="return submit_hotel_recherche()"]')))
btn_rechercher.click()
urls = []
hotels = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//div[starts-with(#id,'produit_affair')]")))
for hotel in hotels:
link = hotel.find_element_by_xpath(".//span[#class='tittre_hotel']/a").get_attribute("href")
urls.append(link)
for url in urls:
driver.get(url)
try:
name = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[#class='bloc_titre_hotels']/h2"))).text
arropt = driver.find_element_by_xpath("//div[contains(#class,'line_result')][1]")
opt = arropt.find_element_by_tag_name("b").text
num = len(arropt.find_elements_by_tag_name("option"))
optiondata = {}
achats = {}
marges= {}
for i in range(num):
try:
selection = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'arrangement')))).select_by_index(i)
time.sleep(0.5)
arr = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//select[#id='arrangement']/option[#selected='selected']"))).text
prize = driver.find_element_by_id("prix_total").text
optiondata[arr] = int(prize)
except StaleElementReferenceException:
pass
print("{} : {} - {}".format(name, opt, optiondata))
except NoSuchElementException:
pass
driver.quit()
Result:
Byzance Nabeul : Chambre Double - {'All Inclusive soft': 93, 'Demi Pension': 38, 'Petit Dejeuner': 28, 'Pension Complete': 78}
Palmyra Club Nabeul Nabeul : Double Standard - {'All Inclusive soft': 92}
The following code goes to the payment page and extracts all the info there:
#!/usr/bin/env python
# coding: utf-8
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.common.exceptions import StaleElementReferenceException, NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome("/usr/local/bin/chromedriver")
driver.get('https://tn.tunisiebooking.com/')
# params to select
params = {
'destination': 'Nabeul',
'date_from': '29/08/2021',
'date_to': '30/08/2021',
'bedroom': '1'
}
# select destination
destination_select = Select(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, 'ville_des'))))
destination_select.select_by_value(params['destination'])
# select bedroom
bedroom_select = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'select_ch'))))
bedroom_select.select_by_value(params['bedroom'])
# select dates
script = f"document.getElementById('checkin').value ='{params['date_from']}';"
script += f"document.getElementById('checkout').value ='{params['date_to']}';"
script += f"document.getElementById('depart').value ='{params['date_from']}';"
script += f"document.getElementById('arrivee').value ='{params['date_to']}';"
driver.execute_script(script)
# submit form
btn_rechercher = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//div[#onclick="return submit_hotel_recherche()"]')))
btn_rechercher.click()
urls = []
hotels = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//div[starts-with(#id,'produit_affair')]")))
for hotel in hotels:
link = hotel.find_element_by_xpath(".//span[#class='tittre_hotel']/a").get_attribute("href")
urls.append(link)
for url in urls:
driver.get(url)
try:
name = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[#class='bloc_titre_hotels']/h2"))).text
arropt = driver.find_element_by_xpath("//div[contains(#class,'line_result')][1]")
opt = arropt.find_element_by_tag_name("b").text
num = len(arropt.find_elements_by_tag_name("option"))
optiondata = {}
achats = {}
marges= {}
try:
selection = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'arrangement'))))
time.sleep(0.5)
arr = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//select[#id='arrangement']/option[#selected='selected']"))).text
prize = driver.find_element_by_id("prix_total").text
optiondata[arr] = (int(prize))
btn_passe = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'resa')))
btn_passe.click()
tot = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'montant_total_apres_code')))
total = int(tot.text.replace(' €', ''))
# params to select
params = {
'civilite_acheteur': 'Mlle',
'prenom_acheteur': 'test',
'nom_acheteur': 'test',
'e_mail_acheteur': 'test#gmail.com',
'portable_acheteur': '22222222',
'ville_acheteur': 'Test',
}
# select civilite
civilite_acheteur = Select(WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.NAME, 'civilite_acheteur'))))
civilite_acheteur.select_by_value(params['civilite_acheteur'])
# saisir prenom
script = f"document.getElementsByName('prenom_acheteur')[0].value ='{params['prenom_acheteur']}';"
script += f"document.getElementsByName('nom_acheteur')[0].value ='{params['nom_acheteur']}';"
script += f"document.getElementsByName('e_mail_acheteur')[0].value ='{params['e_mail_acheteur']}';"
script += f"document.getElementsByName('portable_acheteur')[0].value ='{params['portable_acheteur']}';"
script += f"document.getElementsByName('ville_acheteur')[0].value ='{params['ville_acheteur']}';"
driver.execute_script(script)
# submit form
btn_agence = driver.find_element_by_class_name('continuez_resa')
btn_agence.click()
achat1 = int(driver.find_element_by_id('montant_a_payer').text.replace(' €', ''))
achat = int(driver.find_element_by_id('montant_restant').text.replace(' €', ''))
achat3 = float(driver.find_element_by_xpath('//div[#class="ligne_interne_total"]/div[3]/div[#class="prix_total1 text_shadow"]').text.replace(' TND', ''))
achats[arr]=achat
marge =int(((float(prize) - float(achat)) / float(achat)) * 100);
marges[arr]=marge
optiondata[arr]=prize,total,achat1,achat,achat3,marge
except StaleElementReferenceException:
pass
print("{} : {} - {}".format(name, opt, optiondata))
except NoSuchElementException:
pass
driver.quit()
Output:
Byzance Nabeul : Chambre Double - {'Petit Dejeuner': (36, 41, 12, 29, 4.0, 24)}
Where:
36 = Prix Total
41 = Montant Total
12 = Montant de l'acompte
29 = Vous payerez le reste à votre arrivée à l'hôtel
4.0 = Total taxe de séjour à payer sur place à l'hôtel est
24 = Marges
Hotel page:

You are using sleeps to load the pages in your first example but not in your second one (the one that you state works just fine).
This is typically not the way you want to actually use selenium and leads me to believe that your timing is off.
This SO answer shows you how to use "Explicit Waits" on "expected_conditions" to not have "specific timings" which can/will fail.
You even create a wait object but never use it.
Use it in conjunction with expected_conditions and remove the specific timed sleeps and things will get better.
expected_conditions docs are here

The problem was that it can't access to the element listing arrangements for the rest of the hotels in the list i've added a function that tests the presence of the data and it workod
for url in urls:
driver.get(url)
def existsElement(xpath):
try:
driver.find_element_by_id(xpath);
except NoSuchElementException:
return "false"
else:
return "true"
if (existsElement('result_par_arrangement')=="false"):
btn_t = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="moteur_rech"]/form/div/div[3]/div')))
btn_t.click()
else :
pass

WebDriverWait is terminated before reach to the last page

I scraped review score data from booking.com. My code is shown as follow with step-by-step explanation
import pandas as pd
import time
from random import randint
from selenium import webdriver
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
def get_random_wait(lower=5, upper=15):
max_wait = randint(lower, upper)
waiting_time = randint(lower - 1, max_wait)
print("waiting for {} seconds".format(waiting_time))
return waiting_time
name = "Eden The Residence at The Sea"
# Invoke Chrome and go to booking.com
driver = webdriver.Chrome(executable_path="../drivers/chromedriver.exe")
driver.get("https://booking.com")
print("Accessing " + driver.title)
print(driver.current_url)
# Enter hotel name
print("Searching for {}.".format(name))
time.sleep(get_random_wait())
driver.find_element_by_css_selector("input[class*='sb-searchbox__input']").send_keys(name)
# Click search
print("Clicking search button.")
time.sleep(get_random_wait())
driver.find_element_by_css_selector("button[type='submit']").click()
# Click the hotel name, i.e. Eden the Residence at The Sea
print("Selecting {} from return list.".format(name))
time.sleep(get_random_wait())
driver.find_element_by_xpath("//span[contains(text(),'" + name + "')]").click()
# Click to show toggle side bar that store reviews
print("Clicking on review tab.")
time.sleep(get_random_wait())
driver.switch_to.window(driver.window_handles[1])
# Click to show all reviews
print("Showing all reviews.")
time.sleep(get_random_wait())
driver.find_element_by_css_selector('#show_reviews_tab').click()
# Choose sorting by date from drop-down menu
print("Sorting review by date in descending order.")
time.sleep(get_random_wait())
dropdown = Select(driver.find_element_by_css_selector("select[id='review_sort'] "))
dropdown.select_by_visible_text("Date (newer to older)")
date_reviews = []
score_reviews = []
scrape_data = pd.DataFrame()
page = 1
while True:
try:
print("Scraping reviews on page {}".format(page))
# time.sleep(get_random_wait())
container = WebDriverWait(driver, 10).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "div[class='c-review-block']")))
# Extracting dates
dates = [c.find_element_by_css_selector("div[class='c-review-block__row'] > span[class='c-review-block__date']").text.split("Reviewed: ")[-1] for c in container]
date_reviews = date_reviews + dates
# Extracting scores
scores = [float(c.find_element_by_css_selector("div[class='bui-review-score__badge']").text) for c in container]
score_reviews = score_reviews + scores
buffer = pd.DataFrame({"date": date_reviews,
"score": score_reviews})
scrape_data = pd.concat([scrape_data, buffer], axis=0, ignore_index=True, sort=False)
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[class='pagenext']"))).click()
page += 1
except Exception:
scrape_data["date"] = pd.to_datetime(scrape_data["date"], format="%B %d, %Y")
break
With this hotel, the code should click nextpage until the 10th page but it stop at the 8th page. I could not figure out what went wrong with the code. May I have your suggestions?

Some reviews are hidden by moderators and looks like "This review is hidden because it doesn't meet our guidelines.". You cannot extract date or score from these reviews and this is why your code breakes
Try to replace below 2 lines
dates = [c.find_element_by_css_selector("div[class='c-review-block__row'] > span[class='c-review-block__date']").text.split("Reviewed: ")[-1] for c in container]
scores = [float(c.find_element_by_css_selector("div[class='bui-review-score__badge']").text) for c in container]
with
dates_selector = "div[class='c-review-block__row'] > span[class='c-review-block__date']"
dates = [c.find_element_by_css_selector(dates_selector).text.split("Reviewed: ")[-1] if c.find_elements_by_css_selector(dates_selector) else None for c in container]
score_selector = "div[class='bui-review-score__badge']"
scores = [float(c.find_element_by_css_selector(score_selector).text) if c.find_elements_by_css_selector(score_selector) else None for c in container]
In case of hidden review you'll get None as date and score (you can of cource set another value)

trying to close popover - python - selenium - Glassdoor

Trying to close a popover while scraping Glassdoor for jobs [It keeps popping up from time to time - need to close it every time].. I've tried quite a few things
Tried closing it by looking for the close button. Please help !
driver.find_element_by_class_name("SVG_Inline modal_closeIcon").click()
Tried looking for a ElementClickInterceptedException when the bot couldn't click on the next company, and everywhere else there was a click
element = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CLASS_NAME, "SVG_Inline-svg modal_closeIcon-svg")))
element.click()
This is the website:
https://www.glassdoor.co.uk/Job/web-developer-jobs-SRCH_KO0,13.htm
This is the complete code:
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException, StaleElementReferenceException
from selenium import webdriver
import time
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def get_jobs(keyword, num_jobs, verbose, place):
'''Gathers jobs as a dataframe, scraped from Glassdoor'''
#Initializing the webdriver
options = webdriver.ChromeOptions()
#Uncomment the line below if you'd like to scrape without a new Chrome window every time.
#options.add_argument('headless')
#Change the path to where chromedriver is in your home folder.
#driver = webdriver.Chrome(executable_path="/Users/omersakarya/Documents/GitHub/scraping-glassdoor-selenium/chromedriver", options=options)
driver = webdriver.Chrome()
driver.set_window_size(1120, 1000)
url = "https://www.glassdoor.co.uk/Job/web-developer-jobs-SRCH_KO0,13.htm"
driver.get(url)
jobs = []
time.sleep(3)
driver.find_element_by_id("onetrust-accept-btn-handler").click()
time.sleep(3)
while len(jobs) < num_jobs: #If true, should be still looking for new jobs.
job_buttons = driver.find_elements_by_class_name("jl") #jl for Job Listing. These are the buttons we're going to click.
try:
for job_button in job_buttons:
if len(jobs) >= num_jobs:
break
print("Progress: {}".format("" + str(len(jobs)) + "/" + str(num_jobs)))
job_button.click()
collected_successfully = False
while not collected_successfully:
try:
company_name = driver.find_element_by_xpath('.//div[#class="employerName"]').text
location = driver.find_element_by_xpath('.//div[#class="location"]').text
job_title = driver.find_element_by_xpath('.//div[contains(#class, "title")]').text
job_description = driver.find_element_by_xpath('.//div[#class="jobDescriptionContent desc"]').text
collected_successfully = True
except:
time.sleep(5)
try:
#salary_estimate = driver.find_element_by_xpath('.//span[#class="gray salary"]').text
salary_estimate = driver.find_element_by_xpath('//*[#id="HeroHeaderModule"]/div[3]/div[1]/div[4]/span').text
except NoSuchElementException:
salary_estimate = -1 #You need to set a "not found value. It's important."
try:
rating = driver.find_element_by_xpath('.//span[#class="rating"]').text
except NoSuchElementException:
rating = -1 #You need to set a "not found value. It's important."
#Printing for debugging
if verbose:
print("Job Title: {}".format(job_title))
print("Salary Estimate: {}".format(salary_estimate))
print("Job Description: {}".format(job_description[:500]))
print("Rating: {}".format(rating))
print("Company Name: {}".format(company_name))
print("Location: {}".format(location))
#Going to the Company tab...
#clicking on this:
#<div class="tab" data-tab-type="overview"><span>Company</span></div>
try:
driver.find_element_by_xpath('.//div[#class="tab" and #data-tab-type="overview"]').click()
try:
#<div class="infoEntity">
# <label>Headquarters</label>
# <span class="value">San Francisco, CA</span>
#</div>
headquarters = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Headquarters"]//following-sibling::*').text
except NoSuchElementException:
headquarters = -1
try:
size = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Size"]//following-sibling::*').text
except NoSuchElementException:
size = -1
try:
founded = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Founded"]//following-sibling::*').text
except (NoSuchElementException, StaleElementReferenceException):
founded = -1
try:
type_of_ownership = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Type"]//following-sibling::*').text
except NoSuchElementException:
type_of_ownership = -1
try:
industry = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Industry"]//following-sibling::*').text
except NoSuchElementException:
industry = -1
try:
sector = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Sector"]//following-sibling::*').text
except NoSuchElementException:
sector = -1
try:
revenue = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Revenue"]//following-sibling::*').text
except NoSuchElementException:
revenue = -1
try:
competitors = driver.find_element_by_xpath('.//div[#class="infoEntity"]//label[text()="Competitors"]//following-sibling::*').text
except NoSuchElementException:
competitors = -1
except (NoSuchElementException,ElementClickInterceptedException,StaleElementReferenceException): #Rarely, some job postings do not have the "Company" tab.
if NoSuchElementException:
time.sleep(1)
headquarters = -1
size = -1
founded = -1
type_of_ownership = -1
industry = -1
sector = -1
revenue = -1
competitors = -1
else:
driver.find_element_by_class_name("selected").click()
driver.find_element_by_class_name("SVG_Inline modal_closeIcon").click()
element = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CLASS_NAME, "SVG_Inline-svg modal_closeIcon-svg")))
element.click()
pass
if verbose:
print("Headquarters: {}".format(headquarters))
print("Size: {}".format(size))
print("Founded: {}".format(founded))
print("Type of Ownership: {}".format(type_of_ownership))
print("Industry: {}".format(industry))
print("Sector: {}".format(sector))
print("Revenue: {}".format(revenue))
print("Competitors: {}".format(competitors))
print("####################################################")
jobs.append({"Job Title" : job_title,
"Salary Estimate" : salary_estimate,
"Job Description" : job_description,
"Rating" : rating,
"Company Name" : company_name,
"Location" : location,
"Headquarters" : headquarters,
"Size" : size,
"Founded" : founded,
"Type of ownership" : type_of_ownership,
"Industry" : industry,
"Sector" : sector,
"Revenue" : revenue,
"Competitors" : competitors})
#You might
#time.sleep(0.5)
except (ElementClickInterceptedException, StaleElementReferenceException):
alertObj = driver.switch_to.alert
alertObj.accept()
alertObj.dismiss()
driver.find_element_by_class_name("selected").click()
driver.find_element_by_class_name("SVG_Inline modal_closeIcon").click()
element = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CLASS_NAME, "SVG_Inline-svg modal_closeIcon-svg")))
element.click()
pass
#add job to jobs
#Clicking on the "next page" button
# try:
# driver.find_element_by_xpath('.//li[#class="page"]//a').click()
# except NoSuchElementException:
# print("Scraping terminated before reaching target number of jobs. Needed {}, got {}.".format(num_jobs, len(jobs)))
# break
# time.sleep(5)
try:
driver.find_element_by_xpath('.//li[#class="next"]//a').click()
except (ElementClickInterceptedException):
#print("Scraping terminated before reaching target number of jobs. Needed {}, got {}.".format(num_jobs, len(jobs)))
driver.find_element_by_class_name("selected").click()
driver.find_element_by_class_name("SVG_Inline modal_closeIcon").click()
element = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CLASS_NAME, "SVG_Inline-svg modal_closeIcon-svg")))
element.click()
element.text
pass
#break
return pd.DataFrame(jobs) #This line converts the dictionary object into a pandas DataFrame.
df = gs.get_jobs(keyword, num_jobs, False, place)
Trying to get rid of this:
enter image description here
[Screenshot of the element I need to close and continue with the loop][2]

Python Selenium + Datepicker Click

I have been banging my head around trying to get the price of a room like this for example by clicking the first available (green) datepicker checkin input and then clicking the first available datepicker checkout input so the price for the minium period is generated.
My code is a mess so i would really appreciate if someone could post a cleaner code to achieve that.
I am using Python selenium + scrapy although something in Java for example would still help.
UPDATE:
here is the code:
def availability(self, doc):
url = doc['url'] + '#calendar'
self.driver.get(url)
is_active = True
# We want to the availability/price for each day in a month.
availabilities = []
# wait for the check in input to load
wait = WebDriverWait(self.driver, 10)
try:
elem = wait.until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, ".dates-group input[name=startDateInput]")
)
)
except TimeoutException:
pass
else:
elem.click() # open calendar
# wait for datepicker to load
wait.until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, '.ui-datepicker:not(.loading)'))
)
days = self.driver.find_elements_by_css_selector(
"#ui-datepicker-div tr td"
)
for cell in days:
day = cell.text.strip()
if not day:
continue
if "full-changeover" not in cell.get_attribute("class"):
available = False
else:
available = True
self.logger.warning('CELL "%s"', cell)
self.logger.warning('DAY "%s"', day)
self.logger.warning('available "%s"', available)
# The first iteration was to list the availability, now we want to
# click the first available element to get the price
for cell in days:
day = cell.text.strip()
if not day:
continue
if "full-changeover" in cell.get_attribute("class"):
self.logger.warning('CLICK IT "%s"', day)
self.driver.implicitly_wait(10)
x = self.driver.find_element_by_xpath("//table/tbody/tr/td/a[text()=" + day + "]")
self.driver.implicitly_wait(10)
x.click() # Element not found in the cache issue here
# import ipdb; ipdb.set_trace()
# self.logger.warning('CELL "%s"', cell)
# self.logger.warning('DAY "%s"', day)
# self.logger.warning('available "%s"', available)
# elem.click() # close checkin calendar
# Now lets click on the checkout input to get the price and minimum
# number of days. We probably don't have to wait for the checkout
# because its already loaded but you never know.
try:
elem = wait.until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR,
".dates-group input[name=endDateInput]")
)
)
except TimeoutException:
pass
else:
# elem.click() # open calendar in checkout input
# wait for datepicker to load
wait.until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, '.ui-datepicker:not(.loading)'))
)
days = self.driver.find_elements_by_css_selector(
"#ui-datepicker-div tr td"
)
for cell in days:
day = cell.text.strip()
if not day:
continue
# This is the first available date to checkout
if "full-changeover" in cell.get_attribute("class"):
self.logger.warning('CLICK IT "%s"', available)
import ipdb; ipdb.set_trace()
# Here we would get the generated price
self.logger.warning('CELL "%s"', cell)
self.logger.warning('DAY "%s"', day)
self.logger.warning('available "%s"', available)
import ipdb; ipdb.set_trace()
return {'availabilities': availabilities, 'is_active': is_active}
Thanks

One tricky thing about this calendar is that you first need to hover a particular day and then relocate the active day and click it. Here is a working implementation that selects the first available start and end dates and prints the calculated price:
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.maximize_window()
wait = WebDriverWait(driver, 10)
url = 'https://www.homeaway.pt/arrendamento-ferias/p1418427a?uni_id=1590648'
driver.get(url)
# pick start date
start_date = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".quotebar-container input[name=startDateInput]")))
start_date.click()
first_available_date = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#ui-datepicker-div td.full-changeover > a")))
ActionChains(driver).move_to_element(first_available_date).perform()
driver.find_element_by_css_selector("#ui-datepicker-div td.full-selected.full-changeover > a").click()
# pick end date (TODO: violates DRY principle, refactor!)
end_date = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".quotebar-container input[name=endDateInput]")))
end_date.click()
first_available_date = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#ui-datepicker-div td.full-changeover > a")))
ActionChains(driver).move_to_element(first_available_date).perform()
driver.find_element_by_css_selector("#ui-datepicker-div td.full-selected.full-changeover > a").click()
# get the calculated price
price = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".price-quote .price-total")))
print(price.text)
driver.close()
At the moment, it selects 20/04/2016 and 23/04/2016 and prints 180€.
Hope that helps.

how to wait for one drop down list to be processed before moving to the next one?

Having an issue with drop down menus in my selenium (python). In this page, what happens is that when selecting a drop down option, it will process the selection, making the other options not selectable until that process is finished. I'm trying to say in my script to wait until a drop down list has processed it's option before moving onto the next one, but it's just hanging. How can I get it to wait for the option to be processed in one drop down menu before moving to the next drop down?
Below is what I've attempted:
#Meal details
try:
pax_one_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDown"))
pax_one_ob_meal.select_by_index(0)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDownIn")))
pax_one_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDownIn"))
pax_one_ib_meal.select_by_index(2)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDown")))
pax_two_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDown"))
pax_two_ob_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDownIn")))
pax_two_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDownIn"))
pax_two_ib_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDown")))
pax_three_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownt"))
pax_three_ob_meal.select_by_index(2)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownIn")))
pax_three_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownIn"))
pax_three_ib_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDown")))
pax_four_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDown"))
pax_four_ob_meal.select_by_index(0)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDownIn")))
pax_four_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDownIn"))
pax_four_ib_meal.select_by_index(1)
You can visit the www.jet2.com page and load up the script below to see what I'm seeing (it's on the meals page as it's the meals drop down options I'm having issues with). After reserving your seats, you will have to manually click on the continue button when 'you have reserved your seats' message comes up as I'm still trying to figure out how to do this.
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from datetime import datetime, timedelta
def select_date(calendar, mininum_date=None):
try:
# check if "Your Date" is there
your_date_elm = calendar.find_element_by_class_name("your-date")
your_date = your_date_elm.get_attribute("data-date")
print("Found 'Your Date': " + your_date)
your_date_elm.click()
# check if your_date against the minimum date if given
your_date = datetime.strptime(your_date, "%Y-%m-%d")
if mininum_date and your_date < mininum_date:
raise NoSuchElementException("Minimum date violation")
return your_date
except NoSuchElementException:
flight_date = None
flight_date_elm = None
while True:
print("Processing " + calendar.find_element_by_css_selector("div.subheader > p").text)
try:
if mininum_date:
flight_date_elms = calendar.find_elements_by_class_name("flights")
flight_date_elm = next(flight_date_elm for flight_date_elm in flight_date_elms
if datetime.strptime(flight_date_elm.get_attribute("data-date"), "%Y-%m-%d") >= mininum_date)
else:
flight_date_elm = calendar.find_element_by_class_name("flights")
except (StopIteration, NoSuchElementException):
calendar.find_element_by_partial_link_text("Next month").click()
# if found - print out the date, click and exit the loop
if flight_date_elm:
flight_date = flight_date_elm.get_attribute("data-date")
print("Found 'Flight Date': " + flight_date)
flight_date_elm.click()
break
return datetime.strptime(flight_date, "%Y-%m-%d")
FROM = "Leeds Bradford"
TO = "Antalya"
PAX1_FORENAME = "Dad"
PAX2_FORENAME = "Mum"
PAX3_FORENAME = "Son"
PAX4_FORENAME = "Daughter"
PAX5_FORENAME = "Baby"
PAX_SURNAME = "Test"
driver = webdriver.Firefox()
#driver = webdriver.Chrome()
driver.get("http://www.jet2.com")
driver.maximize_window()
wait = WebDriverWait(driver, 20)
actions = ActionChains(driver)
# wait for the page to load
wait.until(EC.presence_of_element_located((By.ID, "departure-airport-input")))
# fill out the form
return_flight = driver.find_element_by_id('return-flight-selector').click()
depart_from = driver.find_element_by_id("departure-airport-input").send_keys(FROM)
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#ui-id-1 .ui-menu-item"))).click()
go_to = driver.find_element_by_id("destination-airport-input").send_keys(TO)
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#ui-id-2 .ui-menu-item"))).click()
# select depart date
datepicker = driver.find_element_by_id("departure-date-selector")
actions.move_to_element(datepicker).click().perform()
# find the calendar, month and year picker and the current date
calendar = driver.find_element_by_id("departureDateContainer")
month_picker = Select(calendar.find_element_by_class_name("ui-datepicker-month"))
year_picker = Select(calendar.find_element_by_class_name("ui-datepicker-year"))
current_date = calendar.find_element_by_class_name("ui-datepicker-current-day")
# printing out current date
month = month_picker.first_selected_option.text
year = year_picker.first_selected_option.text
print("Current departure date: {day} {month} {year}".format(day=current_date.text, month=month, year=year))
# see if we have an available date in this month
try:
next_available_date = current_date.find_element_by_xpath("following::td[#data-handler='selectDay' and ancestor::div/#id='departureDateContainer']")
print("Found an available departure date: {day} {month} {year}".format(day=next_available_date.text, month=month, year=year))
next_available_date.click()
except NoSuchElementException:
# looping over until the next available date found
while True:
# click next, if not found, select the next year
try:
calendar.find_element_by_class_name("ui-datepicker-next").click()
except NoSuchElementException:
# select next year
year = Select(calendar.find_element_by_class_name("ui-datepicker-year"))
year.select_by_visible_text(str(int(year.first_selected_option.text) + 1))
# reporting current processed month and year
month = Select(calendar.find_element_by_class_name("ui-datepicker-month")).first_selected_option.text
year = Select(calendar.find_element_by_class_name("ui-datepicker-year")).first_selected_option.text
print("Processing {month} {year}".format(month=month, year=year))
try:
next_available_date = calendar.find_element_by_xpath(".//td[#data-handler='selectDay']")
print("Found an available departure date: {day} {month} {year}".format(day=next_available_date.text, month=month, year=year))
next_available_date.click()
break
except NoSuchElementException:
continue
# select return date
datepicker = driver.find_element_by_id("return-date-selector")
actions.move_to_element(datepicker).click().perform()
# find the calendar, month and year picker and the current date
calendar = driver.find_element_by_id("returnDateContainer")
month_picker = Select(calendar.find_element_by_class_name("ui-datepicker-month"))
year_picker = Select(calendar.find_element_by_class_name("ui-datepicker-year"))
current_date = calendar.find_element_by_class_name("ui-datepicker-current-day")
# printing out current date
month = month_picker.first_selected_option.text
year = year_picker.first_selected_option.text
print("Current return date: {day} {month} {year}".format(day=current_date.text, month=month, year=year))
# see if we have an available date in this month
try:
next_available_date = current_date.find_element_by_xpath("following::td[#data-handler='selectDay' and ancestor::div/#id='returnDateContainer']")
print("Found an available return date: {day} {month} {year}".format(day=next_available_date.text, month=month, year=year))
next_available_date.click()
except NoSuchElementException:
# looping over until the next available date found
while True:
# click next, if not found, select the next year
try:
calendar.find_element_by_class_name("ui-datepicker-next").click()
except NoSuchElementException:
# select next year
year = Select(calendar.find_element_by_class_name("ui-datepicker-year"))
year.select_by_visible_text(str(int(year.first_selected_option.text) + 1))
# reporting current processed month and year
month = Select(calendar.find_element_by_class_name("ui-datepicker-month")).first_selected_option.text
year = Select(calendar.find_element_by_class_name("ui-datepicker-year")).first_selected_option.text
print("Processing {month} {year}".format(month=month, year=year))
try:
next_available_date = calendar.find_element_by_xpath(".//td[#data-handler='selectDay']")
print("Found an available return date: {day} {month} {year}".format(day=next_available_date.text, month=month, year=year))
next_available_date.click()
break
except NoSuchElementException:
continue
#select adults
adults = Select(driver.find_element_by_id("adults-number"))
adults.select_by_visible_text("2")
#select children
children = Select(driver.find_element_by_id("children-number"))
children.select_by_visible_text("2")
#children ages
child_one = Select(driver.find_element_by_id("childSelect1"))
child_one.select_by_visible_text("4")
child_two = Select(driver.find_element_by_id("childSelect2"))
child_two.select_by_visible_text("6")
confirm = driver.find_element_by_link_text("Confirm")
confirm.click()
#select infants
infants = Select(driver.find_element_by_id("infants-number"))
infants.select_by_visible_text("1")
#search flights
search_flight = driver.find_element_by_id("search-flights").click()
# get the outbound date
outbound = wait.until(EC.visibility_of_element_located((By.ID, "outboundsearchresults")))
outbound_date = select_date(outbound)
# get the inbound date
inbound = driver.find_element_by_id("inboundsearchresults")
inbound_minimum_date = outbound_date + timedelta(days=7)
inbound_date = select_date(inbound, mininum_date=inbound_minimum_date)
print(outbound_date, inbound_date)
# continue after flights selected
proceed_to_pax = driver.find_element_by_id('navigateActionNext').click()
pax_page = wait.until(EC.visibility_of_element_located((By.ID, "ctl00_MainContent_passengerList_PassengerGridView_ctl08_butAddBagsForAll")))
#select currency
currency = Select(driver.find_element_by_id("ctl00_MainContent_dynamicCurrencyDropDown"))
currency.select_by_value("1")
#Passenger details
pax_one_title = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl02_TitleDropDownList"))
pax_one_title.select_by_visible_text("Mr")
pax_one_forename = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl02_ForeNameTextBox").send_keys(PAX1_FORENAME)
pax_one_surname = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl02_SurNameTextBox").send_keys(PAX_SURNAME)
pax_two_title = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl03_TitleDropDownList"))
pax_two_title.select_by_visible_text("Mrs")
pax_two_forename = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl03_ForeNameTextBox").send_keys(PAX2_FORENAME)
pax_two_surname = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl03_SurNameTextBox").send_keys(PAX_SURNAME)
pax_three_title = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl04_TitleDropDownList"))
pax_three_title.select_by_visible_text("Mr")
pax_three_forename = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl04_ForeNameTextBox").send_keys(PAX3_FORENAME)
pax_three_surname = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl04_SurNameTextBox").send_keys(PAX_SURNAME)
pax_four_title = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl05_TitleDropDownList"))
pax_four_title.select_by_visible_text("Ms")
pax_four_forename = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl05_ForeNameTextBox").send_keys(PAX4_FORENAME)
pax_four_surname = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl05_SurNameTextBox").send_keys(PAX_SURNAME)
pax_five_title = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl06_TitleDropDownList"))
pax_five_title.select_by_visible_text("Mstr")
pax_five_forename = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl06_ForeNameTextBox").send_keys(PAX5_FORENAME)
pax_five_surname = driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl06_SurNameTextBox").send_keys(PAX_SURNAME)
#pax baggage
pax_one_bags = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl02_baggageOutDropDown"))
pax_one_bags.select_by_value("0")
pax_two_bags = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl03_baggageOutDropDown"))
pax_two_bags.select_by_value("0")
pax_three_bags = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl04_baggageOutDropDown"))
pax_three_bags.select_by_value("0")
pax_four_bags = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl05_baggageOutDropDown"))
pax_four_bags.select_by_value("0")
#infant carer
infant_one_carer = Select(driver.find_element_by_id("ctl00_MainContent_passengerList_PassengerGridView_ctl06_InfantCarerDropDown"))
infant_one_carer.select_by_visible_text("Mr Dad Test")
#checkin
#online = driver.find_element_by_id("ctl00_MainContent_passengerList_checkinOnline").click()
airport = driver.find_element_by_id("ctl00_MainContent_passengerList_checkinAirport").click()
#continue to seats
continue_pax = driver.find_element_by_id("pageContinue").click()
modal_outer = wait.until(EC.invisibility_of_element_located((By.ID, "ctl00_MainContent_PageTransitionDialog_messageWrapper")))
continue_again = wait.until(EC.element_to_be_clickable((By.ID, "pageContinue"))).click();
seats_page = wait.until(EC.visibility_of_element_located((By.ID, "findyourseat")))
#seats selection - outbound
for outbound_passenger in driver.find_elements_by_css_selector("ol[data-flightbound='Outbound'] li[data-personid]"):
outbound_passenger.click()
#driver.find_elements_by_css_selector("ol.passengerlist li[data-personid]"):
outbound_has_infant = outbound_passenger.get_attribute("data-hasinfant")
# choose seats
if outbound_has_infant:
# select a non-selected infant seat
outbound_seat = driver.find_element_by_css_selector(".outbound .planebody a.seat.infant:not(.reserved):not(.selected)")
else:
# select a non-reserved non-selected seat
outbound_seat = driver.find_element_by_css_selector(".outbound .planebody a.seat:not(.reserved):not(.selected)")
print("Passenger: %s, choosing seat: %s" % (outbound_passenger.text.strip(), outbound_seat.get_attribute("data-seat")))
outbound_seat.click()
outbound_plan = wait.until(EC.invisibility_of_element_located((By.CLASS_NAME, "outbound")))
inbound_plan = wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "inbound")))
#seats selection - inbound
for inbound_passenger in driver.find_elements_by_css_selector("ol[data-flightbound='Inbound'] li[data-personid]"):
inbound_passenger.click()
#driver.find_elements_by_css_selector("ol.passengerlist li[data-personid]"):
inbound_has_infant = inbound_passenger.get_attribute("data-hasinfant")
# choose seats
if inbound_has_infant:
# select a non-selected infant seat
inbound_seat = driver.find_element_by_css_selector(".inbound .planebody a.seat.infant:not(.reserved):not(.selected)")
else:
# select a non-reserved non-selected seat
inbound_seat = driver.find_element_by_css_selector(".inbound .planebody a.seat:not(.reserved):not(.selected)")
print("Passenger: %s, choosing seat: %s" % (inbound_passenger.text.strip(), inbound_seat.get_attribute("data-seat")))
try:
inbound_seat.click()
except ElementNotVisibleException:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#AllInboundPassengersSeatedOk button"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".submitseatselection button"))).click()
meals = wait.until(EC.visibility_of_element_located((By.ID, "ctl00_MainContent_InFlightMeal_removeMealButton")))
#Meal details
try:
pax_one_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDown"))
pax_one_ob_meal.select_by_index(0)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDownIn")))
pax_one_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDownIn"))
pax_one_ib_meal.select_by_index(2)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDown")))
pax_two_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDown"))
pax_two_ob_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDownIn")))
pax_two_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl03_mealsDropDownIn"))
pax_two_ib_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDown")))
pax_three_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownt"))
pax_three_ob_meal.select_by_index(2)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownIn")))
pax_three_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownIn"))
pax_three_ib_meal.select_by_index(1)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDown")))
pax_four_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDown"))
pax_four_ob_meal.select_by_index(0)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDownIn")))
pax_four_ib_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl05_mealsDropDownIn"))
pax_four_ib_meal.select_by_index(1)

The logic is a bit different - when you select an item from the dropdown, this dropdown becomes disabled until the price is recalculated - see the disabled attribute appearing on the select element after selecting an option.
So, select an option and wait for the dropdown to be clickable/enabled:
pax_one_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDown"))
pax_one_ob_meal.select_by_index(0)
# wait for the dropdown to be clickable/enabled again
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_MainContent_InFlightMeal_PassengerGridView_ctl02_mealsDropDown")))
# proceed to the next dropdown

Please use the xpath or css locator to locate the element and then use select dropdown using index.Use of Fluent wait is good to use in this case.

I see two issues :-)
The first issue is:
//locator is incorrect it has a t in the end
pax_three_ob_meal = Select(driver.find_element_by_id("ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDownt"))
// it should be
ctl00_MainContent_InFlightMeal_PassengerGridView_ctl04_mealsDropDown
and the second issue is with the modal dialog box which we are forced to manually click on "continue" button is actually seems to be part of a iframe and you can get around this by switching through the iframes.
This is a very crude way but the below code works though it can be optimized( so add the below code above " meals selection")
framecount = driver.find_elements_by_tag_name("iframe")
for frame in framecount:
driver.switch_to.frame(frame)
try:
wait.until(EC.element_to_be_clickable((By.XPATH, ".//*[#id='AllInboundPassengersSeatedOk']/div[2]/p[2]/button"))).click()
driver.switch_to.default_content()
break
except TimeoutException:
driver.switch_to.default_content()
print "Not found in the frame"
// i could not understand why but until you loop thorugh the iframes the below element is not visiable
wait.until(EC.element_to_be_clickable((By.XPATH, ".//*[#id='AllInboundPassengersSeatedOk']/div[2]/p[2]/button"))).click()
//Start of meals section
meals = wait.until(EC.visibility_of_element_located((By.ID, "ctl00_MainContent_InFlightMeal_removeMealButton")))
After the above changes the script worked fine for me :-)