I'm scraping the product reviews from the sephora website which contains javascript(reviews), but I can't scrape. This is my code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.expected_conditions import presence_of_element_located as EC
import time
chrome_path = '/media/danish-khan/New Volume/Web_scraping/rgcrawler2/chromedriver'
driver = webdriver.Chrome(chrome_path)
chrome_options = Options()
url = 'https://www.sephora.com/product/the-porefessional-face-primer-P264900?skuId=1259068&icid2=products%20grid:p264900:product'
driver.get(url)
WebDriverWait(driver, 70)
time.sleep(70)
review = driver.find_element_by_class_name('css-1jg2pb9 eanm77i0')
for post in review:
#try:
# element = WebDriverWait(driver, 50).until(
# EC.presence_of_element_located((By.XPATH, "//div[#class = 'css-1jg2pb9 eanm77i0']"))
# )
#finally:
# driver.quit()
#
print(review)
driver.close()'
The output is:
Traceback (most recent call last):
File "resgt.py", line 15, in
review = driver.find_element_by_class_name('css-1jg2pb9 eanm77i0')
File "/home/danish-khan/miniconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 564, in find_element_by_class_name
return self.find_element(by=By.CLASS_NAME, value=name)
File "/home/danish-khan/miniconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 978, in find_element
'value': value})['value']
File "/home/danish-khan/miniconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/home/danish-khan/miniconda3/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".css-1jg2pb9 eanm77i0"}
(Session info: chrome=85.0.4183.102)
The reviews for that page are being loaded in asynchronously specifically when the section is scrolled into view. you will have to scroll to an element close to where the reviews are and wait until it appears. Only then you will be able to retrieve the element.
I was able to do this with this code
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
time.sleep(10)
review = driver.find_element_by_css_selector('.css-1jg2pb9.eanm77i0')
# review = driver.find_element_by_xpath('/html/body/div[1]/div[2]/div/main/div/div[2]/div[1]/div/div[5]/div/div[2]/div[1]/div[2]')
print(review)
I left the Xpath in there as thats what i used to get it the first time
note* you may have to adjust the timing and the scroll height to get it always correct
Related
I'm using Selenium with Python to input an address into a textbox within the Glovo page for Madrid. The code I wrote can be seen below, and the error I get is also copied after the code.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
url = 'https://glovoapp.com/es/es/madrid/'
# open browser
driver = webdriver.Chrome()
# load page
driver.get(url)
# find field
item = driver.find_element_by_class_name('address-input__container')
# select textbox and input text
time.sleep(2)
item.click()
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "el-input__inner"))
)
item = driver.find_element_by_class_name('el-input__inner')
time.sleep(2)
item.send_keys('Calle')
The error I get is shown below.
Traceback (most recent call last):
item.send_keys('Calle')
File "C:\Users\Usuario\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py", line 477, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT,
File "C:\Users\Usuario\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\Usuario\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Usuario\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
`ElementNotInteractableException: element not interactable`
(Session info: chrome=91.0.4472.124)
Locator you are using is returning 2 different elements. See if this works
item = driver.find_element_by_xpath("//input[#class='el-input__inner' and #data-test-id='address-input-autocomplete']")
item.send_keys("Calle")
Using various xpath for the element address-input__containerand trying to click it is not working while automation and throws ElementNotInteractableException: Message: element not interactable You can refer this question. Applying the same and it works.
from selenium.webdriver.common.action_chains import ActionChains
driver.get("https://glovoapp.com/es/es/madrid/")
box = driver.find_element_by_class_name('address-input__container')
time.sleep(2)
ActionChains(driver).move_to_element(box).click(box).perform()
driver.find_element_by_xpath("//input[#data-test-id='address-input-autocomplete']").send_keys("SampleText")
I'm new to selenium
Here I want to ask about a problem code (actually not mine)
this is the code
aww = email.strip().split('|')
driver = webdriver.Chrome()
driver.get("https://stackoverflow.com/users/signup?ssrc=head&returnurl=%2fusers%2fstory%2fcurrent")
time.sleep(5)
loginform = driver.find_element_by_xpath("//button[#data-provider='google']")
loginform.click()
mailform = driver.find_element_by_id('identifierId')
mailform.send_keys(aww[0])
driver.find_element_by_xpath("//div[#id='identifierNext']").click()
time.sleep(3)
passform = driver.find_element_by_css_selector("input[type='password']")
passform.send_keys(aww[1])
driver.find_element_by_id('passwordNext').click()
time.sleep(3)
driver.get("https://myaccount.google.com/lesssecureapps?pli=1")
open('LIVE.txt', 'a+').write(f"CHECKED : {aww[0]}|{aww[1]}")
time.sleep(3)
lessoff = driver.find_element_by_xpath('//div[#class="hyMrOd "]/div/div/div//div[#class="N9Ni5"]').click()
driver.delete_all_cookies()
driver.close()
I'm using those code for automating turn on the less-secure apps from Gmail
and the error will pop up like this
quote Traceback (most recent call last):
File "C:\Users\ASUS\Downloads\ok\less.py", line 59, in
lessoff = driver.find_element_by_xpath('//div[#class="hyMrOd "]/div/div/div//div[#class="N9Ni5"]').click()
File "C:\Users\ASUS\AppData\Local\Programs\Python\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "C:\Users\ASUS\AppData\Local\Programs\Python\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "C:\Users\ASUS\AppData\Local\Programs\Python\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\ASUS\AppData\Local\Programs\Python\Python39\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//div[#class="hyMrOd "]/div/div/div//div[#class="N9Ni5"]"}
(Session info: chrome=86.0.4240.183)
any help gonna be helpfull,sorry for my english before :)
You can simply target this xpath and .click to toggle the less secure apps.
lessoff = driver.find_element_by_xpath("input[type='checkbox']").click()
It looks like it couldn't find the element it's looking for so give some time to load the element. You can check with Wait().until.
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait as wait
wait(driver, 10).until(EC.presence_of_element_located((By.XPATH, 'YOUR_XPATH')))
when you try to click an element make sure it's there. above code will wait until the element located for the 10s if the element not located then it will throw an exception
I just started learning selenium and I'm facing some issues with some code that should be quite easy to write..
I'm simply trying to enter "Paris" in the input field, but I keep getting the error: "Unable to locate element". Do I need to refer to the div tag or the input tag in order for it to work?
Here's my code at the moment.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC
import time
# Setup webdriver
PATH = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(PATH)
# Open website maximized
driver.get("https://www.corsair.ca/")
driver.maximize_window()
print(driver.title)
# Select and enter destination
wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//span[text()='Votre destination']"))).click()
to_city = driver.find_element_by_class_name("form-control valid")
to_city.send_keys("Paris")
to_city.send_keys(Keys.RETURN)
Here's the error I have
Traceback (most recent call last):
File "C:\Users\Admin\PycharmProjects\FlightFinder\venv\SeleniumTutorial.py", line 26, in <module>
to_city = driver.find_element_by_class_name("form-control valid")
File "C:\Users\Admin\PycharmProjects\FlightFinder\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 564, in find_element_by_class_name
return self.find_element(by=By.CLASS_NAME, value=name)
File "C:\Users\Admin\PycharmProjects\FlightFinder\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "C:\Users\Admin\PycharmProjects\FlightFinder\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Admin\PycharmProjects\FlightFinder\venv\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".form-control valid"}
(Session info: chrome=86.0.4240.183)
I am scraping the title of jobs results on https://www.indeed.ae/jobs-in-dubai through selenium. i think .text is not working.
i am running the code through selenium which go to main website, enter selective keyword and then scrape all titles from result. but i am getting error, how can i solve this error
here is my code
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
Path = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(Path)
driver.get("https://indeed.ae/")
print(driver.title)
search = driver.find_element_by_name("l")
search.send_keys("Dubai")
search.send_keys(Keys.RETURN)
try:
td = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "resultsCol"))
)
divs = td.find_elements_by_tag_name("div")
for div in divs:
header = div.find_element_by_class_name("title")
print(header)
finally:
driver.quit()
driver.quit()
and i am getting following error
Job Search | Indeed
Traceback (most recent call last):
File "C:/Users/hp/Desktop/python projects/selenium-pycharm/selenium-bot.py", line 24, in <module>
header = div.find_element_by_class_name("title")
File "C:\Users\hp\Desktop\python projects\selenium-pycharm\venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 398, in find_element_by_class_name
return self.find_element(by=By.CLASS_NAME, value=name)
File "C:\Users\hp\Desktop\python projects\selenium-pycharm\venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 659, in find_element
{"using": by, "value": value})['value']
File "C:\Users\hp\Desktop\python projects\selenium-pycharm\venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\hp\Desktop\python projects\selenium-pycharm\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\hp\Desktop\python projects\selenium-pycharm\venv\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".title"}
(Session info: chrome=83.0.4103.116)
Process finished with exit code 1
thanks in advance
You can not find title because you get all of the div from the resultsCol. which means some div have title and some not.
Try this :
try:
td = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "resultsCol"))
)
divs = td.find_elements_by_tag_name("div")
#print(divs)
for div in divs:
try:
header = div.find_element_by_class_name("title")
print(header.text)
except:
continue
finally:
driver.quit()
driver.quit()
which give title as output :
Receptionist
Administrative Assistant/ Document Controller
RECEPTIONIST
ADMIN OFFICER IN UAE
Data Entry Assistant (Fresh Graduate)
Receptionist
Replenishment Associate - Light Household - Hypermarket
DOCUMENT CONTROLLER
School Administrative Assistant - Dubai
ACCOUNTANT
I am trying to fill in the search input type box with value "fab" and then I want to display the next url with that keyword but I am getting this error for element not interactable. How can I solve this?
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome(executable_path='C:\\Users\\Mansi Dhingra\\Downloads\\chromedriver')
driver.get("https://www.thenational.ae/search?q=")
print(driver.title)
driver.implicitly_wait(10)
search_bar = driver.find_element_by_xpath('//input[#name="q"]')
print(search_bar)
search_bar.clear()
search_bar.send_keys("fab")
search_bar.send_keys(Keys.RETURN)
print(driver.current_url)
driver.close()
Error:-
Traceback (most recent call last): File "C:/Users/Mansi
Dhingra/Desktop/Projects/api/news/news_python.py", line 10, in
search_bar.clear() File "C:\Users\Mansi Dhingra\Desktop\Projects\api\venv\lib\site-packages\selenium\webdriver\remote\webelement.py",
line 95, in clear
self._execute(Command.CLEAR_ELEMENT) File "C:\Users\Mansi Dhingra\Desktop\Projects\api\venv\lib\site-packages\selenium\webdriver\remote\webelement.py",
line 633, in _execute
return self._parent.execute(command, params) File "C:\Users\Mansi
Dhingra\Desktop\Projects\api\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py",
line 321, in execute
self.error_handler.check_response(response) File "C:\Users\Mansi Dhingra\Desktop\Projects\api\venv\lib\site-packages\selenium\webdriver\remote\errorhandler.py",
line 242, in check_response
raise exception_class(message, screen, stacktrace) selenium.common.exceptions.ElementNotInteractableException: Message:
element not interactable (Session info: chrome=81.0.4044.92)
Whenever webelement is found but not ready to interact then webdriver will throw element not intractable exception.
1.Mostly it occurs, when element is located in bottom of the page, so it can be accessible by scrolling the page down. You can use Action class to scroll to scroll to that element
WebElement element = driver.findElement(By.id("my-id"));
Actions actions = new Actions(driver);
actions.moveToElement(element);
## actions.click();
actions.perform();
2. Sometimes we need to wait few seconds to access the web element, in such situations we can add wait statements.
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.ID, 'someid')))