I am trying to move to page 2 and beyond of this page (pagination) with python selenium and spent a few hours on this. I am getting this error, and would be thankful of any help..Error from chromedriver
is not clickable at point(). Other element would receive the click
My code so far:
class Chezacash:
t1 = time.time()
driver = webdriver.Chrome(chromedriver)
def controller(self):
self.driver.get("https://www.chezacash.com/#/home/")
element = WebDriverWait(self.driver, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "div.panel-heading")))
soup = BeautifulSoup(self.driver.page_source.encode('utf-8'),"html.parser")
self.parser(soup)
self.driver.find_element(By.XPATH, "//li[#class='paginate_button active']/following-sibling::li").click()
time.sleep(2)
soup = BeautifulSoup(self.driver.page_source.encode('utf-8'),"html.parser")
self.parser(soup)
def parser(self, soup):
for i in soup.find("table", {"id":"DataTables_Table_1"}).tbody.contents:
date = i.findAll("td")[0].get_text().strip()
time = i.findAll("td")[1].get_text().strip()
home = i.findAll("td")[4].div.span.get_text().strip().encode("utf-8")
home_odds = i.findAll("td")[4].div.findAll("span")[1].get_text().strip()
draw_odds = i.findAll("td")[5].div.findAll("span")[1].get_text().strip()
away = i.findAll("td")[6].div.span.get_text().strip().encode("utf-8")
away_odds = i.findAll("td")[6].div.findAll("span")[1].get_text().strip()
print home
cheza = Chezacash()
try:
cheza.controller()
except:
cheza.driver.service.process.send_signal(signal.SIGTERM) # kill the specific phantomjs child proc # quit the node proc
cheza.driver.quit()
traceback.print_exc()
What if instead you would locate the "Next" button by link text, scroll into it's view and then click:
next_button = self.driver.find_element_by_link_text("Next")
self.driver.execute_script("arguments[0].scrollIntoView();", next_button)
next_button.click()
I would also maximize the browser window before navigating to the page:
self.driver.maximize_window()
self.driver.get("https://www.chezacash.com/#/home/")
Related
I have an e-commerce page and there are multiple products on a page. I need to click the link of a product then return on the main page and click the link of the next product, but when I return, the elements can't be found anymore.
Path = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(Path)
driver.get("https://www.emag.ro/")
search_bar = driver.find_element_by_id("searchboxTrigger")
search_bar.send_keys("laptopuri")
search_bar.send_keys(Keys.RETURN)
main = None
try:
main = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "main-container"))
)
print("Page loaded,main retrived succesfully")
print(" ")
except:
driver.quit()
products = main.find_elements_by_css_selector("div.card-item.js-product-data")
for product in products:
raw_name = product.text
raw_price = product.find_element_by_css_selector("p.product-new-price").text
link = product.find_element_by_tag_name("a")
#clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
driver.back()
After the first iteration, I get the following error :
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document on line raw_name = product.text,basically at the beginning of the loop.
I assume the page is not loading properly or something like that, I tried using time.sleep before going through the loop but nothing
When you are writing driver.back(), it will go back to previous page and by the time it will reach to original page all the defined elements will become stale. You need to redefined them like below :-
This should handle the exception.
products = len(main.find_elements_by_css_selector("div.card-item.js-product-data"))
j = 0
for product in range(products):
elements = main.find_elements_by_css_selector("div.card-item.js-product-data")
raw_name = elements[j].text
raw_price = elements[j].find_element_by_css_selector("p.product-new-price").text
link = elements[j].find_element_by_tag_name("a")
# clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
j = j + 1
driver.back()
I'm trying to click on a div to get to the next page of a table (the url does not change when the page changes). The go to the next page div has the same class as the go to the previous page's.
ive used:
elem = driver.find_element_by_class_name('cXQSjq')
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text
most = most.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text
last = last.replace(',','')
last7day.append(last)
#loop for the other pages:
for i in range(6):
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text.replace(',','')
most = most.text.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text.replace(',','')
last7day.append(last)
and it worked to get me to page 2, but after page 2 it gives me the error:
selenium.common.exceptions.ElementClickInterceptedException: Message: element click
intercepted: Element <div class="styles__Chevron-sc-1buchb9-1 cXQSjq">...</div> is
not clickable at point (702, 656). Other element would receive the click: <div
id="hs-eu-cookie-confirmation-inner">...</div>
(Session info: chrome=92.0.4515.107)
Do you know if there is an issue that i am able to call elem.click() after using selenium to find other parts of the page. I'm scraping data from
https://nonfungible.com/market/history
I guess the issue here is that the next page button appears on the bottom of the page, so to click on it you should first scroll this element into the view.
See if this will work now:
from selenium.webdriver.common.action_chains import ActionChains
actions = ActionChains(driver)
elem = driver.find_element_by_class_name('cXQSjq')
actions.move_to_element(elem).perform()
time.sleep(0.3)
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text
most = most.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text
last = last.replace(',','')
last7day.append(last)
#loop for the other pages:
for i in range(6):
actions.move_to_element(elem).perform()
time.sleep(0.3)
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text.replace(',','')
most = most.text.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text.replace(',','')
last7day.append(last)
I didn't check correctness of the rest of your code, but clicking the next button should work now
Image is for CSS selector and xpath for pagination.
I also wanted to perform a regex in to to separate Apple, iPhone 12, Neo Galactic Silver like this I wanted to print it in new line.
After finishing the product list of this current page, I want to be able to click next and perform the same procedure with the products on the next page.
This is the problem: when it reaches the 10 items of the current page, I have no idea how to change to another page and start all over again.
import xlwt
from selenium import webdriver
import re
import time
class cometmobiles:
def __init__(self):
self.url='https://www.mediaworld.it/catalogo/telefonia/smartphone-e-cellulari/smartphone'
def comet(self):
try:
driver=webdriver.Chrome()
driver.get(self.url)
time.sleep(5)
cookies = driver.find_element_by_id("onetrust-accept-btn-handler")
cookies.click()
print("accepted cookies")
driver.maximize_window()
print("window maximized")
mylist = []
hasNextPate = True
while hasNextPate:
containers = []
containters =driver.find_elements_by_css_selector('article[class="product clearfix p-list-js"]')
for container in containters:
#Title
try:
title = container.find_element_by_css_selector('h3[class="product-name"]').text
print(title)
except:
pass
#price
try:
price = container.find_element_by_css_selector('span[class="price mw-price enhanced"]').text
print(price)
except:
pass
try:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
nxt=driver.find_elements_by_css_selector('span[class="pages"] a')
time.sleep(5)
nxt.click()
except:
break
except:
pass
comets=cometmobiles()
comets.comet()
Instead of this part
try:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
nxt=driver.find_elements_by_css_selector('span[class="pages"] a')
time.sleep(5)
nxt.click()
except:
break
You can use this and also if the page number doesn't exist website turn the main page so you should add
try:
x=0
while True:
x+=1
driver.get(url+"?pageNumber="+str(x)) #Get the next page
if driver.current_url == url: #If there is no next page it will turn main page and you can break at this time
break
except:
pass
I'm using selenium with python to test my web server. Here is my test code:
i = 0
msg = 'abc'
while i < 10:
driver = webdriver.Chrome()
driver.get("http://www.example.com")
txt = driver.find_element_by_id('input-text')
txt.clear()
txt.send_keys(msg)
btn = driver.find_element_by_id('input-search')
btn.click()
driver.quit()
i += 1
The code works well except only one thing: it executes Chrome, do the test and close it for each time of loop. Obviously it's not necessary. What I need is simply to execute Chrome only one time and do many requests. I've tried as below but it doesn't work:
i = 0
msg = 'abc'
driver = webdriver.Chrome()
while i < 10:
driver.get("http://www.example.com")
txt = driver.find_element_by_id('input-text')
txt.clear()
txt.send_keys(msg)
btn = driver.find_element_by_id('input-search')
btn.click()
i += 1
driver.quit()
I think it's because in my test, there are two things:
1) fill abc in input-text;
2) click a button, submit the abc and open a new web page.
On the new page, there is also an input-text and a button input-search, so it will fill the abc and click the button on the new page, which is not what I want.
At btn.click(), it doesn't wait for page loaded because it is not a hyperlink and it doesn't submit a webform. So your script might be failed. You should fix it by waiting for some elements to determine if it should reload page or not. See code below.
try:
driver.get('http://www.example.com')
txt = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "input-text")))
txt.clear()
txt.send_keys(msg)
btn = driver.find_element_by_id('input-search')
btn.click()
countryDescriptionElement = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "country-description")))
#print(driver.page_source.encode('utf-8'))
except WebDriverException as ex:
print("Enter: " + msg + ", Error: " + str(ex) + ", Found: " + driver.page_source)
Prerequisites.
You need an account at Instagram to use this script.
Setup a test environment:
Log in, open the needed list(works correctly):
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
driver = webdriver.Chrome(
# driver = webdriver.Firefox(
# driver = webdriver.PhantomJS(
service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.get("https://instagram.com/accounts/login")
username = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")
username1 = 'instagram' # change it!
password1 = 'instagrampassword1' # change it!
username.send_keys(username1)
password.send_keys(password1)
submit_button = driver.find_element_by_css_selector(
'#react-root > div > article > div > div:nth-child(1) > div > form > span > button')
submit_button.click()
sleep(2)
link = 'https://www.instagram.com/youtube/'
driver.get(link)
driver.implicitly_wait(2)
driver.find_elements_by_class_name("_218yx")[2].click()
Wrong scroll.
How to fix this block?
How to focus and scroll correctly on this page?
My attempts:
driver.find_element_by_class_name("_cx1ua").send_keys(Keys.NULL) # focus
#The element has been deleted entirely or
#The element is no longer attached to the DOM.
driver.find_element_by_class_name("_q44m8").send_keys(Keys.NULL)
# cannot focus element
driver.find_element_by_class_name("_qjr85").send_keys(Keys.NULL)
# cannot focus element
for i in range(5):
driver.find_element_by_class_name("_cx1ua").send_keys(Keys.END)
=============================================================
to #Moshisho :
We need to focus on some element to activate it.
The question is what the element we need to choose to focus and how?
This is not a "body":
something like that, but not this:
background = driver.find_element_by_css_selector("body")
# background = driver.find_element_by_css_selector("div._2uju6")
for i in range(5):
background.send_keys(Keys.SPACE)
time.sleep(1)
Without it this command do not work.
to #Naveen :
print(driver.find_element_by_css_selector("div._a1rcs").location_once_scrolled_into_view) # {'x': 0, 'y': 0}
print(driver.find_element_by_class_name("_cx1ua").location_once_scrolled_into_view) # {'x': 376, 'y': 229}
print(driver.find_element_by_class_name("_q44m8").location_once_scrolled_into_view) # {'x': 376, 'y': 180}
print(driver.find_element_by_class_name("_qjr85").location_once_scrolled_into_view) # {'x': 376, 'y': 180}
And what's next?
driver.execute_script("window.scrollTo(0, 3000);") # do not working
Try the following code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome(
# driver = webdriver.Firefox(
# driver = webdriver.PhantomJS(
service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
driver.maximize_window()
driver.get("https://instagram.com/accounts/login")
username = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")
username1 = 'instagramlogin1' # change it!
password1 = 'instagrampassword1' # change it!
username.send_keys(username1)
password.send_keys(password1)
submit_button = driver.find_element_by_css_selector(
'#react-root > div > article > div > div:nth-child(1) > div > form > span > button')
submit_button.click()
sleep(2)
link = 'https://www.instagram.com/youtube/'
driver.get(link)
driver.implicitly_wait(2)
following = driver.find_element_by_xpath("//a[#href='/youtube/following/']/span")
total_following = int(following.text)
print "total no. of users following: ", total_following
# click on 239 following, displays 10 users
following.click()
loaded_following = driver.find_elements_by_xpath("//ul[#class='_539vh _4j13h']/li")
loaded_till_now = len(loaded_following)
while(loaded_till_now<total_following):
print "following users loaded till now: ", loaded_till_now
print loaded_following[loaded_till_now-1]
loaded_following[loaded_till_now-1].location_once_scrolled_into_view
# driver.execute_script("arguments[0].focus();", loaded_following[loaded_till_now-1])
driver.find_element_by_tag_name('body').send_keys(Keys.END) # triggers AJAX request to load more users. observed that loading 10 users at a time.
sleep(1) # tried wihtout sleep but throws StaleElementReferenceException. As it takes time to get the resposne and update the DOM
loaded_following = driver.find_elements_by_xpath("//ul[#class='_539vh _4j13h']/li")
loaded_till_now = len(loaded_following)
# All 239 users are loaded.
driver.quit()
Observed that browser is sending AJAX request to load more users. this action is triggered when you scroll using mouse or enter Space or End keys
In order to scroll in the window, you need to execute JavaScript, try this:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
EDIT: in order to focus an element (it needs to be able to get the focus e.g. an anchor, input, button etc...) you also need to use JavaScript executor:
elementToFocus = driver.find_element_by_id("yourID")
driver.execute_script("arguments[0].focus();", elementToFocus)
I'm working with a dynamic React app, I need to scroll to the pages bottom to make react render all the data.
For unknown reasons, solutions based on JS execute_script didn't work. However I got send_keys solution working:
# scroll to bottom to load all
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "//body"))
)
attempt_num = 2
while attempt_num > 0:
try:
elem = driver.find_element_by_xpath("//body")
elem.click()
elem.send_keys(Keys.END)
except StaleElementReferenceException as e:
print(e)
attempt_num = attempt_num - 1
The click() on body and the retry for StaleElementReferenceException are crucial. I haven't found a more elegant way than to retry.
See top answer of How to avoid "StaleElementReferenceException" in Selenium?