Develop Reference

How can I avoid this Stale Element Reference error using Selenium with Python?

I have been attempting to use Selenium to complete a webscrape of product information, and have managed to successfully scrape the information I need from three or four category pages at a time (up to 120 products in total). My goal is to scrape c.4000 products from c.100 category pages.
However, I have been receiving "stale element reference" errors sporadically, which stop the scrape from completing. The more products I try to collect, the more likely it seems that the error will occur and the whole scrape fail.
I have attempted to avoid the error by re-adding the webdriver to "refresh" the "hrefs" element which seems to be causing the problem, but this doesn't seem to have helped.
This approach seemed to fix the error when it was triggered by the individual product elements. I changed my code to write web elements directly into a dictionary, rather than assigning them to a variable and then writing them to a dictionary (See lower down my code for example of the previous approach commented out), and this seems to have worked.
The problem now seems to be focussed on line72: href_list.append(href.get_attribute('href'))
This is the error I repeatedly receive:
Traceback (most recent call last):
File "h:\Python\aldi\aldi\selenium_test5.py", line 79, in <module>
href_scraper(all_urls)
File "h:\Python\aldi\aldi\selenium_test5.py", line 72, in href_scraper
href_list.append(href.get_attribute('href'))
File "H:\Python\aldi\aldi_venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 172, in get_attribute
attribute_value = self.parent.execute_script(
File "H:\Python\aldi\aldi_venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 491, in execute_script
return self.execute(command, {
File "H:\Python\aldi\aldi_venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 428, in execute
self.error_handler.check_response(response)
File "H:\Python\aldi\aldi_venv\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 243, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
The csv file 'url_list.csv' contains the following lines:
https://groceries.aldi.co.uk/en-GB/specially-selected
https://groceries.aldi.co.uk/en-GB/vegan-range
https://groceries.aldi.co.uk/en-GB/bakery
https://groceries.aldi.co.uk/en-GB/fresh-food
https://groceries.aldi.co.uk/en-GB/drinks
https://groceries.aldi.co.uk/en-GB/food-cupboard
https://groceries.aldi.co.uk/en-GB/frozen
https://groceries.aldi.co.uk/en-GB/chilled-food
https://groceries.aldi.co.uk/en-GB/baby-toddler
https://groceries.aldi.co.uk/en-GB/health-beauty
https://groceries.aldi.co.uk/en-GB/household
https://groceries.aldi.co.uk/en-GB/pet-care
And here is my code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from fp.fp import FreeProxy
import pandas as pd
from datetime import datetime
import time
import csv
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-logging"])
options.add_argument('--disable-blink-features=AutomationControlled')
proxy = FreeProxy(rand=True, country_id=['GB'], elite=True).get()
with open('url_list.csv') as f:
base_urls = [line.strip() for line in f]
# Collect category page links from main site
cat_urls = []
def cat_url_collector(base_urls):
for base_url in base_urls:
options.add_argument('--proxy-server=%s' % proxy)
driver = webdriver.Chrome(options=options)
driver.get(base_url)
time.sleep(1.5)
driver.find_element('xpath','//*[#id="onetrust-accept-btn-handler"]').click()
time.sleep(2)
extra_cat_urls = driver.find_elements(By.CLASS_NAME, 'dropdown-item')
def cat_href_lister(extra_cat_urls, cat_urls):
for extra_url in extra_cat_urls:
text_url = extra_url.get_attribute('href')
if '?sortDirection=asc&page=' not in text_url:
pass
else:
cat_urls.append(text_url)
return cat_urls
cat_href_lister(extra_cat_urls, cat_urls)
return cat_urls
cat_url_collector(base_urls)
# Combine base urls & extra pages, then de-dupe
all_urls = base_urls + cat_urls
all_urls = list(set(all_urls))
# Collect product page links from category pages
href_list=[]
def href_scraper(all_urls):
for cat_url in all_urls:
options.add_argument('--proxy-server=%s' % proxy)
driver = webdriver.Chrome(options=options)
driver.get(cat_url)
time.sleep(1.5)
driver.find_element('xpath','//*[#id="onetrust-accept-btn-handler"]').click()
time.sleep(2)
hrefs = driver.find_elements(By.CLASS_NAME, 'p.text-default-font')
for href in hrefs:
hrefs = driver.find_elements(By.CLASS_NAME, 'p.text-default-font')
href_list.append(href.get_attribute('href'))
return href_list
href_scraper(all_urls)
# Collect product information from product page links
prod_info = []
def prod_scraper(href_list):
prod_count = 0
for href in href_list:
options.add_argument('--proxy-server=%s' % proxy)
driver = webdriver.Chrome(options=options)
driver.get(href)
driver.maximize_window()
time.sleep(1.5)
driver.find_element('xpath','//*[#id="onetrust-accept-btn-handler"]').click()
time.sleep(2)
# prod_id = driver.find_element(By.CSS_SELECTOR, 'span.sku.small').text[prod_id.text.rindex(' ')+1:]
prod_name = driver.find_element(By.CLASS_NAME, 'my-0')
# prod_size = driver.find_element(By.CSS_SELECTOR, 'span.text-black-50.font-weight-bold')
# prod_price = driver.find_element(By.CLASS_NAME, 'product-price.h4.m-0.font-weight-bold')
if driver.find_element(By.CLASS_NAME, 'product-price.h4.m-0.font-weight-bold') == None:
with open('rescrape_list.csv') as f:
rescrape_writer = csv.writer(f)
rescrape_writer.writerow(href)
else:
try:
prod_prev_price = driver.find_element(By.XPATH, '/html/body/div[4]/div[1]/div/div/div[2]/div/div/div[1]/div/div/del').text
except:
prod_prev_price = ""
prod = {
"URL": href,
"ID": driver.find_element(By.CSS_SELECTOR, 'span.sku.small').text,
"Description": driver.find_element(By.CLASS_NAME, 'my-0').text,
"Size": driver.find_element(By.CSS_SELECTOR, 'span.text-black-50.font-weight-bold').text,
"Price": driver.find_element(By.CLASS_NAME, 'product-price.h4.m-0.font-weight-bold').text.strip('£'),
"Previous Price": prod_prev_price.strip('£'),
"Scrape Date": datetime.now().strftime('%d/%m/%Y')
}
prod_info.append(prod)
# report progress of scrape (number of products and latest product name)
prod_count +=1
print(str(prod_count) +') '+prod_name.text)
return prod_info
prod_scraper(href_list)
# use pandas to write csv
df = pd.DataFrame.from_dict(prod_info, orient='columns')
df.to_csv("scrape_data_{}.csv".format(pd.datetime.now().strftime("%Y-%m-%d %H%M%S")))
I'd be very grateful for any advice you can provide, as I can't understand how to implement a fix for the "stale element reference" within the for loops in my code.
Thank you.

getting 'attribute error while using tuple arguments python selenium

I am trying to "until(EC.visibility_of_element_located" function in my method.But getting error like this,what is the reason ?
by = (<selenium.webdriver.chrome.webdriver.WebDriver (session="0df329cbca657f67360d8fa26fc0ca1c")>, ('xpath', "//div[#title='Kapat'])"))
def _find_element(driver, by):
"""Looks up an element. Logs and re-raises ``WebDriverException``
if thrown."""
try:
> return driver.find_element(*by)
E AttributeError: 'BasePage' object has no attribute 'find_element'
Here is my code,BasePage.py::
def click(driver,*by_locator):
WebDriverWait(driver, 60) .until(EC.visibility_of_element_located(by_locator)).click()
and I am calling this function:
driver = webdriver.Chrome()
driver.get("https://www.trendyol.com/")
driver.maximize_window()
website_is_up_and_running("https://www.trendyol.com/")
time.sleep(5)
bp.click(driver, (By.XPATH,"//div[#title='Kapat'])"))

InvalidArgumentException: Message: invalid argument: 'using' must be a string

im very new to python, trying to create reusable code. when i try to call the class Login and function login_user in test_main.py by passing all the arguments that were used under Login class, im getting an error as InvalidArgumentException: Message: invalid argument: 'using' must be a string.
test_main.py file which runs on pytest.
Locators_test is the class of test_Locators.py file where i have all my xpaths
test_Locators.py
class Locators_test():
loginlink_xpath = "//a[#id='login-link']"
login_email = "xxxxx"
login_password = "xxxxx"
loginemail_id = "dnn_ctr1179_Login_txtEmail"
loginpassword_id = "dnn_ctr1179_Login_txtPassword"
clicklogin_id = "dnn_ctr1179_Login_btnLogin"
test_login.py
from Smoketest.locatorfile.test_Locators import Locators_test
class Login():
def __init__(self,driver):
self.driver = driver
def login_user(self,driver):
try:
loginButton = self.driver.find_element((By.XPATH, Locators_test.loginlink_xpath))
while loginButton.click() is True:
break
time.sleep(3)
self.driver.execute_script("window.scrollBy(0,300);")
EmailField = self.driver.find_element((By.ID, Locators_test.loginemail_id))
EmailField.send_keys(Locators_test.login_email)
PasswordField = self.driver.find_element((By.ID, Locators_test.loginpassword_id))
PasswordField.send_keys(Locators_test.login_password)
ClickLogin = self.driver.find_element((By.ID, Locators_test.clicklogin_id))
while ClickLogin.click() is True:
break
time.sleep(5)
userName = self.driver.find_element((By.XPATH, Locators_test.username_xpath))
print("Logged in as", userName.text)
except StaleElementReferenceException or ElementClickInterceptedException or TimeoutException as ex:
print(ex.message)
test_main.py
def test_setup():
driver = webdriver.Chrome(executable_path= Locators_test.browser_path)
driver.maximize_window()
driver.delete_all_cookies()
driver.get(homePage)
driver.implicitly_wait(5)
yield
print("test complete")
def test_login(test_setup):
from Smoketest.pages.test_login import Login
lo = Login(driver)
lo.login_user(((Locators_test.loginlink_xpath,Locators_test.loginemail_id,Locators_test.login_email,Locators_test.loginpassword_id,Locators_test.login_password,Locators_test.clicklogin_id,Locators_test.username_xpath)))
indentations are all fine

I fixed it myself by removing the extra pair of parenthesis from the line
loginButton = self.driver.find_element((By.XPATH, Locators_test.loginlink_xpath))
Right way is
loginButton = self.driver.find_element(By.XPATH, Locators_test.loginlink_xpath)
ps: this applies to all the lines.

This worked for me,
locator = (By.XPATH, Locators_test.loginlink_xpath)
self.driver.find_element(*locator).click()
Explanation: In *<arguments>, all positional arguments other than the first one will be packed in a tuple, as they won't get Changed, the exact property will be reflected in the second step.

How to send text to username and password field in Instagram through Selenium and Python

Im having the error messages for a python program I got they are :
C:\Users\chanm\AppData\Local\Programs\Python\Python37-32\python.exe C:/Users/chanm/OneDrive/Desktop/bot/Commenter.py
Traceback (most recent call last):
File "C:/Users/chanm/OneDrive/Desktop/bot/Commenter.py", line 133, in <module>
com.login()
File "C:/Users/chanm/OneDrive/Desktop/bot/Commenter.py", line 28, in login
login_button = driver.find_element_by_xpath("//a[#href='/accounts/login/']")
File "C:\Users\chanm\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "C:\Users\chanm\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 978, in find_element
'value': value})['value']
File "C:\Users\chanm\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\chanm\AppData\Local\Programs\Python\Python37-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//a[#href='/accounts/login/']"}
(Session info: chrome=71.0.3578.80)
(Driver info: chromedriver=2.44.609538 (b655c5a60b0b544917107a59d4153d4bf78e1b90),platform=Windows NT 10.0.17134 x86_64)
Process finished with exit code 1
This is my code that I have
Commenter.py
import time
import random
import re
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException
from chatterbot.trainers import ListTrainer
from chatterbot import ChatBot
class Commenter:
def __init__(self, username, password):
self.username = username
self.password = password
self.driver = webdriver.Chrome()
self.driver.set_window_size(700, 900)
"""closing browser"""
def closeBrowser(self):
self.driver.close()
"""login in to Instagram"""
def login(self) -> object:
driver = self.driver
driver.get("https://www.instagram.com/")
time.sleep(2)
login_button = driver.find_element_by_xpath("//a[#href='/accounts/login/']")
login_button.click()
time.sleep(2)
user_name_elem = driver.find_element_by_xpath("//input[#name='username']")
user_name_elem.clear()
user_name_elem.send_keys(self.username)
passworword_elem = driver.find_element_by_xpath("//input[#name='password']")
passworword_elem.clear()
passworword_elem.send_keys(self.password)
passworword_elem.send_keys(Keys.RETURN)
time.sleep(2)
"""getting pictures on a hashtag page"""
def get_pictures_on_page(self, hashtag, scrolls=int):
self.driver.get("https://www.instagram.com/explore/tags/" + hashtag + "/")
time.sleep(2)
# gathering photos
pic_hrefs = []
for i in range(1, scrolls):
try:
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2)
# get tags
hrefs_in_view = self.driver.find_elements_by_tag_name('a')
# finding relevant hrefs
hrefs_in_view = [elem.get_attribute('href') for elem in hrefs_in_view if
hashtag in elem.get_attribute('href')]
# building list of unique photos
[pic_hrefs.append(href) for href in hrefs_in_view if href not in pic_hrefs]
# print("Check: pic href length " + str(len(pic_hrefs)))
except Exception:
continue
return pic_hrefs
"""write comment in text area using lambda function"""
def write_comment(self, comment_text):
try:
comment_button = lambda: self.driver.find_element_by_link_text('Comment')
comment_button().click()
except NoSuchElementException:
pass
try:
comment_box_elem = lambda: self.driver.find_element_by_xpath("//textarea[#aria-label='Add a comment…']")
comment_box_elem().send_keys('')
comment_box_elem().clear()
for letter in comment_text:
comment_box_elem().send_keys(letter)
time.sleep((random.randint(1, 7) / 30))
return comment_box_elem
except StaleElementReferenceException and NoSuchElementException as e:
print(e)
return False
"""actually post a comment"""
def post_comment(self, comment_text):
time.sleep(random.randint(1,5))
comment_box_elem = self.write_comment(comment_text)
if comment_text in self.driver.page_source:
comment_box_elem().send_keys(Keys.ENTER)
try:
post_button = lambda: self.driver.find_element_by_xpath("//button[#type='Post']")
post_button().click()
print('clicked post button')
except NoSuchElementException:
pass
time.sleep(random.randint(4, 6))
self.driver.refresh()
if comment_text in self.driver.page_source:
return True
return False
"""grab comments from a picture page"""
def get_comments(self):
# load more comments if button exists
time.sleep(3)
try:
comments_block = self.driver.find_element_by_class_name('Xl2Pu')
comments_in_block = comments_block.find_elements_by_class_name('gElp9')
comments = [x.find_element_by_tag_name('span') for x in comments_in_block]
user_comment = re.sub(r'#.\w*', '', comments[0].text)
except NoSuchElementException:
return ''
return user_comment
"""have bot comment on picture"""
def comment_on_picture(self):
bot = ChatBot('YouTubeChatBot')
bot.set_trainer(ListTrainer)
picture_comment = self.get_comments()
# user's comment and bot's response
response = bot.get_response(picture_comment).__str__()
print("User's Comment", picture_comment)
print("Bot's Response", response)
return self.post_comment(response)
com: Commenter = Commenter(username='username', password='password')
com.login()
for pic in com.get_pictures_on_page(hashtag='gaming', scrolls=5)[1:]:
com.driver.get(pic)
time.sleep(3)
print('Posted Comment:', com.comment_on_picture())
time.sleep(3)
that is the script I have the most problems I have I have tried things along the lines of changing extensions and other little things it resolved most of them now im stuck with these ones

The username and password field within Instagram are JavaScript enabled element so you have to induce WebDriverWait for the desired element to be clickable and you can use the following solution:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = Options()
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get('https://www.instagram.com')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[href='/accounts/login/?source=auth_switcher']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='username'][aria-label='Phone number, username, or email']"))).send_keys("JRProgrammer")
driver.find_element_by_css_selector("input[name='password'][aria-label='Password']").send_keys("JRProgrammer")
driver.find_element_by_xpath("//button[text()='Log in']").click()

If you read the stack trace, it says that it is failing to find the login button, so you've entered a bad selector. I'm not really sure why you're calling that anyway, since you wouldn't want to click the login button before entering user info.
Try:
def login(self) -> object:
driver = self.driver
driver.get("https://www.instagram.com/")
time.sleep(2)
user_name_elem = driver.find_element_by_xpath("//input[#name='username']")
user_name_elem.clear()
user_name_elem.send_keys(self.username)
passworword_elem = driver.find_element_by_xpath("//input[#name='password']")
passworword_elem.clear()
passworword_elem.send_keys(self.password)
passworword_elem.send_keys(Keys.RETURN)
time.sleep(2)
login_button = driver.find_element_by_css_selector("button[type='submit']")
login_button.click()
time.sleep(2)

using Enum to build locator for selenium test

I come out one question about selenium web test. Since all elements require their own xpath or css selector to be action by selenium webdriver.
I have tried to use python Enum and create something like
file: elementEnum.py
from enum import Enum
class PageA(Enum):
pElementA = '//div[{}]'
pElementB = '//a[.="{}"]'
pElementC = '//div[#class={}]'
class PageB(Enum):
pElementA = '//button[.="{}""]'
pElementB = '//table/tr[{}]/td[{}]'
but it turns out lots of time I require to build the string in python format function and it does not see pythonoic.
from elementEnum import *
driver.find_element_by_xpath('//div[#class="aaaaaa"]/{}'.format((PageA.pElementA.value).format(1)))
driver.find_element_by_xpath('{}/{}'.format(PageA.pElementA.value).format(1), PageA.pElementB.value.format(2)))
driver.find_element_by_xpath('{}/{}'.format(PageB.pElementB.value).format(1, 3), PageA.pElementA.value.format(2)))
What is the best way for me to list out all corresponse element and their locator.

you could use the
EC.visibility_of_element_located to locate the element
http://selenium-python.readthedocs.io/waits.html
sample code :
class SeleniumBaseClass(object):
def __init__(self,driver):
self.driver = driver
def open(self,URL):
self.driver.get(URL)
def driverURLChange(self,URL):
print("change URL" + URL)
self.driver.get(URL)
def currentUrl(self):
print("URL " + self.driver.current_url)
return self.driver.current_url
def switchNewWindow(self):
self.driver.switch_to_window(self.driver.window_handles[1])
return self.driver.title
def locateElement(self, loc):
try:
print(loc)
element = WebDriverWait(self.driver,10).until(EC.visibility_of_element_located(loc))
return element
except:
print ("cannot find {0} element".format(loc))
return None
and you could
password_loc =(By.NAME,'password')
webdriver = SeleniumBaseClass(driver)
webdriver.locateElement(password_loc )
this could you pass the tuple to locate the element