Develop Reference

Stale Element - Selenium - Python

So I'll start by saying that this has became such a mess with me trying to solve this issue, other times I have been able to resolve the stale element issue.
Problem all starts after the first players stats are stored ( Everything it should be doing up this point ), and then once it goes back to loop and find the next player we have the issue.
I'm not sure if its caused by the nested loops or what.
I try reinstating the variable that is giving me the issues I assume all throughout the code.
player_stats
The thing is I did have it previously going through 5 players, and I am not sure what happened, or when the bug first established itself lol, as I was working on getting the rounds won, and played situated.
(We aren't even able to print("Found playerCol element") on the second go around)
All print statements works till it hangs in the while loop after the first iteration.
Here is the full code (with comments):
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions importStaleElementReferenceException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd
import re
import time
# Initialize the webdriver
driver = webdriver.Firefox()
# Navigate to the website
url = "https://www.hltv.org/stats/players"
driver.get(url)
WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll"))).click()
# Find the elements containing the player statistics
player_stats = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".playerCol, .statsDetail"))
)
# Extract the relevant data from the elements
players = []
for i, player_stat in enumerate(player_stats):
try:
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".playerCol, .statsDetail")))
while True:
player_stats = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".playerCol, .statsDetail")))
try:
if "playerCol" in player_stat.get_attribute("class"):
print("Found playerCol element")
name = player_stat.find_element(By.CSS_SELECTOR, "a").text if player_stat.find_elements(By.CSS_SELECTOR, "a") else player_stat.text
print(f"Name: {name}")
elif "statsDetail" in player_stat.get_attribute("class"):
stats = player_stat.text.split()
if len(stats) >= 1 and re.search(r"\d+\.\d+", stats[0]):
kd_ratio = stats[0]
break
except StaleElementReferenceException as e:
player_stats = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".playerCol, .statsDetail")))
player_stats = driver.find_elements(By.CSS_SELECTOR, ".playerCol, .statsDetail")
print(f"An error occurred while processing match stats: {e}")
break
# Extract the player stats
if "statsDetail" in player_stat.get_attribute("class"):
stats = player_stat.text.split()
if len(stats) >= 1 and re.search(r"\d+\.\d+", stats[0]):
kd_ratio = stats[0]
# Process match stats for the player
try:
time.sleep(1)
WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".playerCol, .statsDetail")))
player_link = driver.find_element(By.XPATH, f"//a[contains(text(), '{name}')]")
print(player_link.get_attribute('outerHTML'))
driver.execute_script("arguments[0].click();", player_link)
time.sleep(1)
player_stats = driver.find_elements(By.CSS_SELECTOR, ".playerCol, .statsDetail")
player = [name, kd_ratio]
# Extract additional player stats
headshot_percentage = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//span[contains(text(), 'Headshot %')]/following-sibling::span"))).text
player.append(headshot_percentage)
kpr = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//span[contains(text(), 'Kills / round')]/following-sibling::span"))).text
player.append(kpr)
dpr = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//span[contains(text(), 'Deaths / round')]/following-sibling::span"))).text
player.append(dpr)
# Extract match stats for the player
matches_link = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a[href*='/stats/players/matches/'][data-link-tracking-destination='Click on Matches -> Individual -> Overview [subnavigation]']")))
driver.execute_script("arguments[0].click();", matches_link)
match_stats = WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "tr.group-2, tr.group-1")))
match_scores = []
num_of_matches = 0
rounds_won = 0
rounds_played = 0
# Process match stats for the player
for i, match_stat in enumerate(match_stats):
player_name = player[0]
player_team = driver.find_element(By.CSS_SELECTOR, ".gtSmartphone-only span:last-of-type").text
try:
team_name = ""
score = ""
while team_name == "" or score == "":
try:
team = match_stat.find_element(By.CSS_SELECTOR, ".gtSmartphone-only span:last-of-type").text
team_name = team.strip()
score_span = match_stat.find_element(By.XPATH, ".//div[contains(#class, 'gtSmartphone-only')]//*[contains(text(), '(')]")
score_text = score_span.text.strip()
score = re.search(r'\((\d+)\)', score_text).group(1)
except:
time.sleep(1)
match_stats = WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "tr.group-2, tr.group-1")))
match_stat = match_stats[i]
team_data = match_stat.find_elements(By.CSS_SELECTOR, ".gtSmartphone-only span")
print("Team data:", team_data[3].text)
if team_name.lower() == player_team.lower():
player_score = score
opposing_team_name = team_data[2].text.strip()
print(opposing_team_name)
opposing_team_score = team_data[3].text.strip('()')
print("Score strip: ", opposing_team_score)
rounds_won += int(player_score)
rounds_played += int(player_score) + int(opposing_team_score)
else:
player_score = team_data[1].text.strip('()')
print(player_score)
opposing_team_score = score
print(opposing_team_score)
opposing_team_name = team_data[0].text.strip()
print(opposing_team_name)
rounds_won += int(opposing_team_score)
rounds_played += int(player_score) + int(opposing_team_score)
match_scores.append((team_name, opposing_team_name, player_score, opposing_team_score))
num_of_matches += 1
if num_of_matches == 5: # exit loop after 5 iterations
break
except:
# Refresh the page if the element can't be found
driver.back()
player_stats = driver.find_elements(By.CSS_SELECTOR, ".playerCol, .statsDetail")
time.sleep(1)
match_stats = WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "tr.group-2, tr.group-1")))
except Exception as e:
print(f"An error occurred while processing data for player {name}: {e}")
continue
players.append([name, kd_ratio, headshot_percentage, kpr, dpr, rounds_won, rounds_played])
print(players)
print(f"{player_name}: {rounds_won} rounds won out of {rounds_played} rounds played in {num_of_matches} matches")
driver.get(url)
time.sleep(1)
except StaleElementReferenceException as e:
# handle the exception here
print(f"An error occurred while processing match stats: {e}")
break
# Close the webdriver
driver.quit()
# Store the data in a Pandas dataframe
df = pd.DataFrame(players, columns=["Name", "K/D", "HS %", "KPR", "DPR", "RW", "RP"])
# Clean the data
df["K/D"] = df["K/D"].str.extract(r"(\d+\.\d+)").astype(float)
df["HS %"] = df["HS %"].str.extract(r"(\d+\.\d+)").astype(float)
df["KPR"] = df["KPR"].str.extract(r"(\d+\.\d+)").astype(float)
df["DPR"] = df["DPR"].str.extract(r"(\d+\.\d+)").astype(float)
# Drop any rows that have missing or invalid data
df.dropna(subset=["Name", "K/D", "HS %", "KPR", "DPR"], inplace=True)
# Save the data to a CSV file
df.to_csv("player_stats.csv", index=False, sep='\t')
# Close the webdriver
driver.quit()

Print onclick attribute of several buttons in a webpage

I am trying to get the attribute "onclick" (onclick="openDirection( 41.051777,28.986551)") of the first button element inside all of the li elements. I want to print all the coordinates. In the code above I can manage to print the first two element's coordinates. However, a NoSuchElementException is thrown afterwards.
driver.get('https://migroskurumsal.com/magazalarimiz/')
try:
select = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'stores'))
)
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID, 'stores'))
select.select_by_value('2')
shopList = driver.find_element(By.ID, "shopList")
try:
WebDriverWait(driver, 10).until(
EC.visibility_of_all_elements_located((By.XPATH, "//ul[#id='shopList']/li/div/button"))
)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
print(shopList.get_attribute("class"))
li_items = shopList.find_elements(By.TAG_NAME, 'li')
for item in li_items:
first = item.find_element(By.TAG_NAME, 'div')
second = first.find_element(By.TAG_NAME, 'button')
coord = second.get_attribute("onclick")
print(coord)

There are several issues here:
You are using wrong locators
You will need to scroll the page in order to access the elements initially not in the visible view.
This should work better:
from selenium.webdriver.common.action_chains import ActionChains
driver.get('https://migroskurumsal.com/magazalarimiz/')
actions = ActionChains(driver)
try:
select = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'stores')))
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID, 'stores'))
select.select_by_value('2')
shopList = driver.find_element(By.ID, "shopList")
try:
WebDriverWait(driver, 10).until(EC.visibility_of_all_elements_located((By.XPATH, "//ul[#id='shopList']/li/div/button")))
time.sleep(1)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
li_items = shopList.find_elements(By.CSS_SELECTOR, 'li.m-bg-cream')
for idx, val in enumerate(li_items):
li_items = shopList.find_elements(By.CSS_SELECTOR, 'li.m-bg-cream')
item = li_items[idx]
actions.move_to_element(item).perform()
time.sleep(0.5)
button = item..find_elements(By.CSS_SELECTOR, "button[onclick^='set']")
coord = button.get_attribute("onclick")
print(coord)

selenium stale element reference: element is not attached to the page document error

I have an e-commerce page and there are multiple products on a page. I need to click the link of a product then return on the main page and click the link of the next product, but when I return, the elements can't be found anymore.
Path = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(Path)
driver.get("https://www.emag.ro/")
search_bar = driver.find_element_by_id("searchboxTrigger")
search_bar.send_keys("laptopuri")
search_bar.send_keys(Keys.RETURN)
main = None
try:
main = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "main-container"))
)
print("Page loaded,main retrived succesfully")
print(" ")
except:
driver.quit()
products = main.find_elements_by_css_selector("div.card-item.js-product-data")
for product in products:
raw_name = product.text
raw_price = product.find_element_by_css_selector("p.product-new-price").text
link = product.find_element_by_tag_name("a")
#clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
driver.back()
After the first iteration, I get the following error :
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document on line raw_name = product.text,basically at the beginning of the loop.
I assume the page is not loading properly or something like that, I tried using time.sleep before going through the loop but nothing

When you are writing driver.back(), it will go back to previous page and by the time it will reach to original page all the defined elements will become stale. You need to redefined them like below :-
This should handle the exception.
products = len(main.find_elements_by_css_selector("div.card-item.js-product-data"))
j = 0
for product in range(products):
elements = main.find_elements_by_css_selector("div.card-item.js-product-data")
raw_name = elements[j].text
raw_price = elements[j].find_element_by_css_selector("p.product-new-price").text
link = elements[j].find_element_by_tag_name("a")
# clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
j = j + 1
driver.back()

Trying to Scrape Instagram Post Data from .csv with links - For Masters Thesis

I am trying to scrape instagram post data (number of likes, Caption, Hashtags, Mentions and number of comments) from a collection of links in a .csv for data analysis to put towards my Masters Thesis. however i am coming across an error where the xpath or element cannot be found. Here is the error message:
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//*[#id="react-root"]/section/main/div/div/article/div[2]/section[2]/div/div/button"}
Here is the code block i have written using selenium:
def scrape_post_data():
influencerpostsdata = []
# Specify the path to chromedriver.exe
chromedriver_path = r"C:\\Users\\stuar\\Instagram Scraper\\ChromeDrivers\chromedriver.exe"
driver = webdriver.Chrome(executable_path=chromedriver_path)
time.sleep(2)
# Open the webpage
url = "https://www.instagram.com"
driver.get(url)
time.sleep(3)
# Alert number 1
time.sleep(5)
alert = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Accept All")]'))).click()
# Target Username Entry
username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='username']")))
password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='password']")))
# Enter Username and Password
login_username = str(enter_loginusername_entry.get())
login_password = str(enter_password_entry.get())
username.clear()
username.send_keys(login_username)
password.clear()
password.send_keys(login_password)
button = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']"))).click()
# Alert number 2
time.sleep(5)
alert2 = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
# Alert number 3
time.sleep(5)
alert3 = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
with open(r"C:\\Users\\stuar\\Instagram Scraper\\SourceCode/influencerpostlinks1.csv",'r') as csv_file:
csv_reading = csv.reader(csv_file)
for line in csv_reading:
links = line[1]
try:
Page = driver.get(links)
except Exception as e:
Page = None
time.sleep(20)
try:
# This captures the standard like count.
likes = driver.find_element_by_xpath("""//*[#id="react-root"]/section/main/div/div/article/div[2]/section[2]/div/div/button""").text.split()[0]
post_type = 'photo'
except:
# This captures the like count for videos which is stored
likes = driver.find_element_by_xpath("""//*[#id="react-root"]/section/main/div/div/article/div[2]/section[2]/div/span""").text.split()[0]
post_type = 'video'
age = driver.find_element_by_css_selector('a time').text
comment = driver.find_element_by_xpath("""//*[#id="react-root"]/section/main/div/div/article/div[2]/div[1]/ul/div/li/div/div/div[2]/span""").text
hashtags = find_hashtags(comment)
mentions = find_mentions(comment)
post_details = {'link': url, 'type': post_type, 'likes/views': likes,
'age': age, 'comment': comment, 'hashtags': hashtags,
'mentions': mentions}
time.sleep(10)
#turning data into a .csv file
influencerpostsdata.append(post_details)
df = pd.DataFrame(influencerposts)
print(df)
df.to_csv('influencerpostsdata.csv')
driver.close()

Not To worry i have resolved the problem..
with open(r"C:\\Users\\stuar\\Instagram Scraper\\SourceCode/influencerpostlinks1.csv",'r') as csv_file:
csv_reading = csv.reader(csv_file)
for line in csv_reading:
links = line[1]
try:
Page = driver.get(links)
except Exception as e:
Page = None
time.sleep(20)
try:
likes = driver.find_element_by_xpath('/html/body/div[1]/section/main/div/div[1]/article/div[3]/section[2]/div/div/a/span')
except Exception as e:
likes = None
try:
likes2 = likes.text
except Exception as e:
likes2 = None
time.sleep(20)
try:
age = driver.find_element_by_xpath('/html/body/div[1]/section/main/div/div[1]/article/div[3]/div[2]/a/time')
except Exception as e:
age = None
try:
age2 = age.text
except Exception as e:
age2 = None
time.sleep(20)
try:
caption = driver.find_element_by_xpath('/html/body/div[1]/section/main/div/div[1]/article/div[3]/div[1]/ul/div/li/div/div/div[2]/span')
except Exception as e:
caption = None
try:
caption2 = caption.text
except Exception as e:
caption2 = None
time.sleep(20)
try:
AccountName = driver.find_element_by_xpath('/html/body/div[1]/section/main/div/div[1]/article/header/div[2]/div[1]/div/span/a')
except Exception as e:
AccountName = None
try:
AccountName2 = AccountName.text
except Exception as e:
AccountName2 = None
time.sleep(20)
post_details = {'Username': AccountName2,'Caption': caption2, 'Likes/Views': likes2,
'Age': age2 }
#turning data into a .csv file
influencerpostsdata.append(post_details)
df = pd.DataFrame(influencerpostsdata)
print(df)
df.to_csv('influencerpostsdata.csv')
driver.close()

using selenium.click() to change pages but gets error

I'm trying to click on a div to get to the next page of a table (the url does not change when the page changes). The go to the next page div has the same class as the go to the previous page's.
ive used:
elem = driver.find_element_by_class_name('cXQSjq')
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text
most = most.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text
last = last.replace(',','')
last7day.append(last)
#loop for the other pages:
for i in range(6):
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text.replace(',','')
most = most.text.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text.replace(',','')
last7day.append(last)
and it worked to get me to page 2, but after page 2 it gives me the error:
selenium.common.exceptions.ElementClickInterceptedException: Message: element click
intercepted: Element <div class="styles__Chevron-sc-1buchb9-1 cXQSjq">...</div> is
not clickable at point (702, 656). Other element would receive the click: <div
id="hs-eu-cookie-confirmation-inner">...</div>
(Session info: chrome=92.0.4515.107)
Do you know if there is an issue that i am able to call elem.click() after using selenium to find other parts of the page. I'm scraping data from
https://nonfungible.com/market/history

I guess the issue here is that the next page button appears on the bottom of the page, so to click on it you should first scroll this element into the view.
See if this will work now:
from selenium.webdriver.common.action_chains import ActionChains
actions = ActionChains(driver)
elem = driver.find_element_by_class_name('cXQSjq')
actions.move_to_element(elem).perform()
time.sleep(0.3)
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text
most = most.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text
last = last.replace(',','')
last7day.append(last)
#loop for the other pages:
for i in range(6):
actions.move_to_element(elem).perform()
time.sleep(0.3)
elem.click()
timeout = 30
try:
WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.CLASS_NAME, "gxzFwa")))
except TimeoutException:
driver.quit()
names = driver.find_elements_by_class_name('iBSZGH')
for company in names[1:]:
name.append(company.text)
mostdata = driver.find_elements_by_class_name('gvgMSe.gJYnHB')
for most in mostdata:
most = most.text.replace(',','')
most = most.text.replace(',','')
data.append(most)
last7dsales = driver.find_elements_by_class_name('fGpHsy.kpsxyE')
for last in last7dsales:
last = last.text.replace(',','')
last7day.append(last)
I didn't check correctness of the rest of your code, but clicking the next button should work now