Develop Reference

Python Selenium cannot find a way to scroll down (checked other responses an none seem to work)

I'm currently trying to scrape the Kaggle rankings and the page is an infinte loading scroll. I would like to get at least the first 2000 ranked kagglers, and so to solve this I've created this script:
No matter what I do I don't see the browser scrolling and the lista_parseada list always has a length of 20. Can somebody help with this? Thanks!!
My code below:
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
import re
import pandas as pd
import numpy as np
import time
from datetime import date
# Notebook rankings url
url = 'https://www.kaggle.com/rankings?group=notebooks&page=1&pageSize=20'
wait_delay = 10 # seconds
scroll_pause_time = 2 # seconds
firefox_options = webdriver.FirefoxOptions()
firefox_options.add_argument('-private')
driver = webdriver.Firefox(options=firefox_options)
# load page
driver.get(url)
try:
WebDriverWait(driver, wait_delay).until(EC.presence_of_element_located((By.ID, 'site-content')))
print("Page is ready!")
except Exception as e:
print(e)
print("Loading took too much time!")
# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")
scroll_pass = 0
while scroll_pass < 10:
# Scroll down to bottom
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(scroll_pause_time)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
scroll_pass+=1
lista = driver.find_elements_by_xpath('//div[#role="button"]//div[#class="leaderboards__name"]/p/a')
lista_parseada = [link.get_attribute('href') for link in lista]
print(len(lista_parseada))
driver.close()

Kaggle has an api, so it's better to use it:
import requests
import json
import pandas as pd
def post_call(url: str, headers: dict, data: dict) -> dict:
response = requests.post(url=url, headers=headers, json=data)
response.raise_for_status()
return response.json()
url = 'https://www.kaggle.com/api/i/users.ProgressionService/GetUserRankings'
headers = {
"content-type": "application/json",
"cookie": "CSRF-TOKEN=VxwvCfDJ7O7KAEn8EWH0HeK9uT-G89SyETB0-hq9mZZhVsjDDIFJAh4OOhIUFjymST0kO8oX43sl86ZuOudHOoxHlPWV-krcTXNUlSgOQA;",
"x-xsrf-token": "CfDJ7O7VujnuKA6ZuOudEn8ExwsAkR8eU_RQRaWH0HLuA2qYIkNHMeUOWequ-h2j0YuQNki8aAxC0j5tYvo9fI9fL-j9yzhevhI4MPdC9DRHLWnA"
}
tdf = []
for i in range(1, 101):
data = {
"group": "ACHIEVEMENT_SUMMARY_TYPE_NOTEBOOKS",
"page": i,
"pageSize": 20
}
df = pd.json_normalize(post_call(url, headers, data)['list'])
# use drop(columns=["value1", "value2"]) to exclude unnecessary values
tdf.append(df.drop(columns=["thumbnailUrl"]))
# reset indexes
print(pd.concat(tdf).reset_index(drop=True))
Output df with 2000 users:
currentRanking displayName userId userUrl tier points joinTime totalGoldMedals totalSilverMedals totalBronzeMedals
0 1 Chris Deotte 1723677 /cdeotte GRANDMASTER 4943 2018-03-14T22:51:30.630Z 71.0 17.0 3.0
1 2 Marília Prata 3012786 /mpwolke MASTER 3621 2019-03-29T19:09:20.750Z 12.0 39.0 450.0
2 3 Abhishek Thakur 5309 /abhishek GRANDMASTER 3169 2011-01-12T03:44:52Z 65.0 28.0 24.0
3 4 AmbrosM 7917824 /ambrosm GRANDMASTER 2737 2021-07-16T18:36:58.170Z 28.0 8.0 8.0
4 5 Y.Nakama 1695531 /yasufuminakama GRANDMASTER 2630 2018-03-06T11:56:37.560Z 37.0 9.0 6.0
... ... ... ... ... ... ... ... ... ... ...
1995 1996 ayoub chaoui 6625407 /ayoubchaoui EXPERT 51 2021-01-30T15:31:19.840Z NaN 1.0 6.0
1996 1997 micheldc55 6646082 /micheldc55 EXPERT 51 2021-02-02T18:58:13.170Z NaN NaN 5.0
1997 1998 Hugo R. V. Angulo 6910521 /hugovallejo EXPERT 51 2021-03-10T18:29:25.247Z NaN 1.0 7.0
1998 1999 Dina Nabil 7213495 /dinanabil811 EXPERT 51 2021-04-18T11:09:01.470Z NaN NaN 5.0
1999 2000 Naser Al-qaydeh 7424338 /naseralqaydeh EXPERT 51 2021-05-15T13:16:16.093Z NaN NaN 8.0
Cookies and other info can be found on the "Network" tab in "DevTools"
In cookies you only need "CSRF-TOKEN"

Looping through Different Selectors on Webpage to store in one big DF

I came with a question regarding this project today that was answered super quickly so here I am again. The code below scrapes through the provided website, pulls the data, and adds a column for what instance of the table it is scraping. The next battle I am facing with this is getting all of the Game Recency instances loaded into the big_df with a column to replicate what the game recency drop down is currently on. If anyone could help me with the last piece to my puzzle I would grealy appreciate it.
https://www.fantasypros.com/daily-fantasy/nba/fanduel-defense-vs-position.php
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)
big_df = pd.DataFrame()
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service(r'chromedriver\chromedriver') ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 20)
url = "https://www.fantasypros.com/daily-fantasy/nba/fanduel-defense-vs-position.php"
driver.get(url)
sleep(60)
tables_list = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//ul[#class="pills pos-filter pull-left"]/li')))
for x in tables_list:
x.click()
print('selected', x.text)
t.sleep(2)
table = wait.until(EC.element_to_be_clickable((By.XPATH, '//table[#id="data-table"]')))
df = pd.read_html(table.get_attribute('outerHTML'))[0]
df['Category'] = x.text.strip()
big_df = pd.concat([big_df, df], axis=0, ignore_index=True)
print('done, moving to next table')
print(big_df)
big_df.to_csv('fanduel.csv')

This is how you might achieve your end goal:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)
big_df = pd.DataFrame()
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 20)
url = "https://www.fantasypros.com/daily-fantasy/nba/fanduel-defense-vs-position.php"
driver.get(url)
select_recency_options = [x.text for x in wait.until(EC.presence_of_all_elements_located((By.XPATH, '//select[#class="game-change"]/option')))]
for option in select_recency_options:
select_recency = Select(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//select[#class="game-change"]'))))
select_recency.select_by_visible_text(option)
print('selected', option)
t.sleep(2)
tables_list = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//ul[#class="pills pos-filter pull-left"]/li')))
for x in tables_list:
x.click()
print('selected', x.text)
t.sleep(2)
table = wait.until(EC.element_to_be_clickable((By.XPATH, '//table[#id="data-table"]')))
df = pd.read_html(table.get_attribute('outerHTML'))[0]
df['Category'] = x.text.strip()
df['Recency'] = option
big_df = pd.concat([big_df, df], axis=0, ignore_index=True)
print('done, moving to next table')
display(big_df)
big_df.to_csv('fanduel.csv')
The result is a (bigger) dataframe:
Team PTS REB AST 3PM STL BLK TO FD PTS Category Recency
0 HOUHouston Rockets 23.54 9.10 5.10 2.54 1.88 1.15 2.65 48.55 ALL Season
1 OKCOklahoma City Thunder 22.22 9.61 5.19 2.70 1.67 1.18 2.52 47.57 ALL Season
2 PORPortland Trail Blazers 22.96 8.92 5.31 2.74 1.63 0.99 2.65 46.84 ALL Season
3 SACSacramento Kings 23.00 9.10 5.03 2.58 1.61 0.95 2.50 46.65 ALL Season
4 ORLOrlando Magic 22.35 9.39 4.94 2.62 1.57 1.04 2.50 46.36 ALL Season
... ... ... ... ... ... ... ... ... ... ... ...
715 TORToronto Raptors 23.33 13.97 2.77 0.57 0.84 1.88 3.38 49.03 C Last 30
716 NYKNew York Knicks 19.78 15.40 2.94 0.53 0.90 1.92 2.17 48.96 C Last 30
717 BKNBrooklyn Nets 19.69 13.60 3.16 0.86 1.10 2.25 2.06 48.74 C Last 30
718 BOSBoston Celtics 17.79 11.95 3.75 0.41 1.66 1.80 2.54 45.60 C Last 30
719 MIAMiami Heat 17.41 14.19 2.16 0.50 1.01 1.52 1.75 43.52 C Last 30
720 rows × 11 columns

How to find the attribute and element id by selenium.webdriver?

I am learning web scrapping since I need it for my work. I wrote the following code:
from selenium import webdriver
chromedriver='/home/es/drivers/chromedriver'
driver = webdriver.Chrome(chromedriver)
driver.implicitly_wait(30)
driver.get('http://crdd.osdd.net/raghava/hemolytik/submitkey_browse.php?ran=1955')
df = pd.read_html(driver.find_element_by_id("table.example.display.datatable").get_attribute('example'))[0]
However, it is showing the following error:
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":"[id="table.example.display.datatable"]"}
(Session info: chrome=103.0.5060.134)
Then I inspect the table that I wanna scrape this table from this page
what is the attribute that needs to be included in get_attribute() function in the following line?
df = pd.read_html(driver.find_element_by_id("table.example.display.datatable").get_attribute('example'))[0]
what I should write in the driver.find_element_by_id?
EDITED:
Some tables have lots of records in multi-pages.
For example, this page has 2,246 entries, which shows 100 entries on each page. Once I tried to web-scrape it, there were only 320 entries in df and the record ID is from 1232-1713, which means it took entries from the next few pages and it is not starting from the first page to the end at the last page.
What we can do in such cases?

You need to get the outerHTML property of the table first, then call the table element from pandas.
You need to wait for element to be visible. Use explicit wait like WebdriverWait()
driver.get('http://crdd.osdd.net/raghava/hemolytik/submitkey_browse.php?ran=1955')
table=WebDriverWait(driver,10).until(EC.visibility_of_element_located((By.CSS_SELECTOR,"table#example")))
tableRows=table.get_attribute("outerHTML")
df = pd.read_html(tableRows)[0]
print(df)
Import below libraries.
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import pandas as pd
Output:
ID PMID YEAR ... DSSP Natural Structure Final Structure
0 1643 16137634 2005 ... CCCCCCCCCCCSCCCC NaN NaN
1 1644 16137634 2005 ... CCTTSCCSSCCCC NaN NaN
2 1645 16137634 2005 ... CTTTCGGGHHHHHHHHCC NaN NaN
3 1646 16137634 2005 ... CGGGTTTHHHHHHHGGGC NaN NaN
4 1647 16137634 2005 ... CCSCCCSSCHHHHHHHHHTTC NaN NaN
5 1910 16730859 2006 ... CCCCCCCSSCCSHHHHHHHHTTHHHHHHHHSSCCC NaN NaN
6 1911 16730859 2006 ... CCSCC NaN NaN
7 1912 16730859 2006 ... CCSSSCSCC NaN NaN
8 1913 16730859 2006 ... CCCSSCCSSCCSHHHHHTTHHHHTTTCSCC NaN NaN
9 1914 16730859 2006 ... CCSHHHHHHHHHHHHHCCCC NaN NaN
10 2110 11226440 2001 ... CCCSSCCCBTTBTSSSSSSCSCC NaN NaN
11 3799 9204560 1997 ... CCSSCC NaN NaN
12 4149 16137634 2005 ... CCHHHHHHHHHHHC NaN NaN
[13 rows x 17 columns]

If you want to select table by #id you need
driver.find_element_by_id("example")
By.CSS:
driver.find_element_by_css_selector("table#example")
By.XPATH:
driver.find_element_by_xpath("//table[#id='example'])
If you want to extract #id value you need
.get_attribute('id')
Since there is not much sense in searching by #id to extract that exact #id you might use other attribute of table node:
driver.find_element_by_xpath("//table[#aria-describedby='example_info']").get_attribute('id')

Scraping dynamic data selenium - Unable to locate element

I am very new to scraping and have a question. I am scraping worldometers covid data. As it is dynamic - I am doing it with selenium.
The code is the following:
from selenium import webdriver
import time
URL = "https://www.worldometers.info/coronavirus/"
# Start the Driver
driver = webdriver.Chrome(executable_path = r"C:\Webdriver\chromedriver.exe")
# Hit the url and wait for 10 seconds.
driver.get(URL)
time.sleep(10)
#find class element
data= driver.find_elements_by_class_name("odd" and "even")
#for loop
for d in data:
country=d.find_element_by_xpath(".//*[#id='main_table_countries_today']").text
print(country)
current output:
NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":".//*[#id='main_table_countries_today']"}
(Session info: chrome=96.0.4664.45)

To scrape table within worldometers covid data you need to induce WebDriverWait for the visibility_of_element_located() and using DataFrame from Pandas you can use the following Locator Strategy:
Code Block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
options = Options()
options.add_argument("start-maximized")
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://www.worldometers.info/coronavirus/")
data = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "table#main_table_countries_today"))).get_attribute("outerHTML")
df = pd.read_html(data)
print(df)
driver.quit()
Console Output:
[ # Country,Other TotalCases NewCases ... Deaths/1M pop TotalTests Tests/ 1M pop Population
0 NaN World 264359298 632349.0 ... 673.3 NaN NaN NaN
1 1.0 USA 49662381 89259.0 ... 2415.0 756671013.0 2267182.0 3.337495e+08
2 2.0 India 34609741 3200.0 ... 336.0 643510926.0 459914.0 1.399198e+09
3 3.0 Brazil 22118782 12910.0 ... 2865.0 63776166.0 297051.0 2.146975e+08
4 4.0 UK 10329074 53945.0 ... 2124.0 364875273.0 5335159.0 6.839070e+07
.. ... ... ... ... ... ... ... ... ...
221 221.0 Samoa 3 NaN ... NaN NaN NaN 2.002800e+05
222 222.0 Saint Helena 2 NaN ... NaN NaN NaN 6.103000e+03
223 223.0 Micronesia 1 NaN ... NaN NaN NaN 1.167290e+05
224 224.0 Tonga 1 NaN ... NaN NaN NaN 1.073890e+05
225 NaN Total: 264359298 632349.0 ... 673.3 NaN NaN NaN
[226 rows x 15 columns]]

Webscraping jTable with hidden columns?

I am currently trying to setup a webscraper in Python for the following webpage:
https://understat.com/team/Juventus/2018
specifically for the 'team-players jTable'
I have managed to scrape the table successfully with BeautifulSoup and selenium, but there are hidden columns (accessible via the options popup window) that I can't initialize and include in my scraping.
Anyone know how to change this?
import urllib.request
from bs4 import BeautifulSoup
import lxml
import re
import requests
from selenium import webdriver
import pandas as pd
import re
import random
import datetime
base_url = 'https://understat.com/team/Juventus/2018'
url = base_url
data = requests.get(url)
html = data.content
soup = BeautifulSoup(html, 'lxml')
options = webdriver.ChromeOptions()
options.add_argument('headless')
driver = webdriver.Chrome('/Users/kylecaron/Desktop/souptest/chromedriver',options=options)
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'lxml')
headers = soup.find('div', attrs={'class':'players jTable'}).find('table').find_all('th',attrs={'class':'sort'})
headers_list = [header.get_text(strip=True) for header in headers]
body = soup.find('div', attrs={'class':'players jTable'}).table.tbody
all_rows_list = []
for tr in body.find_all('tr'):
row = tr.find_all('td')
current_row = []
for item in row:
current_row.append(item.get_text(strip=True))
all_rows_list.append(current_row)
headers_list = ['№', 'Player', 'Positions', 'Apps', 'Min', 'G', 'A', 'Sh90', 'KP90', 'xG', 'xA', 'xG90', 'xA90']
xg_df = pd.DataFrame(all_rows_list, columns=headers_list)
If you navigate to the website, there are hidden table columns such as 'XGChain'. I want all of these hidden elements scraped, but having trouble doing it.
Best,
Kyle

Here you go. You could still use BeautifulSoup to iterate through the tr and td tags, but I always find pandas much easier to get tables, as it does the work for you.
from selenium import webdriver
import pandas as pd
url = 'https://understat.com/team/Juventus/2018'
driver = webdriver.Chrome()
driver.get(url)
# Click the Options Button
driver.find_element_by_xpath('//*[#id="team-players"]/div[1]/button/i').click()
# Click the fields that are hidden
hidden = [7, 12, 14, 15, 17, 19, 20, 21, 22, 23, 24]
for val in hidden:
x_path = '//*[#id="team-players"]/div[2]/div[2]/div/div[%s]/div[2]/label' %val
driver.find_element_by_xpath(x_path).click()
# Appy the filter
driver.find_element_by_xpath('//*[#id="team-players"]/div[2]/div[3]/a[2]').click()
# get the tables in source
tables = pd.read_html(driver.page_source)
data = tables[1]
data.rename(columns={'Unnamed: 22':"Yellow_Cards", "Unnamed: 23":"Red_Cards"})
driver.close()
Output:
print (data.columns)
Index(['№', 'Player', 'Pos', 'Apps', 'Min', 'G', 'NPG', 'A', 'Sh90', 'KP90',
'xG', 'NPxG', 'xA', 'xGChain', 'xGBuildup', 'xG90', 'NPxG90', 'xA90',
'xG90 + xA90', 'NPxG90 + xA90', 'xGChain90', 'xGBuildup90',
'Yellow_Cards', 'Red_Cards'],
dtype='object')
print (data)
№ Player ... Yellow_Cards Red_Cards
0 1.0 Cristiano Ronaldo ... 2 0
1 2.0 Mario Mandzukic ... 3 0
2 3.0 Paulo Dybala ... 1 0
3 4.0 Federico Bernardeschi ... 2 0
4 5.0 Blaise Matuidi ... 2 0
5 6.0 Rodrigo Bentancur ... 5 1
6 7.0 Juan Cuadrado ... 2 0
7 8.0 Leonardo Bonucci ... 1 0
8 9.0 Miralem Pjanic ... 4 0
9 10.0 Sami Khedira ... 0 0
10 11.0 Giorgio Chiellini ... 1 0
11 12.0 Medhi Benatia ... 2 0
12 13.0 Douglas Costa ... 2 1
13 14.0 Emre Can ... 2 0
14 15.0 Mattia Perin ... 1 0
15 16.0 Mattia De Sciglio ... 0 0
16 17.0 Wojciech Szczesny ... 0 0
17 18.0 Andrea Barzagli ... 0 0
18 19.0 Alex Sandro ... 3 0
19 20.0 Daniele Rugani ... 1 0
20 21.0 Moise Kean ... 0 0
21 22.0 João Cancelo ... 2 0
22 NaN NaN ... 36 2
[23 rows x 24 columns]