Develop Reference

Scraping Gifs Using Selenium

I have a working code that is able to access Tenor.com, scroll through the website and scrape gifs. But my issue is that it only scrapes and saves upto 24 gifs (no matter how many it scrolls past).
This exact same code works for saving images on other websites (without the same issues presented here).
I've also tried using BeautifulSoup to find all divs with the class "Gif " and then extract the img from each class. But that leads to the exact same result (only 24 gifs being downloaded).
Heres my code and output below. What might the issue be?
Output
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import time
import requests
from urllib.parse import urljoin
from selenium.webdriver.common.by import By
import urllib.request
options = Options()
options.add_experimental_option("detach", True)
options.add_argument("--disable-notifications")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
search_url = 'https://tenor.com/'
driver.get(search_url)
time.sleep(5) # Allow 7 seconds for the web page to open
scroll_pause_time = 2 # You can set your own pause time. My laptop is a bit slow so I use 1 sec
screen_height = driver.execute_script("return window.screen.height;") #get the screen height of the web
i = 1
start_time = time.time()
while True:
if time.time() - start_time >= 60:
break
# scroll one screen height each time
driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))
i += 1
time.sleep(scroll_pause_time)
# update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
scroll_height = driver.execute_script("return document.body.scrollHeight;")
# Break the loop when the height we need to scroll to is larger than the total scroll height
if (screen_height) * i > scroll_height:
break
media = []
media_elements = driver.find_elements(By.XPATH,"//div[contains(#class,'Gif ')]//img")
for m in media_elements:
src = m.get_attribute("src")
media.append(src)
print("Total Number of Animated GIFs and Videos Stored is", len(media))
print("The Sequence of Pages we Have Scrolled is", i)
for i in range(len(media)):
urllib.request.urlretrieve(str(media[i]),"tenor/media{}.gif".format(i))

If you scroll down with the DevTools opened, you can see that the number of figure elements doesn't increase after a certain quantity, i.e. old images are removed from the html as new ones are added.
So you have to run .get_attribute("src") inside the scrolling loop. Also, I suggest you using a set instead of a list to save the urls, since by running set.add(url) the url is added only if is not already contained in the set.
The code below scrape the images, get the urls and scroll to the last visible image.
media = set()
for i in range(6):
images = driver.find_elements(By.XPATH,"//div[contains(#class,'Gif ')]//img")
[media.add(img.get_attribute('src')) for img in images]
driver.execute_script('arguments[0].scrollIntoView({block: "center", behavior: "smooth"});', images[-1])
time.sleep(1)

Selenium scraping Issues with site having an popup window with endless scroll

I am trying to scrape a website that populates a list of providers. the site makes you go through a list of options and then finally it populates a list of providers through a pop up that has an endless/continuous scroll.
i have tried:
from selenium.webdriver.common.action_chains import ActionChains
element = driver.find_element_by_id("my-id")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
but this code didn't work.
I tried something similar to this:
driver.execute_script("arguments[0].scrollIntoView();", list )
but this didnt move anything. it just stayed on the first 20 providers.
i tried this alternative:
main = driver.find_element_by_id('mainDiv')
recentList = main.find_elements_by_class_name('nameBold')
for list in recentList :
driver.execute_script("arguments[0].scrollIntoView(true);", list)
time.sleep(20)
but ended up with this error message:
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
The code that worked the best was this one:
while True:
# Scroll down to bottom
element_inside_popup = driver.find_element_by_xpath('//*[#id="mainDiv"]')
element_inside_popup.send_keys(Keys.END)
# Wait to load page
time.sleep(3)
but this is an endless scroll that i dont know how to stop since "while True:" will always be true.
Any help with this would be great and thanks in advance.
This is my code so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
import pandas as pd
PATH = '/Users/AnthemScraper/venv/chromedriver'
driver = webdriver.Chrome(PATH)
#location for the website
driver.get('https://shop.anthem.com/sales/eox/abc/ca/en/shop/plans/medical/snq?execution=e1s13')
print(driver.title)
#entering the zipcode
search = driver.find_element_by_id('demographics.zip5')
search.send_keys(90210)
#making the scraper sleep for 5 seconds while the page loads
time.sleep(5)
#entering first name and DOB then hitting next
search = driver.find_element_by_id('demographics.applicants0.firstName')
search.send_keys('juelz')
search = driver.find_element_by_id('demographics.applicants0.dob')
search.send_keys('01011990')
driver.find_element_by_xpath('//*[#id="button/shop/getaquote/next"]').click()
#hitting the next button
driver.find_element_by_xpath('//*[#id="hypertext/shop/estimatesavings/skipthisstep"]').click()
#making the scraper sleep for 2 seconds while the page loads
time.sleep(2)
#clicking the no option to view all the health plans
driver.find_element_by_xpath('//*[#id="radioNoID"]').click()
driver.find_element_by_xpath('/html/body/div[4]/div[11]/div/button[2]/span').click()
#making the scraper sleep for 2 seconds while the page loads
time.sleep(2)
driver.find_element_by_xpath('//*[#id="hypertext/shop/medical/showmemydoctorlink"]/span').click()
time.sleep(2)
#section to choose the specialist. here we are choosing all
find_specialist=\
driver.find_element_by_xpath('//*[#id="specializedin"]')
#this is the method for a dropdown
select_provider = Select(find_specialist)
select_provider.select_by_visible_text('All Specialties')
#choosing the distance. Here we click on 50 miles
choose_mile_radius=\
driver.find_element_by_xpath('//*[#id="distanceInMiles"]')
select_provider = Select(choose_mile_radius)
select_provider.select_by_visible_text('50 miles')
driver.find_element_by_xpath('/html/body/div[4]/div[11]/div/button[2]/span').click()
#handling the endless scroll
while True:
time.sleep(20)
# Scroll down to bottom
element_inside_popup = driver.find_element_by_xpath('//*[#id="mainDiv"]')
element_inside_popup.send_keys(Keys.END)
# Wait to load page
time.sleep(3)
#block below allows us to grab the majority of the data. we would have to split it up in pandas since this info
#is nested in with classes
time.sleep(5)
main = driver.find_element_by_id('mainDiv')
sections = main.find_elements_by_class_name('firstRow')
pcp_info = []
#print(section.text)
for pcp in sections:
#the site stores the information inside inner classes which make it difficult to scrape.
#the solution would be to pull the entire text in the block and hope to clean it aftewards
#innerText allows to pull just the text inside the blocks
first_blox = pcp.find_element_by_class_name('table_content_colone').get_attribute('innerText')
second_blox = pcp.find_element_by_class_name('table_content_coltwo').get_attribute('innerText')
#creating columns and rows and assigning them
pcp_items = {
'first_block' : [first_blox],
'second_block' : [second_blox]
}
pcp_info.append(pcp_items)
df = pd.DataFrame(pcp_info)
print(df)
df.to_csv('yerp.csv',index=False)
#driver.quit()

How to get screen coordinate of a element in selenium? [Python]

I wanted to know that how can one get the coordinates of a element according to the screen resolution rather than the browser windows size, I have tried this already (code block), but it provides coordinates according to the browser window rather than the screen
element = driver.find_element_by_xpath("//*[#id='search_form_input_homepage']")
print(element.location)
Any alternatives that I can use?
A terrible attempt to explain what I mean :
note: driver.execute_script is not allowed, as the website has a bot blocker :(

You can use .size and .location to get the sizes.
Try this:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep, strftime
url = "some url"
webdriver = webdriver.Chrome()
webdriver.get(url)
webdriver.fullscreen_window()
cookies = webdriver.find_element_by_xpath("xome xpath")
location = cookies.location
size = cookies.size
w, h = size['width'], size['height']
print(location)
print(size)
print(w, h)

print(element.location_once_scrolled_into_view)
Try if this helps , more available methods like size rect etc can be found at:
https://www.selenium.dev/selenium/docs/api/py/webdriver_remote/selenium.webdriver.remote.webelement.html#module-selenium.webdriver.remote.webelement

Assert correct scrolling position with selenium in python

I am running a django app and in that scope I have a page with a navigation bar and when I click on my contact-tab this automatically scrolls down to my contact-section.
I am trying to test this behaviour with selenium but I can't seem to figure out how I can test the actual position of the page. Or, in other words, I want to verify that the page actually scrolled down to the contact-section.
Right now I am doing this:
def test_verify_position(
self, browser, live_server
):
""""""
browser.get(live_server.url)
contact_section = browser.find_element_by_id("contact").click()
assert ????
I think I somehow have to get the current scroll location-coordinates. I know I can get the location of an element using .location. But the relative position of an element is always the same no matter the scroll-position. I tried this to debug:
def test_verify_position(
self, browser, live_server
):
""""""
browser.get(live_server.url)
e = browser.find_element_by_xpath(
"/html/body/section[4]/div/div[1]/h2/strong")
location = e.location
print(location)
browser.find_element_by_css_selector(".nav-contact").click()
e = browser.find_element_by_xpath("/html/body/section[4]/div/div[1]/h2/strong")
location = e.location
print(location)
This prints the same coordinates for before and after the scroll.
I also searched the official doc https://www.selenium.dev/documentation/en/webdriver/web_element/ but couldn't find a better solution or any solution for that matter.
Anyone knows how to do this? Help is very much appreciated. Thanks in advance!

Did you want to click on it and check if it moved there? Cause you can return the current scroll height to match with the element location. You could get x,y offset if you want to as well.
height = driver.execute_script("return document.body.scrollHeight")
print(height)
nav = browser.find_element_by_css_selector(".nav-contact")
location = nav.location
print(location)
browser.execute_script("arguments[0].scrollIntoView();",nav)
nav.click()
Assert.assertTrue(height.equals(location['y']));
#what the answer was
browser.execute_script("return window.pageYOffset")

Edit : You can identify one of element in screen loading after click on Contact nav and wait till its visible.
try:
WebDriverWait(driver.visibility_of_element_located, 60).until(EC.presence_of_element_located((By.XPATH, '<xpath>')))
except TimeoutException:
assert False
Need to Import:
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException

Selection of a specific date in Calendar Date-picker using Selenium with Python

please I need some help with a piece of code. (I am a complete beginner so my code might be a mess, sorry for that)
I am trying to create an automated "date scheduler" to book a tennis court every Sunday of the week because I always seem to be too late and end up sitting around eating apples. -_-
The problem I am facing is selenium fills in all the required fields without a hitch, but when it comes to the calendar I just cannot figure it out.
The date is in a tbody/tr/td element and xpath doesn't seem to identify it.
For reference I am using Python 3
Problem:
Date element is found in the following:
<td class="day">25</td>
Xpath does not identify this element accurately and I have no idea how to select the date as it is not labeled
I have tried everything from locating with class/id/name/text/xpath with no success and using Select(driver.find_element....) throws the error that Select cannot be used with div, tr, td.
Attempted to identify with driver.find_element_by_visible_text("") as suggested by online tutorials and this just does not work.
Current Code is as follows:
import selenium
from selenium import webdriver
from time import sleep
import time
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome()
driver.get('http://gobook.co.za/')
time.sleep(2)
User = driver.find_element_by_xpath("//input[#id='UserName']")
User.send_keys('XXXXXXXXX')
time.sleep(1)
Psw = driver.find_element_by_xpath("//input[#id='Password']")
Psw.send_keys('XXXXXXXXX')
time.sleep(1)
Button = driver.find_element_by_xpath("//input[#type='submit']")
Button.click()
time.sleep(2)
Booking = driver.find_element_by_xpath("//a[#href='/Bookings/Client']")
Booking.click()
time.sleep(0.5)
New = driver.find_element_by_xpath("//a[#data-toggle='collapse']")
New.click()
time.sleep(0.5)
Serv = Select(driver.find_element_by_xpath("//select[#id='ServiceId']"))
Serv.select_by_value("6")
time.sleep(0.5)
Prov = Select(driver.find_element_by_xpath("//select[#id='ProviderId']"))
Prov.select_by_value("22")
time.sleep(0.5)
Date = driver.find_element_by_id("BookingDate")
Date.click()
Date.send_keys('/25')
time.sleep(0.5)
Expected results = Select the date that falls on a Sunday (25th of August)

Just created an account and tested it. Would that be convienent for you?
import selenium
from selenium import webdriver
from time import sleep
import time
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome('drivers/chromedriver72.exe')
driver.get('http://gobook.co.za/')
time.sleep(2)
User = driver.find_element_by_xpath("//input[#id='UserName']")
User.send_keys('testgeo')
time.sleep(1)
Psw = driver.find_element_by_xpath("//input[#id='Password']")
Psw.send_keys('testgeo')
time.sleep(1)
Button = driver.find_element_by_xpath("//input[#type='submit']")
Button.click()
time.sleep(2)
Booking = driver.find_element_by_xpath("//a[#href='/Bookings/Client']")
Booking.click()
time.sleep(0.5)
New = driver.find_element_by_xpath("//a[#data-toggle='collapse']")
New.click()
time.sleep(0.5)
Serv = Select(driver.find_element_by_xpath("//select[#id='ServiceId']"))
Serv.select_by_value("6")
time.sleep(0.5)
Prov = Select(driver.find_element_by_xpath("//select[#id='ProviderId']"))
Prov.select_by_value("22")
time.sleep(0.5)
driver.find_element_by_id("BookingDate").clear()
Date = driver.find_element_by_id("BookingDate").send_keys('2019/08/25')
time.sleep(0.5)