Hi I want to click on a button and then go to next page using selenium python. but instead it get close doesnot show the next tab at all
import time
from Base_Class import *
class Majestic_Domain:
def capture_domain():
web_driver_conn = base_class.web_driver()
web_driver_conn.get("https://tranco-list.eu/")
CSV_Download = web_driver_conn.find_element(By.XPATH, '//body/main/div/div/a')
time.sleep(10)
CSV_Download.click()
#web_driver_conn.execute_script("arguments[0].click();", CSV_Download)
Majestic_Domain.capture_domain()
This is the code I used
Base class
def web_driver():
driver_conn = webdriver.Chrome()
return driver_conn
Add some wait time after the click() action:
def capture_domain():
web_driver_conn = base_class.web_driver()
web_driver_conn.get("https://tranco-list.eu/")
CSV_Download = web_driver_conn.find_element(By.XPATH, '//body/main/div/div/a')
time.sleep(1)
CSV_Download.click()
time.sleep(5)
Related
Here is the code I am using to click next button the problem is after the first page is loded it closes the browser rather than clicking on the next button until it disappears. (I know it is html website but I am learning Playwright so starting light.)
I am using get_by_text() function, I have used this loop to achieve similar results but with selenium python.
Any suggestion how to make this happen?
with sync_playwright() as p:
browser = p.firefox.launch(headless=False)
page = browser.new_page()
page.goto("https://books.toscrape.com/")
while True:
try:
next = page.get_by_text("Next") ## next clicker
next.click()
except:
break
Maybe if you put a break in each loop:
from playwright.sync_api import sync_playwright
from time import sleep
with sync_playwright() as p:
browser = p.firefox.launch(headless=False)
page = browser.new_page()
page.goto("https://books.toscrape.com/")
while True:
try:
next = page.get_by_text("Next") # next clicker
next.click()
sleep(2)
except Exception:
break
I am trying to scrape a website that populates a list of providers. the site makes you go through a list of options and then finally it populates a list of providers through a pop up that has an endless/continuous scroll.
i have tried:
from selenium.webdriver.common.action_chains import ActionChains
element = driver.find_element_by_id("my-id")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
but this code didn't work.
I tried something similar to this:
driver.execute_script("arguments[0].scrollIntoView();", list )
but this didnt move anything. it just stayed on the first 20 providers.
i tried this alternative:
main = driver.find_element_by_id('mainDiv')
recentList = main.find_elements_by_class_name('nameBold')
for list in recentList :
driver.execute_script("arguments[0].scrollIntoView(true);", list)
time.sleep(20)
but ended up with this error message:
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
The code that worked the best was this one:
while True:
# Scroll down to bottom
element_inside_popup = driver.find_element_by_xpath('//*[#id="mainDiv"]')
element_inside_popup.send_keys(Keys.END)
# Wait to load page
time.sleep(3)
but this is an endless scroll that i dont know how to stop since "while True:" will always be true.
Any help with this would be great and thanks in advance.
This is my code so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
import pandas as pd
PATH = '/Users/AnthemScraper/venv/chromedriver'
driver = webdriver.Chrome(PATH)
#location for the website
driver.get('https://shop.anthem.com/sales/eox/abc/ca/en/shop/plans/medical/snq?execution=e1s13')
print(driver.title)
#entering the zipcode
search = driver.find_element_by_id('demographics.zip5')
search.send_keys(90210)
#making the scraper sleep for 5 seconds while the page loads
time.sleep(5)
#entering first name and DOB then hitting next
search = driver.find_element_by_id('demographics.applicants0.firstName')
search.send_keys('juelz')
search = driver.find_element_by_id('demographics.applicants0.dob')
search.send_keys('01011990')
driver.find_element_by_xpath('//*[#id="button/shop/getaquote/next"]').click()
#hitting the next button
driver.find_element_by_xpath('//*[#id="hypertext/shop/estimatesavings/skipthisstep"]').click()
#making the scraper sleep for 2 seconds while the page loads
time.sleep(2)
#clicking the no option to view all the health plans
driver.find_element_by_xpath('//*[#id="radioNoID"]').click()
driver.find_element_by_xpath('/html/body/div[4]/div[11]/div/button[2]/span').click()
#making the scraper sleep for 2 seconds while the page loads
time.sleep(2)
driver.find_element_by_xpath('//*[#id="hypertext/shop/medical/showmemydoctorlink"]/span').click()
time.sleep(2)
#section to choose the specialist. here we are choosing all
find_specialist=\
driver.find_element_by_xpath('//*[#id="specializedin"]')
#this is the method for a dropdown
select_provider = Select(find_specialist)
select_provider.select_by_visible_text('All Specialties')
#choosing the distance. Here we click on 50 miles
choose_mile_radius=\
driver.find_element_by_xpath('//*[#id="distanceInMiles"]')
select_provider = Select(choose_mile_radius)
select_provider.select_by_visible_text('50 miles')
driver.find_element_by_xpath('/html/body/div[4]/div[11]/div/button[2]/span').click()
#handling the endless scroll
while True:
time.sleep(20)
# Scroll down to bottom
element_inside_popup = driver.find_element_by_xpath('//*[#id="mainDiv"]')
element_inside_popup.send_keys(Keys.END)
# Wait to load page
time.sleep(3)
#block below allows us to grab the majority of the data. we would have to split it up in pandas since this info
#is nested in with classes
time.sleep(5)
main = driver.find_element_by_id('mainDiv')
sections = main.find_elements_by_class_name('firstRow')
pcp_info = []
#print(section.text)
for pcp in sections:
#the site stores the information inside inner classes which make it difficult to scrape.
#the solution would be to pull the entire text in the block and hope to clean it aftewards
#innerText allows to pull just the text inside the blocks
first_blox = pcp.find_element_by_class_name('table_content_colone').get_attribute('innerText')
second_blox = pcp.find_element_by_class_name('table_content_coltwo').get_attribute('innerText')
#creating columns and rows and assigning them
pcp_items = {
'first_block' : [first_blox],
'second_block' : [second_blox]
}
pcp_info.append(pcp_items)
df = pd.DataFrame(pcp_info)
print(df)
df.to_csv('yerp.csv',index=False)
#driver.quit()
from selenium import webdriver
from time import sleep
filename = "log.txt"
myfile = open(filename, 'w')
class Search(object):
def __init__(self):
self.driver = webdriver.Chrome('chromedriver.exe')
# "This will open a new chrome instance without being logged in to the site"
self.driver.get("Site2")
sleep(2)
self.driver.find_element_by_xpath("/html/body/div[1]/div[4]/div/div/div[3]/div/div/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[2]/div[1]/a").click()
sleep(2)
Texto = self.driver.find_element_by_xpath("/html/body/div[1]/div[4]/div/div/div[4]/div/div/div[1]/div[3]/div/article[1]/div/div[2]/div/div/div/article/div[1]").text
print(Texto)
myfile.write(Texto)
myfile.close()
sleep(10)
import re
Id = re.compile('[0-9]{9}')
Id2 = re.compile('[0-9]{4}')
DobMonth = re.compile('[0-9]{2}')
DobYear = re.compile('[0-9]{2}')
if Id.match(Texto) and Id2.match(Texto) and DobMonth.match(Texto) and DobYear.match(Texto):
print ("Matches")
else:
print("Not match")
sleep (20)
Search()
class BasicBot(object):
def __init__(self, username, pw):
self.driver = webdriver.Chrome('chromedriver.exe')
self.driver.get("site1")
sleep(2)
self.driver.find_element_by_xpath("/html/body/div[1]/div[1]/div/div/div/div[1]/a[1]/span").click()
sleep(2)
self.driver.find_element_by_xpath("//input[#name=\"login\"]")\
.send_keys(username)
self.driver.find_element_by_xpath("//input[#name=\"password\"]")\
.send_keys(pw)
self.driver.find_element_by_xpath('/html/body/div[4]/div/div[2]/div/form/div[1]/dl/dd/div/div[2]/button').click()
sleep(2)
Search()
BasicBot('username',"password")
So the script runs BasicBot(usr,psswd) logs in to the site, and is supposed to go to a different site while logged in, then searches if X post matches the criteria given, if it does that's it if it does not, I should refresh the site and check again.
In the Search class, you start with creating a new chromedriver instance: self.driver = webdriver.Chrome('chromedriver.exe'), this is why it opens another window with a fresh state.
Instead, modify your Search class to
take an existing instance of webdriver.
open a new window by using script window.open instead of get:
class Search:
def __init__(self, driver):
self.driver = driver
# Open a new window
self.driver.execute_script("window.open('https://site2')")
sleep(2) # visually check to make sure it opened
# Switch to new window
self.driver.switch_to.window(self.driver.window_handles[-1])
# After this you can start doing self.driver.find_element_by_xpath and the rest
In the class BasicBot, modify the last line to pass the driver:
Search(self.driver)
Lastly, you can use refresh to refresh your site2:
else:
print("Not match")
sleep(20)
self.driver.refresh()
Hope, it helps. Good luck!
I am trying to scrape the Google News page in the following way:
from selenium import webdriver
import time
from pprint import pprint
base_url = 'https://www.google.com/'
driver = webdriver.Chrome('/home/vincent/wintergreen/chromedriver') ## change here to your location of the chromedriver
driver.implicitly_wait(30)
driver.get(base_url)
input = driver.find_element_by_id('lst-ib')
input.send_keys("brexit key dates timetable schedule briefing")
click = driver.find_element_by_name('btnK')
click.click()
news = driver.find_element_by_link_text('News')
news.click()
tools = driver.find_element_by_link_text('Tools')
tools.click()
time.sleep(1)
recent = driver.find_element_by_css_selector('div.hdtb-mn-hd[aria-label=Recent]')
recent.click()
# custom = driver.find_element_by_link_text('Custom range...')
custom = driver.find_element_by_css_selector('li#cdr_opt span')
custom.click()
from_ = driver.find_element_by_css_selector('input#cdr_min')
from_.send_keys("9/1/2018")
to_ = driver.find_element_by_css_selector('input#cdr_max')
to_.send_keys("9/2/2018")
time.sleep(1)
go_ = driver.find_element_by_css_selector('form input[type="submit"]')
print(go_)
pprint(dir(go_))
pprint(go_.__dict__)
go_.click()
This script manage to enter search terms, switch to the news tab, open the custom time period tab, fill in start and end date, but fails to click on the 'Go' button after that point.
From the print and pprint statement at the end of the script, I can deduct that it does find the 'go' button succesfully, but is somehow unable to click on it. The error displays as selenium.common.exceptions.ElementNotVisibleException: Message: element not visible
Could anyone experienced with Selenium have a quick run at it and give me hints as why it returns such error?
Thx!
Evaluating the css using developer tools in chrome yields 4 elements.
Click here for the image
use the following css instead:
go_ = driver.find_element_by_css_selector('#cdr_frm > input.ksb.mini.cdr_go')
I am making an automated python script which opens chromedriver on a loop until it finds a specific element on the webpage (using selenium) the driver gets. This obviously eats up recourses eventually as it is constantly opening and closing the driver while on the loop.
Is there a way to use an existing chromedriver window instead of just opening and closing on a loop until a conditional is satisfied?
If that is not possible is there an alternative way to go about this you would reccomend?
Thanks!
Script:
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
import pyautogui
import time
import os
def snkrs():
driver = webdriver.Chrome('/Users/me/Desktop/Random/chromedriver')
driver.get('https://www.nike.com/launch/?s=in-stock')
time.sleep(3)
pyautogui.click(184,451)
pyautogui.click(184,451)
current = driver.current_url
driver.get(current)
time.sleep(3.5)
elem = driver.find_element_by_xpath("//* .
[#id='j_s17368440']/div[2]/aside/div[1]/h1")
ihtml = elem.get_attribute('innerHTML')
if ihtml == 'MOON RACER':
os.system("clear")
print("SNKR has not dropped")
time.sleep(1)
else:
print("SNKR has dropped")
pyautogui.click(1303,380)
pyautogui.hotkey('command', 't')
pyautogui.typewrite('python3 messages.py') # Notifies me by text
pyautogui.press('return')
pyautogui.click(928,248)
pyautogui.hotkey('ctrl', 'z') # Kills the bash loop
snkrs()
Bash loop file:
#!/bin/bash
while [ 1 ]
do
python snkrs.py
done
You are defining a method that contains the chromedriver launch and then running through the method once (not looping) so each method call generates a new browser instance. Instead of doing that, do something more like this...
url = 'https://www.nike.com/launch/?s=in-stock'
driver.get(url)
# toggle grid view
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[aria-label='Show Products as List']"))).click();
# wait for shoes to drop
while not driver.find_elements((By.XPATH, "//div[#class='figcaption-content']//h3[contains(.,'MOON RACER')]"))
print("SNKR has not dropped")
time.sleep(300) // 300s = 5 mins, don't spam their site
driver.get(url)
print("SNKR has dropped")
I simplified your code, changed the locator, and added a loop. The script launches a browser (once), loads the site, clicks the grid view toggle button, and then looks for the desired shoe to be displayed in this list. If the shoes don't exist, it just sleeps for 5 mins, reloads the page, and tries again. There's no need to refresh the page every 1s. You're going to draw attention to yourself and the shoes aren't going to be refreshed on the site that often anyway.
If you're just trying to wait until something changes on the page then this should do the trick:
snkr_has_not_dropped = True
while snkr_has_not_dropped:
elem = driver.find_element_by_xpath("//* .[ # id = 'j_s17368440'] / div[2] / aside / div[1] / h1")
ihtml = elem.get_attribute('innerHTML')
if ihtml == 'MOON RACER':
print("SNKR has not dropped")
driver.refresh()
else:
print("SNKR has dropped")
snkr_has_not_dropped = False
Just need to refresh the page and try again.