I'm trying to make a script that logs into my online grade book to look for any changes (new grades, etc). This is my code so far.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
def main():
options = Options()
# options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(10)
driver.quit()
if __name__ == '__main__':
main()
And there is a problem, I cannot seem to find a way to make webdriver see what I see. What I mean is that I see the webpage like this and webdriver sees the same webpage like this also the source code is different. I've tried using undetected ChromeDriver with no success. This is my code using UC.
import undetected_chromedriver as uc
import time
from selenium.webdriver.common.by import By
def main():
driver = uc.Chrome()
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina/home')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(5)
driver.execute_script("window.print();")
if __name__ == '__main__':
main()
Has anyone had a similar problem and managed to solve it?
Related
I'd like to open a chrome window with my default profile using selenium. I'm using a mac.
This is the code I have so far.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
import time
options = Options()
options.add_argument('user-data-dir=/Users/USERNAME/Library/Application Support/Google/Chrome')
options.add_argument('profile-directory=Default')
##options.add_argument("user-data-dir=/Users/USERNAME/Library/Application Support/Google/Chrome/Default")
driver = webdriver.Chrome(chrome_options=options)
driver.get('https://www.selenium.dev/')
## find element using xpath
l = driver.find_element('xpath', '//a[#href="/documentation/webdriver/"]')
## click button
driver.execute_script("arguments[0].click();", l);
## print resultant page title
print("Page title is: ")
print(driver.title)
When I remove line 8, which is
options.add_argument('user-data-dir=/Users/USERNAME/Library/Application Support/Google/Chrome')
the code works just fine, but the chrome window isn't signed in. When I add that line, the chrome window opens logged in, but doesn't go to any website.
Any help would be appreciated!
I am currently working on a scraper for aniworld.to.
My goal is it to enter the anime name and get all of the Episodes downloaded.
I have everything working except one thing...
The websites has a Watch button. That Button redirects you to https://aniworld.to/redirect/SOMETHING and that Site has a captcha which means the link is not in the html...
Is there a way to bypass this/get the link in python? Or a way to display the captcha so I can solve it?
Because the captcha only appears every lightyear.
The only thing I need from that page is the redirect link. It looks like this:
https://vidoza.net/embed-something.html
My very very wip code is here if it helps: https://github.com/wolfswolke/aniworld_scraper
Mitchdu showed me how to do it.
If anyone else needs help here is my code: https://github.com/wolfswolke/aniworld_scraper/blob/main/src/logic/captcha.py
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from threading import Thread
import os
def open_captcha_window(full_url):
working_dir = os.getcwd()
path_to_ublock = r'{}\extensions\ublock'.format(working_dir)
options = webdriver.ChromeOptions()
options.add_argument("app=" + full_url)
options.add_argument("window-size=423,705")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
if os.path.exists(path_to_ublock):
options.add_argument('load-extension=' + path_to_ublock)
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.get(full_url)
wait = WebDriverWait(driver, 100, 0.3)
wait.until(lambda redirect: redirect.current_url != full_url)
new_page = driver.current_url
Thread(target=threaded_driver_close, args=(driver,)).start()
return new_page
def threaded_driver_close(driver):
driver.close()
So I am trying to scrape a website, and I want to click an element, go to the page that opens from the click, find another element and click that one. The first click seems to work, no errors, but the next page doesn't open, thus I get an error. Here is a screenshot of what I want to click on the fist page: https://prnt.sc/10l8xa4. Clicking that should redirect to the second page. The problem seems to be that the driver clicks the element but nothing happens:
import sys, csv, os
from selenium import webdriver # Selenium 3.141.0
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from datetime import datetime
from time import sleep
class Scraper(object):
''' A lot of the messy code is just playing with the tags from the page'''
def __init__(self, link):
self.link = link
self.driver = self.configure_driver() # The simulated browser
# Configuring the browser simulator, named driver, that will get all the information
def configure_driver(self):
# Add additional Options to the webdriver
chrome_options = Options()
# add the argument and make the browser Headless. It will work smoother& faster but it will miss the first category
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options = chrome_options)
return driver
def click_element(self, selector): # Clicks the provided element from the page, even if not visible
element = WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, selector)))
ActionChains(self.driver).move_to_element(element).click(element).perform()
if __name__ == '__main__':
product_link = 'https://www.action.com/nl-nl/click-and-collect-producten/' # An example of a product
app = Scraper(product_link)
with app.driver:
app.driver.get(product_link)
app.click_element('a.content-card.has-text.card-theme--light.card-size--s.card-align--bottom-left') # This gets clicked and should open new page, but it doesn't
sleep(10)
app.click_element('a.product-card__link') # This throws a Timeout, because the element can't be found, which is obvoius because the second page(which has this element) didn't open
sleep(20)
Try use like that:
with app.driver:
app.driver.get(product_link)
sleep(2)
app.click_element('li.has-submenu')
sleep(2)
app.click_element(
'div.grid-item.grid-item--content') # This gets clicked and should open new page, but it doesn't
sleep(2)
you should add step of openning pop-up and then click on your aimed button
Code with required argument but, by xpath:
def click_element(self, selector, by=By.CSS_SELECTOR): # Clicks the provided element from the page, even if not visible
element = WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((by, selector)))
ActionChains(self.driver).move_to_element(element).click(element).perform()
if __name__ == '__main__':
product_link = 'https://www.action.com/nl-nl/click-and-collect-producten/' # An example of a product
app = Scraper(product_link)
with app.driver:
app.driver.get(product_link)
sleep(2)
app.click_element("//section[#class='grid']/div[#class='grid-item grid-item--content'][1]", By.XPATH)
sleep(2)
I am trying to locate the 'Like' and 'Share' button on facebook pages with xpath, however after the facebook page loads, the python script execution is complete and the like button is not clicked.
I got the like button xpath from facebook page.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
def connect():
driver.set_page_load_timeout("10")
driver.get("https://www.facebook.com/mytheoryofevolution/")
time.sleep(4)
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
driver = webdriver.Chrome(executable_path=r"C:\Users\javed\PycharmProjects\click-website\drivers\chromedriver.exe",chrome_options=chrome_options)
connect()
like_buttons = driver.find_elements_by_xpath('//a[contains(#class,"likeButton")]')
for like in like_buttons:
print(like)
break
tYou don't need
break
because when it will start the for loop you will get the first item and then the loop will break
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
driver = webdriver.Chrome(executable_path=r"C:\Users\javed\PycharmProjects\click-website\drivers\chromedriver.exe",chrome_options=chrome_options)
def connect():
driver.set_page_load_timeout("10")
driver.get("https://www.facebook.com/mytheoryofevolution/")
time.sleep(4)
like_buttons = driver.find_elements_by_xpath('//a[contains(#class,"likeButton")]')
for like in like_buttons:
print(like)
connect()
As a test, I am trying to create a script that goes to my website and clicks on the learn more button, but am having trouble actually automatically clicking the button.
I've tried everything that I've found on stack overflow but nothing has worked.
from selenium import webdriver
import webbrowser
import time
url = 'https://www.mwstan.com'
driver = webbrowser.open_new_tab(url)
element = driver.find_element_by_id('learnmore')
element.click()
You are going to need to install a binary for whatever driver you are going to use
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_driver = os.getcwd() + "/chromedriver"
def get_url_example(url):
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=chrome_driver)
driver.get(url)
button = driver.find_element_by_id("learnmore")
button.click()
# you can access the page source here using driver.page_source
if __name__ == '__main__':
get_url_page_source("https://www.mwstan.com")
This code works for me and hits your button.
This is using chrome webdriver but you can use another webdriver. JUst makesure you move the driver and access the path correctly like in line
chrome_driver = os.getcwd() + "/chromedriver"