I'm trying to make an Instagram bot and it somewhat works but I want it to be able to post pictures, the extension I'm using blocks the page after a few seconds of it being loaded. I found a thread somewhere and someone said if you go into incognito mode it fixes this problem. The problem is I don't know how to make the move the slider with XPath.
Code:
from selenium import webdriver
from time import sleep
from info import pw
from info import email
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_extension('mhh.crx.crx')
chrome_options.add_argument("--disable-extension")
class InstaBot:
def __init__(self, username, pw):
self.username = username
self.driver = webdriver.Chrome("C:/path stuff/chromedriver.exe", chrome_options=chrome_options)
sleep(1)
self.driver.get("chrome://extensions/?id=jkcghkeldalkgfhgenmcblejihijdpha")
sleep(1)
self.driver.find_element_by_xpath("//a[contains(text(), 'crToggle')]")\
.click()
#sleep(2)
self.driver.get("chrome-extension://odlpjhnipdekfkdkadoecooboghijleh/app/index.html")
sleep(2)
self.driver.find_element_by_xpath("//a[contains(text(), 'Log in')]")\
.click()
my_bot = InstaBot('tonka_r_us', pw)
pictures:
https://imgur.com/a/FLfrFxq
To open your Chrome webdriver in incognito mode, best way is to use Options.
options = webdriver.ChromeOptions()
options.add_argument("--incognito")
driver = webdriver.Chrome('..\drivers\chromedriver',options=options)
driver.get("https://stackoverflow.com/")
Related
I am currently working on a scraper for aniworld.to.
My goal is it to enter the anime name and get all of the Episodes downloaded.
I have everything working except one thing...
The websites has a Watch button. That Button redirects you to https://aniworld.to/redirect/SOMETHING and that Site has a captcha which means the link is not in the html...
Is there a way to bypass this/get the link in python? Or a way to display the captcha so I can solve it?
Because the captcha only appears every lightyear.
The only thing I need from that page is the redirect link. It looks like this:
https://vidoza.net/embed-something.html
My very very wip code is here if it helps: https://github.com/wolfswolke/aniworld_scraper
Mitchdu showed me how to do it.
If anyone else needs help here is my code: https://github.com/wolfswolke/aniworld_scraper/blob/main/src/logic/captcha.py
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from threading import Thread
import os
def open_captcha_window(full_url):
working_dir = os.getcwd()
path_to_ublock = r'{}\extensions\ublock'.format(working_dir)
options = webdriver.ChromeOptions()
options.add_argument("app=" + full_url)
options.add_argument("window-size=423,705")
options.add_experimental_option('excludeSwitches', ['enable-logging'])
if os.path.exists(path_to_ublock):
options.add_argument('load-extension=' + path_to_ublock)
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.get(full_url)
wait = WebDriverWait(driver, 100, 0.3)
wait.until(lambda redirect: redirect.current_url != full_url)
new_page = driver.current_url
Thread(target=threaded_driver_close, args=(driver,)).start()
return new_page
def threaded_driver_close(driver):
driver.close()
I'm trying to make a script that logs into my online grade book to look for any changes (new grades, etc). This is my code so far.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
def main():
options = Options()
# options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(10)
driver.quit()
if __name__ == '__main__':
main()
And there is a problem, I cannot seem to find a way to make webdriver see what I see. What I mean is that I see the webpage like this and webdriver sees the same webpage like this also the source code is different. I've tried using undetected ChromeDriver with no success. This is my code using UC.
import undetected_chromedriver as uc
import time
from selenium.webdriver.common.by import By
def main():
driver = uc.Chrome()
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina/home')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(5)
driver.execute_script("window.print();")
if __name__ == '__main__':
main()
Has anyone had a similar problem and managed to solve it?
I have a code for making temp mail automatically but I have a problem.
This is code:
import time
from selenium import webdriver
browser = webdriver.Chrome()
browser.get("https://temp-mail.org/")
time.sleep(10)
browser.close()
The link opens correctly but I can't pass cloudflare.
Also, I see some errors on my console:
Thanks...
Try adding user agent argument in chrome options and set user agent to any value
ops = Options()
ua='me'
ops.add_argument('--user-agent=%s' % ua)
driver=uc.Chrome(executable_path=r"C:\chromedriver.exe",chrome_options=ops)
Alternatively try using undetected-chromedriver
import undetected_chromedriver as uc
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
driver = uc.Chrome(options=options)
driver.get("https://temp-mail.org/")
As a test, I am trying to create a script that goes to my website and clicks on the learn more button, but am having trouble actually automatically clicking the button.
I've tried everything that I've found on stack overflow but nothing has worked.
from selenium import webdriver
import webbrowser
import time
url = 'https://www.mwstan.com'
driver = webbrowser.open_new_tab(url)
element = driver.find_element_by_id('learnmore')
element.click()
You are going to need to install a binary for whatever driver you are going to use
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_driver = os.getcwd() + "/chromedriver"
def get_url_example(url):
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=chrome_driver)
driver.get(url)
button = driver.find_element_by_id("learnmore")
button.click()
# you can access the page source here using driver.page_source
if __name__ == '__main__':
get_url_page_source("https://www.mwstan.com")
This code works for me and hits your button.
This is using chrome webdriver but you can use another webdriver. JUst makesure you move the driver and access the path correctly like in line
chrome_driver = os.getcwd() + "/chromedriver"
I would like to scrape job listings from a Dutch job listings website. However, when I try to open the page with selenium I run into a cookiewall (new GDPR rules). How do I bypass the cookiewall?
import selenium
#launch url
url = "https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO"
# create a new Firefox session
driver = webdriver.Firefox()
driver.implicitly_wait(30)
driver.get(url)
Edit I tried something
import selenium
import pickle
url = "https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO"
driver = webdriver.Firefox()
driver.set_page_load_timeout(20)
driver.get(start_url)
pickle.dump(driver.get_cookies() , open("NVBCookies.pkl","wb"))
after that loading the cookies did not work
for cookie in pickle.load(open("NVBCookies.pkl", "rb")):
driver.add_cookie(cookie)
InvalidCookieDomainException: Message: Cookies may only be set for the current domain (cookiewall.vnumediaonline.nl)
It looks like I don't get the cookies from the cookiewall, correct?
Instead of bypassing why don't you write code to check if it's present then accept it otherwise continue with next operation. Please find below code for more details
import unittest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
class PythonOrgSearch(unittest.TestCase):
def setUp(self):
self.driver = webdriver.Chrome(executable_path="C:\\Users\\USER\\Downloads\\New folder (2)\\chromedriver_win32\\chromedriver.exe")
def test_search_in_python_org(self):
driver = self.driver
driver.get("https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO")
elem = driver.find_element_by_xpath("//div[#class='article__button']//button[#id='form_save']")
elem.click()
def tearDown(self):
self.driver.close()
if __name__ == "__main__":
unittest.main()
driver.find_element_by_xpath('//*[#id="form_save"]').click()
ok I made selenium click the accept button. Also fine by me. Not sure if I'll run into cookiewalls later