I'm trying to webscrape images from the web using selenium. On Linux there do not seem to be any problems however when running on windows the program seems to exit on line 18.
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.firefox.service import Service
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.by import By
import os
import requests
import base64
def webscrape(query, max_count, q):
options = Options()
options.add_argument('-headless')
# Add a GH_token to the environment so there are no limitations on the number of requests.
os.environ['GH_TOKEN'] = "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"
gecko = GeckoDriverManager().install()
driver = webdriver.Firefox(service=Service(gecko), options=options)
driver.get("https://www.google.com/search?q=" + query + "&tbm=isch")
print("test")
the test print doesn't happen. But all print statements before the driver.get call do.
Related
I'm trying to change my code to use an IE headless browser. The automation I'm doing is in a website that only works in internet explorer
My code was working great until I tried to use a headless browser
When I run this code, absolutely nothing happens, no error is thrown
# selenium 4
from selenium import webdriver
from selenium.webdriver.ie.service import Service
from webdriver_manager.microsoft import IEDriverManager
from selenium.webdriver.ie.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import StaleElementReferenceException
from dotenv import load_dotenv
# Inicialização do Selenium
ie_options = Options()
ie_options.ignore_zoom_level = True
## WORKS!
# driver = webdriver.Ie(service=Service(IEDriverManager().install()), options=ie_options)
## NOT WORKING
service = Service(executable_path=constantes.PATH_HEADLESS)
driver = webdriver.Ie(service=service, options=ie_options)
# Acessa a página
driver.get(constantes.URL)
I believe the reason why it seems nothing is happening is because you have no output (not printing anything). I'm not familiar with your process, but I tried it out with mine also using chrome and it worked fine. Context:
chrome_options = Options()
chrome_options.add_argument("--headless")
driver=webdriver.Chrome(service=Service('*executable
path'),options=chrome_options)
driver.get('https://stackoverflow.com/')
print(driver.title)
Selenium does not find the accept cookies button.
Tested: xpath, class and css
Error
Command
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import pandas as pd
import csv
options = Options()
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
navegador = webdriver.Chrome(options=options)
navegador.get('https://app-vlc.hotmart.com/market/search?categoryId=25&page=1&userLanguage=PT_BR')
navegador.implicitly_wait(30)
sleep(30)
navegador.find_element(By.CSS_SELECTOR, ".cookie-policy-accept-all.hot-button.hot-button--primary").click()
navegador.implicitly_wait(30)
elem=navegador.find_element(By.XPATH,"//div[#id='hotmart-cookie-policy']").shadow_root
elem.find_element(By.CSS_SELECTOR, ".cookie-policy-accept-all.hot-button.hot-button--primary").click()
You need to find the shadow root and then find from there.
Since the above didn't work try this one.
navegador.get('https://app-vlc.hotmart.com/market/search?categoryId=25&page=1&userLanguage=PT_BR')
time.sleep(10)
elem=navegador.find_element(By.XPATH,"//div[#id='hotmart-cookie-policy']")
script='''return arguments[0].shadowRoot.querySelector(".cookie-policy-accept-all.hot-button.hot-button--primary")'''
elem1= navegador.execute_script(script, elem)
elem1.click()
import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
#Open browser
driver = webdriver.Chrome(ChromeDriverManager().install())
url='https://www.premierleague.com/players'
driver.get(url)
This the code I have. It only opens for a second and then closes again. Does anyone know why?
I am assuming the problem is with the chrome driver manager, but I can't figure out how to fix it.
You can use time.sleep(10) to keep the window open. I'm using 10 as an example for the number of seconds.
I'm new to selenium I am trying to copy something from one page to another, the page that I copy off of already has it so if you just click on the text once it copies automatically but it is not copying anything I am not sure why
chrome_driver = '/Applications/chromedriver'
driver = webdriver.Chrome(chrome_driver)
driver.get('https://tempail.com/en/')
time.sleep(5)
driver.find_element_by_xpath('//*[#id="eposta_adres"]').click()
driver.get('https://www.instagram.com/')
driver.find_element_by_xpath('//*[#id="user_first_name"]').send_keys('Scott')
driver.find_element_by_xpath('//*[#id="user_email"]').click()
act.key_down(Keys.META).send_key("COMMAND + v").key_up(Keys.META).perform()
On my Linux in Firefox and Chrome works
item.send_keys(Keys.CONTROL, "v")
Like
item = driver.find_element_by_xpath('//*[#id="user_email"]')
item.send_keys(Keys.CONTROL, "v")
I tried to test it with your code but you use xpath which I can't find on Instagam
So I tested with field Search... on current page
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
#from webdriver_manager.firefox import GeckoDriverManager
import time
driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
#driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
driver.get('https://tempail.com/en/')
time.sleep(5)
driver.find_element_by_xpath('//*[#id="eposta_adres"]').click()
driver.get('https://stackoverflow.com/questions/71543113/copy-and-paste-in-selenium-python/')
item = driver.find_element_by_xpath('//*[#name="q"]')
item.send_keys(Keys.CONTROL, "v")
BTW: I described this long time ago in
Selenium: How to send clipboad to field in browser — furas.pl.
I show also how to use module pyperclip to work with clipboard.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
#from webdriver_manager.firefox import GeckoDriverManager
import time
import pyperclip
driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
#driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
driver.get('https://stackoverflow.com/questions/71543113/copy-and-paste-in-selenium-python/')
item = driver.find_element_by_xpath('//*[#name="q"]')
#text = pyperclip.paste() # get text from cliboard
#item.clear()
#item.send_keys(text)
pyperclip.copy("Hello World") # put text in clipboard
item.send_keys(Keys.CONTROL, "v")
I have a javascript which I want to put in console when page loaded but i am unable to open console using selenium in python I can access devtoos using this --auto-open-devtools-for-tabs but not able to open console.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException
import os
import time
from urllib.parse import quote_plus
driverPath = "chromedriver.exe"
dataPath = "whatsapp-assistant-bot-master/Data"
options = webdriver.ChromeOptions()
options.add_argument("--user-data-dir=" + dataPath)
options.add_argument("--auto-open-console-for-tabs=" + dataPath)
driver = webdriver.Chrome(options=options, executable_path=driverPath)
driver.get('http://google.com')
print('Please Scan the QR Code and press enter')
driver.find_element_by_id("gsr").send_keys(Keys.F12)
input()
driver.quit()
Answered here, you don't have to open the console, you can run your js code through selenium
Running javascript in Selenium using Python