Develop Reference

Run Selenium directly on Pydroid3 on Android

my goal is to run a python script that uses Selenium to manipulate the chrome browser, directly with Pydroid 3 app on my android smartphone (without a pc connection).
I followed the instructions at the link https://chromedriver.chromium.org/getting-started/getting-started---android to download the appropriate webdriver and set it correctly for Android.
In detail the version of Chrome on my Smartphone (Samsung S10) is 108.0.5359.128 therefore, following what is indicated here:
https://chromedriver.chromium.org/downloads#h.p_ID_32
I downloaded webdriver "chromedriver_linux64.zip" present at the following link:
https://chromedriver.storage.googleapis.com/index.html?path=108.0.5359.71/
But following this example
https://chromedriver.chromium.org/getting-started/getting-started---android#h.p_ID_390 and running the following Python3 script:
from selenium import webdriver
path_to_chromedriver = os.path.dirname(__file__) + '/chromedriver'
print('path_to_chromedriver: ', path_to_chromedriver)
options = webdriver.ChromeOptions()
options.add_experimental_option('androidPackage', 'com.android.chrome')
driver = webdriver.Chrome(path_to_chromedriver, 0, options=options)
driver.get('https://google.com')
driver.quit()
or this one:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
path_to_chromedriver = os.path.dirname(__file__) + '/chromedriver'
print('path_to_chromedriver: ', path_to_chromedriver)
service = Service(path_to_chromedriver)
options = webdriver.ChromeOptions()
options.add_experimental_option('androidPackage', 'com.android.chrome')
driver = webdriver.Chrome(service=service, options=options)
driver.get('https://google.com')
I get this error:
path_to_chromedriver: /storage/emulated/0/Download/ke-ce-verimme-justwatch-scraper/ke-ce-verimme-justwatch-scraper/scrapers/chromedriver
Traceback (most recent call last):
File "/data/user/0/ru.iiec.pydroid3/files/accomp_files/iiec_run/iiec_run.py", line 31, in <module>
start(fakepyfile,mainpyfile)
File "/data/user/0/ru.iiec.pydroid3/files/accomp_files/iiec_run/iiec_run.py", line 30, in start
exec(open(mainpyfile).read(), __main__.__dict__)
File "<string>", line 124, in <module>
File "/storage/emulated/0/Download/ke-ce-verimme-justwatch-scraper/ke-ce-verimme-justwatch-scraper/scrapers/justwatchScraper.py", line 168, in start
scrolledGenrePage = scrollGenrePageToTheEnd(genrePageURL)
File "/storage/emulated/0/Download/ke-ce-verimme-justwatch-scraper/ke-ce-verimme-justwatch-scraper/scrapers/justwatchScraper.py", line 47, in scrollGenrePageToTheEnd
driver = webdriver.Chrome(service=service, options=options)
File "/data/user/0/ru.iiec.pydroid3/files/aarch64-linux-android/lib/python3.9/site-packages/selenium/webdriver/chrome/webdriver.py", line 81, in __init__
super().__init__(
File "/data/user/0/ru.iiec.pydroid3/files/aarch64-linux-android/lib/python3.9/site-packages/selenium/webdriver/chromium/webdriver.py", line 103, in __init__
self.service.start()
File "/data/user/0/ru.iiec.pydroid3/files/aarch64-linux-android/lib/python3.9/site-packages/selenium/webdriver/common/service.py", line 106, in start
self.assert_process_still_running()
File "/data/user/0/ru.iiec.pydroid3/files/aarch64-linux-android/lib/python3.9/site-packages/selenium/webdriver/common/service.py", line 117, in assert_process_still_running
return_code = self.process.poll()
AttributeError: 'Service' object has no attribute 'process'
[Program finished]
Can anyone help me or suggest a workaround?

"'chromedriver' executable needs to be in PATH" error even though chromedriver is demonstrably in path (executed by typing 'chromedriver' in terminal) [duplicate]

when i run my script , i got this error
Traceback (most recent call last):
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\common\service.py", line 74, in start
stdout=self.log_file, stderr=self.log_file)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\subprocess.py", line 707, in __init__
restore_signals, start_new_session)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\subprocess.py", line 992, in _execute_child
startupinfo)
FileNotFoundError: [WinError 2] The system cannot find the file specified
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/ishaq/AppData/Local/Programs/Python/Python36/headless.py", line 9, in <module>
driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"), chrome_options=chrome_options)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\chrome\webdriver.py", line 62, in __init__
self.service.start()
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\common\service.py", line 81, in start
os.path.basename(self.path), self.start_error_message)
selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home
here is my script
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location =
r'C:\Users\ishaq\Desktop\chrome\chromedriver.exe'
driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"),
chrome_options=chrome_options)
driver.get("http://www.duo.com")
magnifying_glass = driver.find_element_by_id("js-open-icon")
if magnifying_glass.is_displayed():
magnifying_glass.click()
else:
menu_button = driver.find_element_by_css_selector(".menu-trigger.local")
menu_button.click()
search_field = driver.find_element_by_id("site-search")
search_field.clear()
search_field.send_keys("Olabode")
search_field.send_keys(Keys.RETURN)
assert "Looking Back at Android Security in 2016" in driver.page_source
driver.close()

If we analyze the logs it seems the main issue is with in start os.path.basename(self.path) and subsequent error message:
selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH
So it's clear from the error that the Python client was unable to locate the chromedriver executable binary.
You need to take care of a couple of things as follows:
chrome_options.binary_location : The parameter points to the chrome.exe not the chromedriver.exe
os.path.abspath("chromedriver") will pick up the file path of chromedriver but won't append chromedriver.exe at the end.
Here is the sample code on my windows-8 system to start google-chrome-headless:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("http://www.duo.com")
print("Chrome Browser Initialized in Headless Mode")
driver.quit()
print("Driver Exited")

Selenium Cookie Retrieving Warnings

i am using selenium and idle to create and reuse cookies for selenium sicne logging in to some website everytime is a hassle.
sample code i used to save cookies:-
from webserver import keep_alive
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import pickle
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(options=chrome_options)
driver.maximize_window()
driver.get("https://discord.com/login")
time.sleep(120)
pickle.dump( driver.get_cookies() , open("cookies.pkl","wb"))
and after i was able to save cookies in a file i edited the code to retrieve cookies next time i execute
from webserver import keep_alive
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import pickle
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(options=chrome_options)
driver.maximize_window()
driver.get("https://discord.com/login")
cookies = pickle.load(open("cookies.pkl", "rb"))
time.sleep(5)
for cookie in cookies:
driver.add_cookie(cookie)
driver.get("https://discord.com/login")
but it doesnt work as intended giving an error when trying to insert cookies :-
main.py:15: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
username_input = driver.find_element_by_name('email')
main.py:19: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
password_input = driver.find_element_by_name('password')
main.py:23: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
main.py:15: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
username_input = driver.find_element_by_name('email')
main.py:19: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
password_input = driver.find_element_by_name('password')
main.py:23: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
Traceback (most recent call last):
File "main.py", line 19, in <module>
driver.get("discord.com/login")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 436, in get
main.py:15: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
username_input = driver.find_element_by_name('email')
main.py:19: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
password_input = driver.find_element_by_name('password')
main.py:23: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
Traceback (most recent call last):
File "main.py", line 23, in <module>
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 520, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1244, in find_element
return self.execute(Command.FIND_ELEMENT, {
main.py:15: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
username_input = driver.find_element_by_name('email')
main.py:19: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
password_input = driver.find_element_by_name('password')
main.py:23: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
Traceback (most recent call last):
File "main.py", line 23, in <module>
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 520, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1244, in find_element
return self.execute(Command.FIND_ELEMENT, {
main.py:15: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
username_input = driver.find_element_by_name('email')
main.py:19: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead
password_input = driver.find_element_by_name('password')
main.py:23: DeprecationWarning: find_element_by_* commands are deprecated
Traceback (most recent call last):
File "main.py", line 15, in <module>
driver.add_cookie(cookie)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1128, in add_cookie
Traceback (most recent call last):
File "main.py", line 15, in <module>
driver.add_cookie(cookie)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1128, in add_cookie
Traceback (most recent call last):
File "main.py", line 19, in <module>
driver.get("discord.com/login")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 436, in get
Traceback (most recent call last):
File "main.py", line 19, in <module>
driver.get("discord.com/login")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 436, in get
any help will be appreciated

In this line:
driver.get("discord.com/login")
You are passing an invalid parameter to the driver.get method since discord.com/login is not a valid URL.
It should be https://discord.com/login as you yourself use it several lines before.
driver.get("https://discord.com/login")

Unable to run selenium on heroku with python script

I'm trying to run a simple python script that open firefox and search some info on https://biblioteca.aneel.gov.br/.
On my computer the script works, but when i try to run on heroku, it always crashes. I tried all tutorials that I could find, but I don't know what more to do.
I did:
I put as 'Buildpacks' on heroku: https://github.com/pyronlaboratory/heroku-integrated-firefox-geckodriver
In 'Config Vars' on heroku:
FIREFOX_BIN = /app/vendor/firefox/firefox
GECKODRIVER_PATH = /app/vendor/geckodriver/geckodriver
LD_LIBRARY_PATH = /usr/local/lib:/usr/lib:/lib:/app/vendor
PATH = /usr/local/bin:/usr/bin:/bin:/app/vendor/
Adjuste code on heroku:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from bs4 import BeautifulSoup
import re
import yagmail
import time
import os
options = webdriver.FirefoxOptions()
# enable trace level for debugging
options.log.level = "trace"
options.add_argument("-remote-debugging-port=9224")
options.add_argument("-headless")
options.add_argument("-disable-gpu")
options.add_argument("-no-sandbox")
binary = FirefoxBinary(os.environ.get('FIREFOX_BIN'))
rea_inicial='teste'
while True:
driver = webdriver.Firefox(firefox_binary=binary, executable_path=os.environ.get('GECKODRIVER_PATH'), options=options)
#driver.maximize_window()
driver.get("https://biblioteca.aneel.gov.br/Busca/Avancada")
print('entrando no site...')
#time.sleep(5)
# Busca por legislacoo
driver.find_element(By.XPATH, '/html/body/main/div/div/div[2]/div/div/button[2]').click()
wait = WebDriverWait(driver, 30)
print('chegou ate aqui')
Then, the error after deploy on heroku:
2021-11-05T17:40:55.035807+00:00 app[worker.1]: code.py:24: DeprecationWarning: executable_path has been deprecated, please pass in a Service object
2021-11-05T17:40:55.035820+00:00 app[worker.1]: driver = webdriver.Firefox(firefox_binary=binary, executable_path=os.environ.get('GECKODRIVER_PATH'), options=options)
2021-11-05T17:40:55.035825+00:00 app[worker.1]: code.py:24: DeprecationWarning: firefox_binary has been deprecated, please pass in a Service object
2021-11-05T17:40:55.035825+00:00 app[worker.1]: driver = webdriver.Firefox(firefox_binary=binary, executable_path=os.environ.get('GECKODRIVER_PATH'), options=options)
2021-11-05T17:40:55.540760+00:00 app[worker.1]: Traceback (most recent call last):
2021-11-05T17:40:55.540774+00:00 app[worker.1]: File "code.py", line 24, in <module>
2021-11-05T17:40:55.540947+00:00 app[worker.1]: driver = webdriver.Firefox(firefox_binary=binary, executable_path=os.environ.get('GECKODRIVER_PATH'), options=options)
2021-11-05T17:40:55.540949+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.8/site-packages/selenium/webdriver/firefox/webdriver.py", line 175, in __init__
2021-11-05T17:40:55.541113+00:00 app[worker.1]: self.service.start()
2021-11-05T17:40:55.541123+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.8/site-packages/selenium/webdriver/common/service.py", line 101, in start
2021-11-05T17:40:55.541249+00:00 app[worker.1]: self.assert_process_still_running()
2021-11-05T17:40:55.541250+00:00 app[worker.1]: File "/app/.heroku/python/lib/python3.8/site-packages/selenium/webdriver/common/service.py", line 113, in assert_process_still_running
2021-11-05T17:40:55.541359+00:00 app[worker.1]: raise WebDriverException(
2021-11-05T17:40:55.541409+00:00 app[worker.1]: selenium.common.exceptions.WebDriverException: Message: Service /app/vendor/geckodriver/geckodriver unexpectedly exited. Status code was: 64
Can anyone helps me how to solve it ?

The following arguments:
-no-sandbox
-disable-gpu
perhaps are applicable to ChromeDriver / google-chrome combo but Not Applicable to GeckoDriver / firefox combo.
Remove the arguments and re-execute.

selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH error with Headless Chrome

when i run my script , i got this error
Traceback (most recent call last):
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\common\service.py", line 74, in start
stdout=self.log_file, stderr=self.log_file)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\subprocess.py", line 707, in __init__
restore_signals, start_new_session)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\subprocess.py", line 992, in _execute_child
startupinfo)
FileNotFoundError: [WinError 2] The system cannot find the file specified
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/ishaq/AppData/Local/Programs/Python/Python36/headless.py", line 9, in <module>
driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"), chrome_options=chrome_options)
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\chrome\webdriver.py", line 62, in __init__
self.service.start()
File "C:\Users\ishaq\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\common\service.py", line 81, in start
os.path.basename(self.path), self.start_error_message)
selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home
here is my script
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location =
r'C:\Users\ishaq\Desktop\chrome\chromedriver.exe'
driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"),
chrome_options=chrome_options)
driver.get("http://www.duo.com")
magnifying_glass = driver.find_element_by_id("js-open-icon")
if magnifying_glass.is_displayed():
magnifying_glass.click()
else:
menu_button = driver.find_element_by_css_selector(".menu-trigger.local")
menu_button.click()
search_field = driver.find_element_by_id("site-search")
search_field.clear()
search_field.send_keys("Olabode")
search_field.send_keys(Keys.RETURN)
assert "Looking Back at Android Security in 2016" in driver.page_source
driver.close()

If we analyze the logs it seems the main issue is with in start os.path.basename(self.path) and subsequent error message:
selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH
So it's clear from the error that the Python client was unable to locate the chromedriver executable binary.
You need to take care of a couple of things as follows:
chrome_options.binary_location : The parameter points to the chrome.exe not the chromedriver.exe
os.path.abspath("chromedriver") will pick up the file path of chromedriver but won't append chromedriver.exe at the end.
Here is the sample code on my windows-8 system to start google-chrome-headless:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("http://www.duo.com")
print("Chrome Browser Initialized in Headless Mode")
driver.quit()
print("Driver Exited")