I have the following Python script:
from selenium import webdriver
import time
def main():
chrome_options = webdriver.ChromeOptions();
chrome_options.add_argument("user-data-dir=/Users/octavian/Library/Application Support/Google/Chrome/Default");
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get("https://www.facebook.com/")
time.sleep(1000)
main()
I am trying to load my Facebook profile. However, the page that is opened doesn't have my profile(I'm not logged in), which means the browser state is not loaded.
However, my Chrome profile is stored in this file:
/Users/octavian/Library/Application Support/Google/Chrome/Default
Why is the profile not seen by Selenium?
Related
Previously, I use to load and save cookies which is an inconvenient way for most websites. Instead, I use a User Profile to make it run flawlessly.
The problem is I able to load the User Profile but not save it. After I make some changes to the browser such as config addons, changing themes, logging into some websites, etc., exit Firefox via input('exit?') (see the code below), I reopen Firefox with the same User Profile and notice that nothing was saved to the User Profile (I use both methods, open Firefox and select the User Profile (via cmd "Firefox.exe -P"); rerun the script).
Selenium v3.14.0
Python v3.9.7
GeckoDriver v0.30.0 (64-bit)
Firefox v103.0.1 (64-bit)
import os
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
FireFoxPath = r"C:\\Program Files\\Mozilla Firefox\\firefox.exe"
DriverPath = r"BrowserDriver\geckodriver-v0.30.0-win64\geckodriver.exe"
FirefoxProfilePath = os.path.join(os.getcwd(), 'Playground_Temp')
# Initialize
options = webdriver.FirefoxOptions()
options.binary_location = FireFoxPath
options.set_preference('app.update.auto', False)
options.set_preference('app.update.enable', False)
capabilities = DesiredCapabilities.FIREFOX
capabilities["pageLoadStrategy"] = "none"
# Start Firefox
driver = webdriver.Firefox(
webdriver.FirefoxProfile(FirefoxProfilePath),
desired_capabilities=capabilities,
options=options,
executable_path=DriverPath
)
# Wait for user make changes
input('exit?')
# Quit Firefox
driver.quit()
im tryng to load my default chrome profile but instead of opening my profile it open a new istance of chrome
from selenium import webdriver
#object of ChromeOptions class
o = webdriver.ChromeOptions()
#adding Chrome Profile Path
o.add_argument = {'user-data-dir':r'C:\Users\liamm\AppData\Local\Google\Chrome\User Data\Default'}
#set chromedriver.exe path
driver = webdriver.Chrome(executable_path=r"C:\Users\liamm\Desktop\selenium\driver\chromedriver.exe", options=o)
#maximize browser
driver.maximize_window()
#launch URL
driver.get("https://www.tutorialspoint.com/index.htm")
I am trying to acces whatsapp web with python without having to scan the QR code everytime I restart the program (because in my normal browser I also dont have to do that). But how can I do that? Where is the data stored that tells whatsapp web to connect to my phone? And how do I save this data and send it to the browser when I rerun the code?
I already tried this because someone told me I should save the cookies:
from selenium import webdriver
import time
browser = None
cookies = None
def init():
browser = webdriver.Firefox(executable_path=r"C:/Users/Pascal/Desktop/geckodriver.exe")
browser.get("https://web.whatsapp.com/")
time.sleep(5) # in this time I scanned the QR to see if there are cookies
cookies = browser.get_cookies()
print(len(cookies))
print(cookies)
init()
Unfortunately there were no cookies..
The output was 0 and [].
How do I fix this probblem?
As mentioned in the answer to this question, pass your Chrome profile to the Chromedriver in order to avoid this problem. You can do it like this:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = webdriver.ChromeOptions()
options.add_argument("user-data-dir=C:\\Path") #Path to your chrome profile
driver = webdriver.Chrome(executable_path="C:\\Users\\chromedriver.exe", options=options)
This one works for me, I just created a folder, on the home directory of the script and a little modifications and it works perfectly.
###########
E_PROFILE_PATH = "user-data-dir=C:\Users\Denoh\Documents\Project\WhatBOts\SessionSaver"
##################
This is the Config File that I will import later
##################
The main script starts here
##################
from selenium import webdriver
from config import E_PROFILE_PATH
options = webdriver.ChromeOptions()
options.add_argument(E_PROFILE_PATH)
driver = webdriver.Chrome(executable_path='chromedriver_win32_86.0.4240.22\chromedriver.exe', options=options)
driver.get('https://web.whatsapp.com/')
I am trying to scrape LinkedIn, the script was working for 3 months but yesterday it crashed.
I use selenium webdriver, Firefox with fake useragent.
The URL is https://www.linkedin.com/company/my_company/
def init_driver():
"""Initiates selenium webdriver.
:return: Firefox browser instance
"""
try:
# use random UserAgent to avoid captcha
fp = webdriver.FirefoxProfile()
fp.set_preference("general.useragent.override", UserAgent().random)
fp.update_preferences()
# initiate driver
options = FirefoxOptions()
#options.add_argument("--headless")
return webdriver.Firefox(firefox_options=options)
except Exception as e:
logging.error('Exception occurred initiating webdriver', exc_info=True)
And then just open a page driver.get(url)
at this moment it opens it but cannot load
the same situation happens without fake agent and by using chrome.
Has anyone encountered something like this? When I open the link myself everything os ok.
https://www.linkedin.com/authwall?trk=gf&trkInfo=AQFvPeNP8NQIxwAAAXLqc-uI5rnQe1ZIysPcZOgjZCzbrBHZj7q6gd68fPG9NzbX00Rlre_yC0tITChjMDEXSNnD8tZRaMXqcRG-z_3QUMlCvQPR4uVGBQYoSOl3ycoO2E6Jl9w=&originalReferer=&sessionRedirect=https%3A%2F%2Fwww.linkedin.com%2Fcompany%2my_company%2F
Other URLs are opened without problems by the function
This is how you should modify your code.
I modified your code and your code was executed correctly.
from selenium import webdriver
from fake_useragent import UserAgent
import logging
def init_driver():
"""Initiates selenium webdriver.
:return: Firefox browser instance
"""
path = r"your firefox driver path"
try:
# use random UserAgent to avoid captcha
fp = webdriver.FirefoxProfile()
fp.set_preference("general.useragent.override", UserAgent().random)
fp.update_preferences()
# initiate driver
options = webdriver.FirefoxOptions()
# options.add_argument("--headless")
return webdriver.Firefox(firefox_options=options, executable_path=path)
except Exception:
logging.error('Exception occurred initiating webdriver', exc_info=True)
url = "your url"
driver = init_driver()
driver.get(url)
I would like to scrape job listings from a Dutch job listings website. However, when I try to open the page with selenium I run into a cookiewall (new GDPR rules). How do I bypass the cookiewall?
import selenium
#launch url
url = "https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO"
# create a new Firefox session
driver = webdriver.Firefox()
driver.implicitly_wait(30)
driver.get(url)
Edit I tried something
import selenium
import pickle
url = "https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO"
driver = webdriver.Firefox()
driver.set_page_load_timeout(20)
driver.get(start_url)
pickle.dump(driver.get_cookies() , open("NVBCookies.pkl","wb"))
after that loading the cookies did not work
for cookie in pickle.load(open("NVBCookies.pkl", "rb")):
driver.add_cookie(cookie)
InvalidCookieDomainException: Message: Cookies may only be set for the current domain (cookiewall.vnumediaonline.nl)
It looks like I don't get the cookies from the cookiewall, correct?
Instead of bypassing why don't you write code to check if it's present then accept it otherwise continue with next operation. Please find below code for more details
import unittest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
class PythonOrgSearch(unittest.TestCase):
def setUp(self):
self.driver = webdriver.Chrome(executable_path="C:\\Users\\USER\\Downloads\\New folder (2)\\chromedriver_win32\\chromedriver.exe")
def test_search_in_python_org(self):
driver = self.driver
driver.get("https://www.nationalevacaturebank.nl/vacature/zoeken?query=&location=&distance=city&limit=100&sort=relevance&filters%5BcareerLevel%5D%5B%5D=Starter&filters%5BeducationLevel%5D%5B%5D=MBO")
elem = driver.find_element_by_xpath("//div[#class='article__button']//button[#id='form_save']")
elem.click()
def tearDown(self):
self.driver.close()
if __name__ == "__main__":
unittest.main()
driver.find_element_by_xpath('//*[#id="form_save"]').click()
ok I made selenium click the accept button. Also fine by me. Not sure if I'll run into cookiewalls later