I used Selenium to navigate to a URL (i.e. URL_1) with a login/password and provided the login credentials. I'm logged in and the URL (i.e. URL_2) has changed as expected. I don't know how to navigate URL_2 because the driver still refers to URL_1.
Thanks in advance.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
user_name = 'xyz'
password = 'xyz'
def login_process():
driver = webdriver.Firefox()
driver.get("URL_1")
#successfully navigated to URL_1
elem = driver.find_element_by_name("username")
elem.clear()
elem.send_keys(user_name)
elem = driver.find_element_by_name("password")
elem.clear()
elem.send_keys(password)
driver.find_element_by_id("submit").click()
#successfully entered URL_2
def query():
HOW DO I CHANGE THE DRIVER TO URL_2?
#elem = driver.find_element_by_class_name(ticker_box) #this doesn't work, references URL_1 driver
#elem.clear()
#elem.send_keys('xyz')
Instead of having independent functions, create a class with driver instance as an instance variable. Then, use self.driver.get() to navigate to a different URL:
class MyTest(object):
def __init__(self):
self.driver = webdriver.Firefox()
def login_process(self):
self.driver.get("URL_1")
#successfully navigated to URL_1
elem = self.driver.find_element_by_name("username")
elem.clear()
elem.send_keys(user_name)
elem = self.driver.find_element_by_name("password")
elem.clear()
elem.send_keys(password)
self.driver.find_element_by_id("submit").click()
#successfully entered URL_2
def query(self):
self.driver.get("URL2")
# do smth
test = MyTest()
test.login_process()
test.query()
After navigating to the new page if you want to do something on that new page
newURl = driver.window_handles[0]
driver.switch_to.window(newURl)
After doing this you can do something in the new url without getting "no such element exceptions"
First you can assign the url variable as a global:
global url;
url = "firstURL"
At the end of your first function you can change the value of the variable to the new URL:
url = driver.current_url
And then you can get the new url at the beginning of your second function:
driver.get(url)
Related
i keep getting error of "get() missing 1 required positional argument: 'url'" when running following code
import selenium.webdriver as webdriver
def get_results(search_term):
url = "https://www.google.com"
browser = webdriver.Chrome
browser.get(url)
search_box = browser.find_element_by_class_name('gLFyf gsfi')
search_box.send_keys(search_term)
search_box.submit()
try:
links = browser.find_element_by_xpath('//ol[#class="web_regular_results"]//h3//a')
except:
links = browser.find_element_by_xpath('//h3//a')
results = []
for link in links :
href = link.get_attribute('href')
print(href)
results.append(href)
browser.close()
return results
get_results('dog')
the code is supposed to return search results of 'dog' from google, but gets stuck on
browser.get(url)
all help is appreciated
This issue is in the assignment of browser, browser = webdriver.Chrome. It needs to be browser = webdriver.Chrome().
In your code you are not assigning an instance of the chrome webdriver to browser, but the class itself. Thus when you call def get(self, url), your url parameter gets assigned to self and the argument url is not supplied, hence the positional argument error.
You should modify a bit of your code.
Change the import statement:
from selenium import webdriver
Also, you need to create chrome driver instance and provide the path of chromedriver jar file.
driver = webdriver.Chrome(executable_path="C:\\chromedriver.exe")
I have a problem with scraping request list from instagram setting by selenium . Here is my code:
from time import sleep
from selenium import webdriver
#user_info
user = 'xxx'
password = 'xxx'
#login
bot = webdriver.Firefox()
bot.get('https://instagram.com')
sleep(2)
username_input = bot.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[1]/div/label/input')
sleep(1)
username_input.send_keys(user)
sleep(1)
password_input = bot.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[2]/div/label/input')
sleep(1)
password_input.send_keys(password)
sleep(1)
bot.find_element_by_xpath('/html/body/div[1]/section/main/article/div[2]/div[1]/div/form/div/div[3]/button/div').click()
sleep(3)
#try_to_scraping_requested_profile
bot.get('https://instagram.com/accounts/access_tool/current_follow_requests')
sleep(2)
viewmore = 0
while viewmore <4:
bot.find_element_by_xpath('/html/body/div[1]/div/div/section/main/div/article/main/button').click()
viewmore += 1
sleep(1)
#this_is_where_the_problem_happend
requestlist =[]
users = bot.find_elements_by_class_name('-utLF')
print(users.text) #this_line_will_return_nothing
for user in users :
requestlist.append(user.text)
#also_the_requestlist_will_be_empty
print(requestlist)
Since you have mentioned
print(users.text) #this_line_will_return_nothing
users in your case is a list in Python selenium. You can not do .text on a list.
and for this :
for user in users :
requestlist.append(user.text)
#also_the_requestlist_will_be_empty
print(requestlist)
I would suggest first print the size of list :-
print(len(users))
and see if it has something in it. I think it does not have anything, so try a different locator.
I am trying to save the cookies for my selenium automation script in order to avoid login process for future automation.
This is the original script which works fine
class SeleniumDriver(object):
def __init__(
self,
# chromedriver path
driver_path='chromedriver.exe',
# pickle file path to store cookies
cookies_file_path = 'cookies.pkl',
# list of websites to reuse cookies with
cookies_websites=["https://website.com"] # I would like to replace this website by an argument
):
self.driver_path = driver_path
self.cookies_file_path = cookies_file_path
self.cookies_websites = cookies_websites
#PREVIOUS VERSION WITH NORMAL CHROME chrome_options = webdriver.ChromeOptions()
chrome_options = uc.ChromeOptions()
#PREVIOUS VERSION WITH NORMAL CHROME self.driver = webdriver.Chrome(
self.driver = uc.Chrome(
executable_path=self.driver_path,
options=chrome_options
)
try:
# load cookies for given websites
cookies = pickle.load(open(self.cookies_file_path, "rb"))
for website in self.cookies_websites:
self.driver.get(website)
for cookie in cookies:
self.driver.add_cookie(cookie)
self.driver.refresh()
except Exception as e:
# it'll fail for the first time, when cookie file is not present
logger.error(str(e))
logger.error("ERROR : Error loading cookies")
def save_cookies(self):
# save cookies
cookies = self.driver.get_cookies()
pickle.dump(cookies, open(self.cookies_file_path, "wb"))
def close_all(self):
# close all open tabs
if len(self.driver.window_handles) < 1:
return
for window_handle in self.driver.window_handles[:]:
self.driver.switch_to.window(window_handle)
self.driver.close()
def quit(self):
self.save_cookies()
self.close_all()
self.driver.quit()
selenium_object = SeleniumDriver()
driver = selenium_object.driver
You can see in the code during the creation of object, the url of website "https://website.com" is used to create the cookies for this website.
If tomorrow, I use my script for "https://anotherwebsite.com", I would need to pass hthe website as arguments like this for example:
class SeleniumDriver(object,website): # the variable website is passed as argument in class creation
....
website="https://anotherwebsite.com"
selenium_object = SeleniumDriver(website)
My problem is I alsways have this error output:
NameError: name 'website' is not defined
Here is full code which doesn't work:
class SeleniumDriver(object,website):
def __init__(
self,
# chromedriver path
driver_path='chromedriver.exe',
# pickle file path to store cookies
cookies_file_path = 'cookies.pkl',
# list of websites to reuse cookies with
cookies_websites=[website]
):
self.driver_path = driver_path
self.cookies_file_path = cookies_file_path
self.cookies_websites = cookies_websites
#PREVIOUS VERSION WITH NORMAL CHROME chrome_options = webdriver.ChromeOptions()
chrome_options = uc.ChromeOptions()
#PREVIOUS VERSION WITH NORMAL CHROME self.driver = webdriver.Chrome(
self.driver = uc.Chrome(
executable_path=self.driver_path,
options=chrome_options
)
try:
# load cookies for given websites
cookies = pickle.load(open(self.cookies_file_path, "rb"))
for website in self.cookies_websites:
self.driver.get(website)
for cookie in cookies:
self.driver.add_cookie(cookie)
self.driver.refresh()
except Exception as e:
# it'll fail for the first time, when cookie file is not present
logger.error(str(e))
logger.error("ERROR : Error loading cookies")
def save_cookies(self):
# save cookies
cookies = self.driver.get_cookies()
pickle.dump(cookies, open(self.cookies_file_path, "wb"))
def close_all(self):
# close all open tabs
if len(self.driver.window_handles) < 1:
return
for window_handle in self.driver.window_handles[:]:
self.driver.switch_to.window(window_handle)
self.driver.close()
def quit(self):
self.save_cookies()
self.close_all()
self.driver.quit()
website="https://mywebsite.com"
selenium_object = SeleniumDriver(website)
driver = selenium_object.driver
When I run this code, I get this error output:
class SeleniumDriver(object,website):
NameError: name 'website' is not defined
from selenium import webdriver
from time import sleep
filename = "log.txt"
myfile = open(filename, 'w')
class Search(object):
def __init__(self):
self.driver = webdriver.Chrome('chromedriver.exe')
# "This will open a new chrome instance without being logged in to the site"
self.driver.get("Site2")
sleep(2)
self.driver.find_element_by_xpath("/html/body/div[1]/div[4]/div/div/div[3]/div/div/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[2]/div[1]/a").click()
sleep(2)
Texto = self.driver.find_element_by_xpath("/html/body/div[1]/div[4]/div/div/div[4]/div/div/div[1]/div[3]/div/article[1]/div/div[2]/div/div/div/article/div[1]").text
print(Texto)
myfile.write(Texto)
myfile.close()
sleep(10)
import re
Id = re.compile('[0-9]{9}')
Id2 = re.compile('[0-9]{4}')
DobMonth = re.compile('[0-9]{2}')
DobYear = re.compile('[0-9]{2}')
if Id.match(Texto) and Id2.match(Texto) and DobMonth.match(Texto) and DobYear.match(Texto):
print ("Matches")
else:
print("Not match")
sleep (20)
Search()
class BasicBot(object):
def __init__(self, username, pw):
self.driver = webdriver.Chrome('chromedriver.exe')
self.driver.get("site1")
sleep(2)
self.driver.find_element_by_xpath("/html/body/div[1]/div[1]/div/div/div/div[1]/a[1]/span").click()
sleep(2)
self.driver.find_element_by_xpath("//input[#name=\"login\"]")\
.send_keys(username)
self.driver.find_element_by_xpath("//input[#name=\"password\"]")\
.send_keys(pw)
self.driver.find_element_by_xpath('/html/body/div[4]/div/div[2]/div/form/div[1]/dl/dd/div/div[2]/button').click()
sleep(2)
Search()
BasicBot('username',"password")
So the script runs BasicBot(usr,psswd) logs in to the site, and is supposed to go to a different site while logged in, then searches if X post matches the criteria given, if it does that's it if it does not, I should refresh the site and check again.
In the Search class, you start with creating a new chromedriver instance: self.driver = webdriver.Chrome('chromedriver.exe'), this is why it opens another window with a fresh state.
Instead, modify your Search class to
take an existing instance of webdriver.
open a new window by using script window.open instead of get:
class Search:
def __init__(self, driver):
self.driver = driver
# Open a new window
self.driver.execute_script("window.open('https://site2')")
sleep(2) # visually check to make sure it opened
# Switch to new window
self.driver.switch_to.window(self.driver.window_handles[-1])
# After this you can start doing self.driver.find_element_by_xpath and the rest
In the class BasicBot, modify the last line to pass the driver:
Search(self.driver)
Lastly, you can use refresh to refresh your site2:
else:
print("Not match")
sleep(20)
self.driver.refresh()
Hope, it helps. Good luck!
The below request finds the contest id's for the day. I am trying to pass that str into the driver.get url so it will go to each individual contest url and download each contests CSV. I would imagine you have to write a loop but I'm not sure what that would look like with a webdriver.
import time
from selenium import webdriver
import requests
import datetime
req = requests.get('https://www.draftkings.com/lobby/getlivecontests?sport=NBA')
data = req.json()
for ids in data:
contest = ids['id']
driver = webdriver.Chrome() # Optional argument, if not specified will search path.
driver.get('https://www.draftkings.com/account/sitelogin/false?returnurl=%2Flobby');
time.sleep(2) # Let DK Load!
search_box = driver.find_element_by_name('username')
search_box.send_keys('username')
search_box2 = driver.find_element_by_name('password')
search_box2.send_keys('password')
submit_button = driver.find_element_by_xpath('//*[#id="react-mobile-home"]/section/section[2]/div[3]/button/span')
submit_button.click()
time.sleep(2) # Let Page Load, If not it will go to Account!
driver.get('https://www.draftkings.com/contest/exportfullstandingscsv/' + str(contest) + '')
Try in following order:
import time
from selenium import webdriver
import requests
import datetime
req = requests.get('https://www.draftkings.com/lobby/getlivecontests?sport=NBA')
data = req.json()
driver = webdriver.Chrome() # Optional argument, if not specified will search path.
driver.get('https://www.draftkings.com/account/sitelogin/false?returnurl=%2Flobby')
time.sleep(2) # Let DK Load!
search_box = driver.find_element_by_name('username')
search_box.send_keys('Pr0c3ss')
search_box2 = driver.find_element_by_name('password')
search_box2.send_keys('generic1!')
submit_button = driver.find_element_by_xpath('//*[#id="react-mobile-home"]/section/section[2]/div[3]/button/span')
submit_button.click()
time.sleep(2) # Let Page Load, If not it will go to Account!
for ids in data:
contest = ids['id']
driver.get('https://www.draftkings.com/contest/exportfullstandingscsv/' + str(contest) + '')
You do not need to send load selenium for x nos of times to download x nos of files. Requests and selenium can share cookies. This means you can login to site with selenium, retrieve the login details and share them with requests or any other application. Take a moment to check out httpie, https://httpie.org/doc#sessions it seems you manually control sessions like requests does.
For requests look at: http://docs.python-requests.org/en/master/user/advanced/?highlight=sessions
For selenium look at: http://selenium-python.readthedocs.io/navigating.html#cookies
Looking at the Webdriver block,you can add proxies and load the browser headless or live: Just comment the headless line and it should load the browser live, this makes debugging easy, easy to understand movements and changes to site api/html.
import time
from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import requests
import datetime
import shutil
LOGIN = 'https://www.draftkings.com/account/sitelogin/false?returnurl=%2Flobby'
BASE_URL = 'https://www.draftkings.com/contest/exportfullstandingscsv/'
USER = ''
PASS = ''
try:
data = requests.get('https://www.draftkings.com/lobby/getlivecontests?sport=NBA').json()
except BaseException as e:
print(e)
exit()
ids = [str(item['id']) for item in data]
# Webdriver block
driver = webdriver.Chrome()
options.add_argument('headless')
options.add_argument('window-size=800x600')
# options.add_argument('--proxy-server= IP:PORT')
# options.add_argument('--user-agent=' + USER_AGENT)
try:
driver.get(URL)
driver.implicitly_wait(2)
except WebDriverException:
exit()
def login(USER, PASS)
'''
Login to draftkings.
Retrieve authentication/authorization.
http://selenium-python.readthedocs.io/waits.html#implicit-waits
http://selenium-python.readthedocs.io/api.html#module-selenium.common.exceptions
'''
search_box = driver.find_element_by_name('username')
search_box.send_keys(USER)
search_box2 = driver.find_element_by_name('password')
search_box2.send_keys(PASS)
submit_button = driver.find_element_by_xpath('//*[#id="react-mobile-home"]/section/section[2]/div[3]/button/span')
submit_button.click()
driver.implicitly_wait(2)
cookies = driver.get_cookies()
return cookies
site_cookies = login(USER, PASS)
def get_csv_files(id):
'''
get each id and download the file.
'''
session = rq.session()
for cookie in site_cookies:
session.cookies.update(cookies)
try:
_data = session.get(BASE_URL + id)
with open(id + '.csv', 'wb') as f:
shutil.copyfileobj(data.raw, f)
except BaseException:
return
map(get_csv_files, ids)
will this help
for ids in data:
contest = ids['id']
driver.get('https://www.draftkings.com/contest/exportfullstandingscsv/' + str(contest) + '')
May be its time to decompose it a bit.
Create few isolated functions, which are:
0. (optional) Provide authorisation to target url.
1. Collecting all needed id (first part of your code).
2. Exporting CSV for specific id (second part of your code).
3. Loop through list of id and call func #2 for each.
Share chromedriver as input argument for each of them to save driver state and auth-cookies.
Its works fine, make code clear and readable.
I think you can set the URL of a contest to an a element in the landing page, and then click on it. Then repeat the step with other ID.
See my code below.
req = requests.get('https://www.draftkings.com/lobby/getlivecontests?sport=NBA')
data = req.json()
contests = []
for ids in data:
contests.append(ids['id'])
driver = webdriver.Chrome() # Optional argument, if not specified will search path.
driver.get('https://www.draftkings.com/account/sitelogin/false?returnurl=%2Flobby');
time.sleep(2) # Let DK Load!
search_box = driver.find_element_by_name('username')
search_box.send_keys('username')
search_box2 = driver.find_element_by_name('password')
search_box2.send_keys('password')
submit_button = driver.find_element_by_xpath('//*[#id="react-mobile-home"]/section/section[2]/div[3]/button/span')
submit_button.click()
time.sleep(2) # Let Page Load, If not it will go to Account!
for id in contests:
element = driver.find_element_by_css_selector('a')
script1 = "arguments[0].setAttribute('download',arguments[1]);"
driver.execute_script(script1, element, str(id) + '.pdf')
script2 = "arguments[0].setAttribute('href',arguments[1]);"
driver.execute_script(script2, element, 'https://www.draftkings.com/contest/exportfullstandingscsv/' + str(id))
time.sleep(1)
element.click()
time.sleep(3)