Develop Reference

~FIREFOX~ Selenium Instagram Bot is unable to locate Follow button to follow profiles despite trying By. XPATH, LINK_TEXT, CLASS_NAME etc

I am on VS Code and my Selenium Instagram Bot's intentional design is to read from a list of profiles from a .txt file, visit those profiles, follow and like a specified number of their posts(if they are private, it just follows them) then goes on to the next profile in the list, all the while using different pre-made bot accounts who's usernames are also on a list, so the code may iterate over them once a number of profiles have been engaged with by a single bot.
I am able to iterate over target profiles, but right now I am just having problems with locating elements and having them to be clicked by the bot. I got it to work on 1 profile, after going to the next profile, it simply didn't do anything and seems to can't find the follow button to click again(I can't recreate this, after some changes were made lol, just getting back into Python after briefly touching on it in school). I still haven't even seen the bot like a post too. Although, the XPATHS on the Log In and the Pop Ups seems to work. It's now just not interacting with the profiles.
~
Any insights would be highly appreciated!
Source Code:
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time, random
from selenium.webdriver.common.keys import Keys
profilePath = (r'C:\Users\****\AppData\Roaming\Mozilla\Firefox\Profiles\75d4lwz2.3rd')
options = Options()
service = Service('geckodrivere.exe')
firefox = webdriver.Firefox(options=options, service=service)
wait = WebDriverWait(firefox, 20)
file = open('scrape_archivepages.txt', 'r')
data = file.read()
igUsers = data.split('\n')
file.close()
file2 = open('botlist.txt', 'r')
data2 = file2.read()
bots = data2.split('\n')
file2.close()
def startLogIn(user_, pass_,):
firefox.get('https://www.instagram.com/')
while True:
try:
cookiesAccept = firefox.find_element(By.XPATH, '/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/button[2]')
time.sleep(4)
cookiesAccept.click()
time.sleep(4)
break
except:
pass
username = firefox.find_element(By.XPATH, '//*[#id="loginForm"]/div/div[1]/div/label/input')
password = firefox.find_element(By.XPATH, '//*[#id="loginForm"]/div/div[2]/div/label/input')
username.click()
username.send_keys(user_)
time.sleep(random.randint(1, 2))
password.click()
password.send_keys(pass_)
time.sleep(random.randint(1, 2))
log_in=firefox.find_element(By.XPATH, '//*[#id="loginForm"]/div/div[3]')
log_in.click()
time.sleep(5)
# while True:
# try:
# credentials= firefox.find_element(By.XPATH, '//button[text()="Not Now"]')
# time.sleep(3)
# credentials.click()
# break
# except:
# pass
# while True:
# try:
# notifications = firefox.find_element(By.XPATH, '//button[text()="Not Now"]')
# time.sleep(3)
# notifications.click()
# break
# except:
# pass
def interact(igUserLink, n):
firefox.get(igUserLink)
time.sleep(2)
#while True:
# try:
follow = firefox.find_element(By.CSS_SELECTOR, '#mount_0_0_0I > div > div > div > div.x9f619.x1n2onr6.x1ja2u2z > div > div > div > div.x78zum5.xdt5ytf.x10cihs4.x1t2pt76.x1n2onr6.x1ja2u2z > div.x9f619.xnz67gz.x78zum5.x168nmei.x13lgxp2.x5pf9jr.xo71vjh.x1uhb9sk.x1plvlek.xryxfnj.x1c4vz4f.x2lah0s.x1q0g3np.xqjyukv.x1qjc9v5.x1oa3qoh.x1qughib > div.xh8yej3.x1gryazu.x10o80wk.x14k21rp.x1porb0y.x17snn68.x6osk4m > section > main > div > header > section > div.x6s0dn4.x78zum5.x1q0g3np.xs83m0k.xeuugli.x1n2onr6 > div._ab8w._ab94._ab99._ab9f._ab9k._ab9p._abb3._abcm > div > div._ab8w._ab94._ab99._ab9f._ab9m._ab9o._abb0._abcm > button > div > div')
time.sleep(2)
private = firefox.find_element(By.XPATH, '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/section/main/div/div/article/div[1]/div/h2')
if(bool(private)):
print('lol')
follow.click()
time.sleep(2)
##xpath of header from IG saying profile is private
if (not(bool(private))):
print('here')
follow.click()
time.sleep(2)
time.sleep(2)
c = 0
numPosts = firefox.find_element(By.XPATH, '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/div[1]/div[2]/section/main/div/header/section/ul/li[1]/div/span/span')
numPosts = int(numPosts.text)
if n == 0:
#do nothing
if n <= numPosts:
media = firefox.find_element(By.XPATH, '/html/body/div[2]/div/div/div/div[1]/div/div/div/div[1]/div[1]/div[2]/section/main/div/div[2]/article/div/div/div[1]/div[1]')
media.click()
time.sleep(1)
like = firefox.find_element(By.NAME, 'Like')
next = firefox.find_element(By.NAME, 'Next')
while(c<n):
like.click()
time.sleep(3)
c=c+1
next.click()
#break
#except:
#pass
def VibeFinderInteract(listOfBots, passw, userLink):
for userbot in listOfBots:
startLogIn(userbot, passw)
for u in userLink:
interact(u, 2)
print('')
VibeFinderInteract(bots, 'samepasswordforallthebots', igUsers)

How to solve invisible captcha on Selenium

Hi I am trying to write a sign-up bot for UEFA.com using Selenium as requests I find to be too difficult for me to try so I am just working on automating the sign-up process even if it is a lot slower.
I am able to get to the final stage where I click on Create an Account, but faced with a reCaptcha which only appears after clicking on Create an Account. And after solving the captcha there is no 'Submit' button but it will automatically submit the details for you.
I am able to get the captcha token returned from 2captcha solving service, and inputted it into the innerHTML of the g-response-token field using javascript. However I do not know how to submit the captcha and the form.
import requests
import time
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from seleniumwire import webdriver
import pyautogui
from twocaptcha import TwoCaptcha
import random
import os
from random import randint
import sys
firstnames = []
lastnames = []
API_Key = '6LehfZUbAAAAAJhue_6BVqqxLulLiXLP0rEgpdRH'
# Open Names File
with open('firstnames.txt', 'r') as f:
for name in f:
name = name.strip()
firstnames.append(name)
with open('lastnames.txt', 'r') as e:
for name in e:
name = name.strip()
lastnames.append(name)
with open('proxies.txt') as f:
proxy = f.readlines()
proxy_rand = randint(1, 35)
s_proxy = str(proxy[proxy_rand])
p_strip = s_proxy.rstrip()
# Proxy Input and Format
bare_proxy = p_strip.split(':')
username = bare_proxy[2]
password = bare_proxy[3]
ip = bare_proxy[0]
port = bare_proxy[1]
options = {
'proxy': {
'http': f'http://{username}:{password}#{ip}:{port}',
'https': f'https://{username}:{password}#{ip}:{port}',
'no_proxy': 'localhost,127.0.0.1'
}
}
os.environ['PATH'] += 'C:/SeleniumDrivers'
homepage_URL = 'https://www.uefa.com/tickets/'
driver = webdriver.Chrome(seleniumwire_options=options)
driver.get(homepage_URL)
# Accessing Register Page
reject_cookies = driver.find_element(By.ID, 'onetrust-reject-all-handler')
reject_cookies.click()
time.sleep(1)
login_button = driver.find_element(By.CSS_SELECTOR, "a[class='btn btn-secondary tickets__btn js-tracking-card']")
login_button.click()
time.sleep(10)
create_account = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[4]/a')
create_account.click()
time.sleep(10)
# Inputting SignUp Details
letters = 'abcdefghijklmnopqrstuvwxyz'
a = random.choice(letters)
b = random.choice(letters)
c = random.choice(letters)
d = random.choice(letters)
email = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[6]/input')
email.send_keys(f'{a}{b}{c}{d}#nottingham.pro')
time.sleep(2)
password = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[7]/input')
password.send_keys('19741002Rw!')
time.sleep(2)
first_name = driver.find_element(By.XPATH, '//*[#id="gigya-textbox-130722358975432270"]')
first_range = len(firstnames) - 1
random_first = randint(1, first_range)
f_name = firstnames[random_first]
first_name.send_keys(f'{f_name}')
time.sleep(2)
last_name = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[9]/input')
last_range = len(lastnames) - 1
random_last = randint(1, first_range)
l_name = lastnames[random_last]
last_name.send_keys(f'{l_name}')
time.sleep(2)
day_of_birth = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[10]/div[1]/input')
day = randint(1, 28)
day_of_birth.send_keys(f'{day}')
time.sleep(2)
month_of_birth = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[10]/div[2]/input')
month = randint(1, 12)
month_of_birth.send_keys(f'{month}')
time.sleep(2)
year_of_birth = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div/form/div[1]/div[10]/div[3]/input')
year = randint(1940, 2000)
year_of_birth.send_keys(f'{year}')
driver.execute_script("window.scrollTo(0, 500)")
time.sleep(2)
pyautogui.moveTo(353, 619)
time.sleep(2)
pyautogui.click()
time.sleep(5)
current_url = driver.current_url
print(current_url)
g_key = '6LehfZUbAAAAAJhue_6BVqqxLulLiXLP0rEgpdRH'
def SolveCaptcha():
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
api_key = os.getenv(g_key, 'a733edea49a8327795d56edc9f06d391')
solver = TwoCaptcha(api_key)
try:
result = solver.recaptcha(
sitekey=g_key,
url=current_url)
except Exception as e:
print(e)
else:
return result
result = SolveCaptcha()
code = result['code']
print(code)
token = f'document.getElementById("g-recaptcha-response").innerHTML="{code}";'
driver.execute_script(token)
time.sleep(10000)
As you can see by the end of the code I have managed to input the captcha token but not sure how to submit as there is no submit button
I have tried to look for a callback function but can't seem to find it when I inspect the page.

submit the first form on the page:
driver.execute_script('document.forms[0].submit()')

How to stop selenium scraper from redirecting to another internal weblink of the scraped website?

Was wondering if anyone knows of a way for instructing a selenium script to avoid visiting/redirecting to an internal page that wasn't part of the code. Essentially, my code opens up this page:
https://cryptwerk.com/companies/?coins=1,6,11,2,3,8,17,7,13,4,25,29,24,32,9,38,15,30,43,42,41,12,40,44,20
keeps clicking on show more button until there's none (at end of page) - which by then - it should have collected the links of all the products listed on the page it scrolled through till the end, then visit each one respectively.
What happens instead, it successfully clicks on show more till the end of the page, but then it visits this weird promotion page of the same website instead of following each of the gathered links respectively and then scraping further data points located off each of those newly opened ones.
In a nutshell, would incredibly appreciate it if someone can explain how to avoid this automated redirection on its own! And this is the code in case someone can gratefully nudge me in the right direction :)
from selenium.webdriver import Chrome
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
import json
import selenium.common.exceptions as exception
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
webdriver = '/Users/karimnabil/projects/selenium_js/chromedriver-1'
driver = Chrome(webdriver)
driver.implicitly_wait(5)
url = 'https://cryptwerk.com/companies/?coins=1,6,11,2,3,8,17,7,13,4,25,29,24,32,9,38,15,30,43,42,41,12,40,44,20'
driver.get(url)
links_list = []
coins_list = []
all_names = []
all_cryptos = []
all_links = []
all_twitter = []
all_locations = []
all_categories = []
all_categories2 = []
wait = WebDriverWait(driver, 2)
sign_in = driver.find_element_by_xpath("//li[#class='nav-item nav-guest']/a")
sign_in.click()
time.sleep(2)
user_name = wait.until(EC.presence_of_element_located((By.XPATH, "//input[#name='login']")))
user_name.send_keys("karimnsaber95#gmail.com")
password = wait.until(EC.presence_of_element_located((By.XPATH, "//input[#name='password']")))
password.send_keys("PleomaxCW#2")
signIn_Leave = driver.find_element_by_xpath("//div[#class='form-group text-center']/button")
signIn_Leave.click()
time.sleep(3)
while True:
try:
loadMoreButton = driver.find_element_by_xpath("//button[#class='btn btn-outline-primary']")
time.sleep(2)
loadMoreButton.click()
time.sleep(2)
except exception.StaleElementReferenceException:
print('stale element')
break
print('no more elements to show')
try:
company_links = driver.find_elements_by_xpath("//div[#class='companies-list items-infinity']/div[position() > 3]/div[#class='media-body']/div[#class='title']/a")
for link in company_links:
links_list.append(link.get_attribute('href'))
except:
pass
try:
with open("links_list.json", "w") as f:
json.dump(links_list, f)
with open("links_list.json", "r") as f:
links_list = json.load(f)
except:
pass
try:
for link in links_list:
driver.get(link)
name = driver.find_element_by_xpath("//div[#class='title']/h1").text
try:
show_more_coins = driver.find_element_by_xpath("//a[#data-original-title='Show more']")
show_more_coins.click()
time.sleep(1)
except:
pass
try:
categories = driver.find_elements_by_xpath("//div[contains(#class, 'categories-list')]/a")
categories_list = []
for category in categories:
categories_list.append(category.text)
except:
pass
try:
top_page_categories = driver.find_elements_by_xpath("//ol[#class='breadcrumb']/li/a")
top_page_categories_list = []
for category in top_page_categories:
top_page_categories_list.append(category.text)
except:
pass
coins_links = driver.find_elements_by_xpath("//div[contains(#class, 'company-coins')]/a")
all_coins = []
for coin in coins_links:
all_coins.append(coin.get_attribute('href'))
try:
location = driver.find_element_by_xpath("//div[#class='addresses mt-3']/div/div/div/div/a").text
except:
pass
try:
twitter = driver.find_element_by_xpath("//div[#class='links mt-2']/a[2]").get_attribute('href')
except:
pass
try:
print('-----------')
print('Company name is: {}'.format(name))
print('Potential Categories are: {}'.format(categories_list))
print('Potential top page categories are: {}'.format(top_page_categories_list))
print('Supporting Crypto is:{}'.format(all_coins))
print('Registered location is: {}'.format(location))
print('Company twitter profile is: {}'.format(twitter))
time.sleep(1)
except:
pass
all_names.append(name)
all_categories.append(categories_list)
all_categories2.append(top_page_categories_list)
all_cryptos.append(all_coins)
all_twitter.append(twitter)
all_locations.append(location)
except:
pass
df = pd.DataFrame(list(zip(all_names, all_categories, all_categories2, all_cryptos, all_twitter, all_locations)), columns=['Company name', 'Categories1', 'Categories2', 'Supporting Crypto', 'Twitter Handle', 'Registered Location'])
CryptoWerk_Data = df.to_csv('CryptoWerk4.csv', index=False)

Redirect calls happen for two reasons, in your case either by executing some javascript code when clicking the last time on the load more button or by receiving an HTTP 3xx code, which is the least likely in your case.
So you need to identify when this javascript code is executed and send an ESC_KEY before it loads and then executing the rest of your script.
You could also scrape the links and append them to your list before clicking the load more button and each time it is clicked, make an if statement the verify the link of the page you're in, if it is that of the promotion page then execute the rest of your code, else click load more.
while page_is_same:
scrape_elements_add_to_list()
click_load_more()
verify_current_page_link()
if current_link_is_same != link_of_scraped_page:
page_is_same = False
# rest of the code here

InvalidArgumentException: Message: invalid argument: 'using' must be a string

im very new to python, trying to create reusable code. when i try to call the class Login and function login_user in test_main.py by passing all the arguments that were used under Login class, im getting an error as InvalidArgumentException: Message: invalid argument: 'using' must be a string.
test_main.py file which runs on pytest.
Locators_test is the class of test_Locators.py file where i have all my xpaths
test_Locators.py
class Locators_test():
loginlink_xpath = "//a[#id='login-link']"
login_email = "xxxxx"
login_password = "xxxxx"
loginemail_id = "dnn_ctr1179_Login_txtEmail"
loginpassword_id = "dnn_ctr1179_Login_txtPassword"
clicklogin_id = "dnn_ctr1179_Login_btnLogin"
test_login.py
from Smoketest.locatorfile.test_Locators import Locators_test
class Login():
def __init__(self,driver):
self.driver = driver
def login_user(self,driver):
try:
loginButton = self.driver.find_element((By.XPATH, Locators_test.loginlink_xpath))
while loginButton.click() is True:
break
time.sleep(3)
self.driver.execute_script("window.scrollBy(0,300);")
EmailField = self.driver.find_element((By.ID, Locators_test.loginemail_id))
EmailField.send_keys(Locators_test.login_email)
PasswordField = self.driver.find_element((By.ID, Locators_test.loginpassword_id))
PasswordField.send_keys(Locators_test.login_password)
ClickLogin = self.driver.find_element((By.ID, Locators_test.clicklogin_id))
while ClickLogin.click() is True:
break
time.sleep(5)
userName = self.driver.find_element((By.XPATH, Locators_test.username_xpath))
print("Logged in as", userName.text)
except StaleElementReferenceException or ElementClickInterceptedException or TimeoutException as ex:
print(ex.message)
test_main.py
def test_setup():
driver = webdriver.Chrome(executable_path= Locators_test.browser_path)
driver.maximize_window()
driver.delete_all_cookies()
driver.get(homePage)
driver.implicitly_wait(5)
yield
print("test complete")
def test_login(test_setup):
from Smoketest.pages.test_login import Login
lo = Login(driver)
lo.login_user(((Locators_test.loginlink_xpath,Locators_test.loginemail_id,Locators_test.login_email,Locators_test.loginpassword_id,Locators_test.login_password,Locators_test.clicklogin_id,Locators_test.username_xpath)))
indentations are all fine

I fixed it myself by removing the extra pair of parenthesis from the line
loginButton = self.driver.find_element((By.XPATH, Locators_test.loginlink_xpath))
Right way is
loginButton = self.driver.find_element(By.XPATH, Locators_test.loginlink_xpath)
ps: this applies to all the lines.

This worked for me,
locator = (By.XPATH, Locators_test.loginlink_xpath)
self.driver.find_element(*locator).click()
Explanation: In *<arguments>, all positional arguments other than the first one will be packed in a tuple, as they won't get Changed, the exact property will be reflected in the second step.

AttributeError: 'unicode' object has no attribute 'sleep'

I am getting AttributeError: 'unicode' object has no attribute 'sleep' as specified in the title of this question and I cannot figure out why it is throwing that error message. this is automation script which will automatic post facebook. if someone know what's wrong with my code my tell me. code is here
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import traceback
import string
import xlrd
#taking inputs from user-----------------------------------
delay=input('Set Time: ')
limit=input('Set Post Limit: ')
valuee=limit
count=0
start=0
end=valuee
#----------------------------------------------------------
#open file for taking multiple accounts-------------------
f=open('Accounts.csv')
data=f.readlines()
i=0
for value in data:
values=value.split(',')
password=values[1]
user_name=values[0]
#Condition for next URLs--------------------------
if count>=1:
start=limit
end=start + valuee
limit=end
count+=1
#-----------------------------------------------
chrome_options = webdriver.ChromeOptions() #going to chrome options
chrome_options.add_argument("--start-maximized")
prefs = {"profile.default_content_setting_values.notifications" : 2 #turn off all notifications
,"profile.managed_default_content_settings.images": 2} #disable images
chrome_options.add_experimental_option("prefs",prefs)
driver = webdriver.Chrome(chrome_options=chrome_options) # passing paramaters to chrome
#----------------------------------------------------------------------------------------------
try:
driver.get('https://www.facebook.com/')
time.sleep(2)
#puting userName and password---------------------------------------------------------
driver.find_element_by_css_selector('#email').send_keys(user_name)
time.sleep(1)
driver.find_element_by_css_selector('#pass').send_keys(password, Keys.RETURN)
time.sleep(2)
driver.get('https://www.facebook.com/pages/?category=your_pages')
driver.find_element_by_css_selector('._1vgt.ellipsis._349g a').click()
pageUrl=driver.current_url
raw_input('>>')
except:
traceback.print_exc()
raw_input('Something Wrong..! please hit Enter >>\n')
driver.quit()
continue
#file for report-----
report=open('Report/'+user_name+'.csv','w')
#reading file of excel--------------------------------------------------------------------
workbook = xlrd.open_workbook('links.xlsx')
sheet = workbook.sheet_by_index(0)
#print commands:
print '\n******************** Auto Posting Start ********************\n'
#print 'Using Proxy:',proxy
print 'Using Email:',user_name
for row in range(start, end):
driver.get(pageUrl)
time.sleep(2)
try:
#title=value=sheet.cell(row, 0).value
disc=value=sheet.cell(row, 0).value
try:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/%s);" % 5)
time.sleep(2)
try:
driver.find_element_by_css_selector('._1hib._4bl9').click()
time.sleep(4)
except:
pass
try:
driver.find_element_by_css_selector('.notranslate._5rpu').send_keys(disc)
time.sleep(3)
except:
driver.find_element_by_css_selector('._4h98.navigationFocus').send_keys(disc)
time.sleep(3)
driver.find_element_by_css_selector('._1mf7._4jy0._4jy3._4jy1._51sy.selected._42ft').click()
time.sleep(6)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(4)
#get latest post url---------------------------
for elem in driver.find_elements_by_css_selector('._5pcq'):
try:
tim=elem.find_element_by_css_selector('._5ptz.timestamp.livetimestamp')
time=tim.text
if time=='Just now':
print'great'
href=elem.get_attribute('href')
print href
raw_input('>>')
break
else:
continue
except:
continue
print 'Report Url:',href
report.write(href+'\n')
except:
traceback.print_exc()
pass
time.sleep(delay)
except:
traceback.print_exc()
print'Urls Completed!'
break
report.close()
driver.quit()
print'Thanks For Using..'

This is because you overwrote time with a string in below line
tim=elem.find_element_by_css_selector('._5ptz.timestamp.livetimestamp')
time=tim.text
if time=='Just now':
So time is no more a module it is just a unicode text. You need to rename that variable to something else