I'm trying to log in to facebook using requests module. Although it seems I've prepared payload in the right way but when I send it with post requests, I don't get desired content in the response. I get 200 status code, though. To let you know, If I get response accordingly, I should find my fullname within it.
I initially tried like the following:
import requests
from bs4 import BeautifulSoup
from datetime import datetime
link = 'https://www.facebook.com/'
base_url = 'https://www.facebook.com{}'
time = int(datetime.now().timestamp())
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36',
'referer': 'https://www.facebook.com/',
}
with requests.Session() as s:
r = s.get(link)
soup = BeautifulSoup(r.text,"lxml")
form_url = soup.select_one("form[data-testid='royal_login_form']")['action']
post_url = base_url.format(form_url)
payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
payload['email'] = 'YOUR_EMAIL'
payload['encpass'] = f'#PWD_BROWSER:0:{time}:YOUR_PASSWORD'
payload.pop('pass')
res = s.post(post_url,data=payload,headers=headers)
print(res.url)
print(res.text)
This is another way I tried which didn't work out either:
import requests
from bs4 import BeautifulSoup
from datetime import datetime
login_url = 'https://www.facebook.com/login/device-based/regular/login/?login_attempt=1&lwv=101'
time = int(datetime.now().timestamp())
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'origin': 'https://www.facebook.com',
'referer': 'https://www.facebook.com/login/device-based/regular/login/?login_attempt=1&lwv=101'
}
with requests.Session() as s:
r = s.get(login_url)
soup = BeautifulSoup(r.text,"lxml")
payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
payload['email'] = 'YOUR_EMAIL'
payload['encpass'] = f'#PWD_BROWSER:0:{time}:YOUR_PASSWORD'
payload['had_password_prefilled'] = 'true'
payload['had_cp_prefilled'] = 'true'
payload['prefill_source'] = 'browser_dropdown'
payload['prefill_type'] = 'contact_point'
payload['first_prefill_source'] = 'last_login'
payload['first_prefill_type'] = 'contact_point'
payload['prefill_contact_point'] = 'YOUR_EMAIL'
payload.pop('pass')
r = s.post(login_url,data=payload,headers=headers)
print(r.status_code)
print(r.url)
How can I log in to facebook using requests?
This might be a case of xy problem
I recommend trying Selenium in accessing Facebook programmatically.
More examples using Selenium in logging in.
https://www.askpython.com/python/examples/python-automate-facebook-login
https://www.guru99.com/facebook-login-using-python.html
If logging in is all that you require, then using selenium, you could do it as:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
URL = 'https://www.facebook.com/'
PATH = r'C:\Program Files (x86)\chromedriver.exe'
driver = webdriver.Chrome(PATH)
driver.get(URL)
email = driver.find_element_by_id('email')
email.send_keys('YourEmail')
password = driver.find_element_by_id('pass')
password.send_keys('YourPassword')
password.send_keys(Keys.RETURN)
I would recommend that you use the browser that you frequently use to login for this process.
Related
There is a FIPS website with all registered patents. It works like this: a form is filled in, which determines in which databases the search will take place at https://www.fips.ru/iiss/db.xhtml , then after the post request and pressing submit "go to search" it goes to https: / /www.fips.ru/iiss/search.xhtml with a new form and a new submit. Below is my code
import requests
from bs4 import BeautifulSoup
post_url = 'https://new.fips.ru/iiss/db.xhtml'
with requests.Session() as s:
s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
r = s.get(post_url)
soup = BeautifulSoup(r.text,"lxml")
payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
payload['db-selection-form:dbsGrid1:0:dbsGrid1checkbox'] = 'on'
payload['db-selection-form:j_idt90'] = 'перейти к поиску'
r = s.post(post_url,data=payload)
print(r.text)
The problem is that after my actions in the answer, it again returns https://www.fips.ru/iiss/db.xhtml , and not https://www.fips.ru/iiss/search.xhtml , as it happens in the browser. How can I fix this behavior for what I need?
I have a static .aspx url that I am trying to scrape. All of my attempts yield the raw html data of the regular website instead of the data I am querying.
My understanding is the headers I am using (which I found from another post) are correct and generalizable:
import urllib.request
from bs4 import BeautifulSoup
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17',
'Content-Type': 'application/x-www-form-urlencoded',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
}
class MyOpener(urllib.request.FancyURLopener):
version = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17'
myopener = MyOpener()
url = 'https://www.mytaxcollector.com/trSearch.aspx'
# first HTTP request without form data
f = myopener.open(url)
soup_dummy = BeautifulSoup(f,"html5lib")
# parse and retrieve two vital form values
viewstate = soup_dummy.select("#__VIEWSTATE")[0]['value']
viewstategen = soup_dummy.select("#__VIEWSTATEGENERATOR")[0]['value']
Trying to enter the form data causes nothing to happen:
formData = (
('__VIEWSTATE', viewstate),
('__VIEWSTATEGENERATOR', viewstategen),
('ctl00_contentHolder_trSearchCharactersAPN', '631091430000'),
('__EVENTTARGET', 'ct100$MainContent$calculate')
)
encodedFields = urllib.parse.urlencode(formData)
# second HTTP request with form data
f = myopener.open(url, encodedFields)
soup = BeautifulSoup(f,"html5lib")
trans_emissions = soup.find("span", id="ctl00_MainContent_transEmissions")
print(trans_emissions.text)
This give raw html code almost exactly the same as the "soup_dummy" variable. But what I want to see is the data of the field ('ctl00_contentHolder_trSearchCharactersAPN', '631091430000') being submitted (this is the "parcel number" box.
I would really appreciate the help. If anything, linking me to a good post about HTML requests (one that not only explains but actually walks through scraping aspx) would be great.
To get the result using the parcel number, your parameters have to be somewhat different from what you have already tried with. Moreover, you have to use this url https://www.mytaxcollector.com/trSearchProcess.aspx to send the post requests.
Working code:
from urllib.request import Request, urlopen
from urllib.parse import urlencode
from bs4 import BeautifulSoup
url = 'https://www.mytaxcollector.com/trSearchProcess.aspx'
payload = {
'hidRedirect': '',
'hidGotoEstimate': '',
'txtStreetNumber': '',
'txtStreetName': '',
'cboStreetTag': '(Any Street Tag)',
'cboCommunity': '(Any City)',
'txtParcelNumber': '0108301010000', #your search term
'txtPropertyID': '',
'ctl00$contentHolder$cmdSearch': 'Search'
}
data = urlencode(payload)
data = data.encode('ascii')
req = Request(url,data)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36')
res = urlopen(req)
soup = BeautifulSoup(res.read(),'html.parser')
for items in soup.select("table.propInfoTable tr"):
data = [item.get_text(strip=True) for item in items.select("td")]
print(data)
import requests
from bs4 import BeautifulSoup as soup
import webbrowser
from urllib.request import Request, urlopen
import requests
import urllib
import re
import time
import math ## i know some of the imports are unnecessary, but i just copy paste them everywhere
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}
j=0
proxies = {'http': "http://213.6.68.78:53281",
'https': "https://213.6.68.78:53281"}
login_data = {
'username': 'my_nickname420',
'password': 'password1234',
'phone': ''
}
with requests.Session() as s:
while(1):
try:
s.proxies = proxies
break
except:
print('error in connecting to proxy... ')
url = "https://www.TOPSECRETSITE.com/login"
r = s.get(url)
page_soup = soup(r.content, 'html5lib')
login_data['_csrf'] = page_soup.find('input', attrs={'name': '_csrf'}) ['value']
r = s.post(url, data=login_data, headers=headers)
print(r.content)
When I run the code, it gives me an output but somehow I'm still not connected to site, it shows me as guest user.
Here is the things you see when you click inspect element after logging(normally from browser)
*Form Data
username: my_nickname420
password: password1234
_csrf: m4beGyaG-CGYiSd2PHSjt3yFyHBsK1YP4GbI
phone:
There is also some "set cookie" things in above this, if that helps.
I want to login to the website by requests sessions.
For example:
https://turbobit.net
But can't login normally, the code is as follows:
# coding=utf-8
import lxml
import re
import requests
import socket
import socks
import sys
import time
from bs4 import BeautifulSoup
from urllib.request import urlopen
url = 'https://turbobit.net/user/login'
header = {
'Host': 'turbobit.net',
'Referer': 'https://turbobit.net/user/login',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3673.0 Safari/537.36'
}
form_data = {
'user[login]': 'your_email',
'user[pass]': 'your_password'
}
session = requests.session()
login = session.post(url, data=form_data, headers=header)
page = session.get('https://turbobit.net/').content
soup = BeautifulSoup(page, features='html.parser')
msgs = soup.find('div', {"class": 'logged'})
for msg in msgs:
print(msg.get_text())
BeautifulSoup does not interact with web application in realtime, It just takes data as input (string/ byte-string) and parses it as properly formatted HTML.
If you want to simulate interactions with web applications such as clicking, entering text and logging in. You should try other options like selenium which is a Browser Automation Framework and it comes very handy in such cases.
Here's an example to perform automated login via python script
First of all find out the name of the inputs used on the websites form for usernames
<form ... name=username ... />
and passwords
<form ... name=password ... />
and replace them in the script below. Also replace the URL to point at the desired site to log into.
Example code: login.py
#!/usr/bin/env python
import requests
payload = { 'username': 'user#email.com', 'password': 'sup3rs3cretp4ssw0rd' }
url = 'https://example.com/login.html'
requests.post(url, data=payload, verify=False)
Bonus:
To run this script from the command line on a UNIX based system place it in a directory, i.e. home/scripts and add this directory to your path in ~/.bash_profile or a similar file used by the terminal.
# Custom scripts export
CUSTOM_SCRIPTS=home/scripts
export PATH=$CUSTOM_SCRIPTS:$PATH
Then create a link to this python script inside home/scripts/login.py
ln -s ~/home/scripts/login.py ~/home/scripts/login
Close your terminal, start a new one, run login.
I have successfully logged in, the code is as follows:
# coding=utf-8
import lxml
import re
import requests
import socket
import socks
import sys
import time
from bs4 import BeautifulSoup
from urllib.request import urlopen
from requests import Request, Session
email = "your_email"
password = "yor_password"
s = requests.Session()
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3673.0 Safari/537.36"
headers = {
'Authority':'turbobit.net',
'Method':'POST',
'Path':'/lang/en',
'Host': 'turbobit.net',
'Referer': 'https://turbobit.net/login',
'User-Agent': user_agent
}
def login_site(email, password):
login_url = 'https://turbobit.net/user/login'
form_data = {
'user[login]': email,
'user[pass]': password,
'user[submit]':'Sign in',
'user[memory]':'on'
}
login = s.post(login_url, data=form_data, headers=headers)
# print(f"text = {login.text}")
soup = BeautifulSoup(login.text, "lxml")
'''If the login is successful, there will be "<div class="lang-links in">" in the output.'''
msgs = soup.find('div', {"class": "user-menu"})
print(msgs)
login_site(email, password)
Thanks you for your help!
Referring to this post: Unable to log in to Amazon using Python
I tried using the suggested answer, but still cannot login.
I added code to display what the result is. It's inputting the email into the box, but I'm still seeing "Enter a valid email" in the result. I'm pretty sure I selected the form correctly, and the name's of the input fields are correct, but need a little guidance to debug this.
import bs4, requests
import os
import webbrowser
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36'
}
from bs4 import BeautifulSoup
with requests.Session() as s:
s.headers = headers
r = s.get('https://www.amazon.com/ap/signin?_encoding=UTF8&ignoreAuthState=1&openid.assoc_handle=usflex&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.mode=checkid_setup&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&openid.ns.pape=http%3A%2F%2Fspecs.openid.net%2Fextensions%2Fpape%2F1.0&openid.pape.max_auth_age=0&openid.return_to=https%3A%2F%2Fwww.amazon.com%2F%3Fref_%3Dnav_signin&switch_account=')
soup = BeautifulSoup(r.content, "html.parser")
signin_data = {s["name"]: s["value"]
for s in soup.select("form[name=signIn]")[0].select("input[name]")
if s.has_attr("value")}
signin_data[u'email'] = 'xx'
signin_data[u'password'] = 'xx'
response = s.post('https://www.amazon.com/ap/signin', data=signin_data)
soup = bs4.BeautifulSoup(response.text, "html.parser")
html = response.content
path = os.path.abspath('temp.html')
url = 'file://' + path
with open(path, 'w') as f:
f.write(str(html))
webbrowser.open(url)
I don't know about BeautifulSoup, but here's how I did it using requests.
from getpass import getpass
import webbrowser
import requests
import os
amazon_username = raw_input("Amazon email: ")
amazon_password = getpass()
headers = {
"User-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36",
"action": "sign-in",
"email": amazon_username,
"password": amazon_password
}
r = requests.get("https://www.amazon.com/gp/sign-in.html", headers=headers)
print(r.status_code)
r = requests.get("https://www.amazon.com/gp/flex/sign-in/select.html", headers=headers)
print(r.status_code)
r = requests.get("https://www.amazon.com/", headers=headers)
print(r.status_code)