Develop Reference

How do I loop this webscrape/tweet script 24/7?

Just started learning Python. I am trying to gather data by webscraping and tweet out info. But everytime I rerun the code. I get
Forbidden: 403 Forbidden
187 - Status is a duplicate.
How do I loop this script without getting this error?
Here's my code :
def scrape ():
page = requests.get("https://www.reuters.com/business/future-of-money/")
soup = BeautifulSoup(page.content, "html.parser")
home = soup.find(class_="editorial-franchise-layout__main__3cLBl")
posts = home.find_all(class_="text__text__1FZLe text__dark-grey__3Ml43 text__inherit-font__1Y8w3 text__inherit-size__1DZJi link__underline_on_hover__2zGL4")
top_post = posts[0].find("h3", class_="text__text__1FZLe text__dark-grey__3Ml43 text__medium__1kbOh text__heading_3__1kDhc heading__base__2T28j heading__heading_3__3aL54 hero-card__title__33EFM").find_all("span")[0].text.strip()
tweet (top_post)
def tweet (top_post):
api_key = 'deletedforprivacy'
api_key_secret = 'deletedforprivacy'
access_token = 'deletedforprivacy'
access_token_secret = 'deletedforprivacy'
authenticator = tweepy.OAuthHandler(api_key, api_key_secret)
authenticator.set_access_token(access_token, access_token_secret)
api = tweepy.API(authenticator, wait_on_rate_limit=True)
api.update_status(f"{top_post} \nSource : https://www.reuters.com/business/future-of-money/")
print(top_post)
scrape()

The twitter api checks if the content is duplicate and if it is duplicate it returns:
Request returned an error: 403 {"detail":"You are not allowed to create a Tweet with duplicate content.","type":"about:blank","title":"Forbidden","status":403}
I added an simple function to check if the previous content is same as the one about to be added
** Full Code**
from requests_oauthlib import OAuth1Session
import os
import json
import requests
from bs4 import BeautifulSoup
import time
user_id = 000000000000000 # Get userid from https://tweeterid.com/
bearer_token = "<BEARER_TOKEN>"
consumer_key = "<CONSUMER_KEY>"
consumer_secret = "<CONSUMER_SECRET>"
def init():
# Get request token
request_token_url = "https://api.twitter.com/oauth/request_token?oauth_callback=oob&x_auth_access_type=write"
oauth = OAuth1Session(consumer_key, client_secret=consumer_secret)
try:
fetch_response = oauth.fetch_request_token(request_token_url)
except ValueError:
print(
"There may have been an issue with the consumer_key or consumer_secret you entered."
)
resource_owner_key = fetch_response.get("oauth_token")
resource_owner_secret = fetch_response.get("oauth_token_secret")
print("Got OAuth token and secret")
# Get authorization
base_authorization_url = "https://api.twitter.com/oauth/authorize"
authorization_url = oauth.authorization_url(base_authorization_url)
print("Please go here and authorize: %s" % authorization_url)
verifier = input("Paste the PIN here: ")
# Get the access token
access_token_url = "https://api.twitter.com/oauth/access_token"
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=resource_owner_key,
resource_owner_secret=resource_owner_secret,
verifier=verifier,
)
oauth_tokens = oauth.fetch_access_token(access_token_url)
access_token = oauth_tokens["oauth_token"]
access_token_secret = oauth_tokens["oauth_token_secret"]
# Make the request
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=access_token,
resource_owner_secret=access_token_secret,
)
scraper(oauth, bearer_token)
def bearer_oauth(r):
"""
Method required by bearer token authentication.
"""
r.headers["Authorization"] = f"Bearer {bearer_token}"
r.headers["User-Agent"] = "v2UserTweetsPython"
return r
def previous_tweet():
url = "https://api.twitter.com/2/users/{}/tweets".format(user_id)
# Tweet fields are adjustable.
# Options include:
# attachments, author_id, context_annotations,
# conversation_id, created_at, entities, geo, id,
# in_reply_to_user_id, lang, non_public_metrics, organic_metrics,
# possibly_sensitive, promoted_metrics, public_metrics, referenced_tweets,
# source, text, and withheld
params = {"tweet.fields": "text"}
response = requests.request(
"GET", url, auth=bearer_oauth, params=params)
print(response.status_code)
if response.status_code != 200:
raise Exception(
"Request returned an error: {} {}".format(
response.status_code, response.text
)
)
# checking if this is the first post
if response.json() != {'meta': {'result_count': 0}}:
# Since twitter changes html to small url I am splitting at \n to match to new payload
previous_tweet_text = response.json()["data"][0]["text"].split("\n")[0]
previous_payload = {"text": f"{previous_tweet_text}"}
else:
previous_payload = {"text": f""}
return previous_payload
def scraper(oauth, bearer_token):
while True:
page = requests.get(
"https://www.reuters.com/business/future-of-money/")
soup = BeautifulSoup(page.content, "html.parser")
home = soup.find(class_="editorial-franchise-layout__main__3cLBl")
posts = home.find_all(
class_="text__text__1FZLe text__dark-grey__3Ml43 text__inherit-font__1Y8w3 text__inherit-size__1DZJi link__underline_on_hover__2zGL4")
top_post = posts[0].find(
"h3", class_="text__text__1FZLe text__dark-grey__3Ml43 text__medium__1kbOh text__heading_3__1kDhc heading__base__2T28j heading__heading_3__3aL54 hero-card__title__33EFM").find_all("span")[0].text.strip()
# Be sure to add replace the text of the with the text you wish to Tweet. You can also add parameters to post polls, quote Tweets, Tweet with reply settings, and Tweet to Super Followers in addition to other features.
payload = {
"text": f"{top_post}\nSource:https://www.reuters.com/business/future-of-money/"}
current_checker_payload = {"text": payload["text"].split("\n")[0]}
previous_payload = previous_tweet()
if previous_payload != current_checker_payload:
tweet(payload, oauth)
else:
print("Content hasn't changed")
time.sleep(60)
def tweet(payload, oauth):
# Making the request
response = oauth.post(
"https://api.twitter.com/2/tweets",
json=payload,
)
if response.status_code != 201:
raise Exception(
"Request returned an error: {} {}".format(
response.status_code, response.text)
)
print("Response code: {}".format(response.status_code))
# Showing the response as JSON
json_response = response.json()
print(json.dumps(json_response, indent=4, sort_keys=True))
if __name__ == "__main__":
init()
** Output**
Response code: 201
{
"data": {
"id": "1598558336672497664",
"text": "FTX ex-CEO Bankman-Fried claims he was unaware of improper use of customer funds -ABC News\nSource:URL" #couldn't post short url in stackoverflow
}
}
Content hasn't changed
Content hasn't changed
Content hasn't changed
Hope this helps. Happy Coding :)

Adding location to Twitter sentiment analysis

I am trying to build a twitter sentiment analysis tool and want to add a geolocation - that looks for tweets within 10 miles of NYC. How do I do this? I I tried to add the location to the end of the url but it did not work.
Here is the code that I have so far:
import oauth2 as oauth
import urllib2 as urllib
# See assignment1.html instructions or README for how to get these credentials
api_key = ''
api_secret = ''
access_token_key = '
access_token_secret = ''
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
'''
Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
def fetchsamples():
url = "https://stream.twitter.com/1.1/statuses/filter.json?
track=money&locations"
parameters = []
response = twitterreq(url, "GET", parameters)
for line in response:
print(line.strip())
if __name__ == '__main__':
fetchsamples()

Here is the API doc: https://developer.twitter.com/en/docs/tweets/filter-realtime/api-reference/post-statuses-filter.html
statuses/filter uses POST, not GET.
For tweets around NYC use "locations=-74,40,-73,41". (You will need to expand this bounding box to make it 10 miles around NYC.) However, when used with track the two filters are OR'ed. In other words, you will get tweets that match either your locations filter or your track filter. You wont get only the tweets that match both filters.
EDIT
url = "https://stream.twitter.com/1.1/statuses/filter.json?track=money&locations=-74,40,-73,41"
parameters = []
response = twitterreq(url, "POST", parameters)

Looking for example Python code for Netsuite API using OAuth?

Netsuite's documentation is not forthcoming. Does anyone have code they've written that will help me generate a valid signature.

There is some sample code in the NetSuite Suite answers site, but you'll have to log in to access it.
https://netsuite.custhelp.com/app/answers/detail/a_id/42165/kw/42165
Here is the code from the answer that I was able to make work. The only difference is that their code broke by trying to encode the timestamp as an int. I typecasted it to a str and the encoding worked fine. The keys/tokens/realm are from their demo code. Insert your own and you should be good to go.
import oauth2 as oauth
import requests
import time
url = "https://rest.netsuite.com/app/site/hosting/restlet.nl?script=992&deploy=1"
token = oauth.Token(key="080eefeb395df81902e18305540a97b5b3524b251772adf769f06e6f0d9dfde5", secret="451f28d17127a3dd427898c6b75546d30b5bd8c8d7e73e23028c497221196ae2")
consumer = oauth.Consumer(key="504ee7703e1871f22180441563ad9f01f3f18d67ecda580b0fae764ed7c4fd38", secret="b36d202caf62f889fbd8c306e633a5a1105c3767ba8fc15f2c8246c5f11e500c")
http_method = "GET"
realm="ACCT123456"
params = {
'oauth_version': "1.0",
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': str(int(time.time())),
'oauth_token': token.key,
'oauth_consumer_key': consumer.key
}
req = oauth.Request(method=http_method, url=url, parameters=params)
signature_method = oauth.SignatureMethod_HMAC_SHA1()
req.sign_request(signature_method, consumer, token)
header = req.to_header(realm)
headery = header['Authorization'].encode('ascii', 'ignore')
headerx = {"Authorization": headery, "Content-Type":"application/json"}
print(headerx)
conn = requests.get("https://rest.netsuite.com/app/site/hosting/restlet.nl?script=992&deploy=1",headers=headerx)
print(conn.text)

Just for reference, I recently did this in Python3 using requests_oauthlib and it worked with standard use of the library:
from requests_oauthlib import OAuth1Session
import json
url = 'https://xxx.restlets.api.netsuite.com/app/site/hosting/restlet.nl?script=xxx&deploy=xxx'
oauth = OAuth1Session(
client_key='xxx',
client_secret='xxx',
resource_owner_key='xxx',
resource_owner_secret='xxx',
realm='xxx')
payload = dict(...)
resp = oauth.post(
url,
headers={'Content-Type': 'application/json'},
data=json.dumps(payload),
)
print(resp)

Building off NetSuite's original sample code I was able to get the below working with SHA256, I think you could do a similar thing for SHA512.
import binascii
import hmac
import time
from hashlib import sha256
import oauth2 as oauth
import requests
url = "https://<account>.restlets.api.netsuite.com/app/site/hosting/restlet.nl?script=<scriptId>&deploy=1"
token = oauth.Token(key="080eefeb395df81902e18305540a97b5b3524b251772adf769f06e6f0d9dfde5",
secret="451f28d17127a3dd427898c6b75546d30b5bd8c8d7e73e23028c497221196ae2")
consumer = oauth.Consumer(key="504ee7703e1871f22180441563ad9f01f3f18d67ecda580b0fae764ed7c4fd38",
secret="b36d202caf62f889fbd8c306e633a5a1105c3767ba8fc15f2c8246c5f11e500c")
http_method = "POST"
realm = "CCT123456"
params = {
'oauth_version': "1.0",
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': str(int(time.time())),
'oauth_token': token.key,
'oauth_consumer_key': consumer.key
}
class SignatureMethod_HMAC_SHA256(oauth.SignatureMethod):
name = 'HMAC-SHA256'
def signing_base(self, request, consumer, token):
if (not hasattr(request, 'normalized_url') or request.normalized_url is None):
raise ValueError("Base URL for request is not set.")
sig = (
oauth.escape(request.method),
oauth.escape(request.normalized_url),
oauth.escape(request.get_normalized_parameters()),
)
key = '%s&' % oauth.escape(consumer.secret)
if token:
key += oauth.escape(token.secret)
raw = '&'.join(sig)
return key.encode('ascii'), raw.encode('ascii')
def sign(self, request, consumer, token):
"""Builds the base signature string."""
key, raw = self.signing_base(request, consumer, token)
hashed = hmac.new(key, raw, sha256)
# Calculate the digest base 64.
return binascii.b2a_base64(hashed.digest())[:-1]
req = oauth.Request(method=http_method, url=url, parameters=params)
oauth.SignatureMethod_HMAC_SHA256 = SignatureMethod_HMAC_SHA256
signature_method = oauth.SignatureMethod_HMAC_SHA256()
req.sign_request(signature_method, consumer, token)
header = req.to_header(realm)
header_y = header['Authorization'].encode('ascii', 'ignore')
header_x = {"Authorization": header_y, "Content-Type": "application/json"}
print(header_x)
response = requests.request("POST", url, data={}, headers=header_x)
# conn = requests.post(url, headers=headerx)
print(response.text)

Streaming API statuses/filter.json?track not delivering JSON Python

I am outputing a Twitter stream ("https://stream.twitter.com/1.1/statuses/filter.json?track=term") to a file using Python. However, the file is outputting an "instance" instead of a "json" (see relevant part in bold):
import oauth2 as oauth
import urllib2 as urllib
import json
# Filled correctly, no authentication problem
api_key = "XXX"
api_secret = "XXX"
access_token_key = "XXX"
access_token_secret = "XXX"
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
'''
Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
**def fetchcontinuousstream():
# For streaming of tweets use
url = "https://stream.twitter.com/1.1/statuses/filter.json?track=term"
parameters = []
response = twitterreq(url, "GET", parameters)
print "Type of the response"
print type(response)
for line in response:
print type(line)
if __name__ == '__main__':
fetchcontinuousstream()**
The result is this:
Type of the response
<type 'instance'>
<type 'str'>
So basically the response is an instance, not a json/dict, each line is just a string...
How can I obtain a JSON instead?

Try to parse the response with json.loads
import oauth2 as oauth
import urllib2 as urllib
from json import loads
# Filled correctly, no authentication problem
api_key = "xxxx"
api_secret = "xxxx"
access_token_key = "xxxx"
access_token_secret = "xxxx"
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
def twitterreq(url, method, parameters):
'''Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
return response
def fetchcontinuousstream():
# For streaming of tweets use
url = "https://stream.twitter.com/1.1/statuses/filter.json?track=term"
parameters = []
response = twitterreq(url, "GET", parameters)
for line in response:
print loads(line)
if __name__ == '__main__':
fetchcontinuousstream()

The json loader was reading an empty line and failing to decode, this code will work
def fetchcontinuousstream():
# For streaming of tweets use
url = "https://stream.twitter.com/1.1/statuses/filter.json?track=term"
parameters = []
response = twitterreq(url, "GET", parameters)
for line in response:
if line != "":
tweet = json.loads(line)
print tweet["text"]

Problems with authorization

I have problems with authorization in python. I want automatic enter to website, but i can't. I used many libraries: Grab, urlib2, request, but i never entered(
For check myself i enter pege with account data
It's real site, login and password
URL="http://pin-im.com/accounts/login/"
LOGIN="testuser"
PASSWORD="test12345user"
urlib2:
def authorization():
import urllib2
gh_url = 'http://pin-im.com/accounts/login/'
gh_user= 'testuser'
gh_pass = 'test12345user'
req = urllib2.Request(gh_url)
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, gh_url, gh_user, gh_pass)
auth_manager = urllib2.HTTPBasicAuthHandler(password_manager)
opener = urllib2.build_opener(auth_manager)
urllib2.install_opener(opener)
handler = urllib2.urlopen(req)
Grab:
def autorization():
g = Grab()
g.setup(post={'username':'testuser', 'Password':'test12345user', 'act': 'submit'})
g.go("http://pin-im.com/accounts/login/")
g.go("http://pin-im.com/user/my-profile/")
print g.response.code
Request(i used all methods in Request Lib for authorization, one of them):
from requests.auth import HTTPBasicAuth
requests.get('http://pin-im.com/accounts/login/', auth=HTTPBasicAuth('testuser', 'test12345user'))
r. get("http://pin-im.com/user/my-profile/")
r.status_code
I'm despair, can you help me login to this site? and what i did wrong?

userData = "Basic " + ("testuser:test12345user").encode("base64").rstrip()
req = urllib2.Request('http://pin-im.com/accounts/login')
req.add_header('Accept', 'application/json')
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header('Authorization', userData)
res = urllib2.urlopen(req)

This site uses CSRF protection, so you should get csrftoken cookie and send it back to server with your request:
import Cookie
from urllib import urlencode
import httplib2
URL="http://pin-im.com/accounts/login/"
LOGIN="testuser"
PASSWORD="test12345user"
http = httplib2.Http()
response, _ = http.request(URL)
cookies = Cookie.BaseCookie()
cookies.load(response["set-cookie"])
csrftoken = cookies["csrftoken"].value
headers = {'Content-type': 'application/x-www-form-urlencoded'}
headers['Cookie'] = response['set-cookie']
data = {
"csrfmiddlewaretoken": csrftoken,
"username":LOGIN,
"password": PASSWORD
}
response, _ = http.request(URL, "POST", headers=headers, body=urlencode(data))
response, content = http.request(
"http://pin-im.com/user/my-profile/",
"GET",
headers={'Cookie': response['set-cookie']}
)
print response, content