When I run the code,
I want to save it as output.
But my code is stored differently.
How do I fix it? Thank you for your advice.
"run result" =
1 : {'text': 'Today is sunday! https//abcd'}
2 : {'text': 'hi!!!\nhi!!!\nhi!!! https//abcd}
"Text file saving result"=
Today is sunday! https//abcd
hi!!!
hi!!!
hi!!! https//abcd
import tweepy
import time
import os
import json
import simplejson
search_term = ''
lat = ""
lon = ""
radius = ""
API_key = ""
API_secret = ""
Access_token = ""
Access_token_secret = ""
location = "%s,%s,%s" % (lat, lon, radius)
auth = tweepy.OAuthHandler(API_key, API_secret)
auth.set_access_token(Access_token, Access_token_secret)
api = tweepy.API(auth)
c=tweepy.Cursor(api.search,
q="{}".format(search_term),
rpp=100,
geocode=location,
include_entities=False)
wfile = open("test1.txt", mode='w', encoding='utf8')
data = {}
i = 1
for tweet in c.items():
data['text'] = tweet.text
print(i, ":", data)
wfile.write(data['text']+'\n')
time.sleep(0.35)
i += 1
wfile.close()
you need to clarify your question, i guess nobody really knows what you want to achieve exactly. Do you want a "json like" output of a dictionary in your text file? Give us an example, how your file's content should look like.
I give a guess, maybe you just want the string represenation of your dict, you get the string representation by calling str(data) or data.__str__().
wfile = open("test1.txt", mode='w', encoding='utf8')
data = {}
i = 1
for tweet in c.items():
data['text'] = tweet.text
print(i, ":", data)
wfile.write(str(data) +'\n')
time.sleep(0.35)
i += 1
wfile.close()
Related
I have written the following to automate the parsing of text to a telegram bot from a .txt file that is continuously being updated.
import urllib.parse
import time
import requests
def post_to_telegram(msg):
print(msg)
base_url = 'https://api.telegram.org/bot&text="{}"'.format(msg)
requests.get(base_url)
urr = ""
name = ""
price = ""
ourLines=0
while(True):
file1 = open('example.txt', 'r')
Lines = file1.readlines()
time.sleep(1)
while(True):
if(ourLines==len(Lines)):
break
else:
txt = Lines[ourLines].strip()
tlist = txt.split("&")
ourLines=ourLines+1
for subtxt in tlist:
if "eventurl=" in subtxt:
a = subtxt[9:len(subtxt) - 3]
url = 'www.bbey43.com/#'+a.replace("%23", "/")
#print(url)
urr = url
elif "bet=" in subtxt:
name = urllib.parse.unquote(subtxt[4:len(subtxt)])
#print(name)
elif "price\":" in subtxt:
a = subtxt.split("price")[1]
price = a.split("\"")[2]
#print(price)
post_to_telegram(urr + " "+ name + " " + price)
the 'name' & 'price' is successfully posted to the bot, but the 'url' doesn't post correctly. The only thing that gets through is "bbey43.com/#/
The solution to this was rather simple in the end. As the "#" was apart of a URL it required special formatting when being parsed.
Simply adding %23 instead of # solved it.
so as the title says I hardly tried to figure out how to save tweets using tweepy in python 3.6. I found a solution that I can save it in English but I can't in Arabic. anyone have any ideas how?
the output I get in the CSV file for Arabic tweets is like this
1510123361.875904::\u0623\u0639\u0648\u0630 \u0628\u0643\u0644\u0645\u0627\u062a \u0627\u0644\u0644\u0647 \u0627\u0644/FMsjMi2nvF
Thank you in advance.
This is my code
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
save = open('ExampleNumber4.csv', mode='w', encoding="utf8", newline=None)
class listener(StreamListener) :
def on_data (self , data):
try:
tweet = json.loads(data)['text']
print(tweet.translate(non_bmp_map))
tweet = data.split(',"text":"')[1].split('","source')[0]
savefile = str(time.time()) + "::" + tweet
save.write(savefile)
save.write("\n\n")
return (True)
except KeyError:
pass
def on_error(self , status):
print(status)
auth = OAuthHandler (ConsumerKey , ConsumerSecret)
auth.set_access_token(AccessToken , AccessTokenSecret)
twitterStream = Stream(auth , listener())
twitterStream.filter(track=[u'سيارة'])
save.close()
Here's a working solution. Please try to make working examples that produce the error of your question next time by including some sample JSON data and skipping the twitter code that we can't run as is.
#coding:utf8
import sys
import json
import time
import csv
data = r'{"text": "\u0633\u064a\u0627\u0631\u0629\ud83d\ude00"}' # ASCII JSON
# data = '{"text": "سيارة😀"}' # non-ASCII JSON
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
with open('ExampleNumber4.csv', mode='w', encoding="utf-8-sig", newline='') as save:
writer = csv.writer(save)
tweet = json.loads(data)['text']
print(tweet.translate(non_bmp_map))
savefile = [time.time(),tweet]
writer.writerow(savefile)
Output:
1510208283.7488384,سيارة�
i am collecting twitter data with python tweepy here is code
class listener (StreamListener):
def on_data(self, raw_data):
data = json.loads(raw_data)
print data.keys()
tweet = data['text'].encode("utf-8")
tweet_id = data['id']
time_tweet = data['timestamp_ms']
date = datetime.datetime.fromtimestamp(int(time_tweet) / 1000)
new_date = str(date).split(" ") [0]
print new_date
user_id = data['user']['id']
with open('twitDB.csv','ab') as csvfile:
myfile = csv.writer(csvfile)
myfile.writerow([tweet_id,new_date,tweet,user_id])
return True
def on_error(self, status_code):
print status_code
auth = OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
twitterStream = Stream(auth,listener())
twitterStream.filter(track=["car"])
But in Csv File for tweet_id and user_id are in 8.85132E+17 format how i resolve this ?
Place a tab character in front of tweet_id
tweet_id = '\t' + data['id']
Closed. This question needs details or clarity. It is not currently accepting answers.
Want to improve this question? Add details and clarify the problem by editing this post.
Closed 6 years ago.
Improve this question
I want to analyse twitter data.I have downloaded some tweets and saved them in a .txt file.
When I tried to extract useful information from the tweets data , i was not able to make any progress because for a beginner like me it seems very difficult to extract tweets , location etc.
while googling i found if we convert json into dictionary it would be easy to extract the info.
Now I want to convert my JSON data to python dictionaries. I don't know how to proceed.
Here is the code used to save tweets
import tweepy
import json
import jsonpickle
consumer_key = "*********"
consumer_secret = "*******"
access_token = "************"
access_token_secret = "**********"
auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# It make the Tweepy API call auto wait (sleep) when it hits the rate limit and continue upon expiry of the window.
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
if (not api):
print ("Can't Authenticate")
sys.exit(-1)
searchQuery = 'SomeHashtag'
maxTweets = 10000000 # Some arbitrary large number
tweetsPerQry = 100
fName = 'file.txt'
sinceId = None
max_id = "Latest tweet ID"
tweetCount = 0
print("Downloading max {0} tweets".format(maxTweets))
with open(fName, 'a') as f:
while tweetCount < maxTweets:
try:
if (max_id <= 0):
if (not sinceId):
new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry)
else:
new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
since_id=sinceId)
else:
if (not sinceId):
new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
max_id=str(max_id - 1))
else:
new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
max_id=str(max_id - 1),
since_id=sinceId)
if not new_tweets:
print("No more tweets found")
break
for tweet in new_tweets:
f.write(jsonpickle.encode(tweet._json, unpicklable=False) + '\n')
tweetCount += len(new_tweets)
print("Downloaded {0} tweets".format(tweetCount))
max_id = new_tweets[-1].id
except tweepy.TweepError as e:
# Just exit if any error
print("some error : " + str(e))
break
print ("Downloaded {0} tweets, Saved to {1}".format(tweetCount, fName))
It seems you can just read your file line by line and unpickle it using jsonpickle.decode method:
tweets = []
with open(filename) as f:
for line in f:
tweets.append(jsonpickle.decode(line))
And I think you can bypass third-party library at all:
import json
with open(filename, 'w') as f:
for tweet in new_tweets:
f.write(json.dumps(tweet) + '\n')
tweets = []
with open(filename) as f:
for line in f:
tweets.append(json.loads(line))
I'm using Tweepy to collect tweets from the Twitter API by their Tweet ID.
Im trying to read in a file full of the IDs, get the previous tweet from the conversation stream, then store that tweet and its author's screen name etc in a text file. Some of the tweets have been deleted or the user's profile has been set to private, in which case I want to ignore that tweet and move on to the next. However, for some reason, I'm not collecting all accessible tweets. Its storing maybe 3/4 of all tweets that aren't private and haven't been deleted. Any ideas why its not catching everything?
Thanks in advance.
def getTweet(tweetID, tweetObj, callTweetObj, i):
tweet = callTweetObj.text.encode("utf8")
callUserName = callTweetObj.user.screen_name
callTweetID = tweetObj.in_reply_to_status_id_str
with open("call_tweets.txt", "a") as calltweets:
output = (callTweetObj.text.encode('utf-8')+ "\t" + callTweetID + "\t" + tweetID)
calltweets.write(output)
print output
with open("callauthors.txt", "a") as callauthors:
cauthors = (callUserName+ "\t" + "\t" + callTweetID + "\n")
callauthors.write(cauthors)
with open("callIDs.txt", "a") as callIDs:
callIDs.write(callTweetID + "\n")
with open("newResponseIDs.txt", "a") as responseIDs:
responseIDs.write(tweetID)
count = 0
file = "Response_IDs.txt"
with open(file, 'r+') as f:
lines = f.readlines()
for i in range(0, len(lines)):
tweetID = lines[i]
sleep(5)
try:
tweetObj = api.get_status(tweetID)
callTweetID = tweetObj.in_reply_to_status_id_str
callTweetObj = api.get_status(callTweetID)
getTweet(tweetID, tweetObj, callTweetObj, i)
count = count+1
print count
except:
pass
You haven't specified information regarding the response coming back from api.get_status, so it's hard to detect what the error is.
However, it might be you have reached the rate limit for the statuses/show/:id request. The API specifies this request is limited to 180 requests a window.
You can use Tweepy to call application/rate_limit_status:
response = api.rate_limit_status()
remaining = response['resources']['statuses']['/statuses/show/:id']['remaining']
assert remaining > 0