i am collecting twitter data with python tweepy here is code
class listener (StreamListener):
def on_data(self, raw_data):
data = json.loads(raw_data)
print data.keys()
tweet = data['text'].encode("utf-8")
tweet_id = data['id']
time_tweet = data['timestamp_ms']
date = datetime.datetime.fromtimestamp(int(time_tweet) / 1000)
new_date = str(date).split(" ") [0]
print new_date
user_id = data['user']['id']
with open('twitDB.csv','ab') as csvfile:
myfile = csv.writer(csvfile)
myfile.writerow([tweet_id,new_date,tweet,user_id])
return True
def on_error(self, status_code):
print status_code
auth = OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
twitterStream = Stream(auth,listener())
twitterStream.filter(track=["car"])
But in Csv File for tweet_id and user_id are in 8.85132E+17 format how i resolve this ?
Place a tab character in front of tweet_id
tweet_id = '\t' + data['id']
Related
I can't get the full text of the Arabic tweet when I use tweet_mode='extended' and tweet.full_text.encode('utf-8') I don't know what is the problem
HashValue = "#لقاح"
StartDate = "2022-01-02"
csvFile = open(HashValue+'.csv', 'a',encoding='utf-8')
csvWriter = csv.writer(csvFile)
for tweet in tweepy.Cursor(api.search,
q=HashValue,
count=20,
lang="ar",
since=StartDate,
tweet_mode='extended').items():
print (tweet.created_at, tweet.full_text)
csvWriter.writerow([tweet.created_at, tweet.full_text.encode('utf-8')])
print ("Scraping finished and saved to "+HashValue+".csv")
I'm trying to save the JSON results of an API call to CSV. The API is in a for loop where it subtracts a day and re-runs the API to get the price of a Bitcoin. It's then supposed to save each API call to a row in a CSV but it's currently only saving the last API call.
How can I get it to write a row for each returned API call?
import requests
import json
import csv
timestampCurrent = 1550698057
timestampOneWeekAgo = 1550161800
oneDay = 86400
for timestamp in range(timestampCurrent, timestampOneWeekAgo, -oneDay):
print(timestamp)
base_url = "https://api.gemini.com/v1/trades/btcusd"
payload = {'timestamp' : timestamp,
'limit_trades' : '1'}
r = requests.get(url = base_url, params = payload)
time = r.json()
print(time)
# open a file for writing
bitcoincsv = open('/Users/kanye_west/Desktop/Code/Python/BitcoinTracker/bitcoinyear.csv', 'w')
# create csv writer object
csvwriter = csv.writer(bitcoincsv)
count = 0
for year in time:
if count == 0:
header = year.keys()
csvwriter.writerow(header)
count += 1
csvwriter.writerow(year.values())
bitcoincsv.close()
# append csv results
# save CSV file
This should work.
import requests
import csv
URL = 'https://api.gemini.com/v1/trades/btcusd'
TIMESTAMP_CURRENT = 1550698057
TIMESTAMP_ONE_WEEK_AGO = 1550161800
ONE_DAY = 86400
def get_time_slot_data(timestamp):
r = requests.get(URL, params={'timestamp': timestamp, 'limit_trades': '1'})
if r.status_code == 200:
return r.json()
with open('gemini.csv', 'wb') as out:
headers_written = False
writer = csv.writer(out)
for timestamp in range(TIMESTAMP_CURRENT, TIMESTAMP_ONE_WEEK_AGO, -ONE_DAY):
daily_data = get_time_slot_data(timestamp)
if not headers_written and daily_data:
writer.writerow(daily_data[0].keys())
headers_written = True
if daily_data:
for entry in daily_data:
writer.writerow(entry.values())
so as the title says I hardly tried to figure out how to save tweets using tweepy in python 3.6. I found a solution that I can save it in English but I can't in Arabic. anyone have any ideas how?
the output I get in the CSV file for Arabic tweets is like this
1510123361.875904::\u0623\u0639\u0648\u0630 \u0628\u0643\u0644\u0645\u0627\u062a \u0627\u0644\u0644\u0647 \u0627\u0644/FMsjMi2nvF
Thank you in advance.
This is my code
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
save = open('ExampleNumber4.csv', mode='w', encoding="utf8", newline=None)
class listener(StreamListener) :
def on_data (self , data):
try:
tweet = json.loads(data)['text']
print(tweet.translate(non_bmp_map))
tweet = data.split(',"text":"')[1].split('","source')[0]
savefile = str(time.time()) + "::" + tweet
save.write(savefile)
save.write("\n\n")
return (True)
except KeyError:
pass
def on_error(self , status):
print(status)
auth = OAuthHandler (ConsumerKey , ConsumerSecret)
auth.set_access_token(AccessToken , AccessTokenSecret)
twitterStream = Stream(auth , listener())
twitterStream.filter(track=[u'سيارة'])
save.close()
Here's a working solution. Please try to make working examples that produce the error of your question next time by including some sample JSON data and skipping the twitter code that we can't run as is.
#coding:utf8
import sys
import json
import time
import csv
data = r'{"text": "\u0633\u064a\u0627\u0631\u0629\ud83d\ude00"}' # ASCII JSON
# data = '{"text": "سيارة😀"}' # non-ASCII JSON
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
with open('ExampleNumber4.csv', mode='w', encoding="utf-8-sig", newline='') as save:
writer = csv.writer(save)
tweet = json.loads(data)['text']
print(tweet.translate(non_bmp_map))
savefile = [time.time(),tweet]
writer.writerow(savefile)
Output:
1510208283.7488384,سيارة�
I'm beginner to using Python. It's hard to study alone.
I collected data from twitter. And I can see data through IPython Console(Spyder). I want to print data to text file, but it doesn't work. My code is follow that. What should I do for printing data to text file?
import tweepy
import pandas as pd
consumer_key = ''
consumer_skey = ''
access_token = ''
access_stoken = ''
class listener(tweepy.StreamListener):
def on_data(self, data):
print (data)
return data
def on_err(self, status):
print (status)
auth = tweepy.OAuthHandler(consumer_key, consumer_skey)
auth.set_access_token(access_token, access_stoken)
twitterStreaming = tweepy.Stream(auth, listener())
twitterStreaming.filter(track=(["siri"]))
df = pd.DataFrame()
df.to_csv(r'C:/Users/ID500/Desktop/Sentiment analysis/hi.txt', header=None, index=None, sep=' ', mode='a')
You can open the text file and write the data to it and then close it, you don't need to use Pandas for that. Also, your class listner is not formatted properly.
Here's the code that you need to modify.
class listener(tweepy.StreamListener):
def on_data(self, data):
print (data)
# write to file here.
out_file = open("FILE_PATH_HERE", 'a')
out_file.write(data)
out_file.close()
return data
def on_err(self, status):
print (status)
Hope this helps.
When I run the code,
I want to save it as output.
But my code is stored differently.
How do I fix it? Thank you for your advice.
"run result" =
1 : {'text': 'Today is sunday! https//abcd'}
2 : {'text': 'hi!!!\nhi!!!\nhi!!! https//abcd}
"Text file saving result"=
Today is sunday! https//abcd
hi!!!
hi!!!
hi!!! https//abcd
import tweepy
import time
import os
import json
import simplejson
search_term = ''
lat = ""
lon = ""
radius = ""
API_key = ""
API_secret = ""
Access_token = ""
Access_token_secret = ""
location = "%s,%s,%s" % (lat, lon, radius)
auth = tweepy.OAuthHandler(API_key, API_secret)
auth.set_access_token(Access_token, Access_token_secret)
api = tweepy.API(auth)
c=tweepy.Cursor(api.search,
q="{}".format(search_term),
rpp=100,
geocode=location,
include_entities=False)
wfile = open("test1.txt", mode='w', encoding='utf8')
data = {}
i = 1
for tweet in c.items():
data['text'] = tweet.text
print(i, ":", data)
wfile.write(data['text']+'\n')
time.sleep(0.35)
i += 1
wfile.close()
you need to clarify your question, i guess nobody really knows what you want to achieve exactly. Do you want a "json like" output of a dictionary in your text file? Give us an example, how your file's content should look like.
I give a guess, maybe you just want the string represenation of your dict, you get the string representation by calling str(data) or data.__str__().
wfile = open("test1.txt", mode='w', encoding='utf8')
data = {}
i = 1
for tweet in c.items():
data['text'] = tweet.text
print(i, ":", data)
wfile.write(str(data) +'\n')
time.sleep(0.35)
i += 1
wfile.close()