Develop Reference

Twitter API User timeline stream in python tweepy

The code below is functional and it streams tweets and stores them in a JSON file. I am running it on a virtual machine since I want to collect 2 months worth of data. However, for some reason, the code stopped running after around 48 hours with no error. Is this a tweepy limitation ( streaming rate or something) or should I check my connection:
import os
import sys
from tweepy import API
from tweepy import OAuthHandler
consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
def get_twitter_client():
"""Setup Twitter API client Return: tweepy.API object """
auth = get_twitter_auth()
client = API(auth)
return client
from tweepy import Stream
from tweepy.streaming import StreamListener
class MyListener(StreamListener):
def on_data(self, data):
try:
with open('strategyand_user.json', 'a') as f:
f.write(data)
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
return True
def on_error(self, status):
print(status)
return True
twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(follow=['xxxxxxx'])

How to json dump tweepy stream into text file?

Hi I have looked at many guides and tutorials on how to do this, but I am having trouble with being able to use tweepy to store the JSON data in a text file.
class StreamListener(tweepy.StreamListener):
def on_status(self, status):
print(status)
def on_error(self, status):
print status
if status == 420:
return False
if __name__ == '__main__':
stream_listener = StreamListener()
auth = tweepy.OAuthHandler(consumer_token, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = tweepy.Stream(auth, stream_listener)
I have another python file which is supposed to read data into a list:
import pandas
import json
json_data = 'twitter_data.txt'
data_list = []
#load file
tweets_file = open(json_data, "r")
for line in tweets_file:
try:
tweet = json.loads(line) #this line causes problems
data_list.append(tweet)
except:
continue
print len(data_list)
I thought the data received from twitter comes in JSON format, and the guides I'm following all say it does, but it's actually in another object.
Should I just store everything in a list then json dump that list into the new text file?

It seems like you're on the right track. You can modify the stream listener to write tweets to a file directly.
Edit: this now writes out in JSON format.
#Import the necessary methods from tweepy library
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy import API
#Variables that contains the user credentials to access Twitter API
CONSUMER_KEY = #YOUR CONSUMER KEY
CONSUMER_SECRET = #YOUR CONSUMER SECRET
ACCESS_TOKEN = #YOUR ACCESS TOKEN
ACCESS_TOKEN_SECRET = #YOUR ACCESS TOKEN SECRET
class FileWriteListener(StreamListener):
def __init__(self):
super(StreamListener, self).__init__()
self.save_file = open('tweets.json','w')
self.tweets = []
def on_data(self, tweet):
self.tweets.append(json.loads(tweet))
self.save_file.write(str(tweet))
def on_error(self, status):
print(status)
return True
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
api = API(auth)
twitter_stream = Stream(auth, MyListener())
# Here you can filter the stream by:
# - keywords (as shown)
# - users
twitter_stream.filter(track=['hello'])
This code will run indefinitely, so you either need to exit the process after some time (Ctrl-C) or modify the code.
Then you can load the data:
import json
json_data = []
with open('tweets.json','r') as f:
json_data.append( json.loads(f.readline()) )
Hope this helps!

I think something like this may be what your looking for.
def on_status(self, tweet):
json_dumps = json.dumps(tweet._json)
tweet_json = json.loads(json_dumps)
print(tweet_json['created_at'])
These are all the keys you can use in the tweet_json[ ]
dict_keys(['created_at', 'id', 'id_str', 'text', 'source', 'truncated', 'in_reply_to_status_id', 'in_reply_to_status_id_str', 'in_reply_to_user_id', 'in_reply_to_user_id_str', 'in_reply_to_screen_name', 'user', 'geo', 'coordinates', 'place', 'contributors', 'retweeted_status', 'is_quote_status', 'quote_count', 'reply_count', 'retweet_count', 'favorite_count', 'entities', 'favorited', 'retweeted', 'filter_level', 'lang', 'timestamp_ms'])

How to retrieve user details from mention tweets?

I am in the processing of creating a bot that reads all mentions in userstream Twitter timeline and reply accordingly, however I cannot seem to get the users details who posted the mention in order to reply back, all I receive is the mention status/text.
I am using Python and Tweepy API to achieve this:
import tweepy
from tweepy import Stream
consumer_key = '********'
consumer_secret = '********'
access_token = '***********'
access_secret = '*********'
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
class MyStreamListener (tweepy.StreamListener):
def on_status(self, status):
print(status.text)
def on_error(self, status):
print(status)
def on_direct_message(self, status):
print(status)
twitterStream = Stream(auth, MyStreamListener())
userStream = twitterStream.userstream()
Any help would be greatly appreciated

It needed
print(status.author.screen_name)

twitter timeline streaming with tweepy

i want to stream my own twitter timeline with python and tweepy and use code in below but it just print me some numbers and i dosent print my timeline twitts. Can you help me?
import tweepy
from tweepy import Stream
from tweepy.streaming import StreamListener
consumer_key = 'abc'
consumer_secret = 'abc'
access_token = 'abc'
access_secret = 'abc'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
class listener(StreamListener):
def on_data(self, data):
print (data)
return True
def on_error(self, status):
print (status)
twitterStream = Stream(auth, listener())
twitterStream.userstream(encoding='utf8')

If you take a look at the documentation it says:
The on_data method of Tweepy’s StreamListener conveniently passes data from statuses to the on_status method.
The numbers you are seeing are the object IDs of the Tweets. You will need to do something like:
def on_status(self, status):
print(status.text)

How can I consume tweets from Twitter's streaming api and store them in mongodb

I need to develop an app that lets me track tweets and save them in a mongodb for a research project (as you might gather, I am a noob, so please bear with me). I have found this piece of code that sends tweets streaming through my terminal window:
import sys
import tweepy
consumer_key=""
consumer_secret=""
access_key = ""
access_secret = ""
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth)
class CustomStreamListener(tweepy.StreamListener):
def on_status(self, status):
print status.text
def on_error(self, status_code):
print >> sys.stderr, 'Encountered error with status code:', status_code
return True # Don't kill the stream
def on_timeout(self):
print >> sys.stderr, 'Timeout...'
return True # Don't kill the stream
sapi = tweepy.streaming.Stream(auth, CustomStreamListener())
sapi.filter(track=['Gandolfini'])
Is there a way I can modify this piece of code so that instead of having tweets streaming over my screen, they are sent to my mongodb database?
Thanks

Here's an example:
import json
import pymongo
import tweepy
consumer_key = ""
consumer_secret = ""
access_key = ""
access_secret = ""
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth)
class CustomStreamListener(tweepy.StreamListener):
def __init__(self, api):
self.api = api
super(tweepy.StreamListener, self).__init__()
self.db = pymongo.MongoClient().test
def on_data(self, tweet):
self.db.tweets.insert(json.loads(tweet))
def on_error(self, status_code):
return True # Don't kill the stream
def on_timeout(self):
return True # Don't kill the stream
sapi = tweepy.streaming.Stream(auth, CustomStreamListener(api))
sapi.filter(track=['Gandolfini'])
This will write tweets to the mongodb test database, tweets collection.
Hope that helps.

I have developed a simple command line tool that does exactly this.
https://github.com/janezkranjc/twitter-tap
It allows using the streaming API or the search API.