Trying to check whether connection established or not, but nothing happened
I used on_connect to understand but got nothing:
import tweepy
import time
class InOutStreamListener(tweepy.StreamListener):
def on_connect(self):
print 'Connected'
def disconnect(self):
if self.running is False:
return
self.running = False
def on_friends(self, friends):
print friends[0]
auth = tweepy.OAuthHandler('code', 'code')
auth.set_access_token('code', 'code')
l = InOutStreamListener()
streamer = tweepy.Stream(auth, l)
time.sleep(15)
streamer.disconnect()
You only created a Stream, you didn't start it, see the docs.
In this example we will use filter to stream all tweets containing the
word python. The track parameter is an array of search terms to
stream.
myStream.filter(track=['python'])
Related
I'm trying to generate keys for every message in Kafka, for that purpose I want to create a key generator that joins the topic first two characters and the tweet id.
Here is an example of the messages that get sent in kafka:
{"data":{"created_at":"2022-03-18T09:51:12.000Z","id":"1504757303811231755","text":"#Danielog111 #POTUS #NATO #UNPeacekeeping #UN Yes! Not to minimize Ukraine at all, but to bring attention to a horrific crisis and Tigrayan genocide that targets 7M people, longer time frame, and is largely unacknowledged by western news agencies. And people are being eaten-literally! #maddow #JoyAnnReid help Ethiopians!"},"matching_rules":[{"id":"1502932028618072070","tag":"NATO"},{"id":"1502932021731115013","tag":"Biden"}]}'
And here is my code modified to try generating partition keys (I'm using PyKafka):
from dotenv import load_dotenv
import os
import json
import tweepy
from pykafka import KafkaClient
# Getting credentials:
BEARER_TOKEN=os.getenv("BEARER_TOKEN")
# Setting up pykafka:
def get_kafka_client():
return KafkaClient(hosts='localhost:9092,localhost:9093,localhost:9094')
def send_message(data, name_topic, id):
client = get_kafka_client()
topic = client.topics[name_topic]
producer = topic.get_sync_producer()
producer.produce(data, partition_key=f"{name_topic[:2]}{id}")
# Creating a Twitter stream listener:
class Listener(tweepy.StreamingClient):
def on_data(self, data):
print(data)
message = json.loads(data)
for rule in message['matching_rules']:
send_message(data, rule['tag'], message['data']['id'].encode())
return True
def on_error(self, status):
print(status)
# Start streaming:
Listener(BEARER_TOKEN).filter(tweet_fields=['created_at'])
And this is the error I'm getting:
File "/Users/mac/.local/share/virtualenvs/tweepy_step-Ck3DvAWI/lib/python3.9/site-packages/pykafka/producer.py", line 372, in produce
raise TypeError("Producer.produce accepts a bytes object as partition_key, "
TypeError: ("Producer.produce accepts a bytes object as partition_key, but it got '%s'", <class 'str'>)
I've also tried not encoding it and trying to fetch the id just using the data (that comes in bytes) but none of these options work.
I found the error, I should've been encoding the partition key and not the json id:
def send_message(data, name_topic, id):
client = get_kafka_client()
topic = client.topics[name_topic]
producer = topic.get_sync_producer()
producer.produce(data, partition_key=f"{name_topic[:2]}{id}".encode())
# Creating a Twitter stream listener:
class Listener(tweepy.StreamingClient):
def on_data(self, data):
print(data)
message = json.loads(data)
for rule in message['matching_rules']:
send_message(data, rule['tag'], message['data']['id'])
return True
def on_error(self, status):
print(status)
I made a python script which uses tweepy streaming module to stream mentions to a twitter account and carry some functions based on the status text.
I wanted it to stream until a mention is made, next stop streaming, carry some functions based on the status text and again start streaming.
This is my code:
class StdOutListener(tweepy.StreamListener):
def on_data(self, data):
tweet = json.loads(data.strip())
global d
d=tweet
return False #stops streaming after a tweet is fed to it
def on_error(self, status_code):
print(status_code)
time.sleep(120)
return F # To continue listening
def on_timeout(self)
time.sleep(120)
return True # To continue listening
while True:
d={}
listener = StdOutListener()
stream = tweepy.Stream(twitter_auth(tokens), listener)
stream.filter(track=['#xxx'])
stream.disconnect()
doSomething(d)
But it only works for one loop and later shows 420(Exceeding Rate Limit) errors,even though I just take in a single tweet (per stream, if I'm not wrong).
Can anyone please explain where I'm doing it wrong? And also when should we use async mode in tweepy stream listener?
I am trying to write a DDE server in python which needs to send a continuously changing string to a program which is connected as a DDE client.
The program which connects to a DDE server uses the following DDE settings to connect [Service: Orbitron, Topic: Tracking, Item: Tracking].
The program has to receive information that is sent by the DDE server in the following string format:
"UP0 DN145000001 UMusb DMfm AZ040 EL005 SNNO SATELLITE".
The content of this string changes approximately every second and I want the DDE server to send the new string to the connected DDE client, for example every second.
I am currently using the code below, which is a slightly modified version of the original ddeserver.py file, see here.
import win32ui
from pywin.mfc import object
import dde
class MySystemTopic(object.Object):
def __init__(self):
object.Object.__init__(self, dde.CreateServerSystemTopic())
def Exec(self, cmd):
print "System Topic asked to exec", cmd
class MyOtherTopic(object.Object):
def __init__(self, topicName):
object.Object.__init__(self, dde.CreateTopic(topicName))
def Exec(self, cmd):
print "Other Topic asked to exec", cmd
class MyRequestTopic(object.Object):
def __init__(self, topicName):
topic = dde.CreateTopic(topicName)
topic.AddItem(dde.CreateStringItem(""))
object.Object.__init__(self, topic)
def Request(self, aString):
print "Request Topic sent: ", aString
a="UP0 DN145800001 UMusb DMfm AZ040 EL005 SNNO SATELLITE"
print a
return(a)
server = dde.CreateServer()
server.AddTopic(MyRequestTopic("Tracking"))
server.Create('Orbitron')
while 1:
win32ui.PumpWaitingMessages(0, -1)
When I run the code I can successfully connect with the program and the string (as provided in the code) is received one time. I tried a few different things but I can not think of a way yet how to change to python code in order to have the DDE server continuously resend the string in a loop or similar.
Any suggestions would be very welcome.
P.S. I am relatively new to python, DDE and this forum, my apologies if something is unclear. Just let me know.
# coded by JayleoPlayGround
# use Portable Python 2.7.5.1 + pywin32-214
import time
import win32ui, dde
from pywin.mfc import object
class DDETopic(object.Object):
def __init__(self, topicName):
self.topic = dde.CreateTopic(topicName)
object.Object.__init__(self, self.topic)
self.items = {}
def setData(self, itemName, value):
try:
self.items[itemName].SetData( str(value) )
except KeyError:
if itemName not in self.items:
self.items[itemName] = dde.CreateStringItem(itemName)
self.topic.AddItem( self.items[itemName] )
self.items[itemName].SetData( str(value) )
ddeServer = dde.CreateServer()
ddeServer.Create('Orbitron')
ddeTopic = DDETopic('Tracking')
ddeServer.AddTopic(ddeTopic)
while True:
yourData = time.ctime() + ' UP0 DN145000001 UMusb DMfm AZ040 EL005 SNNO SATELLITE'
ddeTopic.setData('Tracking', yourData)
win32ui.PumpWaitingMessages(0, -1)
time.sleep(0.1)
I'm a bit new to twisted servers and servers in general. I need to have an asynchronous server running on my local retrieving data from twitter tweets (writing in a local file the data) and at the same time, I want to server some simple content such as "Hello world" at "local_ip:8880"
The problem is that the stream api creates a http connection and I dont know how to do the two tasks at the same time. My code look like this:
"imports and connection to twitter api keys"
def pp(o):
return json.dumps(o, indent=3)
class listener(StreamListener):
def on_data(self,data):
#Convert to JSON the input string
data_json = json.loads(data)
#print pp(data_json['user'])
#We get the html from embedly
tweet_id = data_json['id_str']
tweet_user = data_json['user']['screen_name']
tweet_url = "https://twitter.com/"+tweet_user+"/status/"+tweet_id
html="<a class=\"embedly-card\" href=\""+tweet_url+"\">Prueba</a>\n"
print "Datos recibidos en el listener."
#Write the results in the file
f = open('myfile.html','a')
f.write(html) # python will convert \n to os.linesep
f.close()
return True
def on_error(self,status):
print status
class Root(resource.Resource,StreamListener):
isLeaf = True
def __init__(self):
print "Server iniciado"
auth = OAuthHandler(CONSUMER_KEY,CONSUMER_SECRET)
auth.set_access_token(OAUTH_TOKEN,OAUTH_TOKEN_SECRET)
twitterStream = Stream(auth, listener())
twitterStream.filter(track=["query"])
def render_GET(self, request):
print "Recibida peticion"
return '<html><body>Hello world<body></html>'
root = Root()
site = server.Site(root)
reactor.listenTCP(8880, site)
reactor.run()
When I run this, i got stuck in the console, it receives the stream data, but when I access my "local_ip:8880" it cant load the "Hello world".
Is it possible to make this? Thanks in advance for attention and sorry for my English.
If you use blocking APIs - for example, a function that does network I/O using blocking sockets - then you will prevent your Twisted application from performing operations concurrently. Only one blocking operation can be in progress at a time.
Switch to a non-blocking Twitter API - for example, https://github.com/fiorix/twisted-twitter-stream
So far I have the following code that works and inserts the tweets into my mongodb but I had a few questions.
class CustomStreamListener(tweepy.StreamListener):
def __init__(self, api):
self.api = api
super(tweepy.StreamListener, self).__init__()
self.db = pymongo.MongoClient().test
def on_data(self, tweet):
self.db.tweets.insert(json.loads(tweet))
def on_error(self, status_code):
return True # Don't kill the stream
def on_timeout(self):
return True # Don't kill the stream
sapi = tweepy.streaming.Stream(auth, CustomStreamListener(api))
sapi.filter(track=['arsenal'] , languages = ['en'])
Could someone explain how I can get only certain parts of the tweet inserted into the database ie. just the tweet text and location.
Does the twitter streaming api allow displaying just tweets no # reply tweets?
json.loads(tweet) is just a dictionary, you can freely choose what parts of its key-values you process.
You can filter tweets by conditioning them either way you like:
tweet_obj = json.loads(tweet)
if not tweet_obj['in_reply_to_user_id']: # replies has `None` in this field
pass # add some processing here