Develop Reference

Kill and restart Python script every day

I have a Python code that extracts Twitter data via the streaming API. I would like to use separate files for each day so I would like to have the script running for 24 hours, then kill it and restart it as with a restart of the program the name of the file will change.
How can I ensure that the script is stopped at 00:00 and restarts right away?
The code can be found below. If you have any other ideas about how I can create a new text file daily, this would be even better.
import tweepy
import datetime
key_words = ["xx"]
twitter_data_title = "".join([xx, "_", date_today, ".txt"])
class TwitterStreamer():
def __init__(self):
pass
def stream_tweets(self, twitter_data_title, key_words):
listener = StreamListener(twitter_data_title)
auth = tweepy.OAuthHandler(api_key, api_secret_key)
auth.set_access_token(access_token, access_secret_token)
stream = tweepy.Stream(auth, listener)
stream.filter(track=key_words)
class StreamListener(tweepy.StreamListener):
def __init__(self, twitter_data_title):
self.fetched_tweets_filename = twitter_data_title
def on_data(self, data):
try:
print(data)
with open(self.fetched_tweets_filename, 'a') as tf:
tf.write(data)
return True
except BaseException as e:
print("Error on_data %s" % str(e))
return True
def on_exception(self, exception):
print('exception', exception)
stream_tweets(twitter_data_title, key_words)
def on_error(self, status):
print(status)
def stream_tweets(twitter_data_title, key_words):
listener = StreamListener(twitter_data_title)
auth = tweepy.OAuthHandler(api_key, api_secret_key)
auth.set_access_token(access_token, access_secret_token)
stream = tweepy.Stream(auth, listener)
stream.filter(track=key_words)
if __name__ == '__main__':
twitter_streamer = TwitterStreamer()
twitter_streamer.stream_tweets(twitter_data_title, key_words)

It looks like the 'blocking' code in your example comes from another library, so you don't have the opportunity to (easily) change the inner loop to check for a condition and exit.
Using a Background Process (Not Ideal)
You could change your entry point to start the code in a background process, and check to see if the file's title should have changed:
from multiprocessing import Process
from time import sleep
...
if __name__ == "__main__":
twitter_streamer = TwitterStreamer()
twitter_data_title, process = None, None
while True:
new_data_title = "".join([xx, "_", str(datetime.date.today()), ".txt"])
if new_data_title == twitter_data_title: # Nothing to do.
sleep(60) # Sleep for a minute
continue # And check again
# Set the new title.
twitter_data_title = new_data_title
# If the process is already running, terminate and join it.
if process is not None:
process.terminate()
process.join()
process = Process(target=twitter_streamer.stream_tweets, args=[twitter_data_title, key_words])
process.start()
Changing StreamListener
A better alternative would probably be to encode the knowledge of the date into StreamListener. Instead of passing a file name (twitter_data_title), pass a file prefix (xx from your example), and build the filename in a property:
...
class StreamListener(tweepy.StreamListener):
def __init__(self, file_prefix):
self.prefix = file_prefix
#property
def fetched_tweets_filename(self):
"""The file name for the tweets."""
date = datetime.date.today()
return f"{self.prefix}_{date}.txt"
...
...
if __name__ == "__main__":
twitter_streamer = TwitterStreamer()
twitter_streamer.stream_tweets(xx, key_words)
Since StreamListener.on_data grabs the file name from self.fetched_tweets_filename, this should mean the tweets are written to the new file when the date changes.

I would add this to your code:
from threading import Timer
def stopTheScript():
exec(open("anotherscript.py").read())
exit()
Timer(86400, stopTheScript).start() #86400 s = 24 h

Tornado client with stdin

I am trying to build a multiplayer game that uses Python. I am using Tornado to build the client and server. Ideally, what I would like to happen are as follows:
(a) For the client to wait for user input from the command line
(b) When the client gets user input, to send the user input to the server
(c) for the server to simulate some processing(which will be the game engine) on it and send the response back to a client.
The server
"""
Server module for game server
"""
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.websocket
import uuid
import json
class Controller(object):
def __init__(self):
self.players = ()
self.connections = ()
def check(self):
return "Hi"
def run_somthing(self, text):
new_text = "Server: " + text
return new_text
class InitMessageHandler(tornado.web.RequestHandler):
def get(self):
user_data = {}
user_data['user_id'] = str(uuid.uuid4())
self.write(json.dumps(user_data))
class GameHandler(tornado.websocket.WebSocketHandler):
def open(self):
# called anytime a new connection with this server is opened
print("Client connected")
print("Client sent: ", self)
if seif not in self.application.controller.connections:
self.application.controller.connections.add(self)
def on_message(self):
# called anytime a new message is received
pass
def check_origin(self, origin):
return True
def on_close(self):
# called a websocket connection is closed
if self in self.application.controller.connections:
self.application.controller.connections.remove(self)
class Server(tornado.web.Application):
def __init__(self):
self.controller = Controller()
handlers = [
(r"/join", InitMessageHandler),
(r"/game", GameHandler)
]
tornado.web.Application.__init__(self, handlers)
if __name__ == "__main__":
tornado.options.parse_command_line()
try:
application = Server()
server = tornado.httpserver.HTTPServer(application)
server.listen(8888)
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
tornado.ioloop.IOLoop.instance().stop()
print("closed")
The client
"""
Client module for the game clients(Players)
"""
import tornado.ioloop
import tornado.websocket
import requests
import json
import sys
import tornado.gen
class Client(object):
def __init__(self, join_url, play_url):
self.wsconn = None
self.join_url = join_url
self.play_url = play_url
#self.io_loop = tornado.ioloop.IOLoop.instance()
#self.io_loop.add_handler(sys.stdin, self.handle_user_input, tornado.ioloop.IOLoop.READ)
self.user_details = {}
self.has_initialised = False
#self.io_loop.start()
self.main()
def get_user_input(self, question=None):
str_choice = input(question)
while any((str_choice is None, not str_choice.strip())):
print("You entered an empty answer")
str_choice = input(question)
return str_choice
def _merge_dicts(*dict_args):
"""
Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.
"""
result = {}
for dictionary in dict_args:
result.update(dictionary)
return result
def generate_wsmessage(self):
msg_line = input("Enter message to send to server")
while any((msg_line is None, not msg_line.strip())):
print("You entered an empty answer")
msg_line = input("Enter message to send to server")
msg = {}
msg['message'] = msg_line
msg_to_send = self._merge_dicts(self.user_details, msg)
return json.dumps(msg_to-send)
def init(self):
print("Heh")
username = self.get_user_input("What is your username? ")
print("Getting initial user details")
req = requests.get(self.join_url)
response = json.loads(req.text)
print(response)
self.user_details['name'] = username
self.user_details['user_id'] = response['user_id']
self.has_initialised = True
def server_recv(self, msg):
print("Server has connected on websocket socket with msg=", msg)
#tornado.gen.coroutine
def connect_on_websocket(self):
try:
self.wsconn = yield tornado.websocket.websocket_connect(self.play_url, on_message_callback=self.server_recv)
except Exception as e:
print("Connection error: {}".format(e))
else:
print("Connected")
#tornado.gen.coroutine
def send_wsmessage(self):
msg = self.generate_wsmessage()
yield self.wsconn.write_message(msg)
#tornado.gen.coroutine
def communicate_with_websocket(self):
self.send_wsmessage()
while True:
recv_msg = yield self.wsconn.read_message()
if recv_msg is None:
self.wsconn.close()
break
yield tornado.gen.sleep(0.1)
self.send_wsmessage()
print("IoLoop terminate")
def main(self):
choice = input("Do you want to continue(y/n)? ")
if choice == "y" and self.has_initialised == False:
print("Yup")
self.init()
if self.has_initialised == True:
self.connect_on_websocket()
self.communicate_with_websocket()
if __name__ == "__main__":
try:
client = Client("http://localhost:8888/join", "ws://localhost:8888/game")
tornado.ioloop.IOLoop.instance().start()
except (SystemExit, KeyboardInterrupt):
print("Client closed")
From reading the some examples online, I came up the code above, but it is not working. So my main question is
how to make Tornado coroutines work with stdin(command line input)
My other questions are:
(a) Is the code I have written the right way to work with Tornado coroutines or not?
(b) If it is not, could you ELI5? Also I would appreciate code examples that really use Tornado in interesting ways(at any intermediate level) so that I can learn from them.
(c) Is there a more intuitive way to do what I want to do,in Python? Like a Flask+Gevents or Twisted version or just pure sockets version that might be easier to work with?
Thanks for your help.
EDIT: Flan pointed out some errors for me and I fixed it and it works now.

As I can see at the moment, problem is not in stdin interaction, but your wrong way of using coroutines. Your connect_on_websocket and communicate_with_websocket functions are coroutines but you are using them as a plain functions and it won't work. I propose these changes.
Make main() coroutine (add decorator), don't call it, remove from the Client.__init__().
In name=main block schedule client.main() invocation with tornado.ioloop.IOLoop.instance().add_callback(client.main).
In main and all your code change coroutine-functions' (with #tornado.gen.coroutine) calls to yield, for example yield self.connect_on_websocket() instead of just self.connect_on_websocket()
That should be sufficient so you can proceed your development further.

The edited code is
The server:
"""
Server module for game server
"""
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.websocket
import uuid
import json
class Controller(object):
def __init__(self):
self.players = set()
self.connections = set()
def check(self):
return "Hi"
def run_somthing(self, text):
new_text = "Server: " + text
return new_text
class InitMessageHandler(tornado.web.RequestHandler):
def get(self):
print("Client has asked for initial details")
user_data = {}
user_data['user_id'] = str(uuid.uuid4())
self.write(json.dumps(user_data))
class GameHandler(tornado.websocket.WebSocketHandler):
def open(self):
# called anytime a new connection with this server is opened
print("Client connected")
print("Client sent: ", self)
if self not in self.application.controller.connections:
self.application.controller.connections.add(self)
def on_message(self, message):
# called anytime a new message is received
print("Received from client ,msg=", message)
msg = "Server: " + message
self.write_message(json.dumps(msg))
def check_origin(self, origin):
return True
def on_close(self):
# called a websocket connection is closed
if self in self.application.controller.connections:
self.application.controller.connections.remove(self)
class Server(tornado.web.Application):
def __init__(self):
self.controller = Controller()
handlers = [
(r"/join", InitMessageHandler),
(r"/game", GameHandler)
]
tornado.web.Application.__init__(self, handlers)
if __name__ == "__main__":
tornado.options.parse_command_line()
try:
application = Server()
server = tornado.httpserver.HTTPServer(application)
server.listen(8888)
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
tornado.ioloop.IOLoop.instance().stop()
print("Server closed")
The client
import tornado.ioloop
import tornado.websocket
import requests
import json
import sys
import tornado.gen
class Client(object):
def __init__(self, join_url, play_url):
self.wsconn = None
self.join_url = join_url
self.play_url = play_url
self.user_details = {}
def get_user_input(self, question=None):
str_choice = input(question)
while any((str_choice is None, not str_choice.strip())):
print("You entered an empty answer")
str_choice = input(question)
return str_choice
def _merge_dicts(*dict_args):
"""
Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.
"""
result = {}
for dictionary in dict_args:
result.update(dictionary)
return result
def generate_wsmessage(self):
msg_line = self.get_user_input("Enter message to send to server: ")
msg = {}
msg['message'] = msg_line
msg['user_id'] = self.user_details['user_id']
msg['user_name'] = self.user_details['user_name']
print("Message to send: ", msg)
return json.dumps(msg)
def init(self):
print("Heh")
username = self.get_user_input("What is your username? ")
print("Getting initial user details")
req = requests.get(self.join_url)
response = json.loads(req.text)
print(response)
self.user_details['user_name'] = username
self.user_details['user_id'] = response['user_id']
#tornado.gen.coroutine
def connect_on_websocket(self):
try:
self.wsconn = yield tornado.websocket.websocket_connect(self.play_url)
except Exception as e:
print("Connection error: {}".format(e))
else:
print("Server has connected to ")
yield self.send_wsmessage()
#tornado.gen.coroutine
def send_wsmessage(self):
msg = self.generate_wsmessage()
if not self.wsconn:
raise RuntimeError('Web socket connection is closed.')
yield self.wsconn.write_message(json.dumps(msg))
yield self.communicate_with_websocket()
#tornado.gen.coroutine
def communicate_with_websocket(self):
recv_msg = None
while True:
recv_msg = yield self.wsconn.read_message()
if recv_msg is None:
self.wsconn.close()
break
print("Server has replied with message=", recv_msg)
yield self.send_wsmessage()
print("IoLoop terminate")
#tornado.gen.coroutine
def main(self):
choice = input("Do you want to continue(y/n)? ")
if choice == "y":
print("Yup")
self.init()
yield self.connect_on_websocket()
if choice == "n":
sys.exit()
if __name__ == "__main__":
try:
client = Client("http://localhost:8888/join", "ws://localhost:8888/game")
tornado.ioloop.IOLoop.instance().add_callback(client.main)
tornado.ioloop.IOLoop.instance().start()
except (SystemExit, KeyboardInterrupt):
print("Client closed")
For (a), (b) check out here. For (c), another time.

Python, Call a class function from another class

Can you anyone please help me (noob) call the broadcast function from class BroadcastServerFactory in class process, as per attached code
I have tried so many methods of call a function from another class, but no solution
import time, sys
from apscheduler.scheduler import Scheduler
import threading
import socket
from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class process(threading.Thread):
def __init__(self, buffer3):
threading.Thread.__init__(self)
self.setDaemon(True)
self.buffer3 = buffer3
def run(self):
factory.broadcast("I don't know what I'm doing!")
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def onMessage(self, msg, binary):
if not binary:
self.factory.broadcast("'%s' from %s" % (msg, self.peerstr))
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
"""
Simple broadcast server broadcasting any message it receives to all
currently connected clients.
"""
def __init__(self, url, debug = False, debugCodePaths = False):
WebSocketServerFactory.__init__(self, url, debug = debug, debugCodePaths = debugCodePaths)
self.clients = []
self.tickcount = 0
self.tick()
def tick(self):
self.tickcount += 1
self.broadcast("'tick %d' from server" % self.tickcount)
reactor.callLater(1, self.tick)
def register(self, client):
if not client in self.clients:
print "registered client " + client.peerstr
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print "unregistered client " + client.peerstr
self.clients.remove(client)
def broadcast(self, msg):
print "broadcasting message '%s' .." % msg
for c in self.clients:
c.sendMessage(msg)
print "message sent to " + c.peerstr
class BroadcastPreparedServerFactory(BroadcastServerFactory):
"""
Functionally same as above, but optimized broadcast using
prepareMessage and sendPreparedMessage.
"""
def broadcast(self, msg):
print "broadcasting prepared message '%s' .." % msg
preparedMsg = self.prepareMessage(msg)
for c in self.clients:
c.sendPreparedMessage(preparedMsg)
print "prepared message sent to " + c.peerstr
def testing():
buffer2 - "hello"
myDisplay = process(buffer2)
myDisplay.start()
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
level_scheduler = Scheduler()
level_scheduler.add_interval_job(testing, seconds=5)
level_scheduler.start()
#ServerFactory = BroadcastServerFactory
ServerFactory = BroadcastPreparedServerFactory
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
factory.protocol = BroadcastServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory)
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.run()
Thanks

Pass the class instance of BroadcastServerFactory to be called to the class instance that calls it process on creation
class process(threading.Thread):
def __init__(self, buffer3m, broadcast_server_factory):
threading.Thread.__init__(self)
self.setDaemon(True)
self.buffer3 = buffer3
self.factory = broadcast_server_factory
def run(self):
self.factory.broadcast("I don't know what I'm doing!")
and then call it (it's assigned as self.factory in the run statement. I can't see where you create a process class in your __main__ but it will be created with something like
p = process(buffer, factory)
Aside: Using capital letters for class names is considered good form in python process -> Process

Create DBus signal dynamically in python

I've read some topics related to dynamically create python methods, and I followed their instructions, but it does not work. I do not know if it is because I use decorator # or something else.
The code is here, very simple.
When running this code, no error occurred, but when I use D-feet(A tool to check dbus informations), I could not find new signals I created.
#!/usr/bin/python
import dbus
import dbus.service
import dbus.glib
import gobject
from dbus.mainloop.glib import DBusGMainLoop
import psutil
class EventServer(dbus.service.Object):
i = 0
#dbus.service.signal('com.github.bxshi.event')
def singal_example(self,msg):
""" example of singals
"""
print msg
def __init__(self):
bus_name = dbus.service.BusName('com.github.bxshi.event', bus=dbus.SessionBus())
dbus.service.Object.__init__(self, bus_name, '/com/github/bxshi/event')
def create(self):
self.i +=1
setattr(self.__class__, 'signal_'+str(self.i), self.singal_example)
if __name__ == "__main__":
DBusGMainLoop(set_as_default=True)
bus = dbus.SessionBus()
eventserver = EventServer()
gobject.timeout_add(1000,eventserver.create)
loop = gobject.MainLoop()
loop.run()

you have a typo: singal_example instead of signal_example
in your create-method you call setattr on the class. I don't know what you're trying to do, but you should simply emit the signal
This is the fixed example:
#!/usr/bin/python
import dbus
import dbus.service
import dbus.glib
import gobject
from dbus.mainloop.glib import DBusGMainLoop
#import psutil
class EventServer(dbus.service.Object):
i = 0
#dbus.service.signal('com.github.bxshi.event')
def signal_example(self,msg):
""" example of singals
"""
print msg
def __init__(self):
bus_name = dbus.service.BusName('com.github.bxshi.event', bus=dbus.SessionBus())
dbus.service.Object.__init__(self, bus_name, '/com/github/bxshi/event')
def create(self):
self.i +=1
#setattr(self.__class__, 'signal_'+str(self.i), self.singal_example)
self.signal_example('msg: %d' % self.i)
if __name__ == "__main__":
DBusGMainLoop(set_as_default=True)
bus = dbus.SessionBus()
eventserver = EventServer()
gobject.timeout_add(1000,eventserver.create)
loop = gobject.MainLoop()
loop.run()
After that you can connect to the signal:
# ...
bus = dbus.Bus()
service=bus.get_object('com.github.bxshi.event', '/com/github/bxshi/event')
service.connect_to_signal("signal_example", listener)
# ...

Twisted Python getPage

I tried to get support on this but I am TOTALLY confused.
Here's my code:
from twisted.internet import reactor
from twisted.web.client import getPage
from twisted.web.error import Error
from twisted.internet.defer import DeferredList
from sys import argv
class GrabPage:
def __init__(self, page):
self.page = page
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
return result
def listCallback(self, result):
print result
a = GrabPage('http://www.google.com')
data = a.start() # Not the HTML
I wish to get the HTML out which is given to pageCallback when start() is called. This has been a pita for me. Ty! And sorry for my sucky coding.

You're missing the basics of how Twisted operates. It all revolves around the reactor, which you're never even running. Think of the reactor like this:
(source: krondo.com)
Until you start the reactor, by setting up deferreds all you're doing is chaining them with no events from which to fire.
I recommend you give the Twisted Intro by Dave Peticolas a read. It's quick and it really gives you all the missing information that the Twisted documentation doesn't.
Anyways, here is the most basic usage example of getPage as possible:
from twisted.web.client import getPage
from twisted.internet import reactor
url = 'http://aol.com'
def print_and_stop(output):
print output
if reactor.running:
reactor.stop()
if __name__ == '__main__':
print 'fetching', url
d = getPage(url)
d.addCallback(print_and_stop)
reactor.run()
Since getPage returns a deferred, I'm adding the callback print_and_stop to the deferred chain. After that, I start the reactor. The reactor fires getPage, which then fires print_and_stop which prints the data from aol.com and then stops the reactor.
Edit to show a working example of OP's code:
class GrabPage:
def __init__(self, page):
self.page = page
########### I added this:
self.data = None
def start(self, *args):
if args == ():
# We apparently don't need authentication for this
d1 = getPage(self.page)
else:
if len(args) == 2:
# We have our login information
d1 = getPage(self.page, headers={"Authorization": " ".join(args)})
else:
raise Exception('Missing parameters')
d1.addCallback(self.pageCallback)
dl = DeferredList([d1])
d1.addErrback(self.errorHandler)
dl.addCallback(self.listCallback)
def errorHandler(self,result):
# Bad thingy!
pass
def pageCallback(self, result):
########### I added this, to hold the data:
self.data = result
return result
def listCallback(self, result):
print result
# Added for effect:
if reactor.running:
reactor.stop()
a = GrabPage('http://google.com')
########### Just call it without assigning to data
#data = a.start() # Not the HTML
a.start()
########### I added this:
if not reactor.running:
reactor.run()
########### Reference the data attribute from the class
data = a.data
print '------REACTOR STOPPED------'
print
########### First 100 characters of a.data:
print '------a.data[:100]------'
print data[:100]