Develop Reference

Running a Python web scraper every hour [duplicate]

I'm looking for a library in Python which will provide at and cron like functionality.
I'd quite like have a pure Python solution, rather than relying on tools installed on the box; this way I run on machines with no cron.
For those unfamiliar with cron: you can schedule tasks based upon an expression like:
0 2 * * 7 /usr/bin/run-backup # run the backups at 0200 on Every Sunday
0 9-17/2 * * 1-5 /usr/bin/purge-temps # run the purge temps command, every 2 hours between 9am and 5pm on Mondays to Fridays.
The cron time expression syntax is less important, but I would like to have something with this sort of flexibility.
If there isn't something that does this for me out-the-box, any suggestions for the building blocks to make something like this would be gratefully received.
Edit
I'm not interested in launching processes, just "jobs" also written in Python - python functions. By necessity I think this would be a different thread, but not in a different process.
To this end, I'm looking for the expressivity of the cron time expression, but in Python.
Cron has been around for years, but I'm trying to be as portable as possible. I cannot rely on its presence.

If you're looking for something lightweight checkout schedule:
import schedule
import time
def job():
print("I'm working...")
schedule.every(10).minutes.do(job)
schedule.every().hour.do(job)
schedule.every().day.at("10:30").do(job)
while 1:
schedule.run_pending()
time.sleep(1)
Disclosure: I'm the author of that library.

You could just use normal Python argument passing syntax to specify your crontab. For example, suppose we define an Event class as below:
from datetime import datetime, timedelta
import time
# Some utility classes / functions first
class AllMatch(set):
"""Universal set - match everything"""
def __contains__(self, item): return True
allMatch = AllMatch()
def conv_to_set(obj): # Allow single integer to be provided
if isinstance(obj, (int,long)):
return set([obj]) # Single item
if not isinstance(obj, set):
obj = set(obj)
return obj
# The actual Event class
class Event(object):
def __init__(self, action, min=allMatch, hour=allMatch,
day=allMatch, month=allMatch, dow=allMatch,
args=(), kwargs={}):
self.mins = conv_to_set(min)
self.hours= conv_to_set(hour)
self.days = conv_to_set(day)
self.months = conv_to_set(month)
self.dow = conv_to_set(dow)
self.action = action
self.args = args
self.kwargs = kwargs
def matchtime(self, t):
"""Return True if this event should trigger at the specified datetime"""
return ((t.minute in self.mins) and
(t.hour in self.hours) and
(t.day in self.days) and
(t.month in self.months) and
(t.weekday() in self.dow))
def check(self, t):
if self.matchtime(t):
self.action(*self.args, **self.kwargs)
(Note: Not thoroughly tested)
Then your CronTab can be specified in normal python syntax as:
c = CronTab(
Event(perform_backup, 0, 2, dow=6 ),
Event(purge_temps, 0, range(9,18,2), dow=range(0,5))
)
This way you get the full power of Python's argument mechanics (mixing positional and keyword args, and can use symbolic names for names of weeks and months)
The CronTab class would be defined as simply sleeping in minute increments, and calling check() on each event. (There are probably some subtleties with daylight savings time / timezones to be wary of though). Here's a quick implementation:
class CronTab(object):
def __init__(self, *events):
self.events = events
def run(self):
t=datetime(*datetime.now().timetuple()[:5])
while 1:
for e in self.events:
e.check(t)
t += timedelta(minutes=1)
while datetime.now() < t:
time.sleep((t - datetime.now()).seconds)
A few things to note: Python's weekdays / months are zero indexed (unlike cron), and that range excludes the last element, hence syntax like "1-5" becomes range(0,5) - ie [0,1,2,3,4]. If you prefer cron syntax, parsing it shouldn't be too difficult however.

More or less same as above but concurrent using gevent :)
"""Gevent based crontab implementation"""
from datetime import datetime, timedelta
import gevent
# Some utility classes / functions first
def conv_to_set(obj):
"""Converts to set allowing single integer to be provided"""
if isinstance(obj, (int, long)):
return set([obj]) # Single item
if not isinstance(obj, set):
obj = set(obj)
return obj
class AllMatch(set):
"""Universal set - match everything"""
def __contains__(self, item):
return True
allMatch = AllMatch()
class Event(object):
"""The Actual Event Class"""
def __init__(self, action, minute=allMatch, hour=allMatch,
day=allMatch, month=allMatch, daysofweek=allMatch,
args=(), kwargs={}):
self.mins = conv_to_set(minute)
self.hours = conv_to_set(hour)
self.days = conv_to_set(day)
self.months = conv_to_set(month)
self.daysofweek = conv_to_set(daysofweek)
self.action = action
self.args = args
self.kwargs = kwargs
def matchtime(self, t1):
"""Return True if this event should trigger at the specified datetime"""
return ((t1.minute in self.mins) and
(t1.hour in self.hours) and
(t1.day in self.days) and
(t1.month in self.months) and
(t1.weekday() in self.daysofweek))
def check(self, t):
"""Check and run action if needed"""
if self.matchtime(t):
self.action(*self.args, **self.kwargs)
class CronTab(object):
"""The crontab implementation"""
def __init__(self, *events):
self.events = events
def _check(self):
"""Check all events in separate greenlets"""
t1 = datetime(*datetime.now().timetuple()[:5])
for event in self.events:
gevent.spawn(event.check, t1)
t1 += timedelta(minutes=1)
s1 = (t1 - datetime.now()).seconds + 1
print "Checking again in %s seconds" % s1
job = gevent.spawn_later(s1, self._check)
def run(self):
"""Run the cron forever"""
self._check()
while True:
gevent.sleep(60)
import os
def test_task():
"""Just an example that sends a bell and asd to all terminals"""
os.system('echo asd | wall')
cron = CronTab(
Event(test_task, 22, 1 ),
Event(test_task, 0, range(9,18,2), daysofweek=range(0,5)),
)
cron.run()

None of the listed solutions even attempt to parse a complex cron schedule string. So, here is my version, using croniter. Basic gist:
schedule = "*/5 * * * *" # Run every five minutes
nextRunTime = getNextCronRunTime(schedule)
while True:
roundedDownTime = roundDownTime()
if (roundedDownTime == nextRunTime):
####################################
### Do your periodic thing here. ###
####################################
nextRunTime = getNextCronRunTime(schedule)
elif (roundedDownTime > nextRunTime):
# We missed an execution. Error. Re initialize.
nextRunTime = getNextCronRunTime(schedule)
sleepTillTopOfNextMinute()
Helper routines:
from croniter import croniter
from datetime import datetime, timedelta
# Round time down to the top of the previous minute
def roundDownTime(dt=None, dateDelta=timedelta(minutes=1)):
roundTo = dateDelta.total_seconds()
if dt == None : dt = datetime.now()
seconds = (dt - dt.min).seconds
rounding = (seconds+roundTo/2) // roundTo * roundTo
return dt + timedelta(0,rounding-seconds,-dt.microsecond)
# Get next run time from now, based on schedule specified by cron string
def getNextCronRunTime(schedule):
return croniter(schedule, datetime.now()).get_next(datetime)
# Sleep till the top of the next minute
def sleepTillTopOfNextMinute():
t = datetime.utcnow()
sleeptime = 60 - (t.second + t.microsecond/1000000.0)
time.sleep(sleeptime)

I know there are a lot of answers, but another solution could be to go with decorators. This is an example to repeat a function everyday at a specific time. The cool think about using this way is that you only need to add the Syntactic Sugar to the function you want to schedule:
#repeatEveryDay(hour=6, minutes=30)
def sayHello(name):
print(f"Hello {name}")
sayHello("Bob") # Now this function will be invoked every day at 6.30 a.m
And the decorator will look like:
def repeatEveryDay(hour, minutes=0, seconds=0):
"""
Decorator that will run the decorated function everyday at that hour, minutes and seconds.
:param hour: 0-24
:param minutes: 0-60 (Optional)
:param seconds: 0-60 (Optional)
"""
def decoratorRepeat(func):
#functools.wraps(func)
def wrapperRepeat(*args, **kwargs):
def getLocalTime():
return datetime.datetime.fromtimestamp(time.mktime(time.localtime()))
# Get the datetime of the first function call
td = datetime.timedelta(seconds=15)
if wrapperRepeat.nextSent == None:
now = getLocalTime()
wrapperRepeat.nextSent = datetime.datetime(now.year, now.month, now.day, hour, minutes, seconds)
if wrapperRepeat.nextSent < now:
wrapperRepeat.nextSent += td
# Waiting till next day
while getLocalTime() < wrapperRepeat.nextSent:
time.sleep(1)
# Call the function
func(*args, **kwargs)
# Get the datetime of the next function call
wrapperRepeat.nextSent += td
wrapperRepeat(*args, **kwargs)
wrapperRepeat.nextSent = None
return wrapperRepeat
return decoratorRepeat

I like how the pycron package solves this problem.
import pycron
import time
while True:
if pycron.is_now('0 2 * * 0'): # True Every Sunday at 02:00
print('running backup')
time.sleep(60) # The process should take at least 60 sec
# to avoid running twice in one minute
else:
time.sleep(15) # Check again in 15 seconds

There isn't a "pure python" way to do this because some other process would have to launch python in order to run your solution. Every platform will have one or twenty different ways to launch processes and monitor their progress. On unix platforms, cron is the old standard. On Mac OS X there is also launchd, which combines cron-like launching with watchdog functionality that can keep your process alive if that's what you want. Once python is running, then you can use the sched module to schedule tasks.

Another trivial solution would be:
from aqcron import At
from time import sleep
from datetime import datetime
# Event scheduling
event_1 = At( second=5 )
event_2 = At( second=[0,20,40] )
while True:
now = datetime.now()
# Event check
if now in event_1: print "event_1"
if now in event_2: print "event_2"
sleep(1)
And the class aqcron.At is:
# aqcron.py
class At(object):
def __init__(self, year=None, month=None,
day=None, weekday=None,
hour=None, minute=None,
second=None):
loc = locals()
loc.pop("self")
self.at = dict((k, v) for k, v in loc.iteritems() if v != None)
def __contains__(self, now):
for k in self.at.keys():
try:
if not getattr(now, k) in self.at[k]: return False
except TypeError:
if self.at[k] != getattr(now, k): return False
return True

I don't know if something like that already exists. It would be easy to write your own with time, datetime and/or calendar modules, see http://docs.python.org/library/time.html
The only concern for a python solution is that your job needs to be always running and possibly be automatically "resurrected" after a reboot, something for which you do need to rely on system dependent solutions.

Is there a way to cache Python 3.5 definitions using a time-to-live decorator?

Currently, I use functools' lru_cache to handle my caching for the function. The problem is that the size of the cache never grows large enough to make use of the LRU (due to the fact that the function never takes in a parameter). Instead, the function, when called, opens up a specific URL and return its contents.
Is there a way in which I can specify the 'time to live' cache in which after a certain amount of time/certain amount of calls, it refreshes its cache?

I don't know of a decorator, but you can keep track of the last time you got the page and update as needed. If this is a single-threaded app, it can be simple
_cached_page = ''
_cached_page_time = 0
def get_page():
global _cached_page, _cached_page_time
now = time.time()
# invalidate in 1 hour
if not _cached_page or now - _cached_page_time > 60 * 60:
_cached_page = get_the_page_here()
_cached_page_time = time.time()
return _cached_page
You could also age the page with a timer in the background. You need to control access with a lock, but that makes the cache usable in a multithreaded program too.
_cached_page = ''
_cached_page_lock = threading.Lock()
def _invalidate_page():
global _cached_page
with _cached_page_lock:
_cached_page = ''
def get_page():
global _cached_page
with _cached_page_lock:
if not _cached_page:
_cached_page = get_the_page_here()
# invalidate in 1 hour
threading.Timer(60*60, _invalidate_page)
return _cached_page
Finally, the server may include an Expires: ... field in the http header. Depending on how well the service was written, this would be a good reflection of how long the page can be cached.

The functools.lru_cache function accepts a maxsize argument which saves the results up to the maxsize most recent calls.
You can check this by calling the cache_info attribute of your decorated function.
If you want to refresh your cache completely you should implement a cash object manually by counting the number of cache calls and resetting the cache whenever it hits the max size.
from functools import wraps
class Mycache(object):
def __init__(self, maxcount):
self.count = 0
self.maxcount = maxcount
self.cache = {}
def __call__(self, func):
#wraps(func)
def wrapped(*args):
self.count += 1
if self.count > self.maxcount:
self.cache = {}
self.count = 0
result = self.cache[args] = func(*args)
else:
try:
result = self.cache[args]
except KeyError:
result = self.cache[args] = func(*args)
return result
return wrapped
Demo:
#Mycache(3)
def a(arg):
print("arg is : {}".format(arg))
return arg ** 2
print(a(4))
print(a(4))
print(a(4))
print(a(4))
print(a(3))
print(a(4))
output:
arg is : 4
16
16
16
arg is : 4
16
arg is : 3
9
16

Set with expiration not working inside running Python script

I have this class called DecayingSet which is a deque with expiration
class DecayingSet:
def __init__(self, timeout): # timeout in seconds
from collections import deque
self.timeout = timeout
self.d = deque()
self.present = set()
def add(self, thing):
# Return True if `thing` not already in set,
# else return False.
result = thing not in self.present
if result:
self.present.add(thing)
self.d.append((time(), thing))
self.clean()
return result
def clean(self):
# forget stuff added >= `timeout` seconds ago
now = time()
d = self.d
while d and now - d[0][0] >= self.timeout:
_, thing = d.popleft()
self.present.remove(thing)
I'm trying to use it inside a running script, that connects to a streaming api.
The streaming api is returning urls that I am trying to put inside the deque to limit them from entering the next step of the program.
class CustomStreamListener(tweepy.StreamListener):
def on_status(self, status, include_entities=True):
longUrl = status.entities['urls'][0]['expanded_url']
limit = DecayingSet(86400)
l = limit.add(longUrl)
print l
if l == False:
pass
else:
r = requests.get("http://api.some.url/show?url=%s"% longUrl)
When i use this class in an interpreter, everything is good.
But when the script is running, and I repeatedly send in the same url, l returns True every time indicating that the url is not inside the set, when is supposed to be. What gives?

Copying my comment ;-) I think the indentation is screwed up, but it looks like you're creating a brand new limit object every time on_status() is called. Then of course it would always return True: you'd always be starting with an empty limit.
Regardless, change this:
l = limit.add(longUrl)
print l
if l == False:
pass
else:
r = requests.get("http://api.some.url/show?url=%s"% longUrl)
to this:
if limit.add(longUrl):
r = requests.get("http://api.some.url/show?url=%s"% longUrl)
Much easier to follow. It's usually the case that when you're comparing something to a literal True or False, the code can be made more readable.
Edit
i just saw in the interpreter the var assignment is the culprit.
How would I use the same obj?
You could, for example, create the limit object at the module level. Cut and paste ;-)

Python, function quit if it has been run the last 5 minutes

I have a python script that gets data from a USB weather station, now it puts the data into MySQL whenever the data is received from the station.
I have a MySQL class with an insert function, what i want i that the function checks if it has been run the last 5 minutes if it has, quit.
Could not find any code on the internet that does this.
Maybe I need to have a sub-process, but I am not familiar with that at all.
Does anyone have an example that I can use?

Use this timeout decorator.
import signal
class TimeoutError(Exception):
def __init__(self, value = "Timed Out"):
self.value = value
def __str__(self):
return repr(self.value)
def timeout(seconds_before_timeout):
def decorate(f):
def handler(signum, frame):
raise TimeoutError()
def new_f(*args, **kwargs):
old = signal.signal(signal.SIGALRM, handler)
signal.alarm(seconds_before_timeout)
try:
result = f(*args, **kwargs)
finally:
signal.signal(signal.SIGALRM, old)
signal.alarm(0)
return result
new_f.func_name = f.func_name
return new_f
return decorate
Usage:
import time
#timeout(5)
def mytest():
print "Start"
for i in range(1,10):
time.sleep(1)
print "%d seconds have passed" % i
if __name__ == '__main__':
mytest()

Probably the most straight-forward approach (you can put this into a decorator if you like, but that's just cosmetics I think):
import time
import datetime
class MySQLWrapper:
def __init__(self, min_period_seconds):
self.min_period = datetime.timedelta(seconds=min_period_seconds)
self.last_calltime = datetime.datetime.now() - self.min_period
def insert(self, item):
now = datetime.datetime.now()
if now-self.last_calltime < self.min_period:
print "not insert"
else:
self.last_calltime = now
print "insert", item
m = MySQLWrapper(5)
m.insert(1) # insert 1
m.insert(2) # not insert
time.sleep(5)
m.insert(3) # insert 3
As a side-note: Have you noticed RRDTool during your web-search for related stuff? It does apparantly what you want to achieve, i.e.
a database to store the most recent values of arbitrary resolution/update frequency.
extrapolation/interpolation of values if updates are too frequent or missing.
generates graphs from the data.
An approach could be to store all data you can get into your MySQL database and forward a subset to such RRDTool database to generate a nice time series visualization of it. Depending on what you might need.

import time
def timeout(f, k, n):
last_time = [time.time()]
count = [0]
def inner(*args, **kwargs):
distance = time.time() - last_time[0]
if distance > k:
last_time[0] = time.time()
count[0] = 0
return f(*args, **kwargs)
elif distance < k and (count[0]+1) == n:
return False
else:
count[0] += 1
return f(*args, **kwargs)
return inner
timed = timeout(lambda x, y : x + y, 300, 1)
print timed(2, 4)
First argument is the function you want run, second is the time interval, and the third is the number of times it's allowed to run in that time interval.

Each time the function is run save a file with the current time. When the function is run again check the time stored in the file and make sure it is old enough.

Just derive to a new class and override the insert function. In the overwriting function, check last insert time and call father's insert method if it has been more than five minutes, and of course update the most recent insert time.

Why is this recursive statement wrong?

This is a bank simulation that takes into account 20 different serving lines with a single queue, customers arrive following an exponential rate and they are served during a time that follows a normal probability distribution with mean 40 and standard deviation 20.
Things were working just fine till I decided to exclude the negative values given by the normal distribution using this method:
def getNormal(self):
normal = normalvariate(40,20)
if (normal>=1):
return normal
else:
getNormal(self)
Am I screwing up the recursive call? I don't get why it wouldn't work. I have changed the getNormal() method to:
def getNormal(self):
normal = normalvariate(40,20)
while (normal <=1):
normal = normalvariate (40,20)
return normal
But I'm curious on why the previous recursive statement gets busted.
This is the complete source code, in case you're interested.
""" bank21: One counter with impatient customers """
from SimPy.SimulationTrace import *
from random import *
## Model components ------------------------
class Source(Process):
""" Source generates customers randomly """
def generate(self,number):
for i in range(number):
c = Customer(name = "Customer%02d"%(i,))
activate(c,c.visit(tiempoDeUso=15.0))
validateTime=now()
if validateTime<=600:
interval = getLambda(self)
t = expovariate(interval)
yield hold,self,t #esta es la rata de generación
else:
detenerGeneracion=999
yield hold,self,detenerGeneracion
class Customer(Process):
""" Customer arrives, is served and leaves """
def visit(self,tiempoDeUso=0):
arrive = now() # arrival time
print "%8.3f %s: Here I am "%(now(),self.name)
yield (request,self,counter),(hold,self,maxWaitTime)
wait = now()-arrive # waiting time
if self.acquired(counter):
print "%8.3f %s: Waited %6.3f"%(now(),self.name,wait)
tiempoDeUso=getNormal(self)
yield hold,self,tiempoDeUso
yield release,self,counter
print "%8.3f %s: Completed"%(now(),self.name)
else:
print "%8.3f %s: Waited %6.3f. I am off"%(now(),self.name,wait)
## Experiment data -------------------------
maxTime = 60*10.5 # minutes
maxWaitTime = 12.0 # minutes. maximum time to wait
## Model ----------------------------------
def model():
global counter
#seed(98989)
counter = Resource(name="Las maquinas",capacity=20)
initialize()
source = Source('Source')
firstArrival= expovariate(20.0/60.0) #chequear el expovariate
activate(source,
source.generate(number=99999),at=firstArrival)
simulate(until=maxTime)
def getNormal(self):
normal = normalvariate(40,20)
if (normal>=1):
return normal
else:
getNormal(self)
def getLambda (self):
actualTime=now()
if (actualTime <=60):
return 20.0/60.0
if (actualTime>60)and (actualTime<=120):
return 25.0/60.0
if (actualTime>120)and (actualTime<=180):
return 40.0/60.0
if (actualTime>180)and (actualTime<=240):
return 30.0/60.0
if (actualTime>240)and (actualTime<=300):
return 35.0/60.0
if (actualTime>300)and (actualTime<=360):
return 42.0/60.0
if (actualTime>360)and (actualTime<=420):
return 50.0/60.0
if (actualTime>420)and (actualTime<=480):
return 55.0/60.0
if (actualTime>480)and (actualTime<=540):
return 45.0/60.0
if (actualTime>540)and (actualTime<=600):
return 10.0/60.0
## Experiment ----------------------------------
model()

I think you want
return getnormal(self)
instead of
getnormal(self)
If the function exits without hitting a return statement, then it returns the special value None, which is a NoneType object - that's why Python complains about a 'NoneType.' The abs() function wants a number, and it doesn't know what to do with a None.
Also, you could avoid recursion (and the cost of creating a new stack frame) by using
def getNormal(self):
normal = 0
while normal < 1:
normal = normalvariate(40,20)
return normal

I am not entirely sure, but I think you need to change your method to the following:
def getNormal(self):
normal = normalvariate(40,20)
if (normal>=1):
return normal
else:
return getNormal(self)

You need to have:
return getNormal(self)
instead of
getNormal(self)
Really though, there's no need for recursion:
def getNormal(self):
normal = 0
while normal < 1:
normal = normalvariate(40,20)
return normal