Develop Reference

Problem with Python scheduler, can´t get it to execute function every night

In my program I have a scheduler that I can´t get to work. It´s supposed to run a function every night at 00:10 for a web scraper. (def scrape() below) If I change the scrape function to a simple function like this it runs and print out to console every night.
def testTime():
print("test schedule")
But if I run my program looking like this it´s won´t do the scrape every night. If I restart the program and run it everything is fine and I get my data into my variable "fixed_scrape" but restarting the program every day isn´t how I want it.. It´s running on a Raspberry Pi.
What am I´m missing or doing wrong? Have read something about using Ischedule insted but won´t get it to work either.. I´m new into Python...
from multiprocessing.sharedctypes import Value
from optparse import Values
import re, requests, ast
from prompt_toolkit import print_formatted_text
import RPi.GPIO as GPIO
import time
from heapq import nsmallest
import schedule
import pip._vendor.requests
HoursPerDay = 10
PriceLimit = 80
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)
GPIO.setup(18, GPIO.OUT, initial=GPIO.LOW)
def scrape():
r = requests.get('https://www.elbruk.se/timpriser-se3-stockholm')
return dict(zip([i[0] for i in re.findall(r"'((2[0-4]|[01]?[0-9]):([0-5]?[0-9]))'", r.text)],
ast.literal_eval(re.search(r"data: .*(\[.*?\])[\s\S]+(?='Idag snitt')", r.text).group(1))))
fixed_scrape = scrape()
def cheap():
my_dict = fixed_scrape
cheapest_hour = nsmallest(HoursPerDay, my_dict, key=my_dict.get)
return [(i.removesuffix(':00')).lstrip('0') or 0 for i in cheapest_hour]
def threshold():
price_hour = fixed_scrape
price_hour = {k:v for k,v in price_hour.items() if v < PriceLimit}
dick = nsmallest(24, price_hour, key=price_hour.get)
return ([(i.removesuffix(':00')).lstrip('0') or 0 for i in dick])
schedule.every().day.at("00:10").do(scrape)
while True:
schedule.run_pending()
now_time = time.strftime("%H")
cheapest_hours = cheap()
cheap_threshold = threshold()
print(cheapest_hours, "Are the cheapest hours today")
print(cheap_threshold, "Hours when price is under" , PriceLimit, "öre/kwh. (Buying on threshold)")
print("Active hour now is: ",now_time)
if now_time in cheap_threshold:
GPIO.output(18, GPIO.HIGH)
elif now_time in cheapest_hours:
GPIO.output(18, GPIO.HIGH)
else:
GPIO.output(18, GPIO.LOW)
time.sleep(30)

Think I got the solution with implementing this and run it in a separate thread. Seems to work so far.
def run_continuously():
cease_continuous_run = threading.Event()
class ScheduleThread(threading.Thread):
#classmethod
def run(cls):
while not cease_continuous_run.is_set():
schedule.run_pending()
continuous_thread = ScheduleThread()
continuous_thread.start()
return cease_continuous_run
def background_job_1():
logging.info("Background 1 thread running...")
#schedules new scrape with prices from site every day
if __name__ == "__main__":
format = "%(asctime)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO, datefmt="%H:%M:%S")
logging.info("Main - starting thread")
schedule.every(30).seconds.do(background_job_1)
schedule.every().day.at("00:10").do(scrape)
schedule.every().day.at("03:00").do(scrape)
schedule.every().day.at("10:00").do(scrape)
stop_run_continuously = run_continuously()
logging.info("Main: completed !!")
time.sleep(10)
#stop_run_continuously.set()
#run the program
while True:
schedule.run_pending()

Python does something every 5 minutes

i need to check data on an API. The API is refreshed with new data every 5 minutes (10:00, 10:05, 10:10 etc...)
I don't want to use time.sleep(300) because i want my script to do something at 10:05:03, then 10:05:03 etc. and not 5 min avec the script started (maybe it started at 10h12
How can i build this?
Thanks y'all.

UPDATE:
Just wanted to remove the possibility of recursion error, so I have rewritten the code:
from threading import Thread
from time import sleep
import datetime
def check_api():
# ... your code here ...
pass
def schedule_api():
while datetime.datetime.now().minute % 5 != 0:
sleep(1)
check_api()
while True:
sleep(300)
check_api()
thread = Thread(target=schedule_api)
thread.start()
Also if you want your thread to quit when the main program exits you could set daemon as True on the thread like:
thread.daemon = True
But this does not enforce a clean termination of this thread so you could also try this approach below:
# ...
RUNNING = True
# ...
thread = Thread(target=schedule_api)
thread.start()
#...
def main():
# ... all main code ...
pass
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
RUNNING = False
You can use the following code:
import threading
def check_api():
pass
timer_thread = threading.Timer(300, check_api)
timer_thread.start()
# call timer_thread.cancel() when you need it to stop
This will call your check_api function every 5 minutes and will not block your main code's execution.
as mentioned by #scotyy3785 the above code will only run once but I realize what you want and have written the code for it:
from threading import Thread
from time import sleep
import datetime
def check_api():
# ... your code here ...
pass
def caller(callback_func, first=True):
if first:
while not datetime.datetime.now().minute % 5 == 0:
sleep(1)
callback_func()
sleep(300)
caller(callback_func, False)
thread = Thread(target=caller, args=(check_api,))
thread.start()
# you'll have to handle the still running thread on exit
The above code will call check_api at minutes like 00, 05, 10, 15...

Check the time regularly in a loop and do something at certain minute marks:
import time
# returns the next 5 minute mark
# e.g. at minute 2 return 5
def get_next_time():
minute = time.localtime().tm_min
result = 5 - (minute % 5) + minute
if result == 60:
result = 0
return result
next_run = get_next_time()
while True:
now = time.localtime()
# at minute 0, 5, 10... + 3 seconds:
if next_run == now.tm_min and now.tm_sec >= 3:
print("checking api")
next_run = get_next_time()
time.sleep(1)

Python time delay

Alright, I want to know how to delay a portion of a program without pausing the entire program.
I'm not necessarily good at python so if you could give me a relatively simple answer if possible, that would be great.
I want to have a turtle draw a circle on screen every time this function is called, this is what I have:
import time
from random import randint
turtle5 = turtle.Turtle()
coinx = randint(-200, 200)
coiny = randint(-200, 200)
turtle5.pu()
turtle5.goto(coinx, coiny)
turtle5.pd()
turtle5.begin_fill()
turtle5.fillcolor("Gold")
turtle5.circle(5, 360, 8)
turtle5.end_fill()
time.sleep(1)
turtle5.clear()

There is turtle.ontimer() that calls a function with the specified delay:
turtle.ontimer(your_function, delay_in_milliseconds)

You need to put the part of the program you want to delay in its own thread, and then call sleep() in that thread.
I am not sure exactly what you are trying to do in your example, so here is a simple example:
import time
import threading
def print_time(msg):
print 'The time %s is: %s.' % (msg, time.ctime(time.time()))
class Wait(threading.Thread):
def __init__(self, seconds):
super(Wait, self).__init__()
self.seconds = seconds
def run(self):
time.sleep(self.seconds)
print_time('after waiting %d seconds' % self.seconds)
if __name__ == '__main__':
wait_thread = Wait(5)
wait_thread.start()
print_time('now')
Output:
The time now is: Mon Jan 12 01:57:59 2015.
The time after waiting 5 seconds is: Mon Jan 12 01:58:04 2015.
Notice that we started the thread that will wait 5 seconds first, but it did not block the print_time('now') call, rather it waited in the background.
EDIT:
From J.F. Sebastian's comment, the simpler solution with threading is:
import time
import threading
def print_time(msg):
print 'The time %s is: %s.' % (msg, time.ctime(time.time()))
if __name__ == '__main__':
t = threading.Timer(5, print_time, args = ['after 5 seconds'])
t.start()
print_time('now')

Multi threading in python using parallel threads

I created two threads each running different functions.
What i tryed to achieve is if first thread ends then the second should also end ( i tryed achieving it using global variable)
Once both the threads end the same procedure should continue.
The script is not working as expected.
I am using Linux - Centos and python 2.7
#!/usr/bin/python
import threading
import time
import subprocess
import datetime
import os
import thread
command= "strace -o /root/Desktop/a.txt -c ./server"
final_dir = "/root/Desktop/"
exitflag = 0
# Define a function for the thread
def print_time(*args):
os.chdir(final_dir)
print "IN first thread"
proc = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.wait(70)
exitflag=1
def print_time1(*args):
print "In second thread"
global exitflag
while exitflag:
thread.exit()
#proc = subprocess.Popen(command1,shell=True,stdout=subprocess.PIPE, sterr=subprocess.PIPE)
# Create two threads as follows
while (1):
t1=threading.Thread(target=print_time)
t1.start()
t2=threading.Thread(target=print_time1)
t2=start()
time.sleep(80)
z = t1.isAlive()
z1 = t2.isAlive()
if z:
z.exit()
if z1:
z1.exit()
threading.Thread(target=print_time1).start()
threading.Thread(target=print_time1).start()
print "In try"
Where am i going wrong?

You could create an object to share state, and have the dependent thread check that state. Something like:
import threading
import time
import datetime
class Worker1( threading.Thread ):
def __init__(self, state):
super(Worker1, self).__init__()
self.state = state
def run(self):
print_time_helper("Worker1 Start")
time.sleep(4)
print_time_helper("Worker1 End")
self.state.keepOnRunning = False
class Worker2( threading.Thread ):
def __init__(self, state):
super(Worker2, self).__init__()
self.state = state
def run(self):
while self.state.keepOnRunning:
print_time_helper("Worker2")
time.sleep(1)
class State( object ):
def __init__(self):
self.keepOnRunning = True
def main():
state = State()
thread1 = Worker1(state)
thread2 = Worker2(state)
thread1.start()
thread2.start()
thread1.join()
thread2.join()
def print_time_helper(name):
print "{0}: {1}".format(name, datetime.datetime.now().time().strftime("%S"))
which will output something like this (numbers show current time seconds):
Worker1 Start: 39
Worker2: 39
Worker2: 40
Worker2: 41
Worker2: 42
Worker1 End: 43
However, this is a bit simplistic for most situations. You might be better off using message queues - this is a good intro.

Use a threading.Event instead of an int and wait for it to be set.
Also your logic appears to be wrong in print_time1 because your while loop will never run since exitflag is initially 0, but even if it was 1 it would still just exit immediately. It's not actually waiting on anything.

Can this loop be sped up in pure Python?

I was trying out an experiment with Python, trying to find out how many times it could add one to an integer in one minute's time. Assuming two computers are the same except for the speed of the CPUs, this should give an estimate of how fast some CPU operations may take for the computer in question.
The code below is an example of a test designed to fulfill the requirements given above. This version is about 20% faster than the first attempt and 150% faster than the third attempt. Can anyone make any suggestions as to how to get the most additions in a minute's time span? Higher numbers are desireable.
EDIT 1: This experiment is being written in Python 3.1 and is 15% faster than the fourth speed-up attempt.
def start(seconds):
import time, _thread
def stop(seconds, signal):
time.sleep(seconds)
signal.pop()
total, signal = 0, [None]
_thread.start_new_thread(stop, (seconds, signal))
while signal:
total += 1
return total
if __name__ == '__main__':
print('Testing the CPU speed ...')
print('Relative speed:', start(60))
EDIT 2: Regarding using True instead of 1 in the while loop: there should be no speed difference. The following experiment proves that they are the same. First, create a file named main.py and copy the following code into it.
def test1():
total = 0
while 1:
total += 1
def test2():
total = 0
while True:
total += 1
if __name__ == '__main__':
import dis, main
dis.dis(main)
Running the code should produce the following output that shows how the code was actually compiled and what the generated Python Virtual Machine Instructions turned out to be.
Disassembly of test1:
2 0 LOAD_CONST 1 (0)
3 STORE_FAST 0 (total)
3 6 SETUP_LOOP 13 (to 22)
4 >> 9 LOAD_FAST 0 (total)
12 LOAD_CONST 2 (1)
15 INPLACE_ADD
16 STORE_FAST 0 (total)
19 JUMP_ABSOLUTE 9
>> 22 LOAD_CONST 0 (None)
25 RETURN_VALUE
Disassembly of test2:
7 0 LOAD_CONST 1 (0)
3 STORE_FAST 0 (total)
8 6 SETUP_LOOP 13 (to 22)
9 >> 9 LOAD_FAST 0 (total)
12 LOAD_CONST 2 (1)
15 INPLACE_ADD
16 STORE_FAST 0 (total)
19 JUMP_ABSOLUTE 9
>> 22 LOAD_CONST 0 (None)
25 RETURN_VALUE
The emitted PVMIs (byte codes) are exactly the same, so both loops should run without any difference in speed.

I see almost the same but consistently better (~2%) results than the #Amber's one on my machine on Python 3.1.2 for the code:
import signal
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
def jfs_signal(seconds):
# set signal handler
signal.signal(signal.SIGALRM, alarm_handler)
# raise Alarm in `seconds` seconds
signal.alarm(seconds)
total = 0
try:
while 1:
total += 1
finally:
signal.alarm(0) # disable the alarm
return total
Here's variant that uses subprocess module to run interruptible loop:
#!/usr/bin/env python
# save it as `skytower.py` file
import atexit
import os
import signal
import subprocess
import sys
import tempfile
import time
def loop():
#atexit.register
def print_total():
print(total)
total = 0
while 1:
total += 1
def jfs_subprocess(seconds):
# start process, redirect stdout/stderr
f = tempfile.TemporaryFile()
p = subprocess.Popen([sys.executable, "-c",
"from skytower import loop; loop()"],
stdout=f, stderr=open(os.devnull, 'wb'))
# wait
time.sleep(seconds)
# raise KeyboardInterrupt
#NOTE: if it doesn't kill the process then `p.wait()` blocks forever
p.send_signal(signal.SIGINT)
p.wait() # wait for the process to terminate otherwise the output
# might be garbled
# return saved output
f.seek(0) # rewind to the beginning of the file
d = int(f.read())
f.close()
return d
if __name__ == '__main__':
print('total:', jfs_subprocess(60))
It is ~20% slower than the signal.alarm()'s variant on my machine.

About a 20-25% improvement, FWIW - but like others, I'd propose that Python incrementing integers probably isn't the best benchmarking tool.
def start(seconds):
import time, _thread
def stop(seconds):
time.sleep(seconds)
_thread.interrupt_main()
total = 0
_thread.start_new_thread(stop, (seconds,))
try:
while True:
total += 1
except:
return total
if __name__ == '__main__':
print('Testing the CPU speed ...')
print('Relative speed:', start(60))

This exercise on learning more about Python and computers was satisfying. This is the final program:
def start(seconds, total=0):
import _thread, time
def stop():
time.sleep(seconds)
_thread.interrupt_main()
_thread.start_new_thread(stop, ())
try:
while True:
total += 1
except KeyboardInterrupt:
return total
if __name__ == '__main__':
print('Testing the CPU speed ...')
print('Relative speed:', start(60))
Running it on Windows 7 Professional with a 2.16 GHz CPU produced the following output within IDLE:
Python 3.1.3 (r313:86834, Nov 27 2010, 18:30:53) [MSC v.1500 32 bit (Intel)]
on win32
Type "copyright", "credits" or "license()" for more information.
>>> ================================ RESTART ================================
>>>
Testing the CPU speed ...
Relative speed: 673991388
>>>
Edit: The code up above only runs on one core. The following program was written to fix that problem.
#! /usr/bin/env python3
def main(seconds):
from multiprocessing import cpu_count, Barrier, SimpleQueue, Process
def get_all(queue):
while not queue.empty():
yield queue.get()
args = seconds, Barrier(cpu_count()), SimpleQueue()
processes = [Process(target=run, args=args) for _ in range(cpu_count())]
for p in processes:
p.start()
for p in processes:
p.join()
print('Relative speed:', sorted(get_all(args[-1]), reverse=True))
def run(seconds, barrier, queue):
from time import sleep
from _thread import interrupt_main, start_new_thread
def terminate():
sleep(seconds)
interrupt_main()
total = 0
barrier.wait()
start_new_thread(terminate, ())
try:
while True:
total += 1
except KeyboardInterrupt:
queue.put(total)
if __name__ == '__main__':
main(60)