I have following normal function which creates Celery Group and tries to run all the subtasks in the Group at specific time:
def run_sms_task(smstask):
if smstask:
phones = []
for user in smstask.userlist.users.all():
phones.append(user.profile.phone)
tasks = []
for phone in phones:
tasks.append(send_sms_async.s(phone, smstask.text))
job = group(tasks)
result = job.apply_async(eta=smstask.starts_at)
result.save()
return result.id
return None
All the subtasks are fired when I call this function and not at the defined 'starts_at'. What is wrong? Thanks!
P.S. For testing reasons I have wrote a function which works fine for me if to launch tasks separately:
def run_sms_task_test1(smstask):
if smstask:
phones = []
for user in smstask.userlist.users.all():
phones.append(user.profile.phone)
tasks = []
for phone in phones:
send_sms_async.apply_async([phone, smstask.text], eta=smstask.starts_at)
return None
May be this is timezone issue.
Try to use following solution
from pytz import timezone
from datetime import datetime
london_tz = pytz.timezone('Europe/London')
london_dt = london_tz.localize(datetime.datetime(year, month, day, hour, min))
starts_at = london_dt.astimezone(pytz.UTC)
result = job.apply_async(eta=starts_at)
Hope this is helps you.
Related
Is it possible to use multi processing in Django on a request.
#so if I send a request to http://127.0.0.1:8000/wallet_verify
def wallet_verify(request):
walelts = botactive.objects.all()
#here I check if the user want to be included in the process or not so if they set it to True then i'll include them else ignore.
for active in walelts:
check_active = active.active
if check_active == True:
user_is_active = active.user
#for the ones that want to be included I then go to get their key data.
I need to get both api and secret so then I loop through to get the data from active users.
database = Bybitapidatas.objects.filter(user=user_is_active)
for apikey in database:
apikey = apikey.apikey
for apisecret in database:
apisecret = apisecret.apisecret
#since I am making a request to an exchange endpoint I can only include one API and secret at a time . So for 1 person at a time this is why I want to run in parallel.
for a, b in zip(list(Bybitapidatas.objects.filter(user=user_is_active).values("apikey")), list(Bybitapidatas.objects.filter(user=user_is_active).values("apisecret"))):
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=a['apikey'], api_secret=b['apisecret'])
#here I check to see if they have balance to open trades if they have selected to be included.
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
#if they don't have enough balance I skip the user.
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
How can I run this process in parallel while looping through the database to also get data for each individual user.
I am still new to coding so hope I explained that it makes sense.
I have tried multiprocessing and pools but then I get that the app has not started yet and I have to run it outside of wallet_verify is there a way to do it in wallet_verify
and when I send the Post Request.
Any help appreciated.
Filtering the Database to get Users who have set it to True
Listi - [1,3](these are user ID's Returned
processess = botactive.objects.filter(active=True).values_list('user')
listi = [row[0] for row in processess]
Get the Users from the listi and perform the action.
def wallet_verify(listi):
# print(listi)
database = Bybitapidatas.objects.filter(user = listi)
print("---------------------------------------------------- START")
for apikey in database:
apikey = apikey.apikey
print(apikey)
for apisecret in database:
apisecret = apisecret.apisecret
print(apisecret)
start_time = time.time()
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=apikey, api_secret=apisecret)
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
print ("My program took", time.time() - start_time, "to run")
print("---------------------------------------------------- END")
return HttpResponse("Wallets verified")
Verifyt is what I use for the multiprocessing since I don't want it to run without being requested to run. also initialiser starts apps for each loop
def verifyt(request):
with ProcessPoolExecutor(max_workers=4, initializer=django.setup) as executor:
results = executor.map(wallet_verify, listi)
return HttpResponse("done")
```
I really appreciate everyone who contributed to my first question though am new here If I didn't ask in a way you can understand do bear with me.
I have a database and in it a table name contracts which the data in the rows are contract and group, i.e each each group has a single contract.
I need help so that once the bot is started it checks the database and list of contracts in it then it runs task for each group and their own contracts at same time...
This codes below is what I was able to work on but it just works for only one group
from web3 import Web3
import json,os,sys
import datetime
import requests
from db import *
import telebot
bot = telebot.TeleBot("Telegram bot api will be here")
bsc = "https://bsc-dataseed.binance.org/"
web3 = Web3(Web3.HTTPProvider(bsc))
class BuyBot():
def init(self,token,group):
self.token = token
self.group = group
def main(self):
print(f"EVENTS RUNNING FOR {self.token}")
event_filter = web3.eth.filter({"address": web3.toChecksumAddress((self.token).upper()), "block_identifier": 'pending', })
self.log_loop(event_filter, 2)
def log_loop(self,event_filter, poll_interval):
try:
while True:
for event in event_filter.get_new_entries():
self.handle_event(event)
break
time.sleep(poll_interval)
except Exception as p:
print(p)
self.main()
def handle_event(self,event):
txn = json.loads(Web3.toJSON(event))
hash = txn['transactionHash']
if len(hash) > 0:
transaction_recipt = web3.eth.wait_for_transaction_receipt(hash)
trs = web3.toHex(transaction_recipt['transactionHash'])
usl_hash = f"https://bscscan.com/tx/{trs}"
bot.send_message(self.group,url_hash)
def main():
while True:
print("am here")
con = bot_db()
mc= con.cursor()
mc.execute('SELECT * FROM Contracts')
rows = mc.fetchall()
for all in rows:
group = all[1]
ca = all[0]
check_ = BuyBot(ca,group)
check_.main()
bot.polling(none_stop=True)
if name == "main":
try:
print("Started")
main()
except Exception:
print("rebooted")
main()
Please do help I need it to listen to events on the Bsc chain web3 for each group at same time with each group contract.
As I stated below please do bear with me if you don't understand what I meant
Was able to do what I needed multiprocessing did the job I needed ✅
Here is the documentation
https://docs.python.org/3/library/multiprocessing.html
Hello i have a Person Detector script i want to send an info if any person detected by mail.In order to prevent mail spamming i need a timer for sendMail function.Function might be triggered anytime but it will only respond if its not on cooldown.
I tried using async task but couldn't implemented because if a person detected it goes to a loop where it sends email every 5 minutes even if there isn’t any person detected after the first sight.
Example:
Person detection script is running.
Person detected on camera -> Send an email(start the 5 minute cooldown)
Person sighted again after 2 minutes(didn't send any email because there is still 3 minutes cooldown).
Person sighted after 6 minutes send another email(because 5 minute cooldown is over).
Summary of my code.(Necessary parts only detection and sending mail works cooldown (timer) doesn't work
async def sendAlert():
server.sendmail(sender_email, receiver_email, message)
print('sent!')
await asyncio.sleep(300)
if __name__ == "__main__":
while True:
for i in range(len(boxes)):
if classes[i] == 1 and scores[i] > threshold:
with smtplib.SMTP_SSL("smtp.gmail.com", port, context=context) as server:
sendAlert(server)
box = boxes[i]
cv2.rectangle(img,(box[1],box[0]),(box[3],box[2]),(255,0,0),2)
If there is a person detected, script will send an alert by email.Afterwards if a person detected again in 5 minutes sendAlert function shouldn't respond until 5 minutes passed
I agree with #Prune that you need to create a small (minimal) use-case and present your code so that it is not only relevant to you, but also relevant to others. Additionally, your question should have a section with a verifiable example. Without these attributes, your question becomes hard for people to grasp, solve and/or suggest any verifiable solution.
However, as I understand, you have some action (sending email if a person is detected) that you would like to perform after certain cool-off period. So, in other words, you want a mechanism of keeping track of time. Hence, you would need the datetime library.
So, your pseudo code should look something like this:
Pseudo Code
import datetime
start = capture_timestamp()
cutoff = '00:05:00'
dt_cutoff = read_in_cutoff_as_timedelta(cutoff)
if person_detected:
now = capture_timestamp()
dt = now - start
if dt >= dt_cutoff:
# Send notification
send_email_notification()
else:
# Do not Send Notification
print('now: {} | dt: {}'.format(now, dt))
You could use datetime.datetime.utcnow() for timestamp. And datetime.timedelta() for defining dt_cutoff. For reading in a time-string as time you could do this:
tm = datetime.datetime.strptime(cutoff).time()
dt_cutoff = datetime.timedelta(hours = tm.hour, minutes = tm.minute, seconds = tm.second)
I hope this gives you some idea about how to model this.
Additional Resources
https://www.guru99.com/date-time-and-datetime-classes-in-python.html
https://docs.python.org/3/library/datetime.html
https://thispointer.com/python-how-to-convert-a-timestamp-string-to-a-datetime-object-using-datetime-strptime/
Complete Solution
Now, finally if you are in a hurry to use a ready made solution, you may use the following class object as shown. All you would need is to instantiate the class object by specifying your cool-off-period (timer_cutoff) and then call the method is_timeout(). If this returns True, then you send your notification. There is also an obj.istimeout attribute which stores this decision (True/False).
import time
# Set cutoff time to 2 seconds to test the output
# after 5 seconds: expect istimeout = True
# and instantiate the TimeTracker class object.
ttk = TimeTracker(timer_cutoff = '00:00:02') # 'HH:MM:SS'
# Wait for 3 seconds
time.sleep(3)
print('start timestamp: {}'.format(ttk.timestamp_start_str))
print('cutoff timestamp'.format(ttk.timestamp_cutoff_str))
print('timer_cutoff: {}'.format(ttk.timer_cutoff_str))
# Now check if cutoff time reached
ttk.is_timeout()
print('Send Notification: {}'.format(ttk.istimeout))
print('now_timestamp: {}'.format(ttk.timestamp_now_str))
class TimeTracker
Here is the class TimeTracker class:
import datetime
class TimeTracker(object):
def __init__(self,
timer_cutoff = '00:05:00',
cutoff_strformat = '%H:%M:%S'):
self.timer_cutoff_str = timer_cutoff
self.cutoff_strformat = cutoff_strformat
self.timestamp_start, self.timestamp_start_str = self.get_timestamp()
self.dt_cutoff = None # timedelta for cutoff
self.timestamp_cutoff = None
self.timestamp_cutoff_str = None
self.update_timestamp_cutoff()
self.timestamp_now = None
self.timestamp_now_str = None
self.dt_elapsed = None
self.istimeout = False
def get_timestamp(self):
ts = datetime.datetime.utcnow()
tss = str(ts)
return (ts, tss)
def readin_cutoff_as_timedelta(self):
td = datetime.datetime.strptime(self.timer_cutoff_str,
self.cutoff_strformat)
tdm = td.time()
self.dt_cutoff = datetime.timedelta(hours = tdm.hour,
minutes = tdm.minute,
seconds = tdm.second)
def update_timestamp_cutoff(self):
self.readin_cutoff_as_timedelta()
self.timestamp_cutoff = self.timestamp_start + self.dt_cutoff
self.timestamp_cutoff_str = str(self.timestamp_cutoff)
def time_elapsed(self):
self.dt_elapsed = self.timestamp_now - self.timestamp_start
def is_timeout(self):
self.timestamp_now, self.timestamp_now_str = self.get_timestamp()
self.time_elapsed()
if (self.dt_elapsed < self.dt_cutoff):
self.istimeout = False
else:
self.istimeout = True
return self.istimeout
I'm trying to run a unit test on a celery task that I have set to run daily.
I have tried importing the function and calling it in my test but this doesn't work.
The task is:
#shared_task
def create_a_notification_if_a_product_is_in_or_out_of_season():
"""
Send a notification if a product is now in or out of season
"""
julian_date = date.today().timetuple().tm_yday + 1
active_products = Product.objects.filter(status='ACTIVE')
for products in active_products:
in_season_prd = ProductDescription.objects.filter(
product=products,
early_start_julian=julian_date
)
for prd in in_season_prd:
notification = Notification()
notification.type = notification_choices.PRODUCT_IN_SEASON
notification.description = str(prd.product.name) + " will be in season from tomorrow."
notification.save()
and here is an example of one of my tests:
def test_when_product_is_about_to_come_in_to_seasonality(self):
"""
Make a notification when a product is due to come in to seasonality tomorrow
"""
p = Product.objects.first()
p.status = "ACTIVE"
today = date.today().timetuple().tm_yday
p.early_start_julian = today + 1
create_a_notification_if_a_product_is_in_or_out_of_season()
updated_notifications = Notification.objects.all().count()
self.assertNotEqual(self.current_notifications, updated_notifications)
Any help would be appreciated!
Thanks
You can apply() your celery task to execute it synchronously:
def test_when_product_is_about_to_come_in_to_seasonality(self):
"""
Make a notification when a product is due to come in to seasonality tomorrow
"""
p = Product.objects.first()
p.status = "ACTIVE"
today = date.today().timetuple().tm_yday
p.early_start_julian = today + 1
create_a_notification_if_a_product_is_in_or_out_of_season.apply()
updated_notifications = Notification.objects.all().count()
self.assertNotEqual(self.current_notifications, updated_notifications)
I think you're looking for CELERY_ALWAYS_EAGER setting. If set to True it will run your tasks synchronously. You can set it in your test settings or you can decorate only that test with #override_settings(CELERY_ALWAYS_EAGER=True)
import celery
def temptask(n):
header=list(tempsubtask.si(i) for i in range(n))
callback=templink.si('printed at last?')
r = celery.chord(celery.group(header))(callback)
return r
#task()
def tempsubtask(i):
print i
for x in range(i):
time.sleep(2)
current_task.update_state(
state='PROGRESS', meta={'completed': x, 'total': i })
#task()
def templink(x):
print 'this should be run at last %s'%x
#executing temptask
r = temptask(100)
I want acccess to the progress status updated by tempsubtask. How can I go about achieving it?
I've had a similar question. Most examples on the net are outdated, the docs didn't help much, but the docs have links to sources, reading which did help me.
My objective was to organize parallel tasks in groups. The groups would have to be executed sequentially in order.
So I decided to generate the task ids before starting any tasks separately and only assigning them. I'm using Celery 4.3.0
Here's a brief example.
Firstly I needed a dummy task to make execution sequential and to be able to check the state of a certain group. As this is used a callback, it will complete only after all other tasks in the group.
#celery.task(bind=True, name="app.tasks.dummy_task")
def dummy_task( self, results=None, *args, **kwargs ):
return results
My comments here explain how I assign ids.
from celery.utils import uuid
from celery import group, chord, chain
# Generating task ids,
# which can be saved to a db, sent to the client and so on
#
# This is done before executing any tasks
task_id_1 = uuid()
task_id_2 = uuid()
chord_callback_id_1 = uuid()
chord_callback_id_2 = uuid()
workflow_id = None
# Generating goups, using signatures
# the group may contain any number of tasks
group_1 = group(
[
celery.signature(
'app.tasks.real_task',
args=(),
kwargs = { 'email': some_email, 'data':some_data },
options = ( {'task_id': task_id_1 } )
)
]
)
group_2 = group(
[
celery.signature(
'app.tasks.real_task',
args=(),
kwargs = { 'email': some_email, 'data':some_data },
options = ( {'task_id': task_id_2 } )
)
]
)
# Creating callback task which will simply rely the result
# Using the task id, which has been generated before
#
# The dummy task start after all tasks in this group are completed
# This way we know that the group is completed
chord_callback = celery.signature(
'app.tasks.dummy_task',
options=( {'task_id': chord_callback_id_1 } )
)
chord_callback_2 = celery.signature(
'app.tasks.dummy_task',
options=( {'task_id': chord_callback_id_2 } )
)
# we can monitor each step status
# by its chord callback id
# the id of the chord callback
step1 = chord( group_1, body=chord_callback )
# the id of the chord callback
step2 = chord( group_2, body=chord_callback_2 )
# start the workflow execution
# the steps will execute sequentially
workflow = chain( step1, step2 )()
# the id of the last cord callback
workflow_id = workflow.id
# return any ids you need
print( workflow_id )
That's how I can check the status of any task in my app.
# This is a simplified example
# some code is omitted
from celery.result import AsyncResult
def task_status( task_id=None ):
# PENDING
# RECEIVED
# STARTED
# SUCCESS
# FAILURE
# REVOKED
# RETRY
task = AsyncResult(task_id)
response = {
'state': task.state,
}
return jsonify(response), 200
After hours of googling I stumbled upon http://www.manasupo.com/2012/03/chord-progress-in-celery.html . Though the solution there didn't work for me out of the box, it did inspire me to try something similar.
from celery.utils import uuid
from celery import chord
class ProgressChord(chord):
def __call__(self, body=None, **kwargs):
_chord = self.type
body = (body or self.kwargs['body']).clone()
kwargs = dict(self.kwargs, body=body, **kwargs)
if _chord.app.conf.CELERY_ALWAYS_EAGER:
return self.apply((), kwargs)
callback_id = body.options.setdefault('task_id', uuid())
r= _chord(**kwargs)
return _chord.AsyncResult(callback_id), r
and instead of executing celery.chord I use ProgressChord as follows:
def temptask(n):
header=list(tempsubtask.si(i) for i in range(n))
callback=templink.si('printed at last?')
r = celery.Progresschord(celery.group(header))(callback)
return r
returned value of r contained a tuple having both, callback's asyncresult and a group result. So success looked something like this:
In [3]: r
Out[3]:
(<AsyncResult: bf87507c-14cb-4ac4-8070-d32e4ff326a6>,
<GroupResult: af69e131-5a93-492d-b985-267484651d95 [4672cbbb-8ec3-4a9e-971a-275807124fae, a236e55f-b312-485c-a816-499d39d7de41, e825a072-b23c-43f2-b920-350413fd5c9e, e3f8378d-fd02-4a34-934b-39a5a735871d, c4f7093b-9f1a-4e5e-b90d-66f83b9c97c4, d5c7dc2c-4e10-4e71-ba2b-055a33e15f02, 07b1c6f7-fe95-4c1f-b0ba-6bc82bceaa4e, 00966cb8-41c2-4e95-b5e7-d8604c000927, e039c78e-6647-4c8d-b59b-e9baf73171a0, 6cfdef0a-25a2-4905-a40e-fea9c7940044]>)
I inherited and overrode [celery.chord][1] instead of [celery.task.chords.Chord][2] because I couldn't find it's source anywhere.
Old problem and I wasted a several days to find a better and modern solution. In my current project I must to track group progress separately and release lock in final callback.
And current solution is much more simple (but harder to guess), subject lines commented at the end:
#celery_app.task(name="_scheduler", track_started=True, ignore_result=False)
def _scheduler():
lock = cache.lock("test_lock")
if not lock.acquire(blocking=False):
return {"Error": "Job already in progress"}
lock_code = lock.local.token.decode("utf-8")
tasks = []
for x in range(100):
tasks.append(calculator.s())
_group = group(*tasks)
_chord = chord(_group)(_get_results.s(token=lock_code))
group_results = _chord.parent # This is actual group inside chord
group_results.save() # I am saving it to usual results backend, and can track progress inside.
return _chord # can return anything, I need only chord.
I am working in Celery 5.1