I am really new to web scraping.
I am trying to code software that monitors multiple share prices at once and must Identify when there is a change in the price and must show me the new price.
I have got it working that whenever there is a change in price it shows me the new price but it only works with one share and since it is in a while loop it won't break and move on to the next share.
I don't want to jump out of the while loop since it will stop monitoring the price. Then the browser will close and selenium must reopen it when wanting to see the new price which will take longer.
I am trying to create a function for each share I want to monitor.
I can't seem to get it to work that it will monitor more than one share at once. I have tried threading but then only one share price shows up.
Excuse all the imported libraries I just experimented
Any help will be appreciated.
import requests
import time
import tkinter
from selenium import webdriver
import time
from selenium.webdriver.firefox.options import Options
global driver
import threading
from multiprocessing import *
import os
options = Options()
options.headless = True
driver = webdriver.Firefox(options = options, executable_path=...driver path...)
counter_for_mtn = 0 # I only use counter for printing the first price
def MTN():#mtn is the name of the share
global counter_for_mtn
url = 'https://www.google.com/search?q=mtn+shares&safe=strict&sxsrf=ALeKk02Dyba3hHbg1S4YeLOFhw4FPfhqdQ:1588770127528&source=lnms&tbm=fin&sa=X&ved=2ahUKEwjl2dL4pZ_pAhUMZMAKHU1WDGAQ_AUoAXoECBkQAw&biw=1920&bih=969#scso=_XLWyXpmmE8uEhbIPpf-ooAY1:0'
driver.get(url)
while True:
value_mtn = driver.find_element_by_xpath('/html/body/div[6]/div[2]/div[9]/div[1]/div[2]/div/div[2]/div[2]/div/div/div/div/div/sticky-header/div[2]/div/div[1]/div/div[1]/div/div/div/div/div/div/g-card-section/div/g-card-section/span[1]/span/span[1]').text
if counter_for_mtn == 0:
print('MTN: '+ str(value_mtn))
counter_for_mtn =1
try:
if mtn_old_price != value_mtn:
print('MTN: '+str(value_mtn))
except:
pass
mtn_old_price = value_mtn
counter_for_sasol = 0#only use counter to print out the first price
def Sasol():#sasol is the name of the share
global counter_for_sasol
url = 'https://www.google.com/search?q=sasol+shares&safe=strict&sxsrf=ALeKk004QjkQ84tFd8TP8ikaQc3yhNRkrQ:1588772136446&source=lnms&tbm=fin&sa=X&ved=2ahUKEwjpkMm2rZ_pAhV4Q0EAHZ7jCWEQ_AUoAXoECBkQAw&biw=1920&bih=969'
driver.get(url)
while True:
value = driver.find_element_by_xpath('/html/body/div[6]/div[2]/div[9]/div[1]/div[2]/div/div[2]/div[2]/div/div/div/div/div/sticky-header/div[2]/div/div[1]/div/div[1]/div/div/div/div/div/div/g-card-section/div/g-card-section/span[1]/span/span[1]').text
if counter_for_sasol == 0:
print('SASOL: ' + str(value))
counter_for_sasol = 1
try:
if old_price != value:
print('SASOL: ' + str(value))
except:
pass
old_price = value
thread1 = threading.Thread(target = MTN())
thread1.start()
thread2 = threading.Thread(target = Sasol())
thread2.start()
how do I get both Functions Sasol() and MTN() to run at the same exact time?
This works (using toy functions) although technically I'm not sure you can say that they are running at the exact same time:
import sys,time,threading,random
def f():
for i in range(10):
print(f'{i}:f',file=sys.stdout)
time.sleep(random.random())
def g():
for i in range(10):
print(f'{i}:g',file=sys.stdout)
time.sleep(random.random())
if __name__ == '__main__':
threads = [threading.Thread(target=func) for func in [f,g]]
for t in threads:
t.start()
This also works - using single function passing a different argument to each thread - in your case the argument(s) would be unique string(s) specific to the things you want to retrieve.
import sys,time,threading,random
def f(s):
for i in range(10):
print(f'{i}:{s}',file=sys.stdout)
time.sleep(random.random())
if __name__ == '__main__':
threads = [threading.Thread(target=f,args=(arg,)) for arg in ['one','two']]
for t in threads:
t.start()
Related
My trading app written with Python and using the IB TWS API is multithreaded and I'm looking to solve once and for all the issues of corrupt prints and duplicate order ids, both of which can happen once multiple threads are running at the same time.
For the purpose of this question I've put together a simple script that has for goal nothing other than providing a support for solving it. What I'm hoping for is to see how someone more experienced would implement queues in order to make the script do 2 things:
print 'safely' in a non corrupt way
each order should execute without the 'duplicate order id' error code popping up
I know that I need to use queues but I'm just starting to read up on the topic and since the IB TWS API is fairly specific, I'm not sure how to implement. Thank you.
from ibapi.wrapper import EWrapper
from ibapi.client import EClient
from ibapi.order import Order
from ibapi.contract import Contract
import threading
import time
buyOrder1Id = 0
buyOrder2Id = 0
buyOrder3Id = 0
buyOrder4Id = 0
buyOrder5Id = 0
buyOrder6Id = 0
buyOrder7Id = 0
buyOrder8Id = 0
buyOrder9Id = 0
buyOrder10Id = 0
class TradingApp(EWrapper, EClient):
def __init__(self):
self.BidPrice = 0
self.AskPrice = 0
self.LastPrice = 0
EClient.__init__(self,self)
def error(self, reqId, errorCode, errorString):
print("Error. Id: ", reqId, " Code: ", errorCode, " Msg: ", errorString)
def nextValidId(self, orderId):
super().nextValidId(orderId)
self.nextValidOrderId = orderId
def websocket_con():
app.run()
Underlying = Contract()
Underlying.localSymbol = "ESU2"
Underlying.secType = "FUT"
Underlying.currency = "USD"
Underlying.exchange = "GLOBEX"
BuyOrder1 = Order()
BuyOrder1.action = "BUY"
BuyOrder1.totalQuantity = 1
BuyOrder1.orderType = "MKT"
BuyOrder1.account = "DU2312534"
BuyOrder1.tif = "GTC"
app = TradingApp()
app.connect("127.0.0.1", 7497, 2000)
time.sleep(1)
# starting a separate daemon thread to execute the websocket connection
con_thread = threading.Thread(target=websocket_con, daemon=True)
con_thread.start()
time.sleep(1)# some latency added to ensure that the connection is established
def Level1():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder1Id = app.nextValidOrderId
print("buyOrder1Id: ",buyOrder1Id)
app.placeOrder(buyOrder1Id,Underlying,BuyOrder1)
time.sleep(3)
def Level2():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder2Id = app.nextValidOrderId
print("buyOrder2Id: ",buyOrder2Id)
app.placeOrder(buyOrder2Id,Underlying,BuyOrder1)
time.sleep(3)
def Level3():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder3Id = app.nextValidOrderId
print("buyOrder3Id: ",buyOrder3Id)
app.placeOrder(buyOrder3Id,Underlying,BuyOrder1)
time.sleep(3)
def Level4():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder4Id = app.nextValidOrderId
print("buyOrder4Id: ",buyOrder4Id)
app.placeOrder(buyOrder4Id,Underlying,BuyOrder1)
time.sleep(3)
def Level5():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder5Id = app.nextValidOrderId
print("buyOrder5Id: ",buyOrder5Id)
app.placeOrder(buyOrder5Id,Underlying,BuyOrder1)
time.sleep(3)
def Level6():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder6Id = app.nextValidOrderId
print("buyOrder6Id: ",buyOrder6Id)
app.placeOrder(buyOrder6Id,Underlying,BuyOrder1)
time.sleep(3)
def Level7():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder7Id = app.nextValidOrderId
print("buyOrder7Id: ",buyOrder7Id)
app.placeOrder(buyOrder7Id,Underlying,BuyOrder1)
time.sleep(3)
def Level8():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder8Id = app.nextValidOrderId
print("buyOrder8Id: ",buyOrder8Id)
app.placeOrder(buyOrder8Id,Underlying,BuyOrder1)
time.sleep(3)
def Level9():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder9Id = app.nextValidOrderId
print("buyOrder9Id: ",buyOrder9Id)
app.placeOrder(buyOrder9Id,Underlying,BuyOrder1)
time.sleep(3)
def Level10():
while True:
app.reqIds(1)
time.sleep(1)
buyOrder10Id = app.nextValidOrderId
print("buyOrder10Id: ",buyOrder10Id)
app.placeOrder(buyOrder10Id,Underlying,BuyOrder1)
time.sleep(3)
level1_thread = threading.Thread(target=Level1)
level1_thread.start()
level2_thread = threading.Thread(target=Level2)
level2_thread.start()
time.sleep(1)
level3_thread = threading.Thread(target=Level3)
level3_thread.start()
time.sleep(1)
level4_thread = threading.Thread(target=Level4)
level4_thread.start()
time.sleep(1)
level5_thread = threading.Thread(target=Level5)
level5_thread.start()
time.sleep(1)
level6_thread = threading.Thread(target=Level6)
level6_thread.start()
time.sleep(1)
level7_thread = threading.Thread(target=Level7)
level7_thread.start()
time.sleep(1)
level8_thread = threading.Thread(target=Level8)
level8_thread.start()
time.sleep(1)
level9_thread = threading.Thread(target=Level9)
level9_thread.start()
time.sleep(1)
level10_thread = threading.Thread(target=Level10)
level10_thread.start()
time.sleep(1)
I'm not familiar with Python, but I see the logical error - you're using "nextValiOrderId" in your app.1st thing you do - you start a bunch of threads.2nd thing that happens - each thread would send a request app.reqIds(1)
Now you need to understand what's reqIds() function logic (from this discussion):
The nextValidId method is a wrapper method. From the client, you need
to call reqIds to get an Order ID.
ib_api.reqIds(-1) The parameter of reqIds doesn't matter. Also, it doesn't have a return value. Instead, reqIds sends a message to IB,
and when the response is received, the wrapper's nextValidId method
will be called.
So you called reqIds(1) many times (almost at the same time), eventually the response will come in form of nextValidId() callback f-n, which sets app.nextValidOrderId to ever growing number, but the same time you reading that number from all your threads, which is complete mess.
Instead you could reorganize the code logic the following way: - ask for reqIds(1) only once:
then do nothing (no code executed after that request is sent)
the continuation logic will reside inside the body of the nextValidId() callback f-n: upon getting back a valid integer number from IB you would set it to some of you application variable and do not call reqIds(1) anymore.
call your "main_continuation()" function will will use that number as an argument to pass to i-th thread, then in a loop you'd increment this number and pass it to the next thread, and so on.In this case none of the threads will be trying to use the same order ID.
But even this approach is not ideal.. in case you need to start 100s of threads they will exceed 50 requests / second rate (one of many other rate limits by IB API), thus may be add 30 milliseconds sleep between starting new threads (in the loop where you create them one-by-one).
But now if you think over the design of your application a bit more - you'll see that you do NOT need many threads at all.
ps: I would strongly recommend for any Pythonista to carefully look at the ib_insync project, created by Ewald de Wit and it has growing community behind it and everybody is so happy with ease of use of ib_insync (written in python). This is their discussion forum.
and this is the ib_insync on the github
Cheers,Dmitry
I Have a counting function that I would like to start and restart while getting the live variables to use in another function my problem is while using threading it seams like even global variables don't seem to work to pass variables around. What I want the code to do is have a counter be triggered as needed or maybe free running I'm not sure yet. To be able to reset the counter and get the value of the counter.
Right now the counter will start and run fine but the print_stuff function keeps telling me that there is no attribute countval.
The count thread gets started at startup but I don't necessarily want it to start up immediately, I would like to trigger it as needed but I cant put count_thread.start() twice or it will through a error so I'm calling the thread at startup and then calling the function again to restart it as needed. Maybe there is a more elegant way of doing that.?
import threading
import time
def count():
global countval
for countval in range(3):
print('looping')
time.sleep(1)
def print_stuff():
global countval
e = input("press enter to start")
count()
while True:
if countval == 3:
print("time out")
count_thread = threading.Thread(target=count)
print_thread = threading.Thread(target=print_stuff)
print_thread.start()
count_thread.start()
print_stuff is getting to the if statement before the count function is able to create the variable. Just do them in the opposite order. Either that, or create a global countval = 0 to start things off.
To solve the no attribute problem you can use Queue,
and if you want to stop your counting thread you can set a global variable or you can pass a function (using lambda or inner function or ...) to do that.
Here is one way to do that:
import threading
import time
from queue import Queue
from typing import Callable
def count(q, stop_counting):
# type: (Queue, Callable[[], bool]) -> None
for countval in range(3):
if stop_counting():
print('stopped')
break
print(f'looping {countval}')
q.put(countval)
time.sleep(1)
def print_stuff(q):
# type: (Queue) -> None
while True:
countval = q.get()
print(f'countval gotten: {countval}')
if countval == 3:
print("time out")
def main():
flag_stop_counting = False
q = Queue()
def stop_counting():
return flag_stop_counting
count_thread = threading.Thread(target=count, args=(q, stop_counting,))
print_thread = threading.Thread(target=print_stuff, args=(q,))
print_thread.start()
count_thread.start()
time.sleep(1.25)
flag_stop_counting = True
if __name__ == '__main__':
main()
In this code:
counter checks if it should stop counting or not
counter puts the value that it made to q
print_stuff get the value from q (Note: he waits until counter puts his value in q)
To check that program works:
after 1.25 seconds we change the value of flag_stop_counting
But if you want your counter to only have a for, i guess it's better to don't make it as a thread and run it whenever you want.
Hope it was helpful.
I want to write code that does something like-
In a thread call a function that will return two links and in the main thread keep printing
"processing..." until that function, called in secondary thread returns those values
and when we get those return values the while loop of the main thread terminates and print
those
values.
Now I have tried writing few codes in python but couldn't manage to do it!
I have just started python programming so I'm not familiar with it.
BTW the above-mentioned scenario is just a prototype.
The real case looks something like that:-
def search_button(self): #main thread
mname = self.root.ids.name.text
quality = Quality
#print(mname)
#print(quality)
global link4, link5
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = executor.submit(movie_bot, mname, quality) #movie_bot is function
link4, link5 = futures.result() #returning two links
while(link4==None and link5==None):
self.root.ids.status.text = 'Searching, please wait...'
self.root.ids.status.text = ''
print(link4)
print(link5)
In case further details are required then please just let me know.
Thank you for the help.
Tried a lot of things and now finally it got resolved.
Basically what I wanted to do was take two inputs from GUI then call a function (from another python program) with those two parameters and then once the processing is completed then from GUI user and either watch or download that content by pressing the watch or download buttons respectively.
So to do that earlier I was returning the watch and download link from that thread called function and even on calling that function on another thread, as it was returnig values after execution so the GUI freezes and shows not responding
Then after trying a lot of thing I came across daemon thing so I just made that thread daemon and that solved the main problem of freezing but now I wasn't able to take return values (when I tried to take the return values it again started to freeze the GUI)
So then I found an alternative to access those links from the main thread.
Here the point is if the function doesn't return anything that is being called in the thread then just make it daemon thread_name.daemon() = True and now it won't freeze the GUI
Now in case you wanna do something exactly after the thread is finished then this can be used thread_name.is_alive()
MY CODE LOOKS SOMETHING LIKE THAT:-
from selenium_l_headless import movie_bot
from kivy.lang import Builder
from kivymd.app import MDApp
from kivy.properties import ObjectProperty
from selenium_l_headless import *
import threading
import time
from kivy.clock import Clock
class MovieBot(MDApp):
mname = ObjectProperty(None)
quality = ObjectProperty(None)
Quality = ObjectProperty(None)
link4 = ObjectProperty(None)
link5 = ObjectProperty(None)
checks = []
def build(self):
self.theme_cls.theme_style= "Dark"
self.theme_cls.primary_palette = "Teal"
return Builder.load_file('kivy_bot_md.kv')
def checkBox_click(self, instance, value, Q):
global Quality
if value==True:
MovieBot.checks.append(Q)
Quality=''
for x in MovieBot.checks:
Quality = f'{Quality}{x}'
else:
MovieBot.checks.remove(Q)
Quality = ''
for x in MovieBot.checks:
Quality = f'{Quality} {x}'
def complete(self):
self.root.ids.status.text = 'Searching completed, Now you can
download or watch the movie!'
global flag
flag=0
def status_sleep(self, *args):
try:
self.root.ids.status.text = 'Searching, please wait...'
if(t1.is_alive() is False):
self.complete()
except:
pass
def search_button(self):
try:
mname = self.root.ids.name.text
quality = Quality
global link4,link5
global t1, flag
flag=1
t1 = threading.Thread(target = movie_bot, args= [mname, quality])
t1.daemon = True
t1.start()
if(t1.is_alive()):
Clock.schedule_interval(self.status_sleep,1)
except:
pass
def watch(self):
try:
if(flag is 1):
pass
else:
self.root.ids.status.text = ''
t2 = threading.Thread(target=watch_now)
t2.daemon = True
t2.start()
except:
pass
def download(self):
try:
if(flag is 1):
pass
else:
self.root.ids.status.text = ''
t3 = threading.Thread(target=download_movie)
t3.daemon = True
t3.start()
except:
pass
def close(self):
exit(0)
MovieBot().run()
Im trying to run the listener function forever, for example I want any time there is new information available in the stream it updates on the list automatically. Any idea in how to do it would be appreciated.
class Notifications(Screen):
notificationslist = ObjectProperty(None)
def listener(self, event = None):
notifications_screen = self.manager.get_screen('notif')
print(event.event_type) # can be 'put' or 'patch'
print(event.path) # relative to the reference, it seems
print(event.data) # new data at /reference/event.path. None if deleted
notifications = event.data
if notifications.items() == None:
return
else:
for key, value in notifications.items():
thevalue = value
notifications_screen.notificationslist.adapter.data.extend([value[0:17] + '\n' + value[18:]])
print(thevalue)
id = (thevalue[thevalue.index("(") + 1:thevalue.rindex(")")])
print(id)
If you want a function to run forever but that does not block you from doing other functions, then you can use threads.
Here is an example with example_of_process that runs forever, and then the main program with time.sleep(3)
import threading
import time
def example_of_process():
count = 0
while True:
print("count", count)
count += 1
time.sleep(1)
thread_1 = threading.Thread(target=example_of_process)
thread_1.daemon = True # without the daemon parameter, the function in parallel will continue even if your main program ends
thread_1.start()
# Now you can do anything else. I made a time sleep of 3s, otherwise the program ends instantly
time.sleep(3)
how do I update a variable inside a running thread, which is an infinite loop based on such variable?
a simplified version of what I tried is what follows, to no results of course, and I can't find any pointer.
import some_module as mod
import threading
class thr (threading.Thread):
NUM = 5 # set a default value to start the script
def run (self):
mod.NUM = NUM
mod.main_loop()
try:
thr().start()
time.sleep(1)
thr().NUM = 2
time.sleep(1)
thr().NUM = 6
time.sleep(1)
thr().NUM = 8
The problem is that you're creating a new thread each time you "call" (i.e. instantiate) thr. Change your code to
t = thr()
t.start()
time.sleep(1)
t.NUM = 2
time.sleep(1)
t.NUM = 6
time.sleep(1)
t.NUM = 8
time.sleep(1)
Maybe try use queue for change NUM variable.
https://docs.python.org/2/library/queue.html
Check examples here :
https://pymotw.com/2/Queue/
Generally speakinig the queue allows You send data between threads. Use get() for getting data from queue and put() for put data to queue.