I'm currently learning Python and Classes and I have a basic question, but I didn't find any answer to it. Let's say I have this dummy class
class DomainOperations:
def __init__(self, domain):
self.domain = domain
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
#resolve domain to ipv4 and save to self.domain_ip
def generate_website_thumbnail(self):
#generate website thumbnail and save the url to self.website_thumbnail
I want to run simultaneously resolve_domain and generate_website_thumbnail and when the threads are finished I want to print the IP and the thumbnail.
EDIT: I know I should use threads, maybe something like this
r = DomainOperations('google.com')
t1 = threading.Thread(target=r.resolve_domain)
t1.start()
t2 = threading.Thread(target=r.generate_website_thumbnail)
t2.start()
But should I use them outside the Class? Should I write another Class to handle Threads?
What is the right way to do that?
If you call them from the class, it is as simple as:
import threading
class DomainOperations:
def __init__(self):
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
self.domain_ip = 'foo'
def generate_website_thumbnail(self):
self.website_thumbnail= 'bar'
def run(self):
t1 = threading.Thread(target=self.resolve_domain)
t2 = threading.Thread(target=self.generate_website_thumbnail)
t1.start()
t2.start()
t1.join()
t2.join()
print(self.domain_ip, self.website_thumbnail)
if __name__ == '__main__':
d = DomainOperations()
d.run()
You can inherit Thread class in DomainOperation, in this way code would be more clean and easily understandable. you have to override a run() method.
from threading import Thread
class DomainOperations(Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
self.domain_ip = 'foo'
def generate_website_thumbnail(self):
self.website_thumbnail= 'bar'
def run(self):
#domain will be resolved on first thread
self.resolve_domain()
#thumbnail will be resolved on second OR newly created below thread
thread2 = Thread(target=self.generate_website_thumbnail)
thread.start()
# thread1 will wait for thread2
self.join()
# thread2 will wait for thread1, if it's late.
thread2.join()
# here it will print ip and thumbnail before exiting first thread
print(self.domain_ip, self.website_thumbnail)
And you will start your threads in this way.
if __name__ == '__main__':
thread1 = DomainOperations()
thread1.start()
def post_test(tbid, line_num, response_time):
"""
:param tbid: 参数id
:return:
"""
# 请求参数
data = {'tbId': tbid, 'conditions': [{"key": "", "type": 1}], 'pageNum': 1, 'pageSize': 12}
# 请求启动时间
start = time.time()
# post请求
r = requests.post(url=url, data=json.dumps(data), headers=headers)
# 请求结束时间
end = time.time()
# 保留两位小数
finall_time = float('%.2f' % float(end - start))
text = json.loads(r.text)
# IO写入 只写入200的
with open('text6.csv', 'a', newline='') as csvfile:
if text['statusCode'] == '200':
throughput = line_num * response_time / finall_time
throughput = float('%.2f' % float(throughput))
print('the perf_counter time of %s is %s and the content is %s ,throughput is %s' % (
tbid, finall_time, json.loads(r.text), throughput))
spamwriter = csv.writer(csvfile, dialect='excel')
spamwriter.writerow([tbid] + [finall_time] + [throughput])
def start_thread(csv_name):
tbid, response_time_sort, throughput_sort = read_csv(csv_name)
print(tbid)
line_num = len(tbid)
response_times = 5
for j in range(response_times):
for i in tbid:
t = threading.Thread(target=post_test, args=(i, line_num, response_times))
t.start()
t.join()
I don't know how to call a method in a class, especially if it has initialization parameters, but you can try this method。 I'm trying to use multiple processes to solve this problem, right。
Related
I have created multiple threads using for loop and I need to read the name of the thread inside the thread. How do I do that? I used threading.Thread.name but returns some memory address of the thread object. here is part of my code:
def catch_sharp_fall(update, context, x):
global access_token, watchlist, bot, chat_id, market_open_alert, nifty, bot_reset, stop_catch_sharp_fall, \
kite_api_key
ins_code = x
kite = KiteConnect(kite_api_key)
kite.set_access_token(access_token)
kws = KiteTicker(kite_api_key, access_token)
def on_ticks(ws, ticks):
logging.debug("Ticks: {}".format(ticks))
def on_connect(ws, response):
ws.subscribe([ins_code])
ws.set_mode(ws.MODE_FULL, [ins_code])
def on_close(ws, code, reason):
ws.stop()
kws.on_ticks = on_ticks
kws.on_connect = on_connect
kws.on_close = on_close
kws.connect(threaded=True)
while True:
def on_ticks(ws, ticks):
feed_data(ticks)
def feed_data(ticks):
list_of_threads_running = [i.name for i in threading.enumerate()]
logging.info(f'list of threads running {list_of_threads_running}')
# update.message.reply_text(f'<b>I am the stock {ins_code}</b>', parse_mode='HTML')
logging.info(f'{threading.Thread.name}')
kws.on_ticks = on_ticks
for x in watchlist:
t = threading.Thread(name=str(x), target=catch_sharp_fall, args=(update, context, x))
t.setDaemon(True)
t.start()
When you do threading.Thread.name, you're retrieving name on the Thread type itself, not on the thread object for the current thread. You need to look it up on the current thread object:
threading.current_thread().name
You can use threading.current_thread().name, t1.name and t2.name as shown below:
import threading
def test1():
print(threading.current_thread().name) # "Thread-1"
def test2():
print(threading.current_thread().name) # "test_thread"
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2, name="test_thread")
# Thread name
t1.start()
t2.start()
t1.join()
t2.join()
print(t1.name) # "Thread-1"
print(t2.name) # "test_thread"
print(threading.current_thread().name) # "MainThread"
I am working on a rabbitmq consumer application
My consumer should consume and then process the data in a separate thread
I have a script here for that but I cannot make it work. Any help. Thanks so much. It is going to the run function then doing nothing..
import pika, Consumer_Config, queue, threading
import Message_Class
class con:
def __init__(self):
self.config = Consumer_Config._config()
self.path = self.config.path
self.active = 0
self.channel = None
self.tag = None
self.tb = None
def channsel(self):
pika_conn_params = pika.ConnectionParameters(
host=self.config.url, port=self.config.port,
credentials=pika.credentials.PlainCredentials(self.config.user_id, self.config.password))
connection = pika.BlockingConnection(pika_conn_params)
return connection
def runq(self, channel, q):
self.channel = channel
def read_queue(self):
queue = self.channel.queue_declare(
queue="queue", durable=True,
exclusive=False, auto_delete=False)
self.channel.queue_bind(
"queue",
'exchange',
routing_key=str('text'))
self.channel.basic_qos(
prefetch_count=500)
self.tag = self.channel.basic_consume("queue", self.on_msg)
self.channel.start_consuming()
def on_msg(self, _unused_channel, basic_deliver, properties, body):
rk = basic_deliver.routing_key
self.tb = body
q.put(self.tb)
self.acknowledge_message(basic_deliver.delivery_tag)
def acknowledge_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
"""
# LOGGER.info('Acknowledging message %s', delivery_tag) # Acknowledgement is done once the message is delivered
self.channel.basic_ack(delivery_tag)
def printq(self, q):
tb = q.get()
item = Message_Class.Messages().Process_msg(tb, 'text')
print(item)
def run(self):
cn = con()
connct = cn.channsel()
channel = connct.channel()
num_worker_threads = 2
for i in range(num_worker_threads):
t1 = threading.Thread(target=self.runq, args=(channel, q))
t2 = threading.Thread(target=self.printq, args=[q])
t1.daemon = True
t2.daemon = True
t1.start()
t2.start()
t1.join()
t2.join()
q.join()
if __name__ == "__main__":
q = queue.Queue()
r = con()
r.run()
I Have 2 Code That Preferred To Have Faster Processing
First Code
( On My Recent Thread Someone Pointed Out That My Threading Code Is Wrong )
He Stated That No Matter The Value Of TotalThread It Won't Process Faster
class ThreadingPower(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue= queue
def run(self):
while True:
Manager = self.queue.get()
for Numbers,HWID in Manager:
r = requests.post(URL, data=payload) # License Checker Required Numbers ( Buyer Code ) And HWID ( License Code )
data = (r.text)
if ('Verified' in data):
with open(resultsFile,'a+') as results:
results.write("The Number : "+str(Numbers)+" Is Verified By The "+str(HWID))
results.close()
print str(HWID)+" Is Server Verified"
else:
print str(HWID)+" Is Not Server Verified"
self.queue.task_done()
if __name__ == "__main__":
for i in range(TotalThread):
t = ThreadingPower(queue)
t.setDaemon(False)
t.start()
queue.put(credentials)
queue.join()
The Credentials Are Tupples
[
['UniqueHWID', 'BuyerCode'],
['UniqueHWID', 'BuyerCode'],
['UniqueHWID', 'BuyerCode']
]
The Second Code
class ThreadingPower(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue= queue
def run(self):
while True:
Performance = self.queue.get()
for Name,ScoreA,ScoreB,ScoreC in Performance:
TScore = (ScoreA+ScoreB+ScoreC)/3
print Name+" Has Scored :"+TScore
if (TScore >= PassScore):
print Name+" Has Passed"
# Sending Message To The Server With Name Has Pass
else:
print Name+" Has Failed"
# Sending Message To The Server With Name Has Failed
if __name__ == "__main__":
for i in range(TotalThread):
t = ThreadingPower(queue)
t.setDaemon(False)
t.start()
queue.put(TestSheet)
queue.join()
With TestSheet Looking Like This
[
['StudentName', 'Test1', 'Test2', 'Test3'],
['StudentName', 'Test1', 'Test2', 'Test3'],
['StudentName', 'Test1', 'Test2', 'Test3']
]
I'm Still Learning A Few About Python,Sorry For Asking So Much Question
I Haven't Test The Second Code But Since The Threading Part Is Base From The First Code I Think It Has The Same Issue
Go one Step back, try first without threading.
def run(p):
Performance = p
Name, ScoreA, ScoreB, ScoreC = Performance
TScore = int((ScoreA + ScoreB + ScoreC) / 3)
print('{} Has Scored:{}'.format(Name, TScore))
if TScore >= PassScore:
print(Name + " Has Passed")
else:
print(Name + " Has Failed")
if __name__ == "__main__":
PassScore = 10
# Name ScoreA, ScoreB, ScoreC
TestSheet = [['StudentName1', 1, 2, 3],
['StudentName2', 4, 5, 6],
['StudentName3', 7, 8, 9]]
for p in TestSheet:
run(p)
If the result met your needs, add threading.
class ThreadingPower(threading.Thread):
...
def run(self):
Performance = self.queue.get()
...
if __name__ == "__main__":
...
for p in TestSheet:
queue.put(p)
...
Question: He Stated That No Matter The Value Of TotalThread It Won't Process Faster
What you are doing is,
Starting n Threads
All Threads waiting at queue.get()
Put ONE Message into the Queue
Only ONE Thread can get this Message and have to do ALL the jobs.
Change your code to put the 3 jobs from your TestSheet one after the other.
for job in TestSheet:
queue.put(job)
This is invalid Python, you have to change this to your new queue Data:
for in Name,ScoreA,ScoreB,ScoreC in Performance:
Second, you puting a List of str into the queue and want to do some math, this will FAIL also!
How do I do the receive function to print "waiting" while the function send does not send the args?
Well, I really don't know how I can do it, I thought about join() method, but I tried this:
import threading
import random
waiting = True
def receive(a=""):
while waiting:
print "[receive] waiting for args"
print "[receive] Args received: %s" % a # args received from send function
def send(args):
global waiting
if waiting:
if random.randint(1,3) == 2:
waiting = False
print "[send] Args sent"
receive(a=args) # send the args
fargs = ["hello, world", "foo", "bar", "foobar", 12]
t1 = threading.Thread(target=receive, args=[])
t2 = threading.Thread(target=send, args=[fargs])
t1.start()
t2.start()
Sometimes this works, but sometimes I get in a infinite loop.
#EDIT
Now it works fine:
import threading
import random
waiting = True
def receive(a=""):
while waiting:
print "[receive] waiting for args"
if a:
print "[receive] Args received: %s" % a
def send(args):
global waiting
while waiting:
if random.randint(1,3) == 2:
waiting = False
print "[send] Args sent"
receive(a=args)
fargs = ["hello, world", "foo", "bar", "foobar", 12]
t1 = threading.Thread(target=receive, args=[])
t2 = threading.Thread(target=send, args=[fargs])
t1.start()
t2.start()
Is there a better way to do it?
Sorry for my english.
I know the thread is quite old, but expanding on the self answer by OP i created a class that prints a string while a given function is running.
import threading
import queue
import time
import getpass
class DotsPrinter:
def __init__(self, float_dots_frequency=1,
string_to_print_while_waiting="."):
self.float_dots_frequency = float_dots_frequency
self.string_to_print_while_waiting = string_to_print_while_waiting
self.bool_waiting = True
self.output_queue = queue.Queue()
def print_dots(self):
if self.bool_waiting:
print("Waiting ", end="", flush=True)
while self.bool_waiting:
print(self.string_to_print_while_waiting, end="",
flush=True)
time.sleep(self.float_dots_frequency)
def function_wrapper(self, function, *args, **kwargs):
self.output_queue.put(function(*args, **kwargs))
self.bool_waiting = False
def print_dots_while_executing(self, function, *args, **kwargs):
t1 = threading.Thread(target=self.print_dots)
t2 = threading.Thread(target=self.function_wrapper, args=
[function, *args], kwargs={**kwargs})
t1.start()
t2.start()
return self.output_queue.get()
the usage is quite simple:
def count_till_timeout(timeout):
start_time = time.time()
n = 0
while time.time() - start_time < timeout:
n += 1
return n
n = DotsPrinter().print_dots_while_executing(count_till_timeout, 5)
print(n)
I didn't test it exstensively so it might have some bugs, and i am not an expert with threading, therefore I don't know if a thing like this should be done, but I hope it helps someone. I'm using it to print dots while pandas downloads data from a sql DB.
I'm attempting to create python module for getting MAC adresses by IP addresses.
def getMACs(addressesList):
def _processArp(pkt):
spa = _inet_ntoa(pkt.spa)
if pkt.op == dpkt.arp.ARP_OP_REPLY and spa in _cache.macTable:
lock.acquire()
try:
_cache.macTable[spa] = _packedToMacStr(pkt.sha)
_cache.notFilledMacs -= 1
finally:
lock.release()
if _cache.notFilledMacs == 0:
thrd.stop()
addresses = _parseAddresses(addressesList)
_cache.registerCacheEntry("macTable", {})
_cache.registerCacheEntry("notFilledMacs", 0)
_events.arpPacket += _processArp
lock = threading.Lock()
thrd = _CaptureThread(promisc=False, timeout_ms=30, filter="arp")
thrd.start()
for addr in addresses:
if _sendArpQuery(addr):
_cache.macTable[str(addr)] = None
_cache.notFilledMacs += 1
thrd.join(125)
thrd.stop()
return _cache.macTable
if __name__ == "__main__":
macTable = getMACs([IPAddress("192.168.1.1"), IPAddress("192.168.1.3")])
_pprint.pprint(macTable)
When I run this module I get
{'192.168.1.1': '00:11:95:9E:25:B1', '192.168.1.3': None}
When I debug _processArp step by step I get
{'192.168.1.1': '00:11:95:9E:25:B1', '192.168.1.3': '00:21:63:78:98:8E'}
Class CaptureThread:
class CaptureThread(threading.Thread):
def __init__ (self, name=None, snaplen=65535, promisc=True, timeout_ms=0, immediate=False, filter=None):
threading.Thread.__init__(self)
self.__running = True
self.__name = name
self.__snaplen = snaplen
self.__promisc = promisc
self.__timeout_ms = timeout_ms
self.__immediate = immediate
self.__filter = filter
def stop(self):
self.__running = False
def run(self):
self.__pc = pcap.pcap(self.__name, self.__snaplen, self.__promisc, self.__timeout_ms, self.__immediate)
if self.__filter:
self.__pc.setfilter(self.__filter)
while self.__running:
self.__pc.dispatch(1, self.__processPacket)
def __processPacket(self, timestamp, pkt):
peth = dpkt.ethernet.Ethernet(pkt)
if isinstance(peth.data, dpkt.arp.ARP):
_events.arpPacket(peth.data)
Stupid error. As always when working with threads - because of thread synchronization.
One of my conditions for interrupting thread is "_cache.notFilledMacs == 0". In the main thread _cache.notFilledMacs did not have time to get the value of 2 when in the CaptureThread value is decreased.