Develop Reference

Threading in python and Queue leads to memory leak or memory error

I had created simple code using multithreading in python with Queue. I have a main thread which keep on adding the Data in the Queue(Queue maxsize is 2000) and there will be 5 different threads who will take out from the Queue and publish into redis at some specific channel.
The code is working perfectly fine , but after 5 or 6 hours , the publish mechanism becomes slow.
As the threads which is used to remove the data from Queue becomes slow, and started throwing the buffer over flow error, as the Queue size reaches to maxsize. the speed of adding the data to queue is same which was in beginning.
The issue occurs differently on different configuration Linux system. How to identify what kind of error it is throwing? How to debug the problem.
As Stated - the code is very simple , where main thread is required to add the data in the Queue one by one and the 5 other threads can take the data out of the Queue one by one.
Sharing The Code
import redis
import logging
import sys
logging.basicConfig(level = logging.DEBUG)
redisPub = redis.StrictRedis(host='127.0.0.1', port=6379)
def main():
try:
recv_sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_UDP)
recv_sock.bind(("", 8000))
recv_sock.setblocking(0)
recv_sock.settimeout(5)
except Exception,e:
exc_type, exc_obj, exc_tb = sys.exc_info()
logging.error("Socket Connection Unsuccessful")
print "Program halted."
sys.exit()
sendThEvent = threading.Event()
sendThEvent.clear()
sendPktQ = Queue.Queue(maxsize=2000)
for i in range(0,5):
thSend = threading.Thread(name = 'sendThread', target = sendThread , args = (sendPktQ,sendThEvent))
thSend.setDaemon(True)
thSend.start()
packetsCounting = 0
while True:
try:
recvData, recvAddr = recv_sock.recvfrom(2048)
sendPktQ.put(recvData)
packetsCounting += 1
else:
logging.error("There is some error")
except socket.timeout:
continue
except Exception,e:
exc_type, exc_obj, exc_tb = sys.exc_info()
logging.error(e)
def sendThread(sendPktQ,listenEvent):
if sendPktQ == None:
pass
else:
while True:
instanceSentCnt += 1
if sendPktQ.qsize() < 1:
event_is_set = listenEvent.wait(0)
packetDict = sendPktQ.get()
data = redisPub.publish('chnlName', packetToSend)
logging.info("packet size reached to ============================================ %s ------------ %s"%(len(packetToSend),data))
if __name__ == "__main__":
main()
Anyone Inputs will be highly appreciated.

I see a sendPktQ.get(), but no sendPktQ.task_done(), for example:
async def _dispatch_packet(self, destination=None) -> None:
"""Send a command unless in listen_only mode."""
if not self.command_queue.empty():
cmd = self.command_queue.get()
if not (destination is None or self.config.get("listen_only")):
destination.write(bytearray(f"{cmd}\r\n".encode("ascii")))
await asyncio.sleep(0.05)
self.command_queue.task_done()
See queue library docs

use multiprocessing or threading script python

i want use multiprocessing or threading for this code
and I would like to control the threading example threads = (50)
if i chose 50 theards is opend 50 process
please help me
this code :-
import subprocess
import csv
def ping(hostname):
p = subprocess.Popen(["ping", "-n", "1","-w","1000",hostname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pingStatus = 'ok';
for line in p.stdout:
output = line.rstrip().decode('UTF-8')
if (output.endswith('unreachable.')) :
#No route from the local system. Packets sent were never put on the wire.
pingStatus = 'unreacheable'
break
elif (output.startswith('Ping request could not find host')) :
pingStatus = 'host_not_found'
break
if (output.startswith('Request timed out.')) :
#No Echo Reply messages were received within the default time of 1 second.
pingStatus = 'timed_out'
break
#end if
#endFor
return pingStatus
#endDef
def printPingResult(hostname):
statusOfPing = ping(hostname)
if (statusOfPing == 'host_not_found') :
writeToFile('!server-not-found.txt', hostname)
elif (statusOfPing == 'unreacheable') :
writeToFile('!unreachable.txt', hostname)
elif (statusOfPing == 'timed_out') :
writeToFile('!timed_out.txt', hostname)
elif (statusOfPing == 'ok') :
writeToFile('!ok.txt', hostname)
#endIf
#endPing
def writeToFile(filename, data) :
with open(filename, 'a') as output:
output.write(data + '\n')
#endWith
#endDef
'''
servers.txt example
vm8558
host2
server873
google.com
'''
file = open('hosts.txt')
try:
reader = csv.reader(file)
for item in reader:
printPingResult(item[0].strip())
#endFor
finally:
file.close()
#endTry
i want use multiprocessing or threading for this code
and I would like to control the threading example threads = (50)
if i chose 50 theards is opend 50 process

You may want to use pool object from python multiprocessing module.
from https://docs.python.org/2/library/multiprocessing.html
Pool object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following example demonstrates the common practice of defining such functions in a module so that child processes can successfully import that module.
Here is a simple example that demonstrates multiprocessing for ping calls.
Hope this helps.
EDIT1
**host.txt**
google.com
yahoo.com
microsoft.com
cnn.com
stackoverflow.com
github.com
CODE
from multiprocessing import Pool
import subprocess
def ping(hostname):
return hostname, subprocess.call(['ping', '-c', '3', '-w', '1000', hostname])
if __name__ == '__main__':
HOSTFILE = 'server.txt'
POOLCOUNT = 5
#read host name file and load to list
hostfile = open(HOSTFILE, 'r')
hosts = [line.strip() for line in hostfile.readlines()]
#Create pool
p = Pool(POOLCOUNT)
#multiprocess and map ping function to host list
print(p.map(ping, hosts))
Result
Status 1= sucess, 0 = failure
>>>
[('google.com', 1), ('yahoo.com', 1), ('microsoft.com', 1), ('cnn.com', 1), ('stackoverflow.com', 1), ('github.com', 1)]

Run and log the output of 'ping' in a loop in a Python script

i wrote a simple agaent in python that all it dose is just cheacks for the internet connection.
When he finds out that ther is no connection he writes a log file to a text the hour and date and then just exit the program.
I want it to keep testing if there is a connection even if there is not how can i do this ? without the program exit
this is the code:
import os
import time
def Main():
ping =os.system('ping -n 1 -l 1000 8.8.8.8 ')
while ping ==0:
time.sleep(4)
ping = os.system('ping -n 1 -l 1000 8.8.8.8 ')
if ping ==1:
print 'no connection'
CT =time.strftime("%H:%M:%S %d/%m/%y")
alert=' No Connection'
f = open('logfile.txt','a+')
f.write('\n'+CT)
f.write(alert)
f.close()
if __name__ == "__main__":
Main()
Thanx a lot.

Wrap the Main call in an infinite loop?
if __name__ == "__main__":
while True:
Main()
time.sleep(1) # optional, as Main already contains a sleep time

This code should set you on your way. Just substitute the host with that of your choosing in the call to the LogPing object.
Check out the comments inline and please ask me if you have any questions.
from datetime import datetime
import os
import shlex
import subprocess
from time import sleep
class LogPing:
def __init__(self, host, count=1, timeout_seconds=10, logfile="ping_log.txt"):
self.host = host
self.count = count
self.timeout_seconds = timeout_seconds
self.logfile = logfile
self.output_blackhole = open(os.devnull, 'wb')
def _command(self):
command_string = "ping -c {count} -t {timeout} {host}".format(
count=self.count,
timeout=self.timeout_seconds,
host=self.host
)
try:
# we don't actually care about the output, just the return code,
# so trash the output. result == 0 on success
result = subprocess.check_call(
shlex.split(command_string),
stdout=self.output_blackhole,
stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError:
# if here, that means that the host couldn't be reached for some reason.
result = -1
return result
def run(self):
ping_command_result = self._command()
if ping_command_result == 0:
status = "OK"
else:
status = "NOK"
# The time won't be exact, but close enough
message = "{status} : {time} : {host}\n".format(
status=status,
time=datetime.utcnow().strftime("%Y-%m-%d_%T"),
host=self.host
)
# open file in a context manager for writing, creating if not exists
# using a+ so that we append to the end of the last line.
with open(self.logfile, 'a+') as f:
f.write(message)
if __name__ == "__main__":
while True:
ping_instance = LogPing("example.org").run()
sleep(4)

If I understand yous correctly this will do its job:
import os
import time
def Main():
while True:
ping = os.system('ping -n 1 -l 1000 8.8.8.8 ')
if ping:
print 'no connection'
CT =time.strftime("%H:%M:%S %d/%m/%y")
alert=' No Connection'
with open('logfile.txt','a+') as f:
f.write('\n'+CT)
f.write(alert)
time.sleep(4)
if __name__ == "__main__":
Main()

Opening 2 serial ports simultaneously in python (one tx one for rx)

I am making a throughput test for a bluetooth link, and I need to send data through a serial port to one bluetooth device which will then transport that data wirelessly to another bluetooth device. The other device will then complete the circuit by sending the data back to the host PC via a different serial port.
The problem seems to be when I attempt to open up 2 different instances of PySerial, the program simply hangs. I have isolated it down to running vs. hanging when I comment out one of the two serial port instantiations. Anyone see a problem with how I'm doing this? If so, what is the proper way to do this? See code below:
#/usr/bin/python
import serial
import time
import sys
DEFAULT_BAUD = 115200
SEND_SIZE = 100
def addPath(file):
pth, fl = os.path.split(__file__)
return os.path.join(pth, file)
def is_number(s):
try:
int(s, 16)
return True
except:
return False
class SerialReader():
def __init__(self, portRx, portTx):
self.portTx = portTx
self.portRx = portRx
self.start_time__sec = time.time()
self.interval__sec = 0
self.buffer = []
self.sendtext = ''.join([str(i) for i in range(SEND_SIZE)])
# send first batch of data
self.portTx.write(self.sendtext)
def didDataArrive(self):
# Read port
self.buffer.extend(list(self.portRx.read(1024)))
# Step through the buffer byte and byte and see if the tick text
# is at the front.
while len(self.buffer) >= len(self.sendtext):
if self.buffer[:len(self.sendtext)] == self.sendtext:
# Discard the tick text
self.buffer = self.buffer[len(self.sendtext):]
# Record time
snapshot__sec = time.time()
self.interval__sec = snapshot__sec - self.start_time__sec
self.start_time__sec = snapshot__sec
# send more data
self.portTx.write(self.sendtext)
return True
else:
self.buffer.pop(0)
return False
def main(port1, port2, baudrate1 = DEFAULT_BAUD, baudrate2 = DEFAULT_BAUD):
try:
import serial
except:
traceback.print_exc()
print "="*60
print "You need to install PySerial"
print "Windows: easy_install pyserial"
print "Mac/Linux: sudo easy_install pyserial"
try:
s1 = serial.Serial(port1, baudrate1, timeout = 0.1)
s2 = serial.Serial(port2, baudrate2, timeout = 0.1)
print "Loading serial ports"
except:
print "Serial port error"
exit()
plot_stop = False
dataread = SerialReader(s2, s1)
try:
while plot_stop == False:
if dataread.didDataArrive():
print dataread.interval__sec
except KeyboardInterrupt:
print "Keyboard Interrupt"
plot_stop = True
finally:
print "Closing"
s1.close()
s2.close()
if __name__ == '__main__':
if (len(sys.argv) < 3):
print "Usage: python extract_data.py phonelink_serialport phonelinkclient_serialport [baudrate1] [baudrate2]"
else:
main(*sys.argv[1:])
If I remove one of the following lines (doesn't matter which one), the python script runs (although it eventually crashes because in the code it eventually tries to reference both ports). If I leave these lines in, the program seems to just hang (it just seems to sit there and run indefinitely):
s1 = serial.Serial(port1, baudrate1, timeout = 0.1)
s2 = serial.Serial(port2, baudrate2, timeout = 0.1)

Looping in Time Python

I'm new in python programming. I have a Question. I want to save every input data (raw data to list) every 15 minutes. after 15 minutes the list wil be delete and write input data again.can anyone help me please?thank you for your kindness.
from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory
def WriteListtoCSV (data):
with open ('tesdata.csv','a') as csvfile:
writer=csv.writer(csvfile)
for val in data:
writer.writerow([val])
class MyServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
print("Text message received: {0}".format(payload.decode('utf8')))
# echo back message verbatim
self.sendMessage(payload, isBinary)
mins = 0
data_nilai = [ ]
while mins != 60: #change value with seconds
data_nilai.append(payload.decode('utf8'))
time.sleep(1)
mins+=1
WriteListtoCSV(data_nilai)
#ClearCSV()
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import sys
import csv
import time
from twisted.python import log
from twisted.internet import reactor
log.startLogging(sys.stdout)
factory = WebSocketServerFactory(u"ws://192.168.1.23:9000", debug=False)
factory.protocol = MyServerProtocol
# factory.setProtocolOptions(maxConnections=2)
reactor.listenTCP(9000, factory)
reactor.run()
my focus is only onMessage

Following is Algo with small code.
Algo:
Set detail file path where we are saving data.
Get Input from the user and process to create list.
Save data to file.
Wait for some time.
Delete File.
Code:
import pickle
import time
import os
detail_file = "/tmp/test.txt"
while(1):
# Get input from User and split to List.
user_input = raw_input("Enter item of the list separated by comma:")
user_input = user_input.split(",")
print "User List:- ", user_input
#- Save process, We can save your data i.e. list into file or database or any where
with open(detail_file, "wb") as fp:
pickle.dump(user_input, fp)
# Wait for 15 minutes.
time.sleep(900) # 15 * 60 = 900
# delete Save details.
os.remove(detail_file)
Note:
Use input() to get user information for Python 3.x
Use raw_input() to get user information for Python 2.x
[Edit 1]
Crontab
Ref: http://www.thegeekstuff.com/2011/07/cron-every-5-minutes/
Ref: http://www.computerhope.com/unix/ucrontab.htm
OS: CentOS
To edit the crontab, use this command:
crontab -e
*/15 * * * * python /tmp/script.py
where crontab entry structure is:
m h dom mon dow command