My current predicament is that I attempted to make a blocking web serving script non blocking to allow for more than one download to take place at any one time but currently it will hang and wait for the first download to complete before starting the second. Before you go out of your way to down vote this because the answer is odious please know that this is my first ever python script and I am self teaching.
In the example below I only post a single "ConnectionProcesser" Because they all contain the same code
if you need more code please just ask
The script has 3 dependinces
import socket # Networking support
import signal # Signal support (server shutdown on signal receive)
import threading #to make the thing run more than one at a time
Please note that the script has been edited and quite a bit of the code is missing but I believe that it is unrelated to the problem.
def ConnectionProcessorC(self):
connC, AddressC = self.socket.accept()
print("C Got connection from:", AddressC)
DataRecivedC = connC.recv(1024) #receive data from client
DataRecivedC = bytes.decode(DataRecivedC) #decode it to string
print(DataRecivedC)
RequestMethod = DataRecivedC.split(' ')[0]
print ("C Method: ", RequestMethod)
if (RequestMethod == 'GET') | (RequestMethod == 'HEAD'):
Response_Headers = 'HTTP/1.1 200 OK\n'
# Current_Date = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime())
# Response_Headers += 'Date: ' + current_date +'\n'
Response_Headers += 'Server: Moes-Python-Server\n'
Response_Headers += 'Connection: close\n\n' # signal that the conection wil be closed after complting the request
Server_Response = Response_Headers.encode() # return headers for GET and HEAD
file_handler = open('/usr/share/nginx/html/100mb.dump','rb')
Response_Content = file_handler.read() # read file content
file_handler.close()
URL=DataRecivedC.split(' ')
URL = URL[1] # get 2nd element
#Response_Content="<html><body><p>Charlie TEStin this stuff yehURL:"+URL+"</p></body></html>"
Server_Response += Response_Content
connC.send(Server_Response)
print ("C Closing connection with client")
else:
print("C Unknown HTTP request method:", RequestMethod)
connC.close()
return
def Distrabuteconnections(self):
A=0
""" Main loop awaiting connections """
while True:
print ("Awaiting New connection")
self.socket.listen(10) # maximum number of queued connections #changed to 1 from 3 to try and prevent waiting after closing for ther que to clean up
if (A==0):
ConnectionProcessorA = threading.Thread(target=self.ConnectionProcessorA())
ConnectionProcessorA.start()
A=1
elif (A==1):
ConnectionProcessorB = threading.Thread(target=self.ConnectionProcessorB())
ConnectionProcessorB.start()
A=2
else:
ConnectionProcessorC = threading.Thread(target=self.ConnectionProcessorC())
ConnectionProcessorC.start()
A=0
I think that the problem could be solved by changing while true to something that loops 3 times instead of one.
You should pass a reference to the method you wish to start in a thread. Instead, you are calling the thread, and passing the data returned from that method to the threading.Thread() call.
In short your code should become:
if (A==0):
ConnectionProcessorA = threading.Thread(target=self.ConnectionProcessorA)
ConnectionProcessorA.start()
A=1
elif (A==1):
ConnectionProcessorB = threading.Thread(target=self.ConnectionProcessorB)
ConnectionProcessorB.start()
A=2
else:
ConnectionProcessorC = threading.Thread(target=self.ConnectionProcessorC)
ConnectionProcessorC.start()
A=0
Note the removal of the brackets after self.ConnectionProcessorA etc. This passes a reference to the method to start in the thread, which the threading module will call itself.
Note, it is recommended to store a reference to the thread you create so that it doesn't get garbage collected. I would thus recommend your code becomes:
if (A==0):
self.cpa_thread= threading.Thread(target=self.ConnectionProcessorA)
self.cpa_thread.start()
A=1
elif (A==1):
self.cpb_thread= threading.Thread(target=self.ConnectionProcessorB)
self.cpb_thread.start()
A=2
else:
self.cpc_thread= threading.Thread(target=self.ConnectionProcessorC)
self.cpc_thread.start()
A=0
Related
The bounty expires in 5 days. Answers to this question are eligible for a +50 reputation bounty.
Haley Mueller wants to draw more attention to this question.
I'm new to Python so this could be a simple fix.
I am using Flask and sockets for this Python project. I am starting the socket on another thread so I can actively listen for new messages. I have an array variable called 'SocketConnections' that is within my UdpComms class. The variable gets a new 'Connection' appended to it when a new socket connection is made. That works correctly. My issue is that when I try to read 'SocketConnections' from outside of the thread looking in, it is an empty array.
server.py
from flask import Flask, jsonify
import UdpComms as U
app = Flask(__name__)
#app.route('/api/talk', methods=['POST'])
def talk():
global global_server_socket
apples = global_server_socket.SocketConnections
return jsonify(message=apples)
global_server_socket = None
def start_server():
global global_server_socket
sock = U.UdpComms(udpIP="127.0.0.1", portTX=8000, portRX=8001, enableRX=True, suppressWarnings=True)
i = 0
global_server_socket = sock
while True:
i += 1
data = sock.ReadReceivedData() # read data
if data != None: # if NEW data has been received since last ReadReceivedData function call
print(data) # print new received data
time.sleep(1)
if __name__ == '__main__':
server_thread = threading.Thread(target=start_server)
server_thread.start()
app.run(debug=True,host='192.168.0.25')
UdpComms.py
import json
import uuid
class UdpComms():
def __init__(self,udpIP,portTX,portRX,enableRX=False,suppressWarnings=True):
self.SocketConnections = []
import socket
self.udpIP = udpIP
self.udpSendPort = portTX
self.udpRcvPort = portRX
self.enableRX = enableRX
self.suppressWarnings = suppressWarnings # when true warnings are suppressed
self.isDataReceived = False
self.dataRX = None
# Connect via UDP
self.udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # internet protocol, udp (DGRAM) socket
self.udpSock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # allows the address/port to be reused immediately instead of it being stuck in the TIME_WAIT state waiting for late packets to arrive.
self.udpSock.bind((udpIP, portRX))
# Create Receiving thread if required
if enableRX:
import threading
self.rxThread = threading.Thread(target=self.ReadUdpThreadFunc, daemon=True)
self.rxThread.start()
def __del__(self):
self.CloseSocket()
def CloseSocket(self):
# Function to close socket
self.udpSock.close()
def SendData(self, strToSend):
# Use this function to send string to C#
self.udpSock.sendto(bytes(strToSend,'utf-8'), (self.udpIP, self.udpSendPort))
def SendDataAddress(self, strToSend, guid):
# Use this function to send string to C#
print('finding connection: ' + guid)
if self.SocketConnections:
connection = self.GetConnectionByGUID(guid)
print('found connection: ' + guid)
if connection is not None:
self.udpSock.sendto(bytes(strToSend,'utf-8'), connection.Address)
def ReceiveData(self):
if not self.enableRX: # if RX is not enabled, raise error
raise ValueError("Attempting to receive data without enabling this setting. Ensure this is enabled from the constructor")
data = None
try:
data, _ = self.udpSock.recvfrom(1024)
print('Socket data recieved from: ', _)
if self.IsNewConnection(_) == True:
print('New socket')
self.SendDataAddress("INIT:" + self.SocketConnections[-1].GUID, self.SocketConnections[-1].GUID)
data = data.decode('utf-8')
except WindowsError as e:
if e.winerror == 10054: # An error occurs if you try to receive before connecting to other application
if not self.suppressWarnings:
print("Are You connected to the other application? Connect to it!")
else:
pass
else:
raise ValueError("Unexpected Error. Are you sure that the received data can be converted to a string")
return data
def ReadUdpThreadFunc(self): # Should be called from thread
self.isDataReceived = False # Initially nothing received
while True:
data = self.ReceiveData() # Blocks (in thread) until data is returned (OR MAYBE UNTIL SOME TIMEOUT AS WELL)
self.dataRX = data # Populate AFTER new data is received
self.isDataReceived = True
# When it reaches here, data received is available
def ReadReceivedData(self):
data = None
if self.isDataReceived: # if data has been received
self.isDataReceived = False
data = self.dataRX
self.dataRX = None # Empty receive buffer
if data != None and data.startswith('DIALOG:'): #send it info
split = data.split(':')[1]
return data
class Connection:
def __init__(self, gUID, address) -> None:
self.GUID = gUID
self.Address = address
def IsNewConnection(self, address):
for connection in self.SocketConnections:
if connection.Address == address:
return False
print('Appending new connection...')
connection = self.Connection(str(uuid.uuid4()),address)
self.SocketConnections.append(connection)
return True
def GetConnectionByGUID(self, guid):
for connection in self.SocketConnections:
if connection.GUID == guid:
return connection
return None
As mentioned above. When IsNewConnection() is called in UdpComms it does append a new object to SocketConnections. It's just trying to view the SocketConnections in the app.route that is empty. My plans are to be able to send socket messages from the app.routes
For interprocess communication you may try to use something like shared memory documented here
Instead of declaring your self.SocketConnections as a list = []
you'd use self.SocketConnections = Array('i', range(10)) (you are then limited to remembering only 10 connections though).
I have a socket connection which I want to monitor, it recevies market data with high burst.
while 1:
socket.recv()
print('data recevied')
The while loop should only execute print, once in sixty seconds.
Try this:
from datetime import datetime
last = datetime.now()
while 1:
socket.recv()
if (datetime.now() - last).seconds >= 60:
print("data received")
last = datetime.now()
You want some kind of asynchronous processing here: on one hand you want to continuously receive data, on the other hand you want to display a message every 60 seconds.
So threading would be my first idea: foreground display messages while background receives.
def recv_loop(socket, end_cond):
while True:
socket.recv(1024)
# assuming something ends above loop
end_cond[0] = True
end = False
recv_thr = threading.Thread(target = recv_loop, args = (socket,[end]), daemon = True)
recv_thr.start()
while not end:
time.sleep(60)
print('data received')
You haven't show any interesting data in the message, so I haven't either. But as all global variables are shared, it would be trivial to display for example the number of bytes received since last message
Alternatively, you could use select.select because it gives you a timeout. So you change threading for a more complex timeout processing.
last = datetime.datetime.now()
while True:
timeout = (datetime.datetime.now() - last).seconds + 60
if timeout <= 0:
last = datetime.datetime.now()
print("data received")
rl, _, _ = select.select([socket], [], [], timeout)
if (len(rl) > 0):
sockect.recv(1024)
I am try to display data that is sent over a socket via an iteration from a loop. The way I have it at the moment isn't working on the Admin client. What should I do to fix my loop? Thank you
Admin thread in the server -
def HandleAdmin(adminSocket,):
global addressList
(c,a) = adminSocket.accept()
ts = ssl.wrap_socket(c, certfile="5cc515_server.crt",
keyfile="5cc515_server.key",
server_side=True,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs="5cc515-root-ca.cer")
if ts.recv(80).decode() == 'Hello\r\n':
ts.send('Admin-Greetings\r\n'.encode())
if ts.recv(80).decode() == 'Who\r\n':
for i in addressList:
ts.send(i.encode())
ts.close()
return
Admin Client
import ssl
import socket
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
ts = ssl.wrap_socket(s, certfile="100298750.crt",
keyfile="100298750.key",
ca_certs="5cc515-root-ca.cer")
ts.connect(('127.0.0.1', 4001))
ts.send("Hello\r\n".encode())
if ts.recv(80).decode() == "Admin-Greetings\r\n":
print("The players currently online are:\n")
ts.send("Who\r\n".encode())
loop = True
try:
while(loop == True):
if (ts.recv(1000) != Null):
print(ts.recv(1000).decode())
else:
loop = False
ts.close()
except:
pass
The first problem is that you try to do != Null, which will raise a NameError as Null is not valid in python, it's None. You don't see that error because of the raw except. You should only except the errors you are actually expecting.
In addition to that, recv doesn't return None when there is not data or the connection is closed, it returns an empty string.
Then in your loop you call recv twice, throwing away the result of the first call after the comparison. A better way to write your loop would be:
try:
data = ts.recv(1000)
while data:
print(data.decode())
data = ts.recv(1000)
except IOError as e:
pass
finally:
ts.close()
Or, if you want a more pythonic solution, use a function iterator with the empty string as sentinel:
from functools import partial
for data in iter(partial(ts.recv, 1000), b''):
print(data.decode())
I am working on creating a HTTP client which can generate hundreds of connections each second and send up to 10 requests on each of those connections. I am using threading so concurrency can be achieved.
Here is my code:
def generate_req(reqSession):
requestCounter = 0
while requestCounter < requestRate:
try:
response1 = reqSession.get('http://20.20.1.2/tempurl.html')
if response1.status_code == 200:
client_notify('r')
except(exceptions.ConnectionError, exceptions.HTTPError, exceptions.Timeout) as Err:
client_notify('F')
break
requestCounter += 1
def main():
for q in range(connectionPerSec):
s1 = requests.session()
t1 = threading.Thread(target=generate_req, args=(s1,))
t1.start()
Issues:
It is not scaling above 200 connections/sec with requestRate = 1. I ran other available HTTP clients on the same client machine and against the server, test runs fine and it is able to scale.
When requestRate = 10, connections/sec drops to 30.
Reason: Not able to create targeted number of threads every second.
For issue #2, client machine is not able to create enough request sessions and start new threads. As soon as requestRate is set to more than 1, things start to fall apart.
I am suspecting it has something to do with HTTP connection pooling which requests uses.
Please suggest what am I doing wrong here.
I wasn't able to get things to fall apart, however the following code has some new features:
1) extended logging, including specific per-thread information
2) all threads join()ed at the end to make sure the parent process doesntt leave them hanging
3) multithreaded print tends to interleave the messages, which can be unwieldy. This version uses yield so a future version can accept the messages and print them clearly.
source
import exceptions, requests, threading, time
requestRate = 1
connectionPerSec = 2
def client_notify(msg):
return time.time(), threading.current_thread().name, msg
def generate_req(reqSession):
requestCounter = 0
while requestCounter < requestRate:
try:
response1 = reqSession.get('http://127.0.0.1/')
if response1.status_code == 200:
print client_notify('r')
except (exceptions.ConnectionError, exceptions.HTTPError, exceptions.Timeout):
print client_notify('F')
break
requestCounter += 1
def main():
for cnum in range(connectionPerSec):
s1 = requests.session()
th = threading.Thread(
target=generate_req, args=(s1,),
name='thread-{:03d}'.format(cnum),
)
th.start()
for th in threading.enumerate():
if th != threading.current_thread():
th.join()
if __name__=='__main__':
main()
output
(1407275951.954147, 'thread-000', 'r')
(1407275951.95479, 'thread-001', 'r')
I'm making a django app that needs to be able to make emails and then send these out at a given time. I was thinking i could use django-mailer to put things in que and then send it of. But even though theire sample case list, lists that this is a feature, I cant seem to find out how.
What I need is to be able to set a 'when_to_send' field in the message model of django-mailer, and when the cron job fires the send_mail function this needs to filter out the ones that has a 'when_to_send' date that is greater than the current time...
def send_all():
"""
Send all eligible messages in the queue.
"""
lock = FileLock("send_mail")
logging.debug("acquiring lock...")
try:
lock.acquire(LOCK_WAIT_TIMEOUT)
except AlreadyLocked:
logging.debug("lock already in place. quitting.")
return
except LockTimeout:
logging.debug("waiting for the lock timed out. quitting.")
return
logging.debug("acquired.")
start_time = time.time()
dont_send = 0
deferred = 0
sent = 0
try:
for message in prioritize():
if DontSendEntry.objects.has_address(message.to_address):
logging.info("skipping email to %s as on don't send list " % message.to_address)
MessageLog.objects.log(message, 2) # ### avoid using literal result code
message.delete()
dont_send += 1
else:
try:
logging.info("sending message '%s' to %s" % (message.subject.encode("utf-8"), message.to_address.encode("utf-8")))
core_send_mail(message.subject, message.message_body, message.from_address, [message.to_address])
MessageLog.objects.log(message, 1) # ### avoid using literal result code
message.delete()
sent += 1
except (socket_error, smtplib.SMTPSenderRefused, smtplib.SMTPRecipientsRefused, smtplib.SMTPAuthenticationError), err:
message.defer()
logging.info("message deferred due to failure: %s" % err)
MessageLog.objects.log(message, 3, log_message=str(err)) # ### avoid using literal result code
deferred += 1
finally:
logging.debug("releasing lock...")
lock.release()
logging.debug("released.")
logging.info("")
logging.info("%s sent; %s deferred; %s don't send" % (sent, deferred, dont_send))
logging.info("done in %.2f seconds" % (time.time() - start_time))
Anyone see how to customize this function to don't send email's where the message.when_to_send field is greater than the current time?
You need to implement the cron job for django-mailer:
* * * * * (cd $PINAX; /usr/local/bin/python2.5 manage.py send_mail >> $PINAX/cron_mail.log 2>&1)
And then in engine.py line 96:
# Get rid of "while True:"
while not Message.objects.all():
# Get rid of logging.debug("sleeping for %s seconds before checking queue again" % EMPTY_QUEUE_SLEEP)
# Get rid of sleep
send_all()
You can just add another clause to the conditionals under your message processing loop (you will also need to import datetime at the top of your file):
for message in prioritize():
if DontSendEntry.objects.has_address(message.to_address):
logging.info("skipping email to %s as on don't send list " % message.to_address)
MessageLog.objects.log(message, 2) # ### avoid using literal result code
message.delete()
dont_send += 1
elif message.when_to_send > datetime.datetime.now():
continue
else:
try:
... the rest of your code ...