Develop Reference

using select stdin for non blocking input python

I want to create a process that has 3 sub processes, 2 to handle websockets and one to get input from terminal to pass to the other two sockets.
import sys
import time
import select
import asyncio
import threading
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Process
def managment_api():
poller = select.poll()
poller.register(sys.stdin, select.POLLIN)
started = True
count = 0
while started:
print("management api [{:^6}]".format(count))
while sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
_line = sys.stdin.readline()
if _line:
line = _line.strip().lower()
if(line == "exit" or line == "quit"):
started = False
print("got exit message [{}]".format(started))
else:
pass
count += 1
time.sleep(1)
def process_main(loop):
print("process_main BEGIN")
loop.run_in_executor(None, managment_api())
print("process_main ENG ")
if __name__ == "__main__":
#this doesn't work
main = Process(target=managment_api, args=())
main.name = "management api"
main.start()
main.join()
"""
#asyncio.run(managment_api()) #need to add async to management_api
When I make management_api an async function and call asyncio.run(management_api); I can get input.
If I try to run the same code without async in a separate process it, I get stuck in the while sys.stdin in selec... section of the code. I've tried with threads but that doesn't work either.
How can I run this code from a separate process, to get the input in another process?

I was able to solve the problem by first, using fn = sys.stdin.fileno() to get the main process file descriptor, passing that as an argument to the subprocess. Then using sys.stdin = os.fdopen(fn)
import sys
import time
import select
import asyncio
import threading
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Process
def managment_api(fn):
sys.stdin = os.fdopen(fn)
poller = select.poll()
poller.register(sys.stdin, select.POLLIN)
started = True
count = 0
while started:
print("management api [{:^6}]".format(count))
while sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
_line = sys.stdin.readline()
if _line:
line = _line.strip().lower()
if(line == "exit" or line == "quit"):
started = False
print("got exit message [{}]".format(started))
else:
pass
count += 1
time.sleep(1)
def process_main(loop):
print("process_main BEGIN")
loop.run_in_executor(None, managment_api())
print("process_main ENG ")
if __name__ == "__main__":
fn = sys.stdin.fileno()
main = Process(target=managment_api, args=(fn, ))
main.name = "management api"
main.start()
main.join()
"""
#asyncio.run(managment_api()) #need to add async to management_api

use multiprocessing or threading script python

i want use multiprocessing or threading for this code
and I would like to control the threading example threads = (50)
if i chose 50 theards is opend 50 process
please help me
this code :-
import subprocess
import csv
def ping(hostname):
p = subprocess.Popen(["ping", "-n", "1","-w","1000",hostname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pingStatus = 'ok';
for line in p.stdout:
output = line.rstrip().decode('UTF-8')
if (output.endswith('unreachable.')) :
#No route from the local system. Packets sent were never put on the wire.
pingStatus = 'unreacheable'
break
elif (output.startswith('Ping request could not find host')) :
pingStatus = 'host_not_found'
break
if (output.startswith('Request timed out.')) :
#No Echo Reply messages were received within the default time of 1 second.
pingStatus = 'timed_out'
break
#end if
#endFor
return pingStatus
#endDef
def printPingResult(hostname):
statusOfPing = ping(hostname)
if (statusOfPing == 'host_not_found') :
writeToFile('!server-not-found.txt', hostname)
elif (statusOfPing == 'unreacheable') :
writeToFile('!unreachable.txt', hostname)
elif (statusOfPing == 'timed_out') :
writeToFile('!timed_out.txt', hostname)
elif (statusOfPing == 'ok') :
writeToFile('!ok.txt', hostname)
#endIf
#endPing
def writeToFile(filename, data) :
with open(filename, 'a') as output:
output.write(data + '\n')
#endWith
#endDef
'''
servers.txt example
vm8558
host2
server873
google.com
'''
file = open('hosts.txt')
try:
reader = csv.reader(file)
for item in reader:
printPingResult(item[0].strip())
#endFor
finally:
file.close()
#endTry
i want use multiprocessing or threading for this code
and I would like to control the threading example threads = (50)
if i chose 50 theards is opend 50 process

You may want to use pool object from python multiprocessing module.
from https://docs.python.org/2/library/multiprocessing.html
Pool object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following example demonstrates the common practice of defining such functions in a module so that child processes can successfully import that module.
Here is a simple example that demonstrates multiprocessing for ping calls.
Hope this helps.
EDIT1
**host.txt**
google.com
yahoo.com
microsoft.com
cnn.com
stackoverflow.com
github.com
CODE
from multiprocessing import Pool
import subprocess
def ping(hostname):
return hostname, subprocess.call(['ping', '-c', '3', '-w', '1000', hostname])
if __name__ == '__main__':
HOSTFILE = 'server.txt'
POOLCOUNT = 5
#read host name file and load to list
hostfile = open(HOSTFILE, 'r')
hosts = [line.strip() for line in hostfile.readlines()]
#Create pool
p = Pool(POOLCOUNT)
#multiprocess and map ping function to host list
print(p.map(ping, hosts))
Result
Status 1= sucess, 0 = failure
>>>
[('google.com', 1), ('yahoo.com', 1), ('microsoft.com', 1), ('cnn.com', 1), ('stackoverflow.com', 1), ('github.com', 1)]

Can't write to file from child process

I can't wrap my head around this... I have the following code:
def launch(command):
pf = os.path.join(working_directory, '.pid')
pid = os.fork()
if pid == 0:
stdout = os.open(..., os.O_WRONLY | os.O_CREAT)
try:
proc = Popen(command, shell=False, stdout=stdout, cwd=workdir)
print(proc.pid)
with open(pf, 'wb') as p: # pf should not be open as the file is just created.
p.write(proc.pid)
print("Hello World")
except OSError as proc_error:
...
finally:
os._exit(o) # socketserver catches SystemExit exception (flask)
else:
start = time.time()
while not os.path.isfile(pf): # I'm just checking if that file exists here never opened it in the first place.
if time.time() - start >= 30:
raise TimeoutError...
time.sleep(5)
pid = int(open(pf, 'rb').read())
Here's the output:
$pid
TimeoutError occurred
The script seem to be hanging at opening pf for writing. I verified, the file if not created, Hello World never gets printed.
Why is this happening, and how can fix it?
Thanks!

I have reduced your code to this (removing everything I could not reproduce given your code):
import os
import time
s = "foo"
pid = os.fork()
from subprocess import Popen
if pid == 0:
proc = Popen(["sleep", "3"])
with open(s, "w") as p:
p.write(str(proc.pid)) # <- Only real error I could see
os._exit(0)
else:
start = time.time()
while not os.path.isfile(s):
if time.time() - start >= 30:
raise TimeoutError("Command took to long")
time.sleep(5)
print("Read from file: " + open(s, 'r').read())
However, it works just fine, it prints Read from file: 12075. So the issue is not in the part that can be reproduced, given your code.
To read/write the procid to the binary file I succesfully used the pickle module:
pickle.dump(proc.pid,p) # write to file
pickle.load(open(s, "rb")) #read from file

Run and log the output of 'ping' in a loop in a Python script

i wrote a simple agaent in python that all it dose is just cheacks for the internet connection.
When he finds out that ther is no connection he writes a log file to a text the hour and date and then just exit the program.
I want it to keep testing if there is a connection even if there is not how can i do this ? without the program exit
this is the code:
import os
import time
def Main():
ping =os.system('ping -n 1 -l 1000 8.8.8.8 ')
while ping ==0:
time.sleep(4)
ping = os.system('ping -n 1 -l 1000 8.8.8.8 ')
if ping ==1:
print 'no connection'
CT =time.strftime("%H:%M:%S %d/%m/%y")
alert=' No Connection'
f = open('logfile.txt','a+')
f.write('\n'+CT)
f.write(alert)
f.close()
if __name__ == "__main__":
Main()
Thanx a lot.

Wrap the Main call in an infinite loop?
if __name__ == "__main__":
while True:
Main()
time.sleep(1) # optional, as Main already contains a sleep time

This code should set you on your way. Just substitute the host with that of your choosing in the call to the LogPing object.
Check out the comments inline and please ask me if you have any questions.
from datetime import datetime
import os
import shlex
import subprocess
from time import sleep
class LogPing:
def __init__(self, host, count=1, timeout_seconds=10, logfile="ping_log.txt"):
self.host = host
self.count = count
self.timeout_seconds = timeout_seconds
self.logfile = logfile
self.output_blackhole = open(os.devnull, 'wb')
def _command(self):
command_string = "ping -c {count} -t {timeout} {host}".format(
count=self.count,
timeout=self.timeout_seconds,
host=self.host
)
try:
# we don't actually care about the output, just the return code,
# so trash the output. result == 0 on success
result = subprocess.check_call(
shlex.split(command_string),
stdout=self.output_blackhole,
stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError:
# if here, that means that the host couldn't be reached for some reason.
result = -1
return result
def run(self):
ping_command_result = self._command()
if ping_command_result == 0:
status = "OK"
else:
status = "NOK"
# The time won't be exact, but close enough
message = "{status} : {time} : {host}\n".format(
status=status,
time=datetime.utcnow().strftime("%Y-%m-%d_%T"),
host=self.host
)
# open file in a context manager for writing, creating if not exists
# using a+ so that we append to the end of the last line.
with open(self.logfile, 'a+') as f:
f.write(message)
if __name__ == "__main__":
while True:
ping_instance = LogPing("example.org").run()
sleep(4)

If I understand yous correctly this will do its job:
import os
import time
def Main():
while True:
ping = os.system('ping -n 1 -l 1000 8.8.8.8 ')
if ping:
print 'no connection'
CT =time.strftime("%H:%M:%S %d/%m/%y")
alert=' No Connection'
with open('logfile.txt','a+') as f:
f.write('\n'+CT)
f.write(alert)
time.sleep(4)
if __name__ == "__main__":
Main()

Using python how to execute this code concurrently for multiple files?

I want to tail multiple files concurrently and push the logs to scribe.
I am reading the files from a Config file then I want to tail each file and send the logs to scribe.
What I have tried is sends log for only the first file and doesn't for the others.
I want to run the tailing concurrently for each file and send the logs for each one of them at same time.
for l in Config.items('files'):
print l[0]
print l[1]
filename = l[1]
file = open(filename,'r')
st_results = os.stat(l[1])
st_size = st_results[6]
file.seek(st_size)
while 1:
where = file.tell()
line = file.readline()
if not line:
time.sleep(1)
file.seek(where)
else:
print line, # already has newline
category=l[0]
message=line
log_entry = scribe.LogEntry(category, message)
socket = TSocket.TSocket(host='localhost', port=1463)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
result = client.Log(messages=[log_entry])
transport.close()

Try something like this (Inspired by this)
import threading
def monitor_file(l):
print l[0]
print l[1]
filename = l[1]
file = open(filename,'r')
st_results = os.stat(l[1])
st_size = st_results[6]
file.seek(st_size)
while 1:
where = file.tell()
line = file.readline()
if not line:
time.sleep(1)
file.seek(where)
else:
print line, # already has newline
category=l[0]
message=line
log_entry = scribe.LogEntry(category, message)
socket = TSocket.TSocket(host='localhost', port=1463)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
result = client.Log(messages=[log_entry])
transport.close()
for l in Config.items('files'):
thread = threading.Thread(target=monitor_file, args=(l))

A different implementation of #Pengman's idea:
#!/usr/bin/env python
import os
import time
from threading import Thread
def follow(filename):
with open(filename) as file:
file.seek(0, os.SEEK_END) # goto EOF
while True:
for line in iter(file.readline, ''):
yield line
time.sleep(1)
def logtail(category, filename):
print category
print filename
for line in follow(filename):
print line,
log_entry(category, line)
for args in Config.items('files'):
Thread(target=logtail, args=args).start()
Where log_entry() is a copy of the code from the question:
def log_entry(category, message):
entry = scribe.LogEntry(category, message)
socket = TSocket.TSocket(host='localhost', port=1463)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
result = client.Log(messages=[entry])
transport.close()
follow() could be implemented using FS monitoring tools, see tail -f in python with no time.sleep.