Develop Reference

python checking file changes without reading the full file

I have a web app (in the backend) where I am using pysondb (https://github.com/pysonDB/pysonDB) to upload some tasks which will be executed by another program (sniffer).
The sniffer program (a completely separate program) now checks the database for any new unfinished uploaded tasks in an infinite loop and executes them and updates the database.
I don't want to read the database repeatedly, instead want to look for any file changes in the database file (db.json), then read the database only. I have looked into watchdog but was looking for something lightweight and modern to suit my needs.
# infinite loop
import pysondb
import time
from datetime import datetime
# calling aligner with os.system
import os
import subprocess
from pathlib import Path
while True:
# always alive
time.sleep(2)
try:
# process files
db = pysondb.getDb("../tasks_db.json")
tasks = db.getBy({"task_status": "uploaded"})
for task in tasks:
try:
task_path = task["task_path"]
cost = task["cost"]
corpus_folder = task_path
get_output = subprocess.Popen(f"mfa validate {corpus_folder} english english", shell=True, stdout=subprocess.PIPE).stdout
res = get_output.read().decode("utf-8")
# print(type(res))
if "ERROR - There was an error in the run, please see the log." in res:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {str(res)}\n")
f.close()
else:
align_folder = f"{corpus_folder}_aligned"
Path(align_folder).mkdir(parents=True, exist_ok=True)
o = subprocess.Popen(f"mfa align {corpus_folder} english english {align_folder}", shell=True, stdout=subprocess.PIPE).stdout.read().decode("utf-8")
# success
except subprocess.CalledProcessError:
# mfa align ~/mfa_data/my_corpus english english ~/mfa_data/my_corpus_aligned
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: Files not in right format\n")
f.close()
except Exception as e:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {e}\n")
f.close()

Using python-rq would be a much more efficient way of doing this that wouldn't need a database. It has no requirements other then needing a redis install. From there, you could just move all of that into a function:
def task(task_path, cost):
corpus_folder = task_path
get_output = subprocess.Popen(f"mfa validate {corpus_folder} english english", shell=True, stdout=subprocess.PIPE).stdout
res = get_output.read().decode("utf-8")
# print(type(res))
if "ERROR - There was an error in the run, please see the log." in res:
# log errors
f = open("sniffer_log.error", "a+")
f.write(f"{datetime.now()} :: {str(res)}\n")
... #etc
Obviously you would rename that function and put the try-except statement back, but then you could just call that through RQ:
# ... where you want to call the function
from wherever.you.put.your.task.function import task
result = your_redis_queue.enqueue(task, "whatever", "arguments)

Get local DNS settings in Python

Is there any elegant and cross platform (Python) way to get the local DNS settings?
It could probably work with a complex combination of modules such as platform and subprocess, but maybe there is already a good module, such as netifaces which can retrieve it in low-level and save some "reinventing the wheel" effort.
Less ideally, one could probably query something like dig, but I find it "noisy", because it would run an extra request instead of just retrieving something which exists already locally.
Any ideas?

Using subprocess you could do something like this, in a MacBook or Linux system
import subprocess
process = subprocess.Popen(['cat', '/etc/resolv.conf'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
print(stdout, stderr)
or do something like this
import subprocess
with open('dns.txt', 'w') as f:
process = subprocess.Popen(['cat', '/etc/resolv.conf'], stdout=f)
The first output will go to stdout and the second to a file

Maybe this one will solve your problem
import subprocess
def get_local_dns(cmd_):
with open('dns1.txt', 'w+') as f:
with open('dns_log1.txt', 'w+') as flog:
try:
process = subprocess.Popen(cmd_, stdout=f, stderr=flog)
except FileNotFoundError as e:
flog.write(f"Error while executing this command {str(e)}")
linux_cmd = ['cat', '/etc/resolv.conf']
windows_cmd = ['windows_command', 'parameters']
commands = [linux_cmd, windows_cmd]
if __name__ == "__main__":
for cmd in commands:
get_local_dns(cmd)

Thanks #MasterOfTheHouse.
I ended up writing my own function. It's not so elegant, but it does the job for now. There's plenty of room for improvement, but well...
import os
import subprocess
def get_dns_settings()->dict:
# Initialize the output variables
dns_ns, dns_search = [], ''
# For Unix based OSs
if os.path.isfile('/etc/resolv.conf'):
for line in open('/etc/resolv.conf','r'):
if line.strip().startswith('nameserver'):
nameserver = line.split()[1].strip()
dns_ns.append(nameserver)
elif line.strip().startswith('search'):
search = line.split()[1].strip()
dns_search = search
# If it is not a Unix based OS, try "the Windows way"
elif os.name == 'nt':
cmd = 'ipconfig /all'
raw_ipconfig = subprocess.check_output(cmd)
# Convert the bytes into a string
ipconfig_str = raw_ipconfig.decode('cp850')
# Convert the string into a list of lines
ipconfig_lines = ipconfig_str.split('\n')
for n in range(len(ipconfig_lines)):
line = ipconfig_lines[n]
# Parse nameserver in current line and next ones
if line.strip().startswith('DNS-Server'):
nameserver = ':'.join(line.split(':')[1:]).strip()
dns_ns.append(nameserver)
next_line = ipconfig_lines[n+1]
# If there's too much blank at the beginning, assume we have
# another nameserver on the next line
if len(next_line) - len(next_line.strip()) > 10:
dns_ns.append(next_line.strip())
next_next_line = ipconfig_lines[n+2]
if len(next_next_line) - len(next_next_line.strip()) > 10:
dns_ns.append(next_next_line.strip())
elif line.strip().startswith('DNS-Suffix'):
dns_search = line.split(':')[1].strip()
return {'nameservers': dns_ns, 'search': dns_search}
print(get_dns_settings())
By the way... how did you manage to write two answers with the same account?

Running into a Multithreading issue connecting to multiple devices at the same time

I am defining the main function with def get_info(). This function doesn't take the arguments. This program uses argumentParser to parse the arguments from the command line. The argument provided is the CSV file with --csv option. This picks up the csv file from the current directory and read the lines each containing an IP address, logs into devices serially and runs few commands return the output and appends in the text file. When the code runs it removes the old text file from the directory and create a new output text file when executed.
Problem: I want to achieve this using threading module so that it takes 5 devices in parallel and outputs to a file. The problem I am running is with the lock issues as the same object is being used by same process at the same time. Here the sample code I have written. The threading concept is very new to me so please understand.
import getpass
import csv
import time
import os
import netmiko
import paramiko
from argparse import ArgumentParser
from multiprocessing import Process, Queue
def get_ip(device_ip,output_q):
try:
ssh_session = netmiko.ConnectHandler(device_type='cisco_ios', ip=device_row['device_ip'],
username=ssh_username, password=ssh_password)
time.sleep(2)
ssh_session.clear_buffer()
except (netmiko.ssh_exception.NetMikoTimeoutException,
netmiko.ssh_exception.NetMikoAuthenticationException,
paramiko.ssh_exception.SSHException) as s_error:
print(s_error)
def main():
show_vlanfile = "pool.txt"
if os.path.isfile(show_vlanfile):
try:
os.remove(show_vlanfile)
except OSError as e:
print("Error: %s - %s." %(e.filename, e.strerror))
parser = ArgumentParser(description='Arguments for running oneLiner.py')
parser.add_argument('-c', '--csv', required=True, action='store', help='Location of CSV file')
args = parser.parse_args()
ssh_username = input("SSH username: ")
ssh_password = getpass.getpass('SSH Password: ')
with open(args.csv, "r") as file:
reader = csv.DictReader(file)
output_q = Queue(maxsize=5)
procs = []
for device_row in reader:
# print("+++++ {0} +++++".format(device_row['device_ip']))
my_proc = Process(target=show_version_queue, args=(device_row, output_q))
my_proc.start()
procs.append(my_proc)
# Make sure all processes have finished
for a_proc in procs:
a_proc.join()
commands = ["terminal length 0","terminal width 511","show run | inc hostname","show ip int brief | ex una","show
vlan brief","terminal length 70"]
output = ''
for cmd in commands:
output += "\n"
output += ssh_session.send_command(cmd)
output += "\n"
with open("pool.txt", 'a') as outputfile:
while not output_q.empty():
output_queue = output_q.get()
for x in output_queue:
outputfile.write(x)
if name == "main":
main()

Somewhat different take...
I run effectively a main task, and then just fire up a (limited) number of threads; and they communicate via 2 data queues - basically "requests" and "responses".
Main task
dumps the requests into the request queue.
fires up a number (i.e. 10 or so...) worker tasks.
sits on the "response" queue waiting for results. The results can be simple user info messages about status, error messages, or DATA responses to be written out to files.
When all the threads finish, program shuts down.
Workers basically:
get a request. If none, just shut down
connect to the device
send a log message to the response queue that it's started.
does what it has to do.
puts the result as DATA to the response queue
closes the connection to the device
loop back to the start
This way you don't inadvertently flood the processing host, as you have a limited number of concurrent threads going, all doing exactly the same thing in their own sweet time, until there's nothing left to do.
Note that you DON'T do any screen/file IO in the threads, as it will get jumbled with the different tasks running at the same time. Each essentially only sees inputQ, outputQ, and the Netmiko sessions that get cycled through.

It looks like you have code that is from a Django example:
def main():
'''
Use threads and Netmiko to connect to each of the devices in the database.
'''
devices = NetworkDevice.objects.all()
You should move the argument parsing into the main thread. You should read the CSV file in the main thread. You should have each child thread be the Netmiko-SSH connection.
I say this as your current solution--has all of the SSH connections happen in one thread which is not what you intended.
At a high-level main() should have argument parsing, delete old output file, obtain username/password (assuming these are the same for all the devices), loop over CSV file obtaining the IP address for each device.
Once you have the IP address, then you create a thread, the thread uses Netmiko-SSH to connect to each device and retrieve your output. I would then use a Queue to pass back the output from each device (back to the main thread).
Then back in the main thread, you would write all of the output to a single file.
It would look a bit like this:
https://github.com/ktbyers/netmiko/blob/develop/examples/use_cases/case16_concurrency/threads_netmiko.py
Here is an example using a queue (with multiprocessing) though you can probably adapt this using a thread-Queue pretty easily.
https://github.com/ktbyers/netmiko/blob/develop/examples/use_cases/case16_concurrency/processes_netmiko_queue.py

How to print telnet response line by line?

Is it possible to print the telnet response line by line, when a command executed over telnet keeps on responding over console ?
Example: I have executed a command (to collect logs), It keeps on displaying logs on console window. Can we read the response line by line & print it , without missing any single line ?
Below snippet writes the log, but only after certain specified time. If I stop the service/script (CTRL-C) in between, that doesn't write anything.
import sys
import telnetlib
import time
orig_stdout = sys.stdout
f = open('outpuy.txt', 'w')
sys.stdout = f
try:
tn = telnetlib.Telnet(IP)
tn.read_until(b"pattern1")
tn.write(username.encode('ascii') + b"\n")
tn.read_until(b"pattern2")
tn.write(command1.encode('ascii') + b"\n")
z = tn.read_until(b'abcd\b\n',600)
array = z.splitlines( )
except:
sys.exit("Telnet Failed to ", IP)
for i in array:
i=i.strip()
print(i)
sys.stdout = orig_stdout
f.close()

You can use tn.read_until("\n") in a loop in order to read one line durint execution of your telnet command
while True:
line = tn.read_until(b"\n") # Read one line
print(line)
if b'abcd' in line: # last line, no more read
break

You can use the ready_very_eager, read_eager, read_lazy, and ready_very_lazy functions specified in the documentation to read your stream byte-by-byte. You can then handle the "until" logic on your own code and at the same time write the read lines to the console.

tail multiple logfiles in python

This is probably a bit of a silly excercise for me, but it raises a bunch of interesting questions. I have a directory of logfiles from my chat client, and I want to be notified using notify-osd every time one of them changes.
The script that I wrote basically uses os.popen to run the linux tail command on every one of the files to get the last line, and then check each line against a dictionary of what the lines were the last time it ran. If the line changed, it used pynotify to send me a notification.
This script actually worked perfectly, except for the fact that it used a huge amount of cpu (probably because it was running tail about 16 times every time the loop ran, on files that were mounted over sshfs.)
It seems like something like this would be a great solution, but I don't see how to implement that for more than one file.
Here is the script that I wrote. Pardon my lack of comments and poor style.
Edit: To clarify, this is all linux on a desktop.

Not even looking at your source code, there are two ways you could easily do this more efficiently and handle multiple files.
Don't bother running tail unless you have to. Simply os.stat all of the files and record the last modified time. If the last modified time is different, then raise a notification.
Use pyinotify to call out to Linux's inotify facility; this will have the kernel do option 1 for you and call back to you when any files in your directory change. Then translate the callback into your osd notification.
Now, there might be some trickiness depending on how many notifications you want when there are multiple messages and whether you care about missing a notification for a message.
An approach that preserves the use of tail would be to instead use tail -f. Open all of the files with tail -f and then use the select module to have the OS tell you when there's additional input on one of the file descriptors open for tail -f. Your main loop would call select and then iterate over each of the readable descriptors to generate notifications. (You could probably do this without using tail and just calling readline() when it's readable.)
Other areas of improvement in your script:
Use os.listdir and native Python filtering (say, using list comprehensions) instead of a popen with a bunch of grep filters.
Update the list of buffers to scan periodically instead of only doing it at program boot.
Use subprocess.popen instead of os.popen.

If you're already using the pyinotify module, it's easy to do this in pure Python (i.e. no need to spawn a separate process to tail each file).
Here is an example that is event-driven by inotify, and should use very little cpu. When IN_MODIFY occurs for a given path we read all available data from the file handle and output any complete lines found, buffering the incomplete line until more data is available:
import os
import select
import sys
import pynotify
import pyinotify
class Watcher(pyinotify.ProcessEvent):
def __init__(self, paths):
self._manager = pyinotify.WatchManager()
self._notify = pyinotify.Notifier(self._manager, self)
self._paths = {}
for path in paths:
self._manager.add_watch(path, pyinotify.IN_MODIFY)
fh = open(path, 'rb')
fh.seek(0, os.SEEK_END)
self._paths[os.path.realpath(path)] = [fh, '']
def run(self):
while True:
self._notify.process_events()
if self._notify.check_events():
self._notify.read_events()
def process_default(self, evt):
path = evt.pathname
fh, buf = self._paths[path]
data = fh.read()
lines = data.split('\n')
# output previous incomplete line.
if buf:
lines[0] = buf + lines[0]
# only output the last line if it was complete.
if lines[-1]:
buf = lines[-1]
lines.pop()
# display a notification
notice = pynotify.Notification('%s changed' % path, '\n'.join(lines))
notice.show()
# and output to stdout
for line in lines:
sys.stdout.write(path + ': ' + line + '\n')
sys.stdout.flush()
self._paths[path][1] = buf
pynotify.init('watcher')
paths = sys.argv[1:]
Watcher(paths).run()
Usage:
% python watcher.py [path1 path2 ... pathN]

Simple pure python solution (not the best, but doesn't fork, spits out 4 empty lines after idle period and marks everytime the source of the chunk, if changed):
#!/usr/bin/env python
from __future__ import with_statement
'''
Implement multi-file tail
'''
import os
import sys
import time
def print_file_from(filename, pos):
with open(filename, 'rb') as fh:
fh.seek(pos)
while True:
chunk = fh.read(8192)
if not chunk:
break
sys.stdout.write(chunk)
def _fstat(filename):
st_results = os.stat(filename)
return (st_results[6], st_results[8])
def _print_if_needed(filename, last_stats, no_fn, last_fn):
changed = False
#Find the size of the file and move to the end
tup = _fstat(filename)
# print tup
if last_stats[filename] != tup:
changed = True
if not no_fn and last_fn != filename:
print '\n<%s>' % filename
print_file_from(filename, last_stats[filename][0])
last_stats[filename] = tup
return changed
def multi_tail(filenames, stdout=sys.stdout, interval=1, idle=10, no_fn=False):
S = lambda (st_size, st_mtime): (max(0, st_size - 124), st_mtime)
last_stats = dict((fn, S(_fstat(fn))) for fn in filenames)
last_fn = None
last_print = 0
while 1:
# print last_stats
changed = False
for filename in filenames:
if _print_if_needed(filename, last_stats, no_fn, last_fn):
changed = True
last_fn = filename
if changed:
if idle > 0:
last_print = time.time()
else:
if idle > 0 and last_print is not None:
if time.time() - last_print >= idle:
last_print = None
print '\n' * 4
time.sleep(interval)
if '__main__' == __name__:
from optparse import OptionParser
op = OptionParser()
op.add_option('-F', '--no-fn', help="don't print filename when changes",
default=False, action='store_true')
op.add_option('-i', '--idle', help='idle time, in seconds (0 turns off)',
type='int', default=10)
op.add_option('--interval', help='check interval, in seconds', type='int',
default=1)
opts, args = op.parse_args()
try:
multi_tail(args, interval=opts.interval, idle=opts.idle,
no_fn=opts.no_fn)
except KeyboardInterrupt:
pass