I am reading some data from gpsd and writing it using using Python's logging module. I am fairly confident that only one process writes to this file, although I read from it using tail while this logger is running. Every once in a while I am seeing some log entries that look like the image below. I am hoping that somebody can shed some light on what would cause (presumably Python) to insert null control characters into my log file.
The code I am using is:
"""
Read the GPS continuously
"""
import logging
from logging.handlers import RotatingFileHandler
import sys
import gps
LOG = '/sensor_logs/COUNT.csv'
def main():
log_formatter = logging.Formatter('%(asctime)s,%(message)s', "%Y-%m-%d %H:%M:%S")
my_handler = RotatingFileHandler(LOG, mode='a', maxBytes=1024*1024,
backupCount=1, encoding=None, delay=0)
my_handler.setFormatter(log_formatter)
my_handler.setLevel(logging.INFO)
app_log = logging.getLogger('root')
app_log.setLevel(logging.INFO)
app_log.addHandler(my_handler)
session = gps.gps("localhost", "2947")
session.stream(gps.WATCH_ENABLE | gps.WATCH_NEWSTYLE)
while True:
try:
report = session.next()
if hasattr(report, 'gdop'):
satcount = 0
for s in report['satellites']:
if s['used'] == True:
satcount+=1
data = "{}".format(str(satcount))
app_log.info(data)
except KeyError:
pass
except KeyboardInterrupt:
quit()
except StopIteration:
session = None
if __name__ == "__main__":
main()
This ended up being a result of failing hardware. Thank you for your help, #RobertB.
Related
I'm trying to verify logging output in a Python application that writes log messages to two handlers with different levels - stdout and a file.
Using #mock.patch to patch sys.stdout with io.StringIO I'm able to verify that appropriate messages are written there. Can I similarly target TextIO used by the logging module and put my own StringIO object there instead?
import io
import logging
import os
import sys
import unittest.mock
import logutils.colorize
class TestMain(unittest.TestCase):
def _setup_logger(self, stdout_level: str, file_level: str):
try:
os.unlink("test.log")
except FileNotFoundError:
pass
log_stdout_handler = logutils.colorize.ColorizingStreamHandler(sys.stdout)
log_stdout_handler.setFormatter(logging.Formatter("%(message)s"))
log_stdout_handler.setLevel(stdout_level)
log_file_handler = logging.FileHandler(filename="test.log", mode="a", encoding="utf8")
log_file_handler.setFormatter(logging.Formatter("%(message)s"))
log_file_handler.setLevel(file_level)
_root_logger = logging.getLogger()
_root_logger.addHandler(log_stdout_handler)
_root_logger.addHandler(log_file_handler)
_root_logger.setLevel(logging.DEBUG)
def _log_messages(self):
log = logging.getLogger("test")
log.debug("debug")
log.info("info")
log.warning("warning")
log.error("error")
log.critical("critical")
#unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
#unittest.mock.patch("?", new_callable=io.StringIO) # <--- can I target FileHandler's io.TextIOWrapper here?
def test_log_stdout_info_file_debug_then_messages_logged_appropriately(self, mock_file: io.StringIO, mock_stdout: io.StringIO):
self._setup_logger("WARNING", "DEBUG")
self._log_messages()
stdout_output = mock_stdout.getvalue()
stdout_lines = stdout_output.splitlines()
self.assertEqual(len(stdout_lines), 3) # irl i would probably regex match the messages
file_output = mock_file.getvalue()
file_lines = file_output.splitlines()
self.assertEqual(len(file_lines), 5)
Stepping into logging\__init.py:1054 I can see that FileHandler's stream field is an instance of _io.TextIOWrapper, but doing #unittest.mock.patch("_io.TextIOWrapper", new_callable=io.StringIO) does not work. Neither does using io.TextIOWrapper, logging.io.TextIOWrapper, or logging._io.TextIOWrapper.
Or perhaps there's a better way of accomplishing what I need?
I was getting nowhere with that, so what I ended up doing was mocking emit functions of the log handlers (and the _open function of the FileHandler, since I don't want any log files to be created), and doing verification on the mocks.
def _assertEmitted(self, emit_mock: unittest.mock.MagicMock, messages: list[str]) -> typing.NoReturn:
self.assertEqual(emit_mock.call_count, len(messages))
for index, message in enumerate(messages):
self.assertEqual(emit_mock.call_args_list[index].args[0].msg, message)
#unittest.mock.patch.object(logutils.colorize.ColorizingStreamHandler, "emit")
#unittest.mock.patch.object(logging.FileHandler, "emit")
#unittest.mock.patch.object(logging.FileHandler, "_open")
def test_given_log_levels_when_logging_then_appropriate_messages_logged(self, _, file_emit, stdout_emit):
# setup logger
stdout_handler = logutils.colorize.ColorizingStreamHandler(sys.stdout)
stdout_handler.setFormatter(logging.Formatter("..."))
stdout_handler.setLevel("INFO")
file_handler = logging.FileHandler(filename="doesn't matter", mode="a", encoding="utf8")
file_handler.setFormatter(logging.Formatter("..."))
file_handler.setLevel("DEBUG")
root_logger = logging.getLogger()
root_logger.addHandler(log_stdout_handler)
root_logger.addHandler(log_file_handler)
root_logger.setLevel(logging.DEBUG)
# do log calls here
log = logging.getLogger("test")
log.info("info")
log.debug("debug")
self._assertEmitted(stdout_emit, ["info"])
self._assertEmitted(file_emit, ["info", "debug"])
I have requirement to generate new log for each day in my application And i am having two process in application. So i am facing issue as 1 process is not able to write into new day log and another is writing.
For more clarification here is what i am doing..
import logging
import os, sys
import os.path
import datetime
from threading import Thread
import time
firstTime = "false"
def initialize_logger(fileName):
global firstTime
try:
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
output_dir = os.getcwd()
if firstTime == "true":
for handler in logger.handlers[:]: # get rid of existing old handlers
logger.removeHandler(handler)
# create debug file handler and set level to debug
try:
handler = logging.FileHandler(os.path.join(output_dir, fileName), "w")
except:
print("problem to create log")
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("[%(levelname)s] (%(threadName)-30s) %(asctime)s %(message)s ")
handler.setFormatter(formatter)
logger.addHandler(handler)
firstTime = "true"
except Exception as ex:
exc_type, exc_obj, tb = sys.exc_info()
template = "An exception of type {0} occurred at {1}. Arguments:\n{2!r}"
message = template.format(type(ex).__name__, tb.tb_lineno, ex.args)
logging.error(message)
def daily_log(dummyStr):
global firstTime
try:
now = datetime.datetime.now()
log_day = now.day
initialize_logger("Log_start.log")
while True:
currentDate = datetime.datetime.now().day
time.sleep(60)
if currentDate != log_day: # New day started
initialize_logger("Log_continue.log")
except Exception as ex:
exc_type, exc_obj, tb = sys.exc_info()
template = "An exception of type {0} occurred at {1}. Arguments:\n{2!r}"
message = template.format(type(ex).__name__, tb.tb_lineno, ex.args)
logging.error(message)
logThread = Thread(target=daily_log, name="dailyLogThread", args=("dummy",))
logThread.start()
If anyone can help me with understanding this issue and what other alternative i can take to get all logs in new day file..
Your suggestion will be very helpful!!
On Windows, normally opened(*) files can only be accessed by one single process. And on Unix-like system, having more than one thread/process writing to a same file without special synchronization can lead to mixed output (said differently unreadable garbage...).
That means that you cannot simply use FileHandlers for 2 processes to log to the same file.
What can be done:
use the system logging interfaces (SysLogHandler or NTEventLogHandler) because they expect to be used by many processes. syslog can natively use a sequential file for selected sources, and NTEventLog can export selected events
use a relay. For example the main processes could use a DatagramHandler and a relay process listens on the socket, read the packets, format them back into LogRecord with makeLogRecord and finaly write them to the FileHandler.
(*) Of course the API allows special modes to allow concurrent accesses to a file, but this cannot be (easily) done from Python
I have a script that processes csvs and load them to database. My intern mentor wanted us to use log file to capture what's going on and he wanted it to be flexible so one can use a config.ini file to edit where they want the log file to be created. As a result I did just that, using a config file that use key value pairs in a dict that i can extract the path to the log file from. These are excepts from my code where log file is created and used:
dirconfig_file = r"C:\Users\sys_nsgprobeingestio\Documents\dozie\odfs\venv\odfs_tester_history_dirs.ini"
start_time = datetime.now()
def process_dirconfig_file(config_file_from_sysarg):
try:
if Path.is_file(dirconfig_file_Pobj):
parseddict = {}
configsects_set = set()
for sect in config.sections():
configsects_set.add(sect)
for k, v in config.items(sect):
# print('{} = {}'.format(k, v))
parseddict[k] = v
print(parseddict)
try:
if ("log_dir" not in parseddict or parseddict["log_dir"] == "" or "log_dir" not in configsects_set):
raise Exception(f"Error: Your config file is missing 'logfile path' or properly formatted [log_file] section for this script to run. Please edit config file to include logfile path to capture errors")
except Exception as e:
#raise Exception(e)
logging.exception(e)
print(e)
parse_dict = process_dirconfig_file(dirconfig_file)
logfilepath = parse_dict["log_dir"]
log_file_name = start_time.strftime(logfilepath)
print(log_file_name)
logging.basicConfig(
filename=log_file_name,
level=logging.DEBUG,
format='[Probe Data Quality] %(asctime)s - %(name)s %(levelname)-7.7s %(message)s'
# can you explain this Tenzin?
)
if __name__ == '__main__':
try:
startTime = datetime.now()
db_instance = dbhandler(parse_dict["db_string"])
odfs_tabletest_dict = db_instance['odfs_tester_history_files']
odf_history_from_csv_to_dbtable(db_instance)
#print("test exception")
print(datetime.now() - startTime)
except Exception as e:
logging.exception(e)
print(e)
Doing this, no file is created. The script runs with no errors but no log file is created. I've tried several things including using a hardcoded log file name, instead of calling it from the config file but it didn't work
The only thing that works is when the log file is created up top before any method. Why is this?
When you are calling your process_dirconfig_file function, the logging configuration has not been set yet, so no file could have been created. The script executes top to bottom. It would be similar to doing something like this:
import sys
# default logging points to stdout/stderr kind of like this
my_logger = sys.stdout
my_logger.write("Something")
# Then you've pointed logging to a file
my_logger = open("some_file.log", 'w')
my_logger.write("Something else")
Only Something else would be written to our some_file.log, because my_logger pointed somewhere else beforehand.
Much the same is happening here. By default, the logging.<debug/info> functions do nothing because logging won't do anything with them without additional configuration. logging.error, logging.warning, and logging.exception will always at least write to stdout out of the box.
Also, I don't think the inner try is valid Python, you need a matching except. And I wouldn't just print an exception raised by that function, I'd probably raise and have the program crash:
def process_dirconfig_file(config_file_from_sysarg):
try:
# Don't use logging.<anything> yet
~snip~
except Exception as e:
# Just raise or don't use try/except at all until
# you have a better idea of what you want to do in this circumstance
raise
Especially since you are trying to use the logger while validating that its configuration is correct.
The fix? Don't use the logger until after you've determined it's ready.
I am trying to run executescript process in Apache Nifi using python but having problem with passing flow file to next processor in my data flow.
If I run the standalone flow file creation and writing snippet it works and I can read flow file in the next processor but when I try to enrich it, it simply does not pass the flow file. In fact no error is generated and somehow I have no clue how to proceed. I am bit new with python and nifi and appreciate your help with this particular issue.
Below is the code I am using and you can see its very simple. I just want to create and write some string to flow file using some logic. But no luck so far
import urllib2
import json
import datetime
import csv
import time
import sys
import traceback
from org.apache.nifi.processor.io import OutputStreamCallback
from org.python.core.util import StringUtil
class WriteContentCallback(OutputStreamCallback):
def __init__(self, content):
self.content_text = content
def process(self, outputStream):
try:
outputStream.write(StringUtil.toBytes(self.content_text))
except:
traceback.print_exc(file=sys.stdout)
raise
page_id = "dsssssss"
access_token = "sdfsdfsf%sdfsdf"
def scrapeFacebookPageFeedStatus(page_id, access_token):
flowFile = session.create()
flowFile = session.write(flowFile, WriteContentCallback("Hello there this is my data"))
flowFile = session.write()
session.transfer(flowFile, REL_SUCCESS)
print "\nDone!\n%s Statuses Processed in %s" % \
(num_processed, datetime.datetime.now() - scrape_starttime)
if __name__ == '__main__':
scrapeFacebookPageFeedStatus(page_id, access_token)
I believe the problem is the check for __main__:
if __name__ == '__main__':
scrapeFacebookPageFeedStatus(page_id, access_token)
__builtin__ was the actual module name in my experiment. You could either remove that check, or add a different one if you want to preserve your separate testing path.
I'm making my first steps in Python and have come to a point where I need a logging module. The reasons I didn't opt for a rotating file handler are:
I wanted a new folder to be created each time the code is run, hosting the new log files.
Using conventional filenames (myName_X.log, and not the default myName.log.X).
Limit log files by number of lines, and not file size (as done by the rotating file handler).
I've written such a module, using Python's built in logging module, but i have two problems:
The new folder and file are created, and the logging data is printed into the file. However, when main() is run for the second time (see code below), the newly created file is locked, and cannot be deleted from the file explorer unless I close the IDE or release the lock through Process Explorer.
The IPython interpreter freezes the second time I run main(). If I try the pdb module, it freezes as well.
I'm using WinPython 3.3.5 (with Spyder 2.3.0beta). I spent hours trying to find a solution to this problem. I don't know if it is a problem in my code or rather a bug with Spyder.
General coding remarks are always welcome.
main_example.py:
import myLogging
def main():
try:
myLoggerInstance = myLogging.MyLogger()
# Do stuff...
# logging example
for i in range(0, 3):
msg = 'Jose Halapeno on a stick {0}'.format(i)
myLoggerInstance.WriteLog('DEBUG', msg)
print('end of prints...')
finally:
myLoggerInstance._closeFileHandler()
print('closed file handle...')
if __name__ == "__main__":
main()
myLogging.py:
import logging
import time
import os
class MyLogger:
_linesCounter = 0
_nNumOfLinesPerFile = 100000
_fileCounter = 0
_dirnameBase = os.path.dirname(os.path.abspath(__file__))
_dirname = ''
_filenameBase = 'logfile_{0}.log'
_logger = logging.getLogger('innerGnnLogger')
_severityDict = {'CRITICAL' : logging.CRITICAL, 'ERROR': logging.ERROR, 'WARNING':
logging.WARNING, 'INFO': logging.INFO, 'DEBUG': logging.DEBUG}
#staticmethod
def __init__():
# remove file handle
MyLogger._closeFileHandler()
# create folder for session
MyLogger._dirname = MyLogger._dirnameBase + time.strftime("\\logs_%Y_%m_%d-
%H_%M_%S\\")
MyLogger._dirname = MyLogger._dirname.replace('\\\\', '/')
if not os.path.exists(MyLogger._dirname):
os.makedirs(MyLogger._dirname)
# set logger
MyLogger._logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
MyLogger._hConsole = logging.StreamHandler()
MyLogger._hFile = logging.FileHandler(MyLogger._dirname + \
MyLogger._filenameBase.format(MyLogger._fileCounter))
MyLogger._hConsole.setLevel(logging.WARNING)
MyLogger._hFile.setLevel(logging.DEBUG)
# create formatter
MyLogger._formatter = logging.Formatter('%(asctime)s %(filename)s, %(funcName)s, %(lineno)s, %(levelname)s: %(message)s')
# add formatter to handlers
MyLogger._hConsole.setFormatter(MyLogger._formatter)
MyLogger._hFile.setFormatter(MyLogger._formatter)
# add handlers to logger
MyLogger._logger.addHandler(MyLogger._hConsole)
MyLogger._logger.addHandler(MyLogger._hFile)
#staticmethod
def _StartNewFileHandler():
MyLogger._closeFileHandler()
# create new file handler
++MyLogger._fileCounter
MyLogger._hFile = logging.FileHandler(MyLogger._dirname + \
MyLogger._filenameBase.format(MyLogger._fileCounter))
MyLogger._hFile.setLevel(logging.DEBUG)
MyLogger._hFile.setFormatter(MyLogger._formatter)
MyLogger._logger.addHandler(MyLogger._hFile)
#staticmethod
def WriteLog(severity, message):
if (len(MyLogger._logger.handlers) < 2):
MyLogger._StartNewFileHandler()
MyLogger._linesCounter += 1
MyLogger._logger.log(MyLogger._severityDict[severity], message)
if (MyLogger._linesCounter >= MyLogger._nNumOfLinesPerFile):
MyLogger._logger.info('Last line in file')
MyLogger._StartNewFileHandler()
MyLogger._linesCounter = 0
#staticmethod
def _closeFileHandler():
if (len(MyLogger._logger.handlers) > 1):
MyLogger._logger.info('Last line in file')
MyLogger._logger.handlers[1].stream.close()
MyLogger._logger.removeHandler(MyLogger._logger.handlers[1])