I am trying out python's logger module to log info both in console and in a log and it works fine.I wanted that script to get invoked during machine star-tup and hence invoked my script from rc.local.
And my script gets invoked during boot-up, but the problem is, only console logging is working at that time. Logging to a file is not happening. Any clue on what could be the issue.
Is it anything related to syslog daemon?
import time
import logging
import datetime
global logger
global LOG_FILE
def initialize():
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
time = datetime.datetime.now()
LOG_FILE = "sample.log." + time.strftime("%Y%m%d")
log_file_handler = logging.FileHandler(LOG_FILE)
log_file_handler.setLevel(logging.DEBUG)
log_file_formatter = logging.Formatter('%(message)s')
log_file_handler.setFormatter(log_file_formatter)
logger.addHandler(log_file_handler)
log_file_handler = logging.StreamHandler()
log_file_handler.setLevel(logging.INFO)
log_file_formatter = logging.Formatter('%(message)s')
log_file_handler.setFormatter(log_file_formatter)
logger.addHandler(log_file_handler)
if __name__ == '__main__':
initialize()
logger.info("This should go both in console and log")
logger.debug("This should go only to Log")
import time
import logging
import datetime
def initialize():
# because logging is global module, so you can
# always get logger by logging.getLogger(__name__)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
time = datetime.datetime.now()
# if you only use LOG_FILE in function, you don't
# have to declare it outside.
LOG_FILE = "sample.log." + time.strftime("%Y%m%d")
log_file_handler = logging.FileHandler(LOG_FILE)
log_file_handler.setLevel(logging.DEBUG)
log_file_formatter = logging.Formatter('%(message)s')
log_file_handler.setFormatter(log_file_formatter)
logger.addHandler(log_file_handler)
log_file_handler = logging.StreamHandler()
log_file_handler.setLevel(logging.INFO)
log_file_formatter = logging.Formatter('%(message)s')
log_file_handler.setFormatter(log_file_formatter)
logger.addHandler(log_file_handler)
if __name__ == '__main__':
initialize()
logger = logging.getLogger(__name__)
logger.info("This should go both in console and log")
logger.debug("This should go only to Log")
Related
I would like individual .log file for each thread. Unfortunately, after using logging.basicConfig, many different files are created for logs, but finally all logs end up in the last declared file.
What should threads do to have independent log files?
import logging
import threading
import time
from datetime import datetime
def test_printing(name):
logging.basicConfig(
format="%(asctime)s, %(levelname)-8s | %(filename)-23s:%(lineno)-4s | %(threadName)15s: %(message)s", # noqa
datefmt="%Y-%m-%d:%H:%M:%S",
level=logging.INFO,
force=True,
handlers=[
logging.FileHandler(f"{name}.log"),
logging.StreamHandler()])
logging.info(f"Test {name}")
time.sleep(20)
logging.info(f"Test {name} after 20s")
def function_thread():
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
thread = threading.Thread(
target=test_printing,
kwargs={"name": timestamp}
)
thread.start()
for i in range(5):
time.sleep(1)
function_thread()
From https://docs.python.org/3/library/logging.html#logger-objects
Note that Loggers should NEVER be instantiated directly, but always through the module-level function logging.getLogger(name).
So you have to create and configure a new logger inside each thread:
logger = logging.getLogger(name)
logger.basicConfig(...)
more info at: https://docs.python.org/3/howto/logging.html#logging-from-multiple-modules
Edit: Use already defined name as logger identifier, instead of __name__
Edit:
You cannot use logging.basicConfig, instead you need to configure each thread logger on its own.
Full code provided and tested:
import logging
import threading
import time
from datetime import datetime
def test_printing(name):
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt="%(asctime)s, %(levelname)-8s | %(filename)-23s:%(lineno)-4s | %(threadName)15s: %(message)s",
datefmt="%Y-%m-%d:%H:%M:%S")
sh = logging.StreamHandler()
fh = logging.FileHandler(f"{name}.log")
sh.setFormatter(formatter)
fh.setFormatter(formatter)
logger.addHandler(sh)
logger.addHandler(fh)
logger.info(f"Test {name}")
time.sleep(20)
logger.info(f"Test {name} after 20s")
def function_thread():
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
thread = threading.Thread(
target=test_printing,
kwargs={"name": timestamp}
)
thread.start()
for i in range(5):
time.sleep(1)
function_thread()
my Kaggle-Notebook does not print Outputs.
import logging
import sys
# noinspection SpellCheckingInspection,SpellCheckingInspection,SpellCheckingInspection,SpellCheckingInspection,PyArgumentList
def defaultLogger(name="fashiondataset", level=logging.DEBUG, handlers=None,
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s'):
handlers = handlers if handlers else [logging.StreamHandler(sys.stdout)]
logging.basicConfig(level=level, format=format, handlers=handlers)
logger = logging.getLogger(name)
logging.getLogger("matplotlib").setLevel(logging.WARNING)
logging.getLogger("nltk_data").setLevel(logging.WARNING)
logging.getLogger("pysndfx").setLevel(logging.WARNING)
logging.getLogger('selenium.webdriver.remote.remote_connection').setLevel(logging.WARNING)
logging.getLogger('connectionpool').setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
return logger
print("Testing Logger")
logger = defaultLogger()
logger.info("Logging Info Test")
logger.debug("Logging Debug Test")
logger.error("Logging Error Test")
print("Testing Logger [DONE]")
Output:
Testing Logger
Testing Logger [DONE]
I also tried to add logging.getLogger(name).setLevel(logging.DEBUG) but it does not Change anything. TQDM / Tensorflow Progressbars are working.
Edit://
sys.stdout.write("test") Works
I have two python scripts in the same directory. I try to catch logging messages from both of them:
#script.py
import requests
import logging
logger = logging.getLogger(__name__)
class Downloader:
def __init__(self, url):
self.url = url
def download(self):
logger.debug(f'Downloading {self.url}')
req = requests.get(self.url, timeout=1)
return req
#main.py
import logging
from script import Downloader
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
logger.debug('create object')
d = Downloader('https://www.google.com')
res = d.download()
Basically I want to get rid of the debug-messages from the requests-module, so using logging.basicConfig() is not an option. But the way I do it, I do not get the debug-message from the imported script. Apparently because in script.py __name__ is not main.script.
How can I achieve this without hard coding anything to a string?
In a different module (e.g. logger.py):
import logging
def setup_logger(name, logfile, formatter, stream_handler=False, level=logging.DEBUG):
"""Function to create loggers."""
file_handler = logging.FileHandler(log_file)
stdout_handler = logging.StreamHandler()
file_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
logger = logging.getLogger(name)
if not logger.handlers:
logger.setLevel(level)
logger.addHandler(file_handler)
if stream_handler:
logger.addHandler(stdout_handler)
return logger
# Example formatter
formatter = logging.Formatter('%(asctime)s - %(levelname)s -> %(message)s\n')
# Generate the log object
log = setup_logger('logger_name', 'path_to_logfile', formatter)
Import this log object from your other modules to use it: from logger import log
Reading the logging HOWTO (https://docs.python.org/3/howto/logging.html) I came away under the impression that if I configured a logger, then I could subsequently request my logger from the factory via logging.getLogger() and python would know how to get the right logger (the one I configured) and everything would just auto-work, i.e. I wouldn't need to pass the configured logger instance around my code, I could just ask for it wherever I needed it. Instead, I'm observing something different.
File log_tester.py:
from util.logging_custom import SetupLogger
import logging
import datetime
def test():
logger = logging.getLogger()
logger.debug("In test()")
def main():
logger = SetupLogger("logger_test")
logger.setLevel(logging.DEBUG)
logger.info(f"now is {datetime.datetime.now()}", )
logger.debug("In main()")
test()
if __name__ == '__main__':
main()
File util/logging_custom.py:
import os
import time
import logging
from logging.handlers import RotatingFileHandler
def SetupLogger(name_prefix):
if not os.path.exists("log"):
os.makedirs("log")
recfmt = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(message)s')
handler = RotatingFileHandler(time.strftime(f"log/{name_prefix}.log"),maxBytes=5000000, backupCount=10)
handler.setFormatter(recfmt)
handler.setLevel(logging.DEBUG)
logger = logging.getLogger(f"{name_prefix} {__name__}")
logger.addHandler(handler)
return logger
When I run this code only the debug statement that is in main() ends up in the log file. The debug statement from test() ends up I'm not sure where exactly.
Contents of log/logger_test.log:
2019-02-07 09:14:39,906.906 INFO now is 2019-02-07 09:14:39.906848
2019-02-07 09:14:39,906.906 DEBUG In main()
My expectation was that In test() would also show up in my log file. Have I made some assumptions about how python logging works that are untrue? How do I make it so that all of the logging in my program (which has many classes and modules) goes to the same configured logger? Is that possible without passing around a logger instance everywhere, after it's created in main()?
Thanks.
The getLogger function will return a the logger by its name (kind of a singleton):
if it doesn't exist, it creates it
If it already exist, it returns it
Then what you could do is:
util/logging_custom.py
def SetupLogger(logger_name, level=logging.INFO):
if not os.path.exists("log"):
os.makedirs("log")
recfmt = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(message)s')
handler = RotatingFileHandler(time.strftime(f"log/{logger_name}.log"),maxBytes=5000000, backupCount=10)
handler.setFormatter(recfmt)
handler.setLevel(level)
logger = logging.getLogger(logger_name)
logger.addHandler(handler)
# no need to return the logger, I would even advice not to do so
log_tester.py
from util.logging_custom import SetupLogger
import logging
import datetime
logger = SetupLogger("logger_test", logging.DEBUG) # you only need to run this once, in your main script.
logger = logging.getLogger("logger_test")
def test():
logger.debug("In test()")
def main():
logger.info(f"now is {datetime.datetime.now()}", )
logger.debug("In main()")
test()
if __name__ == '__main__':
main()
any_other.py
import logging
logger = logging.getLogger("logger_test") # this will return the logger you already instantiate in log_tester.py
logger.info("that works!")
Update
To set the level and the handling of the root logger instead of the one you setted up, use logging.getLogger() without passing any name:
root_logger = logging.getLogger()
root_logger.addHandler(your_handler)
root_logger.setLevel(logging.DEBUG)
root_logger.info("hello world")
From the docs:
Multiple calls to getLogger() with the same name will return a
reference to the same logger object.
Your assumptions are quite correct. The problem here is the way you are calling getLogger() in test(). You should be passing the name you used in SetupLogger()'s getLogger() i.e. logger = logging.getLogger(f"{name_prefix} {__name__}").
I am trying to use an FTP server stub during tests. I don't want the console output, but I would like to capture the logging to a file.
I want the FTP server to run in a different process, so I use multiprocessing.
My code as follows sets all logging to level WARNING:
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
import pyftpdlib.log as pyftpdliblog
import os
import logging
import multiprocessing as mp
authorizer = DummyAuthorizer()
authorizer.add_user('user', '12345', '.', perm='elradfmwM')
handler = FTPHandler
handler.authorizer = authorizer
pyftpdliblog.LEVEL = logging.WARNING
logging.basicConfig(filename='pyftpd.log', level=logging.INFO)
server = FTPServer(('', 2121), handler)
def main():
p = mp.Process(target=server.serve_forever)
p.start()
if __name__ == '__main__':
main()
How do I set only the console logging to level WARNING, or even better, completely shutdown without giving up the file logging?
So, after digging inside the code, I found out the following hint:
# This is a method of FTPServer and it is called before
# server.serve_forever
def _log_start(self):
if not logging.getLogger('pyftpdlib').handlers:
# If we get to this point it means the user hasn't
# configured logger. We want to log by default so
# we configure logging ourselves so that it will
# print to stderr.
from pyftpdlib.ioloop import _config_logging
_config_logging()
So, all I had to do is to define my own appropriate handlers:
logger = logging.getLogger('pyftpdlib')
logger.setLevel(logging.INFO)
hdlr = logging.FileHandler('pyftpd.log' )
logger.addHandler(hdlr)
Now, there is file logging, but console logging will not start.
Something like this:
import logging
date_format = "%Y/%m/%d %H:%M:%S"
log_file_path = "my_file.txt"
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# own_module_logger = logging.getLogger(__name__)
pyftpdlib_logger = logging.getLogger("pyftpdlib")
# Setup logging to file (Only pyftpdlib)
filehandler = logging.FileHandler(filename = log_file_path)
filehandler.setLevel(logging.DEBUG)
fileformatter = logging.Formatter(fmt = "%(asctime)s - %(levelname)-8s - %(name)s.%(funcName)s - %(message)s",
datefmt = date_format)
filehandler.setFormatter(fileformatter)
pyftpdlib_logger.addHandler(filehandler)
pyftpdlib_logger.propagate = False
# Setup logging to console (All other)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
consoleformatter = logging.Formatter(fmt = "%(asctime)s - %(levelname)-8s - %(name)s.%(funcName)s - %(message)s",
datefmt = date_format)
console.setFormatter(consoleformatter)
root_logger.addHandler(console)
# Do your loggings
a = logging.getLogger()
a.info('root I')
a.debug('root D')
b = logging.getLogger("pyftpdlib")
b.info('P I')
b.debug('P D')
logging.shutdown()
So loggings of pyftpdlib go to file. Everything from your module to console. One of the key thing here is the propagate!