Develop Reference

Log files get reset on the restart of the application instead of appending the new logs to it

import logging
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(message)s')
fileHandler = logging.FileHandler(log_file, mode='w')
fileHandler.setFormatter(formatter)
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(formatter)
l.setLevel(level)
l.addHandler(fileHandler)
l.addHandler(streamHandler)
I'm trying to write the different logs(levels) to the different files the functionality is working fine but when ever the application is restarted the log files get reset instead of appending the data to it.
#method called from the different method
setup_logger('logger', 'login.log')
setup_logger('logger_market', 'transaction.log')
logger = logging.getLogger('logger')
logger_market = logging.getLogger('logger_market')
logger.info(f'Test for the login.log file')
logger_market.info(f'this to test')
I'm new to logging in python and the documentation is a bit complex for me at the moment.

I think you do not need to set mode
just use following: fileHandler = logging.FileHandler(log_file)

Python Logger is not working to write to file?

I'm trying to create logger file inside directory using the below code -
def createLogger(name,log_path=None):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.setLevel(logging.ERROR)
logger.setLevel(logging.INFO)
logger.setLevel(logging.CRITICAL)
formatter = logging.Formatter("%(asctime)s - %(levelname)-8s - %(message)s")
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
if log_path is not None:
file_handler = logging.FileHandler(log_path)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
name = "temp_logs"
file = open(os.path.join(f'{current_date}', f'{name}_{current_timestamp}.txt'), 'w')
log_path = f"{current_date}/{name}_{current_timestamp}.txt"
logger = createLogger(name = name ,log_path = log_path)
write_message = logger.debug('This is a test file')
file.writelines(str(write_message))
file.close()
However, it is just writing None as message inside the file.
Am I missing anything here? Please help.

Several issues. setLevel sets a threshold. When called multiple times, the last one wins (i.e. CRITICAL). You will not see any debug messages, warnings or error.
Also, you are writing to the log file directly. What's the point of setting a logger with a formatter and then bypassing it?
Try this:
Leave only this one setLevel call: logger.setLevel(logging.DEBUG) in createLogger() and then:
name = "temp_logs"
logger = createLogger(name = name ,log_path = './test.log')
logger.debug('This is a debug message')
logger.error('This is an error')

Python3 logging

I'm trying to create my own logger in python 3.6.8 to send output both to stdout and a log file (chosen by date, if the log file doesn't exist yet for today's date it gets created, if there already is a file with the same date just append).
from datetime import date
import logging
import logging.handlers
class Log:
def __init__(self):
pass
def getCleanerLogger(self,moduleName, logFolder, format):
filename = logFolder+ str(date.today()) + '-log.log'
handler = logging.FileHandler(filename)
shandler = logging.StreamHandler()
shandler.setLevel(logging.INFO)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(format)
handler.setFormatter(formatter)
shandler.setFormatter(formatter)
logger = logging.getLogger(moduleName)
logger.addHandler(handler)
logger.addHandler(shandler)
print("I've been called")
return logger
import Conf
conf = Conf.configuration()
print(conf['logFolder'] + " " + conf['logFormat'])
logger = Log()
logger = logger.getCleanerLogger("Log", conf['logFolder'], conf['logFormat'])
logger.info('initializing')
logger.debug('initializing debug')
in the json conf file these are the keys I load
"logFolder": "log/",
"logFormat": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
the log file gets created with the correct logic but there is no logging in either the console or the log file, only the prints go to stdout, no error or exception is raised, I really don't understand why this isn't working. I can only log with logging.root.level('msg') after loading a basiconfig.

Every handler has own logging level but logger has also global logging level which has bigger prioritet so you have to change this level to level which doesn't block handlers - ie.
logger.setLevel(logging.DEBUG)
Mininal working code with few smaller changes.
It doesn't use file with settings so everyone can easily copy it and run it.
from datetime import date
import os
import logging
import logging.handlers
class Log:
def get_cleaner_logger(self, module_name, log_folder, format):
if not os.path.exists(log_folder):
os.makedirs(log_folder)
filename = os.path.join(log_folder, date.today().strftime('%Y-%m-%d-log.log'))
print(filename)
logger = logging.getLogger(module_name)
print('before:', logger.level)
logger.setLevel(logging.DEBUG)
print('after:', logger.level)
formatter = logging.Formatter(format)
handler = logging.FileHandler(filename)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
shandler = logging.StreamHandler()
shandler.setLevel(logging.INFO)
shandler.setFormatter(formatter)
logger.addHandler(shandler)
print("I've been called")
return logger
conf = {
"logFolder": "log/",
"logFormat": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
}
logger = Log()
logger = logger.get_cleaner_logger("Log", conf['logFolder'], conf['logFormat'])
logger.info('initializing')
logger.debug('initializing debug')
BTW: I changed some names based on PEP 8 -- Style Guide for Python Code

RotatingFileHandler with console logging

I've written an init function to configure the python logging module to log to a file and the console. I want to limit the log file size using the RotatingFileHandler. The code below doesn't cause any errors and does everything I want, except it doesn't rotate the logs. I set a low file size to test things out.
How can I configure to use rotating logs and the console with different formats like below?
import logging, logging.handlers
LOG_LEVEL = logging.DEBUG
CONSOLE_LEVEL = logging.DEBUG
def init_logger(fullpath, console_level=CONSOLE_LEVEL, log_level=LOG_LEVEL):
"""
Setup the logger object
Args:
fullpath (str): full path to the log file
"""
logging.basicConfig(level=LOG_LEVEL,
format='%(asctime)s %(threadName)-10s %(name)-12s %
(levelname)-8s %(message)s',
datefmt='%m-%d-%y %H:%M:%S',
filename=fullpath,
filemode='w')
_logger = logging.getLogger('_root')
_logger.setLevel(log_level)
log_handler = logging.handlers.RotatingFileHandler(filename=fullpath,
maxBytes=50, backupCount=10)
log_handler.setLevel(log_level)
_logger.addHandler(log_handler)
console = logging.StreamHandler()
console.setLevel(console_level)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %
(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
logging.debug("Creating log file")

This is your logging's structure:
---root <- I mean real root, which you will get from logging.getLogger([""])
---NormalFileHandler <- Configged by logging.basicConfig, this handler won't rotate file.
---StreamHandler
------_root <- Your root logger, although I don't understand why you want this.
------RotatingFileHandler <- This one will rotate file.
And then you use logging.debug("Creating log file"), which is the same as calling debug on root logger: logging.getLogger().debug(...). So this log passed to StreamHandler and NormalFileHandler.
That's why you see the file isn't got rotated.
The correct configuration should be:
def init_logger(fullpath, console_level=CONSOLE_LEVEL, log_level=LOG_LEVEL):
"""
Setup the logger object
Args:
fullpath (str): full path to the log file
"""
logger = logging.getLogger('YourLogger')
logger.setLevel(log_level)
log_handler = logging.handlers.RotatingFileHandler(filename=fullpath,
maxBytes=50, backupCount=10)
log_handler.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(threadName)-10s %(name)-12s %
(levelname)-8s %(message)s')
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
console = logging.StreamHandler()
console.setLevel(console_level)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %
(message)s')
console.setFormatter(formatter)
logger.addHandler(console) # Or you can add it to root logger, but it is not recommended, you should use your own logger instead of root logger. Or it will cause some problems.
logger.debug("Creating log file")
And then when you want to use logger, you should use:
logger = logging.getLogger('YourLogger')
logger.info(...)
logger.debug(...)

Different logging levels for filehandler and display in Python

I am using the logging module in Python to write debug and error messages.
I want to write to file all messages of logging.DEBUG or greater.
However, I only want to print to the screen messages of logging.WARNING or greater.
Is this possible using just one Logger and one FileHandler?

As it has been mentioned, handlers are so easy to create and add that you're probably better off just using two handlers. If, however, for some reason you want to stick to one, the Python logging cookbook has a section describing more or less what you want to do: logging to both console and file, but at different levels (it even shows you how to do different formatting). It does it with a single StreamHandler rather than a FileHandler, though:
import logging
# set up logging to file - see previous section for more details
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='/temp/myapp.log',
filemode='w')
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Now, we can log to the root logger, or any other logger. First the root...
logging.info('Jackdaws love my big sphinx of quartz.')
# Now, define a couple of other loggers which might represent areas in your
# application:
logger1 = logging.getLogger('myapp.area1')
logger2 = logging.getLogger('myapp.area2')
logger1.debug('Quick zephyrs blow, vexing daft Jim.')
logger1.info('How quickly daft jumping zebras vex.')
logger2.warning('Jail zesty vixen who grabbed pay from quack.')
logger2.error('The five boxing wizards jump quickly.')
Edit: As discussed in the comments this code still generates two handlers, but "hides" one construction through the use of basicConfig(). I would strongly encourage you to create both explicitly.

You can append lots of handlers to the same logger with different loglevel, but root loglevel must be as verbose as the most verbose handler. This example log messages to file with log.debug() and log.info() to console:
log = logging.getLogger("syslog2elastic")
log.setLevel(logging.DEBUG) # this must be DEBUG to allow debug messages through
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s.%(msecs)03d - %(name)s:%(lineno)d - %(levelname)s - %(message)s", "%Y%m%d%H%M%S")
console.setFormatter(formatter)
log.addHandler(console)
fh = RotatingFileHandler(args.logfile, maxBytes=104857600, backupCount=5)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s.%(msecs)03d - %(message)s", "%Y%m%d%H%M%S")
fh.setFormatter(formatter)
log.addHandler(fh)

No. File and Screen output means you need two handlers.

You can do this in the following way, making two loggers, one for file logging, another one for console logging. Make sure to set the root logger to the most verbose of the two.
import logging
logging.getLogger().setLevel(logging.DEBUG) # This must be as verbose as the most verbose handler
formatter = logging.Formatter(
'%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s [%(lineno)s]: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
console_logging = logging.StreamHandler()
console_logging.setLevel(logging.WARNING)
console_logging.setFormatter(formatter)
logging.getLogger().addHandler(console_logging)
file_logging = logging.FileHandler('file.log')
file_logging.setLevel(logging.DEBUG)
file_logging.setFormatter(formatter)
logging.getLogger().addHandler(file_logging)