I want to use AzureLogHandler and AzureEventHandler together it in one file how to use it? But it will print both places traces and customEvents, which is a waste of resources
import logging
from opencensus.ext.azure.log_exporter import AzureLogHandler, AzureEventHandler
EXTRA_INFO = {'custom_dimensions': {'info': 'This is test'}}
logger = logging.getLogger(__name__)
logEvent = logging.getLogger(__name__)
azureEventHandler = AzureEventHandler(
connection_string=f'InstrumentationKey={INSTRUMENTATION_KEY}')
azureEventHandler.setLevel(logging.ERROR)
logEvent.addHandler(azureEventHandler)
azureLogHandler =AzureLogHandler(
connection_string=f'InstrumentationKey={INSTRUMENTATION_KEY}')
azureLogHandler.setLevel(logging.INFO)
logger.addHandler(azureLogHandler)
logger.setLevel(logging.INFO)
def main():
for i in range(10):
logger.info(str(i) + ":logger info Info Log", extra=EXTRA_INFO)
logEvent.info(str(i) + ":logEvent info Info Log", extra=EXTRA_INFO)
if __name__ == "__main__":
main()
Related
I have an rfid that connect to my code shows that it connected. I am using twisted and uhf_reader library for that , but it did not read any data form the tag .
import logging
import binascii
import uhf_reader
from twisted.internet import reactor
UHF_HOST = "192.168.0.178"
UHF_PORT = 4001
FORMAT = ('%(asctime)-15s %(threadName)-15s'
' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s')
logging.basicConfig(format=FORMAT)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
reader_queue = None
reader = None
def sendTestRequest():
deferred = reader.gen2_sec_read_ex(bank=uhf_reader.USER, addr=3, count=16)
deferred.addCallbacks(lambda result: print(binascii.hexlify(result)),
lambda err: print(err))
class UHFReaderProtocol(uhf_reader.protocol.UHFReaderProtocolBase):
def connectionMade(self):
super().connectionMade()
logger.info("Connection established")
reactor.callLater(5, sendTestRequest)
if __name__ == '__main__':
factory = uhf_reader.factory.UHFReaderClientFactory.forProtocol(UHFReaderProtocol)
reader_queue = factory.getQueue("{}:{}".format(UHF_HOST, UHF_PORT))
reader = uhf_reader.AsyncUHFReader(queue=reader_queue)
reactor.connectTCP(UHF_HOST, UHF_PORT, factory)
reactor.run()
this is an error I got
My main program logs to its own log file and the sub-process should have its own log file.
I replaced the logger object inside the multiprocessing process, but the logging data from the sub-process is additionally redirected to the main log file.
How can I prevent this?
The structure looks like this:
import logging
import sys
import os
from pathlib import Path
import multiprocessing
import time
import requests
class ProcessFilter(logging.Filter):
"""Only accept log records from a specific pid."""
def __init__(self, pid):
self._pid = pid
def filter(self, record):
return record.process == self._pid
def create_logger(file):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addFilter(ProcessFilter(pid=os.getpid()))
file_handler = logging.FileHandler(file)
stream_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S')
file_handler.setFormatter(formatter)
stream_handler.setFormatter(formatter)
log.addHandler(file_handler)
log.addHandler(stream_handler)
return log
def subprocess_init():
global log
sub_log_file = str(Path.home()) + '/logfile_sub.log'
log = create_logger(sub_log_file)
do_subprocess_stuff()
def do_subprocess_stuff():
count = 0
while True:
create_log("subprocess", count)
time.sleep(5)
count += 1
def main_tasks():
num = 10
while num > 0:
create_log("main", num)
time.sleep(5)
num -= 1
def create_log(text, num):
log.debug(text + " log %s", num)
if __name__ == '__main__':
file = str(Path.home()) + '/logfile.log'
log = create_logger(file)
sub_process = multiprocessing.Process(target=subprocess_init, args=())
sub_process.daemon = True
sub_process.start()
main_tasks()
I am simply translating this answer to fit multiprocessing.
import logging
class ProcessFilter(logging.Filter):
"""Only accept log records from a specific pid."""
def __init__(self, pid):
self._pid = pid
def filter(self, record):
return record.process == self._pid
import logging
import os
def create_logger(file):
log = logging.getLogger('') # why use this logger and not __name__ ?
log.setLevel(logging.DEBUG)
log.addFilter(ProcessFilter(pid=os.getpid())) # logger wide filter
file_handler = logging.FileHandler(file)
stream_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S')
file_handler.setFormatter(formatter)
stream_handler.setFormatter(formatter)
log.addHandler(file_handler)
log.addHandler(stream_handler)
return log
NB. you can also put the filter on a specific handler
Creating a custom logger for my purpose using python which can be used across different modules just by importing and calling a custom_log method.
This is MyLogger.py script.
import datetime
import logging
import logging.handlers
import os
import colorlog
from pathlib import Path
class MyLogger(logging.Logger):
def __init__(self, verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if (verbose == 1):
max_bytes = 1024 ** 2
backup_count = 100
t = datetime.datetime.now()
file_name = file_name_format.format(year=t.year, month=t.month, day=t.day, hour=t.hour, minute=t.minute,
second=t.second)
file_name = os.path.join(log_dir_path, file_name)
file_handler = logging.handlers.RotatingFileHandler(filename=file_name, maxBytes=max_bytes, backupCount=backup_count)
file_handler.setLevel(logging.DEBUG)
file_formatter = logging.Formatter(file_msg_format)
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
if (verbose == 1):
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
def custom_log(self, level, msg):
logging.log(getattr(logging,level),msg)
I have 2 other scripts within the same directory as below, just need to initialize MyLogger() in Test1.py at beginning of the test and expecting to use custom_log in all other scripts. Missing something here to make this work,either in the way of initializing or the way of importing. Any support how to get this done.
Test1.py
from MyLogger import MyLogger
class StartTest():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
if __name__ == '__main__':
MyLogger()
StartTest()
Test2.py
from MyLogger import MyLogger
class Test():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
def TestMethod(self):
MyLogger.custom_log('INFO','Debug messages are only sent to the logfile.')
You can achieve it by creating a logger with a specific name instead of using root logger. logging.log() uses always the root logger. So, you can define a logger with a specific name instead of creating a new Logging channel.
An example which is inline with your need
class MyLogger:
logger: logging.Logger = None
#staticmethod
def configure(verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger("mylogger")
logger.setLevel(logging.DEBUG)
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
MyLogger.logger = logger
if __name__ == '__main__':
MyLogger.configure()
MyLogger.logger.error("debug message")
Once the MyLogger.configure is called, then you can use the MyLogger.logger.* any where in your app / scripts.
This can be done without a helper class as well. Create a logger configuration file. Configure your custom logger with a different name rather than root, and from your code always call logging.getLogger("name-of-logger") to create a logging instance.
I have written a below code to write the logs into cloudwatch using watchtower.
import os
import sys
import time
import boto3
import watchtower
import logging.handlers
from scripts.config import app_configurations
def fast_logging():
try:
boto3_session = boto3.session.Session()
LOG_GROUP = "Fast-Logging"
log_level = DEBUG
stream_name = os.path.join("fast_logging"+ "_" + time.strftime("%Y%m%d") + '.log')
logger = logging.getLogger("Fast-Logger")
logger.setLevel(log_level)
formatter = logging.Formatter('%(name)s - %(levelname)s - %(filename)s - %(module)s: %(funcName)s: '
'%(lineno)d - %(message)s')
log_handler = watchtower.CloudWatchLogHandler(log_group=LOG_GROUP, boto3_session=boto3_session,
stream_name=stream_name)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
except Exception as e:
raise e
logger = fast_logging()
The above code is working for the normal python code but not able to dump logs into cloudwatch stream for the logs in Fast API services.
I found that this code works for me
import os
import time
import boto3
import watchtower
import logging.handlers
def fast_logging():
try:
LOG_GROUP = "Fast-Logging"
log_level = "INFO"
stream_name = os.path.join("fast_logging"+ "_" + time.strftime("%Y%m%d") + '.log')
logger = logging.getLogger("Fast-Logger")
logger.setLevel(log_level)
formatter = logging.Formatter('%(name)s - %(levelname)s - %(filename)s - %(module)s: %(funcName)s: '
'%(lineno)d - %(message)s')
log_handler = watchtower.CloudWatchLogHandler(log_group=LOG_GROUP,
stream_name=stream_name)
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
except Exception as e:
raise e
logger = fast_logging()
logger.info("test this")
I am trying to log the actions in a function and I have written the following function to log responses to different files based on the type of response i.e. info,error,debug and warning.
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter()
fileHandler = logging.FileHandler(log_file)
fileHandler.setFormatter(formatter)
handler = TimedRotatingFileHandler(logger_name,
when="d",
interval=1,
backupCount=100)
l.setLevel(level)
l.addHandler(fileHandler)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
class Info(APIHandler):
#gen.coroutine
def post(self):
req = json.loads(self.request.body)
resp, content = client(item_id=req['item_id'])
debug.debug(content)
info.info(hello world)
warning.warn('warning message')
error.error('error message')
The problem that I am facing is that a response is printed twice each time I call a function.
for example:
info.log
hello world
hello world
Can anyone tell me why is it happening like. This is the case with all the log files.
Thanks
try:
import logging
import logging.handlers
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.handlers.TimedRotatingFileHandler(str(log_file)+'.log', when="d", interval=1, backupCount=100)
handler.setFormatter(formatter)
l.setLevel(level)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
if __name__ == "__main__":
info.info('hello world')
error.info('hello world')
after run this script, file info.log has one 'hello world' and error.log also has only one 'hello world'.