Develop Reference

python TimedRotatingFileHandler old empty files

I have a problem with the python logging library. I have a flask application that uses the following class (_WrapperLog) to write logs to a file, the library correctly writes logs to today's file and cleans up files older than 7 days correctly. the problem is that when the log rotation occurs, the old files remain saved but their content disappears (as in the example below), only the first 2/4 lines of the file remain, and then nothing, what could be due ?
import logging
from logging.handlers import TimedRotatingFileHandler
import os
from datetime import datetime
from os import name as os_name
from sys import stdout as sys_stdout
class _WrapperLog(object):
class __Log:
logger = None
loggerNameTimeStamp=None
def __init__(self):
self.logger = self.get_logger()
def __str__(self):
return str(self)
def get_logger(self):
if datetime.now().strftime("%Y-%m-%d") == self.loggerNameTimeStamp:
return self.logger
if self.logger:
while len(self.logger.handlers) > 0:
self.logger.handlers.pop()
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
if os_name == 'nt':
handler = logging.StreamHandler(sys_stdout)
else:
handler = TimedRotatingFileHandler(os.path.join('/opt/my_folder', 'my_log.log'),
when='midnight', backupCount=7)
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.loggerNameTimeStamp = datetime.now().strftime("%Y-%m-%d")
return self.logger
def info(self, message:str):
self.get_logger().info(message)
def debug(self, message:str):
print("DEBUG: " + message)
def warning(self, message:str):
self.get_logger().warning(message)
def error(self, message:str):
self.get_logger().error(message)
instance = None
def __new__(cls):
if not _WrapperLog.instance:
_WrapperLog.instance = _WrapperLog.__Log()
return _WrapperLog.instance
def __getattr__(self, name):
return getattr(self._WrapperLog, name)
def __setattr__(self, name):
return setattr(self._WrapperLog, name)
Log = _WrapperLog()
example of what remains of the files from yesterday and the day before yesterday:
my_log.log.2022-04-11:
[2022-04-12 00:00:01] INFO: <censored>
[2022-04-12 00:00:01] INFO: <censored>
my_log.log.2022-04-10:
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <censored>
[2022-04-11 00:00:01] INFO: <this line started like this and stopped after INFO>

Python Decorator Log file not generating

I am new to Python and learning logging technique with Decorator.
For me the below code is not generating required log file. Debugged the code, getting correct message to logger statement but the file is not generating. From Test method i am call the required function where i have implemented Decorator. Please guide where i am doing mistake.
try:
import csv
import requests
import datetime
import os
import sys
import logging
except Exception as e:
print("Some Modules are missing {}".format(e))
class Meta(type):
""" Meta class"""
def __call__(cls, *args, **kwargs):
instance = super(Meta, cls).__call__(*args, **kwargs)
return instance
def __init__(cls, name, base, attr):
super(Meta, cls).__init__(name, base, attr)
class log(object):
def __init__(self, func):
self.func = func
def __call__(self, *args, **kwargs):
""" Wrapper Function"""
start = datetime.datetime.now() #start time
Tem = self.func(*args) #call Function
Argument = args
FunName = self.func.__name__ #get Function name
end = datetime.datetime.now() #end Time
message = """
Function : {}
Execustion Time : {}
Argument : {}
Memory : {} Bytes
Date : {}
""".format(FunName,
end-start,
Argument,
sys.getsizeof(self.func),
start
)
cwd = os.getcwd();
folder = 'Logs'
newPath = os.path.join(cwd, folder)
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
logging.basicConfig(filename='apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
return Tem
class APIHelper(metaclass=Meta):
def __init__(self, *args, **kwargs):
pass
#log
def star_wars_characters(url):
#self.url = url
api_response = requests.get(url)
people = []
if api_response.status_code == 200:
data = api_response.json()
for d in data['results']:
character = []
character.append(d['name'])
character.append(d['height'])
character.append(d['gender'])
people.append(character)
return people
else:
return "Bad Request"
My Test Method:
import unittest
import csv
from com.Script.APIHelper import APIHelper
class TestAPI(unittest.TestCase):
def _setUp(self, file_name):
self.api = APIHelper()
with open(file_name, "w") as self.fd:
self.csvfile = csv.writer(self.fd, delimiter = ',')
self.csvfile.writerow(['Name','Height','Gender'])
def tearDown(self):
self.fd.close()
def test_responseNotEmpty(self):
file_name = 'SWAPI.csv'
self._setUp(file_name)
people = self.api.star_wars_characters("https://swapi.dev/api/people/")
assert type(people) is list
Thanks you in Advance.

Add finally
Change filename='apiRun.log' to filename='{}/apiRun.log'
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
finally:
logging.basicConfig(filename='{}/apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
except is executed only when an exception is raised from try.
finally is always executed.

Stopping log script from being accessed by multiple threads

I have a module called myLog.py which is being accessed by multiple other modules in a project. The myLog.py module has two handlers: file_handler that inputs logs into file and stream_handler that outputs logs to a console. For modules where no threading is occurring i.e myLog.py is only being accessed by a single process the logs are being inserted properly but for modules where threading is being implemented i.e myLog.py is being accessed by multiple processes at the same time I am getting multiple logs of the same line being inserted in my log_file.txt.
While going through logging documentation I found out that logging module is thread_safe but my implementation says things differently. How should I initialize the function setLogger() in myLog.py such that if it gets accessed by multiple threads at the same time it gives the correct output?
#myLog.py
#setup of logger
def setLogger(logfile_name = "log_file.txt"):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_handler = logging.FileHandler(logfile_name)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
return logger
So suppose for example it is being accessed by a module called parser.py which implements threading then the log statements get printed out in a very random duplicated fashion.
#parser.py
import threading
import myLog
logger = myLog.setLogger()
class investigate(threading.Thread):
def __init__(self, section, file, buffer, args):
threading.Thread.__init__(self)
self.section = section
self.file = file
self.buffer = buffer
self.args = args
self.sig = self.pub = None
self.exc = None
def run(self):
aprint("Starting section %d file %d" % (self.section, self.file))
self.exc = None
try:
self.sign()
aprint("Done section %d file %d" % (self.section, self.file))
except:
self.exc = sys.exc_info()
def sign(self):
self.sig, self.pub = sign_hsm(self.buffer, self.args)
if self.sig is None or self.pub is None:
raise Exception("Empty signing result")
def store(self, bot):
sec = filter(lambda x: x.type == self.section, bot.sections)[0]
if self.file == 0xFF:
signature = sec.signature
else:
signature = sec.files[self.file].signature
signature.sig = self.sig
signature.pub = self.pub
def join(self, *args, **kwargs):
threading.Thread.join(self, *args, **kwargs)
if self.exc:
msg = "Thread '%s' threw an exception: %s" % (self.getName(), self.exc[1])
new_exc = Exception(msg)
raise new_exc.__class__, new_exc, self.exc[2]
def PrintVersion():
logger.info("This is output.")
print_lock = threading.RLock()
def aprint(*args, **kwargs):
if verbosityLevel > 0:
with print_lock:
return logger.info(*args, **kwargs)
def multipleTimes():
logger.info("Multiple times.")
if __name__ == "__main__":
PrintVersion()
for investigate in investigations:
investigate.start()
.......
.......
.......
logger.info("This gets repeated")
multipleTimes()
So since multiple threads are trying to access the setLogger() I get logger.info() outputs such as:
This is output.
This is output.
This is output.
This is output.
This is output.
This gets repeated.
This gets repeated.
This gets repeated.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
Multiple times.
What I should be getting:
This is output.
This gets repeated.
Multiple times.

How can I solve Duplicated Data Log Problem in Python stdout to txt file

this is my first question. My code body is below, my data is duplicated and in a short time txt file is filling with too much data to handle, I could not solve the problem, I am new in programming and in python.
import logging
from time import sleep
x = 0
while True:
x += 1
print("you gotta be very careful",x)
class Logger(object):
def __init__(self, filename="Default.log"):
self.terminal = sys.stdout
self.log = open(filename, "a")
def write(self, message):
self.terminal.write(message)
self.log.write(message)
print("x, %.2f " %x)
sys.stdout = Logger("mylogfile.txt")
sleep(0.25)

Shared variables in Python

I am trying to create multiple threads of bot and they share some variables, but I am failing miserably in getingt the shared variables to work.
Here is the code:
import requests
import sys
import threading
import signal
import time
class bot(threading.Thread):
terminate = False
#def __init__(self):
# threading.Thread.__init__(self)
# self.terminate = False
def getCode():
code_lock.acquire()
work_code = code
try:
code += 1
finally:
code_lock.release()
return work_code
def checkCode(code):
try:
#if(code % 1000000 == 0):
print("Code "+str(code)+" is being checked...\n")
html = requests.get(url+str(code))
html.encoding = 'utf-8'
return not 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
print("Connection Error! Retrying...\n")
time.sleep(0.5)
except KeyboardInterrupt:
logCode(code)
sys.exit()
def storeCode(code):
file_lock.acquire()
try:
file.write(code+'\n')
finally:
file_lock.release()
def logCode(code):
log_lock.acquire()
try:
log.write(code+'\n')
finally:
log_lock.release()
#def run(self):
# global bots
# global url
# global file
# global log
# global code_lock
# global file_lock
# global log_lock
while(not terminate):
code = getCode()
if(checkCode(code)):
storeCode(code)
logCode(code)
def main(code = 0, threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
url = 'https://test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
def signal_handler(signal, frame):
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
for bot in bots:
bot.terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot()
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main(736479509787350, 1)
With this code I get this error:
Traceback (most recent call last): File "bot.py", line 7, in
class bot(threading.Thread): File "bot.py", line 59, in bot
code = getCode() File "bot.py", line 14, in getCode
code_lock.acquire() NameError: name 'code_lock' is not defined
I don't know if I should override the run(self) method of bot, but when I tried that it never actually ran the method run and I also receive the same error from all the threads created: that int is not callable (and I can't see where I can possibly be using an int as object).
Additionaly I don't know if I am handling correctly the exit signal from keyboard, as you can see I am trying to deal with that using a terminate variable, but I don't think that this is the problem...
One last thing, the ConnectionError exception is not being appropriately handled, as it's saying "Retrying...", but in fact it will not retry, but I am aware of that and it should be ok, I'll fix it latter.
Worth mentioning that I'm not very used to deal with multi-threading and when I do deal with it, it is in C or C++.
Edit
I can make the code work by using global variables, but I do not want to do that, I prefer to avoid using globals. My attempts of passing the variables directly to the instances of the class bot or by passing an data-object to it weren't successful so far, whenever I pass the variables or the auxiliar object to bot I am unable to access them as attributes using self. and without self. Python claims that the variable was not defined.
Here is the updated code, without success yet:
import requests
import sys
import threading
import signal
import time
class Shared:
def __init__(self, code, url, file, log, code_lock, file_lock, log_lock):
self.code = code
self.url = url
self.file = file
self.log = log
self.code_lock = code_lock
self.file_lock = file_lock
self.log_lock = log_lock
class bot(threading.Thread):
def __init__(self, data):
threading.Thread.__init__(self)
self.terminate = False
self.data = data
#classmethod
def getCode(self):
self.data.code_lock.acquire()
work_code = self.data.code
try:
self.data.code += 1
finally:
self.data.code_lock.release()
return work_code
#classmethod
def checkCode(self, work_code):
try:
#if(code % 1000000 == 0):
print("Code "+str(work_code)+" is being checked...\n")
html = requests.get(self.data.url+str(work_code))
html.encoding = 'utf-8'
return not 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
print("Connection Error! Retrying...\n")
time.sleep(0.5)
except KeyboardInterrupt:
self.logCode(work_code)
sys.exit()
#classmethod
def storeCode(self, work_code):
self.data.file_lock.acquire()
try:
self.data.file.write(work_code+'\n')
finally:
self.data.file_lock.release()
#classmethod
def logCode(self, work_code):
self.data.log_lock.acquire()
try:
self.data.log.write(work_code+'\n')
finally:
self.data.log_lock.release()
#classmethod
def run(self):
while(not self.terminate):
work_code = self.getCode()
if(self.checkCode(work_code)):
self.storeCode(work_code)
self.logCode(work_code)
def main(code = 0, threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
url = 'https://www.test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
data = Shared(code, url, file, log, code_lock, file_lock, log_lock)
def signal_handler(signal, frame):
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
for bot in bots:
bot.terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot(data)
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main(736479509787350, 4)
Yet, the working code with global variables:
import requests
import sys
import threading
import signal
import time
code = 736479509787350
url = 'https://www.test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
terminate = False
class bot(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
#classmethod
def getCode(self):
global code
code_lock.acquire()
work_code = code
try:
code += 1
finally:
code_lock.release()
return work_code
#classmethod
def checkCode(self, work_code):
try:
if(code % 1000000 == 0):
print("Code "+str(work_code)+" is being checked...\n")
html = requests.get(url+str(work_code))
html.encoding = 'utf-8'
if(not 'Page Not Found' in html.text):
time.sleep(0.5)
html = requests.get(url+str(work_code)+":999999999")
html.encoding = 'utf-8'
return 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
#print("Connection Error! Retrying...\n")
time.sleep(1)
return self.checkCode(work_code)
except KeyboardInterrupt:
self.logCode(work_code)
sys.exit()
#classmethod
def storeCode(self, work_code):
global file_lock
global file
file_lock.acquire()
try:
file.write(str(work_code)+'\n')
finally:
file_lock.release()
#classmethod
def logCode(self, work_code):
global log_lock
global log
log_lock.acquire()
try:
log.write(str(work_code)+'\n')
finally:
log_lock.release()
#classmethod
def run(self):
global terminate
while(not terminate):
work_code = self.getCode()
if(self.checkCode(work_code)):
print("Code "+str(work_code)+" is a valid code!\n")
self.storeCode(work_code)
self.logCode(work_code)
def main(threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
#url = 'https://www.facebook.com/leticia.m.demenezes/posts/'
#file = open("valid-codes.txt", "a")
#log = open("log.txt", "a")
#code_lock = threading.Lock()
#file_lock = threading.Lock()
#log_lock = threading.Lock()
def signal_handler(signal, frame):
global terminate
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot()
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main()

You could make the code_lock global as you're trying to do, but why not just pass it into each bot class?
t = bot(code_lock)
Next create a constructor for your class:
class bot(threading.Thread):
def __init__(self, code_lock):
threading.Thread.__init__(self)
self.code_lock = code_lock
Now, whenever you try to use code_lock inside your bot class, always prefix it with self (self.code_lock).
If you really insist on using global variables, then look into the global keyword.

It's clear that you are trying to access code_lock out of it's scope, may be you can follow #MartinKonecny suggestion to fix that.
I could see that even after fixing code_lock problem, your code has lot of problems. as soon as you fix code_lock problem you'll face similar issue with the variable code in the same function getCode.
After fixing all those compiled time issues, you'll face issues with your whole class implementation, this is not the way to implement Python classes.
It's better that you go through this to know more about python name spaces and classes.