I am new to Python and learning logging technique with Decorator.
For me the below code is not generating required log file. Debugged the code, getting correct message to logger statement but the file is not generating. From Test method i am call the required function where i have implemented Decorator. Please guide where i am doing mistake.
try:
import csv
import requests
import datetime
import os
import sys
import logging
except Exception as e:
print("Some Modules are missing {}".format(e))
class Meta(type):
""" Meta class"""
def __call__(cls, *args, **kwargs):
instance = super(Meta, cls).__call__(*args, **kwargs)
return instance
def __init__(cls, name, base, attr):
super(Meta, cls).__init__(name, base, attr)
class log(object):
def __init__(self, func):
self.func = func
def __call__(self, *args, **kwargs):
""" Wrapper Function"""
start = datetime.datetime.now() #start time
Tem = self.func(*args) #call Function
Argument = args
FunName = self.func.__name__ #get Function name
end = datetime.datetime.now() #end Time
message = """
Function : {}
Execustion Time : {}
Argument : {}
Memory : {} Bytes
Date : {}
""".format(FunName,
end-start,
Argument,
sys.getsizeof(self.func),
start
)
cwd = os.getcwd();
folder = 'Logs'
newPath = os.path.join(cwd, folder)
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
logging.basicConfig(filename='apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
return Tem
class APIHelper(metaclass=Meta):
def __init__(self, *args, **kwargs):
pass
#log
def star_wars_characters(url):
#self.url = url
api_response = requests.get(url)
people = []
if api_response.status_code == 200:
data = api_response.json()
for d in data['results']:
character = []
character.append(d['name'])
character.append(d['height'])
character.append(d['gender'])
people.append(character)
return people
else:
return "Bad Request"
My Test Method:
import unittest
import csv
from com.Script.APIHelper import APIHelper
class TestAPI(unittest.TestCase):
def _setUp(self, file_name):
self.api = APIHelper()
with open(file_name, "w") as self.fd:
self.csvfile = csv.writer(self.fd, delimiter = ',')
self.csvfile.writerow(['Name','Height','Gender'])
def tearDown(self):
self.fd.close()
def test_responseNotEmpty(self):
file_name = 'SWAPI.csv'
self._setUp(file_name)
people = self.api.star_wars_characters("https://swapi.dev/api/people/")
assert type(people) is list
Thanks you in Advance.
Add finally
Change filename='apiRun.log' to filename='{}/apiRun.log'
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
finally:
logging.basicConfig(filename='{}/apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
except is executed only when an exception is raised from try.
finally is always executed.
Related
I have a need to fill stdin from code directly when input() is waiting for filling.
Is there to do the next:
# Here suppose to be some code that will automatically fill input() below
string = input("Input something: ")
# Or here
I've heard about subprocess.Popen, but I don't understand how to use it in my case. Thank you.
This code is something:
import sys
from io import StringIO
class File(StringIO):
def __init__(self):
self._origin_out = sys.stdout
self._origin_in = sys.stdin
sys.stdout = self
sys.stdin = self
self._in_data = ''
super(File, self).__init__()
def write(self, data):
if data == 'My name is:':
self._in_data = 'Vasja\n'
else:
self._origin_out.write(data)
def readline(self, *args, **kwargs):
res = self._in_data
if res:
self._in_data = ''
return res
else:
return sys.stdin.readline(*args, **kwargs)
def __del__(self):
sys.stdout = self._origin_out
sys.stdin = self._origin_in
global_out_file = File()
a = input('My name is:')
print('Entered name is:', a)
I'm new to python and trying to remove/trim gevent stacktrace output when an exception is raised. I read somewhere that I can make it happen by using AsyncResult, however it seems like I can't figure out how to use this.
Here is an example I started with and iterated over to make it similar to the real code I'm troubleshooting, but I got stuck in the last phase when I tried to add my_decor to work().
Any help fixing this is much appreciated.
from gevent.event import AsyncResult
import gevent
from functools import wraps
def my_decor(k, *args, **kwargs):
#wraps(k)
def wrapper(*args, **kwargs):
r = AsyncResult()
try:
value = k()
except Exception as e:
r.set_exception(e)
else:
r.set(value)
return r.exception or r.value
result = gevent.spawn(wrapper, k)
return result
def f():
def foo():
if True:
raise Exception('tttttttt')
return foo
def p():
def bar():
if True:
raise Exception('ppppppppppppp')
return bar
#my_decor
def work():
foo1 = gevent.spawn(f())
bar1 = gevent.spawn(p())
gevent.joinall([foo1, bar1])
return foo1.get() or bar1.get()
Found the answer, figured it might be a help to those with the same problem.
from gevent.event import AsyncResult
import gevent
from functools import wraps
def my_decor(k):
#wraps(k)
def wrapper(*args, **kwargs):
r = AsyncResult()
try:
value = k(*args, **kwargs)
except Exception as e:
r.set_exception(e)
else:
r.set(value)
return r.exception or r.value
return wrapper
def f(msg):
#my_decor
def foo():
if True:
raise Exception('tttttttt %s' % msg)
# print('test')
return foo
def p(msg):
#my_decor
def bar():
if True:
raise Exception('ppppppppppppp %s', msg)
return bar
def work():
test = "test"
seti = "set"
foo1 = gevent.spawn(f(test)) # returns a function that coroutine uses
bar1 = gevent.spawn(p(seti))
gevent.joinall([foo1, bar1])
return foo1.get() or bar1.get()
res = work()
print res
In Java, you can make a variable thread safe by just adding the synchronized keyword. Is there anything that can achieve the same results in Python?
You can use with self.lock: and then put your code inside there. See http://theorangeduck.com/page/synchronized-python for more information.
Working code using with self.lock which can take care of exception if occurs:
Inside Manager we are making Manager mehods thread safe :
from threading import RLock
class Manager:
def __init__(self):
self.lock = RLock()
self.hash: dict[str, int] = dict()
def containsToken(self, key) -> bool:
with self.lock:
self.lock.acquire()
return key in self.hash
def addToken(self, token: str):
with self.lock:
token = token.strip()
if token in self.hash:
self.hash[token] = self.hash[token] + 1
else:
self.hash[token] = 1
def removeToken(self, token):
with self.lock:
if token not in self.hash:
raise KeyError(f"token : {token} doesn't exits")
self.hash[token] = self.hash[token] - 1
if self.hash[token] == 0:
self.hash.pop(token)
if __name__ == "__main__":
sync = Manager()
sync.addToken("a")
sync.addToken("a")
sync.addToken("a")
sync.addToken("a")
sync.addToken("B")
sync.addToken("B")
sync.addToken("B")
sync.addToken("B")
sync.removeToken("a")
sync.removeToken("a")
sync.removeToken("a")
sync.removeToken("B")
print(sync.hash)
Output:
{'a': 1, 'B': 3}
You can write your own #synchronized decorator.
The example uses a Mutex Lock:
from functools import wraps
from multiprocessing import Lock
def synchronized(member):
"""
#synchronized decorator.
Lock a method for synchronized access only. The lock is stored to
the function or class instance, depending on what is available.
"""
#wraps(member)
def wrapper(*args, **kwargs):
lock = vars(member).get("_synchronized_lock", None)
result = ""
try:
if lock is None:
lock = vars(member).setdefault("_synchronized_lock", Lock())
lock.acquire()
result = member(*args, **kwargs)
lock.release()
except Exception as e:
lock.release()
raise e
return result
return wrapper
Now your are able to decorate a method like this:
class MyClass:
...
#synchronized
def hello_world(self):
print("synced hello world")
And there is also an excellent Blog post about the missing synchronized decorator.
I have a small script that polls a database to look for status of certain jobs. I decided to use APScheduler to handle the looping call. I created a decorator to timeout a function if taking too long. The issue I am having here is that the decorator is inside a class and even though I create two instances of the class, inside two different functions, they always have the same start_time. I thought maybe if I move the decorator inside of my class and initialize the start_time in the init call it would update the start_time per instance of the class. When I moved the decorator insdie of the class and assigned self.start_time = datetime.now() the start time updates on each call of the class and thus will never time out. The example of the decorator inside of the class is also below.
def timeout(start, min_to_wait):
def decorator(func):
def _handle_timeout():
scheduler.shutdown(wait=False)
#wraps(func)
def wrapper(*args, **kwargs):
expire = start + timedelta(minutes = min_to_wait)
now = datetime.now()
if now > expire:
_handle_timeout()
return func(*args, **kwargs)
return wrapper
return decorator
class Job(object):
def __init__(self, name, run_id, results):
self.name = name
self.run_id = object_id
self.results = results
self.parcel_id= None
self.status = None
start_time = datetime.now()
#timeout(start_time, config.WAIT_TIME)
def wait_for_results(self):
if self.results:
self.pack_id = self.results[0].get('parcel_id')
self.status = self.results[0].get('status')
return self.results[0]
else:
return False
#timeout(start_time, config.WORK_TIME)
def is_done(self):
status = self.results[0].get('status')
status_map = {'done': True,
'failed': FailedError,
'lost': LostError}
def _get_or_throw(s, map_obj):
value = map_obj.get(s)
if s in ['failed', 'lost']:
raise value(s)
else:
self.status = s
return s
return _get_or_throw(status, status_map)
def job_1(mssql, postgres, runid):
res = get_results(mssql, config.MSSQL, first_query_to_call)
first_job= Job('first_job', runid, res)
step_two = pack_job.wait_for_results()
if step_two:
try:
logger.info(first_job)
if first_job.is_done() == 'done':
scheduler.remove_job('first_job')
scheduler.add_job(lambda: job_two(mssql,
postgres, first_job.object_id, runid), 'interval', seconds=config.POLL_RATE, id='second_job')
except LostError as e:
logger.error(e, exc_info=True)
scheduler.shutdown(wait=False)
except FailedError as e:
logger.error(e, exc_info=True)
scheduler.shutdown(wait=False)
def job_two(mssql, postgres, object_id, runid):
res = get_results(mssql, config.MSSQL, some_other_query_to_run, object_id)
second_job= Job('second_job', runid, res)
step_two = second_job.wait_for_results()
if step_two:
try:
logger.info(second_job)
if second_job.is_done() == 'done':
scheduler.remove_job('second_job')
except LostError as e:
logger.error(e, exc_info=True)
scheduler.shutdown(wait=False)
except FailedError as e:
logger.error(e, exc_info=True)
scheduler.shutdown(wait=False)
if __name__ == '__main__':
runid = sys.argv[1:]
if runid:
runid = runid[0]
scheduler = BlockingScheduler()
run_job = scheduler.add_job(lambda: job_one(pymssql, psycopg2, runid), 'interval', seconds=config.POLL_RATE, id='first_job')
attempt to move decorator inside class:
class Job(object):
def __init__(self, name, run_id, results):
self.name = name
self.run_id = run_id
self.results = results
self.pack_id = None
self.status = None
self.start_time = datetime.now()
def timeout(min_to_wait):
def decorator(func):
def _handle_timeout():
scheduler.shutdown(wait=False)
#wraps(func)
def wrapper(self, *args, **kwargs):
print '**'
print self.start_time
print ''
expire = self.start_time + timedelta(minutes = min_to_wait)
now = datetime.now()
if now > expire:
_handle_timeout()
return func(self, *args, **kwargs)
return wrapper
return decorator
here is an example output from when I use the above decorator.
**
self start time: 2014-10-28 08:57:11.947026
**
self start time: 2014-10-28 08:57:16.976828
**
self start time: 2014-10-28 08:57:21.989064
the start_time needs to stay the same or else I can't timeout the function.
In the first exemple, your start time is initialised when the class statement is executed, which in your case is when the module is first imported in the interpreter.
In the second exemple, the start time is initialized when the class is instanciated. It should not change from one method call to another for a same Job instance. Of course if you keep on creating new instances, the start time will be different for each instance.
Now you didn't post the code using your Job class, so it's hard to tell what the right solution would be.
I have the following setting:
import sys
from flask import Flask
from flask.ext import restful
from model import Model
try:
gModel = Model(int(sys.argv[1]))
except IndexError, pExc:
gModel = Model(100)
def main():
lApp = Flask(__name__)
lApi = restful.Api(lApp)
lApi.add_resource(FetchJob, '/')
lApp.run(debug=True)
class FetchJob(restful.Resource):
def get(self):
lRange = gModel.getRange()
return lRange
if __name__ == '__main__':
main()
Is there a way to instantiate the Model-class inside the main()-function? Here, the Flask framework instantiates the FetchJob-class, so that I cannot provide it the parameters it forwards during the instantiation process.
I don't like to have global variables as this messes up the whole design ...
I think this should work, although I'm not familiar with Flask:
import functools
def main():
try:
gModel = Model(int(sys.argv[1]))
except IndexError as pExc:
gModel = Model(100)
lApp = Flask(__name__)
lApi = restful.Api(lApp)
lApi.add_resource(functools.partial(FetchJob, gModel), '/')
lApp.run(debug=True)
class FetchJob(restful.Resource):
def __init__(self, obj, *args, **kwargs):
restfult.Resource.__init__(self, *args, **kwargs)
self.obj = obj
def get(self):
lRange = self.obj.getRange()
return lRange