Is there a pdb equivalent to disp in gdb?
E.g. when I'm debugging C using gdb I can have variables printed on every 'step' through the code by typing:
disp var
When I'm debugging python using pdb I would like similar functionality, but disp does not seem to be there, the python pdb documentation does not seem to offer an alternative - but it seems like an odd omission?
The code bellow uses Python introspection features to add two new commands to the PDB module 0
just put the given function, and its call in a separate module, and import this module before starting debugging - you should have the 'disp' and 'undisp' commands do add and retract watches to variables.
It works by monkeypatching Python's pdb module, which is written in pure python.
# -*- coding: utf-8 -*-
def patch_pdb():
import pdb
def wrap(func):
def new_postcmd(self, *args, **kw):
result = func(self, *args, **kw)
if hasattr(self, "curframe") and self.curframe and hasattr(self, "watch_list"):
for arg in self.watch_list:
try:
print >> self.stdout, "%s: %s"% (arg, self._getval(arg)) + ", ",
except:
pass
self.stdout.write("\n")
return result #func(self, *args, **kw)
return new_postcmd
pdb.Pdb.postcmd = wrap(pdb.Pdb.postcmd)
def do_disp(self, arg):
if not hasattr(self, "watch_list"):
self.watch_list = []
self.watch_list.append(arg)
pdb.Pdb.do_disp = do_disp
def do_undisp(self, arg):
if hasattr(self, "watch_list"):
try:
self.watch_list.remove(arg)
except:
pass
pdb.Pdb.do_undisp = do_undisp
patch_pdb()
if __name__ == "__main__":
# for testing
import pdb; pdb.set_trace()
a = 0
for i in range(10):
print i
a += 2
Unfortunately I could only make it display the state of the variables as they where previously to the execution of the last command. (I tried a little bit, but monkeypatching the bdb module, which is the base for the Pdb did not seem to work as well). You can try and change the methods in either pdb.Pdb, bdb.Bdb or cmd.Cmd that are decorated by wrap to find one that is called after the debugged frame state has changed.
You can set up some aliases that will do this for you:
alias n next;; p var
alias s step;; p var
Printing a whole list of variable names is left as an exercise to the reader. Unfortunately doing it this way means that when you send the debugger an empty line, the "last command" it executes is p var rather than, for example, n. If you want to fix that, then you can use this somewhat hacky set of Pdb commands instead:
!global __stack; from inspect import stack as __stack
!global __Pdb; from pdb import Pdb as __Pdb
!global __pdb; __pdb = [__framerec[0].f_locals.get("pdb") or __framerec[0].f_locals.get("self") for __framerec in __stack() if (__framerec[0].f_locals.get("pdb") or __framerec[0].f_locals.get("self")).__class__ == __Pdb][-1]
alias s step;; p var;; !__pdb.lastcmd = "!__pdb.cmdqueue.append('s')"
alias n next;; p var;; !__pdb.lastcmd = "!__pdb.cmdqueue.append('n')"
During the pdb debugging you can type normal python code, beyond the one letter commands - so just using print var should work for you.
Related
I am seeing a very weird behavior in which I can't step into or put breakpoint in some of the ElementTree classes. I started with the below code:
from xml.etree import ElementTree as ET
print(f"{ET.__file__}")
et = ET.parse("/tmp/pom.xml")
print(et)
I got the below output:
/usr/local/Cellar/python#3.9/3.9.1_3/Frameworks/Python.framework/Versions/3.9/lib/python3.9/xml/etree/ElementTree.py
<xml.etree.ElementTree.ElementTree object at 0x10fef9a30>
So, I opened the ElementTree.py file and put a breakpoint into class XMLParser.__init__ (here), but the breakpoint didn't get hit.
class XMLParser:
...
def __init__(self, *, target=None, encoding=None):
import pdb; pdb.set_trace()
try:
from xml.parsers import expat
except ImportError:
Then I added a breakpoint into the ElementTree.parse (here):
def parse(self, source, parser=None):
...
close_source = False
if not hasattr(source, "read"):
source = open(source, "rb")
close_source = True
try:
if parser is None:
# If no parser was specified, create a default XMLParser
import pdb; pdb.set_trace()
parser = XMLParser()
if hasattr(parser, '_parse_whole'):
I did get pdb prompt, but when I tried to step into XMLParser, it went straight to the next line. I even ensured that it is referring to the same local class (not some native implementation):
(Pdb) import inspect
(Pdb) inspect.getmodule(XMLParser)
<module 'xml.etree.ElementTree' from '/usr/local/Cellar/python#3.9/3.9.1_3/Frameworks/Python.framework/Versions/3.9/lib/python3.9/xml/etree/ElementTree.py'>
The reason I am doing this is to figure out why overridden _start and _end methods are not getting invoked for my custom class that extends XMLParser. I am instantiating the parser something like this (derived from here, with _start_list changed to _start):
class LineNumberingParser(ET.XMLParser):
def _start(self, *args, **kwargs):
element = super()._start(*args, **kwargs)
element._start_line_number = self.parser.CurrentLineNumber
print(f"----- {element.tag} {element._start_line_number}")
return element
def _end(self, *args, **kwargs):
element = super()._end(*args, **kwargs)
element._end_line_number = self.parser.CurrentLineNumber
print(f"----- {element.tag} {element._end_line_number}")
return element
parser = LineNumberingParser(target=ET.TreeBuilder(insert_comments=True))
et = ET.parse("/tmp/pom.xml", parser)
I even tried adding a constructor to LineNumberingParser and stepping into the super constructor, but I got the same behavior as before, though I can see that the self instance gets initialized properly (e.g., self.target is None before super.__init__ call but initialized after).
What am I missing here?
Update 1: I put some print statements in very obvious places in XMLParser (like __init__ and _start) and got no output, so it seems like it is using a different implementation though inspect.getmodule says otherwise.
Update 2: I just noticed the below at the end of the module:
# Import the C accelerators
try:
# Element is going to be shadowed by the C implementation. We need to keep
# the Python version of it accessible for some "creative" by external code
# (see tests)
_Element_Py = Element
# Element, SubElement, ParseError, TreeBuilder, XMLParser, _set_factories
from _elementtree import *
from _elementtree import _set_factories
except ImportError:
pass
else:
_set_factories(Comment, ProcessingInstruction)
I guess it indeed was a C native implementation and that is why pdb wasn't stepping in (the answer to my original question). Now I am back to square one to find a solution for line numbers.
Update 3: I found the code used in the test module to skip native module and _start and _end do get called with the python implementations, but there are some significant differences in the write code path.
Need a help with the next situation. I want to implement debug mode in my script through printing small completion report in functions with command executed name and ellapsed time like:
def cmd_exec(cmd):
if isDebug:
commandStart = datetime.datetime.now()
print commandStart
print cmd
...
... exucuting commands
...
if isDebug:
print datetime.datetime.now() - command_start
return
def main():
...
if args.debug:
isDebug = True
...
cmd_exec(cmd1)
...
cmd_exec(cmd2)
...
How can isDebug variable be simply passed to functions?
Should I use "global isDebug"?
Because
...
cmd_exec(cmd1, isDebug)
...
cmd_exec(cmd2, isDebug)
...
looks pretty bad. Please help me find more elegant way.
isDebug is state that applies to the application of a function cmd_exec. Sounds like a use-case for a class to me.
class CommandExecutor(object):
def __init__(self, debug):
self.debug = debug
def execute(self, cmd):
if self.debug:
commandStart = datetime.datetime.now()
print commandStart
print cmd
...
... executing commands
...
if self.debug:
print datetime.datetime.now() - command_start
def main(args):
ce = CommandExecutor(args.debug)
ce.execute(cmd1)
ce.execute(cmd2)
Python has a built-in __debug__ variable that could be useful.
if __debug__:
print 'information...'
When you run your program as python test.py, __debug__ is True. If you run it as python -O test.py, it will be False.
Another option which I do in my projects is set a global DEBUG var at the beginning of the file, after importing:
DEBUG = True
You can then reference this DEBUG var in the scope of the function.
You can use a module to create variables that are shared. This is better than a global because it only affects code that is specifically looking for the variable, it doesn't pollute the global namespace. It also lets you define something without your main module needing to know about it.
This works because modules are shared objects in Python. Every import gets back a reference to the same object, and modifications to the contents of that module get shared immediately, just like a global would.
my_debug.py:
isDebug = false
main.py:
import my_debug
def cmd_exec(cmd):
if my_debug.isDebug:
# ...
def main():
# ...
if args.debug:
my_debug.isDebug = True
Specifically for this, I would use partials/currying, basically pre-filling a variable.
import sys
from functools import partial
import datetime
def _cmd_exec(cmd, isDebug=False):
if isDebug:
command_start = datetime.datetime.now()
print command_start
print cmd
else:
print 'isDebug is false' + cmd
if isDebug:
print datetime.datetime.now() - command_start
return
#default, keeping it as is...
cmd_exec = _cmd_exec
#switch to debug
def debug_on():
global cmd_exec
#pre-apply the isDebug optional param
cmd_exec = partial(_cmd_exec, isDebug=True)
def main():
if "-d" in sys.argv:
debug_on()
cmd_exec("cmd1")
cmd_exec("cmd2")
main()
In this case, I check for -d on the command line to turn on debug mode and I do pre-populate isDebug on the function call by creating a new function with isDebug = True.
I think even other modules will see this modified cmd_exec, because I replaced the function at the module level.
output:
jluc#explore$ py test_so64.py
isDebug is falsecmd1
isDebug is falsecmd2
jluc#explore$ py test_so64.py -d
2016-10-13 17:00:33.523016
cmd1
0:00:00.000682
2016-10-13 17:00:33.523715
cmd2
0:00:00.000009
I have a file called sub.py, and I want to be able to call functions in it from the iPython shell. The iPython autoreload functionality has not been working very well, though. Sometimes it detects changes, sometimes it doesn't.
Instead of debugging autoreload, I was wondering if there's a way to just manually reload, or unload and load, modules in iPython. Currently I terminate the shell, start it again, re-import my module, and go from there. It would be great to be able to do a manual reload without killing the iPython shell.
I find my homebrewed %reimport to be very useful in this context:
def makemagic(f):
name = f.__name__
if name.startswith('magic_'): name = name[6:]
def wrapped(throwaway, *pargs, **kwargs): return f(*pargs,**kwargs)
if hasattr(f, '__doc__'): wrapped.__doc__ = f.__doc__
get_ipython().define_magic(name, wrapped)
return f
#makemagic
def magic_reimport(dd):
"""
The syntax
%reimport foo, bar.*
is a shortcut for the following:
import foo; foo = reload(foo)
import bar; bar = reload(bar); from bar import *
"""
ipython = get_ipython().user_ns
for d in dd.replace(',', ' ').split(' '):
if len(d):
bare = d.endswith('.*')
if bare: d = d[:-2]
exec('import xx; xx = reload(xx)'.replace('xx', d), ipython)
if bare: exec('from xx import *'.replace('xx', d), ipython)
Once gotcha is that, when there are sub-modules of packages involved, you have to reimport the sub-module, and then the top-level package:
reimport foo.bar, foo
In C++, I can print debug output like this:
printf(
"FILE: %s, FUNC: %s, LINE: %d, LOG: %s\n",
__FILE__,
__FUNCTION__,
__LINE__,
logmessage
);
How can I do something similar in Python?
There is a module named inspect which provides these information.
Example usage:
import inspect
def PrintFrame():
callerframerecord = inspect.stack()[1] # 0 represents this line
# 1 represents line at caller
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
print(info.filename) # __FILE__ -> Test.py
print(info.function) # __FUNCTION__ -> Main
print(info.lineno) # __LINE__ -> 13
def Main():
PrintFrame() # for this line
Main()
However, please remember that there is an easier way to obtain the name of the currently executing file:
print(__file__)
For example
import inspect
frame = inspect.currentframe()
# __FILE__
fileName = frame.f_code.co_filename
# __LINE__
fileNo = frame.f_lineno
There's more here http://docs.python.org/library/inspect.html
Building on geowar's answer:
class __LINE__(object):
import sys
def __repr__(self):
try:
raise Exception
except:
return str(sys.exc_info()[2].tb_frame.f_back.f_lineno)
__LINE__ = __LINE__()
If you normally want to use __LINE__ in e.g. print (or any other time an implicit str() or repr() is taken), the above will allow you to omit the ()s.
(Obvious extension to add a __call__ left as an exercise to the reader.)
You can refer my answer:
https://stackoverflow.com/a/45973480/1591700
import sys
print sys._getframe().f_lineno
You can also make lambda function
I was also interested in a __LINE__ command in python.
My starting point was https://stackoverflow.com/a/6811020 and I extended it with a metaclass object. With this modification it has the same behavior like in C++.
import inspect
class Meta(type):
def __repr__(self):
# Inspiration: https://stackoverflow.com/a/6811020
callerframerecord = inspect.stack()[1] # 0 represents this line
# 1 represents line at caller
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
# print(info.filename) # __FILE__ -> Test.py
# print(info.function) # __FUNCTION__ -> Main
# print(info.lineno) # __LINE__ -> 13
return str(info.lineno)
class __LINE__(metaclass=Meta):
pass
print(__LINE__) # print for example 18
wow, 7 year old question :)
Anyway, taking Tugrul's answer, and writing it as a debug type method, it can look something like:
def debug(message):
import sys
import inspect
callerframerecord = inspect.stack()[1]
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
print(info.filename, 'func=%s' % info.function, 'line=%s:' % info.lineno, message)
def somefunc():
debug('inside some func')
debug('this')
debug('is a')
debug('test message')
somefunc()
Output:
/tmp/test2.py func=<module> line=12: this
/tmp/test2.py func=<module> line=13: is a
/tmp/test2.py func=<module> line=14: test message
/tmp/test2.py func=somefunc line=10: inside some func
import inspect
.
.
.
def __LINE__():
try:
raise Exception
except:
return sys.exc_info()[2].tb_frame.f_back.f_lineno
def __FILE__():
return inspect.currentframe().f_code.co_filename
.
.
.
print "file: '%s', line: %d" % (__FILE__(), __LINE__())
Here is a tool to answer this old yet new question!
I recommend using icecream!
Do you ever use print() or log() to debug your code? Of course, you
do. IceCream, or ic for short, makes print debugging a little sweeter.
ic() is like print(), but better:
It prints both expressions/variable names and their values.
It's 40% faster to type.
Data structures are pretty printed.
Output is syntax highlighted.
It optionally includes program context: filename, line number, and parent function.
For example, I created a module icecream_test.py, and put the following code inside it.
from icecream import ic
ic.configureOutput(includeContext=True)
def foo(i):
return i + 333
ic(foo(123))
Prints
ic| icecream_test.py:6 in <module>- foo(123): 456
To get the line number in Python without importing the whole sys module...
First import the _getframe submodule:
from sys import _getframe
Then call the _getframe function and use its' f_lineno property whenever you want to know the line number:
print(_getframe().f_lineno) # prints the line number
From the interpreter:
>>> from sys import _getframe
... _getframe().f_lineno # 2
Word of caution from the official Python Docs:
CPython implementation detail: This function should be used for internal and specialized purposes only. It is not guaranteed to exist in all implementations of Python.
In other words: Only use this code for personal testing / debugging reasons.
See the Official Python Documentation on sys._getframe for more information on the sys module, and the _getframe() function / submodule.
Based on Mohammad Shahid's answer (above).
I decided to try to preprocess function text before it's compilation into byte-code and following execution. This is merely for training. I hardly imagine situations where it'll be a satisfactory solution to be used. I have faced one problem which I wanted to solve in this way, but eventually a better way was found. So this is just for training and to learn something new, not for real usage.
Assume we have a function, which source code we want to be modified quite a bit before compilation:
def f():
1;a()
print('Some statements 1')
1;a()
print('Some statements 2')
Let, for example, mark some lines of it with 1;, for them to be sometimes commented and sometimes not. I just take it for example, modifications of the function may be different.
To comment these lines I made a decorator. The whole code it bellow:
from __future__ import print_function
def a():
print('a()')
def comment_1(s):
lines = s.split('\n')
return '\n'.join(line.replace(';','#;',1) if line.strip().startswith('1;') else line for line in lines)
def remove_1(f):
import inspect
source = inspect.getsource(f)
new_source = comment_1(source)
with open('temp.py','w') as file:
file.write(new_source)
from temp import f as f_new
return f_new
def f():
1;a()
print('Some statements 1')
1;a()
print('Some statements 2')
f = remove_1(f) #If decorator #remove is used above f(), inspect.getsource includes #remove inside the code.
f()
I used inspect.getsourcelines to retrieve function f code. Then I made some text-processing (in this case commenting lines starting with 1;). After that I saved it to temp.py module, which is then imported. And then a function f is decorated in the main module.
The output, when decorator is applied, is this:
Some statements 1
Some statements 2
when NOT applied is this:
a()
Some statements 1
a()
Some statements 2
What I don't like is that I have to use hard drive to load compiled function. Can it be done without writing it to temporary module temp.py and importing from it?
The second question is about placing decorator above f: #replace. When I do this, inspect.getsourcelines returns f text with this decorator. I could manually be deleted from f's text. but that would be quite dangerous, as there may be more than one decorator applied. So I resorted to the old-style decoration syntax f = remove_1(f), which does the job. But still, is it possible to allow normal decoration technique with #replace?
One can avoid creating a temporary file by invoking the exec statement on the source. (You can also explicitly call compile prior to exec if you want additional control over compilation, but exec will do the compilation for you, so it's not necessary.) Correctly calling exec has the additional benefit that the function will work correctly if it accesses global variables from the namespace of its module.
The problem described in the second question can be resolved by temporarily blocking the decorator while it is running. That way the decorator remains, along all the other ones, but is a no-op.
Here is the updated source.
from __future__ import print_function
import sys
def a():
print('a()')
def comment_1(s):
lines = s.split('\n')
return '\n'.join(line.replace(';','#;',1) if line.strip().startswith('1;') else line for line in lines)
_blocked = False
def remove_1(f):
global _blocked
if _blocked:
return f
import inspect
source = inspect.getsource(f)
new_source = comment_1(source)
env = sys.modules[f.__module__].__dict__
_blocked = True
try:
exec new_source in env
finally:
_blocked = False
return env[f.__name__]
#remove_1
def f():
1;a()
print('Some statements 1')
1;a()
print('Some statements 2')
f()
def remove_1(f):
import inspect
source = inspect.getsource(f)
new_source = comment_1(source)
env = sys.modules[f.__module__].__dict__.copy()
exec new_source in env
return env[f.__name__]
I'll leave a modified version of the solution given in the answer by user4815162342. It uses ast module to delete some parts of f, as was suggested in the comment to the question. To make it I majorly relied on the information in this article.
This implementation deletes all occurrences of a as standalone expression.
from __future__ import print_function
import sys
import ast
import inspect
def a():
print('a() is called')
_blocked = False
def remove_1(f):
global _blocked
if _blocked:
return f
import inspect
source = inspect.getsource(f)
a = ast.parse(source) #get ast tree of f
class Transformer(ast.NodeTransformer):
'''Will delete all expressions containing 'a' functions at the top level'''
def visit_Expr(self, node): #visit all expressions
try:
if node.value.func.id == 'a': #if expression consists of function with name a
return None #delete it
except(ValueError):
pass
return node #return node unchanged
transformer = Transformer()
a_new = transformer.visit(a)
f_new_compiled = compile(a_new,'<string>','exec')
env = sys.modules[f.__module__].__dict__
_blocked = True
try:
exec(f_new_compiled,env)
finally:
_blocked = False
return env[f.__name__]
#remove_1
def f():
a();a()
print('Some statements 1')
a()
print('Some statements 2')
f()
The output is:
Some statements 1
Some statements 2