Collect python source code comments along execution path - python

E.g. I've got the following python function:
def func(x):
"""Function docstring."""
result = x + 1
if result > 0:
# comment 2
return result
else:
# comment 3
return -1 * result
And I want to have some function that would print all function docstrings and comments that are met along the execution path, e.g.
> trace(func(2))
Function docstring.
Comment 2
3
In fact what I try to achieve is to provide some comments how the result has been calculated.
What could be used? AST as far as I understand does not keep comment in the tree.

I thought this was an interesting challenge, so I decided to give it a try. Here is what I came up with:
import ast
import inspect
import re
import sys
import __future__
if sys.version_info >= (3,5):
ast_Call = ast.Call
else:
def ast_Call(func, args, keywords):
"""Compatibility wrapper for ast.Call on Python 3.4 and below.
Used to have two additional fields (starargs, kwargs)."""
return ast.Call(func, args, keywords, None, None)
COMMENT_RE = re.compile(r'^(\s*)#\s?(.*)$')
def convert_comment_to_print(line):
"""If `line` contains a comment, it is changed into a print
statement, otherwise nothing happens. Only acts on full-line comments,
not on trailing comments. Returns the (possibly modified) line."""
match = COMMENT_RE.match(line)
if match:
return '{}print({!r})\n'.format(*match.groups())
else:
return line
def convert_docstrings_to_prints(syntax_tree):
"""Walks an AST and changes every docstring (i.e. every expression
statement consisting only of a string) to a print statement.
The AST is modified in-place."""
ast_print = ast.Name('print', ast.Load())
nodes = list(ast.walk(syntax_tree))
for node in nodes:
for bodylike_field in ('body', 'orelse', 'finalbody'):
if hasattr(node, bodylike_field):
for statement in getattr(node, bodylike_field):
if (isinstance(statement, ast.Expr) and
isinstance(statement.value, ast.Str)):
arg = statement.value
statement.value = ast_Call(ast_print, [arg], [])
def get_future_flags(module_or_func):
"""Get the compile flags corresponding to the features imported from
__future__ by the specified module, or by the module containing the
specific function. Returns a single integer containing the bitwise OR
of all the flags that were found."""
result = 0
for feature_name in __future__.all_feature_names:
feature = getattr(__future__, feature_name)
if (hasattr(module_or_func, feature_name) and
getattr(module_or_func, feature_name) is feature and
hasattr(feature, 'compiler_flag')):
result |= feature.compiler_flag
return result
def eval_function(syntax_tree, func_globals, filename, lineno, compile_flags,
*args, **kwargs):
"""Helper function for `trace`. Execute the function defined by
the given syntax tree, and return its return value."""
func = syntax_tree.body[0]
func.decorator_list.insert(0, ast.Name('_trace_exec_decorator', ast.Load()))
ast.increment_lineno(syntax_tree, lineno-1)
ast.fix_missing_locations(syntax_tree)
code = compile(syntax_tree, filename, 'exec', compile_flags, True)
result = [None]
def _trace_exec_decorator(compiled_func):
result[0] = compiled_func(*args, **kwargs)
func_locals = {'_trace_exec_decorator': _trace_exec_decorator}
exec(code, func_globals, func_locals)
return result[0]
def trace(func, *args, **kwargs):
"""Run the given function with the given arguments and keyword arguments,
and whenever a docstring or (whole-line) comment is encountered,
print it to stdout."""
filename = inspect.getsourcefile(func)
lines, lineno = inspect.getsourcelines(func)
lines = map(convert_comment_to_print, lines)
modified_source = ''.join(lines)
compile_flags = get_future_flags(func)
syntax_tree = compile(modified_source, filename, 'exec',
ast.PyCF_ONLY_AST | compile_flags, True)
convert_docstrings_to_prints(syntax_tree)
return eval_function(syntax_tree, func.__globals__,
filename, lineno, compile_flags, *args, **kwargs)
It is a bit long because I tried to cover most important cases, and the code might not be the most readable, but I hope it is nice enough to follow.
How it works:
First, read the function's source code using inspect.getsourcelines. (Warning: inspect does not work for functions that were defined interactively. If you need that, maybe you can use dill instead, see this answer.)
Search for lines that look like comments, and replace them with print statements. (Right now only whole-line comments are replaced, but it shouldn't be difficult to extend that to trailing comments if desired.)
Parse the source code into an AST.
Walk the AST and replace all docstrings with print statements.
Compile the AST.
Execute the AST. This and the previous step contain some trickery to try to reconstruct the context that the function was originally defined in (e.g. globals, __future__ imports, line numbers for exception tracebacks). Also, since just executing the source would only re-define the function and not call it, we fix that with a simple decorator.
It works in Python 2 and 3 (at least with the tests below, which I ran in 2.7 and 3.6).
To use it, simply do:
result = trace(func, 2) # result = func(2)
Here is a slightly more elaborate test that I used while writing the code:
#!/usr/bin/env python
from trace_comments import trace
from dateutil.easter import easter, EASTER_ORTHODOX
def func(x):
"""Function docstring."""
result = x + 1
if result > 0:
# comment 2
return result
else:
# comment 3
return -1 * result
if __name__ == '__main__':
result1 = trace(func, 2)
print("result1 = {}".format(result1))
result2 = trace(func, -10)
print("result2 = {}".format(result2))
# Test that trace() does not permanently replace the function
result3 = func(42)
print("result3 = {}".format(result3))
print("-----")
print(trace(easter, 2018))
print("-----")
print(trace(easter, 2018, EASTER_ORTHODOX))

Related

How to know the name of a classs loade like parameter on other class - Pyhton [duplicate]

This question already has answers here:
Getting the name of a variable as a string
(32 answers)
Closed 4 months ago.
Is it possible to get the original variable name of a variable passed to a function? E.g.
foobar = "foo"
def func(var):
print var.origname
So that:
func(foobar)
Returns:
>>foobar
EDIT:
All I was trying to do was make a function like:
def log(soup):
f = open(varname+'.html', 'w')
print >>f, soup.prettify()
f.close()
.. and have the function generate the filename from the name of the variable passed to it.
I suppose if it's not possible I'll just have to pass the variable and the variable's name as a string each time.
EDIT: To make it clear, I don't recommend using this AT ALL, it will break, it's a mess, it won't help you in any way, but it's doable for entertainment/education purposes.
You can hack around with the inspect module, I don't recommend that, but you can do it...
import inspect
def foo(a, f, b):
frame = inspect.currentframe()
frame = inspect.getouterframes(frame)[1]
string = inspect.getframeinfo(frame[0]).code_context[0].strip()
args = string[string.find('(') + 1:-1].split(',')
names = []
for i in args:
if i.find('=') != -1:
names.append(i.split('=')[1].strip())
else:
names.append(i)
print names
def main():
e = 1
c = 2
foo(e, 1000, b = c)
main()
Output:
['e', '1000', 'c']
To add to Michael Mrozek's answer, you can extract the exact parameters versus the full code by:
import re
import traceback
def func(var):
stack = traceback.extract_stack()
filename, lineno, function_name, code = stack[-2]
vars_name = re.compile(r'\((.*?)\).*$').search(code).groups()[0]
print vars_name
return
foobar = "foo"
func(foobar)
# PRINTS: foobar
Looks like Ivo beat me to inspect, but here's another implementation:
import inspect
def varName(var):
lcls = inspect.stack()[2][0].f_locals
for name in lcls:
if id(var) == id(lcls[name]):
return name
return None
def foo(x=None):
lcl='not me'
return varName(x)
def bar():
lcl = 'hi'
return foo(lcl)
bar()
# 'lcl'
Of course, it can be fooled:
def baz():
lcl = 'hi'
x='hi'
return foo(lcl)
baz()
# 'x'
Moral: don't do it.
Another way you can try if you know what the calling code will look like is to use traceback:
def func(var):
stack = traceback.extract_stack()
filename, lineno, function_name, code = stack[-2]
code will contain the line of code that was used to call func (in your example, it would be the string func(foobar)). You can parse that to pull out the argument
You can't. It's evaluated before being passed to the function. All you can do is pass it as a string.
#Ivo Wetzel's answer works in the case of function call are made in one line, like
e = 1 + 7
c = 3
foo(e, 100, b=c)
In case that function call is not in one line, like:
e = 1 + 7
c = 3
foo(e,
1000,
b = c)
below code works:
import inspect, ast
def foo(a, f, b):
frame = inspect.currentframe()
frame = inspect.getouterframes(frame)[1]
string = inspect.findsource(frame[0])[0]
nodes = ast.parse(''.join(string))
i_expr = -1
for (i, node) in enumerate(nodes.body):
if hasattr(node, 'value') and isinstance(node.value, ast.Call)
and hasattr(node.value.func, 'id') and node.value.func.id == 'foo' # Here goes name of the function:
i_expr = i
break
i_expr_next = min(i_expr + 1, len(nodes.body)-1)
lineno_start = nodes.body[i_expr].lineno
lineno_end = nodes.body[i_expr_next].lineno if i_expr_next != i_expr else len(string)
str_func_call = ''.join([i.strip() for i in string[lineno_start - 1: lineno_end]])
params = str_func_call[str_func_call.find('(') + 1:-1].split(',')
print(params)
You will get:
[u'e', u'1000', u'b = c']
But still, this might break.
You can use python-varname package
from varname import nameof
s = 'Hey!'
print (nameof(s))
Output:
s
Package below:
https://github.com/pwwang/python-varname
For posterity, here's some code I wrote for this task, in general I think there is a missing module in Python to give everyone nice and robust inspection of the caller environment. Similar to what rlang eval framework provides for R.
import re, inspect, ast
#Convoluted frame stack walk and source scrape to get what the calling statement to a function looked like.
#Specifically return the name of the variable passed as parameter found at position pos in the parameter list.
def _caller_param_name(pos):
#The parameter name to return
param = None
#Get the frame object for this function call
thisframe = inspect.currentframe()
try:
#Get the parent calling frames details
frames = inspect.getouterframes(thisframe)
#Function this function was just called from that we wish to find the calling parameter name for
function = frames[1][3]
#Get all the details of where the calling statement was
frame,filename,line_number,function_name,source,source_index = frames[2]
#Read in the source file in the parent calling frame upto where the call was made
with open(filename) as source_file:
head=[source_file.next() for x in xrange(line_number)]
source_file.close()
#Build all lines of the calling statement, this deals with when a function is called with parameters listed on each line
lines = []
#Compile a regex for matching the start of the function being called
regex = re.compile(r'\.?\s*%s\s*\(' % (function))
#Work backwards from the parent calling frame line number until we see the start of the calling statement (usually the same line!!!)
for line in reversed(head):
lines.append(line.strip())
if re.search(regex, line):
break
#Put the lines we have groked back into sourcefile order rather than reverse order
lines.reverse()
#Join all the lines that were part of the calling statement
call = "".join(lines)
#Grab the parameter list from the calling statement for the function we were called from
match = re.search('\.?\s*%s\s*\((.*)\)' % (function), call)
paramlist = match.group(1)
#If the function was called with no parameters raise an exception
if paramlist == "":
raise LookupError("Function called with no parameters.")
#Use the Python abstract syntax tree parser to create a parsed form of the function parameter list 'Name' nodes are variable names
parameter = ast.parse(paramlist).body[0].value
#If there were multiple parameters get the positional requested
if type(parameter).__name__ == 'Tuple':
#If we asked for a parameter outside of what was passed complain
if pos >= len(parameter.elts):
raise LookupError("The function call did not have a parameter at postion %s" % pos)
parameter = parameter.elts[pos]
#If there was only a single parameter and another was requested raise an exception
elif pos != 0:
raise LookupError("There was only a single calling parameter found. Parameter indices start at 0.")
#If the parameter was the name of a variable we can use it otherwise pass back None
if type(parameter).__name__ == 'Name':
param = parameter.id
finally:
#Remove the frame reference to prevent cyclic references screwing the garbage collector
del thisframe
#Return the parameter name we found
return param
If you want a Key Value Pair relationship, maybe using a Dictionary would be better?
...or if you're trying to create some auto-documentation from your code, perhaps something like Doxygen (http://www.doxygen.nl/) could do the job for you?
I wondered how IceCream solves this problem. So I looked into the source code and came up with the following (slightly simplified) solution. It might not be 100% bullet-proof (e.g. I dropped get_text_with_indentation and I assume exactly one function argument), but it works well for different test cases. It does not need to parse source code itself, so it should be more robust and simpler than previous solutions.
#!/usr/bin/env python3
import inspect
from executing import Source
def func(var):
callFrame = inspect.currentframe().f_back
callNode = Source.executing(callFrame).node
source = Source.for_frame(callFrame)
expression = source.asttokens().get_text(callNode.args[0])
print(expression, '=', var)
i = 1
f = 2.0
dct = {'key': 'value'}
obj = type('', (), {'value': 42})
func(i)
func(f)
func(s)
func(dct['key'])
func(obj.value)
Output:
i = 1
f = 2.0
s = string
dct['key'] = value
obj.value = 42
Update: If you want to move the "magic" into a separate function, you simply have to go one frame further back with an additional f_back.
def get_name_of_argument():
callFrame = inspect.currentframe().f_back.f_back
callNode = Source.executing(callFrame).node
source = Source.for_frame(callFrame)
return source.asttokens().get_text(callNode.args[0])
def func(var):
print(get_name_of_argument(), '=', var)
If you want to get the caller params as in #Matt Oates answer answer without using the source file (ie from Jupyter Notebook), this code (combined from #Aeon answer) will do the trick (at least in some simple cases):
def get_caller_params():
# get the frame object for this function call
thisframe = inspect.currentframe()
# get the parent calling frames details
frames = inspect.getouterframes(thisframe)
# frame 0 is the frame of this function
# frame 1 is the frame of the caller function (the one we want to inspect)
# frame 2 is the frame of the code that calls the caller
caller_function_name = frames[1][3]
code_that_calls_caller = inspect.findsource(frames[2][0])[0]
# parse code to get nodes of abstract syntact tree of the call
nodes = ast.parse(''.join(code_that_calls_caller))
# find the node that calls the function
i_expr = -1
for (i, node) in enumerate(nodes.body):
if _node_is_our_function_call(node, caller_function_name):
i_expr = i
break
# line with the call start
idx_start = nodes.body[i_expr].lineno - 1
# line with the end of the call
if i_expr < len(nodes.body) - 1:
# next expression marks the end of the call
idx_end = nodes.body[i_expr + 1].lineno - 1
else:
# end of the source marks the end of the call
idx_end = len(code_that_calls_caller)
call_lines = code_that_calls_caller[idx_start:idx_end]
str_func_call = ''.join([line.strip() for line in call_lines])
str_call_params = str_func_call[str_func_call.find('(') + 1:-1]
params = [p.strip() for p in str_call_params.split(',')]
return params
def _node_is_our_function_call(node, our_function_name):
node_is_call = hasattr(node, 'value') and isinstance(node.value, ast.Call)
if not node_is_call:
return False
function_name_correct = hasattr(node.value.func, 'id') and node.value.func.id == our_function_name
return function_name_correct
You can then run it as this:
def test(*par_values):
par_names = get_caller_params()
for name, val in zip(par_names, par_values):
print(name, val)
a = 1
b = 2
string = 'text'
test(a, b,
string
)
to get the desired output:
a 1
b 2
string text
Since you can have multiple variables with the same content, instead of passing the variable (content), it might be safer (and will be simpler) to pass it's name in a string and get the variable content from the locals dictionary in the callers stack frame. :
def displayvar(name):
import sys
return name+" = "+repr(sys._getframe(1).f_locals[name])
If it just so happens that the variable is a callable (function), it will have a __name__ property.
E.g. a wrapper to log the execution time of a function:
def time_it(func, *args, **kwargs):
start = perf_counter()
result = func(*args, **kwargs)
duration = perf_counter() - start
print(f'{func.__name__} ran in {duration * 1000}ms')
return result

Is it possible to get the source code of a (possibly decorated) Python function body, including inline comments? [duplicate]

This question already has answers here:
How can I get the source code of a Python function?
(13 answers)
Closed 1 year ago.
I am trying to figure out how to only get the source code of the body of the function.
Let's say I have:
def simple_function(b = 5):
a = 5
print("here")
return a + b
I would want to get (up to indentation):
"""
a = 5
print("here")
return a + b
"""
While it's easy in the case above, I want it to be agnostic of decorators/function headers, etc. However, still include inline comments. So for example:
#decorator1
#decorator2
def simple_function(b: int = 5):
""" Very sophisticated docs
"""
a = 5
# Comment on top
print("here") # And in line
return a + b
Would result in:
"""
a = 5
# Comment on top
print("here") # And in line
return a + b
"""
I was not able to find any utility and have been trying to play with inspect.getsourcelines for few hours now, but with no luck.
Any help appreciated!
Why is it different from How can I get the source code of a Python function?
This question asks for a whole function source code, which includes both decorators, docs, def, and body itself. I'm interested in only the body of the function.
I wrote a simple regex that does the trick. I tried this script with classes and without. It seemed to work fine either way. It just opens whatever file you designate in the Main call, at the bottom, rewrites the entire document with all function/method bodies doc-stringed and then save it as whatever you designated as the second argument in the Main call.
It's not beautiful, and it could probably have more efficient regex statements. It works though. The regex finds everything from a decorator (if one) to the end of a function/method, grouping tabs and the function/method body. It then uses those groups in finditer to construct a docstring and place it before the entire chunk it found.
import re
FUNC_BODY = re.compile(r'^((([ \t]+)?#.+\n)+)?(?P<tabs>[\t ]+)?def([^\n]+)\n(?P<body>(^([\t ]+)?([^\n]+)\n)+)', re.M)
BLANK_LINES = re.compile(r'^[ \t]+$', re.M)
class Main(object):
def __init__(self, file_in:str, file_out:str) -> None:
#prime in/out strings
in_txt = ''
out_txt = ''
#open resuested file
with open(file_in, 'r') as f:
in_txt = f.read()
#remove all lines that just have space characters on them
#this stops FUNC_BODY from finding the entire file in one shot
in_txt = BLANK_LINES.sub('', in_txt)
last = 0 #to keep track of where we are in the file
#process all matches
for m in FUNC_BODY.finditer(in_txt):
s, e = m.span()
#make sure we catch anything that was between our last match and this one
out_txt = f"{out_txt}{in_txt[last:s]}"
last = e
tabs = m.group('tabs') if not m.group('tabs') is None else ''
#construct the docstring and inject it before the found function/method
out_txt = f"{out_txt}{tabs}'''\n{m.group('body')}{tabs}'''\n{m.group()}"
#save as requested file name
with open(file_out, 'w') as f:
f.write(out_txt)
if __name__ == '__main__':
Main('test.py', 'test_docd.py')
EDIT:
Apparently, I "missed the entire point" so I wrote it again a different way. Now you can get the body while the code is running and decorators don't matter, at all. I left my other answer here because it is also a solution, just not a "real time" one.
import re, inspect
FUNC_BODY = re.compile('^(?P<tabs>[\t ]+)?def (?P<name>[a-zA-Z0-9_]+)([^\n]+)\n(?P<body>(^([\t ]+)?([^\n]+)\n)+)', re.M)
class Source(object):
#staticmethod
def investigate(focus:object, strfocus:str) -> str:
with open(inspect.getsourcefile(focus), 'r') as f:
for m in FUNC_BODY.finditer(f.read()):
if m.group('name') == strfocus:
tabs = m.group('tabs') if not m.group('tabs') is None else ''
return f"{tabs}'''\n{m.group('body')}{tabs}'''"
def decorator(func):
def inner():
print("I'm decorated")
func()
return inner
#decorator
def test():
a = 5
b = 6
return a+b
print(Source.investigate(test, 'test'))

Python: monkey patch a function's source code

Can I add a prefix and suffix to the source code of functions?
I know about decorators and do not want to use them (the minimal example below doesn't make clear why, but I have my reasons).
def f():
print('world')
g = patched(f,prefix='print("Hello, ");',suffix='print("!");')
g() # Hello, world!
Here is what I have so far:
import inspect
import ast
import copy
def patched(f,prefix,suffix):
source = inspect.getsource(f)
tree = ast.parse(source)
new_body = [
ast.parse(prefix).body[0],
*tree.body[0].body,
ast.parse(suffix).body[0]
]
tree.body[0].body = new_body
g = copy.deepcopy(f)
g.__code__ = compile(tree,g.__code__.co_filename,'exec')
return g
Unfortunately, nothing happens if I use this and then call g() as above; neither world nor Hello, world! are printed.
Here is a rough version of what can be done:
import inspect
import ast
import copy
def patched(f,prefix,suffix):
source = inspect.getsource(f)
tree = ast.parse(source)
new_body = [
ast.parse(prefix).body[0],
*tree.body[0].body,
ast.parse(suffix).body[0]
]
tree.body[0].body = new_body
code = compile(tree,filename=f.__code__.co_filename,mode='exec')
namespace = {}
exec(code,namespace)
g = namespace[f.__name__]
return g
def temp():
pass
def f():
print('world',end='')
g = patched(f,prefix='print("Hello, ",end="")',suffix='print("!",end="")')
g() # Hello, world!
The call of compile compiles an entire module (represented by tree). This module is then executed in an empty namespace from which the desired function is finally extracted. (Warning: the namespace will need to be filled with some globals from where f comes from if f uses those.)
After some more work, here is a real example of what can be done with this. It uses some extended version of the principle above:
import numpy as np
from playground import graphexecute
#graphexecute(verbose=True)
def my_algorithm(x,y,z):
def SumFirstArguments(x,y)->sumxy:
sumxy = x+y
def SinOfThird(z)->sinz:
sinz = np.sin(z)
def FinalProduct(sumxy,sinz)->prod:
prod = sumxy*sinz
def Return(prod):
return prod
print(my_algorithm(x=1,y=2,z=3))
#OUTPUT:
#>>Executing part SumFirstArguments
#>>Executing part SinOfThird
#>>Executing part FinalProduct
#>>Executing part Return
#>>0.4233600241796016
The clou is that I get the exact same output if I reshuffle the parts of my_algorithm, for example like this:
#graphexecute(verbose=True)
def my_algorithm2(x,y,z):
def FinalProduct(sumxy,sinz)->prod:
prod = sumxy*sinz
def SumFirstArguments(x,y)->sumxy:
sumxy = x+y
def SinOfThird(z)->sinz:
sinz = np.sin(z)
def Return(prod):
return prod
print(my_algorithm2(x=1,y=2,z=3))
#OUTPUT:
#>>Executing part SumFirstArguments
#>>Executing part SinOfThird
#>>Executing part FinalProduct
#>>Executing part Return
#>>0.4233600241796016
This works by (1) grabbing the source of my_algorithm and turning it into an ast (2) patching each function defined within my_algorithm (e.g. SumFirstArguments) to return locals (3) deciding based on the inputs and the outputs (as defined by the type hints) in which order the parts of my_algorithm should be executed. Furthermore, a possibility that I do not have implemented yet is to execute independent parts in parallel (such as SumFirstArguments and SinOfThird). Let me know if you want the sourcecode of graphexecute, I haven't included it here because it contains a lot of stuff that is not relevant to this question.
For your problem, you don't need to recompile your functions. Just define a list of functions, you inspect for arguments and return variable name:
def FinalProduct(sumxy, sinz) -> "prod":
return sumxy * sinz
def SumFirstArguments(x, y) -> "sumxy":
return x + y
def SinOfThird(z) -> "sinz":
return np.sin(z)
def execute(funcs, **args):
result = None
while funcs:
func = funcs.pop(0)
try:
kw = {a: args[a]
for a in func.__code__.co_varnames[:func.__code__.co_argcount]
}
except KeyError:
# not all arguments found
funcs.append(func)
else:
print(func,kw)
result = func(**kw)
args[func.__annotations__['return']] = result
return result
print(execute([FinalProduct, SumFirstArguments, SinOfThird], x=1,y=2,z=3))

Get value of last expression in `exec` call

Let's say I have some python code in a string
code = """
a = 42
a
"""
and I exec that string of code:
result = exec(code)
Then result will always be None. Is there any way at all to get the value of the last expression evaluated? In this case, that would be 5, since a was the last expression.
EDIT: Here's another example of the functionality I'm asking about. Let's say we have the python code (stored in the variable code)
a = 100
sqrt(a)
Then how can I execute the code in such a way as to give me the result 10 - that is, sqrt(a)?
EDIT EDIT: A further example: the code I wish to exec is
function_a()
function_b()
function_c()
Is there any way I can define some kind of magic_exec function so that
magic_exec(code)
will provide me with the value of function_c()?
The request is certainly valid because I need such a function as well during the creation of a Python-based environment. I solved the problem with the following code that utilizes the Python ast mechanism:
def my_exec(script, globals=None, locals=None):
'''Execute a script and return the value of the last expression'''
stmts = list(ast.iter_child_nodes(ast.parse(script)))
if not stmts:
return None
if isinstance(stmts[-1], ast.Expr):
# the last one is an expression and we will try to return the results
# so we first execute the previous statements
if len(stmts) > 1:
exec(compile(ast.Module(body=stmts[:-1]), filename="<ast>", mode="exec"), globals, locals)
# then we eval the last one
return eval(compile(ast.Expression(body=stmts[-1].value), filename="<ast>", mode="eval"), globals, locals)
else:
# otherwise we just execute the entire code
return exec(script, globals, locals)
The code should be pretty self-explanatory, basically it
separate the script into multiple statements
if the last one is an expression, execute the first part as statements, and the last part as expression.
Otherwise execute the entire script as statements.
This doesn't get you the last evaluated value, but gets the whole list of local variables.
>>> loc = {}
>>> exec(code, {}, loc)
>>> loc
{'a': 42}
exec('a = 4')
print a % prints 4
>>> code = """
... a = 42
... b = 53"""
>>> exec(code)
>>> a
42
>>> b
53
Or if you're saying you don't know the last thing is b for instance, then you can have this:
code = """
a = 4
b = 12
abc_d=13
"""
t = re.findall(r'''.*?([A-Za-z0-9_]+)\s*?=.*?$''', code)
assert(len(t)==1)
print t[0] % prints 13
To be honest I can't say I'm very happy with this. It feels very hacky and I haven't tested it all that heavily. On the other hand I'm quite pleased with it. Was quite fun to do. Anyway, hope this helps you or at least comes close to what you want. locals() gives a dict so the output list order does not match the input order for the items that failed the first eval. If you don't want ';' as delimiters then you can change it to '\n'.
import math
def magic_exec(_command):
_command = _command.split(';')
_result = None
_before = list(locals()) # Get list of current local variables
for _code in _command:
_code = _code.strip() # .strip() prevent IndentationError
try:
if eval(_code) != None: # For functions with no return
_result = eval(_code)
except (NameError, SyntaxError):
try:
_before = list(locals())
exec(_code)
except NameError as e: # For undefined variables in _command
print("An Error Occurred with line ' {0} ' as was skipped: {1}".format(_code, e))
del _code # delete temp var _code
# Get new list of locals that didn't exist at the start
_after = [val for val in list(locals()) if val not in _before]
if _after:
return eval(_after[0])
else:
return _result
#Dummy class and functions
class Class1(object):
def __init__(self, x):
self._x = x
def get_val(self):
return self._x
def __repr__(self):
return type(self).__name__
def func1(x):
return x + x
def func2(x):
print(x*x)
if __name__ == '__main__':
code = \
"""
a = 42; a; v; y = 2; b = func1(5); s = 'Hello'; func2(10); c = 25; l = []; l.append('Value');
t = math.sqrt(c); pass; 20*10; print('TEST'); math.sqrt(c); d = Class1('World'); d.get_val();
def func3(x): return x ** 2; s = func3(15)
"""
values = magic_exec(code)
print(values)
I would like to add to user2283347's excellent answer that it works only up to Python 3.7. In Python 3.8 the signature of ast.Module.__init__ has changed. It now requires a second argument which in our case can be an empty list.
Details: ast.Module(body=stmts[:-1]) in
if len(stmts) > 1:
exec(compile(ast.Module(body=stmts[:-1]), filename="<ast>", mode="exec"), globals, locals)
has to be changed to
ast.Module(stmts[:-1], []) if you use Python 3.8 or above (note the second argument []). Otherwise the following TypeError will be raised:
TypeError: required field "type_ignores" missing from Module
Unfortunately this change is not very well documented. I found the solution after extensive Googling here: "IPython broken on 3.8-dev" .

How to change this tree-recursion to a tail-recursion?

I am writing a function ChrNumber that converts Arab number string to Chinese financial number string. I work out a tree recursion form. But when I tried to get a tail-recursion form, it is really difficult for me to handle the situation bit equals 6,7 or 8 or 10 and bigger ones.
You can see how it works at the end of my question.
Here's the tree-recursion solution. It works:
# -*- coding:utf-8 -*-
unitArab=(2,3,4,5,9)
#unitStr=u'十百千万亿' #this is an alternative
unitStr=u'拾佰仟万亿'
unitDic=dict(zip(unitArab,(list(unitStr))))
numArab=list(u'0123456789')
#numStr=u'零一二三四五六七八九' #this is an alternative
numStr=u'零壹贰叁肆伍陆柒捌玖'
numDic=dict(zip(numArab,list(numStr)))
def ChnNumber(s):
def wrapper(v):
'this is to adapt the string to a abbreviation'
if u'零零' in v:
return wrapper(v.replace(u'零零',u'零'))
return v[:-1] if v[-1]==u'零' else v
def recur(s,bit):
'receives the number sting and its length'
if bit==1:
return numDic[s]
if s[0]==u'0':
return wrapper(u'%s%s' % (u'零',recur(s[1:],bit-1)))
if bit<6 or bit==9:
return wrapper(u'%s%s%s' % (numDic[s[0]],unitDic[bit],recur(s[1:],bit-1)))
'below is the hard part to be converted to tail-recurion'
if bit<9:
return u'%s%s%s' % (recur(s[:-4],bit-4),u"万",recur(s[-4:],4))
if bit>9:
return u'%s%s%s' % (recur(s[:-8],bit-8),u"亿",recur(s[-8:],8))
return recur(s,len(s))
My attempt version is only in recur function, I use a closure res and move the bit inside the recur so there is less arguments.:
res=[]
def recur(s):
bit=len(s)
print s,bit,res
if bit==0:
return ''.join(res)
if bit==1:
res.append(numDic[s])
return recur(s[1:])
if s[0]==u'0':
res.append(u'零')
return recur(s[1:])
if bit<6 or bit==9:
res.append(u'%s%s' %(numDic[s[0]],unitDic[bit]))
return recur(s[1:])
if bit<9:
#...can't work it out
if bit>9:
#...can't work it out
the test code is:
for i in range(17):
v1='9'+'0'*(i+1)
v2='9'+'0'*i+'9'
v3='1'*(i+2)
print '%s->%s\n%s->%s\n%s->%s'% (v1,ChnNumber(v1),v2,ChnNumber(v2),v3,ChnNumber(v3))
which should output:
>>>
90->玖拾
99->玖拾玖
11->壹拾壹
900->玖佰
909->玖佰零玖
111->壹佰壹拾壹
9000->玖仟
9009->玖仟零玖
1111->壹仟壹佰壹拾壹
90000->玖万
90009->玖万零玖
11111->壹万壹仟壹佰壹拾壹
900000->玖拾万
900009->玖拾万零玖
111111->壹拾壹万壹仟壹佰壹拾壹
9000000->玖佰万
9000009->玖佰万零玖
1111111->壹佰壹拾壹万壹仟壹佰壹拾壹
90000000->玖仟万
90000009->玖仟万零玖
11111111->壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
900000000->玖亿
900000009->玖亿零玖
111111111->壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
9000000000->玖拾亿
9000000009->玖拾亿零玖
1111111111->壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
90000000000->玖佰亿
90000000009->玖佰亿零玖
11111111111->壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
900000000000->玖仟亿
900000000009->玖仟亿零玖
111111111111->壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
9000000000000->玖万亿
9000000000009->玖万亿零玖
1111111111111->壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
90000000000000->玖拾万亿
90000000000009->玖拾万亿零玖
11111111111111->壹拾壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
900000000000000->玖佰万亿
900000000000009->玖佰万亿零玖
111111111111111->壹佰壹拾壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
9000000000000000->玖仟万亿
9000000000000009->玖仟万亿零玖
1111111111111111->壹仟壹佰壹拾壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
90000000000000000->玖亿亿
90000000000000009->玖亿亿零玖
11111111111111111->壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
900000000000000000->玖拾亿亿
900000000000000009->玖拾亿亿零玖
111111111111111111->壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹亿壹仟壹佰壹拾壹万壹仟壹佰壹拾壹
Python doesn't support tail call elimination nor tail call optimizations. However, there are a number of ways in which you can mimic this approach (Trampolines being the most widely used in other languages.)
Tail call recursive functions should look like the following pseudo code:
def tail_call(*args, acc):
if condition(*args):
return acc
else:
# Operations happen here, producing new_args and new_acc
return tail_call(*new_args, new_acc)
For your example I would not form a closure over anything as your are introducing side-effects and stateful manipulation. Instead, anything that needs to be modified should be modified in isolation of everything else. That makes it easier to reason about.
Copy whatever you're attempting to change (using string.copy for the final output) and pass it in as an argument to the next recursive call. That's where the acc variable comes into play. It's "accumulating" all your changes up to that point.
A classical trampoline can be had from this snippet. There, they are wrapping the function in an object which will eventually either result a result or return another function object which should be called. I prefer this approach as I find it easier to reason about.
This isn't the only way. Take a look at this code snippet. The "magic" occurs when it reaches a point which "solves" the condition and it throws an exception to escape the infinite loop.
Finally, you can read about Trampolines here, here and here.
I keep studying this question off and on these days. and now, I work it out!
NOTE,not just tail-recursion, it's also pure Functional Programming!
The key is to think in a different way (tree-recursion version is processing numbers from left to right while this version is from right to left)
unitDic=dict(zip(range(8),u'拾佰仟万拾佰仟亿'))
numDic=dict(zip('0123456789',u'零壹贰叁肆伍陆柒捌玖'))
wapDic=[(u'零拾',u'零'),(u'零佰',u'零'),(u'零仟',u'零'),
(u'零万',u'万'),(u'零亿',u'亿'),(u'亿万',u'亿'),
(u'零零',u'零'),]
#pure FP
def ChnNumber(s):
def wrapper(s,wd=wapDic):
def rep(s,k,v):
if k in s:
return rep(s.replace(k,v),k,v)
return s
if not wd:
return s
return wrapper(rep(s,*wd[0]),wd[1:])
def recur(s,acc='',ind=0):
if s=='':
return acc
return recur(s[:-1],numDic[s[-1]]+unitDic[ind%8]+acc,ind+1)
def end(s):
if s[-1]!='0':
return numDic[s[-1]]
return ''
def result(start,end):
if end=='' and start[-1]==u'零':
return start[:-1]
return start+end
return result(wrapper(recur(s[:-1])),end(s))
for i in range(18):
v1='9'+'0'*(i+1)
v2='9'+'0'*i+'9'
v3='1'*(i+2)
print ('%s->%s\n%s->%s\n%s->%s'% (v1,ChnNumber(v1),v2,ChnNumber(v2),v3,ChnNumber(v3)))
if any one say that it won't work when facing a huge number(something like a billion-figure number), yeah, I admit that, but this version can solve it(while it will not be pure FP but pure FP won't need this version so..):
class TailCaller(object) :
def __init__(self, f) :
self.f = f
def __call__(self, *args, **kwargs) :
ret = self.f(*args, **kwargs)
while type(ret) is TailCall :
ret = ret.handle()
return ret
class TailCall(object) :
def __init__(self, call, *args, **kwargs) :
self.call = call
self.args = args
self.kwargs = kwargs
def handle(self) :
if type(self.call) is TailCaller :
return self.call.f(*self.args, **self.kwargs)
else :
return self.f(*self.args, **self.kwargs)
def ChnNumber(s):
def wrapper(s,wd=wapDic):
#TailCaller
def rep(s,k,v):
if k in s:
return TailCall(rep,s.replace(k,v),k,v)
return s
if not wd:
return s
return wrapper(rep(s,*wd[0]),wd[1:])
#TailCaller
def recur(s,acc='',ind=0):
if s=='':
return acc
return TailCall(recur,s[:-1],numDic[s[-1]]+unitDic[ind%8]+acc,ind+1)
def end(s):
if s[-1]!='0':
return numDic[s[-1]]
return ''
def result(start,end):
if end=='' and start[-1]==u'零':
return start[:-1]
return start+end
return result(wrapper(recur(s[:-1])),end(s))

Categories

Resources