My goal is to generate functions dynamically and then save them in a file. For e.g, in my current attempt, On calling create_file
import io
def create_file():
nested_func = make_nested_func()
write_to_file([nested_func, a_root_func], '/tmp/code.py')
def a_root_func(x):
pass
def make_nested_func():
def a_nested_func(b, k):
return b, k
return a_nested_func
def write_to_file(code_list, path):
import inspect
code_str_list = [inspect.getsource(c) for c in code_list]
with open(path, 'w') as ofh:
for c in code_str_list:
fh = io.StringIO(c)
ofh.writelines(fh.readlines())
ofh.write('\n')
create_file()
The output I want is('/tmp/code.py'):
def a_nested_func(b, k):
return b, k
def a_root_func(x):
pass
The output I get is('/tmp/code.py'):
def a_nested_func(b, k):
return b, k
def a_root_func(x):
pass
a_nested_func is indented. How can I reduce the indentation? I can do lstrip etc. but I wonder if there is a better way.
Use the textwrap.dedent() function to remove the common leading whitespace:
import inspect
from textwrap import dedent
def write_to_file(code_list, path):
code_str_list = [inspect.getsource(c) for c in code_list]
with open(path, 'w') as ofh:
for c in code_str_list:
dedented = dedent(c)
ofh.write(dedented + '\n')
Note that there is no need for a StringIO(string).readlines() dance here.
There's a function in built-in module, textwrap.dedent.
import textwrap
s = """
abc
def
"""
s2 = """
abc
def
"""
assert textwrap.dedent(s) == s2
Related
Can I add a prefix and suffix to the source code of functions?
I know about decorators and do not want to use them (the minimal example below doesn't make clear why, but I have my reasons).
def f():
print('world')
g = patched(f,prefix='print("Hello, ");',suffix='print("!");')
g() # Hello, world!
Here is what I have so far:
import inspect
import ast
import copy
def patched(f,prefix,suffix):
source = inspect.getsource(f)
tree = ast.parse(source)
new_body = [
ast.parse(prefix).body[0],
*tree.body[0].body,
ast.parse(suffix).body[0]
]
tree.body[0].body = new_body
g = copy.deepcopy(f)
g.__code__ = compile(tree,g.__code__.co_filename,'exec')
return g
Unfortunately, nothing happens if I use this and then call g() as above; neither world nor Hello, world! are printed.
Here is a rough version of what can be done:
import inspect
import ast
import copy
def patched(f,prefix,suffix):
source = inspect.getsource(f)
tree = ast.parse(source)
new_body = [
ast.parse(prefix).body[0],
*tree.body[0].body,
ast.parse(suffix).body[0]
]
tree.body[0].body = new_body
code = compile(tree,filename=f.__code__.co_filename,mode='exec')
namespace = {}
exec(code,namespace)
g = namespace[f.__name__]
return g
def temp():
pass
def f():
print('world',end='')
g = patched(f,prefix='print("Hello, ",end="")',suffix='print("!",end="")')
g() # Hello, world!
The call of compile compiles an entire module (represented by tree). This module is then executed in an empty namespace from which the desired function is finally extracted. (Warning: the namespace will need to be filled with some globals from where f comes from if f uses those.)
After some more work, here is a real example of what can be done with this. It uses some extended version of the principle above:
import numpy as np
from playground import graphexecute
#graphexecute(verbose=True)
def my_algorithm(x,y,z):
def SumFirstArguments(x,y)->sumxy:
sumxy = x+y
def SinOfThird(z)->sinz:
sinz = np.sin(z)
def FinalProduct(sumxy,sinz)->prod:
prod = sumxy*sinz
def Return(prod):
return prod
print(my_algorithm(x=1,y=2,z=3))
#OUTPUT:
#>>Executing part SumFirstArguments
#>>Executing part SinOfThird
#>>Executing part FinalProduct
#>>Executing part Return
#>>0.4233600241796016
The clou is that I get the exact same output if I reshuffle the parts of my_algorithm, for example like this:
#graphexecute(verbose=True)
def my_algorithm2(x,y,z):
def FinalProduct(sumxy,sinz)->prod:
prod = sumxy*sinz
def SumFirstArguments(x,y)->sumxy:
sumxy = x+y
def SinOfThird(z)->sinz:
sinz = np.sin(z)
def Return(prod):
return prod
print(my_algorithm2(x=1,y=2,z=3))
#OUTPUT:
#>>Executing part SumFirstArguments
#>>Executing part SinOfThird
#>>Executing part FinalProduct
#>>Executing part Return
#>>0.4233600241796016
This works by (1) grabbing the source of my_algorithm and turning it into an ast (2) patching each function defined within my_algorithm (e.g. SumFirstArguments) to return locals (3) deciding based on the inputs and the outputs (as defined by the type hints) in which order the parts of my_algorithm should be executed. Furthermore, a possibility that I do not have implemented yet is to execute independent parts in parallel (such as SumFirstArguments and SinOfThird). Let me know if you want the sourcecode of graphexecute, I haven't included it here because it contains a lot of stuff that is not relevant to this question.
For your problem, you don't need to recompile your functions. Just define a list of functions, you inspect for arguments and return variable name:
def FinalProduct(sumxy, sinz) -> "prod":
return sumxy * sinz
def SumFirstArguments(x, y) -> "sumxy":
return x + y
def SinOfThird(z) -> "sinz":
return np.sin(z)
def execute(funcs, **args):
result = None
while funcs:
func = funcs.pop(0)
try:
kw = {a: args[a]
for a in func.__code__.co_varnames[:func.__code__.co_argcount]
}
except KeyError:
# not all arguments found
funcs.append(func)
else:
print(func,kw)
result = func(**kw)
args[func.__annotations__['return']] = result
return result
print(execute([FinalProduct, SumFirstArguments, SinOfThird], x=1,y=2,z=3))
My goal is to generate functions dynamically and then save them in a file. For e.g, in my current attempt, On calling create_file
import io
def create_file(a_value):
a_func = make_concrete_func(a_value)
write_to_file([a_func], '/tmp/code.py')
def make_concrete_func(a_value):
def concrete_func(b, k):
return b + k + a_value
return concrete_func
def write_to_file(code_list, path):
import inspect
code_str_list = [inspect.getsource(c) for c in code_list]
with open(path, 'w') as ofh:
for c in code_str_list:
fh = io.StringIO(c)
ofh.writelines(fh.readlines())
ofh.write('\n')
create_file('my_value')
The output I want is (file /tmp/code.py):
def concrete_func(b, k):
return b + k + 'my_value'
The output I get is (file '/tmp/code.py'):
def concrete_func(b, k):
return b + k + a_value
UPDATE: My solution uses inspect.getsource which returns a string. I wonder if I have limited your options as most solutions below suggest a string replacement. The solution need not use inspect.getsource. You could write it anyhow to get the desired output.
UPDATE 2: The reason I am doing this is because I want to generate a file for Amazon Lambda. Amazon Lambda takes a python file and its virtual environment and will execute it for you(relieving you from worrying about scalability and fault tolerance). You have to tell Lambda which file and which function to call and Lambda will execute it for you.
A function definition doesn't look up its free variables (variables that are not defined in the function itself) at time of definition. I.e. concrete_func here:
def make_concrete_func(a_value):
def concrete_func(b, k):
return b + k + a_value
return concrete_func
doesn't look up a_value when it is defined, instead it will contain code to load a_value from its closure (simplified the enclosing function) at runtime.
You can see this by disassembling the returned function:
f = make_concrete_func(42)
import dis
print dis.dis(f)
3 0 LOAD_FAST 0 (b)
3 LOAD_FAST 1 (k)
6 BINARY_ADD
7 LOAD_DEREF 0 (a_value)
10 BINARY_ADD
11 RETURN_VALUE
None
You can maybe do what you want by editing the byte code.. it's been done before (http://bytecodehacks.sourceforge.net/bch-docs/bch/module-bytecodehacks.macro.html ..shudder).
Use getsource to convert the function to a string, and replace the variable names with simple string manipulation.
from inspect import getsource
def write_func(fn, path, **kwargs):
fn_as_string = getsource(fn)
for var in kwargs:
fn_as_string = fn_as_string.replace(var, kwargs[var])
with open(path, 'a') as fp: # append to file
fp.write('\n' + fn_as_string)
def base_func(b, k):
return b + k + VALUE
# add quotes to string literals
write_func(base_func, '/tmp/code.py', VALUE="'my value'")
# you should replace the function name if you write multiple functions to the file
write_func(base_func, '/tmp/code.py', base_func='another_func', VALUE='5')
Output is as expected in /tmp/code.py:
def base_func(b, k):
return b + k + 'my value'
def another_func(b, k):
return b + k + 5
Try this. Note that I have added another parameter to write_to_file
def write_to_file(code_list, path,a_value):
print "lc",code_list
code_str_list = [inspect.getsource(c) for c in code_list]
with open(path, 'w') as ofh:
for c in code_str_list:
c= c.replace('a_value','\''+a_value+'\'')
fh = io.StringIO(c)
ofh.writelines(fh.readlines())
ofh.write('\n')
If the file doesn't have to be human readable and you trust it won't be manipulated by attackers, combining functools.partial and pickle might be the most pythonic approach. However it comes with disadvantages I don't completely understand: for one thing it doesn't seem to work with locally defined functions (or maybe locally defined variables in general?).
I might just ask my own question about this.
import functools
import pickle
def write_file_not_working():
def concrete_func_not_working(b, k, a_value):
return b + k + a_value
with open('data.pickle', 'wb') as f:
data = functools.partial(concrete_func_not_working, a_value='my_value')
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
def use_file_not_working():
with open('data.pickle', 'rb') as f:
resurrected_data = pickle.load(f)
print(resurrected_data('hi', 'there'))
def top_level_concrete_func(b, k, a_value):
return a_value + b + k
def write_file_working():
with open('working.pickle', 'wb') as f:
data = functools.partial(top_level_concrete_func, a_value='my_working_value')
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
def use_file_working():
with open('working.pickle', 'rb') as f:
resurrected_data = pickle.load(f)
print(resurrected_data('hi', 'there'))
if __name__ == "__main__":
write_file_working()
use_file_working()
write_file_not_working()
use_file_not_working()
#Ben made me realize that I didn't need to use a string based approach for code generation and that I could use serialization. Instead of the limited pickle library, I used dill which overcomes the limitation as mentioned by Ben
So, I finally did something like.
import dill
def create_file(a_value, path):
a_func = make_concrete_func(a_value)
dill.dump(a_func, open(path, "wb"))
return path
def make_concrete_func(a_value):
def concrete_func(b, k):
return b + k + a_value
return concrete_func
if __name__ == '__main__':
path = '/tmp/code.dill'
create_file('Ben', path)
a_func = dill.load(open(path, "rb"))
print(a_func('Thank ', 'You '))
if the function you want to create all have a determinate pattern, I would create a template for it and use it to mass produce the functions
>>> def test(*values):
template="""
def {name}(b,k):
return b + k + {value}
"""
for i,v in enumerate(values):
print( template.format(name="func{}".format(i),value=repr(v)) )
>>> test("my_value",42,[1])
def func0(b,k):
return b + k + 'my_value'
def func1(b,k):
return b + k + 42
def func2(b,k):
return b + k + [1]
>>>
import pep8
s = """
def a:
pass
def b:
pass
"""
pep8.StyleGuide().is_such_method_exists_for_string_?(s)
// and then, get out put as list ?
import pep8
lines = """
def a:
pass
def b:
pass
"""
checker = pep8.Checker(
lines=lines.strip().splitlines(),
filename=None,
show_source=True
)
result = checker.check_all()
All I get from this code is that, print in python is a wrapper function of write method of stdout so if I give it a return type it must return that as well, right? Then why can't I do that?
import sys
class CustomPrint():
def __init__(self):
self.old_stdout=sys.stdout
def write(self, text):
text = text.rstrip()
if len(text) == 0: return
self.old_stdout.write('custom Print--->' + text + '\n')
return text
sys.stdout=CustomPrint()
print "ab" //works
a=print "ab" //error! but why?
In python2.x, print is a statement. So, a = print "ab" is illegal syntax. Try just print "ab".
In python3, print is a function -- So you'd write: a = print("ab"). Note that starting at python2.6, you can get access to python3's print function via from __future__ import print_function.
Ultimately, what you want is something like:
#Need this to use `print` as a function name.
from __future__ import print_function
import sys
class CustomPrint(object):
def __init__(self):
self._stdout = sys.stdout
def write(self,text):
text = text.rstrip()
if text:
self._stdout.write('custom Print--->{0}\n'.format(text))
return text
__call__ = write
print = CustomPrint()
a = print("ab")
I want to closely monitor the chain of function calls which are called from a certain function.
import pdb; pdb.set_trace()
res = api.InsertVideoEntry(video_entry, video)
I'm looking for a way to easily see that api.insertVideoEntry(video_entry, video) calls foo() which calls bar() which calls baz(),
Here's a really crude diagram to show what I mean. I don't need it in this form, but this is the kind kind of information I'm looking for.
api.insertVideoEntry()
foo()
bar()
baz()
baz2()
log()
finish()
This was an interesting learning experience to write up. Maybe you can use the code shown here? This demonstration should give you an idea of the type of output you can expect when using trace.
# Functions to trace
# ==================
def baz():
pass
def baz2():
pass
def bar():
baz()
baz2()
def log():
pass
def foo():
bar()
log()
def finish():
pass
def insertVideoEntry():
foo()
finish()
# Names to trace
# ==============
names = list(locals())
# Machinery for tracing
# =====================
import os
import sys
def trace(start, *names):
def tracefunc(frame, event, arg):
if event == 'call':
code = frame.f_code
name = code.co_name
if name in names:
level = -start
while frame:
frame = frame.f_back
level += 1
print('{}{}.{}()'.format(
' ' * level,
os.path.splitext(os.path.basename(code.co_filename))[0],
name))
return tracefunc
sys.settrace(tracefunc)
# Demonstration of tracing
# ========================
trace(2, *names)
insertVideoEntry()
If you are interested in a recursive demo, you might like this variation with a called arguments readout:
import os
import sys
def main(discs):
a, b, c = list(range(discs, 0, -1)), [], []
line = '-' * len(repr(a))
print(a, b, c, sep='\n')
for source, destination in towers_of_hanoi(discs, a, b, c):
destination.append(source.pop())
print(line, a, b, c, sep='\n')
def towers_of_hanoi(count, source, via, destination):
if count > 0:
count -= 1
yield from towers_of_hanoi(count, source, destination, via)
yield source, destination
yield from towers_of_hanoi(count, via, source, destination)
def trace(start, *names):
def tracefunc(frame, event, arg):
if event == 'call':
code = frame.f_code
name = code.co_name
if name in names:
level = -start
args = ', '.join(repr(frame.f_locals[name]) for name in
code.co_varnames[:code.co_argcount])
while frame:
frame = frame.f_back
level += 1
print('{}{}.{}({})'.format(
' ' * (level * 4),
os.path.splitext(os.path.basename(code.co_filename))[0],
name, args))
return tracefunc
sys.settrace(tracefunc)
if __name__ == '__main__':
trace(3, 'main', 'towers_of_hanoi')
main(3)