Develop Reference

Re-serializing values and functions into a new environment in Python

I'm interested in moving a "session" across a boundary where I can only pass text.
I've been using Dill and this ALMOST seems to work.
import dill
from base64 import urlsafe_b64encode
def fun1():
value_one = "abcdef"
def sub_fun_one(x):
return x * x
__context_export = {}
for val in globals():
if not val.startswith("_"):
__context_export[val] = dill.dumps(globals()[val])
for val in locals():
if not val.startswith("_"):
__context_export[val] = dill.dumps(locals()[val])
# __context_export["globals"] = dill.dumps(globals())
# __context_export["locals"] = dill.dumps(locals())
b64_string = str(urlsafe_b64encode(dill.dumps(__context_export)), encoding="ascii")
print(b64_string)
fun1()
This outputs gASVHAQAAAAAAAB9lCiMBGRpbGyUQzeABJUsAAAAAAAAAIwKZGlsbC5fZGlsbJSMDl9pbXBvcnRfbW9kdWxllJOUjARkaWxslIWUUpQulIwRdXJsc2FmZV9iNjRlbmNvZGWUQyuABJUgAAAAAAAAAIwGYmFzZTY0lIwRdXJsc2FmZV9iNjRlbmNvZGWUk5QulIwEZnVuMZRCYwIAAIAElVgCAAAAAAAAjApkaWxsLl9kaWxslIwQX2NyZWF0ZV9mdW5jdGlvbpSTlChoAIwMX2NyZWF0ZV9jb2RllJOUKEsASwBLAEsFSwVLQ0OGZAF9AGQCZAOEAH0BaQB9AnQAgwBEAF0ifQN8A6ABZAShAXMWdAKgA3QAgwB8AxkAoQF8AnwDPABxFnQEgwBEAF0ifQN8A6ABZAShAXNAdAKgA3QEgwB8AxkAoQF8AnwDPABxQHQFdAZ0AqADfAKhAYMBZAVkBo0CfQR0B3wEgwEBAGQAUwCUKE6MBmFiY2RlZpRoBChLAUsASwBLAUsCS1NDCHwAfAAUAFMAlE6FlCmMAXiUhZSMMS9ob21lL2RhYXJvbmNoL2NvZGUvc2FtZS1jbGkvZXhwZXJpbWVudGFsL2Z1bjEucHmUjAtzdWJfZnVuX29uZZRLCEMCAAGUKSl0lFKUjBlmdW4xLjxsb2NhbHM-LnN1Yl9mdW5fb25llIwBX5SMBWFzY2lplIwIZW5jb2RpbmeUhZR0lCiMB2dsb2JhbHOUjApzdGFydHN3aXRolIwEZGlsbJSMBWR1bXBzlIwGbG9jYWxzlIwDc3RylIwRdXJsc2FmZV9iNjRlbmNvZGWUjAVwcmludJR0lCiMCXZhbHVlX29uZZRoDIwQX19jb250ZXh0X2V4cG9ydJSMA3ZhbJSMCmI2NF9zdHJpbmeUdJRoC4wEZnVuMZRLBUMWAAEEAggDBAEKAQoBFgIKAQoBFgQWApQpKXSUUpRjX19idWlsdGluX18KX19tYWluX18KaCROTn2UTnSUUpQulIwJdmFsdWVfb25llEMVgASVCgAAAAAAAACMBmFiY2RlZpQulIwLc3ViX2Z1bl9vbmWUQ9SABJXJAAAAAAAAAIwKZGlsbC5fZGlsbJSMEF9jcmVhdGVfZnVuY3Rpb26Uk5QoaACMDF9jcmVhdGVfY29kZZSTlChLAUsASwBLAUsCS1NDCHwAfAAUAFMAlE6FlCmMAXiUhZSMMS9ob21lL2RhYXJvbmNoL2NvZGUvc2FtZS1jbGkvZXhwZXJpbWVudGFsL2Z1bjEucHmUjAtzdWJfZnVuX29uZZRLCEMCAAGUKSl0lFKUY19fYnVpbHRpbl9fCl9fbWFpbl9fCmgKTk59lE50lFKULpSMA3ZhbJRDEoAElQcAAAAAAAAAjAN2YWyULpR1Lg==" as expected.
But in the second function - mocked up here - "value_one" doesn't seem to work at all.
import dill
from base64 import urlsafe_b64decode
def fun2(import_string):
__base64_decode = urlsafe_b64decode(import_string)
__context_import_dict = dill.loads(__base64_decode)
# __globals_import = dill.loads(__context_import_dict["globals"])
# __locals_import = dill.loads(__context_import_dict["locals"])
print(__context_import_dict)
for k in __context_import_dict:
print(f"local = {k}")
if locals().get(k) is None:
g = dill.loads(__context_import_dict[k])
locals()[k] = g
print(value_one)
print(f"Square value: {sub_fun_one(5)}")
import_value = "gASVHAQAAAAAAAB9lCiMBGRpbGyUQzeABJUsAAAAAAAAAIwKZGlsbC5fZGlsbJSMDl9pbXBvcnRfbW9kdWxllJOUjARkaWxslIWUUpQulIwRdXJsc2FmZV9iNjRlbmNvZGWUQyuABJUgAAAAAAAAAIwGYmFzZTY0lIwRdXJsc2FmZV9iNjRlbmNvZGWUk5QulIwEZnVuMZRCYwIAAIAElVgCAAAAAAAAjApkaWxsLl9kaWxslIwQX2NyZWF0ZV9mdW5jdGlvbpSTlChoAIwMX2NyZWF0ZV9jb2RllJOUKEsASwBLAEsFSwVLQ0OGZAF9AGQCZAOEAH0BaQB9AnQAgwBEAF0ifQN8A6ABZAShAXMWdAKgA3QAgwB8AxkAoQF8AnwDPABxFnQEgwBEAF0ifQN8A6ABZAShAXNAdAKgA3QEgwB8AxkAoQF8AnwDPABxQHQFdAZ0AqADfAKhAYMBZAVkBo0CfQR0B3wEgwEBAGQAUwCUKE6MBmFiY2RlZpRoBChLAUsASwBLAUsCS1NDCHwAfAAUAFMAlE6FlCmMAXiUhZSMMS9ob21lL2RhYXJvbmNoL2NvZGUvc2FtZS1jbGkvZXhwZXJpbWVudGFsL2Z1bjEucHmUjAtzdWJfZnVuX29uZZRLCEMCAAGUKSl0lFKUjBlmdW4xLjxsb2NhbHM-LnN1Yl9mdW5fb25llIwBX5SMBWFzY2lplIwIZW5jb2RpbmeUhZR0lCiMB2dsb2JhbHOUjApzdGFydHN3aXRolIwEZGlsbJSMBWR1bXBzlIwGbG9jYWxzlIwDc3RylIwRdXJsc2FmZV9iNjRlbmNvZGWUjAVwcmludJR0lCiMCXZhbHVlX29uZZRoDIwQX19jb250ZXh0X2V4cG9ydJSMA3ZhbJSMCmI2NF9zdHJpbmeUdJRoC4wEZnVuMZRLBUMWAAEEAggDBAEKAQoBFgIKAQoBFgQWApQpKXSUUpRjX19idWlsdGluX18KX19tYWluX18KaCROTn2UTnSUUpQulIwJdmFsdWVfb25llEMVgASVCgAAAAAAAACMBmFiY2RlZpQulIwLc3ViX2Z1bl9vbmWUQ9SABJXJAAAAAAAAAIwKZGlsbC5fZGlsbJSMEF9jcmVhdGVfZnVuY3Rpb26Uk5QoaACMDF9jcmVhdGVfY29kZZSTlChLAUsASwBLAUsCS1NDCHwAfAAUAFMAlE6FlCmMAXiUhZSMMS9ob21lL2RhYXJvbmNoL2NvZGUvc2FtZS1jbGkvZXhwZXJpbWVudGFsL2Z1bjEucHmUjAtzdWJfZnVuX29uZZRLCEMCAAGUKSl0lFKUY19fYnVpbHRpbl9fCl9fbWFpbl9fCmgKTk59lE50lFKULpSMA3ZhbJRDEoAElQcAAAAAAAAAjAN2YWyULpR1Lg=="
fun2(import_value)
I'd prefer to do this more elegantly than eval 'k = v' because that will be hard to reserialize functions and dictionaries into.
Is this possible?
To be clear, I've read about the dangers of modifying locals() and I do NOT want to do this. But I also cannot rewrite the code that I'm accessing elsewhere to use a custom dict.
E.g. in fun2, I can NOT change print(value_one) to print(my_dict['value_one'])

Ok, so it APPEARS that doing the following works:
for k in __context_import_dict:
if globals().get(k) is None:
globals()[k] = dill.loads(__context_import_dict[k])
What kind of pain am i signing myself up for?

From the Python documentation about locals():
Note: The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter.
Function namespaces are not commmon dictionaries like module namespaces for performance reasons. I was bitten by this same quirk just yesterday.

Python - How to obtain a dictionary(or even a list) of variables in a specific scope; more specific than locals/globals()

So, the title pretty much says it all.
for instance let's look at the below example code:
## How can I obtain a dict/list (like locals()) of all the variables in second and/or third layer scopes via a command
# coming from the first layer?
## Or another example would be how could I obtain the variables "locals() style" from the thirdlayer via a
# command from the second layer?
# essentially can a parent function/class access a list/dict of a child function
# or class??
def firstLayer():
a = 4.7
q = locals()
print(q)
# local vars of 1st layer
def secondlayer():
b = 7
r = locals()
print(r)
# local vars of 2nd layer
def thirdlayer():
c = False
s = locals()
i = globals()
print('c:\n', c, "\nglobals from 3rd layer:\n\t", i)
# local vars of 3rd layer
thirdlayer()
secondlayer()
firstLayer()
sample_var = globals()
print(sample_var)
# returns the list of global variables
to reiterate what I said in the comments in the code, essentially is their any way I can get a list of all the variables local to a 'child' scope? I know functions are shut off, but if their is no way to do this is their any more complicated code that could achieve this and I could integrate it into a function or class if necessary.
EDIT:
To elaborate further; here's the situation i'm in.
def varsfunc():
font1 = "Harlow Solid"
grey = '#454545'
font2 = 'Mistral'
font3 = 'Italic 35px Times New Roman'
pnk = 'pink'
grn = 'green'
return locals()
Essentially, I am creating a module and the user must create some type of function that they list all of they variables they would like to declare to be used to modify a css file. Essentially, I would like to allow the user to not have to type "return locals()". I want to achieve it by having the end-users wrap the above example function in a decorator that will do the equivalent of returning locals() of the exact scope I want. The decorator does not work for me because it is in an outer scope.
TO BE EVEN MORE CLEAR:
I need a decorator/function that wraps another function(i.e. a decorator), that can access and create a list of a child element.
def module_decorator_func_thing():
r = command_that_acts_like_locals()_but_for_child_scopes
def user_var_list():
font1 = 'green'
font2 = 'pink'
# back in "module_decorator_func_thing"'s scope
print(r) # this variable should contain only a dict/list containing the
# the following:
# r = {'font1': 'green', 'font2': 'pink')
currently users need to do this:
def vars_func_container():
font1 = 'green'
font2 = 'pink'
return locals() # <---- I want the user to not have to type this and for
# a function decorator to take care of it instead possibly.
Info for #aguy and others wishing for more info.
The dictionary/list that I am obtaining via your guys' tips will be sent to this function to do the real job of the program.
(If I were to start using lists, i'd need to convert to a dictionary but that's no problem for me to solve.)
The dict of variables is used with this function to "compile/compyle"(Pun on the word 'Python' + 'compile) and is insert in the "variables" parameter. e.g. you execute the function like this.
compyle("My sample title", return_stylesheet_from_func(*insert .css filename),
return_variables_from_function(*insert function containing variables*), "**True/False to turn on compilation**",
"**True/False to turn on annotations/suggestions**")
def compyle(title, style_sheet, variables, boolean=False, boolean2=True):
"""
:param title: The name you wish your .css file to be named.
:param style_sheet: The name of the multi-line string that will compose your .css file
:param variables: The name of the dictionary containing your .pcss variables
:param boolean: A.K.A the "Compiler Parameter" - Turns the compiler on or off
:param boolean2: A.K.A the "Annotation Parameter" - Turns annotations on or off
:return: returns compiled .pcss text as normal .css style text to be utilized with .html
"""
# -----------------------------------
file_name = title + ".css"
replace_num = len(variables.keys())
counter = replace_num
content = style_sheet
# -----------------------------------
# add theme support with namedtuple's formatted to mimic structs in C/C++
# this will be a major feature update as well as a nice way to allow the future prospect of integrating C/C++ into
# the compiler. Info: https://stackoverflow.com/questions/35988/c-like-structures-in-python
for k, v in variables.items():
counter -= 1
content = content.replace(k, v, replace_num)
if counter == 0:
break
else:
pass
looped_content = str(content)
id_content = looped_content.replace("hash_", "#")
output = id_content.replace("dot_", ".")
if boolean is True:
if boolean2 is True:
output = " /* --- Pyle Sheet --- */\n" + output
with open(file_name, 'w') as writ:
writ.write(output)
writ.close()
print('compiled successfully; The file was saved as ' + "\"" + file_name + "\".")
elif boolean2 is False:
pass
else:
logging.warning("An Error Occurred - see module, documentation, or online Q&A for assistance.")
elif boolean is False:
if boolean2 is True:
print('compiled successfully; The file ' + "\"" + file_name + "\"" + "was not saved/created.")
elif boolean2 is False:
pass
else:
logging.warning("An Error Occurred - see module, documentation, or online Q&A for assistance.")
else:
logging.warning('An Error Occurred with the Compile Parameter (See: boolean in pyle_sheets source file) - \ '
'see module, documentation, or online Q&A for assistance.')

I can't see any way to do this without getting pretty deep; what follows is the simplest solution I've come up with.
how it works
Using the ast module, we go through the code of the given function and find all the assignments. These are evaluated in a given namespace and this namespace is returned.
the code
import ast
import functools
import inspect
def returnAssignments(f):
#functools.wraps(f)
def returner():
assignments = dict()
for node in ast.walk(ast.parse(inspect.getsource(f))):
if isinstance(node, ast.Assign):
exec(compile(ast.Module([node]), '<ast>', 'exec'),
globals(),
assignments)
return assignments
return returner
usage
from ra import returnAssignments
#returnAssignments
def foo():
this = 'something'
that = 37
the_other = object()
print(foo())
output
rat#pandion:~/tmp$ python test.py
{'this': 'something', 'that': 37, 'the_other': <object object at 0x10205b130>}

I wonder if such a crude solution as I provide here might be useful to you. Note that I haven't tested it on all cases, so it might be a bit rough. Also, it returns everything as a string, a behavior which you might want to further change.
Define the function:
def get_local_vars_from_function(f):
import inspect
s = inspect.getsourcelines(f)[0]
d = {}
for l in s:
if '=' in l:
var, val = l.split('=')
var = var.strip()
val = val.strip()
d[var] = val
return d
Then to use it:
In[91]: get_local_vars_from_function(user_var_list)
Out[91]: {'font1': "'green'", 'font2': "'pink'"}

Convert python objects to python AST-nodes

I have a need to dump the modified python object back into source. So I try to find something to convert real python object to python ast.Node (to use later in astor lib to dump source)
Example of usage I want, Python 2:
import ast
import importlib
import astor
m = importlib.import_module('something')
# modify an object
m.VAR.append(123)
ast_nodes = some_magic(m)
source = astor.dump(ast_nodes)
Please help me to find that some_magic

There's no way to do what you want, because that's not how ASTs work.
When the interpreter runs your code, it will generate an AST out of the source files, and interpret that AST to generate python objects.
What happen to those objects once they've been generated has nothing to do with the AST.
It is however possible to get the AST of what generated the object in the first place.
The module inspect lets you get the source code of some python objects:
import ast
import importlib
import inspect
m = importlib.import_module('pprint')
s = inspect.getsource(m)
a = ast.parse(s)
print(ast.dump(a))
# Prints the AST of the pprint module
But getsource() is aptly named.
If I were to change the value of some variable (or any other object) in m, it wouldn't change its source code.
Even if it was possible to regenerate an AST out of an object, there wouldn't be a single solution some_magic() could return.
Imagine I have a variable x in some module, that I reassign in another module:
# In some_module.py
x = 0
# In __main__.py
m = importlib.import_module('some_module')
m.x = 1 + 227
Now, the value of m.x is 228, but there's no way to know what kind of expression led to that value (well, without reading the AST of __main__.py but this would quickly get out of hand). Was it a mere literal? The result of a function call?
If you really have to get a new AST after modifying some value of a module, the best solution would be to transform the original AST by yourself.
You can find where your identifier got its value, and replace the value of the assignment with whatever you want.
For instance, in my small example x = 0 is represented by the following AST:
Assign(targets=[Name(id='x', ctx=Store())], value=Num(n=0))
And to get the AST matching the reassignment I did in __main__.py, I would have to change the value of the above Assign node as the following:
value=BinOp(left=Num(n=1), op=Add(), right=Num(n=227))
If you'd like to go that way, I recommend you check Python's documentation of the AST node transformer (ast.NodeTransformer), as well as this excellent manual that documents all the nodes you can meet in Python ASTs Green Tree Snakes - the missing Python AST docs.

What Vladimir is asking about is certainly useful for compiler optimizations. Indeed, there are ways to accomplish that using the ast library. Here is a simple example demonstrating evaluation of constant functions:
from ast import *
import numpy as np
PURE_FUNS = {'arange' : np.arange}
PROG = '''
A=arange(5)
B=[0, 1, 2, 3, 4]
A[2:3] = 1
C = [A[1], 2, m]
'''
def py_to_ast(o):
if type(o) == np.ndarray:
return List(elts=[py_to_ast(e) for e in o], ctx=Load())
elif type(o) == np.int64:
return Constant(value=o)
# Add elifs for more types here
else:
assert False
class EvalPureFuns(NodeTransformer):
def visit_Call(self, node):
is_const_args = all(type(a) == Constant for a in node.args)
if node.func.id in PURE_FUNS and is_const_args:
res = eval(unparse(node), PURE_FUNS)
return py_to_ast(res)
return node
node = parse(PROG)
node = EvalPureFuns().visit(node)
print(unparse(node))

Export function that names the export file after the input variable

I'm looking to get a function to export Numpy arrays, but to use the name of the variable input to the function as the name of the exported file. Something like:
MyArray = [some numbers]
def export(Varb):
return np.savetxt("%s.dat" %Varb.name, Varb)
export(MyArray)
that will output a file called 'MyArray.dat' filled with [some numbers]. I can't work out how to do the 'Varb.name' bit. Any suggestions?
I'm new to Python and programming so I hope there is something simple I've missed!
Thanks.

I don't recommend such a code style, but you can do it this way:
import copy
myarray = range(4)
for k, v in iter(copy.copy(locals()).items()):
if myarray == v:
print k
This gives myarray as output. To do this in a function useful for exports use:
import copy
def export_with_name(arg):
""" Export variable's string representation with it's name as filename """
for k, v in iter(copy.copy(globals()).items()):
if arg == v:
with open(k, 'w') as handle:
handle.writelines(repr(arg))
locals() and globals() both give dictionaries holding the variable names as keys and the variable values as values.
Use the function the following way:
some_data = list(range(4))
export_with_name(some_data)
gives a file called some_data with
[0, 1, 2, 3]
as content.
Tested and compatible with Python 2.7 and 3.3

You can't. Python objects don't know what named variables happen to be referencing them at any particular time. By the time the variable has been dereferenced and sent to the function, you don't know where it came from. Consider this bit of code:
map(export, myvarbs)
Here, the varbs were in some sort of container and didn't even have a named variable referencing them.

import os
import inspect
import re
number_array = [8,3,90]
def varname(p):
for line in inspect.getframeinfo(inspect.currentframe().f_back)[3]:
m = re.search(r'\bvarname\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)', line)
if m:
return m.group(1)
def export(arg):
file_name = "%s.dat" % varname(arg)
fd = open(file_name, "w")
fd.writelines(str(arg))
fd.close()
export(number_array)
Refer How can you print a variable name in python? to get more details about def varname()

Capturing **vars() pattern in string formatting

I frequently find myself using the following pattern for string formatting.
a = 3
b = 'foo'
c = dict(mykey='myval')
#prints a is 3, b is foo, mykey is myval
print('a is {a}, b is {b}, mykey is {c[mykey]}'.format(**vars()))
That is, I often have the values I need to print in the local namespace, represented by a call to vars(). As I look over my code, however, it seems awfully unpythonic to be constantly repeating the .format(**vars()) pattern.
I'd like to create a function that will capture this pattern. It would be something like the following.
# doesn't work
def lfmt(s):
"""
lfmt (local format) will format the string using variables
in the caller's local namespace.
"""
return s.format(**vars())
Except that by the time I'm in the lfmt namespace, vars() is no longer what I want.
How can I write lfmt so that it executes vars() in the caller's namespace such that the following code would work as the example above?
print(lfmt('a is {a}, b is {b}, mykey is {c[mykey]}'))

Edit: In order for lfmt to work when called from different namespaces, you'll need the inspect module. Note, as the documentation warns, the inspect module may not be suitable for production code since it may not work with all implementations of Python
import inspect
def lfmt(s):
caller = inspect.currentframe().f_back
return s.format(**caller.f_locals)
a = 3
b = 'foo'
c = dict(mykey='myval')
print(lfmt('a is {a}, b is {b}, mykey is {c[mykey]}'))
# a is 3, b is foo, mykey is myval

You have to inspect the variables from the calling frames.
This will get you started:
import inspect
import pprint
def lfmt(s):
for frame in inspect.getouterframes(inspect.currentframe()):
f = frame[0]
print pprint.pformat(f.f_locals)
return '???'
if __name__ == '__main__':
a = 10
b = 20
c = 30
lfmt('test')

Here you are:
import sys
def lfmt(s):
"""
lfmt (local format) will format the string using variables
in the caller's local namespace.
"""
if hasattr(sys, "tracebacklimit") and sys.tracebacklimit == 0:
raise Exception, "failfailfail"
try:
raise ZeroDivisionError
except ZeroDivisionError:
f = sys.exc_info()[2].tb_frame.f_back
return s.format(**f.f_locals)
a = 5
somestring = "text"
print lfmt("{a} {somestring}")
The fact that it works doesn't mean you should use it. This is what developers call "major hack", usually shipped with a comment "XXX fix me XXX".

Is it so bad to type ,vars each time you call the function?
def lfmt(s,v):
"""
lfmt (local format) will format the string using variables
from the dict returned by calling v()"""
return s.format(**v())
print(lfmt('a is {a}, b is {b}, mykey is {c[mykey]}',vars))

You could also use sys instead of inspect, but I don't know if it has the same problem with different implementations as inspect has.
import sys
def lfmt(s):
caller = sys._getframe(1)
return s.format(**caller.f_locals)
This is as far as I got: Python string interpolation implementation