Converting Python objects to JavaScript for PyV8 - python

I'm trying to pass Python data (lists, dicts, strings..., arbitrarily nested) to PyV8.
class Global(object):
def __init__(self, data):
self.data = data
ctx = PyV8.JSContext(Global([{'a':1}]))
ctx.enter()
res = ctx.eval('data.length')
js_len = PyV8.convert(res)
print js_len
The code above prints None, presumably because the data object is not transformed to a JSArray and thus data.length evaluates to undefined. Is there a reliable way to do the necessary conversion in PyV8 other than using JSON?

Apparently PyV8 doesn't correctly convert python lists to Javascript arrays, which leads my_list.length to return undefined, which is getting converted to None.
ctx = PyV8.JSContext()
ctx.enter()
ctx.locals.a = [{'a':1}]
print ctx.locals.a
#> [{'a': 1}]
print ctx.eval("a.length")
#> None
print ctx.eval("a[0].a")
#> 1
ctx.locals.blub = {'a':1}
print ctx.eval("blub.a")
#> 1
print ctx.eval("Object.keys(blub)")
#> a
ctx.locals.blub = {'a':[1,2,3]}
print ctx.eval("Object.keys(blub)")
#> a
print ctx.eval("blub.a")
#> [1, 2, 3]
ctx.locals.blub2 = [{'a':[1,2,3]}]
print ctx.eval("blub2")
#> [{'a': [1, 2, 3]}]
print ctx.eval("blub2.length")
#> None
print ctx.eval("Array.isArray(blub2)")
#> False
print ctx.eval("typeof(blub2)")
#> object
print ctx.eval("blub2[0].a")
#> [1, 2, 3]
print ctx.eval("typeof(blub.a)")
#> object
print ctx.eval("Array.isArray(blub.a)")
#> False
The answer is to use PyV8.JSArray(my_list). I've written the following helper functions for my project that deal with various little problems and make it easy to convert back and forth between python and js objects. These are targeted at a specific version of PyV8 however (which is the only version I can recommend, see discussion in the linked issues), so your results may vary if you use them as-is. Example usage:
ctx.locals.blub3 = get_js_obj({'a':[1,2,3]})
ctx.locals.blub4 = get_js_obj([1,2,3])
ctx.eval("blub3.a.length")
#> 3
ctx.eval("blub4.length")
#> 3
And here are the functions.
def access_with_js(ctx, route):
if len(route) == 0:
raise Exception("route must have at least one element")
accessor_string = route[0]
for elem in route[1:]:
if type(elem) in [str, unicode]:
accessor_string += "['" + elem + "']"
elif type(elem) == int:
accessor_string += "[" + str(elem) + "]"
else:
raise Exception("invalid element in route, must be text or number")
return ctx.eval(accessor_string)
def get_py_obj(ctx, obj, route=[]):
def dict_is_empty(dict):
for key in dict:
return False
return True
def access(obj, key):
if key in obj:
return obj[key]
return None
cloned = None
if isinstance(obj, list) or isinstance(obj, PyV8.JSArray):
cloned = []
temp = str(access_with_js(ctx, route)) #working around a problem with PyV8 r429
num_elements = len(obj)
for index in range(num_elements):
elem = obj[index]
cloned.append(get_py_obj(ctx, elem, route + [index]))
elif isinstance(obj, dict) or isinstance(obj, PyV8.JSObject):
cloned = {}
for key in obj.keys():
cloned_val = None
if type(key) == int:
#workaround for a problem with PyV8 where it won't let me access
#objects with integer accessors
val = None
try:
val = access(obj, str(key))
except KeyError:
pass
if val == None:
val = access(obj, key)
cloned_val = get_py_obj(ctx, val, route + [key])
else:
cloned_val = get_py_obj(ctx, access(obj, key), route + [key])
cloned[key] = cloned_val
elif type(obj) == str:
cloned = obj.decode('utf-8')
else:
cloned = obj
return cloned
def get_js_obj(ctx,obj):
#workaround for a problem with PyV8 where it will implicitely convert python lists to js objects
#-> we need to explicitely do the conversion. see also the wrapper classes for JSContext above.
if isinstance(obj, list):
js_list = []
for entry in obj:
js_list.append(get_js_obj(ctx,entry))
return PyV8.JSArray(js_list)
elif isinstance(obj, dict):
js_obj = ctx.eval("new Object();") # PyV8.JSObject cannot be instantiated from Python
for key in obj.keys():
try:
js_obj[key] = get_js_obj(ctx,obj[key])
except Exception, e:
# unicode keys raise a Boost.Python.ArgumentError
# which can't be caught directly:
# https://mail.python.org/pipermail/cplusplus-sig/2010-April/015470.html
if (not str(e).startswith("Python argument types in")):
raise
import unicodedata
js_obj[unicodedata.normalize('NFKD', key).encode('ascii','ignore')] = get_js_obj(ctx,obj[key])
return js_obj
else:
return obj

Related

xmltodict does not return a list for one element

The following Code produces an error, if there is only one "car" in "garage":
import xmltodict
mydict = xmltodict.parse(xmlstringResults)
for carsInGarage in mydict['garage']['car']:
# do something...
The Reason is that mydict['garage']['car'] is only a list if there is more than one element of "car". So I did something like this:
import xmltodict
mydict = xmltodict.parse(xmlstringResults)
if isinstance(mydict['garage']['car'], list):
for carsInGarage in mydict['garage']['car']:
# do something for each car...
else:
# do something for the car
to get the code to run. But for more advanced operations this is no solution.
Does someone know some kind of function to use, even if there is only one element?
This problem is discussed in this issue on Github. The xmltodict package now supports
d = xmltodict.parse(s, force_list={'car'})
Although this still doesn't create an empty list if the field is absent.
This is of course not an elegant way, but this is what i have done to get the code run (if someone hase the same probleme an found this via google):
import xmltodict
def guaranteed_list(x):
if not x:
return []
elif isinstance(x, list):
return x
else:
return [x]
mydict = xmltodict.parse(xmlstringResults)
for carsInGarage in guaranteed_list(mydict['garage']['car']):
# do something...
but i thing i will write my code again and "use XML directly" as one of the comments said.
I am using the combination of
1)
json_dict = xmltodict.parse(s, force_list={'item'})
And
2)
# Removes a level in python dict if it has only one specific key
#
# Examples:
# recursive_skip_dict_key_level({"c": {"a": "b"}}, "c") # -> {"a", "b"}
# recursive_skip_dict_key_level({"c": ["a", "b"]}, "c") # -> ["a", "b"]
#
def recursive_skip_dict_key_level(d, skipped_key):
if issubclass(type(d), dict):
if list(d.keys()) == [skipped_key]:
return recursive_skip_dict_key_level(d[skipped_key], skipped_key)
else:
for key in d.keys():
d[key] = recursive_skip_dict_key_level(d[key], skipped_key)
return d
elif issubclass(type(d), list):
new_list = []
for e in d:
new_list.append(recursive_skip_dict_key_level(e, skipped_key))
return new_list
else:
return d
# Removes None values from a dict
#
# Examples:
# recursive_remove_none({"a": None}) # -> {}
# recursive_remove_none([None]) # -> []
#
def recursive_remove_none(d):
if issubclass(type(d), dict):
new_dict = {}
for key in d.keys():
if not (d[key] is None):
new_dict[key] = recursive_remove_none(d[key])
return new_dict
elif issubclass(type(d), list):
new_list = []
for e in d:
if not (e is None):
new_list.append(recursive_remove_none(e))
return new_list
else:
return d
json_dict = recursive_skip_dict_key_level(json_dict, "item")
json_dict = recursive_remove_none(json_dict)
to interpret any "item" XML-elements as lists.
In addition to the existing answers, xmltodict now also supports the following to force everything to be a list:
xml = xmltodict.parse(s, force_list=True)

Recursively dir() a python object to find values of a certain type or with a certain value

I have a complex Python data structure (if it matters, it's a large music21 Score object) which will not pickle due to the presence of a weakref somewhere deep inside the object structure. I've debugged such problems with the stack trace and python debugger before, but it's always a big pain. Is there a tool which runs dir() recursively on all attributes of an object, finding objects hidden in lists, tuples, dicts, etc., and returns those that match a certain value (a lambda function or something like that). A big problem is recursive references, so some sort of memo function (like copy.deepcopy uses) is needed. I tried:
import weakref
def findWeakRef(streamObj, memo=None):
weakRefList = []
if memo is None:
memo = {}
for x in dir(streamObj):
xValue = getattr(streamObj, x)
if id(xValue) in memo:
continue
else:
memo[id(xValue)] = True
if type(xValue) is weakref.ref:
weakRefList.append(x, xValue, streamObj)
if hasattr(xValue, "__iter__"):
for i in xValue:
if id(i) in memo:
pass
else:
memo[id(i)] = True
weakRefList.extend(findWeakRef(i), memo)
else:
weakRefList.extend(findWeakRef(xValue), memo)
return weakRefList
I can probably continue plugging holes in this (the iter isn't what I'd want for dicts, for instance), but before I throw more time into it, wondering if someone knows an easier answer. It could be a pretty useful general tool.
This seems to be the start of an answer. I had to backport some items from the Python 3.2 inspect.getattr_static to make it work so it didn't call properties that just kept generating new objects. Here's the code I came up with:
#-------------------------------------------------------------------------------
# Name: treeYield.py
# Purpose: traverse a complex datastructure and yield elements
# that fit a given criteria
#
# Authors: Michael Scott Cuthbert
#
# Copyright: Copyright © 2012 Michael Scott Cuthbert
# License: CC-BY
#-------------------------------------------------------------------------------
import types
class TreeYielder(object):
def __init__(self, yieldValue = None):
'''
`yieldValue` should be a lambda function that
returns True/False or a function/method call that
will be passed the value of a current attribute
'''
self.currentStack = []
self.yieldValue = yieldValue
self.stackVals = []
t = types
self.nonIterables = [t.IntType, t.StringType, t.UnicodeType, t.LongType,
t.FloatType, t.NoneType, t.BooleanType]
def run(self, obj, memo = None):
'''
traverse all attributes of an object looking
for subObjects that meet a certain criteria.
yield them.
`memo` is a dictionary to keep track of objects
that have already been seen
The original object is added to the memo and
also checked for yieldValue
'''
if memo is None:
memo = {}
self.memo = memo
if id(obj) in self.memo:
self.memo[id(obj)] += 1
return
else:
self.memo[id(obj)] = 1
if self.yieldValue(obj) is True:
yield obj
### now check for sub values...
self.currentStack.append(obj)
tObj = type(obj)
if tObj in self.nonIterables:
pass
elif tObj == types.DictType:
for keyX in obj:
dictTuple = ('dict', keyX)
self.stackVals.append(dictTuple)
x = obj[keyX]
for z in self.run(x, memo=memo):
yield z
self.stackVals.pop()
elif tObj in [types.ListType, types.TupleType]:
for i,x in enumerate(obj):
listTuple = ('listLike', i)
self.stackVals.append(listTuple)
for z in self.run(x, memo=memo):
yield z
self.stackVals.pop()
else: # objects or uncaught types...
### from http://bugs.python.org/file18699/static.py
try:
instance_dict = object.__getattribute__(obj, "__dict__")
except AttributeError:
## probably uncaught static object
return
for x in instance_dict:
try:
gotValue = object.__getattribute__(obj, x)
except: # ?? property that relies on something else being set.
continue
objTuple = ('getattr', x)
self.stackVals.append(objTuple)
try:
for z in self.run(gotValue, memo=memo):
yield z
except RuntimeError:
raise Exception("Maximum recursion on:\n%s" % self.currentLevel())
self.stackVals.pop()
self.currentStack.pop()
def currentLevel(self):
currentStr = ""
for stackType, stackValue in self.stackVals:
if stackType == 'dict':
if isinstance(stackValue, str):
currentStr += "['" + stackValue + "']"
elif isinstance(stackValue, unicode):
currentStr += "[u'" + stackValue + "']"
else: # numeric key...
currentStr += "[" + str(stackValue) + "]"
elif stackType == 'listLike':
currentStr += "[" + str(stackValue) + "]"
elif stackType == 'getattr':
currentStr += ".__getattribute__('" + stackValue + "')"
else:
raise Exception("Cannot get attribute of type %s" % stackType)
return currentStr
This code lets you run something like this:
class Mock(object):
def __init__(self, mockThing, embedMock = True):
self.abby = 30
self.mocker = mockThing
self.mockList = [mockThing, mockThing, 40]
self.embeddedMock = None
if embedMock is True:
self.embeddedMock = Mock(mockThing, embedMock = False)
mockType = lambda x: x.__class__.__name__ == 'Mock'
subList = [100, 60, -2]
myList = [5, 20, [5, 12, 17], 30, {'hello': 10, 'goodbye': 22, 'mock': Mock(subList)}, -20, Mock(subList)]
myList.append(myList)
ty = TreeYielder(mockType)
for val in ty.run(myList):
print(val, ty.currentLevel())
And get:
(<__main__.Mock object at 0x01DEBD10>, "[4]['mock']")
(<__main__.Mock object at 0x01DEF370>, "[4]['mock'].__getattribute__('embeddedMock')")
(<__main__.Mock object at 0x01DEF390>, '[6]')
(<__main__.Mock object at 0x01DEF3B0>, "[6].__getattribute__('embeddedMock')")
Or run:
high = lambda x: isinstance(x, (int, float)) and x > 10
ty = TreeYielder(high)
for val in ty.run(myList):
print(val, ty.currentLevel())
And get:
(20, '[1]')
(12, '[2][1]')
(17, '[2][2]')
(30, '[3]')
(22, "[4]['goodbye']")
(100, "[4]['mock'].__getattribute__('embeddedMock').__getattribute__('mocker')[0]")
(60, "[4]['mock'].__getattribute__('embeddedMock').__getattribute__('mocker')[1]")
(40, "[4]['mock'].__getattribute__('embeddedMock').__getattribute__('mockList')[2]")
I'm still trying to figure out why .abby isn't found, but I figure it's worth posting even at this point, since it's much more on the right track than I was when I started.
Here is a simpler solution that is somewhat naive. I.e. just a depth first search under the attribute tree. If its a primitive then stop , otherwise go deeper in the tree. It will get you the call tree and the value at the leaf.
def recursive_dir(obj, path):
if ((obj!=None) and (not isinstance(obj, (str,float,int,list,dict,set)))):
for attr, val in obj.__dict__.iteritems():
temp_path = path[:]
temp_path.append(attr)
recursive_dir(getattr(obj, attr), temp_path)
else:
print (path, "--->", obj)
print("")
recursive_dir(x,[])

Hash Map in Python

I want to implement a HashMap in Python. I want to ask a user for an input. depending on his input I am retrieving some information from the HashMap. If the user enters a key of the HashMap, I would like to retrieve the corresponding value.
How do I implement this functionality in Python?
HashMap<String,String> streetno=new HashMap<String,String>();
streetno.put("1", "Sachin Tendulkar");
streetno.put("2", "Dravid");
streetno.put("3","Sehwag");
streetno.put("4","Laxman");
streetno.put("5","Kohli")
Python dictionary is a built-in type that supports key-value pairs. It's the nearest builtin data structure relative to Java's HashMap.
You can declare a dict with key-value pairs set to values:
streetno = {
"1": "Sachin Tendulkar",
"2": "Dravid",
"3": "Sehwag",
"4": "Laxman",
"5": "Kohli"
}
You can also set a key-value mapping after creation:
streetno = {}
streetno["1"] = "Sachin Tendulkar"
print(streetno["1"]) # => "Sachin Tendulkar"
Another way to create a dictionary is with the dict() builtin function, but this only works when your keys are valid identifiers:
streetno = dict(one="Sachin Tendulkar", two="Dravid")
print(streetno["one"]) # => "Sachin Tendulkar"
All you wanted (at the time the question was originally asked) was a hint. Here's a hint: In Python, you can use dictionaries.
It's built-in for Python. See dictionaries.
Based on your example:
streetno = {"1": "Sachine Tendulkar",
"2": "Dravid",
"3": "Sehwag",
"4": "Laxman",
"5": "Kohli" }
You could then access it like so:
sachine = streetno["1"]
Also worth mentioning: it can use any non-mutable data type as a key. That is, it can use a tuple, boolean, or string as a key.
Hash maps are built-in in Python, they're called dictionaries:
streetno = {} #create a dictionary called streetno
streetno["1"] = "Sachin Tendulkar" #assign value to key "1"
Usage:
"1" in streetno #check if key "1" is in streetno
streetno["1"] #get the value from key "1"
See the documentation for more information, e.g. built-in methods and so on. They're great, and very common in Python programs (unsurprisingly).
streetno = { 1 : "Sachin Tendulkar",
2 : "Dravid",
3 : "Sehwag",
4 : "Laxman",
5 : "Kohli" }
And to retrieve values:
name = streetno.get(3, "default value")
Or
name = streetno[3]
That's using number as keys, put quotes around the numbers to use strings as keys.
Here is the implementation of the Hash Map using python
For the simplicity hash map is of a fixed size 16.
This can be changed easily.
Rehashing is out of scope of this code.
class Node:
def __init__(self, key, value):
self.key = key
self.value = value
self.next = None
class HashMap:
def __init__(self):
self.store = [None for _ in range(16)]
def get(self, key):
index = hash(key) & 15
if self.store[index] is None:
return None
n = self.store[index]
while True:
if n.key == key:
return n.value
else:
if n.next:
n = n.next
else:
return None
def put(self, key, value):
nd = Node(key, value)
index = hash(key) & 15
n = self.store[index]
if n is None:
self.store[index] = nd
else:
if n.key == key:
n.value = value
else:
while n.next:
if n.key == key:
n.value = value
return
else:
n = n.next
n.next = nd
hm = HashMap()
hm.put("1", "sachin")
hm.put("2", "sehwag")
hm.put("3", "ganguly")
hm.put("4", "srinath")
hm.put("5", "kumble")
hm.put("6", "dhoni")
hm.put("7", "kohli")
hm.put("8", "pandya")
hm.put("9", "rohit")
hm.put("10", "dhawan")
hm.put("11", "shastri")
hm.put("12", "manjarekar")
hm.put("13", "gupta")
hm.put("14", "agarkar")
hm.put("15", "nehra")
hm.put("16", "gawaskar")
hm.put("17", "vengsarkar")
print(hm.get("1"))
print(hm.get("2"))
print(hm.get("3"))
print(hm.get("4"))
print(hm.get("5"))
print(hm.get("6"))
print(hm.get("7"))
print(hm.get("8"))
print(hm.get("9"))
print(hm.get("10"))
print(hm.get("11"))
print(hm.get("12"))
print(hm.get("13"))
print(hm.get("14"))
print(hm.get("15"))
print(hm.get("16"))
print(hm.get("17"))
Output:
sachin
sehwag
ganguly
srinath
kumble
dhoni
kohli
pandya
rohit
dhawan
shastri
manjarekar
gupta
agarkar
nehra
gawaskar
vengsarkar
class HashMap:
def __init__(self):
self.size = 64
self.map = [None] * self.size
def _get_hash(self, key):
hash = 0
for char in str(key):
hash += ord(char)
return hash % self.size
def add(self, key, value):
key_hash = self._get_hash(key)
key_value = [key, value]
if self.map[key_hash] is None:
self.map[key_hash] = list([key_value])
return True
else:
for pair in self.map[key_hash]:
if pair[0] == key:
pair[1] = value
return True
else:
self.map[key_hash].append(list([key_value]))
return True
def get(self, key):
key_hash = self._get_hash(key)
if self.map[key_hash] is not None:
for pair in self.map[key_hash]:
if pair[0] == key:
return pair[1]
return None
def delete(self, key):
key_hash = self._get_hash(key)
if self.map[key_hash] is None :
return False
for i in range(0, len(self.map[key_hash])):
if self.map[key_hash][i][0] == key:
self.map[key_hash].pop(i)
return True
def print(self):
print('---Phonebook---')
for item in self.map:
if item is not None:
print(str(item))
h = HashMap()
Python Counter is also a good option in this case:
from collections import Counter
counter = Counter(["Sachin Tendulkar", "Sachin Tendulkar", "other things"])
print(counter)
This returns a dict with the count of each element in the list:
Counter({'Sachin Tendulkar': 2, 'other things': 1})
In python you would use a dictionary.
It is a very important type in python and often used.
You can create one easily by
name = {}
Dictionaries have many methods:
# add entries:
>>> name['first'] = 'John'
>>> name['second'] = 'Doe'
>>> name
{'first': 'John', 'second': 'Doe'}
# you can store all objects and datatypes as value in a dictionary
# as key you can use all objects and datatypes that are hashable
>>> name['list'] = ['list', 'inside', 'dict']
>>> name[1] = 1
>>> name
{'first': 'John', 'second': 'Doe', 1: 1, 'list': ['list', 'inside', 'dict']}
You can not influence the order of a dict.
A dictionary in Python is the best way to implement this. We can create the following dictionary using the given <key,value> pairs:
d = {"1": "Sachin Tendulkar", "2": "Dravid", "3": "Sehwag", "4": "Laxman", "5": "Kohli"}
To extract the value of a particular key, we can directly use d[key]:
name = d["1"] # The value of name would be "Sachin Tendulkar" here
That is my solution for LeetCode problem 706:
A hash map class with three methods: get, put and remove
class Item:
def __init__(self, key, value):
self.key = key
self.value = value
self.next = None
class MyHashMap:
def __init__(self, size=100):
self.items = [None] * size
self.size = size
def _get_index(self, key):
return hash(key) & self.size-1
def put(self, key: int, value: int) -> None:
index = self._get_index(key)
item = self.items[index]
if item is None:
self.items[index] = Item(key, value)
else:
if item.key == key:
item.value = value
else:
while True:
if item.key == key:
item.value = value
return
else:
if not item.next:
item.next = Item(key, value)
return
item = item.next
def get(self, key: int) -> int:
index = self._get_index(key)
if self.items[index] is None:
return -1
item = self.items[index]
while True:
if item.key == key:
return item.value
else:
if item.next:
item = item.next
else:
return -1
def remove(self, key: int) -> None:
value = self.get(key)
if value > -1:
index = self._get_index(key)
item = self.items[index]
if item.key == key:
self.items[index] = item.next if item.next else None
return
while True:
if item.next and item.next.key == key:
item.next = item.next.next
return
else:
if item.next:
item = item.next
else:
return

MongoAlchemy Document Encode to JSON via Flask-MongoAlchemy

I think I am missing something small here. I am testing out Python framework Flask and Flask-MongoAlchemy, and want to convert an entity into JSON output. Here is my code (abstracted):
from flask import Flask
from flaskext.mongoalchemy import MongoAlchemy
try:
from bson.objectid import ObjectId
except:
pass
#a bunch of code to open the mongoDB
class ClassA(db.Document):
title = db.StringField()
field1 = db.StringField()
field2 = db.BoolField()
#app.route('/api/classA', methods=['GET'])
def api_list_all
a = ClassA.query.all()
result = []
for b in a:
result.append(b.wrap())
print result
return json.dumps(result)
Without the json.dumps line, the print statement prompt the right result. But only if I run the json.dumps on result, it yields:
TypeError: ObjectId('...') is not JSON serializable
What am I missing?
The result is a mongo document of some sort that contains ObjectId-type content, which you'll have to tell json how to deserialize. You'll have the same problem with other mongo-specific types, such as ReferenceField(), EmbeddedDocumentField(), etc.
You have to write a deserialization function that you can pass to json. What I use is:
def encode_model(obj, recursive=False):
if obj is None:
return obj
if isinstance(obj, (mongoengine.Document, mongoengine.EmbeddedDocument)):
out = dict(obj._data)
for k,v in out.items():
if isinstance(v, ObjectId):
if k is None:
out['_id'] = str(v)
del(out[k])
else:
# Unlikely that we'll hit this since ObjectId is always NULL key
out[k] = str(v)
else:
out[k] = encode_model(v)
elif isinstance(obj, mongoengine.queryset.QuerySet):
out = encode_model(list(obj))
elif isinstance(obj, ModuleType):
out = None
elif isinstance(obj, groupby):
out = [ (g,list(l)) for g,l in obj ]
elif isinstance(obj, (list)):
out = [encode_model(item) for item in obj]
elif isinstance(obj, (dict)):
out = dict([(k,encode_model(v)) for (k,v) in obj.items()])
elif isinstance(obj, datetime.datetime):
out = str(obj)
elif isinstance(obj, ObjectId):
out = {'ObjectId':str(obj)}
elif isinstance(obj, (str, unicode)):
out = obj
elif isinstance(obj, float):
out = str(obj)
else:
raise TypeError, "Could not JSON-encode type '%s': %s" % (type(obj), str(obj))
return out
Then you'd process the result as:
return json.dumps(result, default=encode_model)
or something to this effect.
You can also use the query.raw_output() method to have that query instance return raw Python dictionaries instead of a Python object. With a dictionary it becomes easy to encode to JSON using json.dumps():
import json
q=db.query(MyObject)
q.raw_output()
json.dumps(q.first())
Reference http://www.mongoalchemy.org/api/expressions/query.html#mongoalchemy.query.Query.raw_output
Combining the previous two answers, you should be able to do something like this:
from bson import json_util
# ...
#app.route('/api/classA', methods=['GET'])
def api_list_all
a = ClassA.query.all()
result = []
for b in a:
result.append(b.wrap())
print result
return json_utils.dumps(result) # Change here.

Serializing a suds object in python

Ok I'm working on getting better with python, so I'm not sure this is the right way to go about what I'm doing to begin with, but here's my current problem...
I need to get some information via a SOAP method, and only use part of the information now but store the entire result for future uses (we need to use the service as little as possible). Looking up the best way to access the service I figured suds was the way to go, and it was simple and worked like a charm to get the data. But now I want to save the result somehow, preferably serialized / in a database so I can pull it out later and use it the same.
What's the best way to do this, it looks like pickle/json isn't an option? Thanks!
Update
Reading the top answer at How can I pickle suds results? gives me a better idea of why this isn't an option, I guess I'm stuck recreating a basic object w/ the information I need?
I have been using following approach to convert Suds object into JSON:
from suds.sudsobject import asdict
def recursive_asdict(d):
"""Convert Suds object into serializable format."""
out = {}
for k, v in asdict(d).items():
if hasattr(v, '__keylist__'):
out[k] = recursive_asdict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(recursive_asdict(item))
else:
out[k].append(item)
else:
out[k] = v
return out
def suds_to_json(data):
return json.dumps(recursive_asdict(data))
Yep, I confirm the explanation I gave in the answer you refer to -- dynamically generated classes are not easily picklable (nor otherwise easily serializable), you need to extract all the state information, pickle that state, and reconstruct the tricky sudsobject on retrieval if you really insist on using it;-).
Here is what I came up with before researching and finding this answer. This actually works well for me on complex suds responses and also on other objects such as __builtins__ since the solution is suds agnostic:
import datetime
def object_to_dict(obj):
if isinstance(obj, (str, unicode, bool, int, long, float, datetime.datetime, datetime.date, datetime.time)):
return obj
data_dict = {}
try:
all_keys = obj.__dict__.keys() # vars(obj).keys()
except AttributeError:
return obj
fields = [k for k in all_keys if not k.startswith('_')]
for field in fields:
val = getattr(obj, field)
if isinstance(val, (list, tuple)):
data_dict[field] = []
for item in val:
data_dict[field].append(object_to_dict(item))
else:
data_dict[field] = object_to_dict(val)
return data_dict
This solution works and is actually faster. It also works on objects that don't have the __keylist__ attribute.
I ran a benchmark 100 times on a complex suds output object, this solutions run time was 0.04 to .052 seconds (0.045724287 average). While recursive_asdict solution above ran in .082 to 0.102 seconds so nearly double (0.0829765582 average).
I then went back to the drawing board and re-did the function to get more performance out of it, and it does not need the datetime import. I leveraged in using the __keylist__ attribute, so this will not work on other objects such as __builtins__ but works nicely for suds object output:
def fastest_object_to_dict(obj):
if not hasattr(obj, '__keylist__'):
return obj
data = {}
fields = obj.__keylist__
for field in fields:
val = getattr(obj, field)
if isinstance(val, list): # tuple not used
data[field] = []
for item in val:
data[field].append(fastest_object_to_dict(item))
else:
data[field] = fastest_object_to_dict(val)
return data
The run time was 0.18 - 0.033 seconds (0.0260889721 average), so nearly 4x as faster than the recursive_asdict solution.
I made an implementation of a dummy class for Object intance of suds, and then being able to serialize. The FakeSudsInstance behaves like an original Suds Object instance, see below:
from suds.sudsobject import Object as SudsObject
class FakeSudsNode(SudsObject):
def __init__(self, data):
SudsObject.__init__(self)
self.__keylist__ = data.keys()
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, FakeSudsNode(value))
elif isinstance(value, list):
l = []
for v in value:
if isinstance(v, list) or isinstance(v, dict):
l.append(FakeSudsNode(v))
else:
l.append(v)
setattr(self, key, l)
else:
setattr(self, key, value)
class FakeSudsInstance(SudsObject):
def __init__(self, data):
SudsObject.__init__(self)
self.__keylist__ = data.keys()
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, FakeSudsNode(value))
else:
setattr(self, key, value)
#classmethod
def build_instance(cls, instance):
suds_data = {}
def node_to_dict(node, node_data):
if hasattr(node, '__keylist__'):
keys = node.__keylist__
for key in keys:
if isinstance(node[key], list):
lkey = key.replace('[]', '')
node_data[lkey] = node_to_dict(node[key], [])
elif hasattr(node[key], '__keylist__'):
node_data[key] = node_to_dict(node[key], {})
else:
if isinstance(node_data, list):
node_data.append(node[key])
else:
node_data[key] = node[key]
return node_data
else:
if isinstance(node, list):
for lnode in node:
node_data.append(node_to_dict(lnode, {}))
return node_data
else:
return node
node_to_dict(instance, suds_data)
return cls(suds_data)
Now, after a suds call, for example below:
# Now, after a suds call, for example below
>>> import cPickle as pickle
>>> suds_intance = client.service.SomeCall(account, param)
>>> fake_suds = FakeSudsInstance.build_instance(suds_intance)
>>> dumped = pickle.dumps(fake_suds)
>>> loaded = pickle.loads(dumped)
I hope it helps.
The solutions suggesed above lose valuable information about class names - it can be of value in some libraries like DFP client https://github.com/googleads/googleads-python-lib where entity types might be encoded in dynamically generated class names (i.e. TemplateCreative/ImageCreative)
Here's the solution I used that preserves class names and restores dict-serialized objects without data loss (except suds.sax.text.Text which would be converted into regular unicode objects and maybe some other types I haven't run into)
from suds.sudsobject import asdict, Factory as SudsFactory
def suds2dict(d):
"""
Suds object serializer
Borrowed from https://stackoverflow.com/questions/2412486/serializing-a-suds-object-in-python/15678861#15678861
"""
out = {'__class__': d.__class__.__name__}
for k, v in asdict(d).iteritems():
if hasattr(v, '__keylist__'):
out[k] = suds2dict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(suds2dict(item))
else:
out[k].append(item)
else:
out[k] = v
return out
def dict2suds(d):
"""
Suds object deserializer
"""
out = {}
for k, v in d.iteritems():
if isinstance(v, dict):
out[k] = dict2suds(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if isinstance(item, dict):
out[k].append(dict2suds(item))
else:
out[k].append(item)
else:
out[k] = v
return SudsFactory.object(out.pop('__class__'), out)
I updated the recursive_asdict example above to be compatible with python3 (items instead of iteritems).
from suds.sudsobject import asdict
from suds.sax.text import Text
def recursive_asdict(d):
"""
Recursively convert Suds object into dict.
We convert the keys to lowercase, and convert sax.Text
instances to Unicode.
Taken from:
https://stackoverflow.com/a/15678861/202168
Let's create a suds object from scratch with some lists and stuff
>>> from suds.sudsobject import Object as SudsObject
>>> sudsobject = SudsObject()
>>> sudsobject.Title = "My title"
>>> sudsobject.JustAList = [1, 2, 3]
>>> sudsobject.Child = SudsObject()
>>> sudsobject.Child.Title = "Child title"
>>> sudsobject.Child.AnotherList = ["4", "5", "6"]
>>> childobject = SudsObject()
>>> childobject.Title = "Another child title"
>>> sudsobject.Child.SudObjectList = [childobject]
Now see if this works:
>>> result = recursive_asdict(sudsobject)
>>> result['title']
'My title'
>>> result['child']['anotherlist']
['4', '5', '6']
"""
out = {}
for k, v in asdict(d).items():
k = k.lower()
if hasattr(v, '__keylist__'):
out[k] = recursive_asdict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(recursive_asdict(item))
else:
out[k].append(
item.title() if isinstance(item, Text) else item)
else:
out[k] = v.title() if isinstance(v, Text) else v
return out
I like this way. We don't do the iteration ourselves, it is python that iterates when converting it to string
class Ob:
def __init__(self, J) -> None:
self.J = J
def __str__(self):
if hasattr(self.J, "__keylist__"):
self.J = {key: Ob(value) for key, value in dict(self.J).items()}
if hasattr(self.J, "append"):
self.J = [Ob(data) for data in sefl.J]
return str(self.J)
result = Ob(result_soap)

Categories

Resources