for p in db.collection.find({"test_set":"abc"}):
posts.append(p)
thejson = json.dumps({'results':posts})
return HttpResponse(thejson, mimetype="application/javascript")
In my Django/Python code, I can't return a JSON from a mongo query because of "ObjectID". The error says that "ObjectID" is not serializable.
What do I have to do?
A hacky way would be to loop through:
for p in posts:
p['_id'] = ""
The json module won't work due to things like the ObjectID.
Luckily PyMongo provides json_util which ...
... allow[s] for specialized encoding and
decoding of BSON documents into Mongo
Extended JSON's Strict mode. This lets
you encode / decode BSON documents to
JSON even when they use special BSON
types.
Here is a simple sample, using pymongo 2.2.1
import os
import sys
import json
import pymongo
from bson import BSON
from bson import json_util
if __name__ == '__main__':
try:
connection = pymongo.Connection('mongodb://localhost:27017')
database = connection['mongotest']
except:
print('Error: Unable to Connect')
connection = None
if connection is not None:
database["test"].insert({'name': 'foo'})
doc = database["test"].find_one({'name': 'foo'})
return json.dumps(doc, sort_keys=True, indent=4, default=json_util.default)
It's pretty easy to write a custom serializer which copes with the ObjectIds. Django already includes one which handles decimals and dates, so you can extend that:
from django.core.serializers.json import DjangoJSONEncoder
from bson import objectid
class MongoAwareEncoder(DjangoJSONEncoder):
"""JSON encoder class that adds support for Mongo objectids."""
def default(self, o):
if isinstance(o, objectid.ObjectId):
return str(o)
else:
return super(MongoAwareEncoder, self).default(o)
Now you can just tell json to use your custom serializer:
thejson = json.dumps({'results':posts}, cls=MongoAwareEncoder)
Something even simpler which works for me on Python 3.6 using
motor==1.1
pymongo==3.4.0
from bson.json_util import dumps, loads
for mongo_doc in await cursor.to_list(length=10):
# mongo_doc is a <class 'dict'> returned from the async mongo driver, in this acse motor / pymongo.
# result of executing a simple find() query.
json_string = dumps(mongo_doc)
# serialize the <class 'dict'> into a <class 'str'>
back_to_dict = loads(json_string)
# to unserialize, thus return the string back to a <class 'dict'> with the original 'ObjectID' type.
Related
so far in my code bellow I managed to store my data into mongoDB.
Now I want to be able to retrieve the data I have stored.
As you can see I have been trying but keep on getting an error.
With BSON do I have to first decode the data to retrieve it from mongoDB?
Any help would be greatly appreciated!
(Apologies for the messy code, I am just practicing through trial and error)
import json
from json import JSONEncoder
import pymongo
from pymongo import MongoClient
from bson.binary import Binary
import pickle
#Do this for each
client = MongoClient("localhost", 27017)
db = client['datacampdb']
coll = db.personpractice4_collection #creating a collection in the database
#my collection on the database is called personpractice4_collection
class Person:
def __init__(self, norwegian, dame, brit, german, sweed):
self.__norwegian = norwegian
self.__dame = dame
self.__brit = brit
self.__german = german #private variable
self.__sweed = sweed
# create getters and setters later to make OOP
personone = Person("norwegian", "dame", "brit", "german","sweed")
class PersonpracticeEncoder(JSONEncoder):
def default(self, o):
return o.__dict__
#Encode Person Object into JSON"
personpracticeJson = json.dumps(personone, indent=4, cls=PersonpracticeEncoder)
practicedata = pickle.dumps(personpracticeJson)
coll.insert_one({'bin-data': Binary(practicedata)})
#print(personpracticeJson)
#print(db.list_collection_names()) #get then names of my collections in DB
#retriving data from mongodb
#Retrieving a Single Document with find_one()
print(({'bin-data': Binary(practicedata)}).find_one()) #not working
the find_one method should be called on a collection
{'bin-data': Binary(practicedata)} is a query to find a document
coll.find_one({'bin-data': Binary(practicedata)})
Witch means : Find a document in the collection coll where bin-data is equal to Binary(practicedata)
I've been trying to send some integer values over socket to a LabVIEW client using json.dumps but as the numbers change the size of each field may change, I would like to know if there is a way to pad the number with '0' without turning it into a string when I do the json dump, as it adds " " to the packet send around each number.
Example:
data = json.dumps({"Data": str(52).zfill(4)]})
self.sock.send(data.encode())
This sends
'"Data":"0052"'
I want
'"Data": 0052'
As #jsonharper mentioned, technically what you're asking for is no longer JSON, more on that here
However, that doesn't mean you can't use the json library to do the bulk of the work for you!
You can achieve this by passing a custom encoder class to json.dumps like this:
>>> import json
>>> class MyInt(int):
... def __str__(self):
... return '{:0>4}'.format(self)
>>> class MyEncoder(json.encoder.JSONEncoder):
... def default(self, o):
... if isinstance(o, MyInt):
... return str(o)
... return super(MyEncoder, self).default(o)
>>> obj = {'Data': MyInt(52)}
>>> json.dumps(obj, cls=MyEncoder)
'{"Data": 0052}'
You can do this with any class, but this can result in something that can't be decoded again with a strict JSON decoder.
See if you can get LabVIEW to read standard JSON, but if not, the above should work.
Trying to define a function in python that can search for a given UUID like so:
def getid(in_id):
return list(CollectionVar.find({"_id":UUID(in_id)}))
And passing in a UUID. I can take a UUID I know exists from Studio 3T like so:
db.getCollection("CollectionName").find({"_id":UUID("5002aa11-eeb7-4e68-a121-dd51497d2572")})
And the above query returns precisely one document. That same UUID in the python query returns absolutely nothing. I can find documents on other (non UUID) fields easily enough, for example the following works fine on that same document from earlier:
def getname(fn,sn):
return list(CollectionVar.find({"Firstname":re.compile(fn, re.IGNORECASE), "Surname":re.compile(sn, re.IGNORECASE)}))
This seems like a problem with the uuid.UUID class rather than a pymongo issue? Can anyone see the problem?
PyMongo Version 3.6.1
The issue is that PyMongo uses a legacy method of encoding/decoding UUID values by default. You probably want to configure the PyMongo client to use the more modern, cross-language compatible "standard" UUID representation:
client = MongoClient(MONGODB_URI, uuidRepresentation="standard")
Now you should be able to query directly using Python uuid.UUID instances:
from uuid import UUID
items = client["item_database"]["items"].find_one({
"uuid": UUID("187382af-1369-43e6-a0ba-d345886c986c")
})
I've solved this. For anyone else who hits this issue the solution is below:
from bson.binary import Binary, UUID_SUBTYPE
def getcust(inid):
newuuid=uuid.UUID(inid).bytes
return list(DealershipConsumer.find({"_id": Binary(bytes(bytearray(newuuid)), UUID_SUBTYPE)}))
UUID_SUBTYPE needs to be set to whatever subtype of UUID you use - in my case it's 4.
You could specify the UUID type you're using when getting the db :
import bson
import pymongo
mongo_client = pymongo.MongoClient(mongo_uri, document_class=dict)
db = mongo_client.get_database(
"my_db_name",
bson.codec_options.CodecOptions(uuid_representation=bson.binary.UUID_SUBTYPE),
)
If you want to read more about Mongo best practices while using UUIDs, this article might help.
Also, here are the docs about codec_options
I've scraped this from pymongo-2.8.1 in the bson.binary.UUIDLegacy class's docstring/comment, might be usefull
>>> import uuid
>>> from bson.binary import Binary, UUIDLegacy, UUID_SUBTYPE
>>> my_uuid = uuid.uuid4()
>>> coll = db.test
>>> coll.uuid_subtype = UUID_SUBTYPE
>>> coll.insert({'uuid': Binary(my_uuid.bytes, 3)})
ObjectId('...')
>>> coll.find({'uuid': my_uuid}).count()
0
>>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count()
1
>>> coll.find({'uuid': UUIDLegacy(my_uuid)})[0]['uuid']
UUID('...')
>>>
>>> # Convert from subtype 3 to subtype 4
>>> doc = coll.find_one({'uuid': UUIDLegacy(my_uuid)})
>>> coll.save(doc)
ObjectId('...')
>>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count()
0
>>> coll.find({'uuid': {'$in': [UUIDLegacy(my_uuid), my_uuid]}}).count()
1
>>> coll.find_one({'uuid': my_uuid})['uuid']
UUID('...')
You need to use ObjectId instead of UUID.
Try this, it works for me:
from bson.objectid import ObjectId
def getid(in_id):
return list(CollectionVar.find({"_id":ObjectId(in_id)}))
I have this String that I need to pass into a REST request
{"notification":{"tag":"MyTag"}}
I'm trying to turn into an object using the JSON module in python.
This is my attempt so far
import json
obj = json.dumps([{'notification': ('{tag : MyTag}')}])
But it isn't parsed correctly so the REST request won't work. Anyone have any ideas?
Just dump your dictionary as is, replace:
obj = json.dumps([{'notification': ('{tag : MyTag}')}])
with:
obj = json.dumps({"notification": {"tag": "MyTag"}})
I've using feedparser module to parse the RSS feeds. I need to pass the feedparser object to a celery task.
Upon trying to pass the object, I receive an error saying time.struct_time(tm_year=2015, tm_mon=2, tm_mday=12, tm_hour=8, tm_min=19, tm_sec=11, tm_wday=3, tm_yday=43, tm_isdst=0) is not JSON serializable
How do I pass the feedparser object to a celery task?
Here is my code:-
rss_content = feedparser.parse(rss_link)
content_entries = rss_content['entries']
for content in content_entries:
parse_link.apply_async(args=[content, link, json_id, news_category], queue= news_source) #celery task
How do I do it?
You need to create your custom encoder and decoder that will basically convert your time.time_struct object into something serializable (a dict), and then register them in the kombu serializer registry as described in the docs, in order to let celery use your new serializer in its task.
import json
import time
import types
import datetime
class FeedContentEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, time_struct):
epoch = int(time.mktime(time_struct))
return {'__type__': '__time__', 'time': epoch}
else:
return json.FeedContentEncoder.default(self, obj)
def decode_feed_content(obj):
if isinstance(obj, types.DictionaryType) and '__type__' in obj:
if obj['__type__'] == '__time__':
return datetime.datetime.fromtimestamp(obj['time']).timetuple()
return obj
You need to notify kombu about your new serialization by registering them into the serializer registry.
from kombu.serialization import register
def feed_content_json_dumps(obj):
return json.dumps(obj, cls=FeedContentEncoder)
def feed_content_json_loads(obj):
return json.loads(obj, object_hook=decode_feed_content)
register('feedcontentjson',
feed_content_json_dumps,
feed_content_json_loads,
content_type='application/x-feedcontent-json',
content_encoding='utf-8')
Finally, you should tell celery to use the new serializer for serializing the task just like the celery docs; you should call your task with the serializer parameter.
parse_link.apply_async(args=[content, link, json_id, news_category], queue= news_source, serializer='feedcontentjson')
Hope this helps.