Develop Reference

Updating a MongoDB document if field doesn't exist

Whenever I updated my insert_one with a new field to use, I had to always delete the old posts in the collection. I know there are manual methods of updating such fields using update_many but I know it's inefficient.
For example:
posts.insert_one({
"id": random.randint(1,10000)
"value1": "value1",
"value2": "value2"
})
I use the following code to check if the document exists or not. How would this work for a field?
if posts.find({'id': 12312}).count() > 0:
I know I can easily overwrite the previous data but I know people won't enjoy having their data wiped every other month.
Is there a way to add the field to a document in Python?

How would this work for a field?
You can use $exists to check whether a field exists in a doc.
In your case, you can combine this with find
find({ 'id':1, "fieldToCheck":{$exists:"true"}})
It will return the doc if it exists with id = 1, fieldToCheck is present in doc with id = 1
You can skip id=1, in that case, it will return all docs where fieldToCheck exists
Is there a way to add the field to a document in Python?
You could use update with new field, it will update if it is present else it will insert.
update({"_id":1}, {field:"x"})
If field is present, it will set to x else it will add with field:x
Beware of update options like multi, upsert

Yes you can you use update command in mongoDB shell to do that. check here
This is the command to use...
db.collection.update({},{$set : {"newfield":1}},false,true)
The above will work in the mongoDB shell. It will add newfield in all the documents, if it is not present.
If you want to use Python, use pymongo.
For python, following command should work
db.collection.update({},{"$set" : {"newfield":1}},False, True)

Thanks to john's answer I have made an entire solution that automatically updates documents without the need to run a task meaning you don't update inactive documents.
import datetime
import pymongo
database = pymongo.MongoClient("mongodb://localhost:27017") # Mongodb connection
db = database.maindb # Database
posts = db.items # Collection within a database
# A schema equivalent function that returns the object
def user_details(name, dob):
return {
"username": name, # a username/id
"dob": dob, # some data
"level": 0, # some other data
"latest_update": datetime.datetime.fromtimestamp(1615640176)
# Must be kept to ensure you aren't doing it that often
}
# The first schema changed for example after adding a new feature
def user_details2(name, dob, cake):
return {
"username": name, # a username/id
"dob": dob, # Some data
"level": 0, # Some other data
"cake": cake, # Some new data that isn't in the document
"latest_update": datetime.datetime.utcnow() # Must be kept to ensure you aren't doing it that often
}
def check_if_update(find, main_document,
collection): # parameters: What you find a document with, the schema dictionary, then the mongodb collection
if collection.count_documents(find) > 0: # How many documents match, only proceed if it exists
fields = {} # Init a dictionary
for x in collection.find(find): # You only want one for this to work
fields = x
if "latest_update" in fields: # Just in case it doesn't exist yet
last_time = fields["latest_update"] # Get the time that it was last updated
time_diff = datetime.datetime.utcnow() - last_time # Get the time difference between the utc time now and the time it was last updated
if time_diff.total_seconds() < 3600: # If the total seconds of the difference is smaller than an hour
print("return")
return
db_schema = main_document # Better naming
db_schema["_id"] = 0 # Adds the _id schema_key into the dictionary
if db_schema.keys() != fields:
print("in")
for schema_key, schema_value in db_schema.items():
if schema_key not in fields.keys(): # Main key for example if cake is added and doesn't exist in db fetched fields
collection.update_one(find, {"$set": {schema_key: schema_value}})
else: # Everything exists and you want to check for if a dictionary within that dictionary is changed
try:
sub_dict = dict(schema_value) # Make the value of it a dictionary
# It exists in the schema dictionary but not in the db fetched document
for key2, value2 in sub_dict.items():
if key2 not in fields[schema_key].keys():
new_value = schema_value
new_value[
key2] = value2 # Adding the key and value from the schema dictionary that was added
collection.update_one(find,
{"$set": {schema_key: new_value}})
# It exists in the db fetched document but not in the schema dictionary
for key2, value2 in fields[schema_key].items():
if key2 not in sub_dict.keys():
new_dict = {} # Get all values, filter then so that only the schema existent ones are passed back
for item in sub_dict:
if item != key2:
new_dict[item] = sub_dict.get(item)
collection.update_one(find, {"$set": {schema_key: new_dict}})
except: # Wasn't a dict
pass
# You removed a value from the schema dictionary and want to update it in the db
for key2, value2 in fields.items():
if key2 not in db_schema:
collection.update_one(find, {"$unset": {key2: 1}})
else:
collection.insert_one(main_document) # Insert it because it doesn't exist yet
print("start")
print(posts.find_one({"username": "john"}))
check_if_update({"username": "john"}, user_details("john", "13/03/2021"), posts)
print("inserted")
print(posts.find_one({"username": "john"}))
check_if_update({"username": "john"}, user_details2("john", "13/03/2021", "Lemon drizzle"), posts)
print("Results:")
print(posts.find_one({"username": "john"}))
It is available as a gist

Postgresql Json column not saving utf-8 character

Hi i'm trying to save data that i get from this api into my Json column in my postgresql using sqlalchemy and python requests.
r = requests.get(api)
content = r.content
data = json.loads(content)
crawl_item = {}
crawl_item = session.query(CrawlItem).filter_by(site_id=3, href=list_id).first()
crawl_item.description = data['ad']['body']
crawl_item.meta_data = {}
crawl_item.meta_data["ward"] = data['ad_params']['ward']['value']
try:
session.commit()
except:
session.rollback()
raise
finally:
ret_id = crawl_item.id
session.close()
my model:
class CrawlItem(Base):
...
description = Column(Text)
meta_data = Column(postgresql.JSON)
i want to get the value of ward :
"ward": {
"id": "ward",
"value": "Thị trấn Trạm Trôi",
"label": " Phường, thị xã, thị trấn"
}
I already encoding my postgresql to utf-8 so other fields that are not json column (description = Column(Text)) save utf-8 characters normally only my json column data are not decode:
{
"ward":"Th\u1ecb tr\u1ea5n Tr\u1ea1m Tr\u00f4i"
}
description column:
meta_data column:
i had tried using :
crawl_item.meta_data["ward"] = data['ad_params']['ward']['value'].decode('utf-8')
but the ward data don't get save
I have no idea what is wrong, hope someone can help me
EDIT:
i checked the data with psql and got these:
description column:
meta_data column:
It seems like only meta_data json column have trouble with the characters

Sqlalchemy serializes JSON field before save to db (see url and url and url).
json_serializer = dialect._json_serializer or json.dumps
By default, the PostgreSQL' dialect uses json.dumps and json.loads.
When you work with Text column, the data is converted in the following flow:
str -> bytes in utf-8 encoding
When you work with JSON column for PostgreSQL dialect, the data is converted in the following flow:
dict -> str with escaped non-ascii symbols -> bytes in utf-8 encoding
You can override the serializer in your engine configuration using json_serializer field:
json_serializer=partial(json.dumps, ensure_ascii=False)

use "jsonb" data type for your json column or cast "meta_data" field to "jsonb" like this:
select meta_data::jsonb from your_table;

How to save data from an API into a dictionary

I'm getting data from an API and saving into a list called data[].
After this, I'm sending these data to another class to format it.
I want to create a dict, so I can save these data. I'm trying to do something like this:
import json
import datetime
class MovieFormatter():
def format(self, data):
data = {
"movie_info_name": data['title']
"movie_info_duration": data['duration']
"movie_info_description": data['synopsis']
"movie_info_genres": data['genres']
"movie_info_actor": data['cast']
"movie_info_director": data['director']
data['trailers'] = data.get('trailers') or []
"dictionary": [{'url': x['url'], 'type': x['type']} for x in data['images'] + data['trailers']]
}
return data
Is this the right way to do?

It seems that the data object passed to your function is already a dictionary, from the way it has been indexed e.g. data['title'].

Try this :
_in_data = ["a","b","c"]
_out_data = ["x","y","z"]
_dict={}
print(_dict)
for i in range (len(_in_data)):
_dict[_in_data[i]]=_out_data[i]
print(_dict)

Django serializing separate variables into json

I have min max variables that are a result of query on model
args.aggregate(Min('price'))
args.aggregate(Max('price'))
returning the serialized data like this
return HttpResponse(json.dumps([{"maxPrice":args.aggregate(Max('price')),
"minPrice":args.aggregate(Min('price'))}]), content_type ='application/json')
the result looks like this:
minPrice = {
"price__min" = 110;
};
maxPrice = {
"price__max" = 36000;
};
and extracting the data looks like this
...
success:^(AFHTTPRequestOperation *operation, id responseObject){
NSDictionary *elements = responseObject;
int minPrice = elements[0][#"minPrice"][#"price__min"];
}
The Question: how to change the django/python code in order for the objective-c code to look like this: int minPrice = elements[#"minPrice"];

data = args.aggregate(minPrice=Min('price'), maxPrice=Max('price'))
return HttpResponse(json.dumps(data), content_type='application/json')
data variable is a dictionary with "minPrice" and "maxPrice" keys.

Dump to JSON a dictionary instead of a list:
values = args.aggregate(Min('price'), Max('price'))
return HttpResponse(json.dumps({'maxPrice': values['price__max'],
'minPrice': values['price__min']}),
content_type ='application/json')

Well you could do something like this to rearrange the json dump:
data = {'maxPrice': args.aggregate(Max('price'))['price__max'],
'minPrice': args.aggregate(Min('price'))['price__min']}
return HttpResponse(json.dumps(data), content_type ='application/json')
That should give you a json dict of that form '{"maxPrice": xxx, "minPrice": yyy}'.

Python - How to parse JSON string

I am trying to find a way to parse a JSON string and save them into mysql.
This is my json!
{"title": My title, "desc": mydesc, "url": http//example.com}
From now i don't have problem to save all json into one column usining json.dumps() so actually I'm trying to parse each joson data string to send him to mysql table. Title | Desc | Url.
This is my python code for desc example (pyspider-resultdb.py)
def _parse(self, data):
for key, value in list(six.iteritems(data)):
if isinstance(value, (bytearray, six.binary_type)):
data[key] = utils.text(value)
if 'result' in data:
decoded = json.loads(data['result'])
data['result'] = json.dumps(decoded['desc'])
return data
def _stringify(self, data):
if 'result' in data:
decoded = json.loads(data['result'])
data['result'] = json.dumps(decoded['desc'])
return data

It's unclear from your question what you trying to achieve, but If your question is how to convert JSON to python dict and then load to the table, then that's how you can do it:
my_dict = json.loads('{"title": "foo", "dest": "bar"}')
curs.execute('INSERT INTO test (title, dest) values(%(title)s, %(dest)s)', my_dict)