Related
I have the following piece of code in python:
def pushHashtagPosts(hashtagPosts):
from bson.json_util import loads
myclient = pymongo.MongoClient(mongoUri)
mydb = myclient["myDB"]
mycol = mydb["hashtags"]
data = loads(hashtagPosts)
posts = mycol.insert_many(data)
Whereas, the content of hashtagPosts looks something like this:
hashtagPosts = [{'hashtag': '###!', 'PostHashHex': '13fc9904028fb62490a3b5dc2111689376e52a06dc636c3322cfa16e33a41398', 'post': {'_id': {'$oid': '608f8eb73718c7977f9c0a43'}, 'PostHashHex': '13fc9904028fb62490a3b5dc2111689376e52a06dc636c3322cfa16e33a41398', 'PosterPublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'ParentStakeID': '', 'Body': 'Need hashtags ####! Or else it’s a bit difficult to create personal brand and niche on this platform. \n\nDevs are u listening?', 'ImageURLs': [], 'RecloutedPostEntryResponse': None, 'CreatorBasisPoints': 0, 'StakeMultipleBasisPoints': 12500, 'TimestampNanos': 1.6177643730879583e+18, 'IsHidden': False, 'ConfirmationBlockHeight': 13248, 'InMempool': False, 'StakeEntry': {'TotalPostStake': 0, 'StakeList': []}, 'StakeEntryStats': {'TotalStakeNanos': 0, 'TotalStakeOwedNanos': 0, 'TotalCreatorEarningsNanos': 0, 'TotalFeesBurnedNanos': 0, 'TotalPostStakeNanos': 0}, 'ProfileEntryResponse': None, 'Comments': None, 'LikeCount': 5, 'PostEntryReaderState': None, 'InGlobalFeed': False, 'IsPinned': False, 'PostExtraData': {}, 'CommentCount': 2, 'RecloutCount': 0, 'ParentPosts': None, 'PublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'Username': ''}},
{'hashtag': 'investementstrategy', 'PostHashHex': '92f2d08ac8f2b47fe5868b748c7f472e13ad12c284bb0e327cf317b4c2514f83', 'post': {'_id': {'$oid': '608f8eb73718c7977f9c0a3f'}, 'PostHashHex': '92f2d08ac8f2b47fe5868b748c7f472e13ad12c284bb0e327cf317b4c2514f83', 'PosterPublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'ParentStakeID': '', 'Body': 'Don’t say that you are going to buy ur own coin to have a steady growth of ur coin \U0001fa99. That doesn’t show the strength of ur investment nor the coin.📉📈 Strength lies in others believing in ur talent, creativity and passion enough to invest in U. 🚀🚀🚀\n#InvestementStrategy', 'ImageURLs': [], 'RecloutedPostEntryResponse': None, 'CreatorBasisPoints': 0, 'StakeMultipleBasisPoints': 12500, 'TimestampNanos': 1.6178065064906166e+18, 'IsHidden': False, 'ConfirmationBlockHeight': 13397, 'InMempool': False, 'StakeEntry': {'TotalPostStake': 0, 'StakeList': []}, 'StakeEntryStats': {'TotalStakeNanos': 0, 'TotalStakeOwedNanos': 0, 'TotalCreatorEarningsNanos': 0, 'TotalFeesBurnedNanos': 0, 'TotalPostStakeNanos': 0}, 'ProfileEntryResponse': None, 'Comments': None, 'LikeCount': 2, 'PostEntryReaderState': None, 'InGlobalFeed': False, 'IsPinned': False, 'PostExtraData': {}, 'CommentCount': 1, 'RecloutCount': 0, 'ParentPosts': None, 'PublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'Username': ''}},
{'hashtag': 'productivity', 'PostHashHex': 'c8fabd96f5d624d06ec8d23e90de19cf07ad4b6696dac321fda815c3000fbf1b', 'post': {'_id': {'$oid': '608f8eb73718c7977f9c0a3d'}, 'PostHashHex': 'c8fabd96f5d624d06ec8d23e90de19cf07ad4b6696dac321fda815c3000fbf1b', 'PosterPublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'ParentStakeID': '', 'Body': 'What is the most productive thing u have done in last 24 hours apart from Bitclout???\n\n\U0001f9d0😏🙌🏼 #productivity', 'ImageURLs': [], 'RecloutedPostEntryResponse': None, 'CreatorBasisPoints': 0, 'StakeMultipleBasisPoints': 12500, 'TimestampNanos': 1.6178362054980055e+18, 'IsHidden': False, 'ConfirmationBlockHeight': 13487, 'InMempool': False, 'StakeEntry': {'TotalPostStake': 0, 'StakeList': []}, 'StakeEntryStats': {'TotalStakeNanos': 0, 'TotalStakeOwedNanos': 0, 'TotalCreatorEarningsNanos': 0, 'TotalFeesBurnedNanos': 0, 'TotalPostStakeNanos': 0}, 'ProfileEntryResponse': None, 'Comments': None, 'LikeCount': 30, 'PostEntryReaderState': None, 'InGlobalFeed': True, 'IsPinned': False, 'PostExtraData': {}, 'CommentCount': 59, 'RecloutCount': 0, 'ParentPosts': None, 'PublicKeyBase58Check': 'BC1YLhKJZZcPB2WbZSSekFF19UshsmmPoEjtEqrYakzusLmL25xxAJv', 'Username': ''}}]
When I try to insert this data as insert_many() into mongodb I get the following error:
File "test.py", line X, in pushHashtagPosts
data = loads(hashtagPosts) TypeError: the JSON object must be str, bytes or bytearray, not 'list'
However, I have inserted the line 'data = loads(hashtagPosts)' based on the solution at bson.errors.InvalidDocument: key '$oid' must not start with '$' trying to insert document with pymongo because without the 'data = loads(hashtagPosts)' I was getting the following error:
bson.errors.InvalidDocument: key '$oid' must not start with '$'
How to resolve this and successfully insert many documents in the collection?
Your issue is that hashtagPosts is a list but loads expects to work on a string.
So working backwards, the question becomes how did you construct hashtagPosts in the first place? As it contains $oid values, it looks like an output from dumps; but an output from dumps is a string. not a list. So how did it become a list?
If you are creating it manually, then just set it using ObjectId, e.g.
from bson import ObjectId
item = {'_id': ObjectId('608f8eb73718c7977f9c0a43')}
and then you won't need to use loads.
I am trying to indent and assort the format of balance so that it is easier to read. I want to print the RequestResponse like the Expected output. The balance variable is of type tuple. How could I do such a thing?
import bybit
import json
balance = client.Wallet.Wallet_getBalance(coin="BTC").result()
print(balance)
Output:
({'ret_code': 0, 'ret_msg': 'OK', 'ext_code': '', 'ext_info': '', 'result': {'BTC': {'equity': 0.00208347, 'available_balance': 0.00208347, 'used_margin': 0, 'order_margin': 0, 'position_margin': 0, 'occ_closing_fee': 0, 'occ_funding_fee': 0, 'wallet_balance': 0.00208347, 'realised_pnl': 0, 'unrealised_pnl': 0, 'cum_realised_pnl': 8.347e-05, 'given_cash': 0, 'service_cash': 0}}, 'time_now': '1616685310.655072', 'rate_limit_status': 118, 'rate_limit_reset_ms': 1616685310652, 'rate_limit': 120}, <bravado.requests_client.RequestsResponseAdapter object at 0x000001F5E92EB048>)
Expected Output:
{
"cross_seq": 11518,
"data": [
{
"price": "2999.00",
"side": "Buy",
"size": 9,
"symbol": "BTCUSD"
},
{
"price": "3001.00",
"side": "Sell",
"size": 10,
"symbol": "BTCUSD"
}
],
"timestamp_e6": 1555647164875373,
"topic": "orderBookL2_25.BTCUSD",
"type": "snapshot"
}
I think you provided the wrong expected output since the fields between your output and expected output don't match but in general if you want a better display of a dictionary you can use the json package:
response = {'ret_code': 0, 'ret_msg': 'OK', 'ext_code': '', 'ext_info': '', 'result': {'BTC': {'equity': 0.00208347, 'available_balance': 0.00208347, 'used_margin': 0, 'order_margin': 0, 'position_margin': 0, 'occ_closing_fee': 0, 'occ_funding_fee': 0, 'wallet_balance': 0.00208347, 'realised_pnl': 0, 'unrealised_pnl': 0, 'cum_realised_pnl': 8.347e-05, 'given_cash': 0, 'service_cash': 0}}, 'time_now': '1616685310.655072', 'rate_limit_status': 118, 'rate_limit_reset_ms': 1616685310652, 'rate_limit': 120}
import json
json.loads(json.dumps(response, indent=4, sort_keys=True))
This will give you the following output:
{'ext_code': '',
'ext_info': '',
'rate_limit': 120,
'rate_limit_reset_ms': 1616685310652,
'rate_limit_status': 118,
'result': {'BTC': {'available_balance': 0.00208347,
'cum_realised_pnl': 8.347e-05,
'equity': 0.00208347,
'given_cash': 0,
'occ_closing_fee': 0,
'occ_funding_fee': 0,
'order_margin': 0,
'position_margin': 0,
'realised_pnl': 0,
'service_cash': 0,
'unrealised_pnl': 0,
'used_margin': 0,
'wallet_balance': 0.00208347}},
'ret_code': 0,
'ret_msg': 'OK',
'time_now': '1616685310.655072'}
Another solution is to use pprint
import pprint
pprint.pprint(response)
This will give you the following output:
{'ext_code': '',
'ext_info': '',
'rate_limit': 120,
'rate_limit_reset_ms': 1616685310652,
'rate_limit_status': 118,
'result': {'BTC': {'available_balance': 0.00208347,
'cum_realised_pnl': 8.347e-05,
'equity': 0.00208347,
'given_cash': 0,
'occ_closing_fee': 0,
'occ_funding_fee': 0,
'order_margin': 0,
'position_margin': 0,
'realised_pnl': 0,
'service_cash': 0,
'unrealised_pnl': 0,
'used_margin': 0,
'wallet_balance': 0.00208347}},
'ret_code': 0,
'ret_msg': 'OK',
'time_now': '1616685310.655072'}
Import JSON, then using json.dumps(balance, indent=4) would get you that format.
You could add keyword argument of sort_keys=True if you want them sorted.
I want to convert one json file format to another format.
input json file looks like,
json1 = {"Roll Number":{"0":"12345675890"},"Exam Code":{"0":"125"},"q1":{"0":"A"},"q2":{"0":"B"},"q3":{"0":"B"},"q4":{"0":"C"},"q5":{"0":"C"}
Here you can consider 0 as the 0th student (answersheet of first student)
I want this output format,
json2 = {
"id": 1,
"Roll Number": 12345675890,
"Exam Code": 125,
"qtn_ans_id": 1, #this is question number
"field1": 0,
"field2": 0,
"field3": 1, #if ans is C then field3=1 and else is 0
"field4": 0,
}
in json1 it contains 5 question answers and details and i want to convert it into json2, and my code is
import json
Stored json file into data
with open('json1.json') as data_file:
data = json.load(data_file)
write default format
format = {"table": "omr",
"rows":
[
{
"id": 1,
"regist_no": 1215152,
"exam_id": 125,
"qtn_ans_id": 1,
"field1": 0,
"field2": 0,
"field3": 0,
"field4": 0,
}
]
}
My dummy Code
json2 = {}
for (k, v) in data.items():
if(k=='Roll Number'):
format['rows'][0]['regist_no']=v['0']
if(k=='Exam Code'):
format['rows'][0]['exam_id'] = v['0']
if(v['0']=='A'):
format['rows'][0]['field1'] = 1
elif(v['0']=='B'):
format['rows'][0]['field2'] = 1
elif(v['0']=='C'):
format['rows'][0]['field3'] = 1
elif(v['0']=='D'):
format['rows'][0]['field4'] = 1
json2.append(format['rows'])
wrong output i get is,
[{
'field1': 1,
'field2': 1,
'field3': 1,
'field4': 1
}]
All the field values are 1.
For answer A Right output can be,
[{
'field1': 1,
'field2': 0,
'field3': 0,
'field4': 0
}]
or is there any other way like using pandas data frame.
I know this is long and dummy question but i will be glade if anyone can help. Thank you!
How about something like this?
# assuming your json data looks like this:
json1 = [
{
"Roll Number" : {"0":"12345675890"},
"Exam Code" : {"0":"125"},
"q1" : {"0":"A"},
"q2" : {"0":"B"},
"q3" : {"0":"B"},
"q4" : {"0":"C"},
"q5" : {"0":"C"},
},
{
"Roll Number" : {"0":"12345675891"},
"Exam Code" : {"0":"125"},
"q1" : {"0":"C"},
"q2" : {"0":"B"},
"q3" : {"0":"A"},
"q4" : {"0":"C"},
"q5" : {"0":"D"},
},
]
Then:
# converts old question field to new question field:
def ConvertAnswer(question):
# get letter answer from question:
for key, value in question.items():
letter = value
# pack fields object:
fields = {
'field1' : 1 if letter == 'A' else 0,
'field2' : 1 if letter == 'B' else 0,
'field3' : 1 if letter == 'C' else 0,
'field4' : 1 if letter == 'D' else 0,
}
return fields
# iterate over each student:
for student in json1:
# iterate over each field in student:
for key, value in student.items():
# check for questions (check if key starts with 'q'):
if key[0] == 'q':
# replace question field in student object:
student[key] = ConvertAnswer(value)
Will result in:
# output of json1:
[{'Roll Number': {'0': '12345675890'},
'Exam Code': {'0': '125'},
'q1': {'field1': 1, 'field2': 0, 'field3': 0, 'field4': 0},
'q2': {'field1': 0, 'field2': 1, 'field3': 0, 'field4': 0},
'q3': {'field1': 0, 'field2': 1, 'field3': 0, 'field4': 0},
'q4': {'field1': 0, 'field2': 0, 'field3': 1, 'field4': 0},
'q5': {'field1': 0, 'field2': 0, 'field3': 1, 'field4': 0}},
{'Roll Number': {'0': '12345675891'},
'Exam Code': {'0': '125'},
'q1': {'field1': 0, 'field2': 0, 'field3': 1, 'field4': 0},
'q2': {'field1': 0, 'field2': 1, 'field3': 0, 'field4': 0},
'q3': {'field1': 1, 'field2': 0, 'field3': 0, 'field4': 0},
'q4': {'field1': 0, 'field2': 0, 'field3': 1, 'field4': 0},
'q5': {'field1': 0, 'field2': 0, 'field3': 0, 'field4': 1}}]
My solution in more "pandasonic" than the other answer.
Start from defining a function, which will be used to change
the answer code to 4 answer columns:
def ans(code):
return pd.Series([ int(code == x) for x in 'ABCD' ],
index=[ 'field' + str(i) for i in range(1, 5)])
Read your input file into a DataFrame as follows:
with open('input.json') as data_file:
df = pd.read_json(data_file, orient='columns')
Then reformat it, saving in another DataFrame:
df2 = pd.wide_to_long(df.rename_axis(index='id').reset_index(),
stubnames='q', i='Roll Number', j='qtn_ans_id').reset_index()\
.reindex(columns=['id', 'Roll Number', 'Exam Code', 'qtn_ans_id', 'q'])
For now it contains one column q with answer codes.
So to change it into 4 columns for each field, run:
df2 = df2.join(df2.q.apply(ans)).drop(columns=['q'])
And the last step is to create the output JSON string:
df2.to_json(orient='records')
which you can e.g. save in an output file.
Note: If your input contained e.g. data for 2 students,
the JSON file should have the following form:
{
"Roll Number":{"0":12345675890,"1":23456758901},
"Exam Code":{"0":125,"1":125},
"q1":{"0":"A","1":"A"},
"q2":{"0":"B","1":"B"},
"q3":{"0":"B","1":"C"},
"q4":{"0":"C","1":"D"},
"q5":{"0":"C","1":"C"}
}
(different than the input format proposed in the other answer).
After updating MongoDB from version 3.4.17 to 4.0.12 from time to time, when executing mongo.conn.COLLECTION_NAME.find({'email': ['1'] * 2000}), while email is a string value, the following error is thrown:
File "/usr/local/lib/python2.7/dist-packages/pymongo/cursor.py", line 787, in count
cmd, self.__collation, session=self.__session)
File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 1600, in _count
_cmd, self._read_preference_for(session), session)
File "/usr/local/lib/python2.7/dist-packages/pymongo/mongo_client.py", line 1465, in _retryable_read
return func(session, server, sock_info, slave_ok)
File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 1594, in _cmd
session=session)
File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 250, in _command
user_fields=user_fields)
File "/usr/local/lib/python2.7/dist-packages/pymongo/pool.py", line 613, in command
user_fields=user_fields)
File "/usr/local/lib/python2.7/dist-packages/pymongo/network.py", line 167, in command
parse_write_concern_error=parse_write_concern_error)
File "/usr/local/lib/python2.7/dist-packages/pymongo/helpers.py", line 159, in _check_command_response
raise OperationFailure(msg % errmsg, code, response)
OperationFailure: The key is too long
The returned error code is: 17280
Python: 2.7.16
Pymongo: 3.9.0
The query execution plan is:
{'executionStats': {'allPlansExecution': [],
'executionStages': {'advanced': 0,
'alreadyHasObj': 0,
'docsExamined': 0,
'executionTimeMillisEstimate': 0,
'filter': {'email': {'$eq': ['1',...,'1']}},
'inputStage': {'advanced': 0,
'direction': 'forward',
'dupsDropped': 0,
'dupsTested': 0,
'executionTimeMillisEstimate': 0,
'indexBounds': {'email': ['["1", "1"]', '[[ "1", ... , "1" ]]']},
'indexName': 'email_1',
'indexVersion': 2,
'invalidates': 0,
'isEOF': 1,
'isMultiKey': False,
'isPartial': False,
'isSparse': False,
'isUnique': False,
'keyPattern': {'email': 1},
'keysExamined': 1,
'multiKeyPaths': {'email': []},
'nReturned': 0,
'needTime': 1,
'needYield': 0,
'restoreState': 0,
'saveState': 0,
'seeks': 2,
'seenInvalidated': 0,
'stage': 'IXSCAN',
'works': 2},
'invalidates': 0,
'isEOF': 1,
'nReturned': 0,
'needTime': 1,
'needYield': 0,
'restoreState': 0,
'saveState': 0,
'stage': 'FETCH',
'works': 2},
'executionSuccess': True,
'executionTimeMillis': 0,
'nReturned': 0,
'totalDocsExamined': 0,
'totalKeysExamined': 1},
'ok': 1.0,
'queryPlanner': {'indexFilterSet': False,
'namespace': 'DATABASE_NAME.COLLECTION_NAME',
'parsedQuery': {'email': {'$eq': ['1', ... , '1']}},
'plannerVersion': 1,
'rejectedPlans': [],
'winningPlan': {'filter': {'email': {'$eq': ['1', ... , '1']}},
'inputStage': {'direction': 'forward',
'indexBounds': {'email': ['["1", "1"]', '[[ "1", ... ,"1" ]]']},
'indexName': 'email_1',
'indexVersion': 2,
'isMultiKey': False,
'isPartial': False,
'isSparse': False,
'isUnique': False,
'keyPattern': {'email': 1},
'multiKeyPaths': {'email': []},
'stage': 'IXSCAN'},
'stage': 'FETCH'}},
'serverInfo': {'gitVersion': '5776e3cbf9e7afe86e6b29e22520ffb6766e95d4',
'host': '*****',
'port': 27037,
'version': '4.0.12'}}
The error probably caused by a tremendous value (a long list) comparison, but I didn't find any documentation about it.
Is it a restriction that was added in version 4.0?
I'm making simple django app.
I made database and I want to call it from javascript code.
Everything was fine, but I had some problem with 'u' string..
models.py
class FileDB(models.Model):
fileName = models.CharField(max_length=200)
fileState = models.IntegerField(default=0)
views.py
from .models import FileDB
def GetFileList(request, state):
list = FileDB.objects.filter(fileState=state).values()
return HttpsResponse(list)
urls.py
urlpatterns = [
...
url(r^getfilelist/(?P<state>[0-2]+)/$', view.GetFileList),
]
In Chrome browser : x.x.x.x:8000/getfilelist/0/
{'fileState': 0, u'id': 1, 'fileName': u'image.jpg'}{'fileState': 0, u'id': 2, 'fileName': u'image2.jpg'}{'fileState': 0, u'id': 3, 'fileName': u'picture1.jpg'}{'fileState': 0, u'id': 4, 'fileName': u'video1.avi'}
How can I remove annoying 'u' string efficiently?
You can to use json library.
import json
from .models import FileDB
def get_file_list(request, state):
list = list(FileDB.objects.filter(fileState=state).values())
return HttpsResponse(json.dumps(list))
This should give you something like this
[{"fileState": 0, "id": 1, "fileName": "image.jpg"}, {"fileState": 0, "id": 2, "fileName": "image2.jpg"}, {"fileState": 0, "id": 3, "fileName": "picture1.jpg"}, {"fileState": 0, "id": 4, "fileName": "video1.avi"}]
You can remove the u characters from the keys and values of a dictionary by using a dictionary comprehension and casting the
>>> x = {'fileState': 0, u'id': 1, 'fileName': u'image.jpg'}
>>> {str(k): str(v) for k, v in x.iteritems()}
{'fileState': '0', 'id': '1', 'fileName': 'image.jpg'}
If you have a list of dictionaries, you can simply do the same but for each element in the list.
>>> y = [{'fileState': 0, u'id': 1, 'fileName': u'image.jpg'}, {'fileState': 0, u'id': 2, 'fileName': u'image2.jpg'}, {'fileState': 0, u'id': 3, 'fileName': u'picture1.jpg'}, {'fileState': 0, u'id': 4, 'fileName': u'video1.avi'}]
>>> newList = []
>>> for dct in y:
... newList.append({str(k): str(v) for k, v in dct.iteritems()})
...
>>> newList
[{'fileState': '0', 'id': '1', 'fileName': 'image.jpg'}, {'fileState': '0', 'id': '2', 'fileName': 'image2.jpg'}, {'fileState': '0', 'id': '3', 'fileName': 'picture1.jpg'}, {'fileState': '0', 'id': '4', 'fileName': 'video1.avi'}]