I have file.json that contains
{"list" : [
]}
I have this dict
lead = {'title': i.css('article>.sales-info>h1::text').extract_first(),
'address': i.css('article>.primary-info>.contact>h2::text').extract_first(),
'phone': i.css('article>.primary-info>.contact>.phone::text').extract_first(),
'time': i.css('article>.primary-info>.contact>.time-info>div:nth-child(2)::text').extract_first(),
'website': i.css('.business-card-footer>.website-link::attr(href)').extract_first(),
'email': i.css('.business-card-footer>.email-business::attr(href)').extract_first(),
'portfolio_item': ''}
I need to append this dict to the json list.
I've tried basic writing to file
with open('leads.json', 'a') as f:
f.write(json.dumps(item))
f.close()
Have no idea how should i append it to the json list
I think this could work for you:
EDIT: forgot to add the write_json() function
def write_json(path, data, indent=4):
with open(path, 'w') as file:
json.dump(data, file, indent=indent)
def append_json(path, data, key=None):
if not os.path.exists(path):
write_json(path, data)
else:
with open(path) as file:
# load a dict of the json data
json_data = json.load(file)
# key should be a list of objects so you can __add__ them
if key is None:
json_data += data
else:
json_data[key] += data
write_json(path, json_data)
appen_json('./leads.json', [lead], 'list') # you need to pass in a list, not just dict
The way this is written requires you to pass in a list rather than a dictionary but that has an advantage because you can pass in multiple items to append rather than one-at-a-time
This will write a new file if one doesn't exist in the director OR append if it does exist
Related
import csv
import json
def csv_to_json(csvFilePath):
jsonArray = []
decoded_file = csvFilePath.read().decode('utf-8').splitlines()
#load csv file data using csv library's dictionary reader
csvReader = csv.DictReader(decoded_file)
#convert each csv row into python dict
for row in csvReader:
#add this python dict to json array
option = [row['option1'], row['option2'], row['option3'], row['option4']]
option = filter(None, option)
newJson = {
'numb': row['numb'],
'question': row['question'],
'answer': row['answer'],
'options': option
}
jsonArray.append(newJson)
print('>>>>>>>jsonArray>>>>>>>', jsonArray)
jsonString = json.dumps(jsonArray)
print('>>>>>>>jsonString>>>>>>>', jsonString)
return jsonArray
Error
TypeError at /path/path
Object of type filter is not JSON serializable
>>>>>>>jsonArray>>>>>>> [{'numb': '1', 'question': 'What does HTML stand for?', 'answer': 'Hyper Text Markup Language', 'options': <filter object at 0x7f3d3459fa00>}, {'numb': '2', 'question': 'What does CSS stand for?', 'answer': 'Cascading Style Sheet', 'options': <filter object at 0x7f3d34317400>}]
If i comment out # 'options': option from newJson then it works perfectly
As the error says, a filter object is not serializable.
But a list is
'options': list(option)
However, it's not really clear to me what you're filtering. A list with None types will become null in the JSON array
JSON does not know how to convert a filter object into a JSON string. In order to put option into the JSON string, you need to convert it to a list first:
import csv
import json
def csv_to_json(csvFilePath):
jsonArray = []
decoded_file = csvFilePath.read().decode('utf-8').splitlines()
# load csv file data using csv library's dictionary reader
csvReader = csv.DictReader(decoded_file)
# convert each csv row into python dict
for row in csvReader:
# add this python dict to json array
option = [row['option1'], row['option2'], row['option3'], row['option4']]
option = list(filter(None, option))
newJson = {
'numb': row['numb'],
'question': row['question'],
'answer': row['answer'],
'options': option
}
jsonArray.append(newJson)
print('>>>>>>>jsonArray>>>>>>>', jsonArray)
jsonString = json.dumps(jsonArray)
print('>>>>>>>jsonString>>>>>>>', jsonString)
return jsonArray
The built-in function filter() has no default Encoder as per documentation: https://docs.python.org/3/library/json.html.
You need to implement custom Encode Decoder depending on what you want to store and retrieve.
Another helpful link:
How to make a class JSON serializable
I know how I can save kwargs in a .txt file.
with open(filename, "w") as file:
for key, value in kwargs.items():
file.write("{}: {}\n".format(key, value))
Let's says kwargs = {'page': 1, 'name': 'xyz', 'title': 'xyz'}
But how can I read the file and save the data back to kwargs how it was before it got saved to the file so that the result is kwargs = {'page': 1, 'name': 'xyz', 'title': 'xyz'} after reading the file?
if you're okay with saving it in a slightly different format, you can use the in-built eval function.
Writing your dictionary to a file:
with open(filename, "w") as file:
file.write(str(kwargs))
Reading your dictionary back from the file:
with open(filename, "r") as file:
kwargs = eval(file.read())
Assuming you aren't concerned about the ambiguities, this would be the inverse of what you presented in your question:
with open(filename, "r") as file:
kwargs = {}
for ln in file:
key, _, value = ln.partition(': ')
if value.isdigit():
value = int(value)
kwargs[key] = value
so here is the simple way to go about that, and before we get to the talk about ambiguity with that its simply to show how you could extract what you save in the file. it will be a key/value pair with key and pair are both strings. to cast the value part into the correct type isn't the scope of the solution
the test.txt content:
foo: bar
bas: buz
the code example:
result = {}
with open("/home/mr/test.txt", "r+") as f:
for line in f.readlines():
k, v = line.replace('\n','').split(':')
result.update({k.strip(): v.strip()})
the content of result dict:
{'foo': 'bar', 'bas': 'buz'}
I'm trying to add user info to a nested dictionary within a json file. Here's the code.
import json
dictionary = {'name': 'Tony Stark', 'attributes': ['genius', 'billionaire', 'playboy', 'philanthropist']}
with open('info.json', 'a+') as file:
json.dump(dictionary, file)
The info.json file
{'marvel': [
{'name': 'Bill Gates', 'attributes': ['philanthropist', 'programmer']}
]
}
Now, I am unable to dump the dictionary as a value for marvel which is a list. I'm trying to make it dynamic, adding Tony Stark's info to the json file.
Please help me with that, thanks.
Alternative:
import json
dictionary = {'name': 'Tony Stark', 'attributes': ['genius', 'billionaire', 'playboy', 'philanthropist']}
def write_json(data, filename='file.json'):
# function to add to JSON
with open(filename,'w') as f:
json.dump(data, f, indent=4)
with open('file.json') as json_file:
data = json.load(json_file)
data['marvel'].append(dictionary) # appending data to Marvel
write_json(data)
Edited as per observation of juanpa.arrivillaga
The following code is giving me the error:
Traceback (most recent call last): File "AMZGetPendingOrders.py", line 66, in <module>
item_list.append(item['SellerSKU']) TypeError: string indices must be integers
The code:
from mws import mws
import time
import json
import xmltodict
access_key = 'xx' #replace with your access key
seller_id = 'yy' #replace with your seller id
secret_key = 'zz' #replace with your secret key
marketplace_usa = '00'
orders_api = mws.Orders(access_key, secret_key, seller_id)
orders = orders_api.list_orders(marketplaceids=[marketplace_usa], orderstatus=('Pending'), fulfillment_channels=('MFN'), created_after='2018-07-01')
#save as XML file
filename = 'c:order.xml'
with open(filename, 'w') as f:
f.write(orders.original)
#ConvertXML to JSON
dictString = json.dumps(xmltodict.parse(orders.original))
#Write new JSON to file
with open("output.json", 'w') as f:
f.write(dictString)
#Read JSON and parse our order number
with open('output.json', 'r') as jsonfile:
data = json.load(jsonfile)
#initialize blank dictionary
id_list = []
for order in data['ListOrdersResponse']['ListOrdersResult']['Orders']['Order']:
id_list.append(order['AmazonOrderId'])
#This "gets" the orderitem info - this code actually is similar to the initial Amazon "get" though it has fewer switches
orders_api = mws.Orders(access_key, secret_key, seller_id)
#opens and empties the orderitem.xml file
open('c:orderitem.xml', 'w').close()
#iterated through the list of AmazonOrderIds and writes the item information to orderitem.xml
for x in id_list:
orders = orders_api.list_order_items(amazon_order_id = x)
filename = 'c:orderitem.xml'
with open(filename, 'a') as f:
f.write(orders.original)
#ConvertXML to JSON
amz_items_pending = json.dumps(xmltodict.parse(orders.original))
#Write new JSON to file
with open("pending.json", 'w') as f:
f.write(amz_items_pending)
#read JSON and parse item_no and qty
with open('pending.json', 'r') as jsonfile1:
data1 = json.load(jsonfile1)
#initialize blank dictionary
item_list = []
for item in data1['ListOrderItemsResponse']['ListOrderItemsResult']['OrderItems']['OrderItem']:
item_list.append(item['SellerSKU'])
#print(item)
#print(id_list)
#print(data1)
#print(item_list)
time.sleep(10)
I don't understand why Python thinks this is a list and not a dictionary. When I print id_list it looks like a dictionary (curly braces, single quotes, colons, etc)
print(data1) shows my dictionary
{
'ListOrderItemsResponse':{
'#xmlns':'https://mws.amazonservices.com/Orders/201 3-09-01',
'ListOrderItemsResult':{
'OrderItems':{
'OrderItem':{
'QuantityOrdered ':'1',
'Title':'Delta Rothko Rolling Bicycle Stand',
'ConditionId':'New',
'Is Gift':'false',
'ASIN':'B00XXXXTIK',
'SellerSKU':'9934638',
'OrderItemId':'49 624373726506',
'ProductInfo':{
'NumberOfItems':'1'
},
'QuantityShipped':'0',
'C onditionSubtypeId':'New'
}
},
'AmazonOrderId':'112-9XXXXXX-XXXXXXX'
},
'ResponseM etadata':{
'RequestId':'8XXXXX8-0866-44a4-96f5-XXXXXXXXXXXX'
}
}
}
Any ideas?
because you are iterating over each key value in dict:
{'QuantityOrdered ': '1', 'Title': 'Delta Rothko Rolling Bicycle Stand', 'ConditionId': 'New', 'Is Gift': 'false', 'ASIN': 'B00XXXXTIK', 'SellerSKU': '9934638', 'OrderItemId': '49 624373726506', 'ProductInfo': {'NumberOfItems': '1'}, 'QuantityShipped': '0', 'C onditionSubtypeId': 'New'}
so first value in item will be 'QuantityOrdered ' and you are trying to access this string as if it is dictionary
you can just do:
id_list.append(data1['ListOrderItemsResponse']['ListOrderItemsResult']['OrderItems']['OrderItem']['SellerSKU']))
and avoid for loop in dictionary
I guess you are trying to iterate OrderItems and finding their SellerSKU values.
for item in data1['ListOrderItemsResponse']['ListOrderItemsResult']['OrderItems']:
item_list.append(item['SellerSKU'])
This is just a part of my json file which looks like:
"network_lo": "127.0.0.0",
"ec2_block_device_mapping_root": "/dev/sda1",
"selinux": "false",
"uptime_seconds": 127412,
"ec2_reservation_id": "r-cd786568",
"sshdsakey": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"ec2_block_device_mapping_ami": "/dev/sda1",
"memorysize": "3.66 GB",
"swapsize": "0.00 kB",
"netmask": "255.255.255.192",
"uniqueid": "24wq0see",
"kernelmajversion": "3.2",
I have a Python scipt which download this file.. i want to parse this file and remove a number of objects like "swapsize","sshdsakey"
sqs = boto.sqs.connect_to_region("ap-west-1")
q = sqs.get_queue("deathvally")
m = q.read(visibility_timeout=15)
if m == None:
print "No message!"
else:
with open('download.json', 'w') as json_data:
print m.get_body()
json_data.write(m.get_body())
json_data.close()
# I want a logic here which can simply delete the specific json objects
# Something like this is what i tried but didn't work...
# clean_data = json.load(json_data)
# for element in clean_data: ##
# del element['sshdsakey']
# json_data.write(clean_data)
I basically need to parse the fetched json file and then remove the specific objects and then just write this new modified stuff in a file.
json.loads will decode JSON string into Python dictionary (Although format you provided is not a valid JSON format, there have to be curly braces on each side), then you can delete the needed keys with del , encode dictionary back to JSON string with json.dumps and write the resultit
clean_data = json.loads(json_data.read())
del clean_data[your_key]
with open(your_file_to_write, 'w') as f:
f.write(json.dumps(clean_data))
You can parse your json using loads from native json module.
Then delete an element from the dict using del
import json
keys_to_remove = ['sshdsakey', 'selinux']
json_str = '''{
"network_lo": "127.0.0.0",
"ec2_block_device_mapping_root": "/dev/sda1",
"selinux": "false",
"uptime_seconds": 127412,
"ec2_reservation_id": "r-cd786568",
"sshdsakey": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}'''
data = json.loads(json_str)
for key in keys_to_remove:
if key in data:
del data[key]
print data
You need to first convert the JSON object string into a Python dict, delete the keys from it, and then write to to the output file.
import json
sqs = boto.sqs.connect_to_region("ap-west-1")
q = sqs.get_queue("deathvally")
m = q.read(visibility_timeout=15)
if m is None:
print "No message!"
else:
KEYS_TO_REMOVE = "swapsize", "sshdsakey", "etc"
with open('download.json', 'w') as json_data:
json_obj = json.loads(m.get_body())
for key in KEYS_TO_REMOVE:
try:
del json_obj[key]
except KeyError:
pass
json_data.write(json.dumps(json_obj, indent=4))