I need a function which will allow setting a nested value in YAML file. For instance, for a YAML like:
LEVEL_1:
LEVEL_2:
LEVEL_3: some_value
I would do something like:
update_yaml_value("LEVEL_1.LEVEL_2.LEVEL_3", "new_value")
I'd appreciate any help. Thanks in advance.
First of all you need to import yaml:
import yaml
When you load some yaml file you will get a python dict object.
with open('/path/to/smth.yaml', 'r') as f:
yaml_data = yaml.safe_load(f)
To have the ability to change it in way you described you can create function like next:
def update_yaml_value(long_key, data):
keys = long_key.split('.')
accessable = yaml_data
for k in keys[:-1]:
accessable = accessable[k]
accessable[keys[-1]] = data
And then save this yaml file:
with open('/path/to/smth.yaml', 'w+') as f:
yaml.dump(yaml_data, f, default_flow_style=False)
Your python code will load the YAML into a dict.
With the dict you will be able to update the value.
a_dict = load_yaml() # to be implemented by the OP
a_dict['level_1']['level_2']['level_3'] = 'a_new_value'
You can use this function to change a dict value recersivly.
test.py:
#!/usr/bin/env python3
import yaml
def change_config(conf, key, value):
if isinstance(conf, dict):
for k, v in conf.items():
if k == key:
conf[k] = value
elif isinstance(v, dict):
change_config(v, key, value)
with open("smth.yaml", "r") as f:
yaml_data = yaml.load(f, Loader=yaml.Loader)
print(yaml_data)
change_config(yaml_data, "level3", "hassan")
print(yaml_data)
smth.yaml:
level0:
level1:
level3: test
Terminal Output:
{'level0': {'level1': {'level3': 'test'}}}
{'level0': {'level1': {'level3': 'hassan'}}}
Related
I'm doing something wrong in the code below. I have a method (update_dictonary) that changes a value or values in a dictionary based on what is specificed in a tuple (new_points).
Before I update the dictionary, I want to save that version in a list (history) in order to be able to access previous versions. However, my attempt below updates all dictionaries in history to be like the latest version.
I can't figure out what I'm doing wrong here.
test_dict = {'var0':{'var1':{'cond1':1,
'cond2':2,
'cond3':3}
}
}
class version_control:
def __init__ (self, dictionary):
self.po = dictionary
self.history = list()
self.version = 0
def update_dictionary(self, var0, var1, new_points):
po_ = self.po
self.history.append(po_)
for i in new_points:
self.po[var0][var1][i[0]] = i[1]
self.version += 1
def get_history(self, ver):
return self.history[ver]
a = version_control(test_dict)
new_points = [('cond1', 2),
('cond2', 0)]
a.update_dictionary('var0', 'var1', new_points)
new_points = [('cond3', -99),
('cond2', 1)]
a.update_dictionary('var0', 'var1', new_points)
print(a.get_history(0))
print(a.get_history(1))
Try this
from copy import deepcopy
...
def update_dictionary(self, var0, var1, new_points):
po_ = deepcopy(self.po)
self.history.append(po_)
for i in new_points:
self.po[var0][var1][i[0]] = i[1]
self.version += 1
...
The problem here is that when you assign po_= self.po you expect po_ to a new variable with a new memory id but actually, you just make a shallow copy(same memory id) of your dictionary. This means if you update the self.po then op_ will automatically update.
To solve this problem by using deepcopy from the copy module(Built-in). It will create a new variable.
You can use this code to save the data into a JSON file.
import json
class version_control:
def __init__(self, dictionary):
self.po = dictionary
self.version = 0
self.ZEROth_version()
def update_dictionary(self, var0, var1, new_points, version=None):
self.version += 1
for i in new_points:
self.po[var0][var1][i[0]] = i[1]
# set the ver to version if given else set to self.version
ver = self.version if version is None else version
with open("version.json", "r") as jsonFile:
# loading the data from the file.
data = json.load(jsonFile)
data[str(ver)] = self.po
with open("version.json", "w") as jsonFile:
# save the updated dictionary in json file
json.dump(data, jsonFile, indent=4)
def get_history(self, ver):
try:
with open("version.json", "r") as jsonFile:
# I don't use .get here. I will catch key errors in except block. I don't want to add an if statement to check for None. But string if you want you can add that.
return json.load(jsonFile)[str(ver)]
# Checking if the file not exists or is empty
except (json.decoder.JSONDecodeError, FileNotFoundError, KeyError) as e:
print("File or Version not found")
def ZEROth_version(self):
with open("version.json", "w") as f:
data = {0: self.po}
json.dump(data, f, indent=4)
I have explained some main points if you want more explanation then comment, I will reply as soon as possible.
I got a complex nested JSON file and I need to convert it completely, but my code can only convert part of the information,
How can I modify my code, or do you guys have a better code?
my json file
import csv
import json
import sys
import codecs
def trans(path):
jsonData = codecs.open('H://14.json', 'r', 'utf-8')
# csvfile = open(path+'.csv', 'w')
# csvfile = open(path+'.csv', 'wb')
csvfile = open('H://11.csv', 'w', encoding='utf-8', newline='')
writer = csv.writer(csvfile, delimiter=',')
flag = True
for line in jsonData:
dic = json.loads(line)
if flag:
keys = list(dic.keys())
print(keys)
writer.writerow(keys)
flag = False
writer.writerow(list(dic.values()))
jsonData.close()
csvfile.close()
if __name__ == '__main__':
path=str(sys.argv[0])
print(path)
trans(path)
my json file
{"id":"aa","sex":"male","name":[{"Fn":"jeri","Ln":"teri"}],"age":45,"info":[{"address":{"State":"NY","City":"new york"},"start_date":"2001-09","title":{"name":"Doctor","Exp":"head"},"year":"2001","month":"05"}],"other":null,"Hobby":[{"smoking":null,"gamble":null}],"connect":[{"phone":"123456789","email":"info#gmail.com"}],"Education":"MBA","School":{"State":"NY","City":"new york"}}
{"id":"aa","sex":"female","name":[{"Fn":"lo","Ln":"li"}],"age":34,"info":[{"address":{"State":"NY","City":"new york"},"start_date":"2008-11","title":{"name":"Doctor","Exp":"hand"},"year":"2008","month":"02"}],"other":null,"Hobby":[{"smoking":null,"gamble":null}],"connect":[{"phone":"123456789","email":"info#gmail.com"}],"Education":"MBA","School":{"State":"NY","City":"new york"}}
It only converts part of the information, 'name''info''Hobby''connect''School' these information are not converted,I need to convert all information completely,
You could use the below function to treat each dic. It will flatten the dict by recursive calls until there is no dict or list in the values. In order to avoid issues with 2 keys having the same name, I concatenate with the previous level.
WARNING: it is based on your format so if you have lists with more than one element in the middle, it will only take care of the first element.
def flatten_dict(input_dict, result = None):
result = result or {}
for key, value in input_dict.items():
if isinstance(value, list):
current_dict = {key+"_"+k: v for k, v in value[0].items()}
flatten_dict(current_dict, result)
elif isinstance(value, dict):
current_dict = {key+"_"+k: v for k, v in value.items()}
flatten_dict(current_dict, result)
else:
result[key] = value
return result
Then apply this function on each dic, transform to Dataframe and save as CSV.
res = []
for line in jsonData:
dic = json.loads(line)
res.append(flatten_dict(dic))
res_df = pd.DataFrame(res)
res_df.to_csv("result.csv")
Result:
I need to edit a object within a dict that is located inside a json file. but whenever I attempt to do it, it would delete the entire json file and only add the thing I've edited.
Here's my function.
async def Con_Manage(keys):
with open('keys.json') as config_file:
config = json.load(config_file)[keys]
try:
Current_Con = config["curCons"] + 1
with open('keys.json', 'w') as config_file:
json.dump(Current_Con, config_file)
return True
except:
return False
heres my json file before i run it
{
"key1": {
"time": 1500,
"maxCons": 15,
"curCons": 2,
"coolDown": 2
}
}
and here's what it looks like after its ran
3
is there any way that i can run this and not delete all my progress?
config["curCons"] gets you just the value which you then increment it and assign to Current_Con. Instead you need to increment and set the value to +1. From there you would want to save the entire json object that you just read in and not just the value that was updated.
async def Con_Manage(keys):
with open('keys.json') as config_file:
config = json.load(config_file)
config[keys]["curCons"] += 1 # mutates the value in place
with open('keys.json', 'w') as keys:
json.dump(config, keys) # saves the entire dict not just the value
You need to be writing the entire config file
you can do something like this...
with open('keys.json') as config_file:
config = json.load(config_file)
for key in keys:
try:
config[key]["curCons"] += 1
except KeyError:
pass
with open('keys.json', 'w') as config_file:
json.dump(config, config_file)
This is a heavily abstracted example where I build objects from variables stored in a .yaml file. I'm writing the reverse method to save them back as a new .yaml
I may create further objects via script, so the output yaml will in general be different.
I'm using .locals() to build a dictionary from the kwargs, and then .pop() to strip the ones I will not want to save.
This seems to work and do what I want, but I feel it is ugly. Am I missing a better, safer, or more pythonic way to do this?
I understand there is pickle and dill, but for the current question I'd like to restrict this to reading and writing yamls. (because)
note: if attributes are added later I don't want them saved. This is why I create ob.L right after instantiation.
Input .yaml:
bob:
args: {'x':1, 'y':2}
sue:
args: {'x':3, 'y':4}
Output .yaml:
bob:
args:
x: 1
y: 2
new:
args:
x: 5
y: 6
sue:
args:
x: 3
y: 4
Current script:
class A(object):
wow = 77
def __init__(self, name, x, y):
self.name = name
self.x = x
self.y = y
self.L = locals()
self.L.pop('name')
self.L.pop('self')
import yaml
with open('thing.yaml', 'r') as infile:
d = yaml.load(infile)
obs = []
for name, info in d.items():
ob = A(name, **info['args'])
obs.append(ob)
newob = A('new', 5, 6)
obs.append(newob)
newob.ignore_me = 777 # this should not be saved
# rebuild the yaml
d = dict()
for ob in obs:
info = dict()
info['args'] = ob.L
d[ob.name] = info
with open('newthing.yaml', 'w') as outfile:
yaml.dump(d, outfile, default_flow_style=False, allow_unicode=True)
I can't understand why you're doing any of this. All you need to do is to load the YAML, add your new items, and then dump it again.
with open('thing.yaml', 'r') as infile:
d = yaml.load(infile)
d['new'] = {'x': 5, 'y': 6}
with open('newthing.yaml', 'w') as outfile:
yaml.dump(d, outfile, default_flow_style=False, allow_unicode=True)
We have a sample .cfg file consist of key value pair. Based on user input, we need to update the value. I was thinking to update the value using configParser but file doesn't have any section (e.g. [My Section]). Based on the documentation it needs three values to set - section, key and value. Unfortunately, I will not be able to add any section marker, as this file is used by other tasks.
What would be the another way we can set the value based on key?
File example
some.status_file_mode = 1 # Some comment
some.example_time = 7200 # Some comment
As per the requirement, no change in the line. Spaces and comments needs to be same as is.
Use NamedTemporaryFile from the tempfile module it is not too hard to build a simple parser to update a file that looks like that:
Code:
def replace_key(filename, key, value):
with open(filename, 'rU') as f_in, tempfile.NamedTemporaryFile(
'w', dir=os.path.dirname(filename), delete=False) as f_out:
for line in f_in.readlines():
if line.startswith(key):
line = '='.join((line.split('=')[0], ' {}'.format(value)))
f_out.write(line)
# remove old version
os.unlink(filename)
# rename new version
os.rename(f_out.name, filename)
Test Code:
import os
import tempfile
replace_key('file1', 'some.example_time', 3)
Results:
some.status_file_mode = 1
some.example_time = 3
If you don't care about spacing, this works well for your case.
def replace_config(filename, key, value):
d = {}
with open(filename, "r+") as f:
for line in f:
k, v = line.split('=')
c = ""
try:
v, c = v.split('#')
except ValueError:
c = ""
d[k.strip()] = {'v': v.strip(), 'c': c.strip()}
f.seek(0)
f.truncate()
d[key]['v'] = value
for k, v in d.items():
if v["c"]:
text = "{} = {} # {}\n".format(k, v['v'], v['c'])
else:
text = "{} = {}\n".format(k, v['v'])
f.write(text)
replace_config('config.cfg', 'some.example_time', 3)