Related
I have a list of dictionaries that have other dictionaries on them.
Dictionary:
[[{'id': 1, 'networkId': 'L_1111', 'name': 'VLAN1', 'applianceIp': '1.1.1.1', 'subnet': '1.1.1.0/24', 'fixedIpAssignments': {}, 'reservedIpRanges': [], 'dnsNameservers': 'upstream_dns', 'dhcpHandling': 'Run a DHCP server', 'dhcpLeaseTime': '1 day', 'dhcpBootOptionsEnabled': False, 'dhcpOptions': [], 'interfaceId': '1', 'networkName': 'NETWORK1'}, {'id': 2, 'networkId': 'L_2222', 'name': 'VLAN2', 'applianceIp': '2.2.2.2', 'subnet': '2.2.2.0/24', 'fixedIpAssignments': {}, 'reservedIpRanges': [], 'dnsNameservers': 'upstream_dns', 'dhcpHandling': 'Do not respond to DHCP requests', 'interfaceId': '2', 'networkName': 'NETWORK2'}]]
JSON version:
[
[
{
"id": 1,
"networkId": "L_1111",
"name": "VLAN1",
"applianceIp": "1.1.1.1",
"subnet": "1.1.1.0/24",
"fixedIpAssignments": {},
"reservedIpRanges": [],
"dnsNameservers": "upstream_dns",
"dhcpHandling": "Run a DHCP server",
"dhcpLeaseTime": "1 day",
"dhcpBootOptionsEnabled": false,
"dhcpOptions": [],
"interfaceId": "1",
"networkName": "NETWORK1"
},
{
"id": 2,
"networkId": "L_2222",
"name": "VLAN2",
"applianceIp": "2.2.2.2",
"subnet": "2.2.2.0/24",
"fixedIpAssignments": {},
"reservedIpRanges": [],
"dnsNameservers": "upstream_dns",
"dhcpHandling": "Do not respond to DHCP requests",
"interfaceId": "2",
"networkName": "NETWORK2"
},
]
]
I am trying to move this to a CSV file. However, I haven't figured out how to do it. I tried with pandas library but it isn't giving me the output that I am looking for.
Something like this:
id,networkId,name,applianceIp,subnet,fixedIpAssignments,reservedIpRanges,dnsNameservers,dhcpHandling,interfaceId,networkName
1,L_1111,VLAN1,1.1.1.1,1.1.1.0/24,{},[],upstream_dns,Do not respond to DHCP requests,1,NETWORK1
2,L_2222,VLAN2,2.2.2.2,2.2.2.0/24,{},[],upstream_dns,Do not respond to DHCP requests,2,NETWORK2
Expected Output:
id networkId name applianceIP subnet
1 L_1111 VLAN1 1.1.1.1 1.1.1.0/24
2 L_2222 VLAN2 2.2.2.2 2.2.2.0/24
I'd look at using pandas to convert the list to a dataframe and then you'll be able to export that to a csv file.
import pandas as pd
data = [[{'id': 1, 'networkId': 'L_1111', 'name': '1', 'applianceIp': '1.1.1.1', 'subnet': '1.1.1.0/24', 'fixedIpAssignments': {}, 'reservedIpRanges': [], 'dnsNameservers': 'upstream_dns', 'dhcpHandling': 'Run a DHCP server', 'dhcpLeaseTime': '1 day', 'dhcpBootOptionsEnabled': False, 'dhcpOptions': [], 'interfaceId': '1', 'networkName': '1'}, {'id': 2, 'networkId': 'L_2222', 'name': '2', 'applianceIp': '2.2.2.2', 'subnet': '2.2.2.0/24', 'fixedIpAssignments': {}, 'reservedIpRanges': [], 'dnsNameservers': 'upstream_dns', 'dhcpHandling': 'Do not respond to DHCP requests', 'interfaceId': '2', 'networkName': '2'}]]
df = pd.DataFrame(data[0])
df.to_csv("output.csv")
I used the csv module.
import json
import csv
import os
PATH = os.path.dirname(__file__) # Get the path of the used directory
with open(PATH+r"\input.json", "r") as file: # Access the data
json_data = json.load(file)
json_data = [item for item in json_data[0]]
with open(PATH+r"\output.csv", "w+", newline='') as file:
writer = csv.writer(file)
headers = [list(data.keys()) for data in json_data] # Divide the data in
rows = [list(data.values()) for data in json_data] # headers and rows
for i in range(len(json_data)):
writer.writerow(headers[i]) # Write everything
writer.writerow(rows[i])
If you don't want to have headers just remove this line writer.writerow(headers[i])
Here is the data I get as output:
id,networkId,name,applianceIp,subnet,fixedIpAssignments,reservedIpRanges,dnsNameservers,dhcpHandling,dhcpLeaseTime,dhcpBootOptionsEnabled,dhcpOptions,interfaceId,networkName
1,L_1111,VLAN1,1.1.1.1,1.1.1.0/24,{},[],upstream_dns,Run a DHCP server,1 day,False,[],1,NETWORK1
id,networkId,name,applianceIp,subnet,fixedIpAssignments,reservedIpRanges,dnsNameservers,dhcpHandling,interfaceId,networkName
2,L_2222,VLAN2,2.2.2.2,2.2.2.0/24,{},[],upstream_dns,Do not respond to DHCP requests,2,NETWORK2
If you use pandas dataframe, you can easily write csv file. Save each column of DataFrame to seperated column in csv file.
df.to_csv(r'myData.csv',sep=';',encoding="utf-8")
I have a json file structured like this:
[
{"ID":"fjhgj","Label":{"objects":[{"featureId":"jhgd","schemaId":"hgkl","title":"Kuh","}],"classifications":[]},"Created By":"xxx_xxx","Project Name":"Tiererkennung"},
{"ID":"jhgh","Label":{"objects":[{"featureId":"jhgd","schemaId":"erzl","title":"Kuh","}],"classifications":[]},"Created By":"xxx_xxx","Project Name":"Tiererkennung"},
...
and I would like to read all IDs and all schemaIds for each entry in the json file. I am codin in python.
What I tried is this:
import json
with open('Tierbilder.json') as f:
data=json.load(f)
data1 =data[0]
print(data1.values)
server_dict = {k:v for d in data for k,v in d.items()}
host_list = server_dict
Now I have the Problem that in host_list only the last row of my json file is saved. How can I get another row, like the first one?
Thanks for your help.
structure your JSON so it's readable and structure is clear
simple list comprehension
data you will have been read from your file
data = [{'ID': 'fjhgj',
'Label': {'objects': [{'featureId': 'jhgd','schemaId': 'hgkl','title': 'Kuh'}], 'classifications': []},
'Created By': 'xxx_xxx','Project Name': 'Tiererkennung'},
{'ID': 'jhgh', 'Label': {'objects': [{'featureId': 'jhgd','schemaId': 'erzl','title': 'Kuh'}], 'classifications': []},
'Created By': 'xxx_xxx','Project Name': 'Tiererkennung'}]
projschema = [{"ID":proj["ID"], "schemaId":schema["schemaId"]}
for proj in data
for schema in proj["Label"]["objects"]]
output
[{'ID': 'fjhgj', 'schemaId': 'hgkl'}, {'ID': 'jhgh', 'schemaId': 'erzl'}]
I'm trying to add user info to a nested dictionary within a json file. Here's the code.
import json
dictionary = {'name': 'Tony Stark', 'attributes': ['genius', 'billionaire', 'playboy', 'philanthropist']}
with open('info.json', 'a+') as file:
json.dump(dictionary, file)
The info.json file
{'marvel': [
{'name': 'Bill Gates', 'attributes': ['philanthropist', 'programmer']}
]
}
Now, I am unable to dump the dictionary as a value for marvel which is a list. I'm trying to make it dynamic, adding Tony Stark's info to the json file.
Please help me with that, thanks.
Alternative:
import json
dictionary = {'name': 'Tony Stark', 'attributes': ['genius', 'billionaire', 'playboy', 'philanthropist']}
def write_json(data, filename='file.json'):
# function to add to JSON
with open(filename,'w') as f:
json.dump(data, f, indent=4)
with open('file.json') as json_file:
data = json.load(json_file)
data['marvel'].append(dictionary) # appending data to Marvel
write_json(data)
Edited as per observation of juanpa.arrivillaga
I have a dictionary now:
data = [{'position': 1, 'name':'player1:', 'number': 524}, {'position':2, 'name': 'player2:','number': 333}]
(just list two group of number first to simplify the problem)
I want to read and print it in the order of positions: "position 1", "position 2" ... "position n" in a text or csv file.
something like:
position name number
1 player1 524
2 player2 333
I tried:
data = [{'position': 1, 'name':'player1', 'number': 524}, {'position':2, 'name': 'player2:','number': 333}]
keys = data[0].keys()
with open(output.csv", 'r') as output_file:
dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writeheader()
dict_writer.writerows(data)
Seems like I should create a csv instead of open it first. Also, is there any better ways? Thanks.
The easiest thing to do would probably be to read it into a pandas Dataframe and then write it to a csv.
import pandas as pd
data = [
{
'position': 1,
'name':'player1',
'number': 524
}, {
'position': 2,
'name': 'player2',
'number': 333
}
]
df = pd.DataFrame.from_records(data, columns=['position', 'name', 'number'])
df = df.sort_values('position')
df.to_csv('data.csv')
use pandas
import pandas as pd
data = [
{
'position': 1,
'name':'player1:',
'number': 524
}, {
'position':2,
'name':'player2:',
'number': 333
}
]
df = pd.DataFrame.from_records(data, columns=['position', 'name', 'number'])
df = df.sort_values('position')
df.head()
I have around 10 EBS volumes attached to a single instance. Below is e.g., of lsblk for some of them. Here we can't simply mount xvdf or xvdp to some location but actual point is xvdf1, xvdf2, xvdp which are to be mounted. I want to have a script that would allow me to iterate through all the points under xvdf, xvdp etc. using python. I m newbie to python.
[root#ip-172-31-1-65 ec2-user]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
xvdf 202:80 0 35G 0 disk
├─xvdf1 202:81 0 350M 0 part
└─xvdf2 202:82 0 34.7G 0 part
xvdp 202:0 0 8G 0 disk
└─xvdp1 202:1 0 8G 0 part
If you have a relatively new lsblk, you can easily import its json output into a python dictionary, which then open all possibilities for iterations.
# lsblk --version
lsblk from util-linux 2.28.2
For example, you could run the following command to gather all block devices and their children with their name and mount point. Use --help to get a list of all supported columns.
# lsblk --json -o NAME,MOUNTPOINT
{
"blockdevices": [
{"name": "vda", "mountpoint": null,
"children": [
{"name": "vda1", "mountpoint": null,
"children": [
{"name": "pv-root", "mountpoint": "/"},
{"name": "pv-var", "mountpoint": "/var"},
{"name": "pv-swap", "mountpoint": "[SWAP]"},
]
},
]
}
]
}
So you just have to pipe that output into a file and use python's json parser. Or run the command straight within your script as the example below shows:
#!/usr/bin/python3.7
import json
import subprocess
process = subprocess.run("/usr/bin/lsblk --json -o NAME,MOUNTPOINT".split(),
capture_output=True, text=True)
# blockdevices is a dictionary with all the info from lsblk.
# Manipulate it as you wish.
blockdevices = json.loads(process.stdout)
print(json.dumps(blockdevices, indent=4))
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
def parse(file_name):
result = []
with open(file_name) as input_file:
for line in input_file:
temp_arr = line.split(' ')
for item in temp_arr:
if '└─' in item or '├─' in item:
result.append(item.replace('└─','').replace('├─',''))
return result
def main(argv):
if len(argv)>1:
print 'Usage: ./parse.py input_file'
return
result = parse(argv[0])
print result
if __name__ == "__main__":
main(sys.argv[1:])
The above is what you need. You can modify it to parse the output of lsblk better.
Usage:
1. Save the output of lsblk to a file.
E.g. run this command: lsblk > output.txt
2. python parse.py output.txt
I remixed minhhn2910's answer for my own purposes to work with encrypted partitions, labels and build the output in a tree-like dict object. I'll probably keep a more updated version as I hit edge-cases on GitHub, but here is the basic code:
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
import re
import pprint
def parse_blk(blk_filename):
result = []
with open(blk_filename) as blk_file:
disks = []
for line in blk_file:
if line.startswith('NAME'): # skip first line
continue
blk_list = re.split('\s+', line)
node_type = blk_list[5]
node_size = blk_list[3]
if node_type in set(['disk', 'loop']):
# new disk
disk = {'name': blk_list[0], 'type': node_type, 'size': node_size}
if node_type == 'disk':
disk['partitions'] = []
disks.append(disk)
# get size info if relevant
continue
if node_type in set(['part', 'dm']):
# new partition (or whatever dm is)
node_name = blk_list[0].split('\x80')[1]
partition = {'name': node_name, 'type': node_type, 'size': node_size}
disk['partitions'].append(partition)
continue
if len(blk_list) > 8: # if node_type == 'crypt':
# crypt belonging to a partition
node_name = blk_list[1].split('\x80')[1]
partition['crypt'] = node_name
return disks
def main(argv):
if len(argv)>1:
print 'Usage: ./parse.py blk_filename'
return
result = parse_blk(argv[0])
pprint.PrettyPrinter(indent=4).pprint(result)
if __name__ == "__main__":
main(sys.argv[1:])
It works for your output as well:
$ python check_partitions.py blkout2.txt
[ { 'name': 'xvdf',
'partitions': [ { 'name': 'xvdf1', 'size': '350M', 'type': 'part'},
{ 'name': 'xvdf2', 'size': '34.7G', 'type': 'part'}],
'size': '35G',
'type': 'disk'},
{ 'name': 'xvdp',
'partitions': [{ 'name': 'xvdp1', 'size': '8G', 'type': 'part'}],
'size': '8G',
'type': 'disk'}]
This is how it works on a slightly more complicated scenario with docker loopback devices and encrypted partitions.
$ python check_partitions.py blkout.txt
[ { 'name': 'sda',
'partitions': [ { 'crypt': 'cloudfleet-swap',
'name': 'sda1',
'size': '2G',
'type': 'part'},
{ 'crypt': 'cloudfleet-storage',
'name': 'sda2',
'size': '27.7G',
'type': 'part'}],
'size': '29.7G',
'type': 'disk'},
{ 'name': 'loop0', 'size': '100G', 'type': 'loop'},
{ 'name': 'loop1', 'size': '2G', 'type': 'loop'},
{ 'name': 'mmcblk0',
'partitions': [{ 'name': 'mmcblk0p1',
'size': '7.4G',
'type': 'part'}],
'size': '7.4G',
'type': 'disk'}]