I have a bunch of json files with multiple lines who look like this:
file1
{"id":1,"name":"Eric","height":1.80, ...},
{"id":2,"name":"Bob","height":1.90, ...}
...
file2
{"id":3,"name":"Jenny","height":1.50, ...},
{"id":4,"name":"Marlene","height":1.60, ...}
...
I want to build a generator to yield each line as a dictionary. My current code:
from typing import Iterator, Dict, Any, Optional
import io
import os
def json_gen2(file_list: list) -> Iterator[Dict[str, Any]]:
import json
for file in file_list:
with open(file) as json_file:
data = []
for line in json_file:
data = json.load(line)
if not data:
break
yield data
datapath = os.path.normcase(os.getcwd()) + '/data/log_data'
file_list = get_files(datapath) # create path list of json files
jsonfile = json_gen2(file_list)
next(jsonfile)
i get the following
Error Message
pls help :)
Oops, I misread. You are doing the same thing I was saying. Your error is due to using 'load' instead of 'loads'. Each line returned by
for line in json_file:
data = json.load(line)
is a string, and you're attempting to read it as a file pointer.
Related
How to dump data into Json file
*as can see in the below python code I am trying the dump data in Json file so but I am struggling to do it in python code *
import time
import json
import os
def long_function(name):
cache_path = 'cache.json'
if not os.path.isfile(cache_path):
with open(cache_path, 't') as json_file:
cache_file_data = [name]
jsondump(cache_file_data, json_file)
else:
with open(cache_path, 'r') as json_file:
cache_file_data = json.load(json_file)
if name in cache_file_data:
print("Name already exist")
return name
else:
cache_file_data.append(name)
for e in range(5):
time.sleep(1)
print(e+1)
with open(cache_path, 'w') as json_file:
jsondump(cache_file_data, json_file)
print("New Name added in cache")
return name
print(long_function('nitu'))
so please resolve my problem......please help me
import json
# JSON data:
x = '{ "organization":"New_holn",
"city":"Noida",
"country":"India"}'
# python object to be appended
y = {"pin":117845}
# parsing JSON string:
z = json.loads(x)
# appending the data
z.update(y)
# the result is a JSON string:
print(json.dumps(z))
This is nothing but follow this pattern and your so your code error is ..you are not defined file mode correctly in if condition
with open (cache_path. "t") as json_file:
Instead of
with open (cache_path. "w") as json_file:
And second thing is you are not doing dump data
I am trying to read the file from the path and add to the dictionary, however, i end up with first json-line and there are about 230MB of data.
I have folder like here
# folder path
folder = os.path.expanduser("~/topic_2022-05-30T23-15-20.jl")
#Initialise the dictionary
items = {}
with open(folder, 'rb') as f:
for item in json_lines.reader(f):
# adding next item to items
items.update(item)
import json
file = open(folder)
items = {}
data = json.load(file)
for k, v in data.items():
items.update(k:v)
file.close()
itmes = data could also work.
I was working on a piece of code that reads a json file ever second or so in accordance with a pyqt5 graph. This was my code:
import csv
from getmac import get_mac_address as gma
import time
macaddr = gma()
mac_name = macaddr.replace(":", "")
weathersensor_id = "BME20"
rainsensor_id = "GCCG41"
def readJSONLatestAllMQTT():
with open(f"{mac_name}_{weathersensor_id}.json", "r") as myfile:
dataRead = json.load(myfile)
time.sleep(0.01)
return dataRead, True
When I went to access an element of the json file by doing something like print(dataRead["Temperature"]), I got an error message that said that dataRead was being output as a string, not a dict/json file like I intented. How would I go about turning dataRead into a dict instead of a string?
Here is the json file for some background:
"{\"dateTime\": \"2021-06-18 07:47:33.710631\", \"Temperature\": 69.26586754195563, \"Pressure\": 346.63102628054014, \"Humidity\": 80.54066707990641, \"Altitude\": 125.3640651860123}"
I believe you are having this problem because your json file is a string instead of a dictionary, I think it should be:
{"dateTime": "2021-06-18 07:47:33.710631", "Temperature": 69.26586754195563, "Pressure": 346.63102628054014, "Humidity": 80.54066707990641, "Altitude": 125.3640651860123}
instead of:
"{\"dateTime\": \"2021-06-18 07:47:33.710631\", \"Temperature\": 69.26586754195563, \"Pressure\": 346.63102628054014, \"Humidity\": 80.54066707990641, \"Altitude\": 125.3640651860123}"
I want to replace the values from YAML file into a JSON file using Python 3.
I have many JSON files where I want to get the values from a master YAML file and replace only certain values like source server ip, email, hostname.
E.g. I have this YAML file (mast_conf.yaml):
- sourcesystem1:
sourceServer: 1.2.3.500
MailTo: gokul#gmail.com
- sourcesystem2:
sourceServer1: 2.2.3.500
sourceServer2: 3.2.3.500
MailTo: gokul#gmail.com
A JSON file (sourcesystem1.json):
{
"source":"sourcesystem1",
"frequency":"daily",
"sourceServer":"1.2.1.2",
"hostName":"1.2.1.3",
"fileFormat":"csv",
"delimiterType":"semicolon"
}
Another JSON file (sourcesystem2.json):
{
"source":"sourcesystem2",
"frequency":"daily",
"sourceServer":"1.2.3.2",
"hostName":"1.2.1.7",
"fileFormat":"csv",
"delimiterType":"commaseperated"
}
Below is my code I am trying out to parse the value from the json file
import json
import yaml
with open("master_conf.yaml", 'r') as f:
yaml_config = yaml.safe_load(f)
yaml_config = {
list(config.keys()[0]): list(config[config.keys()[0]])
for config in yaml_config
}
json_files = ( "sourcesystem1.json",
"sourcesystem2.json",
)
for json_file in json_files:
with open(json_file, "r") as f:
sourcesystem_conf = json.load(f)
sourcesystem = sourcesystem_conf["source"]
if sourcesystem in yaml_config:
for key, value in yaml_config[sourcesystem].items():
sourcesystem_conf[key] = value
with open(json_file, "w") as f:
json.dump(sourcesystem_conf, f, indent=2)
I am getting the below error by program
TypeError: 'dict_keys' object does not support indexing
When I run indivudually I get this issue for yaml
>>> yaml_config = { ... config.keys()[0]: config[config.keys()[0]] ... for config in yaml_config ... } Traceback (most recent call last): File "<stdin>", line 3, in <module> File "<stdin>", line 3, in <dictcomp> TypeError: 'dict_keys' object is not subscriptable >>>
Is there easier method to achieve my end goal where I want to replace the values in the JSON file from the Yaml configuration file
This is needed to update 1000s of Json file in a automated way for updating it from a master Yaml file
The easiest way is to use pyyaml, see Jon's answer.
Then you can load you yaml file using it :
>>> import yaml
>>> yaml_config = yaml.safe_load(yaml_file)
>>> yaml_config
[{'sourcesystem1': {'MailTo': 'gokul#gmail.com', 'sourceServer': '1.2.3.500'}},
{'sourcesystem2': {'MailTo': 'gokul#gmail.com',
'sourceServer1': '2.2.3.500',
'sourceServer2': '3.2.3.500'}}]
It will be easier to manipulate a dict with source systems as keys.
In python 2 aDict.keys() returns a list so the following will work :
>>> yaml_config = {
config.keys()[0]: config[config.keys()[0]]
for config in yaml_config
}
>>> yaml_config
{'sourcesystem1': {'MailTo': 'gokul#gmail.com', 'sourceServer': '1.2.3.500'},
'sourcesystem2': {'MailTo': 'gokul#gmail.com',
'sourceServer1': '2.2.3.500',
'sourceServer2': '3.2.3.500'}}
In python 3 aDict.keys() no longer returns a list so you can simply use a for loop :
yaml_config = {}
for config in yaml_config_raw:
source = [key for key in config][0]
yaml_config[source] = config[source]
Then you can just iterate over your json files to update them :
import json
import yaml
with open("mast_conf.yaml", 'r') as f:
yaml_config_raw = yaml.safe_load(f)
yaml_config = {}
for config in yaml_config_raw:
source = [key for key in config][0]
yaml_config[source] = config[source]
json_files = (
"sourcesystem1.json",
"sourcesystem2.json",
)
for json_file in json_files:
with open(json_file, "r") as f:
sourcesystem_conf = json.load(f)
sourcesystem = sourcesystem_conf["source"]
if sourcesystem in yaml_config:
for key, value in yaml_config[sourcesystem].items():
sourcesystem_conf[key] = value
with open(json_file, "w") as f:
json.dump(sourcesystem_conf, f, indent=2)
I'm trying to remove the very first character (") from a file which contains a JSON String. I'm using Python for this. Below is my code:
jsonOutput = 'JsonString_{}.{}'.format(str(uuid.uuid1()), "json")
jsonOutput_File = os.path.join(arcpy.env.scratchFolder, jsonOutput)
with open(jsonOutput_File, 'w') as json_file:
json.dump(jsonString, json_file)
// I was able to remove the very last character using the code below
with open(jsonOutput_File, 'r+') as read_json_file:
read_json_file.seek(-1, os.SEEK_END)
read_json_file.truncate()
Basically when I dump the JSON String to a file, the String is getting surrounded by double quotes. I'm trying to remove these double quotes from the first & last position of the file.
If you already have a JSON string, simply write it to the file.
Encoding the JSON string to JSON again using json.dump() is a bad idea and will not be fixed as simple as removing a leading and a trailing quote.
Consider the following minimal and complete example:
import json
import os
import uuid
myobject = {"hello": "world"}
jsonString = json.dumps(myobject)
jsonOutput = 'JsonString_{}.{}'.format(str(uuid.uuid1()), "json")
jsonOutput_File = os.path.join("d:\\", jsonOutput)
with open(jsonOutput_File, 'w') as json_file:
json.dump(jsonString, json_file)
The output is a file with the content:
"{\"hello\": \"world\"}"
Removing the quotes will not make it valid JSON.
Instead, avoid the duplicate JSON creation, either by removing json.dumps() which converts the object to JSON one time, or by removing json.dump(), which does it a second time.
Solution 1:
import json
import os
import uuid
myobject = {"hello": "world"}
# <-- deleted line here
jsonOutput = 'JsonString_{}.{}'.format(str(uuid.uuid1()), "json")
jsonOutput_File = os.path.join("d:\\", jsonOutput)
with open(jsonOutput_File, 'w') as json_file:
json.dump(myobject, json_file) # <-- changed to object here
Solution 2:
import json
import os
import uuid
myobject = {"hello": "world"}
jsonString = json.dumps(myobject)
jsonOutput = 'JsonString_{}.{}'.format(str(uuid.uuid1()), "json")
jsonOutput_File = os.path.join("d:\\", jsonOutput)
with open(jsonOutput_File, 'w') as json_file:
json_file.write(jsonString) # <-- Note this line