Develop Reference

gzipped jsonlines file read and write in python

While this code reads and writes a jsonlines file. How to compress it? I tried directly using gzip.open but I am getting various errors.
import json
def dump_jsonl(data, output_path, append=False):
"""
Write list of objects to a JSON lines file.
"""
mode = 'a+' if append else 'w'
with open(output_path, mode, encoding='utf-8') as f:
for line in data:
json_record = json.dumps(line, ensure_ascii=False)
f.write(json_record + '\n')
print('Wrote {} records to {}'.format(len(data), output_path))
def load_jsonl(input_path) -> list:
"""
Read list of objects from a JSON lines file.
"""
data = []
with open(input_path, 'r', encoding='utf-8') as f:
for line in f:
data.append(json.loads(line.rstrip('\n|\r')))
print('Loaded {} records from {}'.format(len(data), input_path))
return data
This is what I am doing to compress but I am unable to read it.
def dump_jsonl(data, output_path, append=False):
with gzip.open(output_path, "a+") as f:
for line in data:
json_record = json.dumps(line, ensure_ascii = False)
encoded = json_record.encode("utf-8") + ("\n").encode("utf-8")
compressed = gzip.compress(encoded)
f.write(compressed)

Use the gzip module's compress function.
import gzip
with open('file.jsonl') as f_in:
with gzip.open('file.jsonl.gz', 'wb') as f_out:
f_out.writelines(f_in)
gzip.open() is for opening gzipped files, not jsonl.
Read:
gzip a file in Python
Python support for Gzip

Converting JSON to CSV, CSV is empty

I'm attempting to convert yelps data set that is in JSON to a csv format. The new csv file that is created is empty.
I've tried different ways to iterate through the JSON but they all give me a zero bytes file.
The json file looks like this:
{"business_id":"1SWheh84yJXfytovILXOAQ","name":"Arizona Biltmore Golf Club","address":"2818 E Camino Acequia Drive","city":"Phoenix","state":"AZ","postal_code":"85016","latitude":33.5221425,"longitude":-112.0184807,"stars":3.0,"review_count":5,"is_open":0,"attributes":{"GoodForKids":"False"},"categories":"Golf, Active Life","hours":null}
import json
import csv
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
out = csv.writer(outfile)
out.writerow(data[0].keys())
for row in data:
out.writerow(row.values())
I get an "extra data" message when the code runs. The new business2 csv file is empty and the size is zero bytes.

if you JSON has only one row.. then try this
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
out = csv.writer(outfile)
#print(data.keys())
out.writerow(data.keys())
out.writerow(data.values())

Hi Please try the below code, by using with command the file access will automatically get closed when the control moves out of scope of with
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
headers = list(data.keys())
values = list(data.values())
with open("business2.csv","w") as outfile:
out = csv.writer(outfile)
out.writerow(headers)
out.writerow(values)

You need to use with to close file.
import json
import csv
infile = open("business.json","r")
data = json.load(infile)
infile.close()
with open("business2.csv","w") as outfile:
out = csv.writer(outfile)
out.writerow(list(data.keys()))
out.writerow(list(data.values()))

keep data in the file python

hi guys how are you I hope that well, I'm new using python and I'm doing a program but I dont know how to save the data permanently in a file. I only know how to create the file but i dont know how can i keep the data on the file eventhough the program be closed and when i open it back i be able to add more data and keep it in the file too.I have also tried several methods to upload the file on python but they didnt work for me. Can someone please help me?
This is my code:
file = open ('file.txt','w')
t = input ('name :')
p= input ('last name: ')
c = input ('nickname: ')
file.write('name :')
file.write(t)
file.write(' ')
file.write('last name: ')
file.write(p)
file.write('nickname: ')
file.write(c)
file.close()
with open('archivo.txt','w') as file:
data = load(file)
print(data)

Here is a demonstration of how file writing works, and the difference between w and a. The comments represent the text in the file that is written to the drive at each given point.
f1 = open('appending.txt', 'w')
f1.write('first string\n')
f1.close()
# first string
f2 = open('appending.txt', 'a')
f2.write('second string\n')
f2.close()
# first string
# second string
f3 = open('appending.txt', 'w')
f3.write('third string\n')
f3.close()
# third string

There are three type of File operation mode can happen on file like read, write and append.
Read Mode: In this you only able to read the file like
#content in file.txt "Hi I am Python Developer"
with open('file.txt', 'r') as f:
data = f.read()
print(data)
#output as : Hi I am Python Developer
Write Mode: In this you are able to write information into files, but it will always overwrite the content of the file like for example.
data = input('Enter string to insert into file:')
with open('file.txt', 'w') as f:
f.write(data)
with open('file.txt', 'r') as f:
data = f.read()
print('out_data:', data)
# Output : Enter string to insert into file: Hi, I am developer
# out_data: Hi, I am developer
When you open file for next time and do same write operation, it will overwrite whole information into file.
Append Mode: In this you will able to write into file but the contents is append into this files. Like for example:
data = input('Enter string to insert into file:')
with open('file.txt', 'a') as f:
f.write(data)
with open('file.txt', 'r') as f:
data = f.read()
print('out_data:', data)
# Output : Enter string to insert into file: Hi, I am developer
# out_data: Hi, I am developer
# Now perform same operation:
data = input('Enter string to insert into file:')
with open('file.txt', 'a') as f:
f.write(data)
with open('file.txt', 'r') as f:
data = f.read()
print('out_data:', data)
# Output : Enter string to insert into file: Hi, I am Python developer
# out_data: Hi, I am developer Hi, I am Python Developer

How to write the list to file?

I am unable to write the result of the following code to a file
import boto3
ACCESS_KEY= "XXX"
SECRET_KEY= "XXX"
regions = ['us-east-1','us-west-1','us-west-2','eu-west-1','sa-east-1','ap-southeast-1','ap-southeast-2','ap-northeast-1']
for region in regions:
client = boto3.client('ec2',aws_access_key_id=ACCESS_KEY,aws_secret_access_key=SECRET_KEY,region_name=region,)
addresses_dict = client.describe_addresses()
#f = open('/root/temps','w')
for eip_dict in addresses_dict['Addresses']:
with open('/root/temps', 'w') as f:
if 'PrivateIpAddress' in eip_dict:
print eip_dict['PublicIp']
f.write(eip_dict['PublicIp'])
This results in printing the IP's but nothing gets written in file, the result of print is :
22.1.14.1
22.1.15.1
112.121.41.41
....
I just need to write the content in this format only

for eip_dict in addresses_dict['Addresses']:
with open('/root/temps', 'w') as f:
if 'PrivateIpAddress' in eip_dict:
print eip_dict['PublicIp']
f.write(eip_dict['PublicIp'])
You are re-opening the file for writing at each iteration of the loop. Perhaps the last iteration has no members with 'PrivateIpAddress' in its dict, so the file gets opened, truncated, and left empty. Write it this way instead:
with open('/root/temps', 'w') as f:
for eip_dict in addresses_dict['Addresses']:
if 'PrivateIpAddress' in eip_dict:
print eip_dict['PublicIp']
f.write(eip_dict['PublicIp'])

open file in append mode
with open('/root/temps', 'a') as f:
or
declare the file outside the loop

Append JSON to file

I am trying to append values to a json file. How can i append the data? I have been trying so many ways but none are working ?
Code:
def all(title,author,body,type):
title = "hello"
author = "njas"
body = "vgbhn"
data = {
"id" : id,
"author": author,
"body" : body,
"title" : title,
"type" : type
}
data_json = json.dumps(data)
#data = ast.literal_eval(data)
#print data_json
if(os.path.isfile("offline_post.json")):
with open('offline_post.json','a') as f:
new = json.loads(f)
new.update(a_dict)
json.dump(new,f)
else:
open('offline_post.json', 'a')
with open('offline_post.json','a') as f:
new = json.loads(f)
new.update(a_dict)
json.dump(new,f)
How can I append data to json file when this function is called?

I suspect you left out that you're getting a TypeError in the blocks where you're trying to write the file. Here's where you're trying to write:
with open('offline_post.json','a') as f:
new = json.loads(f)
new.update(a_dict)
json.dump(new,f)
There's a couple of problems here. First, you're passing a file object to the json.loads command, which expects a string. You probably meant to use json.load.
Second, you're opening the file in append mode, which places the pointer at the end of the file. When you run the json.load, you're not going to get anything because it's reading at the end of the file. You would need to seek to 0 before loading (edit: this would fail anyway, as append mode is not readable).
Third, when you json.dump the new data to the file, it's going to append it to the file in addition to the old data. From the structure, it appears you want to replace the contents of the file (as the new data contains the old data already).
You probably want to use r+ mode, seeking back to the start of the file between the read and write, and truncateing at the end just in case the size of the data structure ever shrinks.
with open('offline_post.json', 'r+') as f:
new = json.load(f)
new.update(a_dict)
f.seek(0)
json.dump(new, f)
f.truncate()
Alternatively, you can open the file twice:
with open('offline_post.json', 'r') as f:
new = json.load(f)
new.update(a_dict)
with open('offline_post.json', 'w') as f:
json.dump(new, f)

This is a different approach, I just wanted to append without reloading all the data. Running on a raspberry pi so want to look after memory. The test code -
import os
json_file_exists = 0
filename = "/home/pi/scratch_pad/test.json"
# remove the last run json data
try:
os.remove(filename)
except OSError:
pass
count = 0
boiler = 90
tower = 78
while count<10:
if json_file_exists==0:
# create the json file
with open(filename, mode = 'w') as fw:
json_string = "[\n\t{'boiler':"+str(boiler)+",'tower':"+str(tower)+"}\n]"
fw.write(json_string)
json_file_exists=1
else:
# append to the json file
char = ""
boiler = boiler + .01
tower = tower + .02
while(char<>"}"):
with open(filename, mode = 'rb+') as f:
f.seek(-1,2)
size=f.tell()
char = f.read()
if char == "}":
break
f.truncate(size-1)
with open(filename, mode = 'a') as fw:
json_string = "\n\t,{'boiler':"+str(boiler)+",'tower':"+str(tower)+"}\n]"
fw.seek(-1, os.SEEK_END)
fw.write(json_string)
count = count + 1