i have written a python program which makes an api call to a webserver once every minute and then parse the json response and saves parsed values in to the csv files.
here is the code that is saving the values into the csv file :
with open('data.csv', 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow([current_time,SHORTPERC, LONGPERC, SHORTvolume, longVolume, longPositions, shortPositions])
how can i make it so that it saves the header only once on the top most row and not on every row ?
UPDATE:
here is a bit of more code to make api call and write the data to file :
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.triggers.cron import CronTrigger
import requests
import json
import csv
from datetime import datetime
def fn():
print("Starting...")
session_id = "auZsJ4F2RsQNJxSPTMDt2238324"
Outlook='http://www.myfxbook.com/api/get-community-outlook.json?session=' + session_id
Outlook_response = requests.get(Outlook)
Outlook_data = Outlook_response.json()['symbols']
now = datetime.now()
current_time = now.strftime("%H:%M")
EURUSD=Outlook_data[0]
SHORTPERC=EURUSD['shortPercentage']
LONGPERC =EURUSD['longPercentage']
SHORTvolume=EURUSD['shortVolume']
longVolume=EURUSD['longVolume']
longPositions=EURUSD['longPositions']
shortPositions=EURUSD['shortPositions']
with open('data.csv', 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow([current_time,SHORTPERC, LONGPERC, SHORTvolume, longVolume, longPositions, shortPositions])
with open('data1.csv', 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow([SHORTvolume, longVolume])
with open('data2.csv', 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow([SHORTPERC, LONGPERC])
i cant post the full code cuz it will be very ugly since its around 700 lines long , but the above mentioned code should work to create the csv file
this is how one of my csv files look :
07:11,31,69,555.55,1265.14,4750,2607
07:12,31,69,555.55,1265.16,4751,2607
07:13,31,69,555.55,1265.16,4751,2607
07:14,30,70,555.56,1267.36,4752,2608
07:15,30,70,555.56,1267.36,4752,2608
07:16,30,70,555.56,1267.36,4752,2608
07:17,30,70,555.46,1267.36,4752,2607
07:18,31,69,558.61,1267.36,4752,2610
07:19,31,69,558.61,1267.37,4753,2610
07:20,31,69,561.58,1267.37,4753,2611
07:21,31,69,561.61,1267.37,4753,2613
07:22,31,69,561.65,1267.37,4753,2614
07:23,31,69,561.65,1267.36,4752,2614
this is just part of the csv file , more rows keep adding as time passes
EDIT 2:
answer suggested by Sparkofska seems to work but somehow it ends up giving an empty row in between every line like this:
Time,ShortPer,LongPer,ShortVolume,LongVolume,ShortPosition,LongPosition
05:47,44,56,19528.8,24789.27,65223,48630
05:48,44,56,19529.04,24789.27,65223,48633
code :
EURUSD=Outlook_data[0]
SHORTPERC=EURUSD['shortPercentage']
LONGPERC =EURUSD['longPercentage']
SHORTvolume=EURUSD['shortVolume']
longVolume=EURUSD['longVolume']
longPositions=EURUSD['longPositions']
shortPositions=EURUSD['shortPositions']
filename='EURUSD.csv';
def write_row_header_aware(filename, row):
if not os.path.exists(filename) or os.stat(filename).st_size == 0:
with open(filename, 'a') as file:
writer = csv.writer(file)
writer.writerow(['Time', 'ShortPer', 'LongPer','ShortVolume','LongVolume','ShortPosition','LongPosition'])
with open(filename, 'a') as file:
writer = csv.writer(file)
writer.writerow([current_time,SHORTPERC, LONGPERC, SHORTvolume, longVolume, longPositions, shortPositions])
write_row_header_aware(filename, [current_time,SHORTPERC, LONGPERC, SHORTvolume, longVolume, longPositions, shortPositions])
print("done...")
You could wrap the writerow function to have it automatically add the header if needed.
If your output csv file is not empty, we can assert the header was already written and simply append the row. Otherwise (file not exist or empty) we write the header before appending the row.
import os
def write_row_header_aware(filename, row):
# in case file doesn't exist or is empty
if not os.path.exists(filename) or os.stat(filename).st_size == 0:
# write header
with open(filename, 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow(['current_time', 'SHORTPERC', 'LONGPERC', ...])
# write line as usual
with open(filename, 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow(row)
write_row_header_aware('data.csv', [current_time, SHORTPERC, LONGPERC, ...])
Please make a check to know if the file exists, if it already exists use append to write rows to file else write the headers. By this way you could avoid writing headers multiple times. Please refer to this link
I am trying to write some output to csv file line by line
Here what I tried:
import csv
today = datetime.datetime.now().date()
filter = "eventTimestamp ge {}".format(today)
select = ",".join([
"eventTimestamp",
"eventName",
"operationName",
"resourceGroupName",
])
activity_logs = client.activity_logs.list(
filter=filter,
select=select
)
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
for log in activity_logs:
result = (" ".join([
str(log.event_timestamp),
str(log.resource_group_name),
log.event_name.localized_value,
log.operation_name.localized_value
]))
f.writerow(result)
Its throwing error:
AttributeError: '_io.TextIOWrapper' object has no attribute 'writerow'
How can i fix this error, possibly any other module ?
This:
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
is creating just text file handle. You need to create csv.writer using f and then you might use writerow, that is:
import csv
...
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
writer = csv.writer(f)
for log in activity_logs:
result = (str(log.event_timestamp),str(log.resource_group_name),log.event_name.localized_value,log.operation_name.localized_value)
writer.writerow(result)
You might find useful examples of usage in csv article at PyMOTW-3
The error is coming from the line:
f.writerow(result)
and it's telling you that the f object does not have a function named writerow.
As Jannes has commented, use the write function instead:
f.write(result)
CSV.writer is required when your trying to write into CSV . then the code can be
import csv
today = datetime.datetime.now().date()
filter = "eventTimestamp ge {}".format(today)
select = ",".join([
"eventTimestamp",
"eventName",
"operationName",
"resourceGroupName",
])
activity_logs = client.activity_logs.list(
filter=filter,
select=select
)
with open(r"C:\scripts\logs.csv", 'w', newline='') as file:
f=csv.writer(file)
for log in activity_logs:
result = (str(log.event_timestamp),
str(log.resource_group_name),
log.event_name.localized_value,
log.operation_name.localized_value)
f.writerow(result)
When the csv.writer is added after opening the csv file it will work without TextIOwrapper error
I'm attempting to convert yelps data set that is in JSON to a csv format. The new csv file that is created is empty.
I've tried different ways to iterate through the JSON but they all give me a zero bytes file.
The json file looks like this:
{"business_id":"1SWheh84yJXfytovILXOAQ","name":"Arizona Biltmore Golf Club","address":"2818 E Camino Acequia Drive","city":"Phoenix","state":"AZ","postal_code":"85016","latitude":33.5221425,"longitude":-112.0184807,"stars":3.0,"review_count":5,"is_open":0,"attributes":{"GoodForKids":"False"},"categories":"Golf, Active Life","hours":null}
import json
import csv
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
out = csv.writer(outfile)
out.writerow(data[0].keys())
for row in data:
out.writerow(row.values())
I get an "extra data" message when the code runs. The new business2 csv file is empty and the size is zero bytes.
if you JSON has only one row.. then try this
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
out = csv.writer(outfile)
#print(data.keys())
out.writerow(data.keys())
out.writerow(data.values())
Hi Please try the below code, by using with command the file access will automatically get closed when the control moves out of scope of with
infile = open("business.json","r")
outfile = open("business2.csv","w")
data = json.load(infile)
infile.close()
headers = list(data.keys())
values = list(data.values())
with open("business2.csv","w") as outfile:
out = csv.writer(outfile)
out.writerow(headers)
out.writerow(values)
You need to use with to close file.
import json
import csv
infile = open("business.json","r")
data = json.load(infile)
infile.close()
with open("business2.csv","w") as outfile:
out = csv.writer(outfile)
out.writerow(list(data.keys()))
out.writerow(list(data.values()))
I'm creating a Django app and I need to import several *.csv files.
One's of this file has this structure:
id|value (header)
12|¤this is the
value¤
34|¤this is another
value¤
I use this code for parse the file:
try:
csvfile = open(path, "r", encoding='utf-16')
except IOError:
return False
cursor.copy_from(csvfile , tblname, columns=['id', 'value'], sep='|')
But when I try to parse this file, it gave me this error:
psycopg2.DataError: ERROR: missing data for the column "value"
Is there a way to parse this file keeping carriage return inside text identifier ('¤')?
You could use Pythons csv module for reading that.
import csv
try:
csvfile = open(path, newline='')
except IOError:
return False
csvreader = csv.reader(csvfile, delimiter='|', quotechar='¤')
for row in csvreader:
print(', '.join(row)) # or do something else with the row of data.
One approach would be to build up the entries yourself as follows:
blocks = []
block = []
with open('input.csv') as f_input:
for row in f_input:
if '|' in row:
if len(block):
blocks.append(''.join(block).strip('\n').split('|'))
block = []
block.append(row)
else:
block.append(row)
if len(block):
blocks.append(''.join(block).strip('\n').split('|'))
print(blocks)
This would produce a list of blocks as follows:
[['id', 'value (header)'], ['12', '¤this is the\nvalue¤'], ['34', '¤this is another\nvalue¤']]
I want to run this program to convert the Json file into a dictionary, Im using linux mint, what commands do i use to run the program and convert the file.
import csv
import json
class Coverters:
def covert_to_dict(self, filename):
""" Read data from file and transform it to dictionary """
out = []
with open(filename, "r") as file:
output = csv.DictReader(file, fieldnames=self.__labels)
print(output)
for row in output:
print(row)
out.append(row)
# for line in file:
# out.append(dict(zip(self.__labels, line.split('#'))))
return out
def json_to_csv_file(self, csv_filename, json_filename):
""" Helper function to conver JSON to CSV file"""
with open(json_filename) as file:
data = json.load(file)
with open(csv_filename, "wb+") as file:
csv_file = csv.writer(file)
for item in data:
# Need to add all indexes for items
csv_file.writerow([item['ts'], item['visitor_uuid']] + item['fields'].values())
import csv
import json
def covert_to_dict(filename):
""" Read data from file and transform it to dictionary """
out = []
with open(filename, "r") as file:
output = csv.DictReader(file, fieldnames=self.__labels)
print(output)
for row in output:
print(row)
out.append(row)
# for line in file:
# out.append(dict(zip(self.__labels, line.split('#'))))
return out
covert_to_dict("filename") #change ot to the file name
save the above code in a file name it somthing( xyz.py)
place the file to convert in same directory
open terminal ->> go the directory ->> and run this command --> python xyz.py