Develop Reference

Continues columns in a csv-file with python

I have a Problem with continues writing my datas in a csv-file. I want a program that detects, if there is a csv-file for my measurements-data. If not it would be generated. When the csv-file is new generated the datas are written in the csv-file on the column after the header with the variable cycle = 0.
If the csv-file exists, the datas should be written continuously after the last line of the csv. Also the variable cycle should continue.
I have written a program that can detect if there is a file or not but with the continuously lines I have problems.
I hope someone can help me.
# mes = Array with 20 spaces filled with the Numbers 0-19
date = time.strftime("%d/%m/%Y")
def write(cycle, mes):
if os.path.exists('/home/pi/Documents/Ventilatorprüfstand_Programm/out.csv') is True: #does the out.csv existate?
print("Do something")
out = open('out.csv', 'w')
data = [[cycle, mes[0],mes[1],mes[2],mes[3],mes[4],mes[5],mes[6],mes[7],mes[8],mes[9],mes[10],mes[11],mes[12],mes[13],mes[14],mes[15],mes[16],mes[17],mes[18],mes[19], date]]
line = cycle+1
for row in data:
for line in row:
out.write('%s;' % line)
out.write('\n')
out.close()
else:
print("Do another something")
header = lookuptable.names()
out = open('out.csv', 'w')
for row in header:
for column in row:
out.write('%s' % column)
out.write('\t')
out.write('\n')
data = [[cycle, mes[0],mes[1],mes[2],mes[3],mes[4],mes[5],mes[6],mes[7],mes[8],mes[9],mes[10],mes[11],mes[12],mes[13],mes[14],mes[15],mes[16],mes[17],mes[18],mes[19], date]]
for row in data:
for column in row:
out.write('%s;' % column)
out.write('\n')
out.close()`

When opening the file with open() there is the option 'a' to append the new lines to the end:
'a' open for writing, appending to the end of the file if it exists
Here is an example using the csv Python standard library:
import csv
import os
import random
headers = ['cycle', 'date', 'speed', 'temp', 'power']
new_data = [[random.randint(0, 100) for _ in range(3)] for _ in range(2)]
date = '00/01/02'
cycle = 1
# Copy the data and include the date and the cycle number:
full_rows = [ [cycle, date, *row] for row in new_data ]
filename = 'example.csv'
# Check if the file exist, if not create the file with header
if not os.path.exists(filename):
print('creating a new file')
with open(filename, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',')
csvwriter.writerow(headers) # add the header
# Append the data to the file
with open(filename, 'a', newline='') as csvfile: # note the 'a' option
csvwriter = csv.writer(csvfile, delimiter=',')
csvwriter.writerows(full_rows)

Write variable output to a specific column in a CSV?

I'm working on a Python script that scrapes data from an Excel doc, then writes the output to a .csv.
I was able to grab the data and get it to write to the .csv, but all of the data goes into the first column.
I need the bar data to go into the 4th and the foo to go into the 5th column, so I tried to use csv.reader to select the row, and this runs without error but doesn't actually write to the .csv file.
Here's my code:
import xlrd
import csv
###Grab the data
def get_row_values(workSheet, row):
to_return = []
num_cells = myWorksheet.ncols - 1
curr_cell = -1
while curr_cell < num_cells:
curr_cell += 1
cell_value = myWorksheet.cell_value(row, curr_cell)
to_return.append(cell_value)
return to_return
file_path = 'map_test.xlsx'
output = []
output_bar = []
output_foo = []
myWorkbook = xlrd.open_workbook(file_path)
myWorksheet = myWorkbook.sheet_by_name('Sheet1')
num_rows = myWorksheet.nrows - 1
curr_row = 0
column_names = get_row_values(myWorksheet, curr_row)
print len(column_names)
while curr_row < num_rows:
curr_row += 1
row = myWorksheet.row(curr_row)
this_row = get_row_values(myWorksheet, curr_row)
x = 0
while x <len(this_row):
if this_row[x] == 'x':
output.append([this_row[0], column_names[x]])
output_bar.append([column_names[x]])
output_foo.append([this_row[0]])
print output
myData = [["number", "name", "version", "bar",
"foo"]]
##### Next section is the code in question, it
####doesn't error out, but won't write to the .csv######
myFile = open("test123.csv", "w")
writer = csv.writer(myFile)
with open('test123.csv', 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
for row in reader:
row[5] = myFile.readline()
writer.writerows(output_foo)
row[4] = myFile.readline()
writer.writerows(outpu_bar)
#####This successfully writes to the csv, but
#####all data to first column#####
# myFile = open('test123.csv', 'w')
# with myFile:
# writer = csv.writer(myFile)
# writer.writerows(myData)
# #writer.writerows(output)
# writer.writerows(output_foo)
# writer.writerows(output_bar)
x += 1
print ("CSV Written")

There's no value in my output file

my file contains "Name" and 5 eye movement values (TFF, TFD, TVD, FB, FC). I want to sum up each eye movement values if the rows under Name column are the same. It seems like the code is working, there's no error happened, but my output files stayed empty. Could anyone give me some pointers where went wrong? Here's the code:
import csv
file = open("P01_All.csv", "r") #Open CSV File in Read Mode
reader = csv.reader(file) #Create reader object which iterates over lines
outfile = open("Name.csv","w")
outfile2 = open("TFF.csv","w")
outfile3 = open("TFD.csv","w")
outfile4 = open("TVD.csv","w")
outfile5 = open("FB.csv","w")
outfile6 = open("FC.csv","w")
class Object: #Object to store unique data
def __init__(self, Name, TFF, TFD, TVD, FB, FC):
self.Name = Name
self.TFF = TFF
self.TFD = TFD
self.TVD = TVD
self.FB = FB
self.FC = FC
rownum = 0 #Row Number currently iterating over
list = [] #List to store objects
def checkList(Name, TFF, TFD, TVD, FB, FC):
for object in list: #Iterate through list
if object.Name == Name:
object.TFF += float(TFF)
object.TFD += float(TFD)
object.TVD += float(TVD)
object.FB += float(FB)
object.FC += float(FC)
return
newObject = Object(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC)) #Create a new object with new eye and TFF
list.append(newObject) #Add to list and break out
for row in reader: #Iterate through all the rows
if rownum == 0: #Store header row seperately to not get confused
header = row
else:
Name = row[0]
TFF = row[1]
TFD = row[2]
TVD = row[3]
FB = row[4]
FC = row[5]
if len(list) == 0: #Default case if list = 0
newObject = Object(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC))
list.append(newObject)
else: #If not...
checkList(Name, TFF, TFD, TVD, FB, FC)
rownum += 1
for each in list: #Print out result
# print(each.Name, each.TFF, each.TFD, each.TVD, each.FB, each.FC)
outfile.write(each.Name + "\n" )
outfile2.write(str(each.TFF)+ "\n" )
outfile3.write(str(each.TFD)+ "\n" )
outfile4.write(str(each.TVD)+ "\n" )
outfile5.write(str(each.FB)+ "\n" )
outfile6.write(str(each.FC)+ "\n" )
file.close() #Close file
outfile.close()
outfile2.close()
outfile3.close()
outfile4.close()
outfile5.close()
outfile6.close()

Like #zwer said, the reason why you have nothing in your output file is because you don't increment rownum while you are iterating the rows from your input file. By indenting the line rownum += 1 you put it inside your loop where you read each row. So with minimal modification it would look
import csv
file = open("P01_All.csv", "r") #Open CSV File in Read Mode
reader = csv.reader(file) #Create reader object which iterates over lines
outfile = open("Name.csv","w")
outfile2 = open("TFF.csv","w")
outfile3 = open("TFD.csv","w")
outfile4 = open("TVD.csv","w")
outfile5 = open("FB.csv","w")
outfile6 = open("FC.csv","w")
class Movement_value: #Object to store unique data
def __init__(self, Name, TFF, TFD, TVD, FB, FC):
self.Name = Name
self.TFF = TFF
self.TFD = TFD
self.TVD = TVD
self.FB = FB
self.FC = FC
rownum = 0 #Row Number currently iterating over
notebook = [] #List to store objects
def checkList(Name, TFF, TFD, TVD, FB, FC):
for value in notebook: #Iterate through list
if value.Name == Name:
value.TFF += float(TFF)
value.TFD += float(TFD)
value.TVD += float(TVD)
value.FB += float(FB)
value.FC += float(FC)
return
newObject = Movement_value(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC)) #Create a new object with new eye and TFF
notebook.append(newObject) #Add to list and break out
for row in reader: #Iterate through all the rows
if rownum == 0: #Store header row seperately to not get confused
header = row
else:
Name = row[0]
TFF = row[1]
TFD = row[2]
TVD = row[3]
FB = row[4]
FC = row[5]
if len(notebook) == 0: #Default case if list = 0
newObject = Movement_value(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC))
notebook.append(newObject)
else: #If not...
checkList(Name, TFF, TFD, TVD, FB, FC)
rownum += 1
for each in notebook: #Print out result
# print(each.Name, each.TFF, each.TFD, each.TVD, each.FB, each.FC)
outfile.write(each.Name + "\n" )
outfile2.write(str(each.TFF)+ "\n" )
outfile3.write(str(each.TFD)+ "\n" )
outfile4.write(str(each.TVD)+ "\n" )
outfile5.write(str(each.FB)+ "\n" )
outfile6.write(str(each.FC)+ "\n" )
file.close() #Close file
outfile.close()
outfile2.close()
outfile3.close()
outfile4.close()
outfile5.close()
outfile6.close()
I have made some additional change: It's better that you don't use list or object as variable names because they are already used in Python and by doing so you'll override their meaning. You could have a bad surprise eventually.
But we can do more.
We don't need to create a class to hold the values
We can work with files using context managers to make sure that our file is not kept open for not relevant reasons.
Here's a version that is shorter than yours:
import csv
import pathlib
input_filepath = pathlib.Path("Input.csv")
output_filepath = pathlib.Path("")
with open(input_filepath, newline="") as input_file:
# Where our data will be kept
input_data = {}
csv_reader = csv.reader(input_file)
# Skip the first line
next(csv_reader)
for (Name, *rest_of_data) in csv_reader:
if Name in input_data:
for (index_of_data_to_update, data_to_update) in enumerate(rest_of_data):
input_data[Name][index_of_data_to_update] += float(data_to_update)
else:
input_data[Name] = [float(x) for x in rest_of_data]
output_rows = ([name] + list(data) for (name, data) in input_data.items())
output_filenames = [
"Name.csv",
"TFF.csv",
"TFD.csv",
"TVD.csv",
"FB.csv",
"FC.csv"
]
output_files = [open(output_filepath / filename, "w") for filename in output_filenames]
# Open all the files
with output_files[0], output_files[1], output_files[2], output_files[3], \
output_files[4], output_files[5]:
for row in output_rows:
for (output_file, data) in zip(output_files, row):
output_file.write("{}\n".format(data))

Txt file to excel conversion in python

I'm trying to convert text file to excel sheet in python. The txt file contains data in the below specified formart
Column names: reg no, zip code, loc id, emp id, lastname, first name. Each record has one or more error numbers. Each record have their column names listed above the values. I would like to create an excel sheet containing reg no, firstname, lastname and errors listed in separate rows for each record.
How can I put the records in excel sheet ? Should I be using regular expressions ? And how can I insert error numbers in different rows for that corresponding record?
Expected output:
Here is the link to the input file:
https://github.com/trEaSRE124/Text_Excel_python/blob/master/new.txt
Any code snippets or suggestions are kindly appreciated.

Here is a draft code. Let me know if any changes needed:
# import pandas as pd
from collections import OrderedDict
from datetime import date
import csv
with open('in.txt') as f:
with open('out.csv', 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
#Remove inital clutter
while("INPUT DATA" not in f.readline()):
continue
header = ["REG NO", "ZIP CODE", "LOC ID", "EMP ID", "LASTNAME", "FIRSTNAME", "ERROR"]; data = list(); errors = list()
spamwriter.writerow(header)
print header
while(True):
line = f.readline()
errors = list()
if("END" in line):
exit()
try:
int(line.split()[0])
data = line.strip().split()
f.readline() # get rid of \n
line = f.readline()
while("ERROR" in line):
errors.append(line.strip())
line = f.readline()
spamwriter.writerow(data + errors)
spamwriter.flush()
except:
continue
# while(True):
# line = f.readline()
Use python-2 to run. The errors are appended as subsequent columns. It's slightly complicated the way you want it. I can fix it if still needed
Output looks like:

You can do this using the openpyxl library which is capable of depositing items directly into a spreadsheet. This code shows how to do that for your particular situation.
NEW_PERSON, ERROR_LINE = 1,2
def Line_items():
with open('katherine.txt') as katherine:
for line in katherine:
line = line.strip()
if not line:
continue
items = line.split()
if items[0].isnumeric():
yield NEW_PERSON, items
elif items[:2] == ['ERROR', 'NUM']:
yield ERROR_LINE, line
else:
continue
from openpyxl import Workbook
wb = Workbook()
ws = wb.active
ws['A2'] = 'REG NO'
ws['B2'] = 'LASTNAME'
ws['C2'] = 'FIRSTNAME'
ws['D2'] = 'ERROR'
row = 2
for kind, data in Line_items():
if kind == NEW_PERSON:
row += 2
ws['A{:d}'.format(row)] = int(data[0])
ws['B{:d}'.format(row)] = data[-2]
ws['C{:d}'.format(row)] = data[-1]
first = True
else:
if first:
first = False
else:
row += 1
ws['D{:d}'.format(row)] = data
wb.save(filename='katherine.xlsx')
This is a screen snapshot of the result.

How to split code into smaller functions

I have an application that works. But in the interest of attempting to understand functions and python better. I am trying to split it out into various functions.
I"m stuck on the file_IO function. I'm sure the reason it does not work is because the main part of the application does not understand reader or writer. To better explain. Here is a full copy of the application.
Also I'm curious about using csv.DictReader and csv.DictWriter. Do either provide any advantages/disadvantages to the current code?
I suppose another way of doing this is via classes which honestly I would like to know how to do it that way as well.
#!/usr/bin/python
""" Description This script will take a csv file and parse it looking for specific criteria.
A new file is then created based of the original file name containing only the desired parsed criteria.
"""
import csv
import re
import sys
searched = ['aircheck', 'linkrunner at', 'onetouch at']
def find_group(row):
"""Return the group index of a row
0 if the row contains searched[0]
1 if the row contains searched[1]
etc
-1 if not found
"""
for col in row:
col = col.lower()
for j, s in enumerate(searched):
if s in col:
return j
return -1
#Prompt for File Name
def file_IO():
print "Please Enter a File Name, (Without .csv extension): ",
base_Name = raw_input()
print "You entered: ",base_Name
in_Name = base_Name + ".csv"
out_Name = base_Name + ".parsed.csv"
print "Input File: ", in_Name
print "OutPut Files: ", out_Name
#Opens Input file for read and output file to write.
in_File = open(in_Name, "rU")
reader = csv.reader(in_File)
out_File = open(out_Name, "wb")
writer = csv.writer(out_File, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
return (reader, writer)
file_IO()
# Read header
header = reader.next()
stored = []
writer.writerow([header[0], header[3]])
for i, row in enumerate(reader):
g = find_group(row)
if g >= 0:
stored.append((g, i, row))
stored.sort()
for g, i, row in stored:
writer.writerow([row[0], row[3]])
# Closing Input and Output files.
in_File.close()
out_File.close()

If I were you, I'd only separate find_group.
import csv
def find_group(row):
GROUPS = ['aircheck', 'linkrunner at', 'onetouch at']
for idx, group in enumerate(GROUPS):
if group in map(str.lower, row):
return idx
return -1
def get_filenames():
# this might be the only other thing you'd want to factor
# into a function, and frankly I don't really like getting
# user input this way anyway....
basename = raw_input("Enter a base filename (no extension): ")
infilename = basename + ".csv"
outfilename = basename + ".parsed.csv"
return infilename, outfilename
# notice that I don't open the files yet -- let main handle that
infilename, outfilename = get_filenames()
with open(infilename, 'rU') as inf, open(outfilename, 'wb') as outf:
reader = csv.reader(inf)
writer = csv.writer(outf, delimiter=',',
quotechar='"', quoting=csv.QUOTE_ALL)
header = next(reader)
writer.writerow([[header[0], header[3]])
stored = sorted([(find_group(row),idx,row) for idx,row in
enumerate(reader)) if find_group(row) >= 0])
for _, _, row in stored:
writer.writerow([row[0], row[3]])