Develop Reference

python code for replacing character and remove header

I am trying code to remove the header from multiple CSV files and add | delimiter by replacing, here is my code but it's getting out-
import time, os
from datetime import datetime
def remove_header_replace_delimiter():
src_folder = 'path'
src_files = os.listdir(src_folder)
print(src_files)
for file_name in src_files:
with open('path' + file_name, 'r') as inp, open('path' + file_name, 'w') as out:
next(inp)
for line in inp:
line = line.replace(',', '|')
print(line)
out.write(line)
Myfile content -
Date,Runner Name,Automation,Order Number,SON,Account Name,Quote Number,Product Code,Status
01/02/2021 10:43:25,dsadsa,AS Silver,444,3323,aaapp,W-3342,AQS-11-L,Failed
01/02/2021 10:57:52,dsfsdds,AS Silver,34333,3213,defsd,A-1222,fdsfds-L,Success

You are missing a "/" between path and file_name causing it to try and open a file called "pathMyfile.txt" when src_folder="path" and your file is called "Myfile.txt"
Also you might want to use src_folder instead of hardcoding "path" in your file open line.
Lastly, you could try to use f-strings instead of string concatenation with + for clarity and performance.
Example:
import time, os
from datetime import datetime
def remove_header_replace_delimiter():
src_folder = 'path'
src_files = os.listdir(src_folder)
print(src_files)
for file_name in src_files:
with open(f'{src_folder}/{file_name}', 'r') as inp, open(f'{src_folder}/fixed_{file_name}', 'w') as out:
next(inp)
for line in inp:
line = line.replace(',', '|')
print(line)
out.write(line)
remove_header_replace_delimiter()

How to write the output to a file, the name of which is passed as a second parameter?

def generate_daily_totals(input_filename, output_filename):
"""result in the creation of a file blahout.txt containing the two lines"""
with open(input_filename, 'r') as reader, open(output_filename, 'w') as writer: #updated
for line in reader: #updated
pieces = line.split(',')
date = pieces[0]
rainfall = pieces[1:] #each data in a line
total_rainfall = 0
for data in rainfall:
pure_data = data.rstrip()
total_rainfall = total_rainfall + float(pure_data)
writer.write(date + "=" + '{:.2f}'.format(total_rainfall) + '\n') #updated
#print(date, "=", '{:.2f}'.format(total_rainfall)) #two decimal point format,
generate_daily_totals('data60.txt', 'totals60.txt')
checker = open('totals60.txt')
print(checker.read())
checker.close()
By reading a file, the original program runs well but I was required to convert it by writing the file. I am confused as the write method applies to string only so does that mean only the print section can be replaced by write method? This is the first time I am trying to use the write method. Thanks!
EDIT: the above codes have been updated based on the blhsing instruction which helped a lot! But still not running well as the for loop which gets skipped for some reason. Proper suggestions would be appreciated!
expected output:
2006-04-10 = 1399.46
2006-04-11 = 2822.36
2006-04-12 = 2803.81
2006-04-13 = 1622.71
2006-04-14 = 3119.60
2006-04-15 = 2256.14
2006-04-16 = 3120.05
2006-04-20 = 1488.00

You should open both the input file for reading, and the output file for writing, so change:
with open(input_filename, 'w') as writer:
for line in writer: # error not readable
to:
with open(input_filename, 'r') as reader, open(output_filename, 'w') as writer:
for line in reader:
Also, unlike the print function, the write method of a file object does not automatically add a trailing newline character to the output, so you would have to add it on your own.
Change:
writer.write(date + "=" + '{:.2f}'.format(total_rainfall))
to:
writer.write(date + "=" + '{:.2f}'.format(total_rainfall) + '\n')
or you can use print with the outputting file object specified as the file argument:
print(date, "=", '{:.2f}'.format(total_rainfall), file=writer)

Reading comma separated values from text file in python

I have a text file consisting of 100 records like
fname,lname,subj1,marks1,subj2,marks2,subj3,marks3.
I need to extract and print lname and marks1+marks2+marks3 in python. How do I do that?
I am a beginner in python.
Please help
When I used split, i got an error saying
TypeError: Can't convert 'type' object to str implicitly.
The code was
import sys
file_name = sys.argv[1]
file = open(file_name, 'r')
for line in file:
fname = str.split(str=",", num=line.count(str))
print fname

If you want to do it that way, you were close. Is this what you were trying?
file = open(file_name, 'r')
for line in file.readlines():
fname = line.rstrip().split(',') #using rstrip to remove the \n
print fname

Note: its not a tested code. but it tries to solve your problem. Please give it a try
import csv
with open(file_name, 'rb') as csvfile:
marksReader = csv.reader(csvfile)
for row in marksReader:
if len(row) < 8: # 8 is the number of columns in your file.
# row has some missing columns or empty
continue
# Unpack columns of row; you can also do like fname = row[0] and lname = row[1] and so on ...
(fname,lname,subj1,marks1,subj2,marks2,subj3,marks3) = *row
# you can use float in place of int if marks contains decimals
totalMarks = int(marks1) + int(marks2) + int(marks3)
print '%s %s scored: %s'%(fname, lname, totalMarks)
print 'End.'

"""
sample file content
poohpool#signet.com; meixin_kok#hotmail.com; ngai_nicole#hotmail.com; isabelle_gal#hotmail.com; michelle-878#hotmail.com;
valerietan98#gmail.com; remuskan#hotmail.com; genevieve.goh#hotmail.com; poonzheng5798#yahoo.com; burgergirl96#hotmail.com;
insyirah_powergals#hotmail.com; little_princess-angel#hotmail.com; ifah_duff#hotmail.com; tweety_butt#hotmail.com;
choco_ela#hotmail.com; princessdyanah#hotmail.com;
"""
import pandas as pd
file = open('emaildump.txt', 'r')
for line in file.readlines():
fname = line.split(';') #using split to form a list
#print(fname)
df1 = pd.DataFrame(fname,columns=['Email'])
print(df1)

Using Argparse to create file converter in Python

I have to use the command prompt and python to recieve an input in the form of a csv file, then read it and convert it into a xml file with the same name as the csv file except with .xml file extension or the user can set the ouput file name and path using the -o --output optional command line argument. Well i have searched on google for days, and so far my program allows me to input command line arguments and i can convert the csv to an xml file but it doesn't print it using the same name as the csv file or when the user sets the name. Instead it just prints out a blank file. Here is my code:
import sys, argparse
import csv
import indent
from xml.etree.ElementTree import ElementTree, Element, SubElement, Comment, tostring
parser=argparse.ArgumentParser(description='Convert wordlist text files to various formats.', prog='Text Converter')
parser.add_argument('-v','--verbose',action='store_true',dest='verbose',help='Increases messages being printed to stdout')
parser.add_argument('-c','--csv',action='store_true',dest='readcsv',help='Reads CSV file and converts to XML file with same name')
parser.add_argument('-x','--xml',action='store_true',dest='toxml',help='Convert CSV to XML with different name')
parser.add_argument('-i','--inputfile',type=argparse.FileType('r'),dest='inputfile',help='Name of file to be imported',required=True)
parser.add_argument('-o','--outputfile',type=argparse.FileType('w'),dest='outputfile',help='Output file name')
args = parser.parse_args()
def main(argv):
reader = read_csv()
if args.verbose:
print ('Verbose Selected')
if args.toxml:
if args.verbose:
print ('Convert to XML Selected')
generate_xml(reader)
if args.readcsv:
if args.verbose:
print ('Reading CSV file')
read_csv()
if not (args.toxml or args.readcsv):
parser.error('No action requested')
return 1
def read_csv():
with open ('1250_12.csv', 'r') as data:
return list(csv.reader(data))
def generate_xml(reader):
root = Element('Solution')
root.set('version','1.0')
tree = ElementTree(root)
head = SubElement(root, 'DrillHoles')
head.set('total_holes', '238')
description = SubElement(head,'description')
current_group = None
i = 0
for row in reader:
if i > 0:
x1,y1,z1,x2,y2,z2,cost = row
if current_group is None or i != current_group.text:
current_group = SubElement(description, 'hole',{'hole_id':"%s"%i})
collar = SubElement (current_group, 'collar',{'':', '.join((x1,y1,z1))}),
toe = SubElement (current_group, 'toe',{'':', '.join((x2,y2,z2))})
cost = SubElement(current_group, 'cost',{'':cost})
i+=1
indent.indent(root)
tree.write(open('hole.xml','w'))
if (__name__ == "__main__"):
sys.exit(main(sys.argv))
for the generate_xml() function, you can ignore it since it accepts csv files formatted a certain way so you might not understand it but, i think the problem lies in tree.write() since that part generates the xml file with a name that is written in the code itself and not the arguments at the command prompt.

You need to pass a file argument to generate_xml(). You appear to have the output file in args.outputfile.
generate_xml(reader, args.outputfile)
...
def generate_xml(reader, outfile):
...
tree.write(outfile)
You should probably also make use of args.inputfile:
reader = read_csv(args.inputfile)
...
def read_csv(inputfile):
return list(csv.reader(inputfile))
And this line does not do anything useful, it processes the .csv file, but doesn't do anything with the results:
read_csv()

The following code has been adapted from FB36's recipie on code.activestate.com
It will do what you need and you don't have to worry about the headers in the csv file, though there should only be one header (the first row) in the csv file. Have a look at the bottom of this page if you want to do batch conversion.
'''Convert csv to xml file
csv2xml.py takes two arguments:
1. csvFile: name of the csv file (may need to specify path to file)
2. xmlFile: name of the desired xml file (path to destination can be specified)
If only the csv file is provided, its name is used for the xml file.
Command line usage:
example1: python csv2xml.py 'fileName.csv' 'desiredName.xml'
example2: python csv2xml.py '/Documents/fileName.csv' '/NewFolder/desiredName.xml'
example3: python csv2xml.py 'fileName.csv'
This code has been adapted from: http://code.activestate.com/recipes/577423/
'''
import csv
def converter(csvFile, xmlFile):
csvData = csv.reader(open(csvFile))
xmlData = open(xmlFile, 'w')
xmlData.write('<?xml version="1.0"?>' + "\n")
# there must be only one top-level tag
xmlData.write('<csv_data>' + "\n")
rowNum = 0
for row in csvData:
if rowNum == 0:
tags = row
# replace spaces w/ underscores in tag names
for i in range(len(tags)):
tags[i] = tags[i].replace(' ', '_')
else:
xmlData.write('<row>' + "\n")
for i in range(len(tags)):
xmlData.write(' ' + '<' + tags[i] + '>' \
+ row[i] + '</' + tags[i] + '>' + "\n")
xmlData.write('</row>' + "\n")
rowNum +=1
xmlData.write('</csv_data>' + "\n")
xmlData.close()
## for using csv2xml.py from the command line
if __name__ == '__main__':
import sys
if len(sys.argv)==2:
import os
csvFile = sys.argv[1]
xmlFile = os.path.splitext(csvFile)[0] + '.xml'
converter(csvFile,xmlFile)
elif len(sys.argv)==3:
csvFile = sys.argv[1]
xmlFile = sys.argv[2]
converter(csvFile,xmlFile)
else:
print __doc__

get file size and append to new column of CSV file

Python 2.4
For my example I have a 2 column csv file
Eg:
HOST, FILE
server1, /path/to/file1
server2, /path/to/file2
server3, /path/to/file3
I would like to get the file size of the object at PATH for each row in the csv FILE, then add that value to the csv FILE on a new column.
Making it:
HOST, PATH, FILESIZE
server1, /path/to/file1, 6546542
server2, /path/to/file2, 46546343
server3, /path/to/file3, 87523
Ive tried a couple methods but havnt had much success.
The code below executes fileSizeCmd (du -b) on the PATH and outputs the filezie correctly, but I havnt figured out how to use the data to add to the csv FILE
import datetime
import csv
import os, time
from subprocess import Popen, PIPE, STDOUT
now = datetime.datetime.now()
fileSizeCmd = "du -b"
SP = " "
# Try to get disk size and append to another row after entry above
#st = os.stat(row[3])
#except IOError:
#print "failed to get information about", file
#else:
#print "file size:", st[ST_SIZE]
#print "file modified:", time.asctime(time.localtime(st[ST_MTIME]))
incsv = open('my_list.csv', 'rb')
try:
reader = csv.reader(incsv)
outcsv = open('results/results_' + now.strftime("%m-%d-%Y") + '.csv', 'wb')
try:
writer = csv.writer(outcsv)
for row in reader:
p = Popen(fileSizeCmd + SP + row[1], shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, empty = p.communicate()
print 'Command: %s\nOutput: %s\n' % (fileSizeCmd + SP + row[1], stdout)
# Results in bytes example
#
# Output:
# 8589935104 /path/to/file
#
# Write 8589935104 to new column of csv FILE
finally:
outcsv.close()
finally:
incsv.close()

Sketch w/o error handling:
#!/usr/bin/env python
import csv
import os
filename = "sample.csv"
# localhost, 01.html.bak
# localhost, 01.htmlbak
# ...
def filesize(filename):
# no need to shell out for filesize
return os.stat(filename).st_size
with open(filename, 'rb') as handle:
reader = csv.reader(handle)
# result is written to sample.csv.updated.csv
writer = csv.writer(open('%s.updated.csv' % filename, 'w'))
for row in reader:
# need to strip filename, just in case
writer.writerow(row + [ filesize(row[1].strip()) ])
# result
# localhost, 01.html.bak,10021
# localhost, 01.htmlbak,218982
# ...

You can
1) read the cvs content into a list of tuple of (server, filename)
2) collect the file size for each element of this list
3) package the result into another tuple (server, filename, filesize) into another list ('result')
4) write out the result to new file

First, getting file size is a lot easier than using subprocess (see os.stat):
>>> os.stat('/tmp/file').st_size
100
Second, you're on the right track with your writer object writing to a different file, but you just need to add a column to the row lists you're getting back from the reader and then feed them to writerow on the writer (see here). Something like this:
>>> writerfp = open('out.csv', 'w')
>>> writer = csv.writer(writerfp)
>>> for row in csv.reader(open('in.csv', 'r')):
... row.append('column')
... writer.writerow(row)
...
>>> writerfp.close()