My python program loops through a bunch of csv-files, read them, and write specific columns in the file to another csv file. While the program runs, i can see the files being written in the correct manner, but once the program is finished, all the files i've just written become empty.
The solution to all the other similar threads seems to be closing the file you write to properly, but i cant seem to figure out what im doing wrong. Anyone?
import os
import csv
def ensure_dir(f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
readpath = os.path.join("d:\\", "project")
savepath=os.path.join("d:\\", "save")
ensure_dir(savepath)
contents_1=os.listdir(readpath)
for i in contents_1[1:len(contents_1)]:
readpath_2=os.path.join(readpath, i)
if os.path.isdir(readpath_2)== True :
contents_2=os.listdir(readpath_2)
for i in contents_2:
readpath_3=os.path.join(readpath_2, i)
if os.path.isfile(readpath_3)== True :
savefile=savepath + "\\" + i
savefile = open(savefile, 'wb')
writer = csv.writer(savefile, delimiter=';')
readfile=open(readpath_3, 'rb')
reader = csv.reader(readfile, delimiter=';')
try:
for row in reader:
writer.writerow([row[0], row[3]])
except:
print(i)
finally:
savefile.close()
readfile.close()
savefile=savepath + "\\" + i is the error. If both "d:\\project\a\x.csv" and "d:\\project\b\x.csv" exist, then you will write to savepath + "\\" + i more than once. If the second path as an empty "x.csv", then it would overwrite the result with an empty file.
Try this instead:
import os
import csv
def ensure_dir(f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
readpath = os.path.join("d:\\", "project")
savepath = os.path.join("d:\\", "save")
ensure_dir(savepath)
for dname in os.listdir(readpath)[1:]:
readpath_2 = os.path.join(dname, fname)
if not os.path.isdir(readpath_2):
continue
for fname in os.listdir(readpath_2)
fullfname = os.path.join(readpath_2, fname)
if not os.path.isfile(fullfname):
continue
savefile = open(savepath + "\\" + dname + "_" + fname, wb)
writer = csv.writer(savefile, delimiter=';')
readfile=open(fullfname, 'rb')
reader = csv.reader(readfile, delimiter=';')
try:
for row in reader:
writer.writerow([row[0], row[3]])
except:
print(i)
finally:
savefile.close()
readfile.close()
This code could be greatly improved with os.walk
Quoting from the python documentation:
If csvfile is a file object, it must be opened with the ‘b’ flag on platforms where that makes a difference.
Change the 'w' and 'r' flags to 'wb' and 'rb'.
(1) Your outer loop AND your inner loop both use i as the loop variable. This has no hope of (a) being understood by a human (b) working properly.
(2) except: print(i) ... What??? I'd suggest you remove the try/except and fix any bugs that you come across.
Related
I wrote a program that makes a list
But when the content of the imported files is high
The program is off(crash)
This is the code I wrote
In this photo, a file with low content is used
enter image description here
This photo uses a file with a lot of content
enter image description here
file zip
def btn_start():
try:
if open_file1_text and open_file2_text :
file_1 = open(open_file1_text.get(), 'r')
file_2 = open(open_file2_text.get(), 'r')
lines_1 = file_1.readlines()
lines_2 = file_2.readlines()
global text
text = ("")
demo_text_box.delete(1.0,'end')
demo_text_box_a.delete(1.0,'end')
demo_text_box_a.insert(INSERT,start_text_demo )
for pline in lines_2:
for uline in lines_1:
demo_text_box.insert(INSERT,uline.rstrip('\n') + separator_text_.get() + pline)
text += (uline.rstrip('\n') + separator_text_.get() + pline)
file_1.close()
file_2.close()
except FileNotFoundError :
demo_text_box.delete(1.0,'end')
demo_text_box_a.delete(1.0,'end')
demo_text_box_a.insert(INSERT,File_Not_Found_Error )
Your code has another problem: if file_2 is not found, then file_1 will be left open, which may be bad (you want to close files as soon as you no longer need them).
You can solve this with with statements, which automatically close your files even when an exception occurs.
As for your memory problem, I guess text does not fit in memory, so you may want writing its content to another file.
def btn_start(open_file1_text, open_file2_text):
if not (open_file1_text and open_file2_text):
return
try:
with open(open_file1_text.get(), 'r') as file_1:
lines_1 = file_1.readlines()
with open(open_file1_text.get(), 'r') as file_2:
lines_2 = file_2.readlines()
demo_text_box.delete(1.0, 'end')
demo_text_box_a.delete(1.0, 'end')
demo_text_box_a.insert(INSERT, start_text_demo)
with open('text.txt', 'w') as text_file:
for pline in lines_2:
for uline in lines_1:
demo_text_box.insert(INSERT,uline.rstrip('\n') + separator_text_.get() + pline)
text_file.write(uline.rstrip('\n') + separator_text_.get() + pline)
except FileNotFoundError :
demo_text_box.delete(1.0,'end')
demo_text_box_a.delete(1.0,'end')
demo_text_box_a.insert(INSERT,File_Not_Found_Error )
If the files themselves do not fit in memory (meaning you cannot call file.readlines()),
you can also read them at each iteration of the loop:
with open('text.txt', 'w') as text_file:
with open(open_file1_text.get(), 'r') as file_2:
for pline in file_2:
with open(open_file1_text.get(), 'r') as file_1:
for uline in file_1:
demo_text_box.insert(INSERT,uline.rstrip('\n') + separator_text_.get() + pline)
text_file.write(uline.rstrip('\n') + separator_text_.get() + pline)
Hello i want to write my list to a .csv file.
This is my code
def writeCsv(self, content):
filename = 'data.csv'
f = open(filename, 'w')
header = 'index;title;img;link;views;brand;\n'
f.write(header)
#print(len(content))
i = 0
for c in content:
f.write(c['index'] + ";" + c['title'] + ';' + c['img'] + ';' + c['link'] + ';' + c['views'] + ";\n")
#i += 1
#print(i)
f.close()
My problem is that len(content) returns 72 but the loop only runs 21 times. (I print i every time in the loop and my .csv file only has 21 lines.)
Is there some limit or unknown parameter i miss on the write() function?
Update: I used Sayse´s solution but added encoding='utf-8'. The probem was an illegal charater in line 22
As noted in the comments, the only thing that could cause this is malformed data (probably line 22) and you catching the broader exception.
Regardless, you should just use the csv modules DictWriter
from csv import DictWriter
def writeCsv(self, content):
filename = 'data.csv'
with open(filename, 'w') as f:
field_names = ["index","title","img","link","views","brand"]
dict_writer = DictWriter(f, field_names, delimiter=";")
dict_writer.writeheader()
dict_writer.writerows(content)
Try this perhaps:
def writeCsv(self, content):
filename = 'data.csv'
f = open(filename, 'w')
header = 'index;title;img;link;views;brand'
f.write(header)
#print(len(content))
i = 0
for c in content:
try:
f.write(";\n"+";".join([c[k] for k in header.split(";")]))
except KeyError:
print(c)
i += 1
print(i)
f.write(";")
f.close()
Using the header as your indexes is cleaner imo and wrapping your explicit key access in error handling could help you get through some snags. Also based on how you are writing you output file you will have an empty line at the end of your file, presuming that you have amalgamated your data from some number of similar files you likely have empty elements in your list.
I am just following a simple Python script to write to a text file. The suggetsed method; adding "\n" to the end didn't work. It is printing within a loopAs I am using Windows, I also tried "\r\n." Still it only prints the last item. I have tried moving everything inside and outside the loop (starting with path and ending with file.close() but no go. What's going on here?
#Assign variables to the shapefiles
park = "Parks_sd.shp"
school = "Schools_sd.shp"
sewer = "Sewer_Main_sd.shp"
#Create a list of shapefile variables
shapeList = [park, school, sewer]
path = r"C:/EsriTraining/PythEveryone/CreatingScripts/SanDiegoUpd.txt"
open(path, 'w')
for shp in shapeList:
shp = shp.replace("sd", "SD")
print shp
file = open(path, 'w')
file.write(shp + "\r\n")
file.close()
Open the file outside the loop
Ex:
with open(path, "w") as infile:
for shp in shapeList:
shp = shp.replace("sd", "SD")
infile.write(shp + "\n")
You can 1) open file outside of the for loop and 2) use writelines
with open(path, 'w+') as f:
f.writelines([shp.replace("sd", "SD")+'\n' for shp in shaplist])
or
with open(path, 'w+') as f:
f.writelines(map(lambda s: s.replace("sd", "SD")+'\n', shaplist))
In this way, you open the file once and once the lines are written, the file is automatically closed (because of the [with]).
I have an application that works. But in the interest of attempting to understand functions and python better. I am trying to split it out into various functions.
I"m stuck on the file_IO function. I'm sure the reason it does not work is because the main part of the application does not understand reader or writer. To better explain. Here is a full copy of the application.
Also I'm curious about using csv.DictReader and csv.DictWriter. Do either provide any advantages/disadvantages to the current code?
I suppose another way of doing this is via classes which honestly I would like to know how to do it that way as well.
#!/usr/bin/python
""" Description This script will take a csv file and parse it looking for specific criteria.
A new file is then created based of the original file name containing only the desired parsed criteria.
"""
import csv
import re
import sys
searched = ['aircheck', 'linkrunner at', 'onetouch at']
def find_group(row):
"""Return the group index of a row
0 if the row contains searched[0]
1 if the row contains searched[1]
etc
-1 if not found
"""
for col in row:
col = col.lower()
for j, s in enumerate(searched):
if s in col:
return j
return -1
#Prompt for File Name
def file_IO():
print "Please Enter a File Name, (Without .csv extension): ",
base_Name = raw_input()
print "You entered: ",base_Name
in_Name = base_Name + ".csv"
out_Name = base_Name + ".parsed.csv"
print "Input File: ", in_Name
print "OutPut Files: ", out_Name
#Opens Input file for read and output file to write.
in_File = open(in_Name, "rU")
reader = csv.reader(in_File)
out_File = open(out_Name, "wb")
writer = csv.writer(out_File, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
return (reader, writer)
file_IO()
# Read header
header = reader.next()
stored = []
writer.writerow([header[0], header[3]])
for i, row in enumerate(reader):
g = find_group(row)
if g >= 0:
stored.append((g, i, row))
stored.sort()
for g, i, row in stored:
writer.writerow([row[0], row[3]])
# Closing Input and Output files.
in_File.close()
out_File.close()
If I were you, I'd only separate find_group.
import csv
def find_group(row):
GROUPS = ['aircheck', 'linkrunner at', 'onetouch at']
for idx, group in enumerate(GROUPS):
if group in map(str.lower, row):
return idx
return -1
def get_filenames():
# this might be the only other thing you'd want to factor
# into a function, and frankly I don't really like getting
# user input this way anyway....
basename = raw_input("Enter a base filename (no extension): ")
infilename = basename + ".csv"
outfilename = basename + ".parsed.csv"
return infilename, outfilename
# notice that I don't open the files yet -- let main handle that
infilename, outfilename = get_filenames()
with open(infilename, 'rU') as inf, open(outfilename, 'wb') as outf:
reader = csv.reader(inf)
writer = csv.writer(outf, delimiter=',',
quotechar='"', quoting=csv.QUOTE_ALL)
header = next(reader)
writer.writerow([[header[0], header[3]])
stored = sorted([(find_group(row),idx,row) for idx,row in
enumerate(reader)) if find_group(row) >= 0])
for _, _, row in stored:
writer.writerow([row[0], row[3]])
I'm a newbie in python scripting and would like to know how to fix the code below. I would like to know how to solve the Invalid Syntax of Else statement, thank you!
import os
import zipfile
f = open("filelist.txt","w+")
path=("pathtofile")
directory=os.fsencode(path)
filenames = []
for file in os.listdir(directory):
filename = os.fsdecode(file)
if filename.endswith(".zip"):
with zipfile.ZipFile(filename, 'r') as zipObj:
contents = zipObj.namelist()
listfile = (str(contents))
remchar = "'[]"
for char in remchar:
a = listfile.replace(char,"")
f.write (a)
continue
else:
pass
f.close()
Remember that indentation in python is significant. Your else: statement has no matching if statement on the same level. Make sure to indent properly to achieve what you're looking for:
if filename.endswith(".zip"):
with zipfile.ZipFile(filename, 'r') as zipObj:
contents = zipObj.namelist()
listfile = (str(contents))
remchar = "'[]"
for char in remchar:
a = listfile.replace(char,"")
f.write (a)
continue
else:
pass
The else statement is outside the end of the if one. You have to include an else right after you ended the other one. It's like this:
if filename.endswith(".zip"):
with zipfile.ZipFile(filename, 'r') as zipObj:
contents = zipObj.namelist()
else:
pass
If you're just going to write pass, then you can directly ommit the else.