I have written this code
import os
import csv
import time
class upload_CSV:
def __init__(self):
coup = []
self.coup = coup
self.csv_name = 'Coup.csv'
def loader(self,rw):
with open(self.csv_name,'rb') as csv_file:
reader = csv.reader(csv_file,delimiter=',')
for row in reader:
self.coup.append(row[rw])
self.coup = self.coup[1:]
csv_file.flush()
csv_file.close()
return self.coup
def update(self,rw,message):
#try:
with open(self.csv_name,'rb') as csv_file1:
reader = csv.reader(csv_file1,delimiter=',')
csv_file1.flush()#To clean the register for reuse
csv_file1.close()
#except Exception as ex:
#error = 'An error occured loading data in the reader'
# #raise
# return ex
os.remove(self.csv_name)
writer = csv.writer(open(self.csv_name,'wb'))
for row in reader:
if row[rw]==message:
print str(row),' has been removed'
else:
writer.writerow(row)
return message
I am trying to read the content of a csv to a list first. Once i get the relevant data, i need to go back to my csv and create a new entry without that record. I keep getting the single error
Line 27 in update
with open(csv_name,'rb')as csvfile1:
Python: IOError: [Errno 2] No such file or directory 'Coup.csv'
when i call the Update function
I have looked at this question Python: IOError: [Errno 2] No such file or directory but nothing seems to work. Its as if the first function has a lock on the file. Any help would be appreciated
It would be enormously helpful if we saw the traceback to know exactly what line is producing the error...but here is a start...
First, you have two spots in your code where you are working with a filename that expects to only be available in the current directory. That is one possible point of failure in your code if you run it outside the directory containing the file:
self.csv_name = 'Coup.csv'
...
with open(self.csv_name,'rb') as csv_file:
...
with open('Coup.csv~','rb') as csv_file1:
...
And then, you are also referring to a variable that won't exist:
def update(self,rw,message):
...
# self.csv_name? or csv_file1?
os.remove(csv_name)
writer = csv.writer(open(csv_name,'wb'))
Also, how can you be sure this temp file will exist? Is it guaranteed? I normally wouldn't recommend relying on a system-temporary file.
with open('Coup.csv~','rb') as csv_file1:
Related
I need to test my methods using the Pytest library. I have written such tests.
def test_read_data_from_file(): # check for missing file.(1)
try:
read_data_from_file('example.csv')
except FileNotFoundError:
pytest.fail('File not found.')
def test_roots(): # check for lack of access to the file.(2)
try:
read_data_from_file('err.csv')
except Permission Error:
py test.fail('The file is not readable.')
The method itself
def read_data_from_file(filename):
# we will return it.
rows_csv = []
with open(filename) as csvfile:
read_csv = csv.reader(csvfile)
# for convenience, when performing split, we will separate the data in lines with a space.
for row in read_csv:
rows_csv.append(' '.join(row))
rows_csv.pop(0)
return rows_csv
How can I change the tests so that "ExceptionInfo" is used in their implementation?(with pytest.raises(FileNotFoundError) as exception_info:)
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 2 years ago.
Improve this question
After the user provides the source directory, the following script reads in a list of csvs. It then takes one csv and copies its contents row by row to a new csv until it reaches 100,000 rows at which point a new csv is created to continue the process until the original csv has been copied completely. The process is then repeated for the next csv file in the directory.
I will sometimes encounter the above PermissionError and am not sure how to go about fixing it, but sometimes I will run the script and I encounter no issues. I've verified that both the input and output files are NOT open on my machine. I've also tried to change the properties of my directory folder to not be read-only, though this always reverts back. When the error does occur, it is always within a few seconds of first starting to process a csv. Once you are about 5 seconds in, it won't give the error for that csv. But it could later once it gets to a new input csv.
"""
Script processes all csv's in a provided directory and returns
csv's with a maximum of 100,000 rows
"""
import csv
import pathlib
import argparse
import os
import glob
def _get_csv_list(
*, description: str = "Process csv file directory.",
):
"""
Uses argument parser to set up working directory, then
extracts list of csv file names from directory
Args: Directory string
Returns list of csv file name strings
"""
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"SRC", type=pathlib.Path, help="source (input) directory"
)
parsed_arg = parser.parse_args()
os.chdir(parsed_arg.SRC)
return glob.glob("*.{}".format("csv"))
def _process_csv(file_name):
"""
Iterates through csv file and copies each row to output
file. Once 100,000 rows is reached, a new file is started
Args: file name string
"""
file_index = 0
max_records_per_file = 100_000
with open(file_name) as _file:
reader = csv.reader(_file)
first_line = _file.readline()
first_line_list = first_line.split(",")
for index, row in enumerate(reader):
if index % max_records_per_file == 0:
file_index += 1
with open(
f"output_{file_name.strip('.csv')}_{file_index}.csv",
mode="xt",
encoding="utf-8",
newline="\n",
) as buffer:
writer = csv.writer(buffer)
writer.writerow(first_line_list)
else:
try:
with open(
f"output_{file_name.strip('.csv')}_{file_index}.csv",
mode="at",
encoding="utf-8",
newline="\n",
) as buffer:
writer = csv.writer(buffer)
writer.writerow(row)
except FileNotFoundError as error:
print(error)
with open(
f"output_{file_name.strip('.csv')}_{file_index}.csv",
mode="xt",
encoding="utf-8",
newline="\n",
) as buffer:
writer = csv.writer(buffer)
writer.writerow(first_line_list)
writer.writerow(row)
def main():
"""
Primary function for limiting csv file size
Cmd Line: python csv_row_limiter.py . (Replace '.' with other path
if csv_row_limiter.py directory and csv directory are different)
"""
csv_list = _get_csv_list()
for file_name in csv_list:
_process_csv(file_name)
if __name__ == "__main__":
main()
Also, please note that the only requirement for the contents of the input csv's is that they have a large number of rows (100,000+) with some amount of data.
Any ideas of how I might resolve this issue?
try opening it as root i.e try running this python script through root or su privileges. What i mean is login as root and then run this python script . Hope this helps.
I have written script that parses a web page and saves data of interest in a CSV file. Before I open the data and use it in a second script I check if the file with data exist and if not I am running the parser script first. The odd behaviour of the second script is, that it is able to detect that there is no file, then the file is created, but when it is read for the first time it is empty (part of else statement). I tried to provide some delay by using the time.sleep() method, but it does not work. The explorer clearly shows that the file is not empty, but at the first run, script recognizes the file as empty. At the subsequent runs the scripts clearly sees the file and is able to properly recognize it content.
Maybe You have some explanation for this behaviour.
def open_file():
# TARGET_DIR and URL are global variables.
all_lines = []
try:
current_file = codecs.open(TARGET_DIR, 'r', 'utf-8')
except FileNotFoundError:
procesed_data = parse_site(URL)
save_parsed(procesed_data)
compare_parsed()
open_file()
else:
time.sleep(10)
data = csv.reader(current_file, delimiter=';')
for row in data:
all_lines.append(row)
current_file.close()
return all_lines
You got some recursion going on.
Another way to do it—assuming I understand correctly—is this:
import os
def open_file():
# TARGET_DIR and URL are global variables.
all_lines = []
# If the file is not there, make it.
if not os.path.isfile(TARGET_DIR):
procesed_data = parse_site(URL)
save_parsed(procesed_data)
compare_parsed()
# Here I am assuming the file has been created.
current_file = codecs.open(TARGET_DIR, 'r', 'utf-8')
data = csv.reader(current_file, delimiter=';')
for row in data:
all_lines.append(row)
current_file.close()
return all_lines
you should return the result of your internal open_file call, or just opening the file in your except block:
def open_file():
# TARGET_DIR and URL are hopefully constants
try:
current_file = codecs.open(TARGET_DIR, 'r', 'utf-8')
except FileNotFoundError:
procesed_data = parse_site(URL)
save_parsed(procesed_data)
compare_parsed()
current_file = codecs.open(TARGET_DIR, 'r', 'utf-8')
data = csv.reader(current_file, delimiter=';')
all_lines = list(data)
current_file.close()
return all_lines
Using Python 3 on a windows machine:
I have a function to take a list of lists and open it as a csv file using my default application (excel). Despite closing the file after writing, I get a 'locked for editing' message when excel starts.
def opencsv(data):
"""saves a list of lists as a csv and opens"""
import tempfile
import os
import csv
handle, fn = tempfile.mkstemp(suffix='.csv')
with open(fn,"w", encoding='utf8',errors='surrogateescape',\
newline='') as f:
writer=csv.writer(f)
for row in data:
try:
writer.writerow(row)
except Exception as e:
print ('Error in writing row:',e)
f.close()
url = 'file://' + fn.replace(os.path.sep, '/')
os.startfile(fn)
opencsv([['d1','d2'],['d3','d4','d5']])
How can I fix this?
Answer from swstephe's input:
The issue is that mkstemp opens the file and associates it with an os handle. In my original code I was not closing this file properly. See below for updated code.
def opencsv(data):
"""saves a list of lists as a csv and opens"""
import tempfile
import os
import csv
handle, fn = tempfile.mkstemp(suffix='.csv')
with os.fdopen(handle,"w", encoding='utf8',errors='surrogateescape',\
newline='') as f:
writer=csv.writer(f)
for row in data:
try:
writer.writerow(row)
except Exception as e:
print ('Error in writing row:',e)
print (fn)
os.startfile(fn)
My program uses a csv file to load up some initialization info.
I have a config file that loads data from this csv file. I am building a web app as part of this program, and the config file is accessed from various points in the entire application.
This program must operate cross-platform.
Problem: Depending on who calls the config file and where the caller lives in the file tree, the csv operation is throwing IOError errors. The csv data isn't even accessed, but on import of the config file, the csv read portion executes anyway.
The code below is rife with Band-Aids...
# print os.getcwd()
try:
with open('value_addresses.csv') as file: # located in code folder. used extensively below
reader = csv.reader(file)
lines = [l for l in reader]
except IOError:
try:
with open('_code/value_addresses.csv') as file: #
reader = csv.reader(file)
lines = [l for l in reader]
except IOError:
with open('../_code/value_addresses.csv') as file: #
reader = csv.reader(file)
lines = [l for l in reader]
I would refactor the common code into a function.
def read_file(path):
with open(path) as file:
reader = csv.reader(file)
lines = [l for l in reader]
return lines
try:
read_file("value_addresses.csv")
except IOError:
try:
read_file('_code/value_addresses.csv')
except IOError:
read_file('../_code/value_addresses.csv')
You can further simplify this by recursively figuring out the path to value_addresses.csv.
This is how I would do it:
from os.path import join
import csv
def myopen( filename, possible_dirs, mode ):
for dir in possible_dirs:
try:
return open(join(dir,filename),mode)
except IOError:
pass
raise IOError('File not found.')
with myopen('value_addresses.csv',['.','_code','../_code'],'r') as file:
reader = csv.reader(file)
lines = [l for l in reader]
Although maybe you want to look more into the specific IOError you are getting, but either way this is the general approach I would take.