Develop Reference

Overwrite a .txt file in python

I have a .txt file that is being written to by a python script.
Adam,3,2,4
Sorin,3,2,4
Sorin,0,0,0
new_record = studentName+","+str(Score1) +","+str(Score2) +","+str(Score3)
student_class = 0
while student_class != 1 or student_class != 2 or student_class != 3:
student_class=input("What class are you in?(1/2/3): ")
if student_class == "1":
file=open("Class1.txt", "a+")
file.write(new_record)
file.write("\n")
file.close()
with open("Class1.txt", 'r') as fp:
for count, line in enumerate(fp):
pass
break
I want the scores to be overwritten if the student name is the same. For example if I run the script again, and Sorin gets a score of "3,3,3" the .txt file would look like this:
Adam,3,2,4
Sorin,3,2,4
Sorin,0,0,0
Sorin 3,3,3
However I want it to turn out like this:
Adam,3,2,4
Sorin 3,3,3

There are some things missing in your code, like how we know which student we are working on, etc.
But either way, this is the approach I would take if the files you are working on are not too big, as the file contents will be stored in memory while editing.
It uses a StringIO as intermediary location where the rows are appended, except if the name matches the current student, and then the content of the StringIO as put in place of the original file.
Starting with:
Adam,3,2,4
Sorin,3,2,4
Sorin,0,0,0
And running the following
import csv
from io import StringIO
current_student = "Sorin"
current_scores = (3, 3, 3)
# obtain a temporary file-like object in memory with a csv writer
with StringIO() as f_tmp:
writer = csv.writer(f_tmp)
# open the input file for reading with a csv reader
with open("/tmp/classes.csv", "r", newline="") as f_in:
reader = csv.reader(f_in)
for row in reader:
# skip the rows of current_student
if row[0] == current_student:
continue
writer.writerow(row)
# add current_student and their scores
writer.writerow((current_student,) + current_scores)
# open the input file for writing
with open("/tmp/classes.csv", "w") as f_out:
f_out.write(f_tmp.getvalue())
You get
Adam,3,2,4
Sorin,3,3,3

error while writing to csv file with python

I am trying to write some output to csv file line by line
Here what I tried:
import csv
today = datetime.datetime.now().date()
filter = "eventTimestamp ge {}".format(today)
select = ",".join([
"eventTimestamp",
"eventName",
"operationName",
"resourceGroupName",
])
activity_logs = client.activity_logs.list(
filter=filter,
select=select
)
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
for log in activity_logs:
result = (" ".join([
str(log.event_timestamp),
str(log.resource_group_name),
log.event_name.localized_value,
log.operation_name.localized_value
]))
f.writerow(result)
Its throwing error:
AttributeError: '_io.TextIOWrapper' object has no attribute 'writerow'
How can i fix this error, possibly any other module ?

This:
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
is creating just text file handle. You need to create csv.writer using f and then you might use writerow, that is:
import csv
...
with open(r"C:\scripts\logs.csv", 'w', newline='') as f:
writer = csv.writer(f)
for log in activity_logs:
result = (str(log.event_timestamp),str(log.resource_group_name),log.event_name.localized_value,log.operation_name.localized_value)
writer.writerow(result)
You might find useful examples of usage in csv article at PyMOTW-3

The error is coming from the line:
f.writerow(result)
and it's telling you that the f object does not have a function named writerow.
As Jannes has commented, use the write function instead:
f.write(result)

CSV.writer is required when your trying to write into CSV . then the code can be
import csv
today = datetime.datetime.now().date()
filter = "eventTimestamp ge {}".format(today)
select = ",".join([
"eventTimestamp",
"eventName",
"operationName",
"resourceGroupName",
])
activity_logs = client.activity_logs.list(
filter=filter,
select=select
)
with open(r"C:\scripts\logs.csv", 'w', newline='') as file:
f=csv.writer(file)
for log in activity_logs:
result = (str(log.event_timestamp),
str(log.resource_group_name),
log.event_name.localized_value,
log.operation_name.localized_value)
f.writerow(result)
When the csv.writer is added after opening the csv file it will work without TextIOwrapper error

Python requests data in file, line by line

I try to get the output of this request (https://api.opendota.com/api/players/7841909) in a file, line by line.
For some reason the output is stored in byte and not str, which I can change by str().
I tried to use a regualar expression to just store the information between the {} and also tried the csv module, which lead to just store digits.
What did I do wrong? The following version ignores the linebreak and the delimiters. :/
import requests
import csv
import re
dotaId = "7841909" #somit als string gespeichert
pfad = "https://api.opendota.com/api/players/" + dotaId + "/matches"
req = requests.get(pfad)
with open('%s.csv' % dotaId, 'w') as file:
clean_line = re.findall(r'\{(.*?)\}', req.text)
file.write(str(clean_line))

Your object clean_line is a list which you are writing as a one liner into the file.
It is better to use the csv writer module and write the content row by row:
with open('new_file.csv', 'w', newline='') as file:
writer = csv.writer(file, quotechar="'")
clean_lines = re.findall(r'\{(.*?)\}', req.text)
for line in clean_lines:
writer.writerow([str(line)])

How can I pickle a python object into a csv file?

I am trying to pickle a python object into a csv file. I want to write the pickle of an object as the third column in my file. I want to use pickle to avoid writing serialization for my complex objects.
Code to write to csv :
with open(self.file_path, 'a') as csv_file:
wr = csv.writer(csv_file, delimiter='|')
row = ['klines', symbol]
row.extend(pickle.dumps(object))
wr.writerow(row)
Code to read csv :
with open(self.simulation_file_name, 'r') as csv_file:
line = csv_file.readline()
while line != '':
line = line.strip('\n')
columns = line.split('|')
event_type = line.pop(0)
symbol = line.pop(0)
pickled = line.pop(0)
klines = pickle.loads(klines)
I get the following error :
TypeError: a bytes-like object is required, not 'str'

To write bytes/binary in text file like CSV, use base64 or other methods to avoid any escaping problem. Code simplified & python3 assumed.
import base64
with open('a.csv', 'a', encoding='utf8') as csv_file:
wr = csv.writer(csv_file, delimiter='|')
pickle_bytes = pickle.dumps(obj) # unsafe to write
b64_bytes = base64.b64encode(pickle_bytes) # safe to write but still bytes
b64_str = b64_bytes.decode('utf8') # safe and in utf8
wr.writerow(['col1', 'col2', b64_str])
# the file contains
# col1|col2|gANdcQAu
with open('a.csv', 'r') as csv_file:
for line in csv_file:
line = line.strip('\n')
b64_str = line.split('|')[2] # take the pickled obj
obj = pickle.loads(base64.b64decode(b64_str)) # retrieve
P.S. If you are not writing a utf8 file (e.g. ascii file), simply replace the encoding method.
P.S. Writing bytes in CSV is possible yet hardly elegant. One alternative is dumping a whole dict with dumped objects as values and storing keys in the CSV.

How to divide csv file with condition?

I have this csv file:
89,Network activity,ip-dst,80.179.42.44,,1,20160929
89,Payload delivery,md5,4ad2924ced722ab65ff978f83a40448e,,1,20160929
89,Network activity,domain,alkamaihd.net,,1,20160929
90,Payload delivery,md5,197c018922237828683783654d3c632a,,1,20160929
90,Network activity,domain,dnsrecordsolver.tk,,1,20160929
90,Network activity,ip-dst,178.33.94.47,,1,20160929
90,Payload delivery,filename,Airline.xls,,1,20160929
91,Payload delivery,md5,23a9bbf8d64ae893db17777bedccdc05,,1,20160929
91,Payload delivery,md5,07e47f06c5ed05a062e674f8d11b01d8,,1,20160929
91,Payload delivery,md5,bd75af219f417413a4e0fae8cd89febd,,1,20160929
91,Payload delivery,md5,9f4023f2aefc8c4c261bfdd4bd911952,,1,20160929
91,Network activity,domain,mailsinfo.net,,1,20160929
91,Payload delivery,md5,1e4653631feebf507faeb9406664792f,,1,20160929
92,Payload delivery,md5,6fa869f17b703a1282b8f386d0d87bd4,,1,20160929
92,Payload delivery,md5,24befa319fd96dea587f82eb945f5d2a,,1,20160929
I need to divide this csv file to 4 csv files where as the condition is the event number at the beginning of every row. so far I created a set that includes al the event numbers {89,90,91,92}, and I know that I need to make loop in a loop and copy each row to its dedicated csv file.

data = {
'89': [],
'90': [],
'91': [],
'92': []
}
with open('yourfile.csv') as infile:
for line in infile:
prefix = line[:2]
data[prefix].append(line)
for prefix in data.keys():
with open('csv' + prefix + '.csv', 'w') as csv:
csv.writelines(''.join(data[prefix]))
However if your are open to solutions other than python then this can be easily accomplished by running four commands
grep ^89 file.csv > 89.csv
grep ^90 file.csv > 90.csv
Similarly for other values.

It would be best to not hardcode the event numbers in your code so it's not dependent on the values of the data. I also prefer to use the csv module which has been optimized to read and write .csv files.
Here's a way to do that:
import csv
prefix = 'events' # of output csv file names
data = {}
with open('conditions.csv', 'rb') as conditions:
reader = csv.reader(conditions)
for row in reader:
data.setdefault(row[0], []).append(row)
for event in sorted(data):
csv_filename = '{}_{}.csv'.format(prefix, event)
print(csv_filename)
with open(csv_filename, 'wb') as csvfile:
writer = csv.writer(csvfile)
writer.writerows(data[event])
Update
The approach implemented above first reads the entire csv file into memory, and then writes the all the rows associated with each event value into a separate output file, one at a time.
A more memory-efficient approach would be to open multiple output files simultaneously and write each row immediately after it has been read out to the proper destination file. Doing this requires keeping track of what files are already open. Something else the file managing code needs to do is make sure all the files are closed when processing is complete.
In the code below all of this has been accomplished by defining and using a Python Context Manager type to centralize the handling of all the csv output files that might be generated depending on how many different event values there are in the input file.
import csv
import sys
PY3 = sys.version_info.major > 2
class MultiCSVOutputFileManager(object):
"""Context manager to open and close multiple csv files and csv writers.
"""
def __enter__(self):
self.files = {}
return self
def __exit__(self, exc_type, exc_value, traceback):
for file, csv_writer in self.files.values():
print('closing file: {}'.format(file.name))
file.close()
self.files.clear()
return None
def get_csv_writer(self, filename):
if filename not in self.files: # new file?
open_kwargs = dict(mode='w', newline='') if PY3 else dict(mode='wb')
print('opening file: {}'.format(filename))
file = open(filename, **open_kwargs)
self.files[filename] = file, csv.writer(file)
return self.files[filename][1] # return associated csv.writer object
And here's how to use it:
prefix = 'events' # to name of each csv output file
with open('conditions.csv', 'rb') as conditions:
reader = csv.reader(conditions)
with MultiCSVOutputFileManager() as file_manager:
for row in reader:
csv_filename = '{}_{}.csv'.format(prefix, row[0]) # row[0] is event
writer = file_manager.get_csv_writer(csv_filename)
writer.writerow(row)

You can even dynamically create the resulting files if the first field has not been encountered by keeping a mapping of that id and the associated file:
files = {}
with open('file.csv') as fd:
for line in fd:
if 0 == len(line.strip()): continue # skip empty lines
try:
id_field = line.split(',', 1)[0] # extract first field
if not id in files.keys(): # if not encountered open a new result file
files[id] = open(id + '.csv')
files[id].write(line) # write the line in proper file
except Exception as e:
print('ERR', line, e) # catchall in case of problems...