Related
This is one of the MIT python project questions, but it's basically written for python 2.x users, so is there any way to fix the following code to operate in the latest python 3?
The current code is raising "ValueError: can't have unbuffered text I/O"
WORDLIST_FILENAME = "words.txt"
def load_words():
print("Loading word list from file...")
inFile = open(WORDLIST_FILENAME, 'r', 0)
# wordlist: list of strings
wordlist = []
for line in inFile:
wordlist.append(line.strip().lower())
print(" ", len(wordlist), "words loaded.")
return wordlist
From open's docstring:
... buffering is an optional integer used to set the buffering policy.
Pass 0 to switch buffering off (only allowed in binary mode) ...
So change inFile = open(WORDLIST_FILENAME, 'r', 0)
to
inFile = open(WORDLIST_FILENAME, 'r'), or to
inFile = open(WORDLIST_FILENAME, 'rb', 0) if you really need it (which I doubt).
I could overcome this error by using code from this answer:
class Unbuffered(object):
def __init__(self, stream):
self.stream = stream
def write(self, data):
self.stream.write(data)
self.stream.flush()
def writelines(self, datas):
self.stream.writelines(datas)
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
import sys
sys.stdout = Unbuffered(sys.stdout)
You can flush to the file after every write.
outFile = open(outputfile, "w",encoding='utf-8')
inFile = open(inputfile, "r",encoding='utf-8')
while inFile:
line = inFile.readline()[:-1]
outFile.write(line)
outFile.flush()
#I wrote the following code for making a text editor.
print "This is a simple text editor"
from sys import argv
script, name = argv
print "You have selected the file : %s" %name
print "Opening the file...."
t = open(name, 'r+')
print "The contents of the file are"
print t.read()
f = open(name, 'w+')
print "Now we will truncate the file and empty it of it's contents"
f.truncate()
print "Now let us write something into our file\n"
x = raw_input('What do you want to write\n') #Works fine till here
f.write(x)
print "Now we read our file again"
print f.read()
print "And finally we close the file"
f.close()
After the promp to write something in the file, the script goes awry and produces strange symbols instead of the typed text. Please help
You need to close and re-open your file.
print "This is a simple text editor"
from sys import argv
script, name = argv
print "You have selected the file : %s" %name
print "Opening the file...."
t = open(name, 'r+')
print "The contents of the file are"
print t.read()
t.close() ##########
f = open(name, 'w+')
print "Now we will truncate the file and empty it of it's contents"
f.truncate()
print "Now let us write something into our file\n"
x = raw_input('What do you want to write\n') #Works fine till here
f.write(x)
f.close() ##########
f = open(name, 'r+') ##########
print "Now we read our file again"
print f.read()
print "And finally we close the file"
f.close()
so I'm having this trouble with the decode. I found it in other threads how to do it for simple strings, with the u'string'.encode. But I can't find a way to make it work with files.
Any help would be appreciated!
Here's the code.
text = file.read()
text.replace(txt.encode('utf-8'), novo_txt.encode('utf-8'))
file.seek(0) # rewind
file.write(text.encode('utf-8'))
and here's the whole code, should it help.
#!/usr/bin/env python
# coding: utf-8
"""
Script to helps on translate some code's methods from
portuguese to english.
"""
from multiprocessing import Pool
from mock import MagicMock
from goslate import Goslate
import fnmatch
import logging
import os
import re
import urllib2
_MAX_PEERS = 1
try:
os.remove('traducoes.log')
except OSError:
pass
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler('traducoes.log')
logger.addHandler(handler)
def fileWalker(ext, dirname, names):
"""
Find the files with the correct extension
"""
pat = "*" + ext[0]
for f in names:
if fnmatch.fnmatch(f, pat):
ext[1].append(os.path.join(dirname, f))
def encontre_text(file):
"""
find on the string the works wich have '_' on it
"""
text = file.read().decode('utf-8')
return re.findall(r"\w+(?<=_)\w+", text)
#return re.findall(r"\"\w+\"", text)
def traduza_palavra(txt):
"""
Translate the word/phrase to english
"""
try:
# try connect with google
response = urllib2.urlopen('http://google.com', timeout=2)
pass
except urllib2.URLError as err:
print "No network connection "
exit(-1)
if txt[0] != '_':
txt = txt.replace('_', ' ')
txt = txt.replace('media'.decode('utf-8'), 'média'.decode('utf-8'))
gs = Goslate()
#txt = gs.translate(txt, 'en', gs.detect(txt))
txt = gs.translate(txt, 'en', 'pt-br') # garantindo idioma tupiniquim
txt = txt.replace(' en ', ' br ')
return txt.replace(' ', '_') # .lower()
def subistitua(file, txt, novo_txt):
"""
should rewrite the file with the new text in the future
"""
text = file.read()
text.replace(txt.encode('utf-8'), novo_txt.encode('utf-8'))
file.seek(0) # rewind
file.write(text.encode('utf-8'))
def magica(File):
"""
Thread Pool. Every single thread should play around here with
one element from list os files
"""
global _DONE
if _MAX_PEERS == 1: # inviavel em multithread
logger.info('\n---- File %s' % File)
with open(File, "r+") as file:
list_txt = encontre_text(file)
for txt in list_txt:
novo_txt = traduza_palavra(txt)
if txt != novo_txt:
logger.info('%s -> %s [%s]' % (txt, novo_txt, File))
subistitua(file, txt, novo_txt)
file.close()
print File.ljust(70) + '[OK]'.rjust(5)
if __name__ == '__main__':
try:
response = urllib2.urlopen('http://www.google.com.br', timeout=1)
except urllib2.URLError as err:
print "No network connection "
exit(-1)
root = './app'
ex = ".py"
files = []
os.path.walk(root, fileWalker, [ex, files])
print '%d files found to be translated' % len(files)
try:
if _MAX_PEERS > 1:
_pool = Pool(processes=_MAX_PEERS)
result = _pool.map_async(magica, files)
result.wait()
else:
result = MagicMock()
result.successful.return_value = False
for f in files:
pass
magica(f)
result.successful.return_value = True
except AssertionError, e:
print e
else:
pass
finally:
if result.successful():
print 'Translated all files'
else:
print 'Some files were not translated'
Thank you all for the help!
In Python 2, reading from files produces regular (byte) string objects, not unicode objects. There is no need to call .encode() on these; in fact, that'll only trigger an automatic decode to Unicode first, which can fail.
Rule of thumb: use a unicode sandwich. Whenever you read data, you decode to unicode at that stage. Use unicode values throughout your code. Whenever you write data, encode at that point. You can use io.open() to open file objects that encode and decode automatically for you.
That also means you can use unicode literals everywhere; for your regular expressions, for your string literals. So use:
def encontre_text(file):
text = file.read() # assume `io.open()` was used
return re.findall(ur"\w+(?<=_)\w+", text) # use a unicode pattern
and
def subistitua(file, txt, novo_txt):
text = file.read() # assume `io.open()` was used
text = text.replace(txt, novo_txt)
file.seek(0) # rewind
file.write(text)
as all string values in the program are already unicode, and
txt = txt.replace(u'media', u'média')
as u'..' unicode string literals don't need decoding anymore.
I have a CSV file which has three columns. The third column "row[02]" has a listing of IP's which contains a list of IP's that are comma delimited. I would like to turn the following:
"column01", "column02", "192.168.1.1, 192.168.2.1, 192.168.3.1"
into (minus the bullets):
column01, column02, 192.168.1.1
column01, column02, 192.168.2.1
column01, column02, 192.168.3.1
Provided is my code, but the output keeps displaying the following:
row01, row02, 192.168.1.1, 192.168.2.1, 192.168.3.1
Please let me know if any other information is needed. Thanks in advance.
Original Code:
#!/usr/bin/python
import csv, sys, time, logging
s = time.strftime('%Y%m%d%H%M%S')
# Create exception file from standard output
class Logger(object):
def __init__(self):
self.terminal = sys.stdout
self.log = open((s) + "_" + sys.argv[1], "a")
def write(self, message):
self.terminal.write(message)
self.log.write(message)
def main():
# Start screen capture to output into CSV file
sys.stdout = Logger()
# File input argument
with open(sys.argv[1], "rb") as f:
reader = csv.reader(f, delimiter=',')
for row in reader:
print row[0], ",", row[1], ",", row[2]
if __name__=='__main__':
main()
Updated Code:
#!/usr/bin/python
import csv, sys, time, logging
s = time.strftime('%Y%m%d%H%M%S')
# Create exception file from standard output
class Logger(object):
def __init__(self):
self.terminal = sys.stdout
self.log = open((s) + "_" + sys.argv[1], "a")
def write(self, message):
self.terminal.write(message)
self.log.write(message)
def main():
# Start screen capture to output file
sys.stdout = Logger()
# File input argument
with open(sys.argv[1], "rb") as f:
reader = csv.reader(f, delimiter=',', skipinitialspace=True)
for row in reader:
network = row[0].replace(" ","")
network_group = row[1].replace(" ","")
address = row[2].replace(',','\n').replace(" ","")
if "-" in address: #parse network ranges
try:
print IP(address), network, network_group
except:
netrange = address.replace('-'," ").replace(" ",",")
print netrange, ",", network, ",", network_group
else:
print address, ",", network, ",", network_group
if __name__=='__main__':
main()
The reason it's printing this:
row01, row02, 192.168.1.1, 192.168.2.1, 192.168.3.1
Is that you asked it to do exactly that for each row:
for row in reader:
print row[0], ",", row[1], ",", row[2]
If you want it to do something different, you have to tell it to do something different. It can't read your mind. So, if you want to, e.g., split row[2] on commas, you need to write some code that does that. Like this:
for row in reader:
addresses = row[2].split(", ")
for address in addresses:
print row[0], ",", row[1], ",", address
I want to append a newline to my string every time I call file.write(). What's the easiest way to do this in Python?
Use "\n":
file.write("My String\n")
See the Python manual for reference.
You can do this in two ways:
f.write("text to write\n")
or, depending on your Python version (2 or 3):
print >>f, "text to write" # Python 2.x
print("text to write", file=f) # Python 3.x
You can use:
file.write(your_string + '\n')
If you use it extensively (a lot of written lines), you can subclass 'file':
class cfile(file):
#subclass file to have a more convienient use of writeline
def __init__(self, name, mode = 'r'):
self = file.__init__(self, name, mode)
def wl(self, string):
self.writelines(string + '\n')
Now it offers an additional function wl that does what you want:
with cfile('filename.txt', 'w') as fid:
fid.wl('appends newline charachter')
fid.wl('is written on a new line')
Maybe I am missing something like different newline characters (\n, \r, ...) or that the last line is also terminated with a newline, but it works for me.
you could do:
file.write(your_string + '\n')
as suggested by another answer, but why using string concatenation (slow, error-prone) when you can call file.write twice:
file.write(your_string)
file.write("\n")
note that writes are buffered so it amounts to the same thing.
Another solution that writes from a list using fstring
lines = ['hello','world']
with open('filename.txt', "w") as fhandle:
for line in lines:
fhandle.write(f'{line}\n')
And as a function
def write_list(fname, lines):
with open(fname, "w") as fhandle:
for line in lines:
fhandle.write(f'{line}\n')
write_list('filename.txt', ['hello','world'])
file_path = "/path/to/yourfile.txt"
with open(file_path, 'a') as file:
file.write("This will be added to the next line\n")
or
log_file = open('log.txt', 'a')
log_file.write("This will be added to the next line\n")
Unless write to binary files, use print. Below example good for formatting csv files:
def write_row(file_, *columns):
print(*columns, sep='\t', end='\n', file=file_)
Usage:
PHI = 45
with open('file.csv', 'a+') as f:
write_row(f, 'header', 'phi:', PHI, 'serie no. 2')
write_row(f) # additional empty line
write_row(f, data[0], data[1])
You can also use partial as a more pythonic way of creating this kind of wrappers. In the example below, row is print with predefined kwargs.
from functools import partial
with open('file.csv', 'a+') as f:
row = partial(print, sep='\t', end='\n', file=f)
row('header', 'phi:', PHI, 'serie no. 2', end='\n\n')
row(data[0], data[1])
Notes:
print documentation
'{}, {}'.format(1, 'the_second') - https://pyformat.info/, PEP-3101
'\t' - tab character
*columns in function definition - dispatches any number of arguments to list - see question on *args & **kwargs
Just a note, file isn't supported in Python 3 and was removed. You can do the same with the open built-in function.
f = open('test.txt', 'w')
f.write('test\n')
Ok, here is a safe way of doing it.
with open('example.txt', 'w') as f:
for i in range(10):
f.write(str(i+1))
f.write('\n')
This writes 1 to 10 each number on a new line.
I really didn't want to type \n every single time and #matthause's answer didn't seem to work for me, so I created my own class
class File():
def __init__(self, name, mode='w'):
self.f = open(name, mode, buffering=1)
def write(self, string, newline=True):
if newline:
self.f.write(string + '\n')
else:
self.f.write(string)
And here it is implemented
f = File('console.log')
f.write('This is on the first line')
f.write('This is on the second line', newline=False)
f.write('This is still on the second line')
f.write('This is on the third line')
This should show in the log file as
This is on the first line
This is on the second lineThis is still on the second line
This is on the third line
This is the solution that I came up with trying to solve this problem for myself in order to systematically produce \n's as separators. It writes using a list of strings where each string is one line of the file, however it seems that it may work for you as well. (Python 3.+)
#Takes a list of strings and prints it to a file.
def writeFile(file, strList):
line = 0
lines = []
while line < len(strList):
lines.append(cheekyNew(line) + strList[line])
line += 1
file = open(file, "w")
file.writelines(lines)
file.close()
#Returns "\n" if the int entered isn't zero, otherwise "".
def cheekyNew(line):
if line != 0:
return "\n"
return ""
You can decorate method write in specific place where you need this behavior:
#Changed behavior is localized to single place.
with open('test1.txt', 'w') as file:
def decorate_with_new_line(method):
def decorated(text):
method(f'{text}\n')
return decorated
file.write = decorate_with_new_line(file.write)
file.write('This will be on line 1')
file.write('This will be on line 2')
file.write('This will be on line 3')
#Standard behavior is not affected. No class was modified.
with open('test2.txt', 'w') as file:
file.write('This will be on line 1')
file.write('This will be on line 1')
file.write('This will be on line 1')
Using append (a) with open() on a print() statement looks easier for me:
save_url = ".\test.txt"
your_text = "This will be on line 1"
print(your_text, file=open(save_url, "a+"))
another_text = "This will be on line 2"
print(another_text, file=open(save_url, "a+"))
another_text = "This will be on line 3"
print(another_text, file=open(save_url, "a+"))
You could use C-style string formatting:
file.write("%s\n" % "myString")
More about String Formatting.
If write is a callback, you may need a custom writeln.
def writeln(self, string):
self.f.write(string + '\n')
Itself inside a custom opener. See answers and feedback for this question : subclassing file objects (to extend open and close operations) in python 3
(Context Manager)
I faced this when using ftplib to "retrieve lines" from a file that was "record based" (FB80):
with open('somefile.rpt', 'w') as fp:
ftp.retrlines('RETR USER.REPORT', fp.write)
and ended up with one long record with no newlines, this is likely a problem with ftplib, but obscure.
So this became:
with OpenX('somefile.rpt') as fp:
ftp.retrlines('RETR USER.REPORT', fp.writeln)
It does the job. This is a use case a few people will be looking for.
Complete declaration (only the last two lines are mine):
class OpenX:
def __init__(self, filename):
self.f = open(filename, 'w')
def __enter__(self):
return self.f
def __exit__(self, exc_type, exc_value, traceback):
self.f.close()
def writeln(self, string):
self.f.write(string + '\n')
in order to suspport multiple operating systems use:
file.write(f'some strings and/or {variable}. {os.linesep}')
Actually, when you use the multiline syntax, like so:
f.write("""
line1
line2
line2""")
You don't need to add \n!
Usually you would use \n but for whatever reason in Visual Studio Code 2019 Individual it won't work. But you can use this:
# Workaround to \n not working
print("lorem ipsum", file=f) # Python 3.0 onwards only
print >>f, "Text" # Python 2.0 and under