Develop Reference

IndexError: list index out of range - PythonError

I'm creating a program that should create a file (.txt) based on each line of 'clouds.txt'. This is my code:
def CreateFile():
global file_name
f = open(file_name,"w+")
f.write(list_email + ":")
f.close()
def WriteInConfig():
f = open("config/config.txt","a")
f.write(list_name + "\n")
f.close()
with open("clouds.txt","r") as f:
list_lines = sum(1 for line in open('clouds.txt'))
lines = f.readline()
for line in lines:
first_line = f.readline().strip()
list_email = first_line.split('|')[1] #email
print("Email: " + list_email)
list_pass = first_line.split('|')[2] #pass
print("Pass: " + list_pass)
list_name = first_line.split('|')[3] #name
print(list_name)
global file_name
file_name = "config/." + list_name + ".txt"
with open('clouds.txt', 'r') as fin:
data = fin.read().splitlines(True)
with open('clouds.txt', 'w') as fout:
fout.writelines(data[1:])
CreateFile()
WriteInConfig()
The clouds.txt file looks like this:
>|clouds.n1c0+mega01#gmail.com|cwSHklDIybllCD1OD4M|Mega01|15|39.91|FdUkLiW0ThDeDkSlqRThMQ| |x
|clouds.n1c0+mega02#gmail.com|tNFVlux4ALC|Mega02|50|49.05|lq1cTyp13Bh9-hc6cZp1RQ|xxx|x
|clouds.n1c0+mega03#gmail.com|7fe4196A4CUT3V|Mega03|50|49.94|BzW7NOGmfhQ01cy9dAdlmg|xxx|xxx >
Everything works fine until 'Mega48'. There I get "IndexError: list index out of range"
>|clouds.n1c0+mega47#gmail.com|bd61t9zxcuC1Yx|Mega47|50|10|Xjff6C8mzEqpa3VcaalUuA|xxx|x
|clouds.n1c0+mega48#gmail.com|kBdnyB6i0PUyUb|Mega48|50|0|R6YfuGP2hvE-uds0ylbQtQ|xxx|x
|clouds.n1c0+mega49#gmail.com|OcAdgpS4tmSLTO|Mega49|50|28.65|xxx| >
I checked and there are no spaces/other characters. As you could see, after creating the file, the program deletes the line. After the error, if I'm starting the program again (and starts from 'Mega47') it doesn't show the error, and everything works as planned.
Any ideas how to fix this?

I see many mistakes in your code. First, what do you want with this list_lines = sum(1 for line in open('clouds.txt'))?
You have a problem in your for loop because you did lines = f.readline() so lines is the first line, then you do for line in lines where line will be each character of the first line and there are more character in the first line than lines in your file to read.
[edited]
you don't need to know the number of lines in the file to do a for loop. You can just do for line in f:, then you don't need to read the line again with readline it is already in the variable line

Function not returning an output file

I have the following python code whose purpose is to remove blank lines from an input text file. It should return an output file with all blank lines removed but it doesn't. What's the bug? Thank you!
import sys
def main():
inputFileName = sys.argv[1]
outputFileName = sys.argv[2]
inputFile = open(inputFileName, "r")
outputFile = open(inputFileName, "w")
for line in inputFile:
if "\n" in line:
removeBlank = line.replace("\n", "")
outputFile.write(removeBlank)
else:
outputFile.write(line)
inputFile.close()
outputFile.close()
main()

You have a lot of problem with your code. Specially the condition you check with empty line. People has rightly pointed out some problems.
Here is the solutions that should work and generate the output file with no empty lines.
import sys
def main():
inputFileName = sys.argv[1]
outputFileName = sys.argv[2]
with open(inputFileName) as inputFile, open(inputFileName, "w") as outputFile:
for line in inputFile.readlines():
if line.strip() != '':
outputFile.write(line)
if __name__ == '__main__':
main()

At present your code appears to truncate its input file immediately after opening it. At best this might give differing results on different platforms. On some platforms the file might be empty. I presume that opening the input file for writing was a typo.
A better way to approach this problem is to use a generator. Also, the correct test for an empty line is line == '\n', not '\n' in line, which will be true for all returned lines except perhaps the last.
def noblanks(file):
for line in file:
if line != '\n':
yield line
You can use this like so:
with open(inputFileName, "r") as inf, open(outputFilename, 'w') as outf:
for line in noblanks(inf):
outf.write(line)
The context managers in the with statement will ensure that your files are properly closed without further action on your part.

How to replace string in a file text based on regex?

Assume:
self.base_version = 1000
self.target_version = 2000
I have a file as follows:
some text...
<tsr_args> \"upgrade_test test_mode=upgrade base_sw=1000 target_sw=2000 system_profile=eth\"</tsr_args>
some text...
<tsr_args> \"upgrade_test test_mode=rollback base_sw=2000 target_sw=1000 system_profile=eth manufacture_type=no-manufacture\"</tsr_args>
some text...
<tsr_args> \"upgrade_test test_mode=downgrade base_sw=2000 target_sw=1000 system_profile=eth no_boot_next_enable_flag=True\"</tsr_args>
I need the base and target version values to be placed as specified above (Note that on the 2nd and 3rd entry, the base and target are opposite).
I tried to do it as follows, but it does not work:
base_regex = re.compile('.*test_mode.*base_sw=(.*)')
target_regex = re.compile('.*test_mode.*target_sw=(.*)')
o = open(file,'a')
for line in open(file):
if 'test_mode' in line:
if 'upgrade' in line:
new_line = (re.sub(base_regex, self.base_version, line))
new_line = (re.sub(target_regex, self.target_version, line))
o.write(new_line)
elif 'rollback' in line or 'downgrade' in line):
new_line = (re.sub(base_regex, self.target_version, line))
new_line = (re.sub(target_regex, self.base_version, line))
o.write(new_line)
o.close()
Assume the above code runs properly without any syntax errors.
The file is not modified at all.
The complete line is modified instead of just the captured group. How can I make re.sub to substitute only the captured group?

You are opening file with a -> append. So, your changes should be at the end of file. You should create a new file and replace old_one at the end of your script.
There is only one way I know if you want replace several matching groups: first of all you find word using regexp and replace it like a string without regexp.

Thanks Jimilan for your remarks. I fixed my code, and now it`s working:
base_regex = re.compile(.*test_mode.*base_sw=(\S*))
target_regex = re.compile(.*test_mode.*target_sw=(\S*))
for file in self.upgrade_cases_files_list:
file_handle = open(file, 'r')
file_string = file_handle.read()
file_handle.close()
base_version_result = base_regex.search(file_string)
target_version_result = target_regex.search(file_string)
if base_version_result is not None:
current_base_version = base_version_result.group(1)
else:
raise Exception("Could not detect base version in the following file: -> %s \n" % (file))
if target_version_result is not None:
current_target_version = target_version_result.group(1)
else:
raise Exception("Could not detect target version in the following file: -> %s \n" % (file))
file_string = file_string.replace(current_base_version, self.base_version)
file_string = file_string.replace(current_target_version, self.target_version)
file_handle = open(file, 'w')
file_handle.write(file_string)
file_handle.close()

Python - delete uuencoding lines

I am processing many text files which (some of them) contain uuencoding which can be .jpg or .pdf or .zip of .xlsx etc. I don't care about the embedded UUencoded data, so I would just like to discard these passages and keep the rest of the text. I'm struggling with how to come up with a method to skip only just enough, but not too much.
To summarize http://en.wikipedia.org/wiki/Uuencoding each blob begins with
begin 644 filename.extension
every line after the begin 644 seems to start by the letter
M
so this might also help. Any idea how to have a function that deletes all these lines for all .txt files in a folder (directory)?
For example, the following is a .jpg uuencoding
GRAPHIC
18
g438975g32h99a01.jpg
begin 644 g438975g32h99a01.jpg
M_]C_X``02D9)1#`!`#$`8`!#``#_[0G64&AO;=&]S:&]P(#,N,``X0DE-`^T`
M`````!``8`````$``0!#`````0`!.$))300-```````$````'CA"24T$&0``
M````!````!XX0DE-`_,```````D```````````$`.$))300*```````!```X
M0DE-)Q````````H``0`````````".$))30/U``````!(`"]F9#`!`&QF9;#`&
M```````!`"]F9#`!`*&9F#`&```````!`#(````!`%H````&```````!`#4`
M```!`"T````&```````!.$))30/X``````!P``#_____________________
M________`^#`````_____________________________P/H`````/______
M______________________\#Z`````#_____________________________
M`^#``#A"24T$"```````$`````$```)````"0``````X0DE-!!X```````0`
M````.$))300:``````!M````!#``````````````)P```+`````&`&<`,P`R
M`&#`.0`Y`````0`````````````````````````!``````````````"P````
M)P`````````````````````````````````````````````X0DE-!!$`````
M``$!`#A"24T$%```````!`````(X0DE-!`P`````!SH````!````<````!D`
M``%0```#T```!QX`&``!_]C_X``02D9)1#`!`#$`2`!(``#_[#`.061O8F4`
M9(`````!_]L`A``,"`#("0#,"0D,$0L*"Q$5#PP,#Q48$Q,5$Q,8$0P,#`P,
M#!$,#`P,#`P,#`P,#`P,#`P,#`P,#`P,#`P,#`P,`0T+"PT.#1`.#A`4##X.
M%!0.##X.%!$,#`P,#!$1#`P,#`P,$0P,#`P,#`P,#`P,#`P,#`P,#`P,#`P,
M#`P,#`S_P``1"``9`'`#`2(``A$!`Q$!_]T`!``'_\0!/P```04!`0$!`0$`
M`````````P`!`#0%!#<("0H+`0`!!0$!`0$!`0`````````!``(#!`4&!P#)
M"#L0``$$`0,"!`(%!P8(!0,,,P$``A$#!"$2,05!46$3(G&!,#84D:&Q0B;,D
M%5+!8C,T<H+10P)E\K.$P]-U
MX_-&)Y2DA;25Q-3D]*6UQ=7E]59F=H:6IK;&UN;V-T=79W>'EZ>WQ]?G]Q$`
M`#(!`#0$`P0%!#<'!#4U`0`"$0,A,1($05%A<2(3!3*!D12AL4(CP5+1\#,D
M8N%R#I)#4Q5C<S3Q)086HK*#!R8UPM)$DU2C%V1%539T9>+RLX3#TW7C\T:4
MI(6TE<34Y/2EM<75Y?569G:&EJ;:VQM;F]B
I would like to be left with just
GRAPHIC
18
g438975g32h99a01.jpg
For background, see also my earlier question How to remove weird encoding from txt file
EDIT : Here is a try
start_marker = 'begin 644'
with open('fileWithBegin644.txt') as inf:
ignoreLines = False
for line in inf:
if start_marker in line:
print line,
ignoreLines = True
if not ignoreLines:
with open("strip_" + inf, "w") as f:
f.write(line.get_text().encode('utf-8'))
But I am getting the following error
File "removeUuencodingFromAll.py", line 10
with open("strip_" + inf, "w") as f:
^
IndentationError: expected an indented block

I coded up what was supposed to be a rather simple generator. Because the spec is slightly tedious (why two separate end markers on different lines?) it is rather bulky, but here goes. It should work as a validator for uuencode at the same time, but I have only tested it in very limited settings.
import re
def unuuencode (iterator, collector=None, ignore_length_errors=False):
"""
Yield lines from iterator except when they are in an uuencode blob.
If collector is not None, append to it the uuencoded blobs as a list
of a list of lines, one for each uuencoded blob.
"""
state = None # one of { None, 'in_blob', 'closing', 'closed' }
collectitem = None
regex = re.compile(r'^begin\s+[0-7]{3,6}\s+.*?(?:\r?\n)?$')
for line in iterator:
if state == None:
if regex.match(line):
if collector != None:
collectitem = [line]
state = 'in_blob'
continue
else:
yield line
else:
stripped = line.rstrip('\r\n')
if state == 'in_blob' and line.startswith('`'):
state = 'closing'
if state == 'closing':
if stripped != '`':
raise ValueError('Expected "`" but got "%s"' % line)
state = 'closed'
elif state == 'closed':
if stripped != 'end':
raise ValueError('Expected "end" but got "%s"' % line)
state = None
else:
expect = ord(line[0:1])-32
actual = len(stripped)
seen = (len(stripped)-1)*6/8
if seen != expect:
if not ignore_length_errors:
raise ValueError('Wrong prefix on line: %s '
'(indicated %i, 6/8 %i, actual length %i)' % (
line, expect, seen, actual))
if line[0:1] != 'M':
state = 'closing'
if collectitem:
collectitem.append(line)
if state is None:
if collectitem:
collector.append(collectitem)
collectitem = None
continue
Use it like this:
with open(file, 'r') as f:
lines = [x for x in unuuencode(f)]
or like this:
with open(file, 'r') as f:
blobs = []
lines = [x for x in unuuencode(f, collector=blobs)]
or like this:
with open(file, 'r') as f:
lines = f.read().split('\n')
# ... or whichever way you obtained your content as an array of lines
lines = [x for x in unuuencode(lines)]
or in the case of the code you seem to be using:
for fi in sys.argv[1:]:
with open(fi) as markup:
soup = BeautifulSoup(''.join(unuuencode(markup, ignore_length_errors=True)))
with open("strip_" + fi, "w") as f:
f.write(soup.get_text().encode('utf-8'))
The sample you linked to had an invalid length indicator in the second uuencoded blob, so I added an option to ignore that.

how to replace (update) text in a file line by line

I am trying to replace text in a text file by reading each line, testing it, then writing if it needs to be updated. I DO NOT want to save as a new file, as my script already backs up the files first and operates on the backups.
Here is what I have so far... I get fpath from os.walk() and I guarantee that the pathmatch var returns correctly:
fpath = os.path.join(thisdir, filename)
with open(fpath, 'r+') as f:
for line in f.readlines():
if '<a href="' in line:
for test in filelist:
pathmatch = file_match(line, test)
if pathmatch is not None:
repstring = filelist[test] + pathmatch
print 'old line:', line
line = line.replace(test, repstring)
print 'new line:', line
f.write(line)
But what ends up happening is that I only get a few lines (updated correctly, mind you, but repeated from earlier in the file) corrected. I think this is a scoping issue, afaict.
*Also: I would like to know how to only replace the text upon the first instance of the match, for ex., I don't want to match the display text, only the underlying href.

First, you want to write the line whether it matches the pattern or not. Otherwise, you're writing out only the matched lines.
Second, between reading the lines and writing the results, you'll need to either truncate the file (can f.seek(0) then f.truncate()), or close the original and reopen. Picking the former, I'd end up with something like:
fpath = os.path.join(thisdir, filename)
with open(fpath, 'r+') as f:
lines = f.readlines()
f.seek(0)
f.truncate()
for line in lines:
if '<a href="' in line:
for test in filelist:
pathmatch = file_match(line, test)
if pathmatch is not None:
repstring = filelist[test] + pathmatch
line = line.replace(test, repstring)
f.write(line)

Open the file for read and copy all of the lines into memory. Close the file.
Apply your transformations on the lines in memory.
Open the file for write and write out all the lines of text in memory.
with open(filename, "r") as f:
lines = (line.rstrip() for line in f)
altered_lines = [some_func(line) if regex.match(line) else line for line in lines]
with open(filename, "w") as f:
f.write('\n'.join(altered_lines) + '\n')

A (relatively) safe way to replace a line in a file.
#!/usr/bin/python
# defensive programming style
# function to replace a line in a file
# and not destroy data in case of error
def replace_line(filepath, oldline, newline ):
"""
replace a line in a temporary file,
then copy it over into the
original file if everything goes well
"""
# quick parameter checks
assert os.exists(filepath) # !
assert ( oldline and str(oldline) ) # is not empty and is a string
assert ( newline and str(newline) )
replaced = False
written = False
try:
with open(filepath, 'r+') as f: # open for read/write -- alias to f
lines = f.readlines() # get all lines in file
if oldline not in lines:
pass # line not found in file, do nothing
else:
tmpfile = NamedTemporaryFile(delete=True) # temp file opened for writing
for line in lines: # process each line
if line == oldline: # find the line we want
tmpfile.write(newline) # replace it
replaced = True
else:
tmpfile.write(oldline) # write old line unchanged
if replaced: # overwrite the original file
f.seek(0) # beginning of file
f.truncate() # empties out original file
for tmplines in tmpfile:
f.write(tmplines) # writes each line to original file
written = True
tmpfile.close() # tmpfile auto deleted
f.close() # we opened it , we close it
except IOError, ioe: # if something bad happened.
printf ("ERROR" , ioe)
f.close()
return False
return replaced and written # replacement happened with no errors = True
(note: this replaces entire lines only , and all of the lines that match in the file)