The goal of this code is to find the frequency of words used in a book.
I am tying to read in the text of a book but the following line keeps throwing my code off:
precious protégés. No, gentlemen; he'll always show 'em a clean pair
specifically the é character
I have looked at the following documentation, but I don't quite understand it: https://docs.python.org/3.4/howto/unicode.html
Heres my code:
import string
# Create word dictionary from the comprehensive word list
word_dict = {}
def create_word_dict ():
# open words.txt and populate dictionary
word_file = open ("./words.txt", "r")
for line in word_file:
line = line.strip()
word_dict[line] = 1
# Removes punctuation marks from a string
def parseString (st):
st = st.encode("ascii", "replace")
new_line = ""
st = st.strip()
for ch in st:
ch = str(ch)
if (n for n in (1,2,3,4,5,6,7,8,9,0)) in ch or ' ' in ch or ch.isspace() or ch == u'\xe9':
print (ch)
new_line += ch
else:
new_line += ""
# now remove all instances of 's or ' at end of line
new_line = new_line.strip()
print (new_line)
if (new_line[-1] == "'"):
new_line = new_line[:-1]
new_line.replace("'s", "")
# Conversion from ASCII codes back to useable text
message = new_line
decodedMessage = ""
for item in message.split():
decodedMessage += chr(int(item))
print (decodedMessage)
return new_line
# Returns a dictionary of words and their frequencies
def getWordFreq (file):
# Open file for reading the book.txt
book = open (file, "r")
# create an empty set for all Capitalized words
cap_words = set()
# create a dictionary for words
book_dict = {}
total_words = 0
# remove all punctuation marks other than '[not s]
for line in book:
line = line.strip()
if (len(line) > 0):
line = parseString (line)
word_list = line.split()
# add words to the book dictionary
for word in word_list:
total_words += 1
if (word in book_dict):
book_dict[word] = book_dict[word] + 1
else:
book_dict[word] = 1
print (book_dict)
# close the file
book.close()
def main():
wordFreq1 = getWordFreq ("./Tale.txt")
print (wordFreq1)
main()
The error that I received is as follows:
Traceback (most recent call last):
File "Books.py", line 80, in <module>
main()
File "Books.py", line 77, in main
wordFreq1 = getWordFreq ("./Tale.txt")
File "Books.py", line 60, in getWordFreq
line = parseString (line)
File "Books.py", line 36, in parseString
decodedMessage += chr(int(item))
OverflowError: Python int too large to convert to C long
When you open a text file in python, the encoding is ANSI by default, so it doesn't contain your é chartecter. Try
word_file = open ("./words.txt", "r", encoding='utf-8')
The best way I could think of is to read each character as an ASCII value, into an array, and then take the char value. For example, 97 is ASCII for "a" and if you do char(97) it will output "a". Check out some online ASCII tables that provide values for special characters also.
Try:
def parseString(st):
st = st.encode("ascii", "replace")
# rest of code here
The new error you are getting is because you are calling isalpha on an int (i.e. a number)
Try this:
for ch in st:
ch = str(ch)
if (n for n in (1,2,3,4,5,6,7,8,9,0) if n in ch) or ' ' in ch or ch.isspace() or ch == u'\xe9':
print (ch)
Related
I am trying to create a function in which a filename is taken as a parameter and the function returns the longest word in the file with the line number attached to the front of it.
This is what I have so far but it is not producing the expected output I need.
def word_finder(file_name):
with open(file_name) as f:
lines = f.readlines()
line_num = 0
longest_word = None
for line in lines:
line = line.strip()
if len(line) == 0:
return None
else:
line_num += 1
tokens = line.split()
for token in tokens:
if longest_word is None or len(token) > len(longest_word):
longest_word = token
return (str(line_num) + ": " + str(longest_word))
I think this is the shortest way to find the word, correct if not
def wordFinder(filename):
with open(filename, "r") as f:
words = f.read().split() # split() returns a list with the words in the file
longestWord = max(words, key = len) # key = len returns the word size
print(longestWord) # prints the longest word
Issue
Exactly what ewong diagnosed:
last return statement is too deep indented
Currently:
the longest word in the first line only
Solution
Should be aligned with the loop's column, to be executed after the loop.
def word_finder(file_name):
with open(file_name) as f:
lines = f.readlines()
line_num = 0
longest_word = None
for line in lines:
line = line.strip()
if len(line) == 0:
return None
else:
line_num += 1
tokens = line.split()
for token in tokens:
if longest_word is None or len(token) > len(longest_word):
longest_word = token
# return here would exit the loop too early after 1st line
# loop ended
return (str(line_num) + ": " + str(longest_word))
Then:
the longest word in the file with the line number attached to the front of it.
Improved
def word_finder(file_name):
with open(file_name) as f:
line_word_longest = None # global max: tuple of (line-index, longest_word)
for i, line in enumerate(f): # line-index and line-content
line = line.strip()
if len(line) > 0: # split words only if line present
max_token = max(token for token in line.split(), key = len) # generator then max of tokens by length
if line_word_longest is None or len(max_token) > len(line_word_longest[1]):
line_word_longest = (i, max_token)
# loop ended
if line_word_longest is None:
return "No longest word found!"
return f"{line_word_longest[0]}: '{line_word_longest[1]}' ({len(line_word_longest[1])} chars)"
See also:
Basic python file-io variables with enumerate
List Comprehensions in Python to compute minimum and maximum values of a list
Some SO research for similar questions:
inspiration from all languages: longest word in file
only python: [python] longest word in file
non python: -[python] longest word in file
I wrote this python script for handling dictionary file (which is a text file) like this:
Word goes after the '#' and the meanings of that word goes under the word seperated by \n.
...
#hello
xxx
yyy
zzz
#another
ooo
I'want to add words to file with this python script. But I don't know how to do that. And most confusing part is if the file has that word then add meanings to it.
./main.py add 'hello' 'xxx;yyy;zzz;'
This is how I can add the word and it's meanings to the file. How could I do that. I wrote many scripts to accomplish and they're:
def add(d,w,m):
dic = dictPath+d
nwords = '\n'.join(m.split(";"))
wmean = '#'+w+'\n'+nwords
here = False
lnum=0
with open(dic,"r") as rf:
for line in rf:
if line == "#"+w+"\n": ## 'cause every line ends with \n
here="yes"
lnum+=1
else:
lnum+=1
if here == "no":
with open(dic,"a") as af:
af.writelines(wmean)
else:
with open(dic,"w") as wf:
data = rf.readlines()
data[lnum+1] = nwords
wf.writelines(data)
wf.close()
rf.close()
af.close()
and
def add(d,w,m):
dic = dictPath+d
nwords = '\n'.join(m.split(";"))
wmean = '#'+w+'\n'+nwords
with open(dic,"r+") as f:
for line in f:
if line == "#"+w+"\n":
f.write(wmean)
I wrote a function that is supposed to add the words from a .txt to a list but it is supposed to ignore empty lines, how ever my function outputs ['',] at an empty line.
def words(filename):
word = []
file = open(filename)
for line in file:
word.append(line.strip())
return word
How can i fix this thanks
what about a simple if test?
def words(filename):
word = []
file = open(filename)
for line in file:
if line.strip() != ' ':
word.append(line.strip())
return word
EDIT: I forgot the .strip() after line
Besides, you could also use if line.strip():
Last, if you want to get a list of words but have several words per line, you need to split them. Assuming your separator is ' ':
def words(filename):
word = []
file = open(filename)
for line in file:
if line.strip() != ' ':
word.extend(line.strip().split())
return word
You can fix this like that:
def words(filename):
word = []
file = open(filename)
for line in file:
if not line.strip():
word.append(line)
return word
Your problem is that you're adding line.strip(), but what happens if line is actually an empty string? Look:
In [1]: line = ''
In [2]: line.strip()
Out[2]: ''
''.strip() returns an empty string.
You need to test for an empty line and skip the append in that case.
def words(filename):
word = []
file = open(filename)
for line in file:
line=line.strip()
if len(line):
word.append(line)
return word
I want to replace string in a line which contain patternB, something like this:
from:
some lines
line contain patternA
some lines
line contain patternB
more lines
to:
some lines
line contain patternA
some lines
line contain patternB xx oo
more lines
I have code like this:
inputfile = open("d:\myfile.abc", "r")
outputfile = open("d:\myfile_renew.abc", "w")
obj = "yaya"
dummy = ""
item = []
for line in inputfile:
dummy += line
if line.find("patternA") != -1:
for line in inputfile:
dummy += line
if line.find("patternB") != -1:
item = line.split()
dummy += item[0] + " xx " + item[-1] + "\n"
break
outputfile.write(dummy)
It do not replace the line contain "patternB" as expected, but add an new line below it like :
some lines
line contain patternA
some lines
line contain patternB
line contain patternB xx oo
more lines
What can I do with my code?
Of course it is, since you append line to dummy in the beginning of the for loop and then the modified version again in the "if" statement. Also why check for Pattern A if you treat is as you treat everything else?
inputfile = open("d:\myfile.abc", "r")
outputfile = open("d:\myfile_renew.abc", "w")
obj = "yaya"
dummy = ""
item = []
for line in inputfile:
if line.find("patternB") != -1:
item = line.split()
dummy += item[0] + " xx " + item[-1] + "\n"
else:
dummy += line
outputfile.write(dummy)
The simplest will be:
1. Read all File into string
2. Call string.replace
3. Dump string to file
If you want to keep line by line iterator
(for a big file)
for line in inputfile:
if line.find("patternB") != -1:
dummy = line.replace('patternB', 'patternB xx oo')
outputfile.write(dummy)
else:
outputfile.write(line)
This is slower than other responses, but enables big file processing.
This should work
import os
def replace():
f1 = open("d:\myfile.abc","r")
f2 = open("d:\myfile_renew.abc","w")
ow = raw_input("Enter word you wish to replace:")
nw = raw_input("Enter new word:")
for line in f1:
templ = line.split()
for i in templ:
if i==ow:
f2.write(nw)
else:
f2.write(i)
f2.write('\n')
f1.close()
f2.close()
os.remove("d:\myfile.abc")
os.rename("d:\myfile_renew.abc","d:\myfile.abc")
replace()
You can use str.replace:
s = '''some lines
line contain patternA
some lines
line contain patternB
more lines'''
print(s.replace('patternB', 'patternB xx oo'))
I have a text file that contains the contents of a book. I want to take this file and build an index which allows the user to search through the file to make searches.
The search would consist of entering a word. Then, the program would return the following:
Every chapter which includes that word.
The line number of the line
which contains the word.
The entire line the word is on.
I tried the following code:
infile = open(file)
Dict = {}
word = input("Enter a word to search: ")
linenum = 0
line = infile.readline()
for line in infile
linenum += 1
for word in wordList:
if word in line:
Dict[word] = Dict.setdefault(word, []) + [linenum]
print(count, word)
line = infile.readline()
return Dict
Something like this does not work and seems too awkward for handling the other modules which would require:
An "or" operator to search for one word or another
An "and" operator to search for one word and another in the same chapter
Any suggestions would be great.
def classify_lines_on_chapter(book_contents):
lines_vs_chapter = []
for line in book_contents:
if line.isupper():
current_chapter = line.strip()
lines_vs_chapter.append(current_chapter)
return lines_vs_chapter
def classify_words_on_lines(book_contents):
words_vs_lines = {}
for i, line in enumerate(book_contents):
for word in set([word.strip(string.punctuation) for word in line.split()]):
if word:
words_vs_lines.setdefault(word, []).append(i)
return words_vs_lines
def main():
skip_lines = 93
with open('book.txt') as book:
book_contents = book.readlines()[skip_lines:]
lines_vs_chapter = classify_lines_on_chapter(book_contents)
words_vs_lines = classify_words_on_lines(book_contents)
while True:
word = input("Enter word to search - ")
# Enter a blank input to exit
if not word:
break
line_numbers = words_vs_lines.get(word, None)
if not line_numbers:
print("Word not found!!\n")
continue
for line_number in line_numbers:
line = book_contents[line_number]
chapter = lines_vs_chapter[line_number]
print("Line " + str(line_number + 1 + skip_lines))
print("Chapter '" + str(chapter) + "'")
print(line)
if __name__ == '__main__':
main()
Try it on this input file. Rename it as book.txt before running it.