Develop Reference

Python MRJob Script Sorting Results - Top Ten Words Syllable Count

I am trying to make a job that takes in a text file, only processes words that are not in the STOPWORDS set, counts the number of syllables in each word, then returns the top 10 words with the most syllables, sorting the results.
I believe everything is correct, I just am not sure how to make the reducer sort the results.
Here is my code:
%%file top_10_syllable_count.py
import re
from sys import stderr
from mrjob.job import MRJob
from mrjob.step import MRStep
WORD_RE = re.compile(r"[\w']+")
import syllables
def splitter(text):
WORD_RE = re.compile(r"[\w']+")
return WORD_RE.findall(text)
def sort_results(results):
"""
Sorts a list of 2-tuples descending by the first value in the
tuple, ascending by the second value in the tuple.
"""
return sorted(results, key=lambda k: (-k[0], k[1]))
STOPWORDS = {
'i', 'we', 'ourselves', 'hers', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', 'during',
'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such',
'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each',
'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me',
'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up',
'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been',
'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so',
'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself',
'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by',
'doing', 'it', 'how', 'further', 'was', 'here', 'than'
}
class MRMostTenSyllables(MRJob):
def steps(self):
return[
MRStep(mapper=self.mapper_get_words),
MRStep(reducer=self.reducer_find_max_word)
]
def mapper_get_words(self, _, line):
for word in WORD_RE.findall(line):
if word.lower() not in STOPWORDS:
syllable_count = syllables.estimate(word)
yield None, (syllable_count, word.lower())
def reducer_find_max_word(self, key, values):
self.list = []
for value in values:
self.list.append(value)
self.new = []
for i in range(10):
self.new.append(max(self.list))
self.list.remove(max(self.list))
for i in range(10):
yield self.new[i]
if __name__ == '__main__':
import time
start = time.time()
MRMostTenSyllables.run()
end = time.time()
debug("Run time:", end - start, "seconds")

Write a function which removes english stop words from a tweet

I want to write a function that removes English stop words from a tweet.
Function Specifications:
It should take a pandas dataframe as input.
Should tokenise the sentences according to the definition in function 6. Note that function 6 cannot be called within this function.
Should remove all stop words in the tokenised list. The stopwords are defined in the stop_words_dict variable defined at the top of this notebook.
The resulting tokenised list should be placed in a column named "Without Stop Words".
The function should modify the input dataframe.
The function should return the modified dataframe.
Here is the twitter dataframe:
twitter_url = 'https://raw.githubusercontent.com/Explore-AI/Public-Data/master/Data/twitter_nov_2019.csv'
twitter_df = pd.read_csv(twitter_url)
twitter_df.head()
Here are the 'stop_words' in a dictionary:
stop_words_dict = {
'stopwords':[
'where', 'done', 'if', 'before', 'll', 'very', 'keep', 'something', 'nothing', 'thereupon',
'may', 'why', '’s', 'therefore', 'you', 'with', 'towards', 'make', 'really', 'few', 'former',
'during', 'mine', 'do', 'would', 'of', 'off', 'six', 'yourself', 'becoming', 'through',
'seeming', 'hence', 'us', 'anywhere', 'regarding', 'whole', 'down', 'seem', 'whereas', 'to',
'their', 'various', 'thereafter', '‘d', 'above', 'put', 'sometime', 'moreover', 'whoever', 'although',
'at', 'four', 'each', 'among', 'whatever', 'any', 'anyhow', 'herein', 'become', 'last', 'between', 'still',
'was', 'almost', 'twelve', 'used', 'who', 'go', 'not', 'enough', 'well', '’ve', 'might', 'see', 'whose',
'everywhere', 'yourselves', 'across', 'myself', 'further', 'did', 'then', 'is', 'except', 'up', 'take',
'became', 'however', 'many', 'thence', 'onto', '‘m', 'my', 'own', 'must', 'wherein', 'elsewhere', 'behind',
'becomes', 'alone', 'due', 'being', 'neither', 'a', 'over', 'beside', 'fifteen', 'meanwhile', 'upon', 'next',
'forty', 'what', 'less', 'and', 'please', 'toward', 'about', 'below', 'hereafter', 'whether', 'yet', 'nor',
'against', 'whereupon', 'top', 'first', 'three', 'show', 'per', 'five', 'two', 'ourselves', 'whenever',
'get', 'thereby', 'noone', 'had', 'now', 'everyone', 'everything', 'nowhere', 'ca', 'though', 'least',
'so', 'both', 'otherwise', 'whereby', 'unless', 'somewhere', 'give', 'formerly', '’d', 'under',
'while', 'empty', 'doing', 'besides', 'thus', 'this', 'anyone', 'its', 'after', 'bottom', 'call',
'n’t', 'name', 'even', 'eleven', 'by', 'from', 'when', 'or', 'anyway', 'how', 'the', 'all',
'much', 'another', 'since', 'hundred', 'serious', '‘ve', 'ever', 'out', 'full', 'themselves',
'been', 'in', "'d", 'wherever', 'part', 'someone', 'therein', 'can', 'seemed', 'hereby', 'others',
"'s", "'re", 'most', 'one', "n't", 'into', 'some', 'will', 'these', 'twenty', 'here', 'as', 'nobody',
'also', 'along', 'than', 'anything', 'he', 'there', 'does', 'we', '’ll', 'latterly', 'are', 'ten',
'hers', 'should', 'they', '‘s', 'either', 'am', 'be', 'perhaps', '’re', 'only', 'namely', 'sixty',
'made', "'m", 'always', 'those', 'have', 'again', 'her', 'once', 'ours', 'herself', 'else', 'has', 'nine',
'more', 'sometimes', 'your', 'yours', 'that', 'around', 'his', 'indeed', 'mostly', 'cannot', '‘ll', 'too',
'seems', '’m', 'himself', 'latter', 'whither', 'amount', 'other', 'nevertheless', 'whom', 'for', 'somehow',
'beforehand', 'just', 'an', 'beyond', 'amongst', 'none', "'ve", 'say', 'via', 'but', 'often', 're', 'our',
'because', 'rather', 'using', 'without', 'throughout', 'on', 'she', 'never', 'eight', 'no', 'hereupon',
'them', 'whereafter', 'quite', 'which', 'move', 'thru', 'until', 'afterwards', 'fifty', 'i', 'itself', 'n‘t',
'him', 'could', 'front', 'within', '‘re', 'back', 'such', 'already', 'several', 'side', 'whence', 'me',
'same', 'were', 'it', 'every', 'third', 'together'
]
}
Here is the code I have tried writing:
def stop_words_remover(df):
df['With Stop Words'] = df['Tweets'].str.split()
df['With Stop Words']
stop_words = stop_words_dict.values()
stop_words
df['Without Stop Words'] = df['With Stop Words'].replace(stop_words, '')
df = df[['Tweets', 'Date', 'Without Stop Words']]
return df
stop_words_remover(twitter_df.copy())
This is the output i got
TypeError Traceback (most recent call last)
C:\Users\DATASC~1\AppData\Local\Temp/ipykernel_5696/4217028502.py in <module>
15
16
---> 17 stop_words_remover(twitter_df.copy())
18 ### END FUNCTION
C:\Users\DATASC~1\AppData\Local\Temp/ipykernel_5696/4217028502.py in stop_words_remover(df)
4 stop_words = stop_words_dict.values()
5
----> 6 df['Without Stop Words'] = df['With Stop Words'].replace(stop_words, '', stop_words())
7
8 df = df[['Tweets', 'Date', 'Without Stop Words']]
TypeError: 'dict_values' object is not callable
This is the expected output
stop_words_remover(twitter_df.copy())
Tweets Date Without Stop Words
0 #BongaDlulane Please send an email to mediades... 2019-11-29 12:50:54 [#bongadlulane, send, email, mediadesk#eskom.c...
1 #saucy_mamiie Pls log a call on 0860037566 2019-11-29 12:46:53 [#saucy_mamiie, pls, log, 0860037566]
2 #BongaDlulane Query escalated to media desk. 2019-11-29 12:46:10 [#bongadlulane, query, escalated, media, desk.]
3 Before leaving the office this afternoon, head... 2019-11-29 12:33:36 [leaving, office, afternoon,, heading, weekend...
4 #ESKOMFREESTATE #MEDIASTATEMENT : ESKOM SUSPEN... 2019-11-29 12:17:43 [#eskomfreestate, #mediastatement, :, eskom, s...
... ... ... ...
195 Eskom's Visitors Centres’ facilities include i... 2019-11-20 10:29:07 [eskom's, visitors, centres’, facilities, incl...
196 #Eskom connected 400 houses and in the process... 2019-11-20 10:25:20 [#eskom, connected, 400, houses, process, conn...
197 #ArthurGodbeer Is the power restored as yet? 2019-11-20 10:07:59 [#arthurgodbeer, power, restored, yet?]
198 #MuthambiPaulina #SABCNewsOnline #IOL #eNCA #e... 2019-11-20 10:07:41 [#muthambipaulina, #sabcnewsonline, #iol, #enc...
199 RT #GP_DHS: The #GautengProvince made a commit... 2019-11-20 10:00:09 [rt, #gp_dhs:, #gautengprovince, commitment, e...
Please can someone help me?

there a simple way to do this in a single command using apply lambda:
twitter_df["Tweets"].apply(lambda x: " ".join([word for word in x.split() if word not in stop_words_dict["stopwords"]]))
If you prefer create a function to do this, the function could be:
def remove_stop_words(tweet, stop_words_dict):
sentence = tweet.split()
output = []
for word in sentence:
if word not in stop_words_dict["stopwords"]:
output.append(word)
return " ".join(output)
twitter_df["Tweets"].apply(lambda x: remove_stop_words(x, stop_words_dict))

How to find keywords in a text file using python's sklearn

I want to create a way to optimize my resume using a python script. To do this, I am trying to find keywords used in the job listing that I can add to my resume to make it stand out when it is run through ATS. Currently, I am using the following code to find what percent match my resume is for the job. How can I use this comparison and find how to improve my resume with specific keywords from the job listing?
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
resume = open("resume.txt", encoding='latin-1')
reference = open("reference.txt", encoding='latin-1')
compare = [resume.read(),reference.read()]
cMatrix = CountVectorizer().fit_transform(compare)
#prints how well the resume matches as a percentage
matPercent = cosine_similarity(cMatrix)[0][1] * 100
matPercent = round(matPercent, 2) # round to two decimal
print("Resume is a "+ str(matPercent)+ "% match to the job.")
I am using the following to generate keywords, however, this omits important words and is a long list that I think could be optimized better using sklearn. Instead of using FindKeywords(), how can I access information from the CountVectorizer().fit_transform(compare)
def FindKeywords():
file = open("reference.txt", encoding='latin-1')
string = file.read().replace("\n", " ").replace("\t", " ").lower()
kwDict = {}
avoidables = set(['skilled','skills','skill','minimum','tools','work','features','looking','highly','', ' ','','the', 'of', 'to', 'and', 'a', 'in', 'is', 'it', 'you', 'that', 'he', 'was', 'for', 'on', 'are', 'with', 'as', 'I', 'his', 'they', 'be', 'at', 'one', 'have', 'this', 'from', 'or', 'had', 'by', 'not', 'word', 'but', 'what', 'some', 'we', 'can', 'out', 'other', 'were', 'all', 'there', 'when', 'up', 'use', 'your', 'how', 'said', 'an', 'each', 'she', 'which', 'do', 'their', 'time', 'if', 'will', 'way', 'about', 'many', 'then', 'them', 'write', 'would', 'like', 'so', 'these', 'her', 'long', 'make', 'thing', 'see', 'him', 'two', 'has', 'look', 'more', 'day', 'could', 'go', 'come', 'did', 'number', 'sound', 'no', 'most', 'people', 'my', 'over', 'know', 'water', 'than', 'call', 'first', 'who', 'may', 'down', 'side', 'been', 'now', 'find', 'any', 'new', 'work', 'part', 'take', 'get', 'place', 'made', 'live', 'where', 'after', 'back', 'little', 'only', 'round', 'man', 'year', 'came', 'show', 'every', 'good', 'me', 'give', 'our', 'under', 'name', 'very', 'through', 'just', 'form', 'sentence', 'great', 'think', 'say', 'help', 'low', 'line', 'differ', 'turn', 'cause', 'much', 'mean', 'before', 'move', 'right', 'boy', 'old', 'too', 'same', 'tell', 'does', 'set', 'three', 'want', 'air', 'well', 'also', 'play', 'small', 'end', 'put', 'home', 'read', 'hand', 'port', 'large', 'spell', 'add', 'even', 'land', 'here', 'must', 'big', 'high', 'such', 'follow', 'act', 'why', 'ask', 'men', 'change', 'went', 'light', 'kind', 'off', 'need', 'house', 'picture', 'try', 'us', 'again', 'animal', 'point', 'mother', 'world', 'near', 'build', 'self', 'earth', 'father', 'head', 'stand', 'own', 'page', 'should', 'country', 'found', 'answer', 'school', 'grow', 'study', 'still', 'learn', 'plant', 'cover', 'food', 'sun', 'four', 'between', 'state', 'keep', 'eye', 'never', 'last', 'let', 'thought', 'city', 'tree', 'cross', 'farm', 'hard', 'start', 'might', 'story', 'saw', 'far', 'sea', 'draw', 'left', 'late', 'run', "don't", 'while', 'press', 'close', 'night', 'real', 'life', 'few', 'north', 'open', 'seem', 'together', 'next', 'white', 'children', 'begin', 'got', 'walk', 'example', 'ease', 'paper', 'group', 'always', 'music', 'those', 'both', 'mark', 'often', 'letter', 'until', 'mile', 'river', 'car', 'feet', 'care', 'second', 'book', 'carry', 'took', 'science', 'eat', 'room', 'friend', 'began', 'idea', 'fish', 'mountain', 'stop', 'once', 'base', 'hear', 'horse', 'cut', 'sure', 'watch', 'color', 'face', 'wood', 'main', 'enough', 'plain', 'girl', 'usual', 'young', 'ready', 'above', 'ever', 'red', 'list', 'though', 'feel', 'talk', 'bird', 'soon', 'body', 'dog', 'family', 'direct', 'pose', 'leave', 'song', 'measure', 'door', 'product', 'black', 'short', 'numeral', 'class', 'wind', 'question', 'happen', 'complete', 'ship', 'area', 'half', 'rock', 'order', 'fire', 'south', 'problem', 'piece', 'told', 'knew', 'pass', 'since', 'top', 'whole', 'king', 'space', 'heard', 'best', 'hour', 'better', 'true', 'during', 'hundred', 'five', 'remember', 'step', 'early', 'hold', 'west', 'ground', 'interest', 'reach', 'fast', 'verb', 'sing', 'listen', 'six', 'table', 'travel', 'less', 'morning', 'ten', 'simple', 'several', 'vowel', 'toward', 'war', 'lay', 'against', 'pattern', 'slow', 'center', 'love', 'person', 'money', 'serve', 'appear', 'road', 'map', 'rain', 'rule', 'govern', 'pull', 'cold', 'notice', 'voice', 'unit', 'power', 'town', 'fine', 'certain', 'fly', 'fall', 'lead', 'cry', 'dark', 'machine', 'note', 'wait', 'plan', 'figure', 'star', 'box', 'noun', 'field', 'rest', 'correct', 'able', 'pound', 'done', 'beauty', 'drive', 'stood', 'contain', 'front', 'teach', 'week', 'final', 'gave', 'green', 'oh', 'quick', 'develop', 'ocean', 'warm', 'free', 'minute', 'strong', 'special', 'mind', 'behind', 'clear', 'tail', 'produce', 'fact', 'street', 'inch', 'multiply', 'nothing', 'course', 'stay', 'wheel', 'full', 'force', 'blue', 'object', 'decide', 'surface', 'deep', 'moon', 'island', 'foot', 'system', 'busy', 'test', 'record', 'boat', 'common', 'gold', 'possible', 'plane', 'stead', 'dry', 'wonder', 'laugh', 'thousand', 'ago', 'ran', 'check', 'game', 'shape', 'equate', 'hot', 'miss', 'brought', 'heat', 'snow', 'tire', 'bring', 'yes', 'distant', 'fill', 'east', 'paint', 'language', 'among', 'grand', 'ball', 'yet', 'wave', 'drop', 'heart', 'am', 'present', 'heavy', 'dance', 'engine', 'position', 'arm', 'wide', 'sail', 'material', 'size', 'vary', 'settle', 'speak', 'weight', 'general', 'ice', 'matter', 'circle', 'pair', 'include', 'divide', 'syllable', 'felt', 'perhaps', 'pick', 'sudden', 'count', 'square', 'reason', 'length', 'represent', 'art', 'subject', 'region', 'energy', 'hunt', 'probable', 'bed', 'brother', 'egg', 'ride', 'cell', 'believe', 'fraction', 'forest', 'sit', 'race', 'window', 'store', 'summer', 'train', 'sleep', 'prove', 'lone', 'leg', 'exercise', 'wall', 'catch', 'mount', 'wish', 'sky', 'board', 'joy', 'winter', 'sat', 'written', 'wild', 'instrument', 'kept', 'glass', 'grass', 'cow', 'job', 'edge', 'sign', 'visit', 'past', 'soft', 'fun', 'bright', 'gas', 'weather', 'month', 'million', 'bear', 'finish', 'happy', 'hope', 'flower', 'clothe', 'strange', 'gone', 'jump', 'baby', 'eight', 'village', 'meet', 'root', 'buy', 'raise', 'solve', 'metal', 'whether', 'push', 'seven', 'paragraph', 'third', 'shall', 'held', 'hair', 'describe', 'cook', 'floor', 'either', 'result', 'burn', 'hill', 'safe', 'cat', 'century', 'consider', 'type', 'law', 'bit', 'coast', 'copy', 'phrase', 'silent', 'tall', 'sand', 'soil', 'roll', 'temperature', 'finger', 'industry', 'value', 'fight', 'lie', 'beat', 'excite', 'natural', 'view', 'sense', 'ear', 'else', 'quite', 'broke', 'case', 'middle', 'kill', 'son', 'lake', 'moment', 'scale', 'loud', 'spring', 'observe', 'child', 'straight', 'consonant', 'nation', 'dictionary', 'milk', 'speed', 'method', 'organ', 'pay', 'age', 'section', 'dress', 'cloud', 'surprise', 'quiet', 'stone', 'tiny', 'climb', 'cool', 'design', 'poor', 'lot', 'experiment', 'bottom', 'key', 'iron', 'single', 'stick', 'flat', 'twenty', 'skin', 'smile', 'crease', 'hole', 'trade', 'melody', 'trip', 'office', 'receive', 'row', 'mouth', 'exact', 'symbol', 'die', 'least', 'trouble', 'shout', 'except', 'wrote', 'seed', 'tone', 'join', 'suggest', 'clean', 'break', 'lady', 'yard', 'rise', 'bad', 'blow', 'oil', 'blood', 'touch', 'grew', 'cent', 'mix', 'team', 'wire', 'cost', 'lost', 'brown', 'wear', 'garden', 'equal', 'sent', 'choose', 'fell', 'fit', 'flow', 'fair', 'bank', 'collect', 'save', 'control', 'decimal', 'gentle', 'woman', 'captain', 'practice', 'separate', 'difficult', 'doctor', 'please', 'protect', 'noon', 'whose', 'locate', 'ring', 'character', 'insect', 'caught', 'period', 'indicate', 'radio', 'spoke', 'atom', 'human', 'history', 'effect', 'electric', 'expect', 'crop', 'modern', 'element', 'hit', 'student', 'corner', 'party', 'supply', 'bone', 'rail', 'imagine', 'provide', 'agree', 'thus', 'capital', "won't", 'chair', 'danger', 'fruit', 'rich', 'thick', 'soldier', 'process', 'operate', 'guess', 'necessary', 'sharp', 'wing', 'create', 'neighbor', 'wash', 'bat', 'rather', 'crowd', 'corn', 'compare', 'poem', 'string', 'bell', 'depend', 'meat', 'rub', 'tube', 'famous', 'dollar', 'stream', 'fear', 'sight', 'thin', 'triangle', 'planet', 'hurry', 'chief', 'colony', 'clock', 'mine', 'tie', 'enter', 'major', 'fresh', 'search', 'send', 'yellow', 'gun', 'allow', 'print', 'dead', 'spot', 'desert', 'suit', 'current', 'lift', 'rose', 'continue', 'block', 'chart', 'hat', 'sell', 'success', 'company', 'subtract', 'event', 'particular', 'deal', 'swim', 'term', 'opposite', 'wife', 'shoe', 'shoulder', 'spread', 'arrange', 'camp', 'invent', 'cotton', 'born', 'determine', 'quart', 'nine', 'truck', 'noise', 'level', 'chance', 'gather', 'shop', 'stretch', 'throw', 'shine', 'property', 'column', 'molecule', 'select', 'wrong', 'gray', 'repeat', 'require', 'broad', 'prepare', 'salt', 'nose', 'plural', 'anger', 'claim', 'continent', 'oxygen', 'sugar', 'death', 'pretty', 'skill', 'women', 'season', 'solution', 'magnet', 'silver', 'thank', 'branch', 'match', 'suffix', 'especially', 'fig', 'afraid', 'huge', 'sister', 'steel', 'discuss', 'forward', 'similar', 'guide', 'experience', 'score', 'apple', 'bought', 'led', 'pitch', 'coat', 'mass', 'card', 'band', 'rope', 'slip', 'win', 'dream', 'evening', 'condition', 'feed', 'tool', 'total', 'basic', 'smell', 'valley', 'nor', 'double', 'seat', 'arrive', 'master', 'track', 'parent', 'shore', 'division', 'sheet', 'substance', 'favor', 'connect', 'post', 'spend', 'chord', 'fat', 'glad', 'original', 'share', 'station', 'dad', 'bread', 'charge', 'proper', 'bar', 'offer', 'segment', 'slave', 'duck', 'instant', 'market', 'degree', 'populate', 'chick', 'dear', 'enemy', 'reply', 'drink', 'occur', 'support', 'speech', 'nature', 'range', 'steam', 'motion', 'path', 'liquid', 'log', 'meant', 'quotient', 'teeth', 'shell', 'neck'])
for word in string.split(' '):
if word not in kwDict and word not in avoidables:
kwDict[word] = 1
elif word not in avoidables:
kwDict[word] += 1
returns = [key for key in kwDict.keys() if kwDict[key]>0]
return [kw for kw in returns if kw not in avoidables]

You can use the get_feature_names() method from the CountVectorizer as documented here.
So with a concrete example from your code (adjusted a bit), it could look like this:
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
resume = "This is an example resume for a job"
reference = "This is an example reference for a job advertisement"
compare = [resume,reference]
cVect = CountVectorizer()
cMatrix = cVect.fit_transform(compare)
#prints how well the resume matches as a percentage
matPercent = cosine_similarity(cMatrix)[0][1] * 100
matPercent = round(matPercent, 2) # round to two decimal
print("Resume is a "+ str(matPercent)+ "% match to the job.")
Returns:
Resume is a 80.18% match to the job.
Then to get the keywords:
cVect.get_feature_names()
The returned keywords:
['advertisement',
'an',
'example',
'for',
'is',
'job',
'reference',
'resume',
'this']
If you would want only keywords from your resume or reference without the other, then you can just fit_transform() another CountVectorizer() just on that data and then get the keywords from that.
The important thing to keep in mind, is that you need to 'save' your trained CountVectorizer, so instead of
CountVectorizer().fit_transform(compare)
You need to use
cVect = CountVectorizer()
cVect.fit_transform(compare)
So that you can later still access your CountVectorizer() instance.

How to print a list of tokenized text into a file

from urllib import request
from redditscore.tokenizer import CrazyTokenizer
tokenizer = CrazyTokenizer()
url = "http://www.site.uottawa.ca/~diana/csi5386/A1_2020/microblog2011.txt"
for line in request.urlopen(url):
tokens = tokenizer.tokenize(line.decode('utf-8'))
#print(tokens)
with open('your_file.txt', 'a') as f:
print(tokens)
for item in tokens:
f.write("%s\n" % item)
In the above code my output is in variable tokens in the form of list.
Output sample:
['\ufeffsave', 'bbc', 'world', 'service', 'from', 'savage', 'cuts']
['a', 'lot', 'of', 'people', 'always', 'make', 'fun', 'about', 'the', 'end', 'of', 'the', 'world', 'but', 'the', 'question', 'is', '"are', 'u', 'ready', 'for', 'it']
['rethink', 'group', 'positive', 'in', 'outlook', 'technology', 'staffing', 'specialist', 'the', 'rethink', 'group', 'expects', 'revenues', 'to', 'be']
Now i'm trying to print this output into a text file.
How can i do that? please help..

with open('your_file.txt', 'a') as f:
for line in request.urlopen(url):
tokens = tokenizer.tokenize(line.decode('utf-8'))
#print(tokens)
for item in tokens:
f.write("%s\n" % item)

Just use ' '.join with each token item
like the following (I am assuming that I already have the data in an array):
tokens = [
['\ufeffsave', 'bbc', 'world', 'service', 'from', 'savage', 'cuts'],
['a', 'lot', 'of', 'people', 'always', 'make', 'fun', 'about', 'the', 'end',
'of', 'the', 'world', 'but', 'the', 'question', 'is', '"are', 'u', 'ready',
'for', 'it'],
['rethink', 'group', 'positive', 'in', 'outlook', 'technology', 'staffing',
'specialist', 'the', 'rethink', 'group', 'expects', 'revenues', 'to', 'be']
]
with open('your_file.txt', 'a') as f:
print(tokens)
for item in tokens:
f.write("%s\n" % ' '.join(item))

Tokenizing tweets in Python

enneg3clear.txt is a file with Tweets without punctuation and stopwords on every line.
import re, string
import sys
#this code tokenizes
input_file = 'enneg3clear.txt'
with open(input_file) as f:
lines = f.readlines()
results = []
texts = []
for line in lines:
texts = ([word for word in line.lower().split()])
results.append(texts)
print results
[['\xef\xbb\xbfmy', 'good', 'sis', 'kelly', 'bouta', 'compete', 'with', 'adele', 'that', 'over', 'weinvm'], ['going', 'miss', 'japppaaannnnn'], ['its', 'so', 'hard', 'get', 'out', 'bed', 'morning', 'vote5sos', 'kca'
], ['police', 'fatally', 'shoot', 'homeless', 'man', 'losangeles', 'gtgt'], ['my', 'trumpet', 'has', 'been', 'idle', 'days', 'now'], ['mercenaries', 'was', 'game', 'i', 'lent', 'friend', 'never', 'saw', 'again'], ['
yeah', 'i', 'miss', 'you', 'all', 'so', 'much', 'already'], ['acabou', 'talitaaraujonomaisvoce'], ['im', 'at', 'strain', 'station', 'waiting', 'train', 'arrive', 'sigh', 'im', 'sooo', 'tired']]
#remove words that appear only once
all_tokens = sum(results, [])
tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)
print tokens_once
set(['all', 'already', 'tired', 'sigh', 'over', 'hard', 'sooo', 'yeah', 'strain', '\xef\xbb\xbfmy', 'japppaaannnnn', 'adele', 'at', 'homeless', 'trumpet', 'its', 'out', 'sis', 'again', 'police', 'vote5sos', 'gtgt',
'saw', 'that', 'idle', 'been', 'mercenaries', 'waiting', 'station', 'you', 'has', 'was', 'friend', 'losangeles', 'kca', 'get', 'never', 'much', 'game', 'train', 'lent', 'now', 'with', 'bouta', 'man', 'shoot', 'going
', 'talitaaraujonomaisvoce', 'fatally', 'days', 'bed', 'morning', 'weinvm', 'good', 'compete', 'acabou', 'kelly', 'arrive', 'my'])
results = [[word for word in results if word not in tokens_once]]
print (results)
File "atokenize.py", line 25, in <module>
results = [[word for word in results if word not in tokens_once]]
TypeError: unhashable type: 'list'
So the error can be found in the second line started from the bottom. Any idea how to solve this?

your results contain list of lists.So you have to flatten it.
So simply put
results = [j for i in results for j in i]
above the line
results = [[word for word in results if word not in tokens_once]]
OR another solution
change append to extend
for line in lines:
texts = ([word for word in line.lower().split()])
results.extend(texts) # or results += texts