Develop Reference

Select all string in list and append a string - python

I wanna add "." to the end of any item of doing variable.
and I want output like this => I am watching.
import random
main = ['I', 'You', 'He', 'She']
main0 = ['am', 'are', 'is', 'is']
doing = ['playing', 'watching', 'reading', 'listening']
rd = random.choice(main)
rd0 = random.choice(doing)
result = []
'''
if rd == main[0]:
result.append(rd)
result.append(main0[0])
if rd == main[1]:
result.append(rd)
result.append(main0[1])
if rd == main[2]:
result.append(rd)
result.append(main0[2])
if rd == main[3]:
result.append(rd)
result.append(main0[3])
'''
result.append(rd0)
print(result)
well, I tried those codes.
'.'.append(doing)
'.'.append(doing[0])
'.'.append(rd0)
but no one of them works, and only returns an error that:
Traceback (most recent call last):
File "c:\Users\----\Documents\Codes\Testing\s.py", line 21, in <module>
'.'.append(rd0)
AttributeError: 'str' object has no attribute 'append'

Why not just select a random string from each list and then build the output as you want:
import random
main = ['I', 'You', 'He', 'She']
verb = ['am', 'are', 'is', 'is']
doing = ['playing', 'watching', 'reading', 'listening']
p1 = random.choice(main)
p2 = random.choice(verb)
p3 = random.choice(doing)
output = ' '.join([p1, p2, p3]) + '.'
print(output) # You is playing.
Bad English, but the logic seems to be what you want here.

Just add a period after making the string:
import random
main = ['I', 'You', 'He', 'She']
main0 = ['am', 'are', 'is', 'is']
doing = ['playing', 'watching', 'reading', 'listening']
' '.join([random.choice(i) for i in [main, main0, doing]]) + '.'

Find most common word in a list of sets

I'm currently working in my university projects in NLP. I'd like to display the most common words contained in this list of sets:
[{'allow', 'feel', 'fear', 'situat', 'properti', 'despit', 'face', 'ani'}, {'unpleas', 'someth', 'fear', 'make', 'abil', 'face', 'scar', 'us', 'feel'}]
This is what I've accomplished until now:
def word_list(sent):
if isinstance(sent, str):
tokens = set(word_tokenize(sent.lower()))
else:
tokens = set([t for s in sent for t in word_tokenize(s.lower())])
tokens = set([stemmer.stem(t) for t in tokens])
for w in stopword_final:
tokens.discard(w)
return tokens
def get_most_relevant_words(definitions):
list_of_words = list()
most_common_word_dict = dict()
for d1 in definitions:
list_of_words.append(word_list(d1))
for elem in list_of_words:
for word in elem:
print(word)
word_counter = Counter(word)
most_occurrences = word_counter.most_common(3)
most_common_word_dict.update({word: most_occurrences})
return most_common_word_dict
The desired output should be: {fear: 2, feel: 2}
The output that it prints is: {'feel': [('e', 2), ('f', 1), ('l', 1)]}

Use collections.Counter:
from collections import Counter
list_of_sets = [{'allow', 'feel', 'fear', 'situat', 'properti', 'despit', 'face', 'ani'}, {'unpleas', 'someth', 'fear', 'make', 'abil', 'face', 'scar', 'us', 'feel'}]
words = [word for my_set in list_of_sets for word in my_set]
c = Counter(words)
print(c)
output:
Counter({
'fear': 2,
'face': 2,
'feel': 2,
'properti': 1,
'despit': 1,
'allow': 1,
'situat': 1,
'ani': 1,
'someth': 1,
'unpleas': 1,
'make': 1,
'abil': 1,
'us': 1,
'scar': 1
})

You can simply iterate through the 2 sets, find common terms, and update the count in a dictionary. By the way, 'face' should also be included in your result.
lst = [{'allow', 'feel', 'fear', 'situat', 'properti', 'despit', 'face', 'ani'}, {'unpleas', 'someth', 'fear', 'make', 'abil', 'face', 'scar', 'us', 'feel'}]
dic = {}
for word1 in lst[0]:
for word2 in lst[1]:
if word1 == word2:
dic[word1] = dic.get(word1, 0) + 2
print(dic)
#{'fear': 2, 'feel': 2, 'face': 2}

I want to find the length of my each words in text file

I'm trying to find the length of words individually in my text file. I tried it by following code but this code is showing me up the count of words that how many times this word is used in file.
text = open(r"C:\Users\israr\Desktop\counter\Bigdata.txt")
d = dict()
for line in text:
line = line.strip()
line = line.lower()
words = line.split(" ")
for word in words:
if word in d:
d[word] = d[word] + 1
else:
# Add the word to dictionary with count 1
d[word] = 1
for key in list(d.keys()):
print(key, ":", d[key])
And the output is something like this
china : 14
emerged : 1
as : 16
one : 5
of : 44
the : 108
world's : 7
first : 2
civilizations, : 1
in : 26
fertile : 1
basin : 1
yellow : 1
river : 1
north : 1
plain. : 1
Basically i want a list of words having same length for example china , first , world :5 this 5 is the length of all this words and so on the words having different length in other list

İf you need all word's total lengths seperatly, you can find them using this formula:
len(word) * count(word) for all word in words
equalivent in python:
d[key] * len(key)
Change last 2 lines with below:
for key in list(d.keys()):
print(key, ":", d[key] * len(key))
----EDIT----
It ıs what you asked in comments, I guess. Below code gives you groups whose members are the same length.
for word in words:
if len(word) in d:
if word not in d[len(word)]:
d[len(word)].append(word)
else:
# Add the word to dictionary with count 1
d[len(word)] = [word]
for key in list(d.keys()):
print(key, ":", d[key])
Output of this code:
3 : ['the', 'bc,', '(c.', 'who', 'was', '100', 'bc)', 'and', 'xia', 'but', 'not', 'one', 'due', '8th', '221', 'qin', 'shi', 'for', 'his', 'han', '220', '206', 'has', 'war', 'all', 'far']
8 : ['earliest', 'describe', 'writings', 'indicate', 'commonly', 'however,', 'cultural', 'history,', 'regarded', 'external', 'internal', 'culture,', 'troubled', 'imperial', 'selected', 'replaced', 'republic', 'mainland', "people's", 'peoples,', 'multiple', 'kingdoms', 'xinjiang', 'present.', '(carried']
5 : ['known', 'china', 'early', 'shang', 'texts', 'grand', 'ruled', 'river', 'which', 'along', 'these', 'arose', 'years', 'their', 'rule.', 'began', 'first', 'those', 'huang', 'title', 'after', 'until', '1912,', 'tasks', 'elite', 'young', '1949.', 'unity', 'being', 'civil', 'parts', 'other', 'world', 'waves', 'basis']
7 : ['written', 'records', 'history', 'dynasty', 'ancient', 'century', 'mention', 'writing', 'period,', 'xia.[5]', 'valley,', 'chinese', 'various', 'centers', 'yangtze', "world's", 'cradles', 'concept', 'mandate', 'justify', 'central', 'country', 'smaller', 'period.', 'another', 'warring', 'created', 'himself', 'huangdi', 'marking', 'systems', 'enabled', 'emperor', 'control', 'routine', 'handled', 'special', 'through', "china's", 'between', 'periods', 'culture', 'western', 'foreign']
2 : ['of', 'as', 'wu', 'by', 'no', 'is', 'do', 'in', 'to', 'be', 'at', 'or', 'bc', '21', 'ad']
4 : ['date', 'from', '1250', 'bc),', 'king', 'such', 'book', '11th', '(296', 'held', 'both', 'with', 'zhou', 'into', 'much', 'qin,', 'fell', 'soon', '(206', 'ad).', 'that', 'vast', 'were', 'men,', 'last', 'qing', 'then', 'most', 'whom', 'eras', 'have', 'some', 'asia', 'form']
9 : ['1600–1046', 'mentioned', 'documents', 'chapters,', 'historian', '2070–1600', 'existence', 'neolithic', 'millennia', 'thousands', '(1046–256', 'pressures', 'following', 'developed', 'conquered', '"emperor"', 'beginning', 'dynasties', 'directly.', 'centuries', 'carefully', 'difficult', 'political', 'dominated', 'stretched', 'contact),']
6 : ['during', "ding's", '(early', 'bamboo', 'annals', 'before', 'shang,', 'yellow', 'cradle', 'river.', 'shang.', 'oldest', 'heaven', 'weaken', 'states', 'spring', 'autumn', 'became', 'warred', 'times.', 'china.', 'death,', 'peace,', 'failed', 'recent', 'steppe', 'china;', 'tibet,', 'modern']
12 : ['reign,[1][2]', 'twenty-first', 'longer-lived', 'bureaucratic', 'calligraphy,', '(1644–1912),', '(1927–1949).', 'occasionally', 'immigration,']
11 : ['same.[3][4]', 'independent', 'traditional', 'territories', 'well-versed', 'literature,', 'philosophy,', 'assimilated', 'population.', 'warlordism,']
10 : ['historical', 'originated', 'continuous', 'supplanted', 'introduced', 'government', 'eventually', 'splintered', 'literature', 'philosophy', 'oppressive', 'successive', 'alternated', 'influences', 'expansion,']
1 : ['a', '–']
13 : ['civilization.', 'civilizations', 'examinations.', 'statehood—the', 'assimilation,']
17 : ['civilizations,[6]']
16 : ['civilization.[7]']
0 : ['']
14 : ['administrative']
18 : ['scholar-officials.']
Below is full version of code.
text = open("bigdata.txt")
d = dict()
for line in text:
line = line.strip()
line = line.lower()
words = line.split(" ")
for word in words:
if len(word) in d:
if word not in d[len(word)]:
d[len(word)].append(word)
else:
d[len(word)] = [word]
for key in list(d.keys()):
print(key, ":", d[key])

When you look at the code for dealing with each word, you will see your problem..
for word in words:
if word in d:
d[word] = d[word] + 1
else:
# Add the word to dictionary with count 1
d[word] = 1
Here you are checking if a word is in the dictionary. If it is, add 1 to its key when we find it. If it is not, initialize it at 1. This is the core concept for counting repetitions.
If you want to count the length of the word, you could simply do.
for word in words:
if word not in d:
d[word] = len(word)
And to output your dict, you can do
for k, v in d.items():
print(k, ":", v)

You can create a list of word lengths and then process them through python's built-in Counter:
from collections import Counter
with open("mytext.txt", "r") as f:
words = f.read().split()
words_lengths = [len(word) for word in words]
counter = Counter(words_lengths)
The output would be smth like:
In[1]:counter
Out[1]:Counter({7: 146, 9: 73, 5: 73, 4: 146, 1: 73})
Where keys are words lengths, and values are the number of times they occurred.
You can work with that as with usual dictionary.

AttributeError: 'list' object has no attribute 'isdigit'. Specifying POS of each and every word in sentences list efficiently?

Suppose I am having lists of list of sentences (in a large corpus) as collections of tokenized words. The sample format is as follows:
The format of tokenized_raw_data is as follows:
[['arxiv', ':', 'astro-ph/9505066', '.'], ['seds', 'page', 'on', '``',
'globular', 'star', 'clusters', "''", 'douglas', 'scott', '``', 'independent',
'age', 'estimates', "''", 'krysstal', '``', 'the', 'scale', 'of', 'the',
'universe', "''", 'space', 'and', 'time', 'scaled', 'for', 'the', 'beginner',
'.'], ['icosmos', ':', 'cosmology', 'calculator', '(', 'with', 'graph',
'generation', ')', 'the', 'expanding', 'universe', '(', 'american',
'institute', 'of', 'physics', ')']]
I want to apply the pos_tag.
What I have tried up to now is as follows.
import os, nltk, re
from nltk.corpus import stopwords
from unidecode import unidecode
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.tag import pos_tag
def read_data():
global tokenized_raw_data
with open("path//merge_text_results_pu.txt", 'r', encoding='utf-8', errors = 'replace') as f:
raw_data = f.read()
tokenized_raw_data = '\n'.join(nltk.line_tokenize(raw_data))
read_data()
def function1():
tokens_sentences = sent_tokenize(tokenized_raw_data.lower())
unfiltered_tokens = [[word for word in word_tokenize(word)] for word in tokens_sentences]
tagged_tokens = nltk.pos_tag(unfiltered_tokens)
nouns = [word.encode('utf-8') for word,pos in tagged_tokens
if (pos == 'NN' or pos == 'NNP' or pos == 'NNS' or pos == 'NNPS')]
joined_nouns_text = (' '.join(map(bytes.decode, nouns))).strip()
noun_tokens = [t for t in wordpunct_tokenize(joined_nouns_text)]
stop_words = set(stopwords.words("english"))
function1()
I am getting the following error.
> AttributeError: 'list' object has no attribute 'isdigit'
Please help how to overcome this error in time-efficient manner? Where I am going wrong?
Note: I am using Python 3.7 on Windows 10.

Try this-
word_list=[]
for i in range(len(unfiltered_tokens)):
word_list.append([])
for i in range(len(unfiltered_tokens)):
for word in unfiltered_tokens[i]:
if word[1:].isalpha():
word_list[i].append(word[1:])
then after do
tagged_tokens=[]
for token in word_list:
tagged_tokens.append(nltk.pos_tag(token))
You will get your desired results! Hope this helped.

Find the start of the index of a phrase in a list

Assuming I have this list:
text = ['Malte', 'ex', 'precio', 'empcionis', 'bovum', 'septem', 'laborancium', 'et', 'unius', 'thaurj', 'et', 'unius', 'vacce', 'cum', 'vitulo', 'sequenti', 'et', 'pecudum', 'fetancium', 'sexdecim', 'et', 'duarum', 'caprarum', 'cum', 'duobus', 'et', 'cum', 'vitulo']
And I want to find every index of the beginning of 'cum vitulo' ie: 13 and 26.
At the moment I am getting the start of 'cum' but sometimes this is followed by another word ex: 'duobus' in this case

One way to do it would be like this:
text = ['Malte', 'ex', 'precio', 'empcionis', 'bovum', 'septem', 'laborancium', 'et', 'unius', 'thaurj', 'et', 'unius', 'vacce', 'cum', 'vitulo', 'sequenti', 'et', 'pecudum', 'fetancium', 'sexdecim', 'et', 'duarum', 'caprarum', 'cum', 'duobus', 'et', 'cum', 'vitulo']
target = 'cum vitulo'
target = tuple(target.split())
hits = [i for i, x in enumerate(zip(text, text[1:])) if x == target]
print(hits) # -> [13, 26]

This is the simplest way IMHO (python 2.7 and 3):
text = ['Malte', 'ex', 'precio', 'empcionis', 'bovum', 'septem', 'laborancium', 'et', 'unius', 'thaurj', 'et', 'unius', 'vacce', 'cum', 'vitulo', 'sequenti', 'et', 'pecudum', 'fetancium', 'sexdecim', 'et', 'duarum', 'caprarum', 'cum', 'duobus', 'et', 'cum', 'vitulo'];
result = [i for i, item in enumerate(text[:-1]) if item == 'cum' and text[i+1] == 'vitulo']
print(result) # >>> [13, 26]

text = ['Malte', 'ex', 'precio', 'empcionis', 'bovum', 'septem', 'laborancium', 'et', 'unius', 'thaurj', 'et', 'unius', 'vacce', 'cum', 'vitulo', 'sequenti', 'et', 'pecudum', 'fetancium', 'sexdecim', 'et', 'duarum', 'caprarum', 'cum', 'duobus', 'et', 'cum', 'vitulo']
result = []
for index, value in enumerate(text):
if value == 'cum':
try:
if text[index + 1] == 'vitulo':
result.append(index)
except IndexError: # reached end
break
output
result == [13, 26]
A fancy way:
from itertools import tee
# recipe from itertools
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = tee(iterable)
next(b, None)
return zip(a, b)
[index for (index, (first, second)) in enumerate(pairwise(text)) if first == 'cum' and second == 'vitulo']