My list is formatted like:
gymnastics_school,participant_name,all-around_points_earned
I need to divide it up by schools but keep the scores.
import collections
def main():
names = ["gymnastics_school", "participant_name", "all_around_points_earned"]
Data = collections.namedtuple("Data", names)
data = []
with open('state_meet.txt','r') as f:
for line in f:
line = line.strip()
items = line.split(',')
items[2] = float(items[2])
data.append(Data(*items))
These are examples of how they're set up:
Lanier City Gymnastics,Ben W.,55.301
Lanier City Gymnastics,Alex W.,54.801
Lanier City Gymnastics,Sky T.,51.2
Lanier City Gymnastics,William G.,47.3
Carrollton Boys,Cameron M.,61.6
Carrollton Boys,Zachary W.,58.7
Carrollton Boys,Samuel B.,58.6
La Fayette Boys,Nate S.,63
La Fayette Boys,Kaden C.,62
La Fayette Boys,Cohan S.,59.1
La Fayette Boys,Cooper J.,56.101
La Fayette Boys,Avi F.,53.401
La Fayette Boys,Frederic T.,53.201
Columbus,Noah B.,50.3
Savannah Metro,Levi B.,52.801
Savannah Metro,Taylan T.,52
Savannah Metro,Jacob S.,51.5
SAAB Gymnastics,Dawson B.,58.1
SAAB Gymnastics,Dean S.,57.901
SAAB Gymnastics,William L.,57.101
SAAB Gymnastics,Lex L.,52.501
Suwanee Gymnastics,Colin K.,57.3
Suwanee Gymnastics,Matthew B.,53.201
After processing it should look like:
Lanier City Gymnastics:participants(4)
as it own list
Carrollton Boys(3)
as it own list
La Fayette Boys(6)
etc.
I would recommend putting them in dictionaries:
data = {}
with open('state_meet.txt','r') as f:
for line in f:
line = line.strip()
items = line.split(',')
items[2] = float(items[2])
if items[0] in data:
data[items[0]].append(items[1:])
else:
data[items[0]] = [items[1:]]
Then access schools could be done in the following way:
>>> data['Lanier City Gymnastics']
[['Ben W.',55.301],['Alex W.',54.801],['Sky T'.,51.2],['William G.',47.3]
EDIT:
Assuming you need the whole dataset as a list first, then you want to divide it into smaller lists you can generate the dictionary from the list:
data = []
with open('state_meet.txt','r') as f:
for line in f:
line = line.strip()
items = line.split(',')
items[2] = float(items[2])
data.append(items)
#perform median or other operation on your data
nested_data = {}
for items in data:
if items[0] in data:
data[items[0]].append(items[1:])
else:
data[items[0]] = [items[1:]]
nested_data[item[0]]
When you need to get a subset of a list you can use slicing:
mylist[start:stop:step]
where start, stop and step are optional (see link for more comprehensive introduction)
Related
I'm trying to retrieve wikipedia pages' characters count, for articles in different languages. I'm using a dictionary with as key the name of the page and as value a dictionary with the language as key and the count as value.
The code is:
pages = ["L'arte della gioia", "Il nome della rosa"]
langs = ["it", "en"]
dicty = {}
dicto ={}
numz = 0
for x in langs:
wikipedia.set_lang(x)
for y in pages:
pagelang = wikipedia.page(y)
splittedpage = pagelang.content
dicto[y] = dicty
for char in splittedpage:
numz +=1
dicty[x] = numz
If I print dicto, I get
{"L'arte della gioia": {'it': 72226, 'en': 111647}, 'Il nome della rosa': {'it': 72226, 'en': 111647}}
The count should be different for the two pages.
Please try this code. I didn't run it because I don't have the wikipedia module.
Notes:
Since your expected result is dict[page,dict[lan,cnt]], I think first iterate pages is more natural, then iterate languages. Maybe for performance reason you want first iterate languages, please comment.
Characters count of text can simply be len(text), why iterate and sum again?
Variable names. You will soon be lost in x y like variables.
pages = ["L'arte della gioia", "Il nome della rosa"]
langs = ["it", "en"]
dicto = {}
for page in pages:
lang_cnt_dict = {}
for lang in langs:
wikipedia.set_lang(lang)
page_lang = wikipedia.page(page)
chars_cnt = len(pagelang.content)
lang_cnt_dict[lan] = chars_cnt
dicto[page] = lang_cnt_dict
print(dicto)
update
If you want iterate langs first
pages = ["L'arte della gioia", "Il nome della rosa"]
langs = ["it", "en"]
dicto = {}
for lang in langs:
wikipedia.set_lang(lang)
for page in pages:
page_lang = wikipedia.page(page)
chars_cnt = len(pagelang.content)
if page in dicto:
dicto[page][lang] = chars_cnt
else:
dicto[page] = {lang: chars_cnt}
print(dicto)
I have been practicing on iterating through dictionary and list in Python.
The source file is a csv document containing Country and Capital. It seems I had to go through 2 for loops for country_dict in order to produce the same print result for country_list and capital_list.
Is there a better way to do this in Python dictionary?
The code:
import csv
path = #Path_to_CSV_File
country_list=[]
capital_list=[]
country_dict={'Country':[],'Capital':[]}
with open(path, mode='r') as data:
for line in csv.DictReader(data):
locals().update(line)
country_dict['Country'].append(Country)
country_dict['Capital'].append(Capital)
country_list.append(Country)
capital_list.append(Capital)
i=14 #set pointer value to the 15th row in the csv document
#---------------------- Iterating through Dictionary using for loops---------------------------
if i >= (len(country_dict['Country'])-1):
print("out of bound")
for count1, element in enumerate(country_dict['Country']):
if count1==i:
print('Country = ' + element)
for count2, element in enumerate(country_dict['Capital']):
if count2==i:
print('Capital = ' + element)
#--------------------------------Direct print for list----------------------------------------
print('Country = ' + country_list[i] + '\nCapital = ' + capital_list[i])
The output:
Country = Djibouti
Capital = Djibouti (city)
Country = Djibouti
Capital = Djibouti (city)
The CSV file content:
Country,Capital
Algeria,Algiers
Angola,Luanda
Benin,Porto-Novo
Botswana,Gaborone
Burkina Faso,Ouagadougou
Burundi,Gitega
Cabo Verde,Praia
Cameroon,Yaounde
Central African Republic,Bangui
Chad,N'Djamena
Comoros,Moroni
"Congo, Democratic Republic of the",Kinshasa
"Congo, Republic of the",Brazzaville
Cote d'Ivoire,Yamoussoukro
Djibouti,Djibouti (city)
Egypt,Cairo
Equatorial Guinea,"Malabo (de jure), Oyala (seat of government)"
Eritrea,Asmara
Eswatini (formerly Swaziland),"Mbabane (administrative), Lobamba (legislative, royal)"
Ethiopia,Addis Ababa
Gabon,Libreville
Gambia,Banjul
Ghana,Accra
Guinea,Conakry
Guinea-Bissau,Bissau
Kenya,Nairobi
Lesotho,Maseru
Liberia,Monrovia
Libya,Tripoli
Madagascar,Antananarivo
Malawi,Lilongwe
Mali,Bamako
Mauritania,Nouakchott
Mauritius,Port Louis
Morocco,Rabat
Mozambique,Maputo
Namibia,Windhoek
Niger,Niamey
Nigeria,Abuja
Rwanda,Kigali
Sao Tome and Principe,São Tomé
Senegal,Dakar
Seychelles,Victoria
Sierra Leone,Freetown
Somalia,Mogadishu
South Africa,"Pretoria (administrative), Cape Town (legislative), Bloemfontein (judicial)"
South Sudan,Juba
Sudan,Khartoum
Tanzania,Dodoma
Togo,Lomé
Tunisia,Tunis
Uganda,Kampala
Zambia,Lusaka
Zimbabwe,Harare
I am not sure if I get your point; Please check out the code.
import csv
path = #Path_to_CSV_File
country_dict={}
with open(path, mode='r') as data:
lines = csv.DictReader(data)
for idx,line in enumerate(lines):
locals().update(line)
country_dict[idx] = {"Country":Country,"Capital":}
i=14 #set pointer value to the 15th row in the csv document
#---------------------- Iterating through Dictionary using for loops---------------------------
country_info = country_dict.get(i)
#--------------------------------Direct print for list----------------------------------------
print('Country = ' + country_info['Country'] + '\nCapital = ' + country_info['Capital'])
I have a list that contains pair of keywords ('k1', 'k2'). Here's a sample:
print (word_pairs)
--->[('salaire', 'dépense'), ('gratuité', 'argent'), ('causesmwedemwelamwemort', 'cadres'), ('caractèresmwedumwedispositif', 'historique'), ('psychomotricienmwediplôme', 'infirmier'), ('impôtmwesurmwelesmweréunionsmwesportives', 'compensation'), ('affichage', 'affichagemweopinion'), ('délaimweprorogation', 'défaillance'), ('créancemwenotion', 'généralités')]
I have a text file r_isa.txt (205MB) that contain words that share an "isa" relationship. Here's a sample, where \t represents a literal tab character:
égalité de Parseval\tformule_0.9333\tégalité_1.0
filiation illégitime\tfiliation_1.0
Loi reconnaissant l'égalité\tloi_1.0
égalité entre les sexes\tégalité_1.0
liberté égalité fraternité\tliberté_1.0
This basically means, "égalité de Parseval" isa "formule" with a score of 0.9333 and isa "égalité" with a score of 1. And so go on..
I want to know based on the r_isa file, if the keyword k1 isa k2, and if k2 is-a k1. On the output file, I want to save on each line the pair of words that do have the is-a relationship.
Here's what I did:
#Reading data as list
keywords = [line for line in open('version_final_PMI_espace.txt', encoding='utf8')]
keywords = ast.literal_eval(keywords[0])
word_pairs = []
for k,v in keywords.items():
if v:
word_pairs.append((k,v[0][0]))
len(list(set(word_pairs)))
#####
with open("r_isa.txt",encoding="utf-8") as readfile, open('Hyperonymy_file_pair.txt', 'w') as writefile:
for line in readfile:
firstfield = line.split('\t')[0].lower()
for w in word_pairs:
if w[0]==firstfield:
if w[1] in line:
writefile.write("".join(w[0]) + "\t"+"".join(w[1]) +"\n" )
This returns random pairs to me, for exemple:
salaire\targent
dépense\tcadres
unstead of ( in case of an existing isa relationship)
salaire\tdépense
causesmwedemwelamwemort\tcadres
Where did I go wrong ?
Updated Answer
The statement if w[1] in line: is highly suspect. See the following code for what I believe the logic should be. Since I don't have access to your files, I have turned readfile into a list of strings for testing purposes and instead of writing output to writefile, I am just printing some results. I have added some values to word_pairs and readfile so that I get some results. Also note that if you are converting the input file to lower case, then your word pairs must also be lower case.
This code checks if k1 isa k2 and if not, then checks if k2 isa k1.
word_pairs = [('égalité de parseval', 'égalité'), ('salaire', 'dépense'), ('gratuité', 'argent'), ('causesmwedemwelamwemort', 'cadres'), ('caractèresmwedumwedispositif', 'historique'), ('psychomotricienmwediplôme', 'infirmier'), ('impôtmwesurmwelesmweréunionsmwesportives', 'compensation'), ('affichage', 'affichagemweopinion'), ('délaimweprorogation', 'défaillance'), ('créancemwenotion', 'généralités')]
word_pairs2 = [(pair[1], pair[0]) for pair in word_pairs] # reverse the words
word_dict = dict(word_pairs) # create a dictionary for fast searching
word_dict2 = dict(word_pairs2)
readfile = [
'égalité de Parseval\tformule_0.9333\tégalité_1.0',
'filiation illégitime\tfiliation_1.0',
'Loi reconnaissant l\'égalité\tloi_1.0',
'égalité entre les sexes\tégalité_1.0',
'liberté égalité fraternité\tliberté_1.0',
'dépense\tsalaire_.9'
]
for line in readfile:
fields = line.lower().split('\t')
first_word = fields.pop(0)
isa_word = word_dict.get(first_word, word_dict2.get(first_word)) # check k2 isa k1 if k1 isa k2 is false
if isa_word is not None:
for field in fields: # check each one
fields2 = field.split('_')
second_word, score = fields2
if second_word == isa_word:
print(first_word, second_word, score)
Prints:
égalité de parseval égalité 1.0
dépense salaire .9
If it is possible that k1 isa k2 and k2 isa k1, then you need the more general (but more complicated) code:
word_pairs = [('égalité de parseval', 'égalité'), ('salaire', 'dépense'), ('gratuité', 'argent'), ('causesmwedemwelamwemort', 'cadres'), ('caractèresmwedumwedispositif', 'historique'), ('psychomotricienmwediplôme', 'infirmier'), ('impôtmwesurmwelesmweréunionsmwesportives', 'compensation'), ('affichage', 'affichagemweopinion'), ('délaimweprorogation', 'défaillance'), ('créancemwenotion', 'généralités')]
word_pairs2 = [(pair[1], pair[0]) for pair in word_pairs] # reverse the words
word_dict = dict(word_pairs) # create a dictionary for fast searching
word_dict2 = dict(word_pairs2)
readfile = [
'égalité de Parseval\tformule_0.9333\tégalité_1.0',
'filiation illégitime\tfiliation_1.0',
'Loi reconnaissant l\'égalité\tloi_1.0',
'égalité entre les sexes\tégalité_1.0',
'liberté égalité fraternité\tliberté_1.0',
'salaire\tdépense_1.0',
'dépense\tsalaire_.9'
]
for line in readfile:
fields = line.lower().split('\t')
first_word = fields.pop(0)
# k1 isa k2?
isa_word = word_dict.get(first_word)
if isa_word is not None:
for field in fields: # check each one
fields2 = field.split('_')
second_word, score = fields2
if second_word == isa_word:
print(first_word, second_word, score)
# k2 isa k1?
isa_word = word_dict2.get(first_word)
if isa_word is not None:
for field in fields: # check each one
fields2 = field.split('_')
second_word, score = fields2
if second_word == isa_word:
print(first_word, second_word, score)
Prints:
égalité de parseval égalité 1.0
salaire dépense 1.0
dépense salaire .9
kw = [('salaire', 'dépense'),
('gratuité', 'argent'),
('causesmwedemwelamwemort', 'cadres'),
('caractèresmwedumwedispositif', 'historique'),
('psychomotricienmwediplôme', 'infirmier'),
('impôtmwesurmwelesmweréunionsmwesportives', 'compensation'),
('affichage', 'affichagemweopinion'),
('délaimweprorogation', 'défaillance'),
('créancemwenotion', 'généralités')]
lines_from_file = ['égalité de Parseval\tformule_0.9333\tégalité_1.0',
'filiation illégitime\tfiliation_1.0',
'Loi reconnaissant l\'égalité\tloi_1.0',
'égalité entre les sexes\tégalité_1.0',
'liberté égalité fraternité\tliberté_1.0',
'créancemwenotion\tgénéralités_1.0',
'généralités\tcréancemwenotion_1.0']
who_is_who_dict = {}
for line in lines_from_file:
words = line.split('\t')
key = words[0]
other_words = [w.split('_')[0] for w in words[1:]]
if key in who_is_who_dict:
who_is_who_dict[key] = who_is_who_dict[key] + other_words
else:
who_is_who_dict[key] = other_words
pairs_to_write = []
for kw1, kw2 in kw:
if (kw1 in who_is_who_dict and kw2 in who_is_who_dict[kw1]
and kw2 in who_is_who_dict and kw1 in who_is_who_dict[kw2]):
pairs_to_write.append((kw1, kw2))
print(pairs_to_write)
output :
[('créancemwenotion', 'généralités')]
I have 2 CSV files. One with city name, population and humidity. In second cities are mapped to states. I want to get state-wise total population and average humidity. Can someone help? Here is the example:
CSV 1:
CityName,population,humidity
Austin,1000,20
Sanjose,2200,10
Sacramento,500,5
CSV 2:
State,city name
Ca,Sanjose
Ca,Sacramento
Texas,Austin
Would like to get output(sum population and average humidity for state):
Ca,2700,7.5
Texas,1000,20
The above solution doesn't work because dictionary will contain one one key value. i gave up and finally used a loop. below code is working, mentioned input too
csv1
state_name,city_name
CA,sacramento
utah,saltlake
CA,san jose
Utah,provo
CA,sanfrancisco
TX,austin
TX,dallas
OR,portland
CSV2
city_name population humidity
sacramento 1000 1
saltlake 300 5
san jose 500 2
provo 100 7
sanfrancisco 700 3
austin 2000 4
dallas 2500 5
portland 300 6
def mapping_within_dataframe(self, file1,file2,file3):
self.csv1 = file1
self.csv2 = file2
self.outcsv = file3
one_state_data = 0
outfile = csv.writer(open('self.outcsv', 'w'), delimiter=',')
state_city = read_csv(self.csv1)
city_data = read_csv(self.csv2)
all_state = list(set(state_city.state_name))
for one_state in all_state:
one_state_cities = list(state_city.loc[state_city.state_name == one_state, "city_name"])
one_state_data = 0
for one_city in one_state_cities:
one_city_data = city_data.loc[city_data.city_name == one_city, "population"].sum()
one_state_data = one_state_data + one_city_data
print one_state, one_state_data
outfile.writerows(whatever)
def output(file1, file2):
f = lambda x: x.strip() #strips newline and white space characters
with open(file1) as cities:
with open(file2) as states:
states_dict = {}
cities_dict = {}
for line in states:
line = line.split(',')
states_dict[f(line[0])] = f(line[1])
for line in cities:
line = line.split(',')
cities_dict[f(line[0])] = (int(f(line[1])) , int(f(line[2])))
for state , city in states_dict.iteritems():
try:
print state, cities_dict[city]
except KeyError:
pass
output(CSV1,CSV2) #these are the names of the files
This gives the output you wanted. Just make sure the names of cities in both files are the same in terms of capitalization.
I have two files and I need to compare both of them & update the value of the 1st file from the 2nd file.
My first file is as below,
SeqNo City State
1 Chicago IL
2 Boston MA
3 New York NY
4 Los Angeles CA
5 Seattle WA
My second file is as below,
SeqNo City State NewSeqNo
005 Seattle WA 001
001 Chicago IL 002
004 Los Angeles CA 003
002 Boston MA 004
003 New York NY 005
I have the following code to update the SEQ Number in the first file with the value in the NewSeqNo from the second file & save it as a third file. But it throws key error as SEQNO is zero padded in the second file where as its not in the first,
import csv
lookup = {}
with open('secondfile') as f:
reader = csv.reader(f)
for line in reader:
oldseq, city, state, newseq = line
lookup[oldseq] = newseq
with open('firstfile') as f, open('outfile','w') as w:
reader = csv.reader(f)
writer = csv.writer(w)
for line in reader:
seq, city, state = line
if seq in lookup:
seq = lookup[seq]
writer.writerow([seq, city, state])
For example, the output of the thirs file should be,
NewSeqNo City State
002 Chicago IL
004 Boston MA
005 New York NY
003 Los Angeles CA
001 Seattle WA
Any help is appreciated
Convert your 'numbers' to integers to remove the padding before storing in the dictionary:
import csv
lookup = {}
with open('secondfile') as f:
reader = csv.reader(f)
for line in reader:
oldseq, city, state, newseq = line
lookup[int(oldseq)] = newseq
with open('firstfile') as f, open('outfile','w') as w:
reader = csv.reader(f)
writer = csv.writer(w)
for line in reader:
seq, city, state = line
if int(seq) in lookup:
seq = lookup[int(seq)]
writer.writerow([seq, city, state])
Now lookup has integer keys, and when looking up matching keys in the second loop, we pass in integer keys again.
If you know that it is always padded for a length of 3, when reading your first file, you can convert your seq to an int and use format to write a padded value:
with open('firstfile') as f, open('outfile','w') as w:
reader = csv.reader(f)
writer = csv.writer(w)
for line in reader:
seq, city, state = line
# Convert to padded value
seq = "{:03}".format(int(seq))
if seq in lookup:
seq = lookup[seq]
writer.writerow([seq, city, state])
#!/usr/bin/python
old_dict = dict()
new_dict = dict()
with open('old', 'r') as fh:
for l in fh.readlines():
r = l.split()
if r:
old_dict.setdefault(int(r[0]), None)
old_dict[int(r[0])] = ' '.join(r[1:])
with open('new', 'r') as fh:
for l in fh.readlines():
r = l.split()
if r:
k = ' '.join(r[1:-1])
new_dict.setdefault(k, None)
new_dict[k] = int(r[-1])
for i,j in old_dict.iteritems():
d = j.split()
print '%0.3d %s %s' % (new_dict[j], ' '.join(d[0:-1]), d[-1])
Output:
002 Chicago IL
004 Boston MA
005 New York NY
003 Los Angeles CA
001 Seattle WA