i am getting this error when trying to access a key value named 'valor' in from various JSON's that i am automatically generating from a website. I get different JSON's every iteration of for example the month code in its cycle. But the JSON's format, i.e. keys names, should be the same every JSON.
My code is:
year_codes = ["2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018", "2019", "2020"]
month_codes = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
city_codes = ["1111609", "1120303", "11A1312", "11D1714", "11E0402", "16D0105", "16E0603", "16F1009", "16G1823",
"16H0502", "16J0907", "1701106", "1701512", "1840205", "1851416", "1861214", "1870705", "1500805"]
valor_anual = [0]
for city_code in city_codes:
for year_code in year_codes:
valor_mensal_aux = []
for month_code in month_codes:
url_imob = Request(
"https://www.ine.pt/ine/json_indicador/pindica.jsp?op=2&varcd=0010042&Dim1=S3A" + year_code + \
month_code + "&Dim2=" + city_code + "&Dim3=T&lang=PT", headers={'User-Agent': 'XYZ/3.0'})
json_date = year_code + month_code
response = urlopen(url_imob)
data = json.loads(response.read())
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
city_name = data[0]['Dados'][json_date][0]['geodsg']
dua = data[0]['DataUltimoAtualizacao']
valor_anual = statistics.mean(valor_mensal_aux)
python_imob = Imob(int(year_code), valor_anual, dua, city_name)
session.add(python_imob)
session.commit()
session.close()
imobs = session.query(Imob).all()
and the JSON's look like is:
[ {
"IndicadorCod" : "0010042",
"IndicadorDsg" : "Valor mediano de avaliação bancária (€/ m²) por Localização geográfica (Município - 2013) e Tipo de construção; Mensal - INE, Inquérito à avaliação bancária na habitação",
"MetaInfUrl" : "https://www.ine.pt/bddXplorer/htdocs/minfo.jsp?var_cd=0010042&lingua=PT",
"DataExtracao" : "2020-06-29T15:55:51.640+01:00",
"DataUltimoAtualizacao" : "2020-06-29",
"UltimoPref" : "Maio de 2020",
"Dados" : {
"202005" : [ {
"geocod" : "1701106",
"geodsg" : "Lisboa",
"dim_3" : "T",
"dim_3_t" : "Total",
"valor" : "3084"
} ]
}
} ]
The KeyError: 'valor' is in this line:
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
Maybe the json_date is not in dados, let try
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
To be
dados_value = data[0]['Dados'].get(json_date, {}) ## get dados_value first
if dados_value and hasattr(dados_value[0], 'valor'): ## if dados_value has valor then append
valor_mensal_aux_value = dados_value and dados_value[0] and dados_value[0].get('valor') or None
if valor_mensal_aux_value:
valor_mensal_aux.append(int(valor_mensal_aux_value))
else: continue
else:
print(json_date, dados_value) # json_date with emply dados_value
continue ## this will skip the record does not have dados_value
More compact version
dados_value = data[0]['Dados'].get(json_date, {})
valor_mensal_aux_value = dados_value and dados_value[0] and dados_value[0].get('valor') or None
if valor_mensal_aux_value:
valor_mensal_aux.append(int(valor_mensal_aux_value))
else:
continue
Related
recently I used a vinted scraping bot on the vinted.fr site and it all worked well when I was on my local computer, but when I put the bot on my Debian VPS, the bot didn't work.
# Author : 2$py#5430
# Discord : discord.gg/spyy
# Github : https://github.com/2spy/
# Updated at : 04/07/2022 12:55
# Version : 4.3
# Description : Vinted Bot
# Language : Python
import os
import json
import shutil
import threading
import time
import sys
try:
import requests
from bs4 import BeautifulSoup
except:
os.system("pip install requests")
os.system("pip install bs4")
proxies = {
"https": "https://MyUser:MyPassword#199.247.15.159:12071",
}
class Spy:
gris = "\033[1;30;1m"
rouge = "\033[1;31;1m"
vert = "\033[1;32;1m"
jaune = "\033[1;33;1m"
bleu = "\033[1;34;1m"
violet = "\033[1;35;1m"
cyan = "\033[1;36;1m"
blanc = "\033[1;0;1m"
def get_info_post(url):
try:
time.sleep(2)
print(f"{Spy.blanc}[{Spy.jaune}RECHERCHE{Spy.blanc}] - Le bot recupere les informations de l'item...")
headers = requests.utils.default_headers()
headers.update({'User-agent': 'Mozilla/5.0'})
reponse = requests.get(str(url), headers=headers, proxies=proxies)
if 429 == reponse.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
soup = BeautifulSoup(reponse.text, "html.parser")
res = soup.findAll('script', {"class": "js-react-on-rails-component"})
descindice = 0
userinfoindice = 0
for i in range(len(res)):
if 'data-component-name="ItemDescription"' in str(res[i]).split():
descindice = i
if 'data-component-name="ItemUserInfo"' in str(res[i]).split():
userinfoindice = i
description = json.loads(res[descindice].text.replace(
'<script class="js-react-on-rails-component" data-component-name="ItemDescription" data-dom-id="ItemDescription-react-component-3d79657d-a1b5-4f1d-b501-2f470f328c66" type="application/json">',
"").replace("</script>", ''))
userinfo = json.loads(res[userinfoindice].text.replace(
'<script class="js-react-on-rails-component" data-component-name="ItemUserInfo" data-dom-id="ItemUserInfo-react-component-2105d904-b161-47d1-bfce-9b897a8c1cc6" type="application/json">',
'').replace("</script>", ''))
titre = description["content"]["title"]
description = description["content"]["description"]
positive = userinfo["user"]["positive_feedback_count"]
negative = userinfo["user"]["negative_feedback_count"]
username = userinfo["user"]["login"]
pays = userinfo["user"]["country_title"]
ville = userinfo["user"]["city"]
lesinfo = {}
if titre == "":
titre = "Pas de donnée"
if description == "":
description = "Pas de donnée"
if positive == "":
positive = "Pas de donnée"
if negative == "":
negative = "Pas de donnée"
if username == "":
username = "Pas de donnée"
if pays == "":
pays = "Pas de donnée"
if ville == "":
ville = "Pas de donnée"
try:
lesinfo["titre"] = titre
lesinfo["description"] = description
lesinfo["positive"] = positive
lesinfo["negative"] = negative
lesinfo["username"] = username
lesinfo["pays"] = pays
lesinfo["ville"] = ville
except Exception as err:
print(err)
return lesinfo
except:
pass
def search(url):
try:
time.sleep(5)
print(f"{Spy.blanc}[{Spy.gris}RECHERCHE{Spy.blanc}] - Le bot cherche des nouveaux items...")
headers = requests.utils.default_headers()
headers.update({'User-agent': 'Mozilla/5.0'})
reponse = requests.get(str(url), headers=headers, proxies=proxies)
if 429 == reponse.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
soup = BeautifulSoup(reponse.text, "html.parser")
res = soup.findAll('script')
indices = 0
for i in range(len(res) + 1):
if 'data-js-react-on-rails-store="MainStore"' in str(res[i]).split():
indices += i
break
value = res[indices].text.replace('<script z-js-react-on-rails-store="MainStore" type="application/json">', "")
z = json.loads(value)
return z
except:
pass
with open("config.json", 'r') as config:
configs = json.load(config)
try:
os.system('cls')
except:
os.system('clear')
asciiart = f"""{Spy.rouge}
██╗ ██╗██████╗ ██████╗ ████████╗
██║ ██║██╔══██╗██╔═══██╗╚══██╔══╝
██║ ██║██████╔╝██║ ██║ ██║
╚██╗ ██╔╝██╔══██╗██║ ██║ ██║
╚████╔╝ ██████╔╝╚██████╔╝ ██║
╚═══╝ ╚═════╝ ╚═════╝ ╚═╝
Thx 2$py#5430 \n\n"""
print(asciiart + "\n\n")
posting = []
sys.stdout.write("\x1b]2;Vinted Bot\x07")
class moniteur:
def __init__(self, weburl, url):
while True:
try:
z = search(str(url))
x = z["items"]["catalogItems"]["byId"]
dictlist = list(x)
for i in range(9, -1, -1):
time.sleep(1)
post = dictlist[i - 1]
if str(post) in posting:
print(f"{Spy.blanc}[{Spy.rouge}{post}{Spy.blanc}] - Item déjà envoyé !")
time.sleep(1)
continue
else:
print(f"{Spy.blanc}[{Spy.vert}{post}{Spy.blanc}] - Nouvel item trouvé !")
info = get_info_post(x[str(post)]["url"])
data = {"username": "Turtle Minor v2",
"avatar_url": "https://i.ibb.co/pWsdXYq/pp-turle-free.png",
"embeds": [
{
"description": f"{configs['embed-color-text']}",
"title": f"**__{x[post]['title']}__**",
"color": configs["embed-color"],
"url": x[post]['url'],
"fields": [
],
"image": {
"url": x[post]["photo"]["thumbnails"][4]["url"]
},
"footer": {
"text": f"https://www.paypal.com/paypalme/daviddurand29",
"icon_url": "https://i.ibb.co/pWsdXYq/pp-turle-free.png"
}
}]}
if configs["embed-config"]["prix"] == "oui":
data["embeds"][0]["fields"].append(
{
"name": "**💵 Prix**",
"value": f"{configs['embed-color-text']}\n{x[post]['price']}€",
"inline": True
})
if configs["embed-config"]["taille"] == "oui":
if x[post]['size_title'] == "":
size_title = "Pas de donnée"
else:
size_title = x[post]['size_title']
data["embeds"][0]["fields"].append({
"name": "**📏 Taille**",
"value": f"{configs['embed-color-text']}\n{size_title}",
"inline": True
})
if configs["embed-config"]["marque"] == "oui":
data["embeds"][0]["fields"].append(
{
"name": "**🔖 Marque**",
"value": f"{configs['embed-color-text']}\n{x[post]['brand_title']}",
"inline": True
}
)
if configs["embed-config"]["avis"] == "oui":
data["embeds"][0]["fields"].append({
"name": "👍/👎 **Note du vendeur**",
"value": f"{configs['embed-color-text']}\n{str(info['positive'])} - {str(info['negative'])}",
"inline": True
})
if configs["embed-config"]["localisation"] == "oui":
data["embeds"][0]["fields"].append({
"name": "**📍 Emplacement **",
"value": f"{configs['embed-color-text']}\n{info['pays']}, {info['ville']}",
"inline": True
})
if configs["embed-config"]["vendeur"] == "oui":
data["embeds"][0]["fields"].append({
"name": "**👥 Vendeur**",
"value": f"{configs['embed-color-text']}\n{info['username']}",
"inline": True
})
result = requests.post(weburl, json=data)
if 429 == result.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
else:
posting.append(str(post))
print(f"{Spy.blanc}[{Spy.bleu}POSTE{Spy.blanc}] - Poste envoyé !")
except:
time.sleep(10)
if len(configs["suburl"]) > 10:
print(
f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Trop de salon veuillez en enlever car le bot se fera rate limit !")
else:
for webhurl in configs["suburl"]:
print(
f"{Spy.blanc}[{Spy.violet}LANCEMENT{Spy.blanc}] - Lancement de la tâche dans le salon {Spy.jaune}{configs['suburl'][webhurl]['salon']}")
t = threading.Thread(target=moniteur, args=[webhurl, configs["suburl"][str(webhurl)]["url"]])
t.start()
First, I checked if some scripts didn't work on Linux but everything was in order. Then I noticed that my VPS was located in Germany so I couldn't search on the French Vinted. So I decided to add proxies. However, the bot still didn't work. I then wrote a script that returned the response from Vinted saying that I didn't have permission to access vinted.fr, even though I could access vinted.com
In the following code (a html response), do I need to remove the "M18.high_52 = {" in the beginning and the "}" (in the very end) in order to assign it under a python json variable? Thanks!
Update:
I just want to get the securityCode and securityShortName. Both already done and I just want to see it could be simply it and not using the
x = r.text.replace("M18.high_52 = " ,"")
to assign to json.
M18.high_52 = {
"stockBeans" : [
{
"securityCode" : "00176",
"yearHigh" : "0.218",
"yearLow" : "0.121",
"percentageChange" : 14.737,
"priceChange" : 0.028,
"securityShortName" : "SUPERACTIVE GP",
"securityNameGCCS" : "先機企業集團",
"previousClosingPrice" : "0.19",
"nominalPrice" : "0.218",
"sharesTraded" : "400000",
"turnover" : "80726",
"highPrice" : "0.22",
"lowPrice" : "0.19"
}
,{
"securityCode" : "00532",
"yearHigh" : "0.71",
"yearLow" : "0.49",
"percentageChange" : 20.339,
"priceChange" : 0.12,
"securityShortName" : "WKK INTL (HOLD)",
"securityNameGCCS" : "王氏港建國際",
"previousClosingPrice" : "0.59",
"nominalPrice" : "0.71",
"sharesTraded" : "1122000",
"turnover" : "749480",
"highPrice" : "0.72",
"lowPrice" : "0.63"
}
],
"stockBeans.size" : 37
}
Update:
import json
import requests
###
def update52wHigh():
r = requests.get("http://money18.on.cc/js/real/52highlow/high_securityCode_b5_asc.js")
r.encoding = 'big5-hkscs'
x = r.text.replace("M18.high_52 = " ,"")
j = json.loads(x)
print (j["stockBeans"][0]["securityNameGCCS"])
importList = []
stockList = []
x=0
for stockBean in j["stockBeans"]:
stockName = stockBean["securityNameGCCS"]
stockNo = stockBean["securityCode"]
stockPrice = stockBean["nominalPrice"]
stockTurnover = stockBean["turnover"]
stockPercentageChange = stockBean["percentageChange"]
print (stockNo + " " +stockName)
if float(stockPrice) > 1:
stockList.append([])
stockList[x].append(stockNo)
stockList[x].append(stockPrice)
importList.append((stockNo,stockName,stockPrice,stockTurnover,stockPercentageChange))
x=x+1
update52wHigh()
No need to remove anything..You already have a response
This show work
import json
import requests
string_res = response.read().decode('utf-8')
json_object = json.loads(string_res)
I am new at python and i need some help to make my code work, i am having some errors with the type of variables like: "TypeError: 'int' object is not subscriptable"
and
"TypeError: can't convert type 'str' to numerator/denominator"
I have asked some questions in here about that, but i cant make the code to work. links of the questions:
TypeError: can't convert type 'str' to numerator/denominator
Python Error: TypeError: list indices must be integers or slices, not str
My code is the following:
year_codes = ["2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018", "2019", "2020"]
month_codes = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
city_codes = ["1111609", "1120303", "11A1312", "11D1714", "11E0402", "16D0105", "16E0603", "16F1009", "16G1823",
"16H0502", "16J0907", "1701106", "1701512", "1840205", "1851416", "1861214", "1870705", "1500805"]
valor_anual = [0][0]
for city_code in city_codes:
for year_code in year_codes:
valor_mensal_aux = []
for month_code in month_codes:
url_imob = Request(
"https://www.ine.pt/ine/json_indicador/pindica.jsp?op=2&varcd=0010042&Dim1=S3A" + year_code + \
month_code + "&Dim2=" + city_code + "&Dim3=T&lang=PT", headers={'User-Agent': 'XYZ/3.0'})
json_date = year_code + month_code
response = urlopen(url_imob)
data = json.loads(response.read())
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
valor_anual[data[0]['Dados'][json_date][0]['geodsg']][year_code] = statistics.mean(valor_mensal_aux)
python_imob = Imob(year_code, valor_anual[city_code][year_code], data['DataUltimoAtualizacao'],
data['Dados'][json_date]['geodsg'])
session.add(python_imob)
session.commit()
session.close()
imobs = session.query(Imob).all()
I have errors with my arrays, the "valor_anual" and the "valor_mensal"
one of the JSON's is the following:
[ {
"IndicadorCod" : "0010042",
"IndicadorDsg" : "Valor mediano de avaliação bancária (€/ m²) por Localização geográfica (Município - 2013) e Tipo de construção; Mensal - INE, Inquérito à avaliação bancária na habitação",
"MetaInfUrl" : "https://www.ine.pt/bddXplorer/htdocs/minfo.jsp?var_cd=0010042&lingua=PT",
"DataExtracao" : "2020-06-29T15:55:51.640+01:00",
"DataUltimoAtualizacao" : "2020-06-29",
"UltimoPref" : "Maio de 2020",
"Dados" : {
"202005" : [ {
"geocod" : "1701106",
"geodsg" : "Lisboa",
"dim_3" : "T",
"dim_3_t" : "Total",
"valor" : "3084"
} ]
}
} ]
So basically i want to get data from JSON's from www.ine.pt that's why i cycled the url string with different codes to have different JSON's. I need to get the value "valor" for each month and use the mean to transform it to year value, but i need to know which value corresponds to each year and to each city, like im trying to with the "valor_anual" array. My "geodsg" variable from JSON contains the name of a city, each iterarion of the city code cycle. Can you guys help me to make this code work? i'm trying to make it work for some time now and i cant go anywhere from here..
Thank you so much and sorry for the long question
'''
Make sure whatever dados date is present in data response and if does not exist handle with ease
You don't need a 2-D array to store your country value..you can store it in local variable if there is no further need of it or if you need further refernce you can have a list of dict
**I'm attaching a method from where you can fetch data from list of dict using any key
'''
'''
data- dataset
key- dict key you want to use the first record
value - value that needs to be searched
'''
def getDataFromListOfDict(data, key, value):
for i in data:
if(i[key] == value):
return i
def getRequest(year_code, month_code, city_code):
string = "https://www.ine.pt/ine/json_indicador/pindica.jsp?op=2&varcd=0010042&Dim1=S3A"
headers = {'User-Agent': 'XYZ/3.0'}
month_qoute = "&Dim2="
city_qoute = "&Dim3=T&lang=PT"
url_imob = Request((string+year_code + "/" + month_code + month_qoute + city_code + city_qoute), headers)
json_date = year_code + month_code
response = urlopen(url_imob)
data = json.loads(response.read())
return data[0]
def getValor(data_item, json_date):
try:
return int(data_item['Dados'][json_date][0]['valor'])
except:
return 0
def getGeodsg(data_item, json_date):
try:
return int(data_item['Dados'][json_date][0]['geodsg'])
except:
return 0
def getMean(array):
from statistics import mean
return mean(array)
def execute():
year_codes = ["2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018", "2019", "2020"]
month_codes = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
city_codes = ["1111609", "1120303", "11A1312", "11D1714", "11E0402", "16D0105", "16E0603", "16F1009", "16G1823",
"16H0502", "16J0907", "1701106", "1701512", "1840205", "1851416", "1861214", "1870705", "1500805"]
valor_anual = []
# assuming this is initialize as a empty list
valor_mensal_aux = []
for city_code in city_codes:
for year_code in year_codes:
valor_mensal_aux = []
for month_code in month_codes:
json_date = year_code + month_code
first_data = getRequest(year_code, month_code, city_code)
valor_mensal_aux.append(getValor(first_data, json_date))
internal_dict = {
'index' : getGeodsg(first_data, json_date),
'year_code' : year_code,
'mean' : getMean(valor_mensal_aux)
}
# for other reference add a dictionary to a list
valor_mensal_aux.append(internal_dict)
# python_imob = Imob(year_code, valor_anual[city_code][year_code], data['DataUltimoAtualizacao'],data['Dados'][json_date]['geodsg'])
# Above is you have written and missing data[0] and ['Dados'][json_date][0]
python_imob = Imob(year_code, internal_dict['mean'], data['DataUltimoAtualizacao'],getGeodsg(first_data, json_date))
session.add(python_imob)
session.commit()
session.close()
imobs = session.query(Imob).all()
execute()
I'm trying to get some data from a government website and store them in two different tables. One will contain the filenames and the release date (let's call this filename) and one will contain the actual data and the key to join with the filename (let's call this datasplit)
These data come into a JSON file I saved from the web page (I don't have an API for that). Here's a little example of how the JSON file looks:
{
"filename": [
{
"id": 2,
"nome": "Societa' controllate di fatto dalla Presidenza del Consiglio dei Ministri e dai Ministeri",
"aggiornamento": "04-02-2020",
"datasplit": [
{
"cf": "00081070591",
"den": "SIOG SOCIETA'ITALIANA OLEODOTTI DI GAETA SPA IN AMM.NE STRAORDINARIA",
"dm": "1513641600"
},
{
"cf": "00103540829",
"den": "INDUSTRIA SICILIANA ACIDO FOSFORICO S.P.A.IN LIQUIDAZIONE",
"dm": "1513641600"
}
]
},
{
"id": 1,
"nome": "Enti o societa' controllate dalle Amministrazioni Centrali",
"aggiornamento": "30-10-2019",
"datasplit": [
{
"cf": "00049100522",
"den": "MPS TENIMENTI POGGIO BONELLI E CHIGI SARACINI - SOC. AGRICOLA SPA",
"dm": "1513641600"
},
{
"cf": "00051010528",
"den": "SOCIETA' AGRICOLA SUVIGNANO S.R.L.",
"dm": "1513641600"
}
]
},
{
"id": 4,
"nome": "Societa' quotate inserite nell'indice FTSE MIB della Borsa italiana",
"aggiornamento": "19-12-2017",
"datasplit": [
{
"cf": "00079760328",
"den": "ASSICURAZIONI GENERALI S.P.A.",
"dm": "1513641600"
},
{
"cf": "00222620163",
"den": "FRENI BREMBO - SPA",
"dm": "1513641600"
}
]
}
]
}
So what I'd like to get is the filename table with id, nome, aggiornamento fields and the datasplit table with id, aggiornamento, cf, den, dm
What I've done so far is to get the JSON file (saving it locally) from the webpage and read it in my python program.
# this works
import json
sqlstatement = ''
with open('splitdata.json', 'r') as f: #this is where I saved the website content I want to Import
jsondata = json.loads(f.read())
I was trying to build something that would go through the json file and build some INSERT INTO table_name SQL queries to later execute them and finally have my data in the database.
So, my problem is how to read the nested JSON first and how to insert the data in my database second (if you have a better solution than creating and running the SQL script).
When trying to cycle inside the JSON it seems that it only finds one element.
for json in jsondata:
keylist = "("
valuelist = "("
firstPair = True
for key, value in jsondata.items():
if not firstPair:
keylist += ", "
valuelist += ", "
firstPair = False
keylist += key
if type(value) in (str, unicode):
valuelist += "'" + value + "'"
else:
valuelist += str(value)
keylist += ")"
valuelist += ")"
sqlstatement += "INSERT INTO " + TABLE_NAME + " " + keylist + " VALUES " + valuelist + "\n"
print(sqlstatement)
I know the code is incomplete to generate the correct SQL statements but i need help on how to get to the nested part of the JSON, the datasplit field. Could it be it's not treating it as a dictionary? If so, how can i eventually fix it?
You can attempt to solve the problem using Pandas which has SQL capabilities:
import pandas as pd
for key, val in json_data.items():
df = pd.json_normalize(val)
df = df.explode('datasplit')
df[['cf', 'den', 'dm']] = df.datasplit.apply(pd.Series)
df = df.drop('datasplit', axis=1)
df.to_sql(<name>, <con>)
This is what each datasplit (df) looks like:
id nome aggiornamento \
0 2 Societa' controllate di fatto dalla Presidenza... 04-02-2020
0 2 Societa' controllate di fatto dalla Presidenza... 04-02-2020
1 1 Enti o societa' controllate dalle Amministrazi... 30-10-2019
1 1 Enti o societa' controllate dalle Amministrazi... 30-10-2019
2 4 Societa' quotate inserite nell'indice FTSE MIB... 19-12-2017
2 4 Societa' quotate inserite nell'indice FTSE MIB... 19-12-2017
cf den dm
0 00081070591 SIOG SOCIETA'ITALIANA OLEODOTTI DI GAETA SPA I... 1513641600
0 00103540829 INDUSTRIA SICILIANA ACIDO FOSFORICO S.P.A.IN L... 1513641600
1 00049100522 MPS TENIMENTI POGGIO BONELLI E CHIGI SARACINI ... 1513641600
1 00051010528 SOCIETA' AGRICOLA SUVIGNANO S.R.L. 1513641600
2 00079760328 ASSICURAZIONI GENERALI S.P.A. 1513641600
2 00222620163 FRENI BREMBO - SPA 1513641600
I am not sure where you are stuck at, but perhaps you aren't familiar with how dictionaries work?
jsondata['filename'] # has everything you want. you array of data.
# get the first element in the array
jsondata['filename'][0]
# result: {'id': 2, 'nome': "Societa' controllate di fatto dalla Presidenza del Consiglio dei Ministri e dai Ministeri", 'aggiornamento': '04-02-2020', 'datasplit': [{'cf': '00081070591', 'den': "SIOG SOCIETA'ITALIANA OLEODOTTI DI GAETA SPA IN AMM.NE STRAORDINARIA", 'dm': '1513641600'}, {'cf': '00103540829', 'den': 'INDUSTRIA SICILIANA ACIDO FOSFORICO S.P.A.IN LIQUIDAZIONE', 'dm': '1513641600'}]}
# second element
jsondata['filename'][1]
# result: {'id': 1, 'nome': "Enti o societa' controllate dalle Amministrazioni Centrali", 'aggiornamento': '30-10-2019', 'datasplit': [{'cf': '00049100522', 'den': 'MPS TENIMENTI POGGIO BONELLI E CHIGI SARACINI - SOC. AGRICOLA SPA', 'dm': '1513641600'}, {'cf': '00051010528', 'den': "SOCIETA' AGRICOLA SUVIGNANO S.R.L.", 'dm': '1513641600'}]}
# access `datasplit` directly like
jsondata['filename'][0]['datasplit']
# result: [{'cf': '00081070591', 'den': "SIOG SOCIETA'ITALIANA OLEODOTTI DI GAETA SPA IN AMM.NE STRAORDINARIA", 'dm': '1513641600'}, {'cf': '00103540829', 'den': 'INDUSTRIA SICILIANA ACIDO FOSFORICO S.P.A.IN LIQUIDAZIONE', 'dm': '1513641600'}]
# or in for loop
for data in jsondata['filename']:
data['datasplit']
im trying to loop over a set of lists and dicts and pull the correct info from them,
should run like:
get random from music, if random is a list then print list, if list contains dict print dict
this is as far as i got before i became confused! please help a noob!
import random
music = ['Band1', 'Band2', 'Band3', 'Band4']
Band1 = ['Album1']
Band2 = ['Album2']
Band3 = ['Album3']
Band4 = ['Album4']
Album1 = {
"01": 'Track1', "02": 'Track2', "03": 'Track3', "04": 'Track4',
"05": 'Track5', "06": 'Track6', "07": 'Track7', "08": 'Track8',
"09": 'Track9', "10": 'Track10', "11": 'Track11'}
i = random.choice(music)
if isinstance(i, list):
print('is instance')
I suggest a different data structure:
music = {
"Band 1": {
"Album A": ["1-Track A1", "1-Track A2", "1-Track A3"],
"Album B": ["1-Track B1", "1-Track B2", "1-Track B3"],
"Album C": ["1-Track C1", "1-Track C2", "1-Track C3"]
},
"Band 2": {
"Album A": ["2-Track A1", "2-Track A2", "2-Track A3"],
"Album B": ["2-Track B1", "2-Track B2", "2-Track B3"],
"Album C": ["2-Track C1", "2-Track C2", "2-Track C3"]
},
"Band 3": {
"Album A": ["3-Track A1", "3-Track A2", "3-Track A3"],
"Album B": ["3-Track B1", "3-Track B2", "3-Track B3"],
"Album C": ["3-Track C1", "3-Track C2", "3-Track C3"]
}
}
This is a dictionary of bands (key: band name) where each band is a dictionary containing albums (key: album name) where each album is a list containing the track names (index: track number - 1).
Then we can assume that our data structure contains only dictionaries, lists and strings. We want a function that picks a random track, i.e. a string.
Here's a recursive approach. If wanted, it could also be adapted to return the keys and indexes where it found the track as well. It's also capable of any nesting depth, so if you would want to group bands by countries or language or genre etc. that would be no problem.
import random
def pick_track(music_collection):
# we must pick a key and look that up if we get a dictionary
if isinstance(music_collection, dict):
chosen = music_collection[random.choice(list(music_collection.keys()))]
else:
chosen = random.choice(music_collection)
if isinstance(chosen, str): # it's a string, so it represents a track
return chosen
else: # it's a collection (list or dict) so we have to pick something from inside it
return pick_track(chosen)
Now we use this method like this to e.g. print 10 random tracks:
for i in range(5):
print(pick_track(music))
This could output the following example:
1-Track C1
2-Track C3
2-Track A3
3-Track A3
2-Track B1
Update:
You want to also get the keys and indexes where a track was found i.e. the band name, album name and track number? No problem, here's a modified function:
def pick_track2(music_collection):
if isinstance(music_collection, dict):
random_key = random.choice(list(music_collection.keys()))
else:
random_key = random.randrange(len(music_collection))
chosen = music_collection[random_key]
if isinstance(chosen, str):
return [random_key, chosen]
else:
return [random_key] + pick_track2(chosen)
It now does not return the track name as string, but a list of keys/indices that create the path to the picked track. You would use it like this:
for i in range(5):
print("Band: '{}' - Album: '{}' - Track {}: '{}'".format(*pick_track2(music)))
An example output:
Band: 'Band 1' - Album: 'Album C' - Track 1: '1-Track C2'
Band: 'Band 2' - Album: 'Album B' - Track 0: '2-Track B1'
Band: 'Band 1' - Album: 'Album B' - Track 0: '1-Track B1'
Band: 'Band 3' - Album: 'Album B' - Track 2: '3-Track B3'
Band: 'Band 3' - Album: 'Album B' - Track 2: '3-Track B3'
See this code running on ideone.com
Twisting your order and using the actual variables (not their names as strings) in your lists should get you started:
Album1 = {
"01": 'Track1', "02": 'Track2', "03": 'Track3', "04": 'Track4',
"05": 'Track5', "06": 'Track6', "07": 'Track7', "08": 'Track8',
"09": 'Track9', "10": 'Track10', "11": 'Track11'
}
Album2 = []
Album3 = ""
Album4 = 0
Band1 = [Album1]
Band2 = [Album2]
Band3 = [Album3]
Band4 = [Album4]
music = [Band1, Band2, Band3, Band4]
When I debug this code, I am getting "i" as string. So first you have to obtain the variable by name using globals function.
This code may help you:
import random
music = ['Band1', 'Band2', 'Band3', 'Band4']
Band1 = ['Album1']
Band2 = ['Album2']
Band3 = ['Album3']
Band4 = ['Album4']
Album1 = {
"01": 'Track1', "02": 'Track2', "03": 'Track3', "04": 'Track4',
"05": 'Track5', "06": 'Track6', "07": 'Track7', "08": 'Track8',
"09": 'Track9', "10": 'Track10', "11": 'Track11'}
Album2 = []
Album3 = ""
Album4 = 0
i = random.choice(music)
print i
#val = eval(i)[0]
#print type(eval(val))
val2 = globals()[i][0]
print type(globals()[val2])