Develop Reference

Importing a JSON object to a Google Cloud SQL instance authentication error

I have been stuck on this for the last 2 days, first time in a long time I had to use stackoverflow. But I love this community
What I am trying to do?
I am trying to upload a JSON object to update a table in a cloud SQL instance.
Error I am getting
I have tried using a service account and 02Auth credentials but I still get these errors.
Code below
from pprint import pprint
from googleapiclient import discovery
from oauth2client.client import GoogleCredentials
from google.oauth2 import service_account
import google.auth.credentials
credentials = GoogleCredentials.get_application_default()
quota_project_id = "alert-groove-360914"
scopes = ["https://www.googleapis.com/auth/cloud-platform","https://www.googleapis.com/auth/sqlservice.admin"]
#credentials, _ = google.auth.load_credentials_from_file(
# "alert-groove-360914-0d5821a7921c.json", scopes=scopes, quota_project_id=quota_project_id
# )
#credentials = service_account.Credentials.from_service_account_file("alert-groove-360914-0d5821a7921c.json")
service = discovery.build('sqladmin', 'v1beta4', credentials=credentials)
# Project ID of the project that contains the instance.
project = 'XXXXXXXX' # TODO: Update placeholder value.
# Cloud SQL instance ID. This does not include the project ID.
instance = 'employment-types' # TODO: Update placeholder value.
database_instance_body = {
"id": 3,
"first_name": "XXXXXXXX",
"last_name": "XXXXXXX",
"email": "XXXXXX28#gmail.com",
"mobile_number": "XXXXXXXXX",
"country_id": 224,
"new_user": False,
"status": 1,
"reason": 'null',
"duration": 'null',
"linkedin_id": 'null',
"sync_contacts": False,
"linkedin_sync": False,
"available": True,
"radar_visibility": 'null',
"details": {
"organization": "Foras",
"isSelfEmployed": False,
"gender_id": 2,
"location_id": 457,
"gender": 'null',
"about": "The best around and nothing will ever get me down",
"image": "https:\/\/foras.tedmob.com\/storage\/users\/image-8596.jpg",
"lat": "25.20082000253009",
"long": "55.280527271443894",
"Gender_Rel": "Male",
"job_title": "Engineer",
"Experience": "10+ years",
"Location": "Dubai"
},
"interests": [
{
"id": 106,
"title": "Nightlife",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
},
{
"id": 102,
"title": "Music",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
},
{
"id": 98,
"title": "Personal finance",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Finance"
}
},
{
"id": 101,
"title": "Crypto currency ",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Finance"
}
},
{
"id": 104,
"title": "Breaking",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
}
],
"languages": [],
"goals": [
{
"id": 2,
"title": "Find a new job",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/find-a-new-job-6804.png",
"pivot": {
"user_id": 3,
"goal_id": 2
},
"category": {
"category": "Career"
}
},
{
"id": 11,
"title": "Explore a career change",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/explore-a-career-change-5885.png",
"pivot": {
"user_id": 3,
"goal_id": 11
},
"category": {
"category": "Career"
}
},
{
"id": 12,
"title": "Find co-founders",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/find-co-founders-2170.png",
"pivot": {
"user_id": 3,
"goal_id": 12
},
"category": {
"category": "Networking"
}
}
],
"country": {
"id": 224,
"name": "United Arab Emirates",
"iso": "AE",
"phone_code": "+971"
},
"academic_levels": [
{
"id": 4,
"title": "Master's degree",
"pivot": {
"user_id": 3,
"academic_level_id": 4
}
}
],
"experiences": [
{
"id": 1,
"title": "test1",
"company_name": "test",
"isCurrentlyWorking": True,
"start_date": "2021-06-15 00:00:00",
"end_date": "2022-09-13 00:00:00",
"end_position": True,
"headline": "test",
"industry": "test",
"description": "description",
"employment_type": 1
}
]
}
request = service.instances().update(project=project, instance=instance, body=database_instance_body)
request.execute()

how do I access this json data in python?

hi I'm pretty new at coding and I was trying to create a program in python that reads and save in another file the data inside a json file (not everything, just what I want). I googled how to parse data but there's something I don't understand.
that's a part of the json file:
`
{
"profileRevision": 548789,
"profileId": "campaign",
"profileChangesBaseRevision": 548789,
"profileChanges": [
{
"changeType": "fullProfileUpdate",
"profile": {
"_id": "2da4f079f8984cc48e84fc99dace495d",
"created": "2018-03-29T11:02:15.190Z",
"updated": "2022-10-31T17:34:43.284Z",
"rvn": 548789,
"wipeNumber": 9,
"accountId": "63881e614ef543b2932c70fed1196f34",
"profileId": "campaign",
"version": "refund_teddy_perks_september_2022",
"items": {
"8ec8f13f-6bf6-4933-a7db-43767a055e66": {
"templateId": "Quest:heroquest_loadout_constructor_2",
"attributes": {
"quest_state": "Claimed",
"creation_time": "min",
"last_state_change_time": "2019-05-18T16:09:12.750Z",
"completion_complete_pve03_diff26_loadout_constructor": 300,
"level": -1,
"item_seen": true,
"sent_new_notification": true,
"quest_rarity": "uncommon",
"xp_reward_scalar": 1
},
"quantity": 1
},
"6940c71b-c74b-4581-9f1e-c0a87e246884": {
"templateId": "Worker:workerbasic_sr_t01",
"attributes": {
"gender": "2",
"personality": "Homebase.Worker.Personality.IsDreamer",
"level": 1,
"item_seen": true,
"squad_slot_idx": -1,
"portrait": "WorkerPortrait:IconDef-WorkerPortrait-Dreamer-F02",
"building_slot_used": -1,
"set_bonus": "Homebase.Worker.SetBonus.IsMeleeDamageLow"
}
}
}
]
}
`
I can access profileChanges. I wrote this to create another json file with only the profileChanges things:
`
myjsonfile= open("file.json",'r')
jsondata=myjsonfile.read()
obj=json.loads(jsondata)
ciso=obj['profileChanges']
for i in ciso:
print(i)
with open("file2", "w") as outfile:
json.dump( ciso, outfile, indent=1)
the issue I have is that I can't access "profile" (inside profileChanges) in the same way by parsing the new file and I have no idea on how to do it

Access to JSON or dict element is realized by list indexes, please look at below example:
a = [
{
"friends": [
{
"id": 0,
"name": "Reba May"
}
],
"greeting": "Hello, Doris Gallagher! You have 2 unread messages.",
"favoriteFruit": "strawberry"
},
]
b = a['friends']['id] # b = 0

I've added a couple of closing braces to make your snippet valid json:
s = '''{
"profileRevision": 548789,
"profileId": "campaign",
"profileChangesBaseRevision": 548789,
"profileChanges": [
{
"changeType": "fullProfileUpdate",
"profile": {
"_id": "2da4f079f8984cc48e84fc99dace495d",
"created": "2018-03-29T11:02:15.190Z",
"updated": "2022-10-31T17:34:43.284Z",
"rvn": 548789,
"wipeNumber": 9,
"accountId": "63881e614ef543b2932c70fed1196f34",
"profileId": "campaign",
"version": "refund_teddy_perks_september_2022",
"items": {
"8ec8f13f-6bf6-4933-a7db-43767a055e66": {
"templateId": "Quest:heroquest_loadout_constructor_2",
"attributes": {
"quest_state": "Claimed",
"creation_time": "min",
"last_state_change_time": "2019-05-18T16:09:12.750Z",
"completion_complete_pve03_diff26_loadout_constructor": 300,
"level": -1,
"item_seen": true,
"sent_new_notification": true,
"quest_rarity": "uncommon",
"xp_reward_scalar": 1
},
"quantity": 1
},
"6940c71b-c74b-4581-9f1e-c0a87e246884": {
"templateId": "Worker:workerbasic_sr_t01",
"attributes": {
"gender": "2",
"personality": "Homebase.Worker.Personality.IsDreamer",
"level": 1,
"item_seen": true,
"squad_slot_idx": -1,
"portrait": "WorkerPortrait:IconDef-WorkerPortrait-Dreamer-F02",
"building_slot_used": -1,
"set_bonus": "Homebase.Worker.SetBonus.IsMeleeDamageLow"
}
}
}
}
}
]
}
'''
d = json.loads(s)
print(d['profileChanges'][0]['profile']['version'])
This prints refund_teddy_perks_september_2022
Explanation:
d is a dict
d['profileChanges'] is a list of dicts
d['profileChanges'][0] is the first dict in the list
d['profileChanges'][0]['profile'] is a dict
d['profileChanges'][0]['profile']['version'] is the value of version key in the profile dict in the first entry of the profileChanges list.

Fast way of adding fields to a nested dict

I need a help with improving my code.
I've got a nested dict with many levels:
{
"11": {
"FacLC": {
"immty": [
"in_mm",
"in_mm"
],
"moood": [
"in_oo",
"in_oo"
]
}
},
"22": {
"FacLC": {
"immty": [
"in_mm",
"in_mm",
"in_mm"
]
}
}
}
And I want to add additional fields on every level, so my output looks like this:
[
{
"id": "",
"name": "11",
"general": [
{
"id": "",
"name": "FacLC",
"specifics": [
{
"id": "",
"name": "immty",
"characteristics": [
{
"id": "",
"name": "in_mm"
},
{
"id": "",
"name": "in_mm"
}
]
},
{
"id": "",
"name": "moood",
"characteristics": [
{
"id": "",
"name": "in_oo"
},
{
"id": "",
"name": "in_oo"
}
]
}
]
}
]
},
{
"id": "",
"name": "22",
"general": [
{
"id": "",
"name": "FacLC",
"specifics": [
{
"id": "",
"name": "immty",
"characteristics": [
{
"id": "",
"name": "in_mm"
},
{
"id": "",
"name": "in_mm"
},
{
"id": "",
"name": "in_mm"
}
]
}
]
}
]
}
]
I managed to write a 4-times nested for loop, what I find inefficient and inelegant:
for main_name, general in my_dict.items():
generals = []
for general_name, specific in general.items():
specifics = []
for specific_name, characteristics in specific.items():
characteristics_dicts = []
for characteristic in characteristics:
characteristics_dicts.append({
"id": "",
"name": characteristic,
})
specifics.append({
"id": "",
"name": specific_name,
"characteristics": characteristics_dicts,
})
generals.append({
"id": "",
"name": general_name,
"specifics": specifics,
})
my_new_dict.append({
"id": "",
"name": main_name,
"general": generals,
})
I am wondering if there is more compact and efficient solution.

In the past I created a function to do it. Basically you call this function everytime that you need to add new fields to a nested dict, independently on how many levels this nested dict have. You only have to inform the 'full path' , that I called the 'key_map'.
Like ['node1','node1a','node1apart3']
def insert_value_using_map(_nodes_list_to_be_appended, _keys_map, _value_to_be_inserted):
for _key in _keys_map[:-1]:
_nodes_list_to_be_appended = _nodes_list_to_be_appended.setdefault(_key, {})
_nodes_list_to_be_appended[_keys_map[-1]] = _value_to_be_inserted

Read a JSON file and select node like in Pandas dataframe

I need to read a JSON config file like the example below and change some of its values with a querying structure like in Pandas.
Ex:
[
{
"_id": "5d1f5d0289725ba2c32695ac",
"index": 0,
"guid": "d1a8c2e2-1011-4db2-97a8-b68777c2d18b",
"isActive": false,
"name": {
"first": "Barnett",
"last": "Obrien"
},
"latitude": "-76.327744",
"longitude": "-131.003501",
"friends": [
{
"friend_id": 0,
"name": "Burnett Burke"
},
{
"friend_id": 1,
"name": "Lawrence Hunt"
},
{
"friend_id": 2,
"name": "Nola Benjamin"
}
]
},
{
"_id": "5d1f5d023ef4523b5e326ae2",
"index": 1,
"guid": "6b0ad8a7-2b10-4892-9b91-fc7445038aca",
"isActive": true,
"name": {
"first": "Valerie",
"last": "Preston"
},
"latitude": "27.995886",
"longitude": "170.930419",
"friends": [
{
"friend_id": 0,
"name": "Gretchen Hobbs"
},
{
"friend_id": 1,
"name": "Irene Fox"
},
{
"friend_id": 2,
"name": "Porter King"
}
]
}
]
Then I wanted to change the value for the friend_id == 1 and object with guid == 6b0ad8a7-2b10-4892-9b91-fc7445038aca from Irene Fox to something else.
With Pandas I can have something like this:
valerie = dataframe['guid'] == '6b0ad8a7-2b10-4892-9b91-fc7445038aca'
friend1 = dataframe['friend_1'] == 1
dataframe[valerie & friend1]['name'] = 'Karen Smith'
How can I achieve this without having to add Pandas dependency?

With simple loop:
import json
import sys
data = json.load(open('input.json'))
for d in data:
if d["guid"] == "6b0ad8a7-2b10-4892-9b91-fc7445038aca":
for f in d["friends"]:
if f["friend_id"] == 1:
f["name"] = "Karen Smith"
# break <- uncomment if only one match is implied
# replace sys.stdout with output file pointer
json.dump(data, sys.stdout, indent=4)
you may also break the outer for loop if items/dicts have unique guids.
The output (for demonstration):
[
{
"_id": "5d1f5d0289725ba2c32695ac",
"index": 0,
"guid": "d1a8c2e2-1011-4db2-97a8-b68777c2d18b",
"isActive": false,
"name": {
"first": "Barnett",
"last": "Obrien"
},
"latitude": "-76.327744",
"longitude": "-131.003501",
"friends": [
{
"friend_id": 0,
"name": "Burnett Burke"
},
{
"friend_id": 1,
"name": "Lawrence Hunt"
},
{
"friend_id": 2,
"name": "Nola Benjamin"
}
]
},
{
"_id": "5d1f5d023ef4523b5e326ae2",
"index": 1,
"guid": "6b0ad8a7-2b10-4892-9b91-fc7445038aca",
"isActive": true,
"name": {
"first": "Valerie",
"last": "Preston"
},
"latitude": "27.995886",
"longitude": "170.930419",
"friends": [
{
"friend_id": 0,
"name": "Gretchen Hobbs"
},
{
"friend_id": 1,
"name": "Karen Smith"
},
{
"friend_id": 2,
"name": "Porter King"
}
]
}
]

Cannot import grafana dashboard via Grafana API

I am trying to import the Grafana dashboard using HTTP API by following Grafana
Grafana Version: 5.1.3
OS -Windows 10
This is what i tried
curl --user admin:admin "http://localhost:3000/api/dashboards/db" -X POST -H "Content-Type:application/json;charset=UTF-8" --data-binary #c:/Users/Mahadev/Desktop/Dashboard.json
and
Here is my python code
import requests
headers = {
'Content-Type': 'application/json;charset=UTF-8',
}
data = open('C:/Users/Mahadev/Desktop/Dashboard.json', 'rb').read()
response = requests.post('http://admin:admin#localhost:3000/api/dashboards/db', headers=headers, data=data)
print (response.text)
And output of both is:
[{"fieldNames":["Dashboard"],"classification":"RequiredError","message":"Required"}]
It is asking for root property called dashboard in my json payload. Can anybody suggest me how to use that porperty and what data should i provide.
If any one want to dig more here are some links.
https://github.com/grafana/grafana/issues/8193
https://github.com/grafana/grafana/issues/2816
https://github.com/grafana/grafana/issues/8193
https://community.grafana.com/t/how-can-i-import-a-dashboard-from-a-json-file/669
https://github.com/grafana/grafana/issues/273
https://github.com/grafana/grafana/issues/5811
https://stackoverflow.com/questions/39968111/unable-to-post-to-grafana-using-python3-module-requests
https://stackoverflow.com/questions/39954475/post-request-works-in-postman-but-not-in-python/39954514#39954514
https://www.bountysource.com/issues/44431991-use-api-to-import-json-file-error
https://github.com/grafana/grafana/issues/7029

Maybe you should try to download your dashboard from the API so you will a "proper" json model to push after?
You can download it with the following command :
curl -H "Authorization: Bearer $TOKEN" https://grafana.domain.tld/api/dashboards/uid/$DASHBOARD_UID
An other way to do it , you can download a dashboard JSON on grafana website => grafana.com/dashboards and try to upload it with your current code? ;)
The dashboard field contain everything that will be display, alerts, graph etc....
Here is an example of dashboard.json :
{
"meta": {
"type": "db",
"canSave": true,
"canEdit": true,
"canAdmin": false,
"canStar": true,
"slug": "status-app",
"url": "/d/lOy3lIImz/status-app",
"expires": "0001-01-01T00:00:00Z",
"created": "2018-06-04T11:40:20+02:00",
"updated": "2018-06-14T17:51:23+02:00",
"updatedBy": "jean",
"createdBy": "jean",
"version": 89,
"hasAcl": false,
"isFolder": false,
"folderId": 0,
"folderTitle": "General",
"folderUrl": "",
"provisioned": false
},
"dashboard": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 182,
"links": [],
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
1
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "60s",
"handler": 1,
"name": "Status of alert",
"noDataState": "alerting",
"notifications": [
{
"id": 7
}
]
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Collectd",
"fill": 1,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 0
},
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "Status",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "processes_processes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(value) FROM \"processes_processes\" WHERE (\"instance\" = '' AND \"host\" = 'Webp01') AND $timeFilter GROUP BY time($interval) fill(null)",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": [
{
"key": "instance",
"operator": "=",
"value": ""
},
{
"condition": "AND",
"key": "host",
"operator": "=",
"value": "Webp01"
}
]
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "lt",
"value": 1
}
],
"timeFrom": null,
"timeShift": null,
"title": "Status of ",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "5m",
"schemaVersion": 16,
"style": "dark",
"tags": [
"web",
"nodejs"
],
"templating": {
"list": []
},
"time": {
"from": "now/d",
"to": "now"
},
"timepicker": {
"hidden": false,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Status APP",
"uid": "lOy3lIImz",
"version": 89
},
}
Edit:
Here is a JSON snipper for templating your dashboard :
"templating": {
"list": [
{
"allValue": null,
"current": {
"text": "PRD_Web01",
"value": "PRD_Web01"
},
"datasource": "Collectd",
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"name": "host",
"options": [],
"query": "SHOW TAG VALUES WITH KEY=host",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"text": "sda",
"value": "sda"
},
"datasource": "Collectd",
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"name": "device",
"options": [],
"query": "SHOW TAG VALUES FROM \"disk_read\" WITH KEY = \"instance\"",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
As I read your answer, I guess you will be OK with this ;). I will try to keep a better eye on this thread

Can you show how your dashboard json looks like ? The json MUST contain a key dashboard in it with all the details inside its value like the following:
{
"dashboard": {
"id": null,
"uid": null,
"title": "Production Overview",
"tags": [ "templated" ],
"timezone": "browser",
"schemaVersion": 16,
"version": 0
},
"folderId": 0,
"overwrite": false
}