Develop Reference

Importing a JSON object to a Google Cloud SQL instance authentication error

I have been stuck on this for the last 2 days, first time in a long time I had to use stackoverflow. But I love this community
What I am trying to do?
I am trying to upload a JSON object to update a table in a cloud SQL instance.
Error I am getting
I have tried using a service account and 02Auth credentials but I still get these errors.
Code below
from pprint import pprint
from googleapiclient import discovery
from oauth2client.client import GoogleCredentials
from google.oauth2 import service_account
import google.auth.credentials
credentials = GoogleCredentials.get_application_default()
quota_project_id = "alert-groove-360914"
scopes = ["https://www.googleapis.com/auth/cloud-platform","https://www.googleapis.com/auth/sqlservice.admin"]
#credentials, _ = google.auth.load_credentials_from_file(
# "alert-groove-360914-0d5821a7921c.json", scopes=scopes, quota_project_id=quota_project_id
# )
#credentials = service_account.Credentials.from_service_account_file("alert-groove-360914-0d5821a7921c.json")
service = discovery.build('sqladmin', 'v1beta4', credentials=credentials)
# Project ID of the project that contains the instance.
project = 'XXXXXXXX' # TODO: Update placeholder value.
# Cloud SQL instance ID. This does not include the project ID.
instance = 'employment-types' # TODO: Update placeholder value.
database_instance_body = {
"id": 3,
"first_name": "XXXXXXXX",
"last_name": "XXXXXXX",
"email": "XXXXXX28#gmail.com",
"mobile_number": "XXXXXXXXX",
"country_id": 224,
"new_user": False,
"status": 1,
"reason": 'null',
"duration": 'null',
"linkedin_id": 'null',
"sync_contacts": False,
"linkedin_sync": False,
"available": True,
"radar_visibility": 'null',
"details": {
"organization": "Foras",
"isSelfEmployed": False,
"gender_id": 2,
"location_id": 457,
"gender": 'null',
"about": "The best around and nothing will ever get me down",
"image": "https:\/\/foras.tedmob.com\/storage\/users\/image-8596.jpg",
"lat": "25.20082000253009",
"long": "55.280527271443894",
"Gender_Rel": "Male",
"job_title": "Engineer",
"Experience": "10+ years",
"Location": "Dubai"
},
"interests": [
{
"id": 106,
"title": "Nightlife",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
},
{
"id": 102,
"title": "Music",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
},
{
"id": 98,
"title": "Personal finance",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Finance"
}
},
{
"id": 101,
"title": "Crypto currency ",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Finance"
}
},
{
"id": 104,
"title": "Breaking",
"icon": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-5908.png",
"image": "https:\/\/foras.tedmob.com\/storage\/interests\/basket-ball-7960.png",
"category": {
"category": "Culture"
}
}
],
"languages": [],
"goals": [
{
"id": 2,
"title": "Find a new job",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/find-a-new-job-6804.png",
"pivot": {
"user_id": 3,
"goal_id": 2
},
"category": {
"category": "Career"
}
},
{
"id": 11,
"title": "Explore a career change",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/explore-a-career-change-5885.png",
"pivot": {
"user_id": 3,
"goal_id": 11
},
"category": {
"category": "Career"
}
},
{
"id": 12,
"title": "Find co-founders",
"icon": "https:\/\/foras.tedmob.com\/storage\/goals\/find-co-founders-2170.png",
"pivot": {
"user_id": 3,
"goal_id": 12
},
"category": {
"category": "Networking"
}
}
],
"country": {
"id": 224,
"name": "United Arab Emirates",
"iso": "AE",
"phone_code": "+971"
},
"academic_levels": [
{
"id": 4,
"title": "Master's degree",
"pivot": {
"user_id": 3,
"academic_level_id": 4
}
}
],
"experiences": [
{
"id": 1,
"title": "test1",
"company_name": "test",
"isCurrentlyWorking": True,
"start_date": "2021-06-15 00:00:00",
"end_date": "2022-09-13 00:00:00",
"end_position": True,
"headline": "test",
"industry": "test",
"description": "description",
"employment_type": 1
}
]
}
request = service.instances().update(project=project, instance=instance, body=database_instance_body)
request.execute()

How to parse complex json with python?

I am trying to parse this json file and I am having trouble.
The json looks like this:
<ListObject list at 0x2161945a860> JSON: {
"data": [
{
"amount": 100,
"available_on": 1621382400,
"created": 1621264875,
"currency": "usd",
"description": "0123456",
"exchange_rate": null,
"fee": 266,
"fee_details": [
{
"amount": 266,
"application": null,
"currency": "usd",
"description": "processing fees",
"type": "fee"
}
],
"id": "txn_abvgd1234",
"net": 9999,
"object": "balance_transaction",
"reporting_category": "charge",
"source": "cust1",
"sourced_transfers": {
"data": [],
"has_more": false,
"object": "list",
"total_count": 0,
"url": "/v1/source"
},
"status": "pending",
"type": "charge"
},
{
"amount": 25984,
"available_on": 1621382400,
"created": 1621264866,
"currency": "usd",
"description": "0326489",
"exchange_rate": null,
"fee": 93,
"fee_details": [
{
"amount": 93,
"application": null,
"currency": "usd",
"description": "processing fees",
"type": "fee"
}
],
"id": "txn_65987jihgf4984oihydgrd",
"net": 9874,
"object": "balance_transaction",
"reporting_category": "charge",
"source": "cust2",
"sourced_transfers": {
"data": [],
"has_more": false,
"object": "list",
"total_count": 0,
"url": "/v1/source"
},
"status": "pending",
"type": "charge"
},
],
"has_more": true,
"object": "list",
"url": "/v1/balance_"
}
I am trying to parse it in python with this script:
import pandas as pd
df = pd.json_normalize(json)
df.head()
but what I am getting is:
What i need is to parse each of these data points in its own column. So i will have 2 row of data with columns for each data points.
Something like this:
How do i do this now?

All but one of your fields are direct copies from the JSON, so you can just make a list of the fields you can copy, and then do the extra processing for the fee_details.
import json
import pandas as pd
inp = """{
"data": [
{
"amount": 100,
"available_on": 1621382400,
"created": 1621264875,
"currency": "usd",
"description": "0123456",
"exchange_rate": null,
"fee": 266,
"fee_details": [
{
"amount": 266,
"application": null,
"currency": "usd",
"description": "processing fees",
"type": "fee"
}
],
"id": "txn_abvgd1234",
"net": 9999,
"object": "balance_transaction",
"reporting_category": "charge",
"source": "cust1",
"sourced_transfers": {
"data": [],
"has_more": false,
"object": "list",
"total_count": 0,
"url": "/v1/source"
},
"status": "pending",
"type": "charge"
},
{
"amount": 25984,
"available_on": 1621382400,
"created": 1621264866,
"currency": "usd",
"description": "0326489",
"exchange_rate": null,
"fee": 93,
"fee_details": [
{
"amount": 93,
"application": null,
"currency": "usd",
"description": "processing fees",
"type": "fee"
}
],
"id": "txn_65987jihgf4984oihydgrd",
"net": 9874,
"object": "balance_transaction",
"reporting_category": "charge",
"source": "cust2",
"sourced_transfers": {
"data": [],
"has_more": false,
"object": "list",
"total_count": 0,
"url": "/v1/source"
},
"status": "pending",
"type": "charge"
}
],
"has_more": true,
"object": "list",
"url": "/v1/balance_"
}"""
copies = [
'id',
'net',
'object',
'reporting_category',
'source',
'amount',
'available_on',
'created',
'currency',
'description',
'exchange_rate',
'fee'
]
data = json.loads(inp)
rows = []
for inrow in data['data']:
outrow = {}
for copy in copies:
outrow[copy] = inrow[copy]
outrow['fee_details'] = inrow['fee_details'][0]['description']
rows.append(outrow)
df = pd.DataFrame(rows)
print(df)
Output:
timr#tims-gram:~/src$ python x.py
id net object reporting_category source amount ... created currency description exchange_rate fee fee_details
0 txn_abvgd1234 9999 balance_transaction charge cust1 100 ... 1621264875 usd 0123456 None 266 processing fees
1 txn_65987jihgf4984oihydgrd 9874 balance_transaction charge cust2 25984 ... 1621264866 usd 0326489 None 93 processing fees
[2 rows x 13 columns]
timr#tims-gram:~/src$

Traversing through a tree (multi nest)

There is a parent, children, grandchildren and many more relationship like the json given below (id's are unique inside each list), if the nesting is up to 10 levels with each level having different lengths. What is the best way to find a uni_code to insert another object (child) into its list?
{
"id": "1",
"status": "active",
"created": "",
"children": [{
"id": "1",
"status": "active",
"created": "",
"children": [{
"id": "1",
"status": "active",
"created": "",
"children": [{
"id": "1",
"status": "active",
"created": "",
"children": [
],
"uni_code": "EGCFJ1"
},
{
"id": "1",
"status": "active",
"created": "",
"children": [
],
"uni_code": "D356RY"
},
],
"uni_code": "EGCFJ1"
},
],
"uni_code": "Y7TUP8"
},
{
"id": "4",
"status": "active",
"created": "",
"children": [
],
"uni_code": "WA1JNS"
},
],
"uni_code": "I429TD"
}

Based on your follow-on question in a comment, I believe I now understand what your were asking…and have updated my answer accordingly.
Often you can traverse a recursive data-structure like a tree by using a function that calls itself recursively.
What I mean:
def insert_into_tree(tree, new_child, target):
if tree['uni_code'] == target:
tree['children'].append(new_child)
return True
for child in tree['children']:
if insert_into_tree(child, new_child, target):
return True
return False # `target` not found.
data = {
"id": "1",
"status": "active",
"created": "",
"children": [
{
"id": "1",
"status": "active",
"created": "",
"children": [
{
"id": "1",
"status": "active",
"created": "",
"children": [
{
"id": "1",
"status": "active",
"created": "",
"children": [],
"uni_code": "EGCFJ1"
},
{
"id": "1",
"status": "active",
"created": "",
"children": [],
"uni_code": "D356RY"
}
],
"uni_code": "EGCFJ1"
}
],
"uni_code": "Y7TUP8"
},
{
"id": "4",
"status": "active",
"created": "",
"children": [],
"uni_code": "WA1JNS"
}
],
"uni_code": "I429TD"
}
new_object = {
"id": "1",
"status": "active",
"created": "",
"children": [],
"uni_code": "THX1138"
}
target = "EGCFJ1"
if insert_into_tree(data, new_object, target):
print(f'new child added to {target!r}')
else:
print(f'{target!r} not found, new child not added')

Read a JSON file and select node like in Pandas dataframe

I need to read a JSON config file like the example below and change some of its values with a querying structure like in Pandas.
Ex:
[
{
"_id": "5d1f5d0289725ba2c32695ac",
"index": 0,
"guid": "d1a8c2e2-1011-4db2-97a8-b68777c2d18b",
"isActive": false,
"name": {
"first": "Barnett",
"last": "Obrien"
},
"latitude": "-76.327744",
"longitude": "-131.003501",
"friends": [
{
"friend_id": 0,
"name": "Burnett Burke"
},
{
"friend_id": 1,
"name": "Lawrence Hunt"
},
{
"friend_id": 2,
"name": "Nola Benjamin"
}
]
},
{
"_id": "5d1f5d023ef4523b5e326ae2",
"index": 1,
"guid": "6b0ad8a7-2b10-4892-9b91-fc7445038aca",
"isActive": true,
"name": {
"first": "Valerie",
"last": "Preston"
},
"latitude": "27.995886",
"longitude": "170.930419",
"friends": [
{
"friend_id": 0,
"name": "Gretchen Hobbs"
},
{
"friend_id": 1,
"name": "Irene Fox"
},
{
"friend_id": 2,
"name": "Porter King"
}
]
}
]
Then I wanted to change the value for the friend_id == 1 and object with guid == 6b0ad8a7-2b10-4892-9b91-fc7445038aca from Irene Fox to something else.
With Pandas I can have something like this:
valerie = dataframe['guid'] == '6b0ad8a7-2b10-4892-9b91-fc7445038aca'
friend1 = dataframe['friend_1'] == 1
dataframe[valerie & friend1]['name'] = 'Karen Smith'
How can I achieve this without having to add Pandas dependency?

With simple loop:
import json
import sys
data = json.load(open('input.json'))
for d in data:
if d["guid"] == "6b0ad8a7-2b10-4892-9b91-fc7445038aca":
for f in d["friends"]:
if f["friend_id"] == 1:
f["name"] = "Karen Smith"
# break <- uncomment if only one match is implied
# replace sys.stdout with output file pointer
json.dump(data, sys.stdout, indent=4)
you may also break the outer for loop if items/dicts have unique guids.
The output (for demonstration):
[
{
"_id": "5d1f5d0289725ba2c32695ac",
"index": 0,
"guid": "d1a8c2e2-1011-4db2-97a8-b68777c2d18b",
"isActive": false,
"name": {
"first": "Barnett",
"last": "Obrien"
},
"latitude": "-76.327744",
"longitude": "-131.003501",
"friends": [
{
"friend_id": 0,
"name": "Burnett Burke"
},
{
"friend_id": 1,
"name": "Lawrence Hunt"
},
{
"friend_id": 2,
"name": "Nola Benjamin"
}
]
},
{
"_id": "5d1f5d023ef4523b5e326ae2",
"index": 1,
"guid": "6b0ad8a7-2b10-4892-9b91-fc7445038aca",
"isActive": true,
"name": {
"first": "Valerie",
"last": "Preston"
},
"latitude": "27.995886",
"longitude": "170.930419",
"friends": [
{
"friend_id": 0,
"name": "Gretchen Hobbs"
},
{
"friend_id": 1,
"name": "Karen Smith"
},
{
"friend_id": 2,
"name": "Porter King"
}
]
}
]

Cannot import grafana dashboard via Grafana API

I am trying to import the Grafana dashboard using HTTP API by following Grafana
Grafana Version: 5.1.3
OS -Windows 10
This is what i tried
curl --user admin:admin "http://localhost:3000/api/dashboards/db" -X POST -H "Content-Type:application/json;charset=UTF-8" --data-binary #c:/Users/Mahadev/Desktop/Dashboard.json
and
Here is my python code
import requests
headers = {
'Content-Type': 'application/json;charset=UTF-8',
}
data = open('C:/Users/Mahadev/Desktop/Dashboard.json', 'rb').read()
response = requests.post('http://admin:admin#localhost:3000/api/dashboards/db', headers=headers, data=data)
print (response.text)
And output of both is:
[{"fieldNames":["Dashboard"],"classification":"RequiredError","message":"Required"}]
It is asking for root property called dashboard in my json payload. Can anybody suggest me how to use that porperty and what data should i provide.
If any one want to dig more here are some links.
https://github.com/grafana/grafana/issues/8193
https://github.com/grafana/grafana/issues/2816
https://github.com/grafana/grafana/issues/8193
https://community.grafana.com/t/how-can-i-import-a-dashboard-from-a-json-file/669
https://github.com/grafana/grafana/issues/273
https://github.com/grafana/grafana/issues/5811
https://stackoverflow.com/questions/39968111/unable-to-post-to-grafana-using-python3-module-requests
https://stackoverflow.com/questions/39954475/post-request-works-in-postman-but-not-in-python/39954514#39954514
https://www.bountysource.com/issues/44431991-use-api-to-import-json-file-error
https://github.com/grafana/grafana/issues/7029

Maybe you should try to download your dashboard from the API so you will a "proper" json model to push after?
You can download it with the following command :
curl -H "Authorization: Bearer $TOKEN" https://grafana.domain.tld/api/dashboards/uid/$DASHBOARD_UID
An other way to do it , you can download a dashboard JSON on grafana website => grafana.com/dashboards and try to upload it with your current code? ;)
The dashboard field contain everything that will be display, alerts, graph etc....
Here is an example of dashboard.json :
{
"meta": {
"type": "db",
"canSave": true,
"canEdit": true,
"canAdmin": false,
"canStar": true,
"slug": "status-app",
"url": "/d/lOy3lIImz/status-app",
"expires": "0001-01-01T00:00:00Z",
"created": "2018-06-04T11:40:20+02:00",
"updated": "2018-06-14T17:51:23+02:00",
"updatedBy": "jean",
"createdBy": "jean",
"version": 89,
"hasAcl": false,
"isFolder": false,
"folderId": 0,
"folderTitle": "General",
"folderUrl": "",
"provisioned": false
},
"dashboard": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 182,
"links": [],
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
1
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "60s",
"handler": 1,
"name": "Status of alert",
"noDataState": "alerting",
"notifications": [
{
"id": 7
}
]
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Collectd",
"fill": 1,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 0
},
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "Status",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "processes_processes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(value) FROM \"processes_processes\" WHERE (\"instance\" = '' AND \"host\" = 'Webp01') AND $timeFilter GROUP BY time($interval) fill(null)",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": [
{
"key": "instance",
"operator": "=",
"value": ""
},
{
"condition": "AND",
"key": "host",
"operator": "=",
"value": "Webp01"
}
]
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "lt",
"value": 1
}
],
"timeFrom": null,
"timeShift": null,
"title": "Status of ",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "5m",
"schemaVersion": 16,
"style": "dark",
"tags": [
"web",
"nodejs"
],
"templating": {
"list": []
},
"time": {
"from": "now/d",
"to": "now"
},
"timepicker": {
"hidden": false,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Status APP",
"uid": "lOy3lIImz",
"version": 89
},
}
Edit:
Here is a JSON snipper for templating your dashboard :
"templating": {
"list": [
{
"allValue": null,
"current": {
"text": "PRD_Web01",
"value": "PRD_Web01"
},
"datasource": "Collectd",
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"name": "host",
"options": [],
"query": "SHOW TAG VALUES WITH KEY=host",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"text": "sda",
"value": "sda"
},
"datasource": "Collectd",
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"name": "device",
"options": [],
"query": "SHOW TAG VALUES FROM \"disk_read\" WITH KEY = \"instance\"",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
As I read your answer, I guess you will be OK with this ;). I will try to keep a better eye on this thread

Can you show how your dashboard json looks like ? The json MUST contain a key dashboard in it with all the details inside its value like the following:
{
"dashboard": {
"id": null,
"uid": null,
"title": "Production Overview",
"tags": [ "templated" ],
"timezone": "browser",
"schemaVersion": 16,
"version": 0
},
"folderId": 0,
"overwrite": false
}