How to flatten complex nested json file using spark dataframe - python

I have a json file as below:
{
"Number": "144532",
"Date": "01/01/2021 07:07",
"resultDetails": {
"Overview": {
"status": "NOT_OK"
},
"liabilities": [
{
"liabilitiesNewName": "cert",
"operator": {
"operatorNewName": "And"
},
"operands": [
{
"liabilitiesNewName": "reg",
"operator": {
"operatorNewName": "Or"
},
"operands": [
{
"liabilitiesNewName": "con",
"operator": {
"operatorNewName": "And"
},
"operands": [
{
"liabilitiesNewName": "pos",
"operator": {
"operatorNewName": "Or"
},
"operands": [
{
"operator": {},
"phase1": {
"NewName": "var_1",
"upadtedNewName": "344RS",
"updateTypes": [],
"valueNumbers": [
{
"valueNumber": "C9981124422",
"category": "DIS",
"fewel": {
"pwdVariant": "344RS",
"Id": "44aa"
}
},
{
"valueNumber": "B3324422344",
"Release": "22/02.09",
"category": "FUNWARE"
}
]
},
"Phase2": {
"codingData": [
{
"Id": "992ndbwbbe",
"valueNumber": "C9981124422",
"NewentsCount": 0,
"Newents": []
}
]
},
"validation": {
"errors": [
{
"affected": "B3324422344",
"error": {
"code": 922,
"description": "not found"
},
"detailedDescription": "B3324422344 not found "
},
{
"affected": "C9981124422",
"error": {
"code": "Ahh3w",
"description": "No data recevied"
},
"detailedDescription": "no data"
}
]
},
"liabilitiesNewName": "verti[var_1]",
"checkedStatus": "NOT_OK"
}
],
"phase1": {
"plainNewName": "344RS",
"isOptional": false,
"upadtedNewName": "344RS",
"Field": "344RS",
"asIsvalueNumbers": [
{
"valueNumber": "N7726364664",
"Field": "344RS",
"FieldVariant": false
},
{
"valueNumber": "C9981124422",
"Field": "344RS",
"FieldVariant": false
},
{
"valueNumber": "N7726364663",
"Field": "344RS",
"FieldVariant": false
},
{
"valueNumber": "N2333453441",
"Field": "344RS",
"FieldVariant": false
}
]
},
"validation": {
"errors": [
{
"affected": "N7726364664",
"error": {
"code": "Nddf",
"description": "valueNumber not found"
},
"detailedDescription": "valueNumber N7726364664 not found."
},
{
"affected": "N2333453441",
"error": {
"code": "Nddf",
"description": "valueNumber not found"
},
"detailedDescription": "valueNumber N2333453441 not found."
}
]
},
"checkedStatus": "NOT_OK"
},
{
"liabilitiesNewName": "pos",
"operator": {
"operatorNewName": "Or"
},
"operands": [
{
"operator": {},
"phase1": {
"NewName": "EP2",
"upadtedNewName": "ESS",
"updateTypes": [],
"valueNumbers": [
{
"valueNumber": "M8828377111228377484",
"Release": "",
"category": "DIS",
"fewel": {
"pwdVariant": "EMMQ",
"revision": "",
"Id": "45qhhq"
}
},
{
"valueNumber": "N2929828282",
"Release": "20",
"category": "FUNWARE"
}
]
},
"validation": {
"errors": [
{
"affected": "N2929828282",
"error": {
"code": 88283771112,
"description": "not found"
},
"detailedDescription": "FUNWARE part N2929828282 not "
},
{
"affected": "M8828377111228377484",
"error": {
"code": 88283771112,
"description": " not found"
},
"detailedDescription": " M8828377111228377484 not "
}
]
},
"liabilitiesNewName": "verti[EP2]",
"checkedStatus": "NOT_OK"
},
{
"operator": {},
"phase1": {
"NewName": "EP2",
"upadtedNewName": "ESS",
"updateTypes": [],
"valueNumbers": [
{
"valueNumber": "M8828377111228377484",
"Release": "00",
"category": "DIS"
},
{
"valueNumber": "P1838828377111248383",
"Release": "2",
"category": "FUNWARE"
}
]
},
"validation": {
"errors": [
{
"affected": "P1838828377111248383",
"error": {
"code": 88283771112,
"description": "not found"
},
"detailedDescription": "FUNWARE part P1838828377111248383 not "
},
{
"affected": "M8828377111228377484",
"error": {
"code": 88283771112,
"description": " not found"
},
"detailedDescription": " M8828377111228377484 not "
}
]
},
"liabilitiesNewName": "verti[EP2]",
"checkedStatus": "NOT_OK"
},
{
"operator": {},
"phase1": {
"NewName": "EP2",
"upadtedNewName": "ESS",
"updateTypes": [],
"valueNumbers": [
{
"valueNumber": "P1838828377111248381",
"Release": "2",
"category": "DIS"
},
{
"valueNumber": "P3222299292",
"Release": "20/45.00",
"category": "FUNWARE"
}
]
},
"validation": {
"errors": [
{
"affected": "P1838828377111248381",
"error": {
"code": 88283771112,
"description": " not found"
},
"detailedDescription": " P1838828377111248381 not "
}
]
},
"liabilitiesNewName": "verti[EP2]",
"checkedStatus": "NOT_OK"
},
],
"phase1": {
"plainNewName": "ESS",
"isOptional": false,
"upadtedNewName": "ESS",
"Field": "ESS",
"asIsvalueNumbers": [
{
"valueNumber": "P3222299292",
"Field": "ESS",
"FieldVariant": false
},
{
"valueNumber": "P3222299291",
"Field": "ESS",
"FieldVariant": false
}
]
},
"validation": {
"errors": [
{
"affected": "P3222299291",
"error": {
"code": "Nddf",
"description": "valueNumber not found"
},
"detailedDescription": "valueNumber P3222299291 not found."
}
]
},
"checkedStatus": "NOT_OK"
}
],
"phase1": {
"label": "Ywrwrw",
"changeDescription": "",
"comment": "",
"submitDate": "2020-01-30",
"publishDate": "2020-01-02",
"status": "PUBLISHED"
},
"validation": {
"errors": [
{
"affected": "con",
"error": {
"code": 99737366262,
"description": "error"
},
"detailedDescription": "not valid: con"
}
]
},
"checkedStatus": "NOT_OK"
}
],
"phase1": {
"rxswin": "UOOE_WIIW",
"NewName": "Rein",
"approvalNumber": "OWWPQ2111",
"validityAreaCode": "(",
"validFrom": "2020-01-23",
"validTo": "2999-12-31",
"status": "APPROVED"
},
"validation": {
"errors": [
{
"affected": "reg",
"error": {
"code": 332211,
"description": "Type approval error"
},
"detailedDescription": "not valid: reg"
}
]
},
"checkedStatus": "NOT_OK"
}
],
"validation": {
"errors": [
{
"affected": "cert",
"error": {
"code": 332212,
"description": "error"
},
"detailedDescription": "not valid: cert"
}
]
},
"checkedStatus": "NOT_OK"
}
],
"hasWarnings": false,
"regulationsMissing": false
},
"checkTrigger": "HHAGGD"
}
I want to flatten this json to leaf level properly using pyspark dataframe.
Need a generic code which automatically recognizes the type and flatten to leaf level.
How to achieve this using pyspark dataframe?
tried converting into list and flatten, but its too slow as the data is huge, need spark approach for processing faster

Related

Best way to parse a JSON to store in SQL database (SQL stored procedure/Python)

I have a table of overly complex JSON files I'm trying to convert to tabular format to store in a SQL database. I'm pulling the JSONs from the quickbooks online API, and the format is messy to say the least.. (We're talking 7x nested JSONs for some bits of it..
The format resembles the code snippet down below. Currently I am using a bunch of OpenJSON's + Cross applys to dig down to the innermost ColData then work my way up but it looks like some of the ColData's get skipped over doing that.
Are there any better ways, using either Python (since I pull the JSON initially in Python before sending the JSON to a SQL database to parse) or SQL to convert it to tabular format besides manually trying to use OpenJSON with Cross applys?
The goal is to get all of the ColData's into a SQL table...
Thanks!
{
"Header": {
"ReportName": "BalanceSheet",
"Option": [
{
"Name": "AccountingStandard",
"Value": "GAAP"
},
{
"Name": "NoReportData",
"Value": "false"
}
],
"DateMacro": "this calendar year-to-date",
"ReportBasis": "Accrual",
"StartPeriod": "2016-01-01",
"Currency": "USD",
"EndPeriod": "2016-10-31",
"Time": "2016-10-31T09:42:21-07:00",
"SummarizeColumnsBy": "Total"
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "ASSETS"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Bank Accounts"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "35",
"value": "Checking"
},
{
"value": "1350.55"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "36",
"value": "Savings"
},
{
"value": "800.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "BankAccounts",
"Summary": {
"ColData": [
{
"value": "Total Bank Accounts"
},
{
"value": "2150.55"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Accounts Receivable"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "84",
"value": "Accounts Receivable (A/R)"
},
{
"value": "6383.12"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "AR",
"Summary": {
"ColData": [
{
"value": "Total Accounts Receivable"
},
{
"value": "6383.12"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Other current assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "81",
"value": "Inventory Asset"
},
{
"value": "596.25"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "4",
"value": "Undeposited Funds"
},
{
"value": "2117.52"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "OtherCurrentAssets",
"Summary": {
"ColData": [
{
"value": "Total Other current assets"
},
{
"value": "2713.77"
}
]
}
}
]
},
"type": "Section",
"group": "CurrentAssets",
"Summary": {
"ColData": [
{
"value": "Total Current Assets"
},
{
"value": "11247.44"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Fixed Assets"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"id": "37",
"value": "Truck"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "38",
"value": "Original Cost"
},
{
"value": "13495.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"Summary": {
"ColData": [
{
"value": "Total Truck"
},
{
"value": "13495.00"
}
]
}
}
]
},
"type": "Section",
"group": "FixedAssets",
"Summary": {
"ColData": [
{
"value": "Total Fixed Assets"
},
{
"value": "13495.00"
}
]
}
}
]
},
"type": "Section",
"group": "TotalAssets",
"Summary": {
"ColData": [
{
"value": "TOTAL ASSETS"
},
{
"value": "24742.44"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "LIABILITIES AND EQUITY"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Accounts Payable"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "33",
"value": "Accounts Payable (A/P)"
},
{
"value": "1984.17"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "AP",
"Summary": {
"ColData": [
{
"value": "Total Accounts Payable"
},
{
"value": "1984.17"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Credit Cards"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "41",
"value": "Mastercard"
},
{
"value": "157.72"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "CreditCards",
"Summary": {
"ColData": [
{
"value": "Total Credit Cards"
},
{
"value": "157.72"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Other Current Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "89",
"value": "Arizona Dept. of Revenue Payable"
},
{
"value": "4.55"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "90",
"value": "Board of Equalization Payable"
},
{
"value": "401.98"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "43",
"value": "Loan Payable"
},
{
"value": "4000.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "OtherCurrentLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Other Current Liabilities"
},
{
"value": "4406.53"
}
]
}
}
]
},
"type": "Section",
"group": "CurrentLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Current Liabilities"
},
{
"value": "6548.42"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Long-Term Liabilities"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "44",
"value": "Notes Payable"
},
{
"value": "25000.00"
}
],
"type": "Data"
}
]
},
"type": "Section",
"group": "LongTermLiabilities",
"Summary": {
"ColData": [
{
"value": "Total Long-Term Liabilities"
},
{
"value": "25000.00"
}
]
}
}
]
},
"type": "Section",
"group": "Liabilities",
"Summary": {
"ColData": [
{
"value": "Total Liabilities"
},
{
"value": "31548.42"
}
]
}
},
{
"Header": {
"ColData": [
{
"value": "Equity"
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"id": "34",
"value": "Opening Balance Equity"
},
{
"value": "-9337.50"
}
],
"type": "Data"
},
{
"ColData": [
{
"id": "2",
"value": "Retained Earnings"
},
{
"value": "91.25"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Net Income"
},
{
"value": "2440.27"
}
],
"type": "Data",
"group": "NetIncome"
}
]
},
"type": "Section",
"group": "Equity",
"Summary": {
"ColData": [
{
"value": "Total Equity"
},
{
"value": "-6805.98"
}
]
}
}
]
},
"type": "Section",
"group": "TotalLiabilitiesAndEquity",
"Summary": {
"ColData": [
{
"value": "TOTAL LIABILITIES AND EQUITY"
},
{
"value": "24742.44"
}
]
}
}
]
},
"Columns": {
"Column": [
{
"ColType": "Account",
"ColTitle": "",
"MetaData": [
{
"Name": "ColKey",
"Value": "account"
}
]
},
{
"ColType": "Money",
"ColTitle": "Total",
"MetaData": [
{
"Name": "ColKey",
"Value": "total"
}
]
}
]
}
}
Here is what I tried to get the ColData (unsuccessfully I might add), I think it might be a little too contrived to do in SQL but I'm not sure if I should continue trying this way or if there's a better way in Python:
declare #json nvarchar(max)
SELECT #json = json FROM QboApiRawJSONData WHERE ID = 2
--Outer layer of JSON breaks into 3 parts - header, columns, rows
SELECT * FROM OPENJSON(#json)
WITH
(
Rows nvarchar(max) AS JSON
) as MainLayer
CROSS APPLY OPENJSON (MainLayer.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as SecondaryLayer
CROSS APPLY OPENJSON (SecondaryLayer.Row)
WITH
(
Rows nvarchar(max) AS JSON
) As ThirdLayer
CROSS APPLY OPENJSON (ThirdLayer.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as FourthLayer
CROSS APPLY OPENJSON (FourthLayer.Row)
WITH
(
Rows nvarchar(max) AS JSON
) as FifthLayer
CROSS APPLY OPENJSON (FifthLayer.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as SixthLayer
CROSS APPLY OPENJSON (SixthLayer.Row)
WITH
(
Rows nvarchar(max) AS JSON
) as SeventhLayer
CROSS APPLY OPENJSON (SeventhLayer.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as EighthLayer
CROSS APPLY OPENJSON (EighthLayer.Row)
WITH
(
Rows nvarchar(max) AS JSON
) as LayerNine
---Things get funky here
CROSS APPLY OPENJSON (LayerNine.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as LayerTen
CROSS APPLY OPENJSON (LayerTen.Row)
WITH
(
Rows nvarchar(max) AS JSON
) as LayerEleven
CROSS APPLY OPENJSON (LayerEleven.Rows)
WITH
(
Row nvarchar(max) AS JSON
) as LayerTwelve
--21 items in last col
There is JSON support for sql-server:
https://learn.microsoft.com/en-us/sql/relational-databases/json/json-data-sql-server?view=sql-server-ver15
There is a JSON storage method here: https://learn.microsoft.com/en-us/sql/relational-databases/json/store-json-documents-in-sql-tables?view=sql-server-ver15. Although it will be fairly complex here I recommend storing the JSON data as a Logs table and then following the tutorial above to see if that solves your issue.
Use Quickbooks API to get the JSON, then refer to this guide:
https://learn.microsoft.com/en-us/sql/relational-databases/json/json-data-sql-server?view=sql-server-ver15
and this guide:
https://learn.microsoft.com/en-us/sql/relational-databases/json/convert-json-data-to-rows-and-columns-with-openjson-sql-server?view=sql-server-ver15
You can also consider setting something like AWS Lambda or Google Cloud Functions if you need something more automated.

How to regex all uri in a specific json key value

I have been trying to be able to regex a special value: "uri": in a HTML that looks like:
test = {
"enrichedEntity": {
"type": "product",
"id": "ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21",
"hints": [
"gallery",
"octopus:b14efd56-cfd9-4fb1-8383-0c37e99f2afb:kubernetes",
"octopus:57a5852b-d16a-4021-8e8b-143bcdb9a815:Without gradient background",
"octopus:46a18f4c-58e4-4fcc-94b5-ffdd14540983:WITHOUT_GRADIENT_BACKGROUND",
]
},
"graphqlCache": {
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"barrythrill.prodpres.too.large.article\",\"barrythrill.prodpres.too.small.article\",\"barrythrill.prodpres.standard.article\",\"barrythrill.prodpres.size_reco.message.recommended_size_shoe\",\"barrythrill.prodpres.size_reco.message.recommended_size\",\"pdp.dialog.tile.sizing.review.cta.modal\",\"pdp.dialog.tile.sizing.sizereco\",\"pdp.dialog.tile.sizing.modal.cta\",\"wardrobe.pdp.bought.notification\",\"wardrobe.pdp.returned.notification\",\"wardrobe.pdp.bought.size\",\"wardrobe.pdp.bought.link\"],\"host\":\"www.barrythrill.se\"}}": {
"data": {
"translations": {
"barrythrill.prodpres.too.large.article": "Produkten är stor i storleken, så vårt tips är att ta en storlek mindre än vanligt.",
"barrythrill.prodpres.too.small.article": "Produkten är liten i storleken, så vårt tips är att ta en storlek större än vanligt.",
"barrythrill.prodpres.standard.article": "Vi rekommenderar vanlig storlek.",
"barrythrill.prodpres.size_reco.message.recommended_size_shoe": "Wir empfehlen dir %3Cb>Größe {0}%3C/b>.",
"barrythrill.prodpres.size_reco.message.recommended_size": "Vi rekommenderar %3Cb>storlek {0} %3C/b>.",
"pdp.dialog.tile.sizing.review.cta.modal": "Varför?",
"pdp.dialog.tile.sizing.sizereco": "Vill du få bättre",
"pdp.dialog.tile.sizing.modal.cta": "storleksrekommendationer?",
"wardrobe.pdp.bought.notification": "Du köpte {0} {1}",
"wardrobe.pdp.returned.notification": "Du returnerade denna produkt {0} {1}",
"wardrobe.pdp.bought.size": "storlek {0}",
"wardrobe.pdp.bought.link": "Se denna produkt i din garderob"
}
}
},
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"barrythrill.prodpres.delivery.long.distance.item\",\"barrythrill.prodpres.size.not_available\",\"barrythrill.prodpres.size.availability.equal.2or3\",\"barrythrill.prodpres.size.availability.equal1\",\"barrythrill.prodpres.size.selection.hint\",\"voucher.form.hl.step2\",\"barrythrill.prodpres.size.local\",\"barrythrill.prodpres.size.supplier\",\"barrythrill.prodpres.add_to_cart\",\"barrythrill.prodpres.size.request.title\",\"barrythrill.prodpres.size_reminder.text\",\"barrythrill.prodpres.size.reminder.submit\",\"barrythrill.prodpres.size.reminder.success\",\"membership.pdp.sizereminder.eligible.link\",\"membership.pdp.sizereminder.eligible.text\",\"membership.pdp.sizereminder.active.text\",\"pdp.dialog.tile.sizing.modal.no.fit.cta\",\"pdp.dialog.tile.out.of.stock.email.address\",\"pdp.dialog.tile.out.of.stock.notify.me.cta\",\"pdp.dialog.tile.out.of.stock.email.error\",\"pdp.dialog.tile.sizing.error\",\"barrythrill.prodpres.size.headline\",\"pdp.dialog.tile.thank.you.headline\",\"pdp.dialog.tile.thank.you.body\",\"pdp.dialog.tile.sizing.modal.no.fit.headline\",\"pdp.dialog.tile.sizing.modal.no.fit.body\",\"pdp.dialog.tile.sizing.success.change.cta\",\"pdp.dialog.tile.out.of.stock.headline\",\"pdp.dialog.tile.out.of.stock.body\",\"pdp.dialog.tile.sizing.headline\",\"pdp.dialog.tile.sizing.body\",\"pdp.dialog.tile.sizing.submit\",\"pdp.dialog.tile.sizing.success.headline\",\"pdp.dialog.tile.sizing.success.body\",\"pdp.dialog.tile.sizing.success.cta.add.to.bag\",\"pdp.dialog.tile.out.of.stock.message\",\"pdp.dialog.tile.sizing.modal.headline\",\"pdp.dialog.tile.sizing.modal.body\",\"pdp.dialog.tile.sizing.modal.body2\",\"barrythrill.prodpres.size_reco.profile.link\",\"pdp.dialog.tile.sizing.modal.change.cta\",\"pdp.dialog.tile.sizing.size\",\"pdp.dialog.tile.sizing.brand\",\"catalog.article.brand\",\"mobile.app.filter.search.brand.placeholderText\"],\"host\":\"www.barrythrill.se\"}}": {
},
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"max.order.limit.notification.pdp\",\"max.customer.limit.notification.pdp\"],\"host\":\"www.barrythrill.se\"}}": {
"data": {
"translations": {
"max.order.limit.notification.pdp": "Es sieht so aus, als hätten Sie bereits die maximale Anzahl exklusiver Artikel gekauft.",
"max.customer.limit.notification.pdp": "Sie haben dieses Produkt bereits gekauft und als Kunde die Höchstgrenze erreicht."
}
}
},
"{\"id\":\"6d02003e11d0bb4d2aa57493c0453715a0e1a20ca8c5ae3e853c57eaaa99621b\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"comingSoon": true,
"condition": null,
"simplesWithStock": [
{
"sku": "JOC12O006-A210070000"
}
],
"relevantEntities": {
"edges": [
{
"node": {
"id": "ern:collection:res:res-reco-products-similar-to-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "SIMILAR"
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-with-same-tag-as-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "SAME_TAG"
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-sponsored:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-xsell-to-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "CROSS_SELL"
}
}
}
]
}
}
}
},
"{\"id\":\"86e6491a72545f4c91d7dc380d5c3f40ee094fdd5037845410f24de0ddd3dcf3\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"displayPrice": {
"original": {
"formatted": "1 095,00 kr"
},
"promotional": null
},
"brand": {
"name": "Clothes"
},
"color": {
"name": "light smoke grey/gym red/white/black",
"label": "vit"
},
"urisInAllLocales": [
{
"locale": "fi-FI",
"uri": "/Clothes-air-1-matalavartiset-tennarit-light-smoke-greygym-redwhiteblack-joc12o006-a21.html"
}
],
"name": "AIR 1 - Sneakers - light smoke grey/gym red/white/black",
"category": "Sneakers",
"sku": "JOC12O006-A21",
"primaryImage": {
"uri": "https://img01.ztat.net/article/spp-media-p1/0034959e2d1a3105aa6c338b9cea6ffd/28bcaf048f714eb8b66d715a43848c6b.jpg?imwidth=103&filter=packshot"
}
}
}
},
"{\"id\":\"c34970edbf9399da7d80f7007b06df1ba5039e994b73558c50f864707704e828\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"name": "AIR 1 - Sneakers - light smoke grey/gym red/white/black",
"sku": "JOC12O006-A21",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21.html",
"comingSoon": true,
"brand": {
"name": "Clothes",
"uri": "https://www.barrythrill.se/Clothes/"
},
"family": {
"products": {
"edges": [
{
"node": {
"sku": "JOC12O006-A15",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a15.html",
"color": {
"name": "white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-Q14",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-q14.html",
"color": {
"name": "gym red/black/white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-A13",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a13.html",
"color": {
"name": "white/hyper crimson/light bone/vintage lichen"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-A12",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a12.html",
"color": {
"name": "white/pure platinum"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-C11",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-c11.html",
"color": {
"name": "black/light smoke grey/white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=762&filter=packshot"
}
]
}
]
}
},
"inWishlist": false,
"sizeAdvice": null,
"benefits": [
]
}
}
}
I have shortened the code quite a lot to be able to show you guys the code. I removed the HTML which contains this JSON snippet.
What I have done right now is following:
test = re.findall(
r'(?<="uri":)"(https:\/\/www.barrythrill[^"]+.html)"',
test,
re.M | re.S
)
The problem with the code is that it will scrape all that contains www.barrythrill.se/* and I do just want to scrape all uri elements in the:
"family": {
"products": {
"edges": [
Output:
It should only print out these links since that's the only links that are inside the "family": { "products": { "edges": [
[https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a15.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-q14.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a13.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a12.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-c11.html]
My question is, how can I scrape only the uri's inside the hierarchy I mentioned above?

How do you deploy a python azure function with an arm template?

The following deploys a azure function that run the specified C#. How do I do the same for a function that should run python?
I tried just changing the name to __init__.py as is generated when you use the azure-function-core-tools func command with the --python switch, but couldn't even find error messages as to why things weren't working.
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"appName": {
"type": "string",
"metadata": {
"description": "The name of the function app that you wish to create."
}
},
"storageAccountType": {
"type": "string",
"defaultValue": "Standard_LRS",
"allowedValues": [
"Standard_LRS",
"Standard_GRS",
"Standard_ZRS",
"Premium_LRS"
],
"metadata": {
"description": "Storage Account type"
}
}
},
"variables": {
"functionAppName": "[parameters('appName')]",
"hostingPlanName": "[parameters('appName')]",
"storageAccountName": "[concat(uniquestring(resourceGroup().id), 'azfunctions')]"
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"name": "[variables('storageAccountName')]",
"apiVersion": "2015-06-15",
"location": "[resourceGroup().location]",
"properties": {
"accountType": "[parameters('storageAccountType')]"
}
},
{
"type": "Microsoft.Web/serverfarms",
"apiVersion": "2015-04-01",
"name": "[variables('hostingPlanName')]",
"location": "[resourceGroup().location]",
"properties": {
"name": "[variables('hostingPlanName')]",
"computeMode": "Dynamic",
"sku": "Dynamic"
}
},
{
"apiVersion": "2015-08-01",
"type": "Microsoft.Web/sites",
"name": "[variables('functionAppName')]",
"location": "[resourceGroup().location]",
"kind": "functionapp",
"properties": {
"name": "[variables('functionAppName')]",
"serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]"
},
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]",
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
],
"resources": [
{
"apiVersion": "2016-03-01",
"name": "appsettings",
"type": "config",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', variables('functionAppName'))]",
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
],
"properties": {
"AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=',variables('storageAccountName'),';AccountKey=',listkeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2015-05-01-preview').key1,';')]",
"AzureWebJobsDashboard": "[concat('DefaultEndpointsProtocol=https;AccountName=',variables('storageAccountName'),';AccountKey=',listkeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2015-05-01-preview').key1,';')]",
"FUNCTIONS_EXTENSION_VERSION": "latest"
}
},
{
"apiVersion": "2015-08-01",
"name": "TestFunctionCM",
"type": "functions",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', variables('functionAppName'))]"
],
"properties": {
"config": {
"bindings": [
{
"authLevel": "anonymous",
"name": "req",
"type": "httpTrigger",
"direction": "in"
},
{
"name": "res",
"type": "http",
"direction": "out"
}
]
},
"files": {
"run.csx": "using System.Net;\r\n\r\n public static HttpResponseMessage Run(HttpRequestMessage req, TraceWriter log)\r\n\r\n {\r\n\r\nreturn req.CreateResponse(\"Hello from MyFunction\", HttpStatusCode.OK);\r\n\r\n }"
}
}
}
]
}
]
}
Thank you.
You will probably need the following:
Runtime under appsettings
"FUNCTIONS_WORKER_RUNTIME": "python"
My template looks bit different but does deploy a python function, here is the resource from the same:
{
"type": "Microsoft.Web/sites",
"apiVersion": "2018-11-01",
"name": "[parameters('name')]",
"location": "[parameters('location')]",
"dependsOn": [
"microsoft.insights/components/mycoolfunction",
"[concat('Microsoft.Web/serverfarms/', parameters('hostingPlanName'))]",
"[concat('Microsoft.Storage/storageAccounts/', parameters('storageAccountName'))]"
],
"tags": {},
"kind": "functionapp,linux",
"properties": {
"name": "[parameters('name')]",
"siteConfig": {
"appSettings": [
{
"name": "FUNCTIONS_WORKER_RUNTIME",
"value": "python"
},
{
"name": "FUNCTIONS_EXTENSION_VERSION",
"value": "~2"
},
{
"name": "APPINSIGHTS_INSTRUMENTATIONKEY",
"value": "[reference('microsoft.insights/components/mycoolfunction', '2015-05-01').InstrumentationKey]"
},
{
"name": "AzureWebJobsStorage",
"value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.windows.net')]"
}
]
},
"serverFarmId": "[concat('/subscriptions/', parameters('subscriptionId'),'/resourcegroups/', parameters('serverFarmResourceGroup'), '/providers/Microsoft.Web/serverfarms/', parameters('hostingPlanName'))]",
"hostingEnvironment": "[parameters('hostingEnvironment')]",
"clientAffinityEnabled": false
}
}

How to extract a a specific object from JSON file using python json

Using Python I need to extract all the API (just the API endpoint names) endpoints by reading from a JSON file.
Below is the sample JSON code,
{
"swagger": "2.0",
"info": {
"title": "None",
"description": "some thing over here sample content\n",
"version": "0.1"
},
"produces": [
"application/json"
],
"basePath": "/busrouting",
"schemes": [
"https"
],
"definitions": {
"OAuth2": {
"description": "some thing over here sample content\n",
"type": "coauthor",
"flow": "implicit",
"authorization": "NONE",
"scopes": {
"Zero": "three",
"One": "two"
}
}
},
"paths": {
"/service-provider/details": {
"get": {
"description": "some thing over here sample content",
"security": [
{
"OAuth2": [
"None",
"None"
]
}
],
"parameters": [
{
"name": "service provider",
"in": "query",
"description": "List of service-provider ID's",
"required": false,
"type": "array",
"items": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Record limit. Default is 20",
"required": false,
"type": "integer"
},
{
"name": "offset",
"in": "query",
"description": "Record offset. Default is 0.",
"required": false,
"type": "integer"
}
],
"tags": [
"Service Providers"
],
"responses": {
"200": {
"description": "OK.",
"headers": {
"Link": {
"description": "some thing over here sample content\n",
"type": "array",
"items": {
"type": "string"
}
}
},
"schema": {
"type": "object",
"properties": {
"names": {
"type": "array",
"items": {
"$ref": "#/definitions/service-provider"
}
}
} #Sample add
}
},
"400": {
"description": "Bad Request"
},
"401": {
"description": "Unauthorized"
},
"403": {
"description": "Forbidden"
},
"404": {
"description": "Zero"
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/Error"
}
}
}
}
},
"/Bus/{bus-id}/names": {
"get": {
"description": "some thing over here sample content\n",
"security": [
{
"OAuth2": [
"None",
"None"
]
}
],
"parameters": [
{
"name": "Bus-id",
"in": "path",
"required": true,
"type": "string"
}
],
"tags": [
"BusNames"
],
"responses": {
"200": {
"description": "OK.",
"schema": {
"$ref": "#/definitions/busnames"
}
},
"400": {
"description": "Bad Request"
},
"401": {
"description": "Unauthorized"
},
"403": {
"description": "Forbidden"
},
"404": {
"description": "Balances not found"
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/Error"
}
}
}
}
},
"/busoperator/{bus-id}/names/routes": {
"get": {
"description": "some thing over here sample content.\n",
"security": [
{
"OAuth2": [
"None",
"None"
]
}
],
"parameters": [
{
"name": "bus-id",
"in": "path",
"required": true,
"type": "string"
},
{
"name": "limit",
"in": "query",
"description": "Record limit. Default is 10",
"required": false,
"type": "integer"
},
{
"name": "offset",
"in": "query",
"description": "Record offset. Default is 0.",
"required": false,
"type": "integer"
}
],
"tags": [
"bus route mapping"
],
"responses": {
"200": {
"description": "Recent Route",
"headers": {
"Link": {
"description": "some thing over here sample content.\n",
"type": "array",
"items": {
"type": "string"
}
}
},
"schema": {
"type": "object",
"properties": {
"Route details": {
"type": "array",
"items": {
"$ref": "#/definitions/Route"
}
}
}
}
},
"400": {
"description": "Bad Request"
},
"401": {
"description": "Unauthorized"
},
"403": {
"description": "Not entitled to this account and its transactions"
},
"404": {
"description": "User has no recent transactions"
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/Error"
}
}
}
}
},
"/operator/{id}/route/path": {
"get": {
"description": "some thing over here sample content.\n",
"security": [
{
"OAuth2": [
"None",
"None"
]
}
],
"parameters": [
{
"name": "bus-id",
"in": "path",
"required": true,
"type": "string"
},
{
"name": "limit",
"in": "query",
"description": "Record limit. Default is 10",
"required": false,
"type": "integer"
},
{
"name": "offset",
"in": "query",
"description": "Record offset. Default is 0.",
"required": false,
"type": "integer"
}
],
"tags": [
"Sample Reporting"
],
"responses": {
"200": {
"description": "sample actions",
"headers": {
"Link": {
"description": "some thing over here sample content\n",
"type": "array",
"items": {
"type": "string"
}
}
},
"schema": {
"type": "object",
"properties": {
"Boisterous": {
"type": "array",
"items": {
"$ref": "#/definitions/salesperson"
}
}
}
}
},
"400": {
"description": "Bad Request"
},
"401": {
"description": "Unauthorized"
},
"403": {
"description": "Not entitled to this account and its transactions"
},
"404": {
"description": "User has no recent transactions"
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/Error"
}
}
}
}
}
},
"definitions": {
"sample": {
"required": [
"demo",
"request"
],
"description": "Holds identifying attributes for an sample\n",
"properties": {
"request": {
"description": "some thing over here sample content\n",
"type": "string"
},
"sample one": {
"type": "string"
},
"sample two": {
"type": "string"
},
"number": {
"description": "some thing over here sample content",
"type": "string"
},
"name": {
"type": "string",
"description": "some thing over here sample content"
}
}
},
"dummy": {
"description": "some thing over here sample content\n",
"properties": {
"dummy": {
"$ref": "#/definitions/samples"
},
"data": {
"type": "string",
"description": "sample data "
},
"country": {
"type": "string",
"description": "some thing over here sample content"
},
"dataone": {
"$ref": "#/definitions/samples"
},
"error": {
"$ref": "#/definitions/Error"
},
"links": {
"description": "some thing over here sample content",
"type": "array",
"items": {
"$ref": "#/definitions/Links"
}
}
}
},
"reference": {
"description": "only description\n",
"properties": {
"available": {
"type": "number",
"format": "double",
"description": "some thing over here sample content"
},
"availableFormatted": {
"type": "string",
"description": "some thing over here sample content"
},
"held": {
"type": "number",
"format": "double",
"description": "some thing over here sample content."
},
"formatted": {
"type": "string",
"description": "some thing over here sample content"
},
"asst": {
"type": "string",
"format": "date-time",
"description": "Timestamp of , in UTC"
},
"sampler": {
"type": "string",
"format": "date-time",
"description": "Timestamp of , in users preferred TZ"
}
}
},
"dummy": {
"description": "Transaction done on an account",
"required": [
"because",
"versioned",
"postdates"
],
"properties": {
"tintype": {
"description": "sample content",
"type": "string"
},
"cringed": {
"description": "some thing over here sample content",
"type": "string",
"menu": [
"D",
"C"
]
},
"reference": {
"description": "Reference information",
"type": "string"
},
"amount": {
"description": "some thing over here sample content\n",
"type": "number",
"format": "double"
},
"formatted": {
"description": "formatted according to user's preferences\n"
},
"post Date": {
"type": "string",
"format": "date-time",
"description": "Date-time in UTC"
},
"postdate": {
"type": "string",
"format": "date-time",
"description": "Post Date-time in user-preferred TZ"
},
"narrative": {
"type": "string"
}
}
},
"Links": {
"description": "Related Links for the resource\n",
"properties": {
"rel": {
"description": "relationship to the resource",
"type": "string"
},
"ref": {
"description": "URL of the related link",
"type": "string"
}
}
},
"Error": { Sample
"description": "Error descriptor",
"properties": {
"code": {
"type": "integer",
"format": "intent"
},
"message": {
"type": "string"
},
"description": {
"type": "string"
},
"severity": {
"type": "string"
},
"location": {
"type": "string"
}
}
}
}
}
I have tried the below code:
import json
with open("example.json", "r") as reading:
data = json.load(reading)
print(data["paths"])
From here i need to go further the code to capture all the API endpoint name only.
In the sample json file, Under paths i need to capture all the endpoints and method type of an API as below,
In addition I would also like to capture the values for the below keys from the sample JSON code.
Under "parameters" i also need to capture the below keys,
name:
in:
required:
Expected output (just an example):
/service-provider/details
get
bus (comment - refers to the key - name under 'parameters')
query (comment - refers to the key - in under 'parameters')
false (comment - refers to the key - required under 'parameters')
/Bus/{bus-id}/names
get
bus-id (comment - refers to the key - name under 'parameters')
query (comment - refers to the key - in under 'parameters')
false (comment - refers to the key - required under 'parameters')
/busoperator/{bus-id}/names/routes
get
routes (comment - refers to the key - name under 'parameters')
query (comment - refers to the key - in under 'parameters')
false (comment - refers to the key - required under 'parameters')
/operator/{id}/route/path
get
id (comment - refers to the key - name under 'parameters')
query (comment - refers to the key - in under 'parameters')
false (comment - refers to the key - required under 'parameters')
Try this:
with open('example.json', 'r') as f:
data = json.load(f)
for path, values in data['paths'].items():
print(path)
for value in values:
print(value)
This should also get all the endpoints functions in a path if there are multiple.
To get a list of two-tuples (endpoint, type) use:
[(endpoint, value.keys[0]) for endpoint, value in data["paths"].items()]

Google Assistant API V2 userstorage

I'm attempting to develop a Google Action with the Dialogflow v2 API
My function saves a value to userstorage as follows
def save_value(value):
res = {
"fulfillmentText": "Set value to {}".format(int(value)),
"payload": {
"google": {
"userStorage": str(value)
}
}
}
print ("Saved value")
response = jsonify(res)
return response
And I get the following back from testing in Dialogflow
{
"fulfillmentText": "Set value to 36237269",
"payload": {
"google": {
"userStorage": "36237269"
}
}
}
This works for the duration of session, I am able to use this in later intents via
value = request_json['originalRequest']['data']['user']['userStorage']
However, the data is only stored for one session - if I invoke my action again, there is nothing saved.
Is this the correct way of using userstorage? Has anyone successfully used it with Python?
Failed "response"
{
"responseMetadata": {
"status": {
"code": 10,
"message": "Failed to parse Dialogflow response into AppResponse because of empty speech response",
"details": [
{
"#type": "type.googleapis.com/google.protobuf.Value",
"value": "{\"id\":\"816605a7-f7e0-4d37-a490-c84ff63fb7dd\",\"timestamp\":\"2018-11-08T17:18:49.422Z\",\"lang\":\"en-us\",\"result\":{},\"alternateResult\":{},\"status\":{\"code\":206,\"errorType\":\"partial_content\",\"errorDetails\":\"Webhook call failed. Error: 500 Internal Server Error\"},\"sessionId\":\"ABwppHHai3qsY2WPZWezmh9Q_bUF45aD51GbQ81sUDF7iSrRLA2m8KFgZ1ZYavnCv3fAckW1tcoJdydZTXQY5Nw\"}"
}
]
}
}
}
Working "response"
{
"conversationToken": "[]",
"finalResponse": {
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Text"
}
}
]
}
},
"responseMetadata": {
"status": {
"message": "Success (200)"
},
"queryMatchInfo": {
"queryMatched": true,
"intent": "047ad9d9-0180-47f9-88bd-e5ffc8936c08"
}
},
"userStorage": "36237269"
}
Working "Request"
{
"user": {
"userId": "ABwppHE5H0FKrXKk8PjJyzZJ12OSMQcjxuT2NnfPAgLvai0UsfWEoYE8R_L8qLQdqY29sOnsZhQhE5G4XVVXiGs",
"locale": "en-US",
"lastSeen": "2018-11-08T17:18:16Z",
"userStorage": "36237269"
},
"conversation": {
"conversationId": "ABwppHE6BwK2zIBKxHA8hc9uBGumVgKbbNGHhRVFz7O6yrxxa1WJ_xtKNqhesj3EwNCVlestM-bF6tDWzZhqUXE",
"type": "ACTIVE",
"conversationToken": "[]"
},
"inputs": [
{
"intent": "actions.intent.TEXT",
"rawInputs": [
{
"inputType": "KEYBOARD",
"query": "when is the next bus"
}
],
"arguments": [
{
"name": "text",
"rawText": "when is the next bus",
"textValue": "when is the next bus"
}
]
}
],
"surface": {
"capabilities": [
{
"name": "actions.capability.MEDIA_RESPONSE_AUDIO"
},
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
}
]
},
"isInSandbox": true,
"availableSurfaces": [
{
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
}
]
}
],
"requestType": "SIMULATOR"
}
Failed "request"
{
"user": {
"userId": "ABwppHE5H0FKrXKk8PjJyzZJ12OSMQcjxuT2NnfPAgLvai0UsfWEoYE8R_L8qLQdqY29sOnsZhQhE5G4XVVXiGs",
"locale": "en-US",
"lastSeen": "2018-11-08T17:18:41Z"
},
"conversation": {
"conversationId": "ABwppHHai3qsY2WPZWezmh9Q_bUF45aD51GbQ81sUDF7iSrRLA2m8KFgZ1ZYavnCv3fAckW1tcoJdydZTXQY5Nw",
"type": "ACTIVE",
"conversationToken": "[]"
},
"inputs": [
{
"intent": "actions.intent.TEXT",
"rawInputs": [
{
"inputType": "KEYBOARD",
"query": "when is the next bus"
}
],
"arguments": [
{
"name": "text",
"rawText": "when is the next bus",
"textValue": "when is the next bus"
}
]
}
],
"surface": {
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.MEDIA_RESPONSE_AUDIO"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
}
]
},
"isInSandbox": true,
"availableSurfaces": [
{
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
}
]
}
],
"requestType": "SIMULATOR"
}

Categories

Resources