How to delete list Items in Python - python

I want to remove the list items "snippet, contentDetails and status" if the "title" of the current object equals "Private video". Is there an easy way to do that?
Here is an example of the response JSON I receive:
{
"nextPageToken": "CDIQAA",
"items": [{
"snippet": {
"title": "Zu Tisch... im Marais Poitevien | ARTE"
},
"contentDetails": {
"videoId": "HjrsjdnpcBk"
},
"status": {
"privacyStatus": "public"
}
},
{
"snippet": {
"title": "Private video"
},
"contentDetails": {
"videoId": "arRStb8Hk00"
},
"status": {
"privacyStatus": "private"
}
}
],
"pageInfo": {
"totalResults": 81
}
}
This is what it should look like after the removal:
{
"nextPageToken": "CDIQAA",
"items": [{
"snippet": {
"title": "Zu Tisch... im Marais Poitevien | ARTE"
},
"contentDetails": {
"videoId": "HjrsjdnpcBk"
},
"status": {
"privacyStatus": "public"
}
}
],
"pageInfo": {
"totalResults": 81
}
}


A simple list comprehension should do the trick
j = {
"nextPageToken": "CDIQAA",
"items": [{
"snippet": {
"title": "Zu Tisch... im Marais Poitevien | ARTE"
},
"contentDetails": {
"videoId": "HjrsjdnpcBk"
},
"status": {
"privacyStatus": "public"
}
},
{
"snippet": {
"title": "Private video"
},
"contentDetails": {
"videoId": "arRStb8Hk00"
},
"status": {
"privacyStatus": "private"
}
}
],
"pageInfo": {
"totalResults": 81
}
}
j['items'] = [item for item in j['items'] if item['snippet']['title'] != 'Private video']
Output:
{'nextPageToken': 'CDIQAA',
'items': [{'snippet': {'title': 'Zu Tisch... im Marais Poitevien | ARTE'},
'contentDetails': {'videoId': 'HjrsjdnpcBk'},
'status': {'privacyStatus': 'public'}}],
'pageInfo': {'totalResults': 81}}

Related

groupby query on joined collection in flask mongoDB

I am currently stuck in this problem, i am relatively new to MongoDB, and i have to retrieve number of reports(count of reports done by users ) for a specific user with his name(name), last reported time(time of last reported post), last reason(report_description) ,
i am stuck here since 2 days now, help will be appreciated .
reported posts collection
{
"created_at": {
"$date": "2021-12-21T18:45:27.489Z"
},
"updated_at": {
"$date": "2021-12-21T18:45:27.489Z"
},
"post_id": {
"$oid": "61955ac35b3475f1d9759255"
},
"user_id": 2,
"report_type": "this is test",
"report_description": "this"
}
Post collection
{
"created_at": {
"$date": "2021-11-17T19:24:53.484Z"
},
"updated_at": {
"$date": "2021-11-17T19:24:53.484Z"
},
"user_id": 8,
"privacy_type": "public",
"post_type": "POST",
"post": "Om Sai Ram",
"total_like": 7,
"total_comment": 0,
"total_share": 0,
"image_url_list": [{
"image_url": "post_images/user-8/a31e39334987463bb9faa964391a935e.jpg",
"image_ratio": "1"
}],
"video_url_list": [],
"tag_list": [],
"is_hidden": false
}
User collection
{
"name": "sathish",
"user_id": 1,
"device_id": "faTOi3aVTjyQnBPFz0L7xm:APA91bHNLE9anWYrKWfwoHgmGWL2BlbWqgiVjU5iy7JooWxu26Atk9yZFxVnNp2OF1IXrXm4I6HdVJPGukEppQjSiUPdMoQ64KbOt78rpctxnYWPWliLrdxc9o1VdKL0DGYwE7Y6hx1H",
"user_name": "sathishkumar",
"updated_at": {
"$date": "2021-11-17T19:13:52.668Z"
},
"profile_picture_url": "1"
}
flask_snip.py
flagged_posts = mb.db_report.aggregate([{
'$group':{
'_id':'$user_id',
}
}])
expected out should be list e.g
[
{
'user_id':1,
'name' :'somename',
'no_of_reports':30,
'last_reported_time':sometime,
'reason':'reason_of lastreported_post',
'post_link':'someurl',
},
{
'user_id':2,
'name' :'somename',
'no_of_reports':30,
'last_reported_time':sometime,
'reason':'reason_of last_reported_post',
'post_link':'someurl',
},
{
'user_id':3,
'name' :'somename',
'no_of_reports':30,
'last_reported_time':sometime,
'reason':'reason_of lastreported_post',
'post_link':'someurl',
},
]

Starting from the reported collection, you can $group to get the last_reason and last_reported_time. Then, perform a $lookup to user collection to get the name.
db.reported.aggregate([
{
"$sort": {
updated_at: -1
}
},
{
"$group": {
"_id": "$user_id",
"last_reported_time": {
"$first": "$updated_at"
},
"last_reason": {
"$first": "$report_description"
},
"no_of_reports": {
$sum: 1
}
}
},
{
"$lookup": {
"from": "user",
"localField": "_id",
"foreignField": "user_id",
"as": "userLookup"
}
},
{
"$unwind": "$userLookup"
},
{
"$project": {
"user_id": "$_id",
"name": "$userLookup.user_name",
"no_of_reports": 1,
"last_reported_time": 1,
"last_reason": 1
}
}
])
Here is the Mongo playground for your reference.

Merge 2 json files with jsonmerge

I want to merge many JSON files with the same nested structure, using jsonmerge, but have been unsuccessful so far. For example, I want to merge base and head:
base = {
"data": [
{
"author_id": "id1",
"id": "1"
},
{
"author_id": "id2",
"id": "2"
}
],
"includes": {
"users": [
{
"id": "user1",
"name": "user1"
},
{
"id": "user2",
"name": "user2"
}
]
}
}
head = {
"data": [
{
"author_id": "id3",
"id": "3"
},
{
"author_id": "id4",
"id": "4"
}
],
"includes": {
"users": [
{
"id": "user3",
"name": "user3"
},
{
"id": "user4",
"name": "user4"
}
]
}
}
The resulting JSON should be:
final_result = {
"data": [
{
"author_id": "id1",
"id": "1"
},
{
"author_id": "id2",
"id": "2"
},
{
"author_id": "id3",
"id": "3"
},
{
"author_id": "id4",
"id": "4"
}
],
"includes": {
"users": [
{
"id": "user1",
"name": "user1"
},
{
"id": "user2",
"name": "user2"
},
{
"id": "user3",
"name": "user3"
},
{
"id": "user4",
"name": "user4"
}
]
}
}
However, I've only managed to merge correctly the data fields, while for users it doesn't seem to work. This is my code:
from jsonmerge import merge
from jsonmerge import Merger
schema = { "properties": {
"data": {
"mergeStrategy": "append"
},
"includes": {
"users": {
"mergeStrategy": "append"
}
}
}
}
merger = Merger(schema)
result = merger.merge(base, head)
The end result is:
{'data': [{'author_id': 'id1', 'id': '1'},
{'author_id': 'id2', 'id': '2'},
{'author_id': 'id3', 'id': '3'},
{'author_id': 'id4', 'id': '4'}],
'includes': {'users': [{'id': 'user3', 'name': 'user3'},
{'id': 'user4', 'name': 'user4'}]}}
The issue is with the definition of the schema, but I do not know if it is possible to do it like that with jsonmerge. Any help is appreciated!
Thank you!

It is based on jsonschema. So when you have an object within an object (e.g. "users" within "includes") then you'll need to tell jsonschema it is dealing with another object like so:
schema = {
"properties": {
"data": {
"mergeStrategy": "append"
},
"includes": {
"type": "object",
"properties": {
"users": {
"mergeStrategy": "append"
}
}
}
}
}
Note that this also happens for your top-level objects, hence you have "properties" argument on the highest level.

How to regex all uri in a specific json key value

I have been trying to be able to regex a special value: "uri": in a HTML that looks like:
test = {
"enrichedEntity": {
"type": "product",
"id": "ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21",
"hints": [
"gallery",
"octopus:b14efd56-cfd9-4fb1-8383-0c37e99f2afb:kubernetes",
"octopus:57a5852b-d16a-4021-8e8b-143bcdb9a815:Without gradient background",
"octopus:46a18f4c-58e4-4fcc-94b5-ffdd14540983:WITHOUT_GRADIENT_BACKGROUND",
]
},
"graphqlCache": {
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"barrythrill.prodpres.too.large.article\",\"barrythrill.prodpres.too.small.article\",\"barrythrill.prodpres.standard.article\",\"barrythrill.prodpres.size_reco.message.recommended_size_shoe\",\"barrythrill.prodpres.size_reco.message.recommended_size\",\"pdp.dialog.tile.sizing.review.cta.modal\",\"pdp.dialog.tile.sizing.sizereco\",\"pdp.dialog.tile.sizing.modal.cta\",\"wardrobe.pdp.bought.notification\",\"wardrobe.pdp.returned.notification\",\"wardrobe.pdp.bought.size\",\"wardrobe.pdp.bought.link\"],\"host\":\"www.barrythrill.se\"}}": {
"data": {
"translations": {
"barrythrill.prodpres.too.large.article": "Produkten är stor i storleken, så vårt tips är att ta en storlek mindre än vanligt.",
"barrythrill.prodpres.too.small.article": "Produkten är liten i storleken, så vårt tips är att ta en storlek större än vanligt.",
"barrythrill.prodpres.standard.article": "Vi rekommenderar vanlig storlek.",
"barrythrill.prodpres.size_reco.message.recommended_size_shoe": "Wir empfehlen dir %3Cb>Größe {0}%3C/b>.",
"barrythrill.prodpres.size_reco.message.recommended_size": "Vi rekommenderar %3Cb>storlek {0} %3C/b>.",
"pdp.dialog.tile.sizing.review.cta.modal": "Varför?",
"pdp.dialog.tile.sizing.sizereco": "Vill du få bättre",
"pdp.dialog.tile.sizing.modal.cta": "storleksrekommendationer?",
"wardrobe.pdp.bought.notification": "Du köpte {0} {1}",
"wardrobe.pdp.returned.notification": "Du returnerade denna produkt {0} {1}",
"wardrobe.pdp.bought.size": "storlek {0}",
"wardrobe.pdp.bought.link": "Se denna produkt i din garderob"
}
}
},
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"barrythrill.prodpres.delivery.long.distance.item\",\"barrythrill.prodpres.size.not_available\",\"barrythrill.prodpres.size.availability.equal.2or3\",\"barrythrill.prodpres.size.availability.equal1\",\"barrythrill.prodpres.size.selection.hint\",\"voucher.form.hl.step2\",\"barrythrill.prodpres.size.local\",\"barrythrill.prodpres.size.supplier\",\"barrythrill.prodpres.add_to_cart\",\"barrythrill.prodpres.size.request.title\",\"barrythrill.prodpres.size_reminder.text\",\"barrythrill.prodpres.size.reminder.submit\",\"barrythrill.prodpres.size.reminder.success\",\"membership.pdp.sizereminder.eligible.link\",\"membership.pdp.sizereminder.eligible.text\",\"membership.pdp.sizereminder.active.text\",\"pdp.dialog.tile.sizing.modal.no.fit.cta\",\"pdp.dialog.tile.out.of.stock.email.address\",\"pdp.dialog.tile.out.of.stock.notify.me.cta\",\"pdp.dialog.tile.out.of.stock.email.error\",\"pdp.dialog.tile.sizing.error\",\"barrythrill.prodpres.size.headline\",\"pdp.dialog.tile.thank.you.headline\",\"pdp.dialog.tile.thank.you.body\",\"pdp.dialog.tile.sizing.modal.no.fit.headline\",\"pdp.dialog.tile.sizing.modal.no.fit.body\",\"pdp.dialog.tile.sizing.success.change.cta\",\"pdp.dialog.tile.out.of.stock.headline\",\"pdp.dialog.tile.out.of.stock.body\",\"pdp.dialog.tile.sizing.headline\",\"pdp.dialog.tile.sizing.body\",\"pdp.dialog.tile.sizing.submit\",\"pdp.dialog.tile.sizing.success.headline\",\"pdp.dialog.tile.sizing.success.body\",\"pdp.dialog.tile.sizing.success.cta.add.to.bag\",\"pdp.dialog.tile.out.of.stock.message\",\"pdp.dialog.tile.sizing.modal.headline\",\"pdp.dialog.tile.sizing.modal.body\",\"pdp.dialog.tile.sizing.modal.body2\",\"barrythrill.prodpres.size_reco.profile.link\",\"pdp.dialog.tile.sizing.modal.change.cta\",\"pdp.dialog.tile.sizing.size\",\"pdp.dialog.tile.sizing.brand\",\"catalog.article.brand\",\"mobile.app.filter.search.brand.placeholderText\"],\"host\":\"www.barrythrill.se\"}}": {
},
"{\"id\":\"cb96571222c2df7160e55fd1765c7044a48026b6d47f61e99f9bdd6afffbe9f8\",\"variables\":{\"keys\":[\"max.order.limit.notification.pdp\",\"max.customer.limit.notification.pdp\"],\"host\":\"www.barrythrill.se\"}}": {
"data": {
"translations": {
"max.order.limit.notification.pdp": "Es sieht so aus, als hätten Sie bereits die maximale Anzahl exklusiver Artikel gekauft.",
"max.customer.limit.notification.pdp": "Sie haben dieses Produkt bereits gekauft und als Kunde die Höchstgrenze erreicht."
}
}
},
"{\"id\":\"6d02003e11d0bb4d2aa57493c0453715a0e1a20ca8c5ae3e853c57eaaa99621b\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"comingSoon": true,
"condition": null,
"simplesWithStock": [
{
"sku": "JOC12O006-A210070000"
}
],
"relevantEntities": {
"edges": [
{
"node": {
"id": "ern:collection:res:res-reco-products-similar-to-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "SIMILAR"
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-with-same-tag-as-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "SAME_TAG"
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-sponsored:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
}
}
},
{
"node": {
"id": "ern:collection:res:res-reco-products-xsell-to-product:_CnULuWVdQq_ynK8PrPQKFN36VqPrkkTiUJGG-zK0FqFzxG1MSggeDNrdK86DA9m.cQJUL5yP5OqWN4tvHgLJlia0redeTqgkT9tuP50i6DPHuyyMwQSJv2Y1VQDQkGqpEA60sOVfeDiVh5_att-9M8_rsMVlko7FgSlG9b_WKCwGVhhk2nvI7ynYVRGu4ocDABzjUVvz-EGWxc8ujmNpis_nJQS4mLqG1_DfWbMHUA7JHAc5zxeyXR9_gF9GzBOdIcXeGK59Ta-1WuSCXHKYBxxkq-2QE5iP7IfTUg7XACOonSBWwMfFtvx6AFYVwq7Z0ZqCc1r8o8J7DGiArW-Wv58T-KiOY4VOGto1F38kzGTGaCDfcFkA0AiQmM52gQDeEbww94i3IZI.v2",
"reason": {
"kind": "CROSS_SELL"
}
}
}
]
}
}
}
},
"{\"id\":\"86e6491a72545f4c91d7dc380d5c3f40ee094fdd5037845410f24de0ddd3dcf3\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"displayPrice": {
"original": {
"formatted": "1 095,00 kr"
},
"promotional": null
},
"brand": {
"name": "Clothes"
},
"color": {
"name": "light smoke grey/gym red/white/black",
"label": "vit"
},
"urisInAllLocales": [
{
"locale": "fi-FI",
"uri": "/Clothes-air-1-matalavartiset-tennarit-light-smoke-greygym-redwhiteblack-joc12o006-a21.html"
}
],
"name": "AIR 1 - Sneakers - light smoke grey/gym red/white/black",
"category": "Sneakers",
"sku": "JOC12O006-A21",
"primaryImage": {
"uri": "https://img01.ztat.net/article/spp-media-p1/0034959e2d1a3105aa6c338b9cea6ffd/28bcaf048f714eb8b66d715a43848c6b.jpg?imwidth=103&filter=packshot"
}
}
}
},
"{\"id\":\"c34970edbf9399da7d80f7007b06df1ba5039e994b73558c50f864707704e828\",\"variables\":{\"id\":\"ern:product:uri:Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21\"}}": {
"data": {
"product": {
"id": "ern:product::JOC12O006-A21",
"name": "AIR 1 - Sneakers - light smoke grey/gym red/white/black",
"sku": "JOC12O006-A21",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-light-smoke-greygym-redwhiteblack-joc12o006-a21.html",
"comingSoon": true,
"brand": {
"name": "Clothes",
"uri": "https://www.barrythrill.se/Clothes/"
},
"family": {
"products": {
"edges": [
{
"node": {
"sku": "JOC12O006-A15",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a15.html",
"color": {
"name": "white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ff8266dcd8363db590f8cfd2e85a6b0a/cf0dc2ddd3884a78835d4ce8f972b56f.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-Q14",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-q14.html",
"color": {
"name": "gym red/black/white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/2e4ae62f312b3cbc9f2892b71c233cfe/adb2bac4283e49dd8297955d4816fa14.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-A13",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a13.html",
"color": {
"name": "white/hyper crimson/light bone/vintage lichen"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/9e3d10b1e7973ce297f8d7064ba68daf/97bef05fea1442f99933d0b968d048ef.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-A12",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a12.html",
"color": {
"name": "white/pure platinum"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/ce31897aec983e1a9f9d3e1b1ff4f30b/c445180f638d44b18c9b40db8331a18f.jpg?imwidth=762&filter=packshot"
}
]
}
},
{
"node": {
"sku": "JOC12O006-C11",
"uri": "https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-c11.html",
"color": {
"name": "black/light smoke grey/white"
},
"beautyColorThumbnail": null,
"standardColorThumbnail": {
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=156&filter=packshot"
},
"fallbackStandardColorThumbnails": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=156&filter=packshot"
}
],
"galleryMedia": [
{
"uri": "https://img01.ztat.net/article/spp-media-p1/dea0b4c2d96c3bb1ad82d69ce8388440/ec826d3cccf24b74ac20d42a20dc6d69.jpg?imwidth=762&filter=packshot"
}
]
}
]
}
},
"inWishlist": false,
"sizeAdvice": null,
"benefits": [
]
}
}
}
I have shortened the code quite a lot to be able to show you guys the code. I removed the HTML which contains this JSON snippet.
What I have done right now is following:
test = re.findall(
r'(?<="uri":)"(https:\/\/www.barrythrill[^"]+.html)"',
test,
re.M | re.S
)
The problem with the code is that it will scrape all that contains www.barrythrill.se/* and I do just want to scrape all uri elements in the:
"family": {
"products": {
"edges": [
Output:
It should only print out these links since that's the only links that are inside the "family": { "products": { "edges": [
[https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a15.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-q14.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a13.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-a12.html,
https://www.barrythrill.se/Clothes-air-1-sneakers-joc12o006-c11.html]
My question is, how can I scrape only the uri's inside the hierarchy I mentioned above?

Google Assistant API V2 userstorage

I'm attempting to develop a Google Action with the Dialogflow v2 API
My function saves a value to userstorage as follows
def save_value(value):
res = {
"fulfillmentText": "Set value to {}".format(int(value)),
"payload": {
"google": {
"userStorage": str(value)
}
}
}
print ("Saved value")
response = jsonify(res)
return response
And I get the following back from testing in Dialogflow
{
"fulfillmentText": "Set value to 36237269",
"payload": {
"google": {
"userStorage": "36237269"
}
}
}
This works for the duration of session, I am able to use this in later intents via
value = request_json['originalRequest']['data']['user']['userStorage']
However, the data is only stored for one session - if I invoke my action again, there is nothing saved.
Is this the correct way of using userstorage? Has anyone successfully used it with Python?
Failed "response"
{
"responseMetadata": {
"status": {
"code": 10,
"message": "Failed to parse Dialogflow response into AppResponse because of empty speech response",
"details": [
{
"#type": "type.googleapis.com/google.protobuf.Value",
"value": "{\"id\":\"816605a7-f7e0-4d37-a490-c84ff63fb7dd\",\"timestamp\":\"2018-11-08T17:18:49.422Z\",\"lang\":\"en-us\",\"result\":{},\"alternateResult\":{},\"status\":{\"code\":206,\"errorType\":\"partial_content\",\"errorDetails\":\"Webhook call failed. Error: 500 Internal Server Error\"},\"sessionId\":\"ABwppHHai3qsY2WPZWezmh9Q_bUF45aD51GbQ81sUDF7iSrRLA2m8KFgZ1ZYavnCv3fAckW1tcoJdydZTXQY5Nw\"}"
}
]
}
}
}
Working "response"
{
"conversationToken": "[]",
"finalResponse": {
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Text"
}
}
]
}
},
"responseMetadata": {
"status": {
"message": "Success (200)"
},
"queryMatchInfo": {
"queryMatched": true,
"intent": "047ad9d9-0180-47f9-88bd-e5ffc8936c08"
}
},
"userStorage": "36237269"
}
Working "Request"
{
"user": {
"userId": "ABwppHE5H0FKrXKk8PjJyzZJ12OSMQcjxuT2NnfPAgLvai0UsfWEoYE8R_L8qLQdqY29sOnsZhQhE5G4XVVXiGs",
"locale": "en-US",
"lastSeen": "2018-11-08T17:18:16Z",
"userStorage": "36237269"
},
"conversation": {
"conversationId": "ABwppHE6BwK2zIBKxHA8hc9uBGumVgKbbNGHhRVFz7O6yrxxa1WJ_xtKNqhesj3EwNCVlestM-bF6tDWzZhqUXE",
"type": "ACTIVE",
"conversationToken": "[]"
},
"inputs": [
{
"intent": "actions.intent.TEXT",
"rawInputs": [
{
"inputType": "KEYBOARD",
"query": "when is the next bus"
}
],
"arguments": [
{
"name": "text",
"rawText": "when is the next bus",
"textValue": "when is the next bus"
}
]
}
],
"surface": {
"capabilities": [
{
"name": "actions.capability.MEDIA_RESPONSE_AUDIO"
},
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
}
]
},
"isInSandbox": true,
"availableSurfaces": [
{
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
}
]
}
],
"requestType": "SIMULATOR"
}
Failed "request"
{
"user": {
"userId": "ABwppHE5H0FKrXKk8PjJyzZJ12OSMQcjxuT2NnfPAgLvai0UsfWEoYE8R_L8qLQdqY29sOnsZhQhE5G4XVVXiGs",
"locale": "en-US",
"lastSeen": "2018-11-08T17:18:41Z"
},
"conversation": {
"conversationId": "ABwppHHai3qsY2WPZWezmh9Q_bUF45aD51GbQ81sUDF7iSrRLA2m8KFgZ1ZYavnCv3fAckW1tcoJdydZTXQY5Nw",
"type": "ACTIVE",
"conversationToken": "[]"
},
"inputs": [
{
"intent": "actions.intent.TEXT",
"rawInputs": [
{
"inputType": "KEYBOARD",
"query": "when is the next bus"
}
],
"arguments": [
{
"name": "text",
"rawText": "when is the next bus",
"textValue": "when is the next bus"
}
]
}
],
"surface": {
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.MEDIA_RESPONSE_AUDIO"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
}
]
},
"isInSandbox": true,
"availableSurfaces": [
{
"capabilities": [
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
}
]
}
],
"requestType": "SIMULATOR"
}

Convert deeply nested json from facebook to dataframe in python

I am trying to get user details of persons who has put likes, comments on Facebook posts. I am using python facebook-sdk package. Code is as follows.
import facebook as fi
import json
graph = fi.GraphAPI('Access Token')
data = json.dumps(graph.get_object('DSIfootcandy/posts'))
From the above, I am getting a highly nested json. Here I will put only a json string for one post in the fb.
{
"paging": {
"next": "https://graph.facebook.com/v2.0/425073257683630/posts?access_token=&limit=25&until=1449201121&__paging_token=enc_AdD0DL6sN3aDZCwfYY25rJLW9IZBZCLM1QfX0venal6rpjUNvAWZBOoxTjbOYZAaFiBImzMqiv149HPH5FBJFo0nSVOPqUy78S0YvwZDZD",
"previous": "https://graph.facebook.com/v2.0/425073257683630/posts?since=1450843741&access_token=&limit=25&__paging_token=enc_AdCYobFJpcNavx6STzfPFyFe6eQQxRhkObwl2EdulwL7mjbnIETve7sJZCPMwVm7lu7yZA5FoY5Q4sprlQezF4AlGfZCWALClAZDZD&__previous=1"
},
"data": [
{
"picture": "https://fbcdn-photos-e-a.akamaihd.net/hphotos-ak-xfa1/v/t1.0-0/p130x130/1285_5066979392443_n.png?oh=b37a42ee58654f08af5abbd4f52b1ace&oe=570898E7&__gda__=1461440649_aa94b9ec60f22004675c4a527e8893f",
"is_hidden": false,
"likes": {
"paging": {
"cursors": {
"after": "MTU3NzQxODMzNTg0NDcwNQ==",
"before": "MTU5Mzc1MjA3NDE4ODgwMA=="
}
},
"data": [
{
"id": "1593752074188800",
"name": "Maduri Priyadarshani"
},
{
"id": "427605680763414",
"name": "Darshi Mashika"
},
{
"id": "599793563453832",
"name": "Shakeer Nimeshani Shashikala"
},
{
"id": "1577418335844705",
"name": "Däzlling Jalali Muishu"
}
]
},
"from": {
"category": "Retail and Consumer Merchandise",
"name": "Footcandy",
"category_list": [
{
"id": "2239",
"name": "Retail and Consumer Merchandise"
}
],
"id": "425073257683630"
},
"name": "Timeline Photos",
"privacy": {
"allow": "",
"deny": "",
"friends": "",
"description": "",
"value": ""
},
"is_expired": false,
"comments": {
"paging": {
"cursors": {
"after": "WTI5dGJXVnVkRjlqZFhKemIzSUVXdNVFExTURRd09qRTBOVEE0TkRRNE5EVT0=",
"before": "WTI5dGJXVnVkRjlqZFhKemIzNE16Y3dNVFExTVRFNE9qRTBOVEE0TkRRME5UVT0="
}
},
"data": [
{
"from": {
"name": "NiFû Shafrà",
"id": "1025030640553"
},
"like_count": 0,
"can_remove": false,
"created_time": "2015-12-23T04:20:55+0000",
"message": "wow lovely one",
"id": "50018692683829_500458145118",
"user_likes": false
},
{
"from": {
"name": "Shamnaz Lukmanjee",
"id": "160625809961884"
},
"like_count": 0,
"can_remove": false,
"created_time": "2015-12-23T04:27:25+0000",
"message": "Nice",
"id": "500186926838929_500450145040",
"user_likes": false
}
]
},
"actions": [
{
"link": "https://www.facebook.com/425073257683630/posts/5001866838929",
"name": "Comment"
},
{
"link": "https://www.facebook.com/42507683630/posts/500186926838929",
"name": "Like"
}
],
"updated_time": "2015-12-23T04:27:25+0000",
"link": "https://www.facebook.com/DSIFootcandy/photos/a.438926536298302.1073741827.4250732576630/50086926838929/?type=3",
"object_id": "50018692838929",
"shares": {
"count": 3
},
"created_time": "2015-12-23T04:09:01+0000",
"message": "Reach new heights in the cute and extremely comfortable \"Silviar\" www.focandy.lk",
"type": "photo",
"id": "425077683630_50018926838929",
"status_type": "added_photos",
"icon": "https://www.facebook.com/images/icons/photo1.gif"
}
]
}
Now I need to get this data into a dataframe as follows(no need to get all).
item | Like_id |Like_username | comments_userid |comments_username|comment(msg)|
-----+---------+--------------+-----------------+-----------------+------------+
Bag | 45546 | noel | 641 | James | nice work |
-----+---------+--------------+-----------------+-----------------+------------+
Any Help will be Highly Appreciated.

Not exactly like your intended format, but here is the making of a solution :
import pandas
DictionaryObject_as_List = str(mydict).replace("{","").replace("}","").replace("[","").replace("]","").split(",")
newlist = []
for row in DictionaryObject_as_List :
row = row.replace('https://',' ').split(":")
exec('newlist.append ( ' + "[" + " , ".join(row)+"]" + ')')
DataFrame_Object = pandas.DataFrame(newlist)
print DataFrame_Object

Categories

Resources