Develop Reference

How do I flat a Nested Json file in Python?

I have a nested Json file with arrays.
I want to flat it so there won't be nested Jsons.
For example:
Code for Json:
https://jsonblob.com/4b255e51-7e9f-11e8-b89c-37203846213e
This Json has sub-Json and also array that contains Json.
The source is:
Output should be:
If there are arrays that contains a single Json they can be ignored. But if they have also sub-json they should be treated as above. Basically from my point of view each array is like a separated Json file.
I know that flating the Json can be done as:
from pandas.io.json import json_normalize
json_normalize(sample_object)
But this won't work with arrays.
Any idea how to make this work?
EDIT:
This is how arrays should be handled:
source:
Output:
Which means first Json in array stays as is {0}, {1} etc... but the sub-jsons are flatted. There are no columns of attributes_0_value ! Basically convert it to array with a single Json. No nesting (unless there is another array).

Try using this:
import pandas as pd
import json
response = {u'total': 1245, u'limit': 2, u'results': [{u'customer': {u'lastName': u'rtyrtyrt', u'userAccountId': None, u'id': 637, u'firstName': u'rtyrtyrty', u'email': u'ddfgdfg#dfsdfgdfg.ggg'}, u'shippingAddress': {u'city': u'rtyrtyrtyrty', u'vatNumber': None, u'firstName': u'rtyrtyrty', u'companyName': None, u'country': {u'defaultCulture': {u'languageName': u'English', u'code': u'en-GB', u'id': 2, u'name': u'English'}, u'onlineStoreActive': True, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'locale': None, u'isO2LetterCode': u'GB', u'vatPercentage': 20.0, u'continent': u'Europe ', u'isoNumericCode': u'826', u'invariantName': u'UNITED KINGDOM', u'id': 2, u'isO3LetterCode': u'GBR'}, u'stateProvince': None, u'lastName': u'rtyrtyrt', u'zipCode': u'5464556', u'email': u'ddfgdfg#dfsdfgdfg.ggg', u'addressLine2': None, u'addressLine1': u'tyrtyrty', u'phoneNumber': u'45644443456456546', u'addressName': None, u'id': 861}, u'orderDateUtc': u'0001-01-01 00:00', u'shoppingCardId': 0, u'paymentType': {u'code': u'SafeCharge', u'invariantName': u'Credit Card', u'id': 50}, u'orderNumber': u'0100000845', u'giftMessage': u'', u'storeId': 1, u'shippingService': {u'deletedOn': None, u'code': u'ROYALSTD', u'courier': None, u'updatedOn': u'2018-01-24 09:23', u'locale': None, u'createdOn': u'2018-01-24 09:23', u'storeId': 1, u'sortOrder': 1, u'invariantName': u'Royal Mail Standard', u'id': 1}, u'referenceOrderNumber': u'', u'totals': {u'shippingChargesNet': 3.95, u'orderLevelDiscount': 0.0, u'grandTotal': 8.95, u'vatPercentage': 20.0, u'shippingChargesDiscount': 0.0, u'shippingCharges': 3.95, u'units': 1, u'salesTaxPerc': 0.0, u'subTotal': 5.0, u'salesTax': 1.4916666666666667}, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'status': {u'invariantName': u'Waiting PackingList', u'id': 4, u'name': None}, u'billingAddress': {u'city': u'rtyrtyrtyrty', u'vatNumber': None, u'firstName': u'rtyrtyrty', u'companyName': None, u'country': {u'defaultCulture': {u'languageName': u'English', u'code': u'en-GB', u'id': 2, u'name': u'English'}, u'onlineStoreActive': True, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'locale': None, u'isO2LetterCode': u'GB', u'vatPercentage': 20.0, u'continent': u'Europe ', u'isoNumericCode': u'826', u'invariantName': u'UNITED KINGDOM', u'id': 2, u'isO3LetterCode': u'GBR'}, u'stateProvince': None, u'lastName': u'rtyrtyrt', u'zipCode': u'5464556', u'email': u'ddfgdfg#dfsdfgdfg.ggg', u'addressLine2': None, u'addressLine1': u'tyrtyrty', u'phoneNumber': u'456456456546', u'addressName': None, u'id': 861}, u'items': [{u'orderId': 844, u'discountEach': 0.0, u'cancellationId': 0, u'orderedQty': 1, u'giftMessage': u'', u'orderLevelDiscountEach': 0.0, u'historicalCategories': [], u'giftFrom': u'', u'netShippingChargesEach': 3.95, u'promotionItemIds': [], u'variantId': 11282, u'attributes': [{u'value': u'', u'key': u'ProductSeason'}], u'priceEach': 5.0, u'isGift': False, u'id': 939, u'giftTo': u''}], u'attributes': [{u'value': u'2', u'key': u'CustomerCultureId'}, {u'value': u'185.13.248.67', u'key': u'IpAddress'}, {u'value': u'UA', u'key': u'IpCountryCode'}, {u'value': u'OLS', u'key': u'OrderSource'}, {u'value': u'111790', u'key': u'SafeCharge_AuthCode'}, {u'value': u'UQBzAGQAaAB3ADgAMgB0AE4AagBHADUAegBpAHMAIwA7AC4ANgA3AFEAXwBMAGAAKwAqAHIAVgBGAEcAKQBFAD0ASQA8AC4ATgA0AD8ANQA+AFAAMwA=', u'key': u'SafeCharge_Token'}, {u'value': u'1512424599', u'key': u'SafeCharge_TransactionId'}, {u'value': u'1', u'key': u'StoreId'}], u'isGift': False, u'id': 844}, {u'customer': {u'lastName': u'dfgdfg', u'userAccountId': None, u'id': 638, u'firstName': u'dfgdfg', u'email': u'hfghfgh#dfdfg.fdg'}, u'shippingAddress': {u'city': u'fghfghhf', u'vatNumber': None, u'firstName': u'dfgdfg', u'companyName': None, u'country': {u'defaultCulture': {u'languageName': u'English', u'code': u'en-GB', u'id': 2, u'name': u'English'}, u'onlineStoreActive': True, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'locale': None, u'isO2LetterCode': u'GB', u'vatPercentage': 20.0, u'continent': u'Europe ', u'isoNumericCode': u'826', u'invariantName': u'UNITED KINGDOM', u'id': 2, u'isO3LetterCode': u'GBR'}, u'stateProvince': None, u'lastName': u'dfgdfg', u'zipCode': u'4564566', u'email': u'hfghfgh#dfdfg.fdg', u'addressLine2': None, u'addressLine1': u'fghfghfgh', u'phoneNumber': u'567567567', u'addressName': None, u'id': 862}, u'orderDateUtc': u'0001-01-01 00:00', u'shoppingCardId': 0, u'paymentType': {u'code': u'SafeCharge', u'invariantName': u'Credit Card', u'id': 50}, u'orderNumber': u'0100000846', u'giftMessage': u'', u'storeId': 1, u'shippingService': {u'deletedOn': None, u'code': u'ROYALSTD', u'courier': None, u'updatedOn': u'2018-01-24 09:23', u'locale': None, u'createdOn': u'2018-01-24 09:23', u'storeId': 1, u'sortOrder': 1, u'invariantName': u'Royal Mail Standard', u'id': 1}, u'referenceOrderNumber': u'', u'totals': {u'shippingChargesNet': 3.95, u'orderLevelDiscount': 0.0, u'grandTotal': 8.95, u'vatPercentage': 20.0, u'shippingChargesDiscount': 0.0, u'shippingCharges': 3.95, u'units': 1, u'salesTaxPerc': 0.0, u'subTotal': 5.0, u'salesTax': 1.4916666666666667}, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'status': {u'invariantName': u'Shipped', u'id': 6, u'name': None}, u'billingAddress': {u'city': u'fghfghhf', u'vatNumber': None, u'firstName': u'dfgdfg', u'companyName': None, u'country': {u'defaultCulture': {u'languageName': u'English', u'code': u'en-GB', u'id': 2, u'name': u'English'}, u'onlineStoreActive': True, u'currency': {u'symbol': u'\xa3', u'code': u'GBP', u'id': 2, u'currencyCulture': u'en-GB', u'numericCode': 826}, u'locale': None, u'isO2LetterCode': u'GB', u'vatPercentage': 20.0, u'continent': u'Europe ', u'isoNumericCode': u'826', u'invariantName': u'UNITED KINGDOM', u'id': 2, u'isO3LetterCode': u'GBR'}, u'stateProvince': None, u'lastName': u'dfgdfg', u'zipCode': u'4563334566', u'email': u'hfghfgh#dfdfg.fdg', u'addressLine2': None, u'addressLine1': u'fghfghfgh', u'phoneNumber': u'567567567', u'addressName': None, u'id': 862}, u'items': [{u'orderId': 845, u'discountEach': 0.0, u'cancellationId': 0, u'orderedQty': 1, u'giftMessage': u'', u'orderLevelDiscountEach': 0.0, u'historicalCategories': [], u'giftFrom': u'', u'netShippingChargesEach': 3.95, u'promotionItemIds': [], u'variantId': 11282, u'attributes': [{u'value': u'', u'key': u'ProductSeason'}], u'priceEach': 5.0, u'isGift': False, u'id': 940, u'giftTo': u''}], u'attributes': [{u'value': u'2', u'key': u'CustomerCultureId'}, {u'value': u'115.11.118.67', u'key': u'IpAddress'}, {u'value': u'UA', u'key': u'IpCountryCode'}, {u'value': u'OLS', u'key': u'OrderSource'}, {u'value': u'111335', u'key': u'SafeCharge_AuthCode'}, {u'value': u'UQA1AEYASgBVAEgAcgBvAE8AWAAlAFMAaABcAGAAMwA0AG4ATABiAHAAcQBoAEkAawB6AHMANQBXAEgAUQApACQATwBpAEQAUABAAGcAKwBcADQAMwA=', u'key': u'SafeCharge_Token'}, {u'value': u'1512424624', u'key': u'SafeCharge_TransactionId'}], u'isGift': False, u'id': 845}], u'offset': 0}
sample_object = pd.DataFrame(response)['results'].to_dict()
def flatten_json(y):
out = {}
def flatten(x, name=''):
if type(x) is dict:
for a in x:
flatten(x[a], name + a + '_')
elif type(x) is list:
out[name[:-1]] = x
else:
out[name[:-1]] = x
flatten(y)
return out
flat = {k: flatten_json(v) for k, v in sample_object.items()}
with open('flat.json', 'w') as jsonfile:
jsonfile.write(json.dumps(flat))

I personally use this procedure.
First store the JSON data as string (or, load from url or file)
use nested_to_record() method from pandas
import json
from pandas.io.json.normalize import nested_to_record
json_dic = json.loads(json_str)
flat = nested_to_record(json_dic, sep='_')
for key in flat:
print key, flat[key]
Output:

Load json using python 2.7 [duplicate]

This question already has answers here:
JSON ValueError: Expecting property name: line 1 column 2 (char 1)
(6 answers)
Closed 5 years ago.
I have the follow string, which I'm trying to load using python 2.7 json.loads.
{
u'Status': {
u'display_name': u'Status',
u'is_updatable': True,
u'type': u'TEXT',
u'val': u'Paying',
u'source': u'API'
}, u'Create Date': {
u'display_name': u'Create Date',
u'is_updatable': True,
u'type': u'DATE',
u'val': u'2017-09-20',
u'source': u'API'
}, u'Total # of Projects': {
u'display_name': u'Total # of Projects',
u'is_updatable': True,
u'type': u'TEXT',
u'val': u'53',
u'source': u'Pixel'
}
}
I'm getting the error:
ValueError: Expecting property name: line 1 column 2 (char 1)
Any ideas?

Following file that you have pasted is not in json format.
you can always check the validity of your JSON file using.
https://jsoneditoronline.org/
OR
import json
a= {
u'Status': {
u'display_name': u'Status',
u'is_updatable': True,
u'type': u'TEXT',
u'val': u'Paying',
u'source': u'API'
}, u'Create Date': {
u'display_name': u'Create Date',
u'is_updatable': True,
u'type': u'DATE',
u'val': u'2017-09-20',
u'source': u'API'
}, u'Total # of Projects': {
u'display_name': u'Total # of Projects',
u'is_updatable': True,
u'type': u'TEXT',
u'val': u'53',
u'source': u'Pixel'
}
}
b=json.dumps(a) #String to json
print (b)
c=json.loads(b)
print (c)
Note :
json loads -> returns an object from a string representing a json object.
json dumps -> returns a string representing a json object from an object.

Extracting table caption/title from a word document

I have to extract content of specific tables from a huge msword doc. After some reading, I have been trying to use win32com to extract the table titles. I used the following code.
from win32com import client
word = client.Dispatch("Word.Application")
document = word.Documents.Open("path_to_doc")
titles = [table.Title for table in document.Tables]
f = open('myfile.doc', 'w')
f.write("%s" %titles)
f.close()
However the resulting file just has:
[u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'']
Where am I going wrong?

Python Tweepy no response on Stream

Hello i try to listen on a tweet channel using python with libary Tweepy.
I use python 2.7.11 and install Tweepy using pip. When i run the following code i get no response an no error. Can you tell me what the problem is and how can i fix this:
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import time
import json
#EDITED 13:25
from tweepy.auth import API
# Twitter Credentials
ckey = 'Consumer Key (API Key)'
csecret = 'Consumer Secret (API Secret)'
atoken = 'Access Token'
asecret = 'Access Token Secret'
class listener(StreamListener):
def on_data(self, data):
try:
tweet = json.loads(data)
if tweet["lang"] == "nl":
print tweet["id"]
return True
except BaseException, e:
print 'failed on_date,', str(e)
time.sleep(5)
def on_error(self, status):
print status
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
#EDITED 13:25
print api.verify_credentials()
# twitterStream.filter( track=lstZoekwaarde, languages="nl" )
twitterStream.filter(track='christmas', languages="nl")
CONSOLE: api.verify_credentials()
User(follow_request_sent=False, has_extended_profile=False, profile_use_background_image=True, _json={u'follow_request_sent': False, u'has_extended_profile': False, u'profile_use_background_image': True, u'default_profile_image': False, u'id': 169505005, u'profile_background_image_url_https': u'https://abs.twimg.com/images/themes/theme1/bg.png', u'verified': False, u'translator_type': u'none', u'profile_text_color': u'333333', u'profile_image_url_https': u'https://pbs.twimg.com/profile_images/1425063736/image_normal.jpg', u'profile_sidebar_fill_color': u'DDEEF6', u'entities': {u'description': {u'urls': []}}, u'followers_count': 7, u'profile_sidebar_border_color': u'C0DEED', u'id_str': u'169505005', u'profile_background_color': u'C0DEED', u'listed_count': 0, u'status': {u'contributors': None, u'truncated': False, u'text': u'aan het werk bij Alfam', u'is_quote_status': False, u'in_reply_to_status_id': None, u'id': 541894460343582720, u'favorite_count': 1, u'source': u'Twitter for Android', u'retweeted': False, u'coordinates': {u'type': u'Point', u'coordinates': [5.207323, 52.0616799]}, u'entities': {u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}, u'in_reply_to_screen_name': None, u'in_reply_to_user_id': None, u'retweet_count': 0, u'id_str': u'541894460343582720', u'favorited': False, u'geo': {u'type': u'Point', u'coordinates': [52.0616799, 5.207323]}, u'in_reply_to_user_id_str': None, u'lang': u'nl', u'created_at': u'Mon Dec 08 09:58:01 +0000 2014', u'in_reply_to_status_id_str': None, u'place': {u'full_name': u'Bunnik, Nederland', u'url': u'https://api.twitter.com/1.1/geo/id/ef77325fbde0f5ad.json', u'country': u'The Netherlands', u'place_type': u'city', u'bounding_box': {u'type': u'Polygon', u'coordinates': [[[5.1532516, 51.9976555], [5.2803233, 51.9976555], [5.2803233, 52.0801935], [5.1532516, 52.0801935]]]}, u'contained_within': [], u'country_code': u'NL', u'attributes': {}, u'id': u'ef77325fbde0f5ad', u'name': u'Bunnik'}}, u'is_translation_enabled': False, u'utc_offset': None, u'statuses_count': 186, u'description': u'', u'friends_count': 7, u'location': u'', u'profile_link_color': u'1DA1F2', u'profile_image_url': u'http://pbs.twimg.com/profile_images/1425063736/image_normal.jpg', u'following': False, u'geo_enabled': True, u'profile_background_image_url': u'http://abs.twimg.com/images/themes/theme1/bg.png', u'screen_name': u'ehoeven', u'lang': u'en', u'profile_background_tile': False, u'favourites_count': 1, u'name': u'Erik Hoeven', u'notifications': False, u'url': None, u'created_at': u'Thu Jul 22 14:12:09 +0000 2010', u'contributors_enabled': False, u'time_zone': None, u'protected': False, u'default_profile': True, u'is_translator': False}, time_zone=None, id=169505005, description=u'', _api=<tweepy.api.API object at 0x7efdf2d5a510>, verified=False, profile_text_color=u'333333', profile_image_url_https=u'https://pbs.twimg.com/profile_images/1425063736/image_normal.jpg', profile_sidebar_fill_color=u'DDEEF6', is_translator=False, geo_enabled=True, entities={u'description': {u'urls': []}}, followers_count=7, protected=False, id_str=u'169505005', default_profile_image=False, listed_count=0, status=Status(contributors=None, truncated=False, text=u'aan het werk bij Alfam', is_quote_status=False, in_reply_to_status_id=None, id=541894460343582720, favorite_count=1, _api=<tweepy.api.API object at 0x7efdf2d5a510>, source=u'Twitter for Android', _json={u'contributors': None, u'truncated': False, u'text': u'aan het werk bij Alfam', u'is_quote_status': False, u'in_reply_to_status_id': None, u'id': 541894460343582720, u'favorite_count': 1, u'source': u'Twitter for Android', u'retweeted': False, u'coordinates': {u'type': u'Point', u'coordinates': [5.207323, 52.0616799]}, u'entities': {u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}, u'in_reply_to_screen_name': None, u'in_reply_to_user_id': None, u'retweet_count': 0, u'id_str': u'541894460343582720', u'favorited': False, u'geo': {u'type': u'Point', u'coordinates': [52.0616799, 5.207323]}, u'in_reply_to_user_id_str': None, u'lang': u'nl', u'created_at': u'Mon Dec 08 09:58:01 +0000 2014', u'in_reply_to_status_id_str': None, u'place': {u'full_name': u'Bunnik, Nederland', u'url': u'https://api.twitter.com/1.1/geo/id/ef77325fbde0f5ad.json', u'country': u'The Netherlands', u'place_type': u'city', u'bounding_box': {u'type': u'Polygon', u'coordinates': [[[5.1532516, 51.9976555], [5.2803233, 51.9976555], [5.2803233, 52.0801935], [5.1532516, 52.0801935]]]}, u'contained_within': [], u'country_code': u'NL', u'attributes': {}, u'id': u'ef77325fbde0f5ad', u'name': u'Bunnik'}}, coordinates={u'type': u'Point', u'coordinates': [5.207323, 52.0616799]}, entities={u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}, in_reply_to_screen_name=None, id_str=u'541894460343582720', retweet_count=0, in_reply_to_user_id=None, favorited=False, source_url=u'http://twitter.com/download/android', geo={u'type': u'Point', u'coordinates': [52.0616799, 5.207323]}, in_reply_to_user_id_str=None, lang=u'nl', created_at=datetime.datetime(2014, 12, 8, 9, 58, 1), in_reply_to_status_id_str=None, place=Place(_api=<tweepy.api.API object at 0x7efdf2d5a510>, country_code=u'NL', url=u'https://api.twitter.com/1.1/geo/id/ef77325fbde0f5ad.json', country=u'The Netherlands', place_type=u'city', bounding_box=BoundingBox(_api=<tweepy.api.API object at 0x7efdf2d5a510>, type=u'Polygon', coordinates=[[[5.1532516, 51.9976555], [5.2803233, 51.9976555], [5.2803233, 52.0801935], [5.1532516, 52.0801935]]]), contained_within=[], full_name=u'Bunnik, Nederland', attributes={}, id=u'ef77325fbde0f5ad', name=u'Bunnik'), retweeted=False), lang=u'en', utc_offset=None, statuses_count=186, profile_background_color=u'C0DEED', friends_count=7, profile_link_color=u'1DA1F2', profile_image_url=u'http://pbs.twimg.com/profile_images/1425063736/image_normal.jpg', notifications=False, default_profile=True, profile_background_image_url_https=u'https://abs.twimg.com/images/themes/theme1/bg.png', profile_background_image_url=u'http://abs.twimg.com/images/themes/theme1/bg.png', name=u'Erik Hoeven', is_translation_enabled=False, profile_background_tile=False, favourites_count=1, screen_name=u'ehoeven', url=None, created_at=datetime.datetime(2010, 7, 22, 14, 12, 9), contributors_enabled=False, location=u'', profile_sidebar_border_color=u'C0DEED', translator_type=u'none', following=False)

Try this syntax:
twitterStream.filter(track=['christmas'], languages=['nl'])

How to convert a JSON string into a Python data structure

'[{"append":null,"appendCanExplainable":false,"appendList":[],"auction":{"aucNumId":"35179051643","auctionPic":"http://img.taobaocdn.com/bao/uploaded/i3/TB12WchGXXXXXb5XpXXXXXXXXXX_!!0-item_pic.jpg_40x40.jpg","link":"http://item.taobao.com/item.htm?id=35179051643","sku":"\xd1\xab\xb7\xd6\xc0\xe0:\xc9\xab\xbb\xd2\xcf\xdf\xbd\xf4\xc9\xed\xb3\xa4\xbf\xe3 &nbsp\xb3\xdf\xc2\xeb:M-170M-55-62KG","thumbnail":"","title":"\xcb\xb9\xbd\xf4\xc9\xed\xbf\xe3 \xb5\xaf\xc1\xa6\xd7\xe3\xc7\xf2\xd4\xaf\xbd\xa1\xc9\xed\xbf\xe3 PRO \xc4\xd0 \xb4\xf2\xb5\xd7\xd1\xb5\xc1\xb7\xb3\xa4\xbf\xe3\xcb\xd9\xb8\xc9"},"award":"","bidPriceMoney":{"amount":35,"cent":3500,"centFactor":100,"currency":{"currencyCode":"CNY","defaultFractionDigits":2,"symbol":"\xa3\xa4"},"currencyCode":"CNY","displayUnit":"\xd4\xaa"},"buyAmount":1,"content":"\xba\xc3\xc6\xc0\xa3\xa1","creditFraudRule":0,"date":"2014\xc4\xea12\xd4\xc220\xc8\xd5 15:41","dayAfterConfirm":0,"enableSNS":false,"from":"","lastModifyFrom":0,"payTime":{"date":18,"day":4,"hours":13,"minutes":4,"month":11,"seconds":37,"time":1418879077000,"timezoneOffset":-480,"year":114},"photos":[],"promotionType":"\xbb\xee\xb6\xaf\xb4\xd9\xcf\xfa ","propertiesAvg":"0.0","rate":"1","rateId":231421178840,"raterType":0,"reply":null,"shareInfo":{"lastReplyTime":"","pic":0,"reply":0,"share":false,"userNumIdBase64":""},"showCuIcon":true,"showDepositIcon":false,"spuRatting":[],"status":0,"tag":"","useful":0,"user":{"anony":true,"avatar":"http://a.tbcdn.cn/app/sns/img/default/avatar-40.png","displayRatePic":"b_red_3.gif","nick":"y***6","nickUrl":"","rank":65,"rankUrl":"","userId":"","vip":"","vipLevel":0},"validscore":1,"vicious":""},{"append":null,"appendCanExplainable":false,"appendList":[],"auction":{"aucNumId":"35179051643","auctionPic":"http://img.taobaocdn.com/bao/uploaded/i3/TB12WchGXXXXXb5XpXXXXXXXXXX_!!0-item_pic.jpg_40x40.jpg","link":"http://item.taobao.com/item.htm?id=35179051643","sku":"\xd1\xd5\xc9\xab\xb7\xd6\xc0\xe0:\xba\xda\xc9\xab\xba\xda\xcf\xdf\xbd\xf4\xc9\xed\xb3\xa4\xbf\xe3 &nbsp\xb3\xdf\xc2\xeb:S-160m-45~55KG","thumbnail":"","title":"\xc7\xf2\xc9\xed\xbf\xe3\xb4\xf2\xb5\xd7\xd1\xb5\xc1\xb7\xb3\xa4\xbf\xe3\xcb\xd9\xb8\xc9"},"award":"","bidPriceMoney":{"amount":35,"cent":3500,"centFactor":100,"currency":{"currencyCode":"CNY","defaultFractionDigits":2,"symbol":"\xa3\xa4"},"currencyCode":"CNY","displayUnit":"\xd4\xaa"},"buyAmount":1,"content":"\xba\xc3\xc6\xc0\xa3\xa1","creditFraudRule":0,"date":"2014\xc4\xea12\xd4\xc220\xc8\xd5 15:37","dayAfterConfirm":0,"enableSNS":false,"from":"","lastModifyFrom":0,"payTime":{"date":17,"day":3,"hours":17,"minutes":43,"month":11,"seconds":47,"time":1418809427000,"timezoneOffset":-480,"year":114},"photos":[],"promotionType":"\xbb\xee\xb6\xaf\xb4\xd9\xcf\xfa ","propertiesAvg":"0.0","rate":"1","rateId":231441191365,"raterType":0,"reply":null,"shareInfo":{"lastReplyTime":"","pic":0,"reply":0,"share":false,"userNumIdBase64":""},"showCuIcon":true,"showDepositIcon":false,"spuRatting":[],"status":0,"tag":"","useful":0,"user":{"anony":true,"avatar":"http://a.tbcdn.cn/app/sns/img/default/avatar-40.png","displayRatePic":"b_blue_3.gif","nick":"\xc2\xb7***0","nickUrl":"","rank":1235,"rankUrl":"","userId":"","vip":"","vipLevel":0},"validscore":1,"vicious":""}]'
How can I convert this str to list of dicts ? I have tried some methods, but failed. The string represents a list containing 2 big dicts, and one dict contains nested small dicts. The expected result is:
[{dict1},{dict2}]

You have a JSON string, so use the json module to decode this:
import json
decoded = json.loads(encoded)
decoded is then a Python list; you can then address each dictionary in a list, or use unpacking to assign two dictionaries to two names:
dictionary1, dictionary2 = decoded
If you are using the requests library then you can use the response.json() method to load the content:
decoded = response.json()
In this specific case you appear to have GBK encoded data however (or perhaps GB2312, a predecessor).
This goes well outside the JSON standard (which actually requires one of the UTF codecs to be used), and you'll need to tell json.loads() about the codec used:
decoded = json.loads(encoded, 'gbk')
The requests library will use whatever codec the server sent along with the response, or will otherwise use a characterset detection technique to try and find the right codec to use.
The result, when decoded, then looks like:
>>> decoded = json.loads(encoded, 'gbk')
>>> pprint(decoded)
[{u'append': None,
u'appendCanExplainable': False,
u'appendList': [],
u'auction': {u'aucNumId': u'35179051643',
u'auctionPic': u'http://img.taobaocdn.com/bao/uploaded/i3/TB12WchGXXXXXb5XpXXXXXXXXXX_!!0-item_pic.jpg_40x40.jpg',
u'link': u'http://item.taobao.com/item.htm?id=35179051643',
u'sku': u'\u52cb\u5206\u7c7b:\u8272\u7070\u7ebf\u7d27\u8eab\u957f\u88e4 &nbsp\u5c3a\u7801:M-170M-55-62KG',
u'thumbnail': u'',
u'title': u'\u65af\u7d27\u8eab\u88e4 \u5f39\u529b\u8db3\u7403\u8f95\u5065\u8eab\u88e4 PRO \u7537 \u6253\u5e95\u8bad\u7ec3\u957f\u88e4\u901f\u5e72'},
u'award': u'',
u'bidPriceMoney': {u'amount': 35,
u'cent': 3500,
u'centFactor': 100,
u'currency': {u'currencyCode': u'CNY',
u'defaultFractionDigits': 2,
u'symbol': u'\uffe5'},
u'currencyCode': u'CNY',
u'displayUnit': u'\u5143'},
u'buyAmount': 1,
u'content': u'\u597d\u8bc4\uff01',
u'creditFraudRule': 0,
u'date': u'2014\u5e7412\u670820\u65e5 15:41',
u'dayAfterConfirm': 0,
u'enableSNS': False,
u'from': u'',
u'lastModifyFrom': 0,
u'payTime': {u'date': 18,
u'day': 4,
u'hours': 13,
u'minutes': 4,
u'month': 11,
u'seconds': 37,
u'time': 1418879077000,
u'timezoneOffset': -480,
u'year': 114},
u'photos': [],
u'promotionType': u'\u6d3b\u52a8\u4fc3\u9500 ',
u'propertiesAvg': u'0.0',
u'rate': u'1',
u'rateId': 231421178840,
u'raterType': 0,
u'reply': None,
u'shareInfo': {u'lastReplyTime': u'',
u'pic': 0,
u'reply': 0,
u'share': False,
u'userNumIdBase64': u''},
u'showCuIcon': True,
u'showDepositIcon': False,
u'spuRatting': [],
u'status': 0,
u'tag': u'',
u'useful': 0,
u'user': {u'anony': True,
u'avatar': u'http://a.tbcdn.cn/app/sns/img/default/avatar-40.png',
u'displayRatePic': u'b_red_3.gif',
u'nick': u'y***6',
u'nickUrl': u'',
u'rank': 65,
u'rankUrl': u'',
u'userId': u'',
u'vip': u'',
u'vipLevel': 0},
u'validscore': 1,
u'vicious': u''},
{u'append': None,
u'appendCanExplainable': False,
u'appendList': [],
u'auction': {u'aucNumId': u'35179051643',
u'auctionPic': u'http://img.taobaocdn.com/bao/uploaded/i3/TB12WchGXXXXXb5XpXXXXXXXXXX_!!0-item_pic.jpg_40x40.jpg',
u'link': u'http://item.taobao.com/item.htm?id=35179051643',
u'sku': u'\u989c\u8272\u5206\u7c7b:\u9ed1\u8272\u9ed1\u7ebf\u7d27\u8eab\u957f\u88e4 &nbsp\u5c3a\u7801:S-160m-45~55KG',
u'thumbnail': u'',
u'title': u'\u7403\u8eab\u88e4\u6253\u5e95\u8bad\u7ec3\u957f\u88e4\u901f\u5e72'},
u'award': u'',
u'bidPriceMoney': {u'amount': 35,
u'cent': 3500,
u'centFactor': 100,
u'currency': {u'currencyCode': u'CNY',
u'defaultFractionDigits': 2,
u'symbol': u'\uffe5'},
u'currencyCode': u'CNY',
u'displayUnit': u'\u5143'},
u'buyAmount': 1,
u'content': u'\u597d\u8bc4\uff01',
u'creditFraudRule': 0,
u'date': u'2014\u5e7412\u670820\u65e5 15:37',
u'dayAfterConfirm': 0,
u'enableSNS': False,
u'from': u'',
u'lastModifyFrom': 0,
u'payTime': {u'date': 17,
u'day': 3,
u'hours': 17,
u'minutes': 43,
u'month': 11,
u'seconds': 47,
u'time': 1418809427000,
u'timezoneOffset': -480,
u'year': 114},
u'photos': [],
u'promotionType': u'\u6d3b\u52a8\u4fc3\u9500 ',
u'propertiesAvg': u'0.0',
u'rate': u'1',
u'rateId': 231441191365,
u'raterType': 0,
u'reply': None,
u'shareInfo': {u'lastReplyTime': u'',
u'pic': 0,
u'reply': 0,
u'share': False,
u'userNumIdBase64': u''},
u'showCuIcon': True,
u'showDepositIcon': False,
u'spuRatting': [],
u'status': 0,
u'tag': u'',
u'useful': 0,
u'user': {u'anony': True,
u'avatar': u'http://a.tbcdn.cn/app/sns/img/default/avatar-40.png',
u'displayRatePic': u'b_blue_3.gif',
u'nick': u'\u8def***0',
u'nickUrl': u'',
u'rank': 1235,
u'rankUrl': u'',
u'userId': u'',
u'vip': u'',
u'vipLevel': 0},
u'validscore': 1,
u'vicious': u''}]