So I'm playing around with the Google Analytics dataset in pandas, and I'm having an issue.
The dataset has a column named hits, which contains a list of dictionaries, saved as a string. When I try to use ast.literat_eval to get the list of dictionaries, I get a syntax error pointing to this part of the dictionary:
'customMetrics': array([], dtype=object)
I'm using the following code to try to convert the string:
df['hits'] = df['hits'].apply(lambda x: ast.literal_eval(x))
I don't really care about this piece of data so it's fine if I could just ignore this and keep that particular dictionary value as a string. Does anyone know how to either ignore the error and set that dictionary value to a string, or properly convert this array?
FYI, an entire cell of the hits column looks like this:
"[{'hitNumber': 1, 'time': 0, 'hour': 23, 'minute': 0, 'isSecure': None, 'isInteraction': True, 'isEntrance': True, 'isExit': None, 'referer': 'https://www.youtube.com/yt/about/', 'page': {'pagePath': '/home', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'Home', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/home', 'pagePathLevel2': '', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': None, 'item': None, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/home', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([], dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(entrance)', 'previousContentGroup2': '(entrance)', 'previousContentGroup3': '(entrance)', 'previousContentGroup4': '(entrance)', 'previousContentGroup5': '(entrance)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}\n {'hitNumber': 2, 'time': 30507, 'hour': 23, 'minute': 1, 'isSecure': None, 'isInteraction': True, 'isEntrance': None, 'isExit': None, 'referer': None, 'page': {'pagePath': '/google+redesign/gift+cards', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'Gift Cards', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/google+redesign/', 'pagePathLevel2': '/gift+cards', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': {'transactionId': None, 'transactionRevenue': None, 'transactionTax': None, 'transactionShipping': None, 'affiliation': None, 'currencyCode': 'USD', 'localTransactionRevenue': None, 'localTransactionTax': None, 'localTransactionShipping': None, 'transactionCoupon': None}, 'item': {'transactionId': None, 'productName': None, 'productCategory': None, 'productSku': None, 'itemQuantity': None, 'itemRevenue': None, 'currencyCode': 'USD', 'localItemRevenue': None}, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/google+redesign/gift+cards', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([{'productSKU': 'GGOEGGCX056299', 'v2ProductName': 'Gift Card - $25.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 25000000, 'localProductPrice': 25000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 1},\n {'productSKU': 'GGOEGGCX056499', 'v2ProductName': 'Gift Card - $50.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 50000000, 'localProductPrice': 50000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 2},\n {'productSKU': 'GGOEGGCX056199', 'v2ProductName': 'Gift Card- $100.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 100000000, 'localProductPrice': 100000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 3},\n {'productSKU': 'GGOEGGCX056399', 'v2ProductName': 'Gift Card - $250.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 250000000, 'localProductPrice': 250000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 4}],\n dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(not set)', 'previousContentGroup2': '(not set)', 'previousContentGroup3': '(not set)', 'previousContentGroup4': '(not set)', 'previousContentGroup5': '(not set)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}\n {'hitNumber': 3, 'time': 53670, 'hour': 23, 'minute': 1, 'isSecure': None, 'isInteraction': True, 'isEntrance': None, 'isExit': True, 'referer': None, 'page': {'pagePath': '/signin.html', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'The Google Merchandise Store - Log In', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/signin.html', 'pagePathLevel2': '', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': None, 'item': None, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/signin.html', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([], dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(not set)', 'previousContentGroup2': '(not set)', 'previousContentGroup3': '(not set)', 'previousContentGroup4': '(not set)', 'previousContentGroup5': '(not set)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}]"
Related
I'm looking to scrape some data from the following web site: https://www.atptour.com/en/scores/stats-centre/archive/2022/414/MS024
The underlying match statistics are loaded by the following javacript:
<script nonce="5e824960-ee11-4894-b33c-9b0a25ace2d3" type="text/javascript">
var initial = {
matchStatsContainer : "#matchStatsContainer",
matchStatsTemplate : "#matchStatsTemplate",
currentMatchStatsToggle : "currentMatchStatsButton",
liveMatchUrl : "",
setPhrase : "set",
matchPhrase : "match",
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
}
require(["modules/modal/modalScoresMatchStats"],
function (ModalScoresMatchStats) {
return new ModalScoresMatchStats({
matchStatsContainer : "#match-stats-container",
matchStatsTemplate : "#matchStatsTemplate",
currentMatchStatsToggle : "currentMatchStatsButton",
liveMatchUrl : "",
setPhrase : "set",
matchPhrase : "match",
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
});
});
</script>
I am using the following code with script set to:
var initial = { .....
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
});
});
html = HTML(html=str(match_stats_html), async_=True)
val = html.render(script=script, reload=False)
I get the following error:
ElementHandleError: Evaluation failed: ReferenceError: require is not defined
at pyppeteer_evaluation_script:16:9
When I remove require from the script I get the following error:
ElementHandleError: Evaluation failed: SyntaxError: Unexpected token (
Any ideas on how I could execute the script and obtain the data returned by the results?
Thanks
import cloudscraper
scraper = cloudscraper.create_scraper()
r = scraper.get('https://www.atptour.com/-/ajax/Scores/GetInitialScores')
print(r.json())
This returns:
{'liveScores': {'Tournaments': [{'EventYear': 2022, 'Name': 'Hamburg European Open', 'EventId': '414', 'TournamentLink': '/en/tournaments/hamburg/414/overview', 'OfficialWebsite': 'www.hamburg-open.com', 'Location': 'Hamburg, Germany', 'TournamentBadgeImage': '/-/media/images/tourtypes/categorystamps_500_118x136.png?sc=0&hash=7431FC4DF8556F6BB9AA56E8578D5342', 'LiveScoresLinkText': 'Scores', 'LiveScoresLink': '/en/scores/current/hamburg/414/live-scores', 'DrawLinkText': 'Draw', 'DrawLink': '/en/scores/current/hamburg/414/draws', 'ScheduleLinkText': 'Schedule', 'ScheduleLink': '/en/scores/current/hamburg/414/daily-schedule', 'H2HLinkText': 'H2H', 'StatsLinkText': 'Stats', 'PlayNotStarted': '#Play Not Started Yet#', 'TournamentPosition': '1', 'IsLaverCup': False, 'Matches': [{'Id': 'MS008', 'ChallengerTVMatchItemId': '', 'KSeq': 415, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS008', 'HeadToHeadLink': '/en/players/atp-head-2-head/c-alcaraz-vs-f-krajinovic/A0E2/KB05', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 1, 'LiveListingPosition': 1, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:52:54', 'MatchInfo': 'Game Set and Match Carlos Alcaraz. \r\n Carlos Alcaraz wins the match 7-6(4) 6-3 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'A0E2', 'PartnerId': None, 'PlayerFirstName': 'C.', 'PlayerLastName': 'Alcaraz', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ESP', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'C. Alcaraz', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Carlos-Alcaraz', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ESP.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/carlos-alcaraz/a0e2/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/A0E2', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '7', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'KB05', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Krajinovic', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'SRB', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Krajinovic', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Filip-Krajinovic', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/SRB.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/filip-krajinovic/kb05/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/KB05', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': '4', 'SetTwo': '3', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS015', 'ChallengerTVMatchItemId': '', 'KSeq': 384, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS015', 'HeadToHeadLink': '/en/players/atp-head-2-head/f-cerundolo-vs-a-rublev/C0AU/RE44', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 2, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:32:08', 'MatchInfo': 'Game Set and Match Francisco Cerundolo. \r\n Francisco Cerundolo wins the match 6-4 6-2 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'C0AU', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Cerundolo', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ARG', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Cerundolo', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Francisco-Cerundolo', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ARG.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/francisco-cerundolo/c0au/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/C0AU', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'RE44', 'PartnerId': None, 'PlayerFirstName': 'A.', 'PlayerLastName': 'Rublev', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'RUS', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'A. Rublev', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Andrey-Rublev', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/RUS.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/andrey-rublev/re44/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/RE44', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '4', 'SetOneTiebreak': None, 'SetTwo': '2', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS009', 'ChallengerTVMatchItemId': '', 'KSeq': 384, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS009', 'HeadToHeadLink': '/en/players/atp-head-2-head/f-fognini-vs-k-khachanov/F510/KE29', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 3, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '3', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:42:52', 'MatchInfo': 'Game Set and Match Karen Khachanov. \r\n Karen Khachanov wins the match 6-3 7-5 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'F510', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Fognini', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Fognini', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Fabio-Fognini', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/fabio-fognini/f510/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/F510', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '3', 'SetOneTiebreak': None, 'SetTwo': '5', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'KE29', 'PartnerId': None, 'PlayerFirstName': 'K.', 'PlayerLastName': 'Khachanov', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'RUS', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'K. Khachanov', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Karen-Khachanov', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/RUS.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/karen-khachanov/ke29/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/KE29', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '7', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}]}, {'EventYear': 2022, 'Name': 'EFG Swiss Open Gstaad', 'EventId': '314', 'TournamentLink': '/en/tournaments/gstaad/314/overview', 'OfficialWebsite': 'swissopengstaad.ch', 'Location': 'Gstaad, Switzerland', 'TournamentBadgeImage': '/-/media/images/tourtypes/categorystamps_250_118x136.png?sc=0&hash=10547153B5ADD7DCF2E0C8DAA42886EE', 'LiveScoresLinkText': 'Scores', 'LiveScoresLink': '/en/scores/current/gstaad/314/live-scores', 'DrawLinkText': 'Draw', 'DrawLink': '/en/scores/current/gstaad/314/draws', 'ScheduleLinkText': 'Schedule', 'ScheduleLink': '/en/scores/current/gstaad/314/daily-schedule', 'H2HLinkText': 'H2H', 'StatsLinkText': 'Stats', 'PlayNotStarted': '#Play Not Started Yet#', 'TournamentPosition': '2', 'IsLaverCup': False, 'Matches': [{'Id': 'MD007', 'ChallengerTVMatchItemId': '', 'KSeq': 218, 'EventId': '314', 'RoundTitle': 'Quarterfinal - Court 1', 'MatchType': 'doubles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MD007', 'HeadToHeadLink': '/en/players/atp-head-2-head/l-sonego-vs-r-jebavy/SU87/J321', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 1, 'LiveListingPosition': 1, 'Status': 'P', 'Winner': '0', 'LastServer': '0', 'NumberOfSets': 3, 'MatchTime': '00:45:31', 'MatchInfo': 'L. Sonego/A. Vavassori win the point on the 1st serve of L. Sonego.', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'SU87', 'PartnerId': 'VA08', 'PlayerFirstName': 'L.', 'PlayerLastName': 'Sonego', 'PartnerFirstName': 'A.', 'PartnerLastName': 'Vavassori', 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': 'ITA', 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'L. Sonego', 'PartnerNameForUrl': 'Andrea-Vavassori', 'PlayerNameForUrl': 'Lorenzo-Sonego', 'PlayerTwoName': 'A. Vavassori', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerOneLink': '/en/players/lorenzo-sonego/su87/overview', 'PlayerTwoLink': '/en/players/andrea-vavassori/va08/overview', 'PlayerOneHeadshot': '/-/media/alias/player-headshot/SU87', 'PlayerTwoHeadshot': '/-/media/alias/player-headshot/VA08', 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'now-serving', 'Scores': {'CurrentScore': '30', 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '3', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'J321', 'PartnerId': 'O474', 'PlayerFirstName': 'R.', 'PlayerLastName': 'Jebavy', 'PartnerFirstName': 'J.', 'PartnerLastName': "O'Mara", 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'CZE', 'PartnerCountryCode': 'GBR', 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'R. Jebavy', 'PartnerNameForUrl': "Jonny-O'Mara", 'PlayerNameForUrl': 'Roman-Jebavy', 'PlayerTwoName': "J. O'Mara", 'PlayerOneFlag': '/en/~/media/images/flags/CZE.svg', 'PlayerTwoFlag': '/en/~/media/images/flags/GBR.svg', 'PlayerOneLink': '/en/players/roman-jebavy/j321/overview', 'PlayerTwoLink': "/en/players/jonny-o'mara/o474/overview", 'PlayerOneHeadshot': '/-/media/alias/player-headshot/J321', 'PlayerTwoHeadshot': '/-/media/alias/player-headshot/O474', 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': '0', 'SetOne': '2', 'SetOneTiebreak': None, 'SetTwo': '1', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS008', 'ChallengerTVMatchItemId': '', 'KSeq': 432, 'EventId': '314', 'RoundTitle': 'Round of 16 - Roy Emerson Arena', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MS008', 'HeadToHeadLink': '/en/players/atp-head-2-head/c-ruud-vs-j-lehecka/RH16/L0BV', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 2, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:39:31', 'MatchInfo': 'Game Set and Match Casper Ruud. \r\n Casper Ruud wins the match 6-3 6-4 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'RH16', 'PartnerId': None, 'PlayerFirstName': 'C.', 'PlayerLastName': 'Ruud', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'NOR', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'C. Ruud', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Casper-Ruud', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/NOR.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/casper-ruud/rh16/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/RH16', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'L0BV', 'PartnerId': None, 'PlayerFirstName': 'J.', 'PlayerLastName': 'Lehecka', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'CZE', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'J. Lehecka', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Jiri-Lehecka', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/CZE.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/jiri-lehecka/l0bv/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/L0BV', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '3', 'SetOneTiebreak': None, 'SetTwo': '4', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS015', 'ChallengerTVMatchItemId': '', 'KSeq': 374, 'EventId': '314', 'RoundTitle': 'Round of 16 - Roy Emerson Arena', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MS015', 'HeadToHeadLink': '/en/players/atp-head-2-head/r-gasquet-vs-m-berrettini/G628/BK40', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 3, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '3', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:29:29', 'MatchInfo': 'Game Set and Match Matteo Berrettini. \r\n Matteo Berrettini wins the match 6-4 6-4 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'G628', 'PartnerId': None, 'PlayerFirstName': 'R.', 'PlayerLastName': 'Gasquet', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'FRA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'R. Gasquet', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Richard-Gasquet', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/FRA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/richard-gasquet/g628/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/G628', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '4', 'SetOneTiebreak': None, 'SetTwo': '4', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'BK40', 'PartnerId': None, 'PlayerFirstName': 'M.', 'PlayerLastName': 'Berrettini', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'M. Berrettini', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Matteo-Berrettini', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/matteo-berrettini/bk40/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/BK40', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}]}]}}
cloudscraper can be installed with pip install cloudscraper
I’m trying to use Python print specific values from a JSON file that I pulled from an API. From what I understand, I am pulling it as a JSON file that has a list of dictionaries of players, with a nested dictionary for each player containing their data (i.e. name, team, etc.).
I’m running into issues printing the values within the JSON file, as each character is printing on a separate line.
The end result I am trying to get to is a Pandas DataFrame containing all the values from the JSON file, but I can’t even seem to iterate through the JSON file correctly.
Here is my code:
url = "https://api-football-v1.p.rapidapi.com/v3/players"
querystring = {"league":"39","season":"2020", "page":"2"}
headers = {
"X-RapidAPI-Host": "api-football-v1.p.rapidapi.com",
"X-RapidAPI-Key": "xxxxxkeyxxxxx"
}
response = requests.request("GET", url, headers=headers, params=querystring).json()
response_dump = json.dumps(response)
for item in response_dump:
for player_item in item:
print(player_item)
This is the output when I print the JSON response (first two items):
{'get': 'players', 'parameters': {'league': '39', 'page': '2', 'season': '2020'}, 'errors': [], 'results': 20, 'paging': {'current': 2, 'total': 37}, 'response': [{'player': {'id': 301, 'name': 'Benjamin Luke Woodburn', 'firstname': 'Benjamin Luke', 'lastname': 'Woodburn', 'age': 23, 'birth': {'date': '1999-10-15', 'place': 'Nottingham', 'country': 'England'}, 'nationality': 'Wales', 'height': '174 cm', 'weight': '72 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/301.png'}, 'statistics': [{'team': {'id': 40, 'name': 'Liverpool', 'logo': 'https://media.api-sports.io/football/teams/40.png'}, 'league': {'id': 39, 'name': 'Premier League', 'country': 'England', 'logo': 'https://media.api-sports.io/football/leagues/39.png', 'flag': 'https://media.api-sports.io/flags/gb.svg', 'season': 2020}, 'games': {'appearences': 0, 'lineups': 0, 'minutes': 0, 'number': None, 'position': 'Attacker', 'rating': None, 'captain': False}, 'substitutes': {'in': 0, 'out': 0, 'bench': 3}, 'shots': {'total': None, 'on': None}, 'goals': {'total': 0, 'conceded': 0, 'assists': None, 'saves': None}, 'passes': {'total': None, 'key': None, 'accuracy': None}, 'tackles': {'total': None, 'blocks': None, 'interceptions': None}, 'duels': {'total': None, 'won': None}, 'dribbles': {'attempts': None, 'success': None, 'past': None}, 'fouls': {'drawn': None, 'committed': None}, 'cards': {'yellow': 0, 'yellowred': 0, 'red': 0}, 'penalty': {'won': None, 'commited': None, 'scored': 0, 'missed': 0, 'saved': None}}]}, {'player': {'id': 518, 'name': 'Meritan Shabani', 'firstname': 'Meritan', 'lastname': 'Shabani', 'age': 23, 'birth': {'date': '1999-03-15', 'place': 'München', 'country': 'Germany'}, 'nationality': 'Germany', 'height': '185 cm', 'weight': '78 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/518.png'}, 'statistics': [{'team': {'id': 39, 'name': 'Wolves', 'logo': 'https://media.api-sports.io/football/teams/39.png'}, 'league': {'id': 39, 'name': 'Premier League', 'country': 'England', 'logo': 'https://media.api-sports.io/football/leagues/39.png', 'flag': 'https://media.api-sports.io/flags/gb.svg', 'season': 2020}, 'games': {'appearences': 0, 'lineups': 0, 'minutes': 0, 'number': None, 'position': 'Midfielder', 'rating': None, 'captain': False}, 'substitutes': {'in': 0, 'out': 0, 'bench': 3}, 'shots': {'total': None, 'on': None}, 'goals': {'total': 0, 'conceded': 0, 'assists': None, 'saves': None}, 'passes': {'total': None, 'key': None, 'accuracy': None}, 'tackles': {'total': None, 'blocks': None, 'interceptions': None}, 'duels': {'total': None, 'won': None}, 'dribbles': {'attempts': None, 'success': None, 'past': None}, 'fouls': {'drawn': None, 'committed': None}, 'cards': {'yellow': 0, 'yellowred': 0, 'red': 0}, 'penalty': {'won': None, 'commited': None, 'scored': 0, 'missed': 0, 'saved': None}}]},
This is the data type of each layer of the JSON file, from when I iterated through it with a For loop:
print(type(response)) <class 'dict'>
print(type(response_dump)) <class 'str'>
print(type(item)) <class 'str'>
print(type(player_item)) <class 'str'>
You do not have to json.dumps() in my opinion, just use the JSON from response to iterate:
for player in response['response']:
print(player)
{'player': {'id': 301, 'name': 'Benjamin Luke Woodburn', 'firstname': 'Benjamin Luke', 'lastname': 'Woodburn', 'age': 23, 'birth': {'date': '1999-10-15', 'place': 'Nottingham', 'country': 'England'}, 'nationality': 'Wales', 'height': '174 cm', 'weight': '72 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/301.png'}, 'statistics': [{'team': {'id': 40, 'name': 'Liverpool', 'logo': 'https://media.api-sports.io/football/teams/40.png'}, 'league': {'id': 39, 'name': 'Premier League', 'country': 'England', 'logo': 'https://media.api-sports.io/football/leagues/39.png', 'flag': 'https://media.api-sports.io/flags/gb.svg', 'season': 2020}, 'games': {'appearences': 0, 'lineups': 0, 'minutes': 0, 'number': None, 'position': 'Attacker', 'rating': None, 'captain': False}, 'substitutes': {'in': 0, 'out': 0, 'bench': 3}, 'shots': {'total': None, 'on': None}, 'goals': {'total': 0, 'conceded': 0, 'assists': None, 'saves': None}, 'passes': {'total': None, 'key': None, 'accuracy': None}, 'tackles': {'total': None, 'blocks': None, 'interceptions': None}, 'duels': {'total': None, 'won': None}, 'dribbles': {'attempts': None, 'success': None, 'past': None}, 'fouls': {'drawn': None, 'committed': None}, 'cards': {'yellow': 0, 'yellowred': 0, 'red': 0}, 'penalty': {'won': None, 'commited': None, 'scored': 0, 'missed': 0, 'saved': None}}]}
{'player': {'id': 518, 'name': 'Meritan Shabani', 'firstname': 'Meritan', 'lastname': 'Shabani', 'age': 23, 'birth': {'date': '1999-03-15', 'place': 'München', 'country': 'Germany'}, 'nationality': 'Germany', 'height': '185 cm', 'weight': '78 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/518.png'}, 'statistics': [{'team': {'id': 39, 'name': 'Wolves', 'logo': 'https://media.api-sports.io/football/teams/39.png'}, 'league': {'id': 39, 'name': 'Premier League', 'country': 'England', 'logo': 'https://media.api-sports.io/football/leagues/39.png', 'flag': 'https://media.api-sports.io/flags/gb.svg', 'season': 2020}, 'games': {'appearences': 0, 'lineups': 0, 'minutes': 0, 'number': None, 'position': 'Midfielder', 'rating': None, 'captain': False}, 'substitutes': {'in': 0, 'out': 0, 'bench': 3}, 'shots': {'total': None, 'on': None}, 'goals': {'total': 0, 'conceded': 0, 'assists': None, 'saves': None}, 'passes': {'total': None, 'key': None, 'accuracy': None}, 'tackles': {'total': None, 'blocks': None, 'interceptions': None}, 'duels': {'total': None, 'won': None}, 'dribbles': {'attempts': None, 'success': None, 'past': None}, 'fouls': {'drawn': None, 'committed': None}, 'cards': {'yellow': 0, 'yellowred': 0, 'red': 0}, 'penalty': {'won': None, 'commited': None, 'scored': 0, 'missed': 0, 'saved': None}}]}
or
for player in response['response']:
print(player['player'])
{'id': 301, 'name': 'Benjamin Luke Woodburn', 'firstname': 'Benjamin Luke', 'lastname': 'Woodburn', 'age': 23, 'birth': {'date': '1999-10-15', 'place': 'Nottingham', 'country': 'England'}, 'nationality': 'Wales', 'height': '174 cm', 'weight': '72 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/301.png'}
{'id': 518, 'name': 'Meritan Shabani', 'firstname': 'Meritan', 'lastname': 'Shabani', 'age': 23, 'birth': {'date': '1999-03-15', 'place': 'München', 'country': 'Germany'}, 'nationality': 'Germany', 'height': '185 cm', 'weight': '78 kg', 'injured': False, 'photo': 'https://media.api-sports.io/football/players/518.png'}
To get a DataFrame simply call pd.json_normalize() - Cause your question is not that clear I am not sure wiche information is needed and how to displayed. This is predestinated to ask a new question with exact that focus.:
pd.json_normalize(response['response'])
EDIT
Based on your comment and improvment:
pd.concat([pd.json_normalize(response,['response'])\
,pd.json_normalize(response,['response','statistics'])], axis=1)\
.drop(['statistics'], axis=1)
player.id
player.name
player.firstname
player.lastname
player.age
player.birth.date
player.birth.place
player.birth.country
player.nationality
player.height
player.weight
player.injured
player.photo
team.id
team.name
team.logo
league.id
league.name
league.country
league.logo
league.flag
league.season
games.appearences
games.lineups
games.minutes
games.number
games.position
games.rating
games.captain
substitutes.in
substitutes.out
substitutes.bench
shots.total
shots.on
goals.total
goals.conceded
goals.assists
goals.saves
passes.total
passes.key
passes.accuracy
tackles.total
tackles.blocks
tackles.interceptions
duels.total
duels.won
dribbles.attempts
dribbles.success
dribbles.past
fouls.drawn
fouls.committed
cards.yellow
cards.yellowred
cards.red
penalty.won
penalty.commited
penalty.scored
penalty.missed
penalty.saved
0
301
Benjamin Luke Woodburn
Benjamin Luke
Woodburn
23
1999-10-15
Nottingham
England
Wales
174 cm
72 kg
False
https://media.api-sports.io/football/players/301.png
40
Liverpool
https://media.api-sports.io/football/teams/40.png
39
Premier League
England
https://media.api-sports.io/football/leagues/39.png
https://media.api-sports.io/flags/gb.svg
2020
0
0
0
Attacker
False
0
0
3
0
0
0
0
0
0
0
1
518
Meritan Shabani
Meritan
Shabani
23
1999-03-15
München
Germany
Germany
185 cm
78 kg
False
https://media.api-sports.io/football/players/518.png
39
Wolves
https://media.api-sports.io/football/teams/39.png
39
Premier League
England
https://media.api-sports.io/football/leagues/39.png
https://media.api-sports.io/flags/gb.svg
2020
0
0
0
Midfielder
False
0
0
3
0
0
0
0
0
0
0
I'm fetchin some data from an api, and I want to insert it in too an azure sql db.
I'm getting the data and adding it to a pandas dataframe before dropping empty columns and such.
After I have done what i need to do I'm trying to insert the data in to the database, but I'm getting this error message: ProgrammingError: (pyodbc.ProgrammingError) ("A TVP's rows must be Sequence objects.", 'HY000')
I've found that some of the columns have nested data (and I would like to keep it that way)
These columns contain lists of dicts here are some examples:
0 [{'note': 'Netto per 45 dgr'}]
1 [{'note': 'Netto per 45 dgr'}]
2 [{'note': 'Netto per 45 dgr'}]
[{'accountingCost': None, 'allowanceCharge': array([], dtype=object), 'billingReference': array([], dtype=object), 'contractDocumentReference': None, 'delivery': {'actualDeliveryDate': None, 'deliveryLocation': None, 'deliveryParty': None, 'despatch': None, 'estimatedDeliveryPeriod': None, 'promisedDeliveryPeriod': None, 'requestedDeliveryPeriod': None, 'trackingId': None}, 'despatchLineReference': {'attachment': None, 'copyIndicator': None, 'documentDescription': None, 'documentType': None, 'documentTypeCode': None, 'id': '', 'issueDate': None, 'uuid': None}, 'documentReference': None, 'id': '1', 'invoicePeriod': None, 'invoicedQuantity': {'unitCode': 'H21', 'unitCodeListId': None, 'value': '1.00'}, 'item': {'additionalItemProperty': array([], dtype=object), 'brandName': None, 'certificate': array([], dtype=object), 'classifiedTaxCategory': None, 'commodityClassification': array([], dtype=object), 'description': array([], dtype=object), 'dimension': array([], dtype=object), 'hazardousItem': array([], dtype=object), 'itemInstance': array([], dtype=object), 'itemSpecificationDocumentReference': array([], dtype=object), 'keyword': array([], dtype=object), 'manufacturerParty': None, 'manufacturersItemIdentification': array([], dtype=object), 'name': 'Ansvarsrett', 'originAddress': None, 'originCountry': None, 'packQuantity': None, 'packSizeNumeric': '', 'sellersItemIdentification': {'extendedId': None, 'id': {'id': 'P550', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}}, 'standardItemIdentification': {'extendedId': None, 'id': {'id': '7043010000953', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': 'GTIN', 'schemeName': None}}, 'transactionConditions': array([], dtype=object)}, 'lineExtensionAmount': {'currency': None, 'value': '504.00'}, 'lineGrossExtensionAmount': None, 'note': '', 'orderLineReference': {'buyersReference': '117230405 Kvileitet', 'lineId': {'id': '0', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'orderReference': None, 'salesOrderLineId': {'id': '1', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'sellersReference': ''}, 'orderedQuantity': None, 'price': {'allowanceCharge': array([], dtype=object), 'amount': {'currency': None, 'value': '504.00'}, 'baseQuantity': {'unitCode': None, 'unitCodeListId': None, 'value': '0'}, 'orderableUnitFactorRate': None, 'priceType': None, 'validityPeriod': array([], dtype=object)}, 'taxTotal': {'roundingAmount': None, 'taxAmount': {'currency': None, 'value': '126.00'}, 'taxSubtotal': array([], dtype=object)}}
{'accountingCost': None, 'allowanceCharge': array([], dtype=object), 'billingReference': array([], dtype=object), 'contractDocumentReference': None, 'delivery': {'actualDeliveryDate': None, 'deliveryLocation': None, 'deliveryParty': None, 'despatch': None, 'estimatedDeliveryPeriod': None, 'promisedDeliveryPeriod': None, 'requestedDeliveryPeriod': None, 'trackingId': None}, 'despatchLineReference': {'attachment': None, 'copyIndicator': None, 'documentDescription': None, 'documentType': None, 'documentTypeCode': None, 'id': '', 'issueDate': None, 'uuid': None}, 'documentReference': None, 'id': '2', 'invoicePeriod': None, 'invoicedQuantity': {'unitCode': 'H21', 'unitCodeListId': None, 'value': '3.00'}, 'item': {'additionalItemProperty': array([], dtype=object), 'brandName': None, 'certificate': array([], dtype=object), 'classifiedTaxCategory': None, 'commodityClassification': array([], dtype=object), 'description': array([], dtype=object), 'dimension': array([], dtype=object), 'hazardousItem': array([], dtype=object), 'itemInstance': array([], dtype=object), 'itemSpecificationDocumentReference': array([], dtype=object), 'keyword': array([], dtype=object), 'manufacturerParty': None, 'manufacturersItemIdentification': array([], dtype=object), 'name': 'Prosjektering', 'originAddress': None, 'originCountry': None, 'packQuantity': None, 'packSizeNumeric': '', 'sellersItemIdentification': {'extendedId': None, 'id': {'id': 'Projj', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}}, 'standardItemIdentification': {'extendedId': None, 'id': {'id': '7043010000298', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': 'GTIN', 'schemeName': None}}, 'transactionConditions': array([], dtype=object)}, 'lineExtensionAmount': {'currency': None, 'value': '7020.00'}, 'lineGrossExtensionAmount': None, 'note': '', 'orderLineReference': {'buyersReference': '117230405 Kvileitet', 'lineId': {'id': '0', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'orderReference': None, 'salesOrderLineId': {'id': '2', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'sellersReference': ''}, 'orderedQuantity': None, 'price': {'allowanceCharge': array([], dtype=object), 'amount': {'currency': None, 'value': '2340.00'}, 'baseQuantity': {'unitCode': None, 'unitCodeListId': None, 'value': '0'}, 'orderableUnitFactorRate': None, 'priceType': None, 'validityPeriod': array([], dtype=object)}, 'taxTotal': {'roundingAmount': None, 'taxAmount': {'currency': None, 'value': '1755.00'}, 'taxSubtotal': array([], dtype=object)}}
is there any way to add this data to mssql?
I am iterating over json file and creating dataframe with the desirable columns. I already implemented the code but now json file has little bit changed. But I am not able to think where to change the code to get the required output.
Explanation:
previous json result:
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-18T11:49:48.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 7,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
previous dataframe:
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-18T11:49:48.000Z,7,,,,1.0,,,
new json result:
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
Now desirable dataframe:
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
What are the changes I need to do to in below code to get the new result.
column_names = []
if(query_result.results != None):
for item in query_result.results:
data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
for met in lst_data.metrics:
metric_name = met.metric
column_names.append('Queue_Id')
column_names.append(metric_name+'_count')
column_names.append(metric_name+'_sum')
column_names.append(metric_name+'.denominator')
column_names.append(metric_name+'.numerator')
column_names.append('Interval Start')
column_names.append('Interval End')
data_lst.append(queue_id)
data_lst.append(met.stats.count)
data_lst.append(met.stats.sum)
data_lst.append(met.stats.denominator)
data_lst.append(met.stats.numerator)
data_lst.append(lst_data.interval.split('/')[0])
data_lst.append(lst_data.interval.split('/')[1])
print(data_lst)
else:
data_lst = []
metric_name = query.metrics[0]
column_names.append('Queue_Id')
column_names.append(metric_name+'_count')
column_names.append(metric_name+'_sum')
column_names.append(metric_name+'.denominator')
column_names.append(metric_name+'.numerator')
column_names.append('Interval Start')
column_names.append('Interval End')
data_lst.append(queue_id)
data_lst.append('')
data_lst.append('')
data_lst.append('')
data_lst.append('')
data_lst.append(query.interval.split('/')[0])
data_lst.append(query.interval.split('/')[1])
print("data_lst", data_lst)
print("column_names", column_names)
return data_lst, column_names
I have modified my code little bit and got the result. The below code is working for me-
lst_of_metrics = ["nOffered", "nTransferred"]
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met1, met in itertools.zip_longest(query.metrics, lst_data.metrics):
#for met in lst_data.metrics:
if(met):
if(met.metric == met1):
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)
I have a json result which I am trying to convert into dataframe but not able to get the correct result. Actually for some cases it is giving correct but for some case it is failing.
Example:
Based on metric API is generating result for specified interval. But this is not certain for that particular interval metric have output or not. And process is running 4 different queue_id.
suppose process is running only for 2 metric. ['nOffered', 'nTransferred']
queue_id = 'a72dba75-0bc6-4a65-b120-8803364f8dc3'
for this queue_id, nOffered is having some values but nTransferred doesn't have. Json result is given below-
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
My code is giving below output-
queue_id nOffered_count nOffered_sum interval_start interval_end
0 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-11T11:46:25.000Z 2021-10-12T11:46:25.000Z
1 a72dba75-0bc6-4a65-b120-8803364f8dc3 1 None 2021-10-12T11:46:25.000Z 2021-10-13T11:46:25.000Z
2 a72dba75-0bc6-4a65-b120-8803364f8dc3 12 None 2021-10-13T11:46:25.000Z 2021-10-14T11:46:25.000Z
3 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-14T11:46:25.000Z 2021-10-15T11:46:25.000Z
4 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-15T11:46:25.000Z 2021-10-16T11:46:25.000Z
But when process is running for 2nd queue_id that time it is not working-
queue_id - 73643cff-799b-41ae-9a67-efcf5e593155
json output for this queue_id -
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
This time both metric having some data. So output would be-
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
And in final result, both the result merge and give the output with all columns and data.
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,6,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-12T11:46:25.000Z,2021-10-13T11:46:25.000Z,1.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,12.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,6.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,6.0,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1.0,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
Currently I am running below logic-
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met in lst_data.metrics:
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)
return df
I used below logic to get the desirable result. It is working for me.
lst_of_metrics = ["nOffered", "nTransferred"]
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met1, met in itertools.zip_longest(query.metrics, lst_data.metrics):
if(met):
if(met.metric == met1):
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)