I'm looking to scrape some data from the following web site: https://www.atptour.com/en/scores/stats-centre/archive/2022/414/MS024
The underlying match statistics are loaded by the following javacript:
<script nonce="5e824960-ee11-4894-b33c-9b0a25ace2d3" type="text/javascript">
var initial = {
matchStatsContainer : "#matchStatsContainer",
matchStatsTemplate : "#matchStatsTemplate",
currentMatchStatsToggle : "currentMatchStatsButton",
liveMatchUrl : "",
setPhrase : "set",
matchPhrase : "match",
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
}
require(["modules/modal/modalScoresMatchStats"],
function (ModalScoresMatchStats) {
return new ModalScoresMatchStats({
matchStatsContainer : "#match-stats-container",
matchStatsTemplate : "#matchStatsTemplate",
currentMatchStatsToggle : "currentMatchStatsButton",
liveMatchUrl : "",
setPhrase : "set",
matchPhrase : "match",
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
});
});
</script>
I am using the following code with script set to:
var initial = { .....
liveMatchDetails : {
isMatchLive : 'True',
eventId : '414',
matchId : 'MS024'
}
});
});
html = HTML(html=str(match_stats_html), async_=True)
val = html.render(script=script, reload=False)
I get the following error:
ElementHandleError: Evaluation failed: ReferenceError: require is not defined
at pyppeteer_evaluation_script:16:9
When I remove require from the script I get the following error:
ElementHandleError: Evaluation failed: SyntaxError: Unexpected token (
Any ideas on how I could execute the script and obtain the data returned by the results?
Thanks
import cloudscraper
scraper = cloudscraper.create_scraper()
r = scraper.get('https://www.atptour.com/-/ajax/Scores/GetInitialScores')
print(r.json())
This returns:
{'liveScores': {'Tournaments': [{'EventYear': 2022, 'Name': 'Hamburg European Open', 'EventId': '414', 'TournamentLink': '/en/tournaments/hamburg/414/overview', 'OfficialWebsite': 'www.hamburg-open.com', 'Location': 'Hamburg, Germany', 'TournamentBadgeImage': '/-/media/images/tourtypes/categorystamps_500_118x136.png?sc=0&hash=7431FC4DF8556F6BB9AA56E8578D5342', 'LiveScoresLinkText': 'Scores', 'LiveScoresLink': '/en/scores/current/hamburg/414/live-scores', 'DrawLinkText': 'Draw', 'DrawLink': '/en/scores/current/hamburg/414/draws', 'ScheduleLinkText': 'Schedule', 'ScheduleLink': '/en/scores/current/hamburg/414/daily-schedule', 'H2HLinkText': 'H2H', 'StatsLinkText': 'Stats', 'PlayNotStarted': '#Play Not Started Yet#', 'TournamentPosition': '1', 'IsLaverCup': False, 'Matches': [{'Id': 'MS008', 'ChallengerTVMatchItemId': '', 'KSeq': 415, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS008', 'HeadToHeadLink': '/en/players/atp-head-2-head/c-alcaraz-vs-f-krajinovic/A0E2/KB05', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 1, 'LiveListingPosition': 1, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:52:54', 'MatchInfo': 'Game Set and Match Carlos Alcaraz. \r\n Carlos Alcaraz wins the match 7-6(4) 6-3 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'A0E2', 'PartnerId': None, 'PlayerFirstName': 'C.', 'PlayerLastName': 'Alcaraz', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ESP', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'C. Alcaraz', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Carlos-Alcaraz', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ESP.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/carlos-alcaraz/a0e2/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/A0E2', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '7', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'KB05', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Krajinovic', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'SRB', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Krajinovic', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Filip-Krajinovic', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/SRB.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/filip-krajinovic/kb05/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/KB05', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': '4', 'SetTwo': '3', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS015', 'ChallengerTVMatchItemId': '', 'KSeq': 384, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS015', 'HeadToHeadLink': '/en/players/atp-head-2-head/f-cerundolo-vs-a-rublev/C0AU/RE44', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 2, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:32:08', 'MatchInfo': 'Game Set and Match Francisco Cerundolo. \r\n Francisco Cerundolo wins the match 6-4 6-2 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'C0AU', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Cerundolo', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ARG', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Cerundolo', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Francisco-Cerundolo', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ARG.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/francisco-cerundolo/c0au/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/C0AU', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'RE44', 'PartnerId': None, 'PlayerFirstName': 'A.', 'PlayerLastName': 'Rublev', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'RUS', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'A. Rublev', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Andrey-Rublev', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/RUS.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/andrey-rublev/re44/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/RE44', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '4', 'SetOneTiebreak': None, 'SetTwo': '2', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS009', 'ChallengerTVMatchItemId': '', 'KSeq': 384, 'EventId': '414', 'RoundTitle': 'Round of 16 - Center Court', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/414/MS009', 'HeadToHeadLink': '/en/players/atp-head-2-head/f-fognini-vs-k-khachanov/F510/KE29', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/414/all', 'TeamTieStatsLink': '', 'RailPosition': 3, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '3', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:42:52', 'MatchInfo': 'Game Set and Match Karen Khachanov. \r\n Karen Khachanov wins the match 6-3 7-5 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'F510', 'PartnerId': None, 'PlayerFirstName': 'F.', 'PlayerLastName': 'Fognini', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'F. Fognini', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Fabio-Fognini', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/fabio-fognini/f510/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/F510', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '3', 'SetOneTiebreak': None, 'SetTwo': '5', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'KE29', 'PartnerId': None, 'PlayerFirstName': 'K.', 'PlayerLastName': 'Khachanov', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'RUS', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'K. Khachanov', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Karen-Khachanov', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/RUS.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/karen-khachanov/ke29/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/KE29', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '7', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}]}, {'EventYear': 2022, 'Name': 'EFG Swiss Open Gstaad', 'EventId': '314', 'TournamentLink': '/en/tournaments/gstaad/314/overview', 'OfficialWebsite': 'swissopengstaad.ch', 'Location': 'Gstaad, Switzerland', 'TournamentBadgeImage': '/-/media/images/tourtypes/categorystamps_250_118x136.png?sc=0&hash=10547153B5ADD7DCF2E0C8DAA42886EE', 'LiveScoresLinkText': 'Scores', 'LiveScoresLink': '/en/scores/current/gstaad/314/live-scores', 'DrawLinkText': 'Draw', 'DrawLink': '/en/scores/current/gstaad/314/draws', 'ScheduleLinkText': 'Schedule', 'ScheduleLink': '/en/scores/current/gstaad/314/daily-schedule', 'H2HLinkText': 'H2H', 'StatsLinkText': 'Stats', 'PlayNotStarted': '#Play Not Started Yet#', 'TournamentPosition': '2', 'IsLaverCup': False, 'Matches': [{'Id': 'MD007', 'ChallengerTVMatchItemId': '', 'KSeq': 218, 'EventId': '314', 'RoundTitle': 'Quarterfinal - Court 1', 'MatchType': 'doubles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MD007', 'HeadToHeadLink': '/en/players/atp-head-2-head/l-sonego-vs-r-jebavy/SU87/J321', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 1, 'LiveListingPosition': 1, 'Status': 'P', 'Winner': '0', 'LastServer': '0', 'NumberOfSets': 3, 'MatchTime': '00:45:31', 'MatchInfo': 'L. Sonego/A. Vavassori win the point on the 1st serve of L. Sonego.', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'SU87', 'PartnerId': 'VA08', 'PlayerFirstName': 'L.', 'PlayerLastName': 'Sonego', 'PartnerFirstName': 'A.', 'PartnerLastName': 'Vavassori', 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': 'ITA', 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'L. Sonego', 'PartnerNameForUrl': 'Andrea-Vavassori', 'PlayerNameForUrl': 'Lorenzo-Sonego', 'PlayerTwoName': 'A. Vavassori', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerOneLink': '/en/players/lorenzo-sonego/su87/overview', 'PlayerTwoLink': '/en/players/andrea-vavassori/va08/overview', 'PlayerOneHeadshot': '/-/media/alias/player-headshot/SU87', 'PlayerTwoHeadshot': '/-/media/alias/player-headshot/VA08', 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'now-serving', 'Scores': {'CurrentScore': '30', 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '3', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'J321', 'PartnerId': 'O474', 'PlayerFirstName': 'R.', 'PlayerLastName': 'Jebavy', 'PartnerFirstName': 'J.', 'PartnerLastName': "O'Mara", 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'CZE', 'PartnerCountryCode': 'GBR', 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'R. Jebavy', 'PartnerNameForUrl': "Jonny-O'Mara", 'PlayerNameForUrl': 'Roman-Jebavy', 'PlayerTwoName': "J. O'Mara", 'PlayerOneFlag': '/en/~/media/images/flags/CZE.svg', 'PlayerTwoFlag': '/en/~/media/images/flags/GBR.svg', 'PlayerOneLink': '/en/players/roman-jebavy/j321/overview', 'PlayerTwoLink': "/en/players/jonny-o'mara/o474/overview", 'PlayerOneHeadshot': '/-/media/alias/player-headshot/J321', 'PlayerTwoHeadshot': '/-/media/alias/player-headshot/O474', 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': '0', 'SetOne': '2', 'SetOneTiebreak': None, 'SetTwo': '1', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS008', 'ChallengerTVMatchItemId': '', 'KSeq': 432, 'EventId': '314', 'RoundTitle': 'Round of 16 - Roy Emerson Arena', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MS008', 'HeadToHeadLink': '/en/players/atp-head-2-head/c-ruud-vs-j-lehecka/RH16/L0BV', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 2, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '2', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:39:31', 'MatchInfo': 'Game Set and Match Casper Ruud. \r\n Casper Ruud wins the match 6-3 6-4 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'RH16', 'PartnerId': None, 'PlayerFirstName': 'C.', 'PlayerLastName': 'Ruud', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'NOR', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'C. Ruud', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Casper-Ruud', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/NOR.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/casper-ruud/rh16/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/RH16', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'L0BV', 'PartnerId': None, 'PlayerFirstName': 'J.', 'PlayerLastName': 'Lehecka', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'CZE', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'J. Lehecka', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Jiri-Lehecka', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/CZE.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/jiri-lehecka/l0bv/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/L0BV', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '3', 'SetOneTiebreak': None, 'SetTwo': '4', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}, {'Id': 'MS015', 'ChallengerTVMatchItemId': '', 'KSeq': 374, 'EventId': '314', 'RoundTitle': 'Round of 16 - Roy Emerson Arena', 'MatchType': 'singles', 'StatsLink': 'en/scores/stats-centre/live/2022/314/MS015', 'HeadToHeadLink': '/en/players/atp-head-2-head/r-gasquet-vs-m-berrettini/G628/BK40', 'SecondScreenLink': '', 'ChallengerTVLink': '/en/atp-challenger-tour/challenger-tv/challenger-tv-search-results//2022/314/all', 'TeamTieStatsLink': '', 'RailPosition': 3, 'LiveListingPosition': None, 'Status': 'F', 'Winner': '3', 'LastServer': None, 'NumberOfSets': 3, 'MatchTime': '01:29:29', 'MatchInfo': 'Game Set and Match Matteo Berrettini. \r\n Matteo Berrettini wins the match 6-4 6-4 .', 'TeamTieResults': {'RoundRobinGroupNumber': 0, 'RoundRobinGroupName': None, 'RoundRobinCityName': None, 'TeamCountryCode': None, 'OpponentTeamCountryCode': None, 'TeamTieMatchWins': 0, 'OpponentTeamTieMatchWins': 0, 'TotalTieMatchesPlayed': 0, 'TeamCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'OpponentCountryFlag': '/en/~/media/images/flags/ghost-flag.svg', 'HasTeamTieStats': False, 'IsRoundRobin': False}, 'TeamOne': {'PlayerId': 'G628', 'PartnerId': None, 'PlayerFirstName': 'R.', 'PlayerLastName': 'Gasquet', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'FRA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'R. Gasquet', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Richard-Gasquet', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/FRA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/richard-gasquet/g628/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/G628', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': '', 'Scores': {'CurrentScore': None, 'SetOne': '4', 'SetOneTiebreak': None, 'SetTwo': '4', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'TeamTwo': {'PlayerId': 'BK40', 'PartnerId': None, 'PlayerFirstName': 'M.', 'PlayerLastName': 'Berrettini', 'PartnerFirstName': '', 'PartnerLastName': None, 'PlayerSeed': None, 'PlayerEntryType': None, 'PlayerCountryCode': 'ITA', 'PartnerCountryCode': None, 'PartnerSeed': None, 'PartnerEntryType': None, 'GamePointsPlayerTeam': None, 'PlayerOneName': 'M. Berrettini', 'PartnerNameForUrl': '', 'PlayerNameForUrl': 'Matteo-Berrettini', 'PlayerTwoName': ' ', 'PlayerOneFlag': '/en/~/media/images/flags/ITA.svg', 'PlayerTwoFlag': None, 'PlayerOneLink': '/en/players/matteo-berrettini/bk40/overview', 'PlayerTwoLink': None, 'PlayerOneHeadshot': '/-/media/alias/player-headshot/BK40', 'PlayerTwoHeadshot': None, 'TeamSeed': None, 'TeamEntryType': None, 'TeamStatus': 'won-game', 'Scores': {'CurrentScore': None, 'SetOne': '6', 'SetOneTiebreak': None, 'SetTwo': '6', 'SetTwoTiebreak': None, 'SetThree': None, 'SetThreeTiebreak': None, 'SetFour': None, 'SetFourTiebreak': None, 'SetFive': None, 'SetFiveTiebreak': None}}, 'IsWatchLive': False, 'HasHawkeyeData': False, 'HasStats': True, 'HasHead2Head': True, 'HasChallengerTVWeb': False}]}]}}
cloudscraper can be installed with pip install cloudscraper
Related
I have two datasets - one as a dataframe and the other as an array of JSON files.
Each line in the df has a string (folio number) that identifies a piece of land (Ex: '0101000000030'), and a date (in datetime) a permit was applied for.
Every JSON file in the array has a corresponding number identifying that land. It also has dates the property was sold, to whom it was sold, and the seller.
I need to take the folio number and the date the permit was applied for and run it through the array of JSON files until it finds the matching folio.
Then, it needs to extract the property's owner information by finding who owned the property when the permit was applied for and append it to the corresponding row in the df.
Desired Output
FirstSubmissionDate Folio PropertyOwner
05/17/2018 '0101000000030' blahblah
Input DF
FirstSubmissionDate Folio
05/17/2018 '0101000000030'
Input JSON
{'Additionals': {'AddtionalInfo': [{'Key': 'LAND USE AND RESTRICTIONS',
'Value': [{'InfoName': 'Community Development District',
'InfoValue': 'COUNTYGIS',
'Message': ''},
{'InfoName': 'Community Redevelopment Area',
'InfoValue': 'COUNTYGIS',
'Message': ''},
{'InfoName': 'Empowerment Zone', 'InfoValue': 'COUNTYGIS', 'Message': ''},
{'InfoName': 'Enterprise Zone', 'InfoValue': 'COUNTYGIS', 'Message': ''},
{'InfoName': 'Urban Development',
'InfoValue': 'COUNTYGIS',
'Message': ''},
{'InfoName': 'Zoning Code', 'InfoValue': 'COUNTYGIS', 'Message': ''},
{'InfoName': 'Existing Land Use',
'InfoValue': 'COUNTYGIS',
'Message': ''},
{'InfoName': 'Government Agencies and Community Services',
'InfoValue': 'http://gisweb.miamidade.gov/communityservices/CommunityServicesAll.html?x=&y=&bufferDistance=5&address=60 SE 2 ST',
'Message': ''}]},
{'Key': 'OTHER GOVERNMENTAL JURISDICTIONS',
'Value': [{'InfoName': 'Business Incentives',
'InfoValue': 'https://gisweb.miamidade.gov/businessincentive/default.aspx?searchtype=address¶mvalue=',
'Message': ''},
{'InfoName': 'Childrens Trust',
'InfoValue': 'https://www.thechildrenstrust.org/',
'Message': ''},
{'InfoName': 'City of Miami',
'InfoValue': 'http://www.miamigov.com/home/',
'Message': ''},
{'InfoName': 'Environmental Considerations',
'InfoValue': 'https://gisweb.miamidade.gov/environmentalconsiderations/default.aspx?searchtype=address¶mvalue=60 SE 2 ST',
'Message': ''},
{'InfoName': 'Florida Inland Navigation District',
'InfoValue': 'http://www.aicw.org',
'Message': ''},
{'InfoName': 'PA Bulletin Board',
'InfoValue': 'http://bbs.miamidade.gov/',
'Message': ''},
{'InfoName': 'Special Taxing District and Other Non-Ad valorem Assessment',
'InfoValue': 'http://www.miamidade.gov/Apps/PA/PAOnlineTools/Taxes/NonAdvalorem.aspx?folio=0101000000030',
'Message': ''},
{'InfoName': 'School Board',
'InfoValue': 'http://www.dadeschools.net/',
'Message': ''},
{'InfoName': 'South Florida Water Mgmt District',
'InfoValue': 'http://www.sfwmd.gov/portal/page/portal/sfwmdmain/home%20page',
'Message': ''},
{'InfoName': 'Tax Collector',
'InfoValue': 'http://www.miamidade.gov/taxcollector/',
'Message': ''}]}],
'FooterMessage': '',
'HeaderMessage': "* The information listed below is not derived from the Property Appraiser's Office records. It is provided for convenience and is derived from other government agencies."},
'Assessment': {'AssessmentInfos': [{'AssessedValue': 5587359,
'BuildingOnlyValue': 0,
'ExtraFeatureValue': 0,
'LandValue': 7618560,
'Message': None,
'TotalValue': 7618560,
'Year': 2021},
{'AssessedValue': 5079418,
'BuildingOnlyValue': 0,
'ExtraFeatureValue': 0,
'LandValue': 6963200,
'Message': None,
'TotalValue': 6963200,
'Year': 2020},
{'AssessedValue': 4617653,
'BuildingOnlyValue': 0,
'ExtraFeatureValue': 0,
'LandValue': 6963200,
'Message': None,
'TotalValue': 6963200,
'Year': 2019}],
'Messages': [{'Message': '', 'Year': 2021},
{'Message': '', 'Year': 2020},
{'Message': '', 'Year': 2019}]},
'Benefit': {'BenefitInfos': [{'Description': 'Non-Homestead Cap',
'Message': None,
'Seq': '5',
'TaxYear': 2021,
'Type': 'Assessment Reduction',
'Url': 'http://www.miamidade.gov/pa/property_value_cap.asp',
'Value': 2031201},
{'Description': 'Non-Homestead Cap',
'Message': None,
'Seq': '5',
'TaxYear': 2020,
'Type': 'Assessment Reduction',
'Url': 'http://www.miamidade.gov/pa/property_value_cap.asp',
'Value': 1883782},
{'Description': 'Non-Homestead Cap',
'Message': None,
'Seq': '5',
'TaxYear': 2019,
'Type': 'Assessment Reduction',
'Url': 'http://www.miamidade.gov/pa/property_value_cap.asp',
'Value': 2345547}],
'Messages': []},
'Building': {'BuildingInfos': [], 'Messages': []},
'ClassifiedAgInfo': {'Acreage': 0,
'CalculatedValue': 0,
'LandCode': None,
'LandUse': None,
'Message': None,
'UnitPrice': 0},
'Completed': True,
'District': 6,
'ExtraFeature': {'ExtraFeatureInfos': [], 'Messages': []},
'GeoParcel': None,
'Land': {'Landlines': [{'AdjustedUnitPrice': 465,
'CalculatedValue': 7618560,
'Depth': 0,
'FrontFeet': 0,
'LandUse': 'GENERAL',
'LandlineType': 'C',
'Message': None,
'MuniZone': 'T6-80-O',
'MuniZoneDescription': None,
'PAZoneDescription': 'COMMERCIAL',
'PercentCondition': 1,
'RollYear': 2021,
'TotalAdjustments': 1,
'UnitType': 'Square Ft.',
'Units': 16384,
'UseCode': '00',
'Zone': '6401'},
{'AdjustedUnitPrice': -1,
'CalculatedValue': -1,
'Depth': 0,
'FrontFeet': 0,
'LandUse': 'GENERAL',
'LandlineType': 'C',
'Message': None,
'MuniZone': 'T6-80-O',
'MuniZoneDescription': None,
'PAZoneDescription': 'COMMERCIAL',
'PercentCondition': 1,
'RollYear': 2020,
'TotalAdjustments': 1,
'UnitType': 'Square Ft.',
'Units': 16384,
'UseCode': '00',
'Zone': '6401'},
{'AdjustedUnitPrice': -1,
'CalculatedValue': -1,
'Depth': 0,
'FrontFeet': 0,
'LandUse': 'GENERAL',
'LandlineType': 'C',
'Message': None,
'MuniZone': 'T6-80-O',
'MuniZoneDescription': None,
'PAZoneDescription': 'COMMERCIAL',
'PercentCondition': 1,
'RollYear': 2019,
'TotalAdjustments': 1,
'UnitType': 'Square Ft.',
'Units': 16384,
'UseCode': '00',
'Zone': '6401'}],
'Messages': [{'Message': '', 'Year': 2021},
{'Message': 'The calculated values for this property have been overridden. Please refer to the Land, Building, and XF Values in the Assessment Section, in order to obtain the most accurate values.',
'Year': 2020},
{'Message': 'The calculated values for this property have been overridden. Please refer to the Land, Building, and XF Values in the Assessment Section, in order to obtain the most accurate values.',
'Year': 2019}]},
'LegalDescription': {'Description': 'MIAMI NORTH PB B-41|BEG 12.2FT W OF X OF S/L OF SE 2|ST & W/L OF SE 1 AVE TH S11.85FT|SWLY A/D 72.55FT S52.71FT|W108.69FT N10FT W4.6FT N123.52FT|E137.4FT TO POB|LOT SIZE 16384 SQ FT|COC 25843-0025 26307-3840 0707 6',
'Message': None,
'Number': None},
'MailingAddress': {'Address1': '1000 BRICKELL AVE STE 400',
'Address2': '',
'Address3': '',
'City': 'MIAMI',
'Country': 'USA',
'Message': None,
'State': 'FL',
'ZipCode': '33131'},
'Message': '',
'OwnerInfos': [{'Description': 'Sole Owner',
'MarriedFlag': '0',
'Message': None,
'Name': '16 SE 2ND STREET DOWNTOWN',
'PercentageOwn': 1,
'Role': None,
'ShortDescription': 'Sole Owner',
'TenancyCd': 'S'},
{'Description': 'Sole Owner',
'MarriedFlag': '0',
'Message': None,
'Name': 'INVESTMENT LLC',
'PercentageOwn': 1,
'Role': None,
'ShortDescription': 'Sole Owner',
'TenancyCd': 'S'}],
'PropertyInfo': {'BathroomCount': 0,
'BedroomCount': 0,
'BuildingActualArea': 0,
'BuildingBaseArea': 0,
'BuildingEffectiveArea': 0,
'BuildingGrossArea': 0,
'BuildingHeatedArea': 0,
'DORCode': '1081',
'DORDescription': 'VACANT LAND - COMMERCIAL : VACANT LAND',
'DORDescriptionCurrent': None,
'EncodedFolioAndTaxYear': 'J1COeydnmm%2fHHVEoyromqjt3GPqH8da%2fsulgVBOgI7w%3d',
'FloorCount': 0,
'FolioNumber': '01-0100-000-0030',
'HalfBathroomCount': 0,
'HxBaseYear': 0,
'LotSize': 16384,
'Message': None,
'Municipality': 'Miami',
'Neighborhood': 69010,
'NeighborhoodDescription': 'Miami CBD',
'ParentFolio': '',
'PercentHomesteadCapped': 0,
'PlatBook': 'B',
'PlatPage': '41',
'PrimaryZone': '6401',
'PrimaryZoneDescription': 'COMMERCIAL',
'ShowCurrentValuesFlag': 'N',
'Status': 'AC Active',
'Subdivision': '010100000',
'SubdivisionDescription': '353017046',
'UnitCount': 0,
'YearBuilt': '0'},
'RollYear1': 2021,
'SalesInfos': [{'DateOfSale': '6/23/2021',
'DocumentStamps': 276000,
'EncodedRecordBookAndPage': 'lHVlhHQhIZoJRUYKiXnhi4goVgjenckUAcgPekALEZ8LlG%2bmH%2bycTA%3d%3d',
'GranteeName1': '16 SE 2ND STREET DOWNTOWN',
'GranteeName2': 'INVESTMENT LLC',
'GrantorName1': '16 SE 2ND STREET LLC',
'GrantorName2': '',
'Message': None,
'OfficialRecordBook': '32602',
'OfficialRecordPage': '3521',
'QualificationDescription': 'Qual on DOS, multi-parcel sale',
'QualifiedFlag': 'Q',
'QualifiedSYear': None,
'QualifiedSourceCode': '',
'ReasonCode': '05',
'ReviewCode': None,
'SaleId': 5,
'SaleInstrument': 'WDE',
'SalePrice': 46000000,
'VacantFlag': '\x00',
'ValidCode': None,
'VerifyCode': None},
{'DateOfSale': '5/24/2013',
'DocumentStamps': 0,
'EncodedRecordBookAndPage': 'lHVlhHQhIZoJRUYKiXnhiyo2fiU6Ad2Yj6ROwqxBp26vA0B1JkALuQ%3d%3d',
'GranteeName1': '16 SE 2ND STREET LLC',
'GranteeName2': '',
'GrantorName1': 'BURDINES 1225 LLC',
'GrantorName2': '',
'Message': None,
'OfficialRecordBook': '28688',
'OfficialRecordPage': '1169',
'QualificationDescription': 'Financial inst or "In Lieu of Forclosure" stated',
'QualifiedFlag': 'U',
'QualifiedSYear': None,
'QualifiedSourceCode': '',
'ReasonCode': '12',
'ReviewCode': None,
'SaleId': 4,
'SaleInstrument': 'DEE',
'SalePrice': 32620638,
'VacantFlag': '\x00',
'ValidCode': None,
'VerifyCode': None},
{'DateOfSale': '8/1/1989',
'DocumentStamps': 0,
'EncodedRecordBookAndPage': 'lHVlhHQhIZoJRUYKiXnhi9bvfovmAqmTIZ5uJf3HEgtQChvRqiPQDw%3d%3d',
'GranteeName1': '',
'GranteeName2': '',
'GrantorName1': '',
'GrantorName2': '',
'Message': None,
'OfficialRecordBook': '14202',
'OfficialRecordPage': '2339',
'QualificationDescription': 'Deeds that include more than one parcel',
'QualifiedFlag': 'Q',
'QualifiedSYear': None,
'QualifiedSourceCode': '',
'ReasonCode': '02',
'ReviewCode': None,
'SaleId': 3,
'SaleInstrument': '',
'SalePrice': 6200000,
'VacantFlag': '\x00',
'ValidCode': None,
'VerifyCode': None},
{'DateOfSale': '9/1/2003',
'DocumentStamps': 0,
'EncodedRecordBookAndPage': 'lHVlhHQhIZoJRUYKiXnhi5pTmn2bXcBBM42%2bwPcIyhry9UhcpSwX4g%3d%3d',
'GranteeName1': '',
'GranteeName2': '',
'GrantorName1': '',
'GrantorName2': '',
'Message': None,
'OfficialRecordBook': '21695',
'OfficialRecordPage': '3500',
'QualificationDescription': 'Deeds that include more than one parcel',
'QualifiedFlag': 'Q',
'QualifiedSYear': None,
'QualifiedSourceCode': '',
'ReasonCode': '02',
'ReviewCode': None,
'SaleId': 2,
'SaleInstrument': '',
'SalePrice': 8800000,
'VacantFlag': '\x00',
'ValidCode': None,
'VerifyCode': None},
{'DateOfSale': '7/1/2007',
'DocumentStamps': 0,
'EncodedRecordBookAndPage': 'lHVlhHQhIZoJRUYKiXnhi5bVa6yEIUa%2bSDngqM2N5YUM89ag%2fj8HOA%3d%3d',
'GranteeName1': '',
'GranteeName2': '',
'GrantorName1': '',
'GrantorName2': '',
'Message': None,
'OfficialRecordBook': '25843',
'OfficialRecordPage': '0025',
'QualificationDescription': 'Other disqualified',
'QualifiedFlag': 'U',
'QualifiedSYear': None,
'QualifiedSourceCode': '',
'ReasonCode': '03',
'ReviewCode': None,
'SaleId': 1,
'SaleInstrument': '',
'SalePrice': 21500000,
'VacantFlag': '\x00',
'ValidCode': None,
'VerifyCode': None}],
'SiteAddress': [{'Address': '60 SE 2 ST, Miami, FL 33131-2103',
'BuildingNumber': 1,
'City': 'Miami',
'HouseNumberSuffix': '',
'Message': None,
'StreetName': '2',
'StreetNumber': 60,
'StreetPrefix': 'SE',
'StreetSuffix': 'ST',
'StreetSuffixDirection': '',
'Unit': '',
'Zip': '33131-2103'}],
'Taxable': {'Messages': [],
'TaxableInfos': [{'CityExemptionValue': 0,
'CityTaxableValue': 5587359,
'CountyExemptionValue': 0,
'CountyTaxableValue': 5587359,
'Message': None,
'RegionalExemptionValue': 0,
'RegionalTaxableValue': 5587359,
'SchoolExemptionValue': 0,
'SchoolTaxableValue': 7618560,
'Year': 2021},
{'CityExemptionValue': 0,
'CityTaxableValue': 5079418,
'CountyExemptionValue': 0,
'CountyTaxableValue': 5079418,
'Message': None,
'RegionalExemptionValue': 0,
'RegionalTaxableValue': 5079418,
'SchoolExemptionValue': 0,
'SchoolTaxableValue': 6963200,
'Year': 2020},
{'CityExemptionValue': 0,
'CityTaxableValue': 4617653,
'CountyExemptionValue': 0,
'CountyTaxableValue': 4617653,
'Message': None,
'RegionalExemptionValue': 0,
'RegionalTaxableValue': 4617653,
'SchoolExemptionValue': 0,
'SchoolTaxableValue': 6963200,
'Year': 2019}]}}
I can distill down to the sales info:
for y in variable_name[1]['SalesInfos']:
y = y['DateOfSale']
y = datetime.strptime(y, '%m/%d/%Y')
print(y)
I'm fetchin some data from an api, and I want to insert it in too an azure sql db.
I'm getting the data and adding it to a pandas dataframe before dropping empty columns and such.
After I have done what i need to do I'm trying to insert the data in to the database, but I'm getting this error message: ProgrammingError: (pyodbc.ProgrammingError) ("A TVP's rows must be Sequence objects.", 'HY000')
I've found that some of the columns have nested data (and I would like to keep it that way)
These columns contain lists of dicts here are some examples:
0 [{'note': 'Netto per 45 dgr'}]
1 [{'note': 'Netto per 45 dgr'}]
2 [{'note': 'Netto per 45 dgr'}]
[{'accountingCost': None, 'allowanceCharge': array([], dtype=object), 'billingReference': array([], dtype=object), 'contractDocumentReference': None, 'delivery': {'actualDeliveryDate': None, 'deliveryLocation': None, 'deliveryParty': None, 'despatch': None, 'estimatedDeliveryPeriod': None, 'promisedDeliveryPeriod': None, 'requestedDeliveryPeriod': None, 'trackingId': None}, 'despatchLineReference': {'attachment': None, 'copyIndicator': None, 'documentDescription': None, 'documentType': None, 'documentTypeCode': None, 'id': '', 'issueDate': None, 'uuid': None}, 'documentReference': None, 'id': '1', 'invoicePeriod': None, 'invoicedQuantity': {'unitCode': 'H21', 'unitCodeListId': None, 'value': '1.00'}, 'item': {'additionalItemProperty': array([], dtype=object), 'brandName': None, 'certificate': array([], dtype=object), 'classifiedTaxCategory': None, 'commodityClassification': array([], dtype=object), 'description': array([], dtype=object), 'dimension': array([], dtype=object), 'hazardousItem': array([], dtype=object), 'itemInstance': array([], dtype=object), 'itemSpecificationDocumentReference': array([], dtype=object), 'keyword': array([], dtype=object), 'manufacturerParty': None, 'manufacturersItemIdentification': array([], dtype=object), 'name': 'Ansvarsrett', 'originAddress': None, 'originCountry': None, 'packQuantity': None, 'packSizeNumeric': '', 'sellersItemIdentification': {'extendedId': None, 'id': {'id': 'P550', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}}, 'standardItemIdentification': {'extendedId': None, 'id': {'id': '7043010000953', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': 'GTIN', 'schemeName': None}}, 'transactionConditions': array([], dtype=object)}, 'lineExtensionAmount': {'currency': None, 'value': '504.00'}, 'lineGrossExtensionAmount': None, 'note': '', 'orderLineReference': {'buyersReference': '117230405 Kvileitet', 'lineId': {'id': '0', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'orderReference': None, 'salesOrderLineId': {'id': '1', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'sellersReference': ''}, 'orderedQuantity': None, 'price': {'allowanceCharge': array([], dtype=object), 'amount': {'currency': None, 'value': '504.00'}, 'baseQuantity': {'unitCode': None, 'unitCodeListId': None, 'value': '0'}, 'orderableUnitFactorRate': None, 'priceType': None, 'validityPeriod': array([], dtype=object)}, 'taxTotal': {'roundingAmount': None, 'taxAmount': {'currency': None, 'value': '126.00'}, 'taxSubtotal': array([], dtype=object)}}
{'accountingCost': None, 'allowanceCharge': array([], dtype=object), 'billingReference': array([], dtype=object), 'contractDocumentReference': None, 'delivery': {'actualDeliveryDate': None, 'deliveryLocation': None, 'deliveryParty': None, 'despatch': None, 'estimatedDeliveryPeriod': None, 'promisedDeliveryPeriod': None, 'requestedDeliveryPeriod': None, 'trackingId': None}, 'despatchLineReference': {'attachment': None, 'copyIndicator': None, 'documentDescription': None, 'documentType': None, 'documentTypeCode': None, 'id': '', 'issueDate': None, 'uuid': None}, 'documentReference': None, 'id': '2', 'invoicePeriod': None, 'invoicedQuantity': {'unitCode': 'H21', 'unitCodeListId': None, 'value': '3.00'}, 'item': {'additionalItemProperty': array([], dtype=object), 'brandName': None, 'certificate': array([], dtype=object), 'classifiedTaxCategory': None, 'commodityClassification': array([], dtype=object), 'description': array([], dtype=object), 'dimension': array([], dtype=object), 'hazardousItem': array([], dtype=object), 'itemInstance': array([], dtype=object), 'itemSpecificationDocumentReference': array([], dtype=object), 'keyword': array([], dtype=object), 'manufacturerParty': None, 'manufacturersItemIdentification': array([], dtype=object), 'name': 'Prosjektering', 'originAddress': None, 'originCountry': None, 'packQuantity': None, 'packSizeNumeric': '', 'sellersItemIdentification': {'extendedId': None, 'id': {'id': 'Projj', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}}, 'standardItemIdentification': {'extendedId': None, 'id': {'id': '7043010000298', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': 'GTIN', 'schemeName': None}}, 'transactionConditions': array([], dtype=object)}, 'lineExtensionAmount': {'currency': None, 'value': '7020.00'}, 'lineGrossExtensionAmount': None, 'note': '', 'orderLineReference': {'buyersReference': '117230405 Kvileitet', 'lineId': {'id': '0', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'orderReference': None, 'salesOrderLineId': {'id': '2', 'schemeAgencyId': None, 'schemeAgencyName': None, 'schemeId': None, 'schemeName': None}, 'sellersReference': ''}, 'orderedQuantity': None, 'price': {'allowanceCharge': array([], dtype=object), 'amount': {'currency': None, 'value': '2340.00'}, 'baseQuantity': {'unitCode': None, 'unitCodeListId': None, 'value': '0'}, 'orderableUnitFactorRate': None, 'priceType': None, 'validityPeriod': array([], dtype=object)}, 'taxTotal': {'roundingAmount': None, 'taxAmount': {'currency': None, 'value': '1755.00'}, 'taxSubtotal': array([], dtype=object)}}
is there any way to add this data to mssql?
I am iterating over json file and creating dataframe with the desirable columns. I already implemented the code but now json file has little bit changed. But I am not able to think where to change the code to get the required output.
Explanation:
previous json result:
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-18T11:49:48.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 7,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
previous dataframe:
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-18T11:49:48.000Z,7,,,,1.0,,,
new json result:
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
Now desirable dataframe:
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
What are the changes I need to do to in below code to get the new result.
column_names = []
if(query_result.results != None):
for item in query_result.results:
data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
for met in lst_data.metrics:
metric_name = met.metric
column_names.append('Queue_Id')
column_names.append(metric_name+'_count')
column_names.append(metric_name+'_sum')
column_names.append(metric_name+'.denominator')
column_names.append(metric_name+'.numerator')
column_names.append('Interval Start')
column_names.append('Interval End')
data_lst.append(queue_id)
data_lst.append(met.stats.count)
data_lst.append(met.stats.sum)
data_lst.append(met.stats.denominator)
data_lst.append(met.stats.numerator)
data_lst.append(lst_data.interval.split('/')[0])
data_lst.append(lst_data.interval.split('/')[1])
print(data_lst)
else:
data_lst = []
metric_name = query.metrics[0]
column_names.append('Queue_Id')
column_names.append(metric_name+'_count')
column_names.append(metric_name+'_sum')
column_names.append(metric_name+'.denominator')
column_names.append(metric_name+'.numerator')
column_names.append('Interval Start')
column_names.append('Interval End')
data_lst.append(queue_id)
data_lst.append('')
data_lst.append('')
data_lst.append('')
data_lst.append('')
data_lst.append(query.interval.split('/')[0])
data_lst.append(query.interval.split('/')[1])
print("data_lst", data_lst)
print("column_names", column_names)
return data_lst, column_names
I have modified my code little bit and got the result. The below code is working for me-
lst_of_metrics = ["nOffered", "nTransferred"]
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met1, met in itertools.zip_longest(query.metrics, lst_data.metrics):
#for met in lst_data.metrics:
if(met):
if(met.metric == met1):
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)
I have a json result which I am trying to convert into dataframe but not able to get the correct result. Actually for some cases it is giving correct but for some case it is failing.
Example:
Based on metric API is generating result for specified interval. But this is not certain for that particular interval metric have output or not. And process is running 4 different queue_id.
suppose process is running only for 2 metric. ['nOffered', 'nTransferred']
queue_id = 'a72dba75-0bc6-4a65-b120-8803364f8dc3'
for this queue_id, nOffered is having some values but nTransferred doesn't have. Json result is given below-
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
My code is giving below output-
queue_id nOffered_count nOffered_sum interval_start interval_end
0 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-11T11:46:25.000Z 2021-10-12T11:46:25.000Z
1 a72dba75-0bc6-4a65-b120-8803364f8dc3 1 None 2021-10-12T11:46:25.000Z 2021-10-13T11:46:25.000Z
2 a72dba75-0bc6-4a65-b120-8803364f8dc3 12 None 2021-10-13T11:46:25.000Z 2021-10-14T11:46:25.000Z
3 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-14T11:46:25.000Z 2021-10-15T11:46:25.000Z
4 a72dba75-0bc6-4a65-b120-8803364f8dc3 6 None 2021-10-15T11:46:25.000Z 2021-10-16T11:46:25.000Z
But when process is running for 2nd queue_id that time it is not working-
queue_id - 73643cff-799b-41ae-9a67-efcf5e593155
json output for this queue_id -
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 2,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}},
{'metric': 'nTransferred',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 3,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None},
{'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z',
'metrics': [{'metric': 'nOffered',
'qualifier': None,
'stats': {'count': 1,
'count_negative': None,
'count_positive': None,
'current': None,
'denominator': None,
'max': None,
'min': None,
'numerator': None,
'ratio': None,
'sum': None,
'target': None}}],
'views': None}],
'group': {'mediaType': 'voice',
'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
This time both metric having some data. So output would be-
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
And in final result, both the result merge and give the output with all columns and data.
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,6,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-12T11:46:25.000Z,2021-10-13T11:46:25.000Z,1.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,12.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,6.0,,,,,,,
a72dba75-0bc6-4a65-b120-8803364f8dc3,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,6.0,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1.0,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,,
73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
Currently I am running below logic-
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met in lst_data.metrics:
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)
return df
I used below logic to get the desirable result. It is working for me.
lst_of_metrics = ["nOffered", "nTransferred"]
out = defaultdict(list)
if(query_result.results != None):
for item in query_result.results:
#data_lst = []
for lst_data in item.data:
print("####################################")
print(lst_data)
print("####################################")
out['queue_id'].append(queue_id)
for met1, met in itertools.zip_longest(query.metrics, lst_data.metrics):
if(met):
if(met.metric == met1):
out[met.metric+"_count"].append(met.stats.count)
out[met.metric+"_sum"].append(met.stats.sum)
out[met.metric+".denominator"].append(met.stats.denominator)
out[met.metric+".numerator"].append(met.stats.numerator)
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
else:
out[met1+"_count"].append('')
out[met1+"_sum"].append('')
out[met1+".denominator"].append('')
out[met1+".numerator"].append('')
interval = lst_data.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print("out", out)
else:
metric_name = query.metrics[0]
out['queue_id'].append(queue_id)
out[metric_name+"_count"].append('')
out[metric_name+"_sum"].append('')
out[metric_name+".denominator"].append('')
out[metric_name+".numerator"].append('')
interval = query.interval.split('/')
out['Interval Start'].append(interval[0])
out['Interval End'].append(interval[1])
print(out)
df = pd.DataFrame(out)
print (df)
So I'm playing around with the Google Analytics dataset in pandas, and I'm having an issue.
The dataset has a column named hits, which contains a list of dictionaries, saved as a string. When I try to use ast.literat_eval to get the list of dictionaries, I get a syntax error pointing to this part of the dictionary:
'customMetrics': array([], dtype=object)
I'm using the following code to try to convert the string:
df['hits'] = df['hits'].apply(lambda x: ast.literal_eval(x))
I don't really care about this piece of data so it's fine if I could just ignore this and keep that particular dictionary value as a string. Does anyone know how to either ignore the error and set that dictionary value to a string, or properly convert this array?
FYI, an entire cell of the hits column looks like this:
"[{'hitNumber': 1, 'time': 0, 'hour': 23, 'minute': 0, 'isSecure': None, 'isInteraction': True, 'isEntrance': True, 'isExit': None, 'referer': 'https://www.youtube.com/yt/about/', 'page': {'pagePath': '/home', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'Home', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/home', 'pagePathLevel2': '', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': None, 'item': None, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/home', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([], dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(entrance)', 'previousContentGroup2': '(entrance)', 'previousContentGroup3': '(entrance)', 'previousContentGroup4': '(entrance)', 'previousContentGroup5': '(entrance)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}\n {'hitNumber': 2, 'time': 30507, 'hour': 23, 'minute': 1, 'isSecure': None, 'isInteraction': True, 'isEntrance': None, 'isExit': None, 'referer': None, 'page': {'pagePath': '/google+redesign/gift+cards', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'Gift Cards', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/google+redesign/', 'pagePathLevel2': '/gift+cards', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': {'transactionId': None, 'transactionRevenue': None, 'transactionTax': None, 'transactionShipping': None, 'affiliation': None, 'currencyCode': 'USD', 'localTransactionRevenue': None, 'localTransactionTax': None, 'localTransactionShipping': None, 'transactionCoupon': None}, 'item': {'transactionId': None, 'productName': None, 'productCategory': None, 'productSku': None, 'itemQuantity': None, 'itemRevenue': None, 'currencyCode': 'USD', 'localItemRevenue': None}, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/google+redesign/gift+cards', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([{'productSKU': 'GGOEGGCX056299', 'v2ProductName': 'Gift Card - $25.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 25000000, 'localProductPrice': 25000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 1},\n {'productSKU': 'GGOEGGCX056499', 'v2ProductName': 'Gift Card - $50.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 50000000, 'localProductPrice': 50000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 2},\n {'productSKU': 'GGOEGGCX056199', 'v2ProductName': 'Gift Card- $100.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 100000000, 'localProductPrice': 100000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 3},\n {'productSKU': 'GGOEGGCX056399', 'v2ProductName': 'Gift Card - $250.00', 'v2ProductCategory': 'Home/Gift Cards/', 'productVariant': '(not set)', 'productBrand': '(not set)', 'productRevenue': None, 'localProductRevenue': None, 'productPrice': 250000000, 'localProductPrice': 250000000, 'productQuantity': None, 'productRefundAmount': None, 'localProductRefundAmount': None, 'isImpression': True, 'isClick': None, 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'productListName': 'Category', 'productListPosition': 4}],\n dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(not set)', 'previousContentGroup2': '(not set)', 'previousContentGroup3': '(not set)', 'previousContentGroup4': '(not set)', 'previousContentGroup5': '(not set)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}\n {'hitNumber': 3, 'time': 53670, 'hour': 23, 'minute': 1, 'isSecure': None, 'isInteraction': True, 'isEntrance': None, 'isExit': True, 'referer': None, 'page': {'pagePath': '/signin.html', 'hostname': 'shop.googlemerchandisestore.com', 'pageTitle': 'The Google Merchandise Store - Log In', 'searchKeyword': None, 'searchCategory': None, 'pagePathLevel1': '/signin.html', 'pagePathLevel2': '', 'pagePathLevel3': '', 'pagePathLevel4': ''}, 'transaction': None, 'item': None, 'contentInfo': None, 'appInfo': {'name': None, 'version': None, 'id': None, 'installerId': None, 'appInstallerId': None, 'appName': None, 'appVersion': None, 'appId': None, 'screenName': 'shop.googlemerchandisestore.com/signin.html', 'landingScreenName': 'shop.googlemerchandisestore.com/home', 'exitScreenName': 'shop.googlemerchandisestore.com/signin.html', 'screenDepth': '0'}, 'exceptionInfo': {'description': None, 'isFatal': True, 'exceptions': None, 'fatalExceptions': None}, 'eventInfo': None, 'product': array([], dtype=object), 'promotion': array([], dtype=object), 'promotionActionInfo': None, 'refund': None, 'eCommerceAction': {'action_type': '0', 'step': 1, 'option': None}, 'experiment': array([], dtype=object), 'publisher': None, 'customVariables': array([], dtype=object), 'customDimensions': array([], dtype=object), 'customMetrics': array([], dtype=object), 'type': 'PAGE', 'social': {'socialInteractionNetwork': None, 'socialInteractionAction': None, 'socialInteractions': None, 'socialInteractionTarget': None, 'socialNetwork': 'YouTube', 'uniqueSocialInteractions': None, 'hasSocialSourceReferral': 'Yes', 'socialInteractionNetworkAction': ' : '}, 'latencyTracking': None, 'sourcePropertyInfo': None, 'contentGroup': {'contentGroup1': '(not set)', 'contentGroup2': '(not set)', 'contentGroup3': '(not set)', 'contentGroup4': '(not set)', 'contentGroup5': '(not set)', 'previousContentGroup1': '(not set)', 'previousContentGroup2': '(not set)', 'previousContentGroup3': '(not set)', 'previousContentGroup4': '(not set)', 'previousContentGroup5': '(not set)', 'contentGroupUniqueViews1': None, 'contentGroupUniqueViews2': None, 'contentGroupUniqueViews3': None, 'contentGroupUniqueViews4': None, 'contentGroupUniqueViews5': None}, 'dataSource': None, 'publisher_infos': array([], dtype=object)}]"