For the Itunes Reporter API, I have an access_token and vendor_number.
https://help.apple.com/itc/appsreporterguide/#/apd68da36164
I found some old Python code that was used to send API requests to this API:
def _make_request(self,
cmd_type: str,
command: str,
credentials: Dict[str, str],
extra_params: Dict[str, str] = None
) -> requests.Response:
if not extra_params:
extra_params = {}
# command does not differ anymore, no matter if the apple id has multiple accoutns or not. a= is an invalid parameter by now.
command = f'[p=Reporter.properties, {cmd_type.capitalize()}.{command}]'
endpoint = ('https://reportingitc-reporter.apple.com'
f'/reportservice/{cmd_type}/v1')
# account needs to be passed as data, not as parameter
if self.account:
data = {
'version': self.version,
'mode': self.mode,
**credentials,
'queryInput': command,
'account': self.account
}
else:
data = {
'version': self.version,
'mode': self.mode,
**credentials,
'queryInput': command
}
data = self._format_data(data)
data.update(extra_params)
response = requests.post(endpoint, data=data)
response.raise_for_status()
return response
def download_sales_report(self,
vendor: str,
report_type: str,
date_type: str,
date: str,
report_subtype: str = '',
report_version: str = '') -> Data:
"""Downloads sales report, puts the TSV file into a Python list
Information on the parameters can be found in the iTunes Reporter
documentation:
https://help.apple.com/itc/appsreporterguide/#/itcbd9ed14ac
:param vendor:
:param report_type:
:param date_type:
:param date:
:param report_subtype:
:param report_version:
:return:
"""
credentials = {
'accesstoken': self.access_token
}
command = (f'getReport, {vendor},{report_type},{report_subtype},'
f'{date_type},{date},{report_version}')
ordered_dict_sales_report = self._process_gzip(self._make_request('sales', command,
credentials))
return ordered_dict_sales_report
Now, I want to replicate this in Postman but I am not sure where to place the parameters from the "command" i.e vendor, reportType etc. Do I pass them as a raw json in the body? Or as query paramas?
The endpoint I am using currently for a POST request is this:
https://reportingitc-reporter.apple.com/reportservice/sales/v1
I am passing a "BearerToken" as the authorization and this as the Body:
{
"version": "1.0",
"mode": "Test",
"queryInput": "[p=Reporter.properties, Sales.getReport, 85040615, sales, Summary, Daily, 20230101]"
}
but i get 400 Bad request error
According to the documentation, this is the Java syntax that i need to convert to Python:
Syntax
$ java -jar Reporter.jar p=[properties file] Sales.getReport [vendor number], [report type], [report subtype], [date type], [date], [version]* (if applicable)
Related
Requirement: To read data from S3 to pass into API
Error: "error": {"code": "ModelStateInvalid", "message": "The request has exceeded the maximum number of validation errors.", "target": "HttpRequest"
When I pass data directly in the code as below document , it works fine as below
def create_doc(self,client):
self.n_docs = int(self.n_docs)
document = {'addresses': {'SingleLocation': {'city': 'ABC',
'country': 'US',
'line1': 'Main',
'postalCode': '00000',
'region': 'CA'
}
},
'commit': False,
}
response = client.cr_transc(document)
jsn = response.json()
But when tried having data in the file in the s3 and read it from s3 , it throws into error
def create_doc(self,client):
self.n_docs = int(self.n_docs)
document = data_from_s3()
response = client.cr_transc(document)
jsn = response.json()
def data_from_s3(self):
s3 = S3Hook()
data = s3.read_key(bucket_name = self.bucket_name, key = self.data_key)
return data
Below link is for read_key method in airflow
https://airflow.apache.org/docs/apache-airflow/1.10.6/_modules/airflow/hooks/S3_hook.html#S3Hook:~:text=%5Bdocs%5D%20%20%20%20def-,read_key,-(self%2C
Checking the source code:
def read_key(self, key, bucket_name=None):
"""
Reads a key from S3
:param key: S3 key that will point to the file
:type key: str
:param bucket_name: Name of the bucket in which the file is stored
:type bucket_name: str
"""
obj = self.get_key(key, bucket_name)
return obj.get()['Body'].read().decode('utf-8')
This returns a str. You might need to use the json module to convert it:
import json
def create_doc(self,client):
self.n_docs = int(self.n_docs)
document = json.loads(data_from_s3()) # <----- convert here
response = client.cr_transc(document)
jsn = response.json()
def data_from_s3(self):
s3 = S3Hook()
data = s3.read_key(bucket_name = self.bucket_name, key = self.data_key)
return data
I am trying to set up an AWS API Gateway that could receive a POST request an upload a csv file to S3. Ideally, I would like to make some transformations to the file before uploading it to S3 (renaming and formatting some columns to normalize their names accross different uploads).
So far, I have set up my API Gateway to receive the request and send it to an AWS Lambda. I use Lambda proxy integration. The triggered lambda is as follows:
import logging
import pandas as pd
import boto3
logger = logging.getLogger()
logger.setLevel(logging.INFO)
s3 = boto3.client("s3")
def handler(event, context):
logger.info(f"Event: {event}")
df = pd.read_csv(event['body']['file'])
logger.info(f"df1: {df}")
# Provided parameters
try:
code = event['body']['code']
except KeyError:
logger.info('Code not provided')
code = 'Code'
try:
date = event['body']['date']
except KeyError:
logger.info('Date not provided')
date = 'Date'
try:
debit = event['body']['debit']
except KeyError:
logger.info('Debit not provided')
debit = 'Debit'
try:
credit = event['body']['credit']
except KeyError:
logger.info('Credit not provided')
credit = 'Credit'
try:
id= event['body']['id']
except KeyError:
logger.info('Id not provided')
id = '001'
df.rename(columns={code: 'Code', date: 'Date', credit: 'Credit', debit: 'Debit'})
df.to_csv(f's3://bucket/{id}/file.csv', line_terminator='\n', sep = ';', date_format='%Y-%m-%d %H:%M:%S')
return {
'statusCode': 200,
'headers': {
'Content-Type': 'text/csv',
'Access-Control-Allow-Origin': '*'
},
'body': {
'uploaded': True
},
'isBase64Encoded': False
}
To test this API, I use the following function:
import requests
csv_file = open("file.csv", 'rb')
headers = {"x-api-key": "xxx", "Content-Type":"text/csv"}
url = "https://xxx.execute-api.xxx.amazonaws.com/xxx"
body = {
"file": csv_file,
"code": "my_code"
}
# files = {
# "file": ("file.csv", open('file.csv', 'r'), 'text/csv')
# }
r = requests.post(url=url, headers=headers, data=body)
print(r.text)
The output is {"message": "Internal server error"}, and if I look in CloudWatch logs, I see that the event is encoded this way:
'body': 'file=%EF%BB%BFCol1%3BCol2%3BCol3%3BCol4%0D%0A&file=11%3B12%3B13%3B14%3B%0D%0A&file=21%3B22%3B23%3B24%3B...'
It looks like the body is encoded and passed row by row into different "file" fields. For a file with about 5000 rows I get the error OSError: [Errno 36] File name too long when trying to read it.
Is there another way to proceed in order to get a full dataset that I can transform into a pandas dataframe?
I have also seen suggestions with multipart/form-data, using files=files in the request or using csv library but I keep getting similar errors.
Thank you
This is my code to extract player data from an endpoint containing basketball data for a Data Science project.NOTE: I changed the name of the actual API key I was given since it's subscription. And I change the username/password because for privacy purposes. Using the correct credentials, I wouldn't receive a syntax error but the status code always returns 401. Since it wasn't accepting the API key, I added my account username, password, and the HTTP authentication header as well, but the status code still returns 401.
In case this is relevant, this is the website's recommendation in the developer portal: **The API key can be passed either as a query parameter or using the following HTTP request header.
Please let me know what changes I can make to my code. Any help is appreciated.
Ocp-Apim-Subscription-Key: {key}**
PS: My code got fragmented while posting this, but it is all in one function.
def getData():
user_name = "name#gmail.com"
api_endpoint = "https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7"
api_key = "a45;lkf"
password = "ksaljd"
header = "Ocp-Apim-Subscription-Key"
PARAMS = {'user': user_name, 'pass': password, 'header': header, 'key': api_key}
response = requests.get(url = api_endpoint, data = PARAMS)
print(response.status_code)
file = open("Data.csv", "w")
file.write(response.text)
file.close()
def _get_auth_headers() -> dict:
return {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': "`Insert key here`"
}
api_endpoint = "https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7"
PARAMS = {
# Your params here
}
response = requests.get(
api_endpoint,
headers=_get_auth_headers(),
params=PARAMS
)
Instead of just a string, you need to pass dict in the headers parameter and auth param exist so you can use it as follow:
def getData():
[...]
header = {
"Ocp-Apim-Subscription-Key": api_key
}
[...]
response = requests.get(url = api_endpoint, data = PARAMS, headers=header, auth = (user_name, password))
According to the API documentation you don't need to provide email and password. You're only need to add your API Key to header:
import requests
r = requests.get(url='https://api.sportsdata.io/v3/nba/stats/json/PlayerGameStatsByDate/2020-FEB7', headers={'Ocp-Apim-Subscription-Key': 'API_KEY'})
print(r.json())
Output:
[{
'StatID': 768904,
'TeamID': 25,
'PlayerID': 20000788,
'SeasonType': 1,
'Season': 2020,
'Name': 'Tim Hardaway Jr.',
'Team': 'DAL',
'Position': 'SF',
'Started': 1,
'FanDuelSalary': 7183,
'DraftKingsSalary': 7623,
'FantasyDataSalary': 7623,
...
I try to create roles in an automated way in Google Kubernetes (GKE).
For that, I use the python client library, but I don't want to have any dependency to kubectl and kubeconfig, or gcloud,
I use a service account (with a json key file from GCP) which has the permissions to create roles in namespaces (it is a cluster admin). When I use the access token given by this command :
gcloud auth activate-service-account --key-file=credentials.json
gcloud auth print-access-token
It works.
But when I try to generate the token by myself, I can create namespaces and other standard resources, but I have this error when it comes to roles :
E kubernetes.client.rest.ApiException: (403)
E Reason: Forbidden
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b89b0fc2-9350-456e-9eca-730e7ad2cea1', 'Content-Type': 'application/json', 'Date': 'Tue, 26 Feb 2019 20:35:20 GMT', 'Content-Length': '1346'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"roles.rbac.authorization.k8s.io \"developers\" is forbidden: attempt to grant extra privileges: [{[*] [apps] [statefulsets] [] []} {[*] [apps] [deployments] [] []} {[*] [autoscaling] [horizontalpodautoscalers] [] []} {[*] [] [pods] [] []} {[*] [] [pods/log] [] []} {[*] [] [pods/portforward] [] []} {[*] [] [serviceaccounts] [] []} {[*] [] [containers] [] []} {[*] [] [services] [] []} {[*] [] [secrets] [] []} {[*] [] [configmaps] [] []} {[*] [extensions] [ingressroutes] [] []} {[*] [networking.istio.io] [virtualservices] [] []}] user=\u0026{100701357824788592239 [system:authenticated] map[user-assertion.cloud.google.com:[AKUJVp+KNvF6jw9II+AjCdqjbC0vz[...]hzgs0JWXOyk7oxWHkaXQ==]]} ownerrules=[{[create] [authorization.k8s.io] [selfsubjectaccessreviews selfsubjectrulesreviews] [] []} {[get] [] [] [] [/api /api/* /apis /apis/* /healthz /openapi /openapi/* /swagger-2.0.0.pb-v1 /swagger.json /swaggerapi /swaggerapi/* /version /version/]}] ruleResolutionErrors=[]","reason":"Forbidden","details":{"name":"developers","group":"rbac.authorization.k8s.io","kind":"roles"},"code":403}
I'm using the same service account, so I guess gcloud is doing something more than my script.
Here the python code I use to generate the token :
def _get_token(self) -> str:
# See documentation here
# https://developers.google.com/identity/protocols/OAuth2ServiceAccount
epoch_time = int(time.time())
# Generate a claim from the service account file.
claim = {
"iss": self._service_account_key["client_email"],
"scope": "https://www.googleapis.com/auth/cloud-platform",
"aud": "https://www.googleapis.com/oauth2/v4/token",
"exp": epoch_time + 3600,
"iat": epoch_time
}
# Sign claim with JWT.
assertion = jwt.encode(
claim,
self._service_account_key["private_key"],
algorithm='RS256'
).decode()
# Create payload for API.
data = urlencode({
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": assertion
})
# Request the access token.
result = requests.post(
url="https://www.googleapis.com/oauth2/v4/token",
headers={
"Content-Type": "application/x-www-form-urlencoded"
},
data=data
)
result.raise_for_status()
return json.loads(result.text)["access_token"]
def _get_api_client(self) -> client.ApiClient:
configuration = client.Configuration()
configuration.host = self._api_url
configuration.verify_ssl = self._tls_verify
configuration.api_key = {
"authorization": f"Bearer {self._get_token()}"
}
return client.ApiClient(configuration)
And the function to create the role (which generates the 403 error):
def _create_role(self, namespace: str, body: str):
api_client = self._get_api_client()
rbac = client.RbacAuthorizationV1Api(api_client)
rbac.create_namespaced_role(
namespace,
body
)
If I short-circuit the _get_token method with the token extracted from gcloud, it works.
I guess it has something to do with the way I create my token (missing scope ?), but I don't find any documentation about it.
ANSWER :
Adding a scope does the job ! Thanks a lot :
# Generate a claim from the service account file.
claim = {
"iss": self._service_account_key["client_email"],
"scope": " ".join([
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/userinfo.email"
]),
"aud": "https://www.googleapis.com/oauth2/v4/token",
"exp": epoch_time + 3600,
"iat": epoch_time
}
So if you look at the code here for print-access-token you can see that the access token is generally printed without a scope. You see:
try:
creds = client.GoogleCredentials.get_application_default()
except client.ApplicationDefaultCredentialsError as e:
log.debug(e, exc_info=True)
raise c_exc.ToolException(str(e))
if creds.create_scoped_required():
...
and then on this file you see:
def create_scoped_required(self):
"""Whether this Credentials object is scopeless.
create_scoped(scopes) method needs to be called in order to create
a Credentials object for API calls.
"""
return False
Apparently, in your code, you are getting the token with the https://www.googleapis.com/auth/cloud-platform scope. You could try removing it or try with the USER_EMAIL_SCOPE since you are specifying: "iss": self._service_account_key["client_email"].
You can always check what gcloud auth activate-service-account --key-file=credentials.json stores under ~/.config. So you know what gcloud auth print-access-token uses. Note that as per this and this it looks like the store is in sqlite format.
I'm trying to make a simple SubmitFeed, but I'm getting an error that says "Timestamp must be in ISO8601 format". This is what I have:
def make_update_feed(host, seller_id, mws_auth_token, aws_access_key_id, secret_key, xml):
date_now = dt.utcnow().replace(microsecond=0).isoformat() + "Z"
xml_md5_b64 = md5_b64_hash(xml)
timestamp = urllib.quote_plus(date_now)
params = "AWSAccessKeyId={0}&Action=SubmitFeed&ContentMD5Value={1}&FeedType=_POST_INVENTORY_AVAILABILITY_DATA_&MWSAuthToken={2}&Merchant={3}&PurgeAndReplace=false&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp={4}&Version=2009-01-01".format(aws_access_key_id,urllib.quote_plus(xml_md5_b64),mws_auth_token,seller_id,timestamp)
string_2_sign = "POST\n{0}\n/\n{1}".format(host,params)
signature = hmac_sha256_b64(string_2_sign, secret_key)
params_req = {
"AWSAccessKeyId": urllib.quote_plus(aws_access_key_id),
"Action": "SubmitFeed",
"ContentMD5Value": urllib.quote_plus(xml_md5_b64),
"FeedType": "_POST_INVENTORY_AVAILABILITY_DATA_",
"MWSAuthToken": urllib.quote_plus(mws_auth_token),
"Merchant": urllib.quote_plus(seller_id),
"PurgeAndReplace": False,
"SignatureMethod": "HmacSHA256",
"SignatureVersion": 2,
"Timestamp": timestamp,
"Version": "2009-01-01",
"Signature": urllib.quote_plus(signature),
}
headers = {
'Host': urllib.quote_plus(host),
'Content-Type': "text/xml",
"x-amazon-user-agent": "MyPythonApp/1.0 (Language=Python)"
}
data_xml = {
'FeedContent': xml
}
url = "https://%s" % host
r = requests.post(url, params=params_req, headers=headers,data=data_xml)
return r
Now, when I run my function with the respective parameters, I get this response:
<?xml version="1.0"?>
<ErrorResponse xmlns="http://mws.amazonaws.com/doc/2009-01-01/">
<Error>
<Type>Sender</Type>
<Code>InvalidParameterValue</Code>
<Message>Timestamp 2018-02-19T14%3A30%3A50Z must be in ISO8601 format</Message>
</Error>
<RequestID>7c8631e4-8d61-4f46-9256-af0130028d96</RequestID>
</ErrorResponse>
However, I can see that the timestamp is in ISO8601 format. I don't know why it's giving me that error. Also, both the xml hash and the signature are correct (I checked vs Amazon Scratchpad, and they were the same). Any ideas?