Develop Reference

Object of type QueryResponse is not JSON serializable - Python 3.9 fastAPI using Pinecone

Given the following API response from pinecone (https://www.pinecone.io/docs/api/operation/query/)
results = {'matches': [{'id': 'yral5m',
'metadata': {'subreddit': '2qkq6',
'text': 'Black Friday SaaS Deals - 2022'},
'score': 0.772717535,
'sparseValues': {},
'values': []},
{'id': 'yqypa5',
'metadata': {'subreddit': '2qkq6',
'text': 'B2B sales interface.'},
'score': 0.74192214,
'sparseValues': {},
'values': []}],
'namespace': ''}
i am simply trying to return the JSON results from a function. (from a service to a controller) and getting a range of errors:
doing so with:
return results yields RecursionError: maximum recursion depth exceeded in comparison
return json.dumps(results) yeilds TypeError: Object of type QueryResponse is not JSON serializable
*QueryResponse is the type returned from pinecone
return jsonpickle.encode(query_results) yeilds "null"
Very lost any advice would be appreciated!!
full code example:
Controller:
#router.post("/query/")
async def semantic_search(query: str, ):
result = await TopicQueryService.query(query)
return result
Service method:
#staticmethod
async def query(query) -> str:
index = PineConeService.get_or_create_index("openai")
embed = GPT3Service.get_embedding(query)
query_results = index.query(
vector=embed,
top_k=2,
include_metadata=True
)
return json.dumps(query_results)
Replacing Service Method query results with the logged response from index. query works fine eg below. Leading me to believe it is due to the QueryResponse object pinecone returns.
#staticmethod
async def query(query) -> str:
index = PineConeService.get_or_create_index("openai")
embed = GPT3Service.get_embedding(query)
logger.info(embed)
query_results = {'matches': [{'id': 'yral5m',
'metadata': {'subreddit': '2qkq6',
'text': 'Black Friday SaaS Deals - 2022'},
'score': 0.772717535,
'sparseValues': {},
'values': []},
{'id': 'yqypa5',
'metadata': {'subreddit': '2qkq6',
'text': 'B2B sales interface.'},
'score': 0.74192214,
'sparseValues': {},
'values': []}],
'namespace': ''}
return json.dumps(query_results)

Somewhat of a solution -> just iterate over and build an object to return, not ideal though
``blah = []
for x in query_results.matches:
blah.append({
"id": x.id,
"metadata": x.metadata,
"score": x.score
})
json.dumps(blah)``

You can use the method .to_dict() from the QueryResponse object:
#staticmethod
async def query(query) -> str:
index = PineConeService.get_or_create_index("openai")
embed = GPT3Service.get_embedding(query)
query_results = index.query(
vector=embed,
top_k=2,
include_metadata=True
)
return json.dumps(query_results.to_dict())

Python MagicMock.return_value returning MagicMock instead of return_value

I have a function that verifies if a given input string is a proper GCP zone:
def validate_zone(compute, project_id, zone):
try:
zone_response = compute.zones().get(project=project_id, zone=zone).execute()
print(zone_response)
print(zone_response.return_value)
if ['status'] in zone_response:
zone_details = {
'status': zone_response['status'],
'region': zone_response['region'],
'name': zone_response['name']
}
return zone_details
else:
return "Zone {} not found for project {}".format(zone, project_id)
except HttpError as error:
print("Error calling zone {}: \n {}".format(zone, error))
I am trying to write a test to verify that but I can't mock the output of the compute method correctly.
#patch('googleapiclient.discovery')
def test_validate_zone(self, mock_response):
compute = mock_response.build(serviceName='compute', version='v1')
compute.zones().get(project_id=self.project_id, zone=self.zone).execute().return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
This results in the zone_response output being a MagicMock object with its return_value being correct as developed in the test.
zone_response = MagicMock name='discovery.build().zones().get().execute()' id='139870134525456'
zone_response.return_value = {'status': 'status', 'region': 'region', 'name': 'name'}
Any ideas on what I'm doing wrong? I've been trying to write tests for this for quite a while so maybe my approach is just off.

Turns out the issue was the () on the execute method in the test. So the correct test should be:
#patch('inventory.discovery.build', serviceName='compute', version='v1')
def test_validate_zone(self, compute):
print(compute)
compute.zones().get(project_id=self.project_id, zone=self.zone).execute.return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
Source can be found at: https://realpython.com/python-mock-library/#managing-a-mocks-return-value

Looping through a function

I am struggling with figuring out the best way to loop through a function. The output of this API is a Graph Connection and I am a-little out of my element. I really need to obtain ID's from an api output and have them in a dict or some sort of form that I can pass to another API call.
**** It is important to note that the original output is a graph connection.... print(type(api_response) does show it as a list however, if I do a print(type(api_response[0])) it returns a
This is the original output from the api call:
[{'_from': None, 'to': {'id': '5c9941fcdd2eeb6a6787916e', 'type': 'user'}}, {'_from': None, 'to': {'id': '5cc9055fcc5781152ca6eeb8', 'type': 'user'}}, {'_from': None, 'to': {'id': '5d1cf102c94c052cf1bfb3cc', 'type': 'user'}}]
This is the code that I have up to this point.....
api_response = api_instance.graph_user_group_members_list(group_id, content_type, accept,limit=limit, skip=skip, x_org_id=x_org_id)
def extract_id(result):
result = str(result).split(' ')
for i, r in enumerate(result):
if 'id' in r:
id = (result[i+1].translate(str.maketrans('', '', string.punctuation)))
print( id )
return id
extract_id(api_response)
def extract_id(result):
result = str(result).split(' ')
for i, r in enumerate(result):
if 'id' in r:
id = (result[i+8].translate(str.maketrans('', '', string.punctuation)))
print( id )
return id
extract_id(api_response)
def extract_id(result):
result = str(result).split(' ')
for i, r in enumerate(result):
if 'id' in r:
id = (result[i+15].translate(str.maketrans('', '', string.punctuation)))
print( id )
return id
extract_id(api_response)
I have been able to use a function to extract the ID's but I am doing so through a string. I am in need of a scalable solution that I can use to pass these ID's along to another API call.
I have tried to use a for loop but because it is 1 string and i+1 defines the id's position, it is redundant and just outputs 1 of the id's multiple times.
I am receiving the correct output using each of these functions however, it is not scalable..... and just is not a solution. Please help guide me......

So to solve the response as a string issue I would suggest using python's builtin json module. Specifically, the method .loads() can convert a string to a dict or list of dicts. From there you can iterate over the list or dict and check if the key is equal to 'id'. Here's an example based on what you said the response would look like.
import json
s = "[{'_from': None, 'to': {'id': '5c9941fcdd2eeb6a6787916e', 'type': 'user'}}, {'_from': None, 'to': {'id': '5cc9055fcc5781152ca6eeb8', 'type': 'user'}}, {'_from': None, 'to': {'id': '5d1cf102c94c052cf1bfb3cc', 'type': 'user'}}]"
# json uses double quotes and null; there is probably a better way to do this though
s = s.replace("\'", '\"').replace('None', 'null')
response = json.loads(s) # list of dicts
for d in response:
for key, value in d['to'].items():
if key == 'id':
print(value) # or whatever else you want to do
# 5c9941fcdd2eeb6a6787916e
# 5cc9055fcc5781152ca6eeb8
# 5d1cf102c94c052cf1bfb3cc

Python fails to json.loads() nested JSON from Kafka

I can't loads() a stringified JSON into a valid JSON, using json.dumps(). Here is the code and the result. You can see the erroneous input at error logs. Is there any way I can convert that nested string into valid json? Thanks in advance.
async for value in RedisHandler.get_all_keys():
examined_counter += 1
task = None
try:
task = json.loads(value)
except Exception as convertJSONException:
logger.info("Failed converting '%s' into JSON. Skipping." % value)
continue
prints:
[-- TimeoutHandler] Failed converting '{'description': 'dqwdqdq', 'author': 'spp42admin', 'target_project_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6', 'tasklist': [{'opuid': '253e3d0d-5514-11e8-9a7c-d4c9ef52f9c6', 'method': 'split-count', 'parameter': '7', 'optional_parameters': '{"codec":"libx264"}', 's3_credentials': {'service_name': 's3', 'aws_access_key_id': '********', 'aws_secret_access_key': '*********', 'endpoint_url': 'http://localhost:9000', 'bucket_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6'}, 'srcpath': 'videos/397939d2-550c-11e8-9a7c-d4c9ef52f9c6/test.mp4', 'destpath': 'operations/253e3d0d-5514-11e8-9a7c-d4c9ef52f9c6', 'videoname': 'test.mp4', 'codec': 'mpeg4', 'progress': 1, 'parent_operation_uuid': '253f87b6-5514-11e8-9a7c-d4c9ef52f9c6', 'estimated_timeout': 1526042364}], 's3_credentials': {'service_name': 's3', 'aws_access_key_id': '**********', 'aws_secret_access_key': '***********', 'endpoint_url': 'http://localhost:9000', 'bucket_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6'}, 'video_uuid': '397939d2-550c-11e8-9a7c-d4c9ef52f9c6', 'parent_videodocument_uuid': '397939d3-550c-11e8-9a7c-d4c9ef52f9c6', 'parent_project_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6', 'progress': 0, 'operation_uuid': '253f87b6-5514-11e8-9a7c-d4c9ef52f9c6'}' into JSON. Skipping.
optional_parameters is the one causing error.
TLDR: How can I convert string below into JSON, using json.dumps?
'{'description': 'dqwdqdq', 'author': 'spp42admin', 'target_project_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6', 'tasklist': [{'opuid': '253e3d0d-5514-11e8-9a7c-d4c9ef52f9c6', 'method': 'split-count', 'parameter': '7', 'optional_parameters': '{"codec":"libx264"}', 's3_credentials': {'service_name': 's3', 'aws_access_key_id': '********', 'aws_secret_access_key': '*********', 'endpoint_url': 'http://localhost:9000', 'bucket_uuid': '91115b46-54fa-11e8-9a7c-d4c9ef52f9c6'}, 'srcpath': 'videos/397939d2-550c-11e8-9a7c-d4c9ef52f9c6/test.mp4', 'destpath': 'operations/253e3d0d-5514-11e8-9a7c-d4c9ef52f9c6', 'videoname': 'test.mp4', 'codec': 'mpeg4', 'progress': 1, 'parent_operation_uuid': '253f87b6-5514-11e8-9a7c-d4c9ef52f9c6', 'estimated_timeout': 1526042364}]}'

create a python dictionary based on another dictionary

I have the the following dict that contains the following data:
response = {"status":"ERROR","email":"EMAIL_INVALID","name":"NAME_INVALID"}
I am trying to create a new dict based on 'response' that is suposed to look like the following:
{'api_error': {'list': [{'converted_value': 'No special characters allowed.',
'field': 'name',
'value': 'NAME_INVALID'},
{'converted_value': 'invalid email',
'field': 'email',
'value': 'EMAIL_INVALID'}],
'status': 'ERROR'},
'email': 'EMAIL_INVALID',
'email_label': 'invalid email',
'name': 'NAME_INVALID',
'name_label': 'No special characters allowed.',
'status': 'ERROR'}
So far I have been able to do the following:
ret = {}
for k in response:
if k != 'status':
ret[k+"_label"] = convert(response[k])
ret[k] = response[k]
else:
ret[k] = convert(response[k])
where 'convert' function translates each value of response. for example NAME_INVALID is converted to 'No special characters allowed.' and so on. Here is the output of what above code is doing:
{"status":"ERROR","name_label":"No special characters allowed.",
"email_label":"invalid email","name":"NAME_INVALID","email":"EMAIL_INVALID"}
I am getting problem creating the rest of the dictionary. The one whose key is 'api_error'. What will be the most efficient way of doing that?

import pprint
response = {"status": "ERROR", "email": "EMAIL_INVALID", "name":
"NAME_INVALID"}
def convert(label):
return {'NAME_INVALID': 'No special characters allowed',
'EMAIL_INVALID': 'invalid email',
'ERROR': 'ERROR'}[label]
ret = {}
for k in response:
if k != 'status':
ret[k + "_label"] = convert(response[k])
ret[k] = response[k]
info = {'converted_value': ret[k + "_label"],
'field': k,
'value': response[k]}
(ret.setdefault('api_error', {})
.setdefault('list', [])
.append(info))
else:
ret[k] = convert(response[k])
ret.setdefault('api_error', {})['status'] = ret[k]
pprint.pprint(ret)
yields
{'api_error': {'list': [{'converted_value': 'invalid email',
'field': 'email',
'value': 'EMAIL_INVALID'},
{'converted_value': 'No special characters allowed',
'field': 'name',
'value': 'NAME_INVALID'}],
'status': 'ERROR'},
'email': 'EMAIL_INVALID',
'email_label': 'invalid email',
'name': 'NAME_INVALID',
'name_label': 'No special characters allowed',
'status': 'ERROR'}

make one more function that will be like
def make2nddict(response):
for k in response:
if k != 'status':
d = {}
d['converted_value'] = convert(k)
d['field'] = k
d['value'] = response[k]
arr.append(d)
else:
final[k] = response[k]
final['list'] = arr
arr= []
final = {}
def convert(error):
if error == 'NAME_INVALID':
return 'No special characters allowed'
elif error == 'EMAIL_INVALID':
return 'EMAIL_INVALID'
else:
return error
ret = {}
for k in response:
if k != 'status':
ret[k+"_label"] = convert(response[k])
ret[k] = response[k]
else:
ret[k] = convert(response[k])
put output of noth the function in your api_error dictionary
good luck

Here the full code: ---
>>>response = {"status":"ERROR","email":"EMAIL_INVALID","name":"NAME_INVALID"}
>>>def convert(parameter):
if parameter == "NAME_INVALID":
return "No special characters allowed."
if parameter =="EMAIL_INVALID":
return "invalid email"
>>>def newdic(response):
ret={}
response_keys = response.keys()
response_keys.remove("status")
if response['status']=="ERROR":
ret = response
ret['api_error'] ={'list':[],'status':"ERROR"}
for key in response_keys:
ret[key+"_label"] = convert(response[key])
dic={}
dic['converted_value'] = convert(response[key])
dic['field'] = key
dic['value'] = response[key]
ret['api_error']['list'].append(dic)
return ret
Here is the answer:--
>>>newdic(response)
{'status': 'ERROR',
'email_label': 'invalid email',
'name': 'NAME_INVALID',
'name_label': 'No special characters allowed.',
'api_error': {
'email': 'EMAIL_INVALID',
'list': [{'field': 'email', 'value': 'EMAIL_INVALID',
'converted_value': 'invalid email'},
{'field': 'name', 'value': 'NAME_INVALID',
'converted_value': 'No special characters allowed.'}
],
'status': 'ERROR'}
}