Related
I have a base.json file that I would like to split into smaller ones containing only filtered data by my python script where I am using json module. At the moment I can filter the data but I don't know how to save them to the appropriate files like: user, computer, group, printer, etc.
I would like to achieve the effect that after checking the base file, the data was properly saved to smaller files:
users --> users.json
computers --> computers.json
groups --> groups.json
etc.
First, I would like to know how to save data to separate files, then I know that what is repeated can be closed in the function def save_to_string(data_to_save)
My base json looks like this:
{
"entries": [
{
"attributes": {
all attributes from ldap like cn, sn, objectClass, etc
},
"dn": "CN=group1,OU=Groups,OU=UNIVERSUM,DC=universum,DC=local"
},
{
"attributes": {
all attributes from ldap like cn, sn, objectClass, etc
},
"dn": "CN=cmptr-01,OU=Computers,OU=UNIVERSUM,DC=universum,DC=local"
},
{
"attributes": {
all attributes from ldap like cn, sn, objectClass, etc
},
"dn": "CN=John Doe,OU=Users,OU=UNIVERSUM,DC=universum,DC=local"
}
]
}
and my code looks:
# --- reading encoded accounts
encoded_retrieved_accounts = root_path + data_dir + encoded_accounts_file_name
with open(encoded_retrieved_accounts, 'r', encoding="UTF-8") as file:
data = json.load(file)
retrieved_accounts = data['entries']
def save_to_json(data_to_save, fname):
with open(f"./{fname}.json", 'w', encoding="UTF-8") as file:
return json.dump(data_to_save, file, ensure_ascii=False, indent=4, sort_keys=True)
for account in retrieved_accounts:
attributes = account['attributes']
objectCategory = attributes['objectCategory']
if str(objectCategory[3:-46]) == 'Person':
u_data = account
save_to_json(u_data, 'Person')
elif str(objectCategory[3:-46]) == 'Computer':
c_data = account
save_to_json(c_data, 'Computer')
elif str(objectCategory[3:-46]) == 'Organizational-Unit':
ou_data = account
save_to_json(ou_data, 'Organizational-Unit')
elif str(objectCategory[3:-46]) == 'Group':
g_data = account
save_to_json(g_data, 'Group')
elif str(objectCategory[3:-46]) == 'Print-Queue':
pq_data = account
save_to_json(pq_data, 'Print-Queue')
elif str(objectCategory[3:-46]) == 'MSMQ-Configuration':
msmq_data = account
save_to_json(msmq_data, 'MSMQ-Configuration')
else:
unknow_data = account
save_to_json(unknow_data, 'to-clarify')
but this saves only the last found account in the base file, for example users are 'n', groups are 'm', printers are 'i' and hosts are 'j'
I changed the code and now everything works fine
import os
import json
from dotenv import load_dotenv
# --- loading .env file
load_dotenv('.env')
# ------ variables
root_path = os.environ.get('ROOT_PATH')
data_dir = os.environ.get('DATA_DIR')
tmp_dir = os.environ.get('TMP_DIR')
encoded_accounts_file_name = os.environ.get('ENCODED_ACCOUNTS_FILE_NAME')
print(" - .env loaded")
# --- reading encoded accounts
encoded_retrieved_accounts = root_path + data_dir + tmp_dir + encoded_accounts_file_name
with open(encoded_retrieved_accounts, 'r', encoding="UTF-8") as file:
data = json.load(file)
retrieved_accounts = data['entries']
def save_to_json(data_to_save, fname):
with open(f"./{fname}.json", 'a', encoding="UTF-8") as file:
return json.dump(data_to_save, file, ensure_ascii=False, indent=4, sort_keys=True)
for account in retrieved_accounts:
attributes = account['attributes']
objectCategory = attributes['objectCategory']
if str(objectCategory[3:-46]) == 'Person':
u_data = account
save_to_json(u_data, 'Person')
elif str(objectCategory[3:-46]) == 'Computer':
c_data = account
save_to_json(c_data, 'Computer')
elif str(objectCategory[3:-46]) == 'Organizational-Unit':
ou_data = account
save_to_json(ou_data, 'Organizational-Unit')
elif str(objectCategory[3:-46]) == 'Group':
g_data = account
save_to_json(g_data, 'Group')
elif str(objectCategory[3:-46]) == 'Print-Queue':
pq_data = account
save_to_json(pq_data, 'Print-Queue')
elif str(objectCategory[3:-46]) == 'MSMQ-Configuration':
msmq_data = account
save_to_json(msmq_data, 'MSMQ-Configuration')
else:
unknow_data = account
save_to_json(unknow_data, 'to-clarify')
I am working on a project for a python class, where we have to create a vin number look up tool. This code below works so far for country and year made. However I am having an issue with how I am indexing the user input.
Sometimes a country is the first two characters, and other times the country code is only the first character. I do not how to get around this problem, I tried using an if else kind of method while iterating through the dictionary, but it did not work.
class vinfo():
def __init__(self):
""" Get user input """
vin = input("Enter Vin: ")
""" Stupidity check """
unaccepted_chars = [
"-","/",".",","," ","?",
"^","$","*","(",")","[",
"]","{","}","#","!","_",
"+","'","=","|","#","<",
">","`","~","&"
]
for char in vin:
if char in unaccepted_chars:
vin = vin.replace(char, "")
else:
print("", end = "")
""" Length check and index """
if len(vin) == 17:
self.country_filt1 = vin[0]
self.country_filt2 = vin[0:2]
self.engine_filt = vin[7]
self.year_filt = vin[9]
self.manufact_filt = vin[1:3]
self.manufact_filt1 = vin[1:4]
else:
print("You must've entered something really stupid.\n(must be 17 characters, letters are uppercase)")
""" Vin Code Data """
# each manufacturer seems to have multiple vins for different car styles
# and for different countries
manufact_codes = {
"1G1":"Chevy",
"":"",
}
year_codes = {
"M":"2021","L":"2020","K":"2019",
"J":"2018","H":"2017","G":"2016",
"F":"2015","E":"2014","D":"2013",
"C":"2012","B":"2011","A":"2010",
"9":"2009","8":"2008","7":"2007",
"6":"2006","5":"2005","4":"2004",
"3":"2003","2":"2002","1":"2001",
"Y":"2000","X":"1999","W":"1998",
"V":"1997","T":"1996","S":"1995",
"R":"1994","P":"1993","N":"1992",
"M":"1991","L":"1990","K":"1989",
"J":"1988","H":"1987","G":"1986",
"F":"1985","E":"1984","D":"1983",
"C":"1982","B":"1981","A":"1980"
}
country_codes = {
"1": "USA",
"4": "USA",
"5": "USA",
"7F": "USA",
"3X": "Mexico",
"37": "Mexico",
"3A": "Canada",
"3W": "Canada",
"W": "Germany",
"SA": "United Kingdom",
"SM": "United Kingdom",
"J": "Japan",
"KL": "Korea",
"KR": "Korea"
}
engine_codes = {
}
""" Define the vehicles attributes using the find() function below """
self.year = self.find(self.year_filt, year_codes)[0]
# self.country = self.find(self.country_filt, country_codes)[0]
# self.manufact = self.find(self.manufact_filt, manufact_codes)
self.engine = self.find(self.engine_filt, engine_codes)
""" Find country (different lengths of country codes) """
for key, value in country_codes.items():
if key == self.country_filt1:
country = value
elif key == self.country_filt2:
country = value
else:
country = "Unsupported code"
""" Same for manufacturer """
for key, value in manufact_codes.items():
if key == self.manufact_filt:
manufact = value
elif key == self.manufact_filt1:
manufact = value
else:
manufact = "Unsupported code"
self.info = print(f"Year: {self.year}\nManufacturer: {manufact}\nCountry: {country}\nEngine: {self.engine}")
""" search through the dictionaries """
def find(self, filt, dict_of_codes):
try:
info = [value for key, value in dict_of_codes.items() if key == filt]
except:
info = "Unsupported"
if len(info) > 1:
info += " (Could be any of these)"
return info
I have recently uploaded contact records to HubSpot using Postman. Here is a raw JSON data example and POST method that I use to successfully upload a contact:
https://api.hubapi.com/crm/v3/objects/contacts?hapikey={{hapikey}}
{properties": {
"smbapi": "yes",
"email": "fcgrinding#junkstermail.com",
"business_name":"Forest City Grinding Inc",
"srvc_address_1":"3544 Stenstrom Rd",
"srvc_city_1":"",
"srvc_state_1":"IL",
"srvc_zip_1":"61109",
"proposal_date":"2021-12-07",
"proposal_start_date": "2022-01-01",
"udc_code_1": "COMED",
"eog":"electric",
"fixedprice_1_gas_mcf": 6.63,
"fixedprice_2_gas_mcf": 6.11,
"fixedprice_3_gas_mcf": 5.9,
"term_1": 12,
"term_2": 24,
"term_3": 36,
"smb_bdm_name": "Timothy Chin",
"smb_bdm_phone": "833-999-9999",
"smb_bdm_email": "tim.chin#junkstermail.com"
}
}
Next, I then created a python lambda function to automate this process because we want to ingest CSV files that may have many records to extract. So, I had constructed the dictionary to look the same as the string above which had worked great/fine with Postman. However, when I try and do a Post method API call to HubSpot, using my dictionary payload, I am getting this error:
Invalid input JSON : Cannot build ObjectSchemaEgg, Invalid input JSON
on line 1, column 2: Cannot build ObjectSchemaEgg, some of required
attributes are not set [name, labels]
Here is the processed dictionary string that my code constructed for the API call:
{'properties': '{"smbapi": "yes", "business_name": "Forest City Grinding Inc", "srvc_address_1": "4844 Stenstrom Rd", "srvc_state_1": "IL", "srvc_zip_1": "61109", "proposal_date": "2021-12-07", "proposal_start_date": "2022-01-01", "udc_code_1": "COMED", "fixedprice_1": "6.63", "fixedprice_2": "6.11", "fixedprice_3": "5.9", "term_2": "24", "term_3": "36", "smb_bdm_name": "Gary Wysong", "smb_bdm_phone": "833-389-0881", "smb_bdm_email": "gary.wysong#constellation.com"}'}
Here is my Lambda code in full (give special attention to both the call to post_to_hubspot() and also the post_to_hubspot() function itself). The code that loads the dynamo table is working correctly.:
import boto3
import json
import decimal
from botocore.exceptions import ClientError
from boto3.dynamodb.conditions import Key, Attr
import re
import pandas as pd
import numpy as np
import os
import datetime
from os import urandom
import email
import base64
import requests
from datetime import datetime, timedelta, timezone
import mailparser
import calendar
global payload_data
landing_zone_bucket_name = str(os.environ['BUCKETNAME'])
s3 = boto3.resource('s3')
landing_zone_bucket = s3.Bucket(landing_zone_bucket_name )
s3r = boto3.client('s3')
dynamodb = boto3.resource('dynamodb', region_name='us-west-2')
table = dynamodb.Table(str(os.environ['DYNAMOTABLE']))
unprocessed_records_table = dynamodb.Table(str(os.environ['UNPROCESSEDTABLE']))
email_table = dynamodb.Table(str(os.environ['EMAILSTATUSTABLE']))
endpoint_url=os.environ['ENDPOINT_URL']
access_key = os.environ['ACCESSKEY']
now = datetime.now()
today_date = datetime.strftime(now,'%d')
today_month = datetime.strftime(now,'%m')
today_year = datetime.strftime(now,'%Y')
time_stamp = datetime.now().strftime('%Y%m%d%H%M%S')
payload_data = {}
#WRITE RECORDS TO DYNAMO
def dynamoPut(dObj,table_name=None):
try:
for each in list(dObj['Information']):
if dObj['Information'][each]:
dObj['Information'][each] = str(dObj['Information'][each])
else:
del dObj['Information'][each]
dObj['Month'] = today_month
dObj['Year'] = today_year
dObj['Day'] = today_date
for each in list(dObj):
if dObj[each] != '':
dObj[each] = dObj[each]
else:
del dObj[each]
if table_name != None:
response = unprocessed_records_table.put_item(Item = dObj)
else:
response = table.put_item(Item = dObj)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
return True
else:
return False
except Exception as e:
print(e)
return False
def dynamoPutFileName(filename,source_type):
try:
dObj = {}
dObj['id'] = urandom(20).hex()
dObj['CreatedAt'] = str(datetime.now())
dObj['FileName'] = filename
dObj['Type'] = source_type
dObj['EmailSent'] = False
response = email_table.put_item(Item = dObj)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
return True
else:
return False
except Exception as e:
print(e)
return False
def parse_csv_hubspot(event, obj):
#parsing CSV file to write to dynamo
try:
def auto_truncate(val):
return val[:255 ]
print('<< IN PARSE CSV HUBSPOT >>')
print(event)
csv = pd.read_csv(obj['Body'], encoding = "ISO-8859-1")
csv_nn = csv.replace(np.nan, 'null', regex=True)
d = csv_nn.to_dict(orient='records')
source_id = urandom(20).hex()
file_name = event['file_path'].split('/')[-1]
print('<< FILE NAME >>', file_name)
for each in d:
try:
dbObj = {}
#PASSING THE EXTERNAL KEY
UniqueKey = ''
if 'smbapi' in each and each['smbapi'] != 'null':
dbObj['smbapi' ] = each['smbapi']
print('<< SMB API>>', dbObj['smbapi' ])
if 'business_name' in each and each['business_name'] != 'null':
dbObj['business_name'] = each['business_name']
print('<< BUSINESS NAME >>', dbObj['business_name'])
if 'srvc_address_1' in each and each['srvc_address_1'] != 'null':
dbObj['srvc_address_1'] = each['srvc_address_1']
print('<< ADDRESS 1 >>', dbObj['srvc_address_1'])
if 'srvc_city_1' in each and each['srvc_city_1'] != 'null':
dbObj['srvc_city_1'] = each['srvc_city_1']
if 'srvc_state_1' in each and each['srvc_state_1'] != 'null':
dbObj['srvc_state_1'] = each['srvc_state_1']
if 'srvc_zip_1' in each and each['srvc_zip_1'] != 'null':
dbObj['srvc_zip_1']= str(each['srvc_zip_1']).zfill(5)
if 'proposal_date' in each and each['proposal_date'] != 'null':
dbObj['proposal_date']= try_parsing_date(each['proposal_date']).date().isoformat()
if 'proposal_start_date' in each and each['proposal_start_date'] != 'null':
dbObj['proposal_start_date']= try_parsing_date(each['proposal_start_date']).date().isoformat()
if 'udc_code_1' in each and each['udc_code_1'] != 'null':
dbObj['udc_code_1']= each['udc_code_1']
if 'eog' in each and each['eog'] != 'null':
dbObj['eog']= each['eog']
if 'fixedprice_1' in each and each['fixedprice_1'] != 'null':
dbObj['fixedprice_1']= each['fixedprice_1']
if 'fixedprice_2' in each and each['fixedprice_2'] != 'null':
dbObj['fixedprice_2']= each['fixedprice_2']
if 'fixedprice_3' in each and each['fixedprice_3'] != 'null':
dbObj['fixedprice_3']= each['fixedprice_3']
if 'fixedprice_1_gas_therm' in each and each['fixedprice_1_gas_therm'] != 'null':
dbObj['fixedprice_1_gas_therm']= each['fixedprice_1_gas_therm']
if 'fixedprice_2_gas_therm' in each and each['fixedprice_2_gas_therm'] != 'null':
dbObj['fixedprice_2_gas_therm']= each['fixedprice_2_gas_therm']
if 'fixedprice_3_gas_therm' in each and each['fixedprice_3_gas_therm'] != 'null':
dbObj['fixedprice_3_gas_therm']= each['fixedprice_3_gas_therm']
if 'fixedprice_1_gas_ccf' in each and each['fixedprice_1_gas_ccf'] != 'null':
dbObj['fixedprice_1_gas_ccf']= each['fixedprice_1_gas_ccf']
if 'fixedprice_2_gas_ccf' in each and each['fixedprice_2_gas_ccf'] != 'null':
dbObj['fixedprice_2_gas_ccf']= each['fixedprice_2_gas_ccf']
if 'fixedprice_3_gas_ccf' in each and each['fixedprice_3_gas_ccf'] != 'null':
dbObj['fixedprice_3_gas_ccf']= each['fixedprice_3_gas_ccf']
if 'fixedprice_1_gas_dth' in each and each['fixedprice_1_gas_dth'] != 'null':
dbObj['fixedprice_1_gas_dth']= each['fixedprice_1_gas_dth']
if 'fixedprice_2_gas_dth' in each and each['fixedprice_2_gas_dth'] != 'null':
dbObj['fixedprice_2_gas_dth']= each['fixedprice_2_gas_dth']
if 'fixedprice_3_gas_dth' in each and each['fixedprice_3_gas_dth'] != 'null':
dbObj['fixedprice_3_gas_dth']= each['fixedprice_3_gas_dth']
if 'fixedprice_1_gas_mcf' in each and each['fixedprice_1_gas_mcf'] != 'null':
dbObj['fixedprice_1_gas_mcf']= each['fixedprice_1_gas_mcf']
if 'fixedprice_2_gas_mcf' in each and each['fixedprice_2_gas_mcf'] != 'null':
dbObj['fixedprice_2_gas_mcf']= each['fixedprice_2_gas_mcf']
if 'fixedprice_3_gas_mcf' in each and each['fixedprice_3_gas_mcf'] != 'null':
dbObj['fixedprice_3_gas_mcf']= each['fixedprice_3_gas_mcf']
if 'term_1' in each and each['term_1'] != 'null':
dbObj['term_1']= each['term_1']
if 'term_2' in each and each['term_2'] != 'null':
dbObj['term_2']= each['term_2']
if 'term_3' in each and each['term_3'] != 'null':
dbObj['term_3']= each['term_3']
if 'smb_bdm_name' in each and each['smb_bdm_name'] != 'null':
dbObj['smb_bdm_name']= each['smb_bdm_name']
if 'smb_bdm_phone' in each and each['smb_bdm_phone'] != 'null':
if '.' in str(each['smb_bdm_phone']):
dbObj['smb_bdm_phone']= str(int(float(each['smb_bdm_phone'])))
else:
dbObj['smb_bdm_phone']= str(each['smb_bdm_phone'])
if 'smb_bdm_email' in each and each['smb_bdm_email'] != 'null' and each['smb_bdm_email'].strip() != '' and each['smb_bdm_email'] != None:
dbObj['smb_bdm_email']= each['smb_bdm_email']
print('<< OBJ >> ',dbObj)
N = urandom(20).hex()
now = str(datetime.now())
#<< END of HUBSPOT INGESTION >>
# table.put_item(
Item = {
'CogId' : str(N),
'CreatedAt': now,
'ExternalId': UniqueKey,
'Information' : dbObj,
'SourceBucket': landing_zone_bucket_name,
'SourcePath' : event['file_path'],
'Source' : 'HubSpot',
'SourceId' : source_id,
'SourceFileName': time_stamp + '_' + file_name
}
#WRITE-TO-DYNAMO
files_processing = dynamoPut(Item)
if not files_processing:
print('Writing {} record to dynamodb Failed'.format(Item))
except Exception as e:
print(e)
N = urandom(20).hex()
Item = {
'CogId' : str(N),
'CreatedAt': now,
'Information' : each,
'SourceBucket': landing_zone_bucket_name,
'SourcePath' : event['file_path'],
'Source' : 'HubSpot',
'message': str(e),
'SourceId' : source_id,
'ExternalId': UniqueKey
}
files_processing = dynamoPut(Item,'Fail')
pass
temp_file_name = time_stamp + '_' + file_name
isert_file_name = dynamoPutFileName(temp_file_name,'HubSpot')
post_to_hubspot(dbObj)
return True
except Exception as e:
print(e)
new_folder_path = os.environ['CSV_NEW_FOLDER_HUBSPOT']
unprocessed_folder_path = os.environ['CSV_ERROR_FOLDER_HUBSPOT']
# MOVING PROCESSED FILES FROM NEW TO UNPROCESSED FOLDER
move_file_to_processed = moving_files_new_to_processed(event, new_folder_path,unprocessed_folder_path)
return False
def try_parsing_date(text):
for fmt in ('%m/%d/%Y','%Y-%m-%dT%H:%M:%S-%f', '%m/%d/%y', '%Y-%m-%d', '%m.%d.%Y','%Y-%m-%dT%I', '%Y-%m-%dT%I%p', '%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S.%f+','%Y-%m-%dT%H:%M:%S'):#2018-11-20T08:05:54-0500
try:
return datetime.strptime(text, fmt)
except ValueError:
print('in except')
pass
return ValueError('no valid date format found')
def post_to_hubspot(list_contacts):
print('<< IN POST-To-HUBSPOT >>')
data_string = **json.dumps(list_contacts)**
payload_data = {"properties": data_string}
print('<< dbOBJ LIST >> ',payload_data)
response = requests.request("POST", endpoint_url+access_key, headers={'Content-Type': 'application/json'}, data=payload_data)
token_response=json.loads(response.text)
print('<< TOKEN RESPONSE >>',token_response)
def moving_files_new_to_processed(event, new_folder,processed_folder):
#MOVING-FILES-TO-PROCESSED
try:
copy_source = {
'Bucket': landing_zone_bucket_name,
'Key': event['file_path']
}
path = event['file_path']
processed_folder = processed_folder + time_stamp + '_'
new_key = path.replace(new_folder, processed_folder)
new_obj = landing_zone_bucket.Object(new_key)
new_obj.copy(copy_source)
s3.Object(landing_zone_bucket_name, event['file_path']).delete()
return True
except Exception as e:
print(e)
return False
def lambda_handler(event,context):
print("Starting to Push Records to Dynamo Lambda")
print(event)
try:
parse_flag = False
new_folder_path = ''
processed_folder_path = ''
#Gets file path and calls required function to parse it out
key = str(os.environ['CSV_NEW_FOLDER_HUBSPOT'])
obj = s3r.get_object(Bucket=landing_zone_bucket_name, Key=event['file_path'])
print('after obj')
print(os.environ['CSV_NEW_FOLDER_HUBSPOT'])
print('in HubSpot parse_csv')
parse_csv_func = parse_csv_hubspot(event, obj)
# Checks if parse_csv return empty dictionary
if parse_csv_func:
parse_flag = True
new_folder_path = os.environ['CSV_NEW_FOLDER_HUBSPOT']
processed_folder_path = os.environ['CSV_PROCESSED_FOLDER_HUBSPOT']
else:
print('File Format not Supported for {}'.format(event['file_path']))
if parse_flag:
# UPLOADING CONTACT.MOVING PROCESSED FILES FROM NEW TO PROCESSED FOLDER
#print('<< PAYLOAD >> ',payload)
#response = requests.request("POST", "https://api.hubapi.com/crm/v3/schemas/?hapikey="+access_key, headers={'Content-Type': 'application/json'}, data=json.dumps(str(payload)))
#token_response=json.loads(response.text)
#print('<< TOKEN RESPONSE >>',token_response)
#MOVING PROCESSED FILES FROM NEW TO PROCESSED FOLDER
move_file_to_processed = moving_files_new_to_processed(event, new_folder_path,processed_folder_path)
if move_file_to_processed:
print('File {} moved Successfully from {} to {}'.format(event['file_path'],new_folder_path,processed_folder_path))
else:
print('Moving {} file from new to processing folder Failed'.format(event['file_path']))
except Exception as e:
print(e)
What could be the problem? Thanks for your help.
The problem was caused by two issues:
The dictionary should have been placed in json.dumps() to convert it to JSON string when doing a POST so the dictionary didn't need to change its structure. Here's the response from the POST:
<< TOKEN RESPONSE >> {
"id": "135120801",
"properties": {
"business_name": "Millers Brand Oats",
"createdate": "2021-12-21T02:31:12.452Z",
"fixedprice_1": "6.63",
"fixedprice_2": "6.11",
"fixedprice_3": "5.9",
"hs_all_contact_vids": "135120801",
"hs_is_contact": "true",
"hs_is_unworked": "true",
"hs_marketable_until_renewal": "false",
"hs_object_id": "135120801",
"hs_pipeline": "contacts-lifecycle-pipeline",
"lastmodifieddate": "2021-12-21T02:31:12.452Z",
"proposal_date": "2021-12-07",
"proposal_start_date": "2022-01-01",
"smb_bdm_email": "Tim.Chu#junkster.com",
"smb_bdm_name": "Tim Chu",
"smb_bdm_phone": "833-999-9999",
"smbapi": "yes",
"srvc_address_1": "4844 Stenstrom Rd",
"srvc_state_1": "IL",
"srvc_zip_1": "61109",
"term_2": "24",
"term_3": "36",
"udc_code_1": "COMED"
},
"createdAt": "2021-12-21T02:31:12.452Z",
"updatedAt": "2021-12-21T02:31:12.452Z",
"archived": false
}
I was using the wrong endpoint:
https://api.hubapi.com/crm/v3/schemas/
instead of:
https://api.hubapi.com/crm/v3/objects/contacts/
Now I just need to find out why the AWS Lambda POSTs allow for duplicate contacts to be created in HubSpot while Postman POSTs prohibit duplicate contacts to be created.
My code is working, but I know there has to be a more efficient way to accomplish it and better exception handling.
API data format per item:
{
u'logEnd':False,
u'logBegin':True,
u'links':{
u'self': u'https://192.168.15.140/api/fmc_config/v1/domain/e276abec-e0f2-11e3-8169-6d9ed49b625f/policy/accesspolicies/00505689-7E52-0ed3-0000-184683643989/accessrules/00505689-7E52-0ed3-0000-000268439552'
},
'sourceNetworks':'any-src',
u'name':u'Peer-to-Peer Blocking',
'sourceZones':'any-src-zn',
u'enabled':True,
'sourcePorts':'any-src-port',
u'logFiles':False,
u'vlanTags':{ },
u'applications':{
u'applications':[
{
u'type':u'Application',
u'id':u'61',
u'name':u'BitTorrent'
},
{
u'type':u'Application',
u'id':u'571',
u'name':u'BitTorrent tracker'
},
{
u'type':u'Application',
u'id':u'1214',
u'name':u'ExtraTorrent'
}
]
},
u'sendEventsToFMC':True,
u'action':u'BLOCK',
'destinationPorts':'any-dest-port',
'destinationNetworks':'any-dest',
u'variableSet':{
u'type':u'VariableSet',
u'name':u'Default Set',
u'id':u'76fa83ea-c972-11e2-8be8-8e45bb1343c0'
},
u'type':u'AccessRule',
u'id':u'00505689-7E52-0ed3-0000-000268439552',
'destinationZones':'any-dst-zn',
u'metadata':{
u'category':u'--Undefined--',
u'accessPolicy':{
u'type':u'AccessPolicy',
u'name':u'PVmain1-ACPolicy-201610251131',
u'id':u'00505689-7E52-0ed3-0000-184683643989'
},
u'section':u'Mandatory',
u'domain':{
u'type':u'Domain',
u'name':u'Global',
u'id':u'e276abec-e0f2-11e3-8169-6d9ed49b625f'
},
u'timestamp':1482339574186
}
}
Working Python Script
for i in results:
response = requests.request("GET", i, headers=headers, verify=False)
raw=response.json()
raw.setdefault('name', "noname_rule")
raw.setdefault('action', "no_action")
raw.setdefault('sourceNetworks', "any-src")
raw.setdefault('destinationNetworks', "any-dest")
raw.setdefault('sourcePorts', "any-src-port")
raw.setdefault('destinationPorts', "any-dest-port")
raw.setdefault('sourceZones', "any-src-zn")
raw.setdefault('destinationZones', "any-dst-zn")
interesting_keys = ('name', 'action','sourceZones', 'sourceNetworks', 'sourcePorts', 'destinationZones', 'destinationNetworks', 'destinationPorts' )
subdict = {x: raw.get(x, "any") for x in interesting_keys if x in raw}
if 'objects' in subdict['sourceZones']:
srczn = subdict['sourceZones']['objects'][0]['name']
elif 'literals' in subdict['sourceZones']:
srczn = subdict['sourceZones']['literals'][0]['port']
else :
srczn = subdict['sourceZones']
if 'objects' in subdict['sourceNetworks']:
srcnet = subdict['sourceNetworks']['objects'][0]['name']
elif 'literals' in subdict['sourceNetworks']:
srcnet = subdict['sourceNetworks']['literals'][0]['value']
else :
srcnet = subdict['sourceNetworks']
if 'objects' in subdict['sourcePorts']:
srcprt = subdict['sourcePorts']['objects'][0]['name']
elif 'literals' in subdict['sourcePorts']:
srcprt = subdict['sourcePorts']['literals'][0]['port']
else :
srcprt = subdict['sourcePorts']
if 'objects' in subdict['destinationZones']:
dstzn = subdict['destinationZones']['objects'][0]['name']
elif 'literals' in subdict['destinationZones']:
dstzn = subdict['destinationZones']['literals'][0]['port']
else :
dstzn = subdict['destinationZones']
if 'objects' in subdict['destinationNetworks']:
dstnet = subdict['destinationNetworks']['objects'][0]['name']
elif 'literals' in subdict['destinationNetworks']:
dstnet = subdict['destinationNetworks']['literals'][0]['value']
else :
dstnet = subdict['destinationNetworks']
if 'objects' in subdict['destinationPorts']:
dstprt = subdict['destinationPorts']['objects'][0]['name']
elif 'literals' in subdict['destinationPorts']:
try:
dstprt = subdict['destinationPorts']['literals'][0]['port']
except KeyError:
dstprt = "0"
else :
dstprt = subdict['destinationPorts']
#print >> target, "%d,%s,%s,%s,%s,%s,%s,%s,%s" %(number, subdict['name'],subdict['action'],srczn,srcnet,srcprt,dstzn,dstnet,dstprt)
#print "%d,%s,%s,%s,%s,%s,%s,%s,%s" %(number, subdict['name'],subdict['action'],srczn,srcnet,srcprt,dstzn,dstnet,dstprt)
number+=1
time.sleep(.5)
print raw
Can you suggest other ways that would improve error handling, performance, readability, etc?
Whenever you see repeated code, perhaps it could be collapsed into one function that can reused over and over? Refactoring the code this way probably doesn't make it faster, but it does make it easier to read and understand.
def getSubdictData(subdict, fieldname, prop2):
if 'objects' in subdict[fieldname]:
return subdict[fieldname]['objects'][0]['name']
elif 'literals' in subdict[fieldname]:
if prop2 in subdict[fieldname]['literals'][0]
return subdict[fieldname]['literals'][0][prop2]
else:
return "0"
else :
return subdict[fieldname]
for i in results:
...
srczn = getSubdictData(subdict, 'sourceZones', 'port')
srcnet = getSubdictData(subdict, 'sourceNetworks', 'value')
srcprt = getSubdictData(subdict, 'sourcePorts', 'port')
dstzn = getSubdictData(subdict, 'destinationZones', 'port')
dstnet = getSubdictData(subdict, 'destinationNetworks', 'value')
dstnet = getSubdictData(subdict, 'destinationPorts', 'port')
...
Also note that I removed the try-catch you were using to detect if a field existed or not. If the object/code provides a way for you to determine that this field existed or not, we should use that instead of relying on a sloppy try catch to perform our logical operations. Try catches used like this are hard to read and perform badly. In this case it wouldn't matter, but its good practice not to abuse try catches like this.
I am working on a facebook mini-chat bot and I am encountering a problem which consists on the bot to receive the same message over and over even though it has already answered the message.
it keeps receiving the same text from FB and replying to it over and over
def message_handler(request):
data = json.loads(request.body.decode('utf-8'))
if data and data['object'] == 'page':
for pageEntry in data['entry']:
print "nombre de message", len(pageEntry['messaging'])
for messagingEvent in pageEntry['messaging']:
if messagingEvent.get('optin'):
print "optin", messagingEvent
receivedAuthentication(messagingEvent)
elif messagingEvent.get('message'):
print "message", messagingEvent
receivedMessage(messagingEvent)
elif messagingEvent.get('delivery'):
print "delivery", messagingEvent
receivedDeliveryConfirmation(messagingEvent)
elif messagingEvent.get('postback'):
print "postback", messagingEvent
receivedPostback(messagingEvent)
else:
print "UnHandled"
return HttpResponse(status=200)
def receivedMessage(event):
senderID = event.get('sender').get('id')
message = event.get('message')
messageText = message.get('text')
messageAttachments = message.get('attachments')
if messageText:
if messageText == 'image':
sendImageMessage(senderID)
elif messageText == 'button':
sendButtonMessage(senderID)
elif messageText == 'generic':
sendGenericMessage(senderID)
elif messageText == 'receipt':
sendReceiptMessage(senderID)
elif messageText == 'hey':
sendTextMessage(senderID, "Get it. Gimme a moment to process it :). Will get back to you in a moment")
send_seen()
send_typing()
words = words_gen()
sendTextMessage(senderID, words)
def callSendAPI(messageData):
requests.post(
url='https://graph.facebook.com/v2.6/me/messages?access_token=' + config.page_token,
data=json.dumps(messageData),
headers={"Content-Type":"application/json"}
)
I get that I need to send a status 200 every time, which I did but still receiving the same text over and over
Here are the events I am subscribed to
conversations, message_deliveries, message_reads, messages, messaging_optins, messaging_postbacks, picture
I removed messaging_echoes because I thought it was the problem turned out to not
I have resolved this issue by writing a function and checking duplicate messages in my Web API service.
Here I am generating message unique id either by payload or message received from Facebook which user clicks or types and then comparing with earlier stored unique value from concurrent dictionary.
_messageUniqueKeysBySender is ConcurrentDictionary and I am caching values by Sender Id for 30 minutes.
private bool IsDuplicate(Messaging messaging)
{
var messageUniqueId = string.Empty;
var messageMessaging = messaging as MessageMessaging;
if (messageMessaging != null)
messageUniqueId = messageMessaging.Message.Id + messageMessaging.Message.SequenceNumber;
else if (messaging is PostbackMessaging)
messageUniqueId = ((PostbackMessaging)messaging).Postback.Payload +
((PostbackMessaging)messaging).TimestampUnix;
if (string.IsNullOrEmpty(messageUniqueId)) return false;
string existingUniqueId;
if (_messageUniqueKeysBySender.TryGetValue(messaging.Sender.Id, out existingUniqueId))
{
if (existingUniqueId == messageUniqueId)
{
return true;
}
else
{
_messageUniqueKeysBySender.TryUpdate(messaging.Sender.Id, messageUniqueId, existingUniqueId);
return false;
}
}
_messageUniqueKeysBySender.TryAdd(messaging.Sender.Id, messageUniqueId);
return false;
}
And then by checking in main code
try
{
if (!IsDuplicate(messaging))
{
var conversation = _conversationRepository[messaging.Sender.Id] ?? new Conversation(messaging.Sender.Id);
message = await _bot.RespondToMessagingAsync(conversation, messaging);
_conversationRepository[messaging.Sender.Id] = conversation;
_logger.ForContext("FacebookMessage", messagingJson).LogDuration("Processing Facebook message", sw);
}
else
_logger.ForContext("FacebookMessage", messagingJson).Warning("Duplicate message skipped");
}
catch (Exception ex)
{
_logger.ForContext("FacebookMessage", messagingJson).Error(ex, "Failed to process message");
message = new TextMessage(Resources.Error);
hasError = true;
}