recently I used a vinted scraping bot on the vinted.fr site and it all worked well when I was on my local computer, but when I put the bot on my Debian VPS, the bot didn't work.
# Author : 2$py#5430
# Discord : discord.gg/spyy
# Github : https://github.com/2spy/
# Updated at : 04/07/2022 12:55
# Version : 4.3
# Description : Vinted Bot
# Language : Python
import os
import json
import shutil
import threading
import time
import sys
try:
import requests
from bs4 import BeautifulSoup
except:
os.system("pip install requests")
os.system("pip install bs4")
proxies = {
"https": "https://MyUser:MyPassword#199.247.15.159:12071",
}
class Spy:
gris = "\033[1;30;1m"
rouge = "\033[1;31;1m"
vert = "\033[1;32;1m"
jaune = "\033[1;33;1m"
bleu = "\033[1;34;1m"
violet = "\033[1;35;1m"
cyan = "\033[1;36;1m"
blanc = "\033[1;0;1m"
def get_info_post(url):
try:
time.sleep(2)
print(f"{Spy.blanc}[{Spy.jaune}RECHERCHE{Spy.blanc}] - Le bot recupere les informations de l'item...")
headers = requests.utils.default_headers()
headers.update({'User-agent': 'Mozilla/5.0'})
reponse = requests.get(str(url), headers=headers, proxies=proxies)
if 429 == reponse.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
soup = BeautifulSoup(reponse.text, "html.parser")
res = soup.findAll('script', {"class": "js-react-on-rails-component"})
descindice = 0
userinfoindice = 0
for i in range(len(res)):
if 'data-component-name="ItemDescription"' in str(res[i]).split():
descindice = i
if 'data-component-name="ItemUserInfo"' in str(res[i]).split():
userinfoindice = i
description = json.loads(res[descindice].text.replace(
'<script class="js-react-on-rails-component" data-component-name="ItemDescription" data-dom-id="ItemDescription-react-component-3d79657d-a1b5-4f1d-b501-2f470f328c66" type="application/json">',
"").replace("</script>", ''))
userinfo = json.loads(res[userinfoindice].text.replace(
'<script class="js-react-on-rails-component" data-component-name="ItemUserInfo" data-dom-id="ItemUserInfo-react-component-2105d904-b161-47d1-bfce-9b897a8c1cc6" type="application/json">',
'').replace("</script>", ''))
titre = description["content"]["title"]
description = description["content"]["description"]
positive = userinfo["user"]["positive_feedback_count"]
negative = userinfo["user"]["negative_feedback_count"]
username = userinfo["user"]["login"]
pays = userinfo["user"]["country_title"]
ville = userinfo["user"]["city"]
lesinfo = {}
if titre == "":
titre = "Pas de donnée"
if description == "":
description = "Pas de donnée"
if positive == "":
positive = "Pas de donnée"
if negative == "":
negative = "Pas de donnée"
if username == "":
username = "Pas de donnée"
if pays == "":
pays = "Pas de donnée"
if ville == "":
ville = "Pas de donnée"
try:
lesinfo["titre"] = titre
lesinfo["description"] = description
lesinfo["positive"] = positive
lesinfo["negative"] = negative
lesinfo["username"] = username
lesinfo["pays"] = pays
lesinfo["ville"] = ville
except Exception as err:
print(err)
return lesinfo
except:
pass
def search(url):
try:
time.sleep(5)
print(f"{Spy.blanc}[{Spy.gris}RECHERCHE{Spy.blanc}] - Le bot cherche des nouveaux items...")
headers = requests.utils.default_headers()
headers.update({'User-agent': 'Mozilla/5.0'})
reponse = requests.get(str(url), headers=headers, proxies=proxies)
if 429 == reponse.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
soup = BeautifulSoup(reponse.text, "html.parser")
res = soup.findAll('script')
indices = 0
for i in range(len(res) + 1):
if 'data-js-react-on-rails-store="MainStore"' in str(res[i]).split():
indices += i
break
value = res[indices].text.replace('<script z-js-react-on-rails-store="MainStore" type="application/json">', "")
z = json.loads(value)
return z
except:
pass
with open("config.json", 'r') as config:
configs = json.load(config)
try:
os.system('cls')
except:
os.system('clear')
asciiart = f"""{Spy.rouge}
██╗ ██╗██████╗ ██████╗ ████████╗
██║ ██║██╔══██╗██╔═══██╗╚══██╔══╝
██║ ██║██████╔╝██║ ██║ ██║
╚██╗ ██╔╝██╔══██╗██║ ██║ ██║
╚████╔╝ ██████╔╝╚██████╔╝ ██║
╚═══╝ ╚═════╝ ╚═════╝ ╚═╝
Thx 2$py#5430 \n\n"""
print(asciiart + "\n\n")
posting = []
sys.stdout.write("\x1b]2;Vinted Bot\x07")
class moniteur:
def __init__(self, weburl, url):
while True:
try:
z = search(str(url))
x = z["items"]["catalogItems"]["byId"]
dictlist = list(x)
for i in range(9, -1, -1):
time.sleep(1)
post = dictlist[i - 1]
if str(post) in posting:
print(f"{Spy.blanc}[{Spy.rouge}{post}{Spy.blanc}] - Item déjà envoyé !")
time.sleep(1)
continue
else:
print(f"{Spy.blanc}[{Spy.vert}{post}{Spy.blanc}] - Nouvel item trouvé !")
info = get_info_post(x[str(post)]["url"])
data = {"username": "Turtle Minor v2",
"avatar_url": "https://i.ibb.co/pWsdXYq/pp-turle-free.png",
"embeds": [
{
"description": f"{configs['embed-color-text']}",
"title": f"**__{x[post]['title']}__**",
"color": configs["embed-color"],
"url": x[post]['url'],
"fields": [
],
"image": {
"url": x[post]["photo"]["thumbnails"][4]["url"]
},
"footer": {
"text": f"https://www.paypal.com/paypalme/daviddurand29",
"icon_url": "https://i.ibb.co/pWsdXYq/pp-turle-free.png"
}
}]}
if configs["embed-config"]["prix"] == "oui":
data["embeds"][0]["fields"].append(
{
"name": "**💵 Prix**",
"value": f"{configs['embed-color-text']}\n{x[post]['price']}€",
"inline": True
})
if configs["embed-config"]["taille"] == "oui":
if x[post]['size_title'] == "":
size_title = "Pas de donnée"
else:
size_title = x[post]['size_title']
data["embeds"][0]["fields"].append({
"name": "**📏 Taille**",
"value": f"{configs['embed-color-text']}\n{size_title}",
"inline": True
})
if configs["embed-config"]["marque"] == "oui":
data["embeds"][0]["fields"].append(
{
"name": "**🔖 Marque**",
"value": f"{configs['embed-color-text']}\n{x[post]['brand_title']}",
"inline": True
}
)
if configs["embed-config"]["avis"] == "oui":
data["embeds"][0]["fields"].append({
"name": "👍/👎 **Note du vendeur**",
"value": f"{configs['embed-color-text']}\n{str(info['positive'])} - {str(info['negative'])}",
"inline": True
})
if configs["embed-config"]["localisation"] == "oui":
data["embeds"][0]["fields"].append({
"name": "**📍 Emplacement **",
"value": f"{configs['embed-color-text']}\n{info['pays']}, {info['ville']}",
"inline": True
})
if configs["embed-config"]["vendeur"] == "oui":
data["embeds"][0]["fields"].append({
"name": "**👥 Vendeur**",
"value": f"{configs['embed-color-text']}\n{info['username']}",
"inline": True
})
result = requests.post(weburl, json=data)
if 429 == result.status_code:
print(f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Rate Limit !")
time.sleep(60)
else:
posting.append(str(post))
print(f"{Spy.blanc}[{Spy.bleu}POSTE{Spy.blanc}] - Poste envoyé !")
except:
time.sleep(10)
if len(configs["suburl"]) > 10:
print(
f"{Spy.blanc}[{Spy.rouge}ERREUR{Spy.blanc}] - Trop de salon veuillez en enlever car le bot se fera rate limit !")
else:
for webhurl in configs["suburl"]:
print(
f"{Spy.blanc}[{Spy.violet}LANCEMENT{Spy.blanc}] - Lancement de la tâche dans le salon {Spy.jaune}{configs['suburl'][webhurl]['salon']}")
t = threading.Thread(target=moniteur, args=[webhurl, configs["suburl"][str(webhurl)]["url"]])
t.start()
First, I checked if some scripts didn't work on Linux but everything was in order. Then I noticed that my VPS was located in Germany so I couldn't search on the French Vinted. So I decided to add proxies. However, the bot still didn't work. I then wrote a script that returned the response from Vinted saying that I didn't have permission to access vinted.fr, even though I could access vinted.com
Related
i have list and in list i have 10 city i want to use all of my city to the celery beat but i cant
cityList = ["Tehran", "Shiraz", "Mashhad", "Qom", "Isfahan", "Ardabil", "Hamedan", "Yazd", "Tabriz", "Zavareh"]
app.conf.beat_schedule = {
'call_show_every_one_minute': {
"task": 'ali111.get_weather',
'schedule': crontab(minute='*/1'),
'args': ([cityList], ),
}
}
#app.task()
def get_weather(city):
con = redis.StrictRedis(host='localhost',port=6380, db=0, decode_responses=True)
appid = "b3bf68fdfc6ba46923cd50cb8b9a79c3"
URL = 'https://api.openweathermap.org/data/2.5/weather'
temp = con.get(city)
if temp is not None:
return temp
try:
PARAMS = {'q' :city, 'appid' :appid}
r = requests.get(url=URL, params=PARAMS)
city_temp = (r.json()['main']['temp']) - 273.15
my_temp = f"{round(city_temp,1)}c"
con.set(city, my_temp, ex=60)
return my_temp
except ConnectionError:
return "not internet connection"
I'm trying to see the value of a key in a nested dictionary consuming an api. The code below iterate each pokemon in a the list of all pokemons and print some of its attributes. I know how to print the value of a single key like height, id, name or base_experience but how can I print the name of each ability?? for example:
JSON CODE
"id": 1,
"height": 7,
"name": "bulbasur",
"abilities": [
{
"ability": {
"name": "overgrow",
"url": "https://pokeapi.co/api/v2/ability/65/"
},
"is_hidden": false,
"slot": 1
},
{
"ability": {
"name": "chlorophyll",
"url": "https://pokeapi.co/api/v2/ability/34/"
},
"is_hidden": true,
"slot": 3
}
]
PYTHON CODE
import requests
def print_restul(json_respnse):
pokemones = json_respnse.get("results")
for pokemon in pokemones:
# print(pokemon)
explore_pokemon(pokemon)
def explore_pokemon(pokemon):
url_pokemon = pokemon.get("url")
r = requests.get(url_pokemon)
json_respnse = r.json()
# print(json_respnse.keys())
print("el id del pokemon {} es {}, y la altura es de {}".format(pokemon.get("name"),json_respnse.get("id"),json_respnse.get("height"),))
if __name__ == "__main__":
url = 'http://pokeapi.co/api/v2/pokemon'
r = requests.get(url)
json_respnse = r.json()
print_restul(json_respnse)
for _ in range(10):
print("== nuevo ciclo for === ")
url_next = json_respnse.get("next")
r = requests.get(url_next)
json_respnse = r.json()
print_restul(json_respnse)
Loop over the abilities list and get all the names.
def explore_pokemon(pokemon):
url_pokemon = pokemon.get("url")
r = requests.get(url_pokemon)
json_respnse = r.json()
# print(json_respnse.keys())
print("el id del pokemon {} es {}, y la altura es de {}".format(pokemon.get("name"),json_respnse.get("id"),json_respnse.get("height"),))
abilities = ", ".join(a['ability']['name'] for a in json_response.get("abilities", []))
print(f'abilities: {abilities}')
In long run it might be a better idea to model your data as dataclasses instead of python dict objects. It will also promote code quality and code reuse which I feel is a good idea in general.
To work with nested dataclass model and easily generate it from json, I suggest using wiz command-line from dataclass-wizard. This library can be install with pip install dataclass-wizard and then you have access to wiz utility from a terminal to generate a nested dataclass schema.
In this scenario, your code now becomes like below:
def explore_pokemon(pokemon):
url_pokemon = pokemon.get("url")
r = requests.get(url_pokemon)
json_respnse = Data.from_dict(r.json())
# print(json_respnse.keys())
print("el id del pokemon {} es {}, y la altura es de {}".format(pokemon.get("name"),json_respnse.id,json_respnse.height),)
abilities = ", ".join(a.ability.name for a in json_respnse.abilities)
print(f'abilities: {abilities}')
I have recently uploaded contact records to HubSpot using Postman. Here is a raw JSON data example and POST method that I use to successfully upload a contact:
https://api.hubapi.com/crm/v3/objects/contacts?hapikey={{hapikey}}
{properties": {
"smbapi": "yes",
"email": "fcgrinding#junkstermail.com",
"business_name":"Forest City Grinding Inc",
"srvc_address_1":"3544 Stenstrom Rd",
"srvc_city_1":"",
"srvc_state_1":"IL",
"srvc_zip_1":"61109",
"proposal_date":"2021-12-07",
"proposal_start_date": "2022-01-01",
"udc_code_1": "COMED",
"eog":"electric",
"fixedprice_1_gas_mcf": 6.63,
"fixedprice_2_gas_mcf": 6.11,
"fixedprice_3_gas_mcf": 5.9,
"term_1": 12,
"term_2": 24,
"term_3": 36,
"smb_bdm_name": "Timothy Chin",
"smb_bdm_phone": "833-999-9999",
"smb_bdm_email": "tim.chin#junkstermail.com"
}
}
Next, I then created a python lambda function to automate this process because we want to ingest CSV files that may have many records to extract. So, I had constructed the dictionary to look the same as the string above which had worked great/fine with Postman. However, when I try and do a Post method API call to HubSpot, using my dictionary payload, I am getting this error:
Invalid input JSON : Cannot build ObjectSchemaEgg, Invalid input JSON
on line 1, column 2: Cannot build ObjectSchemaEgg, some of required
attributes are not set [name, labels]
Here is the processed dictionary string that my code constructed for the API call:
{'properties': '{"smbapi": "yes", "business_name": "Forest City Grinding Inc", "srvc_address_1": "4844 Stenstrom Rd", "srvc_state_1": "IL", "srvc_zip_1": "61109", "proposal_date": "2021-12-07", "proposal_start_date": "2022-01-01", "udc_code_1": "COMED", "fixedprice_1": "6.63", "fixedprice_2": "6.11", "fixedprice_3": "5.9", "term_2": "24", "term_3": "36", "smb_bdm_name": "Gary Wysong", "smb_bdm_phone": "833-389-0881", "smb_bdm_email": "gary.wysong#constellation.com"}'}
Here is my Lambda code in full (give special attention to both the call to post_to_hubspot() and also the post_to_hubspot() function itself). The code that loads the dynamo table is working correctly.:
import boto3
import json
import decimal
from botocore.exceptions import ClientError
from boto3.dynamodb.conditions import Key, Attr
import re
import pandas as pd
import numpy as np
import os
import datetime
from os import urandom
import email
import base64
import requests
from datetime import datetime, timedelta, timezone
import mailparser
import calendar
global payload_data
landing_zone_bucket_name = str(os.environ['BUCKETNAME'])
s3 = boto3.resource('s3')
landing_zone_bucket = s3.Bucket(landing_zone_bucket_name )
s3r = boto3.client('s3')
dynamodb = boto3.resource('dynamodb', region_name='us-west-2')
table = dynamodb.Table(str(os.environ['DYNAMOTABLE']))
unprocessed_records_table = dynamodb.Table(str(os.environ['UNPROCESSEDTABLE']))
email_table = dynamodb.Table(str(os.environ['EMAILSTATUSTABLE']))
endpoint_url=os.environ['ENDPOINT_URL']
access_key = os.environ['ACCESSKEY']
now = datetime.now()
today_date = datetime.strftime(now,'%d')
today_month = datetime.strftime(now,'%m')
today_year = datetime.strftime(now,'%Y')
time_stamp = datetime.now().strftime('%Y%m%d%H%M%S')
payload_data = {}
#WRITE RECORDS TO DYNAMO
def dynamoPut(dObj,table_name=None):
try:
for each in list(dObj['Information']):
if dObj['Information'][each]:
dObj['Information'][each] = str(dObj['Information'][each])
else:
del dObj['Information'][each]
dObj['Month'] = today_month
dObj['Year'] = today_year
dObj['Day'] = today_date
for each in list(dObj):
if dObj[each] != '':
dObj[each] = dObj[each]
else:
del dObj[each]
if table_name != None:
response = unprocessed_records_table.put_item(Item = dObj)
else:
response = table.put_item(Item = dObj)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
return True
else:
return False
except Exception as e:
print(e)
return False
def dynamoPutFileName(filename,source_type):
try:
dObj = {}
dObj['id'] = urandom(20).hex()
dObj['CreatedAt'] = str(datetime.now())
dObj['FileName'] = filename
dObj['Type'] = source_type
dObj['EmailSent'] = False
response = email_table.put_item(Item = dObj)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
return True
else:
return False
except Exception as e:
print(e)
return False
def parse_csv_hubspot(event, obj):
#parsing CSV file to write to dynamo
try:
def auto_truncate(val):
return val[:255 ]
print('<< IN PARSE CSV HUBSPOT >>')
print(event)
csv = pd.read_csv(obj['Body'], encoding = "ISO-8859-1")
csv_nn = csv.replace(np.nan, 'null', regex=True)
d = csv_nn.to_dict(orient='records')
source_id = urandom(20).hex()
file_name = event['file_path'].split('/')[-1]
print('<< FILE NAME >>', file_name)
for each in d:
try:
dbObj = {}
#PASSING THE EXTERNAL KEY
UniqueKey = ''
if 'smbapi' in each and each['smbapi'] != 'null':
dbObj['smbapi' ] = each['smbapi']
print('<< SMB API>>', dbObj['smbapi' ])
if 'business_name' in each and each['business_name'] != 'null':
dbObj['business_name'] = each['business_name']
print('<< BUSINESS NAME >>', dbObj['business_name'])
if 'srvc_address_1' in each and each['srvc_address_1'] != 'null':
dbObj['srvc_address_1'] = each['srvc_address_1']
print('<< ADDRESS 1 >>', dbObj['srvc_address_1'])
if 'srvc_city_1' in each and each['srvc_city_1'] != 'null':
dbObj['srvc_city_1'] = each['srvc_city_1']
if 'srvc_state_1' in each and each['srvc_state_1'] != 'null':
dbObj['srvc_state_1'] = each['srvc_state_1']
if 'srvc_zip_1' in each and each['srvc_zip_1'] != 'null':
dbObj['srvc_zip_1']= str(each['srvc_zip_1']).zfill(5)
if 'proposal_date' in each and each['proposal_date'] != 'null':
dbObj['proposal_date']= try_parsing_date(each['proposal_date']).date().isoformat()
if 'proposal_start_date' in each and each['proposal_start_date'] != 'null':
dbObj['proposal_start_date']= try_parsing_date(each['proposal_start_date']).date().isoformat()
if 'udc_code_1' in each and each['udc_code_1'] != 'null':
dbObj['udc_code_1']= each['udc_code_1']
if 'eog' in each and each['eog'] != 'null':
dbObj['eog']= each['eog']
if 'fixedprice_1' in each and each['fixedprice_1'] != 'null':
dbObj['fixedprice_1']= each['fixedprice_1']
if 'fixedprice_2' in each and each['fixedprice_2'] != 'null':
dbObj['fixedprice_2']= each['fixedprice_2']
if 'fixedprice_3' in each and each['fixedprice_3'] != 'null':
dbObj['fixedprice_3']= each['fixedprice_3']
if 'fixedprice_1_gas_therm' in each and each['fixedprice_1_gas_therm'] != 'null':
dbObj['fixedprice_1_gas_therm']= each['fixedprice_1_gas_therm']
if 'fixedprice_2_gas_therm' in each and each['fixedprice_2_gas_therm'] != 'null':
dbObj['fixedprice_2_gas_therm']= each['fixedprice_2_gas_therm']
if 'fixedprice_3_gas_therm' in each and each['fixedprice_3_gas_therm'] != 'null':
dbObj['fixedprice_3_gas_therm']= each['fixedprice_3_gas_therm']
if 'fixedprice_1_gas_ccf' in each and each['fixedprice_1_gas_ccf'] != 'null':
dbObj['fixedprice_1_gas_ccf']= each['fixedprice_1_gas_ccf']
if 'fixedprice_2_gas_ccf' in each and each['fixedprice_2_gas_ccf'] != 'null':
dbObj['fixedprice_2_gas_ccf']= each['fixedprice_2_gas_ccf']
if 'fixedprice_3_gas_ccf' in each and each['fixedprice_3_gas_ccf'] != 'null':
dbObj['fixedprice_3_gas_ccf']= each['fixedprice_3_gas_ccf']
if 'fixedprice_1_gas_dth' in each and each['fixedprice_1_gas_dth'] != 'null':
dbObj['fixedprice_1_gas_dth']= each['fixedprice_1_gas_dth']
if 'fixedprice_2_gas_dth' in each and each['fixedprice_2_gas_dth'] != 'null':
dbObj['fixedprice_2_gas_dth']= each['fixedprice_2_gas_dth']
if 'fixedprice_3_gas_dth' in each and each['fixedprice_3_gas_dth'] != 'null':
dbObj['fixedprice_3_gas_dth']= each['fixedprice_3_gas_dth']
if 'fixedprice_1_gas_mcf' in each and each['fixedprice_1_gas_mcf'] != 'null':
dbObj['fixedprice_1_gas_mcf']= each['fixedprice_1_gas_mcf']
if 'fixedprice_2_gas_mcf' in each and each['fixedprice_2_gas_mcf'] != 'null':
dbObj['fixedprice_2_gas_mcf']= each['fixedprice_2_gas_mcf']
if 'fixedprice_3_gas_mcf' in each and each['fixedprice_3_gas_mcf'] != 'null':
dbObj['fixedprice_3_gas_mcf']= each['fixedprice_3_gas_mcf']
if 'term_1' in each and each['term_1'] != 'null':
dbObj['term_1']= each['term_1']
if 'term_2' in each and each['term_2'] != 'null':
dbObj['term_2']= each['term_2']
if 'term_3' in each and each['term_3'] != 'null':
dbObj['term_3']= each['term_3']
if 'smb_bdm_name' in each and each['smb_bdm_name'] != 'null':
dbObj['smb_bdm_name']= each['smb_bdm_name']
if 'smb_bdm_phone' in each and each['smb_bdm_phone'] != 'null':
if '.' in str(each['smb_bdm_phone']):
dbObj['smb_bdm_phone']= str(int(float(each['smb_bdm_phone'])))
else:
dbObj['smb_bdm_phone']= str(each['smb_bdm_phone'])
if 'smb_bdm_email' in each and each['smb_bdm_email'] != 'null' and each['smb_bdm_email'].strip() != '' and each['smb_bdm_email'] != None:
dbObj['smb_bdm_email']= each['smb_bdm_email']
print('<< OBJ >> ',dbObj)
N = urandom(20).hex()
now = str(datetime.now())
#<< END of HUBSPOT INGESTION >>
# table.put_item(
Item = {
'CogId' : str(N),
'CreatedAt': now,
'ExternalId': UniqueKey,
'Information' : dbObj,
'SourceBucket': landing_zone_bucket_name,
'SourcePath' : event['file_path'],
'Source' : 'HubSpot',
'SourceId' : source_id,
'SourceFileName': time_stamp + '_' + file_name
}
#WRITE-TO-DYNAMO
files_processing = dynamoPut(Item)
if not files_processing:
print('Writing {} record to dynamodb Failed'.format(Item))
except Exception as e:
print(e)
N = urandom(20).hex()
Item = {
'CogId' : str(N),
'CreatedAt': now,
'Information' : each,
'SourceBucket': landing_zone_bucket_name,
'SourcePath' : event['file_path'],
'Source' : 'HubSpot',
'message': str(e),
'SourceId' : source_id,
'ExternalId': UniqueKey
}
files_processing = dynamoPut(Item,'Fail')
pass
temp_file_name = time_stamp + '_' + file_name
isert_file_name = dynamoPutFileName(temp_file_name,'HubSpot')
post_to_hubspot(dbObj)
return True
except Exception as e:
print(e)
new_folder_path = os.environ['CSV_NEW_FOLDER_HUBSPOT']
unprocessed_folder_path = os.environ['CSV_ERROR_FOLDER_HUBSPOT']
# MOVING PROCESSED FILES FROM NEW TO UNPROCESSED FOLDER
move_file_to_processed = moving_files_new_to_processed(event, new_folder_path,unprocessed_folder_path)
return False
def try_parsing_date(text):
for fmt in ('%m/%d/%Y','%Y-%m-%dT%H:%M:%S-%f', '%m/%d/%y', '%Y-%m-%d', '%m.%d.%Y','%Y-%m-%dT%I', '%Y-%m-%dT%I%p', '%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S.%f+','%Y-%m-%dT%H:%M:%S'):#2018-11-20T08:05:54-0500
try:
return datetime.strptime(text, fmt)
except ValueError:
print('in except')
pass
return ValueError('no valid date format found')
def post_to_hubspot(list_contacts):
print('<< IN POST-To-HUBSPOT >>')
data_string = **json.dumps(list_contacts)**
payload_data = {"properties": data_string}
print('<< dbOBJ LIST >> ',payload_data)
response = requests.request("POST", endpoint_url+access_key, headers={'Content-Type': 'application/json'}, data=payload_data)
token_response=json.loads(response.text)
print('<< TOKEN RESPONSE >>',token_response)
def moving_files_new_to_processed(event, new_folder,processed_folder):
#MOVING-FILES-TO-PROCESSED
try:
copy_source = {
'Bucket': landing_zone_bucket_name,
'Key': event['file_path']
}
path = event['file_path']
processed_folder = processed_folder + time_stamp + '_'
new_key = path.replace(new_folder, processed_folder)
new_obj = landing_zone_bucket.Object(new_key)
new_obj.copy(copy_source)
s3.Object(landing_zone_bucket_name, event['file_path']).delete()
return True
except Exception as e:
print(e)
return False
def lambda_handler(event,context):
print("Starting to Push Records to Dynamo Lambda")
print(event)
try:
parse_flag = False
new_folder_path = ''
processed_folder_path = ''
#Gets file path and calls required function to parse it out
key = str(os.environ['CSV_NEW_FOLDER_HUBSPOT'])
obj = s3r.get_object(Bucket=landing_zone_bucket_name, Key=event['file_path'])
print('after obj')
print(os.environ['CSV_NEW_FOLDER_HUBSPOT'])
print('in HubSpot parse_csv')
parse_csv_func = parse_csv_hubspot(event, obj)
# Checks if parse_csv return empty dictionary
if parse_csv_func:
parse_flag = True
new_folder_path = os.environ['CSV_NEW_FOLDER_HUBSPOT']
processed_folder_path = os.environ['CSV_PROCESSED_FOLDER_HUBSPOT']
else:
print('File Format not Supported for {}'.format(event['file_path']))
if parse_flag:
# UPLOADING CONTACT.MOVING PROCESSED FILES FROM NEW TO PROCESSED FOLDER
#print('<< PAYLOAD >> ',payload)
#response = requests.request("POST", "https://api.hubapi.com/crm/v3/schemas/?hapikey="+access_key, headers={'Content-Type': 'application/json'}, data=json.dumps(str(payload)))
#token_response=json.loads(response.text)
#print('<< TOKEN RESPONSE >>',token_response)
#MOVING PROCESSED FILES FROM NEW TO PROCESSED FOLDER
move_file_to_processed = moving_files_new_to_processed(event, new_folder_path,processed_folder_path)
if move_file_to_processed:
print('File {} moved Successfully from {} to {}'.format(event['file_path'],new_folder_path,processed_folder_path))
else:
print('Moving {} file from new to processing folder Failed'.format(event['file_path']))
except Exception as e:
print(e)
What could be the problem? Thanks for your help.
The problem was caused by two issues:
The dictionary should have been placed in json.dumps() to convert it to JSON string when doing a POST so the dictionary didn't need to change its structure. Here's the response from the POST:
<< TOKEN RESPONSE >> {
"id": "135120801",
"properties": {
"business_name": "Millers Brand Oats",
"createdate": "2021-12-21T02:31:12.452Z",
"fixedprice_1": "6.63",
"fixedprice_2": "6.11",
"fixedprice_3": "5.9",
"hs_all_contact_vids": "135120801",
"hs_is_contact": "true",
"hs_is_unworked": "true",
"hs_marketable_until_renewal": "false",
"hs_object_id": "135120801",
"hs_pipeline": "contacts-lifecycle-pipeline",
"lastmodifieddate": "2021-12-21T02:31:12.452Z",
"proposal_date": "2021-12-07",
"proposal_start_date": "2022-01-01",
"smb_bdm_email": "Tim.Chu#junkster.com",
"smb_bdm_name": "Tim Chu",
"smb_bdm_phone": "833-999-9999",
"smbapi": "yes",
"srvc_address_1": "4844 Stenstrom Rd",
"srvc_state_1": "IL",
"srvc_zip_1": "61109",
"term_2": "24",
"term_3": "36",
"udc_code_1": "COMED"
},
"createdAt": "2021-12-21T02:31:12.452Z",
"updatedAt": "2021-12-21T02:31:12.452Z",
"archived": false
}
I was using the wrong endpoint:
https://api.hubapi.com/crm/v3/schemas/
instead of:
https://api.hubapi.com/crm/v3/objects/contacts/
Now I just need to find out why the AWS Lambda POSTs allow for duplicate contacts to be created in HubSpot while Postman POSTs prohibit duplicate contacts to be created.
Is there any way if the status response code of CHANNEL_ID_1comes 200, then it should not send the message again in the same channel.
USERS_TOKEN = "TOKEN"
CHANNEL_ID_1 = "CHANNELID HERE"
CHANNEL_ID_2 = "CHANNELID HERE"
CHANNEL_ID_3 = "CHANNELID HERE"
CHANNEL_ID_4 = "CHANNELID HERE"
MESSAGE_1 = "msg 1 channel 1"
MESSAGE_2 = "msg 2 channel 2"
MESSAGE_3 = "msg 3 channel 3"
MESSAGE_4 = "msg 4 channel 4"
msgcount = "10"
def sendMessage(token, channel_id, message):
url = 'https://discord.com/api/v9/channels/{}/messages'.format(channel_id)
data = {"content": message}
header = {"authorization": token}
r = requests.post(url, data=data, headers=header)
print(r.status_code)
for i in range(int(msgcount)):
time.sleep(0.3)
sendMessage(USERS_TOKEN, CHANNEL_ID_1, MESSAGE_1)
sendMessage(USERS_TOKEN, CHANNEL_ID_2, MESSAGE_2)
sendMessage(USERS_TOKEN, CHANNEL_ID_3, MESSAGE_3)
sendMessage(USERS_TOKEN, CHANNEL_ID_4, MESSAGE_4)
plz modify that code its really helpful for me
If I understand the question correctly, I would suggest this code structure.
[ Note: This is untested ]
import requests
import time
USERS_TOKEN = "TOKEN"
MDICT = {"CHANNEL_1": "message_1",
"CHANNEL_2": "message_2",
"CHANNEL_3": "message_3",
"CHANNEL_4": "message_4"
}
msgcount = 10
def sendMessage(token, channel_id, message):
url = f'https://discord.com/api/v9/channels/{channel_id}/messages'
data = {"content": message}
header = {"authorization": token}
with requests.Session() as session:
return session.post(url, data=data, headers=header).status_code
while msgcount > 0 and MDICT:
time.sleep(0.3)
key = next(iter(MDICT.keys()))
if sendMessage(USERS_TOKEN, key, MDICT[key]) == 200:
del MDICT[key]
msgcount -= 1
i am getting this error when trying to access a key value named 'valor' in from various JSON's that i am automatically generating from a website. I get different JSON's every iteration of for example the month code in its cycle. But the JSON's format, i.e. keys names, should be the same every JSON.
My code is:
year_codes = ["2011", "2012", "2013", "2014", "2015", "2016", "2017", "2018", "2019", "2020"]
month_codes = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
city_codes = ["1111609", "1120303", "11A1312", "11D1714", "11E0402", "16D0105", "16E0603", "16F1009", "16G1823",
"16H0502", "16J0907", "1701106", "1701512", "1840205", "1851416", "1861214", "1870705", "1500805"]
valor_anual = [0]
for city_code in city_codes:
for year_code in year_codes:
valor_mensal_aux = []
for month_code in month_codes:
url_imob = Request(
"https://www.ine.pt/ine/json_indicador/pindica.jsp?op=2&varcd=0010042&Dim1=S3A" + year_code + \
month_code + "&Dim2=" + city_code + "&Dim3=T&lang=PT", headers={'User-Agent': 'XYZ/3.0'})
json_date = year_code + month_code
response = urlopen(url_imob)
data = json.loads(response.read())
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
city_name = data[0]['Dados'][json_date][0]['geodsg']
dua = data[0]['DataUltimoAtualizacao']
valor_anual = statistics.mean(valor_mensal_aux)
python_imob = Imob(int(year_code), valor_anual, dua, city_name)
session.add(python_imob)
session.commit()
session.close()
imobs = session.query(Imob).all()
and the JSON's look like is:
[ {
"IndicadorCod" : "0010042",
"IndicadorDsg" : "Valor mediano de avaliação bancária (€/ m²) por Localização geográfica (Município - 2013) e Tipo de construção; Mensal - INE, Inquérito à avaliação bancária na habitação",
"MetaInfUrl" : "https://www.ine.pt/bddXplorer/htdocs/minfo.jsp?var_cd=0010042&lingua=PT",
"DataExtracao" : "2020-06-29T15:55:51.640+01:00",
"DataUltimoAtualizacao" : "2020-06-29",
"UltimoPref" : "Maio de 2020",
"Dados" : {
"202005" : [ {
"geocod" : "1701106",
"geodsg" : "Lisboa",
"dim_3" : "T",
"dim_3_t" : "Total",
"valor" : "3084"
} ]
}
} ]
The KeyError: 'valor' is in this line:
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
Maybe the json_date is not in dados, let try
valor_mensal_aux.append(int(data[0]['Dados'][json_date][0]['valor']))
To be
dados_value = data[0]['Dados'].get(json_date, {}) ## get dados_value first
if dados_value and hasattr(dados_value[0], 'valor'): ## if dados_value has valor then append
valor_mensal_aux_value = dados_value and dados_value[0] and dados_value[0].get('valor') or None
if valor_mensal_aux_value:
valor_mensal_aux.append(int(valor_mensal_aux_value))
else: continue
else:
print(json_date, dados_value) # json_date with emply dados_value
continue ## this will skip the record does not have dados_value
More compact version
dados_value = data[0]['Dados'].get(json_date, {})
valor_mensal_aux_value = dados_value and dados_value[0] and dados_value[0].get('valor') or None
if valor_mensal_aux_value:
valor_mensal_aux.append(int(valor_mensal_aux_value))
else:
continue