Related
The functionality of this script to shutdown GCP VM's based on some logic. Currently we are trying shut it down at night, but the scripts are failing before it could shutdown the VM. We shutdown based on time. At evenings and night the VM are shutdown using these scripts
Script to turn off instances overnight
Environment: Runs in Google Cloud Functions (Python3.7)
import datetime
import json
from pprint import pformat
import pytz
import re
import modules.common.cfcommon as cfcommon
import modules.utilities.dateutilities as dateutilities
from modules.compute.instances import InstanceList, Instance
from modules.compute.compute_service import ComputeServiceContext
from modules.utilities.printutilities import print_message, debug_message
from modules.pubsub.topic import PublishMessage
from modules.common.labels import VMAUTOSHUTDOWN_LABEL, VMAUTOSHUTDOWN_DEFER_LABEL, ShutdownDeferLabelValueIsValid, ShutdownLabelValueIsValid
from templates.renderer import render_template
# Takes a list in the following format and checks if the 'Instance' object is within it
# list must contain dictionaries in the following format:
# {"name": "instancename", "zone": "zonename"}
# Example: {"name": "test-01", "zone": "us-east4-c"}
#
# Parameters:
# inputList - list of dictionary objects
# instance - Instance object
def isInstanceInList(inputList, instance):
if not isinstance(inputList, list):
raise TypeError("Provided inputList is not a list")
if not isinstance(instance, Instance):
raise TypeError("Provided instance is not of type 'Instance'")
# Iterate over every item in inputList and check if the name and zone match
for cItem in inputList:
if cItem["name"].lower() == instance.properties["name"].lower() and cItem["zone"].lower() == instance.GetShortZoneName().lower():
return True
# No match found
return False
# Takes a list of Instance objects and sees if their shutdown timezone is within the graceperiod of the shutdownHour
#
# Example: is shutdown hour is 23 and the gracePeriodMin is 15 then if the function is called at 23:12, the instance will be included in the shutdown list
#
# Parameters:
# instanceList - List of Instance objects
# shutdownHour - number (0-23) 0 = Midnight, 23 = 11PM
# gracePeriodMin - number
def getInstancesToStop(instanceList, gracePeriodMin):
instancesToStop = []
debug_message("Entering getInstancesToStop")
for cInstance in instanceList:
debug_message("Instance: %s (ID: %s, Zone: %s, Project: %s)" % (cInstance.GetName(), cInstance.GetId(), cInstance.GetShortZoneName(), cInstance.project))
labels = cInstance.GetLabels()
if VMAUTOSHUTDOWN_LABEL in labels.keys():
labelValue = labels.get(VMAUTOSHUTDOWN_LABEL, '')
pattern = '\d\d-\d\d-\d\d'
match = re.match(pattern, labelValue)
if not match or not ShutdownLabelValueIsValid(labelValue):
debug_message(f'Label {labelValue} does not match the correct format')
instancesToStop.append(cInstance)
continue
else:
debug_message(f'Label {VMAUTOSHUTDOWN_LABEL} not found. Adding to shutdown list')
instancesToStop.append(cInstance)
continue
shutdown_deferred_utc_datetime = None
if VMAUTOSHUTDOWN_DEFER_LABEL in labels.keys():
labelValue = labels.get(VMAUTOSHUTDOWN_DEFER_LABEL, '')
pattern = '\d\d\d\d-\d\d-\d\dt\d\d-\d\d-\d\d'
match = re.match(pattern, labelValue)
if match and ShutdownDeferLabelValueIsValid(labelValue):
shutdown_deferred_utc_date, shutdown_deferred_utc_time = labelValue.split('t')
year, month, day = shutdown_deferred_utc_date.split('-')
hour, minute, second = shutdown_deferred_utc_time.split('-')
shutdown_deferred_utc_datetime = datetime.datetime.now(pytz.timezone('GMT')).replace(
year=int(year), month=int(month), day=int(day), hour=int(hour), minute=int(minute), second=int(second)
)
else:
debug_message(f'Label {labels[VMAUTOSHUTDOWN_DEFER_LABEL]} does not match the correct format')
instancesToStop.append(cInstance)
continue
current_utc_time = dateutilities.get_current_datetime()
# If defer date is in the future, and not in grace window time, skip shutting down
if shutdown_deferred_utc_datetime is not None and shutdown_deferred_utc_datetime > current_utc_time:
debug_message(f'Instance {cInstance.GetName()} shutdown deferred until after {labels[VMAUTOSHUTDOWN_DEFER_LABEL]}')
continue
# If defer time is in past, continue with the vm hour shutdown
shutdown_utc_hour = labels[VMAUTOSHUTDOWN_LABEL].split('-')[0]
# Convert shutdown UTC hour into datetime object
shutdown_utc_time = datetime.datetime.now(pytz.timezone('GMT')).replace(hour=int(shutdown_utc_hour), minute=0, second=0)
shutdown_utc_grace_time = shutdown_utc_time + datetime.timedelta(minutes=gracePeriodMin)
debug_message(f"Shutdown UTC time {shutdown_utc_time}")
debug_message(f"Shutdown UTC grace time {shutdown_utc_grace_time}")
# Check if shutdown is within time window
if current_utc_time >= shutdown_utc_time and current_utc_time <= shutdown_utc_grace_time:
debug_message("We're in the time window")
instancesToStop.append(cInstance)
else:
debug_message("We're outside the time window. Not adding to stop list")
return instancesToStop
# This is the main entry point that cloud functions calls
def AutoStopVMInstances(config, policy=None, payload=None, generate_local_report=False):
FUNCTION_NAME = "AutoStopVMInstances"
# Populated by config later...
QUERY_PROJECT_IDS = None # List of project IDs
INSTANCE_WHITELIST = None # List of dictionaries in format {"name": "instancename", "zone": "zonename", "project": "projectid"}
PREVIEW_MODE = True
SHUTDOWN_GRACEPERIOD_MIN = 30
# Start
startTime = datetime.datetime.now()
print_message("Started %s within Cloud Function %s [%s]" % (FUNCTION_NAME, cfcommon.CLOUD_FUNCTION_NAME, startTime))
debug_message("")
# For ease of access, assign from config values
debug_message("Processing Configuration...")
QUERY_PROJECT_IDS = config.get("QueryProjectIDs", []) # Required field
INSTANCE_WHITELIST = config.get("InstanceWhiteList", []) # Optional Field
PREVIEW_MODE = config.get("PreviewMode", True) # Required field
SHUTDOWN_GRACEPERIOD_MIN = config.get("ShutdownGracePeriodMin", None) # Required field
SKIP_INSTANCE_GROUPS = config.get("SkipInstanceGroups", False) # Optional
EMAIL_PUB_SUB_PROJECT = config.get("EmailPubSubProject", None) # Optional
EMAIL_PUB_SUB_TOPIC = config.get("EmailPubSubTopic", None) # Optional
EMAIL_TO = config.get("EmailTo", []) # Optional
EMAIL_CC = config.get("EmailCC", None) # Optional
EMAIL_BCC = config.get("EmailBCC", None) # Optional
EMAIL_FROM = config.get("EmailFrom", "noreply-ei-cs-cloudops-resource-administration#ei-cs-cloudops.local") # Optional
EMAIL_SUBJECT = config.get("EmailSubject", "Nightly VM Instance Shutdown Summary") # Optional
cfLogger = cfcommon.CloudFunctionLog()
# Validate whitelist
if INSTANCE_WHITELIST is None:
raise Exception("Unable to get whitelist")
debug_message("Whitelist loaded:")
debug_message(pformat((INSTANCE_WHITELIST)))
# Re-init Compute service - execution environment in cloud functions can be shared among each other. Let's re-init our connection every execution.
ComputeServiceContext.InitComputeService()
# Build the service object.
allRunningInstances = []
for cProjectId in QUERY_PROJECT_IDS:
debug_message("Checking Project: %s" % (cProjectId))
# Main Loop - Let's get and analyze all instances from our project
# Paginated within the 'request' object
runningInstances = []
allInstances = []
debug_message("Building Instance List...", end="")
instances = InstanceList(cProjectId)
instances.PopulateInstances()
debug_message("Done")
for cInstance in instances.GetAllInstances():
debug_message("Found Instance %s in %s [%s - %s]" % (cInstance.GetName(), cInstance.GetZone(), cInstance.GetId(), cInstance.GetStatus()))
# Check if whitelisted. If it is, skip it
if isInstanceInList(INSTANCE_WHITELIST, cInstance):
debug_message(" Instance is whitelisted. Skipping.")
continue
# Check if we should skip instance groups
if SKIP_INSTANCE_GROUPS and cInstance.IsWithinInstanceGroup():
debug_message(" Instance is within an instance group. Skipping.")
continue
debug_message(" Is Running: %s" % (cInstance.IsRunning()))
owner = cInstance.GetOwner()
if owner in ("devops", "ei devops", "eicsdevopseng"):
debug_message("Skipping instance owned by devops")
continue
# # TODO: FOR USE WHEN TESTING
# if VMAUTOSHUTDOWN_LABEL not in labels.keys():
# continue
# Keep track of this instance
allInstances.append(cInstance)
# If it's running, it's a candidate to stop
if cInstance.IsRunning():
runningInstances.append(cInstance)
# Handle no instances found
if len(allInstances) == 0:
debug_message("INFO: No Instances found.")
# Summarize for user
debug_message("")
if len(runningInstances) > 0:
debug_message("Found %s/%s non-whitelisted instances are running (project: %s)" % (len(runningInstances), len(allInstances), cProjectId))
else:
debug_message("All %s non-whitelisted instances are good (project: %s)" % (len(allInstances), cProjectId))
# Main loop to stop
debug_message("")
allRunningInstances = allRunningInstances + runningInstances
instancesToBeStopped = getInstancesToStop(allRunningInstances, SHUTDOWN_GRACEPERIOD_MIN)
stoppedCount = 0
instanceSummary = []
if len(instancesToBeStopped) == 0:
print_message("No instances are due to be stopped")
else:
for cInstance in instancesToBeStopped:
summaryEntry = {
"Name": cInstance.GetName(),
"ID": cInstance.GetId(),
"Zone": cInstance.GetShortZoneName(),
"Project": cInstance.GetProject(),
"Preview": PREVIEW_MODE,
"Stopped": False,
"InstanceLink": cInstance.GetSelfLinkToConsole()
}
logMessage = "Stopping Instance: {name} (ID: {id}, Zone: {zone}, Project: {project})".format(
name=summaryEntry.get("Name"),
id=summaryEntry.get("ID"),
zone=summaryEntry.get("Zone"),
project=summaryEntry.get("Project")
)
if PREVIEW_MODE:
print_message("(PREVIEW) " + logMessage )
else:
print_message(logMessage)
cInstance.Stop()
summaryEntry["Stopped"] = True
stoppedCount += 1
instanceSummary.append(summaryEntry)
if EMAIL_PUB_SUB_PROJECT is not None and EMAIL_PUB_SUB_TOPIC is not None:
debug_message("It looks like we have an email config. Attempting to send email")
emailBody = render_template(
'shutdown_report',
instance_summary=instanceSummary,
config=json.dumps(config, indent=4, sort_keys=True),
preview_mode=PREVIEW_MODE,
generation_time=datetime.datetime.now().astimezone(pytz.utc)
)
emailPayload = {
"To": EMAIL_TO,
"From": EMAIL_FROM,
"Subject": EMAIL_SUBJECT,
"BodyHtml": emailBody
}
if EMAIL_CC is not None:
emailPayload["CC"] = EMAIL_CC
if EMAIL_BCC is not None:
emailPayload["BCC"] = EMAIL_BCC
if not generate_local_report:
print_message("Sending email...", end="")
PublishMessage(EMAIL_PUB_SUB_PROJECT, EMAIL_PUB_SUB_TOPIC, json.dumps(emailPayload))
else:
print_message('Generating local HTML report')
with open('./html_reports/shutdown_report.html', 'w') as r:
r.write(emailBody)
print_message("Done")
# We want to log a nice structured json line to stackdriver for easy reporting.
cfLogger.log({
"StartTime": startTime.isoformat(),
"InstancesStoppedCount": stoppedCount,
"PreviewMode": PREVIEW_MODE,
"Instances": instanceSummary,
"Whitelist": INSTANCE_WHITELIST,
"EndTime": datetime.datetime.now().isoformat(),
"LogLine": "Summary"
})
print_message("DONE [%s]" % (datetime.datetime.now()))
The error I'm getting when trying to run a VM shutdown script:
Caught exception while running VMNightlyShutdown. Exception Text: Traceback (most recent call last):
File "/workspace/main.py", line 248, in StartCloudFunction
policy=policy_config)
File "/workspace/vm_nightly_shutdown.py", line 256, in AutoStopVMInstances
cInstance.Stop()
File "/workspace/modules/compute/instances.py", line 729, in Stop
stop_instance(self.project, self.GetShortZoneName(), self.GetName(), waitForCompletion=waitForCompletion)
File "/workspace/modules/compute/instances.py", line 45, in stop_instance
return wait_for_operation(project, zone, response["name"])
File "/workspace/modules/compute/instances.py", line 27, in wait_for_operation
operation=operation
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/googleapiclient/http.py", line 932, in execute
headers=self.headers,
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/googleapiclient/http.py", line 222, in _retry_request
raise exception
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/googleapiclient/http.py", line 191, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/google_auth_httplib2.py", line 225, in request
**kwargs
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/httplib2/__init__.py", line 1721, in request
conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/httplib2/__init__.py", line 1440, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/layers/google.python.pip/pip/lib/python3.7/site-packages/httplib2/__init__.py", line 1392, in _conn_request
response = conn.getresponse()
File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 1373, in getresponse
response.begin()
File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 319, in begin
version, status, reason = self._read_status()
File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 280, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/layers/google.python.runtime/python/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/layers/google.python.runtime/python/lib/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/layers/google.python.runtime/python/lib/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:2570)",
}
I am trying to retrieve emails > than 12/1/2020 but get a ValueError. I tried adding .replace(microseconds=0) to str(message.ReceivedTime) but still getting this error.
error
Traceback (most recent call last):
File "C:/Users/SLID/PycharmProjects/PPC_COASplitter/PPC_Ack.py", line 178, in <module>
get_url = readEmail()
File "C:/Users/zSLID/PycharmProjects/PPC_COASplitter/PPC_Ack.py", line 144, in readEmail
if str(message.ReceivedTime) >= '2020-12-1 00:00:00' and 'P1 Cust ID 111111 File ID' in message.Subject:
File "C:\Program Files (x86)\Python37-32\lib\site-packages\win32com\client\dynamic.py", line 516, in __getattr__
ret = self._oleobj_.Invoke(retEntry.dispid,0,invoke_type,1)
ValueError: microsecond must be in 0..999999
code
def readEmail():
outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
folder = outlook.Folders.Item('SharedMailbox, PPC-Investigation')
inbox = folder.Folders.Item('Inbox')
messages = inbox.Items
messages.Sort("[ReceivedTime]", True)
for message in messages:
if str(message.ReceivedTime) >= '2020-12-1 00:00:00' and 'P1 Cust ID 111111 File ID' in message.Subject:
print("")
print('Received from: ',message.Sender)
print("To: ", message.To)
print("Subject: ", message.Subject)
print("Received: ", message.ReceivedTime)
print("Message: ", message.Body)
get_url = re.findall(r'(https?://[^\s]+)', message.Body)[2].strip('>')
return get_url
Firstly, you are comparing DateTime value with a string. Convert the string to a date-time value.
Secondly, never loop through all items in a folder, use Items.Restrict instead (returns a restricted Items collection):
messages = messages.Restrict("[Received] >= '1/12/2020 0:00am' ")
Here's the code snippet:
# Get details from the user.
jira_url = str(raw_input("JIRA URL: ")) # https://jira.mydomain.com
jira_prj = str(raw_input("JIRA Project: ")) # MYPROJ
jira_uid = str(raw_input("JIRA ID: ")) # admin
jira_pas = str(getpass.getpass("JIRA Password: ")) # p#s$wOrd!
# JIRA stuff.
jira = JIRA(server=jira_url, basic_auth=(jira_uid, jira_pas))
search_alert_issues = jira.search_issues("project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)") % jira_prj
search_alert_issues
Objectives:
Connect to JIRA server.
Run the search_alert_issues.
Expectation:
>>> jira = JIRA(server='https://jira.mydomain.com', basic_auth=('admin', 'p#s$wOrd!'))
>>> search_alert_issues = jira.search_issues("project = MYPROJ AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)")
>>> search_alert_issues
[<JIRA Issue: key=u'MYPROJ-35460', id=u'129419'>, <JIRA Issue: key=u'MYPROJ-35459', id=u'129418'>, <JIRA Issue: key=u'MYPROJ-35458', id=u'129417'>, <JIRA Issue: key=u'MYPROJ-35457', id=u'129416'>, <JIRA Issue: key=u'MYPROJ-35456', id=u'129415'>, <JIRA Issue: key=u'MYPROJ-35455', id=u'129414'>, <JIRA Issue: key=u'MYPROJ-35454', id=u'129413'>, <JIRA Issue: key=u'MYPROJ-35453', id=u'129412'>, <JIRA Issue: key=u'MYPROJ-35452', id=u'129411'>, <JIRA Issue: key=u'MYPROJ-35451', id=u'129410'>]
>>>
Reality:
Traceback (most recent call last):
File "./ajat.py", line 82, in <module>
search_alert_issues = jira.search_issues("project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)") % jira_prj
File "/usr/lib/python2.7/site-packages/jira/client.py", line 1587, in search_issues
resource = self._get_json('search', params=search_params)
File "/usr/lib/python2.7/site-packages/jira/client.py", line 2033, in _get_json
r = self._session.get(url, params=params)
File "/usr/lib/python2.7/site-packages/jira/resilientsession.py", line 78, in get
return self.__verb('GET', url, **kwargs)
File "/usr/lib/python2.7/site-packages/jira/resilientsession.py", line 74, in __verb
raise_on_error(r, verb=verb, **kwargs)
File "/usr/lib/python2.7/site-packages/jira/utils.py", line 120, in raise_on_error
r.status_code, error, r.url, request=request, response=r, **kwargs)
jira.utils.JIRAError: JiraError HTTP 400
text: Error in the JQL Query: The character '%' is a reserved JQL character. You must enclose it in a string or use the escape '\u0025' instead. (line 1, character 11)
Questions:
search_alert_issues should be able to use the value stored in jira_prj variable, right?
But why it isn't working? All jira_* variables are showing their values when I did a simple print command on them.
How am I going to use the value stored in jira_prj?
In
search_alert_issues = jira.search_issues("project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)") % jira_prj
the % jira_prj should come before the parenthesis closes -
search_alert_issues = jira.search_issues("project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)" % jira_prj)
Right now, you are sending project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous) to the server, and then formatting the result with the value in jira_prj. When the Jira server sees the %, it complains.
% jura_prj is in wrong position
search_alert_issues = jira.search_issues("project = %s AND issuetype = Alert AND status = Submitted AND reporter in (anonymous)" % jira_prj)
I am doing a location search in Google App Engine and I want my search to be sorted based on proximity. I am getting the following error on the deployed version (production):
Search error:
Traceback (most recent call last):
File "/base/data/home/apps/s~sound-helper-87921/1.385231928987755902/application/search_handler.py", line 68, in doProductSearch
search_results = docs.Product.getIndex().search(search_query)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 105, in positional_wrapper
return wrapped(*args, **kwds)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 3676, in search
return self.search_async(query, deadline=deadline, **kwargs).get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 262, in get_result
return self._get_result_hook()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 3690, in hook
_CheckStatus(response.status())
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/search/search.py", line 517, in _CheckStatus
raise _ERROR_MAP[status.code()](status.error_detail())
InvalidRequest: Failed to parse search request "distance(location, geopoint(30.008164999999998,-95.52959159999999)) < 2000"; Default text value is not appropriate for sort expression 'distance(location, geopoint(30.008165,-95.529592))'
The following is my code, which is pretty much copied from Google's tutorial:
def _buildQueryString(self, params):
userstr = string = params.get('querystr')
userprice = params.get('price')
userdist = params.get('less_than_distance')
loc = params.get('cur_location')
lat = loc.split(',')[0].split()[0]
lng = loc.split(',')[1].split()[0]
if userstr:
string = userstr
if userprice:
string = string + ' price < %s' % userprice
if userdist:
if not os.environ.get('SERVER_SOFTWARE','').startswith('Development'):
string = string + ' distance(%s, geopoint(%s,%s)) < %s' % (
docs.Product.LOCATION,lat,lng,userdist)
return string
def _buildQuery(self, params):
"""Build and return a search query object."""
user_query = self._buildQueryString(params)
doc_limit = self._getDocLimit()
try:
offsetval = int(params.get('offset', 0))
except ValueError:
offsetval = 0
loc = params.get('cur_location')
lat = loc.split(',')[0].split()[0]
lng = loc.split(',')[1].split()[0]
expr = 'distance(%s, geopoint(%f,%f))' % (docs.Product.LOCATION,float(lat),float(lng))
computed_expr_distance = search.FieldExpression(name='actual_distance',
expression=expr)
computed_expr_score = search.FieldExpression(name='actual_score',
expression='score')
returned_fields = [docs.Product.PID]
expr_list = []
expr_list.append(search.SortExpression(
expression=expr,
direction=search.SortExpression.ASCENDING,
default_value='2001'))
sortopts = search.SortOptions(expressions=expr_list, limit = doc_limit)
search_query = search.Query(
query_string=user_query.strip(),
options=search.QueryOptions(
limit=doc_limit,
offset=offsetval,
sort_options=sortopts,
returned_expressions=[computed_expr_distance],
returned_fields=returned_fields
)
)
return search_query
def doProductSearch(self, params):
"""Perform a product search and display the results."""
try:
search_query = self._buildQuery(params)
search_results = docs.Product.getIndex().search(search_query)
returned_count = len(search_results.results)
except search.Error:
logging.exception("Search error:")
msg = 'There was a search error (see logs).'
url = '/'
print('%s' % msg)
return [],[]
psearch_response = []
distances = []
# For each document returned from the search
for doc in search_results:
pdoc = docs.Product(doc)
for expr in doc.expressions:
if expr.name == 'actual_distance':
distances.append(expr.value)
pid = pdoc.getPID()
psearch_response.append(long(pid))
logging.debug('Distances: ' +str(distances))
return psearch_response, distances
Why is the Search API not recognizing my search query?
The problem was in my default_value. I modified the SortExpression to have an integer default_value instead of a string:
expr_list.append(search.SortExpression(
expression=expr,
direction=search.SortExpression.ASCENDING,
default_value=500000))
So I'm attempting to create a program that takes the genre and location of recently uploaded Soundcloud tracks and displays them on a map. I'm trying to gather the data using the Soundcloud API however at random points while the program is executing an HTTPError gets thrown. Here's my code:
import soundcloud
import urllib2
client = soundcloud.Client(client_id="xxxxx",
client_secret="xxxxx",
username="xxxx",
password="xxxx")
def main():
for trackId in getAllTrackId('tracks/'):
try:
trackId = str(trackId)
userId = str(client.get('tracks/' + trackId).user_id)
genre = client.get('tracks/' + trackId).genre
country = client.get('users/' + userId).country
city = client.get('users/' + userId).city
user = client.get('tracks/' + trackId).user
except urllib2.HTTPError, e:
print e.geturl()
if (type(city) != None) & (type(country) != None) & (type(genre) != None):
try:
print 'User: ' + userId + '\t\tLocation: ' + city + ', ' + country + '\t\tGenre: ' + genre
except:
print trackId
def getAllTrackId(path):
items=[]
page_size=200
offset=0
page = client.get(path, limit=page_size, offset=offset)
#while (len(page) != 0):
while (offset<=1000):
for item in page:
items.append(item.id)
offset += page_size
page = client.get(path, limit=page_size, offset=offset)
return items
main()
I'm trying to figure out how to catch the error but also why it is thrown in the first place. I'm new to programming so I'm sure there are many errors in my code but if anyone could help me through this it would be awesome :)
(env)xxxxxxx-MacBook-Pro-2:soundcloudmap xxxxxxxxxx$ python soundcloudmap.py
User: 1278531 Location: Detroit, United States Genre: HSB
User: 1139662 Location: Brasilia, Brazil Genre: Beat
159333532
User: 23129248 Location: Cadiz, Spain Genre: Breaks
159333523
User: 10761166 Location: Wadsworth, United States Genre: Say
Traceback (most recent call last):
File "soundcloudmap.py", line 43, in <module>
main()
File "soundcloudmap.py", line 28, in main
userId = str(client.get('tracks/' + trackId).user_id)
File "/Users/michaelspearman/code/soundcloudmap/env/lib/python2.7/site-packages/soundcloud/client.py", line 130, in _request
return wrapped_resource(make_request(method, url, kwargs))
File "/Users/michaelspearman/code/soundcloudmap/env/lib/python2.7/site-packages/soundcloud/request.py", line 134, in make_request
result.raise_for_status()
File "/Users/michaelspearman/code/soundcloudmap/env/lib/python2.7/site-packages/requests/models.py", line 795, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found
EDIT:
To catch the error I just need to follow the path of the error correctly.. duhh
except urllib2.HTTPError, e:
should be
except requests.exceptions.HTTPError as e:
However I'm still not sure why it's being thrown..
From the Soundclould Python documentation:
tracks = client.get('/tracks', limit=10)
compare this to yours:
client.get('tracks/' + trackId).user_id
More specifically, you are missing the leading /. Adding this should work.