Develop Reference

Python | Multithreading script does not complete

I have this multithreading script, which operates on a data set. Each thread gets a chunk of the data set and then each thread iterates over the data frame and calls and api (MS Graph Create).
What I have seen is that, my script tends to get stuck at almost finish time. I am running this on a linux Ubuntu server. 8vCpus. But this happens only when the total dataset size is in millions. (takes around 9-10 hrs for 2 million records)
I am writing a script (long running) for the first time. Would like to get an opinion if I am doing things correctly.
Please :
I would like to know if my code is the reason why my script hangs.
Have I done multithreading correctly ? Have I created and waited for threads to end correctly ?
UPDATE
Using answers, below, still the threads seems to get stuck at the end.
import pandas as pd
import sys
import os
import logging
import string
import secrets
import random
##### ----- Logging Setup -------
logging.basicConfig(filename="pylogs.log", format='%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
# Creating an object
logger = logging.getLogger()
# Setting the threshold of logger to DEBUG
logger.setLevel(logging.ERROR)
#####------ Function Definitions -------
# generates random password
def generateRandomPassword(lengthOfPassword):
# logic for random password gen
# the most important funtion
#
def createAccounts(splitData, threadID):
batchProgress = 0
batch_size = splitData.shape[0]
for row in splitData.itertuples():
try:
headers = {"Content-Type": "application/json", "Authorization":"Bearer "+access_token}
randomLength = [8,9,12,13,16]
passwordLength = random.choice(randomLength)
password = generateRandomPassword(passwordLength) # will be generated randomly - for debugging purpose
batchProgress+=1
post_request_body = {
"accountEnabled": True,
"displayName": row[5],
"givenName": row[3],
"surname": row[4],
"mobilePhone": row[1],
"mail": row[2],
"passwordProfile" : {
"password": password,
"forceChangePasswordNextSignIn": False
},
"state":"",
"identities": [
{
"signInType": "emailAddress",
"issuer": tenantName,
"issuerAssignedId": row[2]
}
]
}
# if phone number exists then only add - since phone number needs to have length between 1 and 64, cannot leave empty
if(len(row[4])):
post_request_body["identities"].append({"signInType": "phoneNumber","issuer": tenantName,"issuerAssignedId": row[1]})
responseFromApi = requests.post(graph_api_create, headers=headers, json=post_request_body)
status = responseFromApi.status_code
if(status == 201): #success
id = responseFromApi.json().get("id")
print(f" {status} | {batchProgress} / {batch_size} | Success {id}")
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}{row[11]}{row[12]}{row[13]}^Success'
elif(status == 429): #throttling issues
print(f" Thread {threadID} | Throttled by server ! Sleeping for 150 seconds")
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}^Throttled'
time.sleep(150)
elif(status == 401): #token expiry
print(f" Thread {threadID} | Token Expired. Getting it back !")
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}^Token Expired'
getRefreshToken()
else: #any other error
msg = ""
try:
msg = responseFromApi.json().get("error").get("message")
except Exception as e:
msg = f"Error {e}"
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}^{msg}'
print(f" {status} | {batchProgress} / {batch_size} | {msg} {row[2]}")
logger.error(errorDict)
except Exception as e:
# check for refresh token errors
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}^Exception_{e}'
logger.error(errorDict)
msg = " Error "
print(f" {status} | {batchProgress} / {batch_size} | {msg} {row[2]}")
print(f"Thread {threadID} completed ! {batchProgress} / {batch_size}")
batchProgress = 0
###### ------ Main Script ------
if __name__ == "__main__":
# get file name and appid from command line arguments
storageFileName = sys.argv[1]
appId = sys.argv[2]
# setup credentials
bigFilePath = f"./{storageFileName}"
CreatUserUrl = "https://graph.microsoft.com/v1.0/users"
B2C_Tenant_Name = "tenantName"
tenantName = B2C_Tenant_Name + ".onmicrosoft.com"
applicationID = appId
accessSecret = "" # will be taken from command line in future revisions
token_api_body = {
"grant_type": "client_credentials",
"scope": "https://graph.microsoft.com/.default",
"client_Id" : applicationID,
"client_secret": accessSecret
}
# Get initial access token from MS
print("Connecting to MS Graph API")
token_api = "https://login.microsoftonline.com/"+tenantName+"/oauth2/v2.0/token"
response = {}
try:
responseFromApi = requests.post(token_api, data=token_api_body)
responseJson = responseFromApi.json()
print(f"Token API Success ! Expires in {responseJson.get('expires_in')} seconds")
except Exception as e:
print("ERROR | Token auth failed ")
# if we get the token proceed else abort
if(responseFromApi.status_code == 200):
migrationData = pd.read_csv(bigFilePath)
print(" We got the data from Storage !", migrationData.shape[0])
global access_token
access_token = responseJson.get('access_token')
graph_api_create = "https://graph.microsoft.com/v1.0/users"
dataSetSize = migrationData.shape[0]
partitions = 50 # No of partitions # will be taken from command line in future revisions
size = int(dataSetSize/partitions) # No of rows per file
remainder = dataSetSize%partitions
print(f"Data Set Size : {dataSetSize} | Per file size = {size} | Total Files = {partitions} | Remainder: {remainder} | Start...... \n")
##### ------- Dataset partioning.
datasets = []
range_val = partitions + 1 if remainder !=0 else partitions
for partition in range(range_val):
if(partition == partitions):
df = migrationData[size*partition:dataSetSize]
else:
df = migrationData[size*partition:size*(partition+1)]
datasets.append(df)
number_of_threads = len(datasets)
start_time = time.time()
spawned_threads = []
######## ---- Threads are spawned ! here --------
for i in range(number_of_threads): # spawn threads
t = threading.Thread(target=createAccounts, args=(datasets[i], i))
t.start()
spawned_threads.append(t)
number_spawned = len(spawned_threads)
print(f"Started {number_spawned} threads !")
###### - Threads are killed here ! ---------
for thread in spawned_threads: # let the script wait for thread execution
thread.join()
print(f"Done! It took {time.time() - start_time}s to execute") # time check
#### ------ Retry Mechanism -----
print("RETRYING....... !")
os.system(f'python3 retry.py pylogs.log {appId}')
else:
print(f"Token Missing ! API response {responseJson}")```

Here's a refactoring of your code to use the standard library multiprocessing.ThreadPool for simplicity.
Naturally I couldn't have tested it since I don't have your data, but the basic idea should work. I removed the logging and retry stuff, since I really couldn't understand why you'd need it (but feel free to add it back); this will attempt to retry each row if the problem appears to be transient.
import random
import sys
import time
from multiprocessing.pool import ThreadPool
import pandas as pd
import requests
sess = requests.Session()
# globals filled in by `main`
tenantName = None
access_token = None
def submit_user_create(row):
headers = {"Content-Type": "application/json", "Authorization": "Bearer " + access_token}
randomLength = [8, 9, 12, 13, 16]
passwordLength = random.choice(randomLength)
password = generateRandomPassword(passwordLength) # will be generated randomly - for debugging purpose
post_request_body = {
"accountEnabled": True,
"displayName": row[5],
"givenName": row[3],
"surname": row[4],
"mobilePhone": row[1],
"mail": row[2],
"passwordProfile": {"password": password, "forceChangePasswordNextSignIn": False},
"state": "",
"identities": [{"signInType": "emailAddress", "issuer": tenantName, "issuerAssignedId": row[2]}],
}
# if phone number exists then only add - since phone number needs to have length between 1 and 64, cannot leave empty
if len(row[4]):
post_request_body["identities"].append({"signInType": "phoneNumber", "issuer": tenantName, "issuerAssignedId": row[1]})
return sess.post("https://graph.microsoft.com/v1.0/users", headers=headers, json=post_request_body)
def get_access_token(tenantName, applicationID, accessSecret):
token_api_body = {
"grant_type": "client_credentials",
"scope": "https://graph.microsoft.com/.default",
"client_Id": applicationID,
"client_secret": accessSecret,
}
token_api = f"https://login.microsoftonline.com/{tenantName}/oauth2/v2.0/token"
resp = sess.post(token_api, data=token_api_body)
if resp.status_code != 200:
raise RuntimeError(f"Token Missing ! API response {resp.content}")
json = resp.json()
print(f"Token API Success ! Expires in {json.get('expires_in')} seconds")
return json["access_token"]
def process_row(row):
while True:
response = submit_user_create(row)
status = response.status_code
if status == 201: # success
id = response.json().get("id")
print(f"Success {id}")
return True
if status == 429: # throttling issues
print(f"Throttled by server ! Sleeping for 150 seconds")
time.sleep(150)
continue
if status == 401: # token expiry?
print(f"Token Expired. Getting it back !")
getRefreshToken() # TODO
continue
try:
msg = response.json().get("error").get("message")
except Exception as e:
msg = f"Error {e}"
print(f" {status} | {msg} {row[2]}")
return False
def main():
global tenantName, access_token
# get file name and appid from command line arguments
bigFilePath = sys.argv[1]
appId = sys.argv[2]
# setup credentials
B2C_Tenant_Name = "tenantName"
tenantName = f"{B2C_Tenant_Name}.onmicrosoft.com"
accessSecret = "" # will be taken from command line in future revisions
access_token = get_access_token(tenantName, appId, accessSecret)
migrationData = pd.read_csv(bigFilePath)
start_time = time.time()
with ThreadPool(10) as pool:
for i, result in enumerate(pool.imap_unordered(process_row, migrationData.itertuples()), 1):
progress = i / len(migrationData) * 100
print(f"{i} / {len(migrationData)} | {progress:.2f}% | {time.time() - start_time:.2f} seconds")
print(f"Done! It took {time.time() - start_time}s to execute")
if __name__ == "__main__":
main()

un-fair use of MS Graph
Due to possible throttling by the server, the usage of the MS Graph resource might be un-fair between threads. I use fair in the resource starvation sense.
elif(status == 429): #throttling issues
print(f" Thread {threadID} | Throttled by server ! Sleeping for 150 seconds")
errorDict = f'{row[1]}^{row[2]}^{row[3]}^{row[4]}^{row[5]}^{row[6]}^{row[7]}^{row[8]}^{row[9]}^{row[10]}{row[11]}{row[12]}{row[13]}^Throttled'
time.sleep(150)
One thread making a million calls can get a disproportionate amount of 429 responses each followed by a penalty of 150 seconds. This sleep doesn't stop the other threads from making calls though and achieving forward progress.
This would result in one thread lagging far behind the others and giving the appearance of being stuck.

blpapi.exception.UnsupportedOperationException: No subscription management endpoints for snapshot (0x00080013)

I am trying to request snapshot from Bloomberg through Python API using the example programs that Bloomberg provided with the package. Their own program doesn't work properly and I keep getting the error:
WARN blpapi_subscriptionmanager.cpp:1653 blpapi.session.subscriptionmanager.{1} Subscription management endpoints are required for snapshot request templates but none are available
blpapi.exception.UnsupportedOperationException: No subscription management endpoints for snapshot (0x00080013).
The part of the code that has the snapshot request is in main func:
def main():
"""main entry point"""
global options
options = parseCmdLine()
# Create a session and Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
for idx, host in enumerate(options.hosts):
sessionOptions.setServerAddress(host, options.port, idx)
sessionOptions.setAuthenticationOptions(options.auth)
sessionOptions.setAutoRestartOnDisconnection(True)
print("Connecting to port %d on %s" % (
options.port, ", ".join(options.hosts)))
session = blpapi.Session(sessionOptions)
if not session.start():
print("Failed to start session.")
return
subscriptionIdentity = None
if options.auth:
subscriptionIdentity = session.createIdentity()
isAuthorized = False
authServiceName = "//blp/apiauth"
if session.openService(authServiceName):
authService = session.getService(authServiceName)
isAuthorized = authorize(authService, subscriptionIdentity,
session, blpapi.CorrelationId("auth"))
if not isAuthorized:
print("No authorization")
return
else:
print("Not using authorization")
# Snapshot Request Part:
fieldStr = "?fields=" + ",".join(options.fields)
snapshots = []
nextCorrelationId = 0
for i, topic in enumerate(options.topics):
subscriptionString = options.service + topic + fieldStr
snapshots.append(session.createSnapshotRequestTemplate(
subscriptionString,
subscriptionIdentity,
blpapi.CorrelationId(i)))
nextCorrelationId += 1
requestTemplateAvailable = blpapi.Name('RequestTemplateAvailable')
eventCount = 0
try:
while True:
# Specify timeout to give a chance for Ctrl-C
event = session.nextEvent(1000)
for msg in event:
if event.eventType() == blpapi.Event.ADMIN and \
msg.messageType() == requestTemplateAvailable:
for requestTemplate in snapshots:
session.sendRequestTemplate(
requestTemplate,
blpapi.CorrelationId(nextCorrelationId))
nextCorrelationId += 1
elif event.eventType() == blpapi.Event.RESPONSE or \
event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
cid = msg.correlationIds()[0].value()
print("%s - %s" % (cid, msg))
else:
print(msg)
if event.eventType() == blpapi.Event.RESPONSE:
eventCount += 1
if eventCount >= options.maxEvents:
print("%d events processed, terminating." % eventCount)
break
elif event.eventType() == blpapi.Event.TIMEOUT:
for requestTemplate in snapshots:
session.sendRequestTemplate(
requestTemplate,
blpapi.CorrelationId(nextCorrelationId))
nextCorrelationId += 1
I don't know if endpoint and subscription management endpoint are 2 different things because I have one other code working properly and the endpoint is the IP of the server I am pulling the data.

Python: Bloomberg API Failed to get Token. No Authorisation

Trying to use the IntradayTickExample code enclosed below. However I kept getting the following message. It says about authorization so I guess its getting stuck in the main. But I have no idea why this wouldn't work as its a BBG example?
My thoughts:
- It exits with code 0 so everything works well.
- It seems to go debug through the code until the authorisation section
- I cant find any way to change anything else r./e. the authorisation
C:\Users\ME\MyNewEnv\Scripts\python.exe F:/ME/BloombergWindowsSDK/Python/v3.12.2/examples/IntradayTickExample.py
IntradayTickExample
Connecting to localhost:8194
TokenGenerationFailure = {
reason = {
source = "apitkns (apiauth) on ebbdbp-ob-596"
category = "NO_AUTH"
errorCode = 12
description = "User not in emrs userid=DBG\ME firm=8787"
subcategory = "INVALID_USER"
}
}
Failed to get token
No authorization
Process finished with exit code 0
And full code below
# IntradayTickExample.py
from __future__ import print_function
from __future__ import absolute_import
import blpapi
import copy
import datetime
from optparse import OptionParser, Option, OptionValueError
TICK_DATA = blpapi.Name("tickData")
COND_CODE = blpapi.Name("conditionCodes")
TICK_SIZE = blpapi.Name("size")
TIME = blpapi.Name("time")
TYPE = blpapi.Name("type")
VALUE = blpapi.Name("value")
RESPONSE_ERROR = blpapi.Name("responseError")
CATEGORY = blpapi.Name("category")
MESSAGE = blpapi.Name("message")
SESSION_TERMINATED = blpapi.Name("SessionTerminated")
TOKEN_SUCCESS = blpapi.Name("TokenGenerationSuccess")
TOKEN_FAILURE = blpapi.Name("TokenGenerationFailure")
AUTHORIZATION_SUCCESS = blpapi.Name("AuthorizationSuccess")
TOKEN = blpapi.Name("token")
def authOptionCallback(option, opt, value, parser):
vals = value.split('=', 1)
if value == "user":
parser.values.auth = "AuthenticationType=OS_LOGON"
elif value == "none":
parser.values.auth = None
elif vals[0] == "app" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=APPLICATION_ONLY;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "userapp" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=USER_AND_APPLICATION;"\
"AuthenticationType=OS_LOGON;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "dir" and len(vals) == 2:
parser.values.auth = "AuthenticationType=DIRECTORY_SERVICE;"\
"DirSvcPropertyName=" + vals[1]
else:
raise OptionValueError("Invalid auth option '%s'" % value)
def checkDateTime(option, opt, value):
try:
return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
except ValueError as ex:
raise OptionValueError(
"option {0}: invalid datetime value: {1} ({2})".format(
opt, value, ex))
class ExampleOption(Option):
TYPES = Option.TYPES + ("datetime",)
TYPE_CHECKER = copy.copy(Option.TYPE_CHECKER)
TYPE_CHECKER["datetime"] = checkDateTime
def parseCmdLine():
parser = OptionParser(description="Retrieve intraday rawticks.",
epilog="Notes: " +
"1) All times are in GMT. " +
"2) Only one security can be specified.",
option_class=ExampleOption)
parser.add_option("-a",
"--ip",
dest="host",
help="server name or IP (default: %default)",
metavar="ipAddress",
default="localhost")
parser.add_option("-p",
dest="port",
type="int",
help="server port (default: %default)",
metavar="tcpPort",
default=8194)
parser.add_option("-s",
dest="security",
help="security (default: %default)",
metavar="security",
default="IBM US Equity")
parser.add_option("-e",
dest="events",
help="events (default: TRADE)",
metavar="event",
action="append",
default=[])
parser.add_option("--sd",
dest="startDateTime",
type="datetime",
help="start date/time (default: %default)",
metavar="startDateTime",
default=None)
parser.add_option("--ed",
dest="endDateTime",
type="datetime",
help="end date/time (default: %default)",
metavar="endDateTime",
default=None)
parser.add_option("--cc",
dest="conditionCodes",
help="include condition codes",
action="store_true",
default=False)
parser.add_option("--auth",
dest="auth",
help="authentication option: "
"user|none|app=<app>|userapp=<app>|dir=<property>"
" (default: %default)",
metavar="option",
action="callback",
callback=authOptionCallback,
type="string",
default="user")
(options, args) = parser.parse_args()
if not options.host:
options.host = ["localhost"]
if not options.events:
options.events = ["TRADE"]
return options
def authorize(authService, identity, session, cid):
tokenEventQueue = blpapi.EventQueue()
session.generateToken(eventQueue=tokenEventQueue)
# Process related response
ev = tokenEventQueue.nextEvent()
token = None
if ev.eventType() == blpapi.Event.TOKEN_STATUS or \
ev.eventType() == blpapi.Event.REQUEST_STATUS:
for msg in ev:
print(msg)
if msg.messageType() == TOKEN_SUCCESS:
token = msg.getElementAsString(TOKEN)
elif msg.messageType() == TOKEN_FAILURE:
break
if not token:
print("Failed to get token")
return False
# Create and fill the authorization request
authRequest = authService.createAuthorizationRequest()
authRequest.set(TOKEN, token)
# Send authorization request to "fill" the Identity
session.sendAuthorizationRequest(authRequest, identity, cid)
# Process related responses
startTime = datetime.datetime.today()
WAIT_TIME_SECONDS = 10
while True:
event = session.nextEvent(WAIT_TIME_SECONDS * 1000)
if event.eventType() == blpapi.Event.RESPONSE or \
event.eventType() == blpapi.Event.REQUEST_STATUS or \
event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
for msg in event:
print(msg)
if msg.messageType() == AUTHORIZATION_SUCCESS:
return True
print("Authorization failed")
return False
endTime = datetime.datetime.today()
if endTime - startTime > datetime.timedelta(seconds=WAIT_TIME_SECONDS):
return False
def printErrorInfo(leadingStr, errorInfo):
print("%s%s (%s)" % (leadingStr, errorInfo.getElementAsString(CATEGORY),
errorInfo.getElementAsString(MESSAGE)))
def processMessage(msg):
data = msg.getElement(TICK_DATA).getElement(TICK_DATA)
print("TIME\t\t\t\tTYPE\tVALUE\t\tSIZE\tCC")
print("----\t\t\t\t----\t-----\t\t----\t--")
for item in data.values():
time = item.getElementAsDatetime(TIME)
timeString = item.getElementAsString(TIME)
type = item.getElementAsString(TYPE)
value = item.getElementAsFloat(VALUE)
size = item.getElementAsInteger(TICK_SIZE)
if item.hasElement(COND_CODE):
cc = item.getElementAsString(COND_CODE)
else:
cc = ""
print("%s\t%s\t%.3f\t\t%d\t%s" % (timeString, type, value, size, cc))
def processResponseEvent(event):
for msg in event:
print(msg)
if msg.hasElement(RESPONSE_ERROR):
printErrorInfo("REQUEST FAILED: ", msg.getElement(RESPONSE_ERROR))
continue
processMessage(msg)
def sendIntradayTickRequest(session, options, identity = None):
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest("IntradayTickRequest")
# only one security/eventType per request
request.set("security", options.security)
# Add fields to request
eventTypes = request.getElement("eventTypes")
for event in options.events:
eventTypes.appendValue(event)
# All times are in GMT
if not options.startDateTime or not options.endDateTime:
tradedOn = getPreviousTradingDate()
if tradedOn:
startTime = datetime.datetime.combine(tradedOn,
datetime.time(15, 30))
request.set("startDateTime", startTime)
endTime = datetime.datetime.combine(tradedOn,
datetime.time(15, 35))
request.set("endDateTime", endTime)
else:
if options.startDateTime and options.endDateTime:
request.set("startDateTime", options.startDateTime)
request.set("endDateTime", options.endDateTime)
if options.conditionCodes:
request.set("includeConditionCodes", True)
print("Sending Request:", request)
session.sendRequest(request, identity)
def eventLoop(session):
done = False
while not done:
# nextEvent() method below is called with a timeout to let
# the program catch Ctrl-C between arrivals of new events
event = session.nextEvent(500)
if event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
print("Processing Partial Response")
processResponseEvent(event)
elif event.eventType() == blpapi.Event.RESPONSE:
print("Processing Response")
processResponseEvent(event)
done = True
else:
for msg in event:
if event.eventType() == blpapi.Event.SESSION_STATUS:
if msg.messageType() == SESSION_TERMINATED:
done = True
def getPreviousTradingDate():
tradedOn = datetime.date.today()
while True:
try:
tradedOn -= datetime.timedelta(days=1)
except OverflowError:
return None
if tradedOn.weekday() not in [5, 6]:
return tradedOn
def main():
options = parseCmdLine()
# Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
sessionOptions.setServerHost(options.host)
sessionOptions.setServerPort(options.port)
sessionOptions.setAuthenticationOptions(options.auth)
print("Connecting to %s:%s" % (options.host, options.port))
# Create a Session
session = blpapi.Session(sessionOptions)
# Start a Session
if not session.start():
print("Failed to start session.")
return
identity = None
if options.auth:
identity = session.createIdentity()
isAuthorized = False
authServiceName = "//blp/apiauth"
if session.openService(authServiceName):
authService = session.getService(authServiceName)
isAuthorized = authorize(
authService, identity, session,
blpapi.CorrelationId("auth"))
if not isAuthorized:
print("No authorization")
return
try:
# Open service to get historical data from
if not session.openService("//blp/refdata"):
print("Failed to open //blp/refdata")
return
sendIntradayTickRequest(session, options, identity)
print(sendIntradayTickRequest(session, options, identity))
# wait for events from session.
eventLoop(session)
finally:
# Stop the session
session.stop()
if __name__ == "__main__":
print("IntradayTickExample")
try:
main()
except KeyboardInterrupt:
print("Ctrl+C pressed. Stopping...")
__copyright__ = """
Copyright 2012. Bloomberg Finance L.P.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions: The above
copyright notice and this permission notice shall be included in all copies
or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
"""

I realized I have been trying to connect to the Server API instead of connectiong to the local terminal. For example, the following code works perfectly:
# SimpleHistoryExample.py
from __future__ import print_function
from __future__ import absolute_import
import blpapi
from optparse import OptionParser
def parseCmdLine():
parser = OptionParser(description="Retrieve reference data.")
parser.add_option("-a",
"--ip",
dest="host",
help="server name or IP (default: %default)",
metavar="ipAddress",
default="localhost")
parser.add_option("-p",
dest="port",
type="int",
help="server port (default: %default)",
metavar="tcpPort",
default=8194)
(options, args) = parser.parse_args()
return options
def main():
options = parseCmdLine()
# Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
sessionOptions.setServerHost(options.host)
sessionOptions.setServerPort(options.port)
print("Connecting to %s:%s" % (options.host, options.port))
# Create a Session
session = blpapi.Session(sessionOptions)
# Start a Session
if not session.start():
print("Failed to start session.")
return
try:
# Open service to get historical data from
if not session.openService("//blp/refdata"):
print("Failed to open //blp/refdata")
return
# Obtain previously opened service
refDataService = session.getService("//blp/refdata")
# Create and fill the request for the historical data
request = refDataService.createRequest("HistoricalDataRequest")
request.getElement("securities").appendValue("EURUSD Curncy")
request.getElement("fields").appendValue("PX_LAST")
request.set("periodicityAdjustment", "ACTUAL")
request.set("periodicitySelection", "DAILY")
request.set("startDate", "20000102")
request.set("endDate", "20191031")
request.set("maxDataPoints", 100000)
print("Sending Request:", request)
# Send the request
session.sendRequest(request)
# Process received events
while(True):
# We provide timeout to give the chance for Ctrl+C handling:
ev = session.nextEvent(500)
for msg in ev:
print(msg)
if ev.eventType() == blpapi.Event.RESPONSE:
# Response completly received, so we could exit
break
finally:
# Stop the session
session.stop()
if __name__ == "__main__":
print("SimpleHistoryExample")
try:
main()
except KeyboardInterrupt:
print("Ctrl+C pressed. Stopping...")

Python server hangs

I have a problem with python 3.6 HTTPServer
When the browser is sending requests everything is fine. When I send request via PostMan the request is completed (code 200) but then the requests fail. If I ping the server again via postman all pending are completed and then they continue to fail.
This is the JS that send requests to the server
jQuery.post("http://<?php echo $_SERVER['SERVER_ADDR']?>:8081", JSON.stringify({
auth: "security_token",
user_id: <?php echo $_SESSION[CLIENT_LOGGED] ?>
})
And this is how the server process the request
def do_POST(self):
contentLen = int(self.headers['content-length'])
postBody = self.rfile.read(contentLen)
try:
data = json.loads(postBody.decode('utf-8'))
except ValueError:
self.handeValueError()
return
self.processRequest(data)
return
def processRequest(self, data):
if (data["auth"] == self.centralKey):
self.processCentral(data)
elif (data["auth"] == self.clientKey):
self.processClient(data)
else:
self._set_badAuth()
def processCentral(self, inputData):
self._set_validMethod()
phone = standartizePhone(inputData["caller"])
if(inputData["ringing"] == "1"):
result = getCallerInfo(phone)
if (len(result) == 0):
temp = CallerData(phone, -1, 0)
self.callerInfo[inputData["internal"]].append(temp)
for row in result:
if (row[2] == 0):
temp = CallerData(phone, row[0], row[1], 1)
else:
temp = CallerData(phone, row[0], row[1], 1, row[2], row[3], row[4], row[5])
self.callerInfo[inputData["internal"]].append(temp)
elif(inputData["ringing"] == "2"):
for el in self.callerInfo[inputData["internal"]]:
el.c_status = 2
elif(inputData["ringing"] == "3"):
caller = CallerData(phone, 0, 0)
values = [value for value in self.callerInfo[inputData["internal"]] if value != caller]
self.callerInfo[inputData["internal"]] = values
else:
message = '{"success": "false"}'
self.wfile.write(bytes(message, "utf8"))
return False
message = '{"success": "true"}'
self.wfile.write(bytes(message, "utf8"))
return True
def processClient(self, inputData):
self._set_validMethod()
message = "["
internalMsg = []
internalNums = getUserInternal(inputData["user_id"])
for internal in internalNums:
for caller in self.callerInfo[str(internal[0])]:
internalMsg.append(callerInfoToJSON(caller))
message += ", ".join(internalMsg)
message += "]"
self.wfile.write(bytes(message, "utf8"))
Note that it doesn't matter if I am sending the request as central or client via postman. In both ways it just hangs. Also no errors are logged from the python script
Note 2 Updating postman to latest version seems to fix the problem

Bloomberg Api for Python: Parts of result missing in response

I'm using bloomberg api for python to get the option data. Firstly, I got all the symbols of option chain. Then I used them to get the bid and ask prices. Through function getOptionChain, there are more than 400 options and I checked the result , it was fine. However, when I run the getPX function, I got only 10 results in the end. Could anyone help me looking into this? Thanks in advance!
import blpapi
import pandas
import csv
options = blpapi.SessionOptions()
options.setServerHost('localhost')
options.setServerPort(8194)
SECURITY_DATA = blpapi.Name("securityData")
SECURITY = blpapi.Name("security")
FIELD_DATA = blpapi.Name("fieldData")
FIELD_ID = blpapi.Name("fieldId")
OPT_CHAIN = blpapi.Name("OPT_CHAIN")
SECURITY_DES = blpapi.Name("Security Description")
def getOptionChain (sec_list):
session = blpapi.Session(options)
session.start()
session.openService('//blp/refdata')
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest("ReferenceDataRequest")
for s in sec_list:
request.append("securities",s)
request.append("fields", "OPT_CHAIN")
cid = session.sendRequest(request)
try:
# Process received events
while(True):
# We provide timeout to give the chance to Ctrl+C handling:
ev = session.nextEvent(500)
response = []
for msg in ev:
if cid in msg.correlationIds():
securityDataArray = msg.getElement(SECURITY_DATA)
for securityData in securityDataArray.values():
fieldData = securityData.getElement(FIELD_DATA)
for field in fieldData.elements():
for n in range(field.numValues()):
fld = field.getValueAsElement(n)
response.append (fld.getElement(SECURITY_DES).getValueAsString())
# Response completely received, so we could exit
if ev.eventType() == blpapi.Event.RESPONSE:
break
finally:
# Stop the session
session.stop()
return response
def getPX (sec_list, fld_list):
opt_chain_list = getOptionChain(sec_list)
session = blpapi.Session(options)
session.start()
session.openService('//blp/refdata')
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest("ReferenceDataRequest")
for s in opt_chain_list:
request.append("securities",s)
for f in fld_list:
request.append("fields",f)
cid = session.sendRequest(request)
try:
# Process received events
while(True):
# We provide timeout to give the chance to Ctrl+C handling:
ev = session.nextEvent(500)
response = {}
for msg in ev:
if cid in msg.correlationIds():
securityDataArray = msg.getElement(SECURITY_DATA)
for securityData in securityDataArray.values():
secName = securityData.getElementAsString(SECURITY)
fieldData = securityData.getElement(FIELD_DATA)
response[secName] = {}
for field in fieldData.elements():
response[secName][field.name()] = field.getValueAsFloat()
# Response completely received, so we could exit
if ev.eventType() == blpapi.Event.RESPONSE:
break
finally:
# Stop the session
session.stop()
tempdict = {}
for r in response:
tempdict[r] = pandas.Series(response[r])
data = pandas.DataFrame(tempdict)
return data
sec = ["IBM US Equity"]
fld = ["PX_ASK","PX_BID"]
getPX(sec,fld)

It looks like you've got the "response = {}" in the wrong place.
Currently you're clearing at each iteration of your loop so each event coming in refills it.
If you shift the "response = {}" to just before "While(True):" each iteration will append to it rather than clearing and refilling.
The same is true of the first function, but the bulk data comes back in a single event in this case. If you were using multiple securities you would see the same issue (a single Bloomberg refdata (partial) response contains data for at most 10 securities).