Develop Reference

Why Eventhub async receiver is fetching just 30-35 messages per minute?

I have a async_receive method of Eventhub developed in python and also has a checkpoint with it. The code was taken from the official Eventhub sample github repo.
Problem- Using the above-mentioned code, I am just able to receive 20-35 messages per minute if I keep the receiver on for the whole day whereas my Eventhub has a lot of stream data ingested (~200 messages per Minute). The enqueued time at eventhub for a message is now lagging behind by 90 minutes due to poor throughput at the receiver's end which means that the data that got enqueued at X minute in the Eventhub got pulled out of it at X+90 minutes
Investigation- I tried to look at the receive subclass in the Eventhub python SDK and came across a prefetch parameter (line 318) which is set to 300 by default. If this is already set to 300 then I should be able to pull more than 30-35 messages by default.
Any idea on how can I increase my pull capacity? I'm stuck at this point and have no direction forward, any help is highly appreciated.
EDIT 1-
I'm now attaching my Python Code as shown below-
import asyncio
import json
import logging
import os
import sys
import time
from datetime import date
import requests
from azure.eventhub.aio import EventHubConsumerClient
from azure.eventhub.extensions.checkpointstoreblobaio import BlobCheckpointStore
import log_handler
import threading
import traceback
try:
## Set env variables
CONNECTION_STR = os.environ["ECS"].strip()
EVENTHUB_NAME = os.environ['EN'].strip()
EVENTHUB_CONSUMER = os.environ["EC"].strip()
API = os.environ['API_variable'].strip()
AZURE_BLOB_CONNECTION_STR = os.environ["ACS"].strip()
BLOB_CONTAINER_NAME = os.environ["BCN"].strip()
BLOB_ACCOUNT_URL = os.environ["BAU"].strip()
PREFETCH_COUNT = int(os.environ["PREFETCH_COUNT"])
MAX_WAIT_TIME = float(os.environ["MAX_WAIT_TIME"])
except Exception as exception:
logging.debug(traceback.format_exc())
logging.warning(
"*** Please check the environment variables for {}".format(str(exception)))
sys.exit()
def API_CALL(event_data):
"""
Sends the request to the API
"""
try:
url = event_data['image_url']
payload = {"url": url}
## API call to the server
service_response = requests.post(API, json=payload)
logging.info(f"*** service_response.status_code : {service_response.status_code}")
cloud_response = json.loads(
service_response.text) if service_response.status_code == 200 else None
today = date.today()
response_date = today.strftime("%B %d, %Y")
response_time = time.strftime("%H:%M:%S", time.gmtime())
response_data = {
"type": "response_data",
"consumer_group": EVENTHUB_CONSUMER,
'current_date': response_date,
'current_time': response_time,
'image_url': url,
'status_code': service_response.status_code,
'response': cloud_response,
'api_response_time': int(service_response.elapsed.total_seconds()*1000),
"eventhub_data": event_data
}
logging.info(f"*** response_data {json.dumps(response_data)}")
logging.debug(f"*** response_data {json.dumps(response_data)}")
except Exception as exception:
logging.debug(traceback.format_exc())
logging.error(
"**** RaiseError: Failed request url %s, Root Cause of error: %s", url, exception)
async def event_operations(partition_context, event):
start_time = time.time()
data_ = event.body_as_str(encoding='UTF-8')
json_data = json.loads(data_)
## forming data payload
additional_data = {
"type": "event_data",
"consumer_group": EVENTHUB_CONSUMER,
"image_name": json_data["image_url"].split("/")[-1]
}
json_data.update(additional_data)
logging.info(f"*** Data fetched from EH : {json_data}")
logging.debug(f"*** Data fetched from EH : {json_data}")
API_CALL(json_data)
logging.info(f"*** time taken to process an event(ms): {(time.time()-start_time)*1000}")
def between_callback(partition_context, event):
"""
Loop to create threads
"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(event_operations(partition_context, event))
loop.close()
async def on_event(partition_context, event):
"""
Put your code here.
Do some sync or async operations.
If the operation is i/o intensive, async will have better performanceself.
"""
t1 = time.time()
_thread = threading.Thread(target=between_callback, args=(partition_context, event))
_thread.start()
logging.info(f"*** time taken to start a thread(ms): {(time.time()-t1)*1000}")
logging.info("*** Fetching the next event")
## Update checkpoint per event
t2 = time.time()
await partition_context.update_checkpoint(event)
logging.info(f"*** time taken to update checkpoint(ms): {(time.time()-t2)*1000}")
async def main(client):
"""
Run the on_event method for each event received
Args:
client ([type]): Azure Eventhub listener client
"""
try:
async with client:
# Call the receive method. Only read current data (#latest)
logging.info("*** Listening to event")
await client.receive(on_event=on_event,
prefetch=PREFETCH_COUNT,
max_wait_time=MAX_WAIT_TIME)
except KeyboardInterrupt:
print("*** Stopped receiving due to keyboard interrupt")
except Exception as err:
logging.debug(traceback.format_exc())
print("*** some error occured :", err)
if __name__ == '__main__':
## Checkpoint initialization
checkpoint_store = BlobCheckpointStore(
blob_account_url=BLOB_ACCOUNT_URL,
container_name=BLOB_CONTAINER_NAME,
credential=AZURE_BLOB_CONNECTION_STR
)
## Client initialization
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group=EVENTHUB_CONSUMER,
eventhub_name=EVENTHUB_NAME,
checkpoint_store=checkpoint_store, #COMMENT TO RUN WITHOUT CHECKPOINT
logging_enable=True,
on_partition_initialize=on_partition_initialize,
on_partition_close=on_partition_close,
idle_timeout=10,
on_error=on_error,
retry_total=3
)
logging.info("Connecting to eventhub {} consumer {}".format(
EVENTHUB_NAME, EVENTHUB_CONSUMER))
logging.info("Registering receive callback.")
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(client))
except KeyboardInterrupt as exception:
pass
finally:
loop.stop()
Execution-flow main()-->on_event()-->Thread(between_callback-->API_CALL)-->update_checkpoint

Change the function for the receiver in the below format when we can able to get the events and run them until they complete.
import asyncio
from azure.eventhub.aio import EventHubConsumerClient
from azure.eventhub.extensions.checkpointstoreblobaio import BlobCheckpointStore
async def on_event(partition_context, event):
# Print the event data.
print("Received the event: \"{}\" from the partition with ID: \"{}\"".format(event.body_as_str(encoding='UTF-8'), partition_context.partition_id))
# Update the checkpoint so that the program doesn't read the events
# that it has already read when you run it next time.
await partition_context.update_checkpoint(event)
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string("AZURE STORAGE CONNECTION STRING", "BLOB CONTAINER NAME")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string("EVENT HUBS NAMESPACE CONNECTION STRING", consumer_group="$Default", eventhub_name="EVENT HUB NAME", checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
if __name__ == '__main__':
loop = asyncio.get_event_loop()
# Run the main method.
loop.run_until_complete(main())
Also as per the suggest in the comment, call the API on an interval bases.

Can't connect to websocket API

Now I’m working on developing WebSocket to get data (ex.btcusdt) from the website FTX
since the ftx.com you can trade the crypto without having to pay fees and I have a minimum budget
so now I want to try out to get some data and making my own bot
but now I’m having a problem with how I connect to the website
since I watch Binance video I was trying the same way but still didn’t get any message from the " wss://ftx.com/ws/ "
I do not quite understand the document they provide
my question is how can I connect to the data stream for example if I want to get the JSON file of BTCUSDT or BULLUSDT
this is the document they provide
https://docs.ftx.com/#websocket-api
Thank you
My code
import websocket
SOCKET = "wss://ftx.com/ws/"
def on_open(ws):
print('opened connection')
def on_close(ws):
print('closed connection')
def on_message(ws, message):
print("got message")
ws = websocket.WebSocketApp(SOCKET, on_open=on_open, on_close=on_close, on_message=on_message)
ws.run_forever()
This works find with Binance

Using the example code here modified to accept api keys as arguments, here is an example of grabbing ticker data:
if __name__ == '__main__':
# rest = client.FtxClient(api_key=key, api_secret=secret)
ws = ws_client.FtxWebsocketClient(api_key=key, api_secret=secret)
ws.connect()
for i in range(1, 10):
print(ws.get_ticker(market='BTC-PERP'))
time.sleep(1)

As Chev_603 say you can copy the two files into your directory.
Then, import the files at the begging of your app and use:
from client import FtxWebsocketClient
from websocket_manager import WebsocketManager
if __name__ == '__main__':
for i in range(1, 1000):
print(ws.get_ticker(market='BTC-PERP'))
time.sleep(0.2)
# your own api_key and api_secret must be written into the client.py lines 20 and 21

JSON message:
import websocket
import json
this = json.dumps({'op': 'subscribe', 'channel': 'trades', 'market': 'BTC-
PERP'})
def on_open(wsapp):
wsapp.send(this)
def on_message(wsapp, message):
print(message)
wsapp = websocket.WebSocketApp("wss://ftx.com/ws/", on_message=on_message,
on_open=on_open)
wsapp.run_forever()

How to integrate channels and DRF together

I'm currently trying to create a backend server to communicate with some clients with a websocket. The clients makes some request to the backend and the backend responds directly to the client through a consumer.
In addition, I've got an API that needs to send some requests to the client. It has to go through the opened socket of the consumer. I'm using Django Rest Framework for the API. So I've got 2 apps for now. One for the consumer and one for the API. I want to know if it's the correct way or not.
This is actually the code I'm thinking about right now:
# mybackendapp/consumers.py
class MyConsumer(AsyncWebsocketConsumer):
async def connect(self):
self.client_id = self.scope['url_route']['kwargs']['client_id']
# This line I don't get it very well. It comes from:
# [channels doc: single channels][1]
# I don't know if I should create the Clients myself or if it's
# created automatically
Clients.objects.create(channel_name=self.channel_name,
self.client_id)
self.accept()
async def disconnect(self):
Clients.objects.filter(channel_name=self.channel_name).delete()
async def receive(self, text_data):
self.recv_data = json.loads(text_data)
if self.recv_data[0] == CLIENT_REQUEST:
self.handler = ClientRequestHandler(self.client_id,
self.recv_data)
await self.handler.run()
self.sent_data = self.handler.response
self.send(self.sent_data)
elif self.recv_data[0] == CLIENT_RESPONSE:
self.handler = ClientResponseHandler(self.client_id,
self.recv_data)
channel_layer = get_channel_layer()
# Here I'm not sure but I may have several API requests so
# several row with the same client_id.
# I welcome info on how to deal with that.
api_channel_name = self.another_handler.ext_channel_name
channel_layer.send(api_channel_name, {
"text_data": self.handler.response,
})
async def message_from_api(self, event):
self.api_channel_name = event['channel_name_answer']
# this line is for hiding the fact that I'm only manipulating data
# to send it to a correct format to the socket
self.another_handler = ExternalDataHandler(event['json_data'])
query_to_client = another_handler.get_formatted_query()
self.send(query_to_client)
In receive, this consumer handles differently the messages from the client depending if it's initiated by the client or the rest API. You can see that with CLIENT_REQUEST and CLIENT_RESPONSE constants.
Now from the API view:
# myapi/views.py
from channels.layers import get_channel_layer
def my_api_view(request, client_id):
channel_layer = get_channel_layer()
if request.method == 'POST':
ext_request_data_json = request.data
client_channel_name = Clients.objects.get(
client_id=client_id).channel_name
# I don't know what type is listen_channel_name. I assume it's str
listen_channel_name = async_to_sync(channels_layers.new_channel)()
async_to_sync(channel_layer.send)(
client_channel_name, {
'type': 'message.from.api',
'json_data': ext_request_data_json,
'channel_name_answer': listen_channel_name
}
)
received_msg = channel_layer.receive(listen_channel_name)
I believe that this code should work. I want to know if it's the correct way to do it.

See djangochannelsrestframework as a possible alternative solution.
Django Channels Rest Framework provides a DRF like interface for building channels-v3 websocket consumers.

producer based on confluent_kafka: message seems to never get delivered although Kafka is reachable

I'm trying to create a simple Kafka producer based on confluent_kafka. My code is the following:
#!/usr/bin/env python
from confluent_kafka import Producer
import json
def delivery_report(err, msg):
"""Called once for each message produced to indicate delivery result.
Triggered by poll() or flush().
see https://github.com/confluentinc/confluent-kafka-python/blob/master/README.md"""
if err is not None:
print('Message delivery failed: {}'.format(err))
else:
print('Message delivered to {} [{}]'.format(
msg.topic(), msg.partition()))
class MySource:
"""Kafka producer"""
def __init__(self, kafka_hosts, topic):
"""
:kafka_host list(str): hostnames or 'host:port' of Kafka
:topic str: topic to produce messages to
"""
self.topic = topic
# see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
config = {
'metadata.broker.list': ','.join(kafka_hosts),
'group.id': 'mygroup',
}
self.producer = Producer(config)
#staticmethod
def main():
topic = 'my-topic'
message = json.dumps({
'measurement': [1, 2, 3]})
mys = MySource(['kafka'], topic)
mys.producer.produce(
topic, message, on_delivery=delivery_report)
mys.producer.flush()
if __name__ == "__main__":
MySource.main()
The first time I use a topic (here: "my-topic"), Kafka does react with "Auto creation of topic my-topic with 1 partitions and replication factor 1 is successful (kafka.server.KafkaApis)". However, the call-back function (on_delivery=delivery_report) is never called and it hangs at flush() (it terminates if I set a timeout for flush) neither the first time nor subsequent times. The Kafka logs does not show anything if I use an existing topic.

Subscription with selector does not work from python - stomp.py

I have run into a problem where a Python subscriber using stomp (stomp.py) with a message selector does not receive the messages it should. Interestingly enough, it appears to me at least that the problem is somehow with the sending of the message and not the subscription.
I am using ActiveMQ.
Here's the subscriber code:
class Listener(object):
def __init__(self, count):
if count <= 0:
count = float('inf')
self.count = count
def on_error(self, headers, message):
print("=" * 72)
print('RECEIVED AN ERROR.')
print('Message headers:')
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(headers)
print('Message body:')
print(message)
def on_message(self, headers, message):
print("=" * 72)
print('Message headers:')
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(headers)
print('Message body:')
print(message)
def main():
global conn
args = parse_args()
conn = stomp.Connection([(args.host, args.port)])
conn.set_listener('Listener', Listener(args.count))
conn.start()
conn.connect(login=args.user, passcode=args.password)
if (args.selector):
conn.subscribe(
destination=args.destination,
id=1,
ack='auto',
headers={'selector': args.selector}
)
else:
conn.subscribe(
destination=args.destination,
id=1,
ack='auto'
)
Now I can run this subscriber with a selector such as "type = 'test'".
If I publish a message using Java JMS, the message is received just fine. However, if I publish the identical message from Python it is not.
Here's the relevant Python publishing code:
headers = {}
headers['type'] = 'test'
conn = stomp.Connection12([(args.host, args.port)], auto_content_length=False)
conn.start()
conn.connect(login=args.user, passcode=args.password)
conn.send(body=body, headers=headers, destination=args.destination)
conn.disconnect()
print 'Message sent.'
Some interesting notes from my testing and debugging:
Running the subscriber with a selector receives a matching message sent from Java JMS but not from Python.
Running the subscriber with no selector receives a message sent from Java and also a message sent from Python.

fairly old, but I was currently facing the same issue and so I would like to leave a possible solution here.
At first, according to the documentation, you can provide a field called selectorwith SQL like syntax and should be part of the headers. In your example:
headers = {}
headers['selector'] = "type='test'"
conn = stomp.Connection12([(args.host, args.port)], auto_content_length=False)
conn.start()
conn.connect(login=args.user, passcode=args.password)
conn.send(body=body, headers=headers, destination=args.destination)
conn.disconnect()
print 'Message sent.'
I was also facing the error, that I could not receive any message send from JMS, but after a lot of reading, I found here, that there is a field name JMSType. I changed the code to
headers['selector'] = "type='test' OR JMSType='test'"
With that JMSType in there everything works like expected. Hope that helps somebody