I have gotten a few steps further along with my simulation and analysis of the machines that I want to simulate. As of now I am stuck and do not for the life of my know how to go further.
I have decided to add stochastic elements to my machine. Before (see code below) I had a Machine class that has produce, failure and repair process. My professor and I have decided that using stochasticity will be a better representation of reality. Therefore I have the following elements I would like to add and explanation of the process:
All Machines starts in state 1
Every second (tick) a processing speed is chosen from possibilities (% chance)
Immediately a time in state is chosen from distribution (or random.choice() for ease of use)
After the time in state has passed, a new state is chosen from possibilities (% chance)
The process repeats until simulation time is over.
There is no more machine failure and repair process as this is taken up in the different states.
I have an idea that I will need to make 3 functions, for determining state, speed and time in state only I have no idea how to do this.
The different parameters/possibilities per machine are as follows:
Machine 1
If in state 1:
Possible speeds (in ticks): { 3: 54%, 4: 24%, 5 : 22% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 0: 41%, 2: 54%}
If in state 2:
Possible speeds (in ticks): { 2: 90%, 0: 10% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 0: 60%, 1: 40%}
If in state 0:
Possible speeds (in ticks): { 0: 100% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 1: 100% }
Machine 2
If in state 1:
Possible speeds (in ticks): { 5: 42%, 4: 34%, 7 : 24% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 0: 70%, 2: 30%}
If in state 2:
Possible speeds (in ticks): { 1: 80%, 0: 20% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 0: 90%, 1: 10%}
If in state 0:
Possible speeds (in ticks): { 0: 100% }
Duration: Exponential distribution or random.choice()
Possible next state after duration: { 1: 100% }
Any help will be much much appreciated!
I am adding the code of my simulation as I have it now. I do know that a few elements are obsolete due to now needing to fail and repair anymore but I am not sure how to change it up.
# Parameters for the Machines
# Machine 1
speed_1 = 0.1 # Avg. processing time of Machine 1 in minutes
# speed_1_stdev = 0.6 # St. dev. of processing time of Machine 1
MTTF_1 = 9 # Mean time to failure Machine 1
# fail_1 = 1/MTTF_1 # Parameter for exp. distribution
repair_1 = 5 # Time it takes to repair Machine 1
# Machine 2
speed_2 = 0.15 # Processing time of Machine 2 in minutes
# speed_2_stdev = 0.6 # St. dev. of processing time of Machine 2
MTTF_2 = 7 # Mean time to failure Machine 2
# fail_2 = 1/MTTF_2 # Parameter for exp. distribution
repair_2 = 3 # Time it takes to repair Machine 2
# Simulation time
time = 60 # Sim time in minutes
# Class setup for a Machine
class Machine(object):
"""
A machine produces units at a fixed processing speed,
takes units from a store before and puts units into a store after.
Machine has a *name*, a processing speed *speed*, a preceeding buffer *in_q*,
and a proceeding buffer *out_q*.
"""
def __init__(self, env, name, in_q, out_q, speed, mttf, repair):
self.env = env
self.name = name
self.in_q = in_q
self.out_q = out_q
self.speed = speed
self.mttf = mttf
self.repair = repair
self.broken = False
self.count = 0
# Start the producing process
self.process = env.process(self.produce())
# Start the failure process
env.process(self.fail_machine())
def produce(self):
"""
Produce parts as long as the simulation runs.
It is prone to breaking down and being repaired.
"""
while True:
pspeed = self.speed
try:
yield env.timeout(pspeed)
if len(self.out_q.items) >= self.out_q.capacity:
pspeed = 0
print(f'{self.env.now:.2f} {self.name} output buffer full!!!')
# Add the "Tailback" status to the event list
event_list.append({"Time": self.env.now, "Machine": self.name,
"State": "Tailback", "Speed": pspeed,
"Count": self.count, "Tailback": 1})
else:
pspeed = self.speed
part = yield self.in_q.get()
yield self.out_q.put(part)
self.count += 1
# Add the "Running" status to the event list
event_list.append({"Time": self.env.now, "Machine": self.name,
"State": "Running", "Speed": pspeed,
"Count": self.count, "Tailback": 0})
except simpy.Interrupt: # This ensures that the machine breaks down
self.broken = True
pspeed = 0
yield self.env.timeout(self.repair)
print(f'{self.env.now:.2f} {self.name} is fixed')
# Add the "Repaired" event notification to the event list
event_list.append({"Time": self.env.now, "Machine": self.name,
"State": "Repaired", "Speed": pspeed,
"Count": self.count, "Tailback": 0})
self.broken = False
pspeed = self.speed
env.process(self.fail_machine())
def fail_machine(self):
"""
The machine is prone to break down every now and then.
"""
yield self.env.timeout(self.mttf)
pspeed = 0
print(f'{self.env.now:.2f} {self.name} is in failure.')
# Add the "Failure" event notification to the event list
event_list.append({"Time": self.env.now, "Machine": self.name,
"State": "Failure", "Speed": pspeed,
"Count": self.count, "Tailback": 0})
if not self.broken:
# Machine only fails if currently working.
self.process.interrupt(self.mttf)
# Generating the arrival of parts in the entry buffer to be used by machine 1
def gen_arrivals(env, entry_buffer):
"""
Start the process for each part by putting
the part in the starting buffer
"""
while True:
yield env.timeout(random.uniform(0,0.0001))
# print(f'{env.now:.2f} part has arrived')
part = object() # Too lazy to make a real part class, also isn't necessary
yield entry_buffer.put(part)
# Print the status of the number of processed parts per machine every minute
def print_status(env, machine, period):
while True:
yield env.timeout(period)
print(f"{env.now}: {machine.name} has outputted {machine.count}")
# Create environment and start the setup process
env = simpy.Environment()
bufferStart = simpy.Store(env) # Buffer with unlimited capacity
buffer1 = simpy.Store(env) # Buffer between machines with unlimited capacity
bufferEnd = simpy.Store(env) # Last buffer with unlimited capacity
# The machines __init__ starts the machine process so no env.process() is needed here
machine_1 = Machine(env, 'Machine 1', bufferStart, buffer1, speed_1, MTTF_1, repair_1)
machine_2 = Machine(env, 'Machine 2', buffer1, bufferEnd, speed_2, MTTF_2, repair_2)
# Process necessary to run all events
env.process(gen_arrivals(env, bufferStart))
env.process(print_status(env, machine_1, 1))
env.process(print_status(env, machine_2, 1))
# Dictionary to log events
event_list = [{"Time": 0.00, "Machine": "All",
"State": "Simulation start",
"Speed": 0, "Count": 0, "Tailback": 0}]
# Execute
env.run(until = time)
Related
Full disclosure: I am a student who has likely messed up some conventions, notation and best practices. I warmly welcome feedback.
I am trying to implement a version of the gas station example found in the SimPy documentation, attached here:
Covers:
- Resources: Resource
- Resources: Container
- Waiting for other processes
Scenario:
A gas station has a limited number of gas pumps that share a common
fuel reservoir. Cars randomly arrive at the gas station, request one
of the fuel pumps and start refueling from that reservoir.
A gas station control process observes the gas station's fuel level
and calls a tank truck for refueling if the station's level drops
below a threshold.
import itertools
import random
import simpy
RANDOM_SEED = 42
GAS_STATION_SIZE = 200 # liters
THRESHOLD = 10 # Threshold for calling the tank truck (in %)
FUEL_TANK_SIZE = 50 # liters
FUEL_TANK_LEVEL = [5, 25] # Min/max levels of fuel tanks (in liters)
REFUELING_SPEED = 2 # liters / second
TANK_TRUCK_TIME = 300 # Seconds it takes the tank truck to arrive
T_INTER = [30, 300] # Create a car every [min, max] seconds
SIM_TIME = 1000 # Simulation time in seconds
def car(name, env, gas_station, fuel_pump):
"""A car arrives at the gas station for refueling.
It requests one of the gas station's fuel pumps and tries to get the
desired amount of gas from it. If the stations reservoir is
depleted, the car has to wait for the tank truck to arrive.
"""
fuel_tank_level = random.randint(*FUEL_TANK_LEVEL)
print('%s arriving at gas station at %.1f' % (name, env.now))
with gas_station.request() as req:
start = env.now
# Request one of the gas pumps
yield req
# Get the required amount of fuel
liters_required = FUEL_TANK_SIZE - fuel_tank_level
yield fuel_pump.get(liters_required)
# The "actual" refueling process takes some time
yield env.timeout(liters_required / REFUELING_SPEED)
print('%s finished refueling in %.1f seconds.' % (name,
env.now - start))
def gas_station_control(env, fuel_pump):
"""Periodically check the level of the *fuel_pump* and call the tank
truck if the level falls below a threshold."""
while True:
if fuel_pump.level / fuel_pump.capacity * 100 < THRESHOLD:
# We need to call the tank truck now!
print('Calling tank truck at %d' % env.now)
# Wait for the tank truck to arrive and refuel the station
yield env.process(tank_truck(env, fuel_pump))
yield env.timeout(10) # Check every 10 seconds
def tank_truck(env, fuel_pump):
"""Arrives at the gas station after a certain delay and refuels it."""
yield env.timeout(TANK_TRUCK_TIME)
print('Tank truck arriving at time %d' % env.now)
amount = fuel_pump.capacity - fuel_pump.level
print('Tank truck refuelling %.1f liters.' % amount)
yield fuel_pump.put(amount)
def car_generator(env, gas_station, fuel_pump):
"""Generate new cars that arrive at the gas station."""
for i in itertools.count():
yield env.timeout(random.randint(*T_INTER))
env.process(car('Car %d' % i, env, gas_station, fuel_pump))
# Setup and start the simulation
print('Gas Station refuelling')
random.seed(RANDOM_SEED)
# Create environment and start processes
env = simpy.Environment()
gas_station = simpy.Resource(env, 2)
fuel_pump = simpy.Container(env, GAS_STATION_SIZE, init=GAS_STATION_SIZE)
env.process(gas_station_control(env, fuel_pump))
env.process(car_generator(env, gas_station, fuel_pump))
# Execute!
env.run(until=SIM_TIME)
My biggest source of frustration has been to try to implement some sort of way to yield the container to the tank truck such that no cars may get gas while the gas station is replenishing its stock of gas (via the tank truck). I have tried to put if statements in the car definition to check the status of the gas stations fuel stock or yield the resource when the loop in the gas_station_control condition is triggered but have been unsuccessful.
My scenario is:
3 random cars in a set time-frame. The capacity of the gas station is 1.5 tanks of gas, and it takes 3 hours for a car to pump one load. Once that car leaves I need to refill my gas stations reservoir which takes 4 hours(if the level=.5 # a rate of .25 loads/hour, 6 hours to fill all 1.5 tanks). I also need to make sure that if no car is currently waiting in the queue I should be refilling the reservoir, irregardless if I am out of the current time frame where cars may pull up. I need to make sure that I am logging the time that each car waits in the queue.
I am still not sure what you trying to do. Why cannot the cars have a pending request while the truck is replenishing the container? Those requests do not block the truck from adding to the container.
Any way, I took a stab at it and here is what I cam up with
In this version, when the car gets to the pump it checks if the container has enough gas to refuel the car. If it does it will make the container request. If it does not, it will wait for the truck to arrive and replenish the gas station and then make the container request. The car knows when replenishment is done because I added a replenish_event to the gas_station which gets triggered by the truck when the truck finishes replenishment. If the car needs to wait, it can yield to this event. Note that more then one car can yield to this event.
"""
Covers:
- Resources: Resource
- Resources: Container
- Waiting for other processes
Scenario:
A gas station has a limited number of gas pumps that share a common
fuel reservoir. Cars randomly arrive at the gas station, request one
of the fuel pumps and start refueling from that reservoir.
A gas station control process observes the gas station's fuel level
and calls a tank truck for refueling if the station's level drops
below a threshold.
Updated my Michel R. Gibbs
Car does not make a request for gas amount unless it can get full amount.
If cannot get full amount, will wait for truck to replenish before making request
"""
import itertools
import random
import simpy
RANDOM_SEED = 42
GAS_STATION_SIZE = 200 # liters
THRESHOLD = 10 # Threshold for calling the tank truck (in %)
FUEL_TANK_SIZE = 50 # liters
FUEL_TANK_LEVEL = [5, 25] # Min/max levels of fuel tanks (in liters)
REFUELING_SPEED = 2 # liters / second
TANK_TRUCK_TIME = 300 # Seconds it takes the tank truck to arrive
T_INTER = [30, 300] # Create a car every [min, max] seconds
SIM_TIME = 2000 # Simulation time in seconds
REPLENISHMENT_SPEED = 20 # linters / second for tanker truck to refill gas staiion
def car(name, env, gas_station, fuel_pump, replenish_event):
"""A car arrives at the gas station for refueling.
It requests one of the gas station's fuel pumps and tries to get the
desired amount of gas from it. If the stations reservoir is
depleted, the car has to wait for the tank truck to arrive.
"""
fuel_tank_level = random.randint(*FUEL_TANK_LEVEL)
print('%s arriving at gas station at %.1f' % (name, env.now))
with gas_station.request() as req:
start = env.now
# Request one of the gas pumps
yield req
print('%s exit queue and started pumping gas at %.1f' % (name, env.now))
# Get the required amount of fuel
liters_required = FUEL_TANK_SIZE - fuel_tank_level
# check if amount is available
while liters_required > fuel_pump.level:
# use a while in case another car uses up the replenishment
# wait for truck
print('%s waiting for fuel truck before starting to pump %.1f' % (name, env.now))
yield replenish_event
yield fuel_pump.get(liters_required)
# The "actual" refueling process takes some time
yield env.timeout(liters_required / REFUELING_SPEED)
print('%s finished refueling in %.1f seconds.' % (name,
env.now - start))
def gas_station_control(env, gas_station, fuel_pump):
"""
Periodically check the level of the *fuel_pump* and call the tank
truck if the level falls below a threshold.
"""
gas_station.replenish_event = env.event()
while True:
if fuel_pump.level / fuel_pump.capacity * 100 < THRESHOLD:
# We need to call the tank truck now!
print('Calling tank truck at %d' % env.now)
env.process(tank_truck(env, fuel_pump, gas_station.replenish_event))
# yield until replenishment is finished
yield gas_station.replenish_event
# reset event for next replenishment
gas_station.replenish_event = env.event()
yield env.timeout(10) # Check every 10 seconds
def tank_truck(env, fuel_pump, replenish_event):
"""
Arrives at the gas station after a certain delay and refuels it.
"""
yield env.timeout(TANK_TRUCK_TIME)
print('Tank truck arriving at time %d' % env.now)
amount = fuel_pump.capacity - fuel_pump.level
replem_time = amount / REPLENISHMENT_SPEED
yield env.timeout(replem_time)
print('Tank truck replenshied %.1f liters at time %.1f.' % (amount, env.now))
yield fuel_pump.put(amount)
# let any listeners know replenishment is done
replenish_event.succeed()
def car_generator(env, gas_station, fuel_pump):
"""
Generate new cars that arrive at the gas station.
"""
for i in itertools.count():
yield env.timeout(random.randint(*T_INTER))
env.process(car('Car %d' % i, env, gas_station, fuel_pump, gas_station.replenish_event))
# Setup and start the simulation
print('Gas Station refuelling')
random.seed(RANDOM_SEED)
# Create environment and start processes
env = simpy.Environment()
gas_station = simpy.Resource(env, 2)
fuel_pump = simpy.Container(env, GAS_STATION_SIZE, init=GAS_STATION_SIZE)
env.process(gas_station_control(env, gas_station, fuel_pump))
env.process(car_generator(env, gas_station, fuel_pump))
# Execute!
env.run(until=SIM_TIME)
I have been tasked to build a process in python that would extract the data from Elasticsearch, drop data in an Azure Blob after which Snowflake would ingest the data. I have the process running on Azure Functions that extracts an index group (like game_name.*) and for each index in the index group, it creates a thread to scroll on. I save the last date of each result and on the next run parse it in the range query. I am running the process every five minutes and have offset the end of the range by 5 minutes (we have a refresh running every 2 minutes). I let the process run for a while and then I do a gap analysis by taking a count(*) in both Elasticsearch and Snowflake by hour (or by day) and expect to have a max of 1% gap. However, for one index pattern which groups about 127 indexes, when I run a catchup job (for a day or more), the resulting gap is as expected, however, as soon as I let it run on the cron job (every 5 min), after a while I get gaps of 6-10% and only for this index group.
It looks as if the scroller function picks up an N amount of documents within the queried range but then for some reason documents are later added (PUT) with an earlier date. Or I might be wrong and my code is doing something funny. I've talked to our team and they don't cache any docs on the client, and the data is synced to a network clock (not the client's) and sending UTC.
Please see below the query I am using to paginate through elasticsearch:
def query(searchSize, lastRowDateOffset, endDate, pit, keep_alive):
body = {
"size": searchSize,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "baseCtx.date"
}
},
{
"range": {
"baseCtx.date": {
"gt": lastRowDateOffset,
"lte": endDate
}
}
}
]
}
},
"pit": {
"id": pit,
"keep_alive": keep_alive
},
"sort": [
{
"baseCtx.date": {"order": "asc", "unmapped_type": "long"}
},
{
"_shard_doc": "asc"
}
],
"track_total_hits": False
}
return body
def scroller(pit,
threadNr,
index,
lastRowDateOffset,
endDate,
maxThreads,
es,
lastRowCount,
keep_alive="1m",
searchSize=10000):
cumulativeResultCount = 0
iterationResultCount = 0
data = []
dataBytes = b''
lastIndexDate = ''
startScroll = time.perf_counter()
while 1:
if lastRowCount == 0: break
#if lastRowDateOffset == endDate: lastRowCount = 0; break
try:
page = es.search(body=body)
except: # It is believed that the point in time is getting closed, hence the below opens a new one
pit = es.open_point_in_time(index=index, keep_alive=keep_alive)['id']
body = query(searchSize, lastRowDateOffset, endDate, pit, keep_alive)
page = es.search(body=body)
pit = page['pit_id']
data += page['hits']['hits']
body['pit']['id'] = pit
if len(data) > 0: body['search_after'] = [x['sort'] for x in page['hits']['hits']][-1]
cumulativeResultCount += len(page['hits']['hits'])
iterationResultCount = len(page['hits']['hits'])
#print(f"This Iteration Result Count: {iterationResultCount} -- Cumulative Results Count: {cumulativeResultCount} -- {time.perf_counter() - startScroll} seconds")
if iterationResultCount < searchSize: break
if len(data) > rowsPerMB * maxSizeMB / maxThreads: break
if time.perf_counter() - startScroll > maxProcessTimeSeconds: break
if len(data) != 0:
dataBytes = gzip.compress(bytes(json.dumps(data)[1:-1], encoding='utf-8'))
lastIndexDate = max([x['_source']['baseCtx']['date'] for x in data])
response = {
"pit": pit,
"index": index,
"threadNr": threadNr,
"dataBytes": dataBytes,
"lastIndexDate": lastIndexDate,
"cumulativeResultCount": cumulativeResultCount
}
return response
def batch(game_name, env='prod', startDate='auto', endDate='auto', writeDate=True, minutesOffset=5):
es = Elasticsearch(
esUrl,
port=9200,
timeout=300)
lowerFormat = game_name.lower().replace(" ","_")
indexGroup = lowerFormat + "*"
if env == 'dev': lowerFormat, indexGroup = 'dev_' + lowerFormat, 'dev.' + indexGroup
azFormat = re.sub(r'[^0-9a-zA-Z]+', '-', game_name).lower()
storageContainerName = azFormat
curFileName = f"{lowerFormat}_cursors.json"
curBlobFilePath = f"cursors/{curFileName}"
compressedTools = [gzip.compress(bytes('[', encoding='utf-8')), gzip.compress(bytes(',', encoding='utf-8')), gzip.compress(bytes(']', encoding='utf-8'))]
pits = []
lastRowCounts = []
# Parameter and state settings
if os.getenv(f"{lowerFormat}_maxSizeMB") is not None: maxSizeMB = int(os.getenv(f"{lowerFormat}_maxSizeMB"))
if os.getenv(f"{lowerFormat}_maxThreads") is not None: maxThreads = int(os.getenv(f"{lowerFormat}_maxThreads"))
if os.getenv(f"{lowerFormat}_maxProcessTimeSeconds") is not None: maxProcessTimeSeconds = int(os.getenv(f"{lowerFormat}_maxProcessTimeSeconds"))
# Get all indices for the indexGroup
indicesEs = list(set([(re.findall(r"^.*-", x)[0][:-1] if '-' in x else x) + '*' for x in list(es.indices.get(indexGroup).keys())]))
indices = [{"indexName": x, "lastOffsetDate": (datetime.datetime.utcnow()-datetime.timedelta(days=5)).strftime("%Y/%m/%d 00:00:00")} for x in indicesEs]
# Load Cursors
cursors = getCursors(curBlobFilePath, indices)
# Offset the current time by -5 minutes to account for the 2-3 min delay in Elasticsearch
initTime = datetime.datetime.utcnow()
if endDate == 'auto': endDate = f"{initTime-datetime.timedelta(minutes=minutesOffset):%Y/%m/%d %H:%M:%S}"
print(f"Less than or Equal to: {endDate}, {keep_alive}")
# Start Multi-Threading
while 1:
dataBytes = []
dataSize = 0
start = time.perf_counter()
if len(pits) == 0: pits = ['' for x in range(len(cursors))]
if len(lastRowCounts) == 0: lastRowCounts = ['' for x in range(len(cursors))]
with concurrent.futures.ThreadPoolExecutor(max_workers=len(cursors)) as executor:
results = [
executor.submit(
scroller,
pit,
threadNr,
x['indexName'],
x['lastOffsetDate'] if startDate == 'auto' else startDate,
endDate,
len(cursors),
es,
lastRowCount,
keep_alive,
searchSize) for x, pit, threadNr, lastRowCount in (zip(cursors, pits, list(range(len(cursors))), lastRowCounts))
]
for f in concurrent.futures.as_completed(results):
if f.result()['lastIndexDate'] != '': cursors[f.result()['threadNr']]['lastOffsetDate'] = f.result()['lastIndexDate']
pits[f.result()['threadNr']] = f.result()['pit']
lastRowCounts[f.result()['threadNr']] = f.result()['cumulativeResultCount']
dataSize += f.result()['cumulativeResultCount']
if len(f.result()['dataBytes']) > 0: dataBytes.append(f.result()['dataBytes'])
print(f"Thread {f.result()['threadNr']+1}/{len(cursors)} -- Index {f.result()['index']} -- Results pulled {f.result()['cumulativeResultCount']} -- Cumulative Results: {dataSize} -- Process Time: {round(time.perf_counter()-start, 2)} sec")
if dataSize == 0: break
lastRowDateOffsetDT = datetime.datetime.strptime(max([x['lastOffsetDate'] for x in cursors]), '%Y/%m/%d %H:%M:%S')
outFile = f"elasticsearch/live/{lastRowDateOffsetDT:%Y/%m/%d/%H}/{lowerFormat}_live_{lastRowDateOffsetDT:%Y%m%d%H%M%S}_{datetime.datetime.utcnow():%Y%m%d%H%M%S}.json.gz"
print(f"Starting compression of {dataSize} rows -- {round(time.perf_counter()-start, 2)} sec")
dataBytes = compressedTools[0] + compressedTools[1].join(dataBytes) + compressedTools[2]
# Upload to Blob
print(f"Comencing to upload data to blob -- {round(time.perf_counter()-start, 2)} sec")
uploadJsonGzipBlobBytes(outFile, dataBytes, storageContainerName, len(dataBytes))
print(f"File compiled: {outFile} -- {dataSize} rows -- Process Time: {round(time.perf_counter()-start, 2)} sec\n")
# Update cursors
if writeDate: postCursors(curBlobFilePath, cursors)
# Clean Up
print("Closing PITs")
for pit in pits:
try: es.close_point_in_time({"id": pit})
except: pass
print(f"Closing Connection to {esUrl}")
es.close()
return
# Start the process
while 1:
batch("My App")
I think I just need a second pair of eyes to point out where the issue might be in the code. I've tried increasing the minutesOffset argv to 60 (so every 5 minutes it pulls the data from the last run until Now()-60 minutes) but had the same issue. Please help.
So the "baseCtx.date" is triggered by the client and it seems that in some cases there is a delay between when the event is triggered and when it is available to be searched. We fixed this by using the ingest pipeline as follows:
PUT _ingest/pipeline/indexDate
{
"description": "Creates a timestamp when a document is initially indexed",
"version": 1,
"processors": [
{
"set": {
"field": "indexDate",
"value": "{{{_ingest.timestamp}}}",
"tag": "indexDate"
}
}
]
}
And set index.default_pipeline to "indexDate" in the template settings. Every month the index name changes (we append the year and month) and this approach creates a server date we used to scroll.
Python:
I am simulating a call-centre with 2 types of incoming calls: Sales calls, and service calls.
These calls have different, independent distributions, which enter the same system.
I have function, arrivals which contains:
iat_sales = random.expovariate(1/3)
yield env.timeout(iat_sales)
I want to incorporate:
iat_service = random.triangular(0,0,6)
yield env.timeout(iat_service)
how can I yield each event simultaneously?
This is the solution I have come up with:
def arrival_list():
sales_time = 0 #sim time of sales arrival
service_time = 0 #sim time of service arrival
sales_list=[] #list of sequential sales arrivals [arrival time,'sales']
service_list=[] #list of sequential sales arrivals [arrival time,'service']
arrivals = [] #ordered list of arrivals (sales and service merged) [arrival time,arrival type,iat]
while sales_time < sim_end:
iat_sales = random.expovariate(sales_rate)
sales_time += iat_sales
sales=[sales_time,'sales']
sales_list.append(sales)
while service_time < sim_end:
iat_service = random.triangular(0,6,0) ####
service_time += iat_service
service=[service_time,'service']
service_list.append(service)
arrivals = sales_list + service_list
arrivals.sort()
arrivals[0].append(arrivals[0][0])
for i in range(len(arrivals)-1):
arrivals[i+1].append(arrivals[i+1][0]-arrivals[i][0])
return arrivals
As a reference, a simple implementation can be done like this, where a simulation is run indefinitely with 1 second intervals and calls are considered to arrive if their random values exceed some thresholds:
import random
import time
def generate_calls():
return random.expovariate(1/3), random.triangular(10, 20, 5)
def simulation(count, sales_acceptance, services_acceptance):
# run the simulation indefinitely
while True:
print('Time: {}'.format(count))
sales_call, services_call = generate_calls()
# calls arrive if the values exceed some thresholds
if sales_call > sales_acceptance:
print('Sales call arrived!')
if services_call > services_acceptance:
print('Services call arrived!')
time.sleep(1)
count += 1
simulation(1, 2, 13)
You can have three separate parallel processes.
1- One process for making Sales calls.
2- One process for making service calls.
3- One process for handling calls.
import simpy
import random
sim_end = 1000;
def generateSalesCall(env, call_pipe):
while env.now < sim_end:
# put call in the pipe
yield call_pipe.put("sales");
interval = random.expovariate(1/3);
yield env.timeout(interval);
def generateServiceCall(env, call_pipe):
while env.now < sim_end:
# put call in the pipe
yield call_pipe.put("service");
interval = random.triangular(0,6,0);
yield env.timeout(interval);
def handleCalls(env, call_pipe):
while(True):
call = yield call_pipe.get();
if call == "sales":
print(env.now, "sales call");
elif call == "service":
print(env.now, "service call");
env = simpy.Environment();
call_pipe = simpy.Store(env);
env.process(generateSalesCall(env, call_pipe));
env.process(generateServiceCall(env, call_pipe));
env.process(handleCalls(env, call_pipe));
env.run();
When trying to place a buy or sell order with the python-binance api I got the following error:
APIError(code=-1013): Filter failure: LOT_SIZE.
Now I've seen at iceberg_parts that this means there is probably something wrong with my buying or selling quantity. I've tried to increase the quantity by a factor 10 but this only gives me another related error:
APIError(code=-1013): Filter failure: MIN_NOTIONAL.
Here's some of my code:
diff = current_price - prev_price
if diff <= 0.0001:
order = client.order_market_buy(symbol = market , quantity = '0.0001')
print('buy order')
if diff >= 0.00040:
order = client.order_market_sell(symbol =market, quantity ='0.0001')
print('sell order')
Do you know how to fix this?
This error appears because you are trying to create an order with a quantity lower than the minimun required.
You can access the minimun required of a specific pair with:
info = client.get_symbol_info('ETHUSDT')
print(info)
Output a dictionary with information about that pair.
Now you can access the minimun quantity required with:
print(info['filters'][2]['minQty'])
# 0.00001
The buying or selling quantity has to be >= 10.3 USD or 10.3/price, pass the quantity and price to these decimal settings/filters with the amounts set with decimal
from decimal import Decimal as D, ROUND_DOWN, ROUND_UP
import decimal
info = client.get_symbol_info(symbol=pair)
price_filter = float(info['filters'][0]['tickSize'])
ticker = client.get_symbol_ticker(symbol=pair)
price = float(ticker['price'])
price = D.from_float(price).quantize(D(str(price_filter)))
minimum = float(info['filters'][2]['minQty']) # 'minQty'
quant = D.from_float(quantity).quantize(D(str(minimum))) # if quantity >= 10.3/price
I've just gone through this same problem. As a noob, some of the code in these answers seem quite complicated so I came up with a solution.
Code:
def check_decimals(symbol):
info = client.get_symbol_info(symbol)
val = info['filters'][2]['stepSize']
decimal = 0
is_dec = False
for c in val:
if is_dec is True:
decimal += 1
if c == '1':
break
if c == '.':
is_dec = True
return decimal
then when you place the order, just do for ex:
(make sure qty is a float or decimal)
B_order = round(qty / symbol_price, decimal)
order = client.order_market_buy(
symbol=symbol_name,
quantity=B_order)
Maybe this can explain why the server returns this error.
Filters
From the endpoint GET /api/v3/exchangeInfo, you can find all details regarding the trading symbols. It includes many filters that clients need to follow to place an order. For example, the BTCUSDT has the filters as of today(2022-08-31)
"filters": [
{
"filterType": "PRICE_FILTER",
"minPrice": "0.01000000",
"maxPrice": "1000000.00000000",
"tickSize": "0.01000000"
},
{
"filterType": "PERCENT_PRICE",
"multiplierUp": "5",
"multiplierDown": "0.2",
"avgPriceMins": 5
},
{
"filterType": "LOT_SIZE",
"minQty": "0.00001000",
"maxQty": "9000.00000000",
"stepSize": "0.00001000"
},
{
"filterType": "MIN_NOTIONAL",
"minNotional": "10.00000000",
"applyToMarket": true,
"avgPriceMins": 5
},
{
"filterType": "ICEBERG_PARTS",
"limit": 10
},
{
"filterType": "MARKET_LOT_SIZE",
"minQty": "0.00000000",
"maxQty": "282.39806510",
"stepSize": "0.00000000"
},
{
"filterType": "TRAILING_DELTA",
"minTrailingAboveDelta": 10,
"maxTrailingAboveDelta": 2000,
"minTrailingBelowDelta": 10,
"maxTrailingBelowDelta": 2000
},
{
"filterType": "MAX_NUM_ORDERS",
"maxNumOrders": 200
},
{
"filterType": "MAX_NUM_ALGO_ORDERS",
"maxNumAlgoOrders": 5
}
]
LOT_SIZE validation
minQty
If you place an order on this BTCUSDT with parameters:
price=19000
side=BUY
type=LIMIT
quantity=0.000005
that is a LIMIT BUY order with price of $19,000, but the quantity is less than minQty in the LOT_SIZE:
0.000005 < 0.00001000
then the server will reject the order, because the request can't pass this filter validation.
✗ LOT_SIZE.minQty
stepSize
Can I place an order with the same parameters but only change the quantity to 0.000015? That is:
price=19000
side=BUY
type=LIMIT
quantity=0.000015
You will still receive this error, because the quantity is not able to pass the stepSize size validation: (quantity- minQty) % stepSize == 0
(0.000015 - 0.00001) % 0.00001 != 0
✓ LOT_SIZE.minQty
✗ LOT_SIZE.stepSize
MIN_NOTIONAL Validation
Alright, let us change the quantity to 0.00002, with same parameters:
price=19000
side=BUY
type=LIMIT
quantity=0.00002
The order will still be rejected with a different error because it can't pass the filter MIN_NOTIONAL validation.
19000 x 0.00002 = 0.38 < 10 (MIN_NOTIONAL.minNotional)
Note:
minNotional defines the minimum notional value that required for each order.
For MARKET order, the average price is used over the last avgPriceMins minutes.
✓ LOT_SIZE.minQty
✓ LOT_SIZE.stepSize
✗ MIN_NOTIONAL.minNotional
Here is some code.
def round_down(self, coin, number):
info = self.client.get_symbol_info('%sUSDT' % coin)
step_size = [float(_['stepSize']) for _ in info['filters'] if _['filterType'] == 'LOT_SIZE'][0]
step_size = '%.8f' % step_size
step_size = step_size.rstrip('0')
decimals = len(step_size.split('.')[1])
return math.floor(number * 10 ** decimals) / 10 ** decimals
https://python-binance.readthedocs.io/en/latest/account.html
from binance.helpers import round_step_size
# to get a lot size
def getLotSize(self):
info = self.apiCall(lambda: self.client.get_symbol_info(self.pair))
lotSize = float(info['filters'][2]['minQty'])
return lotSize
# get ceiling value and correct format for a lot size
def getCeilingVal(self):
pairData = self.apiCall(lambda:
self.client.get_symbol_ticker(symbol=self.pair))
pairPrice = pairData["price"]
ceilingVal = float(self.dInv) / float(pairPrice)
aLotSize = self.getLotSize()
rounded_amount = round_step_size(ceilingVal, aLotSize)
return rounded_amount
Here's a very helpful code using binance-python package
...
// Full code: https://github.com/ndiecodes/binance-trading-bot/blob/main/main.py
def get_round_step_quantity(self, qty):
info = self.client.get_symbol_info(Config.TRADESYMBOL)
for x in info["filters"]:
if x["filterType"] == "LOT_SIZE":
self.minQty = float(x["minQty"])
self.maxQty = float(x["maxQty"])
self.stepSize= float(x["stepSize"])
if qty < self.minQty:
qty = self.minQty
return round_step_size(quantity=qty, step_size=self.stepSize)
I have read through all of these forum questions and no one has mentioned the fact that Binance charges a 0.1% fee on all transactions. Meaning you do not have your original buying quantity available to sell back when the sell is triggered.
I have attempted to solve this with:
buy_quantity = round(buy_amount * 0.999, len(str(lotsize).split('.')[1]))
Multiplying my original purchase quantity by 0.999 should reduce it by the amount needed to be able to sell it back.
Hi adding a bit further to #stack if I've 20 dollars to buy then my quantity would be as below
I've done like below
decimal_places=abs(Decimal(symbl_info['filters'][2]["stepSize"]).normalize().as_tuple().exponent)
print("DECIMAL PLACES {0}".format(decimal_places))
buy_qauntity= round((20/order_input["askPrice"].values[0]),decimal_places)
print(buy_qauntity)
I write a function like that. It's working for me.
def getPriceLotFormat(self, priceOrg, quantityOrg):
price = float(priceOrg)
quantity = float(quantityOrg)
response = self.get_symbol_info(car.pair) #self is client btw
priceFilterFloat = format(float(response["filters"][0]["tickSize"]), '.20f')
lotSizeFloat = format(float(response["filters"][2]["stepSize"]), '.20f')
# PriceFilter
numberAfterDot = str(priceFilterFloat.split(".")[1])
indexOfOne = numberAfterDot.find("1")
if indexOfOne == -1:
price = int(price)
else:
price = round(float(price), int(indexOfOne - 1))
# LotSize
numberAfterDotLot = str(lotSizeFloat.split(".")[1])
indexOfOneLot = numberAfterDotLot.find("1")
if indexOfOneLot == -1:
quantity = int(quantity)
else:
quantity = round(float(quantity), int(indexOfOneLot))
print(f"""
##### SELL #####
Pair : {str(car.pair)}
Cash : {str(car.price)}
Quantity : {str(car.quantity)}
Price : {str(car.price)}
""")
We can use the Log10 function to get rounding precision from the Binance /api/v3/exchangeinfo endpoint data.
CurrencyRoundNum = int(math.Abs(math.Log10(stepSize)))
PriceRoundNum = int(math.Abs(math.Log10(tickSize)))
The full version on golang is here, or at go playground. I'm sorry that code is not on python.
So I was struggling with the LOT_SIZE error myself.
Previously I was using the round_step_size function from the python-binance library, however, I had to edit this function to deal with this API error.
Here is a function that I use:
from decimal import Decimal, ROUND_DOWN
import math
from typing import Union
def round_step_size(quantity: Union[float, Decimal], step_size: Union[float, Decimal]) -> float:
if step_size == 1.0:
return math.floor(quantity)
elif step_size < 1.0:
return Decimal(f'{quantity}').quantize(Decimal(f'{step_size}'), rounding=ROUND_DOWN)
I am designing a task manager for a month using graphics in python. I have created the calendar for the month of July. I would like to display the tasks for a given date, add new tasks, remove/delete tasks and save the changes to a file.
The maximum number of to do tasks for any day is 5. Initially, the list of tasks are read from a file. Each row in the file contains the date followed by a list of semi-colon separated tasks for that date. The following shows an example file:
1; Go to class;Book Train Ticket
2; Call Home
3; Wash the Clothes
7; Submit the Assignment Online
8; Give Assignment Demo;Prepare for Major
10; Take the Exam
11; Go Home
To the right of calendar, there is a textbox to enter a new task to the list of to-do tasks for the date currently being displayed. After entering the new task in the textbox, I need to click on the ’ok’ button to add it to the list. As soon as the ’ok’ button is pressed, the list of tasks being displayed should be refreshed to reflect the addition of the new task. The textbox should be cleared after the changes have been incorporated into the task list. Here's my code:
from graphics import *
win =GraphWin("Task Manager",800,800)
head = Text(Point(130,15),"July2014")
head.setSize(16)
head.draw(win)
add = Text(Point(440,28),"Add Task :")
add.setSize(17)
add.draw(win)
adde = Entry(Point(560,28),20)
adde.draw(win)
rem = Text(Point(440,78),"Remove Task:")
rem.setSize(17)
rem.draw(win)
reme = Entry(Point(570,78),20)
reme.draw(win)
addr = Rectangle(Point(680,15),Point(640,45))
addr.setFill("yellow")
addr.draw(win)
okt = Text(Point(660,30),"ok")
okt.setSize(16)
okt.draw(win)
remr = Rectangle(Point(690,60),Point(650,90))
remr.setFill("green")
remr.draw(win)
okt = Text(Point(670,75),"ok")
okt.setSize(16)
okt.draw(win)
savr =Rectangle(Point(440,120),Point(540,150))
savr.setFill("orange")
savr.draw(win)
savt = Text(Point(490,135),"Save to file")
savt.setSize(15)
savt.draw(win)
exir = Rectangle(Point(590,120),Point(650,150))
exir.setFill("red")
exir.draw(win)
exis = Text(Point(620,135),"exit")
exis.setSize(15)
exis.draw(win)
def isInside(p,rect):
rectP1 = rect.getP1();
rectP2 = rect.getP2();
if(p.getX() >= rectP1.getX() and p.getX() <= rectP2.getX() and
p.getY() >= rectP1.getY() and p.getY() <= rectP2.getY()):
return True;
else:
return False;
dtext = Text(Point(180,350),"no date selected yet")
dtext.setSize(16)
dtext.draw(win)
def rect(a):
squ = Rectangle(Point(20+((a-1)%7)*40,70+(((a-1)//7)-1)*30),Point(20+(((a- 1)%7)+1)*40,70+((a-1)//7)*30))
return squ
for i in range(1,43):
rect(i).draw(win)
l =['sun','mon','tue','wed','thu','fri','sat']
def elem(a):
p = Point(40+((a-1)%7)*40,85+(((a-1)//7)-1)*30)
sti = Text(p,l[a-1])
sti.setSize(15)
return sti
for i in range(1,8):
elem(i).draw(win)
def num(a):
p = Point(40+((a-1)%7)*40,85+(((a-1)//7)-1)*30)
b = Text(p,a-9)
return b
for j in range(10,41):
num(j).draw(win)
id = Text(Point(50,400),"id")
id.setSize(17)
id.draw(win)
task = Text(Point(200,400),"task")
task.setSize(17)
task.draw(win)
while 9 < a< 41:
inputP = win.getMouse()
if isInside(inputP,rect(a)):
dtext.setText("tasks on July"+rect(a).getText()+"are")
else:
dtext.setText("wrong")
win.getMouse()
win.close()