Related
I have a program that is taking over 2 minutes to run and I'm not sure what I could do to reduce the runtime. It's most definitely the for loop shown in the code.
"CPU Start Address in decimal"
CPU_START_ADDR = 1644167168 #0x62000000
"Import necessary libraries"
import time
import pandas as pd
import numpy as np
"Creates variable for time to display runtime"
start_time = time.time()
"Create a blank dataframe with necessary column headers"
columnNames = ["Settings 1 Values", "Settings 2 Values", "CPU Address", "FPGA Address", "Delta (Decimal)", "Delta (Pos. Difference)", "Register Name", "R/W"]
output = pd.DataFrame(columns = columnNames)
"Fill values from settings files into output dataframe"
df1 = pd.read_csv("50MHzWholeFPGA.csv")
df2 = pd.read_csv("75MHzWholeFPGA.csv")
spec = pd.read_excel("Mozart FPGA Register Specification.xlsx", skiprows=[0,1,2,3])
output.loc[:, "Settings 1 Values"] = df1.iloc[:, 0]
output.loc[:, "Settings 2 Values"] = df2.iloc[:, 0]
output['Delta (Decimal)'] = output['Settings 2 Values'] - output['Settings 1 Values']
"For loop generates CPU Addresses for all values"
for index, row in output.iterrows():
output.loc[index, 'CPU Address'] = hex(CPU_START_ADDR + (2 * index))[2:]
settingXor = bin(int(output.loc[index, "Settings 1 Values"]) ^ int(output.loc[index, "Settings 2 Values"]))
output.loc[index, "Delta (Pos. Difference)"] = settingXor
I know this question has been asked before, but I've tried to implement all the solutions I've found and still haven't solved the problem.
Here is my code
import csv
import pandas as pd
import helperFunctions import pandas tickers = []
f = open("watchlist.txt", "r") for i in f:
tickers.append((helperFunctions.getTicker(i)))
head = ["Name", "Growth", "Recommendation", "Earnings Growth", "Current Ratio",
"Total Cash", "Debt", " Revenue", "Percentage Shares Out", "Percentage Institutions", "Percentage Insiders",
"Price to Book", "Short Ratio", "Regular Market Price"] df = pd.DataFrame() df = pd.DataFrame(columns=head) try:
for i in tickers:
currentTicker = []
currentTicker.append(i.info['longName'])
currentTicker.append(i.info['revenueGrowth'])
currentTicker.append(i.info['recommendationKey'])
currentTicker.append(i.info['earningsGrowth'])
currentTicker.append(i.info['currentRatio'])
currentTicker.append(i.info['totalCash'])
currentTicker.append(i.info['totalDebt'])
currentTicker.append(i.info['totalRevenue'])
currentTicker.append(i.info['sharesPercentSharesOut'])
currentTicker.append(i.info['heldPercentInstitutions'])
currentTicker.append(i.info['heldPercentInsiders'])
currentTicker.append(i.info['priceToBook'])
currentTicker.append(i.info['shortRatio'])
currentTicker.append(i.info['beta'])
currentTicker.append(i.info['regularMarketPrice'])
print(str(currentTicker + "\n"))
'''
# Why Don't these work??
1.
df.append(currentTicker)
2.
df_length = len(df)
df.loc[df_length] = currentTicker
3.
a_series = pd.Series(currentTicker, index=df.columns)
df = df.append(a_series, ignore_index=True)
'''
except Exception as e:
print(str(i) + repr(e))
print(df)
In the section where you see comments in the list, those are all things things I've attempted to add each iteration to the dataframe. Basically the watchlist is a txtfile with some tickers, and I would like to get that data and put it into a dataframe and I'm running into some trouble. Thanks for your time.
This was how I ended up solving it
import csv
import pandas as pd
import helperFunctions
import pandas
tickers = []
f = open("watchlist.txt", "r")
for i in f:
tickers.append((helperFunctions.getTicker(i)))
head = ["Name", "Growth", "Recommendation", "Earnings Growth", "Current Ratio",
"Total Cash", "Debt", "Revenue", "Percentage Shares Out", "Percentage Institutions", "Percentage Insiders",
"Price to Book", "Short Ratio", "Regular Market Price"]
df = pd.DataFrame(columns=head)
try:
for i in tickers:
currentTicker = []
currentTicker.append(i.info['longName'])
currentTicker.append(i.info['revenueGrowth'])
currentTicker.append(i.info['recommendationKey'])
currentTicker.append(i.info['earningsGrowth'])
currentTicker.append(i.info['currentRatio'])
currentTicker.append(i.info['totalCash'])
currentTicker.append(i.info['totalDebt'])
currentTicker.append(i.info['totalRevenue'])
currentTicker.append(i.info['sharesPercentSharesOut'])
currentTicker.append(i.info['heldPercentInstitutions'])
currentTicker.append(i.info['heldPercentInsiders'])
currentTicker.append(i.info['priceToBook'])
currentTicker.append(i.info['shortRatio'])
currentTicker.append(i.info['beta'])
currentTicker.append(i.info['regularMarketPrice'])
df = df.append({'Name': currentTicker[0],
'Growth': currentTicker[1],
"Recommendation":currentTicker[2],
"Earnings Growth":currentTicker[3],
"Current Ratio": currentTicker[4],
"Total Cash":currentTicker[5],
"Debt": currentTicker[6],
"Revenue": currentTicker[7],
"Percentage Shares Out": currentTicker[8],
"Percentage Institutions": currentTicker[9],
"Percentage Insiders": currentTicker[10],
"Price to Book": currentTicker[11],
"Short Ratio": currentTicker[12],
"Regular Market Price": currentTicker[13],
},
ignore_index=True)
#print(currentTicker)
except Exception as e:
print(str(i) + repr(e))
print(df)
The issue in your code seems to be related to the fact that the currentTicker list has an extra column, namely beta, which is not present in the columns of the df.
Once you either add that column to head or remove it from currentTicker, method 2 and 3 will work.
import csv
import pandas as pd
import helperFunctions
with open("watchlist.txt", "r") as f:
tickers = [helperFunctions.getTicker(i) for i in f]
head = [
"Name",
"Growth",
"Recommendation",
"Earnings Growth",
"Current Ratio",
"Total Cash",
"Debt",
"Revenue",
"Percentage Shares Out",
"Percentage Institutions",
"Percentage Insiders",
"Price to Book",
"Short Ratio",
"Regular Market Price"
]
df = pd.DataFrame(columns=head)
columns_to_extract = [
'longName',
'revenueGrowth',
'recommendationKey',
'earningsGrowth',
'currentRatio',
'totalCash',
'totalDebt',
'totalRevenue',
'sharesPercentSharesOut',
'heldPercentInstitutions',
'heldPercentInsiders',
'priceToBook',
'shortRatio',
# 'beta', # <- The column for this value is missing from `head`
'regularMarketPrice'
]
currentTicker = [i.info[key] for key in columns_to_extract]
# 1. Method - THIS DOES NOT WORK
# df.append(currentTicker)
# 2. Method
df_length = len(df)
df.loc[df_length] = currentTicker
# 3. Method
a_series = pd.Series(currentTicker, index=df.columns)
df = df.append(a_series, ignore_index=True)
print(df)
I have been tasked to build a process in python that would extract the data from Elasticsearch, drop data in an Azure Blob after which Snowflake would ingest the data. I have the process running on Azure Functions that extracts an index group (like game_name.*) and for each index in the index group, it creates a thread to scroll on. I save the last date of each result and on the next run parse it in the range query. I am running the process every five minutes and have offset the end of the range by 5 minutes (we have a refresh running every 2 minutes). I let the process run for a while and then I do a gap analysis by taking a count(*) in both Elasticsearch and Snowflake by hour (or by day) and expect to have a max of 1% gap. However, for one index pattern which groups about 127 indexes, when I run a catchup job (for a day or more), the resulting gap is as expected, however, as soon as I let it run on the cron job (every 5 min), after a while I get gaps of 6-10% and only for this index group.
It looks as if the scroller function picks up an N amount of documents within the queried range but then for some reason documents are later added (PUT) with an earlier date. Or I might be wrong and my code is doing something funny. I've talked to our team and they don't cache any docs on the client, and the data is synced to a network clock (not the client's) and sending UTC.
Please see below the query I am using to paginate through elasticsearch:
def query(searchSize, lastRowDateOffset, endDate, pit, keep_alive):
body = {
"size": searchSize,
"query": {
"bool": {
"must": [
{
"exists": {
"field": "baseCtx.date"
}
},
{
"range": {
"baseCtx.date": {
"gt": lastRowDateOffset,
"lte": endDate
}
}
}
]
}
},
"pit": {
"id": pit,
"keep_alive": keep_alive
},
"sort": [
{
"baseCtx.date": {"order": "asc", "unmapped_type": "long"}
},
{
"_shard_doc": "asc"
}
],
"track_total_hits": False
}
return body
def scroller(pit,
threadNr,
index,
lastRowDateOffset,
endDate,
maxThreads,
es,
lastRowCount,
keep_alive="1m",
searchSize=10000):
cumulativeResultCount = 0
iterationResultCount = 0
data = []
dataBytes = b''
lastIndexDate = ''
startScroll = time.perf_counter()
while 1:
if lastRowCount == 0: break
#if lastRowDateOffset == endDate: lastRowCount = 0; break
try:
page = es.search(body=body)
except: # It is believed that the point in time is getting closed, hence the below opens a new one
pit = es.open_point_in_time(index=index, keep_alive=keep_alive)['id']
body = query(searchSize, lastRowDateOffset, endDate, pit, keep_alive)
page = es.search(body=body)
pit = page['pit_id']
data += page['hits']['hits']
body['pit']['id'] = pit
if len(data) > 0: body['search_after'] = [x['sort'] for x in page['hits']['hits']][-1]
cumulativeResultCount += len(page['hits']['hits'])
iterationResultCount = len(page['hits']['hits'])
#print(f"This Iteration Result Count: {iterationResultCount} -- Cumulative Results Count: {cumulativeResultCount} -- {time.perf_counter() - startScroll} seconds")
if iterationResultCount < searchSize: break
if len(data) > rowsPerMB * maxSizeMB / maxThreads: break
if time.perf_counter() - startScroll > maxProcessTimeSeconds: break
if len(data) != 0:
dataBytes = gzip.compress(bytes(json.dumps(data)[1:-1], encoding='utf-8'))
lastIndexDate = max([x['_source']['baseCtx']['date'] for x in data])
response = {
"pit": pit,
"index": index,
"threadNr": threadNr,
"dataBytes": dataBytes,
"lastIndexDate": lastIndexDate,
"cumulativeResultCount": cumulativeResultCount
}
return response
def batch(game_name, env='prod', startDate='auto', endDate='auto', writeDate=True, minutesOffset=5):
es = Elasticsearch(
esUrl,
port=9200,
timeout=300)
lowerFormat = game_name.lower().replace(" ","_")
indexGroup = lowerFormat + "*"
if env == 'dev': lowerFormat, indexGroup = 'dev_' + lowerFormat, 'dev.' + indexGroup
azFormat = re.sub(r'[^0-9a-zA-Z]+', '-', game_name).lower()
storageContainerName = azFormat
curFileName = f"{lowerFormat}_cursors.json"
curBlobFilePath = f"cursors/{curFileName}"
compressedTools = [gzip.compress(bytes('[', encoding='utf-8')), gzip.compress(bytes(',', encoding='utf-8')), gzip.compress(bytes(']', encoding='utf-8'))]
pits = []
lastRowCounts = []
# Parameter and state settings
if os.getenv(f"{lowerFormat}_maxSizeMB") is not None: maxSizeMB = int(os.getenv(f"{lowerFormat}_maxSizeMB"))
if os.getenv(f"{lowerFormat}_maxThreads") is not None: maxThreads = int(os.getenv(f"{lowerFormat}_maxThreads"))
if os.getenv(f"{lowerFormat}_maxProcessTimeSeconds") is not None: maxProcessTimeSeconds = int(os.getenv(f"{lowerFormat}_maxProcessTimeSeconds"))
# Get all indices for the indexGroup
indicesEs = list(set([(re.findall(r"^.*-", x)[0][:-1] if '-' in x else x) + '*' for x in list(es.indices.get(indexGroup).keys())]))
indices = [{"indexName": x, "lastOffsetDate": (datetime.datetime.utcnow()-datetime.timedelta(days=5)).strftime("%Y/%m/%d 00:00:00")} for x in indicesEs]
# Load Cursors
cursors = getCursors(curBlobFilePath, indices)
# Offset the current time by -5 minutes to account for the 2-3 min delay in Elasticsearch
initTime = datetime.datetime.utcnow()
if endDate == 'auto': endDate = f"{initTime-datetime.timedelta(minutes=minutesOffset):%Y/%m/%d %H:%M:%S}"
print(f"Less than or Equal to: {endDate}, {keep_alive}")
# Start Multi-Threading
while 1:
dataBytes = []
dataSize = 0
start = time.perf_counter()
if len(pits) == 0: pits = ['' for x in range(len(cursors))]
if len(lastRowCounts) == 0: lastRowCounts = ['' for x in range(len(cursors))]
with concurrent.futures.ThreadPoolExecutor(max_workers=len(cursors)) as executor:
results = [
executor.submit(
scroller,
pit,
threadNr,
x['indexName'],
x['lastOffsetDate'] if startDate == 'auto' else startDate,
endDate,
len(cursors),
es,
lastRowCount,
keep_alive,
searchSize) for x, pit, threadNr, lastRowCount in (zip(cursors, pits, list(range(len(cursors))), lastRowCounts))
]
for f in concurrent.futures.as_completed(results):
if f.result()['lastIndexDate'] != '': cursors[f.result()['threadNr']]['lastOffsetDate'] = f.result()['lastIndexDate']
pits[f.result()['threadNr']] = f.result()['pit']
lastRowCounts[f.result()['threadNr']] = f.result()['cumulativeResultCount']
dataSize += f.result()['cumulativeResultCount']
if len(f.result()['dataBytes']) > 0: dataBytes.append(f.result()['dataBytes'])
print(f"Thread {f.result()['threadNr']+1}/{len(cursors)} -- Index {f.result()['index']} -- Results pulled {f.result()['cumulativeResultCount']} -- Cumulative Results: {dataSize} -- Process Time: {round(time.perf_counter()-start, 2)} sec")
if dataSize == 0: break
lastRowDateOffsetDT = datetime.datetime.strptime(max([x['lastOffsetDate'] for x in cursors]), '%Y/%m/%d %H:%M:%S')
outFile = f"elasticsearch/live/{lastRowDateOffsetDT:%Y/%m/%d/%H}/{lowerFormat}_live_{lastRowDateOffsetDT:%Y%m%d%H%M%S}_{datetime.datetime.utcnow():%Y%m%d%H%M%S}.json.gz"
print(f"Starting compression of {dataSize} rows -- {round(time.perf_counter()-start, 2)} sec")
dataBytes = compressedTools[0] + compressedTools[1].join(dataBytes) + compressedTools[2]
# Upload to Blob
print(f"Comencing to upload data to blob -- {round(time.perf_counter()-start, 2)} sec")
uploadJsonGzipBlobBytes(outFile, dataBytes, storageContainerName, len(dataBytes))
print(f"File compiled: {outFile} -- {dataSize} rows -- Process Time: {round(time.perf_counter()-start, 2)} sec\n")
# Update cursors
if writeDate: postCursors(curBlobFilePath, cursors)
# Clean Up
print("Closing PITs")
for pit in pits:
try: es.close_point_in_time({"id": pit})
except: pass
print(f"Closing Connection to {esUrl}")
es.close()
return
# Start the process
while 1:
batch("My App")
I think I just need a second pair of eyes to point out where the issue might be in the code. I've tried increasing the minutesOffset argv to 60 (so every 5 minutes it pulls the data from the last run until Now()-60 minutes) but had the same issue. Please help.
So the "baseCtx.date" is triggered by the client and it seems that in some cases there is a delay between when the event is triggered and when it is available to be searched. We fixed this by using the ingest pipeline as follows:
PUT _ingest/pipeline/indexDate
{
"description": "Creates a timestamp when a document is initially indexed",
"version": 1,
"processors": [
{
"set": {
"field": "indexDate",
"value": "{{{_ingest.timestamp}}}",
"tag": "indexDate"
}
}
]
}
And set index.default_pipeline to "indexDate" in the template settings. Every month the index name changes (we append the year and month) and this approach creates a server date we used to scroll.
When trying to place a buy or sell order with the python-binance api I got the following error:
APIError(code=-1013): Filter failure: LOT_SIZE.
Now I've seen at iceberg_parts that this means there is probably something wrong with my buying or selling quantity. I've tried to increase the quantity by a factor 10 but this only gives me another related error:
APIError(code=-1013): Filter failure: MIN_NOTIONAL.
Here's some of my code:
diff = current_price - prev_price
if diff <= 0.0001:
order = client.order_market_buy(symbol = market , quantity = '0.0001')
print('buy order')
if diff >= 0.00040:
order = client.order_market_sell(symbol =market, quantity ='0.0001')
print('sell order')
Do you know how to fix this?
This error appears because you are trying to create an order with a quantity lower than the minimun required.
You can access the minimun required of a specific pair with:
info = client.get_symbol_info('ETHUSDT')
print(info)
Output a dictionary with information about that pair.
Now you can access the minimun quantity required with:
print(info['filters'][2]['minQty'])
# 0.00001
The buying or selling quantity has to be >= 10.3 USD or 10.3/price, pass the quantity and price to these decimal settings/filters with the amounts set with decimal
from decimal import Decimal as D, ROUND_DOWN, ROUND_UP
import decimal
info = client.get_symbol_info(symbol=pair)
price_filter = float(info['filters'][0]['tickSize'])
ticker = client.get_symbol_ticker(symbol=pair)
price = float(ticker['price'])
price = D.from_float(price).quantize(D(str(price_filter)))
minimum = float(info['filters'][2]['minQty']) # 'minQty'
quant = D.from_float(quantity).quantize(D(str(minimum))) # if quantity >= 10.3/price
I've just gone through this same problem. As a noob, some of the code in these answers seem quite complicated so I came up with a solution.
Code:
def check_decimals(symbol):
info = client.get_symbol_info(symbol)
val = info['filters'][2]['stepSize']
decimal = 0
is_dec = False
for c in val:
if is_dec is True:
decimal += 1
if c == '1':
break
if c == '.':
is_dec = True
return decimal
then when you place the order, just do for ex:
(make sure qty is a float or decimal)
B_order = round(qty / symbol_price, decimal)
order = client.order_market_buy(
symbol=symbol_name,
quantity=B_order)
Maybe this can explain why the server returns this error.
Filters
From the endpoint GET /api/v3/exchangeInfo, you can find all details regarding the trading symbols. It includes many filters that clients need to follow to place an order. For example, the BTCUSDT has the filters as of today(2022-08-31)
"filters": [
{
"filterType": "PRICE_FILTER",
"minPrice": "0.01000000",
"maxPrice": "1000000.00000000",
"tickSize": "0.01000000"
},
{
"filterType": "PERCENT_PRICE",
"multiplierUp": "5",
"multiplierDown": "0.2",
"avgPriceMins": 5
},
{
"filterType": "LOT_SIZE",
"minQty": "0.00001000",
"maxQty": "9000.00000000",
"stepSize": "0.00001000"
},
{
"filterType": "MIN_NOTIONAL",
"minNotional": "10.00000000",
"applyToMarket": true,
"avgPriceMins": 5
},
{
"filterType": "ICEBERG_PARTS",
"limit": 10
},
{
"filterType": "MARKET_LOT_SIZE",
"minQty": "0.00000000",
"maxQty": "282.39806510",
"stepSize": "0.00000000"
},
{
"filterType": "TRAILING_DELTA",
"minTrailingAboveDelta": 10,
"maxTrailingAboveDelta": 2000,
"minTrailingBelowDelta": 10,
"maxTrailingBelowDelta": 2000
},
{
"filterType": "MAX_NUM_ORDERS",
"maxNumOrders": 200
},
{
"filterType": "MAX_NUM_ALGO_ORDERS",
"maxNumAlgoOrders": 5
}
]
LOT_SIZE validation
minQty
If you place an order on this BTCUSDT with parameters:
price=19000
side=BUY
type=LIMIT
quantity=0.000005
that is a LIMIT BUY order with price of $19,000, but the quantity is less than minQty in the LOT_SIZE:
0.000005 < 0.00001000
then the server will reject the order, because the request can't pass this filter validation.
✗ LOT_SIZE.minQty
stepSize
Can I place an order with the same parameters but only change the quantity to 0.000015? That is:
price=19000
side=BUY
type=LIMIT
quantity=0.000015
You will still receive this error, because the quantity is not able to pass the stepSize size validation: (quantity- minQty) % stepSize == 0
(0.000015 - 0.00001) % 0.00001 != 0
✓ LOT_SIZE.minQty
✗ LOT_SIZE.stepSize
MIN_NOTIONAL Validation
Alright, let us change the quantity to 0.00002, with same parameters:
price=19000
side=BUY
type=LIMIT
quantity=0.00002
The order will still be rejected with a different error because it can't pass the filter MIN_NOTIONAL validation.
19000 x 0.00002 = 0.38 < 10 (MIN_NOTIONAL.minNotional)
Note:
minNotional defines the minimum notional value that required for each order.
For MARKET order, the average price is used over the last avgPriceMins minutes.
✓ LOT_SIZE.minQty
✓ LOT_SIZE.stepSize
✗ MIN_NOTIONAL.minNotional
Here is some code.
def round_down(self, coin, number):
info = self.client.get_symbol_info('%sUSDT' % coin)
step_size = [float(_['stepSize']) for _ in info['filters'] if _['filterType'] == 'LOT_SIZE'][0]
step_size = '%.8f' % step_size
step_size = step_size.rstrip('0')
decimals = len(step_size.split('.')[1])
return math.floor(number * 10 ** decimals) / 10 ** decimals
https://python-binance.readthedocs.io/en/latest/account.html
from binance.helpers import round_step_size
# to get a lot size
def getLotSize(self):
info = self.apiCall(lambda: self.client.get_symbol_info(self.pair))
lotSize = float(info['filters'][2]['minQty'])
return lotSize
# get ceiling value and correct format for a lot size
def getCeilingVal(self):
pairData = self.apiCall(lambda:
self.client.get_symbol_ticker(symbol=self.pair))
pairPrice = pairData["price"]
ceilingVal = float(self.dInv) / float(pairPrice)
aLotSize = self.getLotSize()
rounded_amount = round_step_size(ceilingVal, aLotSize)
return rounded_amount
Here's a very helpful code using binance-python package
...
// Full code: https://github.com/ndiecodes/binance-trading-bot/blob/main/main.py
def get_round_step_quantity(self, qty):
info = self.client.get_symbol_info(Config.TRADESYMBOL)
for x in info["filters"]:
if x["filterType"] == "LOT_SIZE":
self.minQty = float(x["minQty"])
self.maxQty = float(x["maxQty"])
self.stepSize= float(x["stepSize"])
if qty < self.minQty:
qty = self.minQty
return round_step_size(quantity=qty, step_size=self.stepSize)
I have read through all of these forum questions and no one has mentioned the fact that Binance charges a 0.1% fee on all transactions. Meaning you do not have your original buying quantity available to sell back when the sell is triggered.
I have attempted to solve this with:
buy_quantity = round(buy_amount * 0.999, len(str(lotsize).split('.')[1]))
Multiplying my original purchase quantity by 0.999 should reduce it by the amount needed to be able to sell it back.
Hi adding a bit further to #stack if I've 20 dollars to buy then my quantity would be as below
I've done like below
decimal_places=abs(Decimal(symbl_info['filters'][2]["stepSize"]).normalize().as_tuple().exponent)
print("DECIMAL PLACES {0}".format(decimal_places))
buy_qauntity= round((20/order_input["askPrice"].values[0]),decimal_places)
print(buy_qauntity)
I write a function like that. It's working for me.
def getPriceLotFormat(self, priceOrg, quantityOrg):
price = float(priceOrg)
quantity = float(quantityOrg)
response = self.get_symbol_info(car.pair) #self is client btw
priceFilterFloat = format(float(response["filters"][0]["tickSize"]), '.20f')
lotSizeFloat = format(float(response["filters"][2]["stepSize"]), '.20f')
# PriceFilter
numberAfterDot = str(priceFilterFloat.split(".")[1])
indexOfOne = numberAfterDot.find("1")
if indexOfOne == -1:
price = int(price)
else:
price = round(float(price), int(indexOfOne - 1))
# LotSize
numberAfterDotLot = str(lotSizeFloat.split(".")[1])
indexOfOneLot = numberAfterDotLot.find("1")
if indexOfOneLot == -1:
quantity = int(quantity)
else:
quantity = round(float(quantity), int(indexOfOneLot))
print(f"""
##### SELL #####
Pair : {str(car.pair)}
Cash : {str(car.price)}
Quantity : {str(car.quantity)}
Price : {str(car.price)}
""")
We can use the Log10 function to get rounding precision from the Binance /api/v3/exchangeinfo endpoint data.
CurrencyRoundNum = int(math.Abs(math.Log10(stepSize)))
PriceRoundNum = int(math.Abs(math.Log10(tickSize)))
The full version on golang is here, or at go playground. I'm sorry that code is not on python.
So I was struggling with the LOT_SIZE error myself.
Previously I was using the round_step_size function from the python-binance library, however, I had to edit this function to deal with this API error.
Here is a function that I use:
from decimal import Decimal, ROUND_DOWN
import math
from typing import Union
def round_step_size(quantity: Union[float, Decimal], step_size: Union[float, Decimal]) -> float:
if step_size == 1.0:
return math.floor(quantity)
elif step_size < 1.0:
return Decimal(f'{quantity}').quantize(Decimal(f'{step_size}'), rounding=ROUND_DOWN)
We're working on a Python project, where we are retrieving data from our MySQL database, and we're then sending it back to a new table on our database. We have initialized a list,
list_lpn_temp = []
The problem is that the range of this list varies, and therefore we don't always know how many datapoints we will have in our list. We have this code, and this is where the error occurs:
df2 = pd.DataFrame(columns=['first_temp_lpn', 'first_temp_lpn_validated', 'second_temp_lpn', 'second_temp_lpn_validated', 'third_temp_lpn', 'third_temp_lpn_validated'])
df2 = df2.append({'first_temp_lpn' : list_lpn_temp[0][0], 'first_temp_lpn_validated' : list_validated[0], 'second_temp_lpn' : list_lpn_temp[1][0], 'second_temp_lpn_validated' : list_validated[1], 'third_temp_lpn' : list_lpn_temp[2][0], 'third_temp_lpn_validated' : list_validated[2]}, ignore_index=True).round(2)
with engine.connect() as conn, conn.begin():
df2.to_sql('Raw_Validated', conn, if_exists='append', index=False)
Sometimes it gives us an error saying index out of range, as we sometimes only have 2 values in the list, and therefore list_lpn_temp[3][0] will give us the error. Dream scenario would be, if we could somehow send a null or maybe some text saying that we dont have any value to our database.
Therefore we need 2 things:
Send data, but where it depends on the size of our list, and is not just set static. For example like this (We need something better than this):
'first_temp_lpn' : list_lpn_temp[0][0]
If we are receiving index out of range, then we still need to send something to the database, as it expects 3x columns of temperature. But as there are no values, we could send a null, and therefore this could be nice to implement. Otherwise we will just get another big issue.
BIGGER PART OF THE CODE
engine = create_engine("mysql://xxx:xxx#localhost/xxx")
conn = engine.connect()
list_lpn_temp = []
index = pd.date_range(start=start_range.min(), end=end_range.max(), freq='20T')
for x in index:
a_temp = pd.read_sql('SELECT temperature FROM Raw_Data', conn).astype(float).values
list_lpn_temp.extend(a_temp)
if len(list_lpn_temp) > max_samples:
list_lpn_temp.pop(0)
for i in range (len(list_lpn_temp)):
if -1.5 < 25-list_lpn_temp[i] < 1.5:
validated_lpn = 1
list_validated.append(validated_lpn)
new_list_lpn_temp.extend(list_lpn_temp[i])
else:
validated_lpn = 0
list_validated.append(validated_lpn)
df2 = pd.DataFrame(columns=['first_temp_lpn', 'first_temp_lpn_validated', 'second_temp_lpn', 'second_temp_lpn_validated', 'third_temp_lpn', 'third_temp_lpn_validated'])
df2 = df2.append({'first_temp_lpn' : list_lpn_temp[0][0], 'first_temp_lpn_validated' : list_validated[0], 'second_temp_lpn' : list_lpn_temp[1][0], 'second_temp_lpn_validated' : list_validated[1], 'third_temp_lpn' : list_lpn_temp[2][0], 'third_temp_lpn_validated' : list_validated[2]}, ignore_index=True).round(2)
with engine.connect() as conn, conn.begin():
df2.to_sql('Raw_Validated', conn, if_exists='append', index=False)
NEW (KP)
We have a time_start and time_end value, which is formatted to datetime. We want to send it with the temp, so we have tried to modify the df2.append.
lastTime = pd.read_sql('SELECT MAX(timestamp) FROM Raw_Data', conn).astype(str).values.tolist()
firstTime = pd.read_sql('SELECT MIN(timestamp) FROM Raw_Data', conn).astype(str).values.tolist()
firstTime = (pd.to_datetime(firstTime[0])-datetime.timedelta(minutes=10)).round('20T')
lastTime = (pd.to_datetime(lastTime[0])-datetime.timedelta(minutes=10)).round('20T')
test = lastTime - datetime.timedelta(minutes=40)
time_start = test.astype(str).values[0]
lastTime = lastTime + datetime.timedelta(minutes=20)
time_end = lastTime.astype(str).values[0]
for name, value, valid in zip(['first', 'second', 'third'], list_lpn_temp, list_validated):
temp[name+'_temp_lpn'] = value[0]
temp[name+'_temp_lpn_validated'] = valid
df2 = df2.append({'time_start' : time_start, 'time_end' : time_end}, temp)
print (df2)
But then only datetime is being sent (time_start and time_end)
you can loop over the elements in the list.
Something like
temp = {}
for name, value in zip(['first', 'second', 'third'], list_lpn_temp):
temp[name+'_temp_lpn'] = value[0]
temp[name+'_temp_lpn_validated'] = value[1]
df2 = df2.append(temp)