engine = sqlalchemy.create_engine('sqlite:///CRYPTO.db')
client = Client()
info = client.get_exchange_info()
# Create empty dict and get klines in iteration
columns = [
'Time', 'Open', 'High', 'Low', 'Close', 'Volume', 'Close Time', 'Asset Volume',
'Number Of Trades', 'Buy Base', 'Buy Quote', 'Ignore',
]
klines = {}
for symbol in USDTsymbol:
r_klines = client.get_historical_klines(
symbol, Client.KLINE_INTERVAL_1MONTH,
"1 April 2018", "1 DEC, 2020"
)
r_klines2 = client.get_historical_klines(
symbol, Client.KLINE_INTERVAL_1WEEK,
"1 JAN 2021", "1 JUNE, 2021"
df = pd.DataFrame(r_klines + r_klines2, columns=columns)
klines[symbol] = df
print(df)
df.Time = pd.to_datetime(df.Time, unit='ms')
for df in klines:
data = klines[df]
data[columns].to_sql(symbol, engine, if_exists='replace')
-How to update database from last left off instead of redownloading data?
-->Will appreciate your help. Thank you
Related
def get_price_history(symbol, interval):
session = spot.HTTP(endpoint="https://api.bybit.com")
kline = session.query_kline(symbol=symbol, interval=interval)
kline = kline['result']
df = pd.DataFrame(kline)
df = df.iloc[:, :6]
df.columns = ['Time', 'Open', 'High', 'Low', 'Close', 'Volume']
df = df.set_index('Time')
df.index = pd.to_datetime(df.index, unit='ms')
df = df.astype(float)
return df['Close']
this function gets data from bybit api
symbol is VAIUSDT VAI is crypto token named vaiot
thats an error:
pybit.exceptions.InvalidRequestError: Not supported symbols (ErrCode: -100011) (ErrTime: 19:07:06).
Request → GET https://api.bybit.com/spot/quote/v1/kline: {'symbol': 'VAIUSDT', 'interval': '15m'}.
Month dropdown list unable to display value according to the year selected.
Error message:
ValueError: ('Lengths must match to compare', (36,), (1,))
I had a year dropdown list, require to display the month list based on the year selected and the date list according to the month selected.
Date: 01/02/2021, 29/02/2021, 20/12/2021, 05/06/2021, 25/07/2021, 01/05/2022, 20/03/2022, 21/09/2022, 30/07/2022
Year list : 2021, 2022
Year selected: 2021
Month list: Feb, June, July, Dec
Month selected: Feb
Date list: 01/02/2021, 29/02/2021
Code:
df['year'] = pd.to_datetime(df['Date'], format = '%d/%m/%Y', errors='coerce').dt.year
month_labels = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug',9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'}
df['month'] = pd.to_datetime(df['Date'], format = '%d/%m/%Y', errors='coerce').dt.month
df['monthname']= df['month'].apply(lambda x: month_labels[x])
df['date'] = pd.to_datetime(df['Date'], format = '%d/%m/%Y', errors='coerce')
df.sort_values(by= ['year', 'month', 'Date'], inplace = True)
month_cat = list(df['monthname'].unique())
sort_mm = sorted(month_cat, key=lambda m: datetime.strptime(m, "%b"))
sort_yr = df['year'].drop_duplicates()
dbc.Row([
dbc.Col([
html.H3('Year'),
html.Br(),
dcc.Dropdown(id='year_dd', value= [df['year'].max()],
options = [{'label':x, 'value':x}
for x in sort_yr],
),
html.Br(),
html.H3('Month'),
html.Br(),
dcc.Dropdown(id='month_dd', value= ''
)
#create month selection based on year
#app.callback(
Output('month_dd','options'),
Input('year_dd', 'value')
)
def update_dd (year_dd):
year_month= df.drop_duplicates(['Date'], inplace= False)
relevant_month= year_month[df['year'] == year_dd]['monthname'].values.tolist()
month_opt= [dict(label=x,value=x)for x in relevant_month]
return month_opt
#set latest date as default value according to month dropdown
#app.callback(
Output('month_dd','value'),
Input('month_dd', 'options')
)
def default_value(latest_date):
monthvalue = latest_date[-1]['value']
return monthvalue
####### can ignore this part, is just a reference ############
#create date dropdown according to month dropdown
#app.callback(
Output('date_dd','options'),
Input('month_dd', 'value')
)
def update_dd (month_dd):
month_date= df.drop_duplicates(['Date'], inplace= False)
relevant_date= month_date[df['monthname'] == month_dd]['Date'].values.tolist()
date_option= [dict(label=x,value=x)for x in relevant_date]
return date_option
#set latest date as default value according to month dropdown
#app.callback(
Output('date_dd','value'),
Input('date_dd', 'options')
)
def default_value(latest_date):
datevalue = latest_date[-1]['value']
return datevalue
I have two excel files from two different wholesalers with products and stock quantity information.
Some of the products in the two files are common, so they exist in both files.
The number of products in the files is different e.g. the first has 65000 products and the second has 9000 products.
I need to iterate through the products of the first file based on the common column 'EAN CODE' and check if the current product exists also in the EAN column of the 2nd file.
Afterwards check which product has the lower price (which has stock > 0) and print the matching row of this product to another output excel file.
import os
import re
from datetime import datetime
import pandas
from utils import recognize_excel_type
dataframes = []
input_directory = 'in'
for file in os.listdir(input_directory):
file_path = os.path.join(input_directory, file)
if file.lower().endswith('xlsx') or file.lower().endswith('xls'):
dataframes.append(
pandas.read_excel(file_path)
)
elif file.lower().endswith('csv'):
dataframes.append(
pandas.read_csv(file_path, delimiter=';')
)
combined_dataframe = pandas.DataFrame(columns=['Price', 'Stock', 'EAN Code'])
for dataframe in dataframes:
this_type = recognize_excel_type(dataframe)
if this_type == 'DIFOX':
dataframe.rename(columns={
'retail price': 'Price',
'availability (steps)': 'Stock',
'EAN number 1': 'EAN Code',
}, inplace=True)
tuned_dataframe = pandas.DataFrame(
dataframe[combined_dataframe.columns],
)
combined_dataframe = combined_dataframe.append(tuned_dataframe, ignore_index=True)
elif this_type == 'ECOM_VGA':
headers = dataframe.iloc[2]
dataframe = dataframe[3:]
dataframe.columns = headers
dataframe.rename(columns={
'Price (€)': 'Price',
'Stock': 'Stock',
'EAN Code': 'EAN Code',
}, inplace=True)
tuned_dataframe = pandas.DataFrame(
dataframe[combined_dataframe.columns],
)
combined_dataframe = combined_dataframe.append(tuned_dataframe, ignore_index=True)
elif this_type == 'MAXCOM':
dataframe.rename(columns={
'VK-Preis': 'Price',
'Verfügbar': 'Stock',
'EAN-Code': 'EAN Code',
}, inplace=True)
tuned_dataframe = pandas.DataFrame(
dataframe[combined_dataframe.columns],
)
combined_dataframe = combined_dataframe.append(tuned_dataframe, ignore_index=True)
combined_dataframe.dropna(inplace=True)
combined_dataframe['Stock'].replace('> ?', '', inplace=True, regex=True)
combined_dataframe['Price'].replace('> ?', '', inplace=True, regex=True)
combined_dataframe = combined_dataframe.astype(
{'Stock': 'int32', 'Price': 'float32'}
)
combined_dataframe = combined_dataframe[combined_dataframe['Stock'] > 0]
combined_dataframe = combined_dataframe.loc[combined_dataframe.groupby('EAN Code')['Price'].idxmin()]
combined_dataframe.to_excel('output_backup/output-{}.xlsx'.format(datetime.now().strftime('%Y-%m-%d')), index=False)
if os.path.exists('output/output.xlsx'):
os.remove("output/output.xlsx")
combined_dataframe.to_excel('output/output.xlsx'.format(datetime.now().strftime('%Y-%m-%d')), index=False)
print('Output saved to output directory')
for file in os.listdir(input_directory):
file_path = os.path.join(input_directory, file)
os.remove(file_path)
print('All input files removed')
I have a pandas list called Symbols with 30 ticker symbols for stock e.g., Apple ->> AAPL, and I would like to grab the current stock price for each ticker and populate a data frame with this info. Two columns: the first with ticker symbols and the second with current price. I continue getting the following error message when I run this part of my script:
"ValueError: If using all scalar values, you must pass an index"
Stock = []
Price = []
df_temp = []
for symbol in Symbols:
try:
params = {
'symbols': symbol,
'range': '1d',
'interval': '1d',
'indicators': 'close',
'includeTimestamps': 'false',
'includePrePost': 'false',
'corsDomain': 'finance.yahoo.com',
'.tsrc': 'finance'}
url = 'https://query1.finance.yahoo.com/v7/finance/spark'
r = requests.get(url, params=params)
data = r.json()
df_stock = pd.DataFrame({'Ticker' : symbol,
'Current Price' : data['spark']['result'][0]['response'][0]['indicators']['quote'][0]['close'][0]
})
df_temp.append(df_stock)
df_temp = pd.concat(df_temp, axis = 1)
except KeyError:
continue
Need to change only one part -
df_stock = pd.DataFrame({'Ticker' : [symbol],
'Current Price' : [data['spark']['result'][0]['response'][0]['indicators']['quote'][0]['close'][0]]
})
Output
Ticker Current Price
0 AAPL 118.119
Full Code
Stock = []
Price = []
df_temp = pd.DataFrame()
for symbol in ['AAPL', 'IBM', 'NKE', 'FB']:
try:
params = {
'symbols': symbol,
'range': '1d',
'interval': '1d',
'indicators': 'close',
'includeTimestamps': 'false',
'includePrePost': 'false',
'corsDomain': 'finance.yahoo.com',
'.tsrc': 'finance'}
url = 'https://query1.finance.yahoo.com/v7/finance/spark'
r = requests.get(url, params=params)
data = r.json()
df_stock = pd.DataFrame({'Ticker' : [symbol],
'Current Price' : [data['spark']['result'][0]['response'][0]['indicators']['quote'][0]['close'][0]]
})
df_temp = df_temp.append(df_stock)
except KeyError:
continue
Explanation
What you were passing as values to the df_stock were scalar values, wrapped in a list solves it.
Very simply, this code should be outputting a summary file, which it isn't, in a specified output directory. I can't figure out why
I have tried editing the configuration as well as changing directories.
import os
import pandas as pd
def summarise(indir, outfile):
os.chdir(indir)
filelist = ""
dflist = []
colnames = ["DSP Code", "Report Date", "Initial Date", "End Date", "Transaction Type", "Sale Type",
"Distribution Channel", "Products Origin ID", "Product ID", "Artist", "Title", "Units Sold",
"Retail Price", "Dealer Price", "Additional Revenue", "Warner Share", "Entity to be billed",
"E retailer name", "E retailer Country", "End Consumer Country", "Price Code", "Currency Code"]
for filename in filelist:
print(filename)
df = pd.read_csv('SYB_M_20171001_20171031.txt', header=None, encoding='utf-8', sep='\t',
names=colnames, skiprows=1, usecols=['Units Sold', 'Dealer Price', 'End Consumer Country',
'Currency Code'])
# Multiplying units by dealer price will give you sum of file
df['Sum of Revenue'] = df['Units Sold'] * df['Dealer Price']
# Get those first two columns
d = {'Sum of Revenue': 'Total Revenue', 'Units Sold': 'Total Units'}
for col, newcol in d.items():
df.loc[df.index[0], newcol] = df[col].sum()
# Add the rest for every country:
s = df.groupby('End Consumer Country')['Units Sold'].sum().to_frame().T.add_suffix(' Total')
s.index = [df.index[0]]
df = pd.concat([df, s], 1, sort=False)
df.to_csv(outfile + r"\output.csv", index=None)
dflist.append(filename)
summarise(r"O:\James Upson\Sound Track Your Brand Testing\SYB Test",
r"O:\James Upson\Sound Track Your Brand Testing\SYB Test Formatted")
I am expecting an output file called 'output.csv'
Hmm, ok, I see filelist = "" and then for filename in filelist:
Your trying to loop over an empty list