I am not able to understand the error - AttributeError: 'int' object has no attribute 'to_pydatetime', I will be really grateful, if anyone could please help me out in this?
from datetime import date, datetime
import pandas as pd
import backtrader as bt
file_name = 'fromstart/2021 APR NIFTY.txt'
dataframe = pd.read_csv(file_name)
data = bt.feeds.PandasData(
dataname = dataframe,
fromdate = datetime(2021, 1, 1),
todate = datetime(2021, 4, 30),
)
class AllInOne(bt.Strategy):
def __init__(self):
self.dataopen = self.datas[0].open_p # Keep a reference to the "open" line in the data[0] dataseries
self.datahigh = self.datas[0].high_p # Keep a reference to the "high" line in the data[0] dataseries
self.datalow = self.datas[0].low_p # Keep a reference to the "low" line in the data[0] dataseries
self.dataclose = self.datas[0].close_p # Keep a reference to the "close" line in the data[0] dataseries
def next(self):
pass
if __name__ == '__main__' :
cerebro = bt.Cerebro() # We initialize the `cerebro` backtester.
cerebro.adddata(data) # We add the dataset in the Data cell.
cerebro.addstrategy(AllInOne)
print('Starting Portfolio Value: {0:8.2f}'.format(cerebro.broker.getvalue()))
results = cerebro.run()
print('Final Portfolio Value: {0:8.2f}'.format(cerebro.broker.getvalue()))
Data and stacktrace:
I had the same error and the next steps have worked for me.
This is the CSV File Data that I am using.
I think you should concatenate the columns "date" and "time".
Finally, when reading csv file, specify the index and parse the time.
pathfile = ".../2021-10-11--2021-10-15.csv"
df_data = pd.read_csv(pathfile, delimiter=",", index_col="Datetime", parse_dates= True)
This worked for me, for feeding in my custom dataset.
datapath = '../db/stockData/daily/ONGC.csv'
dataframe = pd.read_csv(datapath,
parse_dates=True,
index_col="timestamp",
)
dataframe.index = pd.to_datetime(dataframe.index,format="%Y-%m-%d",utc=True)
data = bt.feeds.PandasData(dataname=dataframe)
cerebro.adddata(data)
Related
I am unable to fix the error - 'numpy.int64' object has no attribute 'to_pydatetime', I will be really grateful, if anyone could please help me out in this? I have already tried uninstalling pyfolio and itstalling it from git. Please see the complete code below
import os
import glob
import requests
import pandas as pd
from nsepy import *
from datetime import datetime
import backtrader as bt
import backtrader.feeds as btfeeds
from __future__ import (absolute_import, division, print_function,
unicode_literals)
class TestStrategy(bt.Strategy):
def log(self, txt, dt=None):
''' Logging function for this strategy'''
dt = dt or self.datas[0].datetime.date(0)
print('%s, %s' % (dt.isoformat(), txt))
def __init__(self):
# Keep a reference to the "close" line in the data[0] dataseries
self.dataclose = self.datas[0].close
def next(self):
# Simply log the closing price of the series from the reference
self.log('Close, %.2f' % self.dataclose[0])
if __name__ == '__main__':
cerebro = bt.Cerebro()
cerebro.addstrategy(TestStrategy)
#Data feed block
data_path = "/Users/kumarun/Documents/data/files"
joined_files = os.path.join(data_path, "Oct-MONTHLY-Expirydata_2020.csv")
joined_list = glob.glob(joined_files)
df = pd.concat(map(pd.read_csv, joined_list), ignore_index=True)
df.columns=['Ticker','date', 'open', 'high', 'low', 'close', 'volume','Open Interest']
filtered = df[(df['Ticker'] == 'BANKNIFTY')]
#Cerebro block
filtered.date = pd.to_datetime(filtered.date, format='%d-%m-%Y %H:%M:%S')
feed = bt.feeds.PandasData(dataname=filtered)
cerebro.adddata(feed)
cerebro.broker.setcash(100000.0)
print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
I feel like you might be running into issues with the column names. Try ascribing your column names using:
data = bt.feeds.PandasData(
dataname=filtered,
datetime="date",
open='open',
high='high',
low='low',
close='close',
volume='volume',
openinterest="Open Interest"
)
Also, could drop the Ticker name column, and add the name using:
ticker = 'BANKNIFTY'
cerebro.adddata(feed, name=ticker)
I'm trying to make a trading bot and am using backtrader for the same. I have been trying to debug this issue but couldn't find a solution yet. The code is as shown below
# from ast import Constant
from operator import imod
import os, sys
import config
from binance.client import Client
import backtrader
import pandas as pd
import datetime, time
client = Client(config.Binanceapikey, config.BinancesecretKey)
def GetHistoricalData(howLong):
# Calculate the timestamps for the binance api function
untilThisDate = datetime.datetime.now()
sinceThisDate = untilThisDate - datetime.timedelta(days = howLong)
# Execute the query from binance - timestamps must be converted to strings !
candle = client.get_historical_klines("BTCUSDT", Client.KLINE_INTERVAL_1MINUTE, str(sinceThisDate), str(untilThisDate))
# Create a dataframe to label all the columns returned by binance so we work with them later.
df = pd.DataFrame(candle, columns=['dateTime', 'open', 'high', 'low', 'close', 'volume', 'closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol', 'takerBuyQuoteVol', 'ignore'])
# as timestamp is returned in ms, let us convert this back to proper timestamps.
df.dateTime = pd.to_datetime(df.dateTime, unit='ms').dt.strftime("%Y-%m-%d")
df.set_index('dateTime', inplace=True)
# Get rid of columns we do not need
df = df.drop(['closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol','takerBuyQuoteVol', 'ignore'], axis=1)
cerebro = backtrader.Cerebro()
cerebro.broker.set_cash(100000)
cerebro.adddata(GetHistoricalData(1))
print('Starting porfolio value: %.2f' %cerebro.broker.getvalue())
cerebro.run()
print('Final porfolio value: %.2f' %cerebro.broker.getvalue())
The error message is as follows:
File "/TradingBot/tradingBot.py", line 40, in <module>
cerebro.adddata(GetHistoricalData(1))
File "/usr/local/lib/python3.8/site-packages/backtrader/cerebro.py", line 757, in adddata
data._id = next(self._dataid)
AttributeError: 'NoneType' object has no attribute '_id'
Thanks in advance!
You do not have a return in GetHistoricalData so it is sending None to adddata(). Maybe you need to return the dataframe? if not specify your intent.
# from ast import Constant
from operator import imod
import os, sys
import config
from binance.client import Client
import backtrader
import pandas as pd
import datetime, time
client = Client(config.Binanceapikey, config.BinancesecretKey)
def GetHistoricalData(howLong):
# Calculate the timestamps for the binance api function
untilThisDate = datetime.datetime.now()
sinceThisDate = untilThisDate - datetime.timedelta(days = howLong)
# Execute the query from binance - timestamps must be converted to strings !
candle = client.get_historical_klines("BTCUSDT", Client.KLINE_INTERVAL_1MINUTE, str(sinceThisDate), str(untilThisDate))
# Create a dataframe to label all the columns returned by binance so we work with them later.
df = pd.DataFrame(candle, columns=['dateTime', 'open', 'high', 'low', 'close', 'volume', 'closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol', 'takerBuyQuoteVol', 'ignore'])
# as timestamp is returned in ms, let us convert this back to proper timestamps.
df.dateTime = pd.to_datetime(df.dateTime, unit='ms').dt.strftime("%Y-%m-%d")
df.set_index('dateTime', inplace=True)
# Get rid of columns we do not need
df = df.drop(['closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol','takerBuyQuoteVol', 'ignore'], axis=1)
#return the df
return df
cerebro = backtrader.Cerebro()
cerebro.broker.set_cash(100000)
cerebro.adddata(GetHistoricalData(1))
print('Starting porfolio value: %.2f' %cerebro.broker.getvalue())
cerebro.run()
print('Final porfolio value: %.2f' %cerebro.broker.getvalue())
I have a python file called 'clean_data.py' which has all the data frames I need, and I want to import them for use in another python file called 'main.py' to use in creating a dashboard.
Is it possible to create a class in my clean_data.py, and if so can someone direct me to an article (which I struggled to find so far) so that I can figure it out?
The aim is to shift from CSV to an API overtime, so I wanted to keep data side wrangling side of things in a different file while the web app components in the main.py file.
Any help would be much appreciated.
The code from the clean_data.py is:
import pandas as pd
import csv
import os # To access my file directory
print(os.getcwd()) # Let's me know the Current Work Directory
fdi_data = pd.read_csv(r'Data/fdi_data.csv')
fdi_meta = pd.read_csv(r'Data/fdi_metadata.csv')
debt_data = pd.read_csv(r'Data/debt_data.csv')
debt_meta = pd.read_csv(r'Data/debt_metadata.csv')
gdp_percap_data = pd.read_csv(r'Data/gdp_percap_data.csv', header=2)
gdp_percap_meta = pd.read_csv(r'Data/gdp_percap_metadata.csv')
gov_exp_data = pd.read_csv(r'Data/gov_exp_data.csv', header=2)
gov_exp_meta = pd.read_csv(r'Data/gov_exp_metadata.csv')
pop_data = pd.read_csv(r'Data/pop_data.csv', header=2)
pop_meta = pd.read_csv(r'Data/pop_metadata.csv')
"""
'wb' stands for World Bank
"""
def wb_merge_data(data, metadata):
merge = pd.merge(
data,
metadata,
on = 'Country Code',
how = 'inner'
)
return merge
fdi_merge = wb_merge_data(fdi_data, fdi_meta)
debt_merge = wb_merge_data(debt_data, debt_meta)
gdp_percap_merge = wb_merge_data(gdp_percap_data, gdp_percap_meta)
gov_exp_merge = wb_merge_data(gov_exp_data, gov_exp_meta)
pop_merge = wb_merge_data(pop_data, pop_meta)
def wb_drop_data(data):
drop = data.drop(['Country Code','Indicator Name','Indicator Code','TableName','SpecialNotes','Unnamed: 5'], axis=1)
return drop
fdi_merge = wb_drop_data(fdi_merge)
debt_merge = wb_drop_data(debt_merge)
gdp_percap_merge = wb_drop_data(gdp_percap_merge)
gov_exp_merge = wb_drop_data(gov_exp_merge)
pop_merge = wb_drop_data(pop_merge)
def wb_mr_data(data, value_name):
data = data.melt(['Country Name','Region','IncomeGroup']).reset_index()
data = data.rename(columns={'variable': 'Year', 'value': value_name})
data = data.drop('index', axis = 1)
return data
fdi_merge = wb_mr_data(fdi_merge, 'FDI')
debt_merge = wb_mr_data(debt_merge, 'Debt')
gdp_percap_merge = wb_mr_data(gdp_percap_merge, 'GDP per Cap')
gov_exp_merge = wb_mr_data(gov_exp_merge, 'Gov Expend.')
pop_merge = wb_mr_data(pop_merge, 'Population')
def avg_groupby(data, col_cal, cn=False, ig=False, rg=False):
if cn == True:
return data.groupby('Country Name')[col_cal].mean().reset_index()
elif ig == True:
return data.groupby('IncomeGroup')[col_cal].mean().reset_index()
elif rg == True:
return data.groupby('Region')[col_cal].mean().reset_index()
"""
avg_cn_... For country
avg_ig_... Income Group
avg_rg_... Region
"""
avg_cn_fdi = avg_groupby(fdi_merge, 'FDI', cn=True)
avg_ig_fdi = avg_groupby(fdi_merge, 'FDI', ig=True)
avg_rg_fdi = avg_groupby(fdi_merge, 'FDI', rg=True)
avg_cn_debt = avg_groupby(debt_merge, 'Debt', cn=True)
avg_ig_debt = avg_groupby(debt_merge, 'Debt', ig=True)
avg_rg_debt = avg_groupby(debt_merge, 'Debt', rg=True)
avg_cn_gdp_percap = avg_groupby(gdp_percap_merge, 'GDP per Cap', cn=True)
avg_ig_gdp_percap = avg_groupby(gdp_percap_merge, 'GDP per Cap', ig=True)
avg_rg_gdp_percap = avg_groupby(gdp_percap_merge, 'GDP per Cap', rg=True)
avg_cn_gexp = avg_groupby(gov_exp_merge, 'Gov Expend.', cn=True)
avg_ig_gexp = avg_groupby(gov_exp_merge, 'Gov Expend.', ig=True)
avg_rg_gexp = avg_groupby(gov_exp_merge, 'Gov Expend.', rg=True)
avg_cn_pop = avg_groupby(pop_merge, 'Population', cn=True)
avg_ig_pop = avg_groupby(pop_merge, 'Population', ig=True)
avg_rg_pop = avg_groupby(pop_merge, 'Population', rg=True)
In Python, every file is a module. So if you want to re-use your code, you can simple import this module. For example,
# main.py
import clean_data
print(clean_data.avg_cn_fdi)
Maybe you needn't create a class for this
You can import the whole python file like you'd import any other locally created files and have access to the DataFrames in them. Here's an example:
I created a file called temporary.py:
import pandas as pd
data = pd.read_csv("temp.csv")
And then in a separate file I was able to use data like so:
import temporary
print(temporary.data)
Or, you could also do:
from temporary import data
print(data)
All that being said, I don't believe that this would be the best way to handle your data.
I am trying to loop through a list of symbols to get rates for various currencies via the mt5. I use the code below but i get TypeError
d[i] = [y.close for y in rates1]
TypeError: 'NoneType' object is not iterable
I can't see where im going wrong i would like to use this structure to loop through create multiple dataframe and then make a big multiindex of all pairs and time using same kind of loop. I've not been coding long.
sym = ['GBPUSD','USDJPY','USDCHF','AUDUSD','GBPJPY']
# Copying data to dataframe
d = pd.DataFrame()
for i in sym:
rates1 = mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, 5)
d[i] = [y.close for y in rates1]
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 29 18:38:11 2020
#author: DanPc
"""
# -*- coding: utf-8 -*-
"""
"""
import pytz
import pandas as pd
import MetaTrader5 as mt5
import time
from datetime import datetime
from threading import Timer
import talib
import numpy as np
import matplotlib as plt
from multiprocessing import Process
import sys
server_name = "" ENTER DETAILS HERE
server_num =
password = ""
#------------------------------------------------------------------------------
def actualtime():
# datetime object containing current date and time
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
#print("date and time =", dt_string)
return str(dt_string)
#------------------------------------------------------------------------------
def sync_60sec(op):
info_time_new = datetime.strptime(str(actualtime()), '%d/%m/%Y %H:%M:%S')
waiting_time = 60 - info_time_new.second
t = Timer(waiting_time, op)
t.start()
print(actualtime)
#------------------------------------------------------------------------------
def program(symbol):
if not mt5.initialize(login=server_num, server=server_name, password=password):
print("initialize() failed, error code =",mt5.last_error())
quit()
timezone = pytz.timezone("Etc/UTC")
utc_from = datetime.now()
######### Change here the timeframe 525600
# Create currency watchlist for which correlation matrix is to be plotted
sym = ['GBPUSD','USDJPY','USDCHF','AUDUSD','GBPJPY']
# Copying data to dataframe
d = pd.DataFrame()
for i in sym:
rates1 = mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, 5)
d[i] = [y.close for y in rates1]
print(rates1)
mt5.shutdown()
if not mt5.initialize():
print("initialize() failed, error code =",mt5.last_error())
quit()
# starting mt5
if not mt5.initialize(login=server_num, server=server_name, password=password):
print("initialize() failed, error code =",mt5.last_error())
quit()
#------------------------------------------------------------------------------
# S T A R T I N G M T 5
#------------------------------------------------------------------------------
authorized=mt5.login(server_num, password=password)
if authorized:
account_info=mt5.account_info()
if account_info!=None:
account_info_dict = mt5.account_info()._asdict()
df=pd.DataFrame(list(account_info_dict.items()),columns=['property','value'])
print("account_info() as dataframe:")
print(df)
else:
print(mt5.last_error)
mt5.shutdown()
#------------------------------------------------------------------------------
def trading_bot():
symbol_1 = 'EURUSD'
symbol_2 = 'EURCAD'
while True:
program(symbol_1)
program(symbol_2)
time.sleep(59.8) # it depends on your computer and ping
sync_60sec(trading_bot)
copy_rates_from returns None if there is an error. The documentation suggests calling last_error() to find out what that error is.
(And no, I don't know why copy_rates_from doesn't just raise an exception to indicate the error. Apparently, the module is a thin wrapper around a C library.)
I came to this solution that creates a dictionary of dataframes.
sym = ["GBPUSD","USDJPY","USDCHF","AUDUSD","GBPJPY"]
# Copying data to dataframe
utc_from = datetime.now()
for i in sym:
rates = {i:pd.DataFrame(mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, utc_from , 60),
columns=['time', 'open', 'low', 'high', 'close', 'tick_volume', 'spread', 'real_volume']) for i in sym}
I am using two functions, one to load data and another to get a summary of the same data. However in second function analyze() I get the error df not defined. How do I pass df from loader() to analyze() ?
from xlwings import Workbook, Range
import pandas as pd
def Loader():
wb = Workbook.caller()
file_path = Range(1,(1,1)).value
file=pd.read_excel(file_path, sheetname='Sheet1')
df = pd.DataFrame(file)
def analyze():
Range('C1').value=df.describe()
With several ways depending on what you want to do. The simplest way is to return the df from the Loader() and then give it to the analyze() as an argument:
def Loader():
wb = Workbook.caller()
file_path = Range(1,(1,1)).value
file=pd.read_excel(file_path, sheetname='Sheet1')
df = pd.DataFrame(file)
return df
def analyze(df):
Range('C1').value=df.describe()
# Use it this way
dataFrame = Loader()
analyze(dataframe)
Then another way is to have a Loader class like this:
class Loader(object):
def __init__(self):
wb = Workbook.caller()
file_path = Range(1,(1,1)).value
file=pd.read_excel(file_path, sheetname='Sheet1')
self.df = pd.DataFrame(file)
# 1) If you want to analyse when you create the object
# call analyze() here
self.analyze()
def analyze(self):
Range('C1').value=self.df.describe()
loader = Loader()
# 2) Otherwise you can keep control of analyze()
# and call it whenever you want, like this:
loader.analyze()
Of course there are other ways too (like having a global variable for the df).