I am unable to fix the error - 'numpy.int64' object has no attribute 'to_pydatetime', I will be really grateful, if anyone could please help me out in this? I have already tried uninstalling pyfolio and itstalling it from git. Please see the complete code below
import os
import glob
import requests
import pandas as pd
from nsepy import *
from datetime import datetime
import backtrader as bt
import backtrader.feeds as btfeeds
from __future__ import (absolute_import, division, print_function,
unicode_literals)
class TestStrategy(bt.Strategy):
def log(self, txt, dt=None):
''' Logging function for this strategy'''
dt = dt or self.datas[0].datetime.date(0)
print('%s, %s' % (dt.isoformat(), txt))
def __init__(self):
# Keep a reference to the "close" line in the data[0] dataseries
self.dataclose = self.datas[0].close
def next(self):
# Simply log the closing price of the series from the reference
self.log('Close, %.2f' % self.dataclose[0])
if __name__ == '__main__':
cerebro = bt.Cerebro()
cerebro.addstrategy(TestStrategy)
#Data feed block
data_path = "/Users/kumarun/Documents/data/files"
joined_files = os.path.join(data_path, "Oct-MONTHLY-Expirydata_2020.csv")
joined_list = glob.glob(joined_files)
df = pd.concat(map(pd.read_csv, joined_list), ignore_index=True)
df.columns=['Ticker','date', 'open', 'high', 'low', 'close', 'volume','Open Interest']
filtered = df[(df['Ticker'] == 'BANKNIFTY')]
#Cerebro block
filtered.date = pd.to_datetime(filtered.date, format='%d-%m-%Y %H:%M:%S')
feed = bt.feeds.PandasData(dataname=filtered)
cerebro.adddata(feed)
cerebro.broker.setcash(100000.0)
print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
cerebro.run()
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
I feel like you might be running into issues with the column names. Try ascribing your column names using:
data = bt.feeds.PandasData(
dataname=filtered,
datetime="date",
open='open',
high='high',
low='low',
close='close',
volume='volume',
openinterest="Open Interest"
)
Also, could drop the Ticker name column, and add the name using:
ticker = 'BANKNIFTY'
cerebro.adddata(feed, name=ticker)
Related
I'm trying to make a trading bot and am using backtrader for the same. I have been trying to debug this issue but couldn't find a solution yet. The code is as shown below
# from ast import Constant
from operator import imod
import os, sys
import config
from binance.client import Client
import backtrader
import pandas as pd
import datetime, time
client = Client(config.Binanceapikey, config.BinancesecretKey)
def GetHistoricalData(howLong):
# Calculate the timestamps for the binance api function
untilThisDate = datetime.datetime.now()
sinceThisDate = untilThisDate - datetime.timedelta(days = howLong)
# Execute the query from binance - timestamps must be converted to strings !
candle = client.get_historical_klines("BTCUSDT", Client.KLINE_INTERVAL_1MINUTE, str(sinceThisDate), str(untilThisDate))
# Create a dataframe to label all the columns returned by binance so we work with them later.
df = pd.DataFrame(candle, columns=['dateTime', 'open', 'high', 'low', 'close', 'volume', 'closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol', 'takerBuyQuoteVol', 'ignore'])
# as timestamp is returned in ms, let us convert this back to proper timestamps.
df.dateTime = pd.to_datetime(df.dateTime, unit='ms').dt.strftime("%Y-%m-%d")
df.set_index('dateTime', inplace=True)
# Get rid of columns we do not need
df = df.drop(['closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol','takerBuyQuoteVol', 'ignore'], axis=1)
cerebro = backtrader.Cerebro()
cerebro.broker.set_cash(100000)
cerebro.adddata(GetHistoricalData(1))
print('Starting porfolio value: %.2f' %cerebro.broker.getvalue())
cerebro.run()
print('Final porfolio value: %.2f' %cerebro.broker.getvalue())
The error message is as follows:
File "/TradingBot/tradingBot.py", line 40, in <module>
cerebro.adddata(GetHistoricalData(1))
File "/usr/local/lib/python3.8/site-packages/backtrader/cerebro.py", line 757, in adddata
data._id = next(self._dataid)
AttributeError: 'NoneType' object has no attribute '_id'
Thanks in advance!
You do not have a return in GetHistoricalData so it is sending None to adddata(). Maybe you need to return the dataframe? if not specify your intent.
# from ast import Constant
from operator import imod
import os, sys
import config
from binance.client import Client
import backtrader
import pandas as pd
import datetime, time
client = Client(config.Binanceapikey, config.BinancesecretKey)
def GetHistoricalData(howLong):
# Calculate the timestamps for the binance api function
untilThisDate = datetime.datetime.now()
sinceThisDate = untilThisDate - datetime.timedelta(days = howLong)
# Execute the query from binance - timestamps must be converted to strings !
candle = client.get_historical_klines("BTCUSDT", Client.KLINE_INTERVAL_1MINUTE, str(sinceThisDate), str(untilThisDate))
# Create a dataframe to label all the columns returned by binance so we work with them later.
df = pd.DataFrame(candle, columns=['dateTime', 'open', 'high', 'low', 'close', 'volume', 'closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol', 'takerBuyQuoteVol', 'ignore'])
# as timestamp is returned in ms, let us convert this back to proper timestamps.
df.dateTime = pd.to_datetime(df.dateTime, unit='ms').dt.strftime("%Y-%m-%d")
df.set_index('dateTime', inplace=True)
# Get rid of columns we do not need
df = df.drop(['closeTime', 'quoteAssetVolume', 'numberOfTrades', 'takerBuyBaseVol','takerBuyQuoteVol', 'ignore'], axis=1)
#return the df
return df
cerebro = backtrader.Cerebro()
cerebro.broker.set_cash(100000)
cerebro.adddata(GetHistoricalData(1))
print('Starting porfolio value: %.2f' %cerebro.broker.getvalue())
cerebro.run()
print('Final porfolio value: %.2f' %cerebro.broker.getvalue())
The code works fine and prints all the desired values but when I try to save as df to csv all the rows are the same.
a.csv file content
Domains
A.com
AMD.com
AMD.com
AOL.com
AOL.com
AWS.com
AWS.com
3ammagazine.com
3dmail.com
3email.com
3xl.net
444.net
4email.com
4email.net
4mg.com
4newyork.com
whdskcisail.com
wrestlingpages.com
wrexham.net
import whois
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import socket
import os
import csv
import datetime
import time
import requests
from ipwhois import IPWhois
from urllib import request
from ipwhois.utils import get_countries
import tldextract
from ipwhois.utils import get_countries
countries = get_countries(is_legacy_xml=True)
from ipwhois.experimental import bulk_lookup_rdap
from ipwhois.hr import (HR_ASN, HR_ASN_ORIGIN, HR_RDAP_COMMON, HR_RDAP, HR_WHOIS, HR_WHOIS_NIR)
countries = get_countries(is_legacy_xml=True)
import ipaddress
df = pd.read_csv('a.csv', nrows=100)
#TimeOut Setting
s = socket.socket()
s.settimeout(10)
#Date Processing Function
def check_date_type(d):
if type(d) is datetime.datetime:
return d
if type(d) is list:
return d[0]
for index,row in df.iterrows():
try:
DN = df.iloc[index]['Domains']
ip = socket.gethostbyname(DN)
ipwhois = IPWhois(ip).lookup_whois()
print(DN)
print(ip)
print(ipwhois)
df['IPcity'][index] = ipwhois['nets'][0]['city']
df['ASNumber'][index] = ipwhois['asn']
df['NetAddr'][index] = ipwhois['nets'][0]['address']
df['NetCity'][index] = ipwhois['city']
df['NetPostCode'][index] = ipwhois['nets'][0]['postal_code']
df.to_csv('a1.csv', index=False)
except Exception as e:
print(e)
df['e'] = e # want to save the errors in a differet column
df.to_csv('a2.csv', index=False)
This is happening because you're overwriting the .csv file in every iteration of your loop. If you move the df.to_csv() outside of your loop, you will get your desired dataframe:
...
for index,row in df.iterrows():
try:
DN = df.iloc[index]['Domains']
ip = socket.gethostbyname(DN)
ipwhois = IPWhois(ip).lookup_whois()
print(DN)
print(ip)
print(ipwhois)
df['IPcity'][index] = ipwhois['nets'][0]['city']
df['ASNumber'][index] = ipwhois['asn']
df['NetAddr'][index] = ipwhois['nets'][0]['address']
df['NetCity'][index] = ipwhois['city']
df['NetPostCode'][index] = ipwhois['nets'][0]['postal_code']
except Exception as e:
print(e)
df['e'] = e # want to save the errors in a differet column
df.to_csv('a1.csv', index=False)
I am not able to understand the error - AttributeError: 'int' object has no attribute 'to_pydatetime', I will be really grateful, if anyone could please help me out in this?
from datetime import date, datetime
import pandas as pd
import backtrader as bt
file_name = 'fromstart/2021 APR NIFTY.txt'
dataframe = pd.read_csv(file_name)
data = bt.feeds.PandasData(
dataname = dataframe,
fromdate = datetime(2021, 1, 1),
todate = datetime(2021, 4, 30),
)
class AllInOne(bt.Strategy):
def __init__(self):
self.dataopen = self.datas[0].open_p # Keep a reference to the "open" line in the data[0] dataseries
self.datahigh = self.datas[0].high_p # Keep a reference to the "high" line in the data[0] dataseries
self.datalow = self.datas[0].low_p # Keep a reference to the "low" line in the data[0] dataseries
self.dataclose = self.datas[0].close_p # Keep a reference to the "close" line in the data[0] dataseries
def next(self):
pass
if __name__ == '__main__' :
cerebro = bt.Cerebro() # We initialize the `cerebro` backtester.
cerebro.adddata(data) # We add the dataset in the Data cell.
cerebro.addstrategy(AllInOne)
print('Starting Portfolio Value: {0:8.2f}'.format(cerebro.broker.getvalue()))
results = cerebro.run()
print('Final Portfolio Value: {0:8.2f}'.format(cerebro.broker.getvalue()))
Data and stacktrace:
I had the same error and the next steps have worked for me.
This is the CSV File Data that I am using.
I think you should concatenate the columns "date" and "time".
Finally, when reading csv file, specify the index and parse the time.
pathfile = ".../2021-10-11--2021-10-15.csv"
df_data = pd.read_csv(pathfile, delimiter=",", index_col="Datetime", parse_dates= True)
This worked for me, for feeding in my custom dataset.
datapath = '../db/stockData/daily/ONGC.csv'
dataframe = pd.read_csv(datapath,
parse_dates=True,
index_col="timestamp",
)
dataframe.index = pd.to_datetime(dataframe.index,format="%Y-%m-%d",utc=True)
data = bt.feeds.PandasData(dataname=dataframe)
cerebro.adddata(data)
I have this code right here that I'm making a report, and I'm trying to work with the date but I cant cause pycharm says it cant work with "series" format, I`m trying to convert it to simple datetime but nothing works, can u guys help me?
the "DATA" is coming with the format of "datetime n 64" and I need it to be normal datetime, how can I do this?
import pyodbc
import pandas as pd
import matplotlib.pyplot as plt
import datetime
class generate_report():
def __init__(self):
self.csv = "output.csv"
self.sql_conn = pyodbc.connect('Trusted_Connection=yes', driver = '{SQL Server}',
server = 'localhost', database = 'MPWJ_BI')
self.query = "select * from CTP_EXTRATO_GERAL where HISTORICO = 'Aplicação' order by data"
self.df = pd.read_sql(self.query, self.sql_conn)
self.df['DATA'] = pd.to_datetime(self.df['DATA'])
self.df.to_csv(self.csv)
def analyze_data(self):
pd.read_csv(self.csv)
print(self.df.dtypes)
It depends on how your date looks like
for example
from datetime import datetime
datetime_object = datetime.strptime('Jun 1 2020 7:31PM', '%b %d %Y %I:%M%p')
documentation
https://docs.python.org/3/library/datetime.html#datetime.datetime.strptime
EDIT:
To convert from datetime64 to datetime you can do the following:
import datetime
import numpy as np
# Current time UTC
dt = datetime.datetime.utcnow()
# Convert to datetime64
dt64 = np.datetime64(dt)
# convert to epoch
ts = (dt64 - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
# Convert to datetime
print(datetime.datetime.fromtimestamp(ts))
I am trying to loop through a list of symbols to get rates for various currencies via the mt5. I use the code below but i get TypeError
d[i] = [y.close for y in rates1]
TypeError: 'NoneType' object is not iterable
I can't see where im going wrong i would like to use this structure to loop through create multiple dataframe and then make a big multiindex of all pairs and time using same kind of loop. I've not been coding long.
sym = ['GBPUSD','USDJPY','USDCHF','AUDUSD','GBPJPY']
# Copying data to dataframe
d = pd.DataFrame()
for i in sym:
rates1 = mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, 5)
d[i] = [y.close for y in rates1]
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 29 18:38:11 2020
#author: DanPc
"""
# -*- coding: utf-8 -*-
"""
"""
import pytz
import pandas as pd
import MetaTrader5 as mt5
import time
from datetime import datetime
from threading import Timer
import talib
import numpy as np
import matplotlib as plt
from multiprocessing import Process
import sys
server_name = "" ENTER DETAILS HERE
server_num =
password = ""
#------------------------------------------------------------------------------
def actualtime():
# datetime object containing current date and time
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
#print("date and time =", dt_string)
return str(dt_string)
#------------------------------------------------------------------------------
def sync_60sec(op):
info_time_new = datetime.strptime(str(actualtime()), '%d/%m/%Y %H:%M:%S')
waiting_time = 60 - info_time_new.second
t = Timer(waiting_time, op)
t.start()
print(actualtime)
#------------------------------------------------------------------------------
def program(symbol):
if not mt5.initialize(login=server_num, server=server_name, password=password):
print("initialize() failed, error code =",mt5.last_error())
quit()
timezone = pytz.timezone("Etc/UTC")
utc_from = datetime.now()
######### Change here the timeframe 525600
# Create currency watchlist for which correlation matrix is to be plotted
sym = ['GBPUSD','USDJPY','USDCHF','AUDUSD','GBPJPY']
# Copying data to dataframe
d = pd.DataFrame()
for i in sym:
rates1 = mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, 5)
d[i] = [y.close for y in rates1]
print(rates1)
mt5.shutdown()
if not mt5.initialize():
print("initialize() failed, error code =",mt5.last_error())
quit()
# starting mt5
if not mt5.initialize(login=server_num, server=server_name, password=password):
print("initialize() failed, error code =",mt5.last_error())
quit()
#------------------------------------------------------------------------------
# S T A R T I N G M T 5
#------------------------------------------------------------------------------
authorized=mt5.login(server_num, password=password)
if authorized:
account_info=mt5.account_info()
if account_info!=None:
account_info_dict = mt5.account_info()._asdict()
df=pd.DataFrame(list(account_info_dict.items()),columns=['property','value'])
print("account_info() as dataframe:")
print(df)
else:
print(mt5.last_error)
mt5.shutdown()
#------------------------------------------------------------------------------
def trading_bot():
symbol_1 = 'EURUSD'
symbol_2 = 'EURCAD'
while True:
program(symbol_1)
program(symbol_2)
time.sleep(59.8) # it depends on your computer and ping
sync_60sec(trading_bot)
copy_rates_from returns None if there is an error. The documentation suggests calling last_error() to find out what that error is.
(And no, I don't know why copy_rates_from doesn't just raise an exception to indicate the error. Apparently, the module is a thin wrapper around a C library.)
I came to this solution that creates a dictionary of dataframes.
sym = ["GBPUSD","USDJPY","USDCHF","AUDUSD","GBPJPY"]
# Copying data to dataframe
utc_from = datetime.now()
for i in sym:
rates = {i:pd.DataFrame(mt5.copy_rates_from(i, mt5.TIMEFRAME_M1, utc_from , 60),
columns=['time', 'open', 'low', 'high', 'close', 'tick_volume', 'spread', 'real_volume']) for i in sym}