Develop Reference

Conjugate gradient with tensorflow and sparse tensor runs slower than scipy with sparse matrices

I implemented the conjugate gradient method using TensorFlow to invert a sparse matrix.
The matrix I used to test the method is well-conditioned, as it is the sum of a mass matrix and a stiffness matrix obtained with finite elements.
I compared with the same method implemented using scipy and on the same data.
The solutions obtained with either methods are the same, however, TensorFlow is 5 times slower (I tested under colab environment).
Under colab environment, scipy ran in 0.27 s, while TensorFlow required 1.37 s
Why the algorithm is so slow under TensorFlow?
I can not cast to dense matrices, as I want to use the formula with matrices of large size (100k X100k or more).
Thanks,
Cesare
Here is the code I used to test this:
import tensorflow as tf
import numpy as np
from scipy.sparse import coo_matrix,linalg
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
from time import time
from scipy.spatial import Delaunay
def create_mesh(Lx=1,Ly=1,Nx=100,Ny=100):
mesh0=dict()
dx = Lx/Nx
dy = Ly/Ny
XX,YY=np.meshgrid(np.arange(0,Lx+dx,dx),np.arange(0,Ly+dy,dy))
points=np.vstack((XX.ravel(),YY.ravel())).T
#np.random.shuffle(points)
tri = Delaunay(points)
mesh0['Pts']=np.copy(points).astype(np.float32)
mesh0['Tria']=np.copy(tri.simplices).astype(int)
return(mesh0)
def eval_connectivity(mesh0):
print('computing mesh connectivity')
npt=mesh0['Pts'].shape[0]
connectivity = {}
for jpt in range(npt):
connectivity[jpt] = []
for Tria in mesh0['Tria']:
for ilpt in range(3):
iglobalPt=Tria[ilpt]
for jlpt in range(1+ilpt,3):
jglobalPt=Tria[jlpt]
connectivity[iglobalPt].append(jglobalPt)
connectivity[jglobalPt].append(iglobalPt)
for key,value in connectivity.items():
connectivity[key]=np.unique(np.array(value,dtype=int))
return(connectivity)
def eval_local_mass(mesh0,iTri):
lmass = np.zeros(shape=(3,3),dtype=np.float32)
Tria=mesh0['Tria'][iTri]
v10 = mesh0['Pts'][Tria[1],:]-mesh0['Pts'][Tria[0],:]
v20 = mesh0['Pts'][Tria[2],:]-mesh0['Pts'][Tria[0],:]
N12 = np.cross(v10,v20)
Tsurf = 0.5*np.linalg.norm(N12)
for ipt in range(3):
lmass[ipt,ipt]=1.0/12.0
for jpt in range(1+ipt,3):
lmass[ipt,jpt] = 1.0/24.0
lmass[jpt,ipt] = lmass[ipt,jpt]
lmass = 2.0*Tsurf*lmass
return(lmass)
def eval_local_stiffness(mesh0,iTri):
Tria = mesh0['Tria'][iTri]
v10 = mesh0['Pts'][Tria[1],:]-mesh0['Pts'][Tria[0],:]
v20 = mesh0['Pts'][Tria[2],:]-mesh0['Pts'][Tria[0],:]
N12 = np.cross(v10,v20)
Tsurf = 0.5*np.linalg.norm(N12)
covbT = np.zeros(shape=(3,3),dtype=np.float32)
covbT[0,:2] = v10
covbT[1,:2] = v20
covbT[2,2] = N12/(2*Tsurf)
contrb = np.linalg.inv(covbT)
v1 = contrb[:,0]
v2 = contrb[:,1]
a = np.dot(v1,v1)
b = np.dot(v1,v2)
c = np.dot(v2,v2)
gij_c = np.array([[a,b],[b,c]],dtype=np.float32)
lgrad = np.array([[-1.0,1.0,0.0], [-1.0,0.0,1.0] ],dtype=np.float32)
lstif = Tsurf*np.matmul( np.matmul(lgrad.T,gij_c), lgrad )
return(lstif)
def compute_vectors_sparse_matrices(mesh0):
npt = mesh0['Pts'].shape[0]
connect = eval_connectivity(mesh0)
nzero = 0
for key,value in connect.items():
nzero += (1+value.shape[0])
I = np.zeros(shape=(nzero),dtype=int)
J = np.zeros(shape=(nzero),dtype=int)
VM = np.zeros(shape=(nzero),dtype=np.float32)
VS = np.zeros(shape=(nzero),dtype=np.float32)
k0 = np.zeros(shape=(npt+1),dtype=int)
k0[0] = 0
k = -1
for jpt in range(npt):
loc_con = connect[jpt].tolist()[:]
loc_con.append(jpt)
loc_con = np.sort(loc_con)
k0[jpt+1]=k0[jpt]+loc_con.shape[0]
for jloc in range(loc_con.shape[0]):
k=k+1
I[k]= jpt
J[k]= loc_con[jloc]
for iTr, Tria in enumerate(mesh0['Tria']):
lstiff = eval_local_stiffness(mesh0,iTr)
lmass = eval_local_mass(mesh0,iTr)
for iEntry,irow in enumerate(Tria):
loc_con = connect[irow].tolist()[:]
loc_con.append(irow)
loc_con = np.sort(loc_con)
for jEntry,jcol in enumerate(Tria):
indexEntry = k0[irow]+np.where(loc_con==jcol)[0]
VM[indexEntry] = VM[indexEntry]+lmass[iEntry,jEntry]
VS[indexEntry] = VS[indexEntry]+lstiff[iEntry,jEntry]
return(I,J,VM,VS)
def compute_global_sparse_matrices(mesh0):
I,J,VM,VS = compute_vectors_sparse_matrices(mesh0)
npt = mesh0['Pts'].shape[0]
MASS = coo_matrix((VM,(I,J)),shape=(npt,npt))
STIFF = coo_matrix((VS,(I,J)),shape=(npt,npt))
return(MASS,STIFF)
def compute_global_sparse_tensors(mesh0):
I,J,VM,VS = compute_vectors_sparse_matrices(mesh0)
npt = mesh0['Pts'].shape[0]
indices = np.hstack([I[:,np.newaxis], J[:,np.newaxis]])
MASS = tf.sparse.SparseTensor(indices=indices, values=VM.astype(np.float32), dense_shape=[npt, npt])
STIFF = tf.sparse.SparseTensor(indices=indices, values=VS.astype(np.float32), dense_shape=[npt, npt])
return(MASS,STIFF)
def compute_matrices_scipy(mesh0):
MASS,STIFF = compute_global_sparse_matrices(mesh0)
return(MASS,STIFF)
def compute_matrices_tensorflow(mesh0):
MASS,STIFF = compute_global_sparse_tensors(mesh0)
return(MASS,STIFF)
def conjgrad_scipy(A,b,x0,niter=100,toll=1.e-5):
x = np.copy(x0)
r = b - A * x
p = np.copy(r)
rsold = np.dot(r,r)
for it in range(niter):
Ap = A * p
alpha = rsold /np.dot(p,Ap)
x += alpha * p
r -= alpha * Ap
rsnew = np.dot(r,r)
if (np.sqrt(rsnew) < toll):
break
p = r + (rsnew / rsold) * p
rsold = rsnew
return([x,it,np.sqrt(rsnew)])
def conjgrad_tensorflow(A,b,x0,niter=100,toll=1.e-5):
x = x0
r = b - tf.sparse.sparse_dense_matmul(A,x)
p = r
rsold = tf.reduce_sum(tf.multiply(r, r))
for it in range(niter):
Ap = tf.sparse.sparse_dense_matmul(A,p)
alpha = rsold /tf.reduce_sum(tf.multiply(p, Ap))
x += alpha * p
r -= alpha * Ap
rsnew = tf.reduce_sum(tf.multiply(r, r))
if (tf.sqrt(rsnew) < toll):
break
p = r + (rsnew / rsold) * p
rsold = rsnew
return([x,it,tf.sqrt(rsnew)])
mesh = create_mesh(Lx=10,Ly=10,Nx=100,Ny=100)
x0 = tf.constant( (mesh['Pts'][:,0]<5 ).astype(np.float32) )
nit_time = 10
dcoef = 1.0
maxit = x0.shape[0]//2
stoll = 1.e-6
print('nb of nodes:\t{}'.format(mesh['Pts'].shape[0]))
print('nb of trias:\t{}'.format(mesh['Tria'].shape[0]))
t0 = time()
MASS0,STIFF0 = compute_matrices_scipy(mesh)
elapsed_scipy=time()-t0
print('Matrices; elapsed: {:3.5f} s'.format(elapsed_scipy))
A = MASS0+dcoef*STIFF0
x = np.copy(np.squeeze(x0.numpy()) )
t0 = time()
for jt in range(nit_time):
b = MASS0*x
x1,it,tol=conjgrad_scipy(A,b,x,niter=maxit,toll=stoll)
x=np.copy(x1)
print('time {}; iters {}; resid: {:3.2f}'.format(1+jt,it,tol) )
elapsed_scipy=time()-t0
print('elapsed, scipy: {:3.5f} s'.format(elapsed_scipy))
t0 = time()
MASS,STIFF =compute_matrices_tensorflow(mesh)
elapsed=time()-t0
print('Matrices; elapsed: {:3.5f} s'.format(elapsed))
x = None
x1 = None
A = tf.sparse.add(MASS,tf.sparse.map_values(tf.multiply, STIFF, dcoef))
x = tf.expand_dims(tf.identity(x0),axis=1)
t0 = time()
for jt in range(nit_time):
b = tf.sparse.sparse_dense_matmul(MASS,x)
x1,it,tol=conjgrad_tensorflow(A,b,x,niter=maxit,toll=stoll)
x = x1
print('time {}; iters {}; resid: {:3.2f}'.format(1+jt,it,tol) )
elapsed_tf=time()-t0
print('elapsed, tf: {:3.2f} s'.format(elapsed_tf))
print('elapsed times:')
print('scipy: {:3.2f} s\ttf: {:3.2f} s'.format(elapsed_scipy,elapsed_tf))

How can I convert a nested list into a numpy array to prevent the ValueError: Failed to convert a NumPy array to a Tensor in a keras model

Hello I am trying to generate synthetic data to use them as input inside a simple model for now.
Here is the code for creating the generic data:
trajectories = 200
y = 20 #degrees
ns0 = 180 #contact surface initial value
r = 0.003 # roller radiues
vPoisson = 0.3
youngModulus = 2.1 * 1e11
relevantImpalance = 20 *1e-5 #kg.m
systemMass = 5 #kg
rotationSpeed = 2* np.pi*70 #Hz
steps = 1/400
def hertz_elastic_contact(r = r, v = vPoisson, e = youngModulus):
return 2/3 * np.sqrt(r) * e/(1-np.square(v))
def total_contact_stiffness(y = y, ns = ns0):
kp = hertz_elastic_contact()
a = np.power(1/kp, 2/3)
b = np.power(np.cos(y), 5/2)
c = ns * kp
disc = a + np.power(b/c, 2/3)
return 1/disc
def healthy_trajectories(trajectories = trajectories):
healthyTime = []
for i in range(trajectories):
healthyTime.append(random.randint(2,20))
return healthyTime
def stiffness_calculation(healthy, steps = steps, ns = ns0, nsStart = ns0):
Kp = hertz_elastic_contact()
kts = total_contact_stiffness()
ktsThreshold = kts / 5
i = 1
stiffness = []
time = []
while kts > ktsThreshold:
kts = 1/ (( np.power(1/Kp,2/3) + np.power(np.power(np.cos(y),5/2) / (ns * Kp) * steps * i, 2/3) ))
kts += np.random.normal(kts/10, kts/20, None)
ns = np.absolute(nsStart - steps * i)
ns += np.random.normal(ns/10, ns/20, None)
stiffness.append(kts)
time.append(healthy + steps * i)
i += 1
return stiffness, time
def stiffness_degradation(healthy):
stiffnessList = []
daysList = []
gap = 100
for i in range(len(healthy)):
monitoringPoint = []
days = []
stiffness, time = stiffness_calculation(healthy[i])
print(i)
for j in range(int(len(stiffness)/gap)):
monitoringPoint.append(stiffness[j*gap])
days.append(time[gap*j])
monitoringPoint.append(stiffness[-1])
days.append(time[-1])
stiffnessList.append(monitoringPoint)
daysList.append(days)
return stiffnessList, daysList
def total_stiffness(healthy, stiffness, degradationDays):
stiffnessTotal = []
DaysTotal = []
healthyPointMonitoring = []
for i in range(len(stiffness)):
healthyStiffness = stiffness[i][0]
stiffnessHealthy = int(healthy[i] * 4) * [healthyStiffness]
healthyPointMonitoring.append(len(stiffnessHealthy))
days = [healthy[i] / 4] * len(stiffnessHealthy)
stiffnessHealthy.extend(stiffness[i])
days.extend(degradationDays[i])
stiffnessTotal.append(stiffnessHealthy)
DaysTotal.append(days)
return stiffnessTotal, DaysTotal, healthyPointMonitoring
healthyList = healthy_trajectories()
a,b = stiffness_degradation(healthyList)
finalStiff, finalDays, healthyMonitoring = total_stiffness(healthyList, a,b)
RUL = []
for i in range(len(finalDays)):
rulTemp = [np.max(finalDays[i]) - x for x in finalDays[i]]
RUL.append(rulTemp)
Let's say I want to train this following model by using finalStiff as X and RUL as y:
modelTest = keras.Sequential ([
layers.Dense(64, activation='relu'),
layers.Dense(32, activation='relu'),
layers.Dense(1)
])
modelTest.compile(optimizer='adam', loss='mae', metrics=[tf.keras.metrics.RootMeanSquaredError()])
For this I did a finalStiffNp = np.array(finalStiff, dtype = object) and rulArray = np.array(rulList, dtype=object). WHen i am trying to fit the model like this : history=modelTest.fit(finalStiffNp , rulTestArray), I ran into this error:
ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type list).
How can I fix this error. Thanks

Create a 2-hierarchy estimate of (normal) mean and standard deviation

I have a normally distributed variable x (like product demand), an index id_1 (like product number) and a second index id_2 (like product group). My goal is to estimate the mean and the standard deviations for x hierarchically (all > product group > product).
That's my data:
import numpy as np
import pymc3 as pm
import arviz as az
# data
my_step_1 = 0.4
my_step_2 = 4.1
sd_step_1 = 0.1
sd_step_2 = 0.2
my = 10
sd = .1
grp_n = 8
grps_1 = 5
grps_2 = 4
x = np.round(np.concatenate([np.random.normal(my + i * my_step_1 + j * my_step_2, \
sd + i * sd_step_1 + j * sd_step_2, grp_n) \
for i in range(grps_1) for j in range(grps_2)]), 1) # demand
id_1 = np.repeat(np.arange(grps_1 * grps_2), grp_n) # group, product number
id_2 = np.tile(np.repeat(np.arange(grps_2), grp_n), grps_1) # super-group, product group
shape_1 = len(np.unique(id_1))
shape_2 = len(np.unique(id_2))
I've managed a single hierarchy:
with pm.Model() as model_h1:
#
mu_mu_hyper = pm.Normal('mu_mu_hyper', mu = 0, sd = 10)
mu_sd_hyper = pm.HalfNormal('mu_sd_hyper', 10)
sd_hyper = pm.HalfNormal('sd_hyper', 10)
#
mu = pm.Normal('mu', mu = mu_mu_hyper, sd = mu_sd_hyper, shape = shape_1)
sd = pm.HalfNormal('sd', sd = sd_hyper, shape = shape_1)
y = pm.Normal('y', mu = mu[id_1], sd = sd[id_1], observed = x)
trace_h1 = pm.sample(1000)
#az.plot_forest(trace_h1, var_names=['mu', 'sd'], combined = True)
But how can I code 2 hierarchies?
# 2 hierarchies .. doesn't work
with pm.Model() as model_h2:
#
mu_mu_hyper2 = pm.Normal('mu_mu_hyper2', mu = 0, sd = 10)
mu_sd_hyper2 = pm.HalfNormal('mu_sd_hyper2', sd = 10)
sd_mu_sd_hyper2 = pm.HalfNormal('sd_mu_sd_hyper2', sd = 10)
sd_hyper2 = pm.HalfNormal('sd_hyper2', sd = 10)
#
mu_mu_hyper1 = pm.Normal('mu_hyper1', mu = mu_mu_hyper2, sd = mu_sd_hyper2, shape = shape_2)
mu_sd_hyper1 = pm.HalfNormal('mu_sd_hyper1', sd = sd_mu_sd_hyper2, shape = shape_2)
sd_hyper1 = pm.HalfNormal('sd_hyper1', sd = sd_hyper2, shape = shape_2)
#sd_hyper1 = pm.HalfNormal('sd_hyper1', sd = sd_hyper2[id_2], shape = shape_2)??
#
mu = pm.Normal('mu', mu = mu_mu_hyper1, sd = mu_sd_hyper1, shape = shape_1)
sd = pm.HalfNormal('sd', sd = sd_hyper1, shape = shape_1)
y = pm.Normal('y', mu = mu[id_1], sd = sd[id_1], observed = x)
trace_h2 = pm.sample(1000)

You could try looping through product groups and use the mean, std for a group as constraints for the products belonging to this particular group.
# sample product group to product mapping
group_product_mapping = {0: [1, 2, 3], 1: [4, 5, 6]}
total_groups = len(group_product_mapping.keys())
with pm.model() as model_h:
mu_all = pm.Normal('mu_all', 0, 10)
sd_all = pm.HalfNormal('sd_all', 10)
sd_mu_group = pm.HalfNormal('sd_mu_group', 10)
# group parameters constrained to mu, sd from all
mu_group = pm.Normal('mu_group', mu_all, sd_all, shape=total_groups)
sd_group = pm.HalfNormal('sd_group', sd_mu_group, shape=total_groups)
mu_products = dict()
# iterate through groups and constrain product parameters to the product group they belong to
for idx, group in enumerate(group_product_mapping.keys()):
mu_products[group] = pm.Normal(f'mu_products_{group}', mu_group[idx], sd_group[idx], shape=len(group_product_mapping[group]))
sd_produtcs[group] = pm.HalfNormal(f'sd_mu_products_{group}', 10)

How to make the value not null in skewness in the extraction of the color moment feature?

I want to use the color moment feature extraction, which must get the value (mean, standard deviation and skewness) but the skewness value cannot be entered into the excel because there is a negative division value and the skewness result becomes (Nan), how can you still use it?
<<RuntimeWarning: invalid value encountered in double_scalars
hasil_akhir_skewS (skewnessS/totS)**(1/3)>>
import cv2
import numpy as np
import math
import xlsxwriter as xlsx
book = xlsx.Workbook('data_ayam.xlsx')
sheet = book.add_worksheet()
sheet.write(0, 0, 'file')
column = 1
fitur = ['meanH','meanS','meanV', 'stdevH','stdevS','stdevV', 'skewnessH','skewnessS','skewnessV']
for i in fitur :
sheet.write(0, column, i)
column += 1
ayam_type = ['non_segar', 'segar']
sum_each_type = 100
row = 1
for i in ayam_type:
for j in range(1, sum_each_type+1):
column = 0
file_name = 'ayam/' + i + str(j) + '.jpg'
print(file_name)
sheet.write(row, column, file_name)
column += 1
#preprocessing
img = cv2.imread(file_name, 1)
resized_image = cv2.resize(img, (256, 256))
fitur = cv2.cvtColor(resized_image, cv2.COLOR_BGR2HSV)
H = fitur[:, :, 0]
S = fitur[:, :, 1]
V = fitur[:, :, 2]
totH = H.size
totS = S.size
totV = V.size
totalH = H.sum()
totalS = S.sum()
totalV = V.sum()
#Mean
meanH = totalH/totH #equivalent to file_name.mean()
meanS= totalS/totS #equivalent to file_name.mean()
meanV = totalV/totV #equivalent to file_name.mean()
#Stdev
vrH = ((H - meanH)**2).sum()
vrS = ((S - meanS)**2).sum()
vrV = ((V - meanV)**2).sum()
hasil_akhir_stdH = math.sqrt((vrH/totH))
hasil_akhir_stdS = math.sqrt((vrS/totS))
hasil_akhir_stdV = math.sqrt((vrV/totV))
#Skewness
meanijH = (H - meanH)
meanijS = (S - meanS)
meanijV = (V - meanV)
skewnessH = (meanijH**3).sum()
skewnessS = (meanijS**3).sum()
skewnessV = (meanijV**3).sum()
hasil_akhir_skewH = (skewnessH/totH)**(1/3)
hasil_akhir_skewS = (skewnessS/totS)**(1/3)
hasil_akhir_skewV = (skewnessV/totV)**(1/3)
hasil = [meanH,
meanS,
meanV,
hasil_akhir_stdH,
hasil_akhir_stdS,
hasil_akhir_stdV,
hasil_akhir_skewH,
hasil_akhir_skewS,
hasil_akhir_skewV]
feature_props = {
hasil[0],
hasil[1],
hasil[2],
hasil[3],
hasil[4],
hasil[5],
hasil[6],
hasil[7],
hasil[8]
}
for item in feature_props:
sheet.write(row, column, item)
column += 1
row += 1
book.close()
This is the error:

How to find Average directional movement for stocks using Pandas?

I have a dataframe of OHLCV data. I would like to know if anyone knows any tutorial or any way of finding ADX(Average directional movement ) using pandas?
import pandas as pd
import yfinance as yf
import matplotlib.pyplot as plt
import datetime as dt
import numpy as nm
start=dt.datetime.today()-dt.timedelta(59)
end=dt.datetime.today()
df=pd.DataFrame(yf.download("MSFT", start=start, end=end))
The average directional index, or ADX, is the primary technical indicator among the five indicators that make up a technical trading system developed by J. Welles Wilder, Jr. and is calculated using the other indicators that make up the trading system. The ADX is primarily used as an indicator of momentum, or trend strength, but the total ADX system is also used as a directional indicator.
Directional movement is calculated by comparing the difference between two consecutive lows with the difference between their respective highs.
For the excel calculation of ADX this is a really good video:
https://www.youtube.com/watch?v=LKDJQLrXedg&t=387s

I was playing with this a little bit and found something that can help you with the issue:
def ADX(data: pd.DataFrame, period: int):
"""
Computes the ADX indicator.
"""
df = data.copy()
alpha = 1/period
# TR
df['H-L'] = df['High'] - df['Low']
df['H-C'] = np.abs(df['High'] - df['Close'].shift(1))
df['L-C'] = np.abs(df['Low'] - df['Close'].shift(1))
df['TR'] = df[['H-L', 'H-C', 'L-C']].max(axis=1)
del df['H-L'], df['H-C'], df['L-C']
# ATR
df['ATR'] = df['TR'].ewm(alpha=alpha, adjust=False).mean()
# +-DX
df['H-pH'] = df['High'] - df['High'].shift(1)
df['pL-L'] = df['Low'].shift(1) - df['Low']
df['+DX'] = np.where(
(df['H-pH'] > df['pL-L']) & (df['H-pH']>0),
df['H-pH'],
0.0
)
df['-DX'] = np.where(
(df['H-pH'] < df['pL-L']) & (df['pL-L']>0),
df['pL-L'],
0.0
)
del df['H-pH'], df['pL-L']
# +- DMI
df['S+DM'] = df['+DX'].ewm(alpha=alpha, adjust=False).mean()
df['S-DM'] = df['-DX'].ewm(alpha=alpha, adjust=False).mean()
df['+DMI'] = (df['S+DM']/df['ATR'])*100
df['-DMI'] = (df['S-DM']/df['ATR'])*100
del df['S+DM'], df['S-DM']
# ADX
df['DX'] = (np.abs(df['+DMI'] - df['-DMI'])/(df['+DMI'] + df['-DMI']))*100
df['ADX'] = df['DX'].ewm(alpha=alpha, adjust=False).mean()
del df['DX'], df['ATR'], df['TR'], df['-DX'], df['+DX'], df['+DMI'], df['-DMI']
return df
At the beginning the values aren't correct (as always with the EWM approach) but after several computations it converges to the correct value.

Math was taken from here.
def ADX(df):
def getCDM(df):
dmpos = df["High"][-1] - df["High"][-2]
dmneg = df["Low"][-2] - df["Low"][-1]
if dmpos > dmneg:
return dmpos
else:
return dmneg
def getDMnTR(df):
DMpos = []
DMneg = []
TRarr = []
n = round(len(df)/14)
idx = n
while n <= (len(df)):
dmpos = df["High"][n-1] - df["High"][n-2]
dmneg = df["Low"][n-2] - df["Low"][n-1]
DMpos.append(dmpos)
DMneg.append(dmneg)
a1 = df["High"][n-1] - df["High"][n-2]
a2 = df["High"][n-1] - df["Close"][n-2]
a3 = df["Low"][n-1] - df["Close"][n-2]
TRarr.append(max(a1,a2,a3))
n = idx + n
return DMpos, DMneg, TRarr
def getDI(df):
DMpos, DMneg, TR = getDMnTR(df)
CDM = getCDM(df)
POSsmooth = (sum(DMpos) - sum(DMpos)/len(DMpos) + CDM)
NEGsmooth = (sum(DMneg) - sum(DMneg)/len(DMneg) + CDM)
DIpos = (POSsmooth / (sum(TR)/len(TR))) *100
DIneg = (NEGsmooth / (sum(TR)/len(TR))) *100
return DIpos, DIneg
def getADX(df):
DIpos, DIneg = getDI(df)
dx = (abs(DIpos- DIneg) / abs(DIpos + DIneg)) * 100
ADX = dx/14
return ADX
return(getADX(df))
print(ADX(df))

This gives you the exact numbers as Tradingview and Thinkorswim.
import numpy as np
def ema(arr, periods=14, weight=1, init=None):
leading_na = np.where(~np.isnan(arr))[0][0]
arr = arr[leading_na:]
alpha = weight / (periods + (weight-1))
alpha_rev = 1 - alpha
n = arr.shape[0]
pows = alpha_rev**(np.arange(n+1))
out1 = np.array([])
if 0 in pows:
out1 = ema(arr[:int(len(arr)/2)], periods)
arr = arr[int(len(arr)/2) - 1:]
init = out1[-1]
n = arr.shape[0]
pows = alpha_rev**(np.arange(n+1))
scale_arr = 1/pows[:-1]
if init:
offset = init * pows[1:]
else:
offset = arr[0]*pows[1:]
pw0 = alpha*alpha_rev**(n-1)
mult = arr*pw0*scale_arr
cumsums = mult.cumsum()
out = offset + cumsums*scale_arr[::-1]
out = out[1:] if len(out1) > 0 else out
out = np.concatenate([out1, out])
out[:periods] = np.nan
out = np.concatenate(([np.nan]*leading_na, out))
return out
def atr(highs, lows, closes, periods=14, ema_weight=1):
hi = np.array(highs)
lo = np.array(lows)
c = np.array(closes)
tr = np.vstack([np.abs(hi[1:]-c[:-1]),
np.abs(lo[1:]-c[:-1]),
(hi-lo)[1:]]).max(axis=0)
atr = ema(tr, periods=periods, weight=ema_weight)
atr = np.concatenate([[np.nan], atr])
return atr
def adx(highs, lows, closes, periods=14):
highs = np.array(highs)
lows = np.array(lows)
closes = np.array(closes)
up = highs[1:] - highs[:-1]
down = lows[:-1] - lows[1:]
up_idx = up > down
down_idx = down > up
updm = np.zeros(len(up))
updm[up_idx] = up[up_idx]
updm[updm < 0] = 0
downdm = np.zeros(len(down))
downdm[down_idx] = down[down_idx]
downdm[downdm < 0] = 0
_atr = atr(highs, lows, closes, periods)[1:]
updi = 100 * ema(updm, periods) / _atr
downdi = 100 * ema(downdm, periods) / _atr
zeros = (updi + downdi == 0)
downdi[zeros] = .0000001
adx = 100 * np.abs(updi - downdi) / (updi + downdi)
adx = ema(np.concatenate([[np.nan], adx]), periods)
return adx