I'm trying to implement the ILU preconditioner in this GMRES code I wrote (in order to solve the linear sistem Ax = b. I'm trying with an easy tridiagonal SPD matrix of dimension 25x25. As you can see I'm calculating the preconditioner with spilu method. The code is running without error, but the solution is clearly wrong since, at the end of the code, I'm printing the norm of b and the norm of the product A*x. They are not nearly the same..
The code Run fine without preconditioner and converge with 13 iteration for the same matrix.
This is the code I followed
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
'Size controller'
matrixSize =25
'Building a tri-diagonal matrix'
def Atridiag(val_0, val_sup, val_inf, mSize):
cen = np.ones((1, mSize))*val_0
sup = np.ones((1, mSize-1))*val_sup
inf = np.ones((1, mSize-1))*val_inf
diag_cen = np.diagflat(cen, 0)
diag_sup = np.diagflat(sup, 1)
diag_inf = np.diagflat(inf, -1)
return diag_cen + diag_sup + diag_inf
A = Atridiag(2, -1, -1, matrixSize)
A = sp.sparse.csc_matrix (A)
'Plot matrix sparsity'
plt.clf()
plt.spy(A, marker ='.', markersize=2)
plt.show()
'random b and x0 vectors'
b = np.matrix(np.ones((matrixSize, 1)))
x = np.matrix(np.ones((matrixSize, 1)))
'Incomplete LU'
M = sp.sparse.linalg.dsolve.spilu(A)
M1 = lambda x: M.solve(x)
M2=sp.sparse.linalg.LinearOperator((matrixSize,matrixSize),M1)
'Initial Data'
nmax_iter = 30
rstart = 2
tol = 1e-7
e = np.zeros((nmax_iter + 1, 1))
rr = 1
'Starting GMRES'
for rs in range (0, rstart+1):
'first check on residual'
if rr < tol :
break
else:
r0 = (b - A.dot(x))
betha = np.linalg.norm(r0)
e[0] = betha
H = np.zeros((nmax_iter + 1, nmax_iter))
V = np.zeros((matrixSize, nmax_iter+1))
V[:, 0:1] = r0/betha
for k in range (1, nmax_iter+1):
'Appling the Preconditioner'
t = A.dot(V[:, k-1])
V[:, k] = M2.matvec(t)
'Ortogonalizzazione GS'
for j in range (k):
H[j, k-1] = np.dot(V[:, k].T, V[:, j])
V[:, k] = V[:, k] - (np.dot(H[j, k-1], V[:, j]))
H[k, k-1] = np.linalg.norm(V[:, k])
V[:, k] = V[:, k] / H[k, k-1]
'QR Decomposition'
n=k
Q = np.zeros((n+1, n))
R = np.zeros((n, n))
R[0, 0] = np.linalg.norm(H[0:n+2, 0])
Q[:, 0] = H[0:n+1, 0] / R[0,0]
for j in range (0, n+1):
t = H[0:n+1, j-1]
for i in range (0, j-1):
R[i, j-1] = np.dot(Q[:, i], t)
t = t - np.dot(R[i, j-1], Q[:, i])
R[j-1, j-1] = np.linalg.norm(t)
Q[:, j-1] = t / R[j-1, j-1]
g = np.dot(Q.T, e[0:k+1])
Z = np.dot(np.linalg.inv(R), g)
Res = e[0:n] - np.dot(H[0:n, 0:n], Z[0:n])
rr = np.linalg.norm(Res)
'second check on residual'
if rr < tol:
break
'Updating the solution'
x = x + np.dot(V[:, 0:k], Z)
print(sp.linalg.norm(b))
print(sp.linalg.norm(np.dot(A.todense(),x)))
Really Hope somebody can figure it out!!
Maybe it's too late, but for future reference :
You forgot to multiply by the conditioner when updating x :
x = x + M2.dot(np.dot(V[:, 0:k], Z) # M2.matvec() works the same
See here
With that fix, the algorithm converges in 1 iteration.
Other comments:
You can directly do : M2 = sp.sparse.linalg.LinearOperator((matrixSize,matrixSize),M.solve)
At the end, to compare Ax and b, it's better to print the difference (residual) because you will get a much more precise result: print(sp.linalg.norm(b - np.dot(A.todense(),x)))
Related
I am trying to write an algorithm for simulating the steady flow in a windtunnel around a rectangle. I based my code heavily on the beam.py code in the book 'A survey of computational physics' by Landau.
The code in the book is the following:
import matplotlib.pylab as p;
from mpl_toolkits.mplot3d import Axes3D ;
from numpy import *;
import numpy;
print( "Working, look for figure window after 100 iterations")
Nxmax = 70; Nymax = 20; IL = 10; H = 8; T = 8; h = 1.
u = zeros( (Nxmax + 1, Nymax + 1), float) # Stream
w = zeros( (Nxmax + 1, Nymax + 1), float) # Vorticity
V0 = 1.0; omega = 0.1; nu = 1.; iter = 0; R = V0*h/nu # Renold #
def borders(): # Method borders: init & B.C
for i in range(0, Nxmax + 1): # Initialize stream function
for j in range(0, Nymax + 1 ): # And vorticity
w[i, j] = 0.
u[i, j] = j * V0
for i in range(0, Nxmax + 1 ): # Fluid surface
u[i, Nymax] = u[i, Nymax - 1] + V0 * h
w[i, Nymax - 1] = 0.
for j in range(0, Nymax + 1 ):
u[1, j] = u[0, j]
w[0, j] = 0. # Inlet
for i in range(0, Nxmax + 1 ): # Centerline
if i <= IL and i>= IL + T:
u[i, 0] = 0.
w[i, 0] = 0.
for j in range(1, Nymax ): # Outlet
w[Nxmax, j] = w[Nxmax - 1, j]
u[Nxmax, j] = u[Nxmax - 1, j] # Boundary conditions
def beam(): # Method beam; BC for beam
for j in range (0, H + 1): # Beam sides
w[IL, j] = - 2 * u[IL - 1, j]/(h*h) # Front side
w[IL + T, j] = - 2 * u[IL + T + 1, j]/(h*h) # Back side
for i in range(IL, IL + T + 1): w[i, H - 1] = - 2 * u[i, H]/(h*h);
for i in range(IL, IL + T + 1 ):
for j in range(0, H + 1):
u[IL, j] = 0. # Front
u[IL+T, j] = 0. # Back
u[i, H] = 0; # top
def relax(): # Method to relax stream
beam() # Reset conditions at beam
for i in range(1, Nxmax): # Relax stream function
for j in range (1, Nymax):
r1 = omega*((u[i+1,j]+u[i-1,j]+u[i,j+1]+u[i,j-1] + h*h*w[i,j])*0.25-u[i,j])
u[i, j] += r1
for i in range(1, Nxmax): # Relax vorticity
for j in range(1, Nymax):
a1 = w[i+1, j] + w[i-1,j] + w[i,j+1] + w[i,j-1]
a2 = (u[i,j+1] - u[i,j-1])*(w[i+1,j] - w[i - 1, j])
a3 = (u[i+1,j] - u[i-1,j])*(w[i,j+1] - w[i, j - 1])
r2 = omega *( (a1 - (R/4.)*(a2 - a3) )/4.0 - w[i,j])
w[i, j] += r2
borders()
while (iter <= 100):
iter += 1
if iter%10 == 0: print (iter)
relax()
for i in range (0, Nxmax + 1):
for j in range(0, Nymax + 1 ): u[i, j] = u[i, j]/(V0*h) # V0h units
x = range(0, Nxmax - 1); y = range(0, Nymax - 1) # returns stream flow to plot
# for several iterations
X, Y = p.meshgrid(x, y)
def functz(u): # Return transform
Z = u[X, Y]
return Z
Z = functz(u) # here the function is called
fig = p.figure() # creates the figure
ax = Axes3D(fig) # plots the axis for the figure
ax.plot_wireframe(X, Y, Z, color = 'r') # surface of wireframe in red
ax.set_xlabel('X') # label the three axes
ax.set_ylabel('Y')
ax.set_zlabel('Stream Function')
p.show()
I discovered that if I increase the value of V_0 above 2 or 3 then I get a RuntimeWarning: invalid value encountered in double_scalars error when r1 and r2 are calculated.
I want to simulate with much larger velocities and I couldn't find a way to fix this problem. I don't really understand why it is even an error when there are no divisions by really small numbers, close to 0.
Can anyone help me out and spot the problem?
Thanks in advance
I tried to look up the answer but only found special scipy libraries for certain operators but in this code none of them could be used.
This runs the Gram–Schmidt algorithm on a square matrix.
The lines causing problems are the following
R[j, j] = la.norm(Q[:, j])
a = Q[:,j]
b = a/R[j,j]
Q[:,j] = Q[:,j]/R[j,j]
By running these lines the column Q[:,j] is set to 0 instead of the correct value. This doesn't happen if I use a temporary variable. How is this possible?
The full code
import numpy as np
import numpy.linalg as la
import matplotlib.pyplot as plt
def mod_gramschmidt(X):
n = X.shape[0]
R = np.zeros((n, n))
Q = X.copy()
for j in range(n):
R[j, j] = la.norm(Q[:, j])
a = Q[:, j]
b = a / R[j, j]
Q[:, j] = Q[:, j] / R[j, j]
A = R[j, (j + 1) :]
B = Q[:, j].T
R[j, (j + 1) :] = Q[:, j].T # X[:, (j + 1) :]
Q[:, (j + 1) :] = Q[:, (j + 1) :] - Q[:, j] # R[j, (j + 1) :]
I run the code with the following input:
A = np.array([[1,2,3],[4,5,6],[7,5,4]])
print(mod_gramschmidt(A))
After
dropping unnecessary a, b, A, and B variables,
fixing matrix multiplication Q[:, j].reshape((-1,1)) # R[j, (j + 1) :].reshape((1,-1)) by reshaping vectors Q[:, j] and R[j, (j + 1) :] into n x 1and 1 x m matrices,
adding return statement
the presented algorithm reads as
import numpy as np
import numpy.linalg as la
import matplotlib.pyplot as plt
def mod_gramschmidt(X):
n = X.shape[0]
R = np.zeros((n, n))
Q = X.copy()
for j in range(n):
R[j, j] = la.norm(Q[:, j])
Q[:, j] = Q[:, j] / R[j, j]
R[j, (j + 1) :] = Q[:, j].T # X[:, (j + 1) :]
Q[:, (j + 1) :] = Q[:, (j + 1) :] - Q[:, j].reshape((-1,1)) # R[j, (j + 1) :].reshape((1,-1))
return Q
Now applying this algorithm to integer matrix
A = np.array([[1,2,3],[4,5,6],[7,5,4]])
print(mod_gramschmidt(A))
yields
[[0 0 0]
[0 0 0]
[0 0 0]]
However, applying the same algorithm to float matrix
A = np.array([[1.,2.,3.],[4.,5.,6.],[7.,5.,4.]])
print(mod_gramschmidt(A))
yields
[[ 0.12309149 0.52015649 0.84515425]
[ 0.49236596 0.70741282 -0.50709255]
[ 0.86164044 -0.47854397 0.16903085]]
If inside the algorithm the integer matrix are casted astype(float)
def mod_gramschmidt(X):
n = X.shape[0]
R = np.zeros((n, n)).astype(float)
Q = X.copy().astype(float)
...
then the error connected to integer matrix calculations does not appear.
I try to code an LUP (or PLU it's the same) factorization in python. I have a code which works for small matrix (under a 4x4 size). However when I have tried it with a random generated matrix the decomposition has failed.
import numpy as np
def LUP_factorisation(A):
"""Find P, L and U : PA = LU"""
U = A.copy()
shape_a = U.shape
n = shape_a[0]
L = np.eye(n)
P = np.eye(n)
for i in range(n):
print(U)
k = i
comp = abs(U[i, i])
for j in range(i, n):
if abs(U[j, i]) > comp:
k = j
comp = abs(U[j, i])
line_u = U[k, :].copy()
U[k, :] = U[i, :]
U[i, :] = line_u
print(U)
line_p = P[k, :].copy()
P[k, :] = P[i, :]
P[i, :] = line_p
for j in range(i + 1, n):
g = U[j, i] / U[i, i]
L[j, i] = g
U[j, :] -= g * U[i, :]
return L, U, P
if __name__ == "__main__":
A = np.array(
[[1.0, 2.2, 58, 9.5, 42.65], [6.56, 58.789954, 4.45, 23.465, 6.165], [7.84516, 8.9864, 96.546, 4.654, 7.6514],
[45.65, 47.985, 1.56, 3.9845, 8.6], [455.654, 102.615, 63.965, 5.6, 9.456]])
L, U, P = LUP_factorisation(A)
print(L # U)
print(P # A)
With the example I gave it works: we have PA = LU. But when i do for example :
A = np.random.rand(10, 10)
Then, i don't obtain a good result because PA is different of LU. Any ideas ? Thanks.
As #MattTimmermans writes you should swap rows in both L and U.
Normally this is implicitly handled by storing LU in A and then the swaps are automatically applied to both L and U. See https://en.wikipedia.org/wiki/LU_decomposition#C_code_example
But you have split them so you have to add
line_l = L[k, :].copy()
L[k, :] = L[i, :]
L[i, :] = line_l
Only testing it with diagonally dominant matrices is really bad; and only testing linear algebra routines with random matrices is known to be bad as their properties are very specific - and not "random". See work by Trefethen and his students, e.g. http://dspace.mit.edu/handle/1721.1/14322
The goal of testing should be to find bugs - not to make test-cases so simple that it works.
Make sure that the diagonal of the input matrix A is dominant. So add some value to the diagonal of A, e.g.
A = A + np.eye(A.shape)
or
A = A + 100* np.eye(A.shape)
I hope that helps !
The Wikipedia entry for the Arnoldi method provides a Python example that produces basis of the Krylov subspace of a matrix A. Supposedly, if A is Hermitian (i.e. if A == A.conj().T) then the Hessenberg matrix h generated by this algorithm is tridiagonal (source). However, when I use the Wikipedia code on a real-world Hermitian matrix, the Hessenberg matrix is not at all tridiagonal. When I perform the computation on the real part of A (so that A == A.T) then I do get a tridiagonal Hessenberg matrix, so there seems to be a problem with the imaginary components of A. Does anybody know why the Wikipedia code doesn't produce the expected results?
Working example:
import numpy as np
import matplotlib.pyplot as plt
from scipy.linalg import circulant
def arnoldi_iteration(A, b, n):
m = A.shape[0]
h = np.zeros((n + 1, n), dtype=np.complex)
Q = np.zeros((m, n + 1), dtype=np.complex)
q = b / np.linalg.norm(b) # Normalize the input vector
Q[:, 0] = q # Use it as the first Krylov vector
for k in range(n):
v = A.dot(q) # Generate a new candidate vector
for j in range(k + 1): # Subtract the projections on previous vectors
h[j, k] = np.dot(Q[:, j], v)
v = v - h[j, k] * Q[:, j]
h[k + 1, k] = np.linalg.norm(v)
eps = 1e-12 # If v is shorter than this threshold it is the zero vector
if h[k + 1, k] > eps: # Add the produced vector to the list, unless
q = v / h[k + 1, k] # the zero vector is produced.
Q[:, k + 1] = q
else: # If that happens, stop iterating.
return Q, h
return Q, h
# Construct matrix A
N = 2**4
I = np.eye(N)
k = np.fft.fftfreq(N, 1.0 / N) + 0.5
alpha = np.linspace(0.1, 1.0, N)*2e2
c = np.fft.fft(alpha) / N
C = circulant(c)
A = np.einsum("i, ij, j->ij", k, C, k)
# Show that A is Hermitian
print(np.allclose(A, A.conj().T))
# Arbitrary (random) initial vector
np.random.seed(0)
v = np.random.rand(N)
# Perform Arnoldi iteration with complex A
_, h = arnoldi_iteration(A, v, N)
# Perform Arnoldi iteration with real A
_, h2 = arnoldi_iteration(np.real(A), v, N)
# Plot results
plt.subplot(121)
plt.imshow(np.abs(h))
plt.title("Complex A")
plt.subplot(122)
plt.imshow(np.abs(h2))
plt.title("Real A")
plt.tight_layout()
plt.show()
Result:
After browsing through some conference presentation slides, I realised that at some point Q had to be conjugated when A is complex. The correct algorithm is posted below for reference, with the code change marked (note that this correction has also been submitted to the Wikipedia entry):
import numpy as np
def arnoldi_iteration(A, b, n):
m = A.shape[0]
h = np.zeros((n + 1, n), dtype=np.complex)
Q = np.zeros((m, n + 1), dtype=np.complex)
q = b / np.linalg.norm(b)
Q[:, 0] = q
for k in range(n):
v = A.dot(q)
for j in range(k + 1):
h[j, k] = np.dot(Q[:, j].conj(), v) # <-- Q needs conjugation!
v = v - h[j, k] * Q[:, j]
h[k + 1, k] = np.linalg.norm(v)
eps = 1e-12
if h[k + 1, k] > eps:
q = v / h[k + 1, k]
Q[:, k + 1] = q
else:
return Q, h
return Q, h
I am learning some techniques for doing statistics with missing data from a book (Statistical Analysis with Missing Data by Little and Rubin). One particularly useful function for working with monotone non-response data is the Sweep Operator (details on page 148-151). I know that the R module gmm has the swp function which does this but I was wondering if anyone has implemented this function in Python, ideally for Numpy matrices to hold the input data. I searched StackOverflow and also did several web searches without success. Thanks for any help.
Here is the definition.
A PxP symmetric matrix G is said to be swept on row and column k if it is replaced by another symmetric PxP matrix H with elements defined as follows:
h_kk = -1/g_kk
h_jk = h_kj = g_jk/g_kk for j != k
h_jl = g_jl - g_jk g_kl / g_kk j != k, l != k
G = [g11, g12, g13
g12, g22, g23
g13, g23, g33]
H = SWP(1,G) = [-1/g11, g12/g11, g13/g11
g12/g11, g22-g12^2/g11, g23-g13*g12/g11
g13/g11, g23-g13*g12/g11, g33-g13^2/g11]
kvec = [k1,k2,k3]
SWP[kvec,G] = SWP(k1,SWP(k2,SWP(k3,G)))
Inverse function
H = RSW(k,G)
h_kk = -1/g_kk
h_jk = h_kj = -g_jk/g_kk for j != k
h_jl = g_jk g_kl / g_kk j != k, l != k
G == SWP(k,RSW(k,G)) == RSW(k,SWP(k,G))
def sweep(g, k):
g = np.asarray(g)
n = g.shape[0]
if g.shape != (n, n):
raise ValueError('Not a square array')
if not np.allclose(g - g.T, 0):
raise ValueError('Not a symmetrical array')
if k >= n:
raise ValueError('Not a valid row number')
# Fill with the general formula
h = g - np.outer(g[:, k], g[k, :]) / g[k, k]
# h = g - g[:, k:k+1] * g[k, :] / g[k, k]
# Modify the k-th row and column
h[:, k] = g[:, k] / g[k, k]
h[k, :] = h[:, k]
# Modify the pivot
h[k, k] = -1 / g[k, k]
return h
I have no way of testing the above code, but I found an alternativee description here, which is valid for non-symmetrical matrices, which can be calculated as follows:
def sweep_non_sym(a, k):
a = np.asarray(a)
n = a.shape[0]
if a.shape != (n, n):
raise ValueError('Not a square array')
if k >= n:
raise ValueError('Not a valid row number')
# Fill with the general formula
b = a - np.outer(a[:, k], a[k, :]) / a[k, k]
# b = a - a[:, k:k+1] * a[k, :] / a[k, k]
# Modify the k-th row and column
b[k, :] = a[k, :] / a[k, k]
b[:, k] = -a[:, k] / a[k, k]
# Modify the pivot
b[k, k] = 1 / a[k, k]
return b
This one does give the correct results for the examples in that link:
>>> a = [[2,4],[3,1]]
>>> sweep_non_sym(a, 0)
array([[ 0.5, 2. ],
[-1.5, -5. ]])
>>> sweep_non_sym(sweep_non_sym(a, 0), 1)
array([[-0.1, 0.4],
[ 0.3, -0.2]])
>>> np.dot(a, sweep_non_sym(sweep_non_sym(a, 0), 1))
array([[ 1.00000000e+00, 0.00000000e+00],
[ 5.55111512e-17, 1.00000000e+00]])