Develop Reference

convergence issue encountered while implementing policy iteration from scratch

I am trying to implement policy iteration from scratch. I have a 2D grid world environment named GridWorld that returns the successor state and reward from a given action, and it also has a function that returns the transition probability. Below is my code for policy iteration:
import matplotlib
matplotlib.use('Agg')
import random
import numpy as np
import matplotlib.pyplot as plt
import gridworld
from tqdm import tqdm
class PolicyIteration:
def __init__(self, env, gamma):
self.env = env
self.num_states = self.env.num_states
self.num_actions = self.env.num_actions
self.max_num_steps = self.env.max_num_steps
self.gamma = gamma #discount factor
self.values = np.zeros(self.num_states) #Initialize `values` as zeros
self.policy = np.random.randint(0, self.num_actions, self.num_states)
def one_policy_evaluation(self):
"""
Runs one iteration of policy evaluation and updates the value function.
:return: the maximum change in value function
"""
delta = 0
for s in range(self.num_states):
v = self.values[s]
a = self.policy[s]
(s_new, r, _) = self.env.step(a)
p = self.env.p(s_new, s, a)
""" update V(s)"""
self.values[s] = np.sum(p * (r + self.gamma * self.values[s_new]))
delta = max(delta, abs(v - self.values[s]))
return delta
def run_policy_evaluation(self, tol = 1e-3):
"""
Runs policy evaluation until convergence.
:param tol: the tolerance level for convergence
:return: the number of iterations of policy evaluation until convergence
"""
delta = self.one_policy_evaluation()
delta_history = [delta]
while delta > tol:
delta = self.one_policy_evaluation()
delta_history.append(delta)
return len(delta_history)
def run_policy_improvement(self):
update_policy_count = 0
for s in range(self.num_states):
temp = self.policy[s]
v_list = np.zeros(self.num_actions)
for a in range(self.num_actions):
(s_new, r, _) = self.env.step(a)
p = self.env.p(s_new, s, a)
v_list[a] = np.sum(p * (r + self.gamma * self.values[s_new]))
self.policy[s] = np.argmax(v_list)
if temp != self.policy[s]:
update_policy_count += 1
return update_policy_count
def train(self, tol=1e-3, max_iters=100, plot=True):
eval_count = self.run_policy_evaluation(tol)
eval_count_history = [eval_count]
policy_change = self.run_policy_improvement()
policy_change_history = [policy_change]
epoch = 0
val_history= []
for i in tqdm(range(max_iters)):
epoch += 1
new_eval_count = self.run_policy_evaluation(tol)
new_policy_change = self.run_policy_improvement()
eval_count_history.append(new_eval_count)
policy_change_history.append(new_policy_change)
val_history.append(np.mean(self.values))
if new_policy_change == 0:
break
print(f'# epoch: {len(policy_change_history)}')
print(f'eval count = {eval_count_history}')
print(f'policy change = {policy_change_history}')
if plot is True:
plt.figure(dpi=200)
plt.plot(val_history)
plt.tight_layout()
plt.savefig('policy_iteration.png')
plt.show()
def main():
env = gridworld.GridWorld(hard_version=False)
agent = PolicyIteration(env, gamma=0.95)
agent.train()
if __name__ == '__main__':
main()
However, based on the figure generated, the sequence of values is oscillating up and down and never converges. I followed the algorithm in the Sutton book step by step, and can't find any issues with my code yet:
Any help is greatly appreciated!

Python decimal.InvalidOperation Error Using A Large Number

In python, I wrote a program to work out the value of pi using the Chudnovsky Algorithm. It works on numbers under 800. However, if I use a number above 800, it returns a decimal.InvalidOperation error. This is my code:
from math import factorial
from decimal import Decimal, getcontext
getcontext().prec = 1000
pi_input = input("How Many Digits Of Pi Are To Be Represented: ")
num = int(pi_input)
def cal(n):
t = Decimal(0)
pi = Decimal(0)
deno = Decimal(0)
for k in range(n):
t = ((-1) ** k) * (factorial(6 * k)) * (13591409 + 545140134 * k)
deno = factorial(3 * k) * (factorial(k) ** 3) * (640320 ** (3 * k))
pi += Decimal(t) / Decimal(deno)
pi = pi * Decimal(12) / Decimal(640320 ** Decimal(1.5))
pi = 1 / pi
return round(pi, n)
print(cal(num))
Could Anyone Please Help Me?

I wouldn't recommend your method. Try using the gmpy module like this:
import sys
import math
import os.path
from gmpy2 import mpz, sqrt
from time import time, sleep
print_pi = input("Should The Value Of Pi Be Printed: ").lower()
print_pi_bool = 0
if "y" in print_pi:
print_pi_bool = True
elif "n" in print_pi:
print_pi = False
else:
print("Incorrect Input. Please Try Again.")
sys.exit()
def sqrt(n, one):
"""
Return the square root of n as a fixed point number with the one
passed in. It uses a second order Newton-Raphson convgence. This
doubles the number of significant figures on each iteration.
"""
# Use floating point arithmetic to make an initial guess
floating_point_precision = 10**16
n_float = float((n * floating_point_precision) // one) / floating_point_precision
x = (int(floating_point_precision * math.sqrt(n_float)) * one) // floating_point_precision
n_one = n * one
count = 0
while 1:
count += 1
print(count)
x_old = x
x = (x + n_one // x) // 2
if x == x_old:
break
return x
def pi_chudnovsky_bs(digits):
"""
Compute int(pi * 10**digits)
This is done using Chudnovsky's series with binary splitting
"""
C = 640320
C3_OVER_24 = C**3 // 24
def bs(a, b):
"""
Computes the terms for binary splitting the Chudnovsky infinite series
a(a) = +/- (13591409 + 545140134*a)
p(a) = (6*a-5)*(2*a-1)*(6*a-1)
b(a) = 1
q(a) = a*a*a*C3_OVER_24
returns P(a,b), Q(a,b) and T(a,b)
"""
if b - a == 1:
# Directly compute P(a,a+1), Q(a,a+1) and T(a,a+1)
if a == 0:
Pab = Qab = mpz(1)
else:
Pab = mpz((6*a-5)*(2*a-1)*(6*a-1))
Qab = mpz(a*a*a*C3_OVER_24)
Tab = Pab * (13591409 + 545140134*a) # a(a) * p(a)
if a & 1:
Tab = -Tab
else:
# Recursively compute P(a,b), Q(a,b) and T(a,b)
# m is the midpoint of a and b
m = (a + b) // 2
# Recursively calculate P(a,m), Q(a,m) and T(a,m)
Pam, Qam, Tam = bs(a, m)
# Recursively calculate P(m,b), Q(m,b) and T(m,b)
Pmb, Qmb, Tmb = bs(m, b)
# Now combine
Pab = Pam * Pmb
Qab = Qam * Qmb
Tab = Qmb * Tam + Pam * Tmb
return Pab, Qab, Tab
# how many terms to compute
DIGITS_PER_TERM = math.log10(C3_OVER_24/6/2/6)
N = int(digits/DIGITS_PER_TERM + 1)
# Calclate P(0,N) and Q(0,N)
P, Q, T = bs(0, N)
one_squared = mpz(10)**(2*digits)
sqrtC = sqrt(10005*one_squared, one_squared)
return (Q*426880*sqrtC) // T
# The last 5 digits or pi for various numbers of digits
check_digits = {
100 : 70679,
1000 : 1989,
10000 : 75678,
100000 : 24646,
1000000 : 58151,
10000000 : 55897,
}
if __name__ == "__main__":
digits = 100
pi = pi_chudnovsky_bs(digits)
#raise SystemExit
for log10_digits in range(1,11):
digits = 10**log10_digits
start =time()
pi = pi_chudnovsky_bs(digits)
pi = str(pi)
pi = pi[:(len(str(pi)) // 2) + 1]
length = int(len(str(pi))) - 1
print("Chudnovsky Binary Splitting Using GMPY: Digits",f"{digits:,}","\n--------------",time()-start,"--------------")
print("Length Of " + f"{length:,}" + " Digits")
It's my first answer, so I hope it helps!

Multiprocessing a function from main forking main as well in python

I am trying to do multiprocessing of a function 5 times by using python multiprocessing.Process library
To my surprise I am also getting the prints that I have added 5 times more..my intuition is that main is also getting called 5 times.
I added prints in the train function that I need to process 5 times I can see multiprocessing happening from the prints.But I am unable to figure it out why main is also getting called 5 times.
Here is my code..Can someone please help a check what's wrong with this code.
def train(rank,params, shared_model, optimizer,ticker):
try:
print("rank:",str(rank)," ",str(ticker),"\n")
f.write("rank:"+str(rank)+" "+str(ticker)+"\n")
data= pd.read_csv(ticker + '.csv')
data = data.dropna()
count = 0
max_timesteps = int(data.shape[0]*0.8)
data = torch.DoubleTensor(np.asarray(data))
env = ENV(state_dim, action_dim, data)
# init training variables
state = env.reset()
done = True
episode_length = 0
count = 0
while count<max_timesteps-1:
episode_length += 1
if done:
cx = Variable(torch.zeros(1, state_dim))
hx = Variable(torch.zeros(1, state_dim))
else:
cx = Variable(cx.data)
hx = Variable(hx.data)
values = []
log_probs = []
rewards = []
entropies = []
for step in range(max_timesteps):
value, action_values, (hx, cx) = model((Variable(state.unsqueeze(0)), (hx, cx)))
prob = F.softmax(action_values,dim = 0)
log_prob = F.log_softmax(action_values,dim = 0)
entropy = -(log_prob * prob).sum(1)
entropies.append(entropy)
action = prob.multinomial(num_samples=1).data
log_prob = log_prob.gather(1, Variable(action))
values.append(value)
log_probs.append(log_prob)
state, reward, done = env.step(action.numpy())
print(ticker," reward", reward , " rank:",rank)
count+=1
done = (done or count == max_timesteps-2)
reward = max(min(reward, 1), -1)
if done:
episode_length = 0
state = env.reset()
rewards.append(reward)
if done:
break
R = torch.zeros(1, 1)
if not done:
value, _, _ = model((Variable(state.unsqueeze(0)), (hx, cx)))
R = value.data
values.append(Variable(R))
policy_loss = 0
value_loss = 0
R = Variable(R)
gae = torch.zeros(1, 1)
for i in reversed(range(len(rewards))):
R = params.gamma * R + rewards[i]
advantage = R - values[i]
value_loss = value_loss + 0.5 * advantage.pow(2)
TD = rewards[i] + params.gamma * values[i + 1].data - values[i].data
gae = gae * params.gamma * params.tau + TD
policy_loss = policy_loss - log_probs[i] * Variable(gae) - 0.01 * entropies[i]
optimizer.zero_grad()
(policy_loss + 0.5 * value_loss).backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 40)
optimizer.step()
f.flush()
except:
print(ticker)
traceback.print_exc()
var = traceback.format_exc()
f.write(str(ticker)+"\n")
f.write("exception:\n"+str(var))
f.flush()
# Implementing the Adam optimizer with shared states
class Params():
def __init__(self):
self.lr = 0.001
self.gamma = 0.99
self.tau = 1.
self.seed = 1
self.num_processes = 5
self.num_steps = 20
self.max_episode_length = 10000
self.env_name = 'Breakout-v0'
params = Params()
state_dim = 6
action_dim = 3
model = ActorCritic(state_dim, action_dim)
model.share_memory()
optimizer = SharedAdam(model.parameters(), lr=params.lr)
optimizer.share_memory()
# set the parameters
epochs = 1
state_dim = 6
action_dim = 3
max_action = 1
idx = 0
file_name = "%s" % ("computational__reward")
directory="./pytorch_models"
# instantiate policy
tickers = pd.read_csv("tickers.csv")
indices = tickers['Symbol']
jobs = []
#indices = ['FTK']
for ep in range(1):
print("epoch:",ep)
f.write("epoch:"+str(ep)+"\n")
for ticker in indices:
try:
for rank in range(0, params.num_processes): # making a loop to run all the other processes that will be trained by updating the shared model
if __name__ == '__main__':
p = multiprocessing.Process(target=train, args=(rank, params, model, optimizer,ticker,))
jobs.append(p)
p.start()
for p in jobs: # creating a pointer that will allow to kill all the threads when at least one of the threads, or main.py will be killed, allowing to stop the program safely
p.join()
model.save("A3C_multi"+ str(ep)+"_" + file_name, directory="./pytorch_models")
f.flush()
except Exception as e:
print(e)
traceback.print_exc()
var = traceback.format_exc()
f.write(str(ticker)+"\n")
f.write("exception:\n"+str(var))
model.save("A3C"+ str(ep)+"_" + file_name, directory="./pytorch_models")
I am getting this print 6 times:
epoch: 0
epoch: 0
epoch: 0
epoch: 0
epoch: 0
epoch: 0

Short answer
Since you are running this on windows, all code that is not function or class definitions should be inside the if __name__ is "__main__" block.
Long answer
On POSIX operating systems, the multiprocessing module is implemented using the fork() system call, which creates a copy of a process.
This is very handy because the second process is completely initialized out of the box.
Microsoft windows does not have this system call. So Python tries to mimick this by starting a new Python interpreter and importing your program as a module.
For this to work well importing your program should not have side effects. The best way to achieve that is to put anything that is not a class or function definition inside the if __name__ is "__main__" block.
Since part of your code is outside the main block, it will be executed when your program is imported in the newly created Python processes. That is why you are seeing the multiple "epoch" prints.

Print does not fit the data interval settings

I am using this answer here and I have this code which get the gauge strain signal (2 channels) and then I apply a calibration step, then I print the 2 channels and the current time. But it looks like the print does not match the frequency I use in my code.
from time import sleep
from Phidget22.Phidget import *
from Phidget22.Devices.VoltageRatioInput import *
import time
import datetime
TIME_OUT = 5000 #5s beofre it throws a timeout exception
DATA_INTERVAL = 50 #50ms sample frequency
A0 = -6.128983223994E-06
B0 = -0.000059639277340
A1 = -6.101017778744E-06
B1 = -0.000286467338645
def onVoltageRatioChange0(self, voltageRatio):
Masse = (voltageRatio - (B0) ) / (A0)
self.masse = Masse
def onVoltageRatioChange1(self, voltageRatio):
Masse = (voltageRatio - (B1) ) / (A1)
self.masse = Masse
def results():
voltageRatioInput0 = VoltageRatioInput()
voltageRatioInput0.masse = 0
voltageRatioInput0.setChannel(0)
voltageRatioInput0.setOnVoltageRatioChangeHandler(onVoltageRatioChange0)
voltageRatioInput0.openWaitForAttachment(TIME_OUT)
voltageRatioInput0.setBridgeGain(BridgeGain.BRIDGE_GAIN_128)
voltageRatioInput0.setDataInterval(DATA_INTERVAL)
voltageRatioInput1 = VoltageRatioInput()
voltageRatioInput1.masse = 0
voltageRatioInput1.setChannel(1)
voltageRatioInput1.setOnVoltageRatioChangeHandler(onVoltageRatioChange1)
voltageRatioInput1.openWaitForAttachment(TIME_OUT)
voltageRatioInput1.setBridgeGain(BridgeGain.BRIDGE_GAIN_128)
voltageRatioInput1.setDataInterval(DATA_INTERVAL)
print(str(datetime.datetime.now()) + " / " + str(voltageRatioInput0.masse) + " / " + str(voltageRatioInput1.masse))
voltageRatioInput0.close()
voltageRatioInput1.close()
if __name__ == '__main__':
try:
while True:
results()
except KeyboardInterrupt:
print("Goodbye")
pass
So normally using the voltageRatioInput0.setDataInterval(DATA_INTERVAL) it should print the value with a frequency of 20 hz. But it prints me this :
2019-11-22 10:24:59.460503 / -0.03847266852956798 / 0.2918630004986786
2019-11-22 10:25:00.099831 / -0.03695316689942101 / -0.02070779820379342
2019-11-22 10:25:00.772398 / -0.04029613574942367 / 0.28775155534154484
2019-11-22 10:25:01.420283 / -0.043487203384171676 / 0.25676361089420047
So clearly here I do not have 20 Hz...

A lot of time is probably spent in your result() function. I cannot reproduce here, but a simple
python -m cProfile -s tottime your_program.py
should help you to see where most of the time is spent each loop iteration.
I guess that instanciation of a VoltageRatioInput() and VoltageRatioInput.close() are a good way to start. Could you try taking them outside the loop as in:
from Phidget22.Phidget import *
from Phidget22.Devices.VoltageRatioInput import *
import datetime
TIME_OUT = 5000 #5s beofre it throws a timeout exception
DATA_INTERVAL = 50 #50ms sample frequency
A0 = -6.128983223994E-06
B0 = -0.000059639277340
A1 = -6.101017778744E-06
B1 = -0.000286467338645
def onVoltageRatioChange0(self, voltageRatio):
Masse = (voltageRatio - (B0) ) / (A0)
self.masse = Masse
def onVoltageRatioChange1(self, voltageRatio):
Masse = (voltageRatio - (B1) ) / (A1)
self.masse = Masse
def results(voltageRatioInput0, voltageRatioInput1):
voltageRatioInput0.masse = 0
voltageRatioInput0.setChannel(0)
voltageRatioInput0.setOnVoltageRatioChangeHandler(onVoltageRatioChange0)
voltageRatioInput0.openWaitForAttachment(TIME_OUT)
voltageRatioInput0.setBridgeGain(BridgeGain.BRIDGE_GAIN_128)
voltageRatioInput0.setDataInterval(DATA_INTERVAL)
voltageRatioInput1.masse = 0
voltageRatioInput1.setChannel(1)
voltageRatioInput1.setOnVoltageRatioChangeHandler(onVoltageRatioChange1)
voltageRatioInput1.openWaitForAttachment(TIME_OUT)
voltageRatioInput1.setBridgeGain(BridgeGain.BRIDGE_GAIN_128)
voltageRatioInput1.setDataInterval(DATA_INTERVAL)
print(str(datetime.datetime.now()) + " / " + str(voltageRatioInput0.masse) + " / " + str(voltageRatioInput1.masse))
if __name__ == '__main__':
try:
voltageRatioInput0 = VoltageRatioInput()
voltageRatioInput1 = VoltageRatioInput()
while True:
results(voltageRatioInput0, voltageRatioInput1)
voltageRatioInput0.close()
voltageRatioInput1.close()
except KeyboardInterrupt:
print("Goodbye")

How to let a multi-processing python application quit cleanly

When I run a python script that uses multiprocessing I find it hard to get it to stop cleanly when it receives Ctrl-C. Ctrl-C has to be pressed multiple times and all sorts of error messages appear on the screen.
How can you make a python script that uses multiprocessing and quits
cleanly when it receives a Ctrl-C ?
Take this script for example
import numpy as np, time
from multiprocessing import Pool
def countconvolve(N):
np.random.seed() # ensure seed is random
count = 0
iters = 1000000 # 1million
l=12
k=12
l0=l+k-1
for n in range(N):
t = np.random.choice(np.array([-1,1], dtype=np.int8), size=l0 * iters)
v = np.random.choice(np.array([-1,1], dtype=np.int8), size = l * iters)
for i in xrange(iters):
if (not np.convolve(v[(l*i):(l*(i+1))],
t[(l0*i):(l0*(i+1))], 'valid').any()):
count += 1
return count
if __name__ == '__main__':
start = time.clock()
num_processes = 8
N = 13
pool = Pool(processes=num_processes)
res = pool.map(countconvolve, [N] * num_processes)
print res, sum(res)
print (time.clock() - start)

Jon's solution is probably better, but here it is using a signal handler. I tried it in a VBox VM which was extremely slow, but worked. I hope it will help.
import numpy as np, time
from multiprocessing import Pool
import signal
# define pool as global
pool = None
def term_signal_handler(signum, frame):
global pool
print 'CTRL-C pressed'
try:
pool.close()
pool.join()
except AttributeError:
print 'Pool has been already closed'
def countconvolve(N):
np.random.seed() # ensure seed is random
count = 0
iters = 1000000 # 1million
l=12
k=12
l0=l+k-1
for n in range(N):
t = np.random.choice(np.array([-1,1], dtype=np.int8), size=l0 * iters)
v = np.random.choice(np.array([-1,1], dtype=np.int8), size = l * iters)
for i in xrange(iters):
if (not np.convolve(v[(l*i):(l*(i+1))],t[(l0*i):(l0*(i+1))], 'valid').any()):
count += 1
return count
if __name__ == '__main__':
# Register the signal handler
signal.signal(signal.SIGINT, term_signal_handler)
start = time.clock()
num_processes = 8
N = 13
pool = Pool(processes=num_processes)
res = pool.map(countconvolve, [N] * num_processes)
print res, sum(res)
print (time.clock() - start)

I believe the try-catch mentioned in a similar post here on SO could be adapted to cover it.
If you wrap the pool.map call in the try-catch and then call terminate and join I think that would do it.
[Edit]
Some experimentation suggests something along these lines works well:
from multiprocessing import Pool
import random
import time
def countconvolve(N):
try:
sleepTime = random.randint(0,5)
time.sleep(sleepTime)
count = sleepTime
except KeyboardInterrupt as e:
pass
return count
if __name__ == '__main__':
random.seed(0)
start = time.clock()
num_processes = 8
N = 13
pool = Pool(processes=num_processes)
try:
res = pool.map(countconvolve, [N] * num_processes)
print res, sum(res)
print (time.clock() - start)
except KeyboardInterrupt as e:
print 'Stopping..'
I simplified your example somewhat to avoid having to load numpy on my machine to test but the critical part is the two try-except calls which handle the CTRL+C key presses.