In [1]:
import numpy as np
import matplotlib.pyplot as plt
In [202…
gridsize=4
gamma=0.95 #discounted factor
up=0
down=1
left=2
right=3
action={up:"up",down:"down",left:"left",right:'right'}
states=np.arange(gridsize**2).reshape(gridsize,gridsize)
treasure=3
lightning=5
mountain=10
policy=[right,right,down ,up,
up
,up
,right,up,
up
,left ,right,up,
up
,up
,right,up]
reward=np.zeros((gridsize*gridsize,1))
reward[treasure]=1
reward[lightning]=-1
In [3]:
#check if next state is accessible
def accissible(i,j):
if i < 0 or i==gridsize:
return False
if j <0 or j==gridsize:
return False
if states[i,j]==mountain:
return False
return True
In [6]:
#transition probability
def trans(a):
p=0.05*np.ones((4))
p[a]=0.85
return list(p)
In [7]:
#trajectory sampling function
def trajectory_sample(policy):
initial_states=int(np.random.choice(np.arange(16),1))
#initial_states=12
state_now=initial_states
traj=[initial_states]
while True:
if state_now==treasure or state_now==lightning or state_now==mountain:
break
a=policy[state_now]
p=trans(a)
move=np.random.choice(list(action.keys()),1,p=p)
i,j=np.where(states==state_now)
if move == up:
i=i-1
elif move == down:
i=i+1
elif move == left:
j=j-1
elif move == right:
j=j+1
if accissible(i,j):
state_now=int(states[i,j])
traj.append(state_now)
return traj
In [203…
#analytical solution of value function (from piazza)
V_pi=np.array((0.562,0.603,0.737,1.000,0.447,-1.000,0.775,0.930,0.414,0.317,0.000,0.873,0.
print('Analytical Policy Evaluation:')
print(V_pi.reshape(4,4))
Analytical Policy Evaluation:
[[ 0.562 0.603 0.737 1.
]
[ 0.447 -1.
0.775 0.93 ]
[ 0.414 0.317 0.
0.873]
[ 0.386 0.326 0.747 0.818]]
In [197…
#First-visit Monte Carlo
max_eposides=10000
min_eposides=1000
tol=0.01
Diff=[]
Value=np.zeros((gridsize*gridsize,1))
num_G=[0]*(gridsize*gridsize)
diff=0
G=[[] for i in range(gridsize*gridsize)]
#for ep in range(max_eposides):
ep=0
MCvisit=np.zeros((gridsize*gridsize,1))
while True:
ep+=1
traj=trajectory_sample(policy)
T=len(traj)
v=Value
for s in range(gridsize*gridsize):
n_vis=traj.count(s)
MCvisit[s]=n_vis
if s in traj:
ts=traj.index(s) #first-visit
g=0
for t in range(ts,T):
g+=(gamma**(t-ts))*reward[traj[t]]
G[s].append(g)
num_G[s]=num_G[s]+1
v[s]=Value[s]+(1/num_G[s])*(g-Value[s])
diff=max(0,abs(np.linalg.norm(Value-V_pi)))
Diff.append(diff)
if min(num_G) > min_eposides:
if max(0,abs(np.linalg.norm(v-Value)))<tol:
break
Value=v
In [198…
print("Policy Evaluation of First-visit Monte Carlo:")
print(np.round(Value.reshape(4,4),3))
Policy Evaluation of First-visit Monte Carlo:
[[ 0.569 0.608 0.74
1.
]
[ 0.452 -1.
0.777 0.929]
[ 0.418 0.323 0.
0.874]
[ 0.397 0.331 0.742 0.819]]
In [204…
#One-step Temporal Difference
Value_TD=np.ones((gridsize*gridsize,1))
diff_TD=0
Diff_TD=[]
num_G_TD=[0]*(gridsize*gridsize)
alpha=0.01
ep1=0
while True:
ep1+=1
#s0=int(np.random.choice(np.arange(16),1))
vtd=Value_TD
traj=trajectory_sample(policy)
for i in range(len(traj)):
num_G_TD[traj[i]]+=1
if traj[i]==treasure or traj[i]==lightning or traj[i]==mountain:
vtd[traj[i]]+= alpha*(reward[traj[i]] - vtd[traj[i]])
else:
next=traj[i+1]
vtd[traj[i]]+= alpha*(reward[traj[i]] + gamma*vtd[next] - vtd[traj[i]]) #(1/nu
if min(num_G_TD) > min_eposides:
if max(0,abs(np.linalg.norm(Value_TD-vtd,ord=2))) < tol:
diff_TD=max(0,abs(np.linalg.norm(Value_TD-V_pi,ord=2)))
Diff_TD.append(diff_TD)
break
Value_TD=vtd
diff_TD=max(0,abs(np.linalg.norm(Value_TD-V_pi,ord=2)))
Diff_TD.append(diff_TD)
In [205…
print("Policy Evaluation of One-step Temporal Difference:")
print(np.round(Value_TD.reshape(4,4),3))
Policy Evaluation of One-step Temporal Difference:
[[ 0.57
0.624 0.756 1.
]
[ 0.475 -1.
0.799 0.925]
[ 0.413 0.299 0.
0.87 ]
[ 0.353 0.305 0.744 0.818]]
In [206…
Out[206…
plt.plot(range(ep),np.array(Diff),label='First-visit Monte Carlo')
plt.plot(range(ep1),np.array(Diff_TD),label='One-step Temporal Difference')
plt.legend()
plt.xlabel('episodes[-]')
plt.ylabel('Error[-]')
Text(0, 0.5, 'Error[-]')