Skip to content

Commit 0205c4e

Browse files
MorvanZhouMorvanZhou
committed
edit
1 parent 3253cc8 commit 0205c4e

4 files changed

Lines changed: 260 additions & 1 deletion

File tree

‎Reinforcement_learning_TUT/11_Dyna_Q/RL_brain.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
"""
2+
This part of code is the Dyna-Q learning brain, which is a brain of the agent.
3+
All decisions and learning processes are made in here.
4+
5+
View more on 莫烦Python: https://morvanzhou.github.io/tutorials/
6+
"""
7+
8+
import numpy as np
9+
import pandas as pd
10+
11+
12+
class QLearningTable:
13+
def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
14+
self.actions = actions # a list
15+
self.lr = learning_rate
16+
self.gamma = reward_decay
17+
self.epsilon = e_greedy
18+
self.q_table = pd.DataFrame(columns=self.actions)
19+
20+
def choose_action(self, observation):
21+
self.check_state_exist(observation)
22+
# action selection
23+
if np.random.uniform() < self.epsilon:
24+
# choose best action
25+
state_action = self.q_table.ix[observation, :]
26+
state_action = state_action.reindex(np.random.permutation(state_action.index)) # some actions have same value
27+
action = state_action.argmax()
28+
else:
29+
# choose random action
30+
action = np.random.choice(self.actions)
31+
return action
32+
33+
def learn(self, s, a, r, s_):
34+
self.check_state_exist(s_)
35+
q_predict = self.q_table.ix[s, a]
36+
if s_ != 'terminal':
37+
q_target = r + self.gamma * self.q_table.ix[s_, :].max() # next state is not terminal
38+
else:
39+
q_target = r # next state is terminal
40+
self.q_table.ix[s, a] += self.lr * (q_target - q_predict) # update
41+
42+
def check_state_exist(self, state):
43+
if state not in self.q_table.index:
44+
# append new state to q table
45+
self.q_table = self.q_table.append(
46+
pd.Series(
47+
[0]*len(self.actions),
48+
index=self.q_table.columns,
49+
name=state,
50+
)
51+
)
52+
53+
54+
class EnvModel:
55+
"""Similar to the memory buffer of DQN, you can store past experiences in here"""
56+
def __init__(self, actions):
57+
# the simplest case is to think about the model is a memory which has all past transition information
58+
self.actions = actions
59+
self.memory = pd.DataFrame(columns=actions, dtype=np.object)
60+
61+
def store_transition(self, s, a, r, s_):
62+
if s not in self.memory.index:
63+
self.memory = self.memory.append(
64+
pd.Series(
65+
[None] * len(self.actions),
66+
index=self.memory.columns,
67+
name=s,
68+
))
69+
self.memory.set_value(s, a, (r, s_))
70+
71+
def sample_s_a(self):
72+
s = np.random.choice(self.memory.index)
73+
a = np.random.choice(self.memory.ix[s].dropna().index) # filter out the None value
74+
return s, a
75+
76+
def get_r_s_(self, s, a):
77+
r, s_ = self.memory.ix[s, a]
78+
return r, s_

‎Reinforcement_learning_TUT/11_Dyna_Q/maze_env.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
"""
2+
Reinforcement learning maze example.
3+
4+
Red rectangle: explorer.
5+
Black rectangles: hells [reward = -1].
6+
Yellow bin circle: paradise [reward = +1].
7+
All other states: ground [reward = 0].
8+
9+
This script is the environment part of this example. The RL is in RL_brain.py.
10+
11+
View more on 莫烦Python: https://morvanzhou.github.io/tutorials/
12+
"""
13+
14+
15+
import numpy as np
16+
np.random.seed(1)
17+
import tkinter as tk
18+
import time
19+
20+
21+
UNIT = 40 # pixels
22+
MAZE_H = 4 # grid height
23+
MAZE_W = 4 # grid width
24+
25+
26+
class Maze(tk.Tk, object):
27+
def __init__(self):
28+
super(Maze, self).__init__()
29+
self.action_space = ['u', 'd', 'l', 'r']
30+
self.n_actions = len(self.action_space)
31+
self.title('maze')
32+
self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))
33+
self._build_maze()
34+
35+
def _build_maze(self):
36+
self.canvas = tk.Canvas(self, bg='white',
37+
height=MAZE_H * UNIT,
38+
width=MAZE_W * UNIT)
39+
40+
# create grids
41+
for c in range(0, MAZE_W * UNIT, UNIT):
42+
x0, y0, x1, y1 = c, 0, c, MAZE_H * UNIT
43+
self.canvas.create_line(x0, y0, x1, y1)
44+
for r in range(0, MAZE_H * UNIT, UNIT):
45+
x0, y0, x1, y1 = 0, r, MAZE_H * UNIT, r
46+
self.canvas.create_line(x0, y0, x1, y1)
47+
48+
# create origin
49+
origin = np.array([20, 20])
50+
51+
# hell
52+
hell1_center = origin + np.array([UNIT * 2, UNIT])
53+
self.hell1 = self.canvas.create_rectangle(
54+
hell1_center[0] - 15, hell1_center[1] - 15,
55+
hell1_center[0] + 15, hell1_center[1] + 15,
56+
fill='black')
57+
# hell
58+
hell2_center = origin + np.array([UNIT, UNIT * 2])
59+
self.hell2 = self.canvas.create_rectangle(
60+
hell2_center[0] - 15, hell2_center[1] - 15,
61+
hell2_center[0] + 15, hell2_center[1] + 15,
62+
fill='black')
63+
64+
# create oval
65+
oval_center = origin + UNIT * 2
66+
self.oval = self.canvas.create_oval(
67+
oval_center[0] - 15, oval_center[1] - 15,
68+
oval_center[0] + 15, oval_center[1] + 15,
69+
fill='yellow')
70+
71+
# create red rect
72+
self.rect = self.canvas.create_rectangle(
73+
origin[0] - 15, origin[1] - 15,
74+
origin[0] + 15, origin[1] + 15,
75+
fill='red')
76+
77+
# pack all
78+
self.canvas.pack()
79+
80+
def reset(self):
81+
self.update()
82+
time.sleep(0.5)
83+
self.canvas.delete(self.rect)
84+
origin = np.array([20, 20])
85+
self.rect = self.canvas.create_rectangle(
86+
origin[0] - 15, origin[1] - 15,
87+
origin[0] + 15, origin[1] + 15,
88+
fill='red')
89+
# return observation
90+
return self.canvas.coords(self.rect)
91+
92+
def step(self, action):
93+
s = self.canvas.coords(self.rect)
94+
base_action = np.array([0, 0])
95+
if action == 0: # up
96+
if s[1] > UNIT:
97+
base_action[1] -= UNIT
98+
elif action == 1: # down
99+
if s[1] < (MAZE_H - 1) * UNIT:
100+
base_action[1] += UNIT
101+
elif action == 2: # right
102+
if s[0] < (MAZE_W - 1) * UNIT:
103+
base_action[0] += UNIT
104+
elif action == 3: # left
105+
if s[0] > UNIT:
106+
base_action[0] -= UNIT
107+
108+
self.canvas.move(self.rect, base_action[0], base_action[1]) # move agent
109+
110+
s_ = self.canvas.coords(self.rect) # next state
111+
112+
# reward function
113+
if s_ == self.canvas.coords(self.oval):
114+
reward = 1
115+
done = True
116+
elif s_ in [self.canvas.coords(self.hell1), self.canvas.coords(self.hell2)]:
117+
reward = -1
118+
done = True
119+
else:
120+
reward = 0
121+
done = False
122+
123+
return s_, reward, done
124+
125+
def render(self):
126+
# time.sleep(0.1)
127+
self.update()
128+
129+

‎Reinforcement_learning_TUT/11_Dyna_Q/run_this.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
"""
2+
Simplest model-based RL, Dyna-Q.
3+
4+
Red rectangle: explorer.
5+
Black rectangles: hells [reward = -1].
6+
Yellow bin circle: paradise [reward = +1].
7+
All other states: ground [reward = 0].
8+
9+
This script is the main part which controls the update method of this example.
10+
The RL is in RL_brain.py.
11+
12+
View more on 莫烦Python: https://morvanzhou.github.io/tutorials/
13+
"""
14+
15+
from maze_env import Maze
16+
from RL_brain import QLearningTable, EnvModel
17+
18+
19+
def update():
20+
for episode in range(40):
21+
s = env.reset()
22+
while True:
23+
env.render()
24+
a = RL.choose_action(str(s))
25+
s_, r, done = env.step(a)
26+
RL.learn(str(s), a, r, str(s_))
27+
28+
# use a model to output (r, s_) by inputting (s, a)
29+
# the model in dyna Q version is just like a memory replay buffer
30+
env_model.store_transition(str(s), a, r, s_)
31+
for n in range(10): # learn 10 more times using the env_model
32+
ms, ma = env_model.sample_s_a() # ms in here is a str
33+
mr, ms_ = env_model.get_r_s_(ms, ma)
34+
RL.learn(ms, ma, mr, str(ms_))
35+
36+
s = s_
37+
if done:
38+
break
39+
40+
# end of game
41+
print('game over')
42+
env.destroy()
43+
44+
45+
if __name__ == "__main__":
46+
env = Maze()
47+
RL = QLearningTable(actions=list(range(env.n_actions)))
48+
env_model = EnvModel(actions=list(range(env.n_actions)))
49+
50+
env.after(0, update)
51+
env.mainloop()

‎Reinforcement_learning_TUT/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,5 @@ All methods mentioned below have their video and text tutorial in Chinese. Visit
1919
* [Actor Critic](https://github.com/MorvanZhou/tutorials/tree/master/Reinforcement_learning_TUT/8_Actor_Critic_Advantage)
2020
* [Deep Deterministic Policy Gradient](https://github.com/MorvanZhou/tutorials/tree/master/Reinforcement_learning_TUT/9_Deep_Deterministic_Policy_Gradient_DDPG)
2121
* [A3C](https://github.com/MorvanZhou/tutorials/tree/master/Reinforcement_learning_TUT/10_A3C)
22-
* Model-based RL (WIP)
22+
* Model-based RL (WIP)
23+
* [Dyna-Q](https://github.com/MorvanZhou/tutorials/tree/master/Reinforcement_learning_TUT/11_Dyna_Q)

0 commit comments

Comments
 (0)