Pytorch实现强化学习DQN玩迷宫游戏(莫凡强化学习DQN章节pytorch版本)

Pytorch实现强化学习DQN玩迷宫游戏(莫凡强化学习DQN章节pytorch版本)

1.详细的资料可以参考莫凡老师的网页

2.用pytorch实现DQN并用于玩maze

# -*- coding: utf-8 -*-import math
import random
import matplotlib.pyplot as plt
from collections import namedtuple, deque
from itertools import count
import numpy as npimport torch
 as nn
import torch.optim as optim
functional as Ffrom maze_env import Mazerandom.seed(1)
torch.manual_seed(1)
np.random.seed(1)
BATCH_SIZE = 128  # BATCH_SIZE is the number of transitions sampled from the replay buffer
GAMMA = 0.9     # GAMMA is the discount factor as mentioned in the previous section
EPS_START = 0.9   # EPS_START is the starting value of epsilon
EPS_END = 0.05    # EPS_END is the final value of epsilon
EPS_DECAY = 1000  # EPS_DECAY controls the rate of exponential decay of epsilon, higher means a slower decay
TAU = 0.005   # TAU is the update rate of the target network
LR = 1e-4    # LR is the learning rate of the AdamW optimizer
env= Maze()
# Get number of actions from gym action space
n_actions = env.n_actions
# Get the number of state observations
state = set()
n_observations = len(state)
steps_done = 0
episode_durations = []# if gpu is to be used
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")Transition = namedtuple('Transition',('state', 'action', 'next_state', 'reward'))class ReplayMemory(object):def __init__(self, capacity): = deque([], maxlen=capacity)def push(self, *args):"""Save a transition""&#append(Transition(*args))def sample(self, batch_size):return random., batch_size)def __len__(self):return )class DQN(nn.Module):def __init__(self, n_observations, n_actions):super(DQN, self).__init__()self.layer1 = nn.Linear(n_observations, 128)self.layer2 = nn.Linear(128, 128)self.layer3 = nn.Linear(128, n_actions)# Called with either one element to determine next action, or a batch# during optimization. Returns tensor([[left0exp,right0exp]...]).def forward(self, x):x = F.relu(self.layer1(x))x = F.relu(self.layer2(x))return self.layer3(x)policy_net = DQN(n_observations, n_actions).to(device)
target_net = DQN(n_observations, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())optimizer = optim.AdamW(policy_net.parameters(), lr=LR, amsgrad=True)
memory = ReplayMemory(10000)class DeepQNetwork:def __init__(self,n_actions:int,n_observations:int,action_space:[]):self.n_actions=n_actionsself.n_observations=n_observationsself.action_space=action_spacedef select_action(self,state):global steps_donesample = random.random()eps_threshold = EPS_END + (EPS_START - EPS_END) * p(-1. * steps_done / EPS_DECAY)steps_done += 1action_=self.choose_action()if sample > eps_threshold:_grad():return policy_net(state).max(1)[1].view(1, 1)else:sor([[action_]], device=device, dtype=torch.long)def plot_durations(show_result=False):plt.figure(1)durations_t = sor(episode_durations, dtype=torch.float)if show_result:plt.title('Result')else:plt.clf()plt.title(&#')plt.xlabel('Episode')plt.ylabel('Duration')plt.plot(durations_t.numpy())# Take 100 episode averages and plot them tooif len(durations_t) >= 100:means = durations_t.unfold(0, 100, 1).mean(1).view(-1)means = torch.cat((s(99), means))plt.plot(means.numpy())plt.pause(0.001)  # pause a bit so that plots are updated# Training loopdef optimize_model(self):if len(memory) < BATCH_SIZE:returntransitions = memory.sample(BATCH_SIZE)# Transpose the batch (see  for# detailed explanation). This converts batch-array of Transitions# to Transition of batch-arrays.batch = Transition(*zip(*transitions))# Compute a mask of non-final states and concatenate the batch elements# (a final state would've been the one after which simulation ended)non_final_mask = sor(tuple(map(lambda s: s is not _state)), device=device, dtype=torch.bool)non_final_next_states = torch.cat([s for s _stateif s is not None])state_batch = torch.cat(batch.state)action_batch = torch.cat(batch.action)reward_batch = torch.ward)# Compute Q(s_t, a) - the model computes Q(s_t), then we select the# columns of actions taken. These are the actions which would've been taken# for each batch state according to policy_netstate_action_values = policy_net(state_batch).gather(1, action_batch)# Compute V(s_{t+1}) for all next states.# Expected values of actions for non_final_next_states are computed based# on the "older" target_net; selecting their best reward with max(1)[0].# This is merged based on the mask, such that we'll have either the expected# state value or 0 in case the state _state_values = s(BATCH_SIZE, device=device)_grad():next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0]# Compute the expected Q valuesexpected_state_action_values = (next_state_values * GAMMA) + reward_batch# Compute Huber losscriterion = nn.SmoothL1Loss()loss = criterion(state_action_values, expected_state_action_values.unsqueeze(1))# Optimize _grad()loss.backward()# In-place utils.clip_grad_value_(policy_net.parameters(), 100)optimizer.step()def choose_action(self):return np.random.randint(0, self.n_actions)if torch.cuda.is_available():num_episodes = 600
else:num_episodes = 50RL=DeepQNetwork(env.n_actions,set()),env.action_space)for i_episode in range(num_episodes):# Initialize the environment and get it's statestate = set()state = sor(state, dtype=torch.float32, device=device).unsqueeze(0)for t in count():der()action = RL.select_action(state)observation, reward, terminated = env.step(action)reward = sor([reward], device=device)done = terminatedif terminated:next_state = Noneelse:next_state = sor(observation, dtype=torch.float32, device=device).unsqueeze(0)# Store the transition in memorymemory.push(state, action, next_state, reward)# Move to the next statestate = next_state# Perform one step of the optimization (on the policy network)RL.optimize_model()# Soft update of the target network's weights# θ′ ← τ θ + (1 −τ )θ′target_net_state_dict = target_net.state_dict()policy_net_state_dict = policy_net.state_dict()for key in policy_net_state_dict:target_net_state_dict[key] = policy_net_state_dict[key] * TAU + target_net_state_dict[key] * (1 - TAU)target_net.load_state_dict(target_net_state_dict)if done:episode_durations.append(t + 1)break

3.全部代码地址
4.下载后直接运行DQN_new
5.不懂的知识，看1