• 深度强化学习中Double DQN算法（Q-Learning+CNN）的讲解及在Asterix游戏上的实战（超详细 附源码）

    需要源码和环境搭建请点赞关注收藏后评论区留下QQ~~~

    一、核心思想

    针对DQN中出现的高估问题，有人提出深度双Q网络算法（DDQN），该算法是将强化学习中的双Q学习应用于DQN中。在强化学习中，双Q学习的提出能在一定程度上缓解Q学习带来的过高估计问题。

    DDQN的主要思想是在目标值计算时将动作的选择和评估分离，在更新过程中，利用两个网络来学习两组权重，分别是预测网络的权重W和目标网络的权重W'，在DQN中，动作选择和评估都是通过目标网络来实现的，而在DDQN中，计算目标Q值时，采取目标网络获取最优动作，再通过预测网络估计该最优动作的目标Q值，这样就可以将最优动作选额和动作值函数估计分离，采用不用的样本保证独立性

    二、允许结果与分析

    本节实验在Asterix游戏上，通过控制参数变量对DQN,DDQN算法进行性能对比从而验证了在一定程度上，DDQN算法可以缓解DQN算法的高估问题，DDQN需要两个不同的参数网络，每1000步后预测预测网络的参数同步更新给目标网络，实验设有最大能容纳1000000记录的缓冲池，每个Atari游戏，DDQN算法训练1000000时间步

    实战结果如下图所示，图中的DDQN算法最后收敛回报明显大于DQN，并且在实验过程中，可以发现DQN算法容易陷入局部的情况，其问题主要在于Q-Learning中的最大化操作，Agent在选择动作时每次都取最大Q值得动作，对于真实的策略来说，在给定的状态下并不是每次都选择Q值最大的动作，因为一般真实的策略都是随机性策略，所以在这里目标值直接选择动作最大的Q值往往会导致目标值高于真实值

     为了解决值函数高估计的问题，DDQN算法将动作的选择和动作的评估分别用不同的值函数来实现，结果表明DDQN能够估计出更准确的Q值，在一些Atari2600游戏中可获得更稳定有效的策略

    三、代码

    部分源码如下

    1.  
    2.  
    3. import gym, random, pickle, os.path, math, glob
    4.  
    5. import numpy as np
    6. import pandas as pd
    7. import matplotlib.pyplot as plt
    8.  
    9. import torch
    10. import torch.optim as optim
    11. import torch.nn as nn
    12. import torch.nn.functional as F
    13. import torch.autograd as autograd
    14. import pdb
    15.  
    16. from atari_wrappers import make_atari, wrap_deepmind,LazyFrames
    17.  
    18.     def __init__(self, in_channels=4, num_actions=5):
    19.        nnels: number of channel of input.
    20.                 i.e The number of most recent frames stacked together as describe in the paper
    21.             num_actions: number of action-value to output, one-to-one correspondence to action in game.
    22.         """
    23.         super(DQN, self).__init__()
    24.         self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
    25.         self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
    26.         self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
    27.         self.fc4 = nn.Linear(7 * 7 * 64, 512)
    28.         self.fc5 = nn.Linear(512, num_actions)
    30.     def forward(self, x):
    31.         x = F.relu(self.conv1(x))
    32.         x = F.relu(self.conv2(x))
    33.         x = F.relu(self.conv3(x))
    34.         x = F.relu(self.fc4(x.view(x.size(0), -1)))
    35.         return self.fc5(x)
    37. class Memory_Buffer(object):
    38.     def __init__(self, memory_size=1000):
    39.         self.buffer = []
    40.         self.memory_size = memory_size
    41.         self.next_idx = 0
    43.     def push(self, state, action, reward, next_state, done):
    44.         data = (state, action, reward, next_state, done)
    45.         if len(self.buffer) <= self.memory_size: # buffer not full
    46.             self.buffer.append(data)
    47.         else: # buffer is full
    48.             self.buffer[self.next_idx] = data
    49.         self.next_idx = (self.next_idx + 1) % self.memory_size
    51.     def sample(self, batch_size):
    52.         states, actions, rewards, next_states, dones = [], [], [], [], []
    53.         for i in range(batch_size):
    54.             idx = random.randint(0, self.size() - 1)
    55.             data = self.buffer[idx]
    56.             state, action, reward, next_state, done= data
    57.             states.append(state)
    58.             actions.append(action)
    59.             rewards.append(reward)
    60.             next_states.append(next_state)
    61.             dones.append(done)
    64.         return np.concatenate(states), actions, rewards, np.concatenate(next_states), dones
    66.     def size(self):
    67.         return len(self.buffer)
    69. class DDQNAgent:
    70.     def __init__(self, in_channels = 1, action_space = [], USE_CUDA = False, memory_size = 10000, epsilon  = 1, lr = 1e-4):
    71.         self.epsilon = epsilon
    72.         self.action_space = action_space
    73.         self.memory_buffer = Memory_Buffer(memory_size)
    74.         self.DQN = DQN(in_channels = in_channels, num_actions = action_space.n)
    75.         self.DQN_target = DQN(in_channels = in_channels, num_actions = action_space.n)
    76.         self.DQN_target.load_state_dict(self.DQN.state_dict())
    79.         self.USE_CUDA = USE_CUDA
    80.         if USE_CUDA:
    81.             self.DQN = self.DQN.to(device)
    82.             self.DQN_target = self.DQN_target.to(device)
    83.         self.optimizer = optim.RMSprop(self.DQN.parameters(),lr=lr, eps=0.001, alpha=0.95)
    85.     def observe(self, lazyframe):
    86.         # from Lazy frame to tensor
    87.         state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()
    88.         if self.USE_CUDA:
    89.             state = state.to(device)
    90.         return state
    92.     def value(self, state):
    93.         q_values = self.DQN(state)
    94.         return q_values
    96.     def act(self, state, epsilon = None):
    97.         """
    98.         sample actions with epsilon-greedy policy
    99.         recap: with p = epsilon pick random action, else pick action with highest Q(s,a)
    100.         """
    101.         if epsilon is None: epsilon = self.epsilon
    103.         q_values = self.value(state).cpu().detach().numpy()
    104.         if random.random()
    105.             aciton = random.randrange(self.action_space.n)
    106.         else:
    107.             aciton = q_values.argmax(1)[0]
    108.         return aciton
    110.     def compute_td_loss(self, states, actions, rewards, next_states, is_done, gamma=0.99):
    111.         """ Compute td loss using torch operations only. Use the formula above. """
    112.         actions = torch.tensor(actions).long()    # shape: [batch_size]
    113.         rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]
    114.         is_done = torch.tensor(is_done, dtype = torch.uint8)  # shape: [batch_size]
    116.         if self.USE_CUDA:
    117.             actions = actions.to(device)
    118.             rewards = rewards.to(device)
    119.             is_done = is_done.to(device)
    121.         # get q-values for all actions in current states
    122.         predicted_qvalues = self.DQN(states)
    124.         # select q-values for chosen actions
    125.         predicted_qvalues_for_actions = predicted_qvalues[
    126.           range(states.shape[0]), actions
    127.         ]
    129.         # compute q-values for all actions in next states
    130.         ## Where DDQN is different from DQN
    131.         predicted_next_qvalues_current = self.DQN(next_states)
    132.         predicted_next_qvalues_target = self.DQN_target(next_states)
    133.         # compute V*(next_states) using predicted next q-values
    134.         next_state_values =  predicted_next_qvalues_target.gather(1, torch.max(predicted_next_qvalues_current, 1)[1].unsqueeze(1)).squeeze(1)
    136.         # compute "target q-values" for loss - it's what's inside square parentheses in the above formula.
    137.         target_qvalues_for_actions = rewards + gamma *next_state_values
    139.         # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist
    140.         target_qvalues_for_actions = torch.where(
    141.             is_done, rewards, target_qvalues_for_actions)
    143.         # mean squared error loss to minimize
    144.         #loss = torch.mean((predicted_qvalues_for_actions -
    145.         #                   target_qvalues_for_actions.detach()) ** 2)
    146.         loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())
    148.         return loss
    150.     def sample_from_buffer(self, batch_size):
    151.         states, actions, rewards, next_states, dones = [], [], [], [], []
    152.         for i in range(batch_size):
    153.             idx = random.randint(0, self.memory_buffer.size() - 1)
    154.             data = self.memory_buffer.buffer[idx]
    155.             frame, action, reward, next_frame, done= data
    156.             states.append(self.observe(frame))
    157.             actions.append(action)
    158.             rewards.append(reward)
    159.             next_states.append(self.observe(next_frame))
    160.             dones.append(done)
    161.         return torch.cat(states), actions, rewards, torch.cat(next_states), dones
    163.     def learn_from_experience(self, batch_size):
    164.         if self.memory_buffer.size() > batch_size:
    165.             states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)
    166.             td_loss = self.compute_td_loss(states, actions, rewards, next_states, dones)
    167.             self.optimizer.zero_grad()
    168.             td_loss.backward()
    169.             for param in self.DQN.parameters():
    170.                 param.grad.data.clamp_(-1, 1)
    172.             self.optimizer.step()
    173.             return(td_loss.item())
    174.         else:
    175.             return(0)
    177. def moving_average(a, n=3) :
    178.     ret = np.cumsum(a, dtype=float)
    179.     ret[n:] = ret[n:] - ret[:-n]
    180.     return ret[n - 1:] / n
    182. def plot_training(frame_idx, rewards, losses):
    183.     clear_output(True)
    184.     plt.figure(figsize=(20,5))
    185.     plt.subplot(131)
    186.     plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-100:])))
    187.     plt.plot(moving_average(rewards,20))
    188.     plt.subplot(132)
    189.     plt.title('loss, average on 100 stpes')
    190.     plt.plot(moving_average(losses, 100),linewidth=0.2)
    191.     plt.show()
    193. # if __name__ == '__main__':
    195. # Training DQN in PongNoFrameskip-v4
    196. env = make_atari('PongNoFrameskip-v4')
    197. env = wrap_deepmind(env, scale = False, frame_stack=True)
    199. gamma = 0.99
    200. epsilon_max = 1
    201. epsilon_min = 0.01
    202. eps_decay = 30000
    203. frames = 1000000
    204. USE_CUDA = True
    205. learning_rate = 2e-4
    206. max_buff = 100000
    207. update_tar_interval = 1000
    208. batch_size = 32
    209. print_interval = 1000
    210. log_interval = 1000
    211. learning_start = 10000
    212. win_reward = 18     # Pong-v4
    213. win_break = True
    215. action_space = env.action_space
    216. action_dim = env.action_space.n
    217. state_dim = env.observation_space.shape[0]
    218. state_channel = env.observation_space.shape[2]
    219. agent = DDQNAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate)
    221. #frame = env.reset()
    223. episode_reward = 0
    224. all_rewards = []
    225. losses = []
    226. episode_num = 0
    227. is_win = False
    228. # tensorboard
    229. summary_writer = SummaryWriter(log_dir = "DDQN", comment= "good_makeatari")
    231. # e-greedy decay
    232. epsilon_by_frame = lambda frame_idx: epsilon_min + (epsilon_max - epsilon_min) * math.exp(
    233.             -1. * frame_idx / eps_decay)
    234. # plt.plot([epsilon_by_frame(i) for i in range(10000)])
    236. for i in range(frames):
    237.     epsilon = epsilon_by_frame(i)
    238.     #state_tensor = agent.observe(frame)
    239.     #action = agent.act(state_tensor, epsilon)
    241.     #next_frame, reward, done, _ = env.step(action)
    243.     #episode_reward += reward
    244.     #agent.memory_buffer.push(frame, action, reward, next_frame, done)
    245.     #frame = next_frame
    247.     loss = 0
    248.     if agent.memory_buffer.size() >= learning_start:
    249.         loss = agent.learn_from_experience(batch_size)
    250.         losses.append(loss)
    252.     if i % print_interval == 0:
    253.         print("frames: %5d, reward: %5f, loss: %4f, epsilon: %5f, episode: %4d" % (i, np.mean(all_rewards[-10:]), loss, epsilon, episode_num))
    254.         summary_writer.add_scalar("Temporal Difference Loss", loss, i)
    255.         summary_writer.add_scalar("Mean Reward", np.mean(all_rewards[-10:]), i)
    256.         summary_writer.add_scalar("Epsilon", epsilon, i)
    258.     if iQN_dict.pth.tar")
    259. plot_training(i, all_rewards, losses)

    创作不易 觉得有帮助请点赞关注收藏~~~

  • 相关阅读:
    高防服务器和普通服务器之间的区别有哪些
    GB/T28181-2016 SDP定义和音视频传输模式解读
    实时光线追踪（3）Ray Casting
    最小生成树
    c++新闻发布系统(支持登录注册)
    xxxxxxx
    SpringBoot（原理篇 ==＞ 解析起步依赖
    Android使用Banner框架实现轮播图
    Gem5 simpoint 全流程
    【数据结构初阶】一个队列怎么实现栈~~OJ
  • 原文地址：https://blog.csdn.net/jiebaoshayebuhui/article/details/128049271