• PyTorch深度强化学习中蒙特卡洛策略梯度法在短走廊环境（CartPole-v0）中的实战（超详细 附源码）

    需要源码请点赞关注收藏后评论区留下QQ~~~

    一、策略梯度法

    策略梯度法（PG）利用策略函数来选择动作，同时使用值函数来辅助策略函数参数的更新，根据策略类型的不同，可以分为随机策略梯度和确定性策略梯度

    策略梯度法与值函数逼近法相比优点如下

    1：平滑收敛

    在学习过程中，PG法每次更新策略函数，权重参数都会朝着最优值变化，且只发生微小变化，有很强的收敛性，值函数逼近法基于贪心策略对策略进行改进，有些价值函数在后期会一直围绕着最优价值函数持续小的震荡而不收敛 即出现策略退化

    2：处理连续动作空间任务

    PG法可以直接得到一个策略，而值函数逼近法需要对比状态S中的所有动作的价值

    3：学习随机策略

    PG法能够输出随机策略，而值函数逼近法基于贪心方法，每次输出的都是确定性函数

    PG法缺点如下

    1:PG法通常只能收敛到局部最优解

    2：PG法的易收敛性和学习过程平滑优势，都是使Agent尝试过多的无效探索，从而造成学习效率低，整体策略方差偏大，以及存在累积错误带来的过高估计问题 

    二、蒙特卡洛策略梯度法（REINFORCE）

    蒙特卡洛策略梯度法是一种针对情节式问题的，基于MC算法的PG法

    REINFORCE算法采用MC算法来计算动作值函数，只考虑Agent在状态S下实际采取的动作，该方法可以从理论上保存策略参数的收敛性

    1：实验环境设置

    引入短走廊网格世界环境  如下图

    每步的收益是-1，对于三个非终止状态都有两个动作可以选择，向左或者向右，特殊的是，第一个状态向左走会保持原地不动，而在第二个状态执行的动作会导致向相反的方向移动 

     

    部分超参数列表如下

     2：实验结果分析

    每个环境下算法的训练情节数均为1000个情节，这是因为两个环境在1000个情节后都能收敛，如下图所示REINFORCE算法在短走廊环境和CartPole环境的效果整体上都呈现稳步上升，后平稳的学习趋势

    迭代一千次后如下  （每隔一千次输出一次可视化图像）

    迭代两千次后效果如下 

     

     迭代8000次后如下 发现基本已经在高位收敛

     

    三、代码

    部分源码如下

    1. # 代38-REINFORCE算法的实验过程
    2.  
    3.  
    4. #CartPole环境
    5. import argparse
    6. import gym
    7. import numpy as np
    8. from itertools import count
    9.  
    10. import torch
    11. import torch.nn as nn
    12. import torch.nn.functional as F
    13. import torch.optim as optim
    14. from torch.distributions import Categorical
    15. import matplotlib.pyplot as plt
    16.  
    17. import os
    18. os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
    19.  
    20. parser = argparse.ArgumentParser(description='PyTorch REINFORCE example')
    21. parser.add_argument('--gamma', type=float, default=0.99, metavar='G',
    22.                     help='discount factor (default: 0.99)')
    23. parser.add_argument('--seed', type=int, default=543, metavar='N',
    24.                     help='random seed (default: 543)')
    25. parser.add_argument('--render', action='store_true',
    26.                     help='render the environment')
    27. parser.add_argument('--log-interval', type=int, default=10, metavar='N',
    28.                     help='interval between training status logs (default: 10)')
    29. args = parser.parse_args()
    30.  
    31.  
    32. env = gym.make('CartPole-v1')
    33. env.seed(args.seed)
    34. torch.manual_seed(args.seed)
    35.  
    36.  
    37. class Policy(nn.Module):
    38.     def __init__(self):
    39.         super(Policy, self).__init__()
    40.         self.affine1 = nn.Linear(4, 128)
    41.         self.dropout = nn.Dropout(p=0.6)
    42.         self.affine2 = nn.Linear(128, 2)
    43.  
    44.         self.saved_log_probs = []
    45.         self.rewards = []
    46.  
    47.     def forward(self, x):
    48.         x = self.affine1(x)
    49.         x = self.dropout(x)
    50.         x = F.relu(x)
    51.         action_scores = self.affine2(x)
    52.         return F.softmax(action_scores, dim=1)
    53.  
    54.  
    55. policy = Policy()
    56. optimizer = optim.Adam(policy.parameters(), lr=1e-2)
    57. eps = np.finfo(np.float32).eps.item()
    58.  
    59.  
    60. def select_action(state):
    61.     state = torch.from_numpy(state).float().unsqueeze(0)
    62.     probs = policy(state)
    63.     m = Categorical(probs)
    64.     action = m.sample()
    65.     policy.saved_log_probs.append(m.log_prob(action))
    66.     return action.item()
    67.  
    68.  
    69. def finish_episode():
    70.     R = 0
    71.     policy_loss = []
    72.     rewards = []
    73.     for r in policy.rewards[::-1]:
    74.         R = r + args.gamma * R
    75.         rewards.insert(0, R)
    76.     rewards = torch.tensor(rewards)
    77.     rewards = (rewards - rewards.mean()) / (rewards.std() + eps)
    78.     for log_prob, reward in zip(policy.saved_log_probs, rewards):
    79.         policy_loss.append(-log_prob * reward)
    80.     optimizer.zero_grad()
    81.     policy_loss = torch.cat(policy_loss).sum()
    82.     policy_loss.backward()
    83.     optimizer.step()
    84.     del policy.rewards[:]
    85.     del policy.saved_log_probs[:]
    86.  
    87. def plot(epi, run_time):
    88.  
    89.     plt.title('Training')
    90.     plt.xlabel('Episode')
    91.     plt.ylabel('Run Time')
    92.     plt.plot(epi, run_time, color='red')
    93.     plt.show()
    94.  
    95. def main():
    96.     running_reward = 10
    97.     running_rewards = []
    98.     i_episodes = []
    99.     for i_episode in range(10000):
    100.         state, ep_reward = env.reset(), 0
    101.         for t in range(1, 10000):  # Don't infinite loop while learning
    102.             action = select_action(state)
    103.             state, reward, done, _ = env.step(action)
    104.             if args.render:
    105.                 env.render()
    106.             policy.rewards.append(reward)
    107.             ep_reward += reward
    108.             if done:
    109.                 break
    111.         running_reward = 0.05 * ep_reward + (1 - 0.05) * running_reward
    112.         finish_episode()
    113.         running_rewards.append(running_reward)
    114.         i_episodes.append(i_episode)
    115.         if i_episode % args.log_interval == 0:
    117.             print('Episode {}\tLast length: {:2f}\tAverage length: {:.2f}'.format(
    118.                 i_episode, ep_reward, running_reward))
    119.         if (i_episode % 1000 == 0):
    120.             plot(i_episodes, running_rewards)
    121.     np.save(f"putu", running_rewards)
    122. if __name__ == '__main__':
    123.     main()
    127. # 短走廊环境
    128. import numpy as np
    129. import matplotlib
    130. matplotlib.use('Agg')
    131. import matplotlib.pyplot as plt
    132. from tqdm import tqdm
    134. def true_value(p):
    135.     """ True value of the first state
    136.     Args:
    137.         p (float): probability of the action 'right'.
    138.     Returns:
    139.         True value of the first state.
    140.         The expression is obtained by manually solving the easy linear system
    141.         of Bellman equations using known dynamics.
    142.     """
    143.     return (2 * p - 4) / (p * (1 - p))
    145. class ShortCorridor:
    146.     """
    147.     Short corridor environment, see Example 13.1
    148.     """
    149.     def __init__(self):
    150.         self.reset()
    152.     def reset(self):
    153.         self.state = 0
    155.     def step(self, go_right):
    156.         """
    157.         Args:
    158.             go_right (bool): chosen action
    159.         Returns:
    160.             tuple of (reward, episode terminated?)
    161.         """
    162.         if self.state == 0 or self.state == 2:
    163.             if go_right:
    164.                 self.state += 1
    165.             else:
    166.                 self.state = max(0, self.state - 1)
    167.         else:
    168.             if go_right:
    169.                 self.state -= 1
    170.             else:
    171.                 self.state += 1
    173.         if self.state == 3:
    174.             # terminal state
    175.             return 0, True
    176.         else:
    177.             return -1, False
    179. def softmax(x):
    180.     t = np.exp(x - np.max(x))
    181.     return t / np.sum(t)
    183. class ReinforceAgent:
    184.     """
    185.     ReinforceAgent that follows algorithm
    186.     'REINFORNCE Monte-Carlo Policy-Gradient Control (episodic)'
    187.     """
    188.     def __init__(self, alpha, gamma):
    189.         # set values such that initial conditions correspond to left-epsilon greedy
    190.         self.theta = np.array([-1.47, 1.47])
    191.         self.alpha = alpha
    192.         self.gamma = gamma
    193.         # first column - left, second - right
    194.         self.x = np.array([[0, 1],
    195.                            [1, 0]])
    196.         self.rewards = []
    197.         self.actions = []
    199.     def get_pi(self):
    200.         h = np.dot(self.theta, self.x)
    201.         t = np.exp(h - np.max(h))
    202.         pmf = t / np.sum(t)
    203.         # never become deterministic,
    204.         # guarantees episode finish
    205.         imin = np.argmin(pmf)
    206.         epsilon = 0.05
    208.         if pmf[imin] < epsilon:
    209.             pmf[:] = 1 - epsilon
    210.             pmf[imin] = epsilon
    212.         return pmf
    214.     def get_p_right(self):
    215.         return self.get_pi()[1]
    217.     def choose_action(self, reward):
    218.         if reward is not None:
    219.             self.rewards.append(reward)
    221.         pmf = self.get_pi()
    222.         go_right = np.random.uniform() <= pmf[1]
    223.         self.actions.append(go_right)
    225.         return go_right
    227.     def episode_end(self, last_reward):
    228.         self.rewards.append(last_reward)
    230.         # learn theta
    231.         G = np.zeros(len(self.rewards))
    232.         G[-1] = self.rewards[-1]
    234.         for i in range(2, len(G) + 1):
    235.             G[-i] = self.gamma * G[-i + 1] + self.rewards[-i]
    237.         gamma_pow = 1
    239.         for i in range(len(G)):
    240.             j = 1 if self.actions[i] else 0
    241.             pmf = self.get_pi()
    242.             grad_ln_pi = self.x[:, j] - np.dot(self.x, pmf)
    243.             update = self.alpha * gamma_pow * G[i] * grad_ln_pi
    245.             self.theta += update
    246.             gamma_pow *= self.gamma
    248.         self.rewards = []
    249.         self.actions = []
    251. class ReinforceBaselineAgent(ReinforceAgent):
    252.     def __init__(self, alpha, gamma, alpha_w):
    253.         super(ReinforceBaselineAgent, self).__init__(alpha, gamma)
    254.         self.alpha_w = alpha_w
    255.         self.w = 0
    257.     def episode_end(self, last_reward):
    258.         self.rewards.append(last_reward)
    260.         # learn theta
    261.         G = np.zeros(len(self.rewards))
    262.         G[-1] = self.rewards[-1]
    264.         for i in range(2, len(G) + 1):
    265.             G[-i] = self.gamma * G[-i + 1] + self.rewards[-i]
    267.         gamma_pow = 1
    269.         for i in range(len(G)):
    270.             self.w += self.alpha_w * gamma_pow * (G[i] - self.w)
    272.             j = 1 if self.actions[i] else 0
    273.             pmf = self.get_pi()
    274.             grad_ln_pi = self.x[:, j] - np.dot(self.x, pmf)
    275.             update = self.alpha * gamma_pow * (G[i] - self.w) * grad_ln_pi
    277.             self.theta += update
    278.             gamma_pow *= self.gamma
    280.         self.rewards = []
    281.         self.actions = []
    283. def trial(num_episodes, agent_generator):
    284.     env = ShortCorridor()
    285.     agent = agent_generator()
    287.     rewards = np.zeros(num_episodes)
    288.     for episode_idx in range(num_episodes):
    289.         rewards_sum = 0
    290.         reward = None
    291.         env.reset()
    293.         while True:
    294.             go_right = agent.choose_action(reward)
    295.             reward, episode_end = env.step(go_right)
    296.             rewards_sum += reward
    298.             if episode_end:
    299.                 agent.episode_end(reward)
    300.                 break
    302.         rewards[episode_idx] = rewards_sum
    304.     return rewards
    306. def example_13_1():
    307.     epsilon = 0.05
    308.     fig, ax = plt.subplots(1, 1)
    310.     # Plot a graph
    311.     p = np.linspace(0.01, 0.99, 100)
    312.     y = true_value(p)
    313.     ax.plot(p, y, color='red')
    315.     # Find a maximum point, can also be done analytically by taking a derivative
    316.     imax = np.argmax(y)
    317.     pmax = p[imax]
    318.     ymax = y[imax]
    319.     ax.plot(pmax, ymax, color='green', marker="*", label="optimal point: f({0:.2f}) = {1:.2f}".format(pmax, ymax))
    321.     # Plot points of two epsilon-greedy policies
    322.     ax.plot(epsilon, true_value(epsilon), color='magenta', marker="o", label="epsilon-greedy left")
    323.     ax.plot(1 - epsilon, true_value(1 - epsilon), color='blue', marker="o", label="epsilon-greedy right")
    325.     ax.set_ylabel("Value of the first state")
    326.     ax.set_xlabel("Probability of the action 'right'")
    327.     ax.set_title("Short corridor with switched actions")
    328.     ax.set_ylim(ymin=-105.0, ymax=5)
    329.     ax.legend()
    331.     plt.savefig('../images/example_13_1.png')
    332.     plt.close()
    334. def figure_13_1():
    335.     num_trials = 100
    336.     num_episodes = 1000
    337.     gamma = 1
    338.     agent_generators = [lambda : ReinforceAgent(alpha=2e-4, gamma=gamma),
    339.                         lambda : ReinforceAgent(alpha=2e-5, gamma=gamma),
    340.                         lambda : ReinforceAgent(alpha=2e-3, gamma=gamma)]
    341.     labels = ['alpha = 2e-4',
    342.               'alpha = 2e-5',
    343.               'alpha = 2e-3']
    345.     rewards = np.zeros((len(agent_generators), num_trials, num_episodes))
    347.     for agent_index, agent_generator in enumerate(agent_generators):
    348.         for i in tqdm(range(num_trials)):
    349.             reward = trial(num_episodes, agent_generator)
    350.             rewards[agent_index, i, :] = reward
    352.     plt.plot(np.arange(num_episodes) + 1, -11.6 * np.ones(num_episodes), ls='dashed', color='red', label='-11.6')
    353.     for i, label in enumerate(labels):
    354.         plt.plot(np.arange(num_episodes) + 1, rewards[i].mean(axis=0), label=label)
    355.     plt.ylabel('total reward on episode')
    356.     plt.xlabel('episode')
    357.     plt.legend(loc='lower right')
    359.     plt.savefig('../images/figure_13_1.png')
    360.     plt.close()
    362. def figure_13_2():
    363.     num_trials = 100
    364.     num_episodes = 1000
    365.     alpha = 2e-4
    366.     gamma = 1
    367.     ageninforce with baseline']
    368.  
    369.     rewards = np.zeros((len(agent_generators), num_trials, num_episodes))
    370.  
    371.     for agent_index, agent_generator in enumerate(agent_generators):
    372.         for i in tqdm(range(num_trials)):
    373.             reward = trial(num_episodes, agent_generator)
    374.             rewards[agent_index, i, :] = reward
    375.  
    376.     plt.plot(np.arange(num_episodes) + 1, -11.6 * np.ones(num_episodes), ls='dashed', color='red', label='-11.6')
    377.     for i, label in enumerate(labels):
    378.         plt.plot(np.arange(num_episodes) + 1, rewards[i].mean(axis=0), label=label)
    379.   _':
    380.     example_13_1()
    381.     figure_13_1()
    382.     figure_13_2()

    创作不易  觉得有帮助请点赞关注收藏~~~

  • 相关阅读:
    前端代码规范
    0基础学习VR全景平台篇 第101篇：企业版功能-子账号分配管理
    对非均匀采样信号进行重采样
    Spring 框架使用的设计模式
    Redux Toolkit中action派发但state值不更新的原因
    「项目管理工具」进度猫如何实现可视化？
    08:STM32----DMA数据转运
    c# 面试题
    11 月 11 日 ROS 学习笔记——ROS 架构及概念
    Go——数组
  • 原文地址：https://blog.csdn.net/jiebaoshayebuhui/article/details/128055746