• 深度强化学习中利用Q-Learngin和期望Sarsa算法确定机器人最优策略实战（超详细 附源码）

    需要源码和环境搭建请点赞关注收藏后评论区留下QQ~~~

    一、Q-Learning算法

    Q-Learning算法中动作值函数Q的更新方向是最优动作值函数q，而与Agent所遵循的行为策略无关，在评估动作值函数Q时，更新目标为最优动作值函数q的直接近似，故需要遍历当前状态的所有动作，在所有状态都能被无限次访问的前提下，Q-Learning算法能以1的概率收敛到最优动作值函数和最优策略

    下图是估算最优策略的Q-Learning算法流程图

    Q-Learning虽然是异策略，但是从值函数更新迭代式中可以看出，它并没有使用到重要性采样。

     

     使用Q-Learning算法解决确定环境中的扫地机器人问题 参数设置与之前相同 使用贪心策略 

    机器人背景及环境搭建

    输出如下

     

     

     代码如下

    1.  
    2. #Q-learning算法
    3. from 扫地机器人gym环境 import GridWorldEnv
    4. import numpy as np
    5. np.random.seed(1)
    6. env = GridWorldEnv()
    7. #有效动作空间
    8. def vilid_action_space(s):
    9.     action_sacpe = []
    10.     if s % 5 != 0:#左
    11.         action_sacpe.append(0)
    12.     if s % 5 != 4:#右
    13.         action_sacpe.append(1)
    14.     if s <= 19:#上
    15.         action_sacpe.append(2)
    16.     if s >= 5:#下
    17.         action_sacpe.append(3)
    18.     return action_sacpe
    19.  
    20. def policy_epsilon_greedy(s, Q, epsilon):
    21.     Q_s = Q[s]
    22.     action = vilid_action_space(s)
    23.     if np.random.rand() < epsilon:
    24.         a = np.random.choice(action)
    25.     else:
    26.         index_a = np.argmax([Q_s[i] for i in action])
    27.         a = action[index_a]
    28.     return a
    29.  
    30. def trans1(Q_S):
    31.     new_Q = []
    32.     new_Q.append(Q_S[2])
    33.     new_Q.append(Q_S[3])
    34.     new_Q.append(Q_S[0])
    35.     new_Q.append(Q_S[1])
    36.     return new_Q
    37.  
    38. def trans(Q_S):
    39.     new_Q = []
    40.     new_Q.append(round(Q_S[2],3))
    41.     new_Q.append(round(Q_S[3],3))
    42.     new_Q.append(round(Q_S[0],3))
    43.     new_Q.append(round(Q_S[1],3))
    44.     return new_Q
    45.  
    46. def print_dd(s, a, next_s, print_len, episode_i, Q,e_k,a_k):
    47.     for i in range(2):
    48.         if episode_i == int(print_len * (0.1 * i + 1)):
    49.             if s == 15 and a == 3 and next_s == 10:
    50.                 print("*********************************单步的计算过程***************************************")
    51.                 print("alpha:"+str(a_k))
    52.                 print("epsilon:"+str(e_k))
    53.                 print("state:" + str(int(print_len * (0.1 * i + 1))))
    54.                 print("Q(%d,%d)"%(s,a))
    55.                 print(Q[s][a])
    56.                 print("Q(%d,*)"%(next_s))
    57.                 print(trans1(Q[next_s]))
    58.                 print('output:'+str(Q[s][a] + a_k * (0.8 * np.max(Q[next_s]) - Q[s, a])))
    59.  
    60. def print_ff(list_q, Q, episode_i,epsilon_k,alpha_k):
    61.     list_s = range(0,25)
    62.     for em in list_q:
    63.         if em == episode_i:
    64.             print("*******************************情节数:%s*******************************"%(str(em)))
    65.             for state in list_s:
    66.                 print("Q(%d,*)"%(state) + str(trans(Q[state])))
    67.                 action = vilid_action_space(state)
    68.                 len_a = len(action)
    69.                 e_p = epsilon_k / float(len_a)
    70.                 max_a = np.argmax(Q[state])
    71.                 prob = []
    72.                 index_a = np.argmax([Q[state][i] for i in action])
    73.                 for i in range(4):#计算epsilon
    74.                     if i not in action:
    75.                         prob.append(0.0)
    76.                     else:
    77.                         if i == action[index_a]:
    78.                             prob.append(1 - epsilon_k + e_p)
    79.                         else:
    80.                             prob.append(e_p)
    81.                 print('概率值:' + str(trans(prob)))
    82.                 print("epsilon_k: {}".format(epsilon_k))
    83.                 print("alpha_k:{}".format(alpha_k))
    84.  
    85. def Attenuation(epsilon,alpha,episode_sum,episode):
    86.     epsilon = (float(episode_sum) - float(episode)) / float(episode_sum) * epsilon
    87.     alpha = (float(episode_sum) - float(episode)) / float(episode_sum) * alpha
    88.     return epsilon, alpha
    89.  
    90.         while not done:
    91.             a = policy_epsilon_greedy(s, Q, epsilon_k)
    92.             next_s, r, done, _ = env.step(a)
    93.             print_dd(s, a, next_s, 10000, episode_i, Q, epsilon_k, alpha_k)
    94.             Q[s, a] += alpha_k * (r + gamma * np.max(Q[next_s]) - Q[s, a])
    95.             s = next_s
    96.     return Q
    97.  
    98. Q = Q_Learning(env, 25000, 0.05, 0.8, 0.5)

    二、期望Sarsa算法

    通过对Sarsa算法进行改进，得到一种异策略TD算法，该算法考虑当前策略下所有动作的可能性，利用动作值函数的期望值取代某一特定动作值函数来更新估计值，该算法称为期望Sarsa算法。

    相比于Sarsa算法，期望Sarsa算法计算更为复杂，但通过计算能够有效地消除银随机选择而产生的方差，因此通常情况下，期望Sarsa算法明显优于Sarsa算法，另外期望Sarsa算法还可以使用异策略方法，将Q-Learning进行推广并提升性能

    下面利用期望Sarsa算法解决确定环境扫地机器人问题 背景与前面相同 不再赘述

    迭代到20000次后基本Q值已经收敛

     

     代码如下

    1.  
    2. # 期望Sarsa算法
    3. from 扫地机器人gym环境 import GridWorldEnv
    4. import numpy as np
    5. from queue import Queue
    6.  
    7. np.random.seed(1)
    8. env = GridWorldEnv()
    9.  
    10.  
    11. # 有效动作空间
    12. def vilid_action_space(s):
    13.     action_sacpe = []
    14.     if s % 5 != 0:  # 左
    15.         action_sacpe.append(0)
    16.     if s % 5 != 4:  # 右
    17.         action_sacpe.append(1)
    18.     if s <= 19:  # 上
    19.         action_sacpe.append(2)
    20.     if s >= 5:  # 下
    21.         action_sacpe.append(3)
    22.     return action_sacpe
    23.  
    24.  
    25. def policy_epsilon_greedy(s, Q, epsilon):
    26.     Q_s = Q[s]
    27.     action = vilid_action_space(s)
    28.     if np.random.rand() < epsilon:
    29.         a = np.random.choice(action)
    30.     else:
    31.         index_a = np.argmax([Q_s[i] for i in action])
    32.         a = action[index_a]
    33.     return a
    34.  
    35.  
    36. def compute_epsion(s, Q, epsilon):
    37.     max_a = np.argmax(Q[s])
    38.     action = vilid_action_space(s)
    39.     len_all_a = len(action)
    40.     prob_l = [0.0, 0.0, 0.0, 0.0]
    41.     for index_a in action:
    42.         if index_a == max_a:
    43.             prob_l[index_a] = 1.0 - epsilon + (epsilon / len_all_a)
    44.         else:
    45.             prob_l[index_a] = epsilon / len_all_a
    46.     return prob_l
    47.  
    48.  
    49. def compute_e_q(prob, q_n):
    50.     sum = 0.0
    51.     for i in range(4):
    52.         sum += prob[i] * q_n[i]
    53.     return sum
    54.  
    55.  
    56. def trans1(Q_S):
    57.     new_Q = []
    58.     new_Q.append(Q_S[2])
    59.     new_Q.append(Q_S[3])
    60.     new_Q.append(Q_S[0])
    61.     new_Q.append(Q_S[1])
    62.     return new_Q
    63.  
    64.  
    65. def print_dd(s, a, next_s, print_len, episode_i, Q, e_k, a_k):
    66.     for i in range(50):
    67.         if episode_i == int(print_len * ((0.02 * i) + 1)):
    68.             if s == 15 and a == 3 and next_s == 10:
    69.                 print("*****************************单步计算过程****************************************")
    70.                 print("alpha:" + str(a_k))
    71.                 print("epsilon:" + str(e_k))
    72.                 print("state:" + str(int(print_len * (1 + (0.02 * i)))))
    73.                 print("Q(%d,%d)" % (s, a))
    74.                 print(Q[s][a])
    75.                 print("Q(%d,*)" % (next_s))
    76.                 print(trans1(Q[next_s]))
    77.                 prob_l = compute_epsion(next_s, Q, e_k)
    78.                 print('概率' + str(trans1(prob_l)))
    79.                 Q_e = compute_e_q(prob_l, Q[next_s])
    80.                 print('update:' + str(Q[s, a] + a_k * (0.8 * Q_e - Q[s, a])))
    81.  
    82.  
    83. def trans(Q_S):
    84.     new_Q = []
    85.     new_Q.append(round(Q_S[2], 3))
    86.     new_Q.append(round(Q_S[3], 3))
    87.     new_Q.append(round(Q_S[0], 3))
    88.     new_Q.append(round(Q_S[1], 3))
    89.     return new_Q
    90.  
    91.  
    92. def print_ff(list_q, Q, episode_i, epsilon_k, alpha_k):
    93.     list_s = range(0, 25)
    94.     for em in list_q:
    95.         if em == episode_i:
    96.             print("*******************************情节数:%s*******************************" % (str(em)))
    97.             for state in list_s:
    98.                 print("Q(%d,*) " % (state) + str(trans(Q[state])))
    99.                 action = vilid_action_space(state)
    100.                 len_a = len(action)
    101.                 e_p = epsilon_k / float(len_a)
    102.                 prob = []
    103.                 index_a = np.argmax([Q[state][i] for i in action])
    104.                 for i in range(4):  # 计算epsilon
    105.                     if i not in action:
    106.                         prob.append(0.0)
    107.                     else:
    108.                         if i == action[index_a]:
    109.                             prob.append(1 - epsilon_k + e_p)
    110.                         else:
    111.                             prob.append(e_p)
    112.                 print('概率值:' + str(trans(prob)))
    113.                 print("epsilon_k: {}".format(epsilon_k))
    114.                 print("alpha_k:{}".format(alpha_k))
    115.  
    116.  
    117. def Attenuation(epsilon, alpha, episode_sum, episode):
    118.     epsilon = (float(episode_sum) - float(episode)) / float(episode_sum) * epsilon
    119.     alpha = (float(episode_sum) - float(episode)) / float(episode_sum) * alpha
    120.     return epsilon, alpha
    121.  
    122.  
    123. def Expectation_sarsa(env, episode_num, alpha, gamma, epsilon):
    124.     Q = np.zeros((env.n_width * env.n_height, env.action_space.n))
    125.     Q_queue = Queue(maxsize=11)
    126. lon_k, alpha_k)
    127.             prob_l = compute_epsion(next_s, Q, epsilon_k)
    128.             Q_e = compute_e_q(prob_l, Q[next_s])
    129.             Q[s, a] += alpha_k * (r + gamma * Q_e - Q[s, a])
    130.             s = next_s
    131.     return Q
    132.  
    133.  
    134. Q = Expectation_sarsa(env, 20000, 0.05, 0.8, 0.5)

    创作不易 觉得有帮助请点赞关注收藏~~~

  • 相关阅读:
    【Python】read() || readline() || readlines()-笔记
    LeetCode337：打家劫舍III
    如何用Python获取网页指定内容
    JS深浅拷贝
    Git的认识和使用
    go语言 ｜ 图解反射（二）
    大数据中Hadoop、Hive、Spark的关系
    实验三十二、OCL电路的研究
    神机百炼3.52-Prim
    中国女士职业套装行业深度调研及投资前景预测研究报告
  • 原文地址：https://blog.csdn.net/jiebaoshayebuhui/article/details/128012217