TensorFlow强化学习入门（2）——基于策略的Agents

%matplotlib inline
import numpy as np
from matplotlib import animation
from IPython.display import Image
import tensorflow as tf
import matplotlib.pyplot as plt
import math

import gym
env = gym.make("CartPole-v0")

# 尝试随机行动来测试环境
env.reset()
reward_sum = 0
num_games = 10
num_game = 0
while num_game < num_games:
    env.render()
    observation, reward, done, _ = env.step(env.action_space.sample())
    reward_sum += reward
    if done:
        print("本episode的收益：{}".format(reward_sum))
        reward_sum = 0
        num_game += 1
        env.reset()

# 初始化agent的神经网络
# 我们使用基于策略梯度的神经网络来接受观测值并传递给隐藏层来产生选择各个行为（左移/右移）的概率分布
# 神经网络超参数
hidden_layer_neurons = 13
batch_size = 50
learning_rate = 1e-2
gamma = .99
dimen = 4

tf.reset_default_graph()

# 定义输入占位符
observations = tf.placeholder(tf.float32, [None, dimen], name="input_x")

# 第一个权重层
W1 = tf.get_variable("W1", shape=[dimen, hidden_layer_neurons], initializer=tf.contrib.layers.xavier_initializer())
layer1 = tf.nn.relu(tf.matmul(observations, W1))

# 第二个权重层
W2 = tf.get_variable("W2", shape=[hidden_layer_neurons, 1], initializer=tf.contrib.layers.xavier_initializer())
output = tf.nn.sigmoid(tf.matmul(layer1, W2))

# 定义网络用于学习的计算图组件
trainable_vars = [W1, W2]
input_y = tf.placeholder(tf.float32, [None, 1], name="input_y")
advantages = tf.placeholder(tf.float32, name="reward_signal")

# 损失函数
log_lik = tf.log(input_y * (input_y - output) + (1 - input_y) * (input_y + output))
loss = -tf.reduce_mean(log_lik * advantages)

# 梯度
new_grads = tf.gradients(loss, trainable_vars)
W1_grad = tf.placeholder(tf.float32, name="batch_grad1")
W2_grad = tf.placeholder(tf.float32, name="batch_grad2")

# 学习
batch_grad = [W1_grad, W2_grad]
adam = tf.train.AdamOptimizer(learning_rate=learning_rate)
update_grads = adam.apply_gradients(zip(batch_grad, [W1, W2]))

def discount_rewards(r, gamma=0.99):
    """
    输入一维的收益数组，输出折算后的收益值，例：f([1, 1, 1], 0.99) -> [1, 0.99, 0.9801]
    """
    return np.array([val * (gamma ** i) for i, val in enumerate(r)])


reward_sum = 0
init = tf.global_variables_initializer()

# 定义观测值，输出值，收益值的占位符
xs = np.empty(0).reshape(0, dimen)
ys = np.empty(0).reshape(0, 1)
rewards = np.empty(0).reshape(0, 1)

# 初始化环境
sess = tf.Session()
rendering = False
sess.run(init)
observation = env.reset()

# 梯度的占位符
gradients = np.array([np.zeros(var.get_shape()) for var in trainable_vars])

num_episodes = 10000
num_episode = 0

while num_episode < num_episodes:
    # 将观测值作为该批次的输入
    x = np.reshape(observation, [1, dimen])
    
    # 运行神经网络来决定输出
    tf_prob = sess.run(output, feed_dict={observations: x})
    
    # 基于我们的网络来决定输出，允许一定的随机性
    y = 0 if tf_prob > np.random.uniform() else 1
    
    # 将观测值和输出值追加至列表中以供学习
    xs = np.vstack([xs, x])
    ys = np.vstack([ys, y])
    
    # 获取行动的结果
    observation, reward, done, _ = env.step(y)
    reward_sum += reward
    rewards = np.vstack([rewards, reward])
    
    if done:
        # 标准化收益值
        discounted_rewards = discount_rewards(rewards, gamma)
        discounted_rewards -= discounted_rewards.mean()
        discounted_rewards /= discounted_rewards.std()
        
        # 根据实时得到的梯度调整梯度
        gradients += np.array(sess.run(new_grads, feed_dict={observations: xs, input_y: ys, advantages: discounted_rewards}))
        
        # 重置游戏变量
        xs = np.empty(0).reshape(0, dimen)
        ys = np.empty(0).reshape(0, 1)
        rewards = np.empty(0).reshape(0, 1)
        
        # 一个batch运行结束
        if num_episode % batch_size == 0:
            # 更新梯度
            sess.run(update_grads, feed_dict={W1_grad: gradients[0], W2_grad: gradients[1]})
            # 重置梯度
            gradients *= 0
            # 输出本轮运行状态
            print("episode = {} 时的平均收益：{}".format(num_episode, reward_sum / batch_size))
            
            if reward_sum / batch_size > 150:
                print("问题在episode = {} 时解决！".format(num_episode))
                break
            reward_sum = 0
        num_episode += 1
        observation = env.reset()

# 去除随机决策，测试agent的性能
observation = env.reset()
observation
reward_sum = 0

while True:
    env.render()
    
    x = np.reshape(observation, [1, dimen])
    y = sess.run(output, feed_dict={observations: x})
    y = 0 if y > 0.5 else 1
    observation, reward, done, _ = env.step(y)
    reward_sum += reward
    if done:
        print("最终分数: {}".format(reward_sum))
        break

···
episode = 2850 时的平均收益：98.26
episode = 2900 时的平均收益：96.96
episode = 2950 时的平均收益：95.62
episode = 3000 时的平均收益：98.82
episode = 3050 时的平均收益：122.34
episode = 3100 时的平均收益：119.22
episode = 3150 时的平均收益：115.66
episode = 3200 时的平均收益：125.16
episode = 3250 时的平均收益：133.44
episode = 3300 时的平均收益：140.7
episode = 3350 时的平均收益：153.4
问题在episode = 3350 时解决！

最终分数: 200.0