【机器学习】突出强化学习中智能体通过与环境交互不断成长，其决策能力逐步进化提升，展现强化学习核心特点与动态过程。

import numpy as np

# 定义迷宫的大小
maze_size = 4

# 定义动作空间：0 - 上，1 - 下，2 - 左，3 - 右
actions = ['up', 'down', 'left', 'right']

# 初始化 Q 表，形状为 (迷宫状态数, 动作数)
Q = np.zeros((maze_size * maze_size, len(actions)))

# 定义奖励矩阵，-1 表示普通格子，-100 表示陷阱，100 表示目标
rewards = np.full((maze_size, maze_size), -1)
rewards[3, 3] = 100  # 目标位置
rewards[1, 1] = -100  # 陷阱位置

# 定义学习率和折扣因子
alpha = 0.1
gamma = 0.9

# 定义函数获取当前状态的坐标
def get_state_coordinates(state):
    return state // maze_size, state % maze_size

# 定义函数根据动作获取下一个状态
def get_next_state(state, action):
    x, y = get_state_coordinates(state)
    if action == 0 and x > 0:  # 上
        x -= 1
    elif action == 1 and x < maze_size - 1:  # 下
        x += 1
    elif action == 2 and y > 0:  # 左
        y -= 1
    elif action == 3 and y < maze_size - 1:  # 右
        y += 1
    return x * maze_size + y

# 训练循环
for episode in range(1000):
    state = 0  # 初始状态
    while True:
        # 根据 Q 表选择动作，采用 epsilon - greedy 策略
        if np.random.rand() < 0.1:  # 以 0.1 的概率随机选择动作
            action = np.random.choice(len(actions))
        else:
            action = np.argmax(Q[state])

        # 执行动作，获取下一个状态和奖励
        next_state = get_next_state(state, action)
        reward = rewards[get_state_coordinates(next_state)]

        # 更新 Q 值
        Q[state][action] += alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])

        # 更新状态
        state = next_state

        # 判断是否到达目标或陷阱
        if reward == 100 or reward == -100:
            break

# 测试训练好的智能体
state = 0
while True:
    action = np.argmax(Q[state])
    print(f"当前状态: {state}, 采取动作: {actions[action]}")
    next_state = get_next_state(state, action)
    if rewards[get_state_coordinates(next_state)] == 100:
        print("到达目标！")
        break
    state = next_state

import gym
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
import random
import numpy as np

# 创建 CartPole 环境
env = gym.make('CartPole-v1')

# 定义超参数
gamma = 0.99  # 折扣因子
epsilon = 1.0  # 探索率
epsilon_min = 0.01  # 最小探索率
epsilon_decay = 0.995  # 探索率衰减率
learning_rate = 0.001  # 学习率
memory_size = 10000  # 经验回放缓冲区大小
batch_size = 64  # 每次训练的批次大小

# 定义神经网络模型
model = Sequential()
model.add(Dense(24, input_dim=4, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(2, activation='linear'))
model.compile(loss='mse', optimizer=Adam(learning_rate))

# 经验回放缓冲区
memory = []

# 存储经验到缓冲区
def remember(state, action, reward, next_state, done):
    memory.append((state, action, reward, next_state, done))
    if len(memory) > memory_size:
        del memory[0]

# 根据当前状态选择动作
def act(state):
    if np.random.rand() <= epsilon:
        return random.randrange(2)
    else:
        q_values = model.predict(state)[0]
        return np.argmax(q_values)

# 从经验回放缓冲区中采样数据并训练网络
def replay():
    if len(memory) < batch_size:
        return
    minibatch = random.sample(memory, batch_size)
    states = np.array([i[0] for i in minibatch])
    actions = np.array([i[1] for i in minibatch])
    rewards = np.array([i[2] for i in minibatch])
    next_states = np.array([i[3] for i in minibatch])
    dones = np.array([i[4] for i in minibatch])

    # 计算目标 Q 值
    target_q = model.predict(states)
    target_next_q = model.predict(next_states)
    for i in range(batch_size):
        if dones[i]:
            target_q[i][actions[i]] = rewards[i]
        else:
            target_q[i][actions[i]] = rewards[i] + gamma * np.max(target_next_q[i])

    # 训练网络
    model.fit(states, target_q, epochs=1, verbose=0)

# 训练循环
for episode in range(500):
    state = env.reset()[0]
    state = np.reshape(state, [1, 4])
    done = False
    total_reward = 0
    while not done:
        action = act(state)
        next_state, reward, done, _, _ = env.step(action)
        next_state = np.reshape(next_state, [1, 4])
        remember(state, action, reward, next_state, done)
        state = next_state
        total_reward += reward
        replay()
    # 更新探索率
    epsilon = max(epsilon_min, epsilon * epsilon_decay)
    print(f"Episode: {episode}, Total Reward: {total_reward}")

# 测试训练好的智能体
for _ in range(10):
    state = env.reset()[0]
    state = np.reshape(state, [1, 4])
    done = False
    total_reward = 0
    while not done:
        action = np.argmax(model.predict(state)[0])
        env.render()
        next_state, reward, done, _, _ = env.step(action)
        next_state = np.reshape(next_state, [1, 4])
        state = next_state
        total_reward += reward
    print(f"Test Reward: {total_reward}")

env.close()

import numpy as np

# 定义动作数量
num_actions = 3

# 定义策略网络（简单的单层神经网络）
def policy_network(state):
    # 这里简单使用随机权重初始化，实际应用中可使用更复杂的初始化方法
    weights = np.random.rand(state.shape[0], num_actions)
    logits = np.dot(state, weights)
    # 使用 softmax 函数将 logits 转换为动作概率分布
    action_probs = np.exp(logits) / np.sum(np.exp(logits))
    return action_probs

# 定义奖励函数（这里简单模拟）
def get_reward(action):
    # 假设动作 1 有较高奖励，动作 2 和 3 奖励较低
    if action == 0:
        return 1.0
    else:
        return 0.1

# 训练循环
num_episodes = 1000
learning_rate = 0.01
for episode in range(num_episodes):
    state = np.random.rand(2)  # 随机生成一个状态
    action_probs = policy_network(state)
    # 根据动作概率分布选择动作
    action = np.random.choice(num_actions, p=action_probs)
    reward = get_reward(action)
    # 计算策略梯度
    gradient = np.zeros((state.shape[0], num_actions))
    for a in range(num_actions):
        if a == action:
            gradient[:, a] = state * (1 - action_probs[a])
        else:
            gradient[:, a] = -state * action_probs[a]
    # 更新策略网络权重
    weights += learning_rate * reward * gradient

# 测试训练后的策略网络
for _ in range(10):
    state = np.random.rand(2)
    action_probs = policy_network(state)
    action = np.random.choice(num_actions, p=action_probs)
    print(f"State: {state}, Action: {action}")