# 定义环境
env = {
‘states’: [0, 1, 2, 3],
‘actions’: [‘left’, ‘right’],
‘transitions’: {
0: {‘left’: 0, ‘right’: 1},
1: {‘left’: 0, ‘right’: 2},
2: {‘left’: 1, ‘right’: 3},
3: {‘left’: 2, ‘right’: 3}
},
‘rewards’: {
0: {‘left’: 0, ‘right’: 0},
1: {‘left’: 0, ‘right’: 0},
2: {‘left’: 0, ‘right’: 10},
3: {‘left’: 0, ‘right’: 0}
}
}

# 初始化Q表
Q = np.zeros((len(env[‘states’]), len(env[‘actions’]))

# 超参数
alpha = 0.1 # 学习率
gamma = 0.99 # 折扣因子
epsilon = 0.1 # 探索率

# 动作索引
action_idx = {‘left’: 0, ‘right’: 1}

# 训练
for episode in range(1000):
state = 0 # 初始状态
done = False

while not done:
# ε-贪婪策略选择动作
if np.random.uniform(0, 1) < epsilon: action = np.random.choice(env['actions']) else: action = env['actions'][np.argmax(Q[state, :])] # 执行动作 next_state = env['transitions'][state][action] reward = env['rewards'][state][action] # 更新Q值 best_next_action = np.argmax(Q[next_state, :]) Q[state, action_idx[action]] = Q[state, action_idx[action]] + alpha * (reward + gamma * Q[next_state, best_next_action] - Q[state, action_idx[action]]) # 更新状态 state = next_state # 检查是否达到终止状态 if state == 3: done = True # 打印Q表 print("Q表:") print(Q) # 提取最优策略 optimal_policy = {} for state in env['states']: optimal_action = env['actions'][np.argmax(Q[state, :])] optimal_policy[state] = optimal_action print("最优策略:") print(optimal_policy)

# 创建环境
env = gym.make(‘CartPole-v1′)

# 定义Q网络
model = Sequential([
Dense(24, activation=’relu’, input_shape=(env.observation_space.shape[0],)),
Dense(24, activation=’relu’),
Dense(env.action_space.n, activation=’linear’)
])

model.compile(loss=’mse’, optimizer=Adam(learning_rate=0.001))

# 超参数
epsilon = 1.0 # 探索率
epsilon_min = 0.01
epsilon_decay = 0.995
gamma = 0.99 # 折扣因子
batch_size = 32
memory = [] # 经验回放缓冲区

# 训练
for episode in range(1000):
state = env.reset()
state = np.reshape(state, [1, env.observation_space.shape[0]])
done = False
score = 0

while not done:
# ε-贪婪策略选择动作
if np.random.uniform(0, 1) < epsilon: action = env.action_space.sample() else: action = np.argmax(model.predict(state)[0]) # 执行动作 next_state, reward, done, _ = env.step(action) next_state = np.reshape(next_state, [1, env.observation_space.shape[0]]) # 存储经验 memory.append((state, action, reward, next_state, done)) # 更新状态 state = next_state score += reward # 经验回放 if len(memory) > batch_size:
minibatch = np.random.choice(len(memory), batch_size, replace=False)
for i in minibatch:
state_mb, action_mb, reward_mb, next_state_mb, done_mb = memory[i]
target = reward_mb
if not done_mb:
target = reward_mb + gamma * np.amax(model.predict(next_state_mb)[0])
target_f = model.predict(state_mb)
target_f[0][action_mb] = target
model.fit(state_mb, target_f, epochs=1, verbose=0)

# 衰减探索率
if epsilon > epsilon_min:
epsilon *= epsilon_decay

# 打印结果
if episode % 100 == 0:
print(f”Episode: {episode}, Score: {score}, Epsilon: {epsilon:.2f}”)

# 测试
state = env.reset()
state = np.reshape(state, [1, env.observation_space.shape[0]])
done = False
score = 0

while not done:
action = np.argmax(model.predict(state)[0])
next_state, reward, done, _ = env.step(action)
state = np.reshape(next_state, [1, env.observation_space.shape[0]])
score += reward

print(f”Test Score: {score}”)
env.close()

# 创建环境
env = gym.make(‘CartPole-v1’)

# 创建模型
model = DQN(‘MlpPolicy’, env, verbose=1)

# 训练模型
model.learn(total_timesteps=100000)

# 保存模型
model.save(“dqn_cartpole”)

# 加载模型
# model = DQN.load(“dqn_cartpole”)

# 测试模型
obs = env.reset()
done = False
score = 0

while not done:
action, _states = model.predict(obs, deterministic=True)
obs, reward, done, info = env.step(action)
score += reward

print(f”Test Score: {score}”)
env.close()