添加 CARLA 端到端自动驾驶项目 (#5198)

* Create README.md

* feat: add CARLA end-to-end autonomous driving project files

* Create Torcs_End2End_AutonomousDriving

* Create README.md

* Add TORCS DDPG autonomous driving agent

* Delete src/Torcs_End2End_AutonomousDriving

* Delete src/Torcs_End2End directory

* Create README.md

* 修改模块名和脚本入口

* Delete src/CARLA_End2End_AutonomousDriving directory

Author: ovo-ovo-ovo-ovo

src/torcs_end2end/ActorNetwork.py

@@ -0,0 +1,36 @@
+import numpy as np
+import math
+import torch as t
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable as V
+
+
+HIDDEN1_UNITS = 300
+HIDDEN2_UNITS = 600
+
+class ActorNetwork(nn.Module):
+    def __init__(self, state_size):
+        super(ActorNetwork, self).__init__()
+        self.fc1 = nn.Linear(state_size, HIDDEN1_UNITS)
+        self.fc2 = nn.Linear(HIDDEN1_UNITS, HIDDEN2_UNITS)
+        self.steering = nn.Linear(HIDDEN2_UNITS, 1)
+        nn.init.normal_(self.steering.weight, 0, 1e-4)
+        self.acceleration = nn.Linear(HIDDEN2_UNITS, 1)
+        nn.init.normal_(self.acceleration.weight, 0, 1e-4)
+        self.brake = nn.Linear(HIDDEN2_UNITS, 1)
+        nn.init.normal_(self.brake.weight, 0, 1e-4)
+
+
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        out1 = t.tanh(self.steering(x))
+        out2 = t.sigmoid(self.acceleration(x))
+        out3 = t.sigmoid(self.brake(x))
+        out = t.cat((out1, out2, out3), 1) 
+        return out
+
+
+
+

src/torcs_end2end/CriticNetwork.py

@@ -0,0 +1,26 @@
+import numpy as np
+import math
+import torch 
+import torch.nn as nn
+import torch.nn.functional as F
+
+HIDDEN1_UNITS = 300
+HIDDEN2_UNITS = 600
+
+class CriticNetwork(nn.Module):
+    def __init__(self, state_size, action_size):
+        super(CriticNetwork, self).__init__()
+        self.w1 = nn.Linear(state_size, HIDDEN1_UNITS)
+        self.a1 = nn.Linear(action_size, HIDDEN2_UNITS)
+        self.h1 = nn.Linear(HIDDEN1_UNITS, HIDDEN2_UNITS)
+        self.h3 = nn.Linear(HIDDEN2_UNITS, HIDDEN2_UNITS)
+        self.V = nn.Linear(HIDDEN2_UNITS, action_size)
+
+    def forward(self, s, a):
+        w1 = F.relu(self.w1(s))
+        a1 = self.a1(a)
+        h1 = self.h1(w1)
+        h2 = h1 + a1
+        h3 = F.relu(self.h3(h2))
+        out = self.V(h3)
+        return out

src/torcs_end2end/OU.py

@@ -0,0 +1,10 @@
+import random
+import numpy as np
+
+class OU():
+    """
+    Ornstein-Uhlenbeck (OU) 过程类
+    OU过程是一种均值回归的随机过程，常用于模拟具有均值回归特性的随机波动。
+    """
+    def function(self, x, mu, theta, sigma):
+        return theta * (mu - x) + sigma * np.random.randn(1)

src/torcs_end2end/README.md

@@ -0,0 +1,37 @@
+# TORCS 自动驾驶强化学习项目
+
+## 项目概述
+本项目基于 TORCS 模拟器，使用深度确定性策略梯度(DDPG)算法实现自动驾驶。智能体通过强化学习学习驾驶策略，能够在赛道上自主行驶。
+
+本项目是对 https://github.com/jastfkjg/DDPG_Torcs_PyTorch.git 的复现，并针对实验中出现的问题对代码进行改进。
+
+## 文件结构
+```text
+├── snakeoil3_gym.py       # TORCS客户端通信接口
+├── gym_torcs.py           # OpenAI Gym环境封装
+├── main.py                # DDPG算法主训练脚本
+├── ActorNetwork.py        # Actor网络定义
+├── CriticNetwork.py       # Critic网络定义
+├── ReplayBuffer.py        # 经验回放缓冲区
+├── OU.py                  # Ornstein-Uhlenbeck噪声过程
+├── autostart.sh           # TORCS自动启动脚本
+└── Runtime_Screenshot.jpg # 运行时截图
+```
+
+## 环境配置要求
+* Python 3
+* [gym_torcs](https://github.com/ugo-nama-kun/gym_torcs)
+* PyTorch 0.4.1
+
+## 运行方法
+```
+git clone https://github.com/jastfkjg/DDPG_Torcs_PyTorch.git
+cd DDPG_Torcs_PyTorch
+python main.py
+
+```
+
+## 项目运行截图
+<p>
+  <img src="Runtime_Screenshot.jpg" alt="TORCS 仿真运行界面" width="600">
+</p>

src/torcs_end2end/ReplayBuffer.py

@@ -0,0 +1,35 @@
+from collections import deque
+import random
+
+class ReplayBuffer():
+
+    def __init__(self, buffer_size):
+        self.buffer_size = buffer_size
+        self.num_experiences = 0
+        self.buffer = deque()
+
+    def getBatch(self, batch_size):
+        if self.num_experiences < batch_size:
+            return random.sample(self.buffer, self.num_experiences)
+        else:
+            return random.sample(self.buffer, batch_size)
+
+    def size(self):
+        return self.buffer_size
+
+    def add(self, state, action, reward, new_state, done):
+        experience = (state, action, reward, new_state, done)
+        if self.num_experiences < self.buffer_size:
+            self.buffer.append(experience)
+            self.num_experiences += 1
+        else:
+            self.buffer.popleft()
+            self.buffer.append(experience)
+
+    def count(self):
+        return self.num_experiences
+
+    def erase(self):
+        self.buffer = deque()
+        self.num_experiences = 0
+

src/torcs_end2end/Runtime_Screenshot.jpg

@@ -0,0 +1,12 @@
+#!/bin/bash
+xte 'key Return'
+xte 'usleep 100000'
+xte 'key Return'
+xte 'usleep 100000'
+xte 'key Up'
+xte 'usleep 100000'
+xte 'key Up'
+xte 'usleep 100000'
+xte 'key Return'
+xte 'usleep 100000'
+xte 'key Return'