commit on master

Author: JoshuaWu1997

DDPG-agent.py

@@ -12,9 +12,9 @@
 
 cuda = torch.device('cuda')
 
-raw_amount = pd.read_csv('./sh000016/i_amount.csv', header=0, index_col=0).values
-raw_buy = pd.read_csv('./sh000016/o_buy.csv', header=0, index_col=0).values
-raw_sell = pd.read_csv('./sh000016/o_sell.csv', header=0, index_col=0).values
+raw_amount = pd.read_csv('../sh000016/i_amount.csv', header=0, index_col=0).values
+raw_buy = pd.read_csv('../sh000016/o_buy.csv', header=0, index_col=0).values
+raw_sell = pd.read_csv('../sh000016/o_sell.csv', header=0, index_col=0).values
 
 START = 10441
 END = 13899
@@ -30,36 +30,38 @@ def scale(data):
 
 def train(Train_Env, Epoch):
     agent = DDPG(train_env, lb, node)
+    save_iter = [1, 2, 5, 10, 20, 30, 50, 100, 150, 200]
     for t in range(Epoch):
         print('epoch:', t)
         state, done = Train_Env.reset(), False
-        agent.initial()
         while not done:
             action = agent.act(state, Train_Env.portfolio)
             next_state, reward, done, _ = Train_Env.step(action)
             agent.perceive(state, action, reward, next_state, done)
             state = next_state
             if Train_Env.n_step % 300 == 299:
                 print(Train_Env.n_step, ':',
-                      int(Train_Env.rewards[-1]), '\t',
+                      int(Train_Env.rewards[Train_Env.n_step]), '\t',
                       int(sum(Train_Env.cost)), '\t',
-                      int(Train_Env.available_cash[-1]), '\t',
-                      agent.critic_network.loss.data, '\t',
-                      agent.actor_network.loss.data
+                      int(Train_Env.available_cash[Train_Env.n_step]), '\t',
+                      agent.critic_network.loss.data
                       )
         total_reward = Train_Env.rewards[-1]
         total_cost = sum(Train_Env.cost)
         print('DDPG: Evaluation Average Reward:', total_reward)
         print('DDPG: Average Cost: ', total_cost)
+
+        for k in save_iter:
+            if t == k:
+                torch.save(agent.actor_network.target.state_dict(), 'DDPG_model' + str(t) + '.pth')
     return agent
 
 
 if __name__ == '__main__':
-    lb, node, epoch = 36, 2048, 1
+    lb, node, epoch = 12, 1024, 201
     buy_train = raw_buy[:START]
     sell_train = raw_sell[:START]
     amount_train = raw_amount[:START]
 
     train_env = MarketEnv([buy_train, sell_train, amount_train], 0)
     agent = train(train_env, epoch)
-    torch.save(agent.actor_network.target.state_dict(), 'DDPG_model.pth')

DDPG.py

@@ -3,15 +3,12 @@
 @Author :JohsuaWu1997
 @Date   :2020/1/30
 """
-import numpy as np
 import torch
 
 from actor_critic import Actor, Critic
 from ou_noise import OUNoise
 
 cuda = torch.device('cuda')
-torch.backends.cudnn.deterministic = True
-torch.backends.cudnn.benchmark = False
 
 GAMMA = 0.9999999993340943687843739933894
 
@@ -35,7 +32,6 @@ def __init__(self, env, time_steps, hidden_dim):
         self.time_dim = time_steps
         self.state_dim = env.observation_space.shape[1]
         self.action_dim = env.action_space.shape[0]
-        print(self.state_dim,self.action_dim,self.time_dim)
         self.batch_size = 64
         self.memory_size = self.time_dim + self.batch_size * 10
         self.start_size = self.time_dim + self.batch_size * 2
@@ -51,18 +47,16 @@ def __init__(self, env, time_steps, hidden_dim):
         self.replay_reward = torch.zeros((self.start_size - 1,), device=cuda)
 
         # Initialize a random process the Ornstein-Uhlenbeck process for action exploration
-        self.exploration_noise = OUNoise(self.action_dim, sigma=0.05 / self.action_dim)
+        self.exploration_noise = OUNoise(self.action_dim, sigma=0.01 / self.action_dim)
         self.initial()
 
     def initial(self):
         self.steps = 0
-        self.action = np.zeros((self.action_dim,))
+        self.action = torch.zeros(self.action_dim, device=cuda)
         self.replay_state = torch.zeros((self.start_size - 1, 3, self.state_dim), device=cuda)
         self.replay_next_state = torch.zeros((self.start_size - 1, 3, self.state_dim), device=cuda)
         self.replay_action = torch.zeros((self.start_size - 1, self.state_dim), device=cuda)
         self.replay_reward = torch.zeros((self.start_size - 1,), device=cuda)
-        torch.manual_seed(self.seed)
-        np.random.seed(self.seed)
 
     def train_on_batch(self):
         # Sample a random minibatch of N transitions from replay buffer
@@ -84,18 +78,14 @@ def train_on_batch(self):
         reward_batch = torch.index_select(self.replay_reward, 0, sample)
 
         # Calculate y_batch
-        q_batch = self.critic_network.target_q(
-            self.actor_network.target_action(next_state_batch), next_state_batch
-        )
+        next_action_batch = self.actor_network.target_action(next_state_batch)
+        q_batch = self.critic_network.target_q(next_action_batch, next_state_batch)
         y_batch = torch.add(reward_batch, q_batch, alpha=GAMMA).view(-1, 1)
 
-        # train critic by minimizing the loss L
-        self.critic_network.train(y_batch, action_batch, state_batch)
-
-        # train actor by target loss
+        # train actor-critic by target loss
         self.actor_network.train(
-            self.critic_network.critic_loss(
-                self.actor_network.actor_action(state_batch), state_batch
+            self.critic_network.train(
+                y_batch, action_batch, state_batch
             )
         )
 
@@ -104,25 +94,21 @@ def train_on_batch(self):
         self.critic_network.update_target()
 
     def perceive(self, state, action, reward, next_state, done):
-        state_tensor = torch.tensor([state.tolist()], device=cuda)
-        next_state_tensor = torch.tensor([next_state.tolist()], device=cuda)
-        action_tensor = torch.tensor([action.tolist()], device=cuda)
-        reward_tensor = torch.tensor([reward.tolist()], device=cuda)
         if self.steps < self.start_size - 1:
-            self.replay_state[self.steps] = state_tensor
-            self.replay_next_state[self.steps] = next_state_tensor
-            self.replay_action[self.steps] = action_tensor
+            self.replay_state[self.steps] = state
+            self.replay_next_state[self.steps] = next_state
+            self.replay_action[self.steps] = action
             self.replay_reward[self.steps] = reward
         else:
             if self.steps >= self.memory_size:
                 self.replay_state = self.replay_state[1:]
                 self.replay_next_state = self.replay_next_state[1:]
                 self.replay_action = self.replay_action[1:]
                 self.replay_reward = self.replay_reward[1:]
-            self.replay_state = torch.cat((self.replay_state, state_tensor), dim=0)
-            self.replay_next_state = torch.cat((self.replay_next_state, next_state_tensor), dim=0)
-            self.replay_action = torch.cat((self.replay_action, action_tensor), dim=0)
-            self.replay_reward = torch.cat((self.replay_reward, reward_tensor), dim=0)
+            self.replay_state = torch.cat((self.replay_state, state.unsqueeze(0)), dim=0)
+            self.replay_next_state = torch.cat((self.replay_next_state, next_state.unsqueeze(0)), dim=0)
+            self.replay_action = torch.cat((self.replay_action, action.unsqueeze(0)), dim=0)
+            self.replay_reward = torch.cat((self.replay_reward, reward.unsqueeze(0)), dim=0)
         self.steps += 1
 
     def act(self, next_state, portfolio):
@@ -131,11 +117,12 @@ def act(self, next_state, portfolio):
             next_amount_data = min_max_scale(self.replay_next_state[:, 2, :])[-1].view(1, -1)
             next_state_data = torch.cat([next_state_data, next_amount_data], dim=1)
             self.train_on_batch()
-            allocation = self.actor_network.target_action(next_state_data).cpu().data.numpy().ravel()
+            allocation = self.actor_network.target_action(next_state_data).data.view(-1)
+            allocation += torch.tensor(self.exploration_noise.noise().tolist(), device=cuda)
             allocation[allocation < 0] = 0
             allocation /= sum(allocation)
-            allocation = np.floor(
+            allocation = torch.floor(
                 portfolio * allocation / next_state[1, :] / self.unit
             ) * self.unit
             self.action = allocation
-        return np.array(self.action)
+        return self.action.clone()

actor_critic.py

@@ -1,8 +1,3 @@
-"""
-@File   :actor_critic.py
-@Author :JohsuaWu1997
-@Date   :2020/1/30
-"""
 import torch
 
 cuda = torch.device('cuda')
@@ -43,20 +38,24 @@ def forward(self, x):
 class CriticNet(torch.nn.Module):
     def __init__(self, input_dim, hidden_dim, output_dim):
         super(CriticNet, self).__init__()
-
-        self.nn = torch.nn.Sequential(
-            torch.nn.Linear(input_dim + output_dim, hidden_dim),
-            torch.nn.ReLU(),
+        self.nn2 = torch.nn.Sequential(
+            torch.nn.Linear(hidden_dim + output_dim, hidden_dim),
+            torch.nn.Tanh(),
             torch.nn.Linear(hidden_dim, hidden_dim),
+            torch.nn.Tanh(),
+            torch.nn.Linear(hidden_dim, 1)
+        )
+        self.nn1 = torch.nn.Sequential(
+            torch.nn.Linear(input_dim, hidden_dim),
             torch.nn.ReLU(),
             torch.nn.Linear(hidden_dim, hidden_dim),
-            torch.nn.ReLU(),
-            torch.nn.Linear(hidden_dim, 1)
+            torch.nn.Softmax(dim=1),
         )
 
     def forward(self, a, x):
-        ax = torch.cat((a, x), 1)
-        out = self.nn(ax)
+        x_out = self.nn1(x)
+        ax = torch.cat((a, x_out), 1)
+        out = self.nn2(ax)
         return out
 
 
@@ -67,14 +66,12 @@ def __init__(self, time_dim, state_dim, action_dim, hidden_dim):
         self.actor_weights = [params for params in self.actor.parameters()]
         self.target_weights = [params for params in self.target.parameters()]
         self.optimizer = torch.optim.Adam(self.actor.parameters())
-        self.loss = torch.tensor([0], device=cuda)
         hard_copy(self.target_weights, self.actor_weights)
 
-    def train(self, actor_loss):
+    def train(self, loss_grad):
         for _ in range(1):
-            self.loss = actor_loss
             self.optimizer.zero_grad()
-            self.loss.backward()
+            self.actor_weights[-1].backward(-loss_grad)
             self.optimizer.step()
 
     def actor_action(self, state):
@@ -91,6 +88,7 @@ def update_target(self):
 
 class Critic:
     def __init__(self, time_dim, state_dim, action_dim, hidden_dim):
+        self.action_dim = action_dim
         self.critic = CriticNet(state_dim * (time_dim + 1), hidden_dim, action_dim).to(cuda)
         self.target = CriticNet(state_dim * (time_dim + 1), hidden_dim, action_dim).to(cuda)
         self.critic_weights = [params for params in self.critic.parameters()]
@@ -107,14 +105,17 @@ def train(self, y_batch, action_batch, state_batch):
             self.optimizer.zero_grad()
             self.loss.backward()
             self.optimizer.step()
-
-    def critic_loss(self, next_action_batch, next_state_batch):
-        self.critic.zero_grad()
-        return torch.nn.functional.softplus(-self.critic(next_action_batch, next_state_batch).mean())
+        return torch.mean(self.critic_weights[0].grad[:, :self.action_dim], dim=0)
 
     def target_q(self, next_action_batch, next_state_batch):
         self.target.zero_grad()
         return self.target(next_action_batch, next_state_batch).view(-1)
 
     def update_target(self):
         soft_copy(self.target_weights, self.critic_weights)
+
+
+if __name__ == '__main__':
+    critic = CriticNet(50 * (12 + 1), 37, 50).to(cuda)
+    for params in critic.parameters():
+        print(params.shape)