Revert "Split train"

Author: MatthiasBienvenu

.gitignore

@@ -10,4 +10,3 @@ Debug_Wayfinding
 .venv
 *.onnx
 checkpoints
-*.wbproj

README.md

@@ -33,7 +33,7 @@ uv sync --extra rpi
 
 Navigate to the simulator directory.
 ```bash
-cd scripts
+cd src/Simulateur
 ```
 
 Run the multi-process training script.

scripts/launch_train_multiprocessing.py

@@ -2,7 +2,7 @@
 from torch.cuda import is_available
 
 n_map = 2
-n_simulations = 2
+n_simulations = 8
 n_vehicles = 1
 n_stupid_vehicles = 0
 n_actions_steering = 16

src/Simulateur/__init__.py

@@ -2,7 +2,6 @@
 from typing import *
 import numpy as np
 import gymnasium as gym
-import time 
 
 from checkpointmanager import CheckpointManager, checkpoints
 
@@ -220,17 +219,7 @@ def main():
         #Prédiction pour séléctionner une action à partir de l"observation
         for e in envs:
             log(f"CLIENT{simulation_rank}/{e.vehicle_rank} : trying to read from fifo")
-
-            timeout = 10  # seconds
-            start_time = time.time()
-
-            while time.time() - start_time < timeout:
-                raw = e.fifo_r.read(np.dtype(np.int64).itemsize * 2)
-                if len(raw) == np.dtype(np.int64).itemsize * 2:
-                    # We got the full action data
-                    action = np.frombuffer(raw, dtype=np.int64)
-                    break
-                
+            action = np.frombuffer(e.fifo_r.read(np.dtype(np.int64).itemsize * 2), dtype=np.int64)
             log(f"CLIENT{simulation_rank}/{e.vehicle_rank} : received {action=}")
 
             obs, reward, done, truncated, info = e.step(action)

src/Simulateur/config.py

@@ -1,9 +1,27 @@
 import os
+import time
 from typing import *
+
+import matplotlib.pyplot as plt
 import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.multiprocessing as mp
+
+from stable_baselines3 import PPO
+from stable_baselines3.common.env_checker import check_env
+from stable_baselines3.common.vec_env import SubprocVecEnv, DummyVecEnv
+
 import gymnasium as gym
 
+from onnx_utils import export_onnx, test_onnx
 from config import *
+from CNN1DExtractor import CNN1DExtractor
+from TemporalResNetExtractor import TemporalResNetExtractor
+from CNN1DResNetExtractor import CNN1DResNetExtractor
+
+if B_DEBUG: from DynamicActionPlotCallback import DynamicActionPlotDistributionCallback
 
 
 def log(s: str):
@@ -90,4 +108,112 @@ def step(self, action):
         # check if the context is correct
         # if self.simulation_rank == 0:
         #     print(f"{(obs[0] == 0).mean():.3f} {(obs[1] == 0).mean():.3f}")
-        return obs, reward, done, truncated, info
+        return obs, reward, done, truncated, info
+
+
+if __name__ == "__main__":
+    if not os.path.exists("/tmp/autotech/"):
+        os.mkdir("/tmp/autotech/")
+
+    os.system('if [ -n "$(ls /tmp/autotech)" ]; then rm /tmp/autotech/*; fi')
+    if B_DEBUG:
+        print("Webots started", file=open("/tmp/autotech/logs", "w"))
+
+    def make_env(rank: int):
+        log(f"CAREFUL !!! created an SERVER env with {rank=}")
+        return WebotsSimulationGymEnvironment(rank)
+
+    envs = SubprocVecEnv([lambda rank=rank : make_env(rank) for rank in range(n_simulations)])
+
+    ExtractorClass = TemporalResNetExtractor
+
+    policy_kwargs = dict(
+        features_extractor_class=ExtractorClass,
+        features_extractor_kwargs=dict(
+            context_size=context_size,
+            lidar_horizontal_resolution=lidar_horizontal_resolution,
+            camera_horizontal_resolution=camera_horizontal_resolution,
+            device=device
+        ),
+        activation_fn=nn.ReLU,
+        net_arch=[512, 512, 512],
+    )
+
+
+    ppo_args = dict(
+        n_steps=4096,
+        n_epochs=10,
+        batch_size=256,
+        learning_rate=3e-4,
+        gamma=0.99,
+        verbose=1,
+        normalize_advantage=True,
+        device=device
+    )
+
+
+    save_path = __file__.rsplit("/", 1)[0] + "/checkpoints/" + ExtractorClass.__name__ + "/"
+    if not os.path.exists(save_path):
+        os.mkdir(save_path)
+
+    print(save_path)
+    print(os.listdir(save_path))
+
+    valid_files = [x for x in os.listdir(save_path) if x.rstrip(".zip").isnumeric()]
+
+    if valid_files:
+        model_name = max(
+            valid_files,
+            key=lambda x : int(x.rstrip(".zip"))
+        )
+        print(f"Loading model {save_path + model_name}")
+        model = PPO.load(
+            save_path + model_name,
+            envs,
+            **ppo_args,
+            policy_kwargs=policy_kwargs
+        )
+        i = int(model_name.rstrip(".zip")) + 1
+        print(f"----- Model found, loading {model_name} -----")
+
+    else:
+        model = PPO(
+            "MlpPolicy",
+            envs,
+            **ppo_args,
+            policy_kwargs=policy_kwargs
+        )
+
+        i = 0
+        print("----- Model not found, creating a new one -----")
+
+    print("MODEL HAS HYPER PARAMETERS:")
+    print(f"{model.learning_rate=}")
+    print(f"{model.gamma=}")
+    print(f"{model.verbose=}")
+    print(f"{model.n_steps=}")
+    print(f"{model.n_epochs=}")
+    print(f"{model.batch_size=}")
+    print(f"{model.device=}")
+
+    log(f"SERVER : finished executing")
+
+    # obs = envs.reset()
+    # while True:
+    #     action, _states = model.predict(obs, deterministic=True)  # Use deterministic=True for evaluation
+    #     obs, reward, done, info = envs.step(action)
+    #     envs.render()  # Optional: visualize the environment
+
+
+    while True:
+        export_onnx(model)
+        test_onnx(model)
+
+        if B_DEBUG:
+            model.learn(total_timesteps=500_000, callback=DynamicActionPlotDistributionCallback())
+        else:
+            model.learn(total_timesteps=500_000)
+
+        model.save(save_path + str(i))
+
+        i += 1

src/Simulateur/controllers/controllerWorldSupervisor/controllerWorldSupervisor.py

@@ -0,0 +1,9 @@
+Webots Project File version R2023b
+perspectives: 000000ff00000000fd0000000200000001000001cf00000278fc0200000001fb0000001400540065007800740045006400690074006f00720100000016000002780000008900ffffff000000030000078000000176fc0100000001fb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c0100000000000007800000006900ffffff000005af0000027800000001000000020000000100000008fc00000000
+simulationViewPerspectives: 000000ff0000000100000002000001520000045b0100000002010000000100
+sceneTreePerspectives: 000000ff00000001000000030000001f000000c0000000fa0100000002010000000200
+maximizedDockId: -1
+centralWidgetVisible: 1
+orthographicViewHeight: 1
+textFiles: 2 "Bobox.proto" "../../CoVAPSy_Intech/Simulateur/worlds/piste.wbt" "../../CoVAPSy_Intech/Simulateur/protos/Vehicle.proto"
+consoles: Console:All:All

…lateur/WebotsSimulationGymEnvironment.py …mulateur/launch_train_multiprocessing.pysrc/Simulateur/WebotsSimulationGymEnvironment.py renamed to src/Simulateur/launch_train_multiprocessing.py src/Simulateur/WebotsSimulationGymEnvironment.py renamed to src/Simulateur/launch_train_multiprocessing.py

@@ -0,0 +1,11 @@
+Webots Project File version R2025a
+perspectives: 000000ff00000000fd00000002000000010000011c00000177fc0200000001fb0000001400540065007800740045006400690074006f00720000000016000001770000004500ffffff00000003000006c000000216fc0100000001fb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c0100000000000006c00000008700ffffff000006c0000001c400000001000000020000000100000008fc00000000
+simulationViewPerspectives: 000000ff0000000100000002000001000000017d0100000002010000000100
+sceneTreePerspectives: 000000ff00000001000000030000001f0000013e000000fa0100000002010000000200
+maximizedDockId: -1
+centralWidgetVisible: 1
+orthographicViewHeight: 1
+textFiles: -1
+globalOptionalRendering: LidarRaysPaths::LidarPointClouds
+consoles: Console:All:All
+renderingDevicePerspectives: TT02_0:RASPI_Camera_V2;1;32;0;0

src/Simulateur/worlds/.piste.wbproj

@@ -0,0 +1,13 @@
+Webots Project File version R2025a
+perspectives: 000000ff00000000fd0000000200000001000000870000028afc0200000001fb0000001400540065007800740045006400690074006f007201000000000000028a0000004500ffffff00000003000006c000000150fc0100000002fb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c0100000000000006c00000008700fffffffb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c010000000000000a000000000000000000000006370000028a00000001000000020000000100000008fc00000000
+simulationViewPerspectives: 000000ff00000001000000020000012b000005a50100000002010000000100
+sceneTreePerspectives: 000000ff00000001000000030000001e00000364000000fa0100000002010000000200
+minimizedPerspectives: 000000ff00000000fd0000000200000001000000750000017bfc0200000001fb0000001400540065007800740045006400690074006f007201000000160000017b0000003f00ffffff000000030000039b00000039fc0100000002fb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c01000000000000039b0000006900fffffffb0000001a0043006f006e0073006f006c00650041006c006c0041006c006c010000000000000a000000000000000000000003240000017b00000001000000020000000100000008fc00000000
+maximizedDockId: -1
+centralWidgetVisible: 1
+orthographicViewHeight: 1
+textFiles: -1
+globalOptionalRendering: LidarPointClouds::LidarRaysPaths
+consoles: Console:All:All
+renderingDevicePerspectives: TT02_0:RASPI_Camera_V2;1;32;0;0
+renderingDevicePerspectives: sparringpartner_car_0:RASPI_Camera_V2;1;32;0;0