Dev

.gitignore

@@ -204,3 +204,12 @@ example_usage/*
 *.pth
 .gitignore
 CITATION.cff
+CLAUDE.md
+.claude/*
+.reports/*
+.mcp.json
+research/*
+GEMINI.md
+
+#Obsidian notes vault
+gridmind-notes-vault/*

example_usage/control/mountain_car/one_step_actor_critic_example.py

@@ -4,6 +4,7 @@
 
 
 from gridmind.feature_construction.multi_hot import MultiHotEncoder
+from gridmind.feature_construction.normalizer import MinMaxNormalizer
 from gridmind.feature_construction.tile_coding import TileCoding
 from gridmind.policies.parameterized.discrete_action_mlp_policy import (
     DiscreteActionMLPPolicy,
@@ -13,13 +14,20 @@
 )
 import gymnasium as gym
 import torch
+import numpy as np
 
 
 env = gym.make("MountainCar-v0")
+
+# Feature construction pipeline: normalize -> tile coding -> multi-hot encoding
 num_tilings = 7
+normalizer = MinMaxNormalizer(
+    low=np.array([-1.2, -0.07]),  # Mountain Car observation bounds
+    high=np.array([0.6, 0.07]),
+)
 multi_hot_encoder = MultiHotEncoder(num_categories=num_tilings**4)
 tile_encoder = TileCoding(ihtORsize=num_tilings**4, numtilings=num_tilings)
-feature_constructor = lambda x: multi_hot_encoder(tile_encoder(x))
+feature_constructor = lambda x: multi_hot_encoder(tile_encoder(normalizer(x)))
 
 observation, _ = env.reset()
 

src/gridmind/algorithms/base_learning_algorithm.py

@@ -6,8 +6,9 @@
 import dill
 from gridmind.policies.base_policy import BasePolicy
 import logging
+from gridmind.config import get_save_dir
 from gridmind.utils.divergence.base_divergence_detector import BaseDivergenceDetector
-from gridmind.utils.logtools.async_tensorboard_logger import AsyncTensorboardLogger
+from gridmind.utils.logtools.null_logger import NullWriter
 from gridmind.utils.performance_evaluation.base_performance_evaluator import (
     BasePerformanceEvaluator,
 )
@@ -16,12 +17,6 @@
 )
 from gymnasium import Env
 from tqdm import trange
-from torch.utils.tensorboard import SummaryWriter
-
-try:
-    from data import SAVE_DATA_DIR
-except ImportError:
-    SAVE_DATA_DIR = None
 
 
 class BaseLearningAlgorithm(ABC):
@@ -30,7 +25,7 @@ def __init__(
         name: str,
         env: Optional[Env] = None,
         summary_dir: Optional[str] = None,
-        write_summary: bool = True,
+        write_summary: bool = False,
     ) -> None:
         self.name = name
         self.logger = logging.getLogger(self.__class__.__name__)
@@ -48,12 +43,14 @@ def __init__(
         self.monitor_divergence = False
         self.stop_on_divergence = False
 
+        self.summary_writer = NullWriter()
         self.write_summary = write_summary
         if self.write_summary:
-            assert (
-                summary_dir is not None or SAVE_DATA_DIR is not None
-            ), "Please specify summary_dir"
-
+            if summary_dir is None and get_save_dir() is None:
+                raise ValueError(
+                    "write_summary=True requires either summary_dir or "
+                    "gridmind.config.set_save_dir() to be set."
+                )
             self._initialize_summary_writer(summary_dir, env_name)
 
     def _initialize_summary_writer(
@@ -63,7 +60,10 @@ def _initialize_summary_writer(
         extra_info: str = "",
         use_async_writer: bool = False,
     ):
-        summary_dir = summary_dir if summary_dir is not None else SAVE_DATA_DIR
+        from torch.utils.tensorboard import SummaryWriter
+        from gridmind.utils.logtools.async_tensorboard_logger import AsyncTensorboardLogger
+
+        summary_dir = summary_dir if summary_dir is not None else get_save_dir()
 
         log_dir = os.path.join(
             summary_dir,
@@ -291,11 +291,8 @@ def _training_wrapper(
         if save_policy:
             env_name = self.env.spec.id if self.env.spec is not None else "unknown"
 
-        if save_policy:
-            env_name = self.env.spec.id if self.env.spec is not None else "unknown"
-
-            if SAVE_DATA_DIR is not None:
-                saved_policy_dir = os.path.join(SAVE_DATA_DIR, env_name)
+            if get_save_dir() is not None:
+                saved_policy_dir = os.path.join(get_save_dir(), env_name)
                 self.save_policy(saved_policy_dir)
 
     def _report_all_metrics(self):
@@ -314,8 +311,8 @@ def _report_all_metrics(self):
 
         env_name = self.env.spec.id if self.env.spec is not None else "unknown"
 
-        if SAVE_DATA_DIR is not None:
-            saved_policy_dir = os.path.join(SAVE_DATA_DIR, env_name)
+        if get_save_dir() is not None:
+            saved_policy_dir = os.path.join(get_save_dir(), env_name)
             self.save_policy(saved_policy_dir)
 
     def evaluate_policy(self, num_episodes: int):

src/gridmind/algorithms/evolutionary_rl/base_evo_rl_algorithm.py

@@ -1,3 +1,4 @@
+from abc import abstractmethod
 from typing import Optional
 from gridmind.algorithms.base_learning_algorithm import BaseLearningAlgorithm
 from gridmind.policies.base_policy import BasePolicy
@@ -10,7 +11,7 @@ def __init__(
         name: str,
         env: Optional[Env] = None,
         summary_dir: Optional[str] = None,
-        write_summary: bool = True,
+        write_summary: bool = False,
     ) -> None:
         super().__init__(
             name, env, summary_dir=summary_dir, write_summary=write_summary
@@ -34,10 +35,13 @@ def _train_episodes(self, num_episodes: int, prediction_only: bool):
     def _train_steps(self, num_steps: int, prediction_only: bool, *args, **kwargs):
         raise NotImplementedError
 
+    @abstractmethod
+    def _train(self, num_generations: int, *args, **kwargs): ...
+
     def train(self, num_generations: int, save_policy: bool = True):
         self._training_wrapper(
             num_iter=num_generations,
             prediction_only=False,
             save_policy=save_policy,
-            training_fn=self.train,
+            training_fn=self._train,
         )

src/gridmind/algorithms/evolutionary_rl/neuroevolution/neuro_agent.py

@@ -28,6 +28,14 @@ def __init__(
     def __repr__(self):
         return f"NeuroAgent(id={self.id}, fitness={self.fitness}, starting_generation={self.starting_generation})"
 
+    @property
+    def policy(self):
+        return self.network
+
+    @policy.setter
+    def policy(self, value):
+        self.network = value
+
     @property
     def id(self):
         return str(self._id)

src/gridmind/algorithms/evolutionary_rl/neuroevolution/neuroevolution.py

@@ -34,7 +34,7 @@ def __init__(
         num_processes: Optional[int] = None,
         stopping_fitness: Optional[float] = None,
         summary_dir: Optional[str] = None,
-        write_summary: bool = True,
+        write_summary: bool = False,
     ):
         super().__init__(
             name="NeuroEvolution",
@@ -43,8 +43,6 @@ def __init__(
             write_summary=write_summary,
         )
 
-        self.env = env
-        self.name = "NeuroEvolution"
         self.mu = mu
         self._lambda = _lambda
         self.mutation_mean = mutation_mean
@@ -81,7 +79,7 @@ def get_best(self, unwrapped: bool = True):
         ), "No best agent found. Train the algorithm first."
 
         if unwrapped:
-            return self.best_agent.network
+            return self.best_agent.policy
 
         return self.best_agent
 
@@ -163,7 +161,7 @@ def evaluate_fitness(
 
         return sum_episode_return / average_over_episodes
 
-    def train(self, num_generations: int, *args, **kwargs):
+    def _train(self, num_generations: int, *args, **kwargs):
         for num_gen in trange(num_generations):
             agent_to_assess_fitness = []
 
@@ -172,8 +170,7 @@ def train(self, num_generations: int, *args, **kwargs):
                     agent_to_assess_fitness.append(agent)
 
             fitness_scores = [
-                self.evaluate_fitness(agent.network)
-                for agent in agent_to_assess_fitness
+                self.evaluate_fitness(agent.policy) for agent in agent_to_assess_fitness
             ]
 
             for agent, fitness in zip(agent_to_assess_fitness, fitness_scores):
@@ -227,55 +224,15 @@ def train(self, num_generations: int, *args, **kwargs):
             for parent in parents:
                 for _ in range(self._lambda // self.mu):
                     mutated_param_vector = self.mutate(
-                        network=parent.network,
+                        network=parent.policy,
                         mean=self.mutation_mean,
                         std=self.mutation_std,
                     )
                     child = self.spawn_individual()
                     NeuroEvolutionUtil.set_parameters_vector(
-                        child.network, mutated_param_vector
+                        child.policy, mutated_param_vector
                     )
                     self.population.append(child)
 
             self._generation += 1
         return self.best_agent
-
-
-if __name__ == "__main__":
-    from itertools import product
-
-    env = gym.make("CartPole-v1")
-
-    mutation_means = [0, 0.1, 0.2]
-    mutation_stds = [0.1, 0.2, 0.3]
-
-    mutation_rate_combinations = list(product(mutation_means, mutation_stds))
-
-    trained_agents = []
-
-    for mutation_mean, mutation_std in mutation_rate_combinations:
-        algorithm = NeuroEvolution(
-            env=env,
-            mu=5,
-            _lambda=20,
-            stopping_fitness=500,
-            mutation_mean=mutation_mean,
-            mutation_std=mutation_std,
-        )
-        trained_agents.append(algorithm.train(num_generations=1000))
-
-    eval_env = gym.make("CartPole-v1", render_mode="human")
-
-    policy = random.choice(trained_agents).network
-
-    obs, info = eval_env.reset()
-    done = False
-
-    episode_return = 0.0
-
-    while not done:
-        obs = algorithm._preprocess(obs)
-        action = policy.get_action(obs)
-        obs, reward, terminated, truncated, info = eval_env.step(action)
-        episode_return += reward
-        done = terminated or truncated

src/gridmind/algorithms/evolutionary_rl/neuroevolution/neuroevolution_util.py

@@ -48,43 +48,3 @@ def evaluate_fitness(
                 done = terminated or truncated
 
         return sum_episode_return / average_over_episodes
-
-
-if __name__ == "__main__":
-    import torch
-    import torch.nn as nn
-    import numpy as np
-
-    # Define a simple MLP
-    class SimpleNN(nn.Module):
-        def __init__(self, input_size=4, hidden_size=10, output_size=2):
-            super(SimpleNN, self).__init__()
-            self.fc1 = nn.Linear(input_size, hidden_size)
-            self.fc2 = nn.Linear(hidden_size, output_size)
-
-        def forward(self, x):
-            return self.fc2(torch.relu(self.fc1(x)))
-
-    # Create an instance of the model
-    model = SimpleNN()
-    vector = NeuroEvolutionUtil.get_parameters_vector(
-        model
-    )  # Extract weights as a flat vector
-    print(vector)
-    print(vector.shape)
-    NeuroEvolutionUtil.set_parameters_vector(model, vector)
-
-    def mutate(model, mean, std):
-        chromosome = NeuroEvolutionUtil.get_parameters_vector(model)
-        noise = np.random.normal(loc=mean, scale=std, size=chromosome.shape)
-
-        mutated_chromosome = chromosome + noise
-
-        NeuroEvolutionUtil.set_parameters_vector(
-            model, mutated_chromosome
-        )  # Set weights from a flat vector
-
-        return mutated_chromosome
-
-    mutated_vector = mutate(model, 0, 0.01)
-    print(mutated_vector)

src/gridmind/algorithms/function_approximation/actor_critic/one_step_actor_critic.py

@@ -28,7 +28,7 @@ def __init__(
         clip_grads: bool = True,
         grad_clip_value: float = 1.0,
         summary_dir: Optional[str] = None,
-        write_summary: bool = True,
+        write_summary: bool = False,
     ):
         super().__init__(
             "OneStepActorCritic",
@@ -129,9 +129,12 @@ def _train_episodes(self, num_episodes: int, prediction_only: bool = False):
 
                 next_observation = self._preprocess(next_observation)
 
-                next_state_value = (
-                    self.value_estimator(next_observation) if not terminated else 0
-                )
+                with torch.no_grad():
+                    next_state_value = (
+                        self.value_estimator(next_observation)
+                        if not terminated
+                        else torch.tensor(0.0)
+                    )
 
                 cur_state_value = self.value_estimator(observation)
 
@@ -146,7 +149,7 @@ def _train_episodes(self, num_episodes: int, prediction_only: bool = False):
                 self.logger.debug(f"Value grads: {value_grads}")
 
                 policy_grads = torch.autograd.grad(
-                    torch.log(self.policy.get_action_prob(observation, action)),
+                    self.policy.get_log_action_prob(observation, action),
                     self.policy.parameters(),
                 )
                 self.logger.debug(f"Policy grads: {policy_grads}")

src/gridmind/algorithms/function_approximation/base_function_approximation_based_learning_algorithm.py

@@ -13,7 +13,7 @@ def __init__(
         env: Optional[Env] = None,
         feature_constructor=None,
         summary_dir=None,
-        write_summary=True,
+        write_summary=False,
     ):
         super().__init__(name, env, summary_dir, write_summary)
         self.feature_constructor = feature_constructor