|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# |
| 3 | +# This source code is licensed under the MIT license found in the |
| 4 | +# LICENSE file in the root directory of this source tree. |
| 5 | +"""Minimal MAPPO / IPPO recipe on VMAS using the new |
| 6 | +:class:`~torchrl.objectives.multiagent.MAPPOLoss` / |
| 7 | +:class:`~torchrl.objectives.multiagent.IPPOLoss` classes. |
| 8 | +
|
| 9 | +For the full, hydra-configured, wandb-logged version see |
| 10 | +``sota-implementations/multiagent/mappo_ippo.py``. This file is intentionally |
| 11 | +short: it's there to show that the new loss classes collapse the boilerplate |
| 12 | +that previously required ``ClipPPOLoss`` + manual ``set_keys(done=..., |
| 13 | +terminated=...)`` + manual ``make_value_estimator(GAE, ...)`` into a single |
| 14 | +construction call. |
| 15 | +
|
| 16 | +Usage:: |
| 17 | +
|
| 18 | + python examples/multiagent/mappo_vmas.py --algo mappo --frames 200_000 |
| 19 | + python examples/multiagent/mappo_vmas.py --algo ippo --frames 200_000 |
| 20 | +
|
| 21 | +The two should reach similar reward on the easy navigation scenario; MAPPO |
| 22 | +typically pulls ahead on harder coordination tasks (Yu et al. 2022). |
| 23 | +""" |
| 24 | +from __future__ import annotations |
| 25 | + |
| 26 | +import argparse |
| 27 | +import time |
| 28 | + |
| 29 | +import torch |
| 30 | +from tensordict.nn import TensorDictModule |
| 31 | +from tensordict.nn.distributions import NormalParamExtractor |
| 32 | +from torch import nn |
| 33 | + |
| 34 | +from torchrl.collectors import SyncDataCollector |
| 35 | +from torchrl.data import LazyTensorStorage, TensorDictReplayBuffer |
| 36 | +from torchrl.data.replay_buffers.samplers import SamplerWithoutReplacement |
| 37 | +from torchrl.envs import RewardSum, TransformedEnv |
| 38 | +from torchrl.modules import MultiAgentMLP, ProbabilisticActor, TanhNormal, ValueNorm |
| 39 | +from torchrl.objectives import IPPOLoss, MAPPOLoss |
| 40 | + |
| 41 | + |
| 42 | +def make_actor(env, *, share_params: bool = True) -> ProbabilisticActor: |
| 43 | + obs_dim = env.observation_spec["agents", "observation"].shape[-1] |
| 44 | + action_dim = env.action_spec.shape[-1] |
| 45 | + backbone = nn.Sequential( |
| 46 | + MultiAgentMLP( |
| 47 | + n_agent_inputs=obs_dim, |
| 48 | + n_agent_outputs=2 * action_dim, |
| 49 | + n_agents=env.n_agents, |
| 50 | + centralized=False, |
| 51 | + share_params=share_params, |
| 52 | + depth=2, |
| 53 | + num_cells=256, |
| 54 | + activation_class=nn.Tanh, |
| 55 | + ), |
| 56 | + NormalParamExtractor(), |
| 57 | + ) |
| 58 | + module = TensorDictModule( |
| 59 | + backbone, |
| 60 | + in_keys=[("agents", "observation")], |
| 61 | + out_keys=[("agents", "loc"), ("agents", "scale")], |
| 62 | + ) |
| 63 | + return ProbabilisticActor( |
| 64 | + module=module, |
| 65 | + in_keys=[("agents", "loc"), ("agents", "scale")], |
| 66 | + out_keys=[env.action_key], |
| 67 | + distribution_class=TanhNormal, |
| 68 | + distribution_kwargs={ |
| 69 | + "low": env.full_action_spec_unbatched[("agents", "action")].space.low, |
| 70 | + "high": env.full_action_spec_unbatched[("agents", "action")].space.high, |
| 71 | + }, |
| 72 | + return_log_prob=True, |
| 73 | + ) |
| 74 | + |
| 75 | + |
| 76 | +def make_critic( |
| 77 | + env, *, centralized: bool, share_params: bool = True |
| 78 | +) -> TensorDictModule: |
| 79 | + obs_dim = env.observation_spec["agents", "observation"].shape[-1] |
| 80 | + return TensorDictModule( |
| 81 | + MultiAgentMLP( |
| 82 | + n_agent_inputs=obs_dim, |
| 83 | + n_agent_outputs=1, |
| 84 | + n_agents=env.n_agents, |
| 85 | + centralized=centralized, |
| 86 | + share_params=share_params, |
| 87 | + depth=2, |
| 88 | + num_cells=256, |
| 89 | + activation_class=nn.Tanh, |
| 90 | + ), |
| 91 | + in_keys=[("agents", "observation")], |
| 92 | + out_keys=[("agents", "state_value")], |
| 93 | + ) |
| 94 | + |
| 95 | + |
| 96 | +def main(args: argparse.Namespace) -> None: |
| 97 | + try: |
| 98 | + from torchrl.envs.libs.vmas import VmasEnv |
| 99 | + except ImportError as exc: |
| 100 | + raise SystemExit( |
| 101 | + "This example requires VMAS. Install it with `pip install vmas`." |
| 102 | + ) from exc |
| 103 | + |
| 104 | + device = "cuda" if torch.cuda.is_available() else "cpu" |
| 105 | + torch.manual_seed(args.seed) |
| 106 | + |
| 107 | + n_envs = max(1, args.frames_per_batch // args.max_steps) |
| 108 | + env = TransformedEnv( |
| 109 | + VmasEnv( |
| 110 | + scenario=args.scenario, |
| 111 | + num_envs=n_envs, |
| 112 | + continuous_actions=True, |
| 113 | + max_steps=args.max_steps, |
| 114 | + device=device, |
| 115 | + seed=args.seed, |
| 116 | + ), |
| 117 | + RewardSum( |
| 118 | + in_keys=[("next", "agents", "reward")], |
| 119 | + out_keys=[("agents", "episode_reward")], |
| 120 | + ) |
| 121 | + if False |
| 122 | + else RewardSum(in_keys=["reward"], out_keys=["episode_reward"]), |
| 123 | + ) |
| 124 | + |
| 125 | + actor = make_actor(env) |
| 126 | + centralised = args.algo == "mappo" |
| 127 | + critic = make_critic(env, centralized=centralised) |
| 128 | + |
| 129 | + LossCls = MAPPOLoss if args.algo == "mappo" else IPPOLoss |
| 130 | + value_norm = ValueNorm(shape=1, device=device) if args.algo == "mappo" else None |
| 131 | + loss_module = LossCls( |
| 132 | + actor_network=actor, |
| 133 | + critic_network=critic, |
| 134 | + value_norm=value_norm, |
| 135 | + clip_epsilon=0.2, |
| 136 | + entropy_coeff=0.01, |
| 137 | + ) |
| 138 | + loss_module.set_keys( |
| 139 | + value=("agents", "state_value"), |
| 140 | + action=env.action_key, |
| 141 | + reward=env.reward_key, |
| 142 | + ) |
| 143 | + |
| 144 | + collector = SyncDataCollector( |
| 145 | + env, |
| 146 | + actor, |
| 147 | + device=device, |
| 148 | + storing_device=device, |
| 149 | + frames_per_batch=args.frames_per_batch, |
| 150 | + total_frames=args.frames, |
| 151 | + ) |
| 152 | + |
| 153 | + replay_buffer = TensorDictReplayBuffer( |
| 154 | + storage=LazyTensorStorage(args.frames_per_batch, device=device), |
| 155 | + sampler=SamplerWithoutReplacement(), |
| 156 | + batch_size=args.minibatch_size, |
| 157 | + ) |
| 158 | + |
| 159 | + optim = torch.optim.Adam(loss_module.parameters(), lr=args.lr) |
| 160 | + |
| 161 | + total_frames = 0 |
| 162 | + start = time.time() |
| 163 | + for it, td in enumerate(collector): |
| 164 | + with torch.no_grad(): |
| 165 | + loss_module.value_estimator( |
| 166 | + td, |
| 167 | + params=loss_module.critic_network_params, |
| 168 | + target_params=loss_module.target_critic_network_params, |
| 169 | + ) |
| 170 | + replay_buffer.extend(td.reshape(-1)) |
| 171 | + total_frames += td.numel() |
| 172 | + |
| 173 | + for _ in range(args.epochs): |
| 174 | + for _ in range(args.frames_per_batch // args.minibatch_size): |
| 175 | + subdata = replay_buffer.sample() |
| 176 | + losses = loss_module(subdata) |
| 177 | + loss = ( |
| 178 | + losses["loss_objective"] |
| 179 | + + losses["loss_critic"] |
| 180 | + + losses["loss_entropy"] |
| 181 | + ) |
| 182 | + optim.zero_grad() |
| 183 | + loss.backward() |
| 184 | + torch.nn.utils.clip_grad_norm_(loss_module.parameters(), 1.0) |
| 185 | + optim.step() |
| 186 | + |
| 187 | + collector.update_policy_weights_() |
| 188 | + ep_reward = td.get(("next", "episode_reward")).mean().item() |
| 189 | + print( |
| 190 | + f"[{args.algo}] iter={it:03d} frames={total_frames:>7d} " |
| 191 | + f"reward={ep_reward:+.3f} elapsed={time.time() - start:5.1f}s" |
| 192 | + ) |
| 193 | + |
| 194 | + collector.shutdown() |
| 195 | + if not env.is_closed: |
| 196 | + env.close() |
| 197 | + |
| 198 | + |
| 199 | +def parse_args() -> argparse.Namespace: |
| 200 | + p = argparse.ArgumentParser() |
| 201 | + p.add_argument("--algo", choices=("mappo", "ippo"), default="mappo") |
| 202 | + p.add_argument("--scenario", default="navigation") |
| 203 | + p.add_argument("--frames", type=int, default=200_000) |
| 204 | + p.add_argument("--frames_per_batch", type=int, default=6_000) |
| 205 | + p.add_argument("--minibatch_size", type=int, default=400) |
| 206 | + p.add_argument("--max_steps", type=int, default=100) |
| 207 | + p.add_argument("--epochs", type=int, default=4) |
| 208 | + p.add_argument("--lr", type=float, default=3e-4) |
| 209 | + p.add_argument("--seed", type=int, default=0) |
| 210 | + return p.parse_args() |
| 211 | + |
| 212 | + |
| 213 | +if __name__ == "__main__": |
| 214 | + main(parse_args()) |
0 commit comments