diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61206032..955b5fcc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,38 @@ Although common, **mixed case is not allowed** in any code. "Mixed case" here me Run pre-commit at local before submitting merge request. You can also read [.pre-commit-config.yaml]( .pre-commit-config.yaml) to understand what are being forced. The **mypy** settings are inherited from PyTorch. +## Optimizer API + +`closure` is **not supported** on any optimizer's `step` in this repo. The parameter is kept on the signature for compatibility with `torch.optim.Optimizer`, but every `step` implementation raises `ValueError("closure is not supported")` when one is passed and returns `None` otherwise. New optimizers must do the same — do not add code paths that call `closure()` or return its result. + +### Argument-order convention + +When designing or modifying any public function in `emerging_optimizers/` — optimizer `__init__`, `calculate_*_update`, helpers — follow this precedence for argument order: + +1. **PyTorch native first.** Match `torch.optim` / `torch.optim._functional` signatures. For Adam-family signatures that means `betas` and `eps` adjacent (`Adam(lr, betas, eps, weight_decay, ...)`). +2. **HuggingFace next.** For knobs PyTorch doesn't expose, follow HF's order. The clearest case is `correct_bias`: HF's `transformers.AdamW` puts it after the standard Adam scalars (`lr, betas, eps, weight_decay`), so in our update functions `correct_bias` goes **after** `eps`. +3. **Repo-specific extras last.** Anything not in PyTorch or HF — `nesterov` on Adam, the scalar-int `step`, warmup schedulers, `alpha`, `scale_log2`, `use_shape_scaling`, etc. — goes at the end. + +Concrete shape for the scalar `calculate_*_update` family: + +```python +def calculate_*_update( + grad, # positional: tensors only + , + *, # everything below is keyword-only + betas (or momentum), + eps, + correct_bias, + [nesterov], + step, + , +): +``` + +`step` is a repo-specific argument and goes in the trailing bucket. (PyTorch's functional adam takes `state_steps` as a `List[Tensor]` grouped with the buffers — that is *not* the same as our scalar `step: int`, so the PyTorch precedent does not transfer.) + +The `*` keyword-only separator matches the convention used by every optimizer `__init__` in `emerging_optimizers/` (Muon, Soap, PSGDPro, AdaptiveMuon, etc.) and by `torch.optim._functional.adam` — only tensors are positional, scalars and flags must be passed by name. mypy will reject positional scalar passes, so the rule is enforced at type-check time. + ## Test All tests should be placed under [tests](tests). We aim for 100% test coverage for this tiny project. diff --git a/emerging_optimizers/orthogonalized_optimizers/adaptive_muon.py b/emerging_optimizers/orthogonalized_optimizers/adaptive_muon.py index e84854cd..b0f71669 100644 --- a/emerging_optimizers/orthogonalized_optimizers/adaptive_muon.py +++ b/emerging_optimizers/orthogonalized_optimizers/adaptive_muon.py @@ -274,12 +274,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - if closure is None: - loss = None - else: - loss = closure() + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: self._init_group(group) @@ -330,4 +328,4 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: p.add_(update, alpha=-group["lr"]) self.post_weight_update_fn_inplace(p) - return loss + return None diff --git a/emerging_optimizers/orthogonalized_optimizers/orthogonalized_optimizer.py b/emerging_optimizers/orthogonalized_optimizers/orthogonalized_optimizer.py index 00d8e46f..e6ddda50 100644 --- a/emerging_optimizers/orthogonalized_optimizers/orthogonalized_optimizer.py +++ b/emerging_optimizers/orthogonalized_optimizers/orthogonalized_optimizer.py @@ -158,12 +158,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - if closure is None: - loss = None - else: - loss = closure() + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: self._init_group(group) @@ -200,7 +198,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: p.add_(orth_grad, alpha=-group["lr"]) self.post_weight_update_fn_inplace(p) - return loss + return None def orthogonalize(self, p: torch.Tensor, grad: torch.Tensor, **kwargs: Any) -> torch.Tensor: """Orthogonalize the momentum. diff --git a/emerging_optimizers/psgd/psgd.py b/emerging_optimizers/psgd/psgd.py index 28058abc..61fbaed9 100644 --- a/emerging_optimizers/psgd/psgd.py +++ b/emerging_optimizers/psgd/psgd.py @@ -105,12 +105,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - if closure is None: - loss = None - else: - loss = closure() + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: for p in group["params"]: @@ -162,7 +160,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: # Apply weight update p.add_(update, alpha=-group["lr"]) - return loss + return None def _init_psgd_kron_states( diff --git a/emerging_optimizers/riemannian_optimizers/normalized_optimizer.py b/emerging_optimizers/riemannian_optimizers/normalized_optimizer.py index 7441d6ff..503005c6 100644 --- a/emerging_optimizers/riemannian_optimizers/normalized_optimizer.py +++ b/emerging_optimizers/riemannian_optimizers/normalized_optimizer.py @@ -58,13 +58,14 @@ def __init__( lr: float = 1e-3, momentum: float = 0.9, weight_decay: float = 0.0, + *, weight_decay_method: opt_mixin.WeightDecayT = "decoupled", dim: int = 0, eps: float = 1e-8, ) -> None: if lr < 0.0: raise ValueError(f"Invalid learning rate: {lr}") - if momentum < 0.0 or momentum >= 1.0: + if not 0.0 <= momentum < 1.0: raise ValueError(f"Invalid momentum value: {momentum}") if weight_decay < 0.0: raise ValueError(f"Invalid weight_decay value: {weight_decay}") @@ -93,9 +94,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - loss = closure() if closure is not None else None + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: lr = group["lr"] @@ -129,7 +131,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: # Retraction back to the manifold, the hyper-sphere torch.nn.functional.normalize(param, p=2.0, dim=dim, eps=eps, out=param) - return loss + return None @registry.register_optimizer("oblique_adam") @@ -147,6 +149,7 @@ def __init__( lr: float = 1e-3, betas: tuple[float, float] = (0.9, 0.99), weight_decay: float = 0.0, + *, weight_decay_method: opt_mixin.WeightDecayT = "decoupled", dim: int = 0, eps: float = 1e-8, @@ -165,9 +168,9 @@ def __init__( """ if lr < 0.0: raise ValueError(f"Invalid learning rate: {lr}") - if betas[0] < 0.0 or betas[0] >= 1.0: + if not 0.0 <= betas[0] < 1.0: raise ValueError(f"Invalid beta1 value: {betas[0]}") - if betas[1] < 0.0 or betas[1] >= 1.0: + if not 0.0 <= betas[1] < 1.0: raise ValueError(f"Invalid beta2 value: {betas[1]}") if weight_decay < 0.0: raise ValueError(f"Invalid weight_decay value: {weight_decay}") @@ -197,9 +200,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - loss = closure() if closure is not None else None + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: lr = group["lr"] @@ -256,7 +260,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: # Retraction back to the manifold, i.e. the hyper-sphere torch.nn.functional.normalize(param, p=2.0, dim=dim, eps=eps, out=param) - return loss + return None def _compute_riemannian_grad(param: torch.Tensor, grad_like: torch.Tensor, dim: int) -> torch.Tensor: diff --git a/emerging_optimizers/scalar_optimizers/laprop.py b/emerging_optimizers/scalar_optimizers/laprop.py index 16e41aa4..1bf72ee0 100644 --- a/emerging_optimizers/scalar_optimizers/laprop.py +++ b/emerging_optimizers/scalar_optimizers/laprop.py @@ -77,19 +77,20 @@ def __init__( betas: tuple[float, float] = (0.9, 0.999), eps: float = 1e-8, weight_decay: float = 0.0, + *, correct_bias: bool = True, frob_normalize: bool = False, weight_decay_method: WeightDecayT = "decoupled", ) -> None: - if not 0.0 <= lr: + if lr < 0.0: raise ValueError(f"Invalid learning rate: {lr}") if not 0.0 <= betas[0] < 1.0: raise ValueError(f"Invalid beta at index 0: {betas[0]}") if not 0.0 <= betas[1] < 1.0: raise ValueError(f"Invalid beta at index 1: {betas[1]}") - if not 0.0 <= eps: + if eps < 0.0: raise ValueError(f"Invalid epsilon value: {eps}") - if not 0.0 <= weight_decay: + if weight_decay < 0.0: raise ValueError(f"Invalid weight_decay value: {weight_decay}") if frob_normalize and weight_decay != 0.0: logging.warning("LaProp with frob_normalize=True is intended to be used with weight_decay=0.0.") @@ -139,15 +140,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: If you need the original gradient after this call, clone it beforehand. Args: - closure: A closure that reevaluates the model and returns the loss (optional). - - Returns: - The loss from the closure, if provided. + closure: Unsupported; must be ``None``. """ - if closure is None: - loss = None - else: - loss = closure() + if closure is not None: + raise ValueError("closure is not supported") for group in self.param_groups: self._init_group(group) @@ -173,14 +169,14 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: grad, state["exp_avg"], state["exp_avg_sq"], - correct_bias, - betas, - state["step"], - eps, + betas=betas, + eps=eps, + correct_bias=correct_bias, + step=state["step"], ) p.data.add_(update, alpha=-lr) if self.frob_normalize: assert pre_norm is not None p.data.mul_(pre_norm / p.data.norm().clamp_min(eps)) - return loss + return None diff --git a/emerging_optimizers/scalar_optimizers/lion.py b/emerging_optimizers/scalar_optimizers/lion.py index 20cdd700..ae2fb913 100644 --- a/emerging_optimizers/scalar_optimizers/lion.py +++ b/emerging_optimizers/scalar_optimizers/lion.py @@ -71,15 +71,16 @@ def __init__( lr: float = 1e-4, betas: tuple[float, float] = (0.9, 0.99), weight_decay: float = 0.01, + *, weight_decay_method: WeightDecayT = "decoupled", ) -> None: - if not 0.0 <= lr: + if lr < 0.0: raise ValueError(f"Invalid learning rate: {lr}") if not 0.0 <= betas[0] < 1.0: raise ValueError(f"Invalid beta at index 0: {betas[0]}") if not 0.0 <= betas[1] < 1.0: raise ValueError(f"Invalid beta at index 1: {betas[1]}") - if not 0.0 <= weight_decay: + if weight_decay < 0.0: raise ValueError(f"Invalid weight_decay value: {weight_decay}") defaults = dict(lr=lr, betas=betas, weight_decay=weight_decay) self.weight_decay_method = weight_decay_method @@ -124,15 +125,10 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: If you need the original gradient after this call, clone it beforehand. Args: - closure: A closure that reevaluates the model and returns the loss (optional). - - Returns: - The loss from the closure, if provided. + closure: Unsupported; must be ``None``. """ - loss = None if closure is not None: - with torch.enable_grad(): - loss = closure() + raise ValueError("closure is not supported") for group in self.param_groups: self._init_group(group) @@ -154,7 +150,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: # Lion update: sign(beta1 * m + (1-beta1) * g) # Note: different betas per param-group will each trigger a one-time # torch.compile recompilation of calculate_lion_update. - update = calculate_lion_update(grad, exp_avg, betas) + update = calculate_lion_update(grad, exp_avg, betas=betas) p.data.add_(update, alpha=-lr) - return loss + return None diff --git a/emerging_optimizers/scalar_optimizers/update_functions/adam.py b/emerging_optimizers/scalar_optimizers/update_functions/adam.py index 64571537..1161e5f6 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/adam.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/adam.py @@ -26,11 +26,12 @@ def calculate_adam_update( grad: torch.Tensor, exp_avg: torch.Tensor, exp_avg_sq: torch.Tensor, + *, betas: tuple[float, float], + eps: float, correct_bias: bool, nesterov: bool, step: int, - eps: float, ) -> torch.Tensor: """Performs the Adam update. @@ -47,16 +48,16 @@ def calculate_adam_update( Args: grad: The gradient tensor. - exp_avg: The accumulated first moment of the gradient. - exp_avg_sq: The accumulated second moment of the gradient. - betas: The EMA beta coefficients for the Adam update. - correct_bias: Whether to correct the bias of the Adam update. - nesterov: Whether to use nesterov momentum. - step: The current step of the optimizer, used to compute the bias correction terms. - eps: The epsilon for the Adam second moment update. + exp_avg: The accumulated first moment of the gradient (modified in place). + exp_avg_sq: The accumulated second moment of the gradient (modified in place). + betas: The EMA beta coefficients ``(beta1, beta2)``. + eps: Epsilon for the second-moment denominator. + correct_bias: Whether to apply Adam-style bias correction. + nesterov: Whether to use Nesterov momentum. + step: Current optimizer step (1-based), used for bias correction. Returns: - The Adam-update. + The Adam update. """ beta1, beta2 = betas diff --git a/emerging_optimizers/scalar_optimizers/update_functions/ademamix.py b/emerging_optimizers/scalar_optimizers/update_functions/ademamix.py index f72db163..5a94be70 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/ademamix.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/ademamix.py @@ -30,12 +30,13 @@ def calculate_ademamix_update( exp_avg_fast: torch.Tensor, exp_avg_slow: torch.Tensor, exp_avg_sq: torch.Tensor, - num_beta_slow_warmup_steps: int | None, - num_alpha_warmup_steps: int | None, + *, betas: tuple[float, float, float], - step: int, eps: float, correct_bias: bool, + step: int, + num_beta_slow_warmup_steps: int | None, + num_alpha_warmup_steps: int | None, alpha: float = 2, ) -> torch.Tensor: """Performs AdEMAMix update. @@ -56,16 +57,16 @@ def calculate_ademamix_update( Args: grad: The gradient tensor. - exp_avg_fast: The accumulated first moment of the gradient with fast time constant. - exp_avg_slow: The accumulated first moment of the gradient with slow time constant. - exp_avg_sq: The accumulated second moment of the gradient. - num_beta_slow_warmup_steps: Number of warmup steps used to increase beta_slow - num_alpha_warmup_steps: Number of warmup steps used to increase alpha - betas: The EMA beta coefficients for the Adam update. - step: The current step of the optimizer, used to compute the bias correction terms. - eps: The epsilon for the Adam second moment update. - correct_bias: Whether to correct the bias of the AdEMAMix update. - alpha: Coefficient for mixing the current gradient and EMA, the final value to use in case of scheduling. + exp_avg_fast: The accumulated first moment with the fast time constant (modified in place). + exp_avg_slow: The accumulated first moment with the slow time constant (modified in place). + exp_avg_sq: The accumulated second moment of the gradient (modified in place). + betas: The EMA beta coefficients ``(beta_fast, beta2, beta_slow_final)``. + eps: Epsilon for the second-moment denominator. + correct_bias: Whether to apply Adam-style bias correction. + step: Current optimizer step (1-based), used for bias correction and the schedulers below. + num_beta_slow_warmup_steps: Number of warmup steps used to ramp ``beta_slow`` toward ``beta_slow_final``. ``None`` disables the schedule. + num_alpha_warmup_steps: Number of warmup steps used to ramp ``alpha`` toward its final value. ``None`` disables the schedule. + alpha: Coefficient for mixing the slow first moment into the update. When scheduled, this is the final value. Returns: The AdEMAMix update. @@ -113,12 +114,13 @@ def calculate_sim_ademamix_update( grad: torch.Tensor, exp_avg: torch.Tensor, exp_avg_sq: torch.Tensor, - num_beta_fast_warmup_steps: int | None, - min_beta_fast: float, + *, betas: tuple[float, float], - step: int, eps: float, correct_bias: bool, + step: int, + num_beta_fast_warmup_steps: int | None, + min_beta_fast: float, alpha: float = 2, ) -> torch.Tensor: """Performs simplified AdEMAMix update. @@ -138,15 +140,15 @@ def calculate_sim_ademamix_update( Args: grad: The gradient tensor. - exp_avg: The accumulated first moment of the gradient. - exp_avg_sq: The accumulated second moment of the gradient. - num_beta_fast_warmup_steps: Number of warmup steps used to increase beta_fast - min_beta_fast: The minimum beta_fast value used at initialization - betas: The EMA beta coefficients for the Adam update. - step: The current step of the optimizer, used to compute the bias correction terms. - eps: The epsilon for the Adam second moment update. - correct_bias: Whether to correct the bias of the AdEMAMix update. - alpha: Coefficient for mixing the current gradient and EMA. + exp_avg: The accumulated first moment of the gradient (modified in place). + exp_avg_sq: The accumulated second moment of the gradient (modified in place). + betas: The EMA beta coefficients ``(beta_fast_final, beta2)``. + eps: Epsilon for the second-moment denominator. + correct_bias: Whether to apply Adam-style bias correction. + step: Current optimizer step (1-based), used for bias correction and the ``beta_fast`` schedule. + num_beta_fast_warmup_steps: Number of warmup steps used to ramp ``beta_fast`` from ``min_beta_fast`` toward ``beta_fast_final``. ``None`` disables the schedule. + min_beta_fast: Initial ``beta_fast`` value used at the start of the warmup schedule. + alpha: Coefficient for mixing the current gradient into the update. Returns: The simplified-AdEMAMix update. diff --git a/emerging_optimizers/scalar_optimizers/update_functions/laprop.py b/emerging_optimizers/scalar_optimizers/update_functions/laprop.py index 64402181..3d5957b3 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/laprop.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/laprop.py @@ -26,10 +26,11 @@ def calculate_laprop_update( grad: torch.Tensor, exp_avg: torch.Tensor, exp_avg_sq: torch.Tensor, - correct_bias: bool, + *, betas: tuple[float, float], - step: int, eps: float, + correct_bias: bool, + step: int, ) -> torch.Tensor: """Performs the LAProp/Normalized SGD with momentum update. @@ -49,15 +50,15 @@ def calculate_laprop_update( Args: grad: The gradient tensor. - exp_avg: The exponential moving average of the gradient. - exp_avg_sq: The exponential moving average of the gradient squared. - correct_bias: Whether to correct the bias of the Adam update. - betas: The betas for the exponential moving average. - step: The current step. - eps: The epsilon for the second moment update. + exp_avg: The accumulated first moment of the gradient (modified in place). + exp_avg_sq: The accumulated second moment of the gradient (modified in place). + betas: The EMA beta coefficients ``(beta1, beta2)``. + eps: Epsilon for the second-moment denominator. + correct_bias: Whether to apply Adam-style bias correction. + step: Current optimizer step (1-based), used for bias correction. Returns: - The LAProp update. + The LaProp update. """ beta1, beta2 = betas diff --git a/emerging_optimizers/scalar_optimizers/update_functions/lion.py b/emerging_optimizers/scalar_optimizers/update_functions/lion.py index ac88627a..3a15e53c 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/lion.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/lion.py @@ -25,6 +25,7 @@ def calculate_lion_update( grad: torch.Tensor, exp_avg: torch.Tensor, + *, betas: tuple[float, float], ) -> torch.Tensor: """Performs the Lion update. @@ -39,8 +40,8 @@ def calculate_lion_update( Args: grad: The gradient tensor. - exp_avg: The accumulated first moment of the gradient. - betas: The EMA beta coefficients (beta1, beta2) for the Lion update. + exp_avg: The accumulated first moment of the gradient (modified in place). + betas: The EMA beta coefficients ``(beta1, beta2)``. ``beta1`` controls the sign-update interpolation; ``beta2`` controls the momentum EMA. Returns: The Lion update. diff --git a/emerging_optimizers/scalar_optimizers/update_functions/madam.py b/emerging_optimizers/scalar_optimizers/update_functions/madam.py index 2f18d20f..78d5ab08 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/madam.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/madam.py @@ -26,6 +26,7 @@ def calculate_madam_update( grad: torch.Tensor, exp_avg: torch.Tensor, exp_avg_sq_scaled: torch.Tensor, + *, betas: tuple[float, float], correct_bias: bool, step: int, @@ -83,15 +84,15 @@ def calculate_madam_update( ``s * EMA(g_t^2)`` (modified in place). Allocate as ``zeros_like(p)``; the caller is responsible for using the same ``scale`` across steps. betas: The EMA beta coefficients ``(beta1, beta2)``. - correct_bias: Whether to apply Adam's bias correction. - step: Current optimizer step (1-indexed), used for bias correction. + correct_bias: Whether to apply Adam-style bias correction. + step: Current optimizer step (1-based), used for bias correction. scale_log2: ``log2`` of the magnitude scaling factor ``s = 2 ** scale_log2`` used for the second-moment storage. When it is an even integer, ``sqrt(s) = 2 ** (scale_log2 // 2)`` is exactly representable in floating point. Returns: - The MAdam update tensor. + The MAdam update. """ beta1, beta2 = betas diff --git a/emerging_optimizers/scalar_optimizers/update_functions/signum.py b/emerging_optimizers/scalar_optimizers/update_functions/signum.py index 697b8704..c9193471 100644 --- a/emerging_optimizers/scalar_optimizers/update_functions/signum.py +++ b/emerging_optimizers/scalar_optimizers/update_functions/signum.py @@ -25,6 +25,7 @@ def calculate_signum_update( grad: torch.Tensor, exp_avg: torch.Tensor, + *, momentum: float, correct_bias: bool, nesterov: bool, @@ -48,12 +49,12 @@ def calculate_signum_update( Args: grad: The gradient tensor. - exp_avg: The accumulated first moment of the gradient. - momentum: The EMA beta coefficients for the momentum update. - correct_bias: Whether to correct the bias of the momentum update. - nesterov: Whether to use nesterov momentum. - step: The current step of the optimizer, used to compute the bias correction terms. - use_shape_scaling: Whether to scale the update by the shape of the tensor. + exp_avg: The accumulated first moment of the gradient (modified in place). + momentum: The EMA decay coefficient for the momentum buffer (single scalar). + correct_bias: Whether to apply Adam-style bias correction. + nesterov: Whether to use Nesterov momentum. + step: Current optimizer step (1-based), used for bias correction. + use_shape_scaling: Whether to scale the update by ``2 / (m + n)`` for an ``(m, n)`` tensor. Returns: The sign-SGD/Signum update. diff --git a/emerging_optimizers/soap/soap.py b/emerging_optimizers/soap/soap.py index 015a7358..f8b63679 100644 --- a/emerging_optimizers/soap/soap.py +++ b/emerging_optimizers/soap/soap.py @@ -171,12 +171,11 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step. Args: - closure: A closure that reevaluates the model and returns the loss. + closure: Unsupported; must be ``None``. """ - if closure is None: - loss = None - else: - loss = closure() + if closure is not None: + raise ValueError("closure is not supported") + for group in self.param_groups: self._init_group(group) @@ -271,11 +270,11 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: grad_projected, state["exp_avg"], state["exp_avg_sq"], - group["betas"], - self.correct_bias, - self.nesterov, - curr_iter_1_based, # 1-based iteration index is used for bias correction - group["eps"], + betas=group["betas"], + eps=group["eps"], + correct_bias=self.correct_bias, + nesterov=self.nesterov, + step=curr_iter_1_based, # 1-based iteration index is used for bias correction ) # Projecting back the preconditioned (by ADAM) exponential moving average of gradients @@ -295,7 +294,7 @@ def step(self, closure: Callable[[], float] | None = None) -> float | None: for stream in self.stream_list: current_stream.wait_stream(stream) - return loss + return None @torch.no_grad() # type: ignore[misc] diff --git a/tests/test_laprop.py b/tests/test_laprop.py index 883465f8..3c874f79 100644 --- a/tests/test_laprop.py +++ b/tests/test_laprop.py @@ -91,7 +91,9 @@ def test_optimizer_step_matches_update_function(self, shape) -> None: old_param = param.detach().clone() exp_avg = torch.zeros_like(param) exp_avg_sq = torch.zeros_like(param) - expected_update = calculate_laprop_update(grad, exp_avg, exp_avg_sq, True, betas, 1, eps) + expected_update = calculate_laprop_update( + grad, exp_avg, exp_avg_sq, betas=betas, eps=eps, correct_bias=True, step=1 + ) param.grad = grad.clone() optimizer.step() diff --git a/tests/test_scalar_optimizers.py b/tests/test_scalar_optimizers.py index 9e544427..8148bc6c 100644 --- a/tests/test_scalar_optimizers.py +++ b/tests/test_scalar_optimizers.py @@ -59,11 +59,11 @@ def test_calculate_adam_update_simple(self, shape) -> None: grad, exp_avg_for_manual_calc, exp_avg_sq_for_manual_calc, - betas, + betas=betas, + eps=eps, correct_bias=True, nesterov=nesterov, step=step, - eps=eps, ) initial_param_val_tensor = torch.full(shape, 10.0, device=self.device) @@ -117,10 +117,10 @@ def test_calculate_laprop_update_with_zero_momentum_equals_rmsprop(self) -> None grad, exp_avg_for_laprop, exp_avg_sq_for_laprop, - correct_bias, - betas, - step, - eps, + betas=betas, + eps=eps, + correct_bias=correct_bias, + step=step, ) # Manually verify with RMSProp logic @@ -178,12 +178,12 @@ def test_calculate_ademamix_update_with_alpha_zero_equals_adam( exp_avg_fast_for_ademamix, exp_avg_slow_for_ademamix, exp_avg_sq_for_ademamix, - num_beta_slow_warmup_steps=num_beta_slow_warmup_steps, - num_alpha_warmup_steps=None, betas=betas, - step=step, eps=eps, correct_bias=correct_bias, + step=step, + num_beta_slow_warmup_steps=num_beta_slow_warmup_steps, + num_alpha_warmup_steps=None, alpha=0.0, ) @@ -194,11 +194,11 @@ def test_calculate_ademamix_update_with_alpha_zero_equals_adam( grad, exp_avg_for_adam, exp_avg_sq_for_adam, - (betas[0], betas[1]), + betas=(betas[0], betas[1]), + eps=eps, correct_bias=correct_bias, nesterov=False, step=step, - eps=eps, ) torch.testing.assert_close(ademamix_update, adam_update, atol=1e-6, rtol=1e-6) @@ -221,12 +221,12 @@ def test_calculate_sim_ademamix_update_with_zero_momentum_and_alpha_equals_rmspr grad, exp_avg_for_sim_ademamix, exp_avg_sq_for_sim_ademamix, - num_beta_fast_warmup_steps=None, - min_beta_fast=0.0, betas=betas, - step=step, eps=eps, correct_bias=correct_bias, + step=step, + num_beta_fast_warmup_steps=None, + min_beta_fast=0.0, alpha=0.0, ) @@ -390,10 +390,10 @@ def test_calculate_madam_update_matches_adam_eps_zero(self, scale_log2: int, cor exp_avg_adam, exp_avg_sq_adam, betas=betas, + eps=0.0, correct_bias=correct_bias, nesterov=False, step=step, - eps=0.0, ) case = f"scale_log2={scale_log2}, correct_bias={correct_bias}"