Skip to content

Commit 6a5135a

Browse files
authored
fix(litlogger): skip nan or inf when logging metrics (#57)
1 parent 2d8ae66 commit 6a5135a

8 files changed

Lines changed: 46 additions & 19 deletions

File tree

.github/workflows/ci-examples.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,59 +19,59 @@ jobs:
1919
matrix:
2020
include:
2121
- os: "ubuntu-latest"
22-
python-version: "3.10"
22+
python-version: "3.11"
2323
auth_mode: "guest"
2424
name: "standalone_usage.py"
2525
extra_packages: ""
2626
run: "uv run python examples/standalone_usage.py"
2727
- os: "ubuntu-latest"
28-
python-version: "3.10"
28+
python-version: "3.11"
2929
auth_mode: "user"
3030
name: "standalone_usage.py"
3131
extra_packages: ""
3232
run: "uv run python examples/standalone_usage.py"
3333
- os: "ubuntu-latest"
34-
python-version: "3.10"
34+
python-version: "3.11"
3535
auth_mode: "guest"
3636
name: "file_media_model_usage.py"
3737
extra_packages: ""
3838
run: "uv run python examples/file_media_model_usage.py"
3939
- os: "ubuntu-latest"
40-
python-version: "3.10"
40+
python-version: "3.11"
4141
auth_mode: "user"
4242
name: "file_media_model_usage.py"
4343
extra_packages: ""
4444
run: "uv run python examples/file_media_model_usage.py"
4545
- os: "ubuntu-latest"
46-
python-version: "3.10"
46+
python-version: "3.11"
4747
auth_mode: "guest"
4848
name: "lightning_autoencoder.py"
4949
extra_packages: ""
5050
run: "uv run python examples/lightning_autoencoder.py"
5151
- os: "ubuntu-latest"
52-
python-version: "3.10"
52+
python-version: "3.11"
5353
auth_mode: "user"
5454
name: "lightning_autoencoder.py"
5555
extra_packages: ""
5656
run: "uv run python examples/lightning_autoencoder.py"
5757
- os: "ubuntu-latest"
58-
python-version: "3.10"
58+
python-version: "3.11"
5959
auth_mode: "guest"
6060
name: "complete_workflow"
6161
extra_packages: ""
6262
run: |
6363
uv run python examples/complete_workflow/train.py
6464
uv run python examples/complete_workflow/fetch.py
6565
- os: "ubuntu-latest"
66-
python-version: "3.10"
66+
python-version: "3.11"
6767
auth_mode: "user"
6868
name: "complete_workflow"
6969
extra_packages: ""
7070
run: |
7171
uv run python examples/complete_workflow/train.py
7272
uv run python examples/complete_workflow/fetch.py
7373
- os: "ubuntu-latest"
74-
python-version: "3.10"
74+
python-version: "3.11"
7575
auth_mode: "guest"
7676
name: "litserve_inference.py"
7777
extra_packages: "litgpt litserve"

.github/workflows/ci-integrations.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ jobs:
1818
matrix:
1919
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
2020
auth_mode: ["user", "guest"]
21-
python-version: ["3.10", "3.11", "3.12", "3.13"]
21+
python-version: ["3.11", "3.12", "3.13"]
2222
requires: ["latest"]
2323
include:
24-
- python-version: "3.10"
24+
- python-version: "3.11"
2525
requires: "oldest"
2626
os: "ubuntu-latest"
2727
auth_mode: "guest"
28-
- python-version: "3.10"
28+
- python-version: "3.11"
2929
requires: "oldest"
3030
os: "ubuntu-latest"
3131
auth_mode: "user"

.github/workflows/ci-mypy.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
strategy:
2222
fail-fast: false
2323
matrix:
24-
python-version: ["3.10", "3.13"]
24+
python-version: ["3.11", "3.13"]
2525
steps:
2626
- uses: actions/checkout@v5
2727
- name: Install uv and setup python

.github/workflows/ci-unittests.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ jobs:
1919
fail-fast: false
2020
matrix:
2121
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
22-
python-version: ["3.10", "3.11", "3.12", "3.13"]
22+
python-version: ["3.11", "3.12", "3.13"]
2323
dependency: ["lightning"]
2424
requires: ["latest"]
2525
include:
26-
- { os: "ubuntu-latest", python-version: "3.10", dependency: "lightning", requires: "oldest" }
26+
- { os: "ubuntu-latest", python-version: "3.11", dependency: "lightning", requires: "oldest" }
2727
timeout-minutes: 20
2828
env:
2929
LIGHTNING_CLOUD_URL: https://staging.gridai.dev

.github/workflows/general_checks.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
testing-matrix: |
2929
{
3030
"os": ["ubuntu-22.04", "ubuntu-24.04", "macos-14"],
31-
"python-version": ["3.10", "3.11", "3.12", "3.13"],
31+
"python-version": ["3.11", "3.12", "3.13"],
3232
}
3333
3434
pre-commit:

pyproject.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ authors = [
1313
description = "The Lightning Logger of the Lightning.AI Platform"
1414
license = {file = "LICENSE"}
1515
readme = {file = "README.md", content-type = "text/markdown"}
16-
requires-python = ">=3.10"
16+
requires-python = ">=3.11"
1717
dynamic = ["version"]
1818
keywords = ["deep learning", "machine learning", "pytorch", "AI"]
1919
classifiers = [
@@ -22,7 +22,7 @@ classifiers = [
2222

2323
dependencies = [
2424
"blake3",
25-
"lightning-sdk >=2026.04.16",
25+
"lightning-sdk >=2026.05.12",
2626
"lightning-utilities<=0.15.3",
2727
"protobuf",
2828
"psutil",
@@ -72,7 +72,7 @@ where = ["src"]
7272
exclude = ["tests*", "examples*"]
7373

7474
[tool.mypy]
75-
python_version = "3.10"
75+
python_version = "3.11"
7676
strict = true
7777
show_error_codes = true
7878
pretty = true

src/litlogger/series.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
from __future__ import annotations
1717

18+
import math
19+
import warnings
1820
from typing import TYPE_CHECKING, Any, overload
1921

2022
from litlogger.media import File
@@ -63,6 +65,13 @@ def append(self, value: float | File, step: int | None = None) -> None:
6365
self._type = "metric"
6466
self._experiment._register_key_type(self._key, "metric")
6567
float_val = float(value)
68+
if math.isnan(float_val) or math.isinf(float_val):
69+
# FIXME: Remove this when NaN is handled correctly
70+
warnings.warn(
71+
f"Metric '{self._key}' = {float_val} is not a finite number. Skipping.",
72+
stacklevel=2,
73+
)
74+
return
6675
self._values.append(float_val)
6776
self._experiment._log_metric_value(self._key, float_val, step=step)
6877
else:

tests/unittests/test_series.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,3 +302,21 @@ def test_repr(self):
302302
series.extend([1.0, 2.0])
303303

304304
assert repr(series) == "[1.0, 2.0]"
305+
306+
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
307+
def test_append_non_finite_skipped_with_warning(self, value):
308+
"""Test that NaN, Inf, and -Inf values are skipped and subsequent finite appends still work."""
309+
exp = MagicMock(spec=Experiment)
310+
exp._key_types = {}
311+
series = Series(exp, "loss")
312+
313+
with pytest.warns(match="not a finite number"):
314+
series.append(value)
315+
316+
assert len(series) == 0
317+
exp._log_metric_value.assert_not_called()
318+
319+
# Finite value still works after skip
320+
series.append(0.5)
321+
assert len(series) == 1
322+
assert series[0] == 0.5

0 commit comments

Comments
 (0)