From 21910d0f683999945ec7c1442f58ec5bae12e89b Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Wed, 15 Jul 2026 12:54:08 +0545 Subject: [PATCH 1/3] ci: add pytorch 2.12 and 2.13 to cpu tests --- .github/workflows/ci-tests-fabric.yml | 7 ++++++- .github/workflows/ci-tests-pytorch.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml index 8415d3b5c3f5f..36c50e9415c06 100644 --- a/.github/workflows/ci-tests-fabric.yml +++ b/.github/workflows/ci-tests-fabric.yml @@ -40,6 +40,7 @@ jobs: matrix: os: [macOS-14, ubuntu-22.04, windows-2022] config: + # TODO: follow-up - prune the oldest PyTorch minors (2.1-2.5) now that we test up to 2.13 # Test unified "lightning" package with PyTorch 2.1-2.5 - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" } - { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" } @@ -47,17 +48,21 @@ jobs: - { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" } - { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" } - # Test "fabric" package with PyTorch 2.6-2.11 + # Test "fabric" package with PyTorch 2.6-2.13 - { pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" } - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.7" } - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" } - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.9" } - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.10" } - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.11" } + - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.12" } + - { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.13" } - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.8" } - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.9" } - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.10" } - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.11" } + - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.12" } + - { pkg-name: "fabric", python-version: "3.13", pytorch-version: "2.13" } # Test minimum supported versions (oldest) - { pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" } diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index e76cfd72f3e55..b2cf3a2c6e393 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -44,6 +44,7 @@ jobs: matrix: os: [macOS-14, ubuntu-22.04, windows-2022] config: + # TODO: follow-up - prune the oldest PyTorch minors (2.1-2.5) now that we test up to 2.13 # Test unified "lightning" package with PyTorch 2.1-2.5 - { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" } - { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" } @@ -51,17 +52,21 @@ jobs: - { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" } - { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" } - # Test "pytorch" package with PyTorch 2.6-2.11 + # Test "pytorch" package with PyTorch 2.6-2.13 - { pkg-name: "pytorch", python-version: "3.12.7", pytorch-version: "2.6" } - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.7" } - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.8" } - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.9" } - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.10" } - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.11" } + - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.12" } + - { pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.13" } - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.8" } - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.9" } - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.10" } - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.11" } + - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.12" } + - { pkg-name: "pytorch", python-version: "3.13", pytorch-version: "2.13" } # Test minimum supported versions (oldest) - { pkg-name: "pytorch", pytorch-version: "2.1", requires: "oldest" } From 575df9741bb66c6d0642e8e01267818cda744c60 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Wed, 15 Jul 2026 13:05:09 +0545 Subject: [PATCH 2/3] test: avoid global privateuse1 backend pollution breaking CI on PyTorch 2.12/2.13 --- .../utilities/test_distributed.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/tests_fabric/utilities/test_distributed.py b/tests/tests_fabric/utilities/test_distributed.py index 532c7c6d19024..244bc7a88f577 100644 --- a/tests/tests_fabric/utilities/test_distributed.py +++ b/tests/tests_fabric/utilities/test_distributed.py @@ -245,22 +245,18 @@ def test_init_dist_connection_registers_destruction_handler(_, atexit_mock): atexit_mock.register.assert_not_called() -def test_get_default_process_group_backend_for_device(): +def test_get_default_process_group_backend_for_device(monkeypatch): """Test that each device type maps to its correct default process group backend.""" - # register a custom backend for test - torch.utils.rename_privateuse1_backend("pcu") - - def mock_backend(store, group_rank, group_size, timeout): - pass - - torch.distributed.Backend.register_backend( - "pccl", - lambda store, group_rank, group_size, timeout: mock_backend(store, group_rank, group_size, timeout), - devices=["pcu"], - ) + # register a custom backend for the test WITHOUT mutating global torch state: + # `rename_privateuse1_backend` is process-global and irreversible, so doing it here + # leaks a dangling "pcu" device into every subsequent test in the same worker, which + # breaks tensor-moving code paths on newer PyTorch. Patch the map instead. + backend_map = dict(torch.distributed.Backend.default_device_backend_map) + backend_map["pcu"] = "pccl" + monkeypatch.setattr(torch.distributed.Backend, "default_device_backend_map", backend_map) # test that the default backend is correctly set for each device - devices = [torch.device("cpu"), torch.device("cuda:0"), torch.device("pcu:0")] + devices = [torch.device("cpu"), torch.device("cuda:0"), Mock(type="pcu")] backends = ["gloo", "nccl", "pccl"] for device, backend in zip(devices, backends): assert _get_default_process_group_backend_for_device(device) == backend From 2a4b3ce17cd4bc137fe7707d407813edd182300d Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Wed, 15 Jul 2026 13:11:50 +0545 Subject: [PATCH 3/3] test: simplify with monkeypatch.setitem --- tests/tests_fabric/utilities/test_distributed.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/tests_fabric/utilities/test_distributed.py b/tests/tests_fabric/utilities/test_distributed.py index 244bc7a88f577..6f3b07f0ae420 100644 --- a/tests/tests_fabric/utilities/test_distributed.py +++ b/tests/tests_fabric/utilities/test_distributed.py @@ -247,13 +247,9 @@ def test_init_dist_connection_registers_destruction_handler(_, atexit_mock): def test_get_default_process_group_backend_for_device(monkeypatch): """Test that each device type maps to its correct default process group backend.""" - # register a custom backend for the test WITHOUT mutating global torch state: - # `rename_privateuse1_backend` is process-global and irreversible, so doing it here - # leaks a dangling "pcu" device into every subsequent test in the same worker, which - # breaks tensor-moving code paths on newer PyTorch. Patch the map instead. - backend_map = dict(torch.distributed.Backend.default_device_backend_map) - backend_map["pcu"] = "pccl" - monkeypatch.setattr(torch.distributed.Backend, "default_device_backend_map", backend_map) + # patch the map instead of registering a real backend: `rename_privateuse1_backend` is irreversible + # and would leak a dangling "pcu" device into subsequent tests + monkeypatch.setitem(torch.distributed.Backend.default_device_backend_map, "pcu", "pccl") # test that the default backend is correctly set for each device devices = [torch.device("cpu"), torch.device("cuda:0"), Mock(type="pcu")]