diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index af9d08cc..8dafa022 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -62,14 +62,14 @@ jobs: pytest tests \ --ignore=tests/processing \ --ignore=tests/raw \ - -n 2 --cov=litdata --durations=0 --timeout=120 --capture=no --verbose + -n 2 --cov=litdata --durations=0 --timeout=150 --capture=no --verbose - name: Run processing tests sequentially run: | # note that the listed test should match ignored in the previous step pytest \ tests/processing tests/raw \ - --cov=litdata --cov-append --durations=0 --timeout=120 --capture=no --verbose + --cov=litdata --cov-append --durations=0 --timeout=150 --capture=no --verbose - name: Statistics continue-on-error: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 725015af..8cdecce8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: - id: detect-private-key - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell additional_dependencies: [tomli] @@ -55,14 +55,14 @@ repos: #args: ["--write-changes"] # uncomment if you want to get automatic fixing - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: v0.15.9 hooks: - id: ruff args: ["--fix"] - id: ruff-format - repo: https://github.com/executablebooks/mdformat - rev: 0.7.22 + rev: 1.0.0 hooks: - id: mdformat additional_dependencies: @@ -76,8 +76,8 @@ repos: README.md )$ - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + - repo: https://github.com/JoC0de/pre-commit-prettier + rev: v3.8.1 hooks: - id: prettier files: \.(json|yml|yaml|toml) @@ -85,11 +85,11 @@ repos: args: ["--print-width=120"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.8.0 + rev: v2.8.0 # pinned to avoid timeout issues in CI hooks: - id: pyproject-fmt additional_dependencies: [tox] - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.24.1 + rev: v0.25 hooks: - id: validate-pyproject diff --git a/tests/streaming/test_dataset.py b/tests/streaming/test_dataset.py index 15a5babd..953c934b 100644 --- a/tests/streaming/test_dataset.py +++ b/tests/streaming/test_dataset.py @@ -544,7 +544,7 @@ def test_dataset_cache_recreation(tmpdir): assert dataset.shuffler is shuffler # shuffler gets reused -@pytest.mark.timeout(30) +@pytest.mark.timeout(45) def test_len_called_before_dataloader_drop_last(tmpdir): cache = Cache(str(tmpdir), chunk_size=10) for i in range(100): @@ -1535,16 +1535,15 @@ def test_dataset_with_mosaic_mds_data(tmpdir): dataset = StreamingDataset(input_dir=str(tmpdir)) dataloader = DataLoader(dataset, batch_size=4, drop_last=True) - i = 0 - for batch in dataloader: + + for i, batch in enumerate(dataloader): assert len(batch["class"]) == 4 assert len(batch["image"]) == 4 assert list(batch["class"]) == [4 * i, 4 * i + 1, 4 * i + 2, 4 * i + 3] - i += 1 dataloader = DataLoader(dataset, batch_size=4, drop_last=False) - i = 0 - for batch in dataloader: + + for i, batch in enumerate(dataloader): if i == 2: # last batch is smaller than batch_size assert len(batch["class"]) == 2 @@ -1554,7 +1553,6 @@ def test_dataset_with_mosaic_mds_data(tmpdir): assert len(batch["class"]) == 4 assert len(batch["image"]) == 4 assert list(batch["class"]) == [4 * i, 4 * i + 1, 4 * i + 2, 4 * i + 3] - i += 1 @pytest.mark.parametrize("shuffle", [True, False])