diff --git a/front/admin_ui/poetry.lock b/front/admin_ui/poetry.lock index 114c1de1cd..8b52bc0175 100644 --- a/front/admin_ui/poetry.lock +++ b/front/admin_ui/poetry.lock @@ -715,8 +715,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "dill" @@ -1523,7 +1523,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/jobs/cache_maintenance/poetry.lock b/jobs/cache_maintenance/poetry.lock index d51abc1a24..0edb5ef94d 100644 --- a/jobs/cache_maintenance/poetry.lock +++ b/jobs/cache_maintenance/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1203,7 +1203,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/jobs/mongodb_migration/poetry.lock b/jobs/mongodb_migration/poetry.lock index 697f796844..b6273199a3 100644 --- a/jobs/mongodb_migration/poetry.lock +++ b/jobs/mongodb_migration/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1203,7 +1203,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/libs/libapi/poetry.lock b/libs/libapi/poetry.lock index 1152510c09..ece7c48eac 100644 --- a/libs/libapi/poetry.lock +++ b/libs/libapi/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1222,7 +1222,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/libs/libcommon/poetry.lock b/libs/libcommon/poetry.lock index 6d320005fe..37c0d3821d 100644 --- a/libs/libcommon/poetry.lock +++ b/libs/libcommon/poetry.lock @@ -718,8 +718,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -4703,4 +4703,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "3.12.11" -content-hash = "f988acc662acc2821d3be909709005b3fed7051f0a0f13535c8fd374b16e945e" +content-hash = "b43330cecd391bb32c26183d83128051fe4bd87f42d2a4eac88226809d674fec" diff --git a/libs/libcommon/pyproject.toml b/libs/libcommon/pyproject.toml index c999a58184..32986fbe03 100644 --- a/libs/libcommon/pyproject.toml +++ b/libs/libcommon/pyproject.toml @@ -10,7 +10,7 @@ python = "3.12.11" anyio = ">=3.4.0,<5" appdirs = "^1.4.4" cryptography = "^43.0.1" -datasets = { git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01" } +datasets = { git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0" } duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/admin/poetry.lock b/services/admin/poetry.lock index f7a14f713b..98fa94cd1e 100644 --- a/services/admin/poetry.lock +++ b/services/admin/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1226,7 +1226,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/api/poetry.lock b/services/api/poetry.lock index 6e32247d36..519fb5027f 100644 --- a/services/api/poetry.lock +++ b/services/api/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1263,7 +1263,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/rows/poetry.lock b/services/rows/poetry.lock index 98325d1ae6..56a245e6a8 100644 --- a/services/rows/poetry.lock +++ b/services/rows/poetry.lock @@ -704,8 +704,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1284,7 +1284,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/search/poetry.lock b/services/search/poetry.lock index 9945acb4ff..f92333f422 100644 --- a/services/search/poetry.lock +++ b/services/search/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1263,7 +1263,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/sse-api/poetry.lock b/services/sse-api/poetry.lock index 7972bf2ba4..dcaabeae5f 100644 --- a/services/sse-api/poetry.lock +++ b/services/sse-api/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1294,7 +1294,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/webhook/poetry.lock b/services/webhook/poetry.lock index 4447493ef2..64c73d45f2 100644 --- a/services/webhook/poetry.lock +++ b/services/webhook/poetry.lock @@ -683,8 +683,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1263,7 +1263,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/worker/poetry.lock b/services/worker/poetry.lock index 2c6b95b216..10549d78b5 100644 --- a/services/worker/poetry.lock +++ b/services/worker/poetry.lock @@ -1026,8 +1026,8 @@ vision = ["Pillow (>=9.4.0)"] [package.source] type = "git" url = "https://github.com/huggingface/datasets" -reference = "73a880d7950d2611014c429132edad2863947c01" -resolved_reference = "73a880d7950d2611014c429132edad2863947c01" +reference = "2fe39d8ecb03684643a0417c55db490140f986f0" +resolved_reference = "2fe39d8ecb03684643a0417c55db490140f986f0" [[package]] name = "defusedxml" @@ -1700,7 +1700,7 @@ anyio = ">=3.4.0,<5" appdirs = "^1.4.4" async-lru = "^2.0.5" cryptography = "^43.0.1" -datasets = {git = "https://github.com/huggingface/datasets", rev = "73a880d7950d2611014c429132edad2863947c01"} +datasets = {git = "https://github.com/huggingface/datasets", rev = "2fe39d8ecb03684643a0417c55db490140f986f0"} duckdb = "^1.2.2" environs = "^14.3.0" filelock = "^3.18.0" diff --git a/services/worker/tests/fixtures/hub.py b/services/worker/tests/fixtures/hub.py index 681dd700b8..8991b0c5b0 100644 --- a/services/worker/tests/fixtures/hub.py +++ b/services/worker/tests/fixtures/hub.py @@ -477,12 +477,6 @@ def create_dataset_info_response_for_csv(dataset: str, config: str) -> Any: "dataset_name": dataset_name, "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 96, "num_examples": 4, "dataset_name": dataset_name}}, - "download_checksums": { - f"https://hub-ci.huggingface.co/datasets/{dataset}/resolve/__COMMIT__/dataset.csv": { - "num_bytes": 55, - "checksum": None, - } - }, "download_size": 55, "dataset_size": 96, "size_in_bytes": 151, diff --git a/services/worker/tests/job_runners/config/test_info.py b/services/worker/tests/job_runners/config/test_info.py index 42cc9d0067..d613e76899 100644 --- a/services/worker/tests/job_runners/config/test_info.py +++ b/services/worker/tests/job_runners/config/test_info.py @@ -59,24 +59,6 @@ def prepare_and_clean_mongo(app_config: AppConfig) -> None: "builder_name": "dataset_ok", "config_name": "config_1", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, - "download_checksums": { - "https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz": { - "num_bytes": 9912422, - "checksum": "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz": { - "num_bytes": 28881, - "checksum": "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz": { - "num_bytes": 1648877, - "checksum": "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz": { - "num_bytes": 4542, - "checksum": "f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6", - }, - }, "download_size": 11594722, "dataset_size": 20387232, "size_in_bytes": 31981954, @@ -112,12 +94,6 @@ def prepare_and_clean_mongo(app_config: AppConfig) -> None: "builder_name": "dataset_ok", "config_name": "config_2", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, - "download_checksums": { - "https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz": { - "num_bytes": 9912422, - "checksum": "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609", - }, - }, "download_size": 9912422, "dataset_size": 6912, "size_in_bytes": 9919334, diff --git a/services/worker/tests/job_runners/config/test_size.py b/services/worker/tests/job_runners/config/test_size.py index eeb195b84f..300b132612 100644 --- a/services/worker/tests/job_runners/config/test_size.py +++ b/services/worker/tests/job_runners/config/test_size.py @@ -102,24 +102,6 @@ def _get_job_runner( "dataset_name": "dataset_ok", }, }, - "download_checksums": { - "https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz": { - "num_bytes": 9912422, - "checksum": "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz": { - "num_bytes": 28881, - "checksum": "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz": { - "num_bytes": 1648877, - "checksum": "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6", - }, - "https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz": { - "num_bytes": 4542, - "checksum": "f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6", - }, - }, "download_size": 11594722, "dataset_size": 20387232, "size_in_bytes": 31981954,