From e99bd607837569e77aec5d3a8ecaf6afcf20c2c6 Mon Sep 17 00:00:00 2001 From: Felipe Velasquez Date: Wed, 6 May 2026 13:17:14 -0500 Subject: [PATCH 1/2] Default S3Client to boto3 and refactor S3 tests for the new default Flip `S3Client = S3ClientBoto3` (was boto1) and revert the `client=` parameter on `S3PathTask` / `S3EmrTask` / `S3FlagTask` (now redundant). Drop the boto1/boto3 dual-mode shim from `s3_test.py`; add boto3 ports of the SSE / credential / STS tests, boto1-gated test classes, and a `MOTO_LT_2` skip for boto3 round-trips moto<2 corrupts via raw chunked Transfer-Encoding. Document the new default and compatibility matrix. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 46 ++++ PLAN_Py39_P312_DUAL_COMPATIBILITY.md | 97 ++++--- luigi/contrib/s3.py | 307 ++++++++++----------- test/contrib/s3_test.py | 390 +++++++++++++++++++++------ 4 files changed, 558 insertions(+), 282 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e0041b1a2f..3900be1c8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -166,3 +166,49 @@ twine upload --config-file .pypirc -r pypi-local dist/* - boto3 tests require AWS region configuration or proper mocking - SQLAlchemy tests need eager loading for relationships to avoid DetachedInstanceError - Process-related tests may need small delays for `/proc` filesystem to be ready + +## S3 Module: boto3-Only Default + Boto1 Legacy Shim + +`luigi/contrib/s3.py` defaults to boto3: `S3Client = S3ClientBoto3` (and `ReadableS3File = ReadableS3FileBoto3`). The legacy `S3ClientBoto1` class is still defined and importable for callers that need it explicitly, but boto1 is no longer a runtime dependency for the default path. + +`S3PathTask`, `S3EmrTask`, and `S3FlagTask` do **not** accept a `client=` constructor argument — they always use the module default. Callers needing a non-default client (e.g. region-aware boto3) should subclass and override `output()`. (A `client=` parameter was briefly added in commit `05c71137` while the default was still boto1; it was reverted on May 6 2026 once the default flipped to boto3 made it redundant.) + +### Running S3 Tests with uv + +`test/contrib/s3_test.py` is structured for one ephemeral env per Python+moto+boto combo. Use `uv run --no-project --with-editable .` and pin the moto/boto versions you want to validate: + +```bash +# Modern stack — py3.12 + moto5 + boto3 (recommended) +PYTHONPATH=test uv run --python 3.12 --no-project --with-editable . \ + --with pytest --with sqlalchemy --with mock --with hypothesis --with pygments \ + --with 'moto>=5,<6' --with boto3 \ + python -m pytest test/contrib/s3_test.py -q --override-ini addopts='' + +# Legacy stack — py3.9 + moto1 + boto1 + boto3 +arch -x86_64 env PYTHONPATH=test uv run --python 3.9 --no-project --with-editable . \ + --with pytest --with sqlalchemy --with mock --with hypothesis --with pygments \ + --with 'moto==1.3.16' --with boto3 --with boto \ + python -m pytest test/contrib/s3_test.py -q --override-ini addopts='' +``` + +### Compatibility matrix + +| Python | moto | boto1 | boto3 | Result | +| --- | --- | --- | --- | --- | +| 3.12 | 1.x | yes | yes | **Broken**: moto1 calls `ssl.wrap_socket` (removed in 3.12); boto1's vendored `six` also fails to import. | +| 3.12 | ≥5 | — | yes | **60 passed, 13 skipped** (boto1 tests skip cleanly). | +| 3.9 | 1.x | yes | yes | **33 passed, 40 skipped** — boto1 tests run; boto3 round-trip tests skip under `MOTO_LT_2` because moto<2 mishandles boto3 chunked Transfer-Encoding. | +| 3.9 | 1.x | — | yes | Identical to row above — `moto==1.3.16` declares `boto` as a hard runtime dep, so boto1 is always installed transitively. | +| 3.9 | ≥5 | — | yes | **60 passed, 13 skipped**. | + +### Test gating flags + +Defined at the top of `test/contrib/s3_test.py`: + +- `MOTO_LT_2` — true if `moto.__version__` is `<2`. Skips the boto3 round-trip tests (multipart, copy, `test_get`, `test_get_as_string`, the whole `TestS3Target` class) because moto<2 corrupts uploads with raw chunk-size markers. +- `BOTO1_AVAILABLE` — true only if both `boto<3` AND moto's `mock_s3_deprecated`/`mock_sts_deprecated` are importable (the latter exists only in moto<2). Gates `TestS3TargetBoto1` and `TestS3ClientBoto1`. + +### macOS / Apple Silicon notes + +- Python 3.9 builds available locally are x86_64 only (pyenv 3.9.18, CommandLineTools 3.9.6, uv-managed 3.9.x). On arm64 hardware, prefix py3.9 invocations with `arch -x86_64` to load the matching x86_64 wheels via Rosetta. Without it, `cryptography`'s `_cffi_backend.so` fails to load with `incompatible architecture (have 'x86_64', need 'arm64')`. +- Python 3.12 runs natively in either arch; no prefix needed. diff --git a/PLAN_Py39_P312_DUAL_COMPATIBILITY.md b/PLAN_Py39_P312_DUAL_COMPATIBILITY.md index ea2c7892a0..499603cfd5 100644 --- a/PLAN_Py39_P312_DUAL_COMPATIBILITY.md +++ b/PLAN_Py39_P312_DUAL_COMPATIBILITY.md @@ -226,12 +226,13 @@ A module-level `USE_BOTO3` flag was considered but rejected: since both `boto` a  **4\. Add backwards-compatible aliases** at the bottom of the module: - **5\. Add** `**client**` **parameter to** `**S3PathTask**`**,** `**S3EmrTask**`**,** `**S3FlagTask**` -These `ExternalTask` subclasses currently hardcode no client in `output()`, always falling -back to the default. Add a `client` parameter (defaulting to `None`) that is forwarded to -the target constructor. `None` means "use the module default", which after step 4 resolves -to `S3ClientBoto1` — preserving existing behaviour exactly. To use boto3, the caller passes -`client=S3ClientBoto3()` explicitly. + **5\. Add** `**client**` **parameter to** `**S3PathTask**`**,** `**S3EmrTask**`**,** `**S3FlagTask**` — *Reverted May 6 2026; see the May 6 section below. Once the default flipped to boto3 in step 4, the injection plumbing became redundant for boto3 compatibility and these task classes were restored to their pre-`05c71137` form. Original April 13 rationale preserved verbatim below for historical context:* + +> These `ExternalTask` subclasses currently hardcode no client in `output()`, always falling +> back to the default. Add a `client` parameter (defaulting to `None`) that is forwarded to +> the target constructor. `None` means "use the module default", which after step 4 resolves +> to `S3ClientBoto1` — preserving existing behaviour exactly. To use boto3, the caller passes +> `client=S3ClientBoto3()` explicitly. Apply the same pattern to `S3EmrTask` and `S3FlagTask` (forwarding `client` and `flag` where applicable). @@ -247,21 +248,21 @@ forward `self._client` to that `S3Target` call as well. ### Key Design Note -**Existing behaviour is preserved exactly.** Any code that currently works without passing a `client` continues to use boto1 — nothing changes for existing users. +**As of May 2026 the default has flipped to boto3** — see the May 6 2026 section below. The original April 13 plan kept `S3Client = S3ClientBoto1` to preserve existing behavior; that decision was reversed once the upgrade target was clarified to "boto3-only by default, boto1 still importable for legacy paths". -boto3 is strictly opt-in via explicit injection: +Original (April 13) injection model still applies, just with the default swapped: ```python -# existing code — unchanged, still uses boto1 +# default — now boto3 from luigi.contrib.s3 import S3Target target = S3Target("s3://bucket/key") -# new code opting in to boto3 — explicit at the call site -from luigi.contrib.s3 import S3ClientBoto3, S3Target -target = S3Target("s3://bucket/key", client=S3ClientBoto3()) +# explicit boto1 (legacy paths only — requires `boto<3` installed) +from luigi.contrib.s3 import S3ClientBoto1, S3Target +target = S3Target("s3://bucket/key", client=S3ClientBoto1()) ``` -`S3Client` remains as an alias for `S3ClientBoto1` so that existing imports of `S3Client` continue to work without modification. +`S3Client` remains as an alias — but now points to `S3ClientBoto3`. Code that imports `S3Client` keeps working; its underlying backend is now boto3. --- @@ -389,11 +390,7 @@ class S3ClientWithRegion(S3ClientBoto3): `S3PathTaskWithRegion.output()` is unchanged — it still passes `S3ClientWithRegion(region_name=self.region_name)` to `S3Target`. -Once we add a `client` parameter to `S3PathTask` (checklist step 5), simple cases that do not need a custom region can also just pass `client=S3ClientBoto3()` at instantiation time rather than subclassing: - -```python -S3PathTask(path=some_path, client=S3ClientBoto3()) -``` +*(May 6 2026 update — checklist step 5 was reverted, so `S3PathTask` no longer accepts `client=`. Simple cases get boto3 automatically via the new module default. Callers needing a non-default client must subclass and override `output()` as `S3PathTaskWithRegion` does above.)* --- @@ -407,18 +404,7 @@ class NamedS3FlagTask(S3FlagTask): name = luigi.Parameter("A name describing the task instance") ``` -**No structural change needed.** Once we add `client` to `S3FlagTask` (checklist step 5), callers can opt in to boto3 at instantiation: - -```python -from luigi.contrib.s3 import S3ClientBoto3 - -NamedS3FlagTask( - name="chrono-user-views-all-offers", - path="s3://bucket/prefix/", - flag="_SUCCESS", - client=S3ClientBoto3(), # new — opt-in to boto3 -) -``` +**No structural change needed.** With the May 6 2026 default flip, `NamedS3FlagTask(...)` already uses boto3 via the module-level `S3Client = S3ClientBoto3` alias — no opt-in required. (Checklist step 5 was reverted: `S3FlagTask` does not accept a `client=` constructor argument. Callers needing a non-default client should subclass and override `output()` per Pattern 3.) --- @@ -429,7 +415,7 @@ NamedS3FlagTask( | No client (default boto1) | ~850 | Add `client=S3ClientBoto3()` at each call site | | `AffirmS3Client` injected | ~50 | Swap `AffirmS3Client` → `S3ClientBoto3` at injection site; audit any custom methods | | `S3PathTaskWithRegion` | ~20 | In `S3ClientWithRegion`: change base class to `S3ClientBoto3`, remove boto1 `s3` property override, forward `region_name` via `super().__init__()` kwargs; `S3PathTaskWithRegion.output()` unchanged | -| `NamedS3FlagTask` | ~10 | Pass `client=S3ClientBoto3()` at instantiation once step 5 is complete | +| `NamedS3FlagTask` | ~10 | None — picks up boto3 automatically from the module default after May 6 2026 | All changes are localised to the call site. No structural refactoring is required. @@ -456,8 +442,47 @@ class S3PathTask(ExternalTask): ``` ```python -# S3Client keeps the existing default — boto1, same as today. -# To use boto3, pass client=S3ClientBoto3() explicitly at the call site. -S3Client = S3ClientBoto1 -ReadableS3File = ReadableS3FileBoto1 -``` \ No newline at end of file +# Default flipped to boto3 in May 2026 (BATCH-3679 boto3-only upgrade). +# To use boto1, pass client=S3ClientBoto1() explicitly at the call site +# (and ensure `boto<3` is installed — boto1 does not import on Py3.12). +S3Client = S3ClientBoto3 +ReadableS3File = ReadableS3FileBoto3 +``` + +--- + +## May 6 2026 — Boto3-Only Default + Test Refactor + +### S3 client default flipped to boto3 + +`luigi/contrib/s3.py` now aliases `S3Client = S3ClientBoto3` (was `S3ClientBoto1`). Same for `ReadableS3File`. `S3ClientBoto1` is still defined and importable so legacy callers can opt back in explicitly, but the default backend for unparameterized `S3Target(...)`, `S3PathTask(...)`, etc. is now boto3. + +`S3ClientBoto3.__init__` was hardened to fail fast (`import boto3 # noqa: F401`) so that environments missing boto3 surface the issue at construction rather than at first method call. + +### `client=` constructor argument reverted on `S3PathTask` / `S3EmrTask` / `S3FlagTask` + +Commit `05c71137` (April 13 plan, step 5) added a `client=None` constructor parameter to `S3PathTask`, `S3EmrTask`, and `S3FlagTask` so callers could inject `S3ClientBoto3()` while the module default was still boto1. With the default now flipped to boto3, that injection plumbing is no longer needed for boto3 compatibility — these classes are back to inheriting only `path` (and `flag` for `S3FlagTask`) and a one-line `output()` that constructs the target with the module default. Callers that still need a *non-default* client (e.g. region-aware boto3, or explicit boto1) should subclass and override `output()`, matching how `S3PathTaskWithRegion` works in `all-the-things` (Migration Guide, Pattern 3). + +### `test/contrib/s3_test.py` rewritten + +The file previously carried a dual-mode `try/except ImportError` shim (`HAS_BOTO`, boto1-as-fallback assignment, conditional `_create_bucket`, ~12 `@unittest.skipIf(not HAS_BOTO, ...)` decorators on boto1-only behaviors). With the default now being boto3 unconditionally, that shim was incorrect: when boto1 *is* installed `HAS_BOTO=True` would route boto1-style calls (`client.s3.create_bucket('mybucket')`) into a boto3 client. + +The refactor: + +* **Removed** the dual-mode shim entirely — `boto3`, `botocore.exceptions.ClientError`, and `S3Client` are imported unconditionally; `_create_bucket` always uses `boto3.resource('s3', region_name='us-east-1').create_bucket(Bucket=...)`. +* **Added boto3 versions** of the previously-skipped tests, using boto3 idioms: `ServerSideEncryption='AES256'` instead of `encrypt_key=True`; credentials introspected via `s3.meta.client._request_signer._credentials` instead of boto1's `s3.access_key`/`s3.gs_access_key_id`; STS-assumed-role assertions check shape (`ASIA`-prefixed key + token populated) rather than the canonical example values that newer moto no longer returns. +* **Added boto1-gated test classes** — `TestS3TargetBoto1` and `TestS3ClientBoto1` exercise `S3ClientBoto1`/`ReadableS3FileBoto1` directly (basic put/get round-trip, the original `boto.s3.key.Key.BufferSize` line-buffering test, boto1 credential introspection, all four `encrypt_key=True` multipart variants). Gated on `BOTO1_AVAILABLE = HAS_BOTO_1 and HAS_BOTO_1_MOTO` — both `boto<3` and `moto<2`'s `mock_s3_deprecated`/`mock_sts_deprecated` decorators must be importable. Skip cleanly otherwise. +* **Added bonus deprecation tests** — `test_put_encrypt_key_raises`, `test_put_string_encrypt_key_raises`, `test_put_multipart_encrypt_key_raises` verify `S3ClientBoto3` actively rejects the boto1 `encrypt_key=` parameter with `DeprecatedBotoClientException`. +* **Added a `MOTO_LT_2` skip gate** — `moto<2` doesn't decode boto3's `Transfer-Encoding: chunked` upload bodies, so any round-trip through `put_multipart` / `upload_fileobj` / `copy` stores chunk-size markers (`b"4f\n...real-data...\n0\n"`) instead of real content. The class-level `@unittest.skipIf(MOTO_LT_2, ...)` on `TestS3Target` plus `self.skipTest(...)` guards inside `_run_multipart_test` / `_run_copy_test` / `_run_multipart_copy_test`, plus `@unittest.skipIf` on `test_get` / `test_get_as_string`, route around this without false failures. + +### Test matrix validated under uv + +| Scenario | Result | +| --- | --- | +| py3.12 + moto1 + boto + boto3 | Broken environment — moto1's vendored httpretty calls `ssl.wrap_socket` (removed in py3.12) and boto1's vendored `six.moves` is incompatible with py3.12. Cannot run; documented for posterity. | +| py3.12 + moto5 + boto3 | **60 passed, 13 skipped, 0 failed** | +| py3.9 + moto1 + boto + boto3 | **33 passed, 40 skipped, 0 failed** (boto3 round-trip path skipped under MOTO_LT_2) | +| py3.9 + moto1 + boto3 (no boto) | Same as above — moto1 declares `boto` as a hard runtime dep, so boto1 is always installed transitively. Configuration is empirically unachievable. | +| py3.9 + moto5 + boto3 | **60 passed, 13 skipped, 0 failed** | + +Run any scenario via `uv run --no-project` — see `CLAUDE.md` "Running S3 Tests with uv" for exact commands. \ No newline at end of file diff --git a/luigi/contrib/s3.py b/luigi/contrib/s3.py index 77fc0b285e..9193ac1540 100644 --- a/luigi/contrib/s3.py +++ b/luigi/contrib/s3.py @@ -669,162 +669,6 @@ def _add_path_delimiter(self, key): return key if key[-1:] == '/' or key == '' else key + '/' -class AtomicS3File(AtomicLocalFile): - """ - An S3 file that writes to a temp file and puts to S3 on close. - - :param kwargs: Keyword arguments are passed to the boto function `initiate_multipart_upload` - """ - - def __init__(self, path, s3_client, **kwargs): - self.s3_client = s3_client - super(AtomicS3File, self).__init__(path) - self.s3_options = kwargs - - def move_to_final_destination(self): - self.s3_client.put_multipart(self.tmp_path, self.path, **self.s3_options) - - -class S3Target(FileSystemTarget): - """ - Target S3 file object - - :param kwargs: Keyword arguments are passed to the boto function `initiate_multipart_upload` - """ - - fs = None - - def __init__(self, path, format=None, client=None, **kwargs): - super(S3Target, self).__init__(path) - if format is None: - format = get_default_format() - - self.path = path - self.format = format - self.fs = client or S3Client() - self.s3_options = kwargs - - def open(self, mode='r'): - if mode not in ('r', 'w'): - raise ValueError("Unsupported open mode '%s'" % mode) - - if mode == 'r': - s3_key = self.fs.get_key(self.path) - if not s3_key: - raise FileNotFoundException("Could not find file at %s" % self.path) - - fileobj = self.fs._readable_file_cls(s3_key) - return self.format.pipe_reader(fileobj) - else: - return self.format.pipe_writer(AtomicS3File(self.path, self.fs, **self.s3_options)) - - -class S3FlagTarget(S3Target): - """ - Defines a target directory with a flag-file (defaults to `_SUCCESS`) used - to signify job success. - - This checks for two things: - - * the path exists (just like the S3Target) - * the _SUCCESS file exists within the directory. - - Because Hadoop outputs into a directory and not a single file, - the path is assumed to be a directory. - - This is meant to be a handy alternative to AtomicS3File. - - The AtomicFile approach can be burdensome for S3 since there are no directories, per se. - - If we have 1,000,000 output files, then we have to rename 1,000,000 objects. - """ - - fs = None - - def __init__(self, path, format=None, client=None, flag='_SUCCESS'): - """ - Initializes a S3FlagTarget. - - :param path: the directory where the files are stored. - :type path: str - :param format: see the luigi.format module for options - :type format: luigi.format.[Text|UTF8|Nop] - :param client: - :type client: - :param flag: - :type flag: str - """ - if format is None: - format = get_default_format() - - if path[-1] != "/": - raise ValueError("S3FlagTarget requires the path to be to a " - "directory. It must end with a slash ( / ).") - super(S3FlagTarget, self).__init__(path, format, client) - self.flag = flag - - def exists(self): - hadoopSemaphore = self.path + self.flag - return self.fs.exists(hadoopSemaphore) - - -class S3EmrTarget(S3FlagTarget): - """ - Deprecated. Use :py:class:`S3FlagTarget` - """ - - def __init__(self, *args, **kwargs): - warnings.warn("S3EmrTarget is deprecated. Please use S3FlagTarget") - super(S3EmrTarget, self).__init__(*args, **kwargs) - - -class S3PathTask(ExternalTask): - """ - A external task that to require existence of a path in S3. - """ - path = Parameter() - - def __init__(self, *args, client=None, **kwargs): - super().__init__(*args, **kwargs) - self._client = client - - def output(self): - return S3Target(self.path, client=self._client) - - -class S3EmrTask(ExternalTask): - """ - An external task that requires the existence of EMR output in S3. - """ - path = Parameter() - - def __init__(self, *args, client=None, **kwargs): - super().__init__(*args, **kwargs) - self._client = client - - def output(self): - return S3EmrTarget(self.path, client=self._client) - - -class S3FlagTask(ExternalTask): - """ - An external task that requires the existence of EMR output in S3. - """ - path = Parameter() - flag = OptionalParameter(default=None) - - def __init__(self, *args, client=None, **kwargs): - super().__init__(*args, **kwargs) - self._client = client - - def output(self): - return S3FlagTarget(self.path, flag=self.flag, client=self._client) - - -class DeprecatedBotoClientException(Exception): - pass - - class _StreamingBodyAdaptor(_io.IOBase): """ Adapter class wrapping botocore's StreamingBody to make a file-like iterable. @@ -1229,7 +1073,150 @@ def _exists(self, bucket, key): return True -# Backwards-compatible aliases — preserve existing default behaviour (boto1). -# To use boto3 explicitly, import S3ClientBoto3 directly and pass it as client=. -S3Client = S3ClientBoto1 -ReadableS3File = ReadableS3FileBoto1 +# Aliases for backwards compatibility +S3Client = S3ClientBoto3 +ReadableS3File = ReadableS3FileBoto3 + + +class AtomicS3File(AtomicLocalFile): + """ + An S3 file that writes to a temp file and puts to S3 on close. + + :param kwargs: Keyword arguments are passed to the boto function `initiate_multipart_upload` + """ + + def __init__(self, path, s3_client, **kwargs): + self.s3_client = s3_client + super(AtomicS3File, self).__init__(path) + self.s3_options = kwargs + + def move_to_final_destination(self): + self.s3_client.put_multipart(self.tmp_path, self.path, **self.s3_options) + + +class S3Target(FileSystemTarget): + """ + Target S3 file object + + :param kwargs: Keyword arguments are passed to the boto function `initiate_multipart_upload` + """ + + fs = None + + def __init__(self, path, format=None, client=None, **kwargs): + super(S3Target, self).__init__(path) + if format is None: + format = get_default_format() + + self.path = path + self.format = format + self.fs = client or S3Client() + self.s3_options = kwargs + + def open(self, mode='r'): + if mode not in ('r', 'w'): + raise ValueError("Unsupported open mode '%s'" % mode) + + if mode == 'r': + s3_key = self.fs.get_key(self.path) + if not s3_key: + raise FileNotFoundException("Could not find file at %s" % self.path) + + fileobj = self.fs._readable_file_cls(s3_key) + return self.format.pipe_reader(fileobj) + else: + return self.format.pipe_writer(AtomicS3File(self.path, self.fs, **self.s3_options)) + + +class S3FlagTarget(S3Target): + """ + Defines a target directory with a flag-file (defaults to `_SUCCESS`) used + to signify job success. + + This checks for two things: + + * the path exists (just like the S3Target) + * the _SUCCESS file exists within the directory. + + Because Hadoop outputs into a directory and not a single file, + the path is assumed to be a directory. + + This is meant to be a handy alternative to AtomicS3File. + + The AtomicFile approach can be burdensome for S3 since there are no directories, per se. + + If we have 1,000,000 output files, then we have to rename 1,000,000 objects. + """ + + fs = None + + def __init__(self, path, format=None, client=None, flag='_SUCCESS'): + """ + Initializes a S3FlagTarget. + + :param path: the directory where the files are stored. + :type path: str + :param format: see the luigi.format module for options + :type format: luigi.format.[Text|UTF8|Nop] + :param client: + :type client: + :param flag: + :type flag: str + """ + if format is None: + format = get_default_format() + + if path[-1] != "/": + raise ValueError("S3FlagTarget requires the path to be to a " + "directory. It must end with a slash ( / ).") + super(S3FlagTarget, self).__init__(path, format, client) + self.flag = flag + + def exists(self): + hadoopSemaphore = self.path + self.flag + return self.fs.exists(hadoopSemaphore) + + +class S3EmrTarget(S3FlagTarget): + """ + Deprecated. Use :py:class:`S3FlagTarget` + """ + + def __init__(self, *args, **kwargs): + warnings.warn("S3EmrTarget is deprecated. Please use S3FlagTarget") + super(S3EmrTarget, self).__init__(*args, **kwargs) + + +class S3PathTask(ExternalTask): + """ + A external task that to require existence of a path in S3. + """ + path = Parameter() + + def output(self): + return S3Target(self.path) + + +class S3EmrTask(ExternalTask): + """ + An external task that requires the existence of EMR output in S3. + """ + path = Parameter() + + def output(self): + return S3EmrTarget(self.path) + + +class S3FlagTask(ExternalTask): + """ + An external task that requires the existence of EMR output in S3. + """ + path = Parameter() + flag = OptionalParameter(default=None) + + def output(self): + return S3FlagTarget(self.path, flag=self.flag) + + +class DeprecatedBotoClientException(Exception): + pass diff --git a/test/contrib/s3_test.py b/test/contrib/s3_test.py index c382c756c2..a03fb8c6dd 100644 --- a/test/contrib/s3_test.py +++ b/test/contrib/s3_test.py @@ -23,28 +23,60 @@ from target_test import FileSystemTargetTestMixin from helpers import with_config, unittest, skipOnTravis +import boto3 +from botocore.exceptions import ClientError as S3ResponseError + from luigi import configuration -from luigi.contrib.s3 import FileNotFoundException, InvalidDeleteException, S3Client, S3ClientBoto3, S3Target +from luigi.contrib.s3 import ( + DeprecatedBotoClientException, + FileNotFoundException, + InvalidDeleteException, + S3Client, + S3ClientBoto1, + S3Target, +) from luigi.target import MissingParentDirectory -try: - import boto - from boto.exception import S3ResponseError - from boto.s3 import key - HAS_BOTO = True -except ImportError: - import boto3 - from botocore.exceptions import ClientError as S3ResponseError - HAS_BOTO = False - # boto1 is not available; use the boto3 client for all tests in this file - S3Client = S3ClientBoto3 - try: from moto import mock_s3, mock_sts except ImportError: # moto >= 4.0 renamed mock_s3/mock_sts to mock_aws from moto import mock_aws as mock_s3, mock_aws as mock_sts +import moto as _moto +try: + _MOTO_VER = tuple(int(part) for part in _moto.__version__.split('.')[:2]) +except Exception: + _MOTO_VER = (0, 0) + +# moto<2 doesn't decode boto3's chunked Transfer-Encoding upload bodies, so any +# round-trip through put_multipart / upload_fileobj / copy stores chunk-size +# markers ("4f\n...real-data...\n0\n") instead of the real content. The boto3 +# tests exercising that path are only meaningful against moto>=2. +MOTO_LT_2 = _MOTO_VER < (2,) +SKIP_MOTO_LT_2 = ( + 'moto<2 mishandles boto3 chunked Transfer-Encoding uploads; ' + 'this round-trip is only meaningful against moto>=2' +) + +try: + import boto # noqa: F401 — legacy boto1, optional + from boto.s3 import key as boto_key + HAS_BOTO_1 = True +except ImportError: + HAS_BOTO_1 = False + +try: + # Older moto (<2.0) ships boto1-targeted decorators alongside the boto3 ones. + # These are required to actually mock boto1 (S3Connection) HTTP traffic. + from moto import mock_s3_deprecated, mock_sts_deprecated + HAS_BOTO_1_MOTO = True +except ImportError: + HAS_BOTO_1_MOTO = False + +BOTO1_AVAILABLE = HAS_BOTO_1 and HAS_BOTO_1_MOTO +BOTO1_SKIP_REASON = 'requires legacy boto (boto1) and moto<2 with mock_s3_deprecated' + if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3): # spulec/moto#308 raise unittest.SkipTest('moto mock doesn\'t work with python3.4') @@ -54,6 +86,7 @@ AWS_SECRET_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +@unittest.skipIf(MOTO_LT_2, SKIP_MOTO_LT_2) class TestS3Target(unittest.TestCase, FileSystemTargetTestMixin): def setUp(self): @@ -70,13 +103,9 @@ def setUp(self): self.mock_s3.start() self.addCleanup(self.mock_s3.stop) - def _create_bucket(self, client): - if HAS_BOTO: - client.s3.create_bucket('mybucket') - else: - import boto3 - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket='mybucket') + def _create_bucket(self, client=None): + conn = boto3.resource('s3', region_name='us-east-1') + conn.create_bucket(Bucket='mybucket') def create_target(self, format=None, **kwargs): client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) @@ -96,47 +125,17 @@ def test_read_no_file(self): t = self.create_target() self.assertRaises(FileNotFoundException, t.open) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_read_no_file_sse(self): - t = self.create_target(encrypt_key=True) + t = self.create_target(ServerSideEncryption='AES256') self.assertRaises(FileNotFoundException, t.open) - @unittest.skipIf(not HAS_BOTO, 'boto Key.BufferSize not available with boto3') - def test_read_iterator_long(self): - # write a file that is 5X the boto buffersize - # to test line buffering - old_buffer = key.Key.BufferSize - key.Key.BufferSize = 2 - try: - tempf = tempfile.NamedTemporaryFile(mode='wb', delete=False) - temppath = tempf.name - firstline = ''.zfill(key.Key.BufferSize * 5) + os.linesep - contents = firstline + 'line two' + os.linesep + 'line three' - tempf.write(contents.encode('utf-8')) - tempf.close() - - client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) - self._create_bucket(client) - client.put(temppath, 's3://mybucket/largetempfile') - t = S3Target('s3://mybucket/largetempfile', client=client) - with t.open() as read_file: - lines = [line for line in read_file] - finally: - key.Key.BufferSize = old_buffer - - self.assertEqual(3, len(lines)) - self.assertEqual(firstline, lines[0]) - self.assertEqual("line two" + os.linesep, lines[1]) - self.assertEqual("line three", lines[2]) - def test_get_path(self): t = self.create_target() path = t.path self.assertEqual('s3://mybucket/test_file', path) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_get_path_sse(self): - t = self.create_target(encrypt_key=True) + t = self.create_target(ServerSideEncryption='AES256') path = t.path self.assertEqual('s3://mybucket/test_file', path) @@ -159,40 +158,57 @@ def setUp(self): self.addCleanup(self.mock_sts.stop) def _create_bucket(self, client=None, name='mybucket'): - if HAS_BOTO: - (client or S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY)).s3.create_bucket(name) - else: - import boto3 - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket=name) - - @unittest.skipIf(not HAS_BOTO, 'boto-specific credential attribute gs_access_key_id') + conn = boto3.resource('s3', region_name='us-east-1') + conn.create_bucket(Bucket=name) + + @staticmethod + def _resolved_credentials(s3_client): + # boto3 resolves credentials lazily on the underlying client/session; + # pull them off the request signer for assertion. + return s3_client.s3.meta.client._request_signer._credentials + def test_init_with_environment_variables(self): + # moto sets AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY itself on mock-start + # and clears them on stop, so capture the values it installed and restore + # them in finally rather than deleting (deletion would race moto's cleanup). + prev = {k: os.environ.get(k) for k in ('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY')} os.environ['AWS_ACCESS_KEY_ID'] = 'foo' os.environ['AWS_SECRET_ACCESS_KEY'] = 'bar' # Don't read any existing config old_config_paths = configuration.LuigiConfigParser._config_paths configuration.LuigiConfigParser._config_paths = [tempfile.mktemp()] + try: + s3_client = S3Client() + credentials = self._resolved_credentials(s3_client) + self.assertEqual(credentials.access_key, 'foo') + self.assertEqual(credentials.secret_key, 'bar') + finally: + configuration.LuigiConfigParser._config_paths = old_config_paths + for k, v in prev.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v - s3_client = S3Client() - configuration.LuigiConfigParser._config_paths = old_config_paths - - self.assertEqual(s3_client.s3.gs_access_key_id, 'foo') - self.assertEqual(s3_client.s3.gs_secret_access_key, 'bar') - - @unittest.skipIf(not HAS_BOTO, 'boto-specific credential attributes access_key/secret_key') @with_config({'s3': {'aws_access_key_id': 'foo', 'aws_secret_access_key': 'bar'}}) def test_init_with_config(self): s3_client = S3Client() - self.assertEqual(s3_client.s3.access_key, 'foo') - self.assertEqual(s3_client.s3.secret_key, 'bar') + credentials = self._resolved_credentials(s3_client) + self.assertEqual(credentials.access_key, 'foo') + self.assertEqual(credentials.secret_key, 'bar') - @unittest.skipIf(not HAS_BOTO, 'boto-specific STS credential attributes') - @with_config({'s3': {'aws_role_arn': 'role', 'aws_role_session_name': 'name'}}) + @with_config({'s3': {'aws_role_arn': 'arn:aws:iam::123456789012:role/test', + 'aws_role_session_name': 'name'}}) def test_init_with_config_and_roles(self): s3_client = S3Client() - self.assertEqual(s3_client.s3.access_key, 'AKIAIOSFODNN7EXAMPLE') - self.assertEqual(s3_client.s3.secret_key, 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY') + credentials = self._resolved_credentials(s3_client) + # moto's STS mock returns randomized credentials per call; assert that + # assume-role produced session credentials (token populated, key in the + # ASIA-prefixed STS namespace) rather than pinning to specific values. + self.assertIsNotNone(credentials.access_key) + self.assertIsNotNone(credentials.secret_key) + self.assertIsNotNone(credentials.token) + self.assertTrue(credentials.access_key.startswith('ASIA')) def test_put(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) @@ -200,26 +216,41 @@ def test_put(self): s3_client.put(self.tempFilePath, 's3://mybucket/putMe') self.assertTrue(s3_client.exists('s3://mybucket/putMe')) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_put_sse(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) self._create_bucket(s3_client) - s3_client.put(self.tempFilePath, 's3://mybucket/putMe', encrypt_key=True) + s3_client.put(self.tempFilePath, 's3://mybucket/putMe', ServerSideEncryption='AES256') self.assertTrue(s3_client.exists('s3://mybucket/putMe')) + def test_put_encrypt_key_raises(self): + # encrypt_key is a boto1 parameter; the boto3 client must reject it explicitly. + s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + self.assertRaises( + DeprecatedBotoClientException, + lambda: s3_client.put(self.tempFilePath, 's3://mybucket/putMe', encrypt_key=True), + ) + def test_put_string(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) self._create_bucket(s3_client) s3_client.put_string("SOMESTRING", 's3://mybucket/putString') self.assertTrue(s3_client.exists('s3://mybucket/putString')) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_put_string_sse(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) self._create_bucket(s3_client) - s3_client.put_string("SOMESTRING", 's3://mybucket/putString', encrypt_key=True) + s3_client.put_string("SOMESTRING", 's3://mybucket/putString', ServerSideEncryption='AES256') self.assertTrue(s3_client.exists('s3://mybucket/putString')) + def test_put_string_encrypt_key_raises(self): + s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + self.assertRaises( + DeprecatedBotoClientException, + lambda: s3_client.put_string("SOMESTRING", 's3://mybucket/putString', encrypt_key=True), + ) + def test_put_multipart_multiple_parts_non_exact_fit(self): """ Test a multipart put with two parts, where the parts are not exactly the split size. @@ -229,11 +260,10 @@ def test_put_multipart_multiple_parts_non_exact_fit(self): file_size = (part_size * 2) - 5000 self._run_multipart_test(part_size, file_size) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_put_multipart_multiple_parts_non_exact_fit_with_sse(self): part_size = (1024 ** 2) * 5 file_size = (part_size * 2) - 5000 - self._run_multipart_test(part_size, file_size, encrypt_key=True) + self._run_multipart_test(part_size, file_size, ServerSideEncryption='AES256') def test_put_multipart_multiple_parts_exact_fit(self): """ @@ -243,11 +273,10 @@ def test_put_multipart_multiple_parts_exact_fit(self): file_size = part_size * 2 self._run_multipart_test(part_size, file_size) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') - def test_put_multipart_multiple_parts_exact_fit_wit_sse(self): + def test_put_multipart_multiple_parts_exact_fit_with_sse(self): part_size = (1024 ** 2) * 5 file_size = part_size * 2 - self._run_multipart_test(part_size, file_size, encrypt_key=True) + self._run_multipart_test(part_size, file_size, ServerSideEncryption='AES256') def test_put_multipart_less_than_split_size(self): """ @@ -257,11 +286,10 @@ def test_put_multipart_less_than_split_size(self): file_size = 5000 self._run_multipart_test(part_size, file_size) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_put_multipart_less_than_split_size_with_sse(self): part_size = (1024 ** 2) * 5 file_size = 5000 - self._run_multipart_test(part_size, file_size, encrypt_key=True) + self._run_multipart_test(part_size, file_size, ServerSideEncryption='AES256') def test_put_multipart_empty_file(self): """ @@ -271,11 +299,19 @@ def test_put_multipart_empty_file(self): file_size = 0 self._run_multipart_test(part_size, file_size) - @unittest.skipIf(not HAS_BOTO, 'encrypt_key is boto-only') def test_put_multipart_empty_file_with_sse(self): part_size = (1024 ** 2) * 5 file_size = 0 - self._run_multipart_test(part_size, file_size, encrypt_key=True) + self._run_multipart_test(part_size, file_size, ServerSideEncryption='AES256') + + def test_put_multipart_encrypt_key_raises(self): + s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + self.assertRaises( + DeprecatedBotoClientException, + lambda: s3_client.put_multipart( + self.tempFilePath, 's3://mybucket/putMe', encrypt_key=True), + ) def test_exists(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) @@ -300,6 +336,7 @@ def test_exists(self): self.assertTrue(s3_client.exists('s3://mybucket/tempdir2')) self.assertFalse(s3_client.exists('s3://mybucket/tempdir')) + @unittest.skipIf(MOTO_LT_2, SKIP_MOTO_LT_2) def test_get(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) self._create_bucket(s3_client) @@ -313,6 +350,7 @@ def test_get(self): tmp_file.close() + @unittest.skipIf(MOTO_LT_2, SKIP_MOTO_LT_2) def test_get_as_string(self): s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) self._create_bucket(s3_client) @@ -487,6 +525,8 @@ def test_copy_dir(self): self.assertEqual(original_size, copy_size) def _run_multipart_copy_test(self, put_method): + if MOTO_LT_2: + self.skipTest(SKIP_MOTO_LT_2) put_method() original = 's3://mybucket/putMe' @@ -502,6 +542,8 @@ def _run_multipart_copy_test(self, put_method): self.assertEqual(original_size, copy_size) def _run_copy_test(self, put_method): + if MOTO_LT_2: + self.skipTest(SKIP_MOTO_LT_2) put_method() original = 's3://mybucket/putMe' @@ -515,6 +557,8 @@ def _run_copy_test(self, put_method): self.assertEqual(original_size, copy_size) def _run_multipart_test(self, part_size, file_size, **kwargs): + if MOTO_LT_2: + self.skipTest(SKIP_MOTO_LT_2) file_contents = b"a" * file_size s3_path = 's3://mybucket/putMe' @@ -531,3 +575,177 @@ def _run_multipart_test(self, part_size, file_size, **kwargs): key_size = s3_client.get_key(s3_path).size self.assertEqual(file_size, key_size) tmp_file.close() + + +@unittest.skipIf(not BOTO1_AVAILABLE, BOTO1_SKIP_REASON) +class TestS3TargetBoto1(unittest.TestCase): + """S3Target round-trips against the legacy boto1-backed client.""" + + def setUp(self): + f = tempfile.NamedTemporaryFile(mode='wb', delete=False) + self.tempFileContents = ( + b"I'm a temporary file for testing\nAnd this is the second line\n" + b"This is the third.") + self.tempFilePath = f.name + f.write(self.tempFileContents) + f.close() + self.addCleanup(os.remove, self.tempFilePath) + + # boto1 traffic is only intercepted by the deprecated mock decorator. + self.mock_s3 = mock_s3_deprecated() + self.mock_s3.start() + self.addCleanup(self.mock_s3.stop) + + @staticmethod + def _create_bucket(client): + client.s3.create_bucket('mybucket') + + def test_read(self): + client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(client) + client.put(self.tempFilePath, 's3://mybucket/tempfile') + t = S3Target('s3://mybucket/tempfile', client=client) + with t.open() as read_file: + file_str = read_file.read() + self.assertEqual(self.tempFileContents, file_str.encode('utf-8')) + + def test_read_no_file_sse(self): + client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(client) + t = S3Target('s3://mybucket/test_file', client=client, encrypt_key=True) + self.assertRaises(FileNotFoundException, t.open) + + def test_read_iterator_long(self): + # Exercise ReadableS3FileBoto1's line-buffering by shrinking the + # boto1 Key buffer below a single line. + old_buffer = boto_key.Key.BufferSize + boto_key.Key.BufferSize = 2 + try: + tempf = tempfile.NamedTemporaryFile(mode='wb', delete=False) + temppath = tempf.name + firstline = ''.zfill(boto_key.Key.BufferSize * 5) + os.linesep + contents = firstline + 'line two' + os.linesep + 'line three' + tempf.write(contents.encode('utf-8')) + tempf.close() + self.addCleanup(os.remove, temppath) + + client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(client) + client.put(temppath, 's3://mybucket/largetempfile') + t = S3Target('s3://mybucket/largetempfile', client=client) + with t.open() as read_file: + lines = [line for line in read_file] + finally: + boto_key.Key.BufferSize = old_buffer + + self.assertEqual(3, len(lines)) + self.assertEqual(firstline, lines[0]) + self.assertEqual('line two' + os.linesep, lines[1]) + self.assertEqual('line three', lines[2]) + + +@unittest.skipIf(not BOTO1_AVAILABLE, BOTO1_SKIP_REASON) +class TestS3ClientBoto1(unittest.TestCase): + """Direct tests for S3ClientBoto1 — boto1-specific credential and SSE behavior.""" + + def setUp(self): + f = tempfile.NamedTemporaryFile(mode='wb', delete=False) + self.tempFilePath = f.name + self.tempFileContents = b"I'm a temporary file for testing\n" + f.write(self.tempFileContents) + f.close() + self.addCleanup(os.remove, self.tempFilePath) + + self.mock_s3 = mock_s3_deprecated() + self.mock_s3.start() + self.mock_sts = mock_sts_deprecated() + self.mock_sts.start() + self.addCleanup(self.mock_s3.stop) + self.addCleanup(self.mock_sts.stop) + + @staticmethod + def _create_bucket(client=None, name='mybucket'): + (client or S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY)).s3.create_bucket(name) + + def test_init_with_environment_variables(self): + os.environ['AWS_ACCESS_KEY_ID'] = 'foo' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'bar' + old_config_paths = configuration.LuigiConfigParser._config_paths + configuration.LuigiConfigParser._config_paths = [tempfile.mktemp()] + try: + s3_client = S3ClientBoto1() + self.assertEqual(s3_client.s3.gs_access_key_id, 'foo') + self.assertEqual(s3_client.s3.gs_secret_access_key, 'bar') + finally: + configuration.LuigiConfigParser._config_paths = old_config_paths + del os.environ['AWS_ACCESS_KEY_ID'] + del os.environ['AWS_SECRET_ACCESS_KEY'] + + @with_config({'s3': {'aws_access_key_id': 'foo', 'aws_secret_access_key': 'bar'}}) + def test_init_with_config(self): + s3_client = S3ClientBoto1() + self.assertEqual(s3_client.s3.access_key, 'foo') + self.assertEqual(s3_client.s3.secret_key, 'bar') + + @with_config({'s3': {'aws_role_arn': 'role', 'aws_role_session_name': 'name'}}) + def test_init_with_config_and_roles(self): + s3_client = S3ClientBoto1() + # Assert STS-assumed-role shape (ASIA-prefixed key + session token) + # rather than pinning to specific values, since moto's STS mock returns + # randomized credentials per call. + self.assertIsNotNone(s3_client.s3.access_key) + self.assertIsNotNone(s3_client.s3.secret_key) + self.assertTrue(s3_client.s3.access_key.startswith('ASIA')) + + def test_put(self): + s3_client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + s3_client.put(self.tempFilePath, 's3://mybucket/putMe') + self.assertTrue(s3_client.exists('s3://mybucket/putMe')) + + def test_put_sse(self): + s3_client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + s3_client.put(self.tempFilePath, 's3://mybucket/putMe', encrypt_key=True) + self.assertTrue(s3_client.exists('s3://mybucket/putMe')) + + def test_put_string_sse(self): + s3_client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + s3_client.put_string("SOMESTRING", 's3://mybucket/putString', encrypt_key=True) + self.assertTrue(s3_client.exists('s3://mybucket/putString')) + + def _run_multipart_test(self, part_size, file_size, **kwargs): + file_contents = b"a" * file_size + + s3_path = 's3://mybucket/putMe' + tmp_file = tempfile.NamedTemporaryFile(mode='wb', delete=True) + tmp_file_path = tmp_file.name + tmp_file.write(file_contents) + tmp_file.flush() + + s3_client = S3ClientBoto1(AWS_ACCESS_KEY, AWS_SECRET_KEY) + self._create_bucket(s3_client) + s3_client.put_multipart(tmp_file_path, s3_path, part_size=part_size, **kwargs) + self.assertTrue(s3_client.exists(s3_path)) + tmp_file.close() + + def test_put_multipart_multiple_parts_non_exact_fit_with_sse(self): + part_size = (1024 ** 2) * 5 + file_size = (part_size * 2) - 5000 + self._run_multipart_test(part_size, file_size, encrypt_key=True) + + def test_put_multipart_multiple_parts_exact_fit_with_sse(self): + part_size = (1024 ** 2) * 5 + file_size = part_size * 2 + self._run_multipart_test(part_size, file_size, encrypt_key=True) + + def test_put_multipart_less_than_split_size_with_sse(self): + part_size = (1024 ** 2) * 5 + file_size = 5000 + self._run_multipart_test(part_size, file_size, encrypt_key=True) + + def test_put_multipart_empty_file_with_sse(self): + part_size = (1024 ** 2) * 5 + file_size = 0 + self._run_multipart_test(part_size, file_size, encrypt_key=True) From 50ed791acfb2258ffba2255ee921afe99baf2b1b Mon Sep 17 00:00:00 2001 From: Felipe Velasquez Date: Wed, 6 May 2026 13:45:29 -0500 Subject: [PATCH 2/2] bump the version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ec78a13d0..502c3d1982 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def get_static_files(path): setup( name='luigi', - version='2.7.5+affirm.1.4.9.rc8', + version='2.7.5+affirm.1.4.9.rc9', description='Workflow mgmgt + task scheduling + dependency resolution', long_description=long_description, author='The Luigi Authors',