fix: drop Expect header before S3 REST signing

Author: Gayathri Srividya Rajavarapu

pyiceberg/io/fsspec.py

@@ -139,6 +139,12 @@ def __call__(self, request: "AWSRequest", **_: Any) -> None:
 
         signer_headers.update(get_header_properties(self.properties))
 
+        # Some S3-compatible signer services reject requests carrying Expect: 100-continue.
+        # Strip this transport hint before asking the service to sign request headers.
+        for header in list(request.headers):
+            if header.lower() == "expect":
+                del request.headers[header]
+
         signer_body = {
             "method": request.method,
             "region": request.context["client_region"],

tests/io/test_fsspec.py

@@ -1018,6 +1018,59 @@ def test_s3v4_rest_signer_endpoint(requests_mock: Mocker) -> None:
     }
 
 
+def test_s3v4_rest_signer_strips_expect_header(requests_mock: Mocker) -> None:
+    new_uri = "https://other-bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro"
+    requests_mock.post(
+        f"{TEST_URI}/v1/aws/s3/sign",
+        json={
+            "uri": new_uri,
+            "headers": {
+                "Authorization": [
+                    "AWS4-HMAC-SHA256 Credential=ASIAQPRZZYGHUT57DL3I/20221017/us-west-2/s3/aws4_request, "
+                    "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, "
+                    "Signature=430582a17d61ab02c272896fa59195f277af4bdf2121c441685e589f044bbe02"
+                ],
+                "Host": ["bucket.s3.us-west-2.amazonaws.com"],
+                "User-Agent": ["Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"],
+                "x-amz-content-sha256": ["UNSIGNED-PAYLOAD"],
+                "X-Amz-Date": ["20221017T102940Z"],
+                "X-Amz-Security-Token": [
+                    "YQoJb3JpZ2luX2VjEDoaCXVzLXdlc3QtMiJGMEQCID/fFxZP5oaEgQmcwP6XhZa0xSq9lmLSx8ffaWbySfUPAiAesa7sjd/WV4uwRTO0S03y/MWVtgpH+/NyZQ4bZgLVriqrAggTEAEaDDAzMzQwNzIyMjE1OSIMOeFOWhZIurMmAqjsKogCxMCqxX8ZjK0gacAkcDqBCyA7qTSLhdfKQIH/w7WpLBU1km+cRUWWCudan6gZsAq867DBaKEP7qI05DAWr9MChAkgUgyI8/G3Z23ET0gAedf3GsJbakB0F1kklx8jPmj4BPCht9RcTiXiJ5DxTS/cRCcalIQXmPFbaJSqpBusVG2EkWnm1v7VQrNPE2Os2b2P293vpbhwkyCEQiGRVva4Sw9D1sKvqSsK10QCRG+os6dFEOu1kARaXi6pStvR4OVmj7OYeAYjzaFchn7nz2CSae0M4IluiYQ01eQAywbfRo9DpKSmDM/DnPZWJnD/woLhaaaCrCxSSEaFsvGOHFhLd3Rknw1v0jADMILUtJoGOp4BpqKqyMz0CY3kpKL0jfR3ykTf/ge9wWVE0Alr7wRIkGCIURkhslGHqSyFRGoTqIXaxU+oPbwlw/0w/nYO7qQ6bTANOWye/wgw4h/NmJ6vU7wnZTXwREf1r6MF72++bE/fMk19LfVb8jN/qrUqAUXTc8gBAUxL5pgy8+oT/JnI2BkVrrLS4ilxEXP9Ahm+6GDUYXV4fBpqpZwdkzQ/5Gw="
+                ],
+            },
+            "extensions": {},
+        },
+        status_code=200,
+    )
+
+    request = AWSRequest(
+        method="PUT",
+        url="https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro",
+        headers={
+            "User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0",
+            "Expect": "100-continue",
+        },
+        data=b"abc",
+        params={},
+        auth_path="/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro",
+    )
+    request.context = {
+        "client_region": "us-west-2",
+        "has_streaming_input": False,
+        "auth_type": None,
+        "signing": {"bucket": "bucket"},
+        "retries": {"attempt": 1, "invocation-id": "75d143fb-0219-439b-872c-18213d1c8d54"},
+    }
+
+    signer = S3V4RestSigner(properties={"token": "abc", "uri": TEST_URI})
+    signer(request)
+
+    assert requests_mock.last_request is not None
+    signer_request_body = requests_mock.last_request.json()
+    assert "expect" not in {header.lower() for header in signer_request_body["headers"]}
+    assert "expect" not in {header.lower() for header in request.headers}
+
+
 def test_s3v4_rest_signer_forbidden(requests_mock: Mocker) -> None:
     requests_mock.post(
         f"{TEST_URI}/v1/aws/s3/sign",