Skip to content

Commit 65e1c03

Browse files
fix: Download components.zip instead of components.py from registry (#1005)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 3ab17f9 commit 65e1c03

1 file changed

Lines changed: 21 additions & 6 deletions

File tree

airbyte/_executors/util.py

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
from __future__ import annotations
33

44
import hashlib
5+
import io
56
import sys
67
import tempfile
8+
import zipfile
79
from pathlib import Path
810
from typing import TYPE_CHECKING, Literal, cast
911

@@ -33,23 +35,26 @@
3335
"https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/manifest.yaml"
3436
)
3537
DEFAULT_COMPONENTS_URL = (
36-
"https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/components.py"
38+
"https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/components.zip"
3739
)
3840

3941

4042
def _try_get_manifest_connector_files(
4143
source_name: str,
4244
version: str | None = None,
4345
) -> tuple[dict, str | None, str | None]:
44-
"""Try to get source manifest and components.py from URLs.
46+
"""Try to get the source manifest and components.zip from URLs and extract components.py.
4547
4648
Returns tuple of (manifest_dict, components_py_content, components_py_checksum).
47-
Components values are None if components.py is not found (404 is handled gracefully).
49+
Components values are None if components.zip is not found (404 is handled gracefully).
50+
51+
The registry serves custom components as `components.zip` archives. This function
52+
downloads and extracts `components.py` from the zip archive.
4853
4954
Raises:
5055
- `PyAirbyteInputError`: If `source_name` is `None`.
5156
- `AirbyteConnectorInstallationError`: If the manifest cannot be downloaded or parsed,
52-
or if components.py cannot be downloaded (excluding 404 errors).
57+
or if components.zip cannot be downloaded or extracted (excluding 404 errors).
5358
"""
5459
if source_name is None:
5560
raise exc.PyAirbyteInputError(
@@ -104,13 +109,23 @@ def _try_get_manifest_connector_files(
104109
response.raise_for_status()
105110
except requests.exceptions.HTTPError as ex:
106111
raise exc.AirbyteConnectorInstallationError(
107-
message="Failed to download the connector components.py file.",
112+
message="Failed to download the connector components.zip file.",
113+
context={
114+
"components_url": components_url,
115+
},
116+
) from ex
117+
118+
try:
119+
with zipfile.ZipFile(io.BytesIO(response.content)) as zf:
120+
components_content = zf.read("components.py").decode("utf-8")
121+
except (zipfile.BadZipFile, KeyError, UnicodeDecodeError) as ex:
122+
raise exc.AirbyteConnectorInstallationError(
123+
message="Failed to extract components.py from components.zip.",
108124
context={
109125
"components_url": components_url,
110126
},
111127
) from ex
112128

113-
components_content = response.text
114129
components_py_checksum = hashlib.md5(components_content.encode()).hexdigest()
115130

116131
return manifest_dict, components_content, components_py_checksum

0 commit comments

Comments
 (0)