|
2 | 2 | from __future__ import annotations |
3 | 3 |
|
4 | 4 | import hashlib |
| 5 | +import io |
5 | 6 | import sys |
6 | 7 | import tempfile |
| 8 | +import zipfile |
7 | 9 | from pathlib import Path |
8 | 10 | from typing import TYPE_CHECKING, Literal, cast |
9 | 11 |
|
|
33 | 35 | "https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/manifest.yaml" |
34 | 36 | ) |
35 | 37 | DEFAULT_COMPONENTS_URL = ( |
36 | | - "https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/components.py" |
| 38 | + "https://connectors.airbyte.com/files/metadata/airbyte/{source_name}/{version}/components.zip" |
37 | 39 | ) |
38 | 40 |
|
39 | 41 |
|
40 | 42 | def _try_get_manifest_connector_files( |
41 | 43 | source_name: str, |
42 | 44 | version: str | None = None, |
43 | 45 | ) -> tuple[dict, str | None, str | None]: |
44 | | - """Try to get source manifest and components.py from URLs. |
| 46 | + """Try to get the source manifest and components.zip from URLs and extract components.py. |
45 | 47 |
|
46 | 48 | Returns tuple of (manifest_dict, components_py_content, components_py_checksum). |
47 | | - Components values are None if components.py is not found (404 is handled gracefully). |
| 49 | + Components values are None if components.zip is not found (404 is handled gracefully). |
| 50 | +
|
| 51 | + The registry serves custom components as `components.zip` archives. This function |
| 52 | + downloads and extracts `components.py` from the zip archive. |
48 | 53 |
|
49 | 54 | Raises: |
50 | 55 | - `PyAirbyteInputError`: If `source_name` is `None`. |
51 | 56 | - `AirbyteConnectorInstallationError`: If the manifest cannot be downloaded or parsed, |
52 | | - or if components.py cannot be downloaded (excluding 404 errors). |
| 57 | + or if components.zip cannot be downloaded or extracted (excluding 404 errors). |
53 | 58 | """ |
54 | 59 | if source_name is None: |
55 | 60 | raise exc.PyAirbyteInputError( |
@@ -104,13 +109,23 @@ def _try_get_manifest_connector_files( |
104 | 109 | response.raise_for_status() |
105 | 110 | except requests.exceptions.HTTPError as ex: |
106 | 111 | raise exc.AirbyteConnectorInstallationError( |
107 | | - message="Failed to download the connector components.py file.", |
| 112 | + message="Failed to download the connector components.zip file.", |
| 113 | + context={ |
| 114 | + "components_url": components_url, |
| 115 | + }, |
| 116 | + ) from ex |
| 117 | + |
| 118 | + try: |
| 119 | + with zipfile.ZipFile(io.BytesIO(response.content)) as zf: |
| 120 | + components_content = zf.read("components.py").decode("utf-8") |
| 121 | + except (zipfile.BadZipFile, KeyError, UnicodeDecodeError) as ex: |
| 122 | + raise exc.AirbyteConnectorInstallationError( |
| 123 | + message="Failed to extract components.py from components.zip.", |
108 | 124 | context={ |
109 | 125 | "components_url": components_url, |
110 | 126 | }, |
111 | 127 | ) from ex |
112 | 128 |
|
113 | | - components_content = response.text |
114 | 129 | components_py_checksum = hashlib.md5(components_content.encode()).hexdigest() |
115 | 130 |
|
116 | 131 | return manifest_dict, components_content, components_py_checksum |
|
0 commit comments