diff --git a/airbyte_cdk/cli/airbyte_cdk/_secrets.py b/airbyte_cdk/cli/airbyte_cdk/_secrets.py index 7aa3996d4..b2e338317 100644 --- a/airbyte_cdk/cli/airbyte_cdk/_secrets.py +++ b/airbyte_cdk/cli/airbyte_cdk/_secrets.py @@ -43,6 +43,7 @@ from rich.console import Console from rich.table import Table +from airbyte_cdk.cli.airbyte_cdk.exceptions import ConnectorSecretWithNoValidVersionsError from airbyte_cdk.utils.connector_paths import ( resolve_connector_name, resolve_connector_name_and_directory, @@ -131,24 +132,46 @@ def fetch( ) # Fetch and write secrets secret_count = 0 + exceptions = [] + for secret in secrets: secret_file_path = _get_secret_filepath( secrets_dir=secrets_dir, secret=secret, ) - _write_secret_file( - secret=secret, - client=client, - file_path=secret_file_path, + try: + _write_secret_file( + secret=secret, + client=client, + file_path=secret_file_path, + connector_name=connector_name, + gcp_project_id=gcp_project_id, + ) + click.echo(f"Secret written to: {secret_file_path.absolute()!s}", err=True) + secret_count += 1 + except ConnectorSecretWithNoValidVersionsError as e: + exceptions.append(e) + click.echo( + f"Failed to retrieve secret '{e.secret_name}': No enabled version found", err=True + ) + + if secret_count == 0 and not exceptions: + click.echo( + f"No secrets found for connector: '{connector_name}'", + err=True, ) - click.echo(f"Secret written to: {secret_file_path.absolute()!s}", err=True) - secret_count += 1 - if secret_count == 0: + if exceptions: + error_message = f"Failed to retrieve {len(exceptions)} secret(s)" click.echo( - f"No secrets found for connector: '{connector_name}'", + style( + error_message, + fg="red", + ), err=True, ) + if secret_count == 0: + raise exceptions[0] if not print_ci_secrets_masks: return @@ -230,9 +253,8 @@ def list_( table.add_column("Created", justify="left", style="blue", overflow="fold") for secret in secrets: full_secret_name = secret.name - secret_name = full_secret_name.split("/secrets/")[-1] # Removes project prefix - # E.g. https://console.cloud.google.com/security/secret-manager/secret/SECRET_SOURCE-SHOPIFY__CREDS/versions?hl=en&project= - secret_url = f"https://console.cloud.google.com/security/secret-manager/secret/{secret_name}/versions?hl=en&project={gcp_project_id}" + secret_name = _extract_secret_name(full_secret_name) + secret_url = _get_secret_url(secret_name, gcp_project_id) table.add_row( f"[link={secret_url}]{secret_name}[/link]", "\n".join([f"{k}={v}" for k, v in secret.labels.items()]), @@ -242,6 +264,43 @@ def list_( console.print(table) +def _extract_secret_name(secret_name: str) -> str: + """Extract the secret name from a fully qualified secret path. + + Handles different formats of secret names: + - Full path: "projects/project-id/secrets/SECRET_NAME" + - Already extracted: "SECRET_NAME" + + Args: + secret_name: The secret name or path + + Returns: + str: The extracted secret name without project prefix + """ + if "/secrets/" in secret_name: + return secret_name.split("/secrets/")[-1] + return secret_name + + +def _get_secret_url(secret_name: str, gcp_project_id: str) -> str: + """Generate a URL for a secret in the GCP Secret Manager console. + + Note: This URL itself does not contain secrets or sensitive information. + The URL itself is only useful for valid logged-in users of the project, and it + safe to print this URL in logs. + + Args: + secret_name: The name of the secret in GCP. + gcp_project_id: The GCP project ID. + + Returns: + str: URL to the secret in the GCP console + """ + # Ensure we have just the secret name without the project prefix + secret_name = _extract_secret_name(secret_name) + return f"https://console.cloud.google.com/security/secret-manager/secret/{secret_name}/versions?hl=en&project={gcp_project_id}" + + def _fetch_secret_handles( connector_name: str, gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT, @@ -272,9 +331,44 @@ def _write_secret_file( secret: "Secret", # type: ignore client: "secretmanager.SecretManagerServiceClient", # type: ignore file_path: Path, + connector_name: str, + gcp_project_id: str, ) -> None: - version_name = f"{secret.name}/versions/latest" - response = client.access_secret_version(name=version_name) + """Write the most recent enabled version of a secret to a file. + + Lists all enabled versions of the secret and selects the most recent one. + Raises ConnectorSecretWithNoValidVersionsError if no enabled versions are found. + + Args: + secret: The secret to write to a file + client: The Secret Manager client + file_path: The path to write the secret to + connector_name: The name of the connector + gcp_project_id: The GCP project ID + + Raises: + ConnectorSecretWithNoValidVersionsError: If no enabled version is found + """ + # List all enabled versions of the secret. + response = client.list_secret_versions( + request={"parent": secret.name, "filter": "state:ENABLED"} + ) + + # The API returns versions pre-sorted in descending order, with the + # 0th item being the latest version. + versions = list(response) + + if not versions: + secret_name = _extract_secret_name(secret.name) + raise ConnectorSecretWithNoValidVersionsError( + connector_name=connector_name, + secret_name=secret_name, + gcp_project_id=gcp_project_id, + ) + + enabled_version = versions[0] + + response = client.access_secret_version(name=enabled_version.name) file_path.write_text(response.payload.data.decode("UTF-8")) file_path.chmod(0o600) # default to owner read/write only diff --git a/airbyte_cdk/cli/airbyte_cdk/exceptions.py b/airbyte_cdk/cli/airbyte_cdk/exceptions.py new file mode 100644 index 000000000..e736e8661 --- /dev/null +++ b/airbyte_cdk/cli/airbyte_cdk/exceptions.py @@ -0,0 +1,23 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Exceptions for the Airbyte CDK CLI.""" + +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class ConnectorSecretWithNoValidVersionsError(Exception): + """Error when a connector secret has no valid versions.""" + + connector_name: str + secret_name: str + gcp_project_id: str + + def __str__(self) -> str: + """Return a string representation of the exception.""" + from airbyte_cdk.cli.airbyte_cdk._secrets import _get_secret_url + + url = _get_secret_url(self.secret_name, self.gcp_project_id) + return ( + f"No valid versions found for secret '{self.secret_name}' in connector '{self.connector_name}'. " + f"Please check the following URL for more information:\n- {url}" + ) diff --git a/unit_tests/cli/airbyte_cdk/test_secrets.py b/unit_tests/cli/airbyte_cdk/test_secrets.py new file mode 100644 index 000000000..75bf55d15 --- /dev/null +++ b/unit_tests/cli/airbyte_cdk/test_secrets.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from click.testing import CliRunner + +from airbyte_cdk.cli.airbyte_cdk._secrets import ( + _write_secret_file, + fetch, + secretmanager, +) +from airbyte_cdk.cli.airbyte_cdk.exceptions import ConnectorSecretWithNoValidVersionsError + + +class TestWriteSecretFile: + @pytest.fixture + def mock_client(self): + return MagicMock() + + @pytest.fixture + def mock_secret(self): + secret = MagicMock() + secret.name = "projects/test-project/secrets/test-secret" + return secret + + @pytest.fixture + def mock_file_path(self, tmp_path): + return tmp_path / "test_secret.json" + + def test_write_secret_file_with_enabled_version(self, mock_client, mock_secret, mock_file_path): + # Mock list_secret_versions to return an enabled version + mock_version = MagicMock() + mock_version.name = f"{mock_secret.name}/versions/1" + mock_client.list_secret_versions.return_value = [mock_version] + + # Mock access_secret_version to return a payload + mock_response = MagicMock() + mock_response.payload.data.decode.return_value = '{"key": "value"}' + mock_client.access_secret_version.return_value = mock_response + + # Call the function + _write_secret_file( + secret=mock_secret, + client=mock_client, + file_path=mock_file_path, + connector_name="test-connector", + gcp_project_id="test-project", + ) + + # Verify that list_secret_versions was called with the correct parameters + mock_client.list_secret_versions.assert_called_once() + assert "state:ENABLED" in str(mock_client.list_secret_versions.call_args) + + # Verify that access_secret_version was called with the correct version + mock_client.access_secret_version.assert_called_once_with(name=mock_version.name) + + # Verify that the file was created with the correct content + assert mock_file_path.read_text() == '{"key": "value"}' + + def test_write_secret_file_with_no_enabled_versions( + self, mock_client, mock_secret, mock_file_path + ): + # Mock list_secret_versions to return an empty list (no enabled versions) + mock_client.list_secret_versions.return_value = [] + + # Call the function and expect an exception + with pytest.raises(ConnectorSecretWithNoValidVersionsError) as excinfo: + _write_secret_file( + secret=mock_secret, + client=mock_client, + file_path=mock_file_path, + connector_name="test-connector", + gcp_project_id="test-project", + ) + + # Verify that list_secret_versions was called with the correct parameters + mock_client.list_secret_versions.assert_called_once() + assert "state:ENABLED" in str(mock_client.list_secret_versions.call_args) + + # Verify that access_secret_version was not called + mock_client.access_secret_version.assert_not_called() + + # Verify that the file was not created + assert not mock_file_path.exists() + + # Verify the exception details + assert excinfo.value.secret_name == "test-secret" + assert excinfo.value.connector_name == "test-connector" + assert excinfo.value.gcp_project_id == "test-project" + + +@patch("airbyte_cdk.cli.airbyte_cdk._secrets._get_gsm_secrets_client") +@patch("airbyte_cdk.cli.airbyte_cdk._secrets.resolve_connector_name_and_directory") +@patch("airbyte_cdk.cli.airbyte_cdk._secrets._get_secrets_dir") +@patch("airbyte_cdk.cli.airbyte_cdk._secrets._fetch_secret_handles") +class TestFetch: + def test_fetch_with_some_failed_secrets( + self, + mock_fetch_secret_handles, + mock_get_secrets_dir, + mock_resolve, + mock_get_client, + tmp_path, + ): + # Setup mocks + mock_client = MagicMock() + mock_get_client.return_value = mock_client + + mock_resolve.return_value = ("test-connector", tmp_path) + + secrets_dir = tmp_path / "secrets" + mock_get_secrets_dir.return_value = secrets_dir + + # Create two secrets, one that will succeed and one that will fail + secret1 = MagicMock() + secret1.name = "projects/test-project/secrets/test-secret-1" + secret1.labels = {} + + secret2 = MagicMock() + secret2.name = "projects/test-project/secrets/test-secret-2" + secret2.labels = {} + + mock_fetch_secret_handles.return_value = [secret1, secret2] + + # Mock _write_secret_file to succeed for secret1 and fail for secret2 + with patch( + "airbyte_cdk.cli.airbyte_cdk._secrets._write_secret_file" + ) as mock_write_secret_file: + # First call succeeds, second call raises exception + mock_write_secret_file.side_effect = [ + None, # Success for secret1 + ConnectorSecretWithNoValidVersionsError( + connector_name="test-connector", + secret_name="test-secret-2", + gcp_project_id="test-project", + ), # Failure for secret2 + ] + + # Call the function + runner = CliRunner() + result = runner.invoke(fetch) + + # Verify that _write_secret_file was called twice + assert mock_write_secret_file.call_count == 2 + + # Verify that the error message was printed + assert "Failed to retrieve secret 'test-secret-2'" in result.output + assert "Failed to retrieve 1 secret(s)" in result.output + + # Verify that the function did not raise an exception + assert result.exit_code == 0 + + def test_fetch_with_all_failed_secrets( + self, + mock_fetch_secret_handles, + mock_get_secrets_dir, + mock_resolve, + mock_get_client, + tmp_path, + ): + # Setup mocks + mock_client = MagicMock() + mock_get_client.return_value = mock_client + + mock_resolve.return_value = ("test-connector", tmp_path) + + secrets_dir = tmp_path / "secrets" + mock_get_secrets_dir.return_value = secrets_dir + + # Create two secrets that will both fail + secret1 = MagicMock() + secret1.name = "projects/test-project/secrets/test-secret-1" + secret1.labels = {} + + secret2 = MagicMock() + secret2.name = "projects/test-project/secrets/test-secret-2" + secret2.labels = {} + + mock_fetch_secret_handles.return_value = [secret1, secret2] + + # Mock _write_secret_file to fail for both secrets + with patch( + "airbyte_cdk.cli.airbyte_cdk._secrets._write_secret_file" + ) as mock_write_secret_file: + mock_write_secret_file.side_effect = [ + ConnectorSecretWithNoValidVersionsError( + connector_name="test-connector", + secret_name="test-secret-1", + gcp_project_id="test-project", + ), # Failure for secret1 + ConnectorSecretWithNoValidVersionsError( + connector_name="test-connector", + secret_name="test-secret-2", + gcp_project_id="test-project", + ), # Failure for secret2 + ] + + # Call the function + runner = CliRunner() + result = runner.invoke(fetch) + + # Verify that _write_secret_file was called twice + assert mock_write_secret_file.call_count == 2 + + # Verify that the error message was printed + assert "Failed to retrieve secret 'test-secret-1'" in result.output + assert "Failed to retrieve secret 'test-secret-2'" in result.output + assert "Failed to retrieve 2 secret(s)" in result.output + + # Verify that the function raised an exception + assert result.exit_code != 0