diff --git a/airbyte_cdk/manifest_migrations/README.md b/airbyte_cdk/manifest_migrations/README.md index 12ec41837..fc15a9c80 100644 --- a/airbyte_cdk/manifest_migrations/README.md +++ b/airbyte_cdk/manifest_migrations/README.md @@ -20,7 +20,8 @@ This directory contains the logic and registry for manifest migrations in the Ai 3. **Register the Migration:** - Open `migrations/registry.yaml`. - - Add an entry under the appropriate version, or create a new version section if needed. + - Add an entry under the appropriate version, or create a new version section if needed. + - Version can be: "*", "==6.48.3", "~=1.2", ">=1.0.0,<2.0.0", "6.48.3" - Each migration entry should include: - `name`: The filename (without `.py`) - `order`: The order in which this migration should be applied for the version diff --git a/airbyte_cdk/manifest_migrations/migration_handler.py b/airbyte_cdk/manifest_migrations/migration_handler.py index f843c25ce..266059e9a 100644 --- a/airbyte_cdk/manifest_migrations/migration_handler.py +++ b/airbyte_cdk/manifest_migrations/migration_handler.py @@ -5,9 +5,11 @@ import copy import logging +import re from datetime import datetime, timezone -from typing import Type +from typing import Tuple, Type +from packaging.specifiers import SpecifierSet from packaging.version import Version from airbyte_cdk.manifest_migrations.exceptions import ( @@ -25,7 +27,7 @@ METADATA_TAG = "metadata" MANIFEST_VERSION_TAG = "version" APPLIED_MIGRATIONS_TAG = "applied_migrations" - +WILDCARD_VERSION_PATTERN = ".*" LOGGER = logging.getLogger("airbyte.cdk.manifest_migrations") @@ -77,11 +79,14 @@ def _handle_migration( """ try: migration_instance = migration_class() - if self._version_is_valid_for_migration(manifest_version, migration_version): + can_apply_migration, should_bump_version = self._version_is_valid_for_migration( + manifest_version, migration_version + ) + if can_apply_migration: migration_instance._process_manifest(self._migrated_manifest) if migration_instance.is_migrated: - # set the updated manifest version, after migration has been applied - self._set_manifest_version(migration_version) + if should_bump_version: + self._set_manifest_version(migration_version) self._set_migration_trace(migration_class, manifest_version, migration_version) else: LOGGER.info( @@ -112,18 +117,30 @@ def _version_is_valid_for_migration( self, manifest_version: str, migration_version: str, - ) -> bool: + ) -> Tuple[bool, bool]: + """ + Decide whether *manifest_version* satisfies the *migration_version* rule. + + Rules + ----- + 1. ``"*"`` + – Wildcard: anything matches. + 2. String starts with a PEP 440 operator (``==``, ``!=``, ``<=``, ``>=``, + ``<``, ``>``, ``~=``, etc.) + – Treat *migration_version* as a SpecifierSet and test the manifest + version against it. + 3. Plain version + – Interpret both strings as concrete versions and return + ``manifest_version <= migration_version``. """ - Checks if the given manifest version is less than or equal to the specified migration version. + if re.match(WILDCARD_VERSION_PATTERN, migration_version): + return True, False - Args: - manifest_version (str): The version of the manifest to check. - migration_version (str): The migration version to compare against. + if migration_version.startswith(("=", "!", ">", "<", "~")): + spec = SpecifierSet(migration_version) + return spec.contains(Version(manifest_version)), False - Returns: - bool: True if the manifest version is less than or equal to the migration version, False otherwise. - """ - return Version(manifest_version) <= Version(migration_version) + return Version(manifest_version) <= Version(migration_version), True def _set_manifest_version(self, version: str) -> None: """ diff --git a/airbyte_cdk/manifest_migrations/migrations/__init__.py b/airbyte_cdk/manifest_migrations/migrations/__init__.py index 5a567c032..4937a4853 100644 --- a/airbyte_cdk/manifest_migrations/migrations/__init__.py +++ b/airbyte_cdk/manifest_migrations/migrations/__init__.py @@ -2,3 +2,18 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.manifest_migrations.migrations.http_requester_path_to_url import ( + HttpRequesterPathToUrl, +) +from airbyte_cdk.manifest_migrations.migrations.http_requester_request_body_json_data_to_request_body import ( + HttpRequesterRequestBodyJsonDataToRequestBody, +) +from airbyte_cdk.manifest_migrations.migrations.http_requester_url_base_to_url import ( + HttpRequesterUrlBaseToUrl, +) + +__all__ = [ + "HttpRequesterPathToUrl", + "HttpRequesterRequestBodyJsonDataToRequestBody", + "HttpRequesterUrlBaseToUrl", +] diff --git a/airbyte_cdk/manifest_migrations/migrations/registry.yaml b/airbyte_cdk/manifest_migrations/migrations/registry.yaml index 415b82b86..6bad4ba5b 100644 --- a/airbyte_cdk/manifest_migrations/migrations/registry.yaml +++ b/airbyte_cdk/manifest_migrations/migrations/registry.yaml @@ -3,7 +3,7 @@ # manifest_migrations: - - version: 6.48.3 + - version: "*" migrations: - name: http_requester_url_base_to_url order: 1 diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index f296af10d..14caf8865 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -6,11 +6,13 @@ import pytest +from airbyte_cdk.manifest_migrations.manifest_migration import ManifestMigration + @pytest.fixture def manifest_with_url_base_to_migrate_to_url() -> Dict[str, Any]: return { - "version": "0.0.0", + "version": "6.48.3", "type": "DeclarativeSource", "check": { "type": "CheckStream", @@ -493,14 +495,14 @@ def expected_manifest_with_url_base_migrated_to_url() -> Dict[str, Any]: "metadata": { "applied_migrations": [ { - "from_version": "0.0.0", - "to_version": "6.48.3", + "from_version": "6.48.3", + "to_version": ">=6.48.2,<6.50.0", "migration": "HttpRequesterUrlBaseToUrl", "migrated_at": "2025-04-01T00:00:00+00:00", # time freezed in the test }, { - "from_version": "0.0.0", - "to_version": "6.48.3", + "from_version": "6.48.3", + "to_version": ">=6.48.2,<6.50.0", "migration": "HttpRequesterPathToUrl", "migrated_at": "2025-04-01T00:00:00+00:00", # time freezed in the test }, @@ -832,7 +834,7 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic @pytest.fixture def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: return { - "version": "6.48.3", + "version": "0.0.0", "type": "DeclarativeSource", "check": {"type": "CheckStream", "stream_names": ["A"]}, "definitions": { @@ -1195,22 +1197,36 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: "applied_migrations": [ { "from_version": "0.0.0", - "to_version": "6.48.3", + "to_version": "*", "migration": "HttpRequesterUrlBaseToUrl", "migrated_at": "2025-04-01T00:00:00+00:00", }, { "from_version": "0.0.0", - "to_version": "6.48.3", + "to_version": "*", "migration": "HttpRequesterPathToUrl", "migrated_at": "2025-04-01T00:00:00+00:00", }, { "from_version": "0.0.0", - "to_version": "6.48.3", + "to_version": "*", "migration": "HttpRequesterRequestBodyJsonDataToRequestBody", "migrated_at": "2025-04-01T00:00:00+00:00", }, ] }, } + + +class DummyMigration(ManifestMigration): + def _process_manifest(self, manifest): + self.is_migrated = False + + def should_migrate(self, manifest): + return True + + def validate(self, manifest): + return True + + def migrate(self, manifest): + pass diff --git a/unit_tests/manifest_migrations/test_manifest_migration.py b/unit_tests/manifest_migrations/test_manifest_migration.py index 0ad897375..c5e2cebcc 100644 --- a/unit_tests/manifest_migrations/test_manifest_migration.py +++ b/unit_tests/manifest_migrations/test_manifest_migration.py @@ -1,21 +1,39 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # +from unittest.mock import patch from freezegun import freeze_time +from airbyte_cdk.manifest_migrations import migrations_registry from airbyte_cdk.manifest_migrations.migration_handler import ( ManifestMigrationHandler, ) -from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource +from airbyte_cdk.manifest_migrations.migrations import ( + HttpRequesterPathToUrl, + HttpRequesterRequestBodyJsonDataToRequestBody, + HttpRequesterUrlBaseToUrl, +) from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ( ManifestReferenceResolver, ) +from unit_tests.manifest_migrations.conftest import DummyMigration resolver = ManifestReferenceResolver() @freeze_time("2025-04-01") +@patch.dict( + migrations_registry.MANIFEST_MIGRATIONS, + { + ">=6.48.2,<6.50.0": [ + HttpRequesterUrlBaseToUrl, + HttpRequesterPathToUrl, + HttpRequesterRequestBodyJsonDataToRequestBody, + ] + }, + clear=True, +) def test_manifest_resolve_migrate_url_base_and_path_to_url( manifest_with_url_base_to_migrate_to_url, expected_manifest_with_url_base_migrated_to_url, @@ -25,7 +43,9 @@ def test_manifest_resolve_migrate_url_base_and_path_to_url( when the `url_base` is migrated to `url` and the `path` is joined to `url`. """ - resolved_manifest = resolver.preprocess_manifest(manifest_with_url_base_to_migrate_to_url) + resolved_manifest = ManifestReferenceResolver().preprocess_manifest( + manifest_with_url_base_to_migrate_to_url + ) migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations() assert migrated_manifest == expected_manifest_with_url_base_migrated_to_url @@ -50,6 +70,7 @@ def test_manifest_resolve_migrate_request_body_json_and_data_to_request_body( @freeze_time("2025-04-01") +@patch.dict(migrations_registry.MANIFEST_MIGRATIONS, {"0.0.0": [DummyMigration]}, clear=True) def test_manifest_resolve_do_not_migrate( manifest_with_migrated_url_base_and_path_is_joined_to_url, ) -> None: @@ -58,7 +79,7 @@ def test_manifest_resolve_do_not_migrate( after the `url_base` and `path` is joined to `url`. """ - resolved_manifest = resolver.preprocess_manifest( + resolved_manifest = ManifestReferenceResolver().preprocess_manifest( manifest_with_migrated_url_base_and_path_is_joined_to_url ) migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations()