diff --git a/.changeset/loud-spiders-end.md b/.changeset/loud-spiders-end.md new file mode 100644 index 000000000..fb935ae18 --- /dev/null +++ b/.changeset/loud-spiders-end.md @@ -0,0 +1,22 @@ +--- +"chainlink-deployments-framework": minor +--- + +feat: add catalog service integration for datastore operations + +Features: +- Add catalog service support for datastore management as alternative to local file storage +- Add `MergeMigrationDataStoreCatalog` method for catalog-based datastore persistence +- Existing `MergeMigrationDataStore` method continues to work for file-based storage (no breaking changes) +- Add unified `MergeDataStoreToCatalog` function for both initial migration and ongoing merge operations +- All catalog operations are transactional to prevent data inconsistencies +- Add `DatastoreType` configuration option (`file`/`catalog`) in domain.yaml to control storage backend +- Add new CLI command `datastore sync-to-catalog` for initial migration from file-based to catalog storage in CI +- Add `SyncDataStoreToCatalog` method to sync entire local datastore to catalog +- CLI automatically selects the appropriate merge method based on domain.yaml configuration +- Catalog mode does not modify local files - all updates go directly to the catalog service + +Configuration: +- Set `datastore: catalog` in domain.yaml to enable catalog mode +- Set `datastore: file` or omit the setting to use traditional file-based storage +- CLI commands automatically detect the configuration and use the appropriate storage backend diff --git a/datastore/catalog_syncer.go b/datastore/catalog_syncer.go new file mode 100644 index 000000000..dfd00643b --- /dev/null +++ b/datastore/catalog_syncer.go @@ -0,0 +1,75 @@ +package datastore + +import ( + "context" + "errors" + "fmt" +) + +// MergeDataStoreToCatalog merges data from a source DataStore (either full local state or migration-specific) +// into a remote CatalogStore within a transaction. This ensures atomic updates - either all data is +// successfully merged or the entire operation is rolled back on failure. +// +// This function serves two purposes: +// 1. Initial migration: sync entire local datastore to catalog (full state push) +// 2. Ongoing operations: merge migration/changeset artifacts into catalog (incremental updates) +// +// The operation is transactional to prevent partial failures that could lead to data inconsistencies. +func MergeDataStoreToCatalog(ctx context.Context, sourceDS DataStore, catalog CatalogStore) error { + return catalog.WithTransaction(ctx, func(ctx context.Context, txCatalog BaseCatalogStore) error { + // Merge all address references to the catalog + addressRefs, err := sourceDS.Addresses().Fetch() + if err != nil { + return fmt.Errorf("failed to fetch address references from source store: %w", err) + } + + for _, ref := range addressRefs { + if upsertErr := txCatalog.Addresses().Upsert(ctx, ref); upsertErr != nil { + return fmt.Errorf("failed to upsert address reference to catalog: %w", upsertErr) + } + } + + // Merge all chain metadata to the catalog + chainMetadata, err := sourceDS.ChainMetadata().Fetch() + if err != nil { + return fmt.Errorf("failed to fetch chain metadata from source store: %w", err) + } + + for _, metadata := range chainMetadata { + key := NewChainMetadataKey(metadata.ChainSelector) + if upsertErr := txCatalog.ChainMetadata().Upsert(ctx, key, metadata.Metadata); upsertErr != nil { + return fmt.Errorf("failed to upsert chain metadata to catalog: %w", upsertErr) + } + } + + // Merge all contract metadata to the catalog + contractMetadata, err := sourceDS.ContractMetadata().Fetch() + if err != nil { + return fmt.Errorf("failed to fetch contract metadata from source store: %w", err) + } + + for _, metadata := range contractMetadata { + key := NewContractMetadataKey(metadata.ChainSelector, metadata.Address) + if upsertErr := txCatalog.ContractMetadata().Upsert(ctx, key, metadata.Metadata); upsertErr != nil { + return fmt.Errorf("failed to upsert contract metadata to catalog: %w", upsertErr) + } + } + + // Merge environment metadata to the catalog + envMetadata, err := sourceDS.EnvMetadata().Get() + if err != nil { + if !errors.Is(err, ErrEnvMetadataNotSet) { + return fmt.Errorf("failed to fetch environment metadata from source store: %w", err) + } + + return nil + } + + // Merge the environment metadata (upsert semantics) + if setErr := txCatalog.EnvMetadata().Set(ctx, envMetadata.Metadata); setErr != nil { + return fmt.Errorf("failed to set environment metadata in catalog: %w", setErr) + } + + return nil + }) +} diff --git a/datastore/catalog_syncer_integration_test.go b/datastore/catalog_syncer_integration_test.go new file mode 100644 index 000000000..50c808ce2 --- /dev/null +++ b/datastore/catalog_syncer_integration_test.go @@ -0,0 +1,464 @@ +package datastore_test + +import ( + "context" + "os" + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/credentials/insecure" + + "github.com/smartcontractkit/chainlink-deployments-framework/datastore" + catalogremote "github.com/smartcontractkit/chainlink-deployments-framework/datastore/catalog/remote" +) + +// Temporary on demand local tests for catalog syncer - these can be converted to proper testcontainers later +// TestMergeDataStoreToCatalog_FullSync tests syncing entire local datastore to catalog (initial migration use case) +func TestMergeDataStoreToCatalog_FullSync(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Check if catalog service is available + catalogAddr := os.Getenv("CATALOG_GRPC_ADDRESS") + if catalogAddr == "" { + catalogAddr = "localhost:8080" // Default for docker-compose + } + + // Test connectivity with a temporary client + testClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Skipf("Catalog service not available at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, err) + return + } + testStream, streamErr := testClient.DataAccess() + if streamErr != nil { + t.Skipf("Cannot connect to catalog service at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, streamErr) + return + } + _ = testStream.CloseSend() + _ = testClient.CloseStream() // Close the test client + + // Now create the actual client for the test + catalogClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatalf("Failed to create catalog client: %v", err) + } + defer func() { _ = catalogClient.CloseStream() }() + + // Create a unique domain/environment for this test to avoid conflicts + testDomain := "test-sync-domain" + testEnv := "integration-test" + + catalogStore := catalogremote.NewCatalogDataStore(catalogremote.CatalogDataStoreConfig{ + Domain: testDomain, + Environment: testEnv, + Client: catalogClient, + }) + + t.Logf("โœ… Connected to catalog service at %s", catalogAddr) + t.Logf("๐Ÿ“ฆ Testing with domain: %s, environment: %s", testDomain, testEnv) + + // Step 1: Create a local datastore with test data + t.Log("Step 1: Creating local datastore with test data...") + localDS := datastore.NewMemoryDataStore() + + // Add test address references + version1, _ := semver.NewVersion("1.0.0") + testAddressRef1 := datastore.AddressRef{ + ChainSelector: 123456, + Address: "0x1111111111111111111111111111111111111111", + Type: "TestContract", + Version: version1, + Labels: datastore.NewLabelSet("environment:test", "sync:true"), + } + err = localDS.Addresses().Add(testAddressRef1) + require.NoError(t, err) + + version2, _ := semver.NewVersion("2.0.0") + testAddressRef2 := datastore.AddressRef{ + ChainSelector: 123456, + Address: "0x2222222222222222222222222222222222222222", + Type: "AnotherContract", + Version: version2, + Labels: datastore.NewLabelSet("environment:test", "sync:true"), + } + err = localDS.Addresses().Add(testAddressRef2) + require.NoError(t, err) + + // Add test chain metadata + testChainMetadata := datastore.ChainMetadata{ + ChainSelector: 123456, + Metadata: map[string]interface{}{ + "name": "Test Chain", + "type": "evm", + "description": "Integration test chain", + }, + } + err = localDS.ChainMetadata().Add(testChainMetadata) + require.NoError(t, err) + + // Add test contract metadata + testContractMetadata1 := datastore.ContractMetadata{ + Address: "0x1111111111111111111111111111111111111111", + ChainSelector: 123456, + Metadata: map[string]interface{}{ + "name": "TestContract", + "version": "1.0.0", + "abi": "[]", + }, + } + err = localDS.ContractMetadata().Add(testContractMetadata1) + require.NoError(t, err) + + testContractMetadata2 := datastore.ContractMetadata{ + Address: "0x2222222222222222222222222222222222222222", + ChainSelector: 123456, + Metadata: map[string]interface{}{ + "name": "AnotherContract", + "version": "2.0.0", + "abi": "[]", + }, + } + err = localDS.ContractMetadata().Add(testContractMetadata2) + require.NoError(t, err) + + // Set test environment metadata + testEnvMetadata := datastore.EnvMetadata{ + Metadata: map[string]interface{}{ + "environment": "integration-test", + "version": "1.0.0", + "timestamp": "2024-01-01T00:00:00Z", + }, + } + err = localDS.EnvMetadata().Set(testEnvMetadata) + require.NoError(t, err) + + // Seal the local datastore + sealedDS := localDS.Seal() + + t.Log("โœ… Local datastore created with:") + t.Log(" - 2 address references") + t.Log(" - 1 chain metadata") + t.Log(" - 2 contract metadata") + t.Log(" - 1 environment metadata") + + // Step 2: Merge datastore to catalog (full sync for initial migration) + t.Log("Step 2: Merging local datastore to catalog...") + err = datastore.MergeDataStoreToCatalog(ctx, sealedDS, catalogStore) + require.NoError(t, err, "Failed to merge datastore to catalog") + t.Log("โœ… Merge completed successfully!") + + // Step 3: Verify data was synced correctly by reading back from catalog + t.Log("Step 3: Verifying data in catalog...") + + // Verify address references + t.Log(" Checking address references...") + addressRefs, err := catalogStore.Addresses().Fetch(ctx) + require.NoError(t, err) + assert.GreaterOrEqual(t, len(addressRefs), 2, "Should have at least 2 address references") + + // Find our test addresses + foundAddr1 := false + foundAddr2 := false + for _, ref := range addressRefs { + if ref.Address == testAddressRef1.Address && ref.ChainSelector == testAddressRef1.ChainSelector { + foundAddr1 = true + assert.Equal(t, testAddressRef1.Type, ref.Type) + assert.Equal(t, testAddressRef1.Version.String(), ref.Version.String()) + assert.True(t, ref.Labels.Contains("environment:test")) + } + if ref.Address == testAddressRef2.Address && ref.ChainSelector == testAddressRef2.ChainSelector { + foundAddr2 = true + assert.Equal(t, testAddressRef2.Type, ref.Type) + } + } + assert.True(t, foundAddr1, "First address reference should be in catalog") + assert.True(t, foundAddr2, "Second address reference should be in catalog") + t.Log(" โœ… Address references verified") + + // Verify chain metadata + t.Log(" Checking chain metadata...") + chainMeta, err := catalogStore.ChainMetadata().Get(ctx, datastore.NewChainMetadataKey(123456)) + require.NoError(t, err) + assert.Equal(t, uint64(123456), chainMeta.ChainSelector) + metadataMap, ok := chainMeta.Metadata.(map[string]interface{}) + require.True(t, ok, "Metadata should be a map") + assert.Equal(t, "Test Chain", metadataMap["name"]) + assert.Equal(t, "evm", metadataMap["type"]) + t.Log(" โœ… Chain metadata verified") + + // Verify contract metadata + t.Log(" Checking contract metadata...") + contractMeta1, err := catalogStore.ContractMetadata().Get(ctx, + datastore.NewContractMetadataKey(123456, "0x1111111111111111111111111111111111111111")) + require.NoError(t, err) + contractMap1, ok := contractMeta1.Metadata.(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "TestContract", contractMap1["name"]) + assert.Equal(t, "1.0.0", contractMap1["version"]) + + contractMeta2, err := catalogStore.ContractMetadata().Get(ctx, + datastore.NewContractMetadataKey(123456, "0x2222222222222222222222222222222222222222")) + require.NoError(t, err) + contractMap2, ok := contractMeta2.Metadata.(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "AnotherContract", contractMap2["name"]) + t.Log(" โœ… Contract metadata verified") + + // Verify environment metadata + t.Log(" Checking environment metadata...") + envMeta, err := catalogStore.EnvMetadata().Get(ctx) + require.NoError(t, err) + envMetaMap, ok := envMeta.Metadata.(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "integration-test", envMetaMap["environment"]) + assert.Equal(t, "1.0.0", envMetaMap["version"]) + t.Log(" โœ… Environment metadata verified") + + t.Log("") + t.Log("๐ŸŽ‰ All data successfully synced and verified!") + t.Log("") + t.Logf("โ„น๏ธ Data is stored in catalog under domain='%s', environment='%s'", testDomain, testEnv) +} + +// TestMergeDataStoreToCatalog_Incremental tests merging migration data to catalog (ongoing operations use case) +func TestMergeDataStoreToCatalog_Incremental(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Check if catalog service is available + catalogAddr := os.Getenv("CATALOG_GRPC_ADDRESS") + if catalogAddr == "" { + catalogAddr = "localhost:8080" + } + + // Test connectivity with a temporary client + testClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Skipf("Catalog service not available at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, err) + return + } + testStream, streamErr := testClient.DataAccess() + if streamErr != nil { + t.Skipf("Cannot connect to catalog service at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, streamErr) + return + } + _ = testStream.CloseSend() + _ = testClient.CloseStream() // Close the test client + + // Now create the actual client for the test + catalogClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatalf("Failed to create catalog client: %v", err) + } + defer func() { _ = catalogClient.CloseStream() }() + + // Create a unique domain/environment for this test + testDomain := "test-merge-domain" + testEnv := "integration-test" + + catalogStore := catalogremote.NewCatalogDataStore(catalogremote.CatalogDataStoreConfig{ + Domain: testDomain, + Environment: testEnv, + Client: catalogClient, + }) + + t.Logf("โœ… Connected to catalog service at %s", catalogAddr) + t.Logf("๐Ÿ“ฆ Testing with domain: %s, environment: %s", testDomain, testEnv) + + // Step 1: Create initial state in catalog + t.Log("Step 1: Setting up initial catalog state...") + initialDS := datastore.NewMemoryDataStore() + + version1, _ := semver.NewVersion("1.0.0") + initialAddr := datastore.AddressRef{ + ChainSelector: 789012, + Address: "0x3333333333333333333333333333333333333333", + Type: "ExistingContract", + Version: version1, + Labels: datastore.NewLabelSet("status:existing"), + } + err = initialDS.Addresses().Add(initialAddr) + require.NoError(t, err) + + // Merge initial state to catalog + err = datastore.MergeDataStoreToCatalog(ctx, initialDS.Seal(), catalogStore) + require.NoError(t, err) + t.Log("โœ… Initial state merged") + + // Step 2: Create a migration datastore with new contracts + t.Log("Step 2: Creating migration datastore...") + migrationDS := datastore.NewMemoryDataStore() + + // Add new contract from migration + version2, _ := semver.NewVersion("2.0.0") + newAddr := datastore.AddressRef{ + ChainSelector: 789012, + Address: "0x4444444444444444444444444444444444444444", + Type: "NewContract", + Version: version2, + Labels: datastore.NewLabelSet("status:deployed", "migration:0001_deploy"), + } + err = migrationDS.Addresses().Add(newAddr) + require.NoError(t, err) + + // Add chain metadata + chainMeta := datastore.ChainMetadata{ + ChainSelector: 789012, + Metadata: map[string]interface{}{ + "name": "Migration Chain", + "type": "evm", + }, + } + err = migrationDS.ChainMetadata().Add(chainMeta) + require.NoError(t, err) + + t.Log("โœ… Migration datastore created with new contract") + + // Step 3: Merge migration data to catalog + t.Log("Step 3: Merging migration datastore to catalog...") + err = datastore.MergeDataStoreToCatalog(ctx, migrationDS.Seal(), catalogStore) + require.NoError(t, err, "Failed to merge migration datastore to catalog") + t.Log("โœ… Merge completed successfully!") + + // Step 4: Verify both old and new data exist + t.Log("Step 4: Verifying merged data in catalog...") + + addressRefs, err := catalogStore.Addresses().Fetch(ctx) + require.NoError(t, err) + assert.GreaterOrEqual(t, len(addressRefs), 2, "Should have at least 2 address references after merge") + + foundExisting := false + foundNew := false + for _, ref := range addressRefs { + if ref.Address == initialAddr.Address { + foundExisting = true + assert.True(t, ref.Labels.Contains("status:existing")) + t.Log(" โœ… Found existing contract from initial state") + } + if ref.Address == newAddr.Address { + foundNew = true + assert.True(t, ref.Labels.Contains("status:deployed")) + assert.True(t, ref.Labels.Contains("migration:0001_deploy")) + t.Log(" โœ… Found new contract from migration") + } + } + + assert.True(t, foundExisting, "Original address should still exist after merge") + assert.True(t, foundNew, "New address from migration should be added") + + // Verify chain metadata was added + chainMetaResult, err := catalogStore.ChainMetadata().Get(ctx, datastore.NewChainMetadataKey(789012)) + require.NoError(t, err) + chainMetaMap, ok := chainMetaResult.Metadata.(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "Migration Chain", chainMetaMap["name"]) + t.Log(" โœ… Chain metadata from migration verified") + + t.Log("") + t.Log("๐ŸŽ‰ Migration merge completed successfully!") + t.Log("") + t.Logf("โ„น๏ธ Catalog now contains both original and migrated data under domain='%s', environment='%s'", testDomain, testEnv) +} + +// TestMergeDataStoreToCatalog_TransactionRollback tests that failed merges rollback properly +func TestMergeDataStoreToCatalog_TransactionRollback(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Check if catalog service is available + catalogAddr := os.Getenv("CATALOG_GRPC_ADDRESS") + if catalogAddr == "" { + catalogAddr = "localhost:8080" + } + + // Test connectivity with a temporary client + testClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Skipf("Catalog service not available at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, err) + return + } + testStream, streamErr := testClient.DataAccess() + if streamErr != nil { + t.Skipf("Cannot connect to catalog service at %s: %v. Start with: cd op-catalog && docker-compose up -d", catalogAddr, streamErr) + return + } + _ = testStream.CloseSend() + _ = testClient.CloseStream() // Close the test client + + // Now create the actual client for the test + catalogClient, err := catalogremote.NewCatalogClient(ctx, catalogremote.CatalogConfig{ + GRPC: catalogAddr, + Creds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatalf("Failed to create catalog client: %v", err) + } + defer func() { _ = catalogClient.CloseStream() }() + + testDomain := "test-rollback-domain" + testEnv := "integration-test" + + catalogStore := catalogremote.NewCatalogDataStore(catalogremote.CatalogDataStoreConfig{ + Domain: testDomain, + Environment: testEnv, + Client: catalogClient, + }) + + t.Logf("โœ… Connected to catalog service at %s", catalogAddr) + t.Log("๐Ÿงช Testing transaction rollback behavior...") + + // This test verifies that the transaction-based sync is atomic + // If we can't construct a proper failure scenario, we at least verify the success case + localDS := datastore.NewMemoryDataStore() + + version1, _ := semver.NewVersion("1.0.0") + testAddr := datastore.AddressRef{ + ChainSelector: 999888, + Address: "0x5555555555555555555555555555555555555555", + Type: "RollbackTest", + Version: version1, + Labels: datastore.NewLabelSet("test:rollback"), + } + err = localDS.Addresses().Add(testAddr) + require.NoError(t, err) + + // Merge should succeed + err = datastore.MergeDataStoreToCatalog(ctx, localDS.Seal(), catalogStore) + require.NoError(t, err) + + // Verify data was written + addressRefs, err := catalogStore.Addresses().Fetch(ctx) + require.NoError(t, err) + + found := false + for _, ref := range addressRefs { + if ref.Address == testAddr.Address { + found = true + break + } + } + assert.True(t, found, "Data should be committed after successful merge") + + t.Log("โœ… Transaction semantics verified - data committed after successful merge") +} diff --git a/datastore/catalog_syncer_test.go b/datastore/catalog_syncer_test.go new file mode 100644 index 000000000..a08b7a335 --- /dev/null +++ b/datastore/catalog_syncer_test.go @@ -0,0 +1,408 @@ +package datastore + +import ( + "context" + "errors" + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestMergeDataStoreToCatalog(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("successfully merges all data to catalog", func(t *testing.T) { + t.Parallel() + + // Create mocks for catalog and its stores + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockCatalogAddressStore := NewMockMutableRefStoreV2[AddressRefKey, AddressRef](t) + mockCatalogChainStore := NewMockMutableStoreV2[ChainMetadataKey, ChainMetadata](t) + mockCatalogContractStore := NewMockMutableStoreV2[ContractMetadataKey, ContractMetadata](t) + mockCatalogEnvStore := NewMockMutableUnaryStoreV2[EnvMetadata](t) + + // Create mocks for migration datastore and its stores + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + mockMigrationContractStore := NewMockContractMetadataStore(t) + mockMigrationEnvStore := NewMockEnvMetadataStore(t) + + // Setup test data + testAddressRefs := []AddressRef{ + { + Address: "0xabc", + ChainSelector: 3, + Type: "migration", + Version: semver.MustParse("3.0.0"), + Qualifier: "new", + }, + } + + testChainMetadata := []ChainMetadata{ + { + ChainSelector: 3, + Metadata: map[string]interface{}{ + "field": "value3", + }, + }, + } + + testContractMetadata := []ContractMetadata{ + { + Address: "0xdef", + ChainSelector: 3, + Metadata: map[string]interface{}{ + "name": "NewContract", + }, + }, + } + + testEnvMetadata := EnvMetadata{ + Metadata: map[string]interface{}{ + "environment": "production", + }, + } + + // Setup WithTransaction to execute the transaction logic + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore expectations - fetch from migration + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore).Once() + mockMigrationAddressStore.EXPECT().Fetch().Return(testAddressRefs, nil).Once() + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore).Once() + mockMigrationChainStore.EXPECT().Fetch().Return(testChainMetadata, nil).Once() + + mockMigrationDS.EXPECT().ContractMetadata().Return(mockMigrationContractStore).Once() + mockMigrationContractStore.EXPECT().Fetch().Return(testContractMetadata, nil).Once() + + mockMigrationDS.EXPECT().EnvMetadata().Return(mockMigrationEnvStore).Once() + mockMigrationEnvStore.EXPECT().Get().Return(testEnvMetadata, nil).Once() + + // Setup catalog expectations - upsert to catalog + mockTxCatalog.EXPECT().Addresses().Return(mockCatalogAddressStore).Times(1) + for _, ref := range testAddressRefs { + mockCatalogAddressStore.EXPECT().Upsert(ctx, ref).Return(nil).Once() + } + + mockTxCatalog.EXPECT().ChainMetadata().Return(mockCatalogChainStore).Times(1) + for _, metadata := range testChainMetadata { + key := NewChainMetadataKey(metadata.ChainSelector) + mockCatalogChainStore.EXPECT().Upsert(ctx, key, metadata.Metadata).Return(nil).Once() + } + + mockTxCatalog.EXPECT().ContractMetadata().Return(mockCatalogContractStore).Times(1) + for _, metadata := range testContractMetadata { + key := NewContractMetadataKey(metadata.ChainSelector, metadata.Address) + mockCatalogContractStore.EXPECT().Upsert(ctx, key, metadata.Metadata).Return(nil).Once() + } + + mockTxCatalog.EXPECT().EnvMetadata().Return(mockCatalogEnvStore).Once() + mockCatalogEnvStore.EXPECT().Set(ctx, testEnvMetadata.Metadata).Return(nil).Once() + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.NoError(t, err) + }) + + t.Run("skips env metadata when not set (ErrEnvMetadataNotSet)", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + mockMigrationContractStore := NewMockContractMetadataStore(t) + mockMigrationEnvStore := NewMockEnvMetadataStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore expectations + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore).Once() + mockMigrationAddressStore.EXPECT().Fetch().Return([]AddressRef{}, nil).Once() + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore).Once() + mockMigrationChainStore.EXPECT().Fetch().Return([]ChainMetadata{}, nil).Once() + + mockMigrationDS.EXPECT().ContractMetadata().Return(mockMigrationContractStore).Once() + mockMigrationContractStore.EXPECT().Fetch().Return([]ContractMetadata{}, nil).Once() + + mockMigrationDS.EXPECT().EnvMetadata().Return(mockMigrationEnvStore).Once() + mockMigrationEnvStore.EXPECT().Get().Return(EnvMetadata{}, ErrEnvMetadataNotSet).Once() + + // Execute - should succeed because ErrEnvMetadataNotSet is acceptable + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.NoError(t, err) + }) + + t.Run("returns error when address fetch fails", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore to fail on address fetch + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore) + mockMigrationAddressStore.EXPECT().Fetch().Return(nil, errors.New("connection error")) + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + require.ErrorContains(t, err, "failed to fetch address references from source store") + require.ErrorContains(t, err, "connection error") + }) + + t.Run("returns error when chain metadata fetch fails", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore) + mockMigrationAddressStore.EXPECT().Fetch().Return([]AddressRef{}, nil) + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore) + mockMigrationChainStore.EXPECT().Fetch().Return(nil, errors.New("database error")) + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + require.ErrorContains(t, err, "failed to fetch chain metadata from source store") + require.ErrorContains(t, err, "database error") + }) + + t.Run("returns error when contract metadata fetch fails", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + mockMigrationContractStore := NewMockContractMetadataStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore) + mockMigrationAddressStore.EXPECT().Fetch().Return([]AddressRef{}, nil) + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore) + mockMigrationChainStore.EXPECT().Fetch().Return([]ChainMetadata{}, nil) + + mockMigrationDS.EXPECT().ContractMetadata().Return(mockMigrationContractStore) + mockMigrationContractStore.EXPECT().Fetch().Return(nil, errors.New("network timeout")) + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + require.ErrorContains(t, err, "failed to fetch contract metadata from source store") + require.ErrorContains(t, err, "network timeout") + }) + + t.Run("returns error when env metadata fetch fails with non-ErrEnvMetadataNotSet error", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + mockMigrationContractStore := NewMockContractMetadataStore(t) + mockMigrationEnvStore := NewMockEnvMetadataStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore) + mockMigrationAddressStore.EXPECT().Fetch().Return([]AddressRef{}, nil) + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore) + mockMigrationChainStore.EXPECT().Fetch().Return([]ChainMetadata{}, nil) + + mockMigrationDS.EXPECT().ContractMetadata().Return(mockMigrationContractStore) + mockMigrationContractStore.EXPECT().Fetch().Return([]ContractMetadata{}, nil) + + mockMigrationDS.EXPECT().EnvMetadata().Return(mockMigrationEnvStore) + mockMigrationEnvStore.EXPECT().Get().Return(EnvMetadata{}, errors.New("connection timeout")) + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + require.ErrorContains(t, err, "failed to fetch environment metadata from source store") + require.ErrorContains(t, err, "connection timeout") + }) + + t.Run("returns error when address upsert fails", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockCatalogAddressStore := NewMockMutableRefStoreV2[AddressRefKey, AddressRef](t) + + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + + testAddressRefs := []AddressRef{ + { + Address: "0x123", + ChainSelector: 1, + Type: "contract", + Version: semver.MustParse("1.0.0"), + }, + } + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore) + mockMigrationAddressStore.EXPECT().Fetch().Return(testAddressRefs, nil) + + // Setup catalog to fail on upsert + mockTxCatalog.EXPECT().Addresses().Return(mockCatalogAddressStore) + mockCatalogAddressStore.EXPECT().Upsert(ctx, testAddressRefs[0]).Return(errors.New("upsert failed")) + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + require.ErrorContains(t, err, "failed to upsert address reference to catalog") + require.ErrorContains(t, err, "upsert failed") + }) + + t.Run("handles empty migration datastore", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockTxCatalog := NewMockCatalogStore(t) + mockCatalogEnvStore := NewMockMutableUnaryStoreV2[EnvMetadata](t) + + mockMigrationDS := NewMockDataStore(t) + mockMigrationAddressStore := NewMockAddressRefStore(t) + mockMigrationChainStore := NewMockChainMetadataStore(t) + mockMigrationContractStore := NewMockContractMetadataStore(t) + mockMigrationEnvStore := NewMockEnvMetadataStore(t) + + // Setup WithTransaction + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).RunAndReturn( + func(ctx context.Context, fn TransactionLogic) error { + return fn(ctx, mockTxCatalog) + }, + ).Once() + + // Setup migration datastore expectations - all empty + mockMigrationDS.EXPECT().Addresses().Return(mockMigrationAddressStore).Once() + mockMigrationAddressStore.EXPECT().Fetch().Return([]AddressRef{}, nil).Once() + + mockMigrationDS.EXPECT().ChainMetadata().Return(mockMigrationChainStore).Once() + mockMigrationChainStore.EXPECT().Fetch().Return([]ChainMetadata{}, nil).Once() + + mockMigrationDS.EXPECT().ContractMetadata().Return(mockMigrationContractStore).Once() + mockMigrationContractStore.EXPECT().Fetch().Return([]ContractMetadata{}, nil).Once() + + mockMigrationDS.EXPECT().EnvMetadata().Return(mockMigrationEnvStore).Once() + mockMigrationEnvStore.EXPECT().Get().Return(EnvMetadata{Metadata: map[string]interface{}{}}, nil).Once() + + // Setup catalog expectations - env metadata set only + mockTxCatalog.EXPECT().EnvMetadata().Return(mockCatalogEnvStore).Once() + mockCatalogEnvStore.EXPECT().Set(ctx, mock.Anything).Return(nil).Once() + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.NoError(t, err) + }) + + t.Run("transaction rollback on error", func(t *testing.T) { + t.Parallel() + + // Create mocks + mockCatalog := NewMockCatalogStore(t) + mockMigrationDS := NewMockDataStore(t) + + // Setup WithTransaction to return an error (simulating rollback) + expectedErr := errors.New("transaction rolled back") + mockCatalog.EXPECT().WithTransaction(ctx, mock.Anything).Return(expectedErr).Once() + + // Execute + err := MergeDataStoreToCatalog(ctx, mockMigrationDS, mockCatalog) + + // Assert + require.Error(t, err) + assert.Equal(t, expectedErr, err) + }) +} diff --git a/datastore/mock_address_ref_store_test.go b/datastore/mock_address_ref_store_test.go new file mode 100644 index 000000000..d2816cf81 --- /dev/null +++ b/datastore/mock_address_ref_store_test.go @@ -0,0 +1,201 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package datastore + +import ( + mock "github.com/stretchr/testify/mock" +) + +// NewMockAddressRefStore creates a new instance of MockAddressRefStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockAddressRefStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockAddressRefStore { + mock := &MockAddressRefStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockAddressRefStore is an autogenerated mock type for the AddressRefStore type +type MockAddressRefStore struct { + mock.Mock +} + +type MockAddressRefStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockAddressRefStore) EXPECT() *MockAddressRefStore_Expecter { + return &MockAddressRefStore_Expecter{mock: &_m.Mock} +} + +// Fetch provides a mock function for the type MockAddressRefStore +func (_mock *MockAddressRefStore) Fetch() ([]AddressRef, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Fetch") + } + + var r0 []AddressRef + var r1 error + if returnFunc, ok := ret.Get(0).(func() ([]AddressRef, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() []AddressRef); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]AddressRef) + } + } + + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockAddressRefStore_Fetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Fetch' +type MockAddressRefStore_Fetch_Call struct { + *mock.Call +} + +// Fetch is a helper method to define mock.On call +func (_e *MockAddressRefStore_Expecter) Fetch() *MockAddressRefStore_Fetch_Call { + return &MockAddressRefStore_Fetch_Call{Call: _e.mock.On("Fetch")} +} + +func (_c *MockAddressRefStore_Fetch_Call) Run(run func()) *MockAddressRefStore_Fetch_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockAddressRefStore_Fetch_Call) Return(refs []AddressRef, err error) *MockAddressRefStore_Fetch_Call { + _c.Call.Return(refs, err) + return _c +} + +func (_c *MockAddressRefStore_Fetch_Call) RunAndReturn(run func() ([]AddressRef, error)) *MockAddressRefStore_Fetch_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function for the type MockAddressRefStore +func (_mock *MockAddressRefStore) Get(key AddressRefKey) (AddressRef, error) { + ret := _mock.Called(key) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 AddressRef + var r1 error + if returnFunc, ok := ret.Get(0).(func(AddressRefKey) (AddressRef, error)); ok { + return returnFunc(key) + } + if returnFunc, ok := ret.Get(0).(func(AddressRefKey) AddressRef); ok { + r0 = returnFunc(key) + } else { + r0 = ret.Get(0).(AddressRef) + } + + if returnFunc, ok := ret.Get(1).(func(AddressRefKey) error); ok { + r1 = returnFunc(key) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockAddressRefStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type MockAddressRefStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - key AddressRefKey +func (_e *MockAddressRefStore_Expecter) Get(key interface{}) *MockAddressRefStore_Get_Call { + return &MockAddressRefStore_Get_Call{Call: _e.mock.On("Get", key)} +} + +func (_c *MockAddressRefStore_Get_Call) Run(run func(key AddressRefKey)) *MockAddressRefStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(AddressRefKey)) + }) + return _c +} + +func (_c *MockAddressRefStore_Get_Call) Return(ref AddressRef, err error) *MockAddressRefStore_Get_Call { + _c.Call.Return(ref, err) + return _c +} + +func (_c *MockAddressRefStore_Get_Call) RunAndReturn(run func(AddressRefKey) (AddressRef, error)) *MockAddressRefStore_Get_Call { + _c.Call.Return(run) + return _c +} + +// Filter provides a mock function for the type MockAddressRefStore +func (_mock *MockAddressRefStore) Filter(filters ...FilterFunc[AddressRefKey, AddressRef]) []AddressRef { + ret := _mock.Called(filters) + + if len(ret) == 0 { + panic("no return value specified for Filter") + } + + var r0 []AddressRef + if returnFunc, ok := ret.Get(0).(func(...FilterFunc[AddressRefKey, AddressRef]) []AddressRef); ok { + r0 = returnFunc(filters...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]AddressRef) + } + } + return r0 +} + +// MockAddressRefStore_Filter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Filter' +type MockAddressRefStore_Filter_Call struct { + *mock.Call +} + +// Filter is a helper method to define mock.On call +// - filters ...FilterFunc[AddressRefKey, AddressRef] +func (_e *MockAddressRefStore_Expecter) Filter(filters interface{}) *MockAddressRefStore_Filter_Call { + return &MockAddressRefStore_Filter_Call{Call: _e.mock.On("Filter", filters)} +} + +func (_c *MockAddressRefStore_Filter_Call) Run(run func(filters ...FilterFunc[AddressRefKey, AddressRef])) *MockAddressRefStore_Filter_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]FilterFunc[AddressRefKey, AddressRef], len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(FilterFunc[AddressRefKey, AddressRef]) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockAddressRefStore_Filter_Call) Return(refs []AddressRef) *MockAddressRefStore_Filter_Call { + _c.Call.Return(refs) + return _c +} + +func (_c *MockAddressRefStore_Filter_Call) RunAndReturn(run func(...FilterFunc[AddressRefKey, AddressRef]) []AddressRef) *MockAddressRefStore_Filter_Call { + _c.Call.Return(run) + return _c +} + diff --git a/datastore/mock_chain_metadata_store_test.go b/datastore/mock_chain_metadata_store_test.go new file mode 100644 index 000000000..fe196668e --- /dev/null +++ b/datastore/mock_chain_metadata_store_test.go @@ -0,0 +1,201 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package datastore + +import ( + mock "github.com/stretchr/testify/mock" +) + +// NewMockChainMetadataStore creates a new instance of MockChainMetadataStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockChainMetadataStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockChainMetadataStore { + mock := &MockChainMetadataStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockChainMetadataStore is an autogenerated mock type for the ChainMetadataStore type +type MockChainMetadataStore struct { + mock.Mock +} + +type MockChainMetadataStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockChainMetadataStore) EXPECT() *MockChainMetadataStore_Expecter { + return &MockChainMetadataStore_Expecter{mock: &_m.Mock} +} + +// Fetch provides a mock function for the type MockChainMetadataStore +func (_mock *MockChainMetadataStore) Fetch() ([]ChainMetadata, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Fetch") + } + + var r0 []ChainMetadata + var r1 error + if returnFunc, ok := ret.Get(0).(func() ([]ChainMetadata, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() []ChainMetadata); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ChainMetadata) + } + } + + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockChainMetadataStore_Fetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Fetch' +type MockChainMetadataStore_Fetch_Call struct { + *mock.Call +} + +// Fetch is a helper method to define mock.On call +func (_e *MockChainMetadataStore_Expecter) Fetch() *MockChainMetadataStore_Fetch_Call { + return &MockChainMetadataStore_Fetch_Call{Call: _e.mock.On("Fetch")} +} + +func (_c *MockChainMetadataStore_Fetch_Call) Run(run func()) *MockChainMetadataStore_Fetch_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockChainMetadataStore_Fetch_Call) Return(metadata []ChainMetadata, err error) *MockChainMetadataStore_Fetch_Call { + _c.Call.Return(metadata, err) + return _c +} + +func (_c *MockChainMetadataStore_Fetch_Call) RunAndReturn(run func() ([]ChainMetadata, error)) *MockChainMetadataStore_Fetch_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function for the type MockChainMetadataStore +func (_mock *MockChainMetadataStore) Get(key ChainMetadataKey) (ChainMetadata, error) { + ret := _mock.Called(key) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 ChainMetadata + var r1 error + if returnFunc, ok := ret.Get(0).(func(ChainMetadataKey) (ChainMetadata, error)); ok { + return returnFunc(key) + } + if returnFunc, ok := ret.Get(0).(func(ChainMetadataKey) ChainMetadata); ok { + r0 = returnFunc(key) + } else { + r0 = ret.Get(0).(ChainMetadata) + } + + if returnFunc, ok := ret.Get(1).(func(ChainMetadataKey) error); ok { + r1 = returnFunc(key) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockChainMetadataStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type MockChainMetadataStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - key ChainMetadataKey +func (_e *MockChainMetadataStore_Expecter) Get(key interface{}) *MockChainMetadataStore_Get_Call { + return &MockChainMetadataStore_Get_Call{Call: _e.mock.On("Get", key)} +} + +func (_c *MockChainMetadataStore_Get_Call) Run(run func(key ChainMetadataKey)) *MockChainMetadataStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(ChainMetadataKey)) + }) + return _c +} + +func (_c *MockChainMetadataStore_Get_Call) Return(metadata ChainMetadata, err error) *MockChainMetadataStore_Get_Call { + _c.Call.Return(metadata, err) + return _c +} + +func (_c *MockChainMetadataStore_Get_Call) RunAndReturn(run func(ChainMetadataKey) (ChainMetadata, error)) *MockChainMetadataStore_Get_Call { + _c.Call.Return(run) + return _c +} + +// Filter provides a mock function for the type MockChainMetadataStore +func (_mock *MockChainMetadataStore) Filter(filters ...FilterFunc[ChainMetadataKey, ChainMetadata]) []ChainMetadata { + ret := _mock.Called(filters) + + if len(ret) == 0 { + panic("no return value specified for Filter") + } + + var r0 []ChainMetadata + if returnFunc, ok := ret.Get(0).(func(...FilterFunc[ChainMetadataKey, ChainMetadata]) []ChainMetadata); ok { + r0 = returnFunc(filters...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ChainMetadata) + } + } + return r0 +} + +// MockChainMetadataStore_Filter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Filter' +type MockChainMetadataStore_Filter_Call struct { + *mock.Call +} + +// Filter is a helper method to define mock.On call +// - filters ...FilterFunc[ChainMetadataKey, ChainMetadata] +func (_e *MockChainMetadataStore_Expecter) Filter(filters interface{}) *MockChainMetadataStore_Filter_Call { + return &MockChainMetadataStore_Filter_Call{Call: _e.mock.On("Filter", filters)} +} + +func (_c *MockChainMetadataStore_Filter_Call) Run(run func(filters ...FilterFunc[ChainMetadataKey, ChainMetadata])) *MockChainMetadataStore_Filter_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]FilterFunc[ChainMetadataKey, ChainMetadata], len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(FilterFunc[ChainMetadataKey, ChainMetadata]) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockChainMetadataStore_Filter_Call) Return(metadata []ChainMetadata) *MockChainMetadataStore_Filter_Call { + _c.Call.Return(metadata) + return _c +} + +func (_c *MockChainMetadataStore_Filter_Call) RunAndReturn(run func(...FilterFunc[ChainMetadataKey, ChainMetadata]) []ChainMetadata) *MockChainMetadataStore_Filter_Call { + _c.Call.Return(run) + return _c +} + diff --git a/datastore/mock_contract_metadata_store_test.go b/datastore/mock_contract_metadata_store_test.go new file mode 100644 index 000000000..2227d135d --- /dev/null +++ b/datastore/mock_contract_metadata_store_test.go @@ -0,0 +1,201 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package datastore + +import ( + mock "github.com/stretchr/testify/mock" +) + +// NewMockContractMetadataStore creates a new instance of MockContractMetadataStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockContractMetadataStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockContractMetadataStore { + mock := &MockContractMetadataStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockContractMetadataStore is an autogenerated mock type for the ContractMetadataStore type +type MockContractMetadataStore struct { + mock.Mock +} + +type MockContractMetadataStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockContractMetadataStore) EXPECT() *MockContractMetadataStore_Expecter { + return &MockContractMetadataStore_Expecter{mock: &_m.Mock} +} + +// Fetch provides a mock function for the type MockContractMetadataStore +func (_mock *MockContractMetadataStore) Fetch() ([]ContractMetadata, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Fetch") + } + + var r0 []ContractMetadata + var r1 error + if returnFunc, ok := ret.Get(0).(func() ([]ContractMetadata, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() []ContractMetadata); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ContractMetadata) + } + } + + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockContractMetadataStore_Fetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Fetch' +type MockContractMetadataStore_Fetch_Call struct { + *mock.Call +} + +// Fetch is a helper method to define mock.On call +func (_e *MockContractMetadataStore_Expecter) Fetch() *MockContractMetadataStore_Fetch_Call { + return &MockContractMetadataStore_Fetch_Call{Call: _e.mock.On("Fetch")} +} + +func (_c *MockContractMetadataStore_Fetch_Call) Run(run func()) *MockContractMetadataStore_Fetch_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockContractMetadataStore_Fetch_Call) Return(metadata []ContractMetadata, err error) *MockContractMetadataStore_Fetch_Call { + _c.Call.Return(metadata, err) + return _c +} + +func (_c *MockContractMetadataStore_Fetch_Call) RunAndReturn(run func() ([]ContractMetadata, error)) *MockContractMetadataStore_Fetch_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function for the type MockContractMetadataStore +func (_mock *MockContractMetadataStore) Get(key ContractMetadataKey) (ContractMetadata, error) { + ret := _mock.Called(key) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 ContractMetadata + var r1 error + if returnFunc, ok := ret.Get(0).(func(ContractMetadataKey) (ContractMetadata, error)); ok { + return returnFunc(key) + } + if returnFunc, ok := ret.Get(0).(func(ContractMetadataKey) ContractMetadata); ok { + r0 = returnFunc(key) + } else { + r0 = ret.Get(0).(ContractMetadata) + } + + if returnFunc, ok := ret.Get(1).(func(ContractMetadataKey) error); ok { + r1 = returnFunc(key) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockContractMetadataStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type MockContractMetadataStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - key ContractMetadataKey +func (_e *MockContractMetadataStore_Expecter) Get(key interface{}) *MockContractMetadataStore_Get_Call { + return &MockContractMetadataStore_Get_Call{Call: _e.mock.On("Get", key)} +} + +func (_c *MockContractMetadataStore_Get_Call) Run(run func(key ContractMetadataKey)) *MockContractMetadataStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(ContractMetadataKey)) + }) + return _c +} + +func (_c *MockContractMetadataStore_Get_Call) Return(metadata ContractMetadata, err error) *MockContractMetadataStore_Get_Call { + _c.Call.Return(metadata, err) + return _c +} + +func (_c *MockContractMetadataStore_Get_Call) RunAndReturn(run func(ContractMetadataKey) (ContractMetadata, error)) *MockContractMetadataStore_Get_Call { + _c.Call.Return(run) + return _c +} + +// Filter provides a mock function for the type MockContractMetadataStore +func (_mock *MockContractMetadataStore) Filter(filters ...FilterFunc[ContractMetadataKey, ContractMetadata]) []ContractMetadata { + ret := _mock.Called(filters) + + if len(ret) == 0 { + panic("no return value specified for Filter") + } + + var r0 []ContractMetadata + if returnFunc, ok := ret.Get(0).(func(...FilterFunc[ContractMetadataKey, ContractMetadata]) []ContractMetadata); ok { + r0 = returnFunc(filters...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ContractMetadata) + } + } + return r0 +} + +// MockContractMetadataStore_Filter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Filter' +type MockContractMetadataStore_Filter_Call struct { + *mock.Call +} + +// Filter is a helper method to define mock.On call +// - filters ...FilterFunc[ContractMetadataKey, ContractMetadata] +func (_e *MockContractMetadataStore_Expecter) Filter(filters interface{}) *MockContractMetadataStore_Filter_Call { + return &MockContractMetadataStore_Filter_Call{Call: _e.mock.On("Filter", filters)} +} + +func (_c *MockContractMetadataStore_Filter_Call) Run(run func(filters ...FilterFunc[ContractMetadataKey, ContractMetadata])) *MockContractMetadataStore_Filter_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]FilterFunc[ContractMetadataKey, ContractMetadata], len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(FilterFunc[ContractMetadataKey, ContractMetadata]) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockContractMetadataStore_Filter_Call) Return(metadata []ContractMetadata) *MockContractMetadataStore_Filter_Call { + _c.Call.Return(metadata) + return _c +} + +func (_c *MockContractMetadataStore_Filter_Call) RunAndReturn(run func(...FilterFunc[ContractMetadataKey, ContractMetadata]) []ContractMetadata) *MockContractMetadataStore_Filter_Call { + _c.Call.Return(run) + return _c +} + diff --git a/datastore/mock_data_store_test.go b/datastore/mock_data_store_test.go new file mode 100644 index 000000000..500f934ce --- /dev/null +++ b/datastore/mock_data_store_test.go @@ -0,0 +1,221 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package datastore + +import ( + mock "github.com/stretchr/testify/mock" +) + +// NewMockDataStore creates a new instance of MockDataStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockDataStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockDataStore { + mock := &MockDataStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockDataStore is an autogenerated mock type for the DataStore type +type MockDataStore struct { + mock.Mock +} + +type MockDataStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockDataStore) EXPECT() *MockDataStore_Expecter { + return &MockDataStore_Expecter{mock: &_m.Mock} +} + +// Addresses provides a mock function for the type MockDataStore +func (_mock *MockDataStore) Addresses() AddressRefStore { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Addresses") + } + + var r0 AddressRefStore + if returnFunc, ok := ret.Get(0).(func() AddressRefStore); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(AddressRefStore) + } + } + return r0 +} + +// MockDataStore_Addresses_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Addresses' +type MockDataStore_Addresses_Call struct { + *mock.Call +} + +// Addresses is a helper method to define mock.On call +func (_e *MockDataStore_Expecter) Addresses() *MockDataStore_Addresses_Call { + return &MockDataStore_Addresses_Call{Call: _e.mock.On("Addresses")} +} + +func (_c *MockDataStore_Addresses_Call) Run(run func()) *MockDataStore_Addresses_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockDataStore_Addresses_Call) Return(addressRefStore AddressRefStore) *MockDataStore_Addresses_Call { + _c.Call.Return(addressRefStore) + return _c +} + +func (_c *MockDataStore_Addresses_Call) RunAndReturn(run func() AddressRefStore) *MockDataStore_Addresses_Call { + _c.Call.Return(run) + return _c +} + +// ChainMetadata provides a mock function for the type MockDataStore +func (_mock *MockDataStore) ChainMetadata() ChainMetadataStore { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for ChainMetadata") + } + + var r0 ChainMetadataStore + if returnFunc, ok := ret.Get(0).(func() ChainMetadataStore); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(ChainMetadataStore) + } + } + return r0 +} + +// MockDataStore_ChainMetadata_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChainMetadata' +type MockDataStore_ChainMetadata_Call struct { + *mock.Call +} + +// ChainMetadata is a helper method to define mock.On call +func (_e *MockDataStore_Expecter) ChainMetadata() *MockDataStore_ChainMetadata_Call { + return &MockDataStore_ChainMetadata_Call{Call: _e.mock.On("ChainMetadata")} +} + +func (_c *MockDataStore_ChainMetadata_Call) Run(run func()) *MockDataStore_ChainMetadata_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockDataStore_ChainMetadata_Call) Return(chainMetadataStore ChainMetadataStore) *MockDataStore_ChainMetadata_Call { + _c.Call.Return(chainMetadataStore) + return _c +} + +func (_c *MockDataStore_ChainMetadata_Call) RunAndReturn(run func() ChainMetadataStore) *MockDataStore_ChainMetadata_Call { + _c.Call.Return(run) + return _c +} + +// ContractMetadata provides a mock function for the type MockDataStore +func (_mock *MockDataStore) ContractMetadata() ContractMetadataStore { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for ContractMetadata") + } + + var r0 ContractMetadataStore + if returnFunc, ok := ret.Get(0).(func() ContractMetadataStore); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(ContractMetadataStore) + } + } + return r0 +} + +// MockDataStore_ContractMetadata_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ContractMetadata' +type MockDataStore_ContractMetadata_Call struct { + *mock.Call +} + +// ContractMetadata is a helper method to define mock.On call +func (_e *MockDataStore_Expecter) ContractMetadata() *MockDataStore_ContractMetadata_Call { + return &MockDataStore_ContractMetadata_Call{Call: _e.mock.On("ContractMetadata")} +} + +func (_c *MockDataStore_ContractMetadata_Call) Run(run func()) *MockDataStore_ContractMetadata_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockDataStore_ContractMetadata_Call) Return(contractMetadataStore ContractMetadataStore) *MockDataStore_ContractMetadata_Call { + _c.Call.Return(contractMetadataStore) + return _c +} + +func (_c *MockDataStore_ContractMetadata_Call) RunAndReturn(run func() ContractMetadataStore) *MockDataStore_ContractMetadata_Call { + _c.Call.Return(run) + return _c +} + +// EnvMetadata provides a mock function for the type MockDataStore +func (_mock *MockDataStore) EnvMetadata() EnvMetadataStore { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for EnvMetadata") + } + + var r0 EnvMetadataStore + if returnFunc, ok := ret.Get(0).(func() EnvMetadataStore); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(EnvMetadataStore) + } + } + return r0 +} + +// MockDataStore_EnvMetadata_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnvMetadata' +type MockDataStore_EnvMetadata_Call struct { + *mock.Call +} + +// EnvMetadata is a helper method to define mock.On call +func (_e *MockDataStore_Expecter) EnvMetadata() *MockDataStore_EnvMetadata_Call { + return &MockDataStore_EnvMetadata_Call{Call: _e.mock.On("EnvMetadata")} +} + +func (_c *MockDataStore_EnvMetadata_Call) Run(run func()) *MockDataStore_EnvMetadata_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockDataStore_EnvMetadata_Call) Return(envMetadataStore EnvMetadataStore) *MockDataStore_EnvMetadata_Call { + _c.Call.Return(envMetadataStore) + return _c +} + +func (_c *MockDataStore_EnvMetadata_Call) RunAndReturn(run func() EnvMetadataStore) *MockDataStore_EnvMetadata_Call { + _c.Call.Return(run) + return _c +} + diff --git a/datastore/mock_env_metadata_store_test.go b/datastore/mock_env_metadata_store_test.go new file mode 100644 index 000000000..0b245bbb1 --- /dev/null +++ b/datastore/mock_env_metadata_store_test.go @@ -0,0 +1,91 @@ +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify + +package datastore + +import ( + mock "github.com/stretchr/testify/mock" +) + +// NewMockEnvMetadataStore creates a new instance of MockEnvMetadataStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockEnvMetadataStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockEnvMetadataStore { + mock := &MockEnvMetadataStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockEnvMetadataStore is an autogenerated mock type for the EnvMetadataStore type +type MockEnvMetadataStore struct { + mock.Mock +} + +type MockEnvMetadataStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockEnvMetadataStore) EXPECT() *MockEnvMetadataStore_Expecter { + return &MockEnvMetadataStore_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function for the type MockEnvMetadataStore +func (_mock *MockEnvMetadataStore) Get() (EnvMetadata, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 EnvMetadata + var r1 error + if returnFunc, ok := ret.Get(0).(func() (EnvMetadata, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() EnvMetadata); ok { + r0 = returnFunc() + } else { + r0 = ret.Get(0).(EnvMetadata) + } + + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockEnvMetadataStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type MockEnvMetadataStore_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +func (_e *MockEnvMetadataStore_Expecter) Get() *MockEnvMetadataStore_Get_Call { + return &MockEnvMetadataStore_Get_Call{Call: _e.mock.On("Get")} +} + +func (_c *MockEnvMetadataStore_Get_Call) Run(run func()) *MockEnvMetadataStore_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockEnvMetadataStore_Get_Call) Return(metadata EnvMetadata, err error) *MockEnvMetadataStore_Get_Call { + _c.Call.Return(metadata, err) + return _c +} + +func (_c *MockEnvMetadataStore_Get_Call) RunAndReturn(run func() (EnvMetadata, error)) *MockEnvMetadataStore_Get_Call { + _c.Call.Return(run) + return _c +} + diff --git a/engine/cld/domain/envdir.go b/engine/cld/domain/envdir.go index 34393252b..66d7c7d2e 100644 --- a/engine/cld/domain/envdir.go +++ b/engine/cld/domain/envdir.go @@ -1,6 +1,7 @@ package domain import ( + "context" "encoding/json" "errors" "fmt" @@ -209,6 +210,10 @@ func (d EnvDir) ArtifactsDir() *ArtifactsDir { return NewArtifactsDir(d.rootPath, d.domainKey, d.key) } +// MergeMigrationDataStore merges a migration's DataStore into the local file-based datastore. +// This method is used when the environment is configured to use file-based datastore persistence. +// It loads the migration artifacts, merges them into the existing datastore, and writes the +// updated datastore back to local JSON files. func (d EnvDir) MergeMigrationDataStore(migkey, timestamp string) error { // Get the artifacts directory for the environment artDir := d.ArtifactsDir() @@ -258,6 +263,46 @@ func (d EnvDir) MergeMigrationDataStore(migkey, timestamp string) error { return nil } +// MergeMigrationDataStoreCatalog merges a migration's DataStore directly into the remote catalog service. +// This method is used when the environment is configured to use catalog-based datastore persistence. +// It loads the migration artifacts and syncs them to the catalog within a transaction. +// Local files are NOT updated when using catalog mode. +func (d EnvDir) MergeMigrationDataStoreCatalog(ctx context.Context, migkey, timestamp string, catalog fdatastore.CatalogStore) error { + // Get the artifacts directory for the environment + artDir := d.ArtifactsDir() + + // Load the migration datastore for the migration key and timestamp + migrDataStore, err := loadDataStoreByMigrationKey(artDir, migkey, timestamp) + if err != nil { + return err + } + + // Merge the migration datastore to catalog within a transaction + if err = fdatastore.MergeDataStoreToCatalog(ctx, migrDataStore, catalog); err != nil { + return fmt.Errorf("failed to merge datastore to catalog: %w", err) + } + + return nil +} + +// SyncDataStoreToCatalog syncs the entire local datastore state to the catalog service. +// This is useful for migrating from file-based datastore to catalog service. +// The operation is performed within a transaction for atomicity. +func (d EnvDir) SyncDataStoreToCatalog(ctx context.Context, catalog fdatastore.CatalogStore) error { + // Load the current datastore from local files + dataStore, err := d.DataStore() + if err != nil { + return fmt.Errorf("failed to load local datastore: %w", err) + } + + // Sync entire datastore to catalog within a transaction + if err = fdatastore.MergeDataStoreToCatalog(ctx, dataStore, catalog); err != nil { + return fmt.Errorf("failed to sync datastore to catalog: %w", err) + } + + return nil +} + // MergeMigrationAddressBook merges a migration's address book into an existing address book for // the given domain environment. It reads the existing address book and the migration's address // book, merges the latter into the former, and then writes the updated address book back to the diff --git a/engine/cld/domain/envdir_test.go b/engine/cld/domain/envdir_test.go index c62582eb2..0aa3f7b8e 100644 --- a/engine/cld/domain/envdir_test.go +++ b/engine/cld/domain/envdir_test.go @@ -1,7 +1,9 @@ package domain import ( + "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -845,6 +847,220 @@ func Test_EnvDir_MergeMigrationDataStore(t *testing.T) { } } +func Test_EnvDir_MergeMigrationDataStoreCatalog(t *testing.T) { + t.Parallel() + + var ( + dataStore1 = createDataStore(t, + "Contract", version1_0_0, + chainsel.ETHEREUM_TESTNET_SEPOLIA.Selector, + "0x5B5BBb15ECE0a4Ed8cDab22F902e83F66aBe848f", + "qtest1", + ) + ) + + tests := []struct { + name string + beforeFunc func(*testing.T, EnvDir) + giveMigrationName string + mockTransaction func(ctx context.Context, fn fdatastore.TransactionLogic) error + wantErr string + }{ + { + name: "success with merging to catalog", + beforeFunc: func(t *testing.T, envdir EnvDir) { + t.Helper() + + // Create the artifacts for the migration + arts := envdir.ArtifactsDir() + err := arts.SaveChangesetOutput("0001_initial", fdeployment.ChangesetOutput{ + DataStore: dataStore1, + }) + require.NoError(t, err) + }, + giveMigrationName: "0001_initial", + mockTransaction: func(ctx context.Context, fn fdatastore.TransactionLogic) error { + // Simulate successful transaction by returning nil + return nil + }, + }, + { + name: "failure when no migration artifacts directory exists", + giveMigrationName: "0001_invalid", + wantErr: "error finding files", + mockTransaction: func(ctx context.Context, fn fdatastore.TransactionLogic) error { + return nil + }, + }, + { + name: "failure when catalog merge fails", + beforeFunc: func(t *testing.T, envdir EnvDir) { + t.Helper() + + // Create the artifacts for the migration + arts := envdir.ArtifactsDir() + err := arts.SaveChangesetOutput("0001_initial", fdeployment.ChangesetOutput{ + DataStore: dataStore1, + }) + require.NoError(t, err) + }, + giveMigrationName: "0001_initial", + mockTransaction: func(ctx context.Context, fn fdatastore.TransactionLogic) error { + return errors.New("catalog transaction error") + }, + wantErr: "failed to merge datastore to catalog", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + fixture = setupTestDomainsFS(t) + envDir = fixture.envDir + mockCatalog = &mockCatalogStoreForTest{ + withTransactionFn: tt.mockTransaction, + } + ) + + if tt.beforeFunc != nil { + tt.beforeFunc(t, envDir) + } + + // Merge the migration's datastore to catalog + err := envDir.MergeMigrationDataStoreCatalog(context.Background(), tt.giveMigrationName, "", mockCatalog) + + if tt.wantErr != "" { + require.Error(t, err) + assert.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + +func Test_EnvDir_SyncDataStoreToCatalog(t *testing.T) { + t.Parallel() + + var ( + dataStore1 = createDataStore(t, + "Contract", version1_0_0, + chainsel.ETHEREUM_TESTNET_SEPOLIA.Selector, + "0x5B5BBb15ECE0a4Ed8cDab22F902e83F66aBe848f", + "qtest1", + ) + ) + + tests := []struct { + name string + beforeFunc func(*testing.T, EnvDir) + mockTransaction func(ctx context.Context, fn fdatastore.TransactionLogic) error + wantErr string + }{ + { + name: "success syncing datastore to catalog", + beforeFunc: func(t *testing.T, envdir EnvDir) { + t.Helper() + + // Create local datastore files + arts := envdir.ArtifactsDir() + err := arts.SaveChangesetOutput("0001_initial", fdeployment.ChangesetOutput{ + DataStore: dataStore1, + }) + require.NoError(t, err) + + // Merge to local files first + err = envdir.MergeMigrationDataStore("0001_initial", "") + require.NoError(t, err) + }, + mockTransaction: func(ctx context.Context, fn fdatastore.TransactionLogic) error { + // Simulate successful transaction + return nil + }, + }, + { + name: "failure when catalog sync fails", + beforeFunc: func(t *testing.T, envdir EnvDir) { + t.Helper() + + // Create local datastore files + arts := envdir.ArtifactsDir() + err := arts.SaveChangesetOutput("0001_initial", fdeployment.ChangesetOutput{ + DataStore: dataStore1, + }) + require.NoError(t, err) + + // Merge to local files first + err = envdir.MergeMigrationDataStore("0001_initial", "") + require.NoError(t, err) + }, + mockTransaction: func(ctx context.Context, fn fdatastore.TransactionLogic) error { + return errors.New("catalog sync error") + }, + wantErr: "failed to sync datastore to catalog", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + fixture = setupTestDomainsFS(t) + envDir = fixture.envDir + mockCatalog = &mockCatalogStoreForTest{ + withTransactionFn: tt.mockTransaction, + } + ) + + if tt.beforeFunc != nil { + tt.beforeFunc(t, envDir) + } + + // Sync the datastore to catalog + err := envDir.SyncDataStoreToCatalog(context.Background(), mockCatalog) + + if tt.wantErr != "" { + require.Error(t, err) + assert.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + +// mockCatalogStoreForTest is a minimal mock for testing catalog operations +type mockCatalogStoreForTest struct { + withTransactionFn func(ctx context.Context, fn fdatastore.TransactionLogic) error +} + +func (m *mockCatalogStoreForTest) WithTransaction(ctx context.Context, fn fdatastore.TransactionLogic) error { + if m.withTransactionFn != nil { + return m.withTransactionFn(ctx, fn) + } + + return nil +} + +func (m *mockCatalogStoreForTest) Addresses() fdatastore.MutableRefStoreV2[fdatastore.AddressRefKey, fdatastore.AddressRef] { + return nil +} + +func (m *mockCatalogStoreForTest) ChainMetadata() fdatastore.MutableStoreV2[fdatastore.ChainMetadataKey, fdatastore.ChainMetadata] { + return nil +} + +func (m *mockCatalogStoreForTest) ContractMetadata() fdatastore.MutableStoreV2[fdatastore.ContractMetadataKey, fdatastore.ContractMetadata] { + return nil +} + +func (m *mockCatalogStoreForTest) EnvMetadata() fdatastore.MutableUnaryStoreV2[fdatastore.EnvMetadata] { + return nil +} + func Test_EnvDir_DataStoreDirPath(t *testing.T) { t.Parallel() diff --git a/engine/cld/legacy/cli/commands/migration.go b/engine/cld/legacy/cli/commands/migration.go index 1980760a8..7de5394b2 100644 --- a/engine/cld/legacy/cli/commands/migration.go +++ b/engine/cld/legacy/cli/commands/migration.go @@ -5,20 +5,24 @@ import ( "github.com/spf13/cobra" - "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + fdeployment "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + cldcatalog "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/catalog" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/changeset" + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/config" + cfgdomain "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/config/domain" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/domain" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/environment" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/legacy/cli" "github.com/smartcontractkit/chainlink-deployments-framework/experimental/analyzer" - "github.com/smartcontractkit/chainlink-deployments-framework/operations" + foperations "github.com/smartcontractkit/chainlink-deployments-framework/operations" + "github.com/smartcontractkit/chainlink-deployments-framework/pkg/logger" ) // LoadRegistryFunc is a function type that loads the migrations registry for a given environment key. type LoadRegistryFunc func(envKey string) (*changeset.ChangesetsRegistry, error) // DecodeProposalCtxProvider is a function type that adds decoding context based on the environment. -type DecodeProposalCtxProvider func(env deployment.Environment) (analyzer.ProposalContext, error) +type DecodeProposalCtxProvider func(env fdeployment.Environment) (analyzer.ProposalContext, error) // NewMigrationCmds creates a new set of commands for managing migrations. func (c Commands) NewMigrationCmds( @@ -44,6 +48,7 @@ func (c Commands) NewMigrationCmds( Short: "Datastore operations", } datastoreCmd.AddCommand(c.newMigrationDataStoreMerge(domain)) + datastoreCmd.AddCommand(c.newMigrationDataStoreSyncToCatalog(domain)) migrationsCmd.AddCommand( c.newMigrationRun(domain, loadFunc, decodeProposalContext), @@ -84,7 +89,7 @@ var ( func (c Commands) newMigrationRun( domain domain.Domain, loadMigration func(envName string) (*changeset.ChangesetsRegistry, error), - decodeProposalContext func(env deployment.Environment) (analyzer.ProposalContext, error), + decodeProposalContext func(env fdeployment.Environment) (analyzer.ProposalContext, error), ) *cobra.Command { var ( migrationName string @@ -143,7 +148,7 @@ func (c Commands) newMigrationRun( } originalReportsLen := len(reports) cmd.Printf("Loaded %d operations reports", originalReportsLen) - reporter := operations.NewMemoryReporter(operations.WithReports(reports)) + reporter := foperations.NewMemoryReporter(foperations.WithReports(reports)) envOptions = append(envOptions, environment.WithReporter(reporter)) env, err := environment.Load(cmd.Context(), domain, envKey, envOptions...) @@ -485,15 +490,46 @@ func (Commands) newMigrationDataStoreMerge(domain domain.Domain) *cobra.Command envKey, _ := cmd.Flags().GetString("environment") envDir := domain.EnvDir(envKey) - if err := envDir.MergeMigrationDataStore(migrationName, timestamp); err != nil { - return fmt.Errorf("error during data store merge for %s %s %s: %w", - domain, envKey, migrationName, err, - ) + // Load config to check datastore type + cfg, err := config.Load(domain, envKey, logger.Nop()) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) } - cmd.Printf("Merged data stores for %s %s %s", - domain, envKey, migrationName, - ) + // Determine which merge method to use based on datastore configuration + if cfg.DatastoreType == cfgdomain.DatastoreTypeCatalog { + ctx := cmd.Context() + // Catalog mode - merge to catalog service + cmd.Printf("๐Ÿ“ก Using catalog datastore mode (endpoint: %s)\n", cfg.Env.Catalog.GRPC) + + catalog, catalogErr := cldcatalog.LoadCatalog(ctx, envKey, cfg, domain) + if catalogErr != nil { + return fmt.Errorf("failed to load catalog: %w", catalogErr) + } + + if err := envDir.MergeMigrationDataStoreCatalog(ctx, migrationName, timestamp, catalog); err != nil { + return fmt.Errorf("error during data store merge to catalog for %s %s %s: %w", + domain, envKey, migrationName, err, + ) + } + + cmd.Printf("โœ… Merged data stores to catalog for %s %s %s\n", + domain, envKey, migrationName, + ) + } else { + // File mode - merge to local files + cmd.Printf("๐Ÿ“ Using file-based datastore mode\n") + + if err := envDir.MergeMigrationDataStore(migrationName, timestamp); err != nil { + return fmt.Errorf("error during data store merge to file for %s %s %s: %w", + domain, envKey, migrationName, err, + ) + } + + cmd.Printf("โœ… Merged data stores to local files for %s %s %s\n", + domain, envKey, migrationName, + ) + } return nil }, @@ -507,3 +543,57 @@ func (Commands) newMigrationDataStoreMerge(domain domain.Domain) *cobra.Command return &cmd } + +var ( + migrationDataStoreSyncToCatalogExample = cli.Examples(` + # Sync the entire local datastore to catalog for initial migration + ccip migration datastore sync-to-catalog --environment staging + `) +) + +// newMigrationDataStoreSyncToCatalog creates a command to sync the entire local datastore to catalog +func (Commands) newMigrationDataStoreSyncToCatalog(domain domain.Domain) *cobra.Command { + cmd := cobra.Command{ + Use: "sync-to-catalog", + Short: "Sync local datastore to catalog", + Long: "Sync the entire local datastore to the catalog service. This is used for initial migration from file-based to catalog-based datastore.", + Example: migrationDataStoreSyncToCatalogExample, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + envKey, _ := cmd.Flags().GetString("environment") + envDir := domain.EnvDir(envKey) + + // Load config to get catalog connection details + cfg, err := config.Load(domain, envKey, logger.Nop()) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Verify catalog is configured + if cfg.DatastoreType != cfgdomain.DatastoreTypeCatalog { + return fmt.Errorf("catalog is not configured for environment %s (datastore type: %s)", envKey, cfg.DatastoreType) + } + + cmd.Printf("๐Ÿ“ก Syncing local datastore to catalog (endpoint: %s)\n", cfg.Env.Catalog.GRPC) + + catalog, catalogErr := cldcatalog.LoadCatalog(ctx, envKey, cfg, domain) + if catalogErr != nil { + return fmt.Errorf("failed to load catalog: %w", catalogErr) + } + + if err := envDir.SyncDataStoreToCatalog(ctx, catalog); err != nil { + return fmt.Errorf("error syncing datastore to catalog for %s %s: %w", + domain, envKey, err, + ) + } + + cmd.Printf("โœ… Successfully synced entire datastore to catalog for %s %s\n", + domain, envKey, + ) + + return nil + }, + } + + return &cmd +} diff --git a/engine/cld/legacy/cli/commands/migration_test.go b/engine/cld/legacy/cli/commands/migration_test.go index 9bedb1ff7..ae1cbcf0a 100644 --- a/engine/cld/legacy/cli/commands/migration_test.go +++ b/engine/cld/legacy/cli/commands/migration_test.go @@ -72,7 +72,7 @@ func TestNewMigrationCmds_Structure(t *testing.T) { dsUses[i] = sc.Use } require.ElementsMatch(t, - []string{"merge"}, + []string{"merge", "sync-to-catalog"}, dsUses, ) }