diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ed0d6b6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +# Credentials and local config — must never be baked into the image. +config.yml + +# Go build cache and test artifacts +*.test +*.out + +# Git history +.git +.gitignore + +# Editor/IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bb9343a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + name: Build & Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Download dependencies + run: go mod download + + - name: Verify dependencies + run: go mod verify + + - name: Vet + run: go vet ./... + + - name: Test + run: go test -race -coverprofile=coverage.out ./... + + - name: Build + run: go build -o /dev/null . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ded9997 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# --- build stage --- +FROM golang:1.21-alpine AS builder + +WORKDIR /build + +# Download dependencies first (layer-cached until go.mod/go.sum change). +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -o checkout-data-sync . + +# --- runtime stage --- +FROM alpine:3.19 + +# ca-certificates is required for TLS connections to the commercetools APIs. +RUN apk --no-cache add ca-certificates + +WORKDIR /app + +COPY --from=builder /build/checkout-data-sync . + +# Mount your config.yml at /app/config.yml via a volume (see README). +# Credentials must never be baked into the image. +ENTRYPOINT ["./checkout-data-sync"] +CMD ["--config", "config.yml"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5a3eda --- /dev/null +++ b/README.md @@ -0,0 +1,229 @@ +# checkout-data-sync + +[![CI](https://github.com/commercetools/checkout-data-sync/actions/workflows/ci.yml/badge.svg)](https://github.com/commercetools/checkout-data-sync/actions) +[![Docker Pulls](https://img.shields.io/docker/pulls/commercetools/checkout-data-sync)](https://hub.docker.com/r/commercetools/checkout-data-sync) + + + + +- [What is this?](#what-is-this) +- [Prerequisites](#prerequisites) +- [Usage](#usage) + - [Running the Docker Image](#running-the-docker-image) + - [Build](#build) + - [Run](#run) +- [Examples](#examples) +- [Sync Behaviour](#sync-behaviour) + - [Applications](#applications) + - [Payment Integrations](#payment-integrations) +- [Scopes](#scopes) + + + +### What is this? + +A Dockerized CLI application which migrates [commercetools Checkout](https://docs.commercetools.com/checkout) resources between projects — for example when moving between cloud providers or regions. + +The following resource types are supported, synced in this order: + +- **Applications** +- **Payment Integrations** + +Applications are synced first because Payment Integrations reference them by ID. The tool builds a `sourceAppID → targetAppID` map during the application phase and uses it to rewrite references before touching payment integrations. + +### Prerequisites + +- Docker (to run the image) or Go 1.21+ (to build from source) +- A commercetools API client with Checkout permissions on both source and target projects +- A `config.yml` file — copy the example and fill in your credentials: + + ```bash + cp config.example.yml config.yml + ``` + + ```yaml + source: + project_key: "source-project-key" + client_id: "sourceClientId" + client_secret: "sourceClientSecret" + # OAuth token endpoint (Composable Commerce), not the Checkout API host. + auth_url: "https://auth.europe-west1.gcp.commercetools.com/oauth/token" + checkout_api_url: "https://checkout.europe-west1.gcp.commercetools.com" + # Optional: space-separated OAuth scopes. Omit to use the API client's default scopes. + scopes: "manage_project" + + target: + project_key: "target-project-key" + client_id: "targetClientId" + client_secret: "targetClientSecret" + auth_url: "https://auth.eu-central-1.aws.commercetools.com/oauth/token" + checkout_api_url: "https://checkout.eu-central-1.aws.commercetools.com" + scopes: "" + + # Map source connector deployment UUIDs to target deployment UUIDs. + # Required for every Payment Integration that has a connectorDeployment. + # Connector deployment IDs are environment-specific and will not resolve + # across cloud providers without an explicit mapping. + deployment_mapping: + "source-deployment-uuid": "target-deployment-uuid" + ``` + + > **Note:** `auth_url` and `checkout_api_url` must not have a trailing slash. + +- The following fields are **required** to be set on resources that will be synced: + + | Resource | Required Fields | + |---|---| + | Application | `key` | + | Payment Integration | `key` (derived from source ID + name if absent — see [Sync Behaviour](#payment-integrations)) | + +### Usage + +``` +usage: checkout-data-sync + -c, --config Path to config file (default: config.yml) + -f, --full Execute the migration. Omit to perform a dry-run + instead (shows what would be created or updated, + without making any changes). +``` + +By default the tool runs in **dry-run** mode: it compares source and target resources and prints a plan without writing anything. Pass `-f` to execute the migration. + +#### Running the Docker Image + +##### Build + +```bash +docker build -t checkout-data-sync . +``` + +##### Run + +Mount your `config.yml` at `/app/config.yml` via a volume — credentials must never be baked into the image. + +```bash +docker run --rm \ + -v $(pwd)/config.yml:/app/config.yml \ + checkout-data-sync +``` + +### Examples + +- To perform a dry-run (default): + ```bash + docker run --rm -v $(pwd)/config.yml:/app/config.yml checkout-data-sync + ``` + + Example output: + ``` + === DRY RUN — pass -f to execute === + + --- Applications --- + Found 2 application(s) in source project "source-project-key" + + [CREATE] application "my-application" + [SKIP] application "existing-app" — already up to date + + Applications: 1 to create, 0 to update, 1 skipped/errors + + --- Payment Integrations --- + Found 3 payment integration(s) in source project "source-project-key" + + [CREATE] payment integration "credit-card-via-adyen" + [UPDATE] payment integration "paypal" (target id: abc123, version: 2) + • action: setStatus + [SKIP] payment integration "apple-pay" — already up to date + + Payment integrations: 1 to create, 1 to update, 1 skipped/errors + ``` + +- To execute the migration: + ```bash + docker run --rm -v $(pwd)/config.yml:/app/config.yml checkout-data-sync -f + ``` + +- To use a config file at a custom path: + ```bash + docker run --rm \ + -v $(pwd)/staging.yml:/app/staging.yml \ + checkout-data-sync -c /app/staging.yml -f + ``` + +- To build and run without Docker: + ```bash + go build -o checkout-data-sync . + ./checkout-data-sync -c config.yml # dry-run + ./checkout-data-sync -c config.yml -f # execute + ``` + +### Sync Behaviour + +#### Applications + +Applications are matched between source and target by their `key`. + +| Condition | Action | +|---|---| +| Key not found in target | `CREATE` | +| Key found, no fields differ | `SKIP` | +| Key found, fields differ | `UPDATE` | + +The following update actions are used: `setName`, `setStatus`, `setDescription`, `setApplicationLogo`, `setCountries`, `setAllowedOrigins`, `setPaymentsConfiguration`, `setDiscountsConfiguration`. + +Agreements within an application are synced by `name`: + +| Condition | Action | +|---|---| +| Agreement name not in target | `addAgreement` | +| Agreement name not in source | `removeAgreement` | +| Agreement exists but differs | `setAgreementName` / `setAgreementType` / `setAgreementStatus` / `setAgreementText` | + +#### Payment Integrations + +Payment integrations are matched against target resources using a two-step lookup: + +1. **Exact key match** — looks up the source key in the target index. +2. **Name fallback** — if no key match, looks up by `name`. When a name match is found with a different key, the target key is considered stale and a `setKey` action is prepended to the update. This handles key renames in the source project. + +> **Note:** If multiple target payment integrations share the same name, the name index entry is removed to prevent ambiguous matching. A missing key match in that case results in a new resource being created. + +| Condition | Action | +|---|---| +| No match by key or name | `CREATE` | +| Key match, no fields differ | `SKIP` | +| Key match, fields differ | `UPDATE` | +| Name match, key differs | `UPDATE` with `setKey` prepended | + +The following update actions are used: `setKey`, `setName`, `setStatus`, `setComponentType`, `setPredicate`, `setDisplayInfo`, `setSortingInfo`, `setAutomatedReversalConfiguration`, `setConnectorDeployment`. + +**Key derivation for keyless integrations** + +When a payment integration has no `key`, one is derived from the combination of its source **ID** and **name**: `{sanitised-id}-{sanitised-name}`. Using the source ID as a prefix guarantees that two integrations sharing the same display name receive distinct keys. + +**Connector deployment mapping** + +Connector deployment IDs are environment-specific. Any payment integration that has a `connectorDeployment` **must** have its source deployment UUID listed in `deployment_mapping`. Without a mapping entry the integration is skipped with a clear error: + +``` +payment integration "credit-card-via-adyen": connectorDeployment "src-uuid" has no entry +in deployment_mapping — add it under deployment_mapping in config.yml and retry +``` + +**Error handling** + +Errors on individual resources are non-fatal. The tool logs each failure, continues processing the remaining resources, and exits with a non-zero status when any error occurred. This means a single bad resource never blocks the rest of the migration. + +## Scopes + +For least-privilege access, use the following scopes instead of `manage_project`: + +| Operation | Source scope | Target scope | +|---|---|---| +| Read applications | `view_checkout_applications:{projectKey}` | — | +| Write applications | — | `manage_checkout_applications:{projectKey}` | +| Read payment integrations | `view_checkout_payment_integrations:{projectKey}` | — | +| Write payment integrations | — | `manage_checkout_payment_integrations:{projectKey}` | + +____ + +#### Maintained by: [ogwurujohnson](https://github.com/ogwurujohnson) \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 5c60afc..a5ebd20 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,27 +6,27 @@ import ( "github.com/spf13/cobra" - "github.com/commercetools/checkout-cloud-sync/internal/config" - "github.com/commercetools/checkout-cloud-sync/internal/sync" + "github.com/commercetools/checkout-data-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/sync" ) var ( configFile string - full bool + full bool ) var rootCmd = &cobra.Command{ - Use: "checkout-cloud-sync", + Use: "checkout-data-sync", Short: "Migrate commercetools Checkout resources between cloud providers", - Long: `checkout-cloud-sync migrates commercetools Checkout resources + Long: `checkout-data-sync migrates commercetools Checkout resources (applications and payment-integrations) from a source project to a target project. Without -f, runs in dry-run mode and prints what would be migrated. With -f, performs the actual migration of both applications and payment-integrations. Example: - checkout-cloud-sync -c config.yml # dry-run - checkout-cloud-sync -c config.yml -f # execute migration`, + checkout-data-sync -c config.yml # dry-run + checkout-data-sync -c config.yml -f # execute migration`, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := config.Load(configFile) diff --git a/config.example.yml b/config.example.yml index 06a25be..4413a4e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -20,4 +20,4 @@ target: # deployment_mapping: # "source-deployment-uuid": "target-deployment-uuid" deployment_mapping: - "7da2cbeb-d00d-40d3-ab29-5f23f1bdcb21": "5e20cfe4-e849-4abe-b600-c8f569519138" + "source-deployment-uuid": "target-deployment-uuid" diff --git a/go.mod b/go.mod index 1d9b0f1..97b3ef3 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/commercetools/checkout-cloud-sync +module github.com/commercetools/checkout-data-sync go 1.21 diff --git a/internal/client/client.go b/internal/client/client.go index 8d4142d..67a1736 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/commercetools/checkout-cloud-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/config" ) // Client is an authenticated Checkout API client for a single project. diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 809e8ff..943fc4a 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -7,8 +7,8 @@ import ( "net/http/httptest" "testing" - "github.com/commercetools/checkout-cloud-sync/internal/client" - "github.com/commercetools/checkout-cloud-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/client" + "github.com/commercetools/checkout-data-sync/internal/config" ) // tokenHandler always returns a valid token response. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index de760fa..f5a58be 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/commercetools/checkout-cloud-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/config" ) const validConfig = ` diff --git a/internal/sync/payment_integrations.go b/internal/sync/payment_integrations.go index 8c78622..fbfac75 100644 --- a/internal/sync/payment_integrations.go +++ b/internal/sync/payment_integrations.go @@ -72,18 +72,18 @@ func PaymentIntegrationDraftFrom( ) PaymentIntegrationDraft { key := src.Key if key == "" { - key = DeriveKey(src.Name) + key = DeriveKey(src.ID, src.Name) } draft := PaymentIntegrationDraft{ - Key: key, - Type: src.Type, - Name: src.Name, - Status: src.Status, - ComponentType: src.ComponentType, - Predicate: src.Predicate, - DisplayInfo: src.DisplayInfo, - SortingInfo: src.SortingInfo, + Key: key, + Type: src.Type, + Name: src.Name, + Status: src.Status, + ComponentType: src.ComponentType, + Predicate: src.Predicate, + DisplayInfo: src.DisplayInfo, + SortingInfo: src.SortingInfo, AutomatedReversalConfiguration: src.AutomatedReversalConfiguration, Application: Reference{ ID: targetAppID, @@ -134,7 +134,7 @@ func BuildPaymentIntegrationUpdateActions( } if !reflect.DeepEqual(src.AutomatedReversalConfiguration, tgt.AutomatedReversalConfiguration) { actions = append(actions, UpdateAction{ - "action": "setAutomatedReversalConfiguration", + "action": "setAutomatedReversalConfiguration", "automatedReversalConfiguration": src.AutomatedReversalConfiguration, }) } @@ -165,21 +165,13 @@ func BuildPaymentIntegrationUpdateActions( var nonAlphanumDash = regexp.MustCompile(`[^a-z0-9\-_]`) -// DeriveKey turns a human-readable name into a valid commercetools key -// (2-256 chars, ^[A-Za-z0-9_-]+$). Used when a payment integration has no key. -func DeriveKey(name string) string { - key := strings.ToLower(name) - // Replace spaces and most punctuation with dashes. - var b strings.Builder - for _, r := range key { - if unicode.IsLetter(r) || unicode.IsDigit(r) { - b.WriteRune(r) - } else { - b.WriteRune('-') - } - } - key = nonAlphanumDash.ReplaceAllString(b.String(), "") - // Collapse runs of dashes. +// DeriveKey builds a stable commercetools key (2-256 chars, ^[A-Za-z0-9_-]+$) +// for a payment integration that has no explicit key. It combines the source ID +// with the sanitised name so that two integrations sharing the same display name +// still receive distinct derived keys. +func DeriveKey(id, name string) string { + key := sanitiseKeySegment(id) + "-" + sanitiseKeySegment(name) + // Collapse any runs of dashes that resulted from the concatenation. for strings.Contains(key, "--") { key = strings.ReplaceAll(key, "--", "-") } @@ -193,3 +185,17 @@ func DeriveKey(name string) string { } return key } + +// sanitiseKeySegment lowercases s and replaces any character that is not +// a–z, 0–9, or dash/underscore with a dash. +func sanitiseKeySegment(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' { + b.WriteRune(r) + } else { + b.WriteRune('-') + } + } + return nonAlphanumDash.ReplaceAllString(b.String(), "") +} diff --git a/internal/sync/syncer.go b/internal/sync/syncer.go index b1ee6bb..c39e018 100644 --- a/internal/sync/syncer.go +++ b/internal/sync/syncer.go @@ -7,8 +7,8 @@ import ( "io" "net/http" - "github.com/commercetools/checkout-cloud-sync/internal/client" - "github.com/commercetools/checkout-cloud-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/client" + "github.com/commercetools/checkout-data-sync/internal/config" ) type apiClient interface { @@ -176,7 +176,7 @@ func (s *Syncer) syncPaymentIntegrations(ctx context.Context, force bool, appIDM // Determine the key to use for matching in target. key := src.Key if key == "" { - key = DeriveKey(src.Name) + key = DeriveKey(src.ID, src.Name) s.printf(" WARN: payment integration %q has no key; derived key %q\n", src.ID, key) } diff --git a/internal/sync/syncer_test.go b/internal/sync/syncer_test.go index 748a62b..486a590 100644 --- a/internal/sync/syncer_test.go +++ b/internal/sync/syncer_test.go @@ -10,8 +10,8 @@ import ( "strings" "testing" - "github.com/commercetools/checkout-cloud-sync/internal/config" - "github.com/commercetools/checkout-cloud-sync/internal/sync" + "github.com/commercetools/checkout-data-sync/internal/config" + "github.com/commercetools/checkout-data-sync/internal/sync" ) // ---- helpers ----------------------------------------------------------------- @@ -80,16 +80,16 @@ func buildSyncer(t *testing.T, srcHandler, tgtHandler http.HandlerFunc) (*sync.S func TestApplicationDraftFrom_PreservesAllFields(t *testing.T) { app := sync.Application{ - ID: "src-id", - Version: 3, - Key: "my-app", - Name: "My App", - Mode: "CompleteFlow", - Status: "Active", - Description: sync.LocalizedString{"en": "Desc"}, - Logo: &sync.ApplicationLogo{URL: "https://logo.example.com/img.svg"}, - Countries: []string{"DE", "FR"}, - AllowedOrigins: &sync.AllowedOrigins{AllowAll: false, Origins: []string{"https://demo.com"}}, + ID: "src-id", + Version: 3, + Key: "my-app", + Name: "My App", + Mode: "CompleteFlow", + Status: "Active", + Description: sync.LocalizedString{"en": "Desc"}, + Logo: &sync.ApplicationLogo{URL: "https://logo.example.com/img.svg"}, + Countries: []string{"DE", "FR"}, + AllowedOrigins: &sync.AllowedOrigins{AllowAll: false, Origins: []string{"https://demo.com"}}, PaymentsConfiguration: &sync.PaymentsConfiguration{PaymentReturnURL: "https://demo.com/return", ActivePaymentComponentType: "Component"}, DiscountsConfiguration: &sync.DiscountsConfiguration{AllowDiscounts: true}, Agreements: []sync.ApplicationAgreement{ @@ -140,15 +140,15 @@ func TestBuildApplicationUpdateActions_NameChange(t *testing.T) { func TestBuildApplicationUpdateActions_MultipleChanges(t *testing.T) { src := &sync.Application{ - Key: "k", - Name: "New Name", - Status: "Active", + Key: "k", + Name: "New Name", + Status: "Active", Countries: []string{"DE"}, } tgt := &sync.Application{ - Key: "k", - Name: "Old Name", - Status: "Inactive", + Key: "k", + Name: "Old Name", + Status: "Inactive", Countries: []string{"FR"}, } actions := sync.BuildApplicationUpdateActions(src, tgt) @@ -278,28 +278,32 @@ func TestBuildApplicationUpdateActions_AgreementRemove(t *testing.T) { func TestDeriveKey_Basic(t *testing.T) { cases := []struct { + id string name string want string }{ - {"Credit Card via Adyen", "credit-card-via-adyen"}, - {"PayPal", "paypal"}, - {"Apple Pay (Express)", "apple-pay-express"}, - {"My--Service!", "my-service"}, + {"abc-123", "Credit Card via Adyen", "abc-123-credit-card-via-adyen"}, + {"abc-123", "PayPal", "abc-123-paypal"}, + {"abc-123", "Apple Pay (Express)", "abc-123-apple-pay-express"}, + {"abc-123", "My--Service!", "abc-123-my-service"}, + // Two PIs with the same name get different keys because IDs differ. + {"id-001", "Shared Name", "id-001-shared-name"}, + {"id-002", "Shared Name", "id-002-shared-name"}, } for _, tc := range cases { - got := sync.DeriveKey(tc.name) + got := sync.DeriveKey(tc.id, tc.name) if got != tc.want { - t.Errorf("DeriveKey(%q) = %q, want %q", tc.name, got, tc.want) + t.Errorf("DeriveKey(%q, %q) = %q, want %q", tc.id, tc.name, got, tc.want) } } } func TestPaymentIntegrationDraftFrom_TranslatesAppID(t *testing.T) { src := sync.PaymentIntegration{ - Key: "pi-key", - Type: "card", - Name: "Credit Card", - Application: sync.Reference{ID: "src-app-id", TypeID: "application"}, + Key: "pi-key", + Type: "card", + Name: "Credit Card", + Application: sync.Reference{ID: "src-app-id", TypeID: "application"}, ConnectorDeployment: &sync.Reference{ID: "src-dep-id", TypeID: "deployment"}, } mapping := map[string]string{"src-dep-id": "tgt-dep-id"} @@ -319,8 +323,8 @@ func TestPaymentIntegrationDraftFrom_TranslatesAppID(t *testing.T) { func TestPaymentIntegrationDraftFrom_NoKeyDerives(t *testing.T) { src := sync.PaymentIntegration{ - Type: "card", - Name: "Credit Card via Adyen", + Type: "card", + Name: "Credit Card via Adyen", Application: sync.Reference{ID: "src-app-id", TypeID: "application"}, } draft := sync.PaymentIntegrationDraftFrom(src, "tgt-app-id", nil) @@ -389,12 +393,12 @@ func makeApp(id, key, version int) sync.Application { // makePI creates a minimal PaymentIntegration for testing. func makePI(id int, appID string) sync.PaymentIntegration { return sync.PaymentIntegration{ - ID: fmt.Sprintf("pi-%d", id), - Version: 1, - Key: fmt.Sprintf("pi-key-%d", id), - Name: fmt.Sprintf("PI %d", id), - Type: "card", - Status: "Active", + ID: fmt.Sprintf("pi-%d", id), + Version: 1, + Key: fmt.Sprintf("pi-key-%d", id), + Name: fmt.Sprintf("PI %d", id), + Type: "card", + Status: "Active", Application: sync.Reference{ID: appID, TypeID: "application"}, } } diff --git a/main.go b/main.go index 5baa4ae..983fd92 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ package main -import "github.com/commercetools/checkout-cloud-sync/cmd" +import "github.com/commercetools/checkout-data-sync/cmd" func main() { cmd.Execute()