Add postgres_synced_tables bundle resource

.agent/skills/pr-checklist/SKILL.md

@@ -16,8 +16,9 @@ Before submitting a PR, run these commands to match what CI checks. CI uses the
 # 3. Tests (CI runs with both deployment engines)
 ./task test
 
-# 4. If you changed bundle config structs or schema-related code:
+# 4. If you changed bundle config structs, schema, or direct-engine resource code:
 ./task generate-schema
+./task generate-direct
 
 # 5. If you changed files in python/:
 ./task pydabs-codegen pydabs-test pydabs-lint pydabs-docs

NEXT_CHANGELOG.md

@@ -10,5 +10,7 @@
 
 ### Bundles
 * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239))
+* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch ([#5265](https://github.com/databricks/cli/pull/5265)).
+* Add `postgres_synced_tables` resource to sync a Unity Catalog Delta table into a Postgres table on a Lakebase Autoscaling branch ([#5268](https://github.com/databricks/cli/pull/5268)).
 
 ### Dependency updates

acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl

@@ -0,0 +1,15 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  postgres_projects:
+    project:
+      project_id: test-pg-project-$UNIQUE_NAME
+      display_name: Test Postgres Project
+
+  postgres_catalogs:
+    catalog:
+      catalog_id: test_pg_catalog_$UNIQUE_NAME
+      branch: ${resources.postgres_projects.project.name}/branches/production
+      postgres_database: appdb
+      create_database_if_missing: true

acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  postgres_synced_tables:
+    foo:
+      synced_table_id: lakebase_$UNIQUE_NAME.public.trips_synced
+      source_table_full_name: main.raw.trips
+      primary_key_columns: [id]
+      scheduling_policy: SNAPSHOT
+      postgres_database: appdb
+      branch: projects/test-pg-project-$UNIQUE_NAME/branches/production
+      create_database_objects_if_missing: true
+      new_pipeline_spec:
+        storage_catalog: main
+        storage_schema: pipelines

acceptance/bundle/invariant/test.toml

@@ -44,8 +44,10 @@ EnvMatrix.INPUT_CONFIG = [
     "pipeline.yml.tmpl",
     "pipeline_config_dots.yml.tmpl",
     "postgres_branch.yml.tmpl",
+    "postgres_catalog.yml.tmpl",
     "postgres_endpoint.yml.tmpl",
     "postgres_project.yml.tmpl",
+    "postgres_synced_table.yml.tmpl",
     "registered_model.yml.tmpl",
     "schema.yml.tmpl",
     "schema_grant_ref.yml.tmpl",
@@ -67,6 +69,8 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"]
 no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"]
 no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"]
 no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"]
+no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"]
+no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"]
 
 # External locations require actual storage credentials with cloud IAM setup
 # which are environment-specific, so we only test locally with the mock server

acceptance/bundle/refschema/out.fields.txt

@@ -2733,6 +2733,28 @@ resources.postgres_branches.*.ttl	*duration.Duration	INPUT	STATE
 resources.postgres_branches.*.uid	string	REMOTE
 resources.postgres_branches.*.update_time	*time.Time	REMOTE
 resources.postgres_branches.*.url	string	INPUT
+resources.postgres_catalogs.*.branch	string	INPUT	STATE
+resources.postgres_catalogs.*.catalog_id	string	INPUT	STATE
+resources.postgres_catalogs.*.create_database_if_missing	bool	INPUT	STATE
+resources.postgres_catalogs.*.create_time	*time.Time	REMOTE
+resources.postgres_catalogs.*.id	string	INPUT
+resources.postgres_catalogs.*.lifecycle	resources.Lifecycle	INPUT
+resources.postgres_catalogs.*.lifecycle.prevent_destroy	bool	INPUT
+resources.postgres_catalogs.*.modified_status	string	INPUT
+resources.postgres_catalogs.*.name	string	REMOTE
+resources.postgres_catalogs.*.postgres_database	string	INPUT	STATE
+resources.postgres_catalogs.*.spec	*postgres.CatalogCatalogSpec	REMOTE
+resources.postgres_catalogs.*.spec.branch	string	REMOTE
+resources.postgres_catalogs.*.spec.create_database_if_missing	bool	REMOTE
+resources.postgres_catalogs.*.spec.postgres_database	string	REMOTE
+resources.postgres_catalogs.*.status	*postgres.CatalogCatalogStatus	REMOTE
+resources.postgres_catalogs.*.status.branch	string	REMOTE
+resources.postgres_catalogs.*.status.catalog_id	string	REMOTE
+resources.postgres_catalogs.*.status.postgres_database	string	REMOTE
+resources.postgres_catalogs.*.status.project	string	REMOTE
+resources.postgres_catalogs.*.uid	string	REMOTE
+resources.postgres_catalogs.*.update_time	*time.Time	REMOTE
+resources.postgres_catalogs.*.url	string	INPUT
 resources.postgres_endpoints.*.autoscaling_limit_max_cu	float64	INPUT	STATE
 resources.postgres_endpoints.*.autoscaling_limit_min_cu	float64	INPUT	STATE
 resources.postgres_endpoints.*.create_time	*time.Time	REMOTE
@@ -2870,6 +2892,63 @@ resources.postgres_projects.*.permissions[*].group_name	string	ALL
 resources.postgres_projects.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.postgres_projects.*.permissions[*].service_principal_name	string	ALL
 resources.postgres_projects.*.permissions[*].user_name	string	ALL
+resources.postgres_synced_tables.*.branch	string	INPUT	STATE
+resources.postgres_synced_tables.*.create_database_objects_if_missing	bool	INPUT	STATE
+resources.postgres_synced_tables.*.create_time	*time.Time	REMOTE
+resources.postgres_synced_tables.*.existing_pipeline_id	string	INPUT	STATE
+resources.postgres_synced_tables.*.id	string	INPUT
+resources.postgres_synced_tables.*.lifecycle	resources.Lifecycle	INPUT
+resources.postgres_synced_tables.*.lifecycle.prevent_destroy	bool	INPUT
+resources.postgres_synced_tables.*.modified_status	string	INPUT
+resources.postgres_synced_tables.*.name	string	REMOTE
+resources.postgres_synced_tables.*.new_pipeline_spec	*postgres.NewPipelineSpec	INPUT	STATE
+resources.postgres_synced_tables.*.new_pipeline_spec.budget_policy_id	string	INPUT	STATE
+resources.postgres_synced_tables.*.new_pipeline_spec.storage_catalog	string	INPUT	STATE
+resources.postgres_synced_tables.*.new_pipeline_spec.storage_schema	string	INPUT	STATE
+resources.postgres_synced_tables.*.postgres_database	string	INPUT	STATE
+resources.postgres_synced_tables.*.primary_key_columns	[]string	INPUT	STATE
+resources.postgres_synced_tables.*.primary_key_columns[*]	string	INPUT	STATE
+resources.postgres_synced_tables.*.scheduling_policy	postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy	INPUT	STATE
+resources.postgres_synced_tables.*.source_table_full_name	string	INPUT	STATE
+resources.postgres_synced_tables.*.spec	*postgres.SyncedTableSyncedTableSpec	REMOTE
+resources.postgres_synced_tables.*.spec.branch	string	REMOTE
+resources.postgres_synced_tables.*.spec.create_database_objects_if_missing	bool	REMOTE
+resources.postgres_synced_tables.*.spec.existing_pipeline_id	string	REMOTE
+resources.postgres_synced_tables.*.spec.new_pipeline_spec	*postgres.NewPipelineSpec	REMOTE
+resources.postgres_synced_tables.*.spec.new_pipeline_spec.budget_policy_id	string	REMOTE
+resources.postgres_synced_tables.*.spec.new_pipeline_spec.storage_catalog	string	REMOTE
+resources.postgres_synced_tables.*.spec.new_pipeline_spec.storage_schema	string	REMOTE
+resources.postgres_synced_tables.*.spec.postgres_database	string	REMOTE
+resources.postgres_synced_tables.*.spec.primary_key_columns	[]string	REMOTE
+resources.postgres_synced_tables.*.spec.primary_key_columns[*]	string	REMOTE
+resources.postgres_synced_tables.*.spec.scheduling_policy	postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy	REMOTE
+resources.postgres_synced_tables.*.spec.source_table_full_name	string	REMOTE
+resources.postgres_synced_tables.*.spec.timeseries_key	string	REMOTE
+resources.postgres_synced_tables.*.status	*postgres.SyncedTableSyncedTableStatus	REMOTE
+resources.postgres_synced_tables.*.status.detailed_state	postgres.SyncedTableState	REMOTE
+resources.postgres_synced_tables.*.status.last_processed_commit_version	int64	REMOTE
+resources.postgres_synced_tables.*.status.last_sync	*postgres.SyncedTablePosition	REMOTE
+resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info	*postgres.DeltaTableSyncInfo	REMOTE
+resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info.delta_commit_time	*time.Time	REMOTE
+resources.postgres_synced_tables.*.status.last_sync.delta_table_sync_info.delta_commit_version	int64	REMOTE
+resources.postgres_synced_tables.*.status.last_sync.sync_end_time	*time.Time	REMOTE
+resources.postgres_synced_tables.*.status.last_sync.sync_start_time	*time.Time	REMOTE
+resources.postgres_synced_tables.*.status.last_sync_time	*time.Time	REMOTE
+resources.postgres_synced_tables.*.status.message	string	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress	*postgres.SyncedTablePipelineProgress	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress.estimated_completion_time_seconds	float64	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress.latest_version_currently_processing	int64	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress.sync_progress_completion	float64	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress.synced_row_count	int64	REMOTE
+resources.postgres_synced_tables.*.status.ongoing_sync_progress.total_row_count	int64	REMOTE
+resources.postgres_synced_tables.*.status.pipeline_id	string	REMOTE
+resources.postgres_synced_tables.*.status.project	string	REMOTE
+resources.postgres_synced_tables.*.status.provisioning_phase	postgres.ProvisioningPhase	REMOTE
+resources.postgres_synced_tables.*.status.unity_catalog_provisioning_state	postgres.ProvisioningInfoState	REMOTE
+resources.postgres_synced_tables.*.synced_table_id	string	INPUT	STATE
+resources.postgres_synced_tables.*.timeseries_key	string	INPUT	STATE
+resources.postgres_synced_tables.*.uid	string	REMOTE
+resources.postgres_synced_tables.*.url	string	INPUT
 resources.quality_monitors.*.assets_dir	string	ALL
 resources.quality_monitors.*.baseline_table_name	string	ALL
 resources.quality_monitors.*.custom_metrics	[]catalog.MonitorMetric	ALL

acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl

@@ -0,0 +1,19 @@
+bundle:
+  name: deploy-postgres-catalog-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Test Project for Catalog"
+      pg_version: 16
+
+  postgres_catalogs:
+    my_catalog:
+      catalog_id: lakebase_test_$UNIQUE_NAME
+      branch: ${resources.postgres_projects.my_project.id}/branches/production
+      postgres_database: appdb
+      create_database_if_missing: true

acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json

@@ -0,0 +1,31 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "Test Project for Catalog",
+      "pg_version": 16
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/catalogs",
+  "q": {
+    "catalog_id": "lakebase_test_[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+      "create_database_if_missing": true,
+      "postgres_database": "appdb"
+    }
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json

@@ -0,0 +1,31 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "Test Project for Catalog",
+      "pg_version": 16
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/catalogs",
+  "q": {
+    "catalog_id": "lakebase_test_[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+      "create_database_if_missing": true,
+      "postgres_database": "appdb"
+    }
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_catalogs/basic/output.txt

@@ -0,0 +1,74 @@
+
+>>> [CLI] bundle validate
+Name: deploy-postgres-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle summary
+Name: deploy-postgres-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default
+Resources:
+  Postgres catalogs:
+    my_catalog:
+      Name: lakebase_test_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME]
+  Postgres projects:
+    my_project:
+      Name: Test Project for Catalog
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] postgres get-catalog catalogs/lakebase_test_[UNIQUE_NAME]
+{
+  "name": "catalogs/lakebase_test_[UNIQUE_NAME]",
+  "status": {
+    "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+    "catalog_id": "lakebase_test_[UNIQUE_NAME]",
+    "postgres_database": "appdb",
+    "project": "projects/test-pg-proj-[UNIQUE_NAME]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: deploy-postgres-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default
+Resources:
+  Postgres catalogs:
+    my_catalog:
+      Name: lakebase_test_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME]
+  Postgres projects:
+    my_project:
+      Name: Test Project for Catalog
+      URL:  (not deployed)
+
+>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_catalogs.my_catalog
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/postgres_catalogs/basic/script

@@ -0,0 +1,23 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle validate
+
+trace $CLI bundle summary
+
+rm -f out.requests.txt
+trace $CLI bundle deploy
+
+# Get catalog details. Hide volatile fields so cloud and local match.
+catalog_name="catalogs/lakebase_test_${UNIQUE_NAME}"
+trace $CLI postgres get-catalog "${catalog_name}" | jq 'del(.create_time, .update_time, .uid)'
+
+trace $CLI bundle summary
+
+# Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling).
+trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json