From cff69981709dc714327191126d9ce3f54f5dc89c Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 00:43:00 -0500 Subject: [PATCH 1/2] feat: rename repo surface and add source-aware Databricks targets --- .depot/workflows/release.yml | 4 +- .release-please-manifest.json | 2 +- CHANGELOG.md | 12 ++++- README.md | 28 ++++++++++-- apps/convex-sync/CHANGELOG.md | 18 -------- install.sh | 2 +- justfile | 3 ++ .../terraform.tfvars.example | 4 +- .../terraform.tfvars.example | 4 +- .../terraform.tfvars.example | 4 +- .../publisher_user/terraform.tfvars.example | 8 ++-- .../aws/s3_target/terraform/s3_bucket/main.tf | 2 +- .../s3_bucket/terraform.tfvars.example | 2 +- platform/databricks/README.md | 3 ++ platform/databricks/delta/README.md | 18 +++++++- platform/databricks/delta/databricks.yml | 32 ++++++++++---- .../resources/convex_delta_extract.job.yml | 2 +- platform/databricks/s3/README.md | 7 +++ .../terraform.tfvars.example | 12 ++--- release-please-config.json | 38 +++++++++++++--- scripts/bootstrap-databricks-delta.sh | 44 +++++++++++++++++++ scripts/convex-sync-dev | 2 +- scripts/deploy-databricks-delta.sh | 16 +++++-- scripts/ensure-databricks-delta-secret.sh | 5 ++- scripts/load-source-config.sh | 19 ++++++++ scripts/run-databricks-delta-job.sh | 16 +++++-- scripts/run-databricks-delta-smoke.sh | 28 +++++------- scripts/sync-databricks-staging-views.sh | 7 ++- sources/README.md | 27 ++++++++++++ sources/meshix-api/env.sh | 27 ++++++++++++ 30 files changed, 308 insertions(+), 88 deletions(-) delete mode 100644 apps/convex-sync/CHANGELOG.md create mode 100755 scripts/bootstrap-databricks-delta.sh create mode 100755 scripts/load-source-config.sh create mode 100644 sources/README.md create mode 100755 sources/meshix-api/env.sh diff --git a/.depot/workflows/release.yml b/.depot/workflows/release.yml index 3bc2ed9..7eddcd2 100644 --- a/.depot/workflows/release.yml +++ b/.depot/workflows/release.yml @@ -14,8 +14,8 @@ jobs: contents: write pull-requests: write outputs: - release_created: ${{ steps.release.outputs['apps/convex-sync--release_created'] }} - tag_name: ${{ steps.release.outputs['apps/convex-sync--tag_name'] }} + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} paths_released: ${{ steps.release.outputs.paths_released }} steps: - name: run-release-please diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 1fa0cf0..40ff6fe 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - "apps/convex-sync": "0.0.2" + ".": "0.0.2" } diff --git a/CHANGELOG.md b/CHANGELOG.md index ba81188..0031ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ # Changelog -All notable changes to this project will be documented in this file. +## [0.0.2](https://github.com/shpitdev/convex-sync-kit/compare/v0.0.1...v0.0.2) (2026-04-20) +### Features + +* automate stable releases from conventional PR titles ([#9](https://github.com/shpitdev/convex-sync-kit/issues/9)) ([494d7ea](https://github.com/shpitdev/convex-sync-kit/commit/494d7eab5ab33b4067e57d135ffdaa4e93bdb8cc)) + +### Bug Fixes + +* make cargo package versions explicit for release-please ([#10](https://github.com/shpitdev/convex-sync-kit/issues/10)) ([00f3993](https://github.com/shpitdev/convex-sync-kit/commit/00f3993a2fb55728fe2c553d4c9b13dfd34192e9)) +* point release-please at convex-sync package ([#11](https://github.com/shpitdev/convex-sync-kit/issues/11)) ([04e5671](https://github.com/shpitdev/convex-sync-kit/commit/04e56710e464df453a29d2ad228177e290e365c6)) + +All notable changes to `convex-sync-kit` will be documented in this file. diff --git a/README.md b/README.md index 7af99d0..a11672c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# convex-streaming-olap-export +# convex-sync-kit - `Language`: ![Rust](https://img.shields.io/badge/Rust-000000?logo=rust&logoColor=white) - `Source`: ![Convex](https://img.shields.io/badge/Convex-EE342F?logo=convex&logoColor=white) @@ -22,7 +22,7 @@ extraction model: ```mermaid flowchart TD - Root[convex-streaming-olap-export] + Root[convex-sync-kit] Inspect[apps/convex-inspect] CLI[apps/convex-sync] Core[crates/convex-sync-core] @@ -49,13 +49,14 @@ Read the repo by layer: - [`platform/databricks/README.md`](platform/databricks/README.md): Databricks target family overview - [`platform/databricks/s3/README.md`](platform/databricks/s3/README.md): Databricks consuming the S3 export path - [`platform/databricks/delta/README.md`](platform/databricks/delta/README.md): Databricks Delta bronze/silver landing +- [`sources/README.md`](sources/README.md): source-specific defaults layered on top of the shared engine ## Install Release install: ```bash -curl -fsSL https://raw.githubusercontent.com/shpitdev/convex-streaming-olap-export/main/install.sh | bash +curl -fsSL https://raw.githubusercontent.com/shpitdev/convex-sync-kit/main/install.sh | bash ``` Local checkout dev install: @@ -73,6 +74,17 @@ Current release coverage: - command symlinks go in `~/.local/bin` - `convex-inspect` is repo-local today and not part of the release artifact +## Source Configs + +The repo name stays generic. Source-specific defaults live under `sources/`. + +- current source profile: `sources/meshix-api/env.sh` +- activate a source by setting `CONVEX_SYNC_SOURCE=` +- explicit env vars still win over source defaults + +This is the intended scaling model for running the same engine against many +Convex projects without forking the repo or renaming the binaries. + ## Operator Binaries - `convex-inspect`: inspect Convex schemas, snapshot pages, and delta pages directly @@ -145,10 +157,18 @@ Runtime split: Packaged entrypoints: - `just databricks-delta-sync-secret` +- `just databricks-delta-bootstrap ` - `just databricks-delta-deploy` - `just databricks-delta-run` - `just databricks-delta-smoke ` +Recommended production naming: + +- S3-backed Databricks schema: `convex_sync_kit__s3` +- Delta control schema: `convex_sync_kit__delta_control` +- Delta bronze schema: `convex_sync_kit__delta_bronze` +- Delta silver schema: `convex_sync_kit__delta_silver` + ### `Databricks over S3` This variation keeps the existing Rust exporter and S3 publish loop, then adds: @@ -199,7 +219,7 @@ Stable releases are driven by merged PR titles on `main`. - use conventional PR titles such as `feat: ...`, `fix: ...`, or `deps: ...` - `release-please` now starts release history from commit `0cf9f47` -- merge to `main` opens or advances the stable release PR automatically when a releasable PR lands +- merge to `main` opens or advances the stable release PR automatically when a releasable PR lands anywhere the repo-wide release config considers in scope - both release workflows also support manual `workflow_dispatch`, so there is always a button path in GitHub Actions ## References diff --git a/apps/convex-sync/CHANGELOG.md b/apps/convex-sync/CHANGELOG.md deleted file mode 100644 index 81accf8..0000000 --- a/apps/convex-sync/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -## [0.0.2](https://github.com/shpitdev/convex-streaming-olap-export/compare/v0.0.1...v0.0.2) (2026-04-20) - - -### Features - -* automate stable releases from conventional PR titles ([#9](https://github.com/shpitdev/convex-streaming-olap-export/issues/9)) ([494d7ea](https://github.com/shpitdev/convex-streaming-olap-export/commit/494d7eab5ab33b4067e57d135ffdaa4e93bdb8cc)) - - -### Bug Fixes - -* make cargo package versions explicit for release-please ([#10](https://github.com/shpitdev/convex-streaming-olap-export/issues/10)) ([00f3993](https://github.com/shpitdev/convex-streaming-olap-export/commit/00f3993a2fb55728fe2c553d4c9b13dfd34192e9)) -* point release-please at convex-sync package ([#11](https://github.com/shpitdev/convex-streaming-olap-export/issues/11)) ([04e5671](https://github.com/shpitdev/convex-streaming-olap-export/commit/04e56710e464df453a29d2ad228177e290e365c6)) - -## Changelog - -All notable changes to `convex-sync` will be documented in this file. diff --git a/install.sh b/install.sh index 82784fd..dc159b0 100755 --- a/install.sh +++ b/install.sh @@ -3,7 +3,7 @@ set -euo pipefail BINARY_NAME="convex-sync" DEV_BINARY_NAME="${CONVEX_SYNC_DEV_BIN_NAME:-convex-sync-dev}" -REPO="shpitdev/convex-streaming-olap-export" +REPO="shpitdev/convex-sync-kit" VERSION="${CONVEX_SYNC_VERSION:-latest}" INSTALL_MODE="${CONVEX_SYNC_INSTALL_MODE:-auto}" diff --git a/justfile b/justfile index 65acb9a..bea2c9a 100644 --- a/justfile +++ b/justfile @@ -78,6 +78,9 @@ databricks-apply-sql-dir profile warehouse_id sql_dir: databricks-delta-sync-secret *args: ./scripts/ensure-databricks-delta-secret.sh {{args}} +databricks-delta-bootstrap warehouse_id profile="DEFAULT": + ./scripts/bootstrap-databricks-delta.sh {{profile}} {{warehouse_id}} + databricks-delta-deploy profile="DEFAULT" target="dev": ./scripts/deploy-databricks-delta.sh {{profile}} {{target}} diff --git a/platform/aws/s3_consumers/terraform/databricks_reader_role/terraform.tfvars.example b/platform/aws/s3_consumers/terraform/databricks_reader_role/terraform.tfvars.example index 8618634..4b05c4f 100644 --- a/platform/aws/s3_consumers/terraform/databricks_reader_role/terraform.tfvars.example +++ b/platform/aws/s3_consumers/terraform/databricks_reader_role/terraform.tfvars.example @@ -1,7 +1,7 @@ aws_region = "us-east-1" aws_account_id = "123456789012" -bucket_name = "example-convex-streaming-olap-export-bucket" +bucket_name = "example-convex-sync-kit-bucket" current_prefix = "prod/staging/current" -role_name = "example-convex-streaming-olap-export-databricks-prod-reader" +role_name = "example-convex-sync-kit-databricks-prod-reader" databricks_unity_catalog_role_arn = "arn:aws:iam::123456789012:role/unity-catalog-prod-UCMasterRole-EXAMPLE" external_id = "" diff --git a/platform/aws/s3_consumers/terraform/palantir_cloud_identity_reader_role/terraform.tfvars.example b/platform/aws/s3_consumers/terraform/palantir_cloud_identity_reader_role/terraform.tfvars.example index e3b2077..c256a85 100644 --- a/platform/aws/s3_consumers/terraform/palantir_cloud_identity_reader_role/terraform.tfvars.example +++ b/platform/aws/s3_consumers/terraform/palantir_cloud_identity_reader_role/terraform.tfvars.example @@ -1,5 +1,5 @@ aws_region = "us-east-1" -bucket_name = "example-convex-streaming-olap-export-bucket" +bucket_name = "example-convex-sync-kit-bucket" current_prefix = "prod/staging/current" -role_name = "example-convex-streaming-olap-export-palantir-prod-reader" +role_name = "example-convex-sync-kit-palantir-prod-reader" palantir_cloud_identity_role_arn = "arn:aws:iam::123456789012:role/foundry-cloud-identity-example" diff --git a/platform/aws/s3_consumers/terraform/palantir_oidc_reader_role/terraform.tfvars.example b/platform/aws/s3_consumers/terraform/palantir_oidc_reader_role/terraform.tfvars.example index 7e5db92..6c9ec90 100644 --- a/platform/aws/s3_consumers/terraform/palantir_oidc_reader_role/terraform.tfvars.example +++ b/platform/aws/s3_consumers/terraform/palantir_oidc_reader_role/terraform.tfvars.example @@ -1,7 +1,7 @@ aws_region = "us-east-1" -bucket_name = "example-convex-streaming-olap-export-bucket" +bucket_name = "example-convex-sync-kit-bucket" current_prefix = "prod/staging/current" -role_name = "example-convex-streaming-olap-export-palantir-oidc-prod-reader" +role_name = "example-convex-sync-kit-palantir-oidc-prod-reader" oidc_provider_arn = "arn:aws:iam::123456789012:oidc-provider/example" oidc_issuer_url = "https://example.oidc.palantir.com/foundry" oidc_audience = "replace-me" diff --git a/platform/aws/s3_target/terraform/publisher_user/terraform.tfvars.example b/platform/aws/s3_target/terraform/publisher_user/terraform.tfvars.example index 5f3c278..c7b79e4 100644 --- a/platform/aws/s3_target/terraform/publisher_user/terraform.tfvars.example +++ b/platform/aws/s3_target/terraform/publisher_user/terraform.tfvars.example @@ -1,10 +1,10 @@ aws_region = "us-east-1" -bucket_name = "example-convex-streaming-olap-export-bucket" +bucket_name = "example-convex-sync-kit-bucket" allowed_prefixes = [ "*", ] -group_name = "example-convex-streaming-olap-export-s3-publishers" -policy_name = "example-convex-streaming-olap-export-s3-publish" -user_name = "example-convex-streaming-olap-export-publisher" +group_name = "example-convex-sync-kit-s3-publishers" +policy_name = "example-convex-sync-kit-s3-publish" +user_name = "example-convex-sync-kit-publisher" diff --git a/platform/aws/s3_target/terraform/s3_bucket/main.tf b/platform/aws/s3_target/terraform/s3_bucket/main.tf index ebf3280..ccabc54 100644 --- a/platform/aws/s3_target/terraform/s3_bucket/main.tf +++ b/platform/aws/s3_target/terraform/s3_bucket/main.tf @@ -9,7 +9,7 @@ resource "aws_s3_bucket" "this" { { Name = var.bucket_name ManagedBy = "terraform" - Repo = "convex-streaming-olap-export" + Repo = "convex-sync-kit" }, var.tags, ) diff --git a/platform/aws/s3_target/terraform/s3_bucket/terraform.tfvars.example b/platform/aws/s3_target/terraform/s3_bucket/terraform.tfvars.example index c124882..0fe51c8 100644 --- a/platform/aws/s3_target/terraform/s3_bucket/terraform.tfvars.example +++ b/platform/aws/s3_target/terraform/s3_bucket/terraform.tfvars.example @@ -1,5 +1,5 @@ aws_region = "us-east-1" -bucket_name = "example-convex-streaming-olap-export-bucket" +bucket_name = "example-convex-sync-kit-bucket" tags = { Environment = "prod" diff --git a/platform/databricks/README.md b/platform/databricks/README.md index 6bf0500..189f15e 100644 --- a/platform/databricks/README.md +++ b/platform/databricks/README.md @@ -56,4 +56,7 @@ The Databricks-first assets are the starting point for direct Delta landing. The provider is configured from `~/.databrickscfg` by default, typically using the `DEFAULT` profile. +Source-specific defaults are layered in from `sources//env.sh`, starting +with `sources/meshix-api/env.sh`. + Read more: [`platform/databricks/delta/README.md`](delta/README.md) diff --git a/platform/databricks/delta/README.md b/platform/databricks/delta/README.md index 5986358..ed63719 100644 --- a/platform/databricks/delta/README.md +++ b/platform/databricks/delta/README.md @@ -38,6 +38,7 @@ cannot overwrite key, ordering, or delete semantics. Bundle lifecycle: - `scripts/ensure-databricks-delta-secret.sh [scope] [key]` +- `scripts/bootstrap-databricks-delta.sh ` - `scripts/deploy-databricks-delta.sh ` - `scripts/run-databricks-delta-job.sh [job_key]` - `scripts/run-databricks-delta-smoke.sh ` @@ -57,9 +58,22 @@ variable. Helper defaults: -- `DATABRICKS_DELTA_SECRET_SCOPE=convex-streaming-olap-export` +- `DATABRICKS_DELTA_SECRET_SCOPE=convex-sync-kit-meshix-api` - `DATABRICKS_DELTA_SECRET_KEY=convex-deploy-key` +Source-aware defaults come from `sources//env.sh`. The current checked-in +profile is `sources/meshix-api/env.sh`. + +Recommended long-lived naming: + +- `convex_sync_kit__delta_control` +- `convex_sync_kit__delta_bronze` +- `convex_sync_kit__delta_silver` + +The Databricks bundle also uses a source-specific deployment slug so multiple +Convex sources can coexist in one workspace without clobbering each other's +bundle state or extractor job names. + If `CONVEX_DEPLOY_KEY` is available locally, the deploy and run helpers will create or update that Databricks secret automatically before validating, deploying, or running the job. If the local key is not available, the helpers @@ -68,6 +82,7 @@ require the target Databricks secret to already exist. Bootstrap SQL can still be applied directly with: - `scripts/apply-databricks-sql-dir.sh ` +- `scripts/bootstrap-databricks-delta.sh ` The extractor mirrors the Rust source/checkpoint logic and does not depend on the local parquet/S3 path. @@ -89,6 +104,7 @@ Recommended operator entrypoints: ```bash just databricks-delta-sync-secret +just databricks-delta-bootstrap just databricks-delta-deploy just databricks-delta-run just databricks-delta-smoke diff --git a/platform/databricks/delta/databricks.yml b/platform/databricks/delta/databricks.yml index 6512f1e..05e6ea7 100644 --- a/platform/databricks/delta/databricks.yml +++ b/platform/databricks/delta/databricks.yml @@ -1,11 +1,17 @@ bundle: - name: convex-streaming-olap-export-delta + name: convex-sync-kit-delta uuid: 2d5d8d8e-cdb2-41e4-bc7e-7ffae07f628d include: - resources/*.yml variables: + source_slug: + description: Stable source slug used for deployment names and workspace paths. + deployment_slug: + description: Unique deployment slug for this source and target environment. + job_name: + description: Databricks job name for the extractor deployment. convex_deployment_url: description: Convex deployment root URL. convex_deploy_key_secret_scope: @@ -31,25 +37,33 @@ targets: default: true workspace: profile: DEFAULT + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${var.deployment_slug} variables: - convex_deploy_key_secret_scope: convex-streaming-olap-export + source_slug: meshix-api + deployment_slug: meshix-api-dev + job_name: convex-sync-kit-meshix-api-dev-delta-extract + convex_deploy_key_secret_scope: convex-sync-kit-meshix-api convex_deploy_key_secret_key: convex-deploy-key - source_id: convex-streaming-olap-export-dev + source_id: meshix-api-dev table_name: "" catalog: workspace - control_schema: convex_streaming_olap_export_control - bronze_schema: convex_streaming_olap_export_bronze + control_schema: convex_sync_kit_meshix_api_delta_control + bronze_schema: convex_sync_kit_meshix_api_delta_bronze checkpoint_table: connector_checkpoint prod: mode: production workspace: profile: DEFAULT + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${var.deployment_slug} variables: - convex_deploy_key_secret_scope: convex-streaming-olap-export + source_slug: meshix-api + deployment_slug: meshix-api-prod + job_name: convex-sync-kit-meshix-api-prod-delta-extract + convex_deploy_key_secret_scope: convex-sync-kit-meshix-api convex_deploy_key_secret_key: convex-deploy-key - source_id: convex-streaming-olap-export + source_id: meshix-api table_name: "" catalog: workspace - control_schema: convex_streaming_olap_export_control - bronze_schema: convex_streaming_olap_export_bronze + control_schema: convex_sync_kit_meshix_api_delta_control + bronze_schema: convex_sync_kit_meshix_api_delta_bronze checkpoint_table: connector_checkpoint diff --git a/platform/databricks/delta/resources/convex_delta_extract.job.yml b/platform/databricks/delta/resources/convex_delta_extract.job.yml index 7e08082..4cbccf2 100644 --- a/platform/databricks/delta/resources/convex_delta_extract.job.yml +++ b/platform/databricks/delta/resources/convex_delta_extract.job.yml @@ -1,7 +1,7 @@ resources: jobs: convex_delta_extract: - name: convex-delta-extract + name: ${var.job_name} max_concurrent_runs: 1 tasks: - task_key: convex_delta_extract diff --git a/platform/databricks/s3/README.md b/platform/databricks/s3/README.md index f067ec8..dfca849 100644 --- a/platform/databricks/s3/README.md +++ b/platform/databricks/s3/README.md @@ -26,6 +26,13 @@ flowchart LR The published `staging/current` prefix must be covered by a Unity Catalog external location before the view sync is applied. +Recommended long-lived schema naming: + +- `convex_sync_kit__s3` + +The current checked-in source profile is `sources/meshix-api/env.sh`, which +sets `DATABRICKS_S3_SCHEMA=convex_sync_kit_meshix_api_s3`. + Recommended entrypoint: ```bash diff --git a/platform/databricks/s3/terraform/unity_catalog_s3_external_location/terraform.tfvars.example b/platform/databricks/s3/terraform/unity_catalog_s3_external_location/terraform.tfvars.example index 299d171..973870e 100644 --- a/platform/databricks/s3/terraform/unity_catalog_s3_external_location/terraform.tfvars.example +++ b/platform/databricks/s3/terraform/unity_catalog_s3_external_location/terraform.tfvars.example @@ -1,13 +1,13 @@ databricks_profile = "DEFAULT" databricks_config_file = "~/.databrickscfg" -aws_role_arn = "arn:aws:iam::123456789012:role/example-convex-streaming-olap-export-databricks-prod-reader" -storage_credential_name = "example-aws-s3-convex-streaming-olap-export" -external_location_name = "example-s3-convex-streaming-olap-export" -external_location_url = "s3://example-convex-streaming-olap-export-bucket/prod/staging/current/" +aws_role_arn = "arn:aws:iam::123456789012:role/example-convex-sync-kit-databricks-prod-reader" +storage_credential_name = "example-aws-s3-convex-sync-kit" +external_location_name = "example-s3-convex-sync-kit" +external_location_url = "s3://example-convex-sync-kit-bucket/prod/staging/current/" -storage_credential_comment = "AWS S3 access for convex-streaming-olap-export" -external_location_comment = "AWS S3 external location for convex-streaming-olap-export" +storage_credential_comment = "AWS S3 access for convex-sync-kit" +external_location_comment = "AWS S3 external location for convex-sync-kit" storage_credential_read_only = true external_location_read_only = true diff --git a/release-please-config.json b/release-please-config.json index 410db09..f856990 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -5,15 +5,39 @@ "bootstrap-sha": "0cf9f47d6cc225c8080946f774bd98c919e3aec8", "bump-minor-pre-major": true, "bump-patch-for-minor-pre-major": true, - "plugins": [ - "cargo-workspace" - ], "packages": { - "apps/convex-sync": { - "release-type": "rust", - "package-name": "convex-sync", + ".": { + "release-type": "simple", + "package-name": "convex-sync-kit", "changelog-path": "CHANGELOG.md", - "include-component-in-tag": false + "include-component-in-tag": false, + "extra-files": [ + { + "type": "toml", + "path": "Cargo.toml", + "jsonpath": "$.workspace.package.version" + }, + { + "type": "toml", + "path": "apps/convex-sync/Cargo.toml", + "jsonpath": "$.package.version" + }, + { + "type": "toml", + "path": "apps/convex-inspect/Cargo.toml", + "jsonpath": "$.package.version" + }, + { + "type": "toml", + "path": "crates/convex-sync-core/Cargo.toml", + "jsonpath": "$.package.version" + }, + { + "type": "toml", + "path": "crates/convex-export-s3/Cargo.toml", + "jsonpath": "$.package.version" + } + ] } } } diff --git a/scripts/bootstrap-databricks-delta.sh b/scripts/bootstrap-databricks-delta.sh new file mode 100755 index 0000000..a0a2bdd --- /dev/null +++ b/scripts/bootstrap-databricks-delta.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +profile="$1" +warehouse_id="$2" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +bootstrap_src="$repo_root/platform/databricks/delta/sql/bootstrap" +render_dir="$(mktemp -d)" + +catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" +checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" + +"$repo_root/scripts/render-databricks-delta-bootstrap.sh" \ + "$bootstrap_src" \ + "$render_dir" \ + "$catalog" \ + "$control_schema" \ + "$bronze_schema" \ + "$silver_schema" \ + "$checkpoint_table" + +"$repo_root/scripts/apply-databricks-sql-dir.sh" "$profile" "$warehouse_id" "$render_dir" + +echo "bootstrap complete" +echo "catalog=$catalog" +echo "control_schema=$control_schema" +echo "bronze_schema=$bronze_schema" +echo "silver_schema=$silver_schema" +echo "checkpoint_table=$checkpoint_table" diff --git a/scripts/convex-sync-dev b/scripts/convex-sync-dev index e0e6cb8..db67dac 100755 --- a/scripts/convex-sync-dev +++ b/scripts/convex-sync-dev @@ -17,7 +17,7 @@ repo_root="$(cd -- "${script_dir}/.." >/dev/null 2>&1 && pwd)" binary_path="${repo_root}/target/debug/convex-sync" if [[ ! -f "${repo_root}/Cargo.toml" || ! -f "${repo_root}/apps/convex-sync/src/main.rs" ]]; then - printf 'convex-sync-dev must run from a convex-streaming-olap-export checkout\n' >&2 + printf 'convex-sync-dev must run from a convex-sync-kit checkout\n' >&2 exit 1 fi diff --git a/scripts/deploy-databricks-delta.sh b/scripts/deploy-databricks-delta.sh index e1bdd90..e3c6295 100755 --- a/scripts/deploy-databricks-delta.sh +++ b/scripts/deploy-databricks-delta.sh @@ -12,6 +12,9 @@ bundle_engine="${DATABRICKS_BUNDLE_ENGINE:-direct}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" bundle_root="$repo_root/platform/databricks/delta" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" read_env_file_value() { local key="$1" @@ -35,18 +38,25 @@ if [[ -z "$deployment_url" ]]; then fi source_id="${CONVEX_SOURCE_ID:-$deployment_url}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" table_name="${CONVEX_TABLE_NAME:-}" -secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}" +deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +job_name="${DATABRICKS_DELTA_JOB_NAME:-convex-sync-kit-${deployment_slug}-delta-extract}" +secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-sync-kit}" secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" -control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_streaming_olap_export_control}" -bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_streaming_olap_export_bronze}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" "$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" bundle_args=( --var "convex_deployment_url=$deployment_url" + --var "source_slug=$source_slug" + --var "deployment_slug=$deployment_slug" + --var "job_name=$job_name" --var "convex_deploy_key_secret_scope=$secret_scope" --var "convex_deploy_key_secret_key=$secret_key" --var "source_id=$source_id" diff --git a/scripts/ensure-databricks-delta-secret.sh b/scripts/ensure-databricks-delta-secret.sh index c1f3e8d..11ddede 100755 --- a/scripts/ensure-databricks-delta-secret.sh +++ b/scripts/ensure-databricks-delta-secret.sh @@ -11,6 +11,9 @@ scope_arg="${2:-}" key_arg="${3:-}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" read_env_file_value() { local key="$1" @@ -26,7 +29,7 @@ read_env_file_value() { printf '%s' "${line#*=}" } -scope="${scope_arg:-${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}}" +scope="${scope_arg:-${DATABRICKS_DELTA_SECRET_SCOPE:-convex-sync-kit}}" key="${key_arg:-${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}}" deploy_key="${CONVEX_DEPLOY_KEY:-$(read_env_file_value CONVEX_DEPLOY_KEY || true)}" diff --git a/scripts/load-source-config.sh b/scripts/load-source-config.sh new file mode 100755 index 0000000..3a69e74 --- /dev/null +++ b/scripts/load-source-config.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +load_convex_sync_source_config() { + local repo_root="$1" + local source_name="${CONVEX_SYNC_SOURCE:-meshix-api}" + local source_file="$repo_root/sources/$source_name/env.sh" + + if [[ ! -f "$source_file" ]]; then + if [[ -n "${CONVEX_SYNC_SOURCE:-}" ]]; then + echo "unknown Convex source config: $source_name" >&2 + return 1 + fi + return 0 + fi + + # shellcheck source=/dev/null + source "$source_file" +} diff --git a/scripts/run-databricks-delta-job.sh b/scripts/run-databricks-delta-job.sh index b97590c..11d5844 100755 --- a/scripts/run-databricks-delta-job.sh +++ b/scripts/run-databricks-delta-job.sh @@ -13,6 +13,9 @@ bundle_engine="${DATABRICKS_BUNDLE_ENGINE:-direct}" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" bundle_root="$repo_root/platform/databricks/delta" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" read_env_file_value() { local key="$1" @@ -36,18 +39,25 @@ if [[ -z "$deployment_url" ]]; then fi source_id="${CONVEX_SOURCE_ID:-$deployment_url}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" table_name="${CONVEX_TABLE_NAME:-}" -secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}" +deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +job_name="${DATABRICKS_DELTA_JOB_NAME:-convex-sync-kit-${deployment_slug}-delta-extract}" +secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-sync-kit}" secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" -control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_streaming_olap_export_control}" -bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_streaming_olap_export_bronze}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" "$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" bundle_args=( --var "convex_deployment_url=$deployment_url" + --var "source_slug=$source_slug" + --var "deployment_slug=$deployment_slug" + --var "job_name=$job_name" --var "convex_deploy_key_secret_scope=$secret_scope" --var "convex_deploy_key_secret_key=$secret_key" --var "source_id=$source_id" diff --git a/scripts/run-databricks-delta-smoke.sh b/scripts/run-databricks-delta-smoke.sh index 8b78f5a..37e9d04 100755 --- a/scripts/run-databricks-delta-smoke.sh +++ b/scripts/run-databricks-delta-smoke.sh @@ -11,17 +11,20 @@ target="$2" warehouse_id="$3" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -bootstrap_src="$repo_root/platform/databricks/delta/sql/bootstrap" -render_dir="$(mktemp -d)" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" timestamp="$(date +%Y%m%d%H%M%S)" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" -control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_streaming_olap_export_delta_smoke_${timestamp}_control}" -bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_streaming_olap_export_delta_smoke_${timestamp}_bronze}" -silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_streaming_olap_export_delta_smoke_${timestamp}_silver}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_smoke_${timestamp}_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_smoke_${timestamp}_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_smoke_${timestamp}_silver}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" table_name="${CONVEX_TABLE_NAME:-jobs}" -source_id="${CONVEX_SOURCE_ID:-convex-streaming-olap-export-delta-smoke}" +source_id="${CONVEX_SOURCE_ID:-${source_slug}-delta-smoke}" export DATABRICKS_DELTA_CATALOG="$catalog" export DATABRICKS_DELTA_CONTROL_SCHEMA="$control_schema" @@ -30,17 +33,10 @@ export DATABRICKS_DELTA_SILVER_SCHEMA="$silver_schema" export DATABRICKS_DELTA_CHECKPOINT_TABLE="$checkpoint_table" export CONVEX_SOURCE_ID="$source_id" export CONVEX_TABLE_NAME="$table_name" +export DATABRICKS_DELTA_DEPLOYMENT_SLUG="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-smoke}" +export DATABRICKS_DELTA_JOB_NAME="${DATABRICKS_DELTA_JOB_NAME:-convex-sync-kit-${source_slug}-smoke-delta-extract}" -"$repo_root/scripts/render-databricks-delta-bootstrap.sh" \ - "$bootstrap_src" \ - "$render_dir" \ - "$catalog" \ - "$control_schema" \ - "$bronze_schema" \ - "$silver_schema" \ - "$checkpoint_table" - -"$repo_root/scripts/apply-databricks-sql-dir.sh" "$profile" "$warehouse_id" "$render_dir" +"$repo_root/scripts/bootstrap-databricks-delta.sh" "$profile" "$warehouse_id" "$repo_root/scripts/deploy-databricks-delta.sh" "$profile" "$target" "$repo_root/scripts/run-databricks-delta-job.sh" "$profile" "$target" diff --git a/scripts/sync-databricks-staging-views.sh b/scripts/sync-databricks-staging-views.sh index 53ec410..b0911fb 100755 --- a/scripts/sync-databricks-staging-views.sh +++ b/scripts/sync-databricks-staging-views.sh @@ -2,11 +2,16 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" profile="${DATABRICKS_PROFILE:-DEFAULT}" warehouse_id="${DATABRICKS_WAREHOUSE_ID:-}" catalog="${DATABRICKS_CATALOG:-workspace}" -schema="${DATABRICKS_SCHEMA:-convex_streaming_olap_export}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +schema="${DATABRICKS_SCHEMA:-${DATABRICKS_S3_SCHEMA:-convex_sync_kit_${source_slug_sql}_s3}}" bucket="${S3_BUCKET:-}" prefix="${S3_PREFIX:-}" label="${LABEL:-sync}" diff --git a/sources/README.md b/sources/README.md new file mode 100644 index 0000000..af79d7a --- /dev/null +++ b/sources/README.md @@ -0,0 +1,27 @@ +# Sources + +This repo is a generic Convex sync engine. Source-specific defaults live here +so one checkout can manage multiple Convex deployments without renaming the +repo or duplicating the platform logic. + +Each source directory should contain an `env.sh` that exports guarded defaults +using `: "${VAR:=value}"`. + +Recommended contents: + +- `CONVEX_SYNC_SOURCE` +- `CONVEX_SYNC_SOURCE_SLUG` +- `CONVEX_SYNC_SOURCE_SQL` +- `CONVEX_SOURCE_ID` +- `DATABRICKS_S3_SCHEMA` +- `DATABRICKS_DELTA_SECRET_SCOPE` +- `DATABRICKS_DELTA_SECRET_KEY` +- `DATABRICKS_DELTA_CATALOG` +- `DATABRICKS_DELTA_CONTROL_SCHEMA` +- `DATABRICKS_DELTA_BRONZE_SCHEMA` +- `DATABRICKS_DELTA_SILVER_SCHEMA` +- `DATABRICKS_DELTA_CHECKPOINT_TABLE` + +Scripts load `sources/${CONVEX_SYNC_SOURCE:-meshix-api}/env.sh` automatically. +Explicit environment variables still win because the source files only set +defaults. diff --git a/sources/meshix-api/env.sh b/sources/meshix-api/env.sh new file mode 100755 index 0000000..31a8874 --- /dev/null +++ b/sources/meshix-api/env.sh @@ -0,0 +1,27 @@ +: "${CONVEX_SYNC_SOURCE:=meshix-api}" +: "${CONVEX_SYNC_SOURCE_SLUG:=meshix-api}" +: "${CONVEX_SYNC_SOURCE_SQL:=meshix_api}" +: "${CONVEX_SOURCE_ID:=meshix-api}" + +: "${DATABRICKS_S3_SCHEMA:=convex_sync_kit_meshix_api_s3}" + +: "${DATABRICKS_DELTA_SECRET_SCOPE:=convex-sync-kit-meshix-api}" +: "${DATABRICKS_DELTA_SECRET_KEY:=convex-deploy-key}" +: "${DATABRICKS_DELTA_CATALOG:=workspace}" +: "${DATABRICKS_DELTA_CONTROL_SCHEMA:=convex_sync_kit_meshix_api_delta_control}" +: "${DATABRICKS_DELTA_BRONZE_SCHEMA:=convex_sync_kit_meshix_api_delta_bronze}" +: "${DATABRICKS_DELTA_SILVER_SCHEMA:=convex_sync_kit_meshix_api_delta_silver}" +: "${DATABRICKS_DELTA_CHECKPOINT_TABLE:=connector_checkpoint}" + +export CONVEX_SYNC_SOURCE +export CONVEX_SYNC_SOURCE_SLUG +export CONVEX_SYNC_SOURCE_SQL +export CONVEX_SOURCE_ID +export DATABRICKS_S3_SCHEMA +export DATABRICKS_DELTA_SECRET_SCOPE +export DATABRICKS_DELTA_SECRET_KEY +export DATABRICKS_DELTA_CATALOG +export DATABRICKS_DELTA_CONTROL_SCHEMA +export DATABRICKS_DELTA_BRONZE_SCHEMA +export DATABRICKS_DELTA_SILVER_SCHEMA +export DATABRICKS_DELTA_CHECKPOINT_TABLE From 5bb22bd747b735fe6492f386a02747ad1eff99aa Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 01:38:16 -0500 Subject: [PATCH 2/2] fix: keep delta smoke schemas isolated from source defaults --- scripts/run-databricks-delta-smoke.sh | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/scripts/run-databricks-delta-smoke.sh b/scripts/run-databricks-delta-smoke.sh index 37e9d04..da4c549 100755 --- a/scripts/run-databricks-delta-smoke.sh +++ b/scripts/run-databricks-delta-smoke.sh @@ -11,9 +11,54 @@ target="$2" warehouse_id="$3" repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +preserve_or_unset() { + local var_name="$1" + local was_set="$2" + local prior_value="$3" + if [[ "$was_set" == "1" ]]; then + printf -v "$var_name" '%s' "$prior_value" + export "$var_name" + else + unset "$var_name" || true + fi +} + +had_control_schema=0 +prior_control_schema="" +if [[ -n "${DATABRICKS_DELTA_CONTROL_SCHEMA+x}" ]]; then + had_control_schema=1 + prior_control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA}" +fi + +had_bronze_schema=0 +prior_bronze_schema="" +if [[ -n "${DATABRICKS_DELTA_BRONZE_SCHEMA+x}" ]]; then + had_bronze_schema=1 + prior_bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA}" +fi + +had_silver_schema=0 +prior_silver_schema="" +if [[ -n "${DATABRICKS_DELTA_SILVER_SCHEMA+x}" ]]; then + had_silver_schema=1 + prior_silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA}" +fi + +had_source_id=0 +prior_source_id="" +if [[ -n "${CONVEX_SOURCE_ID+x}" ]]; then + had_source_id=1 + prior_source_id="${CONVEX_SOURCE_ID}" +fi + # shellcheck source=/dev/null source "$repo_root/scripts/load-source-config.sh" load_convex_sync_source_config "$repo_root" +preserve_or_unset DATABRICKS_DELTA_CONTROL_SCHEMA "$had_control_schema" "$prior_control_schema" +preserve_or_unset DATABRICKS_DELTA_BRONZE_SCHEMA "$had_bronze_schema" "$prior_bronze_schema" +preserve_or_unset DATABRICKS_DELTA_SILVER_SCHEMA "$had_silver_schema" "$prior_silver_schema" +preserve_or_unset CONVEX_SOURCE_ID "$had_source_id" "$prior_source_id" timestamp="$(date +%Y%m%d%H%M%S)" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}"