fix: move Databricks Delta deploy key into secret scope

README.md

@@ -144,6 +144,7 @@ Runtime split:
 
 Packaged entrypoints:
 
+- `just databricks-delta-sync-secret`
 - `just databricks-delta-deploy`
 - `just databricks-delta-run`
 - `just databricks-delta-smoke <warehouse_id>`

justfile

@@ -75,6 +75,9 @@ databricks-sync-staging-views *args:
 databricks-apply-sql-dir profile warehouse_id sql_dir:
   ./scripts/apply-databricks-sql-dir.sh {{profile}} {{warehouse_id}} {{sql_dir}}
 
+databricks-delta-sync-secret *args:
+  ./scripts/ensure-databricks-delta-secret.sh {{args}}
+
 databricks-delta-deploy profile="DEFAULT" target="dev":
   ./scripts/deploy-databricks-delta.sh {{profile}} {{target}}
 

platform/databricks/delta/README.md

@@ -37,13 +37,34 @@ cannot overwrite key, ordering, or delete semantics.
 
 Bundle lifecycle:
 
+- `scripts/ensure-databricks-delta-secret.sh <profile> [scope] [key]`
 - `scripts/deploy-databricks-delta.sh <profile> <target>`
 - `scripts/run-databricks-delta-job.sh <profile> <target> [job_key]`
 - `scripts/run-databricks-delta-smoke.sh <profile> <target> <warehouse_id>`
 
 These scripts default `DATABRICKS_BUNDLE_ENGINE=direct` so deployment does not
 depend on Terraform downloads.
 
+## Secret Contract
+
+The Databricks bundle never receives the raw Convex deploy key as a bundle
+variable.
+
+- The deploy key lives in a Databricks secret scope.
+- The bundle only carries the secret scope name and secret key name.
+- The extractor resolves the deploy key inside Databricks at runtime with
+  `dbutils.secrets.get(...)`.
+
+Helper defaults:
+
+- `DATABRICKS_DELTA_SECRET_SCOPE=convex-streaming-olap-export`
+- `DATABRICKS_DELTA_SECRET_KEY=convex-deploy-key`
+
+If `CONVEX_DEPLOY_KEY` is available locally, the deploy and run helpers will
+create or update that Databricks secret automatically before validating,
+deploying, or running the job. If the local key is not available, the helpers
+require the target Databricks secret to already exist.
+
 Bootstrap SQL can still be applied directly with:
 
 - `scripts/apply-databricks-sql-dir.sh <profile> <warehouse_id> <rendered_sql_dir>`
@@ -67,6 +88,7 @@ flowchart LR
 Recommended operator entrypoints:
 
 ```bash
+just databricks-delta-sync-secret
 just databricks-delta-deploy
 just databricks-delta-run
 just databricks-delta-smoke <warehouse_id>

platform/databricks/delta/databricks.yml

@@ -8,8 +8,10 @@ include:
 variables:
   convex_deployment_url:
     description: Convex deployment root URL.
-  convex_deploy_key:
-    description: Convex deploy key used by the extractor.
+  convex_deploy_key_secret_scope:
+    description: Databricks secret scope that stores the Convex deploy key.
+  convex_deploy_key_secret_key:
+    description: Secret key name inside the Databricks secret scope.
   source_id:
     description: Source identifier stored in the checkpoint table.
   table_name:
@@ -30,6 +32,8 @@ targets:
     workspace:
       profile: DEFAULT
     variables:
+      convex_deploy_key_secret_scope: convex-streaming-olap-export
+      convex_deploy_key_secret_key: convex-deploy-key
       source_id: convex-streaming-olap-export-dev
       table_name: ""
       catalog: workspace
@@ -41,6 +45,8 @@ targets:
     workspace:
       profile: DEFAULT
     variables:
+      convex_deploy_key_secret_scope: convex-streaming-olap-export
+      convex_deploy_key_secret_key: convex-deploy-key
       source_id: convex-streaming-olap-export
       table_name: ""
       catalog: workspace

platform/databricks/delta/extractor/README.md

@@ -14,7 +14,9 @@ It mirrors the current Rust source/checkpoint behavior:
 ## Required environment
 
 - `CONVEX_DEPLOYMENT_URL`
-- `CONVEX_DEPLOY_KEY`
+- one of:
+  - `CONVEX_DEPLOY_KEY`
+  - `CONVEX_DEPLOY_KEY_SECRET_SCOPE` and `CONVEX_DEPLOY_KEY_SECRET_KEY`
 
 ## Optional environment
 
@@ -25,5 +27,6 @@ It mirrors the current Rust source/checkpoint behavior:
 - `DATABRICKS_BRONZE_SCHEMA`: defaults to `bronze`
 - `DATABRICKS_CHECKPOINT_TABLE`: defaults to `connector_checkpoint`
 
-In the bundled Databricks Delta path, these are usually passed as task
-parameters rather than exported manually.
+In the bundled Databricks Delta path, the job receives the secret scope/key
+names as task parameters and resolves the actual deploy key with
+`dbutils.secrets.get(...)` at runtime.

platform/databricks/delta/extractor/convex_cdc_job.py

@@ -20,6 +20,11 @@
     StructType,
 )
 
+try:
+    from pyspark.dbutils import DBUtils
+except ImportError:  # local syntax checks and non-Databricks execution
+    DBUtils = None  # type: ignore[assignment]
+
 
 spark = SparkSession.builder.getOrCreate()
 
@@ -49,6 +54,33 @@ def opt(value: Optional[str], env_name: str, default: Optional[str] = None) -> s
     return env(env_name, default)
 
 
+def resolve_deploy_key(
+    *,
+    direct_value: Optional[str],
+    secret_scope: Optional[str],
+    secret_key: Optional[str],
+) -> str:
+    if direct_value is not None:
+        return direct_value
+
+    env_value = os.getenv("CONVEX_DEPLOY_KEY")
+    if env_value:
+        return env_value
+
+    scope = secret_scope or os.getenv("CONVEX_DEPLOY_KEY_SECRET_SCOPE")
+    key = secret_key or os.getenv("CONVEX_DEPLOY_KEY_SECRET_KEY")
+    if not scope or not key:
+        raise RuntimeError(
+            "missing Convex deploy key: provide --deploy-key, CONVEX_DEPLOY_KEY, "
+            "or both deploy-key secret scope/key settings"
+        )
+
+    if DBUtils is None:
+        raise RuntimeError("pyspark.dbutils.DBUtils is unavailable outside Databricks runtime")
+
+    return DBUtils(spark).secrets.get(scope=scope, key=key)
+
+
 @dataclass
 class Checkpoint:
     phase: str
@@ -374,6 +406,8 @@ def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Convex CDC Databricks extractor")
     parser.add_argument("--deployment-url")
     parser.add_argument("--deploy-key")
+    parser.add_argument("--deploy-key-secret-scope")
+    parser.add_argument("--deploy-key-secret-key")
     parser.add_argument("--source-id")
     parser.add_argument("--table-name")
     parser.add_argument("--catalog")
@@ -441,7 +475,11 @@ def run_once() -> None:
     args = parse_args()
 
     deployment_url = opt(args.deployment_url, "CONVEX_DEPLOYMENT_URL")
-    deploy_key = opt(args.deploy_key, "CONVEX_DEPLOY_KEY")
+    deploy_key = resolve_deploy_key(
+        direct_value=args.deploy_key,
+        secret_scope=args.deploy_key_secret_scope,
+        secret_key=args.deploy_key_secret_key,
+    )
     source_id = opt(args.source_id, "CONVEX_SOURCE_ID", deployment_url)
     table_name = args.table_name if args.table_name is not None else os.getenv("CONVEX_TABLE_NAME")
 

platform/databricks/delta/resources/convex_delta_extract.job.yml

@@ -11,8 +11,10 @@ resources:
             parameters:
               - --deployment-url
               - ${var.convex_deployment_url}
-              - --deploy-key
-              - ${var.convex_deploy_key}
+              - --deploy-key-secret-scope
+              - ${var.convex_deploy_key_secret_scope}
+              - --deploy-key-secret-key
+              - ${var.convex_deploy_key_secret_key}
               - --source-id
               - ${var.source_id}
               - --table-name

scripts/deploy-databricks-delta.sh

@@ -28,23 +28,27 @@ read_env_file_value() {
 }
 
 deployment_url="${CONVEX_DEPLOYMENT_URL:-$(read_env_file_value CONVEX_DEPLOYMENT_URL || true)}"
-deploy_key="${CONVEX_DEPLOY_KEY:-$(read_env_file_value CONVEX_DEPLOY_KEY || true)}"
 
-if [[ -z "$deployment_url" || -z "$deploy_key" ]]; then
-  echo "CONVEX_DEPLOYMENT_URL and CONVEX_DEPLOY_KEY are required" >&2
+if [[ -z "$deployment_url" ]]; then
+  echo "CONVEX_DEPLOYMENT_URL is required" >&2
   exit 1
 fi
 
 source_id="${CONVEX_SOURCE_ID:-$deployment_url}"
 table_name="${CONVEX_TABLE_NAME:-}"
+secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}"
+secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}"
 catalog="${DATABRICKS_DELTA_CATALOG:-workspace}"
 control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_streaming_olap_export_control}"
 bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_streaming_olap_export_bronze}"
 checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}"
 
+"$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key"
+
 bundle_args=(
   --var "convex_deployment_url=$deployment_url"
-  --var "convex_deploy_key=$deploy_key"
+  --var "convex_deploy_key_secret_scope=$secret_scope"
+  --var "convex_deploy_key_secret_key=$secret_key"
   --var "source_id=$source_id"
   --var "table_name=$table_name"
   --var "catalog=$catalog"

scripts/ensure-databricks-delta-secret.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -lt 1 || "$#" -gt 3 ]]; then
+  echo "usage: $0 <profile> [scope] [key]" >&2
+  exit 1
+fi
+
+profile="$1"
+scope_arg="${2:-}"
+key_arg="${3:-}"
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+read_env_file_value() {
+  local key="$1"
+  local env_file="$repo_root/.env"
+  if [[ ! -f "$env_file" ]]; then
+    return 1
+  fi
+  local line
+  line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)"
+  if [[ -z "$line" ]]; then
+    return 1
+  fi
+  printf '%s' "${line#*=}"
+}
+
+scope="${scope_arg:-${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}}"
+key="${key_arg:-${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}}"
+deploy_key="${CONVEX_DEPLOY_KEY:-$(read_env_file_value CONVEX_DEPLOY_KEY || true)}"
+
+scopes_json="$(databricks secrets list-scopes -p "$profile" -o json)"
+scope_exists=false
+if jq -e --arg scope "$scope" '.[] | select(.name == $scope)' <<<"$scopes_json" >/dev/null; then
+  scope_exists=true
+fi
+
+if [[ -n "$deploy_key" ]]; then
+  if [[ "$scope_exists" == false ]]; then
+    databricks secrets create-scope "$scope" -p "$profile" >/dev/null
+  fi
+  printf '%s' "$deploy_key" | databricks secrets put-secret "$scope" "$key" -p "$profile" >/dev/null
+  echo "synced Databricks secret $scope/$key"
+  exit 0
+fi
+
+if [[ "$scope_exists" == false ]]; then
+  echo "Databricks secret scope $scope does not exist and CONVEX_DEPLOY_KEY is not available to create it" >&2
+  exit 1
+fi
+
+if ! databricks secrets list-secrets "$scope" -p "$profile" -o json | jq -e --arg key "$key" '.[] | select(.key == $key)' >/dev/null; then
+  echo "Databricks secret $scope/$key does not exist and CONVEX_DEPLOY_KEY is not available to create it" >&2
+  exit 1
+fi
+
+echo "using existing Databricks secret $scope/$key"

scripts/run-databricks-delta-job.sh

@@ -29,23 +29,27 @@ read_env_file_value() {
 }
 
 deployment_url="${CONVEX_DEPLOYMENT_URL:-$(read_env_file_value CONVEX_DEPLOYMENT_URL || true)}"
-deploy_key="${CONVEX_DEPLOY_KEY:-$(read_env_file_value CONVEX_DEPLOY_KEY || true)}"
 
-if [[ -z "$deployment_url" || -z "$deploy_key" ]]; then
-  echo "CONVEX_DEPLOYMENT_URL and CONVEX_DEPLOY_KEY are required" >&2
+if [[ -z "$deployment_url" ]]; then
+  echo "CONVEX_DEPLOYMENT_URL is required" >&2
   exit 1
 fi
 
 source_id="${CONVEX_SOURCE_ID:-$deployment_url}"
 table_name="${CONVEX_TABLE_NAME:-}"
+secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-streaming-olap-export}"
+secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}"
 catalog="${DATABRICKS_DELTA_CATALOG:-workspace}"
 control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_streaming_olap_export_control}"
 bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_streaming_olap_export_bronze}"
 checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}"
 
+"$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key"
+
 bundle_args=(
   --var "convex_deployment_url=$deployment_url"
-  --var "convex_deploy_key=$deploy_key"
+  --var "convex_deploy_key_secret_scope=$secret_scope"
+  --var "convex_deploy_key_secret_key=$secret_key"
   --var "source_id=$source_id"
   --var "table_name=$table_name"
   --var "catalog=$catalog"