[WIP] Add databricks dbconnect init / sync commands

NEXT_CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Notable Changes
 
 ### CLI
+* Add `databricks dbconnect sync` to provision a local Python environment (Python version, `databricks-connect` pin, and dependency constraints) matched to the selected Databricks compute target. A project with no `pyproject.toml` is initialized from scratch; an existing one is merged in place. Supports `--cluster`/`--serverless`/`--job` target selection, `--constraints-only` (skip `databricks-connect`), `--check` (dry run), and `--output json` for the VS Code extension.
 
 ### Bundles
 

acceptance/dbconnect/constraints-only/output.txt

@@ -0,0 +1,53 @@
+
+>>> [CLI] dbconnect sync --serverless v4 --constraints-only --check --output json
+{
+  "schemaVersion": 1,
+  "command": "dbconnect sync",
+  "ok": true,
+  "mode": "constraints-only",
+  "dryRun": true,
+  "target": {
+    "source": "serverless",
+    "serverlessVersion": "v4",
+    "envKey": "serverless/serverless-v4"
+  },
+  "resolved": {
+    "pythonVersion": "3.12",
+    "artifactSource": "network"
+  },
+  "greenfield": true,
+  "plan": {
+    "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml",
+    "wouldInstallPython": "3.12",
+    "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,15 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+# managed by databricks dbconnect — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+    \"pyarrow\u003c19\",\n+    \"pandas\u003c3\",\n+]\n+# end managed by databricks dbconnect\n"
+  },
+  "phases": [
+    {
+      "phase": "preflight",
+      "status": "ok"
+    },
+    {
+      "phase": "resolve",
+      "status": "ok"
+    },
+    {
+      "phase": "fetch",
+      "status": "ok"
+    },
+    {
+      "phase": "merge",
+      "status": "ok"
+    },
+    {
+      "phase": "provision",
+      "status": "ok"
+    },
+    {
+      "phase": "validate",
+      "status": "ok"
+    }
+  ],
+  "warnings": [],
+  "error": null,
+  "durationMs": 0
+}

acceptance/dbconnect/constraints-only/script

@@ -0,0 +1 @@
+trace $CLI dbconnect sync --serverless v4 --constraints-only --check --output json

acceptance/dbconnect/constraints-only/test.toml

@@ -0,0 +1,21 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[Env]
+DATABRICKS_DBCONNECT_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
+
+[[Server]]
+Pattern = "GET /serverless/serverless-v4/pyproject.toml"
+Response.Body = '''
+[project]
+requires-python = ">=3.12"
+
+[dependency-groups]
+dev = ["databricks-connect~=17.2.0"]
+
+[tool.uv]
+constraint-dependencies = ["pyarrow<19", "pandas<3"]
+'''
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/env-unsupported/output.txt

@@ -0,0 +1,8 @@
+preflight  ok  uv [UV_VERSION]
+resolve    ok  source=cluster envKey=dbr/15.4.x-scala2.12
+fetch      error  no published environment for "dbr/15.4.x-scala2.12". If this is a new runtime, try the latest LTS target (e.g. --serverless v4 or a supported --cluster DBR): GET [DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml: environment key not found
+merge      pending
+provision  pending
+validate   pending
+For more detail, re-run with --debug, or --output json to share a structured report.
+Error: no published environment for "dbr/15.4.x-scala2.12". If this is a new runtime, try the latest LTS target (e.g. --serverless v4 or a supported --cluster DBR): GET [DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml: environment key not found

acceptance/dbconnect/env-unsupported/script

@@ -0,0 +1 @@
+musterr $CLI dbconnect sync --cluster test-cluster-id --check

acceptance/dbconnect/env-unsupported/test.toml

@@ -0,0 +1,22 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[Env]
+DATABRICKS_DBCONNECT_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
+
+[[Server]]
+Pattern = "GET /api/2.1/clusters/get"
+Response.Body = '''
+{
+  "cluster_id": "test-cluster-id",
+  "spark_version": "15.4.x-scala2.12"
+}
+'''
+
+[[Server]]
+Pattern = "GET /dbr/15.4.x-scala2.12/pyproject.toml"
+Response.StatusCode = 404
+Response.Body = '{"message": "Not found"}'
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/flag-conflict/output.txt

@@ -0,0 +1 @@
+Error: if any flags in the group [cluster serverless job] are set none of the others can be; [cluster serverless] were all set

acceptance/dbconnect/flag-conflict/script

@@ -0,0 +1 @@
+musterr $CLI dbconnect sync --cluster abc --serverless v4

acceptance/dbconnect/flag-conflict/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []

acceptance/dbconnect/help/output.txt

@@ -0,0 +1,46 @@
+Set up a local Python environment matched to your Databricks compute target.
+
+Derives the Python version, databricks-connect version, and dependency
+constraints from the selected compute (cluster, serverless, or job) so that
+local resolution matches the Databricks runtime.
+
+Usage:
+  databricks dbconnect [command]
+
+Available Commands:
+  sync        Provision a local Python environment matched to a Databricks compute target
+
+Flags:
+  -h, --help   help for dbconnect
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+Use "databricks dbconnect [command] --help" for more information about a command.
+Provision (or update) a local Python environment matched to a Databricks compute target.
+
+Resolves the target to an environment key, fetches the pinned Python version,
+databricks-connect version, and dependency constraints published for that key,
+then provisions a matched .venv with uv. A project with no pyproject.toml is
+initialized from scratch; an existing pyproject.toml is merged in place (its
+env-owned sections are refreshed, user-owned content is preserved).
+
+Usage:
+  databricks dbconnect sync [flags]
+
+Flags:
+      --check               compute the plan without writing files or provisioning
+      --cluster string      cluster ID to use as the compute target
+      --constraints-only    apply the Python version and constraints without adding the databricks-connect dependency
+  -h, --help                help for sync
+      --job string          job ID to use as the compute target
+      --serverless string   serverless version to use as the compute target (e.g. v4)
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)

acceptance/dbconnect/help/script

@@ -0,0 +1,2 @@
+$CLI dbconnect --help
+$CLI dbconnect sync --help

acceptance/dbconnect/help/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []

acceptance/dbconnect/json-error/output.txt

@@ -0,0 +1,42 @@
+{
+  "schemaVersion": 1,
+  "command": "dbconnect sync",
+  "ok": false,
+  "mode": "default",
+  "dryRun": false,
+  "greenfield": false,
+  "phases": [
+    {
+      "phase": "preflight",
+      "status": "ok"
+    },
+    {
+      "phase": "resolve",
+      "status": "error"
+    },
+    {
+      "phase": "fetch",
+      "status": "pending"
+    },
+    {
+      "phase": "merge",
+      "status": "pending"
+    },
+    {
+      "phase": "provision",
+      "status": "pending"
+    },
+    {
+      "phase": "validate",
+      "status": "pending"
+    }
+  ],
+  "warnings": [],
+  "error": {
+    "code": "E_NO_TARGET",
+    "failurePhase": "resolve",
+    "message": "No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job",
+    "diskMutated": false
+  },
+  "durationMs": 0
+}

acceptance/dbconnect/json-error/script

@@ -0,0 +1 @@
+musterr $CLI dbconnect sync --output json

acceptance/dbconnect/json-error/test.toml

@@ -0,0 +1,5 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/manager-unsupported/environment.yml

@@ -0,0 +1,3 @@
+name: demo
+dependencies:
+  - python=3.10

acceptance/dbconnect/manager-unsupported/output.txt

@@ -0,0 +1,8 @@
+preflight  error  detected a conda project; automated setup for conda is not yet available (P1). Use a uv project (add a pyproject.toml with a [tool.uv] table, or run `uv init`) to provision automatically
+resolve    pending
+fetch      pending
+merge      pending
+provision  pending
+validate   pending
+For more detail, re-run with --debug, or --output json to share a structured report.
+Error: detected a conda project; automated setup for conda is not yet available (P1). Use a uv project (add a pyproject.toml with a [tool.uv] table, or run `uv init`) to provision automatically

acceptance/dbconnect/manager-unsupported/script

@@ -0,0 +1 @@
+musterr $CLI dbconnect sync --serverless v4 --check

acceptance/dbconnect/manager-unsupported/test.toml

@@ -0,0 +1,5 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/no-target/output.txt

@@ -0,0 +1,8 @@
+preflight  ok  uv [UV_VERSION]
+resolve    error  No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job
+fetch      pending
+merge      pending
+provision  pending
+validate   pending
+For more detail, re-run with --debug, or --output json to share a structured report.
+Error: No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job

acceptance/dbconnect/no-target/script

@@ -0,0 +1 @@
+musterr $CLI dbconnect sync

acceptance/dbconnect/no-target/test.toml

@@ -0,0 +1,5 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/serverless-check/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] dbconnect sync --serverless v4 --check
+preflight  ok  uv [UV_VERSION]
+resolve    ok  source=serverless envKey=serverless/serverless-v4
+fetch      ok  source=[DATABRICKS_URL]/serverless/serverless-v4/pyproject.toml fromCache=false
+merge      ok
+provision  ok
+validate   ok
+Plan: [TEST_TMP_DIR]/pyproject.toml
+  changed region: requires-python
+  changed region: tool.uv.constraint-dependencies
+  changed region: databricks-connect
+Check complete. No files were modified.

acceptance/dbconnect/serverless-check/script

@@ -0,0 +1 @@
+trace $CLI dbconnect sync --serverless v4 --check

acceptance/dbconnect/serverless-check/test.toml

@@ -0,0 +1,21 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []
+
+[Env]
+DATABRICKS_DBCONNECT_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
+
+[[Server]]
+Pattern = "GET /serverless/serverless-v4/pyproject.toml"
+Response.Body = '''
+[project]
+requires-python = ">=3.12"
+
+[dependency-groups]
+dev = ["databricks-connect~=17.2.0"]
+
+[tool.uv]
+constraint-dependencies = ["pyarrow<19", "pandas<3"]
+'''
+
+[[Repls]]
+Old = 'uv uv \S+(?: \([^)]+\))?'
+New = 'uv [UV_VERSION]'

acceptance/dbconnect/serverless-json/output.txt

@@ -0,0 +1,54 @@
+
+>>> [CLI] dbconnect sync --serverless v4 --check --output json
+{
+  "schemaVersion": 1,
+  "command": "dbconnect sync",
+  "ok": true,
+  "mode": "default",
+  "dryRun": true,
+  "target": {
+    "source": "serverless",
+    "serverlessVersion": "v4",
+    "envKey": "serverless/serverless-v4"
+  },
+  "resolved": {
+    "pythonVersion": "3.12",
+    "dbconnectVersion": "17.2.0",
+    "artifactSource": "network"
+  },
+  "greenfield": true,
+  "plan": {
+    "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml",
+    "wouldInstallPython": "3.12",
+    "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+    \"databricks-connect~=17.2.0\",\n+]\n+\n+# managed by databricks dbconnect — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+    \"pyarrow\u003c19\",\n+    \"pandas\u003c3\",\n+]\n+# end managed by databricks dbconnect\n"
+  },
+  "phases": [
+    {
+      "phase": "preflight",
+      "status": "ok"
+    },
+    {
+      "phase": "resolve",
+      "status": "ok"
+    },
+    {
+      "phase": "fetch",
+      "status": "ok"
+    },
+    {
+      "phase": "merge",
+      "status": "ok"
+    },
+    {
+      "phase": "provision",
+      "status": "ok"
+    },
+    {
+      "phase": "validate",
+      "status": "ok"
+    }
+  ],
+  "warnings": [],
+  "error": null,
+  "durationMs": 0
+}

acceptance/dbconnect/serverless-json/script

@@ -0,0 +1 @@
+trace $CLI dbconnect sync --serverless v4 --check --output json