Merge remote-tracking branch 'origin/main' into update-templates-with-script

Author: lennartkats-db

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/fmt.yml

@@ -0,0 +1,23 @@
+name: fmt
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  ruff:
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Run ruff
+        uses: astral-sh/ruff-action@eaf0ecdd668ceea36159ff9d91882c9795d89b49 # v3.4.0
+        with:
+          version: "0.9.1"
+          args: "format --check"

.gitignore

@@ -0,0 +1 @@
+.ruff_cache

.ruff.toml

@@ -0,0 +1,8 @@
+[format]
+exclude = [
+    # Templates are responsible for formatting at the source.
+    "dbt_sql/*",
+    "default_python/*",
+    "default_sql/*",
+    "mlops_stacks/*",
+]

contrib/data_engineering/README.md

@@ -12,11 +12,7 @@ The 'data_engineering' project was generated by using the data-engineering templ
     ```
 
 3. We recommend the UV package manager to install project dependencies. It's a drop-in replacement for `pip`.
-   See https://docs.astral.sh/uv/getting-started/installation/ for full installation instructions,
-   or run:
-   ```
-   $ pip install uv
-   ```
+   UV should be installed as a global package; https://docs.astral.sh/uv/getting-started/installation/ instructions.
 
 4. Install all project dependencies:
    ```

contrib/data_engineering/scripts/add_asset.py

@@ -11,7 +11,10 @@
 
 
 def init_bundle(asset_type: AssetType) -> None:
-    cmd = f"databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/data-engineering/assets/{asset_type}"
+    cmd = (
+        f"databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/data-engineering/assets/{asset_type} "
+        + " ".join(sys.argv[2:])
+    )
     subprocess.run(cmd, shell=True)
 
 

contrib/templates/data-engineering/assets/etl-pipeline/databricks_template_schema.json

@@ -40,6 +40,16 @@
                 "no"
             ],
             "default": "yes"
+        },
+        "use_serverless": {
+            "type": "string",
+            "default": "no",
+            "enum": [
+                "yes",
+                "no"
+            ],
+            "description": "\nWould you like to use serverless compute for this pipeline?\nuse_serverless",
+            "order": 5
         }
     },
     "success_message": "\n\n🪠 New pipeline definition generated under 'assets/{{.pipeline_name}}'!"

contrib/templates/data-engineering/assets/etl-pipeline/template/assets/{{.pipeline_name}}/{{.pipeline_name}}.pipeline.yml.tmpl

@@ -2,7 +2,11 @@ resources:
   pipelines:
     {{.pipeline_name}}:
       name: {{.pipeline_name}}
+      {{- if eq .use_serverless "yes"}}
       serverless: true
+      {{- else}}
+      serverless: false
+      {{- end}}
       {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}}
       ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
       # catalog: ${var.catalog}

contrib/templates/data-engineering/template/{{.project_name}}/README.md.tmpl

@@ -12,11 +12,7 @@ The '{{.project_name}}' project was generated by using the data-engineering temp
     ```
 
 3. We recommend the UV package manager to install project dependencies. It's a drop-in replacement for `pip`.
-   See https://docs.astral.sh/uv/getting-started/installation/ for full installation instructions,
-   or run:
-   ```
-   $ pip install uv
-   ```
+   UV should be installed as a global package; https://docs.astral.sh/uv/getting-started/installation/ instructions.
 
 4. Install all project dependencies:
    ```

contrib/templates/data-engineering/template/{{.project_name}}/scripts/add_asset.py

@@ -11,7 +11,10 @@
 
 
 def init_bundle(asset_type: AssetType) -> None:
-    cmd = f"databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/data-engineering/assets/{asset_type}"
+    cmd = (
+        f"databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/data-engineering/assets/{asset_type} "
+        + " ".join(sys.argv[2:])
+    )
     subprocess.run(cmd, shell=True)