Update CI/CD JFrog integration (#2385)

## Changes

### What does this PR do?

This PR addresses some issues that were preventing the normal
pull-request CI/CD jobs from working properly:

 - The jobs were missing permissions that are needed to access JFrog.
- The project setup needed some changes to the way the dependencies for
build backend (`hatchling`) are resolved.
- A workaround is also put in place to deal with a packaging error with
`databricks-bb-analyzer`: depending on the install order of packages,
this error prevents the import of `databricks.*` and `databricks.labs.*`
modules from other packages.

Acceptance tests still fail, because we can't set up a Spark service at
the moment and some tests depend on this.

### Relevant implementation details

The build backend for a project does not fall within uv's normal
locking/pinning (`uv.lock`) mechanism, which means uv always attempts to
resolve dependencies. When doing so, it will apply the configured
cooldown period: 7 days. Unfortunately JFrog does not provide timestamp
information for PyPi artefacts, and uv therefore disqualifies everything
during resolution. There are two ways of dealing with this: 1) drop the
cooldown configuration; 2) exempt those dependencies from the cooldown
configuration. Of these, this PR implements the latter so that the rest
of the project dependencies are handled properly when locking the
dependencies. (The mirrors we're using _also_ apply the cooldown period,
but for this sort of thing it's best if both apply it just in case
there's a problem with one of them.)

### Caveats/things to watch out for when reviewing:

The scope of this PR is addressing issues in the `build` check
(`push.yml`): other workflows and jobs still fail but are not in scope.

Author: asnare

.github/workflows/acceptance.yml

@@ -20,7 +20,7 @@ jobs:
   integration:
     environment: tool
     permissions:
-      # Access to the integration testing infrastructure.
+      # Access to JFrog and the integration testing infrastructure.
       id-token: write
       # Write test results to the PR.
       pull-requests: write
@@ -48,7 +48,10 @@ jobs:
           $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh
 
       # TODO: Migrate tests to use Databricks clusters instead of Spark local mode
+      # Disabled for now, because access to archives.apache.org is blocked; a new approach will be needed for this.
+      # (The integration tests will still run, but many will fail due to this.)
       - name: Setup spark
+        if: false
         run: |
           chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh
           $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh

.github/workflows/push.yml

@@ -22,6 +22,9 @@ jobs:
     runs-on:
       group: databrickslabs-protected-runner-group
       labels: linux-ubuntu-latest
+    permissions:
+      # JFrog OIDC authentication.
+      id-token: write
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -35,6 +38,9 @@ jobs:
       - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
+      - name: Pre-sync environment to work around analyzer bug
+        run: make dev
+
       - name: Run unit tests
         run: make test
 
@@ -48,6 +54,9 @@ jobs:
     runs-on:
       group: databrickslabs-protected-runner-group
       labels: linux-ubuntu-latest
+    permissions:
+      # JFrog OIDC authentication.
+      id-token: write
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -95,6 +104,9 @@ jobs:
     runs-on:
       group: databrickslabs-protected-runner-group
       labels: linux-ubuntu-latest
+    permissions:
+      # JFrog OIDC authentication.
+      id-token: write
     env:
       INPUT_DIR_PARENT: .
       OUTPUT_DIR: ./test-reports

Makefile

@@ -14,6 +14,16 @@ clean: docs-clean
 
 dev:
 	uv sync --all-extras
+# Workaround: databricks-bb-analyzer is missing databricks/__init__.py in its wheel.
+# If it's installed last the namespace package breaks. Ensure the file exists.
+	@for f in .venv/lib/python*/site-packages/databricks/__init__.py \
+                  .venv/lib/python*/site-packages/databricks/labs/__init__.py; \
+	do \
+	    grep -q 'extend_path' "$$f" 2>/dev/null || { \
+	        printf '__path__ = __import__("pkgutil").extend_path(__path__, __name__)\n' > "$$f"; \
+	        printf 'Warning: workaround needed (and configured) for analyzer packaging bug: %s\n' "$$f"; \
+	    } \
+	done
 
 lint:
 	$(UV_RUN) black --check .

pyproject.toml

@@ -52,7 +52,9 @@ Issues = "https://github.com/databrickslabs/lakebridge/issues"
 Source = "https://github.com/databrickslabs/lakebridge"
 
 [build-system]
-requires = ["hatchling"]
+requires = [
+    "hatchling~=1.29.0"
+]
 build-backend = "hatchling.build"
 
 [tool.hatch.build]
@@ -123,6 +125,19 @@ databricks-labs-pytester = false
 databricks-sdk = false
 databricks-switch-plugin = false
 
+# These packages are here because the backend (hatchling) depends on them, but backend dependencies are not covered
+# by uv.lock. To load the project these must always be resolved, including under CI/CD where JFrog is used. However
+# JFrog does not provide timestamp metadata, which leads to uv excluding them from resolution. The only way to deal
+# with this currently is to exempt them from the 'exclude-newer' policy: the only alternative would be to not have
+# them at all.
+editables = false
+hatchling = false
+packaging = false
+pathspec = false
+pluggy = false
+tomli = false
+trove-classifiers = false
+
 [tool.pytest.ini_options]
 addopts = "-s -p no:warnings -vv --cache-clear"
 cache_dir = ".venv/pytest-cache"