GitHub Actions: ensure pip is configured for CI/CD

.codegen.json

@@ -3,7 +3,7 @@
     "src/databricks/labs/blueprint/__about__.py": "__version__ = \"$VERSION\""
   },
   "toolchain": {
-    "required": ["make","uv"],
+    "required": ["make", "uv"],
     "pre_setup": ["make dev"],
     "prepend_path": ".venv/bin",
     "acceptance_path": "tests/integration",

.github/actions/jfrog-auth/action.yml

@@ -1,5 +1,12 @@
 name: 'Authenticate for JFrog'
 description: 'Authenticate with JFrog using OIDC based on the GitHub repository.'
+# Some things to note:
+#  - Run this _after_ installing any tools that need to use JFrog; auth is configured for all the (supported) tools that
+#    it detects.
+#  - Where possible we avoid exposing tokens in environment variables, preferring to write them into files instead.
+#    (Tokens in environment variables tend to be more exposed and easier to leak than those written into files.)
+#
+# TODO: Factor out into an external action, once releases are allowed.
 outputs:
   jfrog-access-token:
     description: "Access token for JFrog"
@@ -11,4 +18,165 @@ runs:
       name: Authenticate against JFrog
       shell: bash
       run: |
+        if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL}" ]] || [[ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" ]]
+        then
+            printf '::error::%s\n' 'This action uses OIDC: job must have "id-token: write" permission'
+            exit 1
+        fi
         "${GITHUB_ACTION_PATH}/jfrog-auth" "${ACTIONS_ID_TOKEN_REQUEST_URL}" "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}"
+
+    - id: detect-cmds
+      name: Detecting package/project managers.
+      shell: bash
+      run: |
+        for cmd in bun coursier mvn npm pip3 sbt uv
+        do
+            command -v "${cmd}" > /dev/null && found=true || found=false
+            printf '::debug::%s\n' "Found ${cmd}: ${found}"
+            printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}"
+        done
+
+    - name: Configure bun for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_bun == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > ~/.bunfig.toml << 'EOF'
+        [install]
+        registry = { url = "https://databricks.jfrog.io/artifactory/api/npm/db-npm/", token = "$jfrog_access_token" }
+        EOF
+        cat > "${RUNNER_TEMP}/.bun.env" << EOF
+        # Environment variables loaded by bun.
+        jfrog_access_token='${JFROG_ACCESS_TOKEN}'
+        EOF
+        printf '%s=%s\n' 'BUN_OPTIONS' "--env-file=${RUNNER_TEMP}/.bun.env" >> "${GITHUB_ENV}"
+        printf '::debug::%s\n' 'Configured JFrog access for bun.'
+        # There are currently the following issues with JFrog:
+        #  - The default set of CAs doesn't seem to cover the ones used by our JFrog instance.
+        #  - The JSON metadata returned for some NPM artefacts can be invalid JSON.
+        printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it will probably not work.'
+
+    - name: Configure coursier for JFrog
+      # Note: SBT bootstrapping uses Coursier internally.
+      if: "${{ steps.detect-cmds.outputs.command_coursier == 'true' ||
+               steps.detect-cmds.outputs.command_sbt == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > "${RUNNER_TEMP}/.coursier-credentials.properties" << EOF
+        jfrog.host=databricks.jfrog.io
+        jfrog.realm=Artifactory Realm
+        jfrog.username=gha-service-account
+        jfrog.password=${JFROG_ACCESS_TOKEN}
+        EOF
+        printf '%s=%s\n' 'COURSIER_CREDENTIALS' "${RUNNER_TEMP}/.coursier-credentials.properties" >> "${GITHUB_ENV}"
+        printf '::debug::%s\n' 'Configured JFrog access for Coursier.'
+
+    - name: Configure Maven for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_mvn == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        mkdir -p ~/.m2
+        cat > ~/.m2/settings.xml << EOF
+        <settings>
+          <mirrors>
+            <mirror>
+              <id>jfrog-central</id>
+              <mirrorOf>*</mirrorOf>
+              <url>https://databricks.jfrog.io/artifactory/db-maven/</url>
+            </mirror>
+          </mirrors>
+          <servers>
+            <server>
+              <id>jfrog-central</id>
+              <username>gha-service-account</username>
+              <password>${JFROG_ACCESS_TOKEN}</password>
+            </server>
+          </servers>
+        </settings>
+        EOF
+        printf '::debug::%s\n' 'Configured JFrog access for maven.'
+
+    - name: Configure netrc for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' ||
+               steps.detect-cmds.outputs.command_uv == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > "${RUNNER_TEMP}/.netrc" << EOF
+        machine databricks.jfrog.io
+        login gha-service-account
+        password ${JFROG_ACCESS_TOKEN}
+        EOF
+        printf '%s=%s\n' 'NETRC' "${RUNNER_TEMP}/.netrc" >> "${GITHUB_ENV}"
+
+    - name: Configure npm/yarn (classic) for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_npm == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > ~/.npmrc << EOF
+        registry=https://databricks.jfrog.io/artifactory/api/npm/db-npm/
+        always-auth=true
+        ignore-scripts=true
+        //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN}
+        EOF
+        printf '::debug::%s\n' 'Configured JFrog access for npm/yarn (classic).'
+
+    - name: Configure pip for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}"
+      shell: bash
+      run: |
+        cat > "${RUNNER_TEMP}/.pip.conf" << 'EOF'
+        [global]
+        index-url = https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
+        EOF
+        printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}"
+        printf '::debug::%s\n' 'Configured JFrog access for pip.'
+
+    - name: Configure sbt for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_sbt == 'true' }}"
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        mkdir -p ~/.sbt/1.0
+        cat > ~/.sbt/repositories << 'EOF'
+        [repositories]
+          local
+          databricks-jfrog: https://databricks.jfrog.io/artifactory/db-maven/
+        EOF
+
+        cat > "${RUNNER_TEMP}/.sbt.credentials" << EOF
+        realm=Artifactory Realm
+        host=databricks.jfrog.io
+        user=gha-service-account
+        password=${JFROG_ACCESS_TOKEN}
+        EOF
+
+        cat > ~/.sbt/1.0/global.sbt << 'EOF'
+        credentials += Credentials(file(sys.env("RUNNER_TEMP")) / ".sbt.credentials")
+        EOF
+        printf '::debug::%s\n' 'Configured JFrog access for SBT.'
+
+    - name: Configure uv for JFrog
+      if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}"
+      shell: bash
+      env:
+        UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple'
+      run: |
+        printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}"
+        printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}"
+        printf '::debug::%s\n' 'Configured JFrog access for uv.'

.github/actions/jfrog-auth/jfrog-auth

@@ -22,7 +22,7 @@ printf '::add-mask::%s\n' "${_id_token}"
 # Step 2: Exchange it for the JFrog access token.
 #
 printf '::debug::%s\n' "Exchanging OIDC identifier token for JFrog access token..."
-_access_token=$(curl -sLS \
+_access_token=$(curl -fsSL \
     --json "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${_id_token}\", \"provider_name\": \"github-actions\"}" \
     "https://databricks.jfrog.io/access/api/v1/oidc/token" |
     jq -r .access_token)

.github/workflows/acceptance.yml

@@ -19,9 +19,6 @@ jobs:
     runs-on:
       group: larger-runners
       labels: larger
-    env:
-      UV_FROZEN: 1
-      UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple  # Authentication needed, below.
     steps:
       - name: Checkout Code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -34,15 +31,9 @@ jobs:
           version: "0.11.2"
           checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"  # uv-x86_64-unknown-linux-gnu.tar.gz
 
-      - name: Authenticate against JFrog
-        id: jfrog-auth
+      - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
-      - name: Configure uv authentication for JFrog
-        env:
-          JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
-        run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
-
       - name: Acceptance
         uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a  # acceptance/v0.4.4
         with:

.github/workflows/downstreams.yml

@@ -35,9 +35,6 @@ jobs:
       id-token: write
       # Write test results to the PR.
       pull-requests: write
-    env:
-      UV_FROZEN: 1
-      UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple  # Authentication needed, below.
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -50,15 +47,9 @@ jobs:
           version: "0.11.2"
           checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"  # uv-x86_64-unknown-linux-gnu.tar.gz
 
-      - name: Authenticate against JFrog
-        id: jfrog-auth
+      - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
-      - name: Configure uv authentication for JFrog
-        env:
-          JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
-        run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
-
       - name: Acceptance
         run: printf '::error::%s\n' "Downstream tests disabled pending repository lockdown."
       #  uses: databrickslabs/sandbox/downstreams@3313d06ce86227537b3f37f5974f7eecb2a8e59a  # acceptance/v0.4.4

.github/workflows/nightly.yml

@@ -23,9 +23,6 @@ jobs:
     runs-on:
       group: larger-runners
       labels: larger
-    env:
-      UV_FROZEN: 1
-      UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple  # Authentication needed, below.
     steps:
       - name: Checkout Code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -38,15 +35,9 @@ jobs:
           version: "0.11.2"
           checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"  # uv-x86_64-unknown-linux-gnu.tar.gz
 
-      - name: Authenticate against JFrog
-        id: jfrog-auth
+      - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
-      - name: Configure uv authentication for JFrog
-        env:
-          JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
-        run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
-
       - name: Run nightly tests
         uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a  # acceptance/v0.4.4
         with:

.github/workflows/push.yml

@@ -29,8 +29,6 @@ jobs:
     permissions:
       id-token: write  # JFrog OIDC
     env:
-      UV_FROZEN: 1
-      UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple  # Authentication needed, below.
       UV_PYTHON: "${{ matrix.python }}"
     steps:
       - name: Checkout
@@ -42,15 +40,9 @@ jobs:
           version: "0.11.2"
           checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"  # uv-x86_64-unknown-linux-gnu.tar.gz
 
-      - name: Authenticate against JFrog
-        id: jfrog-auth
+      - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
-      - name: Configure uv authentication for JFrog
-        env:
-          JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
-        run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
-
       - name: Initialize the project
         run: make dev
 
@@ -68,9 +60,6 @@ jobs:
       labels: linux-ubuntu-latest
     permissions:
       id-token: write  # JFrog OIDC
-    env:
-      UV_FROZEN: 1
-      UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple  # Authentication needed, below.
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -81,15 +70,9 @@ jobs:
           version: "0.11.2"
           checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"  # uv-x86_64-unknown-linux-gnu.tar.gz
 
-      - name: Authenticate against JFrog
-        id: jfrog-auth
+      - name: Setup for JFrog
         uses: ./.github/actions/jfrog-auth
 
-      - name: Configure uv authentication for JFrog
-        env:
-          JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
-        run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
-
       - name: Initialize the project
         run: make dev