diff --git a/.codegen.json b/.codegen.json index a1e7c89..5609e81 100644 --- a/.codegen.json +++ b/.codegen.json @@ -3,7 +3,7 @@ "src/databricks/labs/blueprint/__about__.py": "__version__ = \"$VERSION\"" }, "toolchain": { - "required": ["make","uv"], + "required": ["make", "uv"], "pre_setup": ["make dev"], "prepend_path": ".venv/bin", "acceptance_path": "tests/integration", diff --git a/.github/actions/jfrog-auth/action.yml b/.github/actions/jfrog-auth/action.yml index d899165..92b617e 100644 --- a/.github/actions/jfrog-auth/action.yml +++ b/.github/actions/jfrog-auth/action.yml @@ -1,5 +1,6 @@ name: 'Authenticate for JFrog' description: 'Authenticate with JFrog using OIDC based on the GitHub repository.' +# TODO: Factor out into an external action, once releases are allowed. outputs: jfrog-access-token: description: "Access token for JFrog" @@ -12,3 +13,38 @@ runs: shell: bash run: | "${GITHUB_ACTION_PATH}/jfrog-auth" "${ACTIONS_ID_TOKEN_REQUEST_URL}" "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" + + - id: detect-cmds + name: Detecting python package/project managers. + shell: bash + run: | + for cmd in pip3 uv + do + command -v "${cmd}" > /dev/null && found=true || found=false + printf '::debug::%s\n' "Found ${cmd}: ${found}" + printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}" + done + + - name: Configure pip for JFrog + if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}" + shell: bash + env: + JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" + run: | + umask 077 + cat > "$RUNNER_TEMP/.pip.conf" <> "${GITHUB_ENV}" + + - name: Configure uv for JFrog + if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + shell: bash + env: + JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" + UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple' + run: | + uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" + printf "%s=%s\n" 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}" + printf "%s=%s\n" 'UV_FROZEN' '1' >> "${GITHUB_ENV}" diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index fbc654d..90bd4c5 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -9,7 +9,7 @@ permissions: jobs: integration: - if: github.event_name == 'pull_request' && github.event.pull_request.draft == false + if: github.event_name == 'pull_request' environment: runtime permissions: # Access to JFrog and the integration testing infrastructure. @@ -19,9 +19,6 @@ jobs: runs-on: group: larger-runners labels: larger - env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. steps: - name: Checkout Code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -34,15 +31,9 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - name: Authenticate against JFrog - id: jfrog-auth + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - name: Acceptance uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4 with: diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 9bf697f..287b51c 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -35,9 +35,6 @@ jobs: id-token: write # Write test results to the PR. pull-requests: write - env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -50,15 +47,9 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - name: Authenticate against JFrog - id: jfrog-auth + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - name: Acceptance run: printf '::error::%s\n' "Downstream tests disabled pending repository lockdown." # uses: databrickslabs/sandbox/downstreams@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ed2dad1..4c95458 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,9 +23,6 @@ jobs: runs-on: group: larger-runners labels: larger - env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. steps: - name: Checkout Code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -38,15 +35,9 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - name: Authenticate against JFrog - id: jfrog-auth + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - name: Run nightly tests uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4 with: diff --git a/.github/workflows/no-cheat.yml b/.github/workflows/no-cheat.yml index aaf942e..8357a23 100644 --- a/.github/workflows/no-cheat.yml +++ b/.github/workflows/no-cheat.yml @@ -20,10 +20,10 @@ jobs: - name: Verify no lint is disabled in the new code run: | - git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF - git diff $GITHUB_BASE_REF...$(git branch --show-current) >> diff_data.txt + git fetch origin "${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" + git diff "${GITHUB_BASE_REF}...$(git branch --show-current)" >> diff_data.txt python tests/unit/no_cheat.py diff_data.txt >> cheats.txt - COUNT=$(cat cheats.txt | wc -c) + COUNT="$(cat cheats.txt | wc -c)" if [ ${COUNT} -gt 1 ]; then cat cheats.txt exit 1 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bc1abdb..28b7a7e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -29,8 +29,6 @@ jobs: permissions: id-token: write # JFrog OIDC env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. UV_PYTHON: "${{ matrix.python }}" steps: - name: Checkout @@ -42,15 +40,9 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - name: Authenticate against JFrog - id: jfrog-auth + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - name: Initialize the project run: make dev @@ -68,9 +60,6 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write # JFrog OIDC - env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -81,15 +70,9 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - name: Authenticate against JFrog - id: jfrog-auth + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - name: Initialize the project run: make dev diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 76dfe3b..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Release - -on: {} - # Disabled for now, pending further work. - #push: - # tags: - # - 'v*' - -permissions: - contents: read - -jobs: - publish: - runs-on: - group: databrickslabs-protected-runner-group - labels: linux-ubuntu-latest - environment: release - permissions: - # Used to access JFrog, authenticate to PyPI via OIDC and sign the release's artifacts with sigstore-python. - id-token: write - # Used to attach signing artifacts to the published release. - contents: write - env: - UV_FROZEN: 1 - UV_INDEX_URL: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple # Authentication needed, below. - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup uv - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - version: "0.11.2" - checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" # uv-x86_64-unknown-linux-gnu.tar.gz - - - name: Authenticate against JFrog - id: jfrog-auth - uses: ./.github/actions/jfrog-auth - - - name: Configure uv authentication for JFrog - env: - JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - run: uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - - - name: Build wheels - run: make build - - - name: Draft release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 - with: - files: | - dist/databricks_*.whl - dist/databricks_*.tar.gz - - - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 - name: Publish package distributions to PyPI - - - name: Sign artifacts with Sigstore - uses: sigstore/gh-action-sigstore-python@a5caf349bc536fbef3668a10ed7f5cd309a4b53d # v3.2.0 - with: - inputs: | - dist/databricks_*.whl - dist/databricks_*.tar.gz - release-signing-artifacts: true diff --git a/src/databricks/labs/blueprint/wheels.py b/src/databricks/labs/blueprint/wheels.py index 1dff46d..93f49f4 100644 --- a/src/databricks/labs/blueprint/wheels.py +++ b/src/databricks/labs/blueprint/wheels.py @@ -280,11 +280,11 @@ def _build_wheel(self, tmp_dir: str, *, verbose: bool = False, no_deps: bool = T :param no_deps: bool: (Default value = True) :param dirs_exist_ok: bool: (Default value = False) """ - stdout = subprocess.STDOUT - stderr = subprocess.STDOUT - if not verbose: - stdout = subprocess.DEVNULL - stderr = subprocess.DEVNULL + stdout: int | None = subprocess.DEVNULL + stderr = subprocess.DEVNULL + if verbose: + stdout = None # Pass-through. + stderr = subprocess.STDOUT # Redirected to stdout checkout_root = self._product_info.checkout_root() if self._product_info.is_git_checkout() and self._product_info.is_unreleased_version(): # working copy becomes project root for building a wheel diff --git a/tests/unit/test_installation.py b/tests/unit/test_installation.py index 2eb6811..b779a23 100644 --- a/tests/unit/test_installation.py +++ b/tests/unit/test_installation.py @@ -24,6 +24,21 @@ ) +@pytest.fixture +def suspend_config_host_metadata_lookup(monkeypatch: pytest.MonkeyPatch) -> None: + """Suspend host metadata lookups from the SDKs Config class. + + As of version 0.102.0 of the Databricks SDK, the `Config` class during __init__() makes a network call with a + timeout of 5 minutes if the target host does not exist. For unit tests, it never does. This fixture therefore + patches over that lookup to ensure it doesn't get in the way of our tests. + """ + + def _fake_resolve_host_metadata(self) -> None: + return + + monkeypatch.setattr(Config, "_resolve_host_metadata", _fake_resolve_host_metadata) + + def test_current_not_found() -> None: ws = create_autospec(WorkspaceClient) ws.current_user.me().user_name = "foo" @@ -180,7 +195,7 @@ def test_save_typed_file_array_csv() -> None: ) -def test_load_typed_file() -> None: +def test_load_typed_file(suspend_config_host_metadata_lookup: None) -> None: ws = create_autospec(WorkspaceClient) ws.current_user.me().user_name = "foo" ws.workspace.download.return_value = io.StringIO(