Skip to content

Commit aeb17d5

Browse files
asnareasnare
authored
Update documentation infrastructure (#2384)
## Changes This PR updates the docusaurus plumbing so that CI/CD works with JFrog. The yarn config/setup has also been updated to ensure that the developer environment matches CI/CD. Incidental changes: - Now building with Node 25 instead of 20. - Sparse checkouts, and only the current version of the repo instead of the complete history. ### Relevant implementation details For some reason I had to pin the version of `webpack` to get the build to succeed: newer versions aren't compatible with the version of `docusaurus` that we are using. ### Caveats/things to watch out for when reviewing: - This PR is only about the documentation workflows; the others are still failing and out of scope for this PR. ### Linked issues Replaces #2381 (bun is incompatible with JFrog) ### Tests - manually tested - CI/CD automation
1 parent 0082e4c commit aeb17d5

8 files changed

Lines changed: 127 additions & 1485 deletions

File tree

‎.github/actions/jfrog-auth/action.yml‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
name: 'Authenticate for JFrog'
22
description: 'Authenticate with JFrog using OIDC based on the GitHub repository.'
3+
# Some things to note:
4+
# - Run this _after_ installing any tools that need to use JFrog; auth is configured all the (supported) tools that it
5+
# detects.
6+
# - Where possible we avoid exposing tokens in environment variables, preferring to write them into files instead.
7+
# (Tokens in environment variables tend to be more exposed and easier to leak than those written into files.)
8+
#
39
# TODO: Factor out into an external action, once releases are allowed.
410
outputs:
511
jfrog-access-token:
@@ -18,13 +24,35 @@ runs:
1824
name: Detecting python package/project managers.
1925
shell: bash
2026
run: |
21-
for cmd in pip3 uv
27+
for cmd in bun npm pip3 uv
2228
do
2329
command -v "${cmd}" > /dev/null && found=true || found=false
2430
printf '::debug::%s\n' "Found ${cmd}: ${found}"
2531
printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}"
2632
done
2733
34+
- name: Configure bun for JFrog
35+
if: "${{ steps.detect-cmds.outputs.command_bun == 'true' }}"
36+
shell: bash
37+
env:
38+
JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
39+
run: |
40+
umask 077
41+
cat > ~/.bunfig.toml << EOF
42+
[install]
43+
registry = { url = "https://databricks.jfrog.io/artifactory/api/npm/db-npm/", token = "$jfrog_access_token" }
44+
EOF
45+
cat > "${RUNNER_TEMP}/.bun.env" << EOF
46+
# Environment variables loaded by bun.
47+
jfrog_access_token='${JFROG_ACCESS_TOKEN}'
48+
EOF
49+
printf '%s=%s\n' 'BUN_OPTIONS' "--env-file=${RUNNER_TEMP}/.bun.env" >> "${GITHUB_ENV}"
50+
printf '::debug::%s\n' 'Configured JFrog access for bun.'
51+
# There are currently the following issues with JFrog:
52+
# - The default set of CAs doesn't seem to cover the ones used by our JFrog instance.
53+
# - The JSON metadata returned for some NPM artefacts can be invalid JSON.
54+
printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it probably won't work.'
55+
2856
- name: Configure pip for JFrog
2957
if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}"
3058
shell: bash
@@ -37,6 +65,7 @@ runs:
3765
index-url = https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
3866
EOF
3967
printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}"
68+
printf '::debug::%s\n' 'Configured JFrog access for pip.'
4069
4170
- name: Configure uv for JFrog
4271
if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}"
@@ -46,5 +75,21 @@ runs:
4675
UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple'
4776
run: |
4877
uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
49-
printf "%s=%s\n" 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}"
50-
printf "%s=%s\n" 'UV_FROZEN' '1' >> "${GITHUB_ENV}"
78+
printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}"
79+
printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}"
80+
printf '::debug::%s\n' 'Configured JFrog access for uv.'
81+
82+
- name: Configure npm/yarn (classic) for JFrog
83+
if: "${{ steps.detect-cmds.outputs.command_npm == 'true' }}"
84+
shell: bash
85+
env:
86+
JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
87+
run: |
88+
umask 077
89+
cat > ~/.npmrc << EOF
90+
registry=https://databricks.jfrog.io/artifactory/api/npm/db-npm/
91+
always-auth=true
92+
ignore-scripts=true
93+
//databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN}
94+
EOF
95+
printf '::debug::%s\n' 'Configured JFrog access for npm/yarn (classic).'

‎.github/workflows/docs-build.yml‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,28 @@ jobs:
1919
runs-on:
2020
group: databrickslabs-protected-runner-group
2121
labels: linux-ubuntu-latest
22-
defaults:
23-
run:
24-
working-directory: docs/lakebridge
22+
permissions:
23+
# JFrog OIDC authentication.
24+
id-token: write
2525
steps:
2626
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
2727
with:
28-
fetch-depth: 0
28+
sparse-checkout: |
29+
.github/actions
30+
docs
31+
sparse-checkout-cone-mode: 'true'
32+
2933
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
3034
with:
31-
node-version: 20
35+
node-version: 25.9.0
3236
cache: yarn
33-
cache-dependency-path: docs/lakebridge/yarn.lock # need to put the lockfile path explicitly
37+
cache-dependency-path: docs/lakebridge/yarn.lock
38+
39+
- name: Setup for JFrog
40+
uses: ./.github/actions/jfrog-auth
3441

3542
- name: Configure documentation environment
36-
run: make dev
43+
run: make docs-dev
3744

3845
- name: Build website
39-
run: yarn build
46+
run: make docs-build

‎.github/workflows/docs-release.yml‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,31 @@ jobs:
1616
group: databrickslabs-protected-runner-group
1717
labels: linux-ubuntu-latest
1818
environment: release
19-
defaults:
20-
run:
21-
working-directory: docs/lakebridge
19+
permissions:
20+
# JFrog OIDC authentication.
21+
id-token: write
2222
steps:
2323
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
2424
with:
25-
fetch-depth: 0
25+
sparse-checkout: |
26+
.github/actions
27+
docs
28+
sparse-checkout-cone-mode: 'true'
29+
2630
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
2731
with:
28-
node-version: 20
32+
node-version: 25.9.0
2933
cache: yarn
30-
cache-dependency-path: docs/lakebridge/yarn.lock # need to put the lockfile path explicitly
34+
cache-dependency-path: docs/lakebridge/yarn.lock
35+
36+
- name: Setup for JFrog
37+
uses: ./.github/actions/jfrog-auth
3138

3239
- name: Configure documentation environment
33-
run: make dev
40+
run: make docs-dev
3441

3542
- name: Build website
36-
run: make build
43+
run: make docs-build
3744

3845
- name: Upload Build Artifact
3946
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0
@@ -43,9 +50,6 @@ jobs:
4350
deploy:
4451
name: Deploy to GitHub Pages
4552
needs: build
46-
defaults:
47-
run:
48-
working-directory: docs/lakebridge
4953

5054
# Grant GITHUB_TOKEN the permissions required to make a Pages deployment
5155
permissions:

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ lock-dependencies:
5252
$(UV_RUN) --group yq tomlq -r '.["build-system"].requires[]' pyproject.toml | \
5353
uv pip compile --generate-hashes --universal --no-header --quiet - > build-constraints-new.txt
5454
mv build-constraints-new.txt .build-constraints.txt
55-
perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple/"|g' uv.lock
55+
@perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple/"|g' uv.lock
56+
@printf 'Stripped registry references from uv.lock.\n'
5657

5758
clean_coverage_dir:
5859
@printf "Deleting: %s\n" "$${OUTPUT_DIR:?must be set}"

‎docs/lakebridge/.yarnrc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
ignore-scripts true
2+
--engine-strict true
3+
--install.check-files true

‎docs/lakebridge/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ serve: build
1818
lock-dependencies:
1919
rm -f yarn.lock
2020
yarn install
21-
perl -pi -e 's|resolved "https://[^/]+/|resolved "https://registry.yarnpkg.com/|' yarn.lock
21+
@perl -ni -e 'print unless /^\s*resolved\s+"https:/' yarn.lock
22+
@printf 'Stripped repository references from yarn.lock.\n'
2223

2324
.DEFAULT: all
2425
.PHONY: all clean dev build serve-dev serve lock-dependencies

‎docs/lakebridge/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@
5959
]
6060
},
6161
"engines": {
62-
"node": ">=18.0"
62+
"node": ">=25.0 <26"
6363
},
64+
"engineStrict": true,
6465
"resolutions": {
6566
"ajv": ">=8.18.0",
6667
"brace-expansion": "^1.1.12",
@@ -74,6 +75,7 @@
7475
"schema-utils": ">=4.0.0",
7576
"serialize-javascript": ">=7.0.3",
7677
"svgo": ">=3.3.3",
78+
"webpack": "~5.97.0",
7779
"webpack-dev-server": "^5.2.1"
7880
}
7981
}

0 commit comments

Comments
 (0)