Skip to content

Overhaul cookbook workflows around Jupyter Book 2 and BinderHub #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
agoose77 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Overhaul cookbook workflows around Jupyter Book 2 and BinderHub #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
278 changes: 44 additions & 234 deletions .github/workflows/build-book.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,267 +2,77 @@ name: build-book

on:
workflow_call:
inputs:
environment_name:
description: 'Name of conda environment to activate (NO LONGER USED)'
required: false
default: 'cookbook-dev'
type: string
environment_file:
description: 'Name of conda environment file'
required: false
default: 'environment.yml'
type: string
path_to_notebooks:
description: 'Location of the JupyterBook source relative to repo root'
required: false
default: '.'
type: string
artifact_name:
description: 'The name to assign to the built book artifact.'
required: false
default: 'book-zip'
type: string
build_command:
description: 'The linux command to build the book or site.'
required: false
default: 'myst build --execute --html'
type: string
output_path:
description: 'Path to the built html content relative to `path_to_notebooks`'
required: false
default: '_build/html'
type: string
build_from_code_artifact:
description: 'Should we try to build from a previously uploaded code artifact?'
required: false
default: 'false'
type: string
code_artifact_name:
description: 'Name of zipped artifact passed in, instead of checking out the repository.'
required: false
default: 'code-zip'
type: string
workflow:
description: 'Identify the workflow that produced the artifact'
required: false
default: trigger-book-build.yaml
type: string
workflow_conclusion:
description: 'Workflow conclusion (options: "failure", "success", "neutral", "cancelled", "skipped", "timed_out", "action_required") or status (options: "completed", "in_progress", "queued") to search for. Use the empty string ("") to ignore status or conclusion in the search.'
required: false
default: success
type: string
base_url:
description: 'Determines where the website is served from, including CSS & JS assets (needed for MyST)'
required: false
default: '/${{ github.event.repository.name }}'
type: string

secrets:
PRIVATE_KEY:
description: 'Google analytics key needed for metrics page on portal site'
required: false
PRIVATE_KEY_ID:
description: 'Google analytics key id needed for metrics page on portal site'
required: false
ARM_USERNAME:
description: 'Username for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)'
required: false
ARM_PASSWORD:
description: 'Password for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)'
required: false
AQS_USERNAME:
description: 'Username for the AQS Data Portal'
required: false
AQS_KEY:
description: 'Key for the AQS Data Portal'
required: false
EARTHDATA_USERNAME:
description: 'NASA Earthdata API Username'
required: false
EARTHDATA_PASSWORD:
description: 'NASA Earthdata API Password'
required: false

env:
# the BASE_URL environment variable needs to be set if building with myst
BASE_URL: ${{ inputs.base_url }}

jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 30
defaults:
run:
shell: bash -leo pipefail {0}
steps:
- name: Checkout the code from the repo
if: inputs.build_from_code_artifact == 'false'
uses: actions/checkout@v6

# The next two steps should replicated checking out the code
- name: Download code artifact
id: get_code_artifact
if: inputs.build_from_code_artifact == 'true'
uses: dawidd6/action-download-artifact@v19
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
workflow: ${{ inputs.workflow }}
run_id: ${{ github.event.workflow_run.id }}
name: ${{ inputs.code_artifact_name }}
workflow_conclusion: ${{ inputs.workflow_conclusion }}

- name: Unzip the code
if: inputs.build_from_code_artifact == 'true'
run: |
unzip pr_code.zip
rm -f pr_code.zip

- name: Get GitHub environment variables
id: get-env
uses: FranzDiebold/github-env-vars-action@v2

- name: Check for config file
id: check_config
uses: andstor/file-existence-action@v3
with:
files: "${{ inputs.path_to_notebooks }}/_config.yml"

- name: Parse config file
id: parse_config
if: steps.check_config.outputs.files_exists == 'true'
uses: CumulusDS/get-yaml-paths-action@v1.0.2
with:
file: ${{ inputs.path_to_notebooks }}/_config.yml
execute_notebooks: execute.execute_notebooks
binderhub_url: sphinx.config.html_theme_options.launch_buttons.binderhub_url
timeout: execute.timeout

- name: Echo values from config file
if: steps.check_config.outputs.files_exists == 'true'
run: |
echo ${{ steps.parse_config.outputs.execute_notebooks }}
echo ${{ steps.parse_config.outputs.binderhub_url }}
echo ${{ steps.parse_config.outputs.timeout }}

- uses: actions/checkout@v6
- name: Test for environment change
id: env_change
uses: tj-actions/changed-files@v47
with:
files: ${{ inputs.environment_file }}
files: environment.yml

- name: Echo environment change test result
run: |
echo '(DEBUG) The value of steps.env_change.outputs.any_changed is:'
echo ${{ steps.env_change.outputs.any_changed }}

- name: Setup environment with micromamba
uses: mamba-org/setup-micromamba@v2
with:
environment-file: ${{ inputs.environment_file }}

- name: Create book build environment
if: |
(inputs.use_cached_environment != 'true'
|| steps.cache.outputs.cache-hit != 'true')
&& steps.parse_config.outputs.execute_notebooks == 'binder'
run: |
conda install -c conda-forge jupyter-book pip
conda install sphinx-pythia-theme
pip install git+https://github.com/pangeo-gallery/binderbot.git
conda list

- name: Get paths to notebook files
if: |
steps.parse_config.outputs.execute_notebooks == 'binder'
# This will find ALL *.ipynb files in the repo
# It would be better to cross-check this against the _toc.yml file
# to avoid unneccesary execution of notebooks that aren't included in the book
shell: python
run: |
import glob
notebooks = glob.glob('**/*.ipynb', recursive=True)
outfile = open("notebook_paths", "w")
for path in notebooks:
outfile.write(path + ' ')
outfile.close() # Writing these out to a file because I can't figure out how to set an environment variable from a python script
environment-file: environment.yml
cache-environment: true

- name: Execute notebooks via binderbot using existing image
if: |
( steps.parse_config.outputs.execute_notebooks == 'binder'
&& steps.env_change.outputs.any_changed != 'true' )
env:
ARM_USERNAME: ${{ secrets.ARM_USERNAME }}
ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }}
AQS_USERNAME: ${{ secrets.AQS_USERNAME }}
AQS_KEY: ${{ secrets.AQS_KEY }}
EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
- name: Check for myst.yml tags
id: myst-tags
run: |
NOTEBOOKS=$(cat notebook_paths)
echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}'
echo "We will now execute these notebooks: $NOTEBOOKS"
echo "using the existing binder image from the main branch"
python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.repository }} --ref main --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD
yq '.project.tags | filter(test("^cookbook:"))[] | sub("cookbook:(.*)","$1=1")' myst.yml | tee "$GITHUB_OUTPUT"

- name: Execute notebooks via binderbot using new image with latest environment
if: |
( steps.parse_config.outputs.execute_notebooks == 'binder'
&& steps.env_change.outputs.any_changed == 'true' )
env:
ARM_USERNAME: ${{ secrets.ARM_USERNAME }}
ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }}
AQS_USERNAME: ${{ secrets.AQS_USERNAME }}
AQS_KEY: ${{ secrets.AQS_KEY }}
EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
run: |
NOTEBOOKS=$(cat notebook_paths)
echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}'
echo "We will now execute these notebooks: $NOTEBOOKS"
echo "using the updated environment file in this branch to build a new image"
python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.actor }}/$CI_REPOSITORY_NAME --ref $CI_ACTION_REF_NAME --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD
- name: Setup execution environment on a BinderHub
if: steps.myst-tags.outputs.build-on-binder
# This action sets up evironment variables for e.g. mystmd to recognise and use
uses: 2i2c-org/clinder@action-v1
with:
hub-url: https://binder.projectpythia.org/
Copy link
Copy Markdown
Member

@brian-rose brian-rose May 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the one place where I think it would be great to avoid hard-coding the solution, in anticipation of more bespoke project-specific binderhubs and/or use of these workflows outside the scope of Project Pythia.

There is a binderhub URL that lives in the myst config as

project:
  jupyter:
    binder:
      url: https://binder.projectpythia.org/

which is actually defined by default in our upstream config file here.

Is there a simple way to parse the myst config to get this URL?


- name: Disable notebook execution during jupyterbook build
if: |
steps.parse_config.outputs.execute_notebooks == 'binder'
shell: python
run: |
import yaml
with open('${{ inputs.path_to_notebooks }}/_config.yml') as f:
data = yaml.safe_load(f)
data['execute']['execute_notebooks'] = 'off'
with open('${{ inputs.path_to_notebooks }}/_config.yml', 'w') as f:
yaml.dump(data, f)
- name: Cache execution
if: "!steps.myst-tags.outputs.build-on-binder"
uses: actions/cache@v4
with:
path: ${{ inputs.path_to_notebooks }}/_build/execute
key: ${{ runner.os }}-${{ hashFiles(env.ENV_FILE) }}-execution

- name: Build the book
# Assumption is that if execute_notebooks != 'binder' then the _config.yml file must be set to execute notebooks during build
env:
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
ARM_USERNAME: ${{ secrets.ARM_USERNAME }}
ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }}
AQS_USERNAME: ${{ secrets.AQS_USERNAME }}
AQS_KEY: ${{ secrets.AQS_KEY }}
EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
SECRETS_VARS: ${{ toJson(secrets) }}
run: |
cd ${{ inputs.path_to_notebooks }}
${{ inputs.build_command }}

- name: Zip the book
run: |
set -x
set -e
if [ -f book.zip ]; then
rm -rf book.zip
# Allow custom build command
if [[ -x "build.sh" ]]; then
./build.sh
else
myst build --execute --site --strict
fi
zip -r book.zip ${{ inputs.path_to_notebooks }}/${{ inputs.output_path }}

- name: Upload zipped book artifact
uses: actions/upload-artifact@v7
- name: Upload zipped AST artifact
id: site-artifact
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./book.zip
name: site
path: _build/site/
retention-days: 7

- name: Stash the artifact URL
run: |
# Use env for safer handling.
echo "URL=${{ steps.site-artifact.outputs.artifact-url }}" >> "$GITHUB_ENV"
- name: Display the preview URL
run: |
import urllib.parse, os;
component = urllib.parse.quote_plus(os.environ['URL'])
preview_url = f"https://hub.projectpythia.org/services/jupyterbook-pub/repo/{component}/"
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(f"Preview this build at {preview_url}")
shell: python {0}
24 changes: 0 additions & 24 deletions .github/workflows/delete-preview.yaml
View file
Open in desktop

This file was deleted.

Loading