diff --git a/.github/workflows/build-book.yaml b/.github/workflows/build-book.yaml index 80f0f5f..c67c779 100644 --- a/.github/workflows/build-book.yaml +++ b/.github/workflows/build-book.yaml @@ -2,267 +2,77 @@ name: build-book on: workflow_call: - inputs: - environment_name: - description: 'Name of conda environment to activate (NO LONGER USED)' - required: false - default: 'cookbook-dev' - type: string - environment_file: - description: 'Name of conda environment file' - required: false - default: 'environment.yml' - type: string - path_to_notebooks: - description: 'Location of the JupyterBook source relative to repo root' - required: false - default: '.' - type: string - artifact_name: - description: 'The name to assign to the built book artifact.' - required: false - default: 'book-zip' - type: string - build_command: - description: 'The linux command to build the book or site.' - required: false - default: 'myst build --execute --html' - type: string - output_path: - description: 'Path to the built html content relative to `path_to_notebooks`' - required: false - default: '_build/html' - type: string - build_from_code_artifact: - description: 'Should we try to build from a previously uploaded code artifact?' - required: false - default: 'false' - type: string - code_artifact_name: - description: 'Name of zipped artifact passed in, instead of checking out the repository.' - required: false - default: 'code-zip' - type: string - workflow: - description: 'Identify the workflow that produced the artifact' - required: false - default: trigger-book-build.yaml - type: string - workflow_conclusion: - description: 'Workflow conclusion (options: "failure", "success", "neutral", "cancelled", "skipped", "timed_out", "action_required") or status (options: "completed", "in_progress", "queued") to search for. Use the empty string ("") to ignore status or conclusion in the search.' - required: false - default: success - type: string - base_url: - description: 'Determines where the website is served from, including CSS & JS assets (needed for MyST)' - required: false - default: '/${{ github.event.repository.name }}' - type: string - - secrets: - PRIVATE_KEY: - description: 'Google analytics key needed for metrics page on portal site' - required: false - PRIVATE_KEY_ID: - description: 'Google analytics key id needed for metrics page on portal site' - required: false - ARM_USERNAME: - description: 'Username for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)' - required: false - ARM_PASSWORD: - description: 'Password for the ARM Data Discovery portal (https://adc.arm.gov/armlive/)' - required: false - AQS_USERNAME: - description: 'Username for the AQS Data Portal' - required: false - AQS_KEY: - description: 'Key for the AQS Data Portal' - required: false - EARTHDATA_USERNAME: - description: 'NASA Earthdata API Username' - required: false - EARTHDATA_PASSWORD: - description: 'NASA Earthdata API Password' - required: false - -env: - # the BASE_URL environment variable needs to be set if building with myst - BASE_URL: ${{ inputs.base_url }} jobs: build: runs-on: ubuntu-latest - timeout-minutes: 30 defaults: run: shell: bash -leo pipefail {0} steps: - - name: Checkout the code from the repo - if: inputs.build_from_code_artifact == 'false' - uses: actions/checkout@v6 - - # The next two steps should replicated checking out the code - - name: Download code artifact - id: get_code_artifact - if: inputs.build_from_code_artifact == 'true' - uses: dawidd6/action-download-artifact@v19 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - workflow: ${{ inputs.workflow }} - run_id: ${{ github.event.workflow_run.id }} - name: ${{ inputs.code_artifact_name }} - workflow_conclusion: ${{ inputs.workflow_conclusion }} - - - name: Unzip the code - if: inputs.build_from_code_artifact == 'true' - run: | - unzip pr_code.zip - rm -f pr_code.zip - - - name: Get GitHub environment variables - id: get-env - uses: FranzDiebold/github-env-vars-action@v2 - - - name: Check for config file - id: check_config - uses: andstor/file-existence-action@v3 - with: - files: "${{ inputs.path_to_notebooks }}/_config.yml" - - - name: Parse config file - id: parse_config - if: steps.check_config.outputs.files_exists == 'true' - uses: CumulusDS/get-yaml-paths-action@v1.0.2 - with: - file: ${{ inputs.path_to_notebooks }}/_config.yml - execute_notebooks: execute.execute_notebooks - binderhub_url: sphinx.config.html_theme_options.launch_buttons.binderhub_url - timeout: execute.timeout - - - name: Echo values from config file - if: steps.check_config.outputs.files_exists == 'true' - run: | - echo ${{ steps.parse_config.outputs.execute_notebooks }} - echo ${{ steps.parse_config.outputs.binderhub_url }} - echo ${{ steps.parse_config.outputs.timeout }} - + - uses: actions/checkout@v6 - name: Test for environment change id: env_change uses: tj-actions/changed-files@v47 with: - files: ${{ inputs.environment_file }} + files: environment.yml - name: Echo environment change test result run: | echo '(DEBUG) The value of steps.env_change.outputs.any_changed is:' echo ${{ steps.env_change.outputs.any_changed }} - + - name: Setup environment with micromamba uses: mamba-org/setup-micromamba@v2 with: - environment-file: ${{ inputs.environment_file }} - - - name: Create book build environment - if: | - (inputs.use_cached_environment != 'true' - || steps.cache.outputs.cache-hit != 'true') - && steps.parse_config.outputs.execute_notebooks == 'binder' - run: | - conda install -c conda-forge jupyter-book pip - conda install sphinx-pythia-theme - pip install git+https://github.com/pangeo-gallery/binderbot.git - conda list - - - name: Get paths to notebook files - if: | - steps.parse_config.outputs.execute_notebooks == 'binder' - # This will find ALL *.ipynb files in the repo - # It would be better to cross-check this against the _toc.yml file - # to avoid unneccesary execution of notebooks that aren't included in the book - shell: python - run: | - import glob - notebooks = glob.glob('**/*.ipynb', recursive=True) - outfile = open("notebook_paths", "w") - for path in notebooks: - outfile.write(path + ' ') - outfile.close() # Writing these out to a file because I can't figure out how to set an environment variable from a python script + environment-file: environment.yml + cache-environment: true - - name: Execute notebooks via binderbot using existing image - if: | - ( steps.parse_config.outputs.execute_notebooks == 'binder' - && steps.env_change.outputs.any_changed != 'true' ) - env: - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} + - name: Check for myst.yml tags + id: myst-tags run: | - NOTEBOOKS=$(cat notebook_paths) - echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}' - echo "We will now execute these notebooks: $NOTEBOOKS" - echo "using the existing binder image from the main branch" - python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.repository }} --ref main --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD + yq '.project.tags | filter(test("^cookbook:"))[] | sub("cookbook:(.*)","$1=1")' myst.yml | tee "$GITHUB_OUTPUT" - - name: Execute notebooks via binderbot using new image with latest environment - if: | - ( steps.parse_config.outputs.execute_notebooks == 'binder' - && steps.env_change.outputs.any_changed == 'true' ) - env: - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} - run: | - NOTEBOOKS=$(cat notebook_paths) - echo 'Retrieved binder_url: ${{ steps.parse_config.outputs.binderhub_url }}' - echo "We will now execute these notebooks: $NOTEBOOKS" - echo "using the updated environment file in this branch to build a new image" - python -m binderbot.cli --binder-url ${{ steps.parse_config.outputs.binderhub_url }} --repo ${{ github.actor }}/$CI_REPOSITORY_NAME --ref $CI_ACTION_REF_NAME --nb-timeout ${{ steps.parse_config.outputs.timeout }} $NOTEBOOKS --pass-env-var ARM_USERNAME --pass-env-var ARM_PASSWORD + - name: Setup execution environment on a BinderHub + if: steps.myst-tags.outputs.build-on-binder + # This action sets up evironment variables for e.g. mystmd to recognise and use + uses: 2i2c-org/clinder@action-v1 + with: + hub-url: https://binder.projectpythia.org/ - - name: Disable notebook execution during jupyterbook build - if: | - steps.parse_config.outputs.execute_notebooks == 'binder' - shell: python - run: | - import yaml - with open('${{ inputs.path_to_notebooks }}/_config.yml') as f: - data = yaml.safe_load(f) - data['execute']['execute_notebooks'] = 'off' - with open('${{ inputs.path_to_notebooks }}/_config.yml', 'w') as f: - yaml.dump(data, f) + - name: Cache execution + if: "!steps.myst-tags.outputs.build-on-binder" + uses: actions/cache@v4 + with: + path: ${{ inputs.path_to_notebooks }}/_build/execute + key: ${{ runner.os }}-${{ hashFiles(env.ENV_FILE) }}-execution - name: Build the book - # Assumption is that if execute_notebooks != 'binder' then the _config.yml file must be set to execute notebooks during build - env: - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} - ARM_USERNAME: ${{ secrets.ARM_USERNAME }} - ARM_PASSWORD: ${{ secrets.ARM_PASSWORD }} - AQS_USERNAME: ${{ secrets.AQS_USERNAME }} - AQS_KEY: ${{ secrets.AQS_KEY }} - EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }} - EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }} - SECRETS_VARS: ${{ toJson(secrets) }} - run: | - cd ${{ inputs.path_to_notebooks }} - ${{ inputs.build_command }} - - - name: Zip the book run: | - set -x - set -e - if [ -f book.zip ]; then - rm -rf book.zip + # Allow custom build command + if [[ -x "build.sh" ]]; then + ./build.sh + else + myst build --execute --site --strict fi - zip -r book.zip ${{ inputs.path_to_notebooks }}/${{ inputs.output_path }} - - name: Upload zipped book artifact - uses: actions/upload-artifact@v7 + - name: Upload zipped AST artifact + id: site-artifact + uses: actions/upload-artifact@v6 with: - name: ${{ inputs.artifact_name }} - path: ./book.zip + name: site + path: _build/site/ + retention-days: 7 + + - name: Stash the artifact URL + run: | + # Use env for safer handling. + echo "URL=${{ steps.site-artifact.outputs.artifact-url }}" >> "$GITHUB_ENV" + - name: Display the preview URL + run: | + import urllib.parse, os; + component = urllib.parse.quote_plus(os.environ['URL']) + preview_url = f"https://hub.projectpythia.org/services/jupyterbook-pub/repo/{component}/" + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(f"Preview this build at {preview_url}") + shell: python {0} diff --git a/.github/workflows/delete-preview.yaml b/.github/workflows/delete-preview.yaml deleted file mode 100644 index 41b99cc..0000000 --- a/.github/workflows/delete-preview.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: delete-preview - -on: - workflow_call: - -jobs: - delete: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - name: Checkout gh-pages branch - uses: actions/checkout@v6 - with: - ref: gh-pages - - name: Delete preview files - run: | - rm -rf _preview/${{ github.event.pull_request.number }} - - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v7 - with: - branch: gh-pages - commit_message: Delete preview for pull request \#${{ github.event.pull_request.number }} \ No newline at end of file diff --git a/.github/workflows/deploy-book.yaml b/.github/workflows/deploy-book.yaml index 09da17a..1a5b69c 100644 --- a/.github/workflows/deploy-book.yaml +++ b/.github/workflows/deploy-book.yaml @@ -2,70 +2,49 @@ name: deploy-book on: workflow_call: - inputs: - artifact_name: - description: 'Name of the artifact (zipped book) created by previous build step' - required: false - default: book-zip - type: string - destination_dir: - description: 'Path to publish to on GitHub Pages, relative to site root. We use this to deploy previews in a subdirectory.' - required: false - default: '' - type: string - is_preview: - description: 'Are we deploying a preview?' - required: false - default: 'false' - type: string - publish_dir: - description: 'Publish dir for the action' - required: false - default: '_build/html' - type: string - + jobs: - deploy: + build-html: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6 - - - name: Download merged artifact - if: inputs.is_preview != 'true' - uses: actions/download-artifact@v8 + shell: bash -leo pipefail {0} + steps: + - uses: actions/setup-python@v6 with: - name: ${{ inputs.artifact_name }} + python-version: "3.13" - # For the preview, the official download-artifact action doesn't work - # because the artifact is created by a different workflow - - name: Download preview artifact - if: inputs.is_preview == 'true' - uses: dawidd6/action-download-artifact@v19 + - name: Install builder + run: pip install "jupyter-book-site-renderer>=0.0.3" + + - name: Download site build + uses: actions/download-artifact@v8 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - workflow: build-book.yaml - run_id: ${{ github.event.workflow_run.id }} - name: ${{ inputs.artifact_name }} + name: site + path: ./site - - name: Unzip the book - run: | - rm -rf _build/html - unzip book.zip - rm -f book.zip + - name: Build HTML + run: jupyter-book-site-renderer --site ./site --html ./html --base-url="/${{ github.event.repository.name }}" - - name: Deploy to GitHub Pages - uses: JamesIves/github-pages-deploy-action@v4 - if: | - (github.ref == 'refs/heads/main') + - name: Upload static files as artifact + id: deployment + uses: actions/upload-pages-artifact@v3 with: - token: ${{ secrets.GITHUB_TOKEN }} - git-config-name: 'github-actions[bot]' - git-config-email: 'github-actions[bot]@users.noreply.github.com' - branch: gh-pages - folder: ${{ inputs.publish_dir }} - clean: true - clean-exclude: _preview/* # keep existing previews from other PRs - target-folder: ${{ inputs.destination_dir }} \ No newline at end of file + path: html/ + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build-html + + # Grant GITHUB_TOKEN the permissions required to make a Pages deployment + permissions: + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/get-code.yaml b/.github/workflows/get-code.yaml deleted file mode 100644 index 235f5af..0000000 --- a/.github/workflows/get-code.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: get-code - -on: - workflow_call: - inputs: - code_artifact_name: - description: "Name of the code zip artifact, usually tagged with a PR number" - required: false - default: 'code-zip' - type: string - -# instead of calling build-book directly, we will create and upload an artifact containing the source -# For security reasons, we strip out the .github directory -# So a PR cannot introduce malicious workflow code - -jobs: - get-code: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6 - - - name: Remove .github directory - id: remove_directory - run: | - rm -rf .github/ - - - name: Zip the code - run: | - set -x - set -e - if [ -f pr_code.zip ]; then - rm -rf pr_code.zip - fi - zip -r pr_code.zip . - - - name: Upload zipped code artifact - uses: actions/upload-artifact@v7 - with: - name: code-zip-${{ github.event.number }} - path: ./pr_code.zip \ No newline at end of file diff --git a/.github/workflows/link-checker.yaml b/.github/workflows/link-checker.yaml deleted file mode 100644 index fd72763..0000000 --- a/.github/workflows/link-checker.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: link-checker - -on: - workflow_call: - inputs: - environment_name: # Ignored! Just here for backwards compatibility - description: 'Name of conda environment to activate' - required: false - default: 'cookbook-dev' - type: string - environment_file: - description: 'Name of conda environment file' - required: false - default: 'environment.yml' - type: string - path_to_notebooks: - description: 'Location of the JupyterBook source relative to repo root' - required: false - default: './' - type: string - use_cached_environment: # Ignored! Just here for backwards compatibility - description: 'Flag for whether we should attempt to retrieve a previously cached environment.' - required: false - default: 'true' - type: string - build_command: - description: 'The linux command to run the link checker for the book or site' - required: false - default: 'myst build --check-links' - type: string - - -concurrency: - group: ${{ github.workflow }}=${{ github.head_ref}} - cancel-in-progress: true - -jobs: - link-checker: - runs-on: ubuntu-latest - timeout-minutes: 10 - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6 - - - name: Setup environment with micromamba - uses: mamba-org/setup-micromamba@v2 - with: - environment-file: ${{ inputs.environment_file }} - - - name: Check for config file - id: check_config - uses: andstor/file-existence-action@v3 - with: - files: "${{ inputs.path_to_notebooks }}/_config.yml" - - - name: Disable notebook execution - if: steps.check_config.outputs.files_exists == 'true' - shell: python - run: | - import yaml - with open('${{ inputs.path_to_notebooks }}/_config.yml') as f: - data = yaml.safe_load(f) - data['execute']['execute_notebooks'] = 'off' - with open('${{ inputs.path_to_notebooks }}/_config.yml', 'w') as f: - yaml.dump(data, f) - - - name: Check external links - run: | - cd ${{ inputs.path_to_notebooks }} - ${{ inputs.build_command }} diff --git a/.github/workflows/preview-comment.yaml b/.github/workflows/preview-comment.yaml index ea85dd7..47d7cb1 100644 --- a/.github/workflows/preview-comment.yaml +++ b/.github/workflows/preview-comment.yaml @@ -66,20 +66,35 @@ jobs: 🔍 Git commit SHA: ${{ inputs.sha }} ✅ Deployment Preview URL: In Progress + - name: Stash the artifact URL + env: + GH_TOKEN: ${{ github.token }} + + run: | + # Get the URL of the form https://api.github.com/repos///actions/artifacts/ + raw_url=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + /repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts \ + -q '.artifacts | map(select(.name == "site"))[0].url' ) + artifact_id="${raw_url##*/}" + # Rewrite into the form https://github.com///actions/runs//artifacts/ + proper_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/artifacts/${artifact_id}" + echo "SITE_URL=${proper_url}" >> "$GITHUB_ENV" + - name: Set preview URL if: | github.event.workflow_run.conclusion == 'success' && steps.fc.outputs.comment-id != '' id: preview-url run: | - shopt -s nocasematch - if [[ ${{ steps.repo-name.outputs.value }} = ${{ github.repository_owner }}.github.io ]] - then - echo "url=https://${{ steps.repo-name.outputs.value }}/_preview/${{ inputs.pull_request_number }}" >> $GITHUB_OUTPUT - else - echo "url=https://${{ github.repository_owner }}.github.io/${{ steps.repo-name.outputs.value }}/_preview/${{ inputs.pull_request_number }}" >> $GITHUB_OUTPUT - fi - shopt -u nocasematch + # Escape component + import urllib.parse, os; + component = urllib.parse.quote_plus(os.environ['SITE_URL']) + preview_url = f"https://hub.projectpythia.org/services/jupyterbook-pub/repo/{component}/" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"url={preview_url}") + shell: python {0} - name: Finalize preview comment if: | @@ -93,3 +108,4 @@ jobs: ${{ env.comment_message }} 🔍 Git commit SHA: ${{ inputs.sha }} ✅ Deployment Preview URL: ${{ steps.preview-url.outputs.url }} + diff --git a/.github/workflows/validate-config.yaml b/.github/workflows/validate-config.yaml new file mode 100644 index 0000000..365d19f --- /dev/null +++ b/.github/workflows/validate-config.yaml @@ -0,0 +1,28 @@ +name: validate-config + +on: + workflow_call: + +concurrency: + group: ${{ github.workflow }}=${{ github.head_ref}} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + with: + repository: ProjectPythia/cookbook-actions + ref: feat/clinder + sparse-checkout: | + myst.schema.json + sparse-checkout-cone-mode: false + - name: Run myst.yml validator + run: | + npx --yes ajv-cli -- \ + validate \ + -s myst.schema.json \ + -d myst.yml \ + --spec=draft2020 diff --git a/myst.schema.json b/myst.schema.json new file mode 100644 index 0000000..772cfef --- /dev/null +++ b/myst.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/projectpythia/pythia-config/myst.schema.json", + "$ref": "#/definitions/myst.yml", + "definitions": { + "myst.yml": { + "type": "object", + "required": ["project"], + "properties": { + "project": { + "$ref": "#/definitions/project" + } + } + }, + "project": { + "type": "object", + "required": ["title", "thumbnail", "tags"], + "properties": { + "title": { + "type": "string" + }, + "thumbnail": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "enum": ["cookbook:build-on-binder"] + }, + { + "not": { + "type": "string", + "pattern": "^cookbook:.*" + } + } + ] + } + } + } + } + } +}