feat: Add runnable examples (#761)

This PR introduces a backward-compatible markdown marker that can be used to mark code examples as runnable.

`doc-builder` won't execute them but provides an helper for upstream Python projects to do so, via a `pytest` plugin and a base class. And few helpers to make sure the example stays clean when rendered as doc.

Author: tarekziade

.github/workflows/build_pr_documentation.yml

@@ -57,6 +57,10 @@ on:
         type: string
         default: "main"
         description: "Debug purposes only. Revision of `doc-builder` repo to use. Useful to test changes on `doc-builder`."
+      fail_on_warning:
+        type: boolean
+        default: false
+        description: "Fail doc build when runnable warnings are emitted. Requires `additional_args` to include `--emit-warning`."
 
 
 jobs:
@@ -203,20 +207,35 @@ jobs:
           source .venv/bin/activate
           echo "doc_folder has been set to ${{ env.doc_folder }}"
           cd doc-builder
+          mkdir -p ../build_dir
+          log_file="../build_dir/doc-build.log"
+          : > "$log_file"
+          set -o pipefail
           args="--build_dir ../build_dir --clean --version pr_${{ inputs.pr_number }} --html ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}"
 
           if [ -z "${{ inputs.languages }}" ];
           then
             echo "languages not provided, defaulting to English"
-            doc-builder build ${{ env.package_name }} ../${{ env.doc_folder }} $args
+            doc-builder build ${{ env.package_name }} ../${{ env.doc_folder }} $args 2>&1 | tee -a "$log_file"
           else
             IFS=', ' read -r -a langs <<< "${{ inputs.languages }}"
             for lang in "${langs[@]}"
             do
                 echo "Generating docs for language $lang"
-                doc-builder build ${{ env.package_name }} ../${{ env.doc_folder }}/$lang $args --language $lang
+                doc-builder build ${{ env.package_name }} ../${{ env.doc_folder }}/$lang $args --language $lang 2>&1 | tee -a "$log_file"
             done
           fi
+
+          warnings_emitted=false
+          if [[ "${{ inputs.additional_args }}" == *"--emit-warning"* ]] && grep -q "Bare assert found in runnable:" "$log_file"; then
+            warnings_emitted=true
+          fi
+          echo "$warnings_emitted" > ../build_dir/warnings_emitted
+
+          if [[ "$warnings_emitted" == "true" && "${{ inputs.fail_on_warning }}" == "true" ]]; then
+            echo "::error::Doc build failed because warnings were emitted (fail_on_warning=true)."
+            exit 1
+          fi
           cd ..
 
       - name: Save commit_sha & pr_number

.github/workflows/upload_pr_documentation.yml

@@ -81,6 +81,23 @@ jobs:
             exit 1
           fi
 
+      - name: Get warning status
+        id: warning-context
+        run: |
+          if [ -f ./build_dir/warnings_emitted ]; then
+            content_warnings_emitted=$(cat ./build_dir/warnings_emitted)
+            rm -rf ./build_dir/warnings_emitted
+          else
+            content_warnings_emitted=false
+          fi
+
+          if [[ $content_warnings_emitted =~ ^(true|false)$ ]]; then
+            echo "warnings_emitted=$content_warnings_emitted" >> $GITHUB_OUTPUT
+          else
+            echo "Encountered an invalid warnings_emitted value"
+            exit 1
+          fi
+
       - name: Set hub_docs_url
         id: hfhub-context
         run: |
@@ -92,6 +109,20 @@ jobs:
             echo "hub_docs_url=${{ inputs.hub_base_path }}/${{ inputs.package_name }}/pr_${{ steps.github-context.outputs.pr_number }}" >> $GITHUB_OUTPUT
           fi
 
+      - name: Compose doc comment message
+        id: doc-comment
+        run: |
+          warning_message=""
+          if [ "${{ steps.warning-context.outputs.warnings_emitted }}" = "true" ]; then
+            warning_message=" Some warnings were emitted during doc build; check the workflow logs for details."
+          fi
+
+          {
+            echo "message<<EOF"
+            echo "The docs for this PR live [here](${{ steps.hfhub-context.outputs.hub_docs_url }}). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.${warning_message}"
+            echo "EOF"
+          } >> $GITHUB_OUTPUT
+
       - name: Push to repositories
         shell: bash
         run: |
@@ -130,16 +161,15 @@ jobs:
         uses: thollander/actions-comment-pull-request@v2
         if: steps.find_comment.outputs.comment-id == ''
         with:
-          message: 'The docs for this PR live [here](${{ steps.hfhub-context.outputs.hub_docs_url }}). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.'
+          message: ${{ steps.doc-comment.outputs.message }}
           pr_number: ${{ steps.github-context.outputs.pr_number }}
           GITHUB_TOKEN: ${{ steps.auth.outputs.method == 'comment_bot_token' && secrets.comment_bot_token || steps.comment_bot_token.outputs.token }}
 
       - name: Update doc comment if necessary
-        if: github.event.action == 'reopened' && steps.find_comment.outputs.comment-id != ''
+        if: steps.find_comment.outputs.comment-id != '' && (github.event.action == 'reopened' || steps.find_comment.outputs.comment-body != steps.doc-comment.outputs.message)
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ steps.find_comment.outputs.comment-id }}
           token: ${{ steps.auth.outputs.method == 'comment_bot_token' && secrets.comment_bot_token || steps.comment_bot_token.outputs.token }}
           edit-mode: replace
-          body: |
-            The docs for this PR live [here](${{ steps.hfhub-context.outputs.hub_docs_url }}). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.
+          body: ${{ steps.doc-comment.outputs.message }}

README.md

@@ -119,6 +119,59 @@ doc-builder build hub ~/git/hub-docs/docs/source --build_dir ~/tmp/test-build --
 - add `[[open-in-colab]]` in the tutorial for which you want to build a notebook
 - add `--notebook_dir {path_to_notebook_folder}` to the build command.
 
+### Runnable code blocks
+
+`doc-builder` recognizes runnable Python fences tagged with `runnable` or `runnable:<label>`:
+
+````md
+```py runnable:quickstart
+from transformers import pipeline
+pipe = pipeline("sentiment-analysis")
+print(pipe("I love this!"))
+```
+````
+
+You can use either `py` or `python` fences:
+
+````md
+```python runnable:my_example
+print("hello")
+```
+````
+
+During conversion:
+- the runnable annotation is removed from the fence in rendered docs
+- the code content is preserved unless lines are hidden with `# doc-builder: hide`
+- `# doc-builder: hide` can hide a single line or a full indented block from rendered docs
+
+The label is used in warning messages (for example, `runnable:quickstart`) to identify the block.
+
+`doc-builder` tags and transforms runnable blocks, but it does not execute them by itself.
+
+Use the reusable helper from `hf-doc-builder`:
+
+```python
+from pathlib import Path
+
+from doc_builder.testing import DocIntegrationTest
+
+
+class MyPageDocIntegrationTest(DocIntegrationTest):
+    doc_path = Path(__file__).resolve().parents[2] / "docs" / "source" / "en" / "my_page.md"
+```
+
+`DocIntegrationTest` finds runnable `py`/`python` fences in the target markdown file, creates one test per block, and executes each block with contextual failure output.
+
+Run locally with:
+
+```bash
+pytest -q tests/docs/test_my_page_docs.py
+```
+
+This executes trusted documentation code with `exec`, so keep it limited to repo-controlled docs and CI.
+
+For continuation blocks, `# pytest-decorator`, bare-assert warnings, and GitHub Actions wiring, see [docs/runnable-code-blocks.md](docs/runnable-code-blocks.md).
+
 ## Writing in notebooks
 
 You can write your docs in jupyter notebooks & use doc-builder to: turn jupyter notebooks into mdx files.

docs/runnable-code-blocks.md

@@ -0,0 +1,207 @@
+# Runnable code blocks
+
+This document describes runnable Python markdown fences handled by `doc-builder`.
+
+## Authoring
+
+`doc-builder` recognizes fenced `py` and `python` blocks tagged with `runnable` or `runnable:<label>`:
+
+During conversion:
+
+- the runnable annotation is removed from the fence in rendered docs
+- the code content is preserved unless lines are hidden with `# doc-builder: hide`
+- `# doc-builder: hide` can hide a single line or a full indented block from rendered docs
+- `# doc-builder: ignore-bare-assert` keeps a bare `assert` in the block while suppressing the bare-assert warning for that line
+- the `# doc-builder: ignore-bare-assert` directive is removed from rendered docs
+
+The label is used in warning messages and generated test names.
+
+Example:
+
+````md
+```py runnable:quickstart
+import torch
+from transformers import AutoProcessor, GlmAsrForConditionalGeneration
+
+checkpoint_name = "zai-org/GLM-ASR-Nano-2512"
+audio_url = "https://huggingface.co/datasets/eustlb/audio-samples/resolve/main/bcn_weather.mp3"
+
+processor = AutoProcessor.from_pretrained(checkpoint_name)
+model = GlmAsrForConditionalGeneration.from_pretrained(checkpoint_name, device_map="auto", dtype="auto")
+
+conversation = [
+    [
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio", "url": audio_url},
+                {"type": "text", "text": "Please transcribe this audio into text"},
+            ],
+        }
+    ]
+]
+
+inputs = processor.apply_chat_template(
+    conversation, tokenize=True, add_generation_prompt=True, return_dict=True
+).to(model.device, dtype=model.dtype)
+
+inputs_transcription = processor.apply_transcription_request([audio_url]).to(model.device, dtype=model.dtype)
+
+for key in inputs:  # doc-builder: hide
+    assert torch.equal(inputs[key], inputs_transcription[key])
+
+outputs = model.generate(**inputs, do_sample=False, max_new_tokens=128)
+decoded_outputs = processor.batch_decode(outputs[:, inputs.input_ids.shape[1] :], skip_special_tokens=True)
+
+print(decoded_outputs)
+assert decoded_outputs == [
+    "Yesterday it was thirty five degrees in Barcelona, but today the temperature will go down to minus twenty degrees."
+]  # doc-builder: ignore-bare-assert
+```
+````
+
+## Continuation blocks
+
+Use `:2`, `:3`, and so on to continue a runnable block in the same namespace:
+
+````md
+```py runnable:test_basic
+processor = AutoProcessor.from_pretrained("suno/bark")
+inputs = processor("Hello, my dog is cute", voice_preset=voice_preset)
+```
+
+```py runnable:test_basic:2
+inputs = processor("Amazing! I can speak English too.")
+```
+````
+
+`runnable:test_basic:2` and later continuations are grouped with `runnable:test_basic`, so later snippets can build on earlier setup.
+
+## Test decorators
+
+Runnable code blocks can declare test decorators with `# pytest-decorator:` comments. The decorator is imported and applied to the generated execution function at runtime.
+
+````md
+```py runnable:test_basic
+# pytest-decorator: transformers.testing_utils.slow
+# pytest-decorator: transformers.testing_utils.require_torch
+from transformers import pipeline
+pipe = pipeline("sentiment-analysis")
+print(pipe("I love this!"))
+```
+````
+
+Multiple decorators on the same line are also supported:
+
+````md
+```py runnable:test_basic
+# pytest-decorator: transformers.testing_utils.slow, transformers.testing_utils.require_torch
+from transformers import pipeline
+pipe = pipeline("sentiment-analysis")
+print(pipe("I love this!"))
+```
+````
+
+How it works:
+
+- each `# pytest-decorator: <dotted.import.path>` line is parsed during collection
+- the decorator is imported and applied to the code block execution function
+- skip-style decorators such as `@slow` or `@require_torch` raise `unittest.SkipTest`, which pytest reports as a skip
+- `# pytest-decorator:` lines are stripped from executed code and rendered documentation
+
+## Running runnable blocks in tests
+
+When `hf-doc-builder` is installed, pytest auto-loads the `doc-builder` plugin. This makes running runnable blocks against `.md` files a supported workflow, and it is the recommended way to execute them in most projects.
+
+You can point pytest directly at a markdown page:
+
+```bash
+pytest -q docs/source/en/my_page.md
+```
+
+Or at a directory of markdown docs:
+
+```bash
+pytest -q docs/source/en/
+```
+
+What the pytest plugin does:
+
+- collects runnable blocks from the `.md` files passed to pytest
+- finds fenced `py`/`python` blocks marked with `runnable` or `runnable:<label>`
+- groups continuation blocks such as `runnable:test_basic:2` with the earlier block
+- creates one pytest item per runnable block
+- applies `# pytest-decorator:` directives before execution
+- reports failures with the markdown file path and runnable code snippet
+
+`DocIntegrationTest` is a lower-level helper. It is not required for markdown-based doc tests. Use it only if you want to manage doc execution from regular Python test files in your project, or if you need tighter control over the Python-side test wrapper.
+
+Example:
+
+```python
+from pathlib import Path
+
+from doc_builder.testing import DocIntegrationTest
+
+
+class MyPageDocIntegrationTest(DocIntegrationTest):
+    doc_path = Path(__file__).resolve().parents[2] / "docs" / "source" / "en" / "my_page.md"
+```
+
+`DocIntegrationTest` reads one markdown file, creates one Python test per runnable block (`runnable:my_case` becomes `test_my_case`), and executes those blocks from a standard pytest test module.
+
+Run locally with:
+
+```bash
+pytest -q tests/docs/test_my_page_docs.py
+```
+
+Both approaches execute trusted documentation code with `exec`, so keep them limited to repo-controlled docs and CI.
+
+Both approaches run raw markdown code blocks. If you want test behavior to match rendered docs exactly, preprocess the blocks first so `# doc-builder: hide` lines are removed before execution.
+
+## Bare-assert warnings
+
+`doc-builder build` can emit warnings for bare `assert` statements inside runnable Python markdown fences.
+
+This behavior is opt-in:
+
+```bash
+doc-builder build {package_name} {path_to_docs} --build_dir {build_dir} --emit-warning
+```
+
+When enabled:
+
+- bare `assert` lines in runnable blocks emit warnings with file and line information
+- `# doc-builder: hide` removes the line from rendered docs and does not warn
+- `# doc-builder: ignore-bare-assert` keeps the line, silences the warning, and is removed from rendered docs
+
+Warning formatting depends on where the build runs:
+
+- local runs: `Warning: docs/source/en/example.md:3: ...`
+- GitHub Actions: `::warning file=docs/source/en/example.md,line=3::...`
+
+## GitHub pipeline integration
+
+If you use the reusable PR documentation workflow, pass `--emit-warning` through `additional_args`:
+
+```yaml
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
+    with:
+      # ...
+      additional_args: --emit-warning
+```
+
+To fail the PR documentation build on warnings, also set `fail_on_warning: true`:
+
+```yaml
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
+    with:
+      # ...
+      additional_args: --emit-warning
+      fail_on_warning: true
+```