|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Execute and convert notebooks while skipping cells that have a given tag. |
| 3 | +
|
| 4 | +This script: |
| 5 | + - finds .ipynb files under --input-dir |
| 6 | + - loads each notebook, removes cells that have the skip tag |
| 7 | + - executes the notebook with nbconvert ExecutePreprocessor |
| 8 | + - exports the executed notebook to a markdown file under --output-dir |
| 9 | +
|
| 10 | +Use this in CI to skip interactive or long-running cells by adding a tag |
| 11 | +to those cells, e.g. tags: ["ci_skip"] |
| 12 | +
|
| 13 | +Example: |
| 14 | + python devtools/scripts/execute_and_convert_notebooks.py \ |
| 15 | + --input-dir notebooks_with_solutions --output-dir notebooks-rendered \ |
| 16 | + --skip-tag ci_skip --timeout 600 |
| 17 | +""" |
| 18 | +import argparse |
| 19 | +import nbformat |
| 20 | +from nbformat import NotebookNode |
| 21 | +import sys |
| 22 | +from pathlib import Path |
| 23 | +from typing import List, Optional |
| 24 | +from nbconvert.preprocessors import ExecutePreprocessor |
| 25 | +from nbconvert.exporters import MarkdownExporter |
| 26 | +from copy import deepcopy |
| 27 | + |
| 28 | + |
| 29 | +def notebook_files(input_dir: str) -> List[Path]: |
| 30 | + p = Path(input_dir) |
| 31 | + return sorted(p.rglob("*.ipynb")) |
| 32 | + |
| 33 | + |
| 34 | +def remove_tagged_cells(nb: NotebookNode, tag: Optional[str]) -> List[NotebookNode]: |
| 35 | + if not tag: |
| 36 | + return list(nb.cells) |
| 37 | + return [ |
| 38 | + cell for cell in nb.cells if tag not in cell.get("metadata", {}).get("tags", []) |
| 39 | + ] |
| 40 | + |
| 41 | + |
| 42 | +def has_skip_tag(cell: NotebookNode, tag: Optional[str]) -> bool: |
| 43 | + if not tag: |
| 44 | + return False |
| 45 | + return tag in cell.get("metadata", {}).get("tags", []) |
| 46 | + |
| 47 | + |
| 48 | +def execute_notebook( |
| 49 | + nb: NotebookNode, |
| 50 | + timeout: int, |
| 51 | + kernel_name: str, |
| 52 | + cwd: Optional[str] = None, |
| 53 | + skip_tag: Optional[str] = None, |
| 54 | +) -> NotebookNode: |
| 55 | + """Execute the notebook but skip execution of cells that have skip_tag. |
| 56 | +
|
| 57 | + Implementation: run a deep copy of the notebook where skipped code cells |
| 58 | + are replaced with a noop (`pass`) so the ExecutePreprocessor executes but |
| 59 | + does nothing for those cells. After execution, copy outputs and |
| 60 | + execution_count back to the original notebook so the original cell |
| 61 | + sources are preserved for conversion. |
| 62 | + """ |
| 63 | + exec_nb = deepcopy(nb) |
| 64 | + # replace skipped code cells with a harmless noop so they won't run |
| 65 | + for cell in exec_nb.cells: |
| 66 | + if cell.get("cell_type") == "code" and has_skip_tag(cell, skip_tag): |
| 67 | + cell.source = "pass\n" |
| 68 | + # clear any existing outputs |
| 69 | + cell.outputs = [] |
| 70 | + cell.execution_count = None |
| 71 | + |
| 72 | + ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name) |
| 73 | + ep.preprocess(exec_nb, {"metadata": {"path": cwd or "."}}) |
| 74 | + |
| 75 | + # copy outputs back to original notebook cells |
| 76 | + for orig_cell, run_cell in zip(nb.cells, exec_nb.cells): |
| 77 | + if orig_cell.get("cell_type") == "code": |
| 78 | + orig_cell["outputs"] = run_cell.get("outputs", []) |
| 79 | + orig_cell["execution_count"] = run_cell.get("execution_count", None) |
| 80 | + |
| 81 | + return nb |
| 82 | + |
| 83 | + |
| 84 | +def convert_to_markdown(nb, out_path: Path): |
| 85 | + exporter = MarkdownExporter() |
| 86 | + body, resources = exporter.from_notebook_node(nb) |
| 87 | + out_path.parent.mkdir(parents=True, exist_ok=True) |
| 88 | + out_path.write_text(body, encoding="utf8") |
| 89 | + |
| 90 | + |
| 91 | +def main(argv=None): |
| 92 | + p = argparse.ArgumentParser() |
| 93 | + p.add_argument("--input-dir", required=True) |
| 94 | + p.add_argument("--output-dir", required=True) |
| 95 | + p.add_argument( |
| 96 | + "--skip-tag", default="ci_skip", help="Cell tag to remove before execution" |
| 97 | + ) |
| 98 | + p.add_argument( |
| 99 | + "--timeout", |
| 100 | + type=int, |
| 101 | + default=600, |
| 102 | + help="ExecutePreprocessor timeout in seconds", |
| 103 | + ) |
| 104 | + p.add_argument( |
| 105 | + "--kernel", default="python3", help="Kernel name to use for execution" |
| 106 | + ) |
| 107 | + args = p.parse_args(argv) |
| 108 | + |
| 109 | + input_dir = Path(args.input_dir) |
| 110 | + output_dir = Path(args.output_dir) |
| 111 | + if not input_dir.exists(): |
| 112 | + print(f"Input directory not found: {input_dir}", file=sys.stderr) |
| 113 | + return 2 |
| 114 | + |
| 115 | + files = notebook_files(input_dir) |
| 116 | + if not files: |
| 117 | + print(f"No notebooks found under {input_dir}") |
| 118 | + return 0 |
| 119 | + |
| 120 | + exit_code = 0 |
| 121 | + for nb_path in files: |
| 122 | + rel = nb_path.relative_to(input_dir) |
| 123 | + out_md = output_dir / rel.with_suffix(".md") |
| 124 | + print(f"Processing {nb_path} -> {out_md}") |
| 125 | + try: |
| 126 | + nb = nbformat.read(str(nb_path), as_version=4) |
| 127 | + # execute in the notebook's parent directory to keep relative paths working |
| 128 | + cwd = str(nb_path.parent) |
| 129 | + nb = execute_notebook( |
| 130 | + nb, |
| 131 | + timeout=args.timeout, |
| 132 | + kernel_name=args.kernel, |
| 133 | + cwd=cwd, |
| 134 | + skip_tag=args.skip_tag, |
| 135 | + ) |
| 136 | + convert_to_markdown(nb, out_md) |
| 137 | + except Exception as e: |
| 138 | + print(f"ERROR processing {nb_path}: {e}", file=sys.stderr) |
| 139 | + exit_code = 1 |
| 140 | + |
| 141 | + return exit_code |
| 142 | + |
| 143 | + |
| 144 | +if __name__ == "__main__": |
| 145 | + raise SystemExit(main()) |
0 commit comments