diff --git a/.gitignore b/.gitignore index dd03479a..76838a30 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ tests/test_jupyter/*.txt .venv docs/jupyter_execute .DS_Store +docs/source/reference_guides/commands.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e906f2ac..3e2fa90b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: mdformat-pyproject>=0.0.2, ] files: (docs/.) - exclude: (docs/source/reference_guides/hookspecs\.md|docs/source/api/.*\.md|docs/source/how_to_guides/capture_warnings\.md|docs/source/how_to_guides/hashing_inputs_of_tasks\.md|docs/source/how_to_guides/provisional_nodes_and_task_generators\.md|docs/source/how_to_guides/remote_files\.md|docs/source/how_to_guides/writing_custom_nodes\.md|docs/source/tutorials/defining_dependencies_products\.md|docs/source/tutorials/using_a_data_catalog\.md) + exclude: (docs/source/reference_guides/hookspecs\.md|docs/source/reference_guides/api/.*\.md|docs/source/how_to_guides/capture_warnings\.md|docs/source/how_to_guides/hashing_inputs_of_tasks\.md|docs/source/how_to_guides/provisional_nodes_and_task_generators\.md|docs/source/how_to_guides/remote_files\.md|docs/source/how_to_guides/writing_custom_nodes\.md|docs/source/tutorials/defining_dependencies_products\.md|docs/source/tutorials/using_a_data_catalog\.md) - repo: https://github.com/kynan/nbstripout rev: 0.9.1 hooks: diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css index 7564f1ae..519593a8 100644 --- a/docs/source/_static/css/custom.css +++ b/docs/source/_static/css/custom.css @@ -127,3 +127,34 @@ body[data-md-color-scheme="slate"] .home-tile p { box-shadow: 0 12px 26px rgba(2, 6, 23, 0.46); } } + +.md-typeset dl.cli-reference { + margin: 0.75rem 0 1.25rem; +} + +.md-typeset dl.cli-reference dt { + margin: 0.85rem 0 0; + font-weight: 500; +} + +.md-typeset dl.cli-reference dd { + margin: 0.2rem 0 0.75rem 1.5rem; +} + +.md-typeset dl.cli-reference dd p { + margin: 0; +} + +.md-typeset dl.cli-reference dd p + p { + margin-top: 0.25rem; + color: var(--md-default-fg-color--light); +} + +.md-typeset dl.cli-reference dt a, +.md-typeset dl.cli-reference dt a:hover, +.md-typeset dl.cli-reference dt a:focus, +.md-typeset dl.cli-reference dt a:visited { + color: inherit; + text-decoration: none; + border-bottom: 0; +} diff --git a/docs/source/commands/build.md b/docs/source/commands/build.md deleted file mode 100644 index 5d5b29f2..00000000 --- a/docs/source/commands/build.md +++ /dev/null @@ -1,39 +0,0 @@ -# build - -Collect tasks, execute them, and report the results. - -## Usage - -```bash -pytask build [OPTIONS] [PATHS] -``` - -`pytask` without a subcommand runs `build` by default. - -## Examples - -```bash -# Run the project in the current directory. -pytask build - -# Select tasks by expression. -pytask build -k random_data - -# Show what would run without executing tasks. -pytask build --dry-run -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/build-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/build-options.md" - -## Related - -- [Invoking pytask](../tutorials/invoking_pytask.md) -- [Selecting tasks](../tutorials/selecting_tasks.md) -- [Debugging](../tutorials/debugging.md) -- [Configuration](../reference_guides/configuration.md) diff --git a/docs/source/commands/clean.md b/docs/source/commands/clean.md deleted file mode 100644 index 3e7b1038..00000000 --- a/docs/source/commands/clean.md +++ /dev/null @@ -1,35 +0,0 @@ -# clean - -Clean provided paths by removing files unknown to pytask. - -## Usage - -```bash -pytask clean [OPTIONS] [PATHS] -``` - -## Examples - -```bash -# Dry-run cleanup. -pytask clean - -# Remove unknown files immediately. -pytask clean --mode force - -# Exclude a path pattern. -pytask clean --exclude obsolete_folder -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/clean-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/clean-options.md" - -## Related - -- [Cleaning projects](../tutorials/cleaning_projects.md) -- [Configuration](../reference_guides/configuration.md) diff --git a/docs/source/commands/collect.md b/docs/source/commands/collect.md deleted file mode 100644 index 4793273d..00000000 --- a/docs/source/commands/collect.md +++ /dev/null @@ -1,35 +0,0 @@ -# collect - -Collect tasks and report information about them. - -## Usage - -```bash -pytask collect [OPTIONS] [PATHS] -``` - -## Examples - -```bash -# Show collected tasks. -pytask collect - -# Also show dependencies and products. -pytask collect --nodes - -# Select collected tasks by marker expression. -pytask collect -m "not slow" -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/collect-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/collect-options.md" - -## Related - -- [Collecting tasks](../tutorials/collecting_tasks.md) -- [Selecting tasks](../tutorials/selecting_tasks.md) diff --git a/docs/source/commands/dag.md b/docs/source/commands/dag.md deleted file mode 100644 index fb962fc6..00000000 --- a/docs/source/commands/dag.md +++ /dev/null @@ -1,34 +0,0 @@ -# dag - -Create a visualization of the directed acyclic graph. - -## Usage - -```bash -pytask dag [OPTIONS] [PATHS] -``` - -## Examples - -```bash -# Create a DAG as PDF. -pytask dag - -# Create a PNG instead. -pytask dag -o dag.png - -# Change the graph layout and direction. -pytask dag --layout dot --rank-direction LR -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/dag-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/dag-options.md" - -## Related - -- [Visualizing the DAG](../tutorials/visualizing_the_dag.md) diff --git a/docs/source/commands/index.md b/docs/source/commands/index.md deleted file mode 100644 index 2e8bde81..00000000 --- a/docs/source/commands/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# Commands - -pytask exposes a command-based CLI. This section documents each command with usage, -examples, arguments, and all available options. - -## Available commands - ---8<-- "docs/source/_static/md/commands/command-list.md" - -## Global options - -Global options are available on the top-level `pytask` command. - ---8<-- "docs/source/_static/md/commands/root-options.md" - -## Help output - ---8<-- "docs/source/_static/md/help-page.md" - -## Default command behavior - -Running `pytask` without a subcommand is equivalent to running -[`pytask build`](build.md). diff --git a/docs/source/commands/markers.md b/docs/source/commands/markers.md deleted file mode 100644 index dced2555..00000000 --- a/docs/source/commands/markers.md +++ /dev/null @@ -1,32 +0,0 @@ -# markers - -Show all registered markers. - -## Usage - -```bash -pytask markers [OPTIONS] [PATHS] -``` - -## Examples - -```bash -# List all registered markers. -pytask markers - -# Load custom marker definitions from a hooks module. -pytask markers --hook-module hooks.py -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/markers-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/markers-options.md" - -## Related - -- [Markers](../tutorials/markers.md) -- [Extending pytask](../how_to_guides/extending_pytask.md) diff --git a/docs/source/commands/profile.md b/docs/source/commands/profile.md deleted file mode 100644 index e9a90312..00000000 --- a/docs/source/commands/profile.md +++ /dev/null @@ -1,34 +0,0 @@ -# profile - -Show information about resource consumption. - -## Usage - -```bash -pytask profile [OPTIONS] [PATHS] -``` - -## Examples - -```bash -# Show profiling information from previous successful runs. -pytask profile - -# Export profiling information as JSON. -pytask profile --export json - -# Export profiling information as CSV. -pytask profile --export csv -``` - -## Arguments - ---8<-- "docs/source/_static/md/commands/profile-arguments.md" - -## Options - ---8<-- "docs/source/_static/md/commands/profile-options.md" - -## Related - -- [Profiling tasks](../tutorials/profiling_tasks.md) diff --git a/docs/source/how_to_guides/bp_complex_task_repetitions.md b/docs/source/how_to_guides/bp_complex_task_repetitions.md index 33ceeffd..7b35a80e 100644 --- a/docs/source/how_to_guides/bp_complex_task_repetitions.md +++ b/docs/source/how_to_guides/bp_complex_task_repetitions.md @@ -99,8 +99,8 @@ When you are using the `.name` attributes of the dimensions and multi-dimensiona objects like in the example above, you ensure that the names of dimensions are included in all downstream tasks. -Thus, you can simply call [`pytask -k`](../commands/build.md#options) with the following -expression to execute all tasks related to the logit model. +Thus, you can simply call [`pytask -k`](../reference_guides/commands.md#pytask-build--k) +with the following expression to execute all tasks related to the logit model. ```console pytask -k logit diff --git a/docs/source/how_to_guides/capture_warnings.md b/docs/source/how_to_guides/capture_warnings.md index 4b01d91e..ef455494 100644 --- a/docs/source/how_to_guides/capture_warnings.md +++ b/docs/source/how_to_guides/capture_warnings.md @@ -71,14 +71,14 @@ configured by the `filterwarnings` configuration option. ## Disabling warnings summary Although not recommended, you can use the -[`--disable-warnings`](../commands/build.md#options) command-line option to suppress the +[`--disable-warnings`](../reference_guides/commands.md#pytask-build--disable-warnings) command-line option to suppress the warning summary entirely from the test run output. ## Debugging warnings Sometimes it is not clear which line of code triggered a warning. To find the location, you can turn warnings into exceptions and then use the -[`pytask build --pdb`](../commands/build.md#options) flag to enter the debugger. +[`pytask build --pdb`](../reference_guides/commands.md#pytask-build--pdb) flag to enter the debugger. You can use the configuration to convert warnings to errors by setting diff --git a/docs/source/how_to_guides/extending_pytask.md b/docs/source/how_to_guides/extending_pytask.md index a57cf375..8466e343 100644 --- a/docs/source/how_to_guides/extending_pytask.md +++ b/docs/source/how_to_guides/extending_pytask.md @@ -19,8 +19,9 @@ More general information about pluggy can be found in its There are two ways to add new [hook implementations](../glossary.md#hook-implementation). -1. Using the [`pytask build --hook-module`](../commands/build.md#options) commandline - option or the `hook_module` configuration value. +1. Using the + [`pytask build --hook-module`](../reference_guides/commands.md#pytask-build--hook-module) + commandline option or the `hook_module` configuration value. 1. Packaging your [plugin](../glossary.md#plugin) as a Python package to publish and share it. @@ -55,8 +56,8 @@ hook_module = ["myproject.hooks"] ``` In `hooks.py` we can add another commandline option to -[`pytask build`](../commands/build.md) by providing an additional -[hook implementation](../glossary.md#hook-implementation) for the +[`pytask build`](../reference_guides/commands.md#pytask-build) by providing an +additional [hook implementation](../glossary.md#hook-implementation) for the [hook specification](../glossary.md#hook-specification) `_pytask.hookspecs.pytask_extend_command_line_interface`. diff --git a/docs/source/how_to_guides/how_to_influence_build_order.md b/docs/source/how_to_guides/how_to_influence_build_order.md index 4c0a9806..100b9160 100644 --- a/docs/source/how_to_guides/how_to_influence_build_order.md +++ b/docs/source/how_to_guides/how_to_influence_build_order.md @@ -35,8 +35,8 @@ def task_second(): print("I'm first.") ``` -The execution yields (use [`-s`](../commands/build.md#options) to make the output -visible in the terminal) +The execution yields (use [`-s`](../reference_guides/commands.md#pytask-build--s) to +make the output visible in the terminal) --8<-- "docs/source/_static/md/try-first.md" diff --git a/docs/source/how_to_guides/portability.md b/docs/source/how_to_guides/portability.md index 57e2a846..6ffb23dd 100644 --- a/docs/source/how_to_guides/portability.md +++ b/docs/source/how_to_guides/portability.md @@ -14,8 +14,8 @@ Use this checklist when you move a project to another machine or environment. 1. **Update state once on the source machine.** -Run a normal build with [`pytask build`](../commands/build.md) so `pytask.lock` is up to -date: +Run a normal build with [`pytask build`](../reference_guides/commands.md#pytask-build) +so `pytask.lock` is up to date: ```` ```console @@ -85,7 +85,7 @@ tasks run. If tasks are removed or renamed, their old entries remain as stale da are ignored. To clean up stale entries without deleting the file, run -[`pytask build --clean-lockfile`](../commands/build.md#options): +[`pytask build --clean-lockfile`](../reference_guides/commands.md#pytask-build--clean-lockfile): ```console $ pytask build --clean-lockfile diff --git a/docs/source/index.md b/docs/source/index.md index 0ac50129..e7ebe7ff 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -70,5 +70,3 @@ If you want to know more about pytask, dive into one of the following topics.

API and implementation details for developers and plugin authors.

- -For command-line usage, see the [CLI reference](commands/index.md). diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md deleted file mode 100644 index 5d073086..00000000 --- a/docs/source/reference_guides/api.md +++ /dev/null @@ -1,12 +0,0 @@ -# API Moved - -The API reference now lives in the top-level **API** section: - -- [API overview](../api/index.md) -- [CLI Imports](../api/cli_and_programmatic.md) -- [Functional Interfaces](../api/functional_interfaces.md) -- [Core Classes and Exceptions](../api/core_classes_and_exceptions.md) -- [Marks](../api/marks.md) -- [Nodes and Tasks](../api/nodes_and_tasks.md) -- [Outcomes, Reports, and Warnings](../api/outcomes_reports_and_warnings.md) -- [Utilities and Typing](../api/utilities_and_typing.md) diff --git a/docs/source/api/cli_and_programmatic.md b/docs/source/reference_guides/api/cli_and_programmatic.md similarity index 86% rename from docs/source/api/cli_and_programmatic.md rename to docs/source/reference_guides/api/cli_and_programmatic.md index b78f9b79..77e18a68 100644 --- a/docs/source/api/cli_and_programmatic.md +++ b/docs/source/reference_guides/api/cli_and_programmatic.md @@ -2,7 +2,8 @@ This page documents the public CLI-related imports from `pytask`. -For command usage and options, see the [CLI reference](../commands/index.md). +For command usage and options, see the +[command reference](../commands.md). ## Command Line Entry Point diff --git a/docs/source/api/core_classes_and_exceptions.md b/docs/source/reference_guides/api/core_classes_and_exceptions.md similarity index 100% rename from docs/source/api/core_classes_and_exceptions.md rename to docs/source/reference_guides/api/core_classes_and_exceptions.md diff --git a/docs/source/api/functional_interfaces.md b/docs/source/reference_guides/api/functional_interfaces.md similarity index 100% rename from docs/source/api/functional_interfaces.md rename to docs/source/reference_guides/api/functional_interfaces.md diff --git a/docs/source/api/index.md b/docs/source/reference_guides/api/index.md similarity index 100% rename from docs/source/api/index.md rename to docs/source/reference_guides/api/index.md diff --git a/docs/source/api/marks.md b/docs/source/reference_guides/api/marks.md similarity index 100% rename from docs/source/api/marks.md rename to docs/source/reference_guides/api/marks.md diff --git a/docs/source/api/nodes_and_tasks.md b/docs/source/reference_guides/api/nodes_and_tasks.md similarity index 100% rename from docs/source/api/nodes_and_tasks.md rename to docs/source/reference_guides/api/nodes_and_tasks.md diff --git a/docs/source/api/outcomes_reports_and_warnings.md b/docs/source/reference_guides/api/outcomes_reports_and_warnings.md similarity index 100% rename from docs/source/api/outcomes_reports_and_warnings.md rename to docs/source/reference_guides/api/outcomes_reports_and_warnings.md diff --git a/docs/source/api/utilities_and_typing.md b/docs/source/reference_guides/api/utilities_and_typing.md similarity index 100% rename from docs/source/api/utilities_and_typing.md rename to docs/source/reference_guides/api/utilities_and_typing.md diff --git a/docs/source/reference_guides/configuration.md b/docs/source/reference_guides/configuration.md index 5cdc6f0f..df2552e0 100644 --- a/docs/source/reference_guides/configuration.md +++ b/docs/source/reference_guides/configuration.md @@ -44,7 +44,7 @@ check_casing_of_paths = false SQLite is the legacy state format. pytask uses `pytask.lock` as the primary state backend for change detection. When no lockfile exists, pytask reads the configured database and writes `pytask.lock`. For downgrade compatibility, -[`pytask build`](../commands/build.md) also keeps the legacy database state updated. +[`pytask build`](commands.md#pytask-build) also keeps the legacy database state updated. The `database_url` option remains for backward compatibility and controls the legacy database location and dialect @@ -114,7 +114,7 @@ pytask can ignore files and directories and exclude some tasks or reduce the dur the collection. To ignore some file/folder via the command line, use the -[`--ignore`](../commands/build.md#options) flag multiple times. +[`--ignore`](commands.md#pytask-build--ignore) flag multiple times. ```console $ pytask --ignore some_file.py --ignore some_directory/* @@ -133,7 +133,7 @@ ignore = ["some_directory/*", "some_file.py"] ### `markers` pytask uses markers to attach additional information to task functions. To see which -markers are available, use the [`pytask markers`](../commands/markers.md) command. +markers are available, use the [`pytask markers`](commands.md#pytask-markers) command. ```console $ pytask markers @@ -201,7 +201,7 @@ pdbcls = "pdbp:Pdb" ``` The custom debugger will be used when you invoke the -[`--pdb`](../commands/build.md#options) flag for post-mortem debugging or when using +[`--pdb`](commands.md#pytask-build--pdb) flag for post-mortem debugging or when using `breakpoint()` in your task code. ### `show_errors_immediately` diff --git a/docs/source/developers_guide.md b/docs/source/reference_guides/developers_guide.md similarity index 100% rename from docs/source/developers_guide.md rename to docs/source/reference_guides/developers_guide.md diff --git a/docs/source/glossary.md b/docs/source/reference_guides/glossary.md similarity index 100% rename from docs/source/glossary.md rename to docs/source/reference_guides/glossary.md diff --git a/docs/source/reference_guides/index.md b/docs/source/reference_guides/index.md index 5f52e080..1bf1c578 100644 --- a/docs/source/reference_guides/index.md +++ b/docs/source/reference_guides/index.md @@ -3,10 +3,11 @@ Reference guides are articles which can be read alongside the code and provide additional details on the inner workings of pytask. +- [API](api/index.md) +- [Commands](commands.md) - [Configuration](configuration.md) -- [Lockfile](lockfile.md) - [Hookspecs](hookspecs.md) -- [Type Hints](../type_hints.md) -- [Developer's Guide](../developers_guide.md) -- [Glossary](../glossary.md) -- [API](../api/index.md) +- [Lockfile](lockfile.md) +- [Type Hints](type_hints.md) +- [Developer's Guide](developers_guide.md) +- [Glossary](glossary.md) diff --git a/docs/source/reference_guides/lockfile.md b/docs/source/reference_guides/lockfile.md index 15b29c39..ea4ac1bc 100644 --- a/docs/source/reference_guides/lockfile.md +++ b/docs/source/reference_guides/lockfile.md @@ -38,8 +38,8 @@ On each run, pytask: 1. Skips tasks whose states match; runs the rest. 1. Updates `pytask.lock` after each completed task (atomic write). 1. Updates `pytask.lock` after skipping unchanged tasks (unless - [`--dry-run`](../commands/build.md#options) or - [`--explain`](../commands/build.md#options) are active). + [`--dry-run`](commands.md#pytask-build--dry-run) or + [`--explain`](commands.md#pytask-build--explain) are active). ## Portability @@ -51,8 +51,8 @@ There are two portability concerns: ## Maintenance -Use [`pytask build --clean-lockfile`](../commands/build.md#options) to rewrite -`pytask.lock` with only currently collected tasks. The rewrite happens after a +Use [`pytask build --clean-lockfile`](commands.md#pytask-build--clean-lockfile) to +rewrite `pytask.lock` with only currently collected tasks. The rewrite happens after a successful build and recomputes current state values without executing tasks again. ## File Format Reference diff --git a/docs/source/type_hints.md b/docs/source/reference_guides/type_hints.md similarity index 100% rename from docs/source/type_hints.md rename to docs/source/reference_guides/type_hints.md diff --git a/docs/source/tutorials/capturing_output.md b/docs/source/tutorials/capturing_output.md index 575c79d0..ea11799a 100644 --- a/docs/source/tutorials/capturing_output.md +++ b/docs/source/tutorials/capturing_output.md @@ -44,7 +44,8 @@ There are three ways in which `pytask` can perform capturing: `sys.stderr`. You can influence output-capturing mechanisms from the command line with -[`-s`](../commands/build.md#options) and [`--capture`](../commands/build.md#options): +[`-s`](../reference_guides/commands.md#pytask-build--s) and +[`--capture`](../reference_guides/commands.md#pytask-build--capture): ```console $ pytask -s # disable all capturing diff --git a/docs/source/tutorials/cleaning_projects.md b/docs/source/tutorials/cleaning_projects.md index 7e7520f4..1d314ed6 100644 --- a/docs/source/tutorials/cleaning_projects.md +++ b/docs/source/tutorials/cleaning_projects.md @@ -2,20 +2,22 @@ Projects usually become cluttered with obsolete files after some time. -To clean the project, type [`pytask clean`](../commands/clean.md). +To clean the project, type +[`pytask clean`](../reference_guides/commands.md#pytask-clean). --8<-- "docs/source/_static/md/clean-dry-run.md" pytask performs a dry-run by default and lists all removable files. If you want to remove the files, use -[`pytask clean --mode`](../commands/clean.md#options) with one of the following modes. +[`pytask clean --mode`](../reference_guides/commands.md#pytask-clean--mode) with one of +the following modes. - `force` removes all files suggested in the `dry-run` without any confirmation. - `interactive` allows you to decide for every file whether to keep it or not. If you want to delete complete folders instead of single files, use -[`pytask clean --directories`](../commands/clean.md#options). +[`pytask clean --directories`](../reference_guides/commands.md#pytask-clean--directories). --8<-- "docs/source/_static/md/clean-dry-run-directories.md" @@ -23,8 +25,9 @@ If you want to delete complete folders instead of single files, use pytask excludes files that are under version control with git. -Use the [`pytask clean --exclude`](../commands/clean.md#options) option or the `exclude` -key in the configuration file to exclude files and directories. +Use the +[`pytask clean --exclude`](../reference_guides/commands.md#pytask-clean--exclude) option +or the `exclude` key in the configuration file to exclude files and directories. Values can be Unix filename patterns that, for example, support the wildcard character `*` for any characters. You find the documentation in `fnmatch`. @@ -44,4 +47,4 @@ exclude = ["obsolete_folder"] ## Further reading -- [command reference for `pytask clean`](../commands/clean.md). +- [command reference for `pytask clean`](../reference_guides/commands.md#pytask-clean). diff --git a/docs/source/tutorials/collecting_tasks.md b/docs/source/tutorials/collecting_tasks.md index 6144fc79..ca3acb39 100644 --- a/docs/source/tutorials/collecting_tasks.md +++ b/docs/source/tutorials/collecting_tasks.md @@ -1,7 +1,7 @@ # Collecting tasks If you want to inspect your project and see a summary of all the tasks, you can use the -[`pytask collect`](../commands/collect.md) command. +[`pytask collect`](../reference_guides/commands.md#pytask-collect) command. Let us take the following task. @@ -9,13 +9,14 @@ Let us take the following task. --8<-- "docs_src/tutorials/collecting_tasks.py" ``` -Now, running [`pytask collect`](../commands/collect.md) will produce the following -output. +Now, running [`pytask collect`](../reference_guides/commands.md#pytask-collect) will +produce the following output. --8<-- "docs/source/_static/md/collect.md" If you want to have more information regarding the dependencies and products of the -task, append the [`pytask collect --nodes`](../commands/collect.md#options) flag. +task, append the +[`pytask collect --nodes`](../reference_guides/commands.md#pytask-collect--nodes) flag. --8<-- "docs/source/_static/md/collect-nodes.md" @@ -24,6 +25,7 @@ patterns as usual. ## Further reading -- Read the [command reference for `pytask collect`](../commands/collect.md). +- Read the + [command reference for `pytask collect`](../reference_guides/commands.md#pytask-collect). - Read the tutorial on [selecting tasks](selecting_tasks.md). - Paths can be ignored with [`ignore`](../reference_guides/configuration.md#ignore). diff --git a/docs/source/tutorials/configuration.md b/docs/source/tutorials/configuration.md index 99de7719..8078809d 100644 --- a/docs/source/tutorials/configuration.md +++ b/docs/source/tutorials/configuration.md @@ -37,7 +37,7 @@ The first option is to let pytask try to find the configuration itself. configuration file with the correct section. Secondly, it is possible to pass the location of the configuration file via -[`pytask build -c`](../commands/build.md#options) like +[`pytask build -c`](../reference_guides/commands.md#pytask-build--config) like ```console $ pytask -c config/pyproject.toml diff --git a/docs/source/tutorials/debugging.md b/docs/source/tutorials/debugging.md index 64e4756d..e33317d5 100644 --- a/docs/source/tutorials/debugging.md +++ b/docs/source/tutorials/debugging.md @@ -5,9 +5,9 @@ help you to gain more information on the root cause. ## Post-mortem debugger -Using [`pytask build --pdb`](../commands/build.md#options) enables the post-mortem -debugger. Whenever an exception is raised inside a task, the prompt will enter the -debugger enabling you to find out the cause of the exception. +Using [`pytask build --pdb`](../reference_guides/commands.md#pytask-build--pdb) enables +the post-mortem debugger. Whenever an exception is raised inside a task, the prompt will +enter the debugger enabling you to find out the cause of the exception. --8<-- "docs/source/_static/md/pdb.md" @@ -19,28 +19,30 @@ debugger enabling you to find out the cause of the exception. !!! note A following tutorial explains [how to select a subset of tasks](selecting_tasks.md). - Combine it with the [`pytask build --pdb`](../commands/build.md#options) flag to debug + Combine it with the + [`pytask build --pdb`](../reference_guides/commands.md#pytask-build--pdb) flag to debug specific tasks. ## Tracing If you want to enter the debugger at the start of every task, use -[`pytask build --trace`](../commands/build.md#options). +[`pytask build --trace`](../reference_guides/commands.md#pytask-build--trace). --8<-- "docs/source/_static/md/trace.md" ## Tracebacks You can enrich the display of tracebacks by showing local variables in each stack frame. -Just execute [`pytask build --show-locals`](../commands/build.md#options). +Just execute +[`pytask build --show-locals`](../reference_guides/commands.md#pytask-build--show-locals). --8<-- "docs/source/_static/md/show-locals.md" ## Custom debugger If you want to use your custom debugger, make sure it is importable and use -[`pytask build --pdbcls`](../commands/build.md#options). Here, we change the standard -`pdb` debugger to IPython's implementation. +[`pytask build --pdbcls`](../reference_guides/commands.md#pytask-build--pdbcls). Here, +we change the standard `pdb` debugger to IPython's implementation. ```console $ pytask --pdbcls=IPython.terminal.debugger:TerminalPdb diff --git a/docs/source/tutorials/invoking_pytask.md b/docs/source/tutorials/invoking_pytask.md index c0a06c40..1834773c 100644 --- a/docs/source/tutorials/invoking_pytask.md +++ b/docs/source/tutorials/invoking_pytask.md @@ -35,9 +35,9 @@ $ pytask -h/--help ## The build command -The [`build`](../commands/build.md) command accepts many -[`options`](../commands/build.md#options) and paths as positional arguments. If no paths -are passed via the command line interface, pytask will look for +The [`build`](../reference_guides/commands.md#pytask-build) command accepts many +[`options`](../reference_guides/commands.md#pytask-build) and paths as positional +arguments. If no paths are passed via the command line interface, pytask will look for [`paths`](../reference_guides/configuration.md#paths) in the configuration file. At last, pytask will collect tasks from the current working directory and subsequent folders. @@ -66,7 +66,8 @@ $ pytask --show-errors-immediately ``` It can be helpful when you have a long-running workflow but want feedback as soon as it -is available. See [`--show-errors-immediately`](../commands/build.md#options). +is available. See +[`--show-errors-immediately`](../reference_guides/commands.md#pytask-build--show-errors-immediately). ### Stopping after the first (N) failures @@ -77,25 +78,28 @@ $ pytask -x | --stop-after-first-failure # Stop after the first failure $ pytask --max-failures 2 # Stop after the second failure ``` -See [`-x, --stop-after-first-failure`](../commands/build.md#options) and -[`--max-failures`](../commands/build.md#options). +See +[`-x, --stop-after-first-failure`](../reference_guides/commands.md#pytask-build--stop-after-first-failure) +and [`--max-failures`](../reference_guides/commands.md#pytask-build--max-failures). ### Performing a dry-run -Do a dry run with [`--dry-run`](../commands/build.md#options) to see which tasks will be -executed without executing them. +Do a dry run with [`--dry-run`](../reference_guides/commands.md#pytask-build--dry-run) +to see which tasks will be executed without executing them. --8<-- "docs/source/_static/md/dry-run.md" ### Explaining why tasks are executed -Use the [`--explain`](../commands/build.md#options) flag to understand why tasks need to -be executed. This shows what changed (source files, dependencies, products, previous -tasks) and helps you understand pytask's execution decisions. +Use the [`--explain`](../reference_guides/commands.md#pytask-build--explain) flag to +understand why tasks need to be executed. This shows what changed (source files, +dependencies, products, previous tasks) and helps you understand pytask's execution +decisions. --8<-- "docs/source/_static/md/explain.md" -The explanation output respects the [`--verbose`](../commands/build.md#options) flag: +The explanation output respects the +[`--verbose`](../reference_guides/commands.md#pytask-build--verbose) flag: - Default verbosity: Shows tasks that would be executed and skipped tasks - `-v` or `--verbose 1`: Same as default, with summary for persisted and unchanged tasks diff --git a/docs/source/tutorials/markers.md b/docs/source/tutorials/markers.md index f34cea64..99c3d599 100644 --- a/docs/source/tutorials/markers.md +++ b/docs/source/tutorials/markers.md @@ -1,7 +1,8 @@ # Markers pytask uses markers to attach additional information to a task. You can see all -available markers by using the [`pytask markers`](../commands/markers.md) command. +available markers by using the +[`pytask markers`](../reference_guides/commands.md#pytask-markers) command. --8<-- "docs/source/_static/md/markers.md" diff --git a/docs/source/tutorials/selecting_tasks.md b/docs/source/tutorials/selecting_tasks.md index 5f61ca07..add3db2b 100644 --- a/docs/source/tutorials/selecting_tasks.md +++ b/docs/source/tutorials/selecting_tasks.md @@ -22,24 +22,26 @@ $ pytask -m wip ``` You can pass more complex expressions to -[`pytask build -m`](../commands/build.md#options) by using multiple markers and `and`, -`or`, `not`, and `()`. The following pattern selects all tasks that belong to the data -management but not those that produce plots and plots for the analysis. +[`pytask build -m`](../reference_guides/commands.md#pytask-build--m) by using multiple +markers and `and`, `or`, `not`, and `()`. The following pattern selects all tasks that +belong to the data management but not those that produce plots and plots for the +analysis. ```console $ pytask -m "(data_management and not plots) or (analysis and plots)" ``` -If you create your markers, use the [`pytask markers`](../commands/markers.md) command -to register and document them. +If you create your markers, use the +[`pytask markers`](../reference_guides/commands.md#pytask-markers) command to register +and document them. ## Expressions Expressions are similar to markers and offer the same syntax but target the task ids -with [`pytask build -k`](../commands/build.md#options). Assume you have the following -tasks. +with [`pytask build -k`](../reference_guides/commands.md#pytask-build--k). Assume you +have the following tasks. ```python def task_1(): diff --git a/docs/source/tutorials/visualizing_the_dag.md b/docs/source/tutorials/visualizing_the_dag.md index 1089178d..cd119971 100644 --- a/docs/source/tutorials/visualizing_the_dag.md +++ b/docs/source/tutorials/visualizing_the_dag.md @@ -31,19 +31,19 @@ $ pytask dag It generates a `dag.pdf` in the current working directory. If you do not want to generate a PDF, use -[`pytask dag --output-path`](../commands/dag.md#options) or, shorter, -[`pytask dag -o`](../commands/dag.md#options) to choose a different format inferred from -the file-ending. Select any format supported by -[graphviz](https://graphviz.org/docs/outputs/). +[`pytask dag --output-path`](../reference_guides/commands.md#pytask-dag--output-path) +or, shorter, [`pytask dag -o`](../reference_guides/commands.md#pytask-dag--output-path) +to choose a different format inferred from the file-ending. Select any format supported +by [graphviz](https://graphviz.org/docs/outputs/). ```console $ pytask dag -o dag.png ``` You can change the graph's layout by using the -[`pytask dag --layout`](../commands/dag.md#options) option. Its default is set to `dot` -and produces a hierarchical structure. graphviz supports other layouts, which are listed -[here](https://graphviz.org/docs/layouts/). +[`pytask dag --layout`](../reference_guides/commands.md#pytask-dag--layout) option. Its +default is set to `dot` and produces a hierarchical structure. graphviz supports other +layouts, which are listed [here](https://graphviz.org/docs/layouts/). ## Programmatic Interface diff --git a/justfile b/justfile index 7b942c45..9efe56dd 100644 --- a/justfile +++ b/justfile @@ -27,11 +27,13 @@ check: lint typing test # Build documentation docs *FLAGS: + uv run python scripts/generate_cli_reference.py uv run --group plugin-list python scripts/update_plugin_list.py uv run --group docs zensical build {{FLAGS}} # Serve documentation with auto-reload docs-serve *FLAGS: + uv run python scripts/generate_cli_reference.py uv run --group plugin-list python scripts/update_plugin_list.py uv run --group docs zensical serve -a 127.0.0.1:8000 {{FLAGS}} diff --git a/mkdocs.yml b/mkdocs.yml index 257741e6..4711b71a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,29 +62,22 @@ nav: - Pluggy: explanations/pluggy.md - Reference Guides: - Overview: reference_guides/index.md + - API: + - Overview: api/index.md + - CLI Imports: api/cli_and_programmatic.md + - Functional Interfaces: api/functional_interfaces.md + - Core Classes And Exceptions: api/core_classes_and_exceptions.md + - Marks: api/marks.md + - Nodes And Tasks: api/nodes_and_tasks.md + - Outcomes, Reports, And Warnings: api/outcomes_reports_and_warnings.md + - Utilities And Typing: api/utilities_and_typing.md + - Commands: reference_guides/commands.md - Configuration: reference_guides/configuration.md - - Lockfile: reference_guides/lockfile.md - Hook Specifications: reference_guides/hookspecs.md - - Type Hints: type_hints.md - - Developer's Guide: developers_guide.md - - Glossary: glossary.md - - API: - - Overview: api/index.md - - CLI Imports: api/cli_and_programmatic.md - - Functional Interfaces: api/functional_interfaces.md - - Core Classes And Exceptions: api/core_classes_and_exceptions.md - - Marks: api/marks.md - - Nodes And Tasks: api/nodes_and_tasks.md - - Outcomes, Reports, And Warnings: api/outcomes_reports_and_warnings.md - - Utilities And Typing: api/utilities_and_typing.md - - CLI: - - Overview: commands/index.md - - build: commands/build.md - - clean: commands/clean.md - - collect: commands/collect.md - - dag: commands/dag.md - - markers: commands/markers.md - - profile: commands/profile.md + - Lockfile: reference_guides/lockfile.md + - Type Hints: reference_guides/type_hints.md + - Developer's Guide: reference_guides/developers_guide.md + - Glossary: reference_guides/glossary.md - Plugin List: plugin_list.md - Changelog: changes.md @@ -95,8 +88,13 @@ theme: features: - content.code.copy - content.code.annotate + - content.tabs.link + - navigation.top - navigation.footer - navigation.path + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.progress - navigation.top - search.highlight palette: diff --git a/scripts/generate_cli_command_docs.py b/scripts/generate_cli_command_docs.py deleted file mode 100644 index 1964c472..00000000 --- a/scripts/generate_cli_command_docs.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Generate CLI command reference tables for docs.""" - -from __future__ import annotations - -import enum -import html -import re -from pathlib import Path - -import click - -from _pytask.cli import cli - -OUT_DIR = Path("docs/source/_static/md/commands") -COMMANDS = ("build", "clean", "collect", "dag", "markers", "profile") - - -def _strip_click_suffixes(help_text: str) -> str: - """Remove click-appended suffixes like default/range metadata.""" - help_text = re.sub(r"\s*\[default: .*?\]\s*$", "", help_text) - help_text = re.sub(r"\s*\[[0-9<>=,\.\-A-Za-z]+\]\s*$", "", help_text) - return help_text.strip() - - -def _escape_table_cell(text: str) -> str: - """Escape markdown table separators inside cell content.""" - return text.replace("|", "|").replace("\n", " ") - - -def _format_code(text: str) -> str: - """Format code-like values for markdown tables.""" - escaped = html.escape(text, quote=False).replace("|", "|") - return f"{escaped}" - - -def _format_default(option: click.Option) -> str: - default = option.default - result = "-" - - if isinstance(default, bool): - if option.secondary_opts: - active = option.opts[0] if default else option.secondary_opts[0] - result = active - else: - result = str(default).lower() - elif default is None or (isinstance(default, tuple | list) and not default): - result = "-" - elif isinstance(default, enum.Enum): - if str(default.value).startswith(" click.Context: - command = cli.commands[command_name] - return click.Context(command, info_name=f"pytask {command_name}") - - -def _write_options(command_name: str) -> None: - command = cli.commands[command_name] - ctx = _command_context(command_name) - - lines = ["| Option | Default | Description |", "|---|---|---|"] - - for param in command.params: - if not isinstance(param, click.Option): - continue - - help_record = param.get_help_record(ctx) - if help_record is None: - continue - - option_decl, description = help_record - default = _format_default(param) - escaped_description = _escape_table_cell(_strip_click_suffixes(description)) - lines.append( - "| " - f"{_format_code(option_decl)}" - " | " - f"{_format_code(default) if default != '-' else '-'}" - " | " - f"{escaped_description}" - " |" - ) - - lines.append("| `-h, --help` | - | Show this message and exit. |") - - output = "\n".join(lines) + "\n" - OUT_DIR.joinpath(f"{command_name}-options.md").write_text(output) - - -def _write_arguments(command_name: str) -> None: - command = cli.commands[command_name] - - lines = ["| Argument | Description |", "|---|---|"] - has_arguments = False - - for param in command.params: - if not isinstance(param, click.Argument): - continue - - has_arguments = True - metavar = param.make_metavar(click.Context(command)).strip() - description = "Paths where pytask looks for task files and configuration." - lines.append(f"| {_format_code(metavar)} | {_escape_table_cell(description)} |") - - if not has_arguments: - lines.append("| - | This command does not take positional arguments. |") - - output = "\n".join(lines) + "\n" - OUT_DIR.joinpath(f"{command_name}-arguments.md").write_text(output) - - -def _write_commands_table() -> None: - lines = ["| Command | Description |", "|---|---|"] - lines.extend( - ( - f"| [`{name}`]({name}.md) | " - f"{_escape_table_cell(cli.commands[name].help or '')} |" - ) - for name in COMMANDS - ) - - output = "\n".join(lines) + "\n" - OUT_DIR.joinpath("command-list.md").write_text(output) - - -def _write_root_options() -> None: - ctx = click.Context(cli, info_name="pytask") - lines = ["| Option | Description |", "|---|---|"] - - for param in cli.params: - if not isinstance(param, click.Option): - continue - help_record = param.get_help_record(ctx) - if help_record is None: - continue - option_decl, description = help_record - lines.append( - "| " - f"{_format_code(option_decl)}" - " | " - f"{_escape_table_cell(_strip_click_suffixes(description))}" - " |" - ) - - lines.append("| `-h, --help` | Show this message and exit. |") - - output = "\n".join(lines) + "\n" - OUT_DIR.joinpath("root-options.md").write_text(output) - - -def main() -> None: - OUT_DIR.mkdir(parents=True, exist_ok=True) - - for command_name in COMMANDS: - _write_arguments(command_name) - _write_options(command_name) - - _write_commands_table() - _write_root_options() - - -if __name__ == "__main__": - main() diff --git a/scripts/generate_cli_reference.py b/scripts/generate_cli_reference.py new file mode 100644 index 00000000..4d243224 --- /dev/null +++ b/scripts/generate_cli_reference.py @@ -0,0 +1,314 @@ +"""Generate a one-page CLI reference for the documentation.""" + +from __future__ import annotations + +import argparse +import enum +import html +import inspect +import re +import sys +from pathlib import Path + +import click + +from _pytask.cli import cli + +OUT_PATH = Path("docs/source/reference_guides/commands.md") + + +def _strip_click_suffixes(help_text: str) -> str: + """Remove click-appended suffixes like default or range metadata.""" + help_text = re.sub(r"\s*\[default: .*?\]\s*$", "", help_text) + help_text = re.sub(r"\s*\[[0-9<>=,\.\-A-Za-z]+\]\s*$", "", help_text) + return help_text.strip() + + +def _format_default(option: click.Option) -> str | None: + result: str | None = None + + if not option.expose_value: + return None + + default = option.default + + if isinstance(default, bool): + if option.secondary_opts: + result = option.opts[0] if default else option.secondary_opts[0] + else: + result = str(default).lower() + elif default is None or (isinstance(default, tuple | list) and not default): + result = None + elif isinstance(default, enum.Enum): + value = str(default.value) + result = None if value.startswith(" str: + help_text = command.help or command.short_help or "" + return inspect.cleandoc(help_text).strip() + + +def _command_anchor(parts: tuple[str, ...]) -> str: + return "-".join(parts) + + +def _option_anchor(parts: tuple[str, ...], option: click.Option) -> str: + if option.opts: + names = [opt for opt in option.opts if opt.startswith("--")] + candidate = names[0] if names else option.opts[0] + suffix = candidate.lstrip("-") + else: + suffix = option.name or "option" + return f"{_command_anchor(parts)}--{suffix}" + + +def _argument_anchor(parts: tuple[str, ...], argument: click.Argument) -> str: + name = (argument.name or "argument").replace("_", "-") + return f"{_command_anchor(parts)}--{name}" + + +def _option_heading(option: click.Option, ctx: click.Context) -> str: + help_record = option.get_help_record(ctx) + if help_record is not None: + return help_record[0] + if option.opts: + return ", ".join((*option.opts, *option.secondary_opts)) + return option.name or "option" + + +def _option_description(option: click.Option, ctx: click.Context) -> str: + help_record = option.get_help_record(ctx) + if help_record is None: + return "" + return _strip_click_suffixes(help_record[1]) + + +def _argument_description(argument: click.Argument) -> str: + if argument.name == "paths": + return "Paths where pytask looks for task files and configuration." + return "Positional argument." + + +def _render_usage(command: click.Command, parts: tuple[str, ...]) -> str: + ctx = click.Context(command, info_name=" ".join(parts)) + usage = command.get_usage(ctx).strip() + return usage.removeprefix("Usage: ").strip() + + +def _command_items(command: click.Command) -> list[tuple[str, click.Command]]: + commands = getattr(command, "commands", {}) + return [ + (name, subcommand) + for name, subcommand in sorted(commands.items()) + if not getattr(subcommand, "hidden", False) + ] + + +def _command_arguments( + command: click.Command, ctx: click.Context +) -> list[click.Argument]: + return [ + param + for param in command.get_params(ctx) + if isinstance(param, click.Argument) and not getattr(param, "hidden", False) + ] + + +def _command_options(command: click.Command, ctx: click.Context) -> list[click.Option]: + return [ + param + for param in command.get_params(ctx) + if isinstance(param, click.Option) and not getattr(param, "hidden", False) + ] + + +def _render_definition_item( + *, + anchor: str, + term: str, + description: str, + details: list[str] | None = None, +) -> str: + escaped_term = html.escape(term, quote=False) + body = [ + f'
{escaped_term}
' + ] + + description_html = html.escape(description) + extra_html = "" + if details: + rendered_details = "
".join(html.escape(detail) for detail in details) + extra_html = f"

{rendered_details}

" + + body.append(f"

{description_html}

{extra_html}
") + return "".join(body) + + +def _render_argument_section( + lines: list[str], + parts: tuple[str, ...], + argument: click.Argument, + ctx: click.Context, +) -> None: + anchor = _argument_anchor(parts, argument) + metavar = argument.make_metavar(ctx).strip() + lines.append( + _render_definition_item( + anchor=anchor, + term=metavar, + description=_argument_description(argument), + ) + ) + + +def _render_option_section( + lines: list[str], parts: tuple[str, ...], option: click.Option, ctx: click.Context +) -> None: + anchor = _option_anchor(parts, option) + heading = _option_heading(option, ctx) + description = _option_description(option, ctx) + + metadata: list[str] = [] + default = _format_default(option) + if default is not None: + metadata.append(f"Default: {default}") + + if isinstance(option.type, click.Choice): + choices = ", ".join(option.type.choices) + metadata.append(f"Accepted values: {choices}") + + lines.append( + _render_definition_item( + anchor=anchor, + term=heading, + description=description, + details=metadata or None, + ) + ) + + +def _render_command( + lines: list[str], command: click.Command, parts: tuple[str, ...] +) -> None: + lines.append(f"## {' '.join(parts)} {{ #{_command_anchor(parts)} }}") + lines.append("") + + description = _command_description(command) + if description: + lines.append(description) + lines.append("") + + lines.append("**Usage**") + lines.append("") + lines.append("```console") + lines.append(_render_usage(command, parts)) + lines.append("```") + lines.append("") + + subcommands = _command_items(command) + if subcommands: + lines.append("**Commands**") + lines.append("") + for name, subcommand in subcommands: + child_parts = (*parts, name) + summary = _command_description(subcommand) + line = f"- [`{' '.join(child_parts)}`](#{_command_anchor(child_parts)})" + if summary: + line += f": {summary}" + lines.append(line) + lines.append("") + + ctx = click.Context(command, info_name=" ".join(parts)) + arguments = _command_arguments(command, ctx) + if arguments: + lines.append("**Arguments**") + lines.append("") + lines.append('
') + for argument in arguments: + _render_argument_section(lines, parts, argument, ctx) + lines.append("
") + lines.append("") + + options = _command_options(command, ctx) + if options: + lines.append("**Options**") + lines.append("") + lines.append('
') + for option in options: + _render_option_section(lines, parts, option, ctx) + lines.append("
") + lines.append("") + + for name, subcommand in subcommands: + _render_command(lines, subcommand, (*parts, name)) + + +def generate_cli_reference() -> str: + """Generate the full markdown document.""" + lines = [ + ( + "" + ), + "", + "# CLI Reference", + "", + "Running `pytask` without a subcommand is equivalent to `pytask build`.", + "", + "## Command Index", + "", + ] + + for name, subcommand in _command_items(cli): + parts = ("pytask", name) + summary = _command_description(subcommand) + line = f"- [`{' '.join(parts)}`](#{_command_anchor(parts)})" + if summary: + line += f": {summary}" + lines.append(line) + + lines.append("") + _render_command(lines, cli, ("pytask",)) + return "\n".join(lines).rstrip() + "\n" + + +def _write_reference(path: Path, content: str) -> bool: + path.parent.mkdir(parents=True, exist_ok=True) + current = path.read_text() if path.exists() else None + if current == content: + return False + path.write_text(content) + return True + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + action="store_true", + help="Fail if the generated page differs from the checked-in file.", + ) + args = parser.parse_args() + + content = generate_cli_reference() + + if args.check: + current = OUT_PATH.read_text() if OUT_PATH.exists() else None + if current != content: + return 1 + return 0 + + _write_reference(OUT_PATH, content) + return 0 + + +if __name__ == "__main__": + sys.exit(main())