From c1200752e795120e9f503bb2a8b844a0cf53f80c Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 13 Aug 2025 11:30:47 +0200 Subject: [PATCH 01/21] refactoring docs --- docs/source/components/configuration.rst | 121 +++------------------ docs/source/components/discover.rst | 132 +++++++++++++++++++++++ docs/src_trace.toml | 6 +- 3 files changed, 150 insertions(+), 109 deletions(-) create mode 100644 docs/source/components/discover.rst diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 751bb40..b5b606a 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -140,54 +140,6 @@ and its corresponding value is a dictionary containing the options specific to t [src_trace.projects.project_name] # Project configuration for "project_name" shall be written here -comment_type -~~~~~~~~~~~~ - -This option defines the comment type used in the source code of the project. - -Default: **cpp** - -.. note:: Currently, only C/C++ is supported - -.. tabs:: - - .. code-tab:: python - - src_trace_projects = { - "project_name": { - "comment_type": "c" - } - } - - .. code-tab:: toml - - [src_trace.projects.project_name] - comment_type = "c" - -.. _source_dir: - -src_dir -~~~~~~~ - -The relative path from the ``conf.py`` or ``.toml`` file to the source code's root directory - -Default: **./** - -.. tabs:: - - .. code-tab:: python - - src_trace_projects = { - "project_name": { - "src_dir": "./../src" - } - } - - .. code-tab:: toml - - [src_trace.projects.project_name] - src_dir = "./../src" - remote_url_pattern ~~~~~~~~~~~~~~~~~~ @@ -229,75 +181,30 @@ with the following setup: "options": [remote_url_field], } -exclude -~~~~~~~ - -The option is a list of glob patterns to exclude the files which are not required to be addressed +source_discover +~~~~~~~~~~~~~~~ -Default: **[]** - -.. tabs:: +This option enables users to discover the source files of the project, which will be processed by ``CodeLinks``. - .. code-tab:: python - - src_trace_projects = { - "project_name": { - "exclude": ["dcdc/src/ubt/ubt.cpp"] - } - } - - .. code-tab:: toml - - [src_trace.projects.project_name] - exclude = ["dcdc/src/ubt/ubt.cpp"] - -include -~~~~~~~ - -The option is a list of glob patterns to include the files which are required to be addressed - -Default: **[]** - -.. tabs:: - - .. code-tab:: python - - src_trace_projects = - { - "project_name": { - "include": ["dcdc/src/ubt/ubt.cpp"] - } - } - - .. code-tab:: toml - - [src_trace.projects.project_name] - include = ["dcdc/src/ubt/ubt.cpp"] - -.. note:: **include** option has the highest priority over **exclude** and **gitignore** options. - -gitignore -~~~~~~~~~ - -The option to respect the .gitignore file. - -Default: **True** +Default: .. tabs:: .. code-tab:: python src_trace_projects = { - "project_name": { - "gitignore": False + "project_name":{ + "source_discover": + { + "src_dir": "./", + "exclude": [], + "include": [], + "gitignore": True, + "comment_type": "cpp" + } } - .. code-tab:: toml - - [src_trace.projects.project_name] - gitignore = false - -.. attention:: This option currently does NOT support nested .gitignore files +The details of each options are explained in :ref:`source discover ` .. _`oneline_comment_style`: diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst new file mode 100644 index 0000000..6ac24e1 --- /dev/null +++ b/docs/source/components/discover.rst @@ -0,0 +1,132 @@ +.. _discover: + +Source Discover +=============== + +SourceDiscover is one of the modules provided in ``Codelinks``. It discovers the source files from the given directory. +It provides users CLI and API to discover the source files. + +Configuration +~~~~~~~~~~~~~ + +When used as CLI, a TOML config file can be provided to configure the source discover module. +The config file contains the following fields: + +comment_type +------------ + +This option defines the comment type used in the source code of the project. + +Default: **cpp** + +.. note:: Currently, only C/C++ is supported + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "comment_type": "c" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + comment_type = "c" + +.. _source_dir: + +src_dir +------- + +The relative path from the ``conf.py`` or ``.toml`` file to the source code's root directory + +Default: **./** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "src_dir": "./../src" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + src_dir = "./../src" + +exclude +------- + +The option is a list of glob patterns to exclude the files which are not required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "exclude": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + exclude = ["dcdc/src/ubt/ubt.cpp"] + + +include +------- + +The option is a list of glob patterns to include the files which are required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = + { + "project_name": { + "include": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + include = ["dcdc/src/ubt/ubt.cpp"] + +.. note:: **include** option has the highest priority over **exclude** and **gitignore** options. + +gitignore +--------- + +The option to respect the .gitignore file. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "gitignore": False + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + gitignore = false + +.. attention:: This option currently does NOT support nested .gitignore files diff --git a/docs/src_trace.toml b/docs/src_trace.toml index 304299d..b1d7d56 100644 --- a/docs/src_trace.toml +++ b/docs/src_trace.toml @@ -5,11 +5,13 @@ local_url_field = "local-url" # Need's field name for local URL set_remote_url = true # Set to true to enable remote url to be generated remote_url_field = "remote-url" # Need's field name for remote URL -[src_trace.projects.dcdc] # Configuration for source tracing project "dcdc" -src_dir = "../tests/data/dcdc" # Relative path from conf.py to the source directory +[src_trace.projects.dcdc] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code +[src_trace.projects.dcdc.source_discover] +src_dir = "../tests/data/dcdc" # Relative path from this TOML config to the source directory + [src_trace.projects.dcdc.oneline_comment_style] # Configuration for oneline comment style start_sequence = "[[" # Start sequence for oneline comments From 887eec468559de58705a5f6b244e08434bc98f8b Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 13 Aug 2025 16:04:50 +0200 Subject: [PATCH 02/21] refactored docs --- docs/source/components/analyse.rst | 363 ++++++++++++++++------- docs/source/components/configuration.rst | 258 +++++++++------- docs/source/components/discover.rst | 204 ++++++++----- 3 files changed, 542 insertions(+), 283 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 8cd9e04..7386f90 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -1,27 +1,150 @@ -Analyse -======= +.. _analyse: -The ``Analyse`` is a :ref:`CLI tool ` that also provides an API for programmatic use. Its primary function is to extract specific, marked content from comments within source code files. +Source Analysis +=============== -It can extract three types of content: +The **Source Analysis** module is a powerful component of **Sphinx-CodeLinks** that extracts documentation-related content from source code comments. It provides both CLI and API interfaces for flexible integration into documentation workflows. -- Sphinx-Needs ID References -- Oneline needs (see :ref:`OneLineCommentStyle `) -- Marked reStructuredText (RST) blocks +**Key Capabilities:** + +- Extract **Sphinx-Needs** ID references from source code comments +- Process custom one-line comment patterns for rapid documentation +- Extract marked reStructuredText (RST) blocks embedded in comments +- Generate structured JSON output for further processing +- Support for multiple programming language comment styles + +Overview +-------- + +Source Analysis works by parsing source code files and identifying specially marked comments that contain documentation information. This enables developers to embed documentation directly in their source code while maintaining clean separation between code and documentation. + +The module supports three primary extraction modes: + +1. **Sphinx-Needs ID References** - Links between code and requirements/specifications +2. **One-line Needs** - Simplified syntax for creating documentation needs +3. **Marked RST Blocks** - Full reStructuredText content embedded in comments + +Supported Content Types +----------------------- + +Sphinx-Needs ID References +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Extract references to **Sphinx-Needs** items directly from source code comments, enabling traceability between code implementations and requirements. + +One-line Needs +~~~~~~~~~~~~~~~ + +Use simplified comment patterns to define **Sphinx-Needs** items without complex RST syntax. See :ref:`OneLineCommentStyle ` for detailed information. + +Marked RST Blocks +~~~~~~~~~~~~~~~~~~ + +Embed complete reStructuredText content within source code comments for rich documentation that can be extracted and processed. + +Limitations +----------- + +**Current Limitations:** + +- **Language Support**: Only C/C++ (``//``, ``/* */``) and Python (``#``) comment styles are supported +- **Single Comment Style**: Each analysis run processes only one comment style at a time Configuration ------------- -The ``Analyse`` is configured using a ``toml`` file. The examples throughout this document are based on the following configuration: +The **Source Analysis** module is configured using TOML files and leverages the :ref:`Source Discovery ` module to locate source files for processing. + +**Complete Configuration Example:** + +.. code-block:: toml + + [source_discover] + src_dir = "./" + exclude = [] + include = [] + gitignore = true + comment_type = "cpp" + + [analyse] + get_need_id_refs = true + get_oneline_needs = false + get_rst = false + outdir = "./output" + + [analyse.oneline_comment_style] + start_sequence = "@" + # End sequences is newline by default. Whether it is "\n" or "\r\n" depending on the platform + end_sequence = "\n" + field_split_char = "," + needs_fields = [ + { name = "title", type = "str" }, + { name = "id", type = "str" }, + { name = "type", type = "str", default = "impl" }, + { name = "links", type = "list[str]", default = [] }, + ] + + [analyse.need_id_refs] + markers = ["@need-ids:"] + + [analyse.marked_rst] + start_sequence = "@rst" + end_sequence = "@endrst" + +Configuration Sections +----------------------- + +analyse +~~~~~~~ + +Main configuration section for the analysis module. + +**Options:** + +- ``get_need_id_refs`` (``bool``) - Enable extraction of Sphinx-Needs ID references +- ``get_oneline_needs`` (``bool``) - Enable extraction of one-line needs +- ``get_rst`` (``bool``) - Enable extraction of marked RST blocks +- ``outdir`` (``str``) - Output directory for generated files + +analyse.need_id_refs +~~~~~~~~~~~~~~~~~~~~ + +Configuration for Sphinx-Needs ID reference extraction. + +**Options:** + +- ``markers`` (``list[str]``) - List of marker strings that identify need ID references + +analyse.marked_rst +~~~~~~~~~~~~~~~~~~ + +Configuration for marked RST block extraction. + +**Options:** + +- ``start_sequence`` (``str``) - Marker that begins an RST block +- ``end_sequence`` (``str``) - Marker that ends an RST block + +analyse.oneline_comment_style +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Configuration for one-line needs extraction. See :ref:`oneline_comment_style` for detailed information. + +**Minimal Configuration Example:** + +The following configuration demonstrates the minimum settings required for basic analysis: .. literalinclude:: ./../../../tests/data/analyse/default_config.toml - :caption: default_config.toml + :caption: minimum_config.toml :language: toml -This configuration instructs the analyse to extract ``Sphinx-Needs ID Refs`` and ``Marked rst text`` using the defined markers. +This configuration enables extraction of **Sphinx-Needs ID References** and **Marked RST blocks** using the specified markers. + +Extraction Examples +------------------- -Sphinx-Needs ID Refs --------------------- +Sphinx-Needs ID References +~~~~~~~~~~~~~~~~~~~~~~~~~~ Below is an example of a C++ source file containing need ID references and the corresponding JSON output from the analyse. @@ -29,112 +152,152 @@ Below is an example of a C++ source file containing need ID references and the c .. code-tab:: cpp - #include + #include - // @need-ids: need_001, need_002, need_003, need_004 - void dummy_func1(){ - //... - } + // @need-ids: need_001, need_002, need_003, need_004 + void dummy_func1(){ + //... + } - // @need-ids: need_003 - int main() { - std::cout << "Starting demo_1..." << std::endl; - dummy_func1(); - std::cout << "Demo_1 finished." << std::endl; - return 0; - } + // @need-ids: need_003 + int main() { + std::cout << "Starting demo_1..." << std::endl; + dummy_func1(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } .. code-tab:: json - [ - { - "filepath": "tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L3", - "source_map": { - "start": { "row": 2, "column": 13 }, - "end": { "row": 2, "column": 51 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "need_ids": ["need_001", "need_002", "need_003", "need_004"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L8", - "source_map": { - "start": { "row": 7, "column": 13 }, - "end": { "row": 7, "column": 21 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "need_ids": ["need_003"], - "marker": "@need-ids:", - "type": "need-id-refs" - } - ] - -Marked RST ----------- + [ + { + "filepath": "tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": { "row": 2, "column": 13 }, + "end": { "row": 2, "column": 51 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["need_001", "need_002", "need_003", "need_004"], + "marker": "@need-ids:", + "type": "need-id-refs" + }, + { + "filepath": "tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L8", + "source_map": { + "start": { "row": 7, "column": 13 }, + "end": { "row": 7, "column": 21 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "need_ids": ["need_003"], + "marker": "@need-ids:", + "type": "need-id-refs" + } + ] + +**Output Structure:** + +- ``filepath`` - Path to the source file containing the reference +- ``remote_url`` - URL to the source code in the remote repository +- ``source_map`` - Location information (row/column) of the marker +- ``tagged_scope`` - The code scope associated with the marker +- ``need_ids`` - List of referenced need IDs +- ``marker`` - The marker string used for identification +- ``type`` - Type of extraction ("need-id-refs") + +Marked RST Blocks +~~~~~~~~~~~~~~~~~ This example demonstrates how the analyse extracts RST blocks from comments. .. tabs:: - .. code-tab:: cpp + .. code-tab:: cpp - #include + #include - /* - @rst - .. impl:: implement dummy function 1 + /* + @rst + .. impl:: implement dummy function 1 :id: IMPL_71 - @endrst - */ - void dummy_func1(){ - //... - } + @endrst + */ + void dummy_func1(){ + //... + } - // @rst..impl:: implement main function @endrst - int main() { - std::cout << "Starting demo_1..." << std::endl; - dummy_func1(); - std::cout << "Demo_1 finished." << std::endl; - return 0; - } + // @rst..impl:: implement main function @endrst + int main() { + std::cout << "Starting demo_1..." << std::endl; + dummy_func1(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } .. code-tab:: json - [ - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", - "source_map": { - "start": { "row": 3, "column": 8 }, - "end": { "row": 3, "column": 61 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", - "type": "rst" - }, - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", - "source_map": { - "start": { "row": 13, "column": 7 }, - "end": { "row": 13, "column": 40 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "rst": "..impl:: implement main function ", - "type": "rst" - } - ] + [ + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" + }, + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" + } + ] -Limitations ------------ +**Output Structure:** + +- ``filepath`` - Path to the source file containing the RST block +- ``remote_url`` - URL to the source code in the remote repository +- ``source_map`` - Location information of the RST markers +- ``tagged_scope`` - The code scope associated with the RST block +- ``rst`` - The extracted reStructuredText content +- ``type`` - Type of extraction ("rst") + +**RST Block Formats:** + +The module supports both multi-line and single-line RST blocks: + +- **Multi-line blocks**: Use ``@rst`` and ``@endrst`` on separate lines +- **Single-line blocks**: Use ``@rst content @endrst`` on the same line + +One-line Needs +-------------- + +**One-line Needs** provide a simplified syntax for creating **Sphinx-Needs** items directly in source code comments without requiring full RST syntax. + +For comprehensive information about one-line needs configuration and usage, see :ref:`OneLineCommentStyle `. + +**Basic Example:** + +.. tabs: + +.. code-tab:: c + + // @Function Implementation, IMPL_001, impl, [REQ_001, REQ_002] + +This single comment line creates a complete **Sphinx-Needs** item equivalent to: -Please be aware of the following limitations: +.. code-tab:: rst -- **Supported Languages**: The analyse only supports comment styles for C/C++ (``//``, ``/*...*/``) and Python (``#``). -- **Single Comment Style**: An analysis run can only process a single comment style at a time. -- **Configuration Incompatibility**: The TOML configuration file cannot be shared with the ``CodeLink`` Sphinx extensions. + .. impl:: Function Implementation + :id: IMPL_001 + :links: REQ_001, REQ_002 diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index b5b606a..9f48d09 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -13,33 +13,39 @@ All configuration options start with the prefix ``src_trace_`` for **Sphinx-Code Global Options -------------- -The options starts with the prefix ``src_trace_`` are globally applied in the scope of Sphinx documentation. +Global options use the ``src_trace_`` prefix and are applied across the entire Sphinx documentation project. src_trace_config_from_toml -~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Specifies the path to a `TOML file `__ containing **Sphinx-CodeLinks** configuration options. This allows you to maintain configuration in a separate file for better organization. -This configuration takes the (relative) path to a `toml file `__ -which contains some or all of the ``CodeLinks`` configuration -(configuration in the toml will override that in the :file:`conf.py`). +**Type:** ``str`` (relative path to the directory where conf.py locates) +**Default:** Not set .. code-block:: python - # Specify the config path for source tracing in conf.py + # In conf.py src_trace_config_from_toml = "src_trace.toml" -Configuration in the toml can contain any of the following options, under a ``[src_trace]`` section, -but with the ``src_trace_`` prefix removed. +When using a TOML configuration file: + +- Configuration options are placed under a ``[src_trace]`` section +- The ``src_trace_`` prefix is omitted in the TOML file +- TOML configuration overrides settings in :file:`conf.py` -.. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the ``toml`` file. +.. caution:: + Relative paths specified in the TOML file are resolved relative to the directory containing the TOML file, not the Sphinx project root. -.. _`src_trace_set_local_url`: +.. _src_trace_set_local_url: src_trace_set_local_url ~~~~~~~~~~~~~~~~~~~~~~~ -Set this option to ``True``, if the local link between a need to the local source code where it is defined is required. +Enables the generation of local file system links to source code locations. When enabled, **Sphinx-CodeLinks** will add a custom field, which contains the local path to the source file, to generated needs. -Default: **False** +**Type:** ``bool`` +**Default:** ``False`` .. tabs:: @@ -52,14 +58,14 @@ Default: **False** [src_trace] set_local_url = true -src_trace_set_local_field +src_trace_local_url_field ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This option is only required if :ref:`src_trace_set_local_url` is set to **True**. +Specifies the custom field name used for local source code links. -Set the desired custom field name for the local link to the source code. - -Default: **local-url** +**Type:** ``str`` +**Default:** ``"local-url"`` +**Required when:** :ref:`src_trace_set_local_url` is ``True`` .. tabs:: @@ -72,17 +78,15 @@ Default: **local-url** [src_trace] local_url_field = "local-url" -.. _`src_trace_set_remote_url`: +.. _src_trace_set_remote_url: src_trace_set_remote_url ~~~~~~~~~~~~~~~~~~~~~~~~ -Set this option to ``True``, if the remote link between a need to the remote source code -where it is defined is required. - -The remote means where the source code is hosted such as GitHub. +Enables the generation of remote repository links to source code locations. When enabled, **Sphinx-CodeLinks** will add a custom field, which contains the URL to the remote repository (e.g., GitHub, GitLab) where the source file is hosted, to needs. -Default: **False** +**Type:** ``bool`` +**Default:** ``False`` .. tabs:: @@ -95,14 +99,14 @@ Default: **False** [src_trace] set_remote_url = true -src_trace_set_remote_field +src_trace_remote_url_field ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This option is only required if :ref:`src_trace_set_remote_url` is set to **True**. - -Set the desired custom field name for the remote link to the source code. +Specifies the custom field name used for remote source code links. -Default: **remote-url** +**Type:** ``str`` +**Default:** ``"remote-url"`` +**Required when:** :ref:`src_trace_set_remote_url` is ``True`` .. tabs:: @@ -115,106 +119,131 @@ Default: **remote-url** [src_trace] remote_url_field = "remote-url" -Project Specific Options ------------------------- +Project-Specific Options +------------------------- -Options defined in **src_trace_projects** are project-specific. +Project-specific options are configured within the ``src_trace_projects`` dictionary, allowing different settings for each source code project being analyzed. src_trace_projects ~~~~~~~~~~~~~~~~~~ -This option contains multiple sets of project-specific options. The project name is defined as the key in a dictionary -and its corresponding value is a dictionary containing the options specific to that project. +Defines configuration for individual source code projects. Each project is identified by a unique name (key) and contains its own set of configuration options (value). + +**Type:** ``dict[str, dict]`` +**Default:** ``{}`` .. tabs:: .. code-tab:: python - project_options = dict() src_trace_projects = { - "project_name": project_options + "my_project": { + # Project-specific options go here + }, + "another_project": { + # Different options for another project + } } .. code-tab:: toml - [src_trace.projects.project_name] - # Project configuration for "project_name" shall be written here + [src_trace.projects.my_project] + # Configuration for "my_project" + + [src_trace.projects.another_project] + # Configuration for "another_project" remote_url_pattern -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~ -This option only works with :ref:`src_trace_set_remote_url` set to **True**. -The pattern to access the source code to the remote repositories such as GitHub. +Defines the URL pattern for generating links to remote source code repositories (e.g., GitHub, GitLab). This pattern uses placeholders that are dynamically replaced with actual values. -Default: **Not set** +**Type:** ``str`` +**Default:** Not set +**Required when:** :ref:`src_trace_set_remote_url` is ``True`` + +**Available placeholders:** + +- ``{commit}`` - Git commit hash +- ``{path}`` - Relative path to the source file +- ``{line}`` - Line number in the source file .. tabs:: .. code-tab:: python src_trace_projects = { - "project_name": { - "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" - } + "my_project": { + "remote_url_pattern": "https://github.com/user/repo/blob/{commit}/{path}#L{line}" + } } .. code-tab:: toml - [src_trace.projects.project_name] - remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" - -This option leverages the configuration of :external+needs:ref:`need_string_links` -with the following setup: + [src_trace.projects.my_project] + remote_url_pattern = "https://github.com/user/repo/blob/{commit}/{path}#L{line}" -.. code-block:: python +**Common patterns:** - remote_url_pattern = remote_url_pattern.format( - commit=commit_id, - path=f"{remote_src_dir}/" + "{{value}}", - line="{{lineno}}", - ) +- **GitHub:** ``https://github.com/user/repo/blob/{commit}/{path}#L{line}`` +- **GitLab:** ``https://gitlab.com/user/repo/-/blob/{commit}/{path}#L{line}`` +- **Bitbucket:** ``https://bitbucket.org/user/repo/src/{commit}/{path}#lines-{line}`` - { - "regex": r"^(?P.+)#L(?P.*)?", - "link_url": remote_url_pattern, - "link_name": "{{value}}#L{{lineno}}", - "options": [remote_url_field], - } +.. note:: + This option integrates with :external+needs:ref:`need_string_links` to automatically generate clickable links in the documentation. source_discover ~~~~~~~~~~~~~~~ -This option enables users to discover the source files of the project, which will be processed by ``CodeLinks``. +Configures how **Sphinx-CodeLinks** discovers and processes source files within a project. This option controls which files are analyzed for extracting documentation needs. -Default: +**Type:** ``dict`` +**Default:** See below .. tabs:: .. code-tab:: python src_trace_projects = { - "project_name":{ - "source_discover": - { - "src_dir": "./", - "exclude": [], - "include": [], - "gitignore": True, - "comment_type": "cpp" - } - } + "my_project": { + "source_discover": { + "src_dir": "./", + "exclude": [], + "include": [], + "gitignore": True, + "comment_type": "cpp" + } + } + } + + .. code-tab:: toml + + [src_trace.projects.my_project.source_discover] + src_dir = "./" + exclude = [] + include = [] + gitignore = true + comment_type = "cpp" + +**Configuration fields:** -The details of each options are explained in :ref:`source discover ` +- ``src_dir`` - Root directory for source file discovery (relative to Sphinx project root or the directory where TOML config file locates if given) +- ``exclude`` - List of glob patterns to exclude from processing +- ``include`` - List of glob patterns to include (if empty, includes all files) +- ``gitignore`` - Whether to respect ``.gitignore`` rules when discovering files (Nested .gitignore is NOT supported yet) +- ``comment_type`` - Comment style for the programming language ("cpp" and "python" are currently supported) -.. _`oneline_comment_style`: +For detailed information about each field, see :ref:`source discover `. + +.. _oneline_comment_style: oneline_comment_style ~~~~~~~~~~~~~~~~~~~~~ -This option enables users to simply define a customized one-line-pattern comment to represent -``Sphinx-Needs`` need items instead of using RST. +Enables the use of simplified one-line comment patterns to represent **Sphinx-Needs** items directly in source code, eliminating the need for embedded RST syntax. -Default: +**Type:** ``dict`` +**Location:** ``src_trace_projects[project_name]["analyse"]["oneline_comment_style"]`` .. tabs:: @@ -222,49 +251,62 @@ Default: import os src_trace_projects = { - "project_name": { - "oneline_comment_style": { - "start_sequence": "@", - "end_sequence": os.linesep, - "field_split_char": ",", - needs_fields = [ - {"name": "title"}, - {"name": "id"}, - {"name": "type", "default": "impl"}, - {"name": "links", "type": "list[str]", "default": []}, - ] - } - } + "my_project": { + "analyse": { + "oneline_comment_style": { + "start_sequence": "@", + "end_sequence": os.linesep, + "field_split_char": ",", + "needs_fields": [ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ] + } + } + } } .. code-tab:: toml - [src_trace.projects.project_name.oneline_comment_style] + [src_trace.projects.my_project.analyse.oneline_comment_style] start_sequence = "@" - # end_sequence for the online comments; default is an os-dependant newline character + end_sequence = "\n" # Platform-specific line ending field_split_char = "," needs_fields = [ - { "name" = "title", "type" = "str" }, - { "name" = "id", "type" = "str" }, - { "name" = "type", "type" = "str", "default" = "impl" }, - { "name" = "links", "type" = "list[str]", "default" = [] }, + { name = "title", type = "str" }, + { name = "id", type = "str" }, + { name = "type", type = "str", default = "impl" }, + { name = "links", type = "list[str]", default = [] }, ] -With the default, the following one-line comment will be extracted by ``CodeLinks`` and -it is equivalent to the following RST +**Configuration fields:** -.. tabs:: +- ``start_sequence`` - Character(s) that begin a one-line comment pattern +- ``end_sequence`` - Character(s) that end a one-line comment pattern (typically line ending) +- ``field_split_char`` - Character used to separate fields within the comment +- ``needs_fields`` - List of field definitions for extracting need information + +**Example usage:** + +The following one-line comment in source code: + +.. code-block:: cpp + + // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] - .. code-tab:: c +Is equivalent to this RST directive: - // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] +.. code-block:: rst - .. code-tab:: RST + .. impl:: Function Bar + :id: IMPL_4 + :links: SPEC_1, SPEC_2 - .. impl:: Function Bar - :id: IMPL_4 - :links: [SPEC_1, SPEC_2] +.. important:: + The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. -.. caution:: **type** and **title** must be configured in **needs_fields** as they are mandatory for Sphinx-Needs +**Additional examples and use cases:** -More uses cases can be found in `tests `__ +For more comprehensive examples and advanced configurations, see the `test cases `__. diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 6ac24e1..8b646c6 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -6,127 +6,181 @@ Source Discover SourceDiscover is one of the modules provided in ``Codelinks``. It discovers the source files from the given directory. It provides users CLI and API to discover the source files. + +.. _`default_discover`: + Configuration ~~~~~~~~~~~~~ When used as CLI, a TOML config file can be provided to configure the source discover module. -The config file contains the following fields: +The default configuration is as follows: -comment_type ------------- +.. code-block:: toml -This option defines the comment type used in the source code of the project. + [source_discover] + src_dir = "./", + exclude = [], + include = [], + gitignore = true, + comment_type = "cpp" -Default: **cpp** +The details of each field are the followings -.. note:: Currently, only C/C++ is supported +src_dir +~~~~~~~ -.. tabs:: +Specifies the root directory for source file discovery. This path is resolved relative to the location of the TOML configuration file. - .. code-tab:: python +**Type:** ``str`` +**Default:** ``"./"`` - src_trace_projects = { - "project_name": { - "comment_type": "c" - } - } +.. code-block:: toml - .. code-tab:: toml + [source_discover] + src_dir = "../src" - [src_trace.projects.project_name] - comment_type = "c" +**Examples:** -.. _source_dir: +- ``"./"`` - Current directory (relative to config file) +- ``"../src"`` - Parent directory's src folder +- ``"./my_project/source"`` - Subdirectory within current directory -src_dir -------- +exclude +~~~~~~~ -The relative path from the ``conf.py`` or ``.toml`` file to the source code's root directory +Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. -Default: **./** +**Type:** ``list[str]`` +**Default:** ``[]`` -.. tabs:: +.. code-block:: toml - .. code-tab:: python + [source_discover] + exclude = [ + "build/**", + "*.tmp", + "tests/fixtures/**", + "vendor/third_party/**" + ] - src_trace_projects = { - "project_name": { - "src_dir": "./../src" - } - } +**Common exclusion patterns:** - .. code-tab:: toml +- ``"build/**"`` - Exclude entire build directory +- ``"*.o"`` - Exclude object files +- ``"**/__pycache__/**"`` - Exclude Python cache directories +- ``"node_modules/**"`` - Exclude Node.js dependencies - [src_trace.projects.project_name] - src_dir = "./../src" +include +~~~~~~~ -exclude -------- +Defines a list of glob patterns for files to explicitly include in discovery. When specified, only files matching these patterns will be processed, regardless of other filtering rules. -The option is a list of glob patterns to exclude the files which are not required to be addressed +**Type:** ``list[str]`` +**Default:** ``[]`` (include all files) -Default: **[]** +.. code-block:: toml -.. tabs:: + [source_discover] + include = [ + "src/**/*.cpp", + "src/**/*.h", + "include/**/*.hpp" + ] - .. code-tab:: python +**Priority:** The ``include`` option has the highest priority and overrides both ``exclude`` and ``gitignore`` settings. - src_trace_projects = { - "project_name": { - "exclude": ["dcdc/src/ubt/ubt.cpp"] - } - } +**Common inclusion patterns:** - .. code-tab:: toml +- ``"**/*.cpp"`` - Include all C++ source files +- ``"**/*.py"`` - Include all Python files +- ``"src/**"`` - Include everything in src directory +- ``"*.{c,h}"`` - Include C source and header files - [src_trace.projects.project_name] - exclude = ["dcdc/src/ubt/ubt.cpp"] +comment_type +~~~~~~~~~~~~ +Specifies the comment syntax style used in the source code files. This determines what file types are discovered and how **Sphinx-CodeLinks** parses comments for documentation extraction. -include -------- +**Type:** ``str`` +**Default:** ``"cpp"`` +**Supported values:** ``"cpp"``, ``"python"`` -The option is a list of glob patterns to include the files which are required to be addressed +.. code-block:: toml -Default: **[]** + [source_discover] + comment_type = "python" -.. tabs:: +**Supported comment styles:** - .. code-tab:: python +.. list-table:: + :header-rows: 1 + :widths: 20 30 50 - src_trace_projects = - { - "project_name": { - "include": ["dcdc/src/ubt/ubt.cpp"] - } - } + * - Language + - comment_type + - Comment Syntax + - discovered file types + * - C/C++ + - ``"cpp"`` + - ``//`` (single-line), ``/* */`` (multi-line) + - ``c``, ``h``, ``.cpp``, and ``.hpp`` + * - Python + - ``"python"`` + - ``#`` (single-line), ``""" """`` (docstrings) + - ``.py`` - .. code-tab:: toml +.. note:: + Future versions may support additional programming languages. Currently, only C/C++ and Python comment styles are supported. - [src_trace.projects.project_name] - include = ["dcdc/src/ubt/ubt.cpp"] +gitignore +~~~~~~~~~ -.. note:: **include** option has the highest priority over **exclude** and **gitignore** options. +Controls whether to respect ``.gitignore`` files when discovering source files. When enabled, files and directories listed in ``.gitignore`` will be automatically excluded from processing. -gitignore ---------- +**Type:** ``bool`` +**Default:** ``true`` + +.. code-block:: toml + + [source_discover] + gitignore = false + +**Behavior:** + +- ``true`` - Respect ``.gitignore`` rules (recommended) +- ``false`` - Ignore ``.gitignore`` files and process all matching files + +.. important:: + **Current Limitation:** This option only supports the root-level ``.gitignore`` file. Nested ``.gitignore`` files in subdirectories or parent directories are not currently processed. + +Usage Examples +-------------- + +**Basic Configuration:** -The option to respect the .gitignore file. +.. code-block:: toml -Default: **True** + [source_discover] + src_dir = "./src" + comment_type = "cpp" -.. tabs:: +**Advanced Filtering:** - .. code-tab:: python +.. code-block:: toml - src_trace_projects = { - "project_name": { - "gitignore": False - } + [source_discover] + src_dir = "./" + include = [] + exclude = ["src/legacy/**", "**/*_test.cpp"] + gitignore = true + comment_type = "cpp" - .. code-tab:: toml +**Python Project:** - [src_trace.projects.project_name] - gitignore = false +.. code-block:: toml -.. attention:: This option currently does NOT support nested .gitignore files + [source_discover] + src_dir = "./my_package" + include = [] + exclude = ["tests/**", "setup.py"] + comment_type = "python" From 56a504351a1cc7a6106247ad341ea82fe36fdd55 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 14 Aug 2025 10:20:56 +0200 Subject: [PATCH 03/21] used comment_type for src discover --- src/sphinx_codelinks/cmd.py | 36 ++++++++++--------- .../source_discover/config.py | 12 +++---- .../source_discover/source_discover.py | 14 +++----- tests/test_source_discover.py | 31 ++++++++++++---- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 1a65422..1a790c6 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -11,6 +11,7 @@ from sphinx_codelinks.analyse.analyse import SourceAnalyse from sphinx_codelinks.analyse.config import ( COMMENT_FILETYPE, + CommentType, MarkedRstConfig, MarkedRstConfigType, NeedIdRefsConfig, @@ -163,43 +164,46 @@ def discover( ), ], exclude: Annotated[ - list[str] | None, + list[str], typer.Option( "--excludes", "-e", help="Glob patterns to be excluded.", ), - ] = None, + ] = [], # noqa: B006 # to show the default value on CLI include: Annotated[ - list[str] | None, + list[str], typer.Option( "--includes", "-i", help="Glob patterns to be included.", ), - ] = None, - gitignore: Annotated[bool, typer.Option(help="Respect .gitignore(s)")] = True, - file_types: Annotated[ - list[str] | None, + ] = [], # noqa: B006 # to show the default value on CLI + gitignore: Annotated[ + bool, typer.Option( - "--file-type", - "-f", - help="The file extension to be discovered. If not specified, all files are discovered.", + help="Respect .gitignore in the given directory. Nested .gitignore Not supported" ), - ] = None, + ] = True, + comment_type: Annotated[ + CommentType, + typer.Option( + "--comment-type", + "-c", + help="The relevant file extensions which use the specified the comment type will be discovered.", + ), + ] = CommentType.cpp, ) -> None: """Discover the filepaths from the given root directory.""" src_discover_dict: SourceDiscoverConfigType = { "src_dir": src_dir, - "exclude": exclude or [], - "include": include or [], + "exclude": exclude, + "include": include, "gitignore": gitignore, + "comment_type": comment_type, } - if file_types is not None: - src_discover_dict["file_types"] = file_types - src_discover_config = SourceDiscoverConfig(**src_discover_dict) errors = src_discover_config.check_schema() diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index d37d854..c25bb47 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -4,7 +4,7 @@ from jsonschema import ValidationError, validate -from sphinx_codelinks.analyse.config import SUPPORTED_COMMENT_TYPES +from sphinx_codelinks.analyse.config import COMMENT_FILETYPE, CommentType class SourceDiscoverConfigType(TypedDict, total=False): @@ -12,7 +12,7 @@ class SourceDiscoverConfigType(TypedDict, total=False): exclude: list[str] include: list[str] gitignore: bool - file_types: list[str] + comment_type: CommentType @dataclass @@ -41,12 +41,12 @@ def field_names(cls) -> set[str]: gitignore: bool = field(default=True, metadata={"schema": {"type": "boolean"}}) """Whether to respect .gitignore to exclude files.""" - file_types: list[str] = field( - default_factory=lambda: list(SUPPORTED_COMMENT_TYPES), + comment_type: str = field( + default="cpp", metadata={ "schema": { - "type": "array", - "items": {"type": "string", "enum": sorted(SUPPORTED_COMMENT_TYPES)}, + "type": "string", + "enum": sorted(COMMENT_FILETYPE), } }, ) diff --git a/src/sphinx_codelinks/source_discover/source_discover.py b/src/sphinx_codelinks/source_discover/source_discover.py index d9e336c..26f7992 100644 --- a/src/sphinx_codelinks/source_discover/source_discover.py +++ b/src/sphinx_codelinks/source_discover/source_discover.py @@ -7,6 +7,7 @@ parse_gitignore, ) +from sphinx_codelinks.analyse.config import COMMENT_FILETYPE from sphinx_codelinks.source_discover.config import SourceDiscoverConfig @@ -22,16 +23,9 @@ def __init__(self, src_discover_config: SourceDiscoverConfig): else None ) # normalize the file types to lower case with leading dot - self.file_types = ( - { - file_type.lower() - if file_type.startswith(".") - else f".{file_type}".lower() - for file_type in src_discover_config.file_types - } - if src_discover_config.file_types - else None - ) + self.file_types = { + f".{ext}" for ext in COMMENT_FILETYPE[src_discover_config.comment_type] + } self.source_paths = self._discover() diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 486fb5e..fac02ad 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -18,7 +18,7 @@ "exclude": ["exclude1", "exclude2"], "include": ["include1", "include2"], "gitignore": True, - "file_types": ["cpp", "hpp"], + "comment_type": "cpp", }, ["Schema validation error in field 'src_dir': 123 is not of type 'string'"], ), @@ -28,7 +28,7 @@ "exclude": ["exclude1", "exclude2"], "include": ["include1", "include2"], "gitignore": "TrueAsString", - "file_types": ["cpp", "hpp"], + "comment_type": "cpp", }, [ "Schema validation error in field 'gitignore': 'TrueAsString' is not of type 'boolean'" @@ -40,10 +40,22 @@ "exclude": ["exclude1", "exclude2"], "include": ["include1", "include2"], "gitignore": True, - "file_types": "py", + "comment_type": "java", }, [ - "Schema validation error in field 'file_types': 'py' is not of type 'array'" + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'python']" + ], + ), + ( + { + "src_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "comment_type": ["cpp", "hpp"], + }, + [ + "Schema validation error in field 'comment_type': ['cpp', 'hpp'] is not of type 'string'" ], ), ], @@ -63,7 +75,14 @@ def test_schema_negative(config, msgs): "exclude": ["exclude1", "exclude2"], "include": ["include1", "include2"], "gitignore": True, - "file_types": ["cpp", "hpp"], + "comment_type": "cpp", + }, + { + "src_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "comment_type": "python", }, ], ) @@ -108,7 +127,7 @@ def test_schema_positive(config): "", ), ( - {"gitignore": False, "file_types": ["cpp"]}, + {"gitignore": False, "comment_type": "cpp"}, 4, "cpp", ), From 68f4044f8699a95cd0ff1574799e12d808883898 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 14 Aug 2025 18:39:02 +0200 Subject: [PATCH 04/21] applied new config setup --- docs/src_trace.toml | 10 +- src/sphinx_codelinks/analyse/config.py | 34 +-- src/sphinx_codelinks/analyse/utils.py | 3 +- src/sphinx_codelinks/cmd.py | 155 +++++------ .../source_discover/config.py | 20 +- .../source_discover/source_discover.py | 6 +- .../sphinx_extension/config.py | 246 +++++++++++------- .../sphinx_extension/directives/src_trace.py | 40 ++- .../sphinx_extension/source_tracing.py | 25 +- tests/data/analyse/default_config.toml | 3 - tests/data/sphinx/src_trace.toml | 8 +- tests/doc_test/recursive_dirs/src_trace.toml | 4 +- tests/test_cmd.py | 51 ++-- tests/test_src_trace.py | 91 ++++--- 14 files changed, 397 insertions(+), 299 deletions(-) delete mode 100644 tests/data/analyse/default_config.toml diff --git a/docs/src_trace.toml b/docs/src_trace.toml index b1d7d56..b1ef17f 100644 --- a/docs/src_trace.toml +++ b/docs/src_trace.toml @@ -12,7 +12,11 @@ remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit [src_trace.projects.dcdc.source_discover] src_dir = "../tests/data/dcdc" # Relative path from this TOML config to the source directory -[src_trace.projects.dcdc.oneline_comment_style] +[src_trace.projects.dcdc.analysis] +get_need_id_refs = false +get_oneline_needs = true + +[src_trace.projects.dcdc.analysis.oneline_comment_style] # Configuration for oneline comment style start_sequence = "[[" # Start sequence for oneline comments end_sequence = "]]" # End sequence for the online comments; default is newline character @@ -25,7 +29,3 @@ needs_fields = [ { "name" = "links", "type" = "list[str]", "default" = [ ] }, ] - -[src_trace.projects.src] -src_dir = "../tests/doc_test/minimum_config" -remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" diff --git a/src/sphinx_codelinks/analyse/config.py b/src/sphinx_codelinks/analyse/config.py index caad889..c94fa40 100644 --- a/src/sphinx_codelinks/analyse/config.py +++ b/src/sphinx_codelinks/analyse/config.py @@ -6,15 +6,14 @@ from jsonschema import ValidationError, validate -UNIX_NEWLINE = "\n" - +from sphinx_codelinks.source_discover.config import ( + CommentType, + SourceDiscoverSectionConfigType, +) -class CommentType(str, Enum): - python = "python" - cpp = "cpp" +UNIX_NEWLINE = "\n" -COMMENT_FILETYPE = {"cpp": ["c", "cpp", "h", "hpp"], "python": ["py"]} COMMENT_MARKERS = {CommentType.cpp: ["//", "/*"], CommentType.python: ["#"]} ESCAPE = "\\" SUPPORTED_COMMENT_TYPES = {"c", "h", "cpp", "hpp", "py"} @@ -25,14 +24,6 @@ class CommentCategory(str, Enum): docstring = "expression_statement" -class SrcDiscoverConfigType4Analyse(TypedDict, total=False): - src_dir: str - exclude: list[str] - include: list[str] - gitignore: bool - comment_type: list[CommentType] - - class NeedIdRefsConfigType(TypedDict): markers: list[str] @@ -298,8 +289,9 @@ def get_pos_list_str(self) -> list[int]: return pos_list_str -class SourceAnalyseConfigFileType(TypedDict, total=False): - src_discover: SrcDiscoverConfigType4Analyse +class AnalyseSectionConfigType(TypedDict, total=False): + """Define typing for loading `analyse` section from the file.""" + get_need_id_refs: bool get_oneline_needs: bool get_rst: bool @@ -309,7 +301,16 @@ class SourceAnalyseConfigFileType(TypedDict, total=False): oneline_comment_style: OneLineCommentStyleType +class SourceAnalyseConfigFileType(TypedDict, total=False): + """Define types to load TOML config file.""" + + src_discover: SourceDiscoverSectionConfigType + analyse: AnalyseSectionConfigType + + class SourceAnalyseConfigType(TypedDict, total=False): + """Define typing for its API configuration.""" + src_files: list[Path] src_dir: Path outdir: Path @@ -329,6 +330,7 @@ def field_names(cls) -> set[str]: return {item.name for item in fields(cls)} src_files: list[Path] = field( + default_factory=list, metadata={"schema": {"type": "array", "items": {"type": "string"}}}, ) """A list of source files to be processed.""" diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 925f86d..1fce95a 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -9,7 +9,8 @@ from tree_sitter import Language, Parser, Point, Query, QueryCursor from tree_sitter import Node as TreeSitterNode -from sphinx_codelinks.analyse.config import UNIX_NEWLINE, CommentCategory, CommentType +from sphinx_codelinks.analyse.config import UNIX_NEWLINE, CommentCategory +from sphinx_codelinks.source_discover.config import CommentType # initialize logger logger = logging.getLogger(__name__) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 1a790c6..728087a 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -10,8 +10,7 @@ from sphinx_codelinks.analyse.analyse import SourceAnalyse from sphinx_codelinks.analyse.config import ( - COMMENT_FILETYPE, - CommentType, + AnalyseSectionConfigType, MarkedRstConfig, MarkedRstConfigType, NeedIdRefsConfig, @@ -21,14 +20,15 @@ SourceAnalyseConfig, SourceAnalyseConfigFileType, SourceAnalyseConfigType, - SrcDiscoverConfigType4Analyse, ) from sphinx_codelinks.source_discover.config import ( + CommentType, SourceDiscoverConfig, SourceDiscoverConfigType, + SourceDiscoverSectionConfigType, ) from sphinx_codelinks.source_discover.source_discover import SourceDiscover -from sphinx_codelinks.sphinx_extension.config import SrcTraceProjectConfigFileType +from sphinx_codelinks.sphinx_extension.config import SrcTraceProjectConfigType app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} @@ -66,12 +66,13 @@ def analyse( errors: deque[str] = deque() # Get source_discover configuration - src_discover_4_analyse_dict = cast( - SrcDiscoverConfigType4Analyse | None, data.get("source_discover") + src_discover_section_config: SourceDiscoverSectionConfigType | None = cast( + SourceDiscoverSectionConfigType | None, data.get("source_discover") ) + src_discover_config = cast( SourceDiscoverConfig, - convert_dict_2_config(src_discover_4_analyse_dict, ConfigType.SourceDiscover), + convert_dict_2_config(src_discover_section_config, ConfigType.SourceDiscover), ) src_discover_errors = src_discover_config.check_schema() @@ -88,57 +89,19 @@ def analyse( ).resolve() src_discover = SourceDiscover(src_discover_config) - # Get oneline_comment_style configuration - oneline_comment_style_dict: OneLineCommentStyleType | None = data.get( - "oneline_comment_style" - ) - oneline_comment_style: OneLineCommentStyle = cast( - OneLineCommentStyle, - convert_dict_2_config( - oneline_comment_style_dict, ConfigType.OneLineCommentStyle - ), - ) - oneline_errors = oneline_comment_style.check_fields_configuration() - if oneline_errors: - errors.appendleft("Invalid oneline comment style configuration:") - errors.extend(oneline_errors) + # Init source analyse config + analyse_config_dict: SourceAnalyseConfigType = {} + analyse_config_dict["src_files"] = src_discover.source_paths + analyse_config_dict["src_dir"] = Path(src_discover.src_discover_config.src_dir) - # Get need_id_refs configuration - need_id_refs_config_dict: NeedIdRefsConfigType | None = data.get("need_id_refs") - need_id_refs_config = cast( - NeedIdRefsConfig, - convert_dict_2_config(need_id_refs_config_dict, ConfigType.NeedIdRefsConfig), - ) + # Get config for `analyse` section + analyse_section_config: AnalyseSectionConfigType | None = data.get("analyse") - # Get marked_rst configuration - marked_rst_config_dict: MarkedRstConfigType | None = data.get("marked_rst") - marked_rst_config = cast( - MarkedRstConfig, - convert_dict_2_config(marked_rst_config_dict, ConfigType.MarkedRstCofig), - ) + src_analyse_config = convert_analyse_config(analyse_section_config, src_discover) - non_root_configs = { - "source_discover", - "oneline_comment_style", - "need_id_refs", - "marked_rst", - } - - # Get root config - src_analyse_dict: SourceAnalyseConfigType = cast( - SourceAnalyseConfigType, - {key: value for key, value in data.items() if key not in non_root_configs}, - ) - src_analyse_dict["src_files"] = src_discover.source_paths - src_analyse_dict["src_dir"] = Path(src_discover.src_discover_config.src_dir) - src_analyse_dict["need_id_refs_config"] = need_id_refs_config - src_analyse_dict["marked_rst_config"] = marked_rst_config - src_analyse_dict["oneline_comment_style"] = oneline_comment_style if outdir: - src_analyse_dict["outdir"] = outdir - - src_analyse_config = SourceAnalyseConfig(**src_analyse_dict) + src_analyse_config.outdir = outdir analyse_errors = src_analyse_config.check_fields_configuration() errors.extend(analyse_errors) @@ -218,7 +181,7 @@ def discover( def load_config_from_toml( toml_file: Path, project: str | None = None -) -> SrcTraceProjectConfigFileType: +) -> SrcTraceProjectConfigType: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) @@ -231,7 +194,7 @@ def load_config_from_toml( f"Failed to load source tracing configuration from {toml_file}" ) from e - return cast(SrcTraceProjectConfigFileType, toml_data) + return cast(SrcTraceProjectConfigType, toml_data) def load_src_analyse_config_from_toml(toml_file: Path) -> SourceAnalyseConfigFileType: @@ -252,11 +215,11 @@ class ConfigType(str, Enum): OneLineCommentStyle = "oneline_comment_style" NeedIdRefsConfig = "need_id_refs" MarkedRstCofig = "marked_rst" - AnalzerConfig = "source_analyse" + Analyse = "source_analyse" def convert_dict_2_config( - config_dict: SrcDiscoverConfigType4Analyse + config_dict: SourceDiscoverSectionConfigType | OneLineCommentStyleType | NeedIdRefsConfigType | MarkedRstConfigType @@ -267,7 +230,7 @@ def convert_dict_2_config( ConfigType, Callable[ [ - SrcDiscoverConfigType4Analyse + SourceDiscoverSectionConfigType | OneLineCommentStyleType | NeedIdRefsConfigType | MarkedRstConfigType @@ -291,30 +254,74 @@ def convert_dict_2_config( def convert_src_discovery_config( - config_dict: SrcDiscoverConfigType4Analyse | None, + config_dict: SourceDiscoverSectionConfigType | None, ) -> SourceDiscoverConfig: if config_dict: - src_dicover_dict = {} - for key, value in config_dict.items(): - if key == "comment_type" and isinstance(value, list): - file_types = [] - for comment_type in value: - file_types.extend(COMMENT_FILETYPE[comment_type]) - src_dicover_dict["file_types"] = file_types - else: - src_dicover_dict[key] = value # type: ignore[assignment] # dynamic assignment - src_dicover_dict["src_dir"] = ( - Path(src_dicover_dict["src_dir"]) - if isinstance(src_dicover_dict["src_dir"], str) - else src_dicover_dict["src_dir"] - ) - src_discover_config = SourceDiscoverConfig(**src_dicover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment + src_discover_dict = { + key: (Path(str(value)) if key == "src_dir" else value) + for key, value in config_dict.items() + } + src_discover_config = SourceDiscoverConfig(**src_discover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment else: src_discover_config = SourceDiscoverConfig() return src_discover_config +def convert_analyse_config( + config_dict: AnalyseSectionConfigType | None, + src_discover: SourceDiscover | None = None, +) -> SourceAnalyseConfig: + if config_dict: + analyse_config_dict: SourceAnalyseConfigType = cast( + SourceAnalyseConfigType, + {k: Path(str(v)) if k == "src_dir" else v for k, v in config_dict.items()}, + ) + + # Get oneline_comment_style configuration + oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( + "oneline_comment_style" + ) + oneline_comment_style: OneLineCommentStyle = cast( + OneLineCommentStyle, + convert_dict_2_config( + oneline_comment_style_dict, ConfigType.OneLineCommentStyle + ), + ) + + # Get need_id_refs configuration + need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get( + "need_id_refs" + ) + need_id_refs_config = cast( + NeedIdRefsConfig, + convert_dict_2_config( + need_id_refs_config_dict, ConfigType.NeedIdRefsConfig + ), + ) + + # Get marked_rst configuration + marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get( + "marked_rst" + ) + marked_rst_config = cast( + MarkedRstConfig, + convert_dict_2_config(marked_rst_config_dict, ConfigType.MarkedRstCofig), + ) + + analyse_config_dict["need_id_refs_config"] = need_id_refs_config + analyse_config_dict["marked_rst_config"] = marked_rst_config + analyse_config_dict["oneline_comment_style"] = oneline_comment_style + else: + analyse_config_dict: SourceAnalyseConfigType = {} + + if src_discover: + analyse_config_dict["src_files"] = src_discover.source_paths + analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir + + return SourceAnalyseConfig(**analyse_config_dict) + + def convert_oneline_comment_style_config( config_dict: OneLineCommentStyleType | None, ) -> OneLineCommentStyle: diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index c25bb47..687f77a 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -1,13 +1,31 @@ from dataclasses import MISSING, dataclass, field, fields +from enum import Enum from pathlib import Path from typing import Any, Required, TypedDict, cast from jsonschema import ValidationError, validate -from sphinx_codelinks.analyse.config import COMMENT_FILETYPE, CommentType +COMMENT_FILETYPE = {"cpp": ["c", "cpp", "h", "hpp"], "python": ["py"]} + + +class CommentType(str, Enum): + python = "python" + cpp = "cpp" + + +class SourceDiscoverSectionConfigType(TypedDict, total=False): + """Define typing for loading configuration from TOML files""" + + src_dir: Required[str] + exclude: list[str] + include: list[str] + gitignore: bool + comment_type: CommentType class SourceDiscoverConfigType(TypedDict, total=False): + """Define typing for its API configuration""" + src_dir: Required[Path] exclude: list[str] include: list[str] diff --git a/src/sphinx_codelinks/source_discover/source_discover.py b/src/sphinx_codelinks/source_discover/source_discover.py index 26f7992..84c5563 100644 --- a/src/sphinx_codelinks/source_discover/source_discover.py +++ b/src/sphinx_codelinks/source_discover/source_discover.py @@ -7,8 +7,10 @@ parse_gitignore, ) -from sphinx_codelinks.analyse.config import COMMENT_FILETYPE -from sphinx_codelinks.source_discover.config import SourceDiscoverConfig +from sphinx_codelinks.source_discover.config import ( + COMMENT_FILETYPE, + SourceDiscoverConfig, +) class SourceDiscover: diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 9310981..77d0cda 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -7,17 +7,21 @@ from sphinx.config import Config as _SphinxConfig from sphinx_codelinks.analyse.config import ( - SUPPORTED_COMMENT_TYPES, - CommentType, + AnalyseSectionConfigType, + MarkedRstConfig, MarkedRstConfigType, + NeedIdRefsConfig, NeedIdRefsConfigType, OneLineCommentStyle, OneLineCommentStyleType, + SourceAnalyseConfig, + SourceAnalyseConfigType, ) from sphinx_codelinks.source_discover.config import ( SourceDiscoverConfig, - SourceDiscoverConfigType, + SourceDiscoverSectionConfigType, ) +from sphinx_codelinks.source_discover.source_discover import SourceDiscover SRC_TRACE_CACHE: str = "src_trace_cache" @@ -32,28 +36,23 @@ def __init__(self) -> None: file_lineno_href = SourceTracingLineHref() -class SrcTraceProjectConfigFileType(TypedDict, total=False): - # only support C/C++ for now - comment_type: CommentType - src_dir: str - remote_url_pattern: str - exclude: list[str] - include: list[str] - gitignore: bool - oneline_comment_style: OneLineCommentStyleType - need_id_refs: NeedIdRefsConfigType - marked_rst: MarkedRstConfigType - - -class SrcTraceProjectConfigType(TypedDict): - # only support C/C++ for now - comment_type: CommentType - src_dir: Path +class SrcTraceProjectConfigType(TypedDict, total=False): + """TypedDict defining the configuration structure for individual SrcTrace projects. + + Contains both user-provided configuration: + - source_discover + - remote_url_pattern + - analyse + and runtime-generated configuration objects + - source_discover_config + - analyse_config + """ + + source_discover: SourceDiscoverSectionConfigType remote_url_pattern: str - exclude: list[str] - include: list[str] - gitignore: bool - oneline_comment_style: OneLineCommentStyle + analyse: AnalyseSectionConfigType + source_discover_config: SourceDiscoverConfig + analyse_config: SourceAnalyseConfig class SrcTraceConfigType(TypedDict): @@ -82,18 +81,8 @@ def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-a src_trace_projects: dict[str, SrcTraceProjectConfigType] = value[ "src_trace_projects" ] - for _config in src_trace_projects.values(): - # overwrite the config into different types on purpose - # covert dict to OneLineCommentStyle class - oneline_comment_style: OneLineCommentStyleType | None = cast( - OneLineCommentStyleType, _config.get("oneline_comment_style") - ) - if not oneline_comment_style: - raise Exception("OneLineCommentStyle is not given") - - _config["oneline_comment_style"] = OneLineCommentStyle( - **oneline_comment_style - ) + generate_project_configs(src_trace_projects) + if name.startswith("__") or name == "_config": return super().__setattr__(name, value) @@ -199,13 +188,11 @@ def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explici "additionalProperties": { "type": "object", "properties": { - "comment_type": {}, - "src_dir": {}, + "source_discover": {}, + "analyse": {}, "remote_url_pattern": {}, - "exclude": {}, - "include": {}, - "gitignore": {}, - "oneline_comment_style": {}, + "source_discover_config": {}, + "analyse_config": {}, }, "additionalProperties": False, }, @@ -225,6 +212,7 @@ def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explici def check_schema(config: SrcTraceSphinxConfig) -> list[str]: + """Check only first layer's of schema, so that the nested dict is not validated here.""" errors = [] for _field_name in SrcTraceSphinxConfig.field_names(): schema = SrcTraceSphinxConfig.get_schema(_field_name) @@ -241,17 +229,29 @@ def check_schema(config: SrcTraceSphinxConfig) -> list[str]: def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: + """Check nested project configurations""" errors = [] for project_name, project_config in config.projects.items(): project_errors: list[str] = [] - oneline_errors = validate_oneline_comment_style(project_config) - src_discover_dict = build_src_discover_dict(project_config) + + # validate source_discover config + src_discover_config: SourceDiscoverConfig | None = project_config.get( + "source_discover_config" + ) src_discover_errors = [] - if src_discover_dict is not None: - src_discover_config = SourceDiscoverConfig(**src_discover_dict) + if src_discover_config: src_discover_errors.extend(src_discover_config.check_schema()) + # validate analyse config + analyse_config: SourceAnalyseConfig | None = project_config.get( + "analyse_config" + ) + analyse_errors = [] + if analyse_config: + analyse_errors = analyse_config.check_fields_configuration() + + # validate src-trace config if config.set_remote_url and "remote_url_pattern" not in project_config: project_errors.append( "remote_url_pattern must be given, as set_remote_url is enabled" @@ -262,9 +262,9 @@ def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: ): project_errors.append("remote_url_pattern must be a string") - if oneline_errors or src_discover_errors or project_errors: + if analyse_errors or src_discover_errors or project_errors: errors.append(f"Project '{project_name}' has the following errors:") - errors.extend(oneline_errors) + errors.extend(analyse_errors) errors.extend(src_discover_errors) errors.extend(project_errors) @@ -278,52 +278,120 @@ def check_configuration(config: SrcTraceSphinxConfig) -> list[str]: return errors -def validate_oneline_comment_style( - project_config: SrcTraceProjectConfigType, -) -> list[str]: - if "oneline_comment_style" in project_config: - style = project_config["oneline_comment_style"] - if isinstance(style, OneLineCommentStyle): - return style.check_fields_configuration() - return [] - - -def build_src_discover_dict( - project_config: SrcTraceProjectConfigType, -) -> SourceDiscoverConfigType | None: - src_discover_dict = cast(SourceDiscoverConfigType, {}) - # adapt the configs between source tracing and source discovery - if "comment_type" in project_config: - # comment type error will be taken care by SourceDiscovery class later - src_discover_dict["file_types"] = ( - list(SUPPORTED_COMMENT_TYPES) - if project_config["comment_type"] in SUPPORTED_COMMENT_TYPES - else [project_config["comment_type"]] +def convert_src_discovery_config( + config_dict: SourceDiscoverSectionConfigType | None, +) -> SourceDiscoverConfig: + if config_dict: + src_discover_dict = { + key: (Path(value) if key == "src_dir" and isinstance(value, str) else value) + for key, value in config_dict.items() + } + src_discover_config = SourceDiscoverConfig(**src_discover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment + else: + src_discover_config = SourceDiscoverConfig() + + return src_discover_config + + +def convert_analyse_config( + config_dict: AnalyseSectionConfigType | None, + src_discover: SourceDiscover | None = None, +) -> SourceAnalyseConfig: + if config_dict: + analyse_config_dict: SourceAnalyseConfigType = cast( + SourceAnalyseConfigType, + {k: Path(str(v)) if k == "src_dir" else v for k, v in config_dict.items()}, ) - for key in ("exclude", "include", "gitignore", "src_dir"): - if key in project_config: - src_discover_dict[key] = project_config[key] - return src_discover_dict + # Get oneline_comment_style configuration + oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( + "oneline_comment_style" + ) + oneline_comment_style: OneLineCommentStyle = ( + convert_oneline_comment_style_config(oneline_comment_style_dict) + ) + + # Get need_id_refs configuration + need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get( + "need_id_refs" + ) + need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict) + # Get marked_rst configuration + marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get( + "marked_rst" + ) + marked_rst_config = convert_marked_rst_config(marked_rst_config_dict) -def adpat_src_discover_config(project_config: SrcTraceProjectConfigType) -> None: - src_discover_dict = build_src_discover_dict(project_config) - if src_discover_dict: - src_discover_config = SourceDiscoverConfig(**src_discover_dict) + analyse_config_dict["need_id_refs_config"] = need_id_refs_config + analyse_config_dict["marked_rst_config"] = marked_rst_config + analyse_config_dict["oneline_comment_style"] = oneline_comment_style else: - src_discover_config = SourceDiscoverConfig() + analyse_config_dict: SourceAnalyseConfigType = {} - for _field in fields(src_discover_config): - key = "comment_type" if _field.name == "file_types" else _field.name + if src_discover: + analyse_config_dict["src_files"] = src_discover.source_paths + analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir - if key == "comment_type": - file_types = getattr(src_discover_config, _field.name) - if set(file_types) == SUPPORTED_COMMENT_TYPES: - comment_type = CommentType.cpp - else: - comment_type = file_types[0] - project_config[key] = comment_type # type: ignore[literal-required] # dynamically assign - continue + return SourceAnalyseConfig(**analyse_config_dict) - project_config[key] = getattr(src_discover_config, _field.name) # type: ignore[literal-required] # dynamically assign + +def convert_oneline_comment_style_config( + config_dict: OneLineCommentStyleType | None, +) -> OneLineCommentStyle: + if config_dict is None: + oneline_comment_style = OneLineCommentStyle() + else: + try: + oneline_comment_style = OneLineCommentStyle(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return oneline_comment_style + + +def convert_need_id_refs_config( + config_dict: NeedIdRefsConfigType | None, +) -> NeedIdRefsConfig: + if not config_dict: + need_id_refs_config = NeedIdRefsConfig() + else: + try: + need_id_refs_config = NeedIdRefsConfig(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return need_id_refs_config + + +def convert_marked_rst_config( + config_dict: MarkedRstConfigType | None, +) -> MarkedRstConfig: + if not config_dict: + marked_rst_config = MarkedRstConfig() + else: + try: + marked_rst_config = MarkedRstConfig(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return marked_rst_config + + +def generate_project_configs( + project_configs: dict[str, SrcTraceProjectConfigType], +) -> None: + """Generate configs of source discover and analyse from their classes dynamically.""" + for project_config in project_configs.values(): + # overwrite the config into different types on purpose + # covert dicts to their own classes + src_discover_section: SourceDiscoverSectionConfigType | None = cast( + SourceDiscoverSectionConfigType, + project_config.get("source_discover"), + ) + source_discover_config = convert_src_discovery_config(src_discover_section) + project_config["source_discover_config"] = source_discover_config + + analyse_section_config: AnalyseSectionConfigType | None = cast( + AnalyseSectionConfigType, project_config.get("analyse") + ) + analyse_config = convert_analyse_config(analyse_section_config) + analyse_config.get_oneline_needs = True # force to get oneline_need + project_config["analyse_config"] = analyse_config diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index 65ed091..923bc35 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -12,7 +12,6 @@ from sphinx_needs.utils import add_doc # type: ignore[import-untyped] from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.analyse.config import COMMENT_FILETYPE, SourceAnalyseConfig from sphinx_codelinks.analyse.models import OneLineNeed from sphinx_codelinks.source_discover.config import SourceDiscoverConfig from sphinx_codelinks.source_discover.source_discover import SourceDiscover @@ -93,34 +92,24 @@ def run(self) -> list[nodes.Node]: src_trace_conf: SrcTraceProjectConfigType = src_trace_sphinx_config.projects[ project ] - comment_type = src_trace_conf["comment_type"] - oneline_comment_style = src_trace_conf["oneline_comment_style"] - - src_dir = self.locate_src_dir(src_trace_sphinx_config, src_trace_conf) + src_discover_config = src_trace_conf["source_discover_config"] + src_dir = self.locate_src_dir(src_trace_sphinx_config, src_discover_config) out_dir = Path(self.env.app.outdir) # the directory where the source files are copied to target_dir = out_dir / src_dir.name extra_options = {"project": project} - source_files = self.get_src_files(self.options, src_dir, src_trace_conf) + source_files = self.get_src_files(self.options, src_dir, src_discover_config) # add source files into the dependency # https://www.sphinx-doc.org/en/master/extdev/envapi.html#sphinx.environment.BuildEnvironment.note_dependency for source_file in source_files: self.env.note_dependency(str(source_file.resolve())) - analyse_config = SourceAnalyseConfig( - source_files, - src_dir, - out_dir, - comment_type, - get_need_id_refs=False, - get_oneline_needs=True, - get_rst=False, - oneline_comment_style=oneline_comment_style, - ) - + analyse_config = src_trace_conf["analyse_config"] + analyse_config.src_dir = src_dir + analyse_config.src_files = source_files src_analyse = SourceAnalyse(analyse_config) src_analyse.run() @@ -204,8 +193,9 @@ def get_src_files( self, extra_options: dict[str, str], src_dir: Path, - src_trace_conf: SrcTraceProjectConfigType, + src_discover_config: SourceDiscoverConfig, ) -> list[Path]: + """Leverage SourceDiscover to find sources files from the given directory.""" source_files = [] if "file" in self.options: file: str = self.options["file"] @@ -220,13 +210,13 @@ def get_src_files( else: extra_options["directory"] = directory dir_path = src_dir / directory - file_types = COMMENT_FILETYPE[src_trace_conf["comment_type"]] + # create a new config for the specified directory src_discover = SourceDiscoverConfig( dir_path, - gitignore=src_trace_conf["gitignore"], - include=src_trace_conf["include"], - exclude=src_trace_conf["exclude"], - file_types=file_types, + gitignore=src_discover_config.gitignore, + include=src_discover_config.include, + exclude=src_discover_config.exclude, + comment_type=src_discover_config.comment_type, ) source_discover = SourceDiscover(src_discover) source_files.extend(source_discover.source_paths) @@ -236,7 +226,7 @@ def get_src_files( def locate_src_dir( self, src_trace_sphinx_config: SrcTraceSphinxConfig, - src_trace_conf: SrcTraceProjectConfigType, + src_discover_config: SourceDiscoverConfig, ) -> Path: """Locate the source directory based on the configuration.""" # src dir in src_trace_conf is relative to conf_dir by default @@ -246,7 +236,7 @@ def locate_src_dir( src_trace_toml_path = Path(src_trace_sphinx_config.config_from_toml) conf_dir = conf_dir / src_trace_toml_path.parent - src_dir = (conf_dir / src_trace_conf["src_dir"]).resolve() + src_dir = (conf_dir / src_discover_config.src_dir).resolve() return src_dir def render_needs( diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index e687cb3..2c6e75a 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -16,16 +16,15 @@ ) from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.analyse.config import OneLineCommentStyle, OneLineCommentStyleType from sphinx_codelinks.sphinx_extension import debug from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, SrcTraceConfigType, SrcTraceProjectConfigType, SrcTraceSphinxConfig, - adpat_src_discover_config, check_configuration, file_lineno_href, + generate_project_configs, ) from sphinx_codelinks.sphinx_extension.directives.src_trace import ( SourceTracing, @@ -150,26 +149,10 @@ def set_config_to_sphinx( if key not in allowed_keys: continue if key == "projects": - for project_config in cast( + src_trace_projects: dict[str, SrcTraceProjectConfigType] = cast( dict[str, SrcTraceProjectConfigType], value - ).values(): - # address SourceDiscovery related config - adpat_src_discover_config(project_config) - - # address OneLoneCommenyStyle config and its default - oneline_comment_style: OneLineCommentStyleType | None = cast( - OneLineCommentStyleType, project_config.get("oneline_comment_style") - ) - if oneline_comment_style: - project_config["oneline_comment_style"] = OneLineCommentStyle( - **cast( - OneLineCommentStyleType, - project_config["oneline_comment_style"], - ) - ) - else: - project_config["oneline_comment_style"] = OneLineCommentStyle() - + ) + generate_project_configs(src_trace_projects) config[f"src_trace_{key}"] = value diff --git a/tests/data/analyse/default_config.toml b/tests/data/analyse/default_config.toml deleted file mode 100644 index 24117a8..0000000 --- a/tests/data/analyse/default_config.toml +++ /dev/null @@ -1,3 +0,0 @@ -[source_discover] -src_dir = "../../data" -gitignore = false diff --git a/tests/data/sphinx/src_trace.toml b/tests/data/sphinx/src_trace.toml index cba8aa2..3b02e1e 100644 --- a/tests/data/sphinx/src_trace.toml +++ b/tests/data/sphinx/src_trace.toml @@ -6,14 +6,18 @@ remote_url_field = "remote-url" debug_measurement = true [src_trace.projects.dcdc] + +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +[src_trace.projects.dcdc.source_discover] comment_type = "cpp" src_dir = "../dcdc" -remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" exclude = ["dcdc/src/ubt/ubt.cpp"] include = ["**/*.cpp", "**/*.hpp"] gitignore = true -[src_trace.projects.dcdc.oneline_comment_style] + +[src_trace.projects.dcdc.analyse.oneline_comment_style] start_sequence = "[[" end_sequence = "]]" # default is newline character field_split_char = "," diff --git a/tests/doc_test/recursive_dirs/src_trace.toml b/tests/doc_test/recursive_dirs/src_trace.toml index fc921f4..77c5ac3 100644 --- a/tests/doc_test/recursive_dirs/src_trace.toml +++ b/tests/doc_test/recursive_dirs/src_trace.toml @@ -6,9 +6,11 @@ remote_url_field = "remote-url" debug_measurement = true [src_trace.projects.dummy_src] +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +[src_trace.projects.dummy_src.source_discover] comment_type = "cpp" src_dir = "./dummy_src_lv1" -remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" exclude = ["dcdc/src/ubt/ubt.cpp"] include = ["**/*.cpp", "**/*.hpp"] gitignore = true diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 0f6e9dc..f592fc7 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -5,8 +5,8 @@ import toml from typer.testing import CliRunner -from sphinx_codelinks.analyse.config import CommentType from sphinx_codelinks.cmd import app +from sphinx_codelinks.source_discover.config import CommentType from .conftest import DATA_DIR, TEST_DIR @@ -25,19 +25,22 @@ "exclude": ["**/charge/demo_1.cpp", "**/discharge/demo_3.cpp"], "include": ["**/charge/demo_2.cpp", "**/supercharge.cpp"], "gitignore": True, - "comment_type": [CommentType.cpp.value], + "comment_type": CommentType.cpp.value, } -ANALYSER_CONFIG_TEMPLATE = { - "source_discover": SRC_DISCOVER_TEMPLATE, +ANALYSE_SECTION_CONFIG_TEMPLATE = { "oneline_comment_style": ONELINE_COMMENT_TEMPLATE, } +ANALYSE_CONFIG_TEMPLATE = { + "source_discover": SRC_DISCOVER_TEMPLATE, + "analyse": ANALYSE_SECTION_CONFIG_TEMPLATE, +} runner = CliRunner() @pytest.mark.parametrize( - ("config_path"), [(DATA_DIR / "analyse" / "default_config.toml")] + ("config_path"), [(DATA_DIR / "analyse" / "minimum_config.toml")] ) def test_analyse(config_path: Path, tmp_path: Path) -> None: options: list[str] = ["analyse", str(config_path), "--outdir", str(tmp_path)] @@ -84,7 +87,7 @@ def test_discover(options, stdout): } if isinstance(value, dict) and key == "source_discover" else value - for key, value in ANALYSER_CONFIG_TEMPLATE.items() + for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid source discovery configuration:", @@ -99,7 +102,7 @@ def test_discover(options, stdout): } if isinstance(value, dict) and key == "source_discover" else value - for key, value in ANALYSER_CONFIG_TEMPLATE.items() + for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid source discovery configuration:", @@ -118,7 +121,7 @@ def test_discover(options, stdout): } if isinstance(value, dict) and key == "source_discover" else value - for key, value in ANALYSER_CONFIG_TEMPLATE.items() + for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid source discovery configuration:", @@ -129,10 +132,17 @@ def test_discover(options, stdout): ), ( { - key: ( - {"not_expected": 123} if key == "oneline_comment_style" else value - ) - for key, value in ANALYSER_CONFIG_TEMPLATE.items() + key: { + oneline_key: ( + {"not_expected": 123} + if oneline_key == "oneline_comment_style" + else oneline_value + ) + for oneline_key, oneline_value in value.items() + } + if isinstance(value, dict) and key == "analyse" + else value + for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid oneline comment style configuration:", @@ -142,12 +152,17 @@ def test_discover(options, stdout): ), ( { - key: ( - {"needs_fields": [{"name": "id"}, {"name": "id"}]} - if key == "oneline_comment_style" - else value - ) - for key, value in ANALYSER_CONFIG_TEMPLATE.items() + key: { + oneline_key: ( + {"needs_fields": [{"name": "id"}, {"name": "id"}]} + if oneline_key == "oneline_comment_style" + else oneline_value + ) + for oneline_key, oneline_value in value.items() + } + if isinstance(value, dict) and key == "analyse" + else value + for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid oneline comment style configuration:", diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 8b98287..b048e46 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -25,27 +25,31 @@ "set_remote_url": "TrueString", "projects": { "dcdc": { - "comment_type": "java", - "src_dir": ["../dcdc"], "remote_url_pattern": 44332, - "exclude": [123], - "include": [345], - "gitignore": "_true", - "oneline_comment_style": { - "start_sequence": "[[", - "end_sequence": "]]", - "field_split_char": ",", - "needs_fields": [ - { - "name": "title", - "type": "list[]", - }, - { - "name": "type", - "default": "impl", - "type": "str", - }, - ], + "source_discover": { + "comment_type": "java", + "src_dir": ["../dcdc"], + "exclude": [123], + "include": [345], + "gitignore": "_true", + }, + "analyse": { + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "list[]", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, }, } }, @@ -53,7 +57,7 @@ [ "Project 'dcdc' has the following errors:", "Schema validation error in field 'exclude': 123 is not of type 'string'", - "Schema validation error in field 'file_types': 'java' is not one of ['c', 'cpp', 'h', 'hpp', 'py']", + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'python']", "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", "Schema validation error in field 'include': 345 is not of type 'string'", "Schema validation error in field 'src_dir': ['../dcdc'] is not of type 'string'", @@ -61,6 +65,7 @@ "Schema validation error in filed 'remote_url_field': 555 is not of type 'string'", "Schema validation error in filed 'set_local_url': 'fdd' is not of type 'boolean'", "Schema validation error in filed 'set_remote_url': 'TrueString' is not of type 'boolean'", + "OneLineCommentStyle configuration errors:", "Schema validation error in need_fields 'title': 'list[]' is not one of ['str', 'list[str]']", "remote_url_pattern must be a string", ], @@ -73,27 +78,31 @@ "set_remote_url": True, "projects": { "dcdc": { - "comment_type": "cpp", - "src_dir": "../dcdc", # intentionally not given "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", - "exclude": [], - "include": [], - "gitignore": True, - "oneline_comment_style": { - "start_sequence": "[[", - "end_sequence": "]]", - "field_split_char": ",", - "needs_fields": [ - { - "name": "title", - "type": "str", - }, - { - "name": "type", - "default": "impl", - "type": "str", - }, - ], + "source_discover": { + "comment_type": "cpp", + "src_dir": "../dcdc", + "exclude": [], + "include": [], + "gitignore": True, + }, + "analyse": { + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, }, } }, From 49bb00c230de1b3b1b3801f5178879f2d693e2f3 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 14 Aug 2025 18:57:23 +0200 Subject: [PATCH 05/21] fixed TC --- tests/test_src_trace.py | 44 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index b048e46..5953ae5 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -138,27 +138,31 @@ def test_src_tracing_config_positive( "set_remote_url": True, "projects": { "dcdc": { - "comment_type": "cpp", - "src_dir": "../dcdc", + "source_discover": { + "comment_type": "cpp", + "src_dir": "../dcdc", + "exclude": ["**/*.hpp"], + "include": ["**/*.cpp"], + "gitignore": True, + }, "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", - "exclude": ["**/*.hpp"], - "include": ["**/*.cpp"], - "gitignore": True, - "oneline_comment_style": { - "start_sequence": "[[", - "end_sequence": "]]", - "field_split_char": ",", - "needs_fields": [ - { - "name": "title", - "type": "str", - }, - { - "name": "type", - "default": "impl", - "type": "str", - }, - ], + "analyse": { + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, }, } }, From 6aef13344250e3ba4ed759fc4ad5d2f588bc3874 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 14 Aug 2025 22:25:35 +0200 Subject: [PATCH 06/21] fixed TC and adapted CLI --- src/sphinx_codelinks/cmd.py | 215 +++--------------- .../sphinx_extension/config.py | 10 +- tests/test_cmd.py | 3 +- 3 files changed, 34 insertions(+), 194 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 728087a..95fbb5a 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -1,6 +1,4 @@ from collections import deque -from collections.abc import Callable -from enum import Enum from os import linesep from pathlib import Path import tomllib @@ -11,13 +9,6 @@ from sphinx_codelinks.analyse.analyse import SourceAnalyse from sphinx_codelinks.analyse.config import ( AnalyseSectionConfigType, - MarkedRstConfig, - MarkedRstConfigType, - NeedIdRefsConfig, - NeedIdRefsConfigType, - OneLineCommentStyle, - OneLineCommentStyleType, - SourceAnalyseConfig, SourceAnalyseConfigFileType, SourceAnalyseConfigType, ) @@ -28,7 +19,11 @@ SourceDiscoverSectionConfigType, ) from sphinx_codelinks.source_discover.source_discover import SourceDiscover -from sphinx_codelinks.sphinx_extension.config import SrcTraceProjectConfigType +from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceProjectConfigType, + convert_analyse_config, + convert_src_discovery_config, +) app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} @@ -47,6 +42,15 @@ def analyse( exists=True, ), ], + project: Annotated[ + str | None, + typer.Option( + "--project", + "-p", + help="Specify the project name of the config when using src-trace config.", + show_default=True, + ), + ] = None, outdir: Annotated[ Path | None, typer.Option( @@ -61,7 +65,7 @@ def analyse( ] = None, ) -> None: """Analyse marked content in source code.""" - data = load_src_analyse_config_from_toml(config) + data = load_config_from_toml(config, project) errors: deque[str] = deque() @@ -70,10 +74,7 @@ def analyse( SourceDiscoverSectionConfigType | None, data.get("source_discover") ) - src_discover_config = cast( - SourceDiscoverConfig, - convert_dict_2_config(src_discover_section_config, ConfigType.SourceDiscover), - ) + src_discover_config = convert_src_discovery_config(src_discover_section_config) src_discover_errors = src_discover_config.check_schema() @@ -98,7 +99,12 @@ def analyse( # Get config for `analyse` section analyse_section_config: AnalyseSectionConfigType | None = data.get("analyse") - src_analyse_config = convert_analyse_config(analyse_section_config, src_discover) + try: + src_analyse_config = convert_analyse_config( + analyse_section_config, src_discover + ) + except TypeError as e: + raise typer.BadParameter(str(e)) from e if outdir: src_analyse_config.outdir = outdir @@ -181,191 +187,22 @@ def discover( def load_config_from_toml( toml_file: Path, project: str | None = None -) -> SrcTraceProjectConfigType: +) -> SrcTraceProjectConfigType | SourceAnalyseConfigFileType: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) - if project: - toml_data = toml_data["src_trace"]["projects"][project] - except Exception as e: raise Exception( f"Failed to load source tracing configuration from {toml_file}" ) from e - return cast(SrcTraceProjectConfigType, toml_data) - - -def load_src_analyse_config_from_toml(toml_file: Path) -> SourceAnalyseConfigFileType: - try: - with toml_file.open("rb") as f: - toml_data = tomllib.load(f) - - except Exception as e: - raise Exception( - f"Failed to load Source analyse configuration from {toml_file}" - ) from e + if project: + toml_data = toml_data["src_trace"]["projects"][project] + return cast(SrcTraceProjectConfigType, toml_data) return cast(SourceAnalyseConfigFileType, toml_data) -class ConfigType(str, Enum): - SourceDiscover = "source_discover" - OneLineCommentStyle = "oneline_comment_style" - NeedIdRefsConfig = "need_id_refs" - MarkedRstCofig = "marked_rst" - Analyse = "source_analyse" - - -def convert_dict_2_config( - config_dict: SourceDiscoverSectionConfigType - | OneLineCommentStyleType - | NeedIdRefsConfigType - | MarkedRstConfigType - | None, - config_type: ConfigType, -) -> SourceDiscoverConfig | OneLineCommentStyle | NeedIdRefsConfig | MarkedRstConfig: - func_map: dict[ - ConfigType, - Callable[ - [ - SourceDiscoverSectionConfigType - | OneLineCommentStyleType - | NeedIdRefsConfigType - | MarkedRstConfigType - | None - ], - SourceDiscoverConfig - | OneLineCommentStyle - | NeedIdRefsConfig - | MarkedRstConfig, - ], - ] = { - ConfigType.SourceDiscover: convert_src_discovery_config, # type: ignore[dict-item] # the type is restrict by its key already - ConfigType.OneLineCommentStyle: convert_oneline_comment_style_config, # type: ignore[dict-item] # the type is restrict by its key already - ConfigType.NeedIdRefsConfig: convert_need_id_refs_config, # type: ignore[dict-item] # the type is restrict by its key already - ConfigType.MarkedRstCofig: convert_marked_rst_config, # type: ignore[dict-item] # the type is restrict by its key already - } - - config = func_map[config_type](config_dict) - - return config - - -def convert_src_discovery_config( - config_dict: SourceDiscoverSectionConfigType | None, -) -> SourceDiscoverConfig: - if config_dict: - src_discover_dict = { - key: (Path(str(value)) if key == "src_dir" else value) - for key, value in config_dict.items() - } - src_discover_config = SourceDiscoverConfig(**src_discover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment - else: - src_discover_config = SourceDiscoverConfig() - - return src_discover_config - - -def convert_analyse_config( - config_dict: AnalyseSectionConfigType | None, - src_discover: SourceDiscover | None = None, -) -> SourceAnalyseConfig: - if config_dict: - analyse_config_dict: SourceAnalyseConfigType = cast( - SourceAnalyseConfigType, - {k: Path(str(v)) if k == "src_dir" else v for k, v in config_dict.items()}, - ) - - # Get oneline_comment_style configuration - oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( - "oneline_comment_style" - ) - oneline_comment_style: OneLineCommentStyle = cast( - OneLineCommentStyle, - convert_dict_2_config( - oneline_comment_style_dict, ConfigType.OneLineCommentStyle - ), - ) - - # Get need_id_refs configuration - need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get( - "need_id_refs" - ) - need_id_refs_config = cast( - NeedIdRefsConfig, - convert_dict_2_config( - need_id_refs_config_dict, ConfigType.NeedIdRefsConfig - ), - ) - - # Get marked_rst configuration - marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get( - "marked_rst" - ) - marked_rst_config = cast( - MarkedRstConfig, - convert_dict_2_config(marked_rst_config_dict, ConfigType.MarkedRstCofig), - ) - - analyse_config_dict["need_id_refs_config"] = need_id_refs_config - analyse_config_dict["marked_rst_config"] = marked_rst_config - analyse_config_dict["oneline_comment_style"] = oneline_comment_style - else: - analyse_config_dict: SourceAnalyseConfigType = {} - - if src_discover: - analyse_config_dict["src_files"] = src_discover.source_paths - analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir - - return SourceAnalyseConfig(**analyse_config_dict) - - -def convert_oneline_comment_style_config( - config_dict: OneLineCommentStyleType | None, -) -> OneLineCommentStyle: - if config_dict is None: - oneline_comment_style = OneLineCommentStyle() - else: - try: - oneline_comment_style = OneLineCommentStyle(**config_dict) - except TypeError as e: - raise typer.BadParameter( - f"Invalid oneline comment style configuration: {e}" - ) from e - return oneline_comment_style - - -def convert_need_id_refs_config( - config_dict: NeedIdRefsConfigType | None, -) -> NeedIdRefsConfig: - if not config_dict: - need_id_refs_config = NeedIdRefsConfig() - else: - try: - need_id_refs_config = NeedIdRefsConfig(**config_dict) - except TypeError as e: - raise typer.BadParameter( - f"Invalid oneline comment style configuration: {e}" - ) from e - return need_id_refs_config - - -def convert_marked_rst_config( - config_dict: MarkedRstConfigType | None, -) -> MarkedRstConfig: - if not config_dict: - marked_rst_config = MarkedRstConfig() - else: - try: - marked_rst_config = MarkedRstConfig(**config_dict) - except TypeError as e: - raise typer.BadParameter( - f"Invalid oneline comment style configuration: {e}" - ) from e - return marked_rst_config - - if __name__ == "__main__": app() diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 77d0cda..f8915b5 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -297,11 +297,13 @@ def convert_analyse_config( config_dict: AnalyseSectionConfigType | None, src_discover: SourceDiscover | None = None, ) -> SourceAnalyseConfig: + analyse_config_dict: SourceAnalyseConfigType = {} if config_dict: - analyse_config_dict: SourceAnalyseConfigType = cast( - SourceAnalyseConfigType, - {k: Path(str(v)) if k == "src_dir" else v for k, v in config_dict.items()}, - ) + for k, v in config_dict.items(): + if k not in {"online_comment_style", "need_id_refs", "marked_rst"}: + analyse_config_dict[k] = ( # type: ignore[literal-required] # dynamical assignment + Path(v) if k == "src_dic" and isinstance(v, str) else v + ) # Get oneline_comment_style configuration oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( diff --git a/tests/test_cmd.py b/tests/test_cmd.py index f592fc7..bd23551 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -28,6 +28,7 @@ "comment_type": CommentType.cpp.value, } ANALYSE_SECTION_CONFIG_TEMPLATE = { + "get_oneline_needs": True, "oneline_comment_style": ONELINE_COMMENT_TEMPLATE, } ANALYSE_CONFIG_TEMPLATE = { @@ -165,7 +166,7 @@ def test_discover(options, stdout): for key, value in ANALYSE_CONFIG_TEMPLATE.items() }, [ - "Invalid value: Invalid oneline comment style configuration:", + "Invalid value: OneLineCommentStyle configuration errors:", "Missing required fields: ['title', 'type']", "Field 'id' is defined multiple times.", ], From ec1f619938c12c2f8339b1dc2e24e86783415138 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 14 Aug 2025 22:36:12 +0200 Subject: [PATCH 07/21] fixed docs --- docs/source/components/analyse.rst | 36 +++++++++++++------------- docs/src_trace.toml | 10 +++++-- tests/data/analyse/minimum_config.toml | 8 ++++++ 3 files changed, 34 insertions(+), 20 deletions(-) create mode 100644 tests/data/analyse/minimum_config.toml diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 7386f90..d3804cd 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -134,7 +134,7 @@ Configuration for one-line needs extraction. See :ref:`oneline_comment_style` fo The following configuration demonstrates the minimum settings required for basic analysis: -.. literalinclude:: ./../../../tests/data/analyse/default_config.toml +.. literalinclude:: ./../../../tests/data/analyse/minimum_config.toml :caption: minimum_config.toml :language: toml @@ -215,25 +215,25 @@ This example demonstrates how the analyse extracts RST blocks from comments. .. code-tab:: cpp - #include + #include - /* - @rst - .. impl:: implement dummy function 1 - :id: IMPL_71 - @endrst - */ - void dummy_func1(){ - //... - } + /* + @rst + .. impl:: implement dummy function 1 + :id: IMPL_71 + @endrst + */ + void dummy_func1(){ + //... + } - // @rst..impl:: implement main function @endrst - int main() { - std::cout << "Starting demo_1..." << std::endl; - dummy_func1(); - std::cout << "Demo_1 finished." << std::endl; - return 0; - } + // @rst..impl:: implement main function @endrst + int main() { + std::cout << "Starting demo_1..." << std::endl; + dummy_func1(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } .. code-tab:: json diff --git a/docs/src_trace.toml b/docs/src_trace.toml index b1ef17f..6dd92a6 100644 --- a/docs/src_trace.toml +++ b/docs/src_trace.toml @@ -12,11 +12,11 @@ remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit [src_trace.projects.dcdc.source_discover] src_dir = "../tests/data/dcdc" # Relative path from this TOML config to the source directory -[src_trace.projects.dcdc.analysis] +[src_trace.projects.dcdc.analyse] get_need_id_refs = false get_oneline_needs = true -[src_trace.projects.dcdc.analysis.oneline_comment_style] +[src_trace.projects.dcdc.analyse.oneline_comment_style] # Configuration for oneline comment style start_sequence = "[[" # Start sequence for oneline comments end_sequence = "]]" # End sequence for the online comments; default is newline character @@ -29,3 +29,9 @@ needs_fields = [ { "name" = "links", "type" = "list[str]", "default" = [ ] }, ] + +[src_trace.projects.src] +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +[src_trace.projects.src.source_discover] +src_dir = "../tests/doc_test/minimum_config" diff --git a/tests/data/analyse/minimum_config.toml b/tests/data/analyse/minimum_config.toml new file mode 100644 index 0000000..74efdc9 --- /dev/null +++ b/tests/data/analyse/minimum_config.toml @@ -0,0 +1,8 @@ +[source_discover] +src_dir = "../../data" +gitignore = false + +[analyse] +get_need_id_refs = true +get_oneline_needs = false +get_rst = true From 8c209e0b2f4829af9a8748b40481e9a5b48feb7a Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 15 Aug 2025 08:32:50 +0200 Subject: [PATCH 08/21] fixed docs --- docs/source/components/analyse.rst | 134 ++++++++++++++-------------- docs/source/components/discover.rst | 2 +- docs/source/components/oneline.rst | 4 +- docs/source/index.rst | 1 + 4 files changed, 70 insertions(+), 71 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index d3804cd..ec7ac2b 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -152,49 +152,49 @@ Below is an example of a C++ source file containing need ID references and the c .. code-tab:: cpp - #include + #include - // @need-ids: need_001, need_002, need_003, need_004 - void dummy_func1(){ - //... - } + // @need-ids: need_001, need_002, need_003, need_004 + void dummy_func1(){ + //... + } - // @need-ids: need_003 - int main() { - std::cout << "Starting demo_1..." << std::endl; - dummy_func1(); - std::cout << "Demo_1 finished." << std::endl; - return 0; - } + // @need-ids: need_003 + int main() { + std::cout << "Starting demo_1..." << std::endl; + dummy_func1(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } .. code-tab:: json - [ - { - "filepath": "tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L3", - "source_map": { - "start": { "row": 2, "column": 13 }, - "end": { "row": 2, "column": 51 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "need_ids": ["need_001", "need_002", "need_003", "need_004"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L8", - "source_map": { - "start": { "row": 7, "column": 13 }, - "end": { "row": 7, "column": 21 } + [ + { + "filepath": "tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": { "row": 2, "column": 13 }, + "end": { "row": 2, "column": 51 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["need_001", "need_002", "need_003", "need_004"], + "marker": "@need-ids:", + "type": "need-id-refs" }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "need_ids": ["need_003"], - "marker": "@need-ids:", - "type": "need-id-refs" - } - ] + { + "filepath": "tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/fa5a9129d60203355ae9fe4a725246a88522c60c/tests/data/need_id_refs/dummy_1.cpp#L8", + "source_map": { + "start": { "row": 7, "column": 13 }, + "end": { "row": 7, "column": 21 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "need_ids": ["need_003"], + "marker": "@need-ids:", + "type": "need-id-refs" + } + ] **Output Structure:** @@ -238,30 +238,30 @@ This example demonstrates how the analyse extracts RST blocks from comments. .. code-tab:: json - [ - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", - "source_map": { - "start": { "row": 3, "column": 8 }, - "end": { "row": 3, "column": 61 } + [ + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", - "type": "rst" - }, - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", - "source_map": { - "start": { "row": 13, "column": 7 }, - "end": { "row": 13, "column": 40 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "rst": "..impl:: implement main function ", - "type": "rst" - } - ] + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" + } + ] **Output Structure:** @@ -288,16 +288,14 @@ For comprehensive information about one-line needs configuration and usage, see **Basic Example:** -.. tabs: - -.. code-tab:: c +.. code-block:: c - // @Function Implementation, IMPL_001, impl, [REQ_001, REQ_002] + // @Function Implementation, IMPL_001, impl, [REQ_001, REQ_002] This single comment line creates a complete **Sphinx-Needs** item equivalent to: -.. code-tab:: rst +.. code-block:: rst - .. impl:: Function Implementation - :id: IMPL_001 - :links: REQ_001, REQ_002 + .. impl:: Function Implementation + :id: IMPL_001 + :links: REQ_001, REQ_002 diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 8b646c6..104984d 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -114,7 +114,7 @@ Specifies the comment syntax style used in the source code files. This determine .. list-table:: :header-rows: 1 - :widths: 20 30 50 + :widths: 40 40 50 * - Language - comment_type diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index b810391..c3f3c51 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -101,7 +101,7 @@ the one-line comment shall be defined as the following .. code-tab:: c - // @ title, id_123, implementation, [link1, link2] + // @ title, id_123, implementation, [link1, link2] .. code-tab:: rst @@ -156,7 +156,7 @@ the first field in the one-line comment .. code-tab:: c - // @ this is title, this is id, this_type, [link1, link2] + // @ this is title, this is id, this_type, [link1, link2] .. code-tab:: rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 1fe53dd..f1bcc25 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -62,6 +62,7 @@ Contents components/directive components/oneline components/analyse + components/discover .. toctree:: :maxdepth: 1 From fd0aeb0a12c944d9eb41bbf88e17e50c33b83b36 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 15 Aug 2025 08:52:41 +0200 Subject: [PATCH 09/21] fixed indentation --- docs/source/basics/quickstart.rst | 2 +- docs/source/components/analyse.rst | 3 +-- docs/source/components/configuration.rst | 2 ++ docs/source/components/discover.rst | 26 ++++++++++++++---------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst index 34e168a..f4f5f48 100644 --- a/docs/source/basics/quickstart.rst +++ b/docs/source/basics/quickstart.rst @@ -24,7 +24,7 @@ Sphinx Config .. literalinclude:: ./../../src_trace.toml :caption: src_trace.toml :language: toml - :lines: 27-29 + One-line comment ---------------- diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index ec7ac2b..a86ffb5 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -235,8 +235,7 @@ This example demonstrates how the analyse extracts RST blocks from comments. return 0; } - - .. code-tab:: json + .. code-tab:: json [ { diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 9f48d09..c2fafc2 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -292,6 +292,8 @@ Enables the use of simplified one-line comment patterns to represent **Sphinx-Ne The following one-line comment in source code: + + .. code-block:: cpp // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 104984d..74c3a74 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -18,14 +18,16 @@ The default configuration is as follows: .. code-block:: toml [source_discover] - src_dir = "./", - exclude = [], - include = [], - gitignore = true, + src_dir = "./" + exclude = [] + include = [] + gitignore = true comment_type = "cpp" The details of each field are the followings +.. _source_dir: + src_dir ~~~~~~~ @@ -57,9 +59,9 @@ Defines a list of glob patterns for files and directories to exclude from discov [source_discover] exclude = [ - "build/**", - "*.tmp", - "tests/fixtures/**", + "build/**" + "*.tmp" + "tests/fixtures/**" "vendor/third_party/**" ] @@ -112,9 +114,9 @@ Specifies the comment syntax style used in the source code files. This determine **Supported comment styles:** -.. list-table:: +.. list-table:: Title :header-rows: 1 - :widths: 40 40 50 + :widths: 25, 25, 30, 50 * - Language - comment_type @@ -122,11 +124,13 @@ Specifies the comment syntax style used in the source code files. This determine - discovered file types * - C/C++ - ``"cpp"`` - - ``//`` (single-line), ``/* */`` (multi-line) + - ``//`` (single-line), + ``/* */`` (multi-line) - ``c``, ``h``, ``.cpp``, and ``.hpp`` * - Python - ``"python"`` - - ``#`` (single-line), ``""" """`` (docstrings) + - ``#`` (single-line), + ``""" """`` (docstrings) - ``.py`` .. note:: From c8f407f69423eee57253ba2d2140602f645a851e Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 15 Aug 2025 08:55:07 +0200 Subject: [PATCH 10/21] fixed mypy --- src/sphinx_codelinks/sphinx_extension/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index f8915b5..c60bff0 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -328,8 +328,6 @@ def convert_analyse_config( analyse_config_dict["need_id_refs_config"] = need_id_refs_config analyse_config_dict["marked_rst_config"] = marked_rst_config analyse_config_dict["oneline_comment_style"] = oneline_comment_style - else: - analyse_config_dict: SourceAnalyseConfigType = {} if src_discover: analyse_config_dict["src_files"] = src_discover.source_paths From 3ce5c2e387e58149e4378998dbffc15a5cf7e17b Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Mon, 18 Aug 2025 12:51:17 +0200 Subject: [PATCH 11/21] Updated intro --- docs/source/basics/introduction.rst | 33 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/source/basics/introduction.rst b/docs/source/basics/introduction.rst index 1004513..e29aa71 100644 --- a/docs/source/basics/introduction.rst +++ b/docs/source/basics/introduction.rst @@ -1,14 +1,37 @@ Introduction ============ -``CodeLinks`` is a Sphinx extension that provides the ``src-trace`` directive to establish traceability between source code and :external+needs:doc:`Sphinx-Needs ` items. +``CodeLinks`` is a versatile utility that enables fast source code tracing and connects it to +the :external+needs:doc:`Sphinx-Needs ` ecosystem. -At its core, ``CodeLinks`` uses a powerful ``Analyse`` to parse source code comments and extract valuable information. The ``Analyse`` can identify and extract three distinct types of content: +It has multiple components: -- **One-line need definitions**: Create new Sphinx-Needs directly from a single, :ref:`customized comment line ` in your source code. +- source code analyzer for multiple programming languages and comment styles +- generator for various output formats that contain the extracted markers or needs +- Sphinx extension that integrates ``CodeLinks`` with Sphinx-Needs +- CLI application to drive the analysis and generation process + +The configuration for ``CodeLinks`` is done via a TOML file, which can be used +for both the :ref:`Sphinx extension ` and the :ref:`CLI application `. + +The configuration determines how markers and languages are ingested and how the Sphinx extension should behave. + +At its core, ``CodeLinks`` parses the source code structure and extracts source markers. +Source markers can be special comments or language-specific constructs like docstrings for Python. + +``CodeLinks`` supports 3 distinct marker types: + +- :ref:`One-line need definitions `: Create new Sphinx-Needs directly from a single customized comment line + in your source code. - **Need ID references**: Link code to existing need items without creating new ones, perfect for tracing implementations to requirements. - **Marked RST text**: Extract blocks of reStructuredText embedded within comments, allowing you to include rich documentation with associated metadata right next to your code. -``src-trace`` directive then consumes ``One-line need definitions`` to generate traceability between source code and your documentation. +When used in a Sphinx context, a new :ref:`directive` creates items at the location where it is placed (for a subset +of the analyzed files/folders). + +For use cases where ``CodeLinks`` should not integrate with Sphinx, but rather generate output files, the +:ref:`cli` can be used. Currently it can write out ``needextend`` directives for the need ID reference comment style. +Other output files are planned such as full need items as RST or needs.json. -The ``Analyse``, along with the ``SourceDiscovery`` module, also provides both a **Python API** for extensibility and a **CLI** for integration into CI/CD pipelines. +The availability of most commands as :ref:`cli` also helps integrate ``CodeLinks`` into build systems and CI/CD pipelines. +Focus is put on performance, portability and caching of processing steps. From 7039cfccb90eb3afa109800ff7165e05ac755d36 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Mon, 18 Aug 2025 10:43:03 +0200 Subject: [PATCH 12/21] fixed typo --- docs/source/components/analyse.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index a86ffb5..16016ce 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -1,9 +1,9 @@ .. _analyse: -Source Analysis +Source Analyse =============== -The **Source Analysis** module is a powerful component of **Sphinx-CodeLinks** that extracts documentation-related content from source code comments. It provides both CLI and API interfaces for flexible integration into documentation workflows. +The **Source Analyse** module is a powerful component of **Sphinx-CodeLinks** that extracts documentation-related content from source code comments. It provides both CLI and API interfaces for flexible integration into documentation workflows. **Key Capabilities:** @@ -16,7 +16,7 @@ The **Source Analysis** module is a powerful component of **Sphinx-CodeLinks** t Overview -------- -Source Analysis works by parsing source code files and identifying specially marked comments that contain documentation information. This enables developers to embed documentation directly in their source code while maintaining clean separation between code and documentation. +Source Analyse works by parsing source code files and identifying specially marked comments that contain documentation information. This enables developers to embed documentation directly in their source code while maintaining clean separation between code and documentation. The module supports three primary extraction modes: @@ -53,7 +53,7 @@ Limitations Configuration ------------- -The **Source Analysis** module is configured using TOML files and leverages the :ref:`Source Discovery ` module to locate source files for processing. +The **Source Analyse** module is configured using TOML files and leverages the :ref:`Source Discovery ` module to locate source files for processing. **Complete Configuration Example:** @@ -97,7 +97,7 @@ Configuration Sections analyse ~~~~~~~ -Main configuration section for the analysis module. +Main configuration section for the ``analyse`` module. **Options:** From c03d48b230114511d253e9c09675dc703f682421 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 10:17:54 +0200 Subject: [PATCH 13/21] unified configuration --- docs/source/components/configuration.rst | 20 +- docs/src_trace.toml | 14 +- src/sphinx_codelinks/analyse/analyse.py | 75 +-- .../analyse/oneline_parser.py | 2 +- src/sphinx_codelinks/analyse/projects.py | 83 ++++ src/sphinx_codelinks/analyse/utils.py | 2 +- src/sphinx_codelinks/cmd.py | 116 +++-- src/sphinx_codelinks/{analyse => }/config.py | 428 +++++++++++++++++- .../sphinx_extension/config.py | 397 ---------------- .../sphinx_extension/directives/src_trace.py | 16 +- .../sphinx_extension/source_tracing.py | 36 +- tests/conftest.py | 6 +- tests/data/analyse/minimum_config.toml | 4 +- .../oneline_comment_basic/analyse_config.toml | 4 +- .../analyse_config.toml | 2 +- tests/data/sphinx/src_trace.toml | 8 +- tests/doc_test/minimum_config/src_trace.toml | 2 +- tests/doc_test/recursive_dirs/src_trace.toml | 6 +- tests/test_analyse.py | 19 +- tests/test_analyse_config.py | 10 +- tests/test_analyse_utils.py | 2 +- tests/test_cmd.py | 102 ++--- tests/test_oneline_parser.py | 2 +- tests/test_src_trace.py | 31 +- 24 files changed, 700 insertions(+), 687 deletions(-) create mode 100644 src/sphinx_codelinks/analyse/projects.py rename src/sphinx_codelinks/{analyse => }/config.py (53%) delete mode 100644 src/sphinx_codelinks/sphinx_extension/config.py diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index c2fafc2..9ea142d 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -30,7 +30,7 @@ Specifies the path to a `TOML file `__ containing **Sphinx-Code When using a TOML configuration file: -- Configuration options are placed under a ``[src_trace]`` section +- Configuration options are placed under a ``[codelinks]`` section - The ``src_trace_`` prefix is omitted in the TOML file - TOML configuration overrides settings in :file:`conf.py` @@ -55,7 +55,7 @@ Enables the generation of local file system links to source code locations. When .. code-tab:: toml - [src_trace] + [codelinks] set_local_url = true src_trace_local_url_field @@ -75,7 +75,7 @@ Specifies the custom field name used for local source code links. .. code-tab:: toml - [src_trace] + [codelinks] local_url_field = "local-url" .. _src_trace_set_remote_url: @@ -96,7 +96,7 @@ Enables the generation of remote repository links to source code locations. When .. code-tab:: toml - [src_trace] + [codelinks] set_remote_url = true src_trace_remote_url_field @@ -116,7 +116,7 @@ Specifies the custom field name used for remote source code links. .. code-tab:: toml - [src_trace] + [codelinks] remote_url_field = "remote-url" Project-Specific Options @@ -147,10 +147,10 @@ Defines configuration for individual source code projects. Each project is ident .. code-tab:: toml - [src_trace.projects.my_project] + [codelinks.projects.my_project] # Configuration for "my_project" - [src_trace.projects.another_project] + [codelinks.projects.another_project] # Configuration for "another_project" remote_url_pattern @@ -180,7 +180,7 @@ Defines the URL pattern for generating links to remote source code repositories .. code-tab:: toml - [src_trace.projects.my_project] + [codelinks.projects.my_project] remote_url_pattern = "https://github.com/user/repo/blob/{commit}/{path}#L{line}" **Common patterns:** @@ -218,7 +218,7 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within .. code-tab:: toml - [src_trace.projects.my_project.source_discover] + [codelinks.projects.my_project.source_discover] src_dir = "./" exclude = [] include = [] @@ -270,7 +270,7 @@ Enables the use of simplified one-line comment patterns to represent **Sphinx-Ne .. code-tab:: toml - [src_trace.projects.my_project.analyse.oneline_comment_style] + [codelinks.projects.my_project.analyse.oneline_comment_style] start_sequence = "@" end_sequence = "\n" # Platform-specific line ending field_split_char = "," diff --git a/docs/src_trace.toml b/docs/src_trace.toml index 6dd92a6..cc7a6a3 100644 --- a/docs/src_trace.toml +++ b/docs/src_trace.toml @@ -1,4 +1,4 @@ -[src_trace] +[codelinks] # Configuration for source tracing set_local_url = true # Set to true to enable local code html and URL generation local_url_field = "local-url" # Need's field name for local URL @@ -6,17 +6,17 @@ set_remote_url = true # Set to true to enable remote url to be generat remote_url_field = "remote-url" # Need's field name for remote URL # Configuration for source tracing project "dcdc" -[src_trace.projects.dcdc] +[codelinks.projects.dcdc] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code -[src_trace.projects.dcdc.source_discover] +[codelinks.projects.dcdc.source_discover] src_dir = "../tests/data/dcdc" # Relative path from this TOML config to the source directory -[src_trace.projects.dcdc.analyse] +[codelinks.projects.dcdc.analyse] get_need_id_refs = false get_oneline_needs = true -[src_trace.projects.dcdc.analyse.oneline_comment_style] +[codelinks.projects.dcdc.analyse.oneline_comment_style] # Configuration for oneline comment style start_sequence = "[[" # Start sequence for oneline comments end_sequence = "]]" # End sequence for the online comments; default is newline character @@ -30,8 +30,8 @@ needs_fields = [ ] }, ] -[src_trace.projects.src] +[codelinks.projects.src] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" -[src_trace.projects.src.source_discover] +[codelinks.projects.src.source_discover] src_dir = "../tests/doc_test/minimum_config" diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index 40cc273..b100118 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -3,16 +3,11 @@ import json import logging from pathlib import Path -from typing import Any +from typing import Any, TypedDict from tree_sitter import Node as TreeSitterNode from sphinx_codelinks.analyse import utils -from sphinx_codelinks.analyse.config import ( - UNIX_NEWLINE, - OneLineCommentStyle, - SourceAnalyseConfig, -) from sphinx_codelinks.analyse.models import ( MarkedContentType, MarkedRst, @@ -26,6 +21,11 @@ OnelineParserInvalidWarning, oneline_parser, ) +from sphinx_codelinks.config import ( + UNIX_NEWLINE, + OneLineCommentStyle, + SourceAnalyseConfig, +) # initialize logger logger = logging.getLogger(__name__) @@ -36,6 +36,14 @@ logger.addHandler(console) +class AnalyseWarningType(TypedDict): + file_path: str + lineno: int + msg: str + type: str + sub_type: str + + @dataclass class AnalyseWarning: file_path: str @@ -46,8 +54,6 @@ class AnalyseWarning: class SourceAnalyse: - warning_filepath: Path = Path("cached_warnings") / "codelinks_warnings.json" - def __init__( self, analyse_config: SourceAnalyseConfig, @@ -70,7 +76,6 @@ def __init__( self.git_root if self.git_root else self.analyse_config.src_dir ) self.oneline_warnings: list[AnalyseWarning] = [] - self.warnings_path = analyse_config.outdir / SourceAnalyse.warning_filepath def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: ignore[explicit-any] """Load source files and extract their content.""" @@ -357,8 +362,8 @@ def merge_marked_content(self) -> None: key=lambda x: (x.filepath, x.source_map["start"]["row"]) ) - def dump_marked_content(self) -> None: - output_path = self.analyse_config.outdir / "marked_content.json" + def dump_marked_content(self, outdir: Path) -> None: + output_path = outdir / "marked_content.json" if not output_path.parent.exists(): output_path.parent.mkdir(parents=True) to_dump = [ @@ -372,51 +377,3 @@ def run(self) -> None: self.create_src_objects() self.extract_marked_content() self.merge_marked_content() - self.dump_marked_content() - self.update_warnings() - - @classmethod - def load_warnings(cls, warnings_dir: Path) -> list[AnalyseWarning] | None: - """Load warnings from the given path. - - It mainly used for other apps or users to load warnings files directly. - """ - warnings_path = warnings_dir / cls.warning_filepath - if not warnings_path.exists(): - return None - with warnings_path.open("r") as f: - # load the json file and convert to AnalyseWarning] - warnings = json.load(f) - loaded_warnings = [AnalyseWarning(**warning) for warning in warnings] - return loaded_warnings - - def _load_warnings(self) -> list[AnalyseWarning] | None: - if not self.warnings_path.exists(): - return None - with self.warnings_path.open("r") as f: - # load the json file and convert to AnalyseWarning] - warnings = json.load(f) - loaded_warnings = [AnalyseWarning(**warning) for warning in warnings] - return loaded_warnings - - def update_warnings(self) -> None: - loaded_warnings = self._load_warnings() - current_warnings = [_warning.__dict__ for _warning in self.oneline_warnings] - if loaded_warnings: - _warnings = [_warning.__dict__ for _warning in loaded_warnings] - cached_warnings = [ - _warning - for _warning in _warnings - if _warning["file_path"] - not in [str(src_file) for src_file in self.src_files] - ] - total_warning = cached_warnings + current_warnings - else: - total_warning = current_warnings - if not self.warnings_path.parent.exists(): - self.warnings_path.parent.mkdir(parents=True) - with self.warnings_path.open("w") as f: - json.dump( - total_warning, - f, - ) diff --git a/src/sphinx_codelinks/analyse/oneline_parser.py b/src/sphinx_codelinks/analyse/oneline_parser.py index 5ac1b16..ff76da0 100644 --- a/src/sphinx_codelinks/analyse/oneline_parser.py +++ b/src/sphinx_codelinks/analyse/oneline_parser.py @@ -2,7 +2,7 @@ from enum import Enum import logging -from sphinx_codelinks.analyse.config import ( +from sphinx_codelinks.config import ( ESCAPE, SUPPORTED_COMMENT_TYPES, UNIX_NEWLINE, diff --git a/src/sphinx_codelinks/analyse/projects.py b/src/sphinx_codelinks/analyse/projects.py new file mode 100644 index 0000000..73e6e0f --- /dev/null +++ b/src/sphinx_codelinks/analyse/projects.py @@ -0,0 +1,83 @@ +import json +import logging +from pathlib import Path + +from sphinx_codelinks.analyse.analyse import ( + AnalyseWarning, + AnalyseWarningType, + SourceAnalyse, +) +from sphinx_codelinks.config import CodeLinksConfig, CodeLinksProjectConfigType + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + + +class AnalyseProjects: + warning_filepath: Path = Path("warnings") / "codelinks_warnings.json" + + def __init__(self, codelink_config: CodeLinksConfig) -> None: + self.projects_configs: dict[str, CodeLinksProjectConfigType] = ( + codelink_config.projects + ) + self.projects_analyse: dict[str, SourceAnalyse] = {} + self.warnings_path = codelink_config.outdir / AnalyseProjects.warning_filepath + self.outdir = codelink_config.outdir + + def run(self) -> None: + for project, config in self.projects_configs.items(): + src_analyse = SourceAnalyse(config["analyse_config"]) + src_analyse.run() + self.projects_analyse[project] = src_analyse + + def dump_markers(self) -> None: + output_path = self.outdir / "marked_content.json" + if not output_path.parent.exists(): + output_path.parent.mkdir(parents=True) + to_dump = { + project: [marker.to_dict() for marker in analyse.all_marked_content] + for project, analyse in self.projects_analyse.items() + } + with output_path.open("w") as f: + json.dump(to_dump, f) + logger.info(f"Marked content dumped to {output_path}") + + @classmethod + def load_warnings(cls, warnings_dir: Path) -> list[AnalyseWarning] | None: + """Load warnings from the given path. + + It mainly used for other apps or users to load warnings files directly. + """ + warnings_path = warnings_dir / cls.warning_filepath + if not warnings_path.exists(): + return None + with warnings_path.open("r") as f: + # load the json file and convert to AnalyseWarning] + warnings = json.load(f) + loaded_warnings: list[AnalyseWarning] = [ + AnalyseWarning(**warning) for warning in warnings + ] + + return loaded_warnings + + def update_warnings(self) -> None: + current_warnings: list[AnalyseWarningType] = [ + _warning.__dict__ + for analyse in self.projects_analyse.values() + for _warning in analyse.oneline_warnings + ] + self.dump_warnings(current_warnings) + + def dump_warnings(self, warnings: dict[str, list[AnalyseWarningType]]) -> None: + if not self.warnings_path.parent.exists(): + self.warnings_path.parent.mkdir(parents=True) + with self.warnings_path.open("w") as f: + json.dump( + warnings, + f, + ) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 1fce95a..a6ab6df 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -9,7 +9,7 @@ from tree_sitter import Language, Parser, Point, Query, QueryCursor from tree_sitter import Node as TreeSitterNode -from sphinx_codelinks.analyse.config import UNIX_NEWLINE, CommentCategory +from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory from sphinx_codelinks.source_discover.config import CommentType # initialize logger diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 95fbb5a..a0d2142 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -6,24 +6,19 @@ import typer -from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.analyse.config import ( - AnalyseSectionConfigType, - SourceAnalyseConfigFileType, - SourceAnalyseConfigType, +from sphinx_codelinks.analyse.projects import AnalyseProjects +from sphinx_codelinks.config import ( + CodeLinksConfig, + CodeLinksConfigType, + CodeLinksProjectConfigType, + generate_project_configs, ) from sphinx_codelinks.source_discover.config import ( CommentType, SourceDiscoverConfig, SourceDiscoverConfigType, - SourceDiscoverSectionConfigType, ) from sphinx_codelinks.source_discover.source_discover import SourceDiscover -from sphinx_codelinks.sphinx_extension.config import ( - SrcTraceProjectConfigType, - convert_analyse_config, - convert_src_discovery_config, -) app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} @@ -42,12 +37,12 @@ def analyse( exists=True, ), ], - project: Annotated[ - str | None, + projects: Annotated[ + list[str] | None, typer.Option( "--project", "-p", - help="Specify the project name of the config when using src-trace config.", + help="Specify the project name of the config. If not specified, take all", show_default=True, ), ] = None, @@ -65,57 +60,57 @@ def analyse( ] = None, ) -> None: """Analyse marked content in source code.""" - data = load_config_from_toml(config, project) - errors: deque[str] = deque() + data: CodeLinksConfigType = load_config_from_toml(config) - # Get source_discover configuration - src_discover_section_config: SourceDiscoverSectionConfigType | None = cast( - SourceDiscoverSectionConfigType | None, data.get("source_discover") - ) + codelinks_config = CodeLinksConfig(**data) + try: + generate_project_configs(codelinks_config.projects) + except TypeError as e: + raise typer.BadParameter(str(e)) from e - src_discover_config = convert_src_discovery_config(src_discover_section_config) + errors: deque[str] = deque() + if outdir: + codelinks_config.outdir = outdir - src_discover_errors = src_discover_config.check_schema() + specifed_project_configs: dict[str, CodeLinksProjectConfigType] = {} + for project, _config in codelinks_config.projects.items(): + if projects and project not in projects: + continue + # Get source_discover configuration + src_discover_config = _config["source_discover_config"] - if src_discover_errors: - errors.appendleft("Invalid source discovery configuration:") - errors.extend(src_discover_errors) - if errors: - raise typer.BadParameter(f"{linesep.join(errors)}") + src_discover_errors = src_discover_config.check_schema() - # src dir shall be relevant to the config file's location - src_discover_config.src_dir = ( - config.parent / src_discover_config.src_dir - ).resolve() + if src_discover_errors: + errors.appendleft("Invalid source discovery configuration:") + errors.extend(src_discover_errors) + if errors: + raise typer.BadParameter(f"{linesep.join(errors)}") - src_discover = SourceDiscover(src_discover_config) + # src dir shall be relevant to the config file's location + src_discover_config.src_dir = ( + config.parent / src_discover_config.src_dir + ).resolve() - # Init source analyse config - analyse_config_dict: SourceAnalyseConfigType = {} - analyse_config_dict["src_files"] = src_discover.source_paths - analyse_config_dict["src_dir"] = Path(src_discover.src_discover_config.src_dir) + src_discover = SourceDiscover(src_discover_config) - # Get config for `analyse` section - analyse_section_config: AnalyseSectionConfigType | None = data.get("analyse") + # Init source analyse config + analyse_config = _config["analyse_config"] + analyse_config.src_files = src_discover.source_paths + analyse_config.src_dir = Path(src_discover.src_discover_config.src_dir) - try: - src_analyse_config = convert_analyse_config( - analyse_section_config, src_discover - ) - except TypeError as e: - raise typer.BadParameter(str(e)) from e - - if outdir: - src_analyse_config.outdir = outdir + analyse_errors = analyse_config.check_fields_configuration() + errors.extend(analyse_errors) + if errors: + raise typer.BadParameter(f"{linesep.join(errors)}") - analyse_errors = src_analyse_config.check_fields_configuration() - errors.extend(analyse_errors) - if errors: - raise typer.BadParameter(f"{linesep.join(errors)}") + specifed_project_configs[project] = {"analyse_config": analyse_config} - src_analyse = SourceAnalyse(src_analyse_config) - src_analyse.run() + codelinks_config.projects = specifed_project_configs + analyse_projects = AnalyseProjects(codelinks_config) + analyse_projects.run() + analyse_projects.dump_markers() @app.command(no_args_is_help=True) @@ -185,23 +180,22 @@ def discover( typer.echo(file_path) -def load_config_from_toml( - toml_file: Path, project: str | None = None -) -> SrcTraceProjectConfigType | SourceAnalyseConfigFileType: +def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) except Exception as e: raise Exception( - f"Failed to load source tracing configuration from {toml_file}" + f"Failed to load CodeLinks configuration from {toml_file}" ) from e - if project: - toml_data = toml_data["src_trace"]["projects"][project] - return cast(SrcTraceProjectConfigType, toml_data) + codelink_dict = toml_data.get("codelinks") + + if not codelink_dict: + raise Exception(f"No 'codelinks' section found in {toml_file}") - return cast(SourceAnalyseConfigFileType, toml_data) + return cast(CodeLinksConfigType, codelink_dict) if __name__ == "__main__": diff --git a/src/sphinx_codelinks/analyse/config.py b/src/sphinx_codelinks/config.py similarity index 53% rename from src/sphinx_codelinks/analyse/config.py rename to src/sphinx_codelinks/config.py index c94fa40..986f5d6 100644 --- a/src/sphinx_codelinks/analyse/config.py +++ b/src/sphinx_codelinks/config.py @@ -5,11 +5,15 @@ from typing import Any, Literal, TypedDict, cast from jsonschema import ValidationError, validate +from sphinx.application import Sphinx +from sphinx.config import Config as _SphinxConfig from sphinx_codelinks.source_discover.config import ( CommentType, + SourceDiscoverConfig, SourceDiscoverSectionConfigType, ) +from sphinx_codelinks.source_discover.source_discover import SourceDiscover UNIX_NEWLINE = "\n" @@ -301,19 +305,11 @@ class AnalyseSectionConfigType(TypedDict, total=False): oneline_comment_style: OneLineCommentStyleType -class SourceAnalyseConfigFileType(TypedDict, total=False): - """Define types to load TOML config file.""" - - src_discover: SourceDiscoverSectionConfigType - analyse: AnalyseSectionConfigType - - class SourceAnalyseConfigType(TypedDict, total=False): """Define typing for its API configuration.""" src_files: list[Path] src_dir: Path - outdir: Path comment_type: CommentType get_need_id_refs: bool get_oneline_needs: bool @@ -323,6 +319,10 @@ class SourceAnalyseConfigType(TypedDict, total=False): oneline_comment_style: OneLineCommentStyle +class ProjectsAnalyseConfigType(TypedDict, total=False): + projects_config: dict[str, SourceAnalyseConfigType] + + @dataclass class SourceAnalyseConfig: @classmethod @@ -338,11 +338,6 @@ def field_names(cls) -> set[str]: default_factory=lambda: Path("./"), metadata={"schema": {"type": "string"}} ) - outdir: Path = field( - default=Path("output"), metadata={"schema": {"type": "string"}} - ) - """The directory where the virtual documents and their caches will be stored.""" - comment_type: CommentType = field( default=CommentType.cpp, metadata={"schema": {"type": "string"}} ) @@ -449,3 +444,410 @@ def check_fields_configuration(self) -> list[str]: errors.appendleft("analyse configuration errors:") errors.extend(analyse_errors) return list(errors) + + +SRC_TRACE_CACHE: str = "src_trace_cache" + + +class SourceTracingLineHref: + """Global class for the mapping between source file line numbers and Sphinx documentation links.""" + + def __init__(self) -> None: + self.mappings: dict[str, dict[int, str]] = {} + + +file_lineno_href = SourceTracingLineHref() + + +class CodeLinksProjectConfigType(TypedDict, total=False): + """TypedDict defining the configuration structure for individual SrcTrace projects. + + Contains both user-provided configuration: + - source_discover + - remote_url_pattern + - analyse + and runtime-generated configuration objects + - source_discover_config + - analyse_config + """ + + source_discover: SourceDiscoverSectionConfigType + remote_url_pattern: str + analyse: AnalyseSectionConfigType + source_discover_config: SourceDiscoverConfig + analyse_config: SourceAnalyseConfig + + +class CodeLinksConfigType(TypedDict): + config_from_toml: str | None + set_local_url: bool + local_url_field: str + set_remote_url: bool + remote_url_field: str + outdir: Path + projects: dict[str, CodeLinksProjectConfigType] + debug_measurement: bool + debug_filters: bool + + +@dataclass +class CodeLinksConfig: + @classmethod + def from_sphinx(cls, sphinx_config: _SphinxConfig) -> "CodeLinksConfig": + obj = cls() + super().__setattr__(obj, "_sphinx_config", sphinx_config) + return obj + + def __getattribute__(self, name: str) -> Any: # type: ignore[explicit-any] + if name.startswith("__") or name == "_sphinx_config": + return super().__getattribute__(name) + sphinx_config = ( + object.__getattribute__(self, "_sphinx_config") + if "_sphinx_config" in self.__dict__ + else None + ) + if sphinx_config: + return getattr( + super().__getattribute__("_sphinx_config"), f"src_trace_{name}" + ) + + return object.__getattribute__(self, name) + + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] + if name == "_sphinx_config" and "src_trace_projects" in value: + src_trace_projects: dict[str, CodeLinksProjectConfigType] = value[ + "src_trace_projects" + ] + generate_project_configs(src_trace_projects) + + if name.startswith("__") or name == "_sphinx_config": + return super().__setattr__(name, value) + + sphinx_config = ( + object.__getattribute__(self, "_sphinx_config") + if "_sphinx_config" in self.__dict__ + else None + ) + + if sphinx_config: + setattr( + super().__getattribute__("_sphinx_config"), f"src_trace_{name}", value + ) + + return object.__setattr__(self, name, value) + + @classmethod + def add_config_values(cls, app: Sphinx) -> None: + """Add all config values to Sphinx application""" + for item in fields(cls): + if item.default_factory is not MISSING: + default = item.default_factory() + elif item.default is not MISSING: + default = item.default + else: + raise Exception(f"Field {item.name} has no default value or factory") + + name = item.name + app.add_config_value( + f"src_trace_{name}", + default, + item.metadata["rebuild"], + types=item.metadata["types"], + ) + + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + """Get the schema for a config item.""" + _field = next(field for field in fields(cls) if field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return _field.metadata["schema"] # type: ignore[no-any-return] + return None + + config_from_toml: str | None = field( + default=None, + metadata={ + "rebuild": "env", + "types": (str, type(None)), + "schema": { + "type": ["string", "null"], + "examples": ["config.toml", None], + }, + }, + ) + """Path to a TOML file to load configuration from.""" + + set_local_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + "schema": { + "type": "boolean", + }, + }, + ) + """Set the file URL in the extracted need.""" + + local_url_field: str = field( + default="local-url", + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, + ) + """The field name for the file URL in the extracted need.""" + + set_remote_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + "schema": { + "type": "boolean", + }, + }, + ) + remote_url_field: str = field( + default="remote-url", + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, + ) + """The field name for the remote URL in the extracted need.""" + + outdir: Path = field( + default=Path("output"), + metadata={"rebuild": "env", "types": (str), "schema": {"type": "string"}}, + ) + """The directory where the generated artifacts and their caches will be stored.""" + + projects: dict[str, CodeLinksProjectConfigType] = field( + default_factory=dict, + metadata={ + "rebuild": "env", + "types": (), + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "source_discover": {}, + "analyse": {}, + "remote_url_pattern": {}, + "source_discover_config": {}, + "analyse_config": {}, + }, + "additionalProperties": False, + }, + }, + }, + ) + """The configuration for the source tracing projects.""" + + debug_measurement: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log runtime information for various functions.""" + debug_filters: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log filter processing runtime information.""" + + +def check_schema(config: CodeLinksConfig) -> list[str]: + """Check only first layer's of schema, so that the nested dict is not validated here.""" + errors = [] + for _field_name in CodeLinksConfig.field_names(): + schema = CodeLinksConfig.get_schema(_field_name) + if not schema: + continue + value = getattr(config, _field_name) + if isinstance(value, Path): # adapt to json schema restriction + value = str(value) + try: + validate(instance=value, schema=schema) + except ValidationError as e: + errors.append( + f"Schema validation error in filed '{_field_name}': {e.message}" + ) + return errors + + +def check_project_configuration(config: CodeLinksConfig) -> list[str]: + """Check nested project configurations""" + errors = [] + + for project_name, project_config in config.projects.items(): + project_errors: list[str] = [] + + # validate source_discover config + src_discover_config: SourceDiscoverConfig | None = project_config.get( + "source_discover_config" + ) + src_discover_errors = [] + if src_discover_config: + src_discover_errors.extend(src_discover_config.check_schema()) + + # validate analyse config + analyse_config: SourceAnalyseConfig | None = project_config.get( + "analyse_config" + ) + analyse_errors = [] + if analyse_config: + analyse_errors = analyse_config.check_fields_configuration() + + # validate src-trace config + if config.set_remote_url and "remote_url_pattern" not in project_config: + project_errors.append( + "remote_url_pattern must be given, as set_remote_url is enabled" + ) + + if "remote_url_pattern" in project_config and not isinstance( + project_config["remote_url_pattern"], str + ): + project_errors.append("remote_url_pattern must be a string") + + if analyse_errors or src_discover_errors or project_errors: + errors.append(f"Project '{project_name}' has the following errors:") + errors.extend(analyse_errors) + errors.extend(src_discover_errors) + errors.extend(project_errors) + + return errors + + +def check_configuration(config: CodeLinksConfig) -> list[str]: + errors = [] + errors.extend(check_schema(config)) + errors.extend(check_project_configuration(config)) + return errors + + +def convert_src_discovery_config( + config_dict: SourceDiscoverSectionConfigType | None, +) -> SourceDiscoverConfig: + if config_dict: + src_discover_dict = { + key: (Path(value) if key == "src_dir" and isinstance(value, str) else value) + for key, value in config_dict.items() + } + src_discover_config = SourceDiscoverConfig(**src_discover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment + else: + src_discover_config = SourceDiscoverConfig() + + return src_discover_config + + +def convert_analyse_config( + config_dict: AnalyseSectionConfigType | None, + src_discover: SourceDiscover | None = None, +) -> SourceAnalyseConfig: + analyse_config_dict: SourceAnalyseConfigType = {} + if config_dict: + for k, v in config_dict.items(): + if k not in {"online_comment_style", "need_id_refs", "marked_rst"}: + analyse_config_dict[k] = ( # type: ignore[literal-required] # dynamical assignment + Path(v) if k == "src_dic" and isinstance(v, str) else v + ) + + # Get oneline_comment_style configuration + oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( + "oneline_comment_style" + ) + oneline_comment_style: OneLineCommentStyle = ( + convert_oneline_comment_style_config(oneline_comment_style_dict) + ) + + # Get need_id_refs configuration + need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get( + "need_id_refs" + ) + need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict) + + # Get marked_rst configuration + marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get( + "marked_rst" + ) + marked_rst_config = convert_marked_rst_config(marked_rst_config_dict) + + analyse_config_dict["need_id_refs_config"] = need_id_refs_config + analyse_config_dict["marked_rst_config"] = marked_rst_config + analyse_config_dict["oneline_comment_style"] = oneline_comment_style + + if src_discover: + analyse_config_dict["src_files"] = src_discover.source_paths + analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir + + return SourceAnalyseConfig(**analyse_config_dict) + + +def convert_oneline_comment_style_config( + config_dict: OneLineCommentStyleType | None, +) -> OneLineCommentStyle: + if config_dict is None: + oneline_comment_style = OneLineCommentStyle() + else: + try: + oneline_comment_style = OneLineCommentStyle(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return oneline_comment_style + + +def convert_need_id_refs_config( + config_dict: NeedIdRefsConfigType | None, +) -> NeedIdRefsConfig: + if not config_dict: + need_id_refs_config = NeedIdRefsConfig() + else: + try: + need_id_refs_config = NeedIdRefsConfig(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return need_id_refs_config + + +def convert_marked_rst_config( + config_dict: MarkedRstConfigType | None, +) -> MarkedRstConfig: + if not config_dict: + marked_rst_config = MarkedRstConfig() + else: + try: + marked_rst_config = MarkedRstConfig(**config_dict) + except TypeError as e: + raise TypeError(f"Invalid oneline comment style configuration: {e}") from e + return marked_rst_config + + +def generate_project_configs( + project_configs: dict[str, CodeLinksProjectConfigType], +) -> None: + """Generate configs of source discover and analyse from their classes dynamically.""" + for project_config in project_configs.values(): + # overwrite the config into different types on purpose + # covert dicts to their own classes + src_discover_section: SourceDiscoverSectionConfigType | None = cast( + SourceDiscoverSectionConfigType, + project_config.get("source_discover"), + ) + source_discover_config = convert_src_discovery_config(src_discover_section) + project_config["source_discover_config"] = source_discover_config + + analyse_section_config: AnalyseSectionConfigType | None = cast( + AnalyseSectionConfigType, project_config.get("analyse") + ) + analyse_config = convert_analyse_config(analyse_section_config) + analyse_config.get_oneline_needs = True # force to get oneline_need + project_config["analyse_config"] = analyse_config diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py deleted file mode 100644 index c60bff0..0000000 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ /dev/null @@ -1,397 +0,0 @@ -from dataclasses import MISSING, dataclass, field, fields -from pathlib import Path -from typing import Any, TypedDict, cast - -from jsonschema import ValidationError, validate -from sphinx.application import Sphinx -from sphinx.config import Config as _SphinxConfig - -from sphinx_codelinks.analyse.config import ( - AnalyseSectionConfigType, - MarkedRstConfig, - MarkedRstConfigType, - NeedIdRefsConfig, - NeedIdRefsConfigType, - OneLineCommentStyle, - OneLineCommentStyleType, - SourceAnalyseConfig, - SourceAnalyseConfigType, -) -from sphinx_codelinks.source_discover.config import ( - SourceDiscoverConfig, - SourceDiscoverSectionConfigType, -) -from sphinx_codelinks.source_discover.source_discover import SourceDiscover - -SRC_TRACE_CACHE: str = "src_trace_cache" - - -class SourceTracingLineHref: - """Global class for the mapping between source file line numbers and Sphinx documentation links.""" - - def __init__(self) -> None: - self.mappings: dict[str, dict[int, str]] = {} - - -file_lineno_href = SourceTracingLineHref() - - -class SrcTraceProjectConfigType(TypedDict, total=False): - """TypedDict defining the configuration structure for individual SrcTrace projects. - - Contains both user-provided configuration: - - source_discover - - remote_url_pattern - - analyse - and runtime-generated configuration objects - - source_discover_config - - analyse_config - """ - - source_discover: SourceDiscoverSectionConfigType - remote_url_pattern: str - analyse: AnalyseSectionConfigType - source_discover_config: SourceDiscoverConfig - analyse_config: SourceAnalyseConfig - - -class SrcTraceConfigType(TypedDict): - config_from_toml: str | None - set_local_url: bool - local_url_field: str - set_remote_url: bool - remote_url_field: str - projects: dict[str, SrcTraceProjectConfigType] - debug_measurement: bool - debug_filters: bool - - -@dataclass -class SrcTraceSphinxConfig: - def __init__(self, config: _SphinxConfig) -> None: - super().__setattr__("_config", config) - - def __getattribute__(self, name: str) -> Any: # type: ignore[explicit-any] - if name.startswith("__") or name == "_config": - return super().__getattribute__(name) - return getattr(super().__getattribute__("_config"), f"src_trace_{name}") - - def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] - if name == "_config" and "src_trace_projects" in value: - src_trace_projects: dict[str, SrcTraceProjectConfigType] = value[ - "src_trace_projects" - ] - generate_project_configs(src_trace_projects) - - if name.startswith("__") or name == "_config": - return super().__setattr__(name, value) - - return setattr(super().__getattribute__("_config"), f"src_trace_{name}", value) - - @classmethod - def add_config_values(cls, app: Sphinx) -> None: - """Add all config values to Sphinx application""" - for item in fields(cls): - if item.default_factory is not MISSING: - default = item.default_factory() - elif item.default is not MISSING: - default = item.default - else: - raise Exception(f"Field {item.name} has no default value or factory") - - name = item.name - app.add_config_value( - f"src_trace_{name}", - default, - item.metadata["rebuild"], - types=item.metadata["types"], - ) - - @classmethod - def field_names(cls) -> set[str]: - return {item.name for item in fields(cls)} - - @classmethod - def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] - """Get the schema for a config item.""" - _field = next(field for field in fields(cls) if field.name is name) - if _field.metadata is not MISSING and "schema" in _field.metadata: - return _field.metadata["schema"] # type: ignore[no-any-return] - return None - - config_from_toml: str | None = field( - default=None, - metadata={ - "rebuild": "env", - "types": (str, type(None)), - "schema": { - "type": ["string", "null"], - "examples": ["config.toml", None], - }, - }, - ) - """Path to a TOML file to load configuration from.""" - - set_local_url: bool = field( - default=False, - metadata={ - "rebuild": "env", - "types": (bool,), - "schema": { - "type": "boolean", - }, - }, - ) - """Set the file URL in the extracted need.""" - - local_url_field: str = field( - default="local-url", - metadata={ - "rebuild": "env", - "types": (str,), - "schema": { - "type": "string", - }, - }, - ) - """The field name for the file URL in the extracted need.""" - - set_remote_url: bool = field( - default=False, - metadata={ - "rebuild": "env", - "types": (bool,), - "schema": { - "type": "boolean", - }, - }, - ) - remote_url_field: str = field( - default="remote-url", - metadata={ - "rebuild": "env", - "types": (str,), - "schema": { - "type": "string", - }, - }, - ) - """The field name for the remote URL in the extracted need.""" - - projects: dict[str, SrcTraceProjectConfigType] = field( - default_factory=dict, - metadata={ - "rebuild": "env", - "types": (), - "schema": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "source_discover": {}, - "analyse": {}, - "remote_url_pattern": {}, - "source_discover_config": {}, - "analyse_config": {}, - }, - "additionalProperties": False, - }, - }, - }, - ) - """The configuration for the source tracing projects.""" - - debug_measurement: bool = field( - default=False, metadata={"rebuild": "html", "types": (bool,)} - ) - """If True, log runtime information for various functions.""" - debug_filters: bool = field( - default=False, metadata={"rebuild": "html", "types": (bool,)} - ) - """If True, log filter processing runtime information.""" - - -def check_schema(config: SrcTraceSphinxConfig) -> list[str]: - """Check only first layer's of schema, so that the nested dict is not validated here.""" - errors = [] - for _field_name in SrcTraceSphinxConfig.field_names(): - schema = SrcTraceSphinxConfig.get_schema(_field_name) - value = getattr(config, _field_name) - if not schema: - continue - try: - validate(instance=value, schema=schema) - except ValidationError as e: - errors.append( - f"Schema validation error in filed '{_field_name}': {e.message}" - ) - return errors - - -def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: - """Check nested project configurations""" - errors = [] - - for project_name, project_config in config.projects.items(): - project_errors: list[str] = [] - - # validate source_discover config - src_discover_config: SourceDiscoverConfig | None = project_config.get( - "source_discover_config" - ) - src_discover_errors = [] - if src_discover_config: - src_discover_errors.extend(src_discover_config.check_schema()) - - # validate analyse config - analyse_config: SourceAnalyseConfig | None = project_config.get( - "analyse_config" - ) - analyse_errors = [] - if analyse_config: - analyse_errors = analyse_config.check_fields_configuration() - - # validate src-trace config - if config.set_remote_url and "remote_url_pattern" not in project_config: - project_errors.append( - "remote_url_pattern must be given, as set_remote_url is enabled" - ) - - if "remote_url_pattern" in project_config and not isinstance( - project_config["remote_url_pattern"], str - ): - project_errors.append("remote_url_pattern must be a string") - - if analyse_errors or src_discover_errors or project_errors: - errors.append(f"Project '{project_name}' has the following errors:") - errors.extend(analyse_errors) - errors.extend(src_discover_errors) - errors.extend(project_errors) - - return errors - - -def check_configuration(config: SrcTraceSphinxConfig) -> list[str]: - errors = [] - errors.extend(check_schema(config)) - errors.extend(check_project_configuration(config)) - return errors - - -def convert_src_discovery_config( - config_dict: SourceDiscoverSectionConfigType | None, -) -> SourceDiscoverConfig: - if config_dict: - src_discover_dict = { - key: (Path(value) if key == "src_dir" and isinstance(value, str) else value) - for key, value in config_dict.items() - } - src_discover_config = SourceDiscoverConfig(**src_discover_dict) # type: ignore[arg-type] # mypy is confused by dynamic assignment - else: - src_discover_config = SourceDiscoverConfig() - - return src_discover_config - - -def convert_analyse_config( - config_dict: AnalyseSectionConfigType | None, - src_discover: SourceDiscover | None = None, -) -> SourceAnalyseConfig: - analyse_config_dict: SourceAnalyseConfigType = {} - if config_dict: - for k, v in config_dict.items(): - if k not in {"online_comment_style", "need_id_refs", "marked_rst"}: - analyse_config_dict[k] = ( # type: ignore[literal-required] # dynamical assignment - Path(v) if k == "src_dic" and isinstance(v, str) else v - ) - - # Get oneline_comment_style configuration - oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get( - "oneline_comment_style" - ) - oneline_comment_style: OneLineCommentStyle = ( - convert_oneline_comment_style_config(oneline_comment_style_dict) - ) - - # Get need_id_refs configuration - need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get( - "need_id_refs" - ) - need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict) - - # Get marked_rst configuration - marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get( - "marked_rst" - ) - marked_rst_config = convert_marked_rst_config(marked_rst_config_dict) - - analyse_config_dict["need_id_refs_config"] = need_id_refs_config - analyse_config_dict["marked_rst_config"] = marked_rst_config - analyse_config_dict["oneline_comment_style"] = oneline_comment_style - - if src_discover: - analyse_config_dict["src_files"] = src_discover.source_paths - analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir - - return SourceAnalyseConfig(**analyse_config_dict) - - -def convert_oneline_comment_style_config( - config_dict: OneLineCommentStyleType | None, -) -> OneLineCommentStyle: - if config_dict is None: - oneline_comment_style = OneLineCommentStyle() - else: - try: - oneline_comment_style = OneLineCommentStyle(**config_dict) - except TypeError as e: - raise TypeError(f"Invalid oneline comment style configuration: {e}") from e - return oneline_comment_style - - -def convert_need_id_refs_config( - config_dict: NeedIdRefsConfigType | None, -) -> NeedIdRefsConfig: - if not config_dict: - need_id_refs_config = NeedIdRefsConfig() - else: - try: - need_id_refs_config = NeedIdRefsConfig(**config_dict) - except TypeError as e: - raise TypeError(f"Invalid oneline comment style configuration: {e}") from e - return need_id_refs_config - - -def convert_marked_rst_config( - config_dict: MarkedRstConfigType | None, -) -> MarkedRstConfig: - if not config_dict: - marked_rst_config = MarkedRstConfig() - else: - try: - marked_rst_config = MarkedRstConfig(**config_dict) - except TypeError as e: - raise TypeError(f"Invalid oneline comment style configuration: {e}") from e - return marked_rst_config - - -def generate_project_configs( - project_configs: dict[str, SrcTraceProjectConfigType], -) -> None: - """Generate configs of source discover and analyse from their classes dynamically.""" - for project_config in project_configs.values(): - # overwrite the config into different types on purpose - # covert dicts to their own classes - src_discover_section: SourceDiscoverSectionConfigType | None = cast( - SourceDiscoverSectionConfigType, - project_config.get("source_discover"), - ) - source_discover_config = convert_src_discovery_config(src_discover_section) - project_config["source_discover_config"] = source_discover_config - - analyse_section_config: AnalyseSectionConfigType | None = cast( - AnalyseSectionConfigType, project_config.get("analyse") - ) - analyse_config = convert_analyse_config(analyse_section_config) - analyse_config.get_oneline_needs = True # force to get oneline_need - project_config["analyse_config"] = analyse_config diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index 923bc35..43a7364 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -13,13 +13,13 @@ from sphinx_codelinks.analyse.analyse import SourceAnalyse from sphinx_codelinks.analyse.models import OneLineNeed -from sphinx_codelinks.source_discover.config import SourceDiscoverConfig -from sphinx_codelinks.source_discover.source_discover import SourceDiscover -from sphinx_codelinks.sphinx_extension.config import ( - SrcTraceProjectConfigType, - SrcTraceSphinxConfig, +from sphinx_codelinks.config import ( + CodeLinksConfig, + CodeLinksProjectConfigType, file_lineno_href, ) +from sphinx_codelinks.source_discover.config import SourceDiscoverConfig +from sphinx_codelinks.source_discover.source_discover import SourceDiscover from sphinx_codelinks.sphinx_extension.debug import measure_time sphinx_version = sphinx.__version__ @@ -86,10 +86,10 @@ def run(self) -> list[nodes.Node]: project = self.options["project"] title = self.arguments[0] # get source tracing config - src_trace_sphinx_config = SrcTraceSphinxConfig(self.env.config) + src_trace_sphinx_config = CodeLinksConfig.from_sphinx(self.env.config) # load config - src_trace_conf: SrcTraceProjectConfigType = src_trace_sphinx_config.projects[ + src_trace_conf: CodeLinksProjectConfigType = src_trace_sphinx_config.projects[ project ] src_discover_config = src_trace_conf["source_discover_config"] @@ -225,7 +225,7 @@ def get_src_files( def locate_src_dir( self, - src_trace_sphinx_config: SrcTraceSphinxConfig, + src_trace_sphinx_config: CodeLinksConfig, src_discover_config: SourceDiscoverConfig, ) -> Path: """Locate the source directory based on the configuration.""" diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index 2c6e75a..5a8f956 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -15,17 +15,17 @@ add_need_type, ) -from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.sphinx_extension import debug -from sphinx_codelinks.sphinx_extension.config import ( +from sphinx_codelinks.analyse.projects import AnalyseProjects +from sphinx_codelinks.config import ( SRC_TRACE_CACHE, - SrcTraceConfigType, - SrcTraceProjectConfigType, - SrcTraceSphinxConfig, + CodeLinksConfig, + CodeLinksConfigType, + CodeLinksProjectConfigType, check_configuration, file_lineno_href, generate_project_configs, ) +from sphinx_codelinks.sphinx_extension import debug from sphinx_codelinks.sphinx_extension.directives.src_trace import ( SourceTracing, SourceTracingDirective, @@ -38,7 +38,7 @@ def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[explicit-any] app.add_node(SourceTracing) app.add_directive("src-trace", SourceTracingDirective) - SrcTraceSphinxConfig.add_config_values(app) + CodeLinksConfig.add_config_values(app) app.connect("config-inited", load_config_from_toml, priority=10) app.connect( @@ -110,7 +110,7 @@ def generate_code_page( def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: """Load the configuration from a TOML file, if defined in conf.py.""" - src_trc_sphinx_config = SrcTraceSphinxConfig(config) + src_trc_sphinx_config = CodeLinksConfig.from_sphinx(config) if src_trc_sphinx_config.config_from_toml is None: return @@ -126,7 +126,7 @@ def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) - toml_data = toml_data["src_trace"] + toml_data = toml_data["codelinks"] if not isinstance(toml_data, dict): raise Exception(f"data must be a dict in {toml_file}") @@ -137,27 +137,27 @@ def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: return set_config_to_sphinx( - src_trace_config=cast(SrcTraceConfigType, toml_data), config=config + src_trace_config=cast(CodeLinksConfigType, toml_data), config=config ) def set_config_to_sphinx( - src_trace_config: SrcTraceConfigType, config: _SphinxConfig + src_trace_config: CodeLinksConfigType, config: _SphinxConfig ) -> None: - allowed_keys = SrcTraceSphinxConfig.field_names() + allowed_keys = CodeLinksConfig.field_names() for key, value in src_trace_config.items(): if key not in allowed_keys: continue if key == "projects": - src_trace_projects: dict[str, SrcTraceProjectConfigType] = cast( - dict[str, SrcTraceProjectConfigType], value + src_trace_projects: dict[str, CodeLinksProjectConfigType] = cast( + dict[str, CodeLinksProjectConfigType], value ) generate_project_configs(src_trace_projects) config[f"src_trace_{key}"] = value def update_sn_extra_options(app: Sphinx, config: _SphinxConfig) -> None: - src_trace_sphinx_config = SrcTraceSphinxConfig(config) + src_trace_sphinx_config = CodeLinksConfig.from_sphinx(config) add_extra_option(app, "project") add_extra_option(app, "file") add_extra_option(app, "directory") @@ -175,7 +175,7 @@ def prepare_env(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> Non """ Prepares the sphinx environment to store stc-trace internal data. """ - src_trace_sphinx_config = SrcTraceSphinxConfig(app.config) + src_trace_sphinx_config = CodeLinksConfig.from_sphinx(app.config) # Set time measurement flag if src_trace_sphinx_config.debug_measurement: @@ -188,7 +188,7 @@ def prepare_env(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> Non def check_sphinx_configuration(app: Sphinx, _config: _SphinxConfig) -> None: - config = SrcTraceSphinxConfig(app.config) + config = CodeLinksConfig.from_sphinx(app.config) errors = check_configuration(config) if errors: raise Exception("\n".join(errors)) @@ -198,7 +198,7 @@ def emit_warnings( app: Sphinx, _env: BuildEnvironment, ) -> None: - warnings = SourceAnalyse.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) + warnings = AnalyseProjects.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) if not warnings: return for warning in warnings: diff --git a/tests/conftest.py b/tests/conftest.py index 819d1c0..3930bcb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,17 +5,13 @@ import pytest from syrupy.extensions.single_file import SingleFileSnapshotExtension, WriteMode -from sphinx_codelinks.analyse.config import OneLineCommentStyle +from sphinx_codelinks.config import OneLineCommentStyle pytest_plugins = "sphinx.testing.fixtures" TEST_DIR = Path(__file__).parent DATA_DIR = TEST_DIR / "data" SRC_TRACE_TOML = TEST_DIR / "data" / "sphinx" / "src_trace.toml" -BASIC_analyse_TOML = TEST_DIR / "data" / "oneline_comment_basic" / "analyse_config.toml" -DEFAULT_analyse_TOML = ( - TEST_DIR / "data" / "oneline_comment_default" / "analyse_config.toml" -) RECURSIVE_DIR_analyse_TOML = TEST_DIR / "doc_test" / "recursive_dirs" / "src_trace.toml" ONELINE_COMMENT_STYLE = OneLineCommentStyle( start_sequence="[[", diff --git a/tests/data/analyse/minimum_config.toml b/tests/data/analyse/minimum_config.toml index 74efdc9..33484f8 100644 --- a/tests/data/analyse/minimum_config.toml +++ b/tests/data/analyse/minimum_config.toml @@ -1,8 +1,8 @@ -[source_discover] +[codelinks.projects.minimum_config.source_discover] src_dir = "../../data" gitignore = false -[analyse] +[codelinks.projects.minimum_config.analyse] get_need_id_refs = true get_oneline_needs = false get_rst = true diff --git a/tests/data/oneline_comment_basic/analyse_config.toml b/tests/data/oneline_comment_basic/analyse_config.toml index 70c64fc..102c0c1 100644 --- a/tests/data/oneline_comment_basic/analyse_config.toml +++ b/tests/data/oneline_comment_basic/analyse_config.toml @@ -1,11 +1,11 @@ -[source_discover] +[codelinks.projects.basic.source_discover] src_dir = "./" comment_type = "c" exclude = [] include = ["**/*.c", "**/*.h"] gitignore = false -[oneline_comment_style] +[codelinks.projects.basic.analyse.oneline_comment_style] start_sequence = "[[" end_sequence = "]]" # default is newline character field_split_char = "," diff --git a/tests/data/oneline_comment_default/analyse_config.toml b/tests/data/oneline_comment_default/analyse_config.toml index c128991..fdc8e15 100644 --- a/tests/data/oneline_comment_default/analyse_config.toml +++ b/tests/data/oneline_comment_default/analyse_config.toml @@ -1,4 +1,4 @@ -[source_discover] +[codelinks.projects.default.source_discover] src_dir = "./" comment_type = "c" exclude = [] diff --git a/tests/data/sphinx/src_trace.toml b/tests/data/sphinx/src_trace.toml index 3b02e1e..c19e368 100644 --- a/tests/data/sphinx/src_trace.toml +++ b/tests/data/sphinx/src_trace.toml @@ -1,15 +1,15 @@ -[src_trace] +[codelinks] set_local_url = true local_url_field = "local-url" set_remote_url = true remote_url_field = "remote-url" debug_measurement = true -[src_trace.projects.dcdc] +[codelinks.projects.dcdc] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" -[src_trace.projects.dcdc.source_discover] +[codelinks.projects.dcdc.source_discover] comment_type = "cpp" src_dir = "../dcdc" exclude = ["dcdc/src/ubt/ubt.cpp"] @@ -17,7 +17,7 @@ include = ["**/*.cpp", "**/*.hpp"] gitignore = true -[src_trace.projects.dcdc.analyse.oneline_comment_style] +[codelinks.projects.dcdc.analyse.oneline_comment_style] start_sequence = "[[" end_sequence = "]]" # default is newline character field_split_char = "," diff --git a/tests/doc_test/minimum_config/src_trace.toml b/tests/doc_test/minimum_config/src_trace.toml index da414a4..0561780 100644 --- a/tests/doc_test/minimum_config/src_trace.toml +++ b/tests/doc_test/minimum_config/src_trace.toml @@ -1,2 +1,2 @@ -[src_trace.projects.src] +[codelinks.projects.src] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" diff --git a/tests/doc_test/recursive_dirs/src_trace.toml b/tests/doc_test/recursive_dirs/src_trace.toml index 77c5ac3..b3e38a3 100644 --- a/tests/doc_test/recursive_dirs/src_trace.toml +++ b/tests/doc_test/recursive_dirs/src_trace.toml @@ -1,14 +1,14 @@ -[src_trace] +[codelinks] set_local_url = true local_url_field = "local-url" set_remote_url = true remote_url_field = "remote-url" debug_measurement = true -[src_trace.projects.dummy_src] +[codelinks.projects.dummy_src] remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" -[src_trace.projects.dummy_src.source_discover] +[codelinks.projects.dummy_src.source_discover] comment_type = "cpp" src_dir = "./dummy_src_lv1" exclude = ["dcdc/src/ubt/ubt.cpp"] diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 6512735..d9897cc 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -4,7 +4,7 @@ import pytest from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.analyse.config import SourceAnalyseConfig +from sphinx_codelinks.config import SourceAnalyseConfig from tests.conftest import ( ONELINE_COMMENT_STYLE, ONELINE_COMMENT_STYLE_DEFAULT, @@ -31,16 +31,16 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): src_analyse_config = SourceAnalyseConfig( src_files=src_paths, src_dir=src_dir, - outdir=tmp_path, get_need_id_refs=True, get_oneline_needs=True, get_rst=True, ) - analyser = SourceAnalyse(src_analyse_config) - analyser.git_remote_url = None - analyser.git_commit_rev = None - analyser.run() + analyse = SourceAnalyse(src_analyse_config) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + analyse.dump_marked_content(tmp_path) dumped_content = tmp_path / "marked_content.json" with dumped_content.open("r") as f: @@ -111,7 +111,6 @@ def test_analyse_oneline_needs( src_analyse_config = SourceAnalyseConfig( src_files=src_paths, src_dir=src_dir, - outdir=tmp_path, get_need_id_refs=False, get_oneline_needs=True, get_rst=False, @@ -122,14 +121,8 @@ def test_analyse_oneline_needs( assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] - assert src_analyse.warnings_path.exists() - - loaded_warnings = SourceAnalyse.load_warnings(tmp_path) cnt_comments = 0 for src_file in src_analyse.src_files: cnt_comments += len(src_file.src_comments) assert cnt_comments == result["num_comments"] - - # use cache - assert SourceAnalyse.load_warnings(tmp_path) == loaded_warnings diff --git a/tests/test_analyse_config.py b/tests/test_analyse_config.py index dd14542..7c81d5d 100644 --- a/tests/test_analyse_config.py +++ b/tests/test_analyse_config.py @@ -1,12 +1,12 @@ import pytest -from sphinx_codelinks.analyse.config import OneLineCommentStyle, SourceAnalyseConfig +from sphinx_codelinks.config import OneLineCommentStyle, SourceAnalyseConfig from .conftest import TEST_DIR @pytest.mark.parametrize( - ("vdocs_config", "result"), + ("analyse_config", "result"), [ ( SourceAnalyseConfig( @@ -14,7 +14,6 @@ TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", ], src_dir=TEST_DIR / "data" / "dcdc", - outdir=TEST_DIR / "output", comment_type=123, ), [ @@ -25,7 +24,6 @@ SourceAnalyseConfig( src_files=None, src_dir=TEST_DIR / "data" / "dcdc", - outdir=TEST_DIR / "output", comment_type=123, ), [ @@ -35,8 +33,8 @@ ), ], ) -def test_config_schema_validator_negative(vdocs_config, result): - errors = vdocs_config.check_schema() +def test_config_schema_validator_negative(analyse_config, result): + errors = analyse_config.check_schema() assert sorted(errors) == sorted(result) diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index c6c61e8..c78feb0 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -9,7 +9,7 @@ import tree_sitter_python from sphinx_codelinks.analyse import utils -from sphinx_codelinks.analyse.config import UNIX_NEWLINE +from sphinx_codelinks.config import UNIX_NEWLINE @pytest.fixture(scope="session") diff --git a/tests/test_cmd.py b/tests/test_cmd.py index bd23551..c03f223 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -27,13 +27,18 @@ "gitignore": True, "comment_type": CommentType.cpp.value, } -ANALYSE_SECTION_CONFIG_TEMPLATE = { +ANALYSE_CONFIG_TEMPLATE = { "get_oneline_needs": True, "oneline_comment_style": ONELINE_COMMENT_TEMPLATE, } -ANALYSE_CONFIG_TEMPLATE = { - "source_discover": SRC_DISCOVER_TEMPLATE, - "analyse": ANALYSE_SECTION_CONFIG_TEMPLATE, +CODELINKS_CONFIG_TEMPLATE = { + "outdir": "set/it/to/somewhere", + "projects": { + "project_1": { + "source_discover": SRC_DISCOVER_TEMPLATE, + "analyse": ANALYSE_CONFIG_TEMPLATE, + } + }, } @@ -77,19 +82,14 @@ def test_discover(options, stdout): @pytest.mark.parametrize( - ("config_dict", "output_lines"), + ("src_discover_dict", "analyse_dict", "output_lines"), [ ( { - key: { - src_key: (123 if src_key == "exclude" else src_value) - for src_key, src_value in value.items() - if isinstance(value, dict) - } - if isinstance(value, dict) and key == "source_discover" - else value - for key, value in ANALYSE_CONFIG_TEMPLATE.items() + src_key: (123 if src_key == "exclude" else src_value) + for src_key, src_value in SRC_DISCOVER_TEMPLATE.items() }, + ANALYSE_CONFIG_TEMPLATE, [ "Invalid value: Invalid source discovery configuration:", "Schema validation error in field 'exclude': 123 is not of type 'array'", @@ -97,14 +97,10 @@ def test_discover(options, stdout): ), ( { - key: { - src_key: (123 if src_key == "include" else src_value) - for src_key, src_value in value.items() - } - if isinstance(value, dict) and key == "source_discover" - else value - for key, value in ANALYSE_CONFIG_TEMPLATE.items() + src_key: (123 if src_key == "include" else src_value) + for src_key, src_value in SRC_DISCOVER_TEMPLATE.items() }, + ANALYSE_CONFIG_TEMPLATE, [ "Invalid value: Invalid source discovery configuration:", "Schema validation error in field 'include': 123 is not of type 'array'", @@ -112,18 +108,12 @@ def test_discover(options, stdout): ), ( { - key: { - src_key: ( - 123 - if src_key in ("exclude", "include", "src_dir") - else src_value - ) - for src_key, src_value in value.items() - } - if isinstance(value, dict) and key == "source_discover" - else value - for key, value in ANALYSE_CONFIG_TEMPLATE.items() + src_key: ( + 123 if src_key in ("exclude", "include", "src_dir") else src_value + ) + for src_key, src_value in SRC_DISCOVER_TEMPLATE.items() }, + ANALYSE_CONFIG_TEMPLATE, [ "Invalid value: Invalid source discovery configuration:", "Schema validation error in field 'src_dir': 123 is not of type 'string'", @@ -132,18 +122,14 @@ def test_discover(options, stdout): ], ), ( + SRC_DISCOVER_TEMPLATE, { - key: { - oneline_key: ( - {"not_expected": 123} - if oneline_key == "oneline_comment_style" - else oneline_value - ) - for oneline_key, oneline_value in value.items() - } - if isinstance(value, dict) and key == "analyse" - else value - for key, value in ANALYSE_CONFIG_TEMPLATE.items() + analyse_key: ( + {"not_expected": 123} + if analyse_key == "oneline_comment_style" + else analyse_value + ) + for analyse_key, analyse_value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: Invalid oneline comment style configuration:", @@ -152,18 +138,14 @@ def test_discover(options, stdout): ], ), ( + SRC_DISCOVER_TEMPLATE, { - key: { - oneline_key: ( - {"needs_fields": [{"name": "id"}, {"name": "id"}]} - if oneline_key == "oneline_comment_style" - else oneline_value - ) - for oneline_key, oneline_value in value.items() - } - if isinstance(value, dict) and key == "analyse" - else value - for key, value in ANALYSE_CONFIG_TEMPLATE.items() + analyse_key: ( + {"needs_fields": [{"name": "id"}, {"name": "id"}]} + if analyse_key == "oneline_comment_style" + else analyse_value + ) + for analyse_key, analyse_value in ANALYSE_CONFIG_TEMPLATE.items() }, [ "Invalid value: OneLineCommentStyle configuration errors:", @@ -173,11 +155,17 @@ def test_discover(options, stdout): ), ], ) -def test_analyse_config_negative(config_dict, output_lines, tmp_path: Path) -> None: - # Force disable Rich styling - config_file = tmp_path / "analyse_config.toml" +def test_analyse_config_negative( + src_discover_dict, analyse_dict, output_lines, tmp_path: Path +) -> None: + config_file = tmp_path / "codelinks_config.toml" + codelink_dict = {"codelinks": CODELINKS_CONFIG_TEMPLATE} + codelink_dict["codelinks"]["projects"]["project_1"]["source_discover"] = ( + src_discover_dict + ) + codelink_dict["codelinks"]["projects"]["project_1"]["analyse"] = analyse_dict with config_file.open("w", encoding="utf-8") as f: - toml.dump(config_dict, f) + toml.dump(codelink_dict, f) options = [ "analyse", diff --git a/tests/test_oneline_parser.py b/tests/test_oneline_parser.py index ebb2b32..aa4ef91 100644 --- a/tests/test_oneline_parser.py +++ b/tests/test_oneline_parser.py @@ -1,11 +1,11 @@ import pytest -from sphinx_codelinks.analyse.config import ESCAPE, UNIX_NEWLINE, OneLineCommentStyle from sphinx_codelinks.analyse.oneline_parser import ( OnelineParserInvalidWarning, WarningSubTypeEnum, oneline_parser, ) +from sphinx_codelinks.config import ESCAPE, UNIX_NEWLINE, OneLineCommentStyle from .conftest import ONELINE_COMMENT_STYLE, ONELINE_COMMENT_STYLE_DEFAULT diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 5953ae5..092cd5e 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -5,17 +5,17 @@ import pytest from sphinx.testing.util import SphinxTestApp -from sphinx_codelinks.analyse.analyse import SourceAnalyse -from sphinx_codelinks.sphinx_extension.config import ( +from sphinx_codelinks.analyse.projects import AnalyseProjects +from sphinx_codelinks.config import ( SRC_TRACE_CACHE, - SrcTraceSphinxConfig, + CodeLinksConfig, check_configuration, ) from sphinx_codelinks.sphinx_extension.source_tracing import set_config_to_sphinx @pytest.mark.parametrize( - ("src_trace_config", "result"), + ("codelinks_config", "result"), [ ( { @@ -116,26 +116,25 @@ ) def test_src_tracing_config_negative( make_app: Callable[..., SphinxTestApp], - src_trace_config, + codelinks_config, result, ): this_file_dir = Path(__file__).parent sphinx_project = Path("data") / "sphinx" app = make_app(srcdir=(this_file_dir / sphinx_project)) - set_config_to_sphinx(src_trace_config, app.env.config) - src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) - errors = check_configuration(src_trace_sphinx_config) + set_config_to_sphinx(codelinks_config, app.env.config) + codelinks_sphinx_config = CodeLinksConfig.from_sphinx(app.env.config) + errors = check_configuration(codelinks_sphinx_config) assert sorted(errors) == sorted(result) -def test_src_tracing_config_positive( - make_app: Callable[..., SphinxTestApp], -): - src_trace_config = { +def test_src_tracing_config_positive(make_app: Callable[..., SphinxTestApp], tmp_path): + codelinks_config = { "remote_url_field": "remote-url", "local_url_field": "local-url", "set_local_url": True, "set_remote_url": True, + "outdir": tmp_path, "projects": { "dcdc": { "source_discover": { @@ -170,9 +169,9 @@ def test_src_tracing_config_positive( this_file_dir = Path(__file__).parent sphinx_project = Path("data") / "sphinx" app = make_app(srcdir=(this_file_dir / sphinx_project)) - set_config_to_sphinx(src_trace_config, app.env.config) - src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) - errors = check_configuration(src_trace_sphinx_config) + set_config_to_sphinx(codelinks_config, app.env.config) + codelinks_sphinx_config = CodeLinksConfig.from_sphinx(app.env.config) + errors = check_configuration(codelinks_sphinx_config) assert not errors @@ -220,7 +219,7 @@ def test_build_html( html = Path(app.outdir, "index.html").read_text() assert html - warnings = SourceAnalyse.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) + warnings = AnalyseProjects.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) assert not warnings assert app.env.get_doctree("index") == snapshot_doctree From 00f9f70fd6874b04eb9f84b00f45ba514865ad5c Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 11:32:36 +0200 Subject: [PATCH 14/21] updated docs --- docs/source/components/analyse.rst | 198 +++------- docs/source/components/configuration.rst | 480 ++++++++++++++++------- docs/source/components/discover.rst | 150 ------- docs/source/components/oneline.rst | 5 + 4 files changed, 399 insertions(+), 434 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 16016ce..4841451 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -1,7 +1,7 @@ .. _analyse: Source Analyse -=============== +============== The **Source Analyse** module is a powerful component of **Sphinx-CodeLinks** that extracts documentation-related content from source code comments. It provides both CLI and API interfaces for flexible integration into documentation workflows. @@ -28,17 +28,17 @@ Supported Content Types ----------------------- Sphinx-Needs ID References -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Extract references to **Sphinx-Needs** items directly from source code comments, enabling traceability between code implementations and requirements. One-line Needs -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~ Use simplified comment patterns to define **Sphinx-Needs** items without complex RST syntax. See :ref:`OneLineCommentStyle ` for detailed information. Marked RST Blocks -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~ Embed complete reStructuredText content within source code comments for rich documentation that can be extracted and processed. @@ -50,99 +50,11 @@ Limitations - **Language Support**: Only C/C++ (``//``, ``/* */``) and Python (``#``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time -Configuration -------------- - -The **Source Analyse** module is configured using TOML files and leverages the :ref:`Source Discovery ` module to locate source files for processing. - -**Complete Configuration Example:** - -.. code-block:: toml - - [source_discover] - src_dir = "./" - exclude = [] - include = [] - gitignore = true - comment_type = "cpp" - - [analyse] - get_need_id_refs = true - get_oneline_needs = false - get_rst = false - outdir = "./output" - - [analyse.oneline_comment_style] - start_sequence = "@" - # End sequences is newline by default. Whether it is "\n" or "\r\n" depending on the platform - end_sequence = "\n" - field_split_char = "," - needs_fields = [ - { name = "title", type = "str" }, - { name = "id", type = "str" }, - { name = "type", type = "str", default = "impl" }, - { name = "links", type = "list[str]", default = [] }, - ] - - [analyse.need_id_refs] - markers = ["@need-ids:"] - - [analyse.marked_rst] - start_sequence = "@rst" - end_sequence = "@endrst" - -Configuration Sections ------------------------ - -analyse -~~~~~~~ - -Main configuration section for the ``analyse`` module. - -**Options:** - -- ``get_need_id_refs`` (``bool``) - Enable extraction of Sphinx-Needs ID references -- ``get_oneline_needs`` (``bool``) - Enable extraction of one-line needs -- ``get_rst`` (``bool``) - Enable extraction of marked RST blocks -- ``outdir`` (``str``) - Output directory for generated files - -analyse.need_id_refs -~~~~~~~~~~~~~~~~~~~~ - -Configuration for Sphinx-Needs ID reference extraction. - -**Options:** - -- ``markers`` (``list[str]``) - List of marker strings that identify need ID references - -analyse.marked_rst -~~~~~~~~~~~~~~~~~~ - -Configuration for marked RST block extraction. - -**Options:** - -- ``start_sequence`` (``str``) - Marker that begins an RST block -- ``end_sequence`` (``str``) - Marker that ends an RST block - -analyse.oneline_comment_style -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Configuration for one-line needs extraction. See :ref:`oneline_comment_style` for detailed information. - -**Minimal Configuration Example:** - -The following configuration demonstrates the minimum settings required for basic analysis: - -.. literalinclude:: ./../../../tests/data/analyse/minimum_config.toml - :caption: minimum_config.toml - :language: toml - -This configuration enables extraction of **Sphinx-Needs ID References** and **Marked RST blocks** using the specified markers. - Extraction Examples ------------------- +The following examples are configured with :ref:`the analyse configuration `, + Sphinx-Needs ID References ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -213,54 +125,54 @@ This example demonstrates how the analyse extracts RST blocks from comments. .. tabs:: - .. code-tab:: cpp + .. code-tab:: cpp - #include + #include + + /* + @rst + .. impl:: implement dummy function 1 + :id: IMPL_71 + @endrst + */ + void dummy_func1(){ + //... + } + + // @rst..impl:: implement main function @endrst + int main() { + std::cout << "Starting demo_1..." << std::endl; + dummy_func1(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } - /* - @rst - .. impl:: implement dummy function 1 - :id: IMPL_71 - @endrst - */ - void dummy_func1(){ - //... - } - - // @rst..impl:: implement main function @endrst - int main() { - std::cout << "Starting demo_1..." << std::endl; - dummy_func1(); - std::cout << "Demo_1 finished." << std::endl; - return 0; - } - - .. code-tab:: json + .. code-tab:: json - [ - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", - "source_map": { - "start": { "row": 3, "column": 8 }, - "end": { "row": 3, "column": 61 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", - "type": "rst" - }, - { - "filepath": "marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", - "source_map": { - "start": { "row": 13, "column": 7 }, - "end": { "row": 13, "column": 40 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "rst": "..impl:: implement main function ", - "type": "rst" - } - ] + [ + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" + }, + { + "filepath": "marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/26b301138eef25c5130518d96eaa7a29a9c6c9fe/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" + } + ] **Output Structure:** @@ -289,12 +201,12 @@ For comprehensive information about one-line needs configuration and usage, see .. code-block:: c - // @Function Implementation, IMPL_001, impl, [REQ_001, REQ_002] + // @Function Implementation, IMPL_001, impl, [REQ_001, REQ_002] This single comment line creates a complete **Sphinx-Needs** item equivalent to: .. code-block:: rst - .. impl:: Function Implementation - :id: IMPL_001 - :links: REQ_001, REQ_002 + .. impl:: Function Implementation + :id: IMPL_001 + :links: REQ_001, REQ_002 diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 9ea142d..5dca457 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -1,22 +1,24 @@ .. _configuration: -Configuration[Sphinx] -===================== +Configuration +============= The configuration for ``CodeLinks`` takes place in the project's :external+sphinx:ref:`conf.py file `. Each source code project may have different configurations because of its programming language or its locations. Therefore, based on such consideration, there are **global options** and **project-specific options** for ``CodeLinks`` -All configuration options start with the prefix ``src_trace_`` for **Sphinx-CodeLinks**. +.. attention:: The configuration options are highly recommended to set in a TOML file, which can be used for both the Sphinx extension and the CLI application. -Global Options --------------- +If the configurations are set in ``conf.py``, the options start with the prefix ``src_trace_``. + +Sphinx Configuration +-------------------- -Global options use the ``src_trace_`` prefix and are applied across the entire Sphinx documentation project. +In ``conf.py``, TOML file can be specified as the source of the configuration for Sphinx Directive ``src-trace``. src_trace_config_from_toml -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Specifies the path to a `TOML file `__ containing **Sphinx-CodeLinks** configuration options. This allows you to maintain configuration in a separate file for better organization. @@ -34,15 +36,17 @@ When using a TOML configuration file: - The ``src_trace_`` prefix is omitted in the TOML file - TOML configuration overrides settings in :file:`conf.py` -.. caution:: - Relative paths specified in the TOML file are resolved relative to the directory containing the TOML file, not the Sphinx project root. +.. caution:: Relative paths specified in the TOML file are resolved relative to the directory containing the TOML file, not the Sphinx project root. + +.. _`set_local_url`: -.. _src_trace_set_local_url: +Global Options +-------------- -src_trace_set_local_url -~~~~~~~~~~~~~~~~~~~~~~~ +set_local_url +~~~~~~~~~~~~~ -Enables the generation of local file system links to source code locations. When enabled, **Sphinx-CodeLinks** will add a custom field, which contains the local path to the source file, to generated needs. +Enables the generation of local file system links to source code locations. When enabled, Sphinx Directive **src-trace** will add a custom field, which contains the local path to the source file, to generated needs. **Type:** ``bool`` **Default:** ``False`` @@ -58,109 +62,109 @@ Enables the generation of local file system links to source code locations. When [codelinks] set_local_url = true -src_trace_local_url_field -~~~~~~~~~~~~~~~~~~~~~~~~~ +local_url_field +~~~~~~~~~~~~~~~ Specifies the custom field name used for local source code links. **Type:** ``str`` **Default:** ``"local-url"`` -**Required when:** :ref:`src_trace_set_local_url` is ``True`` +**Required when:** :ref:`set_local_url` is ``True`` .. tabs:: - .. code-tab:: python - - src_trace_local_url_field = "local-url" - .. code-tab:: toml [codelinks] local_url_field = "local-url" -.. _src_trace_set_remote_url: + .. code-tab:: python + + src_trace_local_url_field = "local-url" + +.. _`set_remote_url`: -src_trace_set_remote_url -~~~~~~~~~~~~~~~~~~~~~~~~ +set_remote_url +~~~~~~~~~~~~~~ -Enables the generation of remote repository links to source code locations. When enabled, **Sphinx-CodeLinks** will add a custom field, which contains the URL to the remote repository (e.g., GitHub, GitLab) where the source file is hosted, to needs. +Enables the generation of remote repository links to source code locations. When enabled, Sphinx Directive **src-trace** will add a custom field, which contains the URL to the remote repository (e.g., GitHub, GitLab) where the source file is hosted, to needs. **Type:** ``bool`` **Default:** ``False`` .. tabs:: - .. code-tab:: python - - src_trace_set_remote_url = True - .. code-tab:: toml [codelinks] set_remote_url = true -src_trace_remote_url_field -~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. code-tab:: python + + src_trace_set_remote_url = True + +remote_url_field +~~~~~~~~~~~~~~~~ Specifies the custom field name used for remote source code links. **Type:** ``str`` **Default:** ``"remote-url"`` -**Required when:** :ref:`src_trace_set_remote_url` is ``True`` +**Required when:** :ref:`set_remote_url` is ``True`` .. tabs:: + .. code-tab:: toml + + [codelinks] + remote_url_field = "remote-url" + .. code-tab:: python src_trace_remote_url_field = "remote-url" - .. code-tab:: toml +outdir +~~~~~~ - [codelinks] - remote_url_field = "remote-url" +Specifies the output directory for generated artifacts such as extracted markers and warnings. + +**Type:** ``str`` +**Default:** ``"./output"`` + +.. code-block:: toml + + [codelinks] + outdir = "output" Project-Specific Options -------------------------- +------------------------ -Project-specific options are configured within the ``src_trace_projects`` dictionary, allowing different settings for each source code project being analyzed. +Project-specific options are configured within the ``projects`` section, allowing different settings for :ref:`SourceDiscver ` and :ref:`SourceAnalyse `. -src_trace_projects -~~~~~~~~~~~~~~~~~~ +projects +~~~~~~~~ Defines configuration for individual source code projects. Each project is identified by a unique name (key) and contains its own set of configuration options (value). **Type:** ``dict[str, dict]`` **Default:** ``{}`` -.. tabs:: +.. code-block:: toml - .. code-tab:: python + [codelinks.projects.my_project] + # Configuration for "my_project" - src_trace_projects = { - "my_project": { - # Project-specific options go here - }, - "another_project": { - # Different options for another project - } - } - - .. code-tab:: toml - - [codelinks.projects.my_project] - # Configuration for "my_project" - - [codelinks.projects.another_project] - # Configuration for "another_project" + [codelinks.projects.another_project] + # Configuration for "another_project" remote_url_pattern -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ -Defines the URL pattern for generating links to remote source code repositories (e.g., GitHub, GitLab). This pattern uses placeholders that are dynamically replaced with actual values. +Defines the URL pattern for Sphinx Directive ``src-trace`` to generate links to remote source code repositories (e.g., GitHub, GitLab). This pattern uses placeholders that are dynamically replaced with actual values. **Type:** ``str`` **Default:** Not set -**Required when:** :ref:`src_trace_set_remote_url` is ``True`` +**Required when:** :ref:`set_remote_url` is ``True`` **Available placeholders:** @@ -168,20 +172,10 @@ Defines the URL pattern for generating links to remote source code repositories - ``{path}`` - Relative path to the source file - ``{line}`` - Line number in the source file -.. tabs:: - - .. code-tab:: python - - src_trace_projects = { - "my_project": { - "remote_url_pattern": "https://github.com/user/repo/blob/{commit}/{path}#L{line}" - } - } - - .. code-tab:: toml +.. code-block:: toml - [codelinks.projects.my_project] - remote_url_pattern = "https://github.com/user/repo/blob/{commit}/{path}#L{line}" + [codelinks.projects.my_project] + remote_url_pattern = "https://github.com/user/repo/blob/{commit}/{path}#L{line}" **Common patterns:** @@ -189,8 +183,9 @@ Defines the URL pattern for generating links to remote source code repositories - **GitLab:** ``https://gitlab.com/user/repo/-/blob/{commit}/{path}#L{line}`` - **Bitbucket:** ``https://bitbucket.org/user/repo/src/{commit}/{path}#lines-{line}`` -.. note:: - This option integrates with :external+needs:ref:`need_string_links` to automatically generate clickable links in the documentation. +.. note:: This option integrates with :external+needs:ref:`need_string_links` to automatically generate clickable links in the documentation. + +.. _`discover_config`: source_discover ~~~~~~~~~~~~~~~ @@ -200,30 +195,14 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within **Type:** ``dict`` **Default:** See below -.. tabs:: - - .. code-tab:: python - - src_trace_projects = { - "my_project": { - "source_discover": { - "src_dir": "./", - "exclude": [], - "include": [], - "gitignore": True, - "comment_type": "cpp" - } - } - } - - .. code-tab:: toml +.. code-block:: toml - [codelinks.projects.my_project.source_discover] - src_dir = "./" - exclude = [] - include = [] - gitignore = true - comment_type = "cpp" + [codelinks.projects.my_project.source_discover] + src_dir = "./" + exclude = [] + include = [] + gitignore = true + comment_type = "cpp" **Configuration fields:** @@ -233,53 +212,243 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within - ``gitignore`` - Whether to respect ``.gitignore`` rules when discovering files (Nested .gitignore is NOT supported yet) - ``comment_type`` - Comment style for the programming language ("cpp" and "python" are currently supported) -For detailed information about each field, see :ref:`source discover `. +.. _`source_dir`: -.. _oneline_comment_style: +src_dir +~~~~~~~ -oneline_comment_style -~~~~~~~~~~~~~~~~~~~~~ +Specifies the root directory for source file discovery. This path is resolved relative to the location of the TOML configuration file. -Enables the use of simplified one-line comment patterns to represent **Sphinx-Needs** items directly in source code, eliminating the need for embedded RST syntax. +**Type:** ``str`` +**Default:** ``"./"`` -**Type:** ``dict`` -**Location:** ``src_trace_projects[project_name]["analyse"]["oneline_comment_style"]`` +.. code-block:: toml -.. tabs:: + [codelinks.projects.my_project.source_discover] + src_dir = "../src" - .. code-tab:: python +**Examples:** - import os - src_trace_projects = { - "my_project": { - "analyse": { - "oneline_comment_style": { - "start_sequence": "@", - "end_sequence": os.linesep, - "field_split_char": ",", - "needs_fields": [ - {"name": "title"}, - {"name": "id"}, - {"name": "type", "default": "impl"}, - {"name": "links", "type": "list[str]", "default": []}, - ] - } - } - } - } +- ``"./"`` - Current directory (relative to config file) +- ``"../src"`` - Parent directory's src folder +- ``"./my_project/source"`` - Subdirectory within current directory - .. code-tab:: toml +exclude +~~~~~~~ + +Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. + +**Type:** ``list[str]`` +**Default:** ``[]`` + +.. code-block:: toml + + [codelinks.projects.my_project.source_discover] + exclude = [ + "build/**" + "*.tmp" + "tests/fixtures/**" + "vendor/third_party/**" + ] + +**Common exclusion patterns:** + +- ``"build/**"`` - Exclude entire build directory +- ``"*.o"`` - Exclude object files +- ``"**/__pycache__/**"`` - Exclude Python cache directories +- ``"node_modules/**"`` - Exclude Node.js dependencies + +include +~~~~~~~ - [codelinks.projects.my_project.analyse.oneline_comment_style] - start_sequence = "@" - end_sequence = "\n" # Platform-specific line ending - field_split_char = "," - needs_fields = [ - { name = "title", type = "str" }, - { name = "id", type = "str" }, - { name = "type", type = "str", default = "impl" }, - { name = "links", type = "list[str]", default = [] }, - ] +Defines a list of glob patterns for files to explicitly include in discovery. When specified, only files matching these patterns will be processed, regardless of other filtering rules. + +**Type:** ``list[str]`` +**Default:** ``[]`` (include all files) + +.. code-block:: toml + + [codelinks.projects.my_project.source_discover] + include = [ + "src/**/*.cpp", + "src/**/*.h", + "include/**/*.hpp" + ] + +**Priority:** The ``include`` option has the highest priority and overrides both ``exclude`` and ``gitignore`` settings. + +**Common inclusion patterns:** + +- ``"**/*.cpp"`` - Include all C++ source files +- ``"**/*.py"`` - Include all Python files +- ``"src/**"`` - Include everything in src directory +- ``"*.{c,h}"`` - Include C source and header files + +comment_type +~~~~~~~~~~~~ + +Specifies the comment syntax style used in the source code files. This determines what file types are discovered and how **Sphinx-CodeLinks** parses comments for documentation extraction. + +**Type:** ``str`` +**Default:** ``"cpp"`` +**Supported values:** ``"cpp"``, ``"python"`` + +.. code-block:: toml + + [codelinks.projects.my_project.source_discover] + comment_type = "python" + +**Supported comment styles:** + +.. list-table:: Title + :header-rows: 1 + :widths: 25, 25, 30, 50 + + * - Language + - comment_type + - Comment Syntax + - discovered file types + * - C/C++ + - ``"cpp"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``c``, ``h``, ``.cpp``, and ``.hpp`` + * - Python + - ``"python"`` + - ``#`` (single-line), + ``""" """`` (docstrings) + - ``.py`` + +.. note:: Future versions may support additional programming languages. Currently, only C/C++ and Python comment styles are supported. + +gitignore +~~~~~~~~~ + +Controls whether to respect ``.gitignore`` files when discovering source files. When enabled, files and directories listed in ``.gitignore`` will be automatically excluded from processing. + +**Type:** ``bool`` +**Default:** ``true`` + +.. code-block:: toml + + [codelinks.projects.my_project.source_discover] + gitignore = false + +**Behavior:** + +- ``true`` - Respect ``.gitignore`` rules (recommended) +- ``false`` - Ignore ``.gitignore`` files and process all matching files + +.. important:: **Current Limitation:** This option only supports the root-level ``.gitignore`` file. Nested ``.gitignore`` files in subdirectories or parent directories are not currently processed. + +For more information about the usage examples, see :ref:`source discover `. + +.. _`analyse_config`: + +analyse +~~~~~~~ + +Configures how **Sphinx-CodeLinks** analyse source files to extract markers from comments. This option defines how the markers in source code are parsed and extracted. + +**Complete Configuration Example:** + +.. code-block:: toml + + [codelinks] + outdir = "output" + + [codelinks.projects.my_project.source_discover] + src_dir = "./" + exclude = [] + include = [] + gitignore = true + comment_type = "cpp" + + [codelinks.projects.my_project.analyse] + get_need_id_refs = true + get_oneline_needs = true + get_rst = true + + [codelinks.projects.my_project.analyse.oneline_comment_style] + start_sequence = "@" + # End sequences is newline by default. Whether it is "\n" or "\r\n" depending on the platform + end_sequence = "\n" + field_split_char = "," + needs_fields = [ + { name = "title", type = "str" }, + { name = "id", type = "str" }, + { name = "type", type = "str", default = "impl" }, + { name = "links", type = "list[str]", default = [] }, + ] + + [codelinks.projects.my_project.analyse.need_id_refs] + markers = ["@need-ids:"] + + [codelinks.projects.my_project.analyse.marked_rst] + start_sequence = "@rst" + end_sequence = "@endrst" + +get_need_id_refs +~~~~~~~~~~~~~~~~ + +Enables the extraction of need IDs from source code comments. When enabled, **SourceAnalyse** will parse comments for specific markers that indicate need IDs, allowing them to be extracted for further usages. + +**Type:** ``bool`` +**Default:** ``False`` + +.. code-block:: toml + + [codelinks.projects.my_project.analyse] + get_need_id_refs = true + +get_oneline_needs +~~~~~~~~~~~~~~~~~ + +Enables the extraction of one-line needs directly from source code comments. When enabled, **SourceAnalyse** will parse comments for simplified :ref:`one-line patterns ` that represent needs, allowing them to be processed without requiring full RST syntax. + +**Type:** ``bool`` +**Default:** ``False`` + +.. code-block:: toml + + [codelinks.projects.my_project.analyse] + get_oneline_needs = false + +get_rst +~~~~~~~ + +Enables the extraction of marked RST text from source code comments. When enabled, **SourceAnalyse** will parse comments for specific markers that indicate RST blocks, allowing them to be extracted. + +**Type:** ``bool`` +**Default:** ``False`` + +.. code-block:: toml + + [codelinks.projects.my_project.analyse] + get_rst = false + +.. _`oneline_comment_style`: + +analyse.oneline_comment_style +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Enables the use of simplified :ref:`one-line comment patterns ` to represent **Sphinx-Needs** items directly in source code, eliminating the need for embedded RST syntax. + +**Type:** ``dict`` +**Default:** See below + +.. code-block:: toml + + [codelinks.projects.my_project.analyse.oneline_comment_style] + start_sequence = "@" + end_sequence = "\n" # Platform-specific line ending + field_split_char = "," + needs_fields = [ + { name = "title", type = "str" }, + { name = "id", type = "str" }, + { name = "type", type = "str", default = "impl" }, + { name = "links", type = "list[str]", default = [] }, + ] **Configuration fields:** @@ -292,8 +461,6 @@ Enables the use of simplified one-line comment patterns to represent **Sphinx-Ne The following one-line comment in source code: - - .. code-block:: cpp // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] @@ -306,9 +473,40 @@ Is equivalent to this RST directive: :id: IMPL_4 :links: SPEC_1, SPEC_2 -.. important:: - The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. +.. important:: The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. -**Additional examples and use cases:** +analyse.need_id_refs +~~~~~~~~~~~~~~~~~~~~ + +Configuration for Sphinx-Needs ID reference extraction. + +**Type:** ``dict`` +**Default:** See below + +.. code-block:: toml + + [codelinks.projects.my_project.analyse.need_id_refs] + markers = ["@need-ids:"] + +**Configuration fields:** + +- ``markers`` (``list[str]``) - List of marker strings that identify need ID references + +analyse.marked_rst +~~~~~~~~~~~~~~~~~~ + +Configuration for marked RST block extraction. + +**Type:** ``dict`` +**Default:** See below + +.. code-block:: toml + + [codelinks.projects.my_project.analyse.marked_rst] + start_sequence = "@rst" + end_sequence = "@endrst" + +**Configuration fields:** -For more comprehensive examples and advanced configurations, see the `test cases `__. +- ``start_sequence`` (``str``) - Marker that begins an RST block +- ``end_sequence`` (``str``) - Marker that ends an RST block diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 74c3a74..db31c72 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -7,156 +7,6 @@ SourceDiscover is one of the modules provided in ``Codelinks``. It discovers the It provides users CLI and API to discover the source files. -.. _`default_discover`: - -Configuration -~~~~~~~~~~~~~ - -When used as CLI, a TOML config file can be provided to configure the source discover module. -The default configuration is as follows: - -.. code-block:: toml - - [source_discover] - src_dir = "./" - exclude = [] - include = [] - gitignore = true - comment_type = "cpp" - -The details of each field are the followings - -.. _source_dir: - -src_dir -~~~~~~~ - -Specifies the root directory for source file discovery. This path is resolved relative to the location of the TOML configuration file. - -**Type:** ``str`` -**Default:** ``"./"`` - -.. code-block:: toml - - [source_discover] - src_dir = "../src" - -**Examples:** - -- ``"./"`` - Current directory (relative to config file) -- ``"../src"`` - Parent directory's src folder -- ``"./my_project/source"`` - Subdirectory within current directory - -exclude -~~~~~~~ - -Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. - -**Type:** ``list[str]`` -**Default:** ``[]`` - -.. code-block:: toml - - [source_discover] - exclude = [ - "build/**" - "*.tmp" - "tests/fixtures/**" - "vendor/third_party/**" - ] - -**Common exclusion patterns:** - -- ``"build/**"`` - Exclude entire build directory -- ``"*.o"`` - Exclude object files -- ``"**/__pycache__/**"`` - Exclude Python cache directories -- ``"node_modules/**"`` - Exclude Node.js dependencies - -include -~~~~~~~ - -Defines a list of glob patterns for files to explicitly include in discovery. When specified, only files matching these patterns will be processed, regardless of other filtering rules. - -**Type:** ``list[str]`` -**Default:** ``[]`` (include all files) - -.. code-block:: toml - - [source_discover] - include = [ - "src/**/*.cpp", - "src/**/*.h", - "include/**/*.hpp" - ] - -**Priority:** The ``include`` option has the highest priority and overrides both ``exclude`` and ``gitignore`` settings. - -**Common inclusion patterns:** - -- ``"**/*.cpp"`` - Include all C++ source files -- ``"**/*.py"`` - Include all Python files -- ``"src/**"`` - Include everything in src directory -- ``"*.{c,h}"`` - Include C source and header files - -comment_type -~~~~~~~~~~~~ - -Specifies the comment syntax style used in the source code files. This determines what file types are discovered and how **Sphinx-CodeLinks** parses comments for documentation extraction. - -**Type:** ``str`` -**Default:** ``"cpp"`` -**Supported values:** ``"cpp"``, ``"python"`` - -.. code-block:: toml - - [source_discover] - comment_type = "python" - -**Supported comment styles:** - -.. list-table:: Title - :header-rows: 1 - :widths: 25, 25, 30, 50 - - * - Language - - comment_type - - Comment Syntax - - discovered file types - * - C/C++ - - ``"cpp"`` - - ``//`` (single-line), - ``/* */`` (multi-line) - - ``c``, ``h``, ``.cpp``, and ``.hpp`` - * - Python - - ``"python"`` - - ``#`` (single-line), - ``""" """`` (docstrings) - - ``.py`` - -.. note:: - Future versions may support additional programming languages. Currently, only C/C++ and Python comment styles are supported. - -gitignore -~~~~~~~~~ - -Controls whether to respect ``.gitignore`` files when discovering source files. When enabled, files and directories listed in ``.gitignore`` will be automatically excluded from processing. - -**Type:** ``bool`` -**Default:** ``true`` - -.. code-block:: toml - - [source_discover] - gitignore = false - -**Behavior:** - -- ``true`` - Respect ``.gitignore`` rules (recommended) -- ``false`` - Ignore ``.gitignore`` files and process all matching files - -.. important:: - **Current Limitation:** This option only supports the root-level ``.gitignore`` file. Nested ``.gitignore`` files in subdirectories or parent directories are not currently processed. - Usage Examples -------------- diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index c3f3c51..963ff5d 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -9,6 +9,11 @@ to simplify the effort required to create a need in source code. :ref:`Here ` is the default one-line comment style. +**Additional examples and use cases:** + +For more comprehensive examples and advanced configurations, see the `test cases `__. + + Start and End sequences ----------------------- From 9bbadbb199878794ce3fab7900018693fa12b606 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 13:52:02 +0200 Subject: [PATCH 15/21] fixed mypy --- src/sphinx_codelinks/analyse/projects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sphinx_codelinks/analyse/projects.py b/src/sphinx_codelinks/analyse/projects.py index 73e6e0f..4a1d873 100644 --- a/src/sphinx_codelinks/analyse/projects.py +++ b/src/sphinx_codelinks/analyse/projects.py @@ -1,6 +1,7 @@ import json import logging from pathlib import Path +from typing import cast from sphinx_codelinks.analyse.analyse import ( AnalyseWarning, @@ -67,13 +68,13 @@ def load_warnings(cls, warnings_dir: Path) -> list[AnalyseWarning] | None: def update_warnings(self) -> None: current_warnings: list[AnalyseWarningType] = [ - _warning.__dict__ + cast(AnalyseWarningType, _warning.__dict__) for analyse in self.projects_analyse.values() for _warning in analyse.oneline_warnings ] self.dump_warnings(current_warnings) - def dump_warnings(self, warnings: dict[str, list[AnalyseWarningType]]) -> None: + def dump_warnings(self, warnings: list[AnalyseWarningType]) -> None: if not self.warnings_path.parent.exists(): self.warnings_path.parent.mkdir(parents=True) with self.warnings_path.open("w") as f: From 9a46c3c38ba7a1d731a8014d581e90c4b8e94cec Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 14:28:10 +0200 Subject: [PATCH 16/21] re-arrange sections --- docs/source/components/configuration.rst | 70 ++++++++---------------- 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 5dca457..c906b17 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -51,16 +51,10 @@ Enables the generation of local file system links to source code locations. When **Type:** ``bool`` **Default:** ``False`` -.. tabs:: - - .. code-tab:: python - - src_trace_set_local_url = True - - .. code-tab:: toml +.. code-block:: toml - [codelinks] - set_local_url = true + [codelinks] + set_local_url = true local_url_field ~~~~~~~~~~~~~~~ @@ -71,16 +65,10 @@ Specifies the custom field name used for local source code links. **Default:** ``"local-url"`` **Required when:** :ref:`set_local_url` is ``True`` -.. tabs:: - - .. code-tab:: toml - - [codelinks] - local_url_field = "local-url" - - .. code-tab:: python +.. code-block:: toml - src_trace_local_url_field = "local-url" + [codelinks] + local_url_field = "local-url" .. _`set_remote_url`: @@ -92,16 +80,10 @@ Enables the generation of remote repository links to source code locations. When **Type:** ``bool`` **Default:** ``False`` -.. tabs:: - - .. code-tab:: toml - - [codelinks] - set_remote_url = true - - .. code-tab:: python +.. code-block:: toml - src_trace_set_remote_url = True + [codelinks] + set_remote_url = true remote_url_field ~~~~~~~~~~~~~~~~ @@ -112,16 +94,10 @@ Specifies the custom field name used for remote source code links. **Default:** ``"remote-url"`` **Required when:** :ref:`set_remote_url` is ``True`` -.. tabs:: - - .. code-tab:: toml - - [codelinks] - remote_url_field = "remote-url" - - .. code-tab:: python +.. code-block:: toml - src_trace_remote_url_field = "remote-url" + [codelinks] + remote_url_field = "remote-url" outdir ~~~~~~ @@ -215,7 +191,7 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within .. _`source_dir`: src_dir -~~~~~~~ +^^^^^^^ Specifies the root directory for source file discovery. This path is resolved relative to the location of the TOML configuration file. @@ -234,7 +210,7 @@ Specifies the root directory for source file discovery. This path is resolved re - ``"./my_project/source"`` - Subdirectory within current directory exclude -~~~~~~~ +^^^^^^^ Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. @@ -259,7 +235,7 @@ Defines a list of glob patterns for files and directories to exclude from discov - ``"node_modules/**"`` - Exclude Node.js dependencies include -~~~~~~~ +^^^^^^^ Defines a list of glob patterns for files to explicitly include in discovery. When specified, only files matching these patterns will be processed, regardless of other filtering rules. @@ -285,7 +261,7 @@ Defines a list of glob patterns for files to explicitly include in discovery. Wh - ``"*.{c,h}"`` - Include C source and header files comment_type -~~~~~~~~~~~~ +^^^^^^^^^^^^ Specifies the comment syntax style used in the source code files. This determines what file types are discovered and how **Sphinx-CodeLinks** parses comments for documentation extraction. @@ -322,7 +298,7 @@ Specifies the comment syntax style used in the source code files. This determine .. note:: Future versions may support additional programming languages. Currently, only C/C++ and Python comment styles are supported. gitignore -~~~~~~~~~ +^^^^^^^^^ Controls whether to respect ``.gitignore`` files when discovering source files. When enabled, files and directories listed in ``.gitignore`` will be automatically excluded from processing. @@ -389,7 +365,7 @@ Configures how **Sphinx-CodeLinks** analyse source files to extract markers from end_sequence = "@endrst" get_need_id_refs -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ Enables the extraction of need IDs from source code comments. When enabled, **SourceAnalyse** will parse comments for specific markers that indicate need IDs, allowing them to be extracted for further usages. @@ -402,7 +378,7 @@ Enables the extraction of need IDs from source code comments. When enabled, **So get_need_id_refs = true get_oneline_needs -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ Enables the extraction of one-line needs directly from source code comments. When enabled, **SourceAnalyse** will parse comments for simplified :ref:`one-line patterns ` that represent needs, allowing them to be processed without requiring full RST syntax. @@ -415,7 +391,7 @@ Enables the extraction of one-line needs directly from source code comments. Whe get_oneline_needs = false get_rst -~~~~~~~ +^^^^^^^ Enables the extraction of marked RST text from source code comments. When enabled, **SourceAnalyse** will parse comments for specific markers that indicate RST blocks, allowing them to be extracted. @@ -430,7 +406,7 @@ Enables the extraction of marked RST text from source code comments. When enable .. _`oneline_comment_style`: analyse.oneline_comment_style -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enables the use of simplified :ref:`one-line comment patterns ` to represent **Sphinx-Needs** items directly in source code, eliminating the need for embedded RST syntax. @@ -476,7 +452,7 @@ Is equivalent to this RST directive: .. important:: The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. analyse.need_id_refs -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Configuration for Sphinx-Needs ID reference extraction. @@ -493,7 +469,7 @@ Configuration for Sphinx-Needs ID reference extraction. - ``markers`` (``list[str]``) - List of marker strings that identify need ID references analyse.marked_rst -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ Configuration for marked RST block extraction. From e7a491accd742d6cabb5b944ecee59a6553cffd2 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 14:45:37 +0200 Subject: [PATCH 17/21] improved docs --- docs/source/components/cli.rst | 4 ++-- docs/source/components/configuration.rst | 10 +++++----- docs/source/components/directive.rst | 19 +++++++++---------- docs/source/components/oneline.rst | 8 ++++---- docs/source/development/contributing.rst | 10 +++++----- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/docs/source/components/cli.rst b/docs/source/components/cli.rst index dfa47bd..6d1d0bf 100644 --- a/docs/source/components/cli.rst +++ b/docs/source/components/cli.rst @@ -3,10 +3,10 @@ Command Line Interface (CLI) ============================ -``Sphinx-CodeLinks`` provides CLI for users to integrate documentation build into CI/CD pipeline +``Sphinx-CodeLinks`` provides a CLI for users to integrate documentation builds into CI/CD pipelines and for local development. -It features help pages. add ``-h`` or ``--help`` to any command to see the available options. +It features help pages. Add ``-h`` or ``--help`` to any command to see the available options. .. typer:: sphinx_codelinks.cmd.app :prog: codelinks diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index c906b17..240e4bc 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -6,23 +6,23 @@ Configuration The configuration for ``CodeLinks`` takes place in the project's :external+sphinx:ref:`conf.py file `. Each source code project may have different configurations because of its programming language or its locations. -Therefore, based on such consideration, there are **global options** and **project-specific options** for ``CodeLinks`` +Therefore, based on such considerations, there are **global options** and **project-specific options** for ``CodeLinks``. -.. attention:: The configuration options are highly recommended to set in a TOML file, which can be used for both the Sphinx extension and the CLI application. +.. attention:: It is highly recommended to set the configuration options in a TOML file, which can be used for both the Sphinx extension and the CLI application. If the configurations are set in ``conf.py``, the options start with the prefix ``src_trace_``. Sphinx Configuration -------------------- -In ``conf.py``, TOML file can be specified as the source of the configuration for Sphinx Directive ``src-trace``. +In ``conf.py``, a TOML file can be specified as the source of the configuration for Sphinx Directive ``src-trace``. src_trace_config_from_toml ~~~~~~~~~~~~~~~~~~~~~~~~~~ Specifies the path to a `TOML file `__ containing **Sphinx-CodeLinks** configuration options. This allows you to maintain configuration in a separate file for better organization. -**Type:** ``str`` (relative path to the directory where conf.py locates) +**Type:** ``str`` (relative path to the directory where conf.py is located) **Default:** Not set .. code-block:: python @@ -182,7 +182,7 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within **Configuration fields:** -- ``src_dir`` - Root directory for source file discovery (relative to Sphinx project root or the directory where TOML config file locates if given) +- ``src_dir`` - Root directory for source file discovery (relative to Sphinx project root or the directory where the TOML config file is located if given) - ``exclude`` - List of glob patterns to exclude from processing - ``include`` - List of glob patterns to include (if empty, includes all files) - ``gitignore`` - Whether to respect ``.gitignore`` rules when discovering files (Nested .gitignore is NOT supported yet) diff --git a/docs/source/components/directive.rst b/docs/source/components/directive.rst index 23aed28..4012aa5 100644 --- a/docs/source/components/directive.rst +++ b/docs/source/components/directive.rst @@ -3,7 +3,7 @@ Directive ========= -``CodeLinks`` provides ``src-trace`` directive and it can be used in the following ways: +``CodeLinks`` provides the ``src-trace`` directive and it can be used in the following ways: .. code-block:: rst @@ -19,17 +19,17 @@ or :project: project_config :directory: ./example -``src-trace`` directive has the following options: +The ``src-trace`` directive has the following options: -* **project**: the project config specified in ``conf.py`` or ``toml`` file to be used for source tracing. +* **project**: the project config specified in ``conf.py`` or TOML file to be used for source tracing. * **file**: the source file to be traced. * **directory**: the source files in the directory to be traced recursively. Regarding the **file** and **directory** options: -- they are optional and mutually exclusive. -- the given paths are relative to ``src_dir`` defined in the source tracing configuration -- if not given, the whole project will be examined. +- They are optional and mutually exclusive. +- The given paths are relative to ``src_dir`` defined in the source tracing configuration. +- If not given, the whole project will be examined. Example ------- @@ -44,9 +44,8 @@ With the following configuration for a demo source code project `dcdc `. diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index 963ff5d..63ee49b 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -17,7 +17,7 @@ For more comprehensive examples and advanced configurations, see the `test cases Start and End sequences ----------------------- -To have a better understanding of the syntax of a one-line comment, we will break it down into the following: +To have a better understanding of the syntax of a one-line comment, we will break it down as follows: **start_sequence** defines the characters where the one-line comment starts. **end_sequence** defines the characters where the one-line comment ends. @@ -100,7 +100,7 @@ For example, with the following **needs_fields** configuration: {"name": "links", "type": "list[str]", "default": []}, ], -the one-line comment shall be defined as the following +the one-line comment shall be defined as follows: .. tabs:: @@ -154,8 +154,8 @@ This means the ``order of needs_fields`` determines ``the position of the field` For example, with the mentioned :ref:`needs_fields definition ` -field ``title`` is the first element is the list, so the string of the title must be -the first field in the one-line comment +field ``title`` is the first element in the list, so the string of the title must be +the first field in the one-line comment. .. tabs:: diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst index 03b3128..4ca31a9 100644 --- a/docs/source/development/contributing.rst +++ b/docs/source/development/contributing.rst @@ -6,11 +6,11 @@ This page provides a guide for developers wishing to contribute to ``Sphinx-Code Bugs, Features and PRs ---------------------- -For **bug reports** and well-described **technical feature request**, please use our issue tracker: +For **bug reports** and well-described **technical feature requests**, please use our issue tracker: https://github.com/useblocks/sphinx-codelinks/issues -If you have already created a PR, you can send it in. Our CI workflow will check (test and code styles) -and a maintainer will perform a review, before we can merge it. +If you have already created a PR, you can send it in. Our CI workflow will check (tests and code styles) +and a maintainer will perform a review before we can merge it. Your PR should conform with the following rules: - A meaningful description or link, which describes the change @@ -24,7 +24,7 @@ Install Dependencies ``CodeLinks`` uses `rye `_ to manage the repository. -For the development, use the following command to install python dependencies into the virtual environment. +For development, use the following command to install Python dependencies into the virtual environment. .. code-block:: bash @@ -40,7 +40,7 @@ To run the formatting and linting, pre-commit is used: pre-commit install # to auto-run on every commit pre-commit run --all-files # to run manually -The CI also checks typing, use the following command locally to see if your code is well-typed +The CI also checks typing. Use the following command locally to see if your code is well-typed: .. code-block:: bash From ba66e1768182a850cf1d23ea8189436a7747a86b Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 20 Aug 2025 15:19:48 +0200 Subject: [PATCH 18/21] Fixed typo --- docs/source/components/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 240e4bc..b078bf1 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -115,7 +115,7 @@ Specifies the output directory for generated artifacts such as extracted markers Project-Specific Options ------------------------ -Project-specific options are configured within the ``projects`` section, allowing different settings for :ref:`SourceDiscver ` and :ref:`SourceAnalyse `. +Project-specific options are configured within the ``projects`` section, allowing different settings for :ref:`SourceDiscover ` and :ref:`SourceAnalyse `. projects ~~~~~~~~ From cb3ffdee3f259e8482d78269750bc294de4d3b6d Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 16:02:31 +0200 Subject: [PATCH 19/21] fixed - outdir type error - exceptions to users --- src/sphinx_codelinks/cmd.py | 6 +-- src/sphinx_codelinks/config.py | 3 ++ tests/data/configs/full_config.toml | 43 +++++++++++++++++++ .../{analyse => configs}/minimum_config.toml | 0 .../oneline_comment_basic/analyse_config.toml | 2 +- .../analyse_config.toml | 2 +- tests/test_cmd.py | 6 ++- 7 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 tests/data/configs/full_config.toml rename tests/data/{analyse => configs}/minimum_config.toml (100%) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index a0d2142..f8c79f7 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -63,8 +63,8 @@ def analyse( data: CodeLinksConfigType = load_config_from_toml(config) - codelinks_config = CodeLinksConfig(**data) try: + codelinks_config = CodeLinksConfig(**data) generate_project_configs(codelinks_config.projects) except TypeError as e: raise typer.BadParameter(str(e)) from e @@ -186,14 +186,14 @@ def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: toml_data = tomllib.load(f) except Exception as e: - raise Exception( + raise typer.BadParameter( f"Failed to load CodeLinks configuration from {toml_file}" ) from e codelink_dict = toml_data.get("codelinks") if not codelink_dict: - raise Exception(f"No 'codelinks' section found in {toml_file}") + raise typer.BadParameter(f"No 'codelinks' section found in {toml_file}") return cast(CodeLinksConfigType, codelink_dict) diff --git a/src/sphinx_codelinks/config.py b/src/sphinx_codelinks/config.py index 986f5d6..6745f86 100644 --- a/src/sphinx_codelinks/config.py +++ b/src/sphinx_codelinks/config.py @@ -534,6 +534,9 @@ def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-a super().__getattribute__("_sphinx_config"), f"src_trace_{name}", value ) + if name == "outdir" and isinstance(value, str): + # Ensure outdir is a Path object + value = Path(value) return object.__setattr__(self, name, value) @classmethod diff --git a/tests/data/configs/full_config.toml b/tests/data/configs/full_config.toml new file mode 100644 index 0000000..4b5adb5 --- /dev/null +++ b/tests/data/configs/full_config.toml @@ -0,0 +1,43 @@ +[codelinks] +# Configuration for source tracing +set_local_url = true # Set to true to enable local code html and URL generation +local_url_field = "local-url" # Need's field name for local URL +set_remote_url = true # Set to true to enable remote url to be generated +remote_url_field = "remote-url" # Need's field name for remote URL +outdir = "codelinks_output" # Output directory for generated files + +# Configuration for source tracing project "dcdc" +[codelinks.projects.dcdc] +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code + +[codelinks.projects.dcdc.source_discover] +src_dir = "../dcdc" # Relative path from this TOML config to the source directory +comment_type = "cpp" +exclude = [] +include = ["**/*.c", "**/*.h"] +gitignore = false + +[codelinks.projects.dcdc.analyse] +get_need_id_refs = false +get_oneline_needs = true +get_rst = false + +[codelinks.projects.dcdc.analyse.oneline_comment_style] +# Configuration for oneline comment style +start_sequence = "[[" # Start sequence for oneline comments +end_sequence = "]]" # End sequence for the online comments; default is newline character +field_split_char = "," # Character to split fields in the comment +# Fields that are defined in the oneline comment style +needs_fields = [ + { "name" = "id", "type" = "str" }, + { "name" = "title", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, +] + +[codelinks.projects.src] +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +[codelinks.projects.src.source_discover] +src_dir = "../doc_test/minimum_config" diff --git a/tests/data/analyse/minimum_config.toml b/tests/data/configs/minimum_config.toml similarity index 100% rename from tests/data/analyse/minimum_config.toml rename to tests/data/configs/minimum_config.toml diff --git a/tests/data/oneline_comment_basic/analyse_config.toml b/tests/data/oneline_comment_basic/analyse_config.toml index 102c0c1..6ceefba 100644 --- a/tests/data/oneline_comment_basic/analyse_config.toml +++ b/tests/data/oneline_comment_basic/analyse_config.toml @@ -1,6 +1,6 @@ [codelinks.projects.basic.source_discover] src_dir = "./" -comment_type = "c" +comment_type = "cpp" exclude = [] include = ["**/*.c", "**/*.h"] gitignore = false diff --git a/tests/data/oneline_comment_default/analyse_config.toml b/tests/data/oneline_comment_default/analyse_config.toml index fdc8e15..fb536d6 100644 --- a/tests/data/oneline_comment_default/analyse_config.toml +++ b/tests/data/oneline_comment_default/analyse_config.toml @@ -1,6 +1,6 @@ [codelinks.projects.default.source_discover] src_dir = "./" -comment_type = "c" +comment_type = "cpp" exclude = [] include = ["**/*.c", "**/*.h"] gitignore = false diff --git a/tests/test_cmd.py b/tests/test_cmd.py index c03f223..4a0e254 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -46,7 +46,11 @@ @pytest.mark.parametrize( - ("config_path"), [(DATA_DIR / "analyse" / "minimum_config.toml")] + ("config_path"), + [ + (DATA_DIR / "configs" / "minimum_config.toml"), + (DATA_DIR / "configs" / "full_config.toml"), + ], ) def test_analyse(config_path: Path, tmp_path: Path) -> None: options: list[str] = ["analyse", str(config_path), "--outdir", str(tmp_path)] From 6596e8b9ede64b1acf540485cea0f1070b445679 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Wed, 20 Aug 2025 17:03:36 +0200 Subject: [PATCH 20/21] added logging --- src/sphinx_codelinks/analyse/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index a6ab6df..027abc0 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -134,6 +134,7 @@ def locate_git_root(src_dir: Path) -> Path | None: for parent in parents: if (parent / ".git").exists() and (parent / ".git").is_dir(): return parent + logger.warning(f"git root is not found in the parent of {src_dir}") return None From 69ca1dc68a1105ae69e6dc8a8be868b9666d6929 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 14:04:05 +0200 Subject: [PATCH 21/21] emit error for unknown projects --- src/sphinx_codelinks/cmd.py | 10 ++++++++++ tests/test_cmd.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index f8c79f7..53dde99 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -73,6 +73,16 @@ def analyse( if outdir: codelinks_config.outdir = outdir + project_errors: list[str] = [] + if projects: + for project in projects: + if project not in codelinks_config.projects: + if not project_errors: + project_errors.append("The following projects are not found:") + project_errors.append(project) + if project_errors: + raise typer.BadParameter(f"{linesep.join(project_errors)}") + specifed_project_configs: dict[str, CodeLinksProjectConfigType] = {} for project, _config in codelinks_config.projects.items(): if projects and project not in projects: diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 4a0e254..6d5adbd 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -179,3 +179,36 @@ def test_analyse_config_negative( assert result.exit_code != 0 for line in output_lines: assert line in result.stdout + + +@pytest.mark.parametrize( + ("projects", "output_lines"), + [ + ( + ["project_1", "project_2"], + [ + "The following projects are not found:", + "project_2", + ], + ), + ], +) +def test_analyse_project_negative(projects, output_lines, tmp_path: Path) -> None: + config_file = tmp_path / "codelinks_config.toml" + codelink_dict = {"codelinks": CODELINKS_CONFIG_TEMPLATE} + with config_file.open("w", encoding="utf-8") as f: + toml.dump(codelink_dict, f) + projects_config = [] + for project in projects: + projects_config.append("--project") + projects_config.append(project) + + options = [ + "analyse", + str(config_file), + ] + options.extend(projects_config) + result = runner.invoke(app, options) + assert result.exit_code != 0 + for line in output_lines: + assert line in result.stdout