From 8557e7d43fd8ab05e1b8e382d84d35906d22f823 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 15:48:51 +0200 Subject: [PATCH 01/14] init impl --- src/sphinx_codelinks/analyse/models.py | 4 +- src/sphinx_codelinks/bridge.py | 195 +++++++++++++++++++++++++ src/sphinx_codelinks/logger.py | 94 ++++++++++++ tests/data/analyse/marked_content.json | 112 ++++++++++++++ tests/test_bridge.py | 8 + 5 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 src/sphinx_codelinks/bridge.py create mode 100644 src/sphinx_codelinks/logger.py create mode 100644 tests/data/analyse/marked_content.json create mode 100644 tests/test_bridge.py diff --git a/src/sphinx_codelinks/analyse/models.py b/src/sphinx_codelinks/analyse/models.py index 66fb3498..856f0a03 100644 --- a/src/sphinx_codelinks/analyse/models.py +++ b/src/sphinx_codelinks/analyse/models.py @@ -7,8 +7,8 @@ class MarkedContentType(str, Enum): - need = ("need",) - need_id_refs = ("need-id-refs",) + need = "need" + need_id_refs = "need-id-refs" rst = "rst" diff --git a/src/sphinx_codelinks/bridge.py b/src/sphinx_codelinks/bridge.py new file mode 100644 index 00000000..61dd5012 --- /dev/null +++ b/src/sphinx_codelinks/bridge.py @@ -0,0 +1,195 @@ +"""Covert the generated JSON file created by CodeLinks anaylse to need-extend in RST.""" + +from collections import deque +from dataclasses import MISSING, dataclass, field, fields +import json +from os import linesep +from pathlib import Path +from string import Template +from typing import Any, TypedDict, cast + +from jsonschema import ValidationError, validate + +from sphinx_codelinks.analyse.models import MarkedContentType, SourceMap +from sphinx_codelinks.logger import logger + +NEEDEXTEND_TEMPLATE = Template(""".. needextend:: $need_id + :$remote_url_field: $remote_url + +""") + + +@dataclass +class MarkedContentSchema: + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + filepath: str = field( + metadata={"schema": {"type": "string"}}, + ) + """filepath where the marked content is located.""" + + remote_url: str | None = field( + metadata={"schema": {"type": "string", "nullable": True}} + ) + """remote url which can be directed to the the marked content.""" + + source_map: SourceMap = field( + metadata={ + "schema": { + "type": "object", + "properties": { + "row": {"type": "integer"}, + "column": {"type": "integer"}, + }, + "required": ["row", "column"], + "additionalProperties": False, + } + } + ) + """Coordinate of the marked content in a file""" + + tagged_scope: str | None = field( + metadata={"schema": {"type": "string", "nullable": True}} + ) + """The scoped tagged by the marked content""" + + type: MarkedContentType = field( + metadata={ + "schema": { + "type": "string", + "enum": [key.value for key in MarkedContentType], + } + } + ) + """Type of the marked content.""" + + marker: str | None = field( + default=None, metadata={"schema": {"type": "string", "nullable": True}} + ) + """Marker of the marked content.""" + + need_ids: list[str] | None = field( + default=None, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """Need id refs which is associated to the need items in the documentation.""" + need: dict[str, str | list[str]] | None = field( + default=None, + metadata={ + "schema": { + "type": "object", + "properties": {"title": {"type": "string"}, "id": {"title": "string"}}, + "required": ["title", "id"], + "additionalProperties": True, + } + }, + ) + """Definition of a need item""" + + rst: str | None = field( + default=None, metadata={"schema": {"type": "string", "nullable": True}} + ) + """Extracted rst text.""" + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type + except ValidationError as e: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors + + def check_conditional_required_fields(self) -> list[str]: + errors = [] + if self.type == MarkedContentType.need.value and not self.need: + errors.append( + "Need definition is required for marked content of type 'need'" + ) + elif self.type == MarkedContentType.need_id_refs.value: + if not self.marker: + errors.append( + "Marker is required for marked content of type 'need_id_refs'" + ) + if not self.need_ids: + errors.append( + "Need id refs are required for marked content of type 'need_id_refs'" + ) + elif self.type == MarkedContentType.need.value and not self.need_ids: + errors.append( + "Need id refs are required for marked content of type 'need_id_refs'" + ) + elif self.type == MarkedContentType.rst.value and not self.rst: + errors.append("RST text is required for marked content of type 'rst'") + return errors + + def check_loaded_objs(self) -> list[str]: + return self.check_schema() + self.check_conditional_required_fields() + + +class MarkedObjType(TypedDict): + filepath: str + remote_url: str | None + source_map: SourceMap + tagged_scope: str | None + type: MarkedContentType + need_ids: list[str] | None + need: dict[str, str | list[str]] | None + rst: str | None + + +def convert_makred_content( + jsonpath: Path, outdir: Path, remote_url_field: str = "remote_url" +) -> None: + """Convert marked objects extracted by anaylse CLI to needextend in RST""" + with jsonpath.open("r") as f: + marked_objs = json.load(f) + + warnings = [] + + need_id_refs = [ + obj + for obj in marked_objs + if obj["type"] == MarkedContentType.need_id_refs.value + ] + + for obj in need_id_refs: + schema = MarkedContentSchema(**obj) + obj_warnings: deque[str] = deque() + obj_warnings.extend(schema.check_schema()) + obj_warnings.extend(schema.check_conditional_required_fields()) + if obj_warnings: + obj_warnings.appendleft(f"Marked object {obj} has the following warnings:") + warnings.extend(list(obj_warnings)) + + if warnings: + logger.warning( + f"Marked objects have the following warnings: {linesep.join(warnings)}" + ) + + needextend_texts: list[str] = [] + for obj in need_id_refs: + for need_id in obj["need_ids"]: + needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( + need_id=need_id, + remote_url_field=remote_url_field, + remote_url=obj[remote_url_field], + ) + needextend_texts.append(needextend_text) + + needextend_path = outdir / "needextend.rst" + with needextend_path.open("w") as f: + f.writelines(needextend_texts) diff --git a/src/sphinx_codelinks/logger.py b/src/sphinx_codelinks/logger.py new file mode 100644 index 00000000..05b3f352 --- /dev/null +++ b/src/sphinx_codelinks/logger.py @@ -0,0 +1,94 @@ +from rich.console import Console +from rich.text import Text +import typer + + +class Logger: + __slots__ = ("console", "err_console", "quiet", "verbose") + + def __init__(self, *, verbose: bool = False, quiet: bool = False) -> None: + self.verbose = verbose + self.quiet = quiet + self.console = Console() + self.err_console = Console(stderr=True) + + def configure(self, verbose: bool = False, quiet: bool = False) -> None: + self.verbose = verbose + self.quiet = quiet + + def debug( + self, + *msg: str | Text, + style: str | None = typer.colors.BRIGHT_BLACK, + highlight: bool = False, + markup: bool = False, + console: Console | None = None, + ) -> None: + """Print a debug message. + + Will only be shown if verbose mode is enabled and not in quiet mode. + """ + if self.verbose and not self.quiet: + (console or self.console).print( + *msg, style=style, highlight=highlight, markup=markup + ) + + def info( + self, + *msg: str | Text, + style: str | None = None, + highlight: bool = False, + markup: bool = False, + no_wrap: bool | None = None, + console: Console | None = None, + ) -> None: + """Print an informational message. + + Will be suppressed in quiet mode. + """ + if not self.quiet: + (console or self.console).print( + *msg, style=style, highlight=highlight, markup=markup, no_wrap=no_wrap + ) + + def result( + self, + *msg: str | Text, + style: str | None = None, + highlight: bool = False, + markup: bool = False, + console: Console | None = None, + ) -> None: + """Print a result message, like info but ignores quiet mode.""" + (console or self.console).print( + *msg, style=style, highlight=highlight, markup=markup + ) + + def warning( + self, + *msg: str | Text, + style: str | None = typer.colors.YELLOW, + highlight: bool = False, + markup: bool = False, + console: Console | None = None, + ) -> None: + """Print a warning message.""" + (console or self.console).print( + *msg, style=style, highlight=highlight, markup=markup + ) + + def error( + self, + *msg: str | Text, + style: str | None = typer.colors.RED, + highlight: bool = False, + markup: bool = False, + console: Console | None = None, + ) -> None: + """Print an error message.""" + (console or self.err_console).print( + *msg, style=style, highlight=highlight, markup=markup + ) + + +logger = Logger() diff --git a/tests/data/analyse/marked_content.json b/tests/data/analyse/marked_content.json new file mode 100644 index 00000000..76460f29 --- /dev/null +++ b/tests/data/analyse/marked_content.json @@ -0,0 +1,112 @@ +[ + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": { "row": 2, "column": 13 }, + "end": { "row": 2, "column": 51 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["need_001", "need_002", "need_003", "need_004"], + "marker": "@need-ids:", + "type": "need-id-refs" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/need_id_refs/dummy_1.cpp#L8", + "source_map": { + "start": { "row": 7, "column": 13 }, + "end": { "row": 7, "column": 21 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "need_ids": ["need_003"], + "marker": "@need-ids:", + "type": "need-id-refs" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L1", + "source_map": { + "start": { "row": 0, "column": 4 }, + "end": { "row": 0, "column": 24 } + }, + "tagged_scope": "void foo() {}", + "need": { + "title": "Function Foo", + "id": "IMPL_1", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L4", + "source_map": { + "start": { "row": 3, "column": 4 }, + "end": { "row": 3, "column": 24 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar", + "id": "IMPL_2", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L7", + "source_map": { + "start": { "row": 6, "column": 4 }, + "end": { "row": 6, "column": 39 } + }, + "tagged_scope": "void baz() {}", + "need": { + "title": "Function Baz, as I want it", + "id": "IMPL_3", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L10", + "source_map": { + "start": { "row": 9, "column": 4 }, + "end": { "row": 9, "column": 51 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar,", + "id": "IMPL_4", + "type": "impl", + "links": ["SPEC_1", "SPEC_2"] + }, + "type": "need" + } +] diff --git a/tests/test_bridge.py b/tests/test_bridge.py new file mode 100644 index 00000000..2a3513c8 --- /dev/null +++ b/tests/test_bridge.py @@ -0,0 +1,8 @@ +from pathlib import Path + +from sphinx_codelinks.bridge import convert_makred_content + + +def test_bridge(): + jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" + convert_makred_content(jsonpath, outdir=Path(__file__).parent.parent / "output") From 71e0a3a58bff732683a2c440efdfc2686b2bb513 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 16:43:00 +0200 Subject: [PATCH 02/14] add CLI for needextend bridge --- src/sphinx_codelinks/cmd.py | 28 +++++++++++++- .../{bridge.py => needextend_bridge.py} | 38 +++++++++++++------ tests/test_bridge.py | 9 +++-- 3 files changed, 59 insertions(+), 16 deletions(-) rename src/sphinx_codelinks/{bridge.py => needextend_bridge.py} (82%) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 53dde992..c02facca 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -2,7 +2,7 @@ from os import linesep from pathlib import Path import tomllib -from typing import Annotated, cast +from typing import Annotated, TypeAlias, cast import typer @@ -13,6 +13,8 @@ CodeLinksProjectConfigType, generate_project_configs, ) +from sphinx_codelinks.logger import logger +from sphinx_codelinks.needextend_bridge import convert_marked_content from sphinx_codelinks.source_discover.config import ( CommentType, SourceDiscoverConfig, @@ -25,6 +27,30 @@ ) +OptVerbose: TypeAlias = Annotated[ # noqa: UP040 # has to be TypeAlias + bool, + typer.Option( + ..., + "-v", + "--verbose", + is_flag=True, + help="Show debug information", + rich_help_panel="Logging", + ), +] +OptQuiet: TypeAlias = Annotated[ # noqa: UP040 # has to be TypeAlias + bool, + typer.Option( + ..., + "-q", + "--quiet", + is_flag=True, + help="Only show errors and warnings", + rich_help_panel="Logging", + ), +] + + @app.command(no_args_is_help=True) def analyse( config: Annotated[ diff --git a/src/sphinx_codelinks/bridge.py b/src/sphinx_codelinks/needextend_bridge.py similarity index 82% rename from src/sphinx_codelinks/bridge.py rename to src/sphinx_codelinks/needextend_bridge.py index 61dd5012..7668d24a 100644 --- a/src/sphinx_codelinks/bridge.py +++ b/src/sphinx_codelinks/needextend_bridge.py @@ -40,19 +40,31 @@ def field_names(cls) -> set[str]: "schema": { "type": "object", "properties": { - "row": {"type": "integer"}, - "column": {"type": "integer"}, + "start": { + "type": "object", + "properties": { + "row": {"type": "integer"}, + "column": {"type": "integer"}, + }, + "required": ["row", "column"], + }, + "end": { + "type": "object", + "properties": { + "row": {"type": "integer"}, + "column": {"type": "integer"}, + }, + "required": ["row", "column"], + }, }, - "required": ["row", "column"], + "required": ["start", "end"], "additionalProperties": False, } } ) """Coordinate of the marked content in a file""" - tagged_scope: str | None = field( - metadata={"schema": {"type": "string", "nullable": True}} - ) + tagged_scope: str | None = field(metadata={"schema": {"type": ["string", "null"]}}) """The scoped tagged by the marked content""" type: MarkedContentType = field( @@ -66,20 +78,20 @@ def field_names(cls) -> set[str]: """Type of the marked content.""" marker: str | None = field( - default=None, metadata={"schema": {"type": "string", "nullable": True}} + default=None, metadata={"schema": {"type": ["string", "null"]}} ) """Marker of the marked content.""" need_ids: list[str] | None = field( default=None, - metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + metadata={"schema": {"type": ["array", "null"], "items": {"type": "string"}}}, ) """Need id refs which is associated to the need items in the documentation.""" need: dict[str, str | list[str]] | None = field( default=None, metadata={ "schema": { - "type": "object", + "type": ["object", "null"], "properties": {"title": {"type": "string"}, "id": {"title": "string"}}, "required": ["title", "id"], "additionalProperties": True, @@ -89,7 +101,7 @@ def field_names(cls) -> set[str]: """Definition of a need item""" rst: str | None = field( - default=None, metadata={"schema": {"type": "string", "nullable": True}} + default=None, metadata={"schema": {"type": ["string", "null"]}} ) """Extracted rst text.""" @@ -151,7 +163,7 @@ class MarkedObjType(TypedDict): rst: str | None -def convert_makred_content( +def convert_marked_content( jsonpath: Path, outdir: Path, remote_url_field: str = "remote_url" ) -> None: """Convert marked objects extracted by anaylse CLI to needextend in RST""" @@ -172,7 +184,7 @@ def convert_makred_content( obj_warnings.extend(schema.check_schema()) obj_warnings.extend(schema.check_conditional_required_fields()) if obj_warnings: - obj_warnings.appendleft(f"Marked object {obj} has the following warnings:") + obj_warnings.appendleft(f"{obj} has the following warnings:") warnings.extend(list(obj_warnings)) if warnings: @@ -193,3 +205,5 @@ def convert_makred_content( needextend_path = outdir / "needextend.rst" with needextend_path.open("w") as f: f.writelines(needextend_texts) + + logger.info(f"Generated needextend.rst in {needextend_path}") diff --git a/tests/test_bridge.py b/tests/test_bridge.py index 2a3513c8..2a45a3e9 100644 --- a/tests/test_bridge.py +++ b/tests/test_bridge.py @@ -1,8 +1,11 @@ from pathlib import Path -from sphinx_codelinks.bridge import convert_makred_content +from sphinx_codelinks.needextend_bridge import convert_marked_content -def test_bridge(): +def test_bridge(tmp_path): jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" - convert_makred_content(jsonpath, outdir=Path(__file__).parent.parent / "output") + convert_marked_content(jsonpath, outdir=tmp_path) + needextend_path = tmp_path / "needextend.rst" + + assert needextend_path.exists() From 32da3d30507ed69a45f241974073d1879f19843e Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 17:07:17 +0200 Subject: [PATCH 03/14] make need id valid and update TC --- .../test_analyse[src_dir0-src_paths0].anchors.json | 10 +++++----- tests/data/need_id_refs/dummy_1.cpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/__snapshots__/test_analyse/test_analyse[src_dir0-src_paths0].anchors.json b/tests/__snapshots__/test_analyse/test_analyse[src_dir0-src_paths0].anchors.json index 02b3285b..1904a625 100644 --- a/tests/__snapshots__/test_analyse/test_analyse[src_dir0-src_paths0].anchors.json +++ b/tests/__snapshots__/test_analyse/test_analyse[src_dir0-src_paths0].anchors.json @@ -48,10 +48,10 @@ }, "tagged_scope": "void dummy_func1(){\n //...\n }", "need_ids": [ - "need_001", - "need_002", - "need_003", - "need_004" + "NEED_001", + "NEED_002", + "NEED_003", + "NEED_004" ], "marker": "@need-ids:", "type": "need-id-refs" @@ -71,7 +71,7 @@ }, "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", "need_ids": [ - "need_003" + "NEED_003" ], "marker": "@need-ids:", "type": "need-id-refs" diff --git a/tests/data/need_id_refs/dummy_1.cpp b/tests/data/need_id_refs/dummy_1.cpp index 86b7b2ff..6d90ed23 100644 --- a/tests/data/need_id_refs/dummy_1.cpp +++ b/tests/data/need_id_refs/dummy_1.cpp @@ -1,11 +1,11 @@ #include -// @need-ids: need_001, need_002, need_003, need_004 +// @need-ids: NEED_001, NEED_002, NEED_003, NEED_004 void dummy_func1(){ //... } - // @need-ids: need_003 + // @need-ids: NEED_003 int main() { std::cout << "Starting demo_1..." << std::endl; dummy_func1(); From 56c82868d495a225367e5fa3453c117c4ee66f25 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 17:08:59 +0200 Subject: [PATCH 04/14] created a demo --- pyproject.toml | 15 +++++++++ tests/data/needextend_demo/README.md | 7 +++++ tests/data/needextend_demo/conf.py | 38 +++++++++++++++++++++++ tests/data/needextend_demo/index.rst | 13 ++++++++ tests/data/needextend_demo/needextend.rst | 14 +++++++++ 5 files changed, 87 insertions(+) create mode 100644 tests/data/needextend_demo/README.md create mode 100644 tests/data/needextend_demo/conf.py create mode 100644 tests/data/needextend_demo/index.rst create mode 100644 tests/data/needextend_demo/needextend.rst diff --git a/pyproject.toml b/pyproject.toml index 4c28a130..ad07e4e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,21 @@ codelinks = "sphinx_codelinks.cmd:app" "docs:rm" = "rm -rf docs/_build/html" "docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" "docs:clean" = { chain = ["docs:rm", "docs"] } +# needextend demo +"analyse" = "codelinks analyse tests/data/analyse/default_config.toml" +"analyse:rm" = "rm -rf output/marked_content.json" +"bridge" = "codelinks bridge output/marked_content.json --outdir tests/data/needextend_demo" +"bridge:rm" = "rm -rf tests/data/needextend_demo/needextend.rst" +"demo:rm" = "rm -rf tests/data/needextend_demo/_build" +"demo:build" = "sphinx-build -nW --keep-going -b html -T -c tests/data/needextend_demo tests/data/needextend_demo tests/data/needextend_demo/_build/html" +"demo:clean" = { chain = [ + "demo:rm", + "analyse:rm", + "bridge:rm", + "analyse", + "bridge", + "demo:build", +] } [tool.ruff.lint] extend-select = [ diff --git a/tests/data/needextend_demo/README.md b/tests/data/needextend_demo/README.md new file mode 100644 index 00000000..40706357 --- /dev/null +++ b/tests/data/needextend_demo/README.md @@ -0,0 +1,7 @@ +# Needextend Demo + +Run the following command in project root to have demo generated + +```bash +rye run demo:clean +``` diff --git a/tests/data/needextend_demo/conf.py b/tests/data/needextend_demo/conf.py new file mode 100644 index 00000000..2cef3d08 --- /dev/null +++ b/tests/data/needextend_demo/conf.py @@ -0,0 +1,38 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "test_needextend" +copyright = "2025, useblocks" +author = "team useblocks" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "furo" +# html_static_path = ["_static"] + +extensions = ["sphinx_needs", "sphinx_codelinks"] + +# for needextend to work with remote url +needs_extra_options = ["remote_url"] + +needs_string_links = { + "custom_name": { + "regex": r"^(?P.+)#L(?P.*)?", + "link_url": "{url}", + "link_name": "{{value}}#L{{lineno}}", + "options": ["remote_url"], + } +} diff --git a/tests/data/needextend_demo/index.rst b/tests/data/needextend_demo/index.rst new file mode 100644 index 00000000..13d2c1de --- /dev/null +++ b/tests/data/needextend_demo/index.rst @@ -0,0 +1,13 @@ +.. impl:: title 1 + :id: NEED_001 + +.. impl:: title 2 + :id: NEED_002 + +.. impl:: title 3 + :id: NEED_003 + +.. impl:: title 4 + :id: NEED_004 + +.. include:: needextend.rst diff --git a/tests/data/needextend_demo/needextend.rst b/tests/data/needextend_demo/needextend.rst new file mode 100644 index 00000000..5d932dde --- /dev/null +++ b/tests/data/needextend_demo/needextend.rst @@ -0,0 +1,14 @@ +.. needextend:: NEED_001 + :remote_url: https://github.com/useblocks/sphinx-codelinks/blob/1759d19e8694c049c68e7f95ce93d951d1ed0336/tests/data/need_id_refs/dummy_1.cpp#L3 + +.. needextend:: NEED_002 + :remote_url: https://github.com/useblocks/sphinx-codelinks/blob/1759d19e8694c049c68e7f95ce93d951d1ed0336/tests/data/need_id_refs/dummy_1.cpp#L3 + +.. needextend:: NEED_003 + :remote_url: https://github.com/useblocks/sphinx-codelinks/blob/1759d19e8694c049c68e7f95ce93d951d1ed0336/tests/data/need_id_refs/dummy_1.cpp#L3 + +.. needextend:: NEED_004 + :remote_url: https://github.com/useblocks/sphinx-codelinks/blob/1759d19e8694c049c68e7f95ce93d951d1ed0336/tests/data/need_id_refs/dummy_1.cpp#L3 + +.. needextend:: NEED_003 + :remote_url: https://github.com/useblocks/sphinx-codelinks/blob/1759d19e8694c049c68e7f95ce93d951d1ed0336/tests/data/need_id_refs/dummy_1.cpp#L8 From 5f849301ef8415822932dcff89c401cd212efdf5 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 17:12:40 +0200 Subject: [PATCH 05/14] removed json --- tests/data/analyse/marked_content.json | 112 ------------------------- 1 file changed, 112 deletions(-) delete mode 100644 tests/data/analyse/marked_content.json diff --git a/tests/data/analyse/marked_content.json b/tests/data/analyse/marked_content.json deleted file mode 100644 index 76460f29..00000000 --- a/tests/data/analyse/marked_content.json +++ /dev/null @@ -1,112 +0,0 @@ -[ - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/marked_rst/dummy_1.cpp#L4", - "source_map": { - "start": { "row": 3, "column": 8 }, - "end": { "row": 3, "column": 61 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", - "type": "rst" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/marked_rst/dummy_1.cpp#L14", - "source_map": { - "start": { "row": 13, "column": 7 }, - "end": { "row": 13, "column": 40 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "rst": "..impl:: implement main function ", - "type": "rst" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/need_id_refs/dummy_1.cpp#L3", - "source_map": { - "start": { "row": 2, "column": 13 }, - "end": { "row": 2, "column": 51 } - }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "need_ids": ["need_001", "need_002", "need_003", "need_004"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/need_id_refs/dummy_1.cpp#L8", - "source_map": { - "start": { "row": 7, "column": 13 }, - "end": { "row": 7, "column": 21 } - }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "need_ids": ["need_003"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L1", - "source_map": { - "start": { "row": 0, "column": 4 }, - "end": { "row": 0, "column": 24 } - }, - "tagged_scope": "void foo() {}", - "need": { - "title": "Function Foo", - "id": "IMPL_1", - "type": "impl", - "links": [] - }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L4", - "source_map": { - "start": { "row": 3, "column": 4 }, - "end": { "row": 3, "column": 24 } - }, - "tagged_scope": "void bar() {}", - "need": { - "title": "Function Bar", - "id": "IMPL_2", - "type": "impl", - "links": [] - }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L7", - "source_map": { - "start": { "row": 6, "column": 4 }, - "end": { "row": 6, "column": 39 } - }, - "tagged_scope": "void baz() {}", - "need": { - "title": "Function Baz, as I want it", - "id": "IMPL_3", - "type": "impl", - "links": [] - }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/481a69e5073a749ff68bc5f01f39dc7d4b1e1c45/tests/data/oneline_comment_default/default_oneliners.c#L10", - "source_map": { - "start": { "row": 9, "column": 4 }, - "end": { "row": 9, "column": 51 } - }, - "tagged_scope": "void bar() {}", - "need": { - "title": "Function Bar,", - "id": "IMPL_4", - "type": "impl", - "links": ["SPEC_1", "SPEC_2"] - }, - "type": "need" - } -] From 42d017634c1246934c488f56176f2b7a70dd1f4d Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Tue, 12 Aug 2025 17:20:56 +0200 Subject: [PATCH 06/14] for TC --- tests/data/analyse/marked_content.json | 112 +++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/data/analyse/marked_content.json diff --git a/tests/data/analyse/marked_content.json b/tests/data/analyse/marked_content.json new file mode 100644 index 00000000..c07d2a18 --- /dev/null +++ b/tests/data/analyse/marked_content.json @@ -0,0 +1,112 @@ +[ + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": { "row": 2, "column": 13 }, + "end": { "row": 2, "column": 51 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["NEED_001", "NEED_002", "NEED_003", "NEED_004"], + "marker": "@need-ids:", + "type": "need-id-refs" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L8", + "source_map": { + "start": { "row": 7, "column": 13 }, + "end": { "row": 7, "column": 21 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "need_ids": ["NEED_003"], + "marker": "@need-ids:", + "type": "need-id-refs" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L1", + "source_map": { + "start": { "row": 0, "column": 4 }, + "end": { "row": 0, "column": 24 } + }, + "tagged_scope": "void foo() {}", + "need": { + "title": "Function Foo", + "id": "IMPL_1", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L4", + "source_map": { + "start": { "row": 3, "column": 4 }, + "end": { "row": 3, "column": 24 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar", + "id": "IMPL_2", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L7", + "source_map": { + "start": { "row": 6, "column": 4 }, + "end": { "row": 6, "column": 39 } + }, + "tagged_scope": "void baz() {}", + "need": { + "title": "Function Baz, as I want it", + "id": "IMPL_3", + "type": "impl", + "links": [] + }, + "type": "need" + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L10", + "source_map": { + "start": { "row": 9, "column": 4 }, + "end": { "row": 9, "column": 51 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar,", + "id": "IMPL_4", + "type": "impl", + "links": ["SPEC_1", "SPEC_2"] + }, + "type": "need" + } +] From 3a621904426e07296a97780984920cbf56b5fc4e Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 14:31:42 +0200 Subject: [PATCH 07/14] adapted new output model --- pyproject.toml | 2 +- src/sphinx_codelinks/cmd.py | 43 +++++++++++++++++++++++ src/sphinx_codelinks/needextend_bridge.py | 7 +++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad07e4e7..41421a6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ codelinks = "sphinx_codelinks.cmd:app" "docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" "docs:clean" = { chain = ["docs:rm", "docs"] } # needextend demo -"analyse" = "codelinks analyse tests/data/analyse/default_config.toml" +"analyse" = "codelinks analyse tests/data/configs/minimum_config.toml" "analyse:rm" = "rm -rf output/marked_content.json" "bridge" = "codelinks bridge output/marked_content.json --outdir tests/data/needextend_demo" "bridge:rm" = "rm -rf tests/data/needextend_demo/needextend.rst" diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index c02facca..1128d8da 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -216,6 +216,49 @@ def discover( typer.echo(file_path) +@app.command(no_args_is_help=True) +def bridge( + jsonpath: Annotated[ + Path, + typer.Argument( + ..., + help="Path of the JSON file which contains the extracted markers", + show_default=False, + dir_okay=False, + file_okay=True, + exists=True, + resolve_path=True, + ), + ], + outdir: Annotated[ + Path, + typer.Option( + "--outdir", + "-o", + help="The output directory for needextend.rst", + show_default=True, + dir_okay=True, + file_okay=False, + exists=True, + ), + ] = Path.cwd(), # noqa: B008 # to show default value in this CLI + remote_url_field: Annotated[ + str, + typer.Option( + "--remote-url-field", + "-r", + help="The field name for the remote url", + show_default=True, + ), + ] = "remote_url", # to show default value in this CLI + verbose: OptVerbose = False, + quiet: OptQuiet = False, +) -> None: + """Generate needextend.rst from the extracted obj in JSON.""" + logger.configure(verbose, quiet) + convert_marked_content(jsonpath, outdir, remote_url_field) + + def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: try: with toml_file.open("rb") as f: diff --git a/src/sphinx_codelinks/needextend_bridge.py b/src/sphinx_codelinks/needextend_bridge.py index 7668d24a..6db2581e 100644 --- a/src/sphinx_codelinks/needextend_bridge.py +++ b/src/sphinx_codelinks/needextend_bridge.py @@ -168,10 +168,15 @@ def convert_marked_content( ) -> None: """Convert marked objects extracted by anaylse CLI to needextend in RST""" with jsonpath.open("r") as f: - marked_objs = json.load(f) + marked_content = json.load(f) warnings = [] + # flatten objects + marked_objs: list[MarkedObjType] = [ + obj for objs in marked_content.values() for obj in objs + ] + need_id_refs = [ obj for obj in marked_objs From d31f2ede2966e0e274790862831ee8cb27ddec27 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 16:37:01 +0200 Subject: [PATCH 08/14] changed bridge to write --- src/sphinx_codelinks/cmd.py | 52 ++++- ...edextend_bridge.py => needextend_write.py} | 48 ++-- tests/data/analyse/marked_content.json | 212 +++++++++--------- tests/test_bridge.py | 11 - tests/test_needextend_write.py | 24 ++ 5 files changed, 205 insertions(+), 142 deletions(-) rename src/sphinx_codelinks/{needextend_bridge.py => needextend_write.py} (82%) delete mode 100644 tests/test_bridge.py create mode 100644 tests/test_needextend_write.py diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 1128d8da..dac64bca 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -1,4 +1,5 @@ from collections import deque +from enum import Enum from os import linesep from pathlib import Path import tomllib @@ -6,6 +7,7 @@ import typer +from sphinx_codelinks.analyse.models import MarkedContentType from sphinx_codelinks.analyse.projects import AnalyseProjects from sphinx_codelinks.config import ( CodeLinksConfig, @@ -14,7 +16,7 @@ generate_project_configs, ) from sphinx_codelinks.logger import logger -from sphinx_codelinks.needextend_bridge import convert_marked_content +from sphinx_codelinks.needextend_write import convert_marked_content from sphinx_codelinks.source_discover.config import ( CommentType, SourceDiscoverConfig, @@ -25,7 +27,10 @@ app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} ) - +write_app = typer.Typer( + help="Export marked content to other formats", no_args_is_help=True +) +app.add_typer(write_app, name="write", rich_help_panel="Sub-menus") OptVerbose: TypeAlias = Annotated[ # noqa: UP040 # has to be TypeAlias bool, @@ -216,8 +221,13 @@ def discover( typer.echo(file_path) -@app.command(no_args_is_help=True) -def bridge( +class SupportedMarkerTypes(str, Enum): + need = "need" + need_id_refs = "need-id-refs" + + +@write_app.command("rst", no_args_is_help=True) +def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires jsonpath: Annotated[ Path, typer.Argument( @@ -230,18 +240,27 @@ def bridge( resolve_path=True, ), ], - outdir: Annotated[ + types: Annotated[ + list[SupportedMarkerTypes], + typer.Option( + "--types", + "-t", + help="The types of marked content to be exported", + show_default=True, + ), + ] = [SupportedMarkerTypes.need_id_refs], # noqa: B006 # to show the default value on CLI + outpath: Annotated[ Path, typer.Option( "--outdir", "-o", - help="The output directory for needextend.rst", + help="The output path for needextend.rst", show_default=True, - dir_okay=True, - file_okay=False, - exists=True, + dir_okay=False, + file_okay=True, + exists=False, ), - ] = Path.cwd(), # noqa: B008 # to show default value in this CLI + ] = Path("needextend.rst"), remote_url_field: Annotated[ str, typer.Option( @@ -251,12 +270,23 @@ def bridge( show_default=True, ), ] = "remote_url", # to show default value in this CLI + title: Annotated[ + str | None, + typer.Option( + "--title", + "-t", + help="Give the title to the generated RST file", + show_default=True, + ), + ] = None, # to show default value in this CLI verbose: OptVerbose = False, quiet: OptQuiet = False, ) -> None: """Generate needextend.rst from the extracted obj in JSON.""" logger.configure(verbose, quiet) - convert_marked_content(jsonpath, outdir, remote_url_field) + convert_marked_content( + jsonpath, outpath, remote_url_field, cast(list[MarkedContentType], types), title + ) def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: diff --git a/src/sphinx_codelinks/needextend_bridge.py b/src/sphinx_codelinks/needextend_write.py similarity index 82% rename from src/sphinx_codelinks/needextend_bridge.py rename to src/sphinx_codelinks/needextend_write.py index 6db2581e..fff33630 100644 --- a/src/sphinx_codelinks/needextend_bridge.py +++ b/src/sphinx_codelinks/needextend_write.py @@ -92,7 +92,7 @@ def field_names(cls) -> set[str]: metadata={ "schema": { "type": ["object", "null"], - "properties": {"title": {"type": "string"}, "id": {"title": "string"}}, + "properties": {"title": {"type": "string"}, "id": {"type": "string"}}, "required": ["title", "id"], "additionalProperties": True, } @@ -164,9 +164,15 @@ class MarkedObjType(TypedDict): def convert_marked_content( - jsonpath: Path, outdir: Path, remote_url_field: str = "remote_url" + jsonpath: Path, + outpath: Path, + remote_url_field: str = "remote-url", + types: list[MarkedContentType] | None = None, + title: str | None = None, ) -> None: """Convert marked objects extracted by anaylse CLI to needextend in RST""" + if not types: + types = [MarkedContentType.need_id_refs] with jsonpath.open("r") as f: marked_content = json.load(f) @@ -177,13 +183,9 @@ def convert_marked_content( obj for objs in marked_content.values() for obj in objs ] - need_id_refs = [ - obj - for obj in marked_objs - if obj["type"] == MarkedContentType.need_id_refs.value - ] + intersted_objs = [obj for obj in marked_objs if obj["type"] in types] - for obj in need_id_refs: + for obj in intersted_objs: schema = MarkedContentSchema(**obj) obj_warnings: deque[str] = deque() obj_warnings.extend(schema.check_schema()) @@ -198,17 +200,33 @@ def convert_marked_content( ) needextend_texts: list[str] = [] - for obj in need_id_refs: - for need_id in obj["need_ids"]: + if title: + needextend_texts.append(f"{title}{linesep}{'=' * len(title)}{linesep}{linesep}") + + for obj in intersted_objs: + if obj["type"] == MarkedContentType.need_id_refs.value and obj["need_ids"]: + for need_id in obj["need_ids"]: + needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( + need_id=need_id, + remote_url_field=remote_url_field, + remote_url=obj["remote_url"], + ) + needextend_texts.append(needextend_text) + elif obj["type"] == MarkedContentType.need.value and obj["need"]: + oneline_need_id = obj["need"].get("id") + if not oneline_need_id: + warnings.append( + f"Oneline-need definition {obj} does not have 'id' field in 'need', skipping." + ) + continue needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( - need_id=need_id, + need_id=oneline_need_id, remote_url_field=remote_url_field, - remote_url=obj[remote_url_field], + remote_url=obj["remote_url"], ) needextend_texts.append(needextend_text) - needextend_path = outdir / "needextend.rst" - with needextend_path.open("w") as f: + with outpath.open("w") as f: f.writelines(needextend_texts) - logger.info(f"Generated needextend.rst in {needextend_path}") + logger.info(f"Generated needextend.rst in {outpath}") diff --git a/tests/data/analyse/marked_content.json b/tests/data/analyse/marked_content.json index c07d2a18..e8ad425b 100644 --- a/tests/data/analyse/marked_content.json +++ b/tests/data/analyse/marked_content.json @@ -1,112 +1,114 @@ -[ - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L4", - "source_map": { - "start": { "row": 3, "column": 8 }, - "end": { "row": 3, "column": 61 } +{ + "full_config": [ + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": { "row": 3, "column": 8 }, + "end": { "row": 3, "column": 61 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "rst" }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", - "type": "rst" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L14", - "source_map": { - "start": { "row": 13, "column": 7 }, - "end": { "row": 13, "column": 40 } + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/marked_rst/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L14", + "source_map": { + "start": { "row": 13, "column": 7 }, + "end": { "row": 13, "column": 40 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "rst": "..impl:: implement main function ", + "type": "rst" }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "rst": "..impl:: implement main function ", - "type": "rst" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L3", - "source_map": { - "start": { "row": 2, "column": 13 }, - "end": { "row": 2, "column": 51 } + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": { "row": 2, "column": 13 }, + "end": { "row": 2, "column": 51 } + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["NEED_001", "NEED_002", "NEED_003", "NEED_004"], + "marker": "@need-ids:", + "type": "need-id-refs" }, - "tagged_scope": "void dummy_func1(){\n //...\n }", - "need_ids": ["NEED_001", "NEED_002", "NEED_003", "NEED_004"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L8", - "source_map": { - "start": { "row": 7, "column": 13 }, - "end": { "row": 7, "column": 21 } + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/need_id_refs/dummy_1.cpp#L8", + "source_map": { + "start": { "row": 7, "column": 13 }, + "end": { "row": 7, "column": 21 } + }, + "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", + "need_ids": ["NEED_003"], + "marker": "@need-ids:", + "type": "need-id-refs" }, - "tagged_scope": "int main() {\n std::cout << \"Starting demo_1...\" << std::endl;\n dummy_func1();\n std::cout << \"Demo_1 finished.\" << std::endl;\n return 0;\n }", - "need_ids": ["NEED_003"], - "marker": "@need-ids:", - "type": "need-id-refs" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L1", - "source_map": { - "start": { "row": 0, "column": 4 }, - "end": { "row": 0, "column": 24 } + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L1", + "source_map": { + "start": { "row": 0, "column": 4 }, + "end": { "row": 0, "column": 24 } + }, + "tagged_scope": "void foo() {}", + "need": { + "title": "Function Foo", + "id": "IMPL_1", + "type": "impl", + "links": [] + }, + "type": "need" }, - "tagged_scope": "void foo() {}", - "need": { - "title": "Function Foo", - "id": "IMPL_1", - "type": "impl", - "links": [] + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L4", + "source_map": { + "start": { "row": 3, "column": 4 }, + "end": { "row": 3, "column": 24 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar", + "id": "IMPL_2", + "type": "impl", + "links": [] + }, + "type": "need" }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L4", - "source_map": { - "start": { "row": 3, "column": 4 }, - "end": { "row": 3, "column": 24 } + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L7", + "source_map": { + "start": { "row": 6, "column": 4 }, + "end": { "row": 6, "column": 39 } + }, + "tagged_scope": "void baz() {}", + "need": { + "title": "Function Baz, as I want it", + "id": "IMPL_3", + "type": "impl", + "links": [] + }, + "type": "need" }, - "tagged_scope": "void bar() {}", - "need": { - "title": "Function Bar", - "id": "IMPL_2", - "type": "impl", - "links": [] - }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L7", - "source_map": { - "start": { "row": 6, "column": 4 }, - "end": { "row": 6, "column": 39 } - }, - "tagged_scope": "void baz() {}", - "need": { - "title": "Function Baz, as I want it", - "id": "IMPL_3", - "type": "impl", - "links": [] - }, - "type": "need" - }, - { - "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", - "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L10", - "source_map": { - "start": { "row": 9, "column": 4 }, - "end": { "row": 9, "column": 51 } - }, - "tagged_scope": "void bar() {}", - "need": { - "title": "Function Bar,", - "id": "IMPL_4", - "type": "impl", - "links": ["SPEC_1", "SPEC_2"] - }, - "type": "need" - } -] + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/oneline_comment_default/default_oneliners.c", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/oneline_comment_default/default_oneliners.c#L10", + "source_map": { + "start": { "row": 9, "column": 4 }, + "end": { "row": 9, "column": 51 } + }, + "tagged_scope": "void bar() {}", + "need": { + "title": "Function Bar,", + "id": "IMPL_4", + "type": "impl", + "links": ["SPEC_1", "SPEC_2"] + }, + "type": "need" + } + ] +} diff --git a/tests/test_bridge.py b/tests/test_bridge.py deleted file mode 100644 index 2a45a3e9..00000000 --- a/tests/test_bridge.py +++ /dev/null @@ -1,11 +0,0 @@ -from pathlib import Path - -from sphinx_codelinks.needextend_bridge import convert_marked_content - - -def test_bridge(tmp_path): - jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" - convert_marked_content(jsonpath, outdir=tmp_path) - needextend_path = tmp_path / "needextend.rst" - - assert needextend_path.exists() diff --git a/tests/test_needextend_write.py b/tests/test_needextend_write.py new file mode 100644 index 00000000..f4edd386 --- /dev/null +++ b/tests/test_needextend_write.py @@ -0,0 +1,24 @@ +from pathlib import Path + +import pytest + +from sphinx_codelinks.analyse.models import MarkedContentType +from sphinx_codelinks.needextend_write import convert_marked_content + + +@pytest.mark.parametrize( + "types", + [ + None, # Default case + [MarkedContentType.need_id_refs], + [MarkedContentType.need], + [MarkedContentType.rst], + [MarkedContentType.need, MarkedContentType.need_id_refs], + ], +) +def test_convert_marked_content(types, tmp_path): + jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" + outpath = tmp_path / "needextend.rst" + convert_marked_content(jsonpath, outpath, types=types) + + assert outpath.exists() From 6e55a2835821ac7ad5f8c94c72b94796a889ce90 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 17:10:42 +0200 Subject: [PATCH 09/14] removed option types --- src/sphinx_codelinks/cmd.py | 24 +++--------------------- src/sphinx_codelinks/needextend_write.py | 24 ++++++------------------ tests/test_needextend_write.py | 22 +++++++--------------- 3 files changed, 16 insertions(+), 54 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index dac64bca..a5943b40 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -1,5 +1,4 @@ from collections import deque -from enum import Enum from os import linesep from pathlib import Path import tomllib @@ -7,7 +6,6 @@ import typer -from sphinx_codelinks.analyse.models import MarkedContentType from sphinx_codelinks.analyse.projects import AnalyseProjects from sphinx_codelinks.config import ( CodeLinksConfig, @@ -221,11 +219,6 @@ def discover( typer.echo(file_path) -class SupportedMarkerTypes(str, Enum): - need = "need" - need_id_refs = "need-id-refs" - - @write_app.command("rst", no_args_is_help=True) def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires jsonpath: Annotated[ @@ -240,21 +233,12 @@ def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires resolve_path=True, ), ], - types: Annotated[ - list[SupportedMarkerTypes], - typer.Option( - "--types", - "-t", - help="The types of marked content to be exported", - show_default=True, - ), - ] = [SupportedMarkerTypes.need_id_refs], # noqa: B006 # to show the default value on CLI outpath: Annotated[ Path, typer.Option( - "--outdir", + "--outpath", "-o", - help="The output path for needextend.rst", + help="The output path for generated rst file", show_default=True, dir_okay=False, file_okay=True, @@ -284,9 +268,7 @@ def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires ) -> None: """Generate needextend.rst from the extracted obj in JSON.""" logger.configure(verbose, quiet) - convert_marked_content( - jsonpath, outpath, remote_url_field, cast(list[MarkedContentType], types), title - ) + convert_marked_content(jsonpath, outpath, remote_url_field, title) def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: diff --git a/src/sphinx_codelinks/needextend_write.py b/src/sphinx_codelinks/needextend_write.py index fff33630..fa6287a6 100644 --- a/src/sphinx_codelinks/needextend_write.py +++ b/src/sphinx_codelinks/needextend_write.py @@ -167,12 +167,9 @@ def convert_marked_content( jsonpath: Path, outpath: Path, remote_url_field: str = "remote-url", - types: list[MarkedContentType] | None = None, title: str | None = None, ) -> None: """Convert marked objects extracted by anaylse CLI to needextend in RST""" - if not types: - types = [MarkedContentType.need_id_refs] with jsonpath.open("r") as f: marked_content = json.load(f) @@ -183,7 +180,11 @@ def convert_marked_content( obj for objs in marked_content.values() for obj in objs ] - intersted_objs = [obj for obj in marked_objs if obj["type"] in types] + intersted_objs = [ + obj + for obj in marked_objs + if obj["type"] == MarkedContentType.need_id_refs.value + ] for obj in intersted_objs: schema = MarkedContentSchema(**obj) @@ -212,21 +213,8 @@ def convert_marked_content( remote_url=obj["remote_url"], ) needextend_texts.append(needextend_text) - elif obj["type"] == MarkedContentType.need.value and obj["need"]: - oneline_need_id = obj["need"].get("id") - if not oneline_need_id: - warnings.append( - f"Oneline-need definition {obj} does not have 'id' field in 'need', skipping." - ) - continue - needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( - need_id=oneline_need_id, - remote_url_field=remote_url_field, - remote_url=obj["remote_url"], - ) - needextend_texts.append(needextend_text) with outpath.open("w") as f: f.writelines(needextend_texts) - logger.info(f"Generated needextend.rst in {outpath}") + logger.info(f"Generated {outpath}") diff --git a/tests/test_needextend_write.py b/tests/test_needextend_write.py index f4edd386..82770629 100644 --- a/tests/test_needextend_write.py +++ b/tests/test_needextend_write.py @@ -1,24 +1,16 @@ from pathlib import Path -import pytest - -from sphinx_codelinks.analyse.models import MarkedContentType from sphinx_codelinks.needextend_write import convert_marked_content -@pytest.mark.parametrize( - "types", - [ - None, # Default case - [MarkedContentType.need_id_refs], - [MarkedContentType.need], - [MarkedContentType.rst], - [MarkedContentType.need, MarkedContentType.need_id_refs], - ], -) -def test_convert_marked_content(types, tmp_path): +def test_convert_marked_content(tmp_path): jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" outpath = tmp_path / "needextend.rst" - convert_marked_content(jsonpath, outpath, types=types) + convert_marked_content(jsonpath, outpath) assert outpath.exists() + + with outpath.open("r") as f: + content = f.readlines() + + assert content From 27f7fdc142f6f97ee0847d34309dc8e8403a4094 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 22:56:07 +0200 Subject: [PATCH 10/14] improve msg for validation --- src/sphinx_codelinks/cmd.py | 22 +++++++- src/sphinx_codelinks/needextend_write.py | 44 ++++++--------- tests/test_cmd.py | 70 ++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 28 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index a5943b40..624c40cf 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -1,4 +1,5 @@ from collections import deque +import json from os import linesep from pathlib import Path import tomllib @@ -14,7 +15,7 @@ generate_project_configs, ) from sphinx_codelinks.logger import logger -from sphinx_codelinks.needextend_write import convert_marked_content +from sphinx_codelinks.needextend_write import MarkedObjType, convert_marked_content from sphinx_codelinks.source_discover.config import ( CommentType, SourceDiscoverConfig, @@ -268,7 +269,24 @@ def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires ) -> None: """Generate needextend.rst from the extracted obj in JSON.""" logger.configure(verbose, quiet) - convert_marked_content(jsonpath, outpath, remote_url_field, title) + try: + with jsonpath.open("r") as f: + marked_content = json.load(f) + except Exception as e: + raise typer.BadParameter( + f"Failed to load marked content from {jsonpath}: {e}" + ) from e + + marked_objs: list[MarkedObjType] = [ + obj for objs in marked_content.values() for obj in objs + ] + + errors = convert_marked_content(marked_objs, outpath, remote_url_field, title) + if errors: + raise typer.BadParameter( + f"Errors occurred during conversion: {linesep.join(errors)}" + ) + typer.echo(f"Generated {outpath}") def load_config_from_toml(toml_file: Path) -> CodeLinksConfigType: diff --git a/src/sphinx_codelinks/needextend_write.py b/src/sphinx_codelinks/needextend_write.py index fa6287a6..09e1d499 100644 --- a/src/sphinx_codelinks/needextend_write.py +++ b/src/sphinx_codelinks/needextend_write.py @@ -2,7 +2,6 @@ from collections import deque from dataclasses import MISSING, dataclass, field, fields -import json from os import linesep from pathlib import Path from string import Template @@ -11,7 +10,6 @@ from jsonschema import ValidationError, validate from sphinx_codelinks.analyse.models import MarkedContentType, SourceMap -from sphinx_codelinks.logger import logger NEEDEXTEND_TEMPLATE = Template(""".. needextend:: $need_id :$remote_url_field: $remote_url @@ -164,21 +162,13 @@ class MarkedObjType(TypedDict): def convert_marked_content( - jsonpath: Path, + marked_objs: list[MarkedObjType], outpath: Path, remote_url_field: str = "remote-url", title: str | None = None, -) -> None: +) -> list[str] | None: """Convert marked objects extracted by anaylse CLI to needextend in RST""" - with jsonpath.open("r") as f: - marked_content = json.load(f) - - warnings = [] - - # flatten objects - marked_objs: list[MarkedObjType] = [ - obj for objs in marked_content.values() for obj in objs - ] + errors = [] intersted_objs = [ obj @@ -187,18 +177,20 @@ def convert_marked_content( ] for obj in intersted_objs: - schema = MarkedContentSchema(**obj) - obj_warnings: deque[str] = deque() - obj_warnings.extend(schema.check_schema()) - obj_warnings.extend(schema.check_conditional_required_fields()) - if obj_warnings: - obj_warnings.appendleft(f"{obj} has the following warnings:") - warnings.extend(list(obj_warnings)) - - if warnings: - logger.warning( - f"Marked objects have the following warnings: {linesep.join(warnings)}" - ) + try: + schema = MarkedContentSchema(**obj) + except TypeError as e: + errors.append(str(e)) + continue + obj_errors: deque[str] = deque() + obj_errors.extend(schema.check_schema()) + obj_errors.extend(schema.check_conditional_required_fields()) + if obj_errors: + obj_errors.appendleft(f"{obj} has the following errors:") + errors.extend(list(obj_errors)) + + if errors: + return errors needextend_texts: list[str] = [] if title: @@ -217,4 +209,4 @@ def convert_marked_content( with outpath.open("w") as f: f.writelines(needextend_texts) - logger.info(f"Generated {outpath}") + return None diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 6d5adbdf..cca3c217 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -212,3 +212,73 @@ def test_analyse_project_negative(projects, output_lines, tmp_path: Path) -> Non assert result.exit_code != 0 for line in output_lines: assert line in result.stdout + + +def test_write_rst_invalid_json(tmp_path: Path) -> None: + json_obj = { + "whatever": "json", + "not": "expected", + } + json_text = json.dumps(json_obj) + json_text = json_text.replace(",", "") + jsonpath = tmp_path / "invalid_json.json" + with jsonpath.open("w") as f: + f.write(json_text) + + options = [ + "write", + "rst", + str(jsonpath), + "--outpath", + str(tmp_path / "out.rst"), + ] + result = runner.invoke(app, options) + + assert result.exit_code != 0 + assert "Expecting ',' delimiter: line 1 column 21 (char 20)" in result.stdout + + +@pytest.mark.parametrize( + ("json_objs", "output_lines"), + [ + ( + [ + { + "filepath": 123, + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/951e40e7845f06d5cfc4ca20ebb984308fdaf985/tests/data/marked_rst/dummy_1.cpp#L4", + "source_map": { + "start": {"row": 3, "column": 8}, + "end": {"row": 3, "column": 61}, + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "rst": ".. impl:: implement dummy function 1\n :id: IMPL_71\n", + "type": "need-id-refs", + } + ], + [ + "Invalid value: Errors occurred", + "Schema validation error in field 'filepath': 123 is not of type 'string'", + "Marker is required for marked content of type 'need_id_refs' ", + "Need id refs are required for marked content of type 'need_id_refs'", + ], + ), + ], +) +def test_write_rst_negative(json_objs: list[dict], output_lines, tmp_path) -> None: + to_dump = {"project_1": json_objs} + jsonpath = Path("invalid_objs.json") + with jsonpath.open("w") as f: + json.dump(to_dump, f) + outpath = tmp_path / "needextend.rst" + options = [ + "write", + "rst", + str(jsonpath), + "--outpath", + str(outpath), + ] + result = runner.invoke(app, options) + + assert result.exit_code != 0 + for line in output_lines: + assert line in result.stdout From a8417edc573f4c9016d12f2d75b2752a59aa402b Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Thu, 21 Aug 2025 23:37:01 +0200 Subject: [PATCH 11/14] fixed n:1 mapping of need_ids and urls --- src/sphinx_codelinks/cmd.py | 6 +++- src/sphinx_codelinks/needextend_write.py | 42 ++++++++++++++---------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 624c40cf..bc0077ff 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -281,11 +281,15 @@ def write_rst( # noqa: PLR0913 # for CLI, so it takes as many as it requires obj for objs in marked_content.values() for obj in objs ] - errors = convert_marked_content(marked_objs, outpath, remote_url_field, title) + needextend_texts, errors = convert_marked_content( + marked_objs, remote_url_field, title + ) if errors: raise typer.BadParameter( f"Errors occurred during conversion: {linesep.join(errors)}" ) + with outpath.open("w") as f: + f.writelines(needextend_texts) typer.echo(f"Generated {outpath}") diff --git a/src/sphinx_codelinks/needextend_write.py b/src/sphinx_codelinks/needextend_write.py index 09e1d499..ee676d4d 100644 --- a/src/sphinx_codelinks/needextend_write.py +++ b/src/sphinx_codelinks/needextend_write.py @@ -3,7 +3,6 @@ from collections import deque from dataclasses import MISSING, dataclass, field, fields from os import linesep -from pathlib import Path from string import Template from typing import Any, TypedDict, cast @@ -163,13 +162,12 @@ class MarkedObjType(TypedDict): def convert_marked_content( marked_objs: list[MarkedObjType], - outpath: Path, remote_url_field: str = "remote-url", title: str | None = None, -) -> list[str] | None: +) -> tuple[list[str], list[str]]: """Convert marked objects extracted by anaylse CLI to needextend in RST""" errors = [] - + needextend_texts: list[str] = [] intersted_objs = [ obj for obj in marked_objs @@ -190,23 +188,31 @@ def convert_marked_content( errors.extend(list(obj_errors)) if errors: - return errors - - needextend_texts: list[str] = [] - if title: - needextend_texts.append(f"{title}{linesep}{'=' * len(title)}{linesep}{linesep}") + return needextend_texts, errors + # handle N:1 mapping of need_id and remote_url + id_urls: dict[str, list[str]] = {} for obj in intersted_objs: if obj["type"] == MarkedContentType.need_id_refs.value and obj["need_ids"]: for need_id in obj["need_ids"]: - needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( - need_id=need_id, - remote_url_field=remote_url_field, - remote_url=obj["remote_url"], - ) - needextend_texts.append(needextend_text) + if need_id not in id_urls: + id_urls[need_id] = [] + if obj["remote_url"]: + id_urls[need_id].append(obj["remote_url"]) - with outpath.open("w") as f: - f.writelines(needextend_texts) + if title: + needextend_texts.append(f"{title}{linesep}{'=' * len(title)}{linesep}{linesep}") - return None + for id, urls in id_urls.items(): + # Connect urls with comma, if there are multiple urls + if not urls: + continue + remote_url = ",".join(urls) + needextend_text = NEEDEXTEND_TEMPLATE.safe_substitute( + need_id=id, + remote_url_field=remote_url_field, + remote_url=remote_url, + ) + needextend_texts.append(needextend_text) + + return needextend_texts, errors From 5b910991ddba1d187be4fe59625f6133cebd4b2c Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 22 Aug 2025 10:11:10 +0200 Subject: [PATCH 12/14] updated TCs --- tests/test_needextend_write.py | 75 +++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/tests/test_needextend_write.py b/tests/test_needextend_write.py index 82770629..4a2342ec 100644 --- a/tests/test_needextend_write.py +++ b/tests/test_needextend_write.py @@ -1,16 +1,71 @@ -from pathlib import Path +import pytest from sphinx_codelinks.needextend_write import convert_marked_content -def test_convert_marked_content(tmp_path): - jsonpath = Path(__file__).parent / "data" / "analyse" / "marked_content.json" - outpath = tmp_path / "needextend.rst" - convert_marked_content(jsonpath, outpath) +@pytest.mark.parametrize( + ("markers", "texts"), + [ + ( + [ + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": {"row": 2, "column": 13}, + "end": {"row": 2, "column": 51}, + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["NEED_001", "NEED_002", "NEED_003", "NEED_004"], + "marker": "@need-ids:", + "type": "need-id-refs", + }, + ], + [ + ".. needextend:: NEED_001\n :remote-url: https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3\n\n", + ".. needextend:: NEED_002\n :remote-url: https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3\n\n", + ".. needextend:: NEED_003\n :remote-url: https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3\n\n", + ".. needextend:: NEED_004\n :remote-url: https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3\n\n", + ], + ), + ( + [ + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3", + "source_map": { + "start": {"row": 2, "column": 13}, + "end": {"row": 2, "column": 51}, + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["NEED_001"], + "marker": "@need-ids:", + "type": "need-id-refs", + }, + { + "filepath": "/home/jui-wen/git_repo/ub/sphinx-codelinks/tests/data/need_id_refs/dummy_1.cpp", + "remote_url": "https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L10", + "source_map": { + "start": {"row": 2, "column": 13}, + "end": {"row": 2, "column": 51}, + }, + "tagged_scope": "void dummy_func1(){\n //...\n }", + "need_ids": ["NEED_001"], + "marker": "@need-ids:", + "type": "need-id-refs", + }, + ], + [ + ".. needextend:: NEED_001\n :remote-url: https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L3,https://github.com/useblocks/sphinx-codelinks/blob/main/tests/data/need_id_refs/dummy_1.cpp#L10\n\n" + ], + ), + ], +) +def test_convert_marked_content(markers, texts): + # Normalize line endings + texts = [line.replace("\r\n", "\n").replace("\r", "\n") for line in texts] + needextend_texts, errors = convert_marked_content(markers) - assert outpath.exists() + assert not errors - with outpath.open("r") as f: - content = f.readlines() - - assert content + assert needextend_texts == texts From 3a94b6b9790194ecd36b38a0d6a116b3dc2f2ed7 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 22 Aug 2025 10:14:32 +0200 Subject: [PATCH 13/14] updated rye script --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41421a6f..67040ff4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,16 +79,16 @@ codelinks = "sphinx_codelinks.cmd:app" # needextend demo "analyse" = "codelinks analyse tests/data/configs/minimum_config.toml" "analyse:rm" = "rm -rf output/marked_content.json" -"bridge" = "codelinks bridge output/marked_content.json --outdir tests/data/needextend_demo" -"bridge:rm" = "rm -rf tests/data/needextend_demo/needextend.rst" +"write" = "codelinks write rst output/marked_content.json --outpath tests/data/needextend_demo/needextend.rst" +"write:rm" = "rm -rf tests/data/needextend_demo/needextend.rst" "demo:rm" = "rm -rf tests/data/needextend_demo/_build" "demo:build" = "sphinx-build -nW --keep-going -b html -T -c tests/data/needextend_demo tests/data/needextend_demo tests/data/needextend_demo/_build/html" "demo:clean" = { chain = [ "demo:rm", "analyse:rm", - "bridge:rm", + "write:rm", "analyse", - "bridge", + "write", "demo:build", ] } From b3add9efebd6ec09ac5f536d01c8e8e6aa141572 Mon Sep 17 00:00:00 2001 From: juiwenchen Date: Fri, 22 Aug 2025 10:21:02 +0200 Subject: [PATCH 14/14] update TC for other platforms --- tests/test_cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index cca3c217..60a4c2e3 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -235,7 +235,7 @@ def test_write_rst_invalid_json(tmp_path: Path) -> None: result = runner.invoke(app, options) assert result.exit_code != 0 - assert "Expecting ',' delimiter: line 1 column 21 (char 20)" in result.stdout + assert "Expecting" in result.stdout @pytest.mark.parametrize( @@ -258,7 +258,7 @@ def test_write_rst_invalid_json(tmp_path: Path) -> None: [ "Invalid value: Errors occurred", "Schema validation error in field 'filepath': 123 is not of type 'string'", - "Marker is required for marked content of type 'need_id_refs' ", + "Marker is required for marked content of type 'need_id_refs'", "Need id refs are required for marked content of type 'need_id_refs'", ], ),