Merge branch 'main' into tracability-improvements

Author: FScholPer

.github/workflows/consumer_test.yml

@@ -25,7 +25,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        consumer: ["process_description", "score", "module_template"]
+        consumer: [
+          "process_description and local",
+          "process_description and remote",
+          "score and local",
+          "score and remote",
+          "module_template and local",
+          "module_template and remote",
+        ]
 
     steps:
       - name: 🛡️ Harden Runner
@@ -35,74 +42,15 @@ jobs:
           egress-policy: audit
 
       - name: Checkout PR
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@v6.0.2
 
       - name: Prepare Python
-        run: |
-          bazel run --lockfile_mode=error //:ide_support
+        run: bazel run //:ide_support
 
-      - name: Prepare report directory
-        run: |
-          mkdir -p reports
-
-      # The pipefail ensures that non 0 exit codes inside the pytest execution get carried into the pipe
-      # & make the tests red in the end. Without this we only would check the exit code of the 'tee' command.
       - name: Run Consumer tests
-
-        run: |
-          pytest_rc=0
-          .venv_docs/bin/python -m pytest -vv src/tests/ \
-          --repo="$CONSUMER" \
-          --junitxml="reports/${{ matrix.consumer }}.xml" \
-          || pytest_rc=$?
-
-
-          if [ -f "consumer_test.log" ]; then
-            src_log="consumer_test.log"
-          else
-            echo "consumer_test.log not found; expected at ./consumer_test.log"
-          exit ${pytest_rc:-1}
-          fi
-
-          dest_log="reports/${{ matrix.consumer }}.log"
-          mv "$src_log" "$dest_log"
-
-          tail -n 15 "$dest_log" >> "$GITHUB_STEP_SUMMARY"
-
-          cat "$dest_log"
-          exit $pytest_rc
+        run: .venv_docs/bin/python -m pytest -vv -s src/tests/test_consumer.py -k "$CONSUMER"
         env:
           FORCE_COLOR: "1"
           TERM: xterm-256color
           PYTHONUNBUFFERED: "1"
           CONSUMER: ${{ matrix.consumer }}
-
-      - name: Upload consumer test report
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: tests-${{ matrix.consumer }}
-          path: reports/
-
-  summarize:
-    needs: test
-    runs-on: ubuntu-latest
-    if: ${{ always() }}
-    steps:
-      - name: Prepare consumer report directory
-        run: |
-          mkdir -p tests-report
-
-      - name: Download individual consumer reports
-        uses: actions/download-artifact@v4
-        with:
-          pattern: tests-*
-          path: tests-report
-          merge-multiple: true
-
-      - name: Upload bundled consumer report
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: tests-report
-          path: tests-report

.gitignore

@@ -27,3 +27,7 @@ __pycache__/
 # bug: This file is created in repo root on test discovery.
 /consumer_test.log
 .clwb
+
+# AI
+/.codex
+/.claude

docs.bzl

@@ -81,25 +81,27 @@ def _merge_sourcelinks(name, sourcelinks, known_good = None):
         extra_srcs = [known_good]
         known_good_arg = "--known_good $(location %s)" % known_good
 
+    merge_sourcelinks_tool = Label("//scripts_bazel:merge_sourcelinks")
+
     native.genrule(
         name = name,
         srcs = sourcelinks + extra_srcs,
         outs = [name + ".json"],
         cmd = """
-        $(location @score_docs_as_code//scripts_bazel:merge_sourcelinks) \
+        $(location {merge_sourcelinks_tool}) \
             --output $@ \
             {known_good_arg} \
             $(SRCS)
-        """.format(known_good_arg = known_good_arg),
-        tools = ["@score_docs_as_code//scripts_bazel:merge_sourcelinks"],
+        """.format(known_good_arg = known_good_arg, merge_sourcelinks_tool = merge_sourcelinks_tool),
+        tools = [merge_sourcelinks_tool],
     )
 
 def _missing_requirements(deps):
     """Add Python hub dependencies if they are missing."""
     found = []
     missing = []
     def _target_to_packagename(target):
-        return target.split("/")[-1].split(":")[0]
+        return str(target).split("/")[-1].split(":")[0]
     all_packages = [_target_to_packagename(pkg) for pkg in all_requirements]
     def _find(pkg):
         for dep in deps:
@@ -156,10 +158,12 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     module_deps = deps
     deps = deps + _missing_requirements(deps)
     deps = deps + [
-        "@score_docs_as_code//src:plantuml_for_python",
-        "@score_docs_as_code//src/extensions/score_sphinx_bundle:score_sphinx_bundle",
+        Label("//src:plantuml_for_python"),
+        Label("//src/extensions/score_sphinx_bundle:score_sphinx_bundle"),
     ]
 
+    incremental_src = Label("//src:incremental.py")
+
     sphinx_build_binary(
         name = "sphinx_build",
         visibility = ["//visibility:private"],
@@ -212,8 +216,9 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
         "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)",
     } | metamodel_env
     if known_good:
-        docs_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")"
-        docs_sources_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")"
+        known_good_str = str(known_good)
+        docs_env["KNOWN_GOOD_JSON"] = "$(location " + known_good_str + ")"
+        docs_sources_env["KNOWN_GOOD_JSON"] = "$(location " + known_good_str + ")"
         docs_data.append(known_good)
         combo_data.append(known_good)
 
@@ -222,7 +227,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "docs",
         tags = ["cli_help=Build documentation:\nbazel run //:docs"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = docs_data,
         deps = deps,
         env = docs_env
@@ -232,7 +237,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "docs_combo",
         tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = combo_data,
         deps = deps,
         env = docs_sources_env
@@ -248,7 +253,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "docs_link_check",
         tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = docs_data,
         deps = deps,
         env = docs_env
@@ -258,7 +263,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "docs_check",
         tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = docs_data,
         deps = deps,
         env = docs_env
@@ -268,7 +273,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "live_preview",
         tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = docs_data,
         deps = deps,
         env = docs_env
@@ -278,7 +283,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     py_binary(
         name = "live_preview_combo_experimental",
         tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"],
-        srcs = ["@score_docs_as_code//src:incremental.py"],
+        srcs = [incremental_src],
         data = combo_data,
         deps = deps,
         env = docs_sources_env
@@ -316,10 +321,18 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
         allow_persistent_workers = False,
     )
 
+    native.genrule(
+        name = "metrics_json",
+        srcs = [":needs_json"],
+        outs = ["metrics.json"],
+        cmd = "cp $(location :needs_json)/metrics.json $@",
+        visibility = ["//visibility:public"],
+    )
+
     native.alias(
         name = "traceability_gate",
-        actual = "@score_docs_as_code//scripts_bazel:traceability_gate",
-        tags = ["cli_help=Enforce traceability coverage thresholds:\nbazel run //:traceability_gate -- --metrics-json bazel-bin/needs_json/_build/needs/metrics.json"],
+        actual = Label("//scripts_bazel:traceability_gate"),
+        tags = ["cli_help=Enforce traceability coverage thresholds:\nbazel run //:traceability_gate -- --metrics-json $(location //:metrics_json)"],
     )
 
 def _sourcelinks_json(name, srcs):
@@ -334,15 +347,17 @@ def _sourcelinks_json(name, srcs):
     """
     output_file = name + ".json"
 
+    generate_sourcelinks_tool = Label("//scripts_bazel:generate_sourcelinks")
+
     native.genrule(
         name = name,
         srcs = srcs,
         outs = [output_file],
         cmd = """
-        $(location @score_docs_as_code//scripts_bazel:generate_sourcelinks) \
+        $(location {generate_sourcelinks_tool}) \
             --output $@ \
             $(SRCS)
-        """,
-        tools = ["@score_docs_as_code//scripts_bazel:generate_sourcelinks"],
+        """.format(generate_sourcelinks_tool = generate_sourcelinks_tool),
+        tools = [generate_sourcelinks_tool],
         visibility = ["//visibility:public"],
     )

docs/how-to/other_modules.rst

@@ -48,7 +48,7 @@ Example `BUILD` snippet (consumer module):
 
 .. code-block:: starlark
 
-    load("@rules_docs//:docs.bzl", "docs")
+    load("@score_docs_as_code//:docs.bzl", "docs")
     docs(
       data = [
          "@score_process//:needs_json",

docs/how-to/setup.md

@@ -23,7 +23,7 @@ designed to enhance documentation capabilities in S-CORE.
 Add the module to your `MODULE.bazel` file:
 
 ```starlark
-bazel_dep(name = "score_docs_as_code", version = "4.0.1")
+bazel_dep(name = "score_docs_as_code", version = "4.1.0")
 ```
 
 And make sure to also add the S-core Bazel registry to your `.bazelrc` file

docs/how-to/source_to_doc_links.rst

@@ -53,7 +53,7 @@ Finally, pass the scan results to the ``docs`` rule as ``scan_code`` attribute.
           "foo.py",
           "bar.cpp",
           "data.yaml",
-      ] + glob(["subdir/**/.py"]),
+      ] + glob(["subdir/**/*.py"]),
    )
 
    docs(

docs/how-to/write_docs.rst

@@ -15,6 +15,9 @@
 Write Documentation
 ===================
 
+Outside Documentation
+^^^^^^^^^^^^^^^^^^^^^
+
 `Sphinx <https://www.sphinx-doc.org/en/master/>`_:
 the documentation generator we use.
 
@@ -23,3 +26,47 @@ the plain-text markup language used for most source files in this project.
 
 `Sphinx-Needs <https://sphinx-needs.readthedocs.io/en/latest/>`_:
 a Sphinx extension that models requirements, tests, tasks and other "needs" inside the docs.
+
+
+Advanced
+^^^^^^^^
+
+Needextend
+~~~~~~~~~~
+
+Needextend allows you to extend needs that are defined in the documentation.
+The scope of allowed behaviour in Docs-As-Code for needextend is limited as we do not allow all of its usecases.
+
+Allowed usecases:
+
+* Setting an attribute or Link **IF** it is not already set in the need that is getting modified
+* Appending to a list of links
+
+Not Allowed:
+
+* Overwriting an attribute or Link that already is set in the need that gets modified
+* Deleting an attribute or Link
+
+.. code-block:: rst
+
+      .. stkh_req:: Test Req Extends 1
+         :id: stkh_req__test__need_extends_1
+         :status: invalid
+
+
+      # ✅ ALLOWED => The replacing of attributes on needs that are NOT set.
+      .. needextend:: c.this_doc() and id == 'stkh_req__test__need_extends_1'
+         :safety: NO
+
+
+      # ❌ NOT ALLOWED => Overwriting attributes that are already set in the need
+      .. needextend:: c.this_doc() and id == 'stkh_req__test__need_extends_1'
+         :status: valid
+
+
+For further documentation on needextends please `look here <https://sphinx-needs.readthedocs.io/en/latest/directives/needextend.html>`_
+
+.. note::
+
+   In the future we will enable a check that needextends will only modify needs in the current document.
+   You can ensure this by adding `c.this_doc()` to the filter string of the need.

docs/internals/extensions/extension_guide.md

@@ -6,7 +6,7 @@ This document will help you with the most important building blocks and provide
 
 ## Getting Started
 
-1. Create a new folder in `docs/_tooling/extensions` called `score_<name of your extension>`
+1. Create a new folder in `src/extensions` called `score_<name of your extension>`
 2. Copy the template inside the `__init__.py`
 3. Adapt to your needs
 
@@ -76,7 +76,7 @@ extensions = [
 ]
 ```
 
-> **Important:** There cannot be any BUILD file inside the entire 'extensions' folder, as that would break the Python imports.
+> **Important:** Each extension needs its own `BUILD` file to declare the `py_library` target and its dependencies.
 
 
 
@@ -90,16 +90,13 @@ We perform testing with unit tests as well as integration tests that validate th
 ### Where to Place Your Test Code
 
 ```bash
-_tooling/
-├── extensions/
-│   ├── README.md
-│   ├── score_draw_uml_funcs/
-│   ├── YOUR_EXTENSION/
-│   │   ├── __init__.py  # your python code (setup needs to be in here)
-│   │   ├── xyz.py       # your python code (if you need/want to split it across different files)
-│   │   └── tests/
-│   │       ├── test_xyz.py             # unit tests
-│   │       └── test_YOUR_EXTENSION.py  # integration tests
+src/extensions/
+├── YOUR_EXTENSION/
+│   ├── __init__.py  # your python code (setup needs to be in here)
+│   ├── xyz.py       # your python code (if you need/want to split it across different files)
+│   └── tests/
+│       ├── test_xyz.py             # unit tests
+│       └── test_YOUR_EXTENSION.py  # integration tests
 ├── score_metamodel/
 ├── score_plantuml.py
 ```

docs/internals/extensions/index.rst

@@ -19,8 +19,6 @@
 Extensions
 ==========
 
-Hello there
-
 
 .. grid:: 1 1 3 3
    :class-container: score-grid