extend source code linker to work with any repo (#84)

so far score was hardcoded

Author: MaximilianSoerenPollak

src/extensions/score_metamodel/metamodel.yaml

@@ -14,7 +14,7 @@
 
 needs_types_base_options:
   optional_options:
-    source_code_link: ^https://github.com/eclipse-score/.*$
+    source_code_link: ^https://github.com/.*
   # Custom semantic validation rules
   prohibited_words:
     # req-Id: tool_req__docs_common_attr_title

src/extensions/score_source_code_linker/__init__.py

@@ -19,7 +19,9 @@
 from sphinx_needs.data import NeedsMutable, SphinxNeedsData, NeedsInfoType
 from sphinx_needs.logging import get_logger
 
-from src.extensions.score_source_code_linker.parse_source_files import GITHUB_BASE_URL
+from src.extensions.score_source_code_linker.parse_source_files import (
+    get_github_base_url,
+)
 
 LOGGER = get_logger(__name__)
 LOGGER.setLevel("DEBUG")
@@ -108,6 +110,7 @@ def add_source_link(app: Sphinx, env: BuildEnvironment) -> None:
     # For some reason the prefix 'sphinx_needs internally' is CAPSLOCKED.
     # So we have to make sure we uppercase the prefixes
     prefixes = [x["id_prefix"].upper() for x in app.config.needs_external_needs]
+    github_base_url = get_github_base_url() + "/blob/"
     try:
         with open(path) as f:
             gh_json = json.load(f)
@@ -117,7 +120,7 @@ def add_source_link(app: Sphinx, env: BuildEnvironment) -> None:
             if need is None:
                 # NOTE: manipulating link to remove git-hash,
                 # making the output file location more readable
-                files = [x.replace(GITHUB_BASE_URL, "").split("/", 1)[-1] for x in link]
+                files = [x.replace(github_base_url, "").split("/", 1)[-1] for x in link]
                 LOGGER.warning(
                     f"Could not find {id} in the needs id's. "
                     + f"Found in file(s): {files}",

src/extensions/score_source_code_linker/parse_source_files.py

@@ -15,8 +15,10 @@
 import json
 import logging
 import os
+import sys
 import subprocess
 
+
 # Importing from collections.abc as typing.Callable is deprecated since Python 3.9
 from collections.abc import Callable
 from pathlib import Path
@@ -28,7 +30,63 @@
     "# req-Id:",
 ]
 
-GITHUB_BASE_URL = "https://github.com/eclipse-score/score/blob/"
+
+def get_github_base_url() -> str:
+    git_root = find_git_root()
+    repo = get_github_repo_info(git_root)
+    return f"https://github.com/{repo}"
+
+
+def parse_git_output(str_line: str) -> str:
+    if len(str_line.split()) < 2:
+        logger.warning(
+            f"Got wrong input line from 'get_github_repo_info'. Input: {str_line}. Expected example: 'origin git@github.com:user/repo.git'"
+        )
+        return ""
+    url = str_line.split()[1]  # Get the URL part
+    # Handle SSH format (git@github.com:user/repo.git)
+    if url.startswith("git@"):
+        path = url.split(":")[1]
+    else:
+        path = "/".join(url.split("/")[3:])  # Get part after github.com/
+    return path.replace(".git", "")
+
+
+def get_github_repo_info(git_root_cwd: Path) -> str:
+    process = subprocess.run(
+        ["git", "remote", "-v"], capture_output=True, text=True, cwd=git_root_cwd
+    )
+    repo = ""
+    for line in process.stdout.split("\n"):
+        if "origin" in line and "(fetch)" in line:
+            repo = parse_git_output(line)
+            break
+    else:
+        # If we do not find 'origin' we just take the first line
+        logger.info(
+            "Did not find origin remote name. Will now take first result from: 'git remote -v'"
+        )
+        repo = parse_git_output(process.stdout.split("\n")[0])
+    assert repo != "", (
+        "Remote repository is not defined. Make sure you have a remote set. Check this via 'git remote -v'"
+    )
+    return repo
+
+
+def find_git_root():
+    """
+    This is copied from 'find_runfiles' as the import does not work for some reason.
+    This should be fixed.
+    """
+    git_root = Path(__file__).resolve()
+    while not (git_root / ".git").exists():
+        git_root = git_root.parent
+        if git_root == Path("/"):
+            sys.exit(
+                "Could not find git root. Please run this script from the "
+                "root of the repository."
+            )
+    return git_root
 
 
 def get_git_hash(file_path: str) -> str:
@@ -48,7 +106,7 @@ def get_git_hash(file_path: str) -> str:
     try:
         abs_path = Path(file_path).resolve()
         if not os.path.isfile(abs_path):
-            print(f"File not found: {abs_path}", flush=True)
+            logger.warning(f"File not found: {abs_path}")
             return "file_not_found"
         result = subprocess.run(
             ["git", "log", "-n", "1", "--pretty=format:%H", "--", abs_path],
@@ -61,12 +119,13 @@ def get_git_hash(file_path: str) -> str:
         assert all(c in "0123456789abcdef" for c in decoded_result)
         return decoded_result
     except Exception as e:
-        print(f"Unexpected error: {abs_path}: {e}", flush=True)
+        logger.warning(f"Unexpected error: {abs_path}: {e}")
         return "error"
 
 
 def extract_requirements(
     source_file: str,
+    github_base_url: str,
     git_hash_func: Callable[[str], str] | None = get_git_hash,
 ) -> dict[str, list[str]]:
     """
@@ -110,7 +169,7 @@ def extract_requirements(
                 check_tag = cleaned_line.split(":")[1].strip()
                 if check_tag:
                     req_id = cleaned_line.split(":")[-1].strip()
-                    link = f"{GITHUB_BASE_URL}{hash}/{source_file}#L{line_number}"
+                    link = f"{github_base_url}/blob/{hash}/{source_file}#L{line_number}"
                     requirement_mapping[req_id].append(link)
     return requirement_mapping
 
@@ -124,11 +183,13 @@ def extract_requirements(
 
     logger.info(f"Parsing source files: {args.inputs}")
 
+    # Finding the GH URL
+    gh_base_url = get_github_base_url()
     requirement_mappings: dict[str, list[str]] = collections.defaultdict(list)
     for input in args.inputs:
         with open(input) as f:
             for source_file in f:
-                rm = extract_requirements(source_file.strip())
+                rm = extract_requirements(source_file.strip(), gh_base_url)
                 for k, v in rm.items():
                     requirement_mappings[k].extend(v)
     with open(args.output, "w") as f:

src/extensions/score_source_code_linker/tests/test_requirement_links.py

@@ -12,14 +12,20 @@
 # *******************************************************************************
 from collections import defaultdict
 from collections.abc import Callable
+from gettext import find
 from pathlib import Path
 
 import pytest
+import logging
 from pytest import TempPathFactory
 from src.extensions.score_source_code_linker.parse_source_files import (
-    GITHUB_BASE_URL,
+    get_github_base_url,
+    find_git_root,
+    get_github_repo_info,
     extract_requirements,
     get_git_hash,
+    parse_git_output,
+    logger as scl_logger,
 )
 
 
@@ -74,39 +80,42 @@ def dummy_git_hash_func(input: str) -> Callable[[str], str]:
 
 def test_extract_requirements(create_tmp_files: Path):
     root_dir = create_tmp_files
-
+    github_base_url = get_github_base_url()
     results_dict1 = extract_requirements(
-        str(root_dir / "testfile.txt"), dummy_git_hash_func("no-hash")
+        str(root_dir / "testfile.txt"), github_base_url, dummy_git_hash_func("no-hash")
     )
     expected_dict1: dict[str, list[str]] = defaultdict(list)
     expected_dict1["TEST_REQ__LINKED_ID"].append(
-        f"{GITHUB_BASE_URL}no-hash/{root_dir}/testfile.txt#L7"
+        f"{github_base_url}/blob/no-hash/{root_dir}/testfile.txt#L7"
     )
     expected_dict1["TEST_REQ__LINKED_TRACE"].append(
-        f"{GITHUB_BASE_URL}no-hash/{root_dir}/testfile.txt#L11"
+        f"{github_base_url}/blob/no-hash/{root_dir}/testfile.txt#L11"
     )
 
     # Assumed random hash here to test if passed correctly
     results_dict2 = extract_requirements(
         str(root_dir / "testfile2.txt"),
+        github_base_url,
         dummy_git_hash_func("aacce4887ceea1f884135242a8c182db1447050"),
     )
     expected_dict2: dict[str, list[str]] = defaultdict(list)
     expected_dict2["TEST_REQ__LINKED_DIFFERENT_FILE"].append(
-        f"{GITHUB_BASE_URL}aacce4887ceea1f884135242a8c182db1447050/{root_dir}/testfile2.txt#L3"
+        f"{github_base_url}/blob/aacce4887ceea1f884135242a8c182db1447050/{root_dir}/testfile2.txt#L3"
     )
 
-    results_dict3 = extract_requirements(str(root_dir / "testfile3.txt"))
+    results_dict3 = extract_requirements(
+        str(root_dir / "testfile3.txt"), github_base_url
+    )
     expected_dict3: dict[str, list[str]] = defaultdict(list)
 
     # if there is no git-hash returned from command.
     # This happens if the file is new and not committed yet.
     results_dict4 = extract_requirements(
-        str(root_dir / "testfile2.txt"), dummy_git_hash_func("")
+        str(root_dir / "testfile2.txt"), github_base_url, dummy_git_hash_func("")
     )
     expected_dict4: dict[str, list[str]] = defaultdict(list)
     expected_dict4["TEST_REQ__LINKED_DIFFERENT_FILE"].append(
-        f"{GITHUB_BASE_URL}/{root_dir}/testfile2.txt#L3"
+        f"{github_base_url}/blob//{root_dir}/testfile2.txt#L3"
     )
 
     assert results_dict1 == expected_dict1
@@ -118,3 +127,62 @@ def test_extract_requirements(create_tmp_files: Path):
 def test_get_git_hash():
     assert get_git_hash("testfile.x") == "file_not_found"
     assert get_git_hash("") == "file_not_found"
+
+
+# These tests aren't great / exhaustive, but an okay first step into the right direction.
+
+
+def test_get_github_repo_info():
+    # I'd argue the happy path is tested with the other ones?
+    with pytest.raises(AssertionError):
+        get_github_repo_info(Path("."))
+
+
+git_test_data_ok = [
+    (
+        "origin  https://github.com/eclipse-score/test-repo.git (fetch)",
+        "eclipse-score/test-repo",
+    ),
+    (
+        "origin  git@github.com:eclipse-score/test-repo.git (fetch)",
+        "eclipse-score/test-repo",
+    ),
+    ("origin  git@github.com:eclipse-score/test-repo.git", "eclipse-score/test-repo"),
+    ("upstream  git@github.com:upstream/repo.git (fetch)", "upstream/repo"),
+]
+
+
+@pytest.mark.parametrize("input,output", git_test_data_ok)
+def test_parse_git_output_ok(input, output):
+    assert output == parse_git_output(input)
+
+
+git_test_data_bad = [
+    ("origin ", ""),
+    (
+        "    ",
+        "",
+    ),
+]
+
+
+@pytest.mark.parametrize("input,output", git_test_data_bad)
+def test_parse_git_output_bad(caplog, input, output):
+    with caplog.at_level(logging.WARNING, logger=scl_logger.name):
+        result = parse_git_output(input)
+    assert len(caplog.messages) == 1
+    assert caplog.records[0].levelname == "WARNING"
+    assert (
+        f"Got wrong input line from 'get_github_repo_info'. Input: {input}. Expected example: 'origin git@github.com:user/repo.git'"
+        in caplog.records[0].message
+    )
+    assert output == result
+
+
+def test_get_github_base_url():
+    # Not really a great test imo.
+    git_root = find_git_root()
+    repo = get_github_repo_info(git_root)
+    expected = f"https://github.com/{repo}"
+    actual = get_github_base_url()
+    assert expected == actual

src/extensions/score_source_code_linker/tests/test_source_link.py

@@ -16,11 +16,18 @@
 
 import pytest
 from pytest import TempPathFactory
-from src.extensions.score_source_code_linker.parse_source_files import GITHUB_BASE_URL
+from src.extensions.score_source_code_linker.parse_source_files import (
+    get_github_base_url,
+)
 from sphinx.testing.util import SphinxTestApp
 from sphinx_needs.data import SphinxNeedsData
 
 
+def construct_gh_url() -> str:
+    gh = get_github_base_url()
+    return f"{gh}/blob/"
+
+
 @pytest.fixture(scope="session")
 def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path:
     return tmp_path_factory.mktemp("sphinx")
@@ -105,22 +112,24 @@ def basic_needs():
 
 @pytest.fixture(scope="session")
 def example_source_link_text_all_ok():
+    github_base_url = construct_gh_url()
     return {
         "TREQ_ID_1": [
-            f"{GITHUB_BASE_URL}aacce4887ceea1f884135242a8c182db1447050/tools/sources/implementation1.py#L2",
-            f"{GITHUB_BASE_URL}/tools/sources/implementation_2_new_file.py#L20",
+            f"{github_base_url}aacce4887ceea1f884135242a8c182db1447050/tools/sources/implementation1.py#L2",
+            f"{github_base_url}/tools/sources/implementation_2_new_file.py#L20",
         ],
         "TREQ_ID_2": [
-            f"{GITHUB_BASE_URL}f53f50a0ab1186329292e6b28b8e6c93b37ea41/tools/sources/implementation1.py#L18"
+            f"{github_base_url}f53f50a0ab1186329292e6b28b8e6c93b37ea41/tools/sources/implementation1.py#L18"
         ],
     }
 
 
 @pytest.fixture(scope="session")
 def example_source_link_text_non_existent():
+    github_base_url = construct_gh_url()
     return {
         "TREQ_ID_200": [
-            f"{GITHUB_BASE_URL}f53f50a0ab1186329292e6b28b8e6c93b37ea41/tools/sources/bad_implementation.py#L17"
+            f"{github_base_url}f53f50a0ab1186329292e6b28b8e6c93b37ea41/tools/sources/bad_implementation.py#L17"
         ],
     }
 
@@ -132,6 +141,7 @@ def test_source_link_integration_ok(
     example_source_link_text_all_ok: dict[str, list[str]],
     sphinx_base_dir: Path,
 ):
+    github_url = construct_gh_url()
     app = sphinx_app_setup(basic_conf, basic_needs, example_source_link_text_all_ok)
     try:
         app.build()