Use weblog metadata to declare weblog's category, and connect to scenario matrix builder

tests/test_the_test/test_ci_orchestrator.py

@@ -1,11 +1,12 @@
 from functools import lru_cache
+from pathlib import Path
 
-from utils import scenarios
+from utils import scenarios, logger
+from utils.const import COMPONENT_GROUPS
 from utils._context.weblog_metadata import WeblogMetaData
-
+from utils._context._scenarios import get_all_scenarios, Scenario
 from utils.scripts.ci_orchestrators.workflow_data import (
     _get_endtoend_weblogs,
-    _is_supported,
     get_endtoend_definitions,
 )
 
@@ -42,9 +43,12 @@ def test_get_endtoend_definitions():
 
 @scenarios.test_the_test
 def test_ipv6_is_not_supported_for_uds_weblogs():
-    assert not _is_supported(get_weblog("dotnet", "uds"), scenarios.ipv6, "dev")
-    assert not _is_supported(get_weblog("python", "uds-flask"), scenarios.ipv6, "dev")
-    assert _is_supported(get_weblog("python", "flask-poc"), scenarios.ipv6, "dev")
+    def _is_supported(weblog: WeblogMetaData, scenario: Scenario) -> bool:
+        return weblog.support_scenario(scenario.name, scenario.weblog_categories)
+
+    assert not _is_supported(get_weblog("dotnet", "uds"), scenarios.ipv6)
+    assert not _is_supported(get_weblog("python", "uds-flask"), scenarios.ipv6)
+    assert _is_supported(get_weblog("python", "flask-poc"), scenarios.ipv6)
 
 
 @scenarios.test_the_test
@@ -139,3 +143,145 @@ def test_otel_collector():
             "weblog_instance": 1,
         }
     ]
+
+
+@scenarios.test_the_test
+def test_legacy_scenario_matrix():
+    has_error = False
+
+    for library in sorted(COMPONENT_GROUPS.all):
+        for weblog in sorted(WeblogMetaData.load(library), key=lambda w: w.name):
+            for scenario in get_all_scenarios():
+                if scenario is scenarios.performances:  # sepcial use case, it is not inside the workflow
+                    continue
+
+                legacy = _is_supported_legacy(weblog, scenario, "")
+                new_value = weblog.support_scenario(scenario.name, scenario.weblog_categories)
+                if legacy is not new_value:
+                    has_error = True
+                    logger.error((library, legacy, new_value, weblog.name, scenario.name, scenario.weblog_categories))
+
+    assert not has_error
+
+
+@scenarios.test_the_test
+def test_all_weblog_has_metadata():
+    for library in sorted(COMPONENT_GROUPS.all):
+        folder = Path(f"utils/build/docker/{library}")
+        if folder.exists():  # some lib does not have any weblog
+            names = [
+                f.name.replace(".Dockerfile", "")
+                for f in folder.iterdir()
+                if f.suffix == ".Dockerfile" and ".base." not in f.name and f.is_file()
+            ]
+
+            known_weblog_names = {w.name.split("@")[0] for w in WeblogMetaData.load(library)}
+
+            for name in names:
+                assert name in known_weblog_names, (
+                    f"Please add {name} in utils/build/docker/{library}/weblog_metadata.yml"
+                )
+
+
+# copy of the old function to test compatibility. To be removed once all good
+def _is_supported_legacy(weblog: WeblogMetaData, scenario: Scenario, _ci_environment: str) -> bool:
+    # this function will remove some couple scenarios/weblog that are not supported
+
+    def _is_uds_weblog(weblog: str) -> bool:
+        return weblog == "uds" or weblog.startswith("uds-")
+
+    if scenario.github_workflow != "endtoend":
+        return False
+
+    library = weblog.library
+    weblog_name = weblog.name
+
+    # Only Allow Lambda scenarios for the lambda libraries
+    is_lambda_library = library in (
+        "python_lambda",
+        "java_lambda",
+        "nodejs_lambda",
+        "ruby_lambda",
+    )
+    is_lambda_scenario = scenario.name in (
+        "APPSEC_LAMBDA_DEFAULT",
+        "APPSEC_LAMBDA_BLOCKING",
+        "APPSEC_LAMBDA_API_SECURITY",
+        "APPSEC_LAMBDA_RASP",
+        "APPSEC_LAMBDA_INFERRED_SPANS",
+    )
+    if is_lambda_library != is_lambda_scenario:
+        return False
+
+    # open-telemetry-automatic
+    if scenario.name == "OTEL_INTEGRATIONS":
+        possible_values: tuple = (
+            ("java_otel", "spring-boot-otel"),
+            ("nodejs_otel", "express4-otel"),
+            ("python_otel", "flask-poc-otel"),
+        )
+        if (library, weblog_name) not in possible_values:
+            return False
+
+    # open-telemetry-manual
+    if scenario.name in ("OTEL_LOG_E2E", "OTEL_METRIC_E2E", "OTEL_TRACING_E2E"):
+        if (library, weblog_name) != ("java_otel", "spring-boot-native"):
+            return False
+
+    if scenario.name in ("GRAPHQL_APPSEC", "GRAPHQL_ERROR_TRACKING"):
+        possible_values: tuple = (
+            ("golang", "gqlgen"),
+            ("golang", "graph-gophers"),
+            ("golang", "graphql-go"),
+            ("ruby", "graphql23"),
+            ("nodejs", "express4"),
+            ("nodejs", "uds-express4"),
+            ("nodejs", "express4-typescript"),
+            ("nodejs", "express5"),
+        )
+        if (library, weblog_name) not in possible_values:
+            return False
+
+    if scenario.name in ("PERFORMANCES",):
+        return False
+
+    if scenario.name == "IPV6":
+        if library == "ruby" or _is_uds_weblog(weblog_name):
+            return False
+
+    if scenario.name in ("CROSSED_TRACING_LIBRARIES",):
+        if weblog_name in ("python3.12", "django-py3.13", "spring-boot-payara"):
+            # python 3.13 issue : APMAPI-1096
+            return False
+
+    if scenario.name in ("APPSEC_MISSING_RULES", "APPSEC_CORRUPTED_RULES") and library in ("cpp_nginx", "cpp_httpd"):
+        # C++ 1.2.0 freeze when the rules file is missing
+        return False
+
+    if weblog_name in ["gqlgen", "graph-gophers", "graphql-go", "graphql23"]:
+        if scenario.name not in ("GRAPHQL_APPSEC",):
+            return False
+
+    # open-telemetry-manual
+    if weblog_name == "spring-boot-native":
+        if scenario.name not in ("OTEL_LOG_E2E", "OTEL_METRIC_E2E", "OTEL_TRACING_E2E"):
+            return False
+
+    # open-telemetry-automatic
+    if weblog_name in ["express4-otel", "flask-poc-otel", "spring-boot-otel"]:
+        if scenario.name not in ("OTEL_INTEGRATIONS",):
+            return False
+
+    # Go proxies
+    if weblog.name in ("envoy", "haproxy"):
+        if scenario.name not in ("DEFAULT", "APPSEC_BLOCKING"):
+            return False
+
+    # otel collector
+    if weblog_name == "otel_collector" or scenario.name in ("OTEL_COLLECTOR", "OTEL_COLLECTOR_E2E"):
+        return weblog_name == "otel_collector" and scenario.name in ("OTEL_COLLECTOR", "OTEL_COLLECTOR_E2E")
+
+    if "@" in weblog_name or scenario.name == "INTEGRATION_FRAMEWORKS":
+        return "@" in weblog_name and scenario.name == "INTEGRATION_FRAMEWORKS"
+
+    return True

tests/test_the_test/test_minimal_number_of_scenarios.py

@@ -4,7 +4,8 @@
 
 import pytest
 
-from utils._context._scenarios import get_all_scenarios, EndToEndScenario, scenarios
+from utils._context._scenarios import get_all_scenarios, scenarios
+from utils._context._scenarios.endtoend import EndToEndScenario
 from utils import logger
 
 # Map of scenario pairs that cannot be merged and their specific reasons