Manage test results logging and metadata via pytest options

Author: ian-sift

python/lib/sift_client/_internal/low_level_wrappers/test_results.py

@@ -4,7 +4,6 @@
 import json
 import logging
 import re
-import sys
 import uuid
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -376,16 +375,7 @@ async def create_test_report(
                 )
             return TestReport._from_proto(simulated_proto)
 
-        try:
-            response = await self._grpc_client.get_stub(TestReportServiceStub).CreateTestReport(
-                request
-            )
-        except Exception:
-            print(
-                f"AAAA {self._grpc_client._config.uri} {self._grpc_client._config.use_ssl}",
-                file=sys.stderr,
-            )
-            raise
+        response = await self._grpc_client.get_stub(TestReportServiceStub).CreateTestReport(request)
         grpc_test_report = cast("CreateTestReportResponse", response).test_report
         return TestReport._from_proto(grpc_test_report)
 

python/lib/sift_client/_tests/util/conftest.py

@@ -1,5 +1,14 @@
-"""Override report_context to disable log file simulation for integration tests in this directory so that we can exercise the context manager when no log file is provided."""
+import pytest
 
-from sift_client.util.test_results.pytest_util import (
-    report_context_no_logging as report_context,  # noqa: F401
-)
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    existing_options = [opt.names() for opt in parser._anonymous.options]
+    # Flatten the list of lists into a single list of strings
+    flat_options = [item for sublist in existing_options for item in sublist]
+    if not any("--sift-test-results-log-file" in name for name in flat_options):
+        parser.addoption("--sift-test-results-log-file", action="store_true", default="false")
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Configure the pytest configuration to disable the Sift test results log file."""
+    config.option.sift_test_results_log_file = False

python/lib/sift_client/util/test_results/__init__.py

@@ -58,7 +58,12 @@ def main(self):
   - If you want each module(file) to be marked as a step w/ each test as a substep, import the `module_substep` fixture as well.
 - The `report_context` fixture requires a fixture `sift_client` returning an `SiftClient` instance to be passed in.
 
-Note: FedRAMP users: report_context will log test results to a temp file to avoid API calls during test execution. If this is a shared environment, you should import the `report_context_no_logging` fixture instead.
+Note: FedRAMP users: report_context will log test results to a temp file to avoid API calls during test execution. If this is a shared environment, you can disable logging by passing ``--sift-test-results-log-file=false``.
+
+#### Configuration
+
+- Git metadata: You can configure the test results by passing the `--sift-test-results-git-metadata` flag to pytest in your commandline, conftest.py file, or as an addopt in your pyproject.toml file (https://docs.pytest.org/en/stable/reference/customize.html#configuration).
+- Log file: You can configure the log file by passing the `--sift-test-results-log-file` flag to pytest in your commandline, conftest.py file, or as an addopt in your pyproject.toml file.
 
 ###### Example at top of your test file or in your conftest.py file:
 
@@ -101,10 +106,10 @@ def test_example(report_context, step):
     client_has_connection,
     module_substep,
     module_substep_check_connection,
+    pytest_addoption,
     pytest_runtest_makereport,
     report_context,
     report_context_check_connection,
-    report_context_no_logging,
     step,
     step_check_connection,
 )
@@ -115,10 +120,10 @@ def test_example(report_context, step):
     "client_has_connection",
     "module_substep",
     "module_substep_check_connection",
+    "pytest_addoption",
     "pytest_runtest_makereport",
     "report_context",
     "report_context_check_connection",
-    "report_context_no_logging",
     "step",
     "step_check_connection",
 ]

python/lib/sift_client/util/test_results/context_manager.py

@@ -38,6 +38,29 @@
 logger = logging.getLogger(__name__)
 
 
+def _git_metadata() -> dict[str, str] | None:
+    """Return git branch and commit hash, or None if not in a git repo."""
+    try:
+        branch = subprocess.check_output(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            stderr=subprocess.DEVNULL,
+            text=True,
+        ).strip()
+        commit = subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"],
+            stderr=subprocess.DEVNULL,
+            text=True,
+        ).strip()
+        repo = subprocess.check_output(
+            ["git", "remote", "get-url", "origin"],
+            stderr=subprocess.DEVNULL,
+            text=True,
+        ).strip()
+        return {"git_repo": repo, "git_branch": branch, "git_commit": commit}
+    except Exception:
+        return None
+
+
 class ReportContext(AbstractContextManager):
     """Context manager for a new TestReport. See usage example in __init__.py."""
 
@@ -59,6 +82,7 @@ def __init__(
         system_operator: str | None = None,
         test_case: str | None = None,
         log_file: str | Path | bool | None = None,
+        include_git_metadata: bool = False,
     ):
         """Initialize a new report context.
 
@@ -70,6 +94,7 @@ def __init__(
             test_case: The name of the test case. Will default to the basename of the file containing the test if not provided.
             log_file: If True, create a temp log file. If a path, use that path.
                 All create/update operations will be logged to this file.
+            include_git_metadata: If True, include git metadata in the report.
         """
         self.client = client
         self.step_is_open = False
@@ -99,6 +124,7 @@ def __init__(
             end_time=datetime.now(timezone.utc),
             status=TestStatus.IN_PROGRESS,
             system_operator=system_operator,
+            metadata=_git_metadata() if include_git_metadata else None,  # type: ignore
         )
         self.report = client.test_results.create(create, log_file=self.log_file)
 

python/lib/sift_client/util/test_results/pytest_util.py

@@ -16,6 +16,39 @@
 REPORT_CONTEXT: ReportContext | None = None
 
 
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Register Sift-specific command-line options."""
+    parser.addoption(
+        "--sift-test-results-log-file",
+        default=None,
+        help="Path to write the Sift test result log file. "
+        "Use 'true' (default) to auto-create a temp file, "
+        "False, 'false', or 'none' to disable logging, "
+        "or a file path to write to a specific location.",
+    )
+    parser.addoption(
+        "--sift-test-results-git-metadata",
+        action="store_true",
+        default=True,
+        help="Include git metadata in the Sift test results.",
+    )
+
+
+def _resolve_log_file(pytestconfig: pytest.Config | None) -> str | Path | bool | None:
+    """Determine log_file value from --sift-test-results-log-file option."""
+    raw: str | None = None
+    if pytestconfig is not None:
+        raw = pytestconfig.getoption("--sift-test-results-log-file", default=None)
+    if raw is None:
+        return True
+    lower = str(raw).lower()
+    if lower in ("true", "1"):
+        return True
+    if lower in ("false", "none"):
+        return None
+    return Path(raw)
+
+
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[Any]):
     """You should import this hook to capture any AssertionErrors that occur during the test. If not included, any assert failures in a test will not automatically fail the step."""
@@ -34,7 +67,7 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[Any]):
 def _report_context_impl(
     sift_client: SiftClient,
     request: pytest.FixtureRequest,
-    log_file: str | Path | bool | None = True,
+    pytestconfig: pytest.Config | None = None,
 ) -> Generator[ReportContext | None, None, None]:
     test_path = Path(request.config.invocation_params.args[0])
     base_name = (
@@ -43,11 +76,18 @@ def _report_context_impl(
         else "pytest " + " ".join(request.config.invocation_params.args)
     )
     test_case = test_path if test_path.exists() else base_name
+    log_file = _resolve_log_file(pytestconfig)
+    include_git_metadata = (
+        bool(pytestconfig.getoption("--sift-test-results-git-metadata", default=False))
+        if pytestconfig
+        else False
+    )
     with ReportContext(
         sift_client,
         name=f"{base_name} {datetime.now(timezone.utc).isoformat()}",
         test_case=str(test_case),
         log_file=log_file,
+        include_git_metadata=include_git_metadata,
     ) as context:
         # Set a global so we can access this in pytest hooks.
         global REPORT_CONTEXT
@@ -57,18 +97,14 @@ def _report_context_impl(
 
 @pytest.fixture(scope="session", autouse=True)
 def report_context(
-    sift_client: SiftClient, request: pytest.FixtureRequest
+    sift_client: SiftClient, request: pytest.FixtureRequest, pytestconfig: pytest.Config
 ) -> Generator[ReportContext | None, None, None]:
-    """Create a report context for the session."""
-    yield from _report_context_impl(sift_client, request, log_file=True)
+    """Create a report context for the session.
 
-
-@pytest.fixture(scope="session", autouse=True)
-def report_context_no_logging(
-    sift_client: SiftClient, request: pytest.FixtureRequest
-) -> Generator[ReportContext | None, None, None]:
-    """Create a report context for the session with logging disabled."""
-    yield from _report_context_impl(sift_client, request, log_file=None)
+    The log file destination is controlled by ``--sift-test-results-log-file``.
+    Defaults to a temp file when not set.
+    """
+    yield from _report_context_impl(sift_client, request, pytestconfig=pytestconfig)
 
 
 def _step_impl(
@@ -127,11 +163,14 @@ def client_has_connection(sift_client):
 
 @pytest.fixture(scope="session", autouse=True)
 def report_context_check_connection(
-    sift_client: SiftClient, client_has_connection: bool, request: pytest.FixtureRequest
+    sift_client: SiftClient,
+    client_has_connection: bool,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
 ) -> Generator[ReportContext | None, None, None]:
     """Create a report context for the session. Doesn't run if the client has no connection to the Sift server."""
     if client_has_connection:
-        yield from _report_context_impl(sift_client, request)
+        yield from _report_context_impl(sift_client, request, pytestconfig=pytestconfig)
     else:
         yield None