feat(quality): add AI-assisted OSPS-QA-06.02 evaluation

.gitignore

@@ -1,7 +1,9 @@
 # ignore the generated artifacts
-pvtr-github-repo
-github-repo
-badge-url
+/pvtr-github-repo
+/pvtr-github-repo-dev
+/github-repo
+/github-repo-dev
+/badge-url
 evaluation_results
 
 # ignore any local dev config file

cmd/badge-url/main_test.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestRunMissingFileFlag(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	code := run([]string{}, &stdout, &stderr)
+	if code != 2 {
+		t.Fatalf("expected exit code 2, got %d", code)
+	}
+	if !strings.Contains(stderr.String(), "requires -f") {
+		t.Fatalf("expected usage message on stderr, got %q", stderr.String())
+	}
+	if stdout.Len() != 0 {
+		t.Fatalf("expected empty stdout, got %q", stdout.String())
+	}
+}
+
+func TestRunInvalidFlag(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"--nonexistent"}, &stdout, &stderr)
+	if code != 2 {
+		t.Fatalf("expected exit code 2, got %d", code)
+	}
+}
+
+func TestRunNonexistentFile(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"-f", "/nonexistent/results.yaml"}, &stdout, &stderr)
+	if code != 1 {
+		t.Fatalf("expected exit code 1, got %d", code)
+	}
+	if !strings.Contains(stderr.String(), "read results file") {
+		t.Fatalf("expected file error on stderr, got %q", stderr.String())
+	}
+}
+
+func TestRunInvalidYAML(t *testing.T) {
+	filePath := writeTestFile(t, "not valid yaml: [")
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"-f", filePath}, &stdout, &stderr)
+	if code != 1 {
+		t.Fatalf("expected exit code 1, got %d", code)
+	}
+}
+
+func TestRunValidResults(t *testing.T) {
+	filePath := writeTestFile(t, minimalResultsYAML())
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"-f", filePath, "-badge", "baseline-1"}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr.String())
+	}
+	output := stdout.String()
+	if !strings.Contains(output, "bestpractices.dev") {
+		t.Fatalf("expected bestpractices.dev URL in stdout, got %q", output)
+	}
+	if !strings.Contains(stderr.String(), "Open this link") {
+		t.Fatalf("expected user instruction on stderr, got %q", stderr.String())
+	}
+}
+
+func TestRunNoJustifications(t *testing.T) {
+	filePath := writeTestFile(t, minimalResultsYAML())
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"-f", filePath, "-justifications=false"}, &stdout, &stderr)
+	if code != 0 {
+		t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr.String())
+	}
+	if strings.Contains(stdout.String(), "justification=") {
+		t.Fatalf("expected no justifications in URL, got %q", stdout.String())
+	}
+}
+
+func TestRunInvalidBadge(t *testing.T) {
+	filePath := writeTestFile(t, minimalResultsYAML())
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"-f", filePath, "-badge", "gold"}, &stdout, &stderr)
+	if code != 1 {
+		t.Fatalf("expected exit code 1, got %d", code)
+	}
+	if !strings.Contains(stderr.String(), "invalid badge") {
+		t.Fatalf("expected invalid badge error, got %q", stderr.String())
+	}
+}
+
+func writeTestFile(t *testing.T, content string) string {
+	t.Helper()
+	filePath := filepath.Join(t.TempDir(), "results.yaml")
+	if err := os.WriteFile(filePath, []byte(content), 0o600); err != nil {
+		t.Fatalf("write test file: %v", err)
+	}
+	return filePath
+}
+
+func minimalResultsYAML() string {
+	return `payload:
+  config:
+    vars:
+      owner: acme
+      repo: rocket
+evaluation-suites:
+  - control-evaluations:
+      evaluations:
+        - assessment-logs:
+            - requirement:
+                entry-id: OSPS-AC-01.01
+              result: Passed
+              message: MFA required
+              applicability:
+                - Maturity Level 1
+`
+}

cmd/list-supported-catalogs/main_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/ossf/pvtr-github-repo-scanner/evaluation_plans"
+)
+
+func TestListSupportedCatalogs(t *testing.T) {
+	// Capture stdout by redirecting os.Stdout to a pipe.
+	old := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Stdout = w
+
+	main()
+
+	_ = w.Close()
+	os.Stdout = old
+
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(r); err != nil {
+		t.Fatal(err)
+	}
+	output := buf.String()
+
+	for _, catalogID := range evaluation_plans.SupportedCatalogIDs {
+		if !strings.Contains(output, catalogID) {
+			t.Errorf("expected output to contain %q, got %q", catalogID, output)
+		}
+	}
+
+	lines := strings.Split(strings.TrimSpace(output), "\n")
+	if len(lines) != len(evaluation_plans.SupportedCatalogIDs) {
+		t.Errorf("expected %d lines, got %d", len(evaluation_plans.SupportedCatalogIDs), len(lines))
+	}
+}

data/payload.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/gemaraproj/go-gemara"
 	"github.com/google/go-github/v74/github"
 	"github.com/privateerproj/privateer-sdk/config"
 	"github.com/shurcooL/githubv4"
@@ -14,6 +15,7 @@ import (
 type Payload struct {
 	*GraphqlRepoData
 	*RestData
+	Evidence                 *gemara.EvidenceCollector
 	Config                   *config.Config
 	RepositoryMetadata       RepositoryMetadata
 	DependencyManifestsCount int
@@ -24,6 +26,27 @@ type Payload struct {
 	cachedTree               *GraphqlRepoTree
 }
 
+// AddEvidence, GetEvidence, and ClearEvidence implement gemara.HasEvidence.
+// Nil-safe: a payload without a collector silently drops evidence.
+func (p Payload) AddEvidence(evidence gemara.Evidence) {
+	if p.Evidence != nil {
+		p.Evidence.AddEvidence(evidence)
+	}
+}
+
+func (p Payload) GetEvidence() []gemara.Evidence {
+	if p.Evidence == nil {
+		return nil
+	}
+	return p.Evidence.GetEvidence()
+}
+
+func (p Payload) ClearEvidence() {
+	if p.Evidence != nil {
+		p.Evidence.ClearEvidence()
+	}
+}
+
 func Loader(config *config.Config) (payload any, err error) {
 	graphql, client, httpClient, err := getGraphqlRepoData(config)
 	if err != nil {
@@ -62,6 +85,7 @@ func Loader(config *config.Config) (payload any, err error) {
 	return any(Payload{
 		GraphqlRepoData:          graphql,
 		RestData:                 rest,
+		Evidence:                 &gemara.EvidenceCollector{},
 		Config:                   config,
 		RepositoryMetadata:       repositoryMetadata,
 		DependencyManifestsCount: dependencyManifestsCount,

evaluation_plans/evaluation-plans.go

@@ -183,7 +183,7 @@ var (
 			quality.HasOneOrMoreStatusChecks,
 		},
 		"OSPS-QA-06.02": {
-			quality.DocumentsTestExecution,
+			quality.TestExecutionDocumentation,
 		},
 		"OSPS-QA-06.03": {
 			reusable_steps.IsCodeRepo,