[DRAFT] preview changes from upcoming SDK upgrade

data/payload.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/gemaraproj/go-gemara"
 	"github.com/google/go-github/v74/github"
 	"github.com/privateerproj/privateer-sdk/config"
 	"github.com/shurcooL/githubv4"
@@ -14,6 +15,7 @@ import (
 type Payload struct {
 	*GraphqlRepoData
 	*RestData
+	Evidence                 *gemara.EvidenceCollector
 	Config                   *config.Config
 	RepositoryMetadata       RepositoryMetadata
 	DependencyManifestsCount int
@@ -24,6 +26,28 @@ type Payload struct {
 	cachedTree               *GraphqlRepoTree
 }
 
+// AddEvidence, GetEvidence, and ClearEvidence satisfy gemara.HasEvidence with
+// nil-safe delegation, so the SDK harvests step-recorded evidence from any
+// Loader-built payload and a collector-less payload degrades to no evidence.
+func (p Payload) AddEvidence(evidence gemara.Evidence) {
+	if p.Evidence != nil {
+		p.Evidence.AddEvidence(evidence)
+	}
+}
+
+func (p Payload) GetEvidence() []gemara.Evidence {
+	if p.Evidence == nil {
+		return nil
+	}
+	return p.Evidence.GetEvidence()
+}
+
+func (p Payload) ClearEvidence() {
+	if p.Evidence != nil {
+		p.Evidence.ClearEvidence()
+	}
+}
+
 func Loader(config *config.Config) (payload any, err error) {
 	graphql, client, httpClient, err := getGraphqlRepoData(config)
 	if err != nil {
@@ -62,6 +86,7 @@ func Loader(config *config.Config) (payload any, err error) {
 	return any(Payload{
 		GraphqlRepoData:          graphql,
 		RestData:                 rest,
+		Evidence:                 &gemara.EvidenceCollector{},
 		Config:                   config,
 		RepositoryMetadata:       repositoryMetadata,
 		DependencyManifestsCount: dependencyManifestsCount,

evaluation_plans/osps/quality/steps.go

@@ -1,13 +1,21 @@
 package quality
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	"github.com/gemaraproj/go-gemara"
 	"github.com/ossf/pvtr-github-repo-scanner/data"
+	sdkai "github.com/privateerproj/privateer-sdk/ai"
 )
 
+const testDocsFallbackMessage = "Review project documentation to ensure it explains when and how tests are run"
+
+// Both vars are seams for tests to stub the AI client and evidence loader.
+var newAIClientFromConfig = sdkai.NewClient
+var loadTestDocsEvidence = testDocsEvidence
+
 func RepoIsPublic(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) {
 	if payload.RepositoryMetadata.IsPublic() {
 		return gemara.Passed, "Repository is public", confidence
@@ -197,9 +205,155 @@
 }
 
 func DocumentsTestExecution(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) {
-	return gemara.NeedsReview, "Review project documentation to ensure it explains when and how tests are run", confidence
+	if payload.Config == nil {
+		return gemara.NeedsReview, testDocsFallbackMessage, confidence
+	}
+	client, err := newAIClientFromConfig(*payload.Config)
+	if err != nil {
+		return testDocsFallback(payload, "AI client construction failed", err)
+	}
+	if client == nil {
+		// AI is not configured; keep the legacy manual-review verdict.
+		return gemara.NeedsReview, testDocsFallbackMessage, confidence
+	}
+
+	material, sources, err := loadTestDocsEvidence(payload)
+	if err != nil {
+		return testDocsFallback(payload, "unable to gather README/CONTRIBUTING evidence", err)
+	}
+
+	// Assist sends the prompt and material to the model against the SDK-owned
+	// verdict schema and hands back both the parsed Response and a
+	// gemara.Evidence that records the exchange verbatim for the audit trail.
+	response, aiEvidence, err := sdkai.Assist(context.Background(), client, sdkai.Question{
+		Prompt:   testDocsPrompt,
+		Material: material,
+	})
+	if err != nil {
+		return testDocsFallback(payload, "AI assessment failed", err)
+	}
+
+	// The message is the SDK-shaped one-liner; the explanation, citations, and
+	// exact prompt are already recorded structured in the evidence payload.
+	// Sources describe where the material came from, so they ride on the
+	// evidence description rather than the message.
+	if len(sources) > 0 {
+		aiEvidence.Description = fmt.Sprintf("AI Assisted Review of %s", strings.Join(sources, ", "))
+	}
+	payload.AddEvidence(aiEvidence)
+
+	return response.GemaraResult(), response.Summary(), response.GemaraConfidence()
+}
+
+// testDocsFallback notes why the AI-assisted path was abandoned and returns
+// the legacy manual-review verdict.
+func testDocsFallback(payload data.Payload, reason string, err error) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) {
+	if payload.Config != nil && payload.Config.Logger != nil {
+		payload.Config.Logger.Warn("OSPS-QA-06.02: "+reason, "err", err)
+	}
+	return gemara.NeedsReview, testDocsFallbackMessage, confidence
 }
 
 func DocumentsTestMaintenancePolicy(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) {
 	return gemara.NeedsReview, "Review project documentation to ensure it contains a clear policy for maintaining tests", confidence
 }
+
+// testDocsEvidence gathers the model input for OSPS-QA-06.02. The control is
+// about contributor-facing test guidance, so README and CONTRIBUTING are the
+// conservative evidence boundary; repository contents at large are too noisy.
+// Sources label where each included part came from, for the evidence record.
+func testDocsEvidence(payload data.Payload) (material string, sources []string, err error) {
+	var parts []string
+
+	if readmePath := testDocsRootFile(payload, "readme"); readmePath != "" {
+		if readme := testDocsFileContent(payload, readmePath); readme != "" {
+			parts = append(parts, "README\n"+readme)
+			sources = append(sources, testDocsSource(payload, readmePath))
+		}
+	}
+
+	if payload.GraphqlRepoData != nil {
+		if contributing := strings.TrimSpace(payload.Repository.ContributingGuidelines.Body); contributing != "" {
+			parts = append(parts, "CONTRIBUTING\n"+contributing)
+			if path := testDocsRootFile(payload, "contributing"); path != "" {
+				sources = append(sources, testDocsSource(payload, path))
+			} else {
+				// GraphQL supplies the body without a path.
+				sources = append(sources, "/CONTRIBUTING")
+			}
+		}
+	}
+
+	if len(parts) == 0 {
+		return "", nil, fmt.Errorf("no README or CONTRIBUTING content available")
+	}
+	return strings.Join(parts, "\n\n"), sources, nil
+}
+
+// testDocsRootFile returns the tree path of the first blob named <name> or
+// <name>.<ext> (case-insensitive), or "".
+func testDocsRootFile(payload data.Payload, name string) string {
+	if payload.GraphqlRepoData == nil {
+		return ""
+	}
+	for _, entry := range payload.Repository.Object.Tree.Entries {
+		if entry.Type != "blob" {
+			continue
+		}
+		lower := strings.ToLower(entry.Name)
+		if lower == name || strings.HasPrefix(lower, name+".") {
+			return entry.Path
+		}
+	}
+	return ""
+}
+
+func testDocsFileContent(payload data.Payload, path string) string {
+	if payload.RestData == nil {
+		return ""
+	}
+	content, err := payload.GetFileContent(path)
+	if err != nil || content == nil {
+		return ""
+	}
+	text, err := content.GetContent()
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(text)
+}
+
+// testDocsSource labels a reviewed file, preferring a commit-pinned GitHub
+// blob URL when owner, repo, and commit are all known.
+func testDocsSource(payload data.Payload, path string) string {
+	path = strings.TrimLeft(strings.TrimSpace(path), "/")
+	if payload.Config != nil && payload.GraphqlRepoData != nil {
+		owner := strings.TrimSpace(payload.Config.GetString("owner"))
+		repo := strings.TrimSpace(payload.Config.GetString("repo"))
+		commit := strings.TrimSpace(payload.Repository.DefaultBranchRef.Target.OID)
+		if owner != "" && repo != "" && commit != "" {
+			return fmt.Sprintf("https://github.com/%s/%s/blob/%s/%s", owner, repo, commit, path)
+		}
+	}
+	return "/" + path
+}
+
+const testDocsPrompt = `You are assessing OSPS-QA-06.02: the project's documentation MUST clearly document WHEN and HOW tests are run. This is a contributor-facing requirement.
+
+Use only the supplied README and CONTRIBUTING content as evidence.
+
+Return result "pass" only when BOTH of the following are clearly explained:
+  - WHEN tests run (e.g. on every pull request, before merge, on a schedule, locally before commit).
+  - HOW tests are run (concrete commands to run tests locally AND/OR a description of how they run in CI/CD).
+
+A pass is stronger when the documentation also explains what the tests cover and how to interpret results, but those are not strictly required.
+
+Return result "fail" when any of the following hold:
+  - The documentation is missing or only implies that tests exist.
+  - It covers WHEN but not HOW, or HOW but not WHEN.
+  - Instructions are vague (e.g. "run the tests" with no command or workflow reference).
+  - The only test discussion is aimed at end users, not contributors.
+
+Reserve result "needs_review" for evidence you genuinely cannot judge either way.
+
+Cite the most relevant section headers or quoted snippets in citations.`