diff --git a/.gitignore b/.gitignore index 7fdb7d1c..a14ca4c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ # ignore the generated artifacts -pvtr-github-repo -github-repo -badge-url +/pvtr-github-repo +/pvtr-github-repo-dev +/github-repo +/github-repo-dev +/badge-url evaluation_results # ignore any local dev config file diff --git a/cmd/badge-url/main_test.go b/cmd/badge-url/main_test.go new file mode 100644 index 00000000..779270b6 --- /dev/null +++ b/cmd/badge-url/main_test.go @@ -0,0 +1,119 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunMissingFileFlag(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{}, &stdout, &stderr) + if code != 2 { + t.Fatalf("expected exit code 2, got %d", code) + } + if !strings.Contains(stderr.String(), "requires -f") { + t.Fatalf("expected usage message on stderr, got %q", stderr.String()) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout, got %q", stdout.String()) + } +} + +func TestRunInvalidFlag(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"--nonexistent"}, &stdout, &stderr) + if code != 2 { + t.Fatalf("expected exit code 2, got %d", code) + } +} + +func TestRunNonexistentFile(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"-f", "/nonexistent/results.yaml"}, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected exit code 1, got %d", code) + } + if !strings.Contains(stderr.String(), "read results file") { + t.Fatalf("expected file error on stderr, got %q", stderr.String()) + } +} + +func TestRunInvalidYAML(t *testing.T) { + filePath := writeTestFile(t, "not valid yaml: [") + var stdout, stderr bytes.Buffer + code := run([]string{"-f", filePath}, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected exit code 1, got %d", code) + } +} + +func TestRunValidResults(t *testing.T) { + filePath := writeTestFile(t, minimalResultsYAML()) + var stdout, stderr bytes.Buffer + code := run([]string{"-f", filePath, "-badge", "baseline-1"}, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, "bestpractices.dev") { + t.Fatalf("expected bestpractices.dev URL in stdout, got %q", output) + } + if !strings.Contains(stderr.String(), "Open this link") { + t.Fatalf("expected user instruction on stderr, got %q", stderr.String()) + } +} + +func TestRunNoJustifications(t *testing.T) { + filePath := writeTestFile(t, minimalResultsYAML()) + var stdout, stderr bytes.Buffer + code := run([]string{"-f", filePath, "-justifications=false"}, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr.String()) + } + if strings.Contains(stdout.String(), "justification=") { + t.Fatalf("expected no justifications in URL, got %q", stdout.String()) + } +} + +func TestRunInvalidBadge(t *testing.T) { + filePath := writeTestFile(t, minimalResultsYAML()) + var stdout, stderr bytes.Buffer + code := run([]string{"-f", filePath, "-badge", "gold"}, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected exit code 1, got %d", code) + } + if !strings.Contains(stderr.String(), "invalid badge") { + t.Fatalf("expected invalid badge error, got %q", stderr.String()) + } +} + +func writeTestFile(t *testing.T, content string) string { + t.Helper() + filePath := filepath.Join(t.TempDir(), "results.yaml") + if err := os.WriteFile(filePath, []byte(content), 0o600); err != nil { + t.Fatalf("write test file: %v", err) + } + return filePath +} + +func minimalResultsYAML() string { + return `payload: + config: + vars: + owner: acme + repo: rocket +evaluation-suites: + - control-evaluations: + evaluations: + - assessment-logs: + - requirement: + entry-id: OSPS-AC-01.01 + result: Passed + message: MFA required + applicability: + - Maturity Level 1 +` +} diff --git a/cmd/list-supported-catalogs/main_test.go b/cmd/list-supported-catalogs/main_test.go new file mode 100644 index 00000000..62ae85e5 --- /dev/null +++ b/cmd/list-supported-catalogs/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/ossf/pvtr-github-repo-scanner/evaluation_plans" +) + +func TestListSupportedCatalogs(t *testing.T) { + // Capture stdout by redirecting os.Stdout to a pipe. + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + + main() + + _ = w.Close() + os.Stdout = old + + var buf bytes.Buffer + if _, err := buf.ReadFrom(r); err != nil { + t.Fatal(err) + } + output := buf.String() + + for _, catalogID := range evaluation_plans.SupportedCatalogIDs { + if !strings.Contains(output, catalogID) { + t.Errorf("expected output to contain %q, got %q", catalogID, output) + } + } + + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) != len(evaluation_plans.SupportedCatalogIDs) { + t.Errorf("expected %d lines, got %d", len(evaluation_plans.SupportedCatalogIDs), len(lines)) + } +} diff --git a/data/payload.go b/data/payload.go index d34d39ec..407d4d60 100644 --- a/data/payload.go +++ b/data/payload.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" + "github.com/gemaraproj/go-gemara" "github.com/google/go-github/v74/github" "github.com/privateerproj/privateer-sdk/config" "github.com/shurcooL/githubv4" @@ -14,6 +15,7 @@ import ( type Payload struct { *GraphqlRepoData *RestData + Evidence *gemara.EvidenceCollector Config *config.Config RepositoryMetadata RepositoryMetadata DependencyManifestsCount int @@ -24,6 +26,27 @@ type Payload struct { cachedTree *GraphqlRepoTree } +// AddEvidence, GetEvidence, and ClearEvidence implement gemara.HasEvidence. +// Nil-safe: a payload without a collector silently drops evidence. +func (p Payload) AddEvidence(evidence gemara.Evidence) { + if p.Evidence != nil { + p.Evidence.AddEvidence(evidence) + } +} + +func (p Payload) GetEvidence() []gemara.Evidence { + if p.Evidence == nil { + return nil + } + return p.Evidence.GetEvidence() +} + +func (p Payload) ClearEvidence() { + if p.Evidence != nil { + p.Evidence.ClearEvidence() + } +} + func Loader(config *config.Config) (payload any, err error) { graphql, client, httpClient, err := getGraphqlRepoData(config) if err != nil { @@ -62,6 +85,7 @@ func Loader(config *config.Config) (payload any, err error) { return any(Payload{ GraphqlRepoData: graphql, RestData: rest, + Evidence: &gemara.EvidenceCollector{}, Config: config, RepositoryMetadata: repositoryMetadata, DependencyManifestsCount: dependencyManifestsCount, diff --git a/evaluation_plans/evaluation-plans.go b/evaluation_plans/evaluation-plans.go index 054dd215..efc7e16f 100644 --- a/evaluation_plans/evaluation-plans.go +++ b/evaluation_plans/evaluation-plans.go @@ -183,7 +183,7 @@ var ( quality.HasOneOrMoreStatusChecks, }, "OSPS-QA-06.02": { - quality.DocumentsTestExecution, + quality.TestExecutionDocumentation, }, "OSPS-QA-06.03": { reusable_steps.IsCodeRepo, diff --git a/evaluation_plans/osps/quality/steps.go b/evaluation_plans/osps/quality/steps.go index f56f491a..cc97d96e 100644 --- a/evaluation_plans/osps/quality/steps.go +++ b/evaluation_plans/osps/quality/steps.go @@ -1,13 +1,22 @@ package quality import ( + "context" "fmt" "strings" "github.com/gemaraproj/go-gemara" "github.com/ossf/pvtr-github-repo-scanner/data" + "github.com/ossf/pvtr-github-repo-scanner/evaluation_plans/reusable_steps" + sdkai "github.com/privateerproj/privateer-sdk/ai" ) +const testExecutionDocumentationFallbackMessage = "Review project documentation to ensure it explains when and how tests are run" + +// Both vars are seams for tests to stub the AI client and evidence loader. +var newAIClientFromConfig = sdkai.NewClient +var loadTestExecutionDocumentationEvidence = testExecutionDocumentationEvidence + func RepoIsPublic(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) { if payload.RepositoryMetadata.IsPublic() { return gemara.Passed, "Repository is public", confidence @@ -196,10 +205,195 @@ func countDependencyManifests(payload data.Payload) (result gemara.Result, messa return gemara.NeedsReview, "No dependency manifests found in the GitHub dependency graph API. Review project to ensure dependencies are managed.", confidence } -func DocumentsTestExecution(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) { - return gemara.NeedsReview, "Review project documentation to ensure it explains when and how tests are run", confidence +// TestExecutionDocumentation assesses OSPS-QA-06.02: whether the project +// documents when and how tests are run. Uses AI when configured, otherwise +// falls back to manual review. +func TestExecutionDocumentation(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) { + if payload.Config == nil { + return gemara.NeedsReview, testExecutionDocumentationFallbackMessage, confidence + } + + client, err := newAIClientFromConfig(*payload.Config) + if err != nil { + return reusable_steps.AIFallback(payload, "OSPS-QA-06.02", testExecutionDocumentationFallbackMessage, "AI client construction failed", err) + } + if client == nil { + // AI is not configured; keep the legacy manual-review verdict. + return gemara.NeedsReview, testExecutionDocumentationFallbackMessage, confidence + } + + material, sources, err := loadTestExecutionDocumentationEvidence(payload) + if err != nil { + return reusable_steps.AIFallback(payload, "OSPS-QA-06.02", testExecutionDocumentationFallbackMessage, "unable to gather README/CONTRIBUTING evidence", err) + } + + response, aiEvidence, err := sdkai.Assist(context.Background(), client, sdkai.Question{ + Prompt: testExecutionDocumentationPrompt, + Material: material, + }) + if err != nil { + return reusable_steps.AIFallback(payload, "OSPS-QA-06.02", testExecutionDocumentationFallbackMessage, "AI assessment failed", err) + } + + // Attach source locations to the evidence so reviewers know what the AI saw. + if len(sources) > 0 { + aiEvidence.Description = fmt.Sprintf("AI Assisted Review of %s", strings.Join(sources, ", ")) + } + payload.AddEvidence(aiEvidence) + + return response.GemaraResult(), response.Summary(), response.GemaraConfidence() } +// DocumentsTestMaintenancePolicy assesses OSPS-QA-06.03: whether the project +// documents a policy for maintaining tests. Currently defers to manual review. func DocumentsTestMaintenancePolicy(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) { return gemara.NeedsReview, "Review project documentation to ensure it contains a clear policy for maintaining tests", confidence } + +// testExecutionDocumentationEvidence gathers README and CONTRIBUTING content +// as AI input for OSPS-QA-06.02. Only these two files are included because the +// control targets contributor-facing test guidance. +func testExecutionDocumentationEvidence(payload data.Payload) (material string, sources []string, err error) { + var parts []string + + if readme := strings.TrimSpace(testExecutionDocumentationReadmeContent(payload)); readme != "" { + parts = append(parts, "README\n"+readme) + if readmePath := testExecutionDocumentationReadmePath(payload); readmePath != "" { + sources = append(sources, testExecutionDocumentationEvidenceSource(payload, readmePath)) + } else { + sources = append(sources, "/README") + } + } + + if payload.GraphqlRepoData != nil { + if contributing := strings.TrimSpace(payload.Repository.ContributingGuidelines.Body); contributing != "" { + parts = append(parts, "CONTRIBUTING\n"+contributing) + if contributingPath := testExecutionDocumentationContributingPath(payload); contributingPath != "" { + sources = append(sources, testExecutionDocumentationEvidenceSource(payload, contributingPath)) + } else { + sources = append(sources, "/CONTRIBUTING") + } + } + } + + if len(parts) == 0 { + return "", nil, fmt.Errorf("no README or CONTRIBUTING content available") + } + + return strings.Join(parts, "\n\n"), sources, nil +} + +func testExecutionDocumentationEvidenceSource(payload data.Payload, path string) string { + if blobURL := testExecutionDocumentationBlobURL(payload, path); blobURL != "" { + return blobURL + } + return testExecutionDocumentationRepoAbsolutePath(path) +} + +func testExecutionDocumentationBlobURL(payload data.Payload, path string) string { + repositoryOwner := "" + repositoryName := "" + commitSHA := "" + if payload.Config != nil { + repositoryOwner = strings.TrimSpace(payload.Config.GetString("owner")) + repositoryName = strings.TrimSpace(payload.Config.GetString("repo")) + } + if payload.GraphqlRepoData != nil { + if strings.TrimSpace(payload.Repository.Name) != "" { + repositoryName = strings.TrimSpace(payload.Repository.Name) + } + commitSHA = strings.TrimSpace(payload.Repository.DefaultBranchRef.Target.OID) + } + trimmedPath := strings.TrimLeft(strings.TrimSpace(path), "/") + if repositoryOwner == "" || repositoryName == "" || commitSHA == "" || trimmedPath == "" { + return "" + } + return fmt.Sprintf("https://github.com/%s/%s/blob/%s/%s", repositoryOwner, repositoryName, commitSHA, trimmedPath) +} + +func testExecutionDocumentationRepoAbsolutePath(path string) string { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return "" + } + return "/" + strings.TrimLeft(trimmed, "/") +} + +func testExecutionDocumentationReadmeContent(payload data.Payload) string { + if payload.GraphqlRepoData == nil || payload.RestData == nil { + return "" + } + + readmePath := testExecutionDocumentationReadmePath(payload) + if readmePath == "" { + return "" + } + + content, err := payload.GetFileContent(readmePath) + if err != nil || content == nil { + return "" + } + + readme, err := content.GetContent() + if err != nil { + return "" + } + + return strings.TrimSpace(readme) +} + +func testExecutionDocumentationReadmePath(payload data.Payload) string { + if payload.GraphqlRepoData == nil { + return "" + } + for _, entry := range payload.Repository.Object.Tree.Entries { + if entry.Type != "blob" { + continue + } + if testExecutionDocumentationReadmeName(entry.Name) { + return entry.Path + } + } + return "" +} + +func testExecutionDocumentationReadmeName(name string) bool { + lower := strings.ToLower(strings.TrimSpace(name)) + return lower == "readme" || strings.HasPrefix(lower, "readme.") +} + +func testExecutionDocumentationContributingPath(payload data.Payload) string { + if payload.GraphqlRepoData == nil { + return "" + } + for _, entry := range payload.Repository.Object.Tree.Entries { + if entry.Type != "blob" { + continue + } + lower := strings.ToLower(strings.TrimSpace(entry.Name)) + if lower == "contributing" || strings.HasPrefix(lower, "contributing.") { + return entry.Path + } + } + return "" +} + +const testExecutionDocumentationPrompt = `You are assessing OSPS-QA-06.02: the project's documentation MUST clearly document WHEN and HOW tests are run. This is a contributor-facing requirement. + +Use only the supplied README and CONTRIBUTING content as evidence. + +Return result "pass" only when BOTH of the following are clearly explained: + - WHEN tests run (e.g. on every pull request, before merge, on a schedule, locally before commit). + - HOW tests are run (concrete commands to run tests locally AND/OR a description of how they run in CI/CD). + +A pass is stronger when the documentation also explains what the tests cover and how to interpret results, but those are not strictly required. + +Return result "fail" when any of the following hold: + - The documentation is missing or only implies that tests exist. + - It covers WHEN but not HOW, or HOW but not WHEN. + - Instructions are vague (e.g. "run the tests" with no command or workflow reference). + - The only test discussion is aimed at end users, not contributors. + +Reserve result "needs_review" for evidence you genuinely cannot judge either way. + +Cite the most relevant section headers or quoted snippets in citations.` diff --git a/evaluation_plans/osps/quality/steps_test.go b/evaluation_plans/osps/quality/steps_test.go index 8065ec94..e4acc2ca 100644 --- a/evaluation_plans/osps/quality/steps_test.go +++ b/evaluation_plans/osps/quality/steps_test.go @@ -1,11 +1,17 @@ package quality import ( + "context" + "encoding/json" + "errors" + "strings" "testing" "github.com/gemaraproj/go-gemara" "github.com/ossf/pvtr-github-repo-scanner/data" "github.com/ossf/si-tooling/v2/si" + sdkai "github.com/privateerproj/privateer-sdk/ai" + sdkconfig "github.com/privateerproj/privateer-sdk/config" ) func Test_InsightsListsRepositories(t *testing.T) { @@ -85,3 +91,230 @@ func Test_NoUnreviewableBinariesInRepo(t *testing.T) { } }) } + +// stubAIClient satisfies sdkai.Client so tests can exercise the AI step +// without a network. +type stubAIClient struct { + response *sdkai.AnalyzeResponse + err error +} + +func (s stubAIClient) Analyze(ctx context.Context, prompt, content string, schema *sdkai.Schema) (*sdkai.AnalyzeResponse, error) { + return s.response, s.err +} + +// assistVerdict wraps a JSON verdict in the AnalyzeResponse shape the SDK's +// Assist accelerator parses. The body must match the SDK-owned assist schema: +// result/confidence/message/explanation/citations. +func assistVerdict(body string) *sdkai.AnalyzeResponse { + return &sdkai.AnalyzeResponse{ + JSON: json.RawMessage(body), + Metadata: sdkai.ResponseMetadata{ + Provider: sdkai.ProviderOpenAI, + Model: "gpt-4o-mini-2024-07-18", + RequestID: "req-123", + }, + } +} + +func stubAIFactory(client sdkai.Client, err error) func(sdkconfig.Config) (sdkai.Client, error) { + return func(cfg sdkconfig.Config) (sdkai.Client, error) { + return client, err + } +} + +func TestTestExecutionDocumentation(t *testing.T) { + originalFactory := newAIClientFromConfig + originalEvidenceLoader := loadTestExecutionDocumentationEvidence + t.Cleanup(func() { + newAIClientFromConfig = originalFactory + loadTestExecutionDocumentationEvidence = originalEvidenceLoader + }) + + payload := data.Payload{Config: &sdkconfig.Config{}} + loadTestExecutionDocumentationEvidence = func(payload data.Payload) (string, []string, error) { + return "README\nRun `go test ./...` before opening a PR.", []string{"/README"}, nil + } + + t.Run("no AI config preserves legacy behavior", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(nil, nil) + + result, msg, _ := TestExecutionDocumentation(payload) + if result != gemara.NeedsReview { + t.Fatalf("result = %v, want NeedsReview", result) + } + if msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("message = %q, want %q", msg, testExecutionDocumentationFallbackMessage) + } + }) + + t.Run("client construction error falls back to needs review", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(nil, errors.New("bad ai config")) + + result, msg, _ := TestExecutionDocumentation(payload) + if result != gemara.NeedsReview || msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("got (%v, %q), want legacy fallback", result, msg) + } + }) + + t.Run("partial live AI config falls back to needs review", func(t *testing.T) { + // Uses the real SDK constructor so this exercises ai.NewClient's + // validation of incomplete ai_* settings end-to-end. + newAIClientFromConfig = sdkai.NewClient + + partialPayload := data.Payload{Config: &sdkconfig.Config{Vars: map[string]interface{}{ + "ai_provider": "openai", + "ai_model": "gpt-4o-mini", + }}} + + result, msg, _ := TestExecutionDocumentation(partialPayload) + if result != gemara.NeedsReview { + t.Fatalf("result = %v, want NeedsReview", result) + } + if msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("message = %q, want %q", msg, testExecutionDocumentationFallbackMessage) + } + }) + + t.Run("ai returns pass verdict and records evidence", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{response: assistVerdict( + `{"result":"pass","confidence":"high","message":"Contributors are told to run go test before opening a PR","explanation":"README explains that contributors run go test before opening a PR.","citations":["README#testing"]}`)}, nil) + + collectingPayload := payload + collectingPayload.Evidence = &gemara.EvidenceCollector{} + + result, msg, confidence := TestExecutionDocumentation(collectingPayload) + if result != gemara.Passed { + t.Fatalf("result = %v, want Passed", result) + } + if confidence != gemara.High { + t.Fatalf("confidence = %v, want High", confidence) + } + if msg != "[AI-Assisted] Contributors are told to run go test before opening a PR" { + t.Fatalf("expected the model-authored one-liner, got %q", msg) + } + if strings.Contains(msg, "README#testing") || strings.Contains(msg, "\n") { + t.Fatalf("citations and newlines belong in the evidence, not the message: %q", msg) + } + + recorded := collectingPayload.GetEvidence() + if len(recorded) != 1 { + t.Fatalf("recorded %d evidence records, want 1", len(recorded)) + } + if recorded[0].Type != sdkai.EvidenceType { + t.Fatalf("evidence type = %q, want %q", recorded[0].Type, sdkai.EvidenceType) + } + if recorded[0].Id != "req-123" { + t.Fatalf("evidence id = %q, want provider request id", recorded[0].Id) + } + if !strings.Contains(recorded[0].Description, "/README") { + t.Fatalf("evidence description should carry the sources, got %q", recorded[0].Description) + } + }) + + t.Run("ai returns fail verdict", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{response: assistVerdict( + `{"result":"fail","confidence":"medium","message":"The docs never explain when or how tests are run","explanation":"The docs mention tests exist but never explain when or how to run them.","citations":["README#development"]}`)}, nil) + + result, _, confidence := TestExecutionDocumentation(payload) + if result != gemara.Failed { + t.Fatalf("result = %v, want Failed", result) + } + if confidence != gemara.Medium { + t.Fatalf("confidence = %v, want Medium", confidence) + } + }) + + t.Run("ai needs_review verdict surfaces the model message", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{response: assistVerdict( + `{"result":"needs_review","confidence":"low","message":"Test guidance lives in external wiki links that were not supplied","explanation":"Test guidance is split across external wiki links that were not supplied.","citations":[]}`)}, nil) + + result, msg, _ := TestExecutionDocumentation(payload) + if result != gemara.NeedsReview { + t.Fatalf("result = %v, want NeedsReview", result) + } + if !strings.HasPrefix(msg, "[AI-Assisted]") { + t.Fatalf("expected the model verdict rather than the fallback, got %q", msg) + } + }) + + t.Run("invalid AI response falls back to needs review", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{response: &sdkai.AnalyzeResponse{JSON: json.RawMessage(`not json`)}}, nil) + + result, msg, _ := TestExecutionDocumentation(payload) + if result != gemara.NeedsReview || msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("got (%v, %q), want legacy fallback", result, msg) + } + }) + + t.Run("ai timeout falls back to needs review", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{err: context.DeadlineExceeded}, nil) + + result, msg, _ := TestExecutionDocumentation(payload) + if result != gemara.NeedsReview || msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("got (%v, %q), want legacy fallback", result, msg) + } + }) + + t.Run("ai provider error falls back and records no evidence", func(t *testing.T) { + newAIClientFromConfig = stubAIFactory(stubAIClient{err: errors.New("provider unavailable")}, nil) + + collectingPayload := payload + collectingPayload.Evidence = &gemara.EvidenceCollector{} + + result, msg, _ := TestExecutionDocumentation(collectingPayload) + if result != gemara.NeedsReview || msg != testExecutionDocumentationFallbackMessage { + t.Fatalf("got (%v, %q), want legacy fallback", result, msg) + } + if recorded := collectingPayload.GetEvidence(); len(recorded) != 0 { + t.Fatalf("expected no evidence on provider failure, got %d records", len(recorded)) + } + }) +} + +func TestTestExecutionDocumentationEvidence(t *testing.T) { + payload := data.Payload{GraphqlRepoData: &data.GraphqlRepoData{}} + payload.Repository.Object.Tree.Entries = []struct { + Name string + Type string + Path string + }{ + {Name: "NOTES.md", Type: "blob", Path: "NOTES.md"}, + {Name: "README.md", Type: "blob", Path: "README.md"}, + {Name: "CONTRIBUTING.md", Type: "blob", Path: "CONTRIBUTING.md"}, + } + payload.Repository.ContributingGuidelines.Body = "Use the documented test workflow before requesting review." + + if got := testExecutionDocumentationReadmePath(payload); got != "README.md" { + t.Fatalf("testExecutionDocumentationReadmePath = %q, want README.md", got) + } + + // No RestData, so README content cannot be fetched: only CONTRIBUTING is + // sent to the model, and only CONTRIBUTING may be claimed as a source. + material, sources, err := testExecutionDocumentationEvidence(payload) + if err != nil { + t.Fatalf("unexpected evidence error: %v", err) + } + if material != "CONTRIBUTING\nUse the documented test workflow before requesting review." { + t.Fatalf("unexpected evidence material: %q", material) + } + if len(sources) != 1 || sources[0] != "/CONTRIBUTING.md" { + t.Fatalf("unexpected sources: %v", sources) + } + + // With owner, repo, and commit known, sources become commit-pinned URLs. + payload.Config = &sdkconfig.Config{Vars: map[string]interface{}{"owner": "test-owner", "repo": "test-repo"}} + payload.Repository.DefaultBranchRef.Target.OID = "abc123def456" + _, sources, err = testExecutionDocumentationEvidence(payload) + if err != nil { + t.Fatalf("unexpected evidence error: %v", err) + } + want := "https://github.com/test-owner/test-repo/blob/abc123def456/CONTRIBUTING.md" + if len(sources) != 1 || sources[0] != want { + t.Fatalf("sources = %v, want [%s]", sources, want) + } + + if _, _, err := testExecutionDocumentationEvidence(data.Payload{}); err == nil { + t.Fatal("expected an error when no documentation is available") + } +} diff --git a/evaluation_plans/reusable_steps/steps.go b/evaluation_plans/reusable_steps/steps.go index 8f78bfc1..6ee95905 100644 --- a/evaluation_plans/reusable_steps/steps.go +++ b/evaluation_plans/reusable_steps/steps.go @@ -76,3 +76,14 @@ func IsCodeRepo(payload data.Payload) (result gemara.Result, message string, con return gemara.Passed, "Repository contains code", confidence } + +// AIFallback logs why an AI-assisted assessment was abandoned and returns +// NeedsReview with the supplied fallback message. Use this when an AI-assisted +// step cannot complete (e.g. client construction failure, missing evidence, +// provider error) and should degrade gracefully to manual review. +func AIFallback(payload data.Payload, controlID string, fallbackMessage string, reason string, err error) (gemara.Result, string, gemara.ConfidenceLevel) { + if payload.Config != nil && payload.Config.Logger != nil { + payload.Config.Logger.Warn(controlID+": "+reason, "err", err) + } + return gemara.NeedsReview, fallbackMessage, 0 +} diff --git a/github-repo-dev b/github-repo-dev deleted file mode 100755 index 7e23de5d..00000000 Binary files a/github-repo-dev and /dev/null differ diff --git a/pvtr-github-repo-dev b/pvtr-github-repo-dev deleted file mode 100755 index 60f8d0a7..00000000 Binary files a/pvtr-github-repo-dev and /dev/null differ