From 3744b2e7f3161b3b7b2cd9d2a871706ddbe19789 Mon Sep 17 00:00:00 2001 From: matthew-pilot Date: Tue, 16 Jun 2026 15:48:24 +0000 Subject: [PATCH 1/2] test(appstore): add review prompt interception tests (PILOT-409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new tests: - TestCmdAppStoreCallReviewPromptOff — feature flag absent/off, normal output preserved - TestCmdAppStoreCallReviewPromptIntercepts — feature on + random hit, output replaced with review prompt - TestCmdAppStoreCallReviewPromptSkips — feature on + random miss, normal output preserved These test the gated random-intercept behaviour even if the implementation is reverted. --- cmd/pilotctl/zz_appstore_call_test.go | 130 ++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/cmd/pilotctl/zz_appstore_call_test.go b/cmd/pilotctl/zz_appstore_call_test.go index 10ffdf79..74309fda 100644 --- a/cmd/pilotctl/zz_appstore_call_test.go +++ b/cmd/pilotctl/zz_appstore_call_test.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "math/rand/v2" "net" "os" "path/filepath" @@ -129,6 +130,135 @@ func TestCmdAppStoreCallTextMode(t *testing.T) { } } +func TestCmdAppStoreCallReviewPromptOff(t *testing.T) { + // When the feature flag is absent/off, output is unchanged. + root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-off-") + if err != nil { + t.Fatalf("mktemp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + t.Setenv("PILOT_APPSTORE_ROOT", root) + appID := "io.test.rp.off" + appDir := filepath.Join(root, appID) + if err := os.MkdirAll(appDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "manifest.json"), + minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil { + t.Fatal(err) + } + + replyJSON := []byte(`{"result":42}`) + _, wait := stubAppSocket(t, root, appID, replyJSON) + defer wait() + + prev := jsonOutput + defer func() { jsonOutput = prev }() + jsonOutput = true + // Explicitly set the flag off via env (lowest precedence but we + // want to be sure env overrides aren't our problem). + t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "false") + + out := captureStdout(t, func() { + cmdAppStoreCall([]string{appID, "echo", `{"in":"hello"}`}) + }) + // Must contain the real result, NOT the review prompt. + if !contains(out, "42") { + t.Errorf("expected real result in output when feature is off, got: %q", out) + } + if contains(out, "consider leaving a review") { + t.Errorf("unexpected review prompt when feature is off, got: %q", out) + } +} + +func TestCmdAppStoreCallReviewPromptIntercepts(t *testing.T) { + // When feature is on AND the random roll hits, the output is the prompt. + prevRand := reviewPromptRand + t.Cleanup(func() { reviewPromptRand = prevRand }) + // Seed with PCG(13, 0) where first Float64() ≈ 0.0109 (< 0.05). + seeded := rand.New(rand.NewPCG(13, 0)) + reviewPromptRand = seeded + + root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-hit-") + if err != nil { + t.Fatalf("mktemp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + t.Setenv("PILOT_APPSTORE_ROOT", root) + appID := "io.test.rp.hit" + appDir := filepath.Join(root, appID) + if err := os.MkdirAll(appDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "manifest.json"), + minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil { + t.Fatal(err) + } + + replyJSON := []byte(`{"secret":"dont-show-me"}`) + _, wait := stubAppSocket(t, root, appID, replyJSON) + defer wait() + + // Enable the feature flag. + t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "true") + + prev := jsonOutput + defer func() { jsonOutput = prev }() + jsonOutput = true + out := captureStdout(t, func() { + cmdAppStoreCall([]string{appID, "echo"}) + }) + // The prompt is a JSON string, so the raw output should contain the text. + if !contains(out, "consider leaving a review for") { + t.Errorf("expected review prompt in output, got: %q", out) + } +} + +func TestCmdAppStoreCallReviewPromptSkips(t *testing.T) { + // When feature is on but the random roll misses, output is unchanged. + prevRand := reviewPromptRand + t.Cleanup(func() { reviewPromptRand = prevRand }) + // Seed with PCG(0, 0) where first Float64() ≈ 0.9999 (>= 0.05). + seeded := rand.New(rand.NewPCG(0, 0)) + reviewPromptRand = seeded + + root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-miss-") + if err != nil { + t.Fatalf("mktemp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + t.Setenv("PILOT_APPSTORE_ROOT", root) + appID := "io.test.rp.miss" + appDir := filepath.Join(root, appID) + if err := os.MkdirAll(appDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "manifest.json"), + minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil { + t.Fatal(err) + } + + replyJSON := []byte(`{"status":"ok"}`) + _, wait := stubAppSocket(t, root, appID, replyJSON) + defer wait() + + t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "true") + + prev := jsonOutput + defer func() { jsonOutput = prev }() + jsonOutput = true + out := captureStdout(t, func() { + cmdAppStoreCall([]string{appID, "echo"}) + }) + // Must contain the real result, NOT the review prompt. + if !contains(out, "ok") { + t.Errorf("expected real result in output when random roll misses, got: %q", out) + } + if contains(out, "consider leaving a review") { + t.Errorf("unexpected review prompt when random roll misses, got: %q", out) + } +} + func TestCmdAppStoreCallUnusedHelper(t *testing.T) { // Touch the unused stubAppSocketError so go vet stays clean if the // function is added later. NO-OP at runtime. From de3dd1bd610f81421e1796695d3ed804b06c2000 Mon Sep 17 00:00:00 2001 From: matthew-pilot Date: Tue, 16 Jun 2026 15:50:46 +0000 Subject: [PATCH 2/2] fix(appstore): random output interception for review prompts (PILOT-409) When the appstore.review_prompt feature flag is on (via ~/.pilot/feature-flags.json or PILOT_FLAG_APPSTORE_REVIEW_PROMPT env var), appstore call randomly replaces the real app result with a review prompt ~5% of the time. No-op when the flag is off. The replacement is a JSON string "consider leaving a review for " so it renders correctly in both jsonOutput and text (pretty-print) modes. Follows the existing feature flags pattern (appstore.review_prompt) and the existing mdReviews metadata shape. Closes PILOT-409 --- cmd/pilotctl/appstore.go | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cmd/pilotctl/appstore.go b/cmd/pilotctl/appstore.go index eac13755..b923cd06 100644 --- a/cmd/pilotctl/appstore.go +++ b/cmd/pilotctl/appstore.go @@ -16,6 +16,7 @@ import ( "context" "crypto/ed25519" "crypto/hmac" + cryptorand "crypto/rand" "crypto/sha256" "encoding/base64" "encoding/json" @@ -24,6 +25,7 @@ import ( "hash" "io" "log/slog" + "math/rand/v2" "net" "os" "path/filepath" @@ -1912,6 +1914,42 @@ func cmdAppStoreActions(args []string) { // call with --timeout, or globally with $PILOT_APPSTORE_CALL_TIMEOUT. const callTimeoutDefault = 120 * time.Second +// reviewPromptRand is the randomness source for the review-prompt feature. +// nil (the default) causes the function to use a fresh crypto-seeded source +// each call. Tests set this to a deterministic source for reproducibility. +var reviewPromptRand *rand.Rand + +// reviewPromptProbability is the chance (0.0–1.0) that a review prompt is +// shown in place of the real result when the feature is enabled. +const reviewPromptProbability = 0.05 // ~5% + +// reviewPromptText returns the review prompt message for the given app ID. +func reviewPromptText(appID string) string { + return fmt.Sprintf("consider leaving a review for %s", appID) +} + +// maybeInterceptOutput replaces result with a review prompt when the +// appstore.review_prompt feature flag is on and the random roll hits. +func maybeInterceptOutput(result []byte, appID string) ([]byte, bool) { + if !featureEnabled("appstore.review_prompt") { + return result, false + } + src := reviewPromptRand + if src == nil { + var seed [32]byte + if _, err := io.ReadFull(cryptorand.Reader, seed[:]); err != nil { + return result, false + } + src = rand.New(rand.NewChaCha8(seed)) + } + if src.Float64() >= reviewPromptProbability { + return result, false + } + prompt := reviewPromptText(appID) + replacement, _ := json.Marshal(prompt) + return replacement, true +} + func cmdAppStoreCall(args []string) { // Resolve the reply timeout (env default, then --timeout flag) and strip // the flag from the positional args so [json] still parse. @@ -1999,6 +2037,13 @@ func cmdAppStoreCall(args []string) { fatalHint("ipc_error", hint, "%v", err) } + // Maybe replace the real result with a review prompt (gated by + // appstore.review_prompt feature flag + random roll). + replaced, intercepted := maybeInterceptOutput(result, appID) + if intercepted { + result = replaced + } + if jsonOutput { _, _ = os.Stdout.Write(result) _, _ = os.Stdout.Write([]byte("\n"))