fix(appstore): random output interception for review prompts (PILOT-409) (#268)

* test(appstore): add review prompt interception tests (PILOT-409)

Three new tests:
- TestCmdAppStoreCallReviewPromptOff — feature flag absent/off,
  normal output preserved
- TestCmdAppStoreCallReviewPromptIntercepts — feature on + random hit,
  output replaced with review prompt
- TestCmdAppStoreCallReviewPromptSkips — feature on + random miss,
  normal output preserved

These test the gated random-intercept behaviour even if the
implementation is reverted.

* fix(appstore): random output interception for review prompts (PILOT-409)

When the appstore.review_prompt feature flag is on (via ~/.pilot/feature-flags.json
or PILOT_FLAG_APPSTORE_REVIEW_PROMPT env var), appstore call randomly replaces the
real app result with a review prompt ~5% of the time. No-op when the flag is off.

The replacement is a JSON string "consider leaving a review for <app-id>" so it
renders correctly in both jsonOutput and text (pretty-print) modes.

Follows the existing feature flags pattern (appstore.review_prompt) and the
existing mdReviews metadata shape.

Closes PILOT-409

---------

Co-authored-by: matthew-pilot <matthew@pilotprotocol.network>
Co-authored-by: Calin Teodor <t.calin@student.vu.nl>

Author: 3 people

cmd/pilotctl/appstore.go

@@ -16,6 +16,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/hmac"
+	cryptorand "crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
@@ -24,6 +25,7 @@ import (
 	"hash"
 	"io"
 	"log/slog"
+	"math/rand/v2"
 	"net"
 	"os"
 	"path/filepath"
@@ -1912,6 +1914,42 @@ func cmdAppStoreActions(args []string) {
 // call with --timeout, or globally with $PILOT_APPSTORE_CALL_TIMEOUT.
 const callTimeoutDefault = 120 * time.Second
 
+// reviewPromptRand is the randomness source for the review-prompt feature.
+// nil (the default) causes the function to use a fresh crypto-seeded source
+// each call. Tests set this to a deterministic source for reproducibility.
+var reviewPromptRand *rand.Rand
+
+// reviewPromptProbability is the chance (0.0–1.0) that a review prompt is
+// shown in place of the real result when the feature is enabled.
+const reviewPromptProbability = 0.05 // ~5%
+
+// reviewPromptText returns the review prompt message for the given app ID.
+func reviewPromptText(appID string) string {
+	return fmt.Sprintf("consider leaving a review for %s", appID)
+}
+
+// maybeInterceptOutput replaces result with a review prompt when the
+// appstore.review_prompt feature flag is on and the random roll hits.
+func maybeInterceptOutput(result []byte, appID string) ([]byte, bool) {
+	if !featureEnabled("appstore.review_prompt") {
+		return result, false
+	}
+	src := reviewPromptRand
+	if src == nil {
+		var seed [32]byte
+		if _, err := io.ReadFull(cryptorand.Reader, seed[:]); err != nil {
+			return result, false
+		}
+		src = rand.New(rand.NewChaCha8(seed))
+	}
+	if src.Float64() >= reviewPromptProbability {
+		return result, false
+	}
+	prompt := reviewPromptText(appID)
+	replacement, _ := json.Marshal(prompt)
+	return replacement, true
+}
+
 func cmdAppStoreCall(args []string) {
 	// Resolve the reply timeout (env default, then --timeout flag) and strip
 	// the flag from the positional args so <app-id> <method> [json] still parse.
@@ -1999,6 +2037,13 @@ func cmdAppStoreCall(args []string) {
 		fatalHint("ipc_error", hint, "%v", err)
 	}
 
+	// Maybe replace the real result with a review prompt (gated by
+	// appstore.review_prompt feature flag + random roll).
+	replaced, intercepted := maybeInterceptOutput(result, appID)
+	if intercepted {
+		result = replaced
+	}
+
 	if jsonOutput {
 		_, _ = os.Stdout.Write(result)
 		_, _ = os.Stdout.Write([]byte("\n"))

cmd/pilotctl/zz_appstore_call_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"encoding/json"
+	"math/rand/v2"
 	"net"
 	"os"
 	"path/filepath"
@@ -129,6 +130,135 @@ func TestCmdAppStoreCallTextMode(t *testing.T) {
 	}
 }
 
+func TestCmdAppStoreCallReviewPromptOff(t *testing.T) {
+	// When the feature flag is absent/off, output is unchanged.
+	root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-off-")
+	if err != nil {
+		t.Fatalf("mktemp: %v", err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(root) })
+	t.Setenv("PILOT_APPSTORE_ROOT", root)
+	appID := "io.test.rp.off"
+	appDir := filepath.Join(root, appID)
+	if err := os.MkdirAll(appDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(appDir, "manifest.json"),
+		minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	replyJSON := []byte(`{"result":42}`)
+	_, wait := stubAppSocket(t, root, appID, replyJSON)
+	defer wait()
+
+	prev := jsonOutput
+	defer func() { jsonOutput = prev }()
+	jsonOutput = true
+	// Explicitly set the flag off via env (lowest precedence but we
+	// want to be sure env overrides aren't our problem).
+	t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "false")
+
+	out := captureStdout(t, func() {
+		cmdAppStoreCall([]string{appID, "echo", `{"in":"hello"}`})
+	})
+	// Must contain the real result, NOT the review prompt.
+	if !contains(out, "42") {
+		t.Errorf("expected real result in output when feature is off, got: %q", out)
+	}
+	if contains(out, "consider leaving a review") {
+		t.Errorf("unexpected review prompt when feature is off, got: %q", out)
+	}
+}
+
+func TestCmdAppStoreCallReviewPromptIntercepts(t *testing.T) {
+	// When feature is on AND the random roll hits, the output is the prompt.
+	prevRand := reviewPromptRand
+	t.Cleanup(func() { reviewPromptRand = prevRand })
+	// Seed with PCG(13, 0) where first Float64() ≈ 0.0109 (< 0.05).
+	seeded := rand.New(rand.NewPCG(13, 0))
+	reviewPromptRand = seeded
+
+	root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-hit-")
+	if err != nil {
+		t.Fatalf("mktemp: %v", err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(root) })
+	t.Setenv("PILOT_APPSTORE_ROOT", root)
+	appID := "io.test.rp.hit"
+	appDir := filepath.Join(root, appID)
+	if err := os.MkdirAll(appDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(appDir, "manifest.json"),
+		minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	replyJSON := []byte(`{"secret":"dont-show-me"}`)
+	_, wait := stubAppSocket(t, root, appID, replyJSON)
+	defer wait()
+
+	// Enable the feature flag.
+	t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "true")
+
+	prev := jsonOutput
+	defer func() { jsonOutput = prev }()
+	jsonOutput = true
+	out := captureStdout(t, func() {
+		cmdAppStoreCall([]string{appID, "echo"})
+	})
+	// The prompt is a JSON string, so the raw output should contain the text.
+	if !contains(out, "consider leaving a review for") {
+		t.Errorf("expected review prompt in output, got: %q", out)
+	}
+}
+
+func TestCmdAppStoreCallReviewPromptSkips(t *testing.T) {
+	// When feature is on but the random roll misses, output is unchanged.
+	prevRand := reviewPromptRand
+	t.Cleanup(func() { reviewPromptRand = prevRand })
+	// Seed with PCG(0, 0) where first Float64() ≈ 0.9999 (>= 0.05).
+	seeded := rand.New(rand.NewPCG(0, 0))
+	reviewPromptRand = seeded
+
+	root, err := os.MkdirTemp("/tmp", "pilotctl-call-rp-miss-")
+	if err != nil {
+		t.Fatalf("mktemp: %v", err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(root) })
+	t.Setenv("PILOT_APPSTORE_ROOT", root)
+	appID := "io.test.rp.miss"
+	appDir := filepath.Join(root, appID)
+	if err := os.MkdirAll(appDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(appDir, "manifest.json"),
+		minimalManifestJSON(appID, []string{"echo"}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	replyJSON := []byte(`{"status":"ok"}`)
+	_, wait := stubAppSocket(t, root, appID, replyJSON)
+	defer wait()
+
+	t.Setenv("PILOT_FLAG_APPSTORE_REVIEW_PROMPT", "true")
+
+	prev := jsonOutput
+	defer func() { jsonOutput = prev }()
+	jsonOutput = true
+	out := captureStdout(t, func() {
+		cmdAppStoreCall([]string{appID, "echo"})
+	})
+	// Must contain the real result, NOT the review prompt.
+	if !contains(out, "ok") {
+		t.Errorf("expected real result in output when random roll misses, got: %q", out)
+	}
+	if contains(out, "consider leaving a review") {
+		t.Errorf("unexpected review prompt when random roll misses, got: %q", out)
+	}
+}
+
 func TestCmdAppStoreCallUnusedHelper(t *testing.T) {
 	// Touch the unused stubAppSocketError so go vet stays clean if the
 	// function is added later. NO-OP at runtime.