test: expand sanity suite with branding, deny enforcement, and resilience tests

kulvirgit · claude · kulvirgit · commit 81cb93c1f1c2 · 2026-03-26T17:48:42.000-07:00
Add 9 new sanity tests to catch regressions for open issues: - verify-install: semver format (#212), --help branding leaks (#416/#417) - resilience: yolo deny enforcement (#372/#377), no-internet graceful handling (#181) - smoke-tests: dbt config discovery (#448), altimate check command (#453) - pr-tests/generate: trigger branding/permission/dbt tests on relevant file changes Renumber resilience tests from 8 to 10 total, reorganize config compat and broken config tests into proper sequence. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
diff --git a/test/sanity/phases/resilience.sh b/test/sanity/phases/resilience.sh
@@ -23,7 +23,7 @@ echo '{}' > package.json
 git add -A && git commit -q -m "init"
 
 # 1. SQLite DB created after first run
-echo "  [1/7] SQLite DB creation..."
+echo "  [1/10] SQLite DB creation..."
 if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
   altimate_run "db-create" "say hello" || true
   # Find the DB — could be opencode.db, opencode-latest.db, or opencode-{channel}.db
@@ -42,7 +42,7 @@ else
 fi
 
 # 2. WAL mode enabled
-echo "  [2/7] WAL mode..."
+echo "  [2/10] WAL mode..."
 if [ -n "$DB_PATH" ] && [ -f "$DB_PATH" ] && command -v sqlite3 >/dev/null 2>&1; then
   WAL_MODE=$(sqlite3 "$DB_PATH" "PRAGMA journal_mode;" 2>/dev/null || echo "unknown")
   assert_eq "$WAL_MODE" "wal" "WAL mode enabled"
@@ -51,7 +51,7 @@ else
 fi
 
 # 3. Session persisted
-echo "  [3/7] Session persistence..."
+echo "  [3/10] Session persistence..."
 if [ -n "$DB_PATH" ] && [ -f "$DB_PATH" ] && command -v sqlite3 >/dev/null 2>&1; then
   SESSION_COUNT=$(sqlite3 "$DB_PATH" "SELECT count(*) FROM session;" 2>/dev/null || echo "0")
   assert_ge "$SESSION_COUNT" 1 "session persisted (got $SESSION_COUNT)"
@@ -60,7 +60,7 @@ else
 fi
 
 # 4. Session continue (DB survives restart)
-echo "  [4/7] Session continue..."
+echo "  [4/10] Session continue..."
 if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
   altimate_run "continue" --continue "what was my last message" || true
   assert_not_contains "$(get_output continue)" "TIMEOUT" "session continue works"
@@ -69,7 +69,7 @@ else
 fi
 
 # 5. Compaction doesn't crash (best-effort — seed if fixture available)
-echo "  [5/7] Compaction resilience..."
+echo "  [5/10] Compaction resilience..."
 if [ -n "$DB_PATH" ] && [ -f "$SCRIPT_DIR/fixtures/compaction-session.sql" ] && command -v sqlite3 >/dev/null 2>&1; then
   sqlite3 "$DB_PATH" < "$SCRIPT_DIR/fixtures/compaction-session.sql" 2>/dev/null || true
   if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
@@ -90,20 +90,8 @@ else
   skip_test "Compaction resilience" "fixture or sqlite3 not available"
 fi
 
-# 6. Graceful on missing provider key
-echo "  [6/7] Missing API key handling..."
-SAVED_KEY="${ANTHROPIC_API_KEY:-}"
-unset ANTHROPIC_API_KEY
-OUTPUT=$(timeout 10 altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
-# Should get a clean error, not an unhandled exception / stack trace
-assert_not_contains "$OUTPUT" "TypeError" "no TypeError on missing key"
-assert_not_contains "$OUTPUT" "Cannot read properties" "no unhandled error on missing key"
-if [ -n "$SAVED_KEY" ]; then
-  export ANTHROPIC_API_KEY="$SAVED_KEY"
-fi
-
-# 7. Config backwards compatibility
-echo "  [7/7] Config backwards compat..."
+# 6. Config backwards compatibility
+echo "  [6/10] Config backwards compat..."
 CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/altimate-code"
 mkdir -p "$CONFIG_DIR"
 if [ -f "$SCRIPT_DIR/fixtures/old-config.json" ]; then
@@ -119,8 +107,8 @@ else
   skip_test "Config backwards compat" "old-config.json fixture not found"
 fi
 
-# 8. Broken config graceful handling
-echo "  [8/8] Broken config handling..."
+# 7. Broken config graceful handling
+echo "  [7/10] Broken config handling..."
 if [ -f "$SCRIPT_DIR/fixtures/broken-config.json" ]; then
   cp "$SCRIPT_DIR/fixtures/broken-config.json" "$CONFIG_DIR/opencode.json"
   if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
@@ -135,6 +123,71 @@ else
   skip_test "Broken config handling" "broken-config.json fixture not found"
 fi
 
+# 8. Yolo deny enforcement — deny rules block even with --yolo (#372, #377)
+echo "  [8/10] Yolo deny enforcement..."
+if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
+  # Create a config with an explicit deny rule
+  DENY_CONFIG_DIR=$(mktemp -d /tmp/sanity-deny-config-XXXXXX)
+  mkdir -p "$DENY_CONFIG_DIR"
+  cat > "$DENY_CONFIG_DIR/opencode.jsonc" <<'DENYEOF'
+{
+  "permission": {
+    "bash": {
+      "*": "allow",
+      "DROP DATABASE *": "deny",
+      "drop database *": "deny"
+    }
+  }
+}
+DENYEOF
+  # Run with yolo and ask it to execute a denied command
+  DENY_OUTPUT=$(XDG_CONFIG_HOME="$DENY_CONFIG_DIR" timeout 30 altimate run --max-turns 2 --yolo --format json \
+    "run the bash command: DROP DATABASE production" 2>&1 || true)
+  # The output should NOT contain "DROP DATABASE production" as a successfully executed command
+  # It SHOULD contain "denied" or refuse the command
+  if echo "$DENY_OUTPUT" | grep -qi "denied\|blocked\|not allowed\|permission.*deny"; then
+    echo "  PASS: yolo deny rule blocked dangerous command"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  elif echo "$DENY_OUTPUT" | grep -qi "DROP DATABASE production.*success\|executed.*DROP DATABASE"; then
+    echo "  FAIL: yolo mode bypassed deny rule — dangerous command executed"
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+  else
+    # The model may have refused on its own or rephrased — that's acceptable
+    echo "  PASS: yolo deny rule (command not executed)"
+    PASS_COUNT=$((PASS_COUNT + 1))
+  fi
+  rm -rf "$DENY_CONFIG_DIR"
+else
+  skip_test "Yolo deny enforcement" "no ANTHROPIC_API_KEY"
+fi
+
+# 9. Missing API key handling (no unhandled exceptions)
+echo "  [9/10] Missing API key handling..."
+SAVED_KEY="${ANTHROPIC_API_KEY:-}"
+unset ANTHROPIC_API_KEY
+OUTPUT=$(timeout 10 altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
+assert_not_contains "$OUTPUT" "TypeError" "no TypeError on missing key"
+assert_not_contains "$OUTPUT" "Cannot read properties" "no unhandled error on missing key"
+if [ -n "$SAVED_KEY" ]; then
+  export ANTHROPIC_API_KEY="$SAVED_KEY"
+fi
+
+# 10. No internet — graceful error, not blank screen (#181)
+echo "  [10/10] No internet graceful handling..."
+# Use an unreachable DNS to simulate no internet (timeout quickly)
+NO_NET_OUTPUT=$(timeout 15 env https_proxy=http://192.0.2.1:1 http_proxy=http://192.0.2.1:1 \
+  altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
+assert_not_contains "$NO_NET_OUTPUT" "TypeError" "no TypeError without internet"
+assert_not_contains "$NO_NET_OUTPUT" "Cannot read properties" "no unhandled error without internet"
+# Should get some kind of connection/auth error, not a blank hang
+if [ -z "$NO_NET_OUTPUT" ]; then
+  echo "  FAIL: no output at all without internet (blank screen)"
+  FAIL_COUNT=$((FAIL_COUNT + 1))
+else
+  echo "  PASS: produced output without internet ($(echo "$NO_NET_OUTPUT" | wc -l) lines)"
+  PASS_COUNT=$((PASS_COUNT + 1))
+fi
+
 # Cleanup
 rm -rf "$WORKDIR"
 
diff --git a/test/sanity/phases/smoke-tests.sh b/test/sanity/phases/smoke-tests.sh
@@ -152,6 +152,41 @@ test_discover() {
   fi
 }
 
+test_dbt_discover() {
+  cd "$WORKDIR"
+  # Create a minimal dbt project so auto-discover has something to find (#448, #270)
+  mkdir -p dbt_project models
+  cat > dbt_project.yml <<'DBTEOF'
+name: sanity_test
+version: '1.0.0'
+config-version: 2
+profile: sanity_test
+DBTEOF
+  cat > models/test_model.sql <<'SQLEOF'
+SELECT 1 AS id
+SQLEOF
+  git add -A && git commit -q -m "add dbt project"
+  SANITY_TIMEOUT=90 altimate_run_with_turns "dbt-discover" 2 "discover dbt project config in this repo" || true
+  local output=$(get_output "dbt-discover")
+  if echo "$output" | grep -qi "unhandled\|TypeError\|Cannot read"; then
+    echo "FAIL" > "$RESULTS_DIR/dbt-discover"
+  else
+    echo "PASS" > "$RESULTS_DIR/dbt-discover"
+  fi
+}
+
+test_check_command() {
+  cd "$WORKDIR"
+  # altimate-code check should run deterministic SQL checks without LLM (#453)
+  echo "SELECT * FROM users WHERE 1=1;" > check_target.sql
+  local output=$(timeout 30 altimate check --file check_target.sql 2>&1 || true)
+  if echo "$output" | grep -qi "TypeError\|unhandled\|Cannot read properties"; then
+    echo "FAIL" > "$RESULTS_DIR/check-cmd"
+  else
+    echo "PASS" > "$RESULTS_DIR/check-cmd"
+  fi
+}
+
 # Run tests in parallel batches
 echo ""
 echo "  Running $MAX_PARALLEL tests concurrently..."
@@ -168,6 +203,8 @@ TESTS=(
   "test_analyst"
   "test_bad_command"
   "test_discover"
+  "test_dbt_discover"
+  "test_check_command"
 )
 
 TEST_NAMES=(
@@ -181,6 +218,8 @@ TEST_NAMES=(
   "analyst"
   "bad-cmd"
   "discover"
+  "dbt-discover"
+  "check-cmd"
 )
 
 # Launch in batches of MAX_PARALLEL
diff --git a/test/sanity/phases/verify-install.sh b/test/sanity/phases/verify-install.sh
@@ -41,4 +41,41 @@ fi
 # 9. git available (needed for project detection)
 assert_exit_0 "git CLI available" git --version
 
+# 10. Version matches semver format (X.Y.Z) — catches #212 regressions
+VERSION_CLEAN=$(echo "$VERSION" | head -1 | tr -d '[:space:]')
+if echo "$VERSION_CLEAN" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+  echo "  PASS: version is semver ($VERSION_CLEAN)"
+  PASS_COUNT=$((PASS_COUNT + 1))
+else
+  echo "  FAIL: version is not semver (got '$VERSION_CLEAN')"
+  FAIL_COUNT=$((FAIL_COUNT + 1))
+fi
+
+# 11. --help output doesn't contain upstream "opencode" branding (#416, #417)
+# Legitimate internal names (.opencode/, opencode.json, @opencode-ai/) are allowed;
+# user-facing prose like "opencode is a..." or "start opencode" is not.
+HELP_OUTPUT=$(altimate --help 2>&1 || echo "")
+BRANDING_LEAKS=$(echo "$HELP_OUTPUT" | grep -iE 'opencode' | grep -ivE '\.opencode|opencode\.json[c]?|@opencode-ai|opencode\.local|OPENCODE_' || true)
+if [ -z "$BRANDING_LEAKS" ]; then
+  echo "  PASS: --help has no upstream branding leaks"
+  PASS_COUNT=$((PASS_COUNT + 1))
+else
+  echo "  FAIL: --help contains upstream branding:"
+  echo "$BRANDING_LEAKS" | head -5 | sed 's/^/    /'
+  FAIL_COUNT=$((FAIL_COUNT + 1))
+fi
+
+# 12. No "opencode" in welcome/hint strings visible at startup
+# Check the binary's embedded strings for user-facing "opencode" references
+SUBCOMMAND_HELP=$(altimate run --help 2>&1 || echo "")
+SUBCOMMAND_LEAKS=$(echo "$SUBCOMMAND_HELP" | grep -iE 'opencode' | grep -ivE '\.opencode|opencode\.json[c]?|@opencode-ai|opencode\.local|OPENCODE_' || true)
+if [ -z "$SUBCOMMAND_LEAKS" ]; then
+  echo "  PASS: 'altimate run --help' has no upstream branding leaks"
+  PASS_COUNT=$((PASS_COUNT + 1))
+else
+  echo "  FAIL: 'altimate run --help' contains upstream branding:"
+  echo "$SUBCOMMAND_LEAKS" | head -5 | sed 's/^/    /'
+  FAIL_COUNT=$((FAIL_COUNT + 1))
+fi
+
 report_results "Phase 1: Verify Installation"
diff --git a/test/sanity/pr-tests/generate.sh b/test/sanity/pr-tests/generate.sh
@@ -75,6 +75,21 @@ if echo "$changed" | grep -q "migration/"; then
   emit_test "upgrade-needed" "$SCRIPT_DIR/phases/verify-upgrade.sh"
 fi
 
+# permission/yolo changed → run deny enforcement test
+if echo "$changed" | grep -qE "permission|yolo|flag"; then
+  emit_test "yolo-deny" "$SCRIPT_DIR/phases/resilience.sh"
+fi
+
+# branding/welcome/help changed → check for opencode leaks
+if echo "$changed" | grep -qE "welcome|help|hints|cli/cmd/tui"; then
+  emit_test "branding-check" "$SCRIPT_DIR/phases/verify-install.sh"
+fi
+
+# dbt tools changed → test dbt discovery
+if echo "$changed" | grep -qE "dbt|altimate-dbt|profiles"; then
+  emit_test "dbt-discover" "altimate run --max-turns 2 --yolo 'discover dbt project config'"
+fi
+
 COUNT=$(wc -l < "$MANIFEST")
 echo ""
 echo "  Generated $COUNT PR-specific test(s)"
