feat: Add new app concept and better query cli command options to limit context usage (#21)

Author: ericallam

.changeset/app-concept-and-pagination.md

@@ -0,0 +1,29 @@
+---
+"agentcrumbs": minor
+---
+
+Add app concept for project-level crumb isolation and cursor-based query pagination.
+
+**App isolation:**
+- Every crumb is stamped with an `app` name, auto-detected from the nearest `package.json`
+- Crumbs stored per-app at `~/.agentcrumbs/<app>/crumbs.jsonl`
+- Collector routes incoming crumbs to per-app stores
+- All CLI commands scope to the current app by default
+- Override with `--app <name>`, `--all-apps`, `AGENTCRUMBS_APP` env var, or `app` field in JSON config
+
+**Query pagination:**
+- New `--cursor` flag for forward pagination with short 8-char cursor IDs
+- New `--after` and `--before` flags for absolute ISO timestamp windows
+- Default limit reduced from 100 to 50 per page
+- Results returned oldest-first with `Next: --cursor <id>` in output when more pages exist
+
+**New files:**
+- `src/cli/app-store.ts` — shared helper for app context resolution across CLI commands
+- `src/cli/cursor.ts` — cursor storage with 1-hour TTL
+
+**Breaking changes:**
+- `Crumb` type now has a required `app: string` field
+- `AgentCrumbsConfig` type now has an optional `app?: string` field
+- `CollectorServer` no longer exposes `getStore()` (routes to per-app stores internally)
+- Storage location changed from `~/.agentcrumbs/crumbs.jsonl` to `~/.agentcrumbs/<app>/crumbs.jsonl`
+- Legacy flat-file crumbs (without `app` field) are still readable as app `"unknown"`

.gitignore

@@ -1,5 +1,6 @@
 node_modules
 dist
+.tshy
 *.tsbuildinfo
 .DS_Store
 .env

README.md

@@ -7,7 +7,7 @@ Crumbs are development-only. They get stripped before merge and cost nothing whe
 ```
 Service A ──┐                                ┌── $ agentcrumbs tail
 Service B ──┤── fetch() ──> Collector :8374 ──┤── $ agentcrumbs query --since 5m
-Service C ──┘  (fire & forget)               └── ~/.agentcrumbs/crumbs.jsonl
+Service C ──┘  (fire & forget)               └── ~/.agentcrumbs/<app>/crumbs.jsonl
 ```
 
 ## Getting started
@@ -54,14 +54,16 @@ When something goes wrong, the agent starts the collector and queries the trail:
 ```bash
 agentcrumbs collect --quiet &
 AGENTCRUMBS=1 node app.js
-agentcrumbs query --since 5m --ns auth-service
+agentcrumbs query --since 5m
 ```
 
 ```
 auth-service  login attempt       +0ms  { tokenPrefix: "eyJhbGci" }
 auth-service  token decode ok     +3ms  { userId: "u_8f3k" }
 auth-service  permissions check   +8ms  { roles: [] }
 auth-service  rejected: no roles  +8ms  { status: 401 }
+
+4 crumbs.
 ```
 
 Now the agent knows: the token is valid, but the user has no roles. The fix is in role assignment, not token validation.
@@ -114,8 +116,11 @@ Everything is controlled by a single `AGENTCRUMBS` environment variable.
 | `auth-*,api-*` | Multiple patterns (comma or space separated) |
 | `* -internal-*` | Match all except excluded patterns |
 | `{"ns":"*","port":9999}` | JSON config with full control |
+| `{"app":"my-app","ns":"*"}` | Explicit app name |
+
+JSON config fields: `app` (app name, default auto-detect from package.json), `ns` (namespace filter, required), `port` (collector port, default 8374), `format` (`"pretty"` or `"json"`, default `"pretty"`).
 
-JSON config fields: `ns` (namespace filter, required), `port` (collector port, default 8374), `format` (`"pretty"` or `"json"`, default `"pretty"`).
+You can also set `AGENTCRUMBS_APP` to override the app name independently.
 
 ## CLI
 
@@ -127,24 +132,27 @@ agentcrumbs collect --quiet &        # Start in background
 agentcrumbs collect --port 9999      # Custom port
 
 # Live tail
-agentcrumbs tail                     # All namespaces
+agentcrumbs tail                     # All namespaces (scoped to current app)
 agentcrumbs tail --ns auth-service   # Filter by namespace
-agentcrumbs tail --tag perf          # Filter by tag
+agentcrumbs tail --app my-app        # Tail a specific app
+agentcrumbs tail --all-apps          # Tail all apps
 
-# Query
-agentcrumbs query --since 5m        # Last 5 minutes
-agentcrumbs query --ns auth-service --since 1h
-agentcrumbs query --tag root-cause
-agentcrumbs query --json --limit 50
+# Query (paginated, 50 per page)
+agentcrumbs query --since 5m         # Last 5 minutes
+agentcrumbs query --since 5m --cursor a1b2c3d4  # Next page
+agentcrumbs query --since 1h --limit 25          # Smaller pages
+agentcrumbs query --session a1b2c3   # Filter by session
+agentcrumbs query --tag root-cause   # Filter by tag
 
 # Strip
 agentcrumbs strip --dry-run          # Preview removals
 agentcrumbs strip                    # Remove all crumb code
 agentcrumbs strip --check            # CI gate (exits 1 if markers found)
 
 # Utilities
-agentcrumbs stats                    # Crumb counts, file size
-agentcrumbs clear                    # Delete stored crumbs
+agentcrumbs stats                    # Crumb counts (current app)
+agentcrumbs stats --all-apps         # Stats for all apps
+agentcrumbs clear                    # Clear crumbs (current app)
 ```
 
 Time units: `s` (seconds), `m` (minutes), `h` (hours), `d` (days).
@@ -160,7 +168,7 @@ The collector is language-agnostic. Any language with HTTP support can send crum
 ```bash
 curl -X POST http://localhost:8374/crumb \
   -H "Content-Type: application/json" \
-  -d '{"ts":"2026-01-01T00:00:00Z","ns":"shell","msg":"hello","type":"crumb","dt":0,"pid":1}'
+  -d '{"app":"my-app","ts":"2026-01-01T00:00:00Z","ns":"shell","msg":"hello","type":"crumb","dt":0,"pid":1}'
 ```
 
 ## Runtime compatibility

docs/content/docs/cli/collect.mdx

@@ -11,7 +11,7 @@ The collector is an HTTP server that receives crumbs via `POST /crumb` and write
 agentcrumbs collect
 # agentcrumbs collector
 #   http:  http://localhost:8374/crumb
-#   store: ~/.agentcrumbs/crumbs.jsonl
+#   crumbs stored per-app in ~/.agentcrumbs/<app>/
 #   press ctrl+c to stop
 ```
 
@@ -50,8 +50,8 @@ agentcrumbs collect --quiet
 
 1. Listens on the specified port for `POST /crumb` requests
 2. Validates the JSON body matches the crumb schema
-3. Appends each crumb as a line to `crumbs.jsonl`
-4. The `tail` and `query` commands read from this file
+3. Routes each crumb to per-app storage at `~/.agentcrumbs/<app>/crumbs.jsonl`
+4. The `tail` and `query` commands read from these files
 
 ### Without the collector
 

docs/content/docs/cli/other.mdx

@@ -3,20 +3,25 @@ title: "Other commands"
 description: "stats, clear, follow, replay, sessions"
 ---
 
+All commands below accept `--app <name>` to scope to a specific app and `--all-apps` to include all apps. Default is auto-detect from `package.json`.
+
 ## `agentcrumbs stats`
 
 Show crumb counts, file size, and active services.
 
 ```bash
-agentcrumbs stats
+agentcrumbs stats              # current app
+agentcrumbs stats --all-apps   # per-app breakdown
 ```
 
 ## `agentcrumbs clear`
 
-Delete all stored crumbs.
+Delete stored crumbs.
 
 ```bash
-agentcrumbs clear
+agentcrumbs clear              # clear current app
+agentcrumbs clear --all-apps   # clear all apps
+agentcrumbs clear --app foo    # clear a specific app
 ```
 
 ## `agentcrumbs sessions`

docs/content/docs/cli/query.mdx

@@ -5,7 +5,7 @@ description: "Search historical crumbs"
 
 ## `agentcrumbs query`
 
-Query historical crumbs with time ranges and filters.
+Query historical crumbs with time windows and cursor-based pagination.
 
 ```bash
 agentcrumbs query --since 5m
@@ -15,34 +15,61 @@ agentcrumbs query --since 5m
 
 | Flag | Description |
 | --- | --- |
-| `--since <duration>` | Time range (e.g., `5m`, `1h`, `24h`, `7d`) |
+| `--since <duration>` | Relative time window (e.g., `5m`, `1h`, `24h`, `7d`) |
+| `--after <timestamp>` | Crumbs after this ISO timestamp |
+| `--before <timestamp>` | Crumbs before this ISO timestamp |
+| `--cursor <id>` | Resume from a previous page (8-char ID from output) |
+| `--limit <n>` | Results per page (default: 50) |
 | `--ns <pattern>` | Filter by namespace |
 | `--tag <tag>` | Filter by tag |
 | `--session <id>` | Filter by session ID |
 | `--match <text>` | Text search |
+| `--app <name>` | Scope to a specific app (default: auto-detect from package.json) |
+| `--all-apps` | Query crumbs from all apps |
 | `--json` | JSON output |
-| `--limit <n>` | Maximum number of results |
 
 Time units: `s` (seconds), `m` (minutes), `h` (hours), `d` (days).
 
+### Pagination
+
+Results are returned oldest-first, capped at `--limit` (default 50). When there are more results, the output includes a short cursor ID for the next page.
+
+```bash
+# First page
+agentcrumbs query --since 5m
+# Output: 50 crumbs (1-50 of 128). Next: --cursor a1b2c3d4
+
+# Next page
+agentcrumbs query --since 5m --cursor a1b2c3d4
+# Output: 50 crumbs (51-100 of 128). Next: --cursor e5f6g7h8
+```
+
+Cursors expire after 1 hour. You can also use `--after` / `--before` with ISO timestamps for explicit time windows without cursors.
+
 ### Examples
 
 ```bash
-# Last 5 minutes
+# Last 5 minutes (all namespaces)
 agentcrumbs query --since 5m
 
-# Last hour, filtered by namespace
-agentcrumbs query --since 1h --ns auth-service
+# Paginate through results
+agentcrumbs query --since 5m --cursor a1b2c3d4
 
-# Filter by tag
-agentcrumbs query --tag root-cause
+# Time window with absolute timestamps
+agentcrumbs query --after 2026-03-11T14:00:00Z --before 2026-03-11T14:05:00Z
+
+# Smaller pages
+agentcrumbs query --since 1h --limit 25
 
 # Filter by session
 agentcrumbs query --session a1b2c3
 
-# JSON output with limit
-agentcrumbs query --since 1h --json --limit 50
+# Filter by tag
+agentcrumbs query --tag root-cause
+
+# Query a specific app
+agentcrumbs query --since 1h --app my-project
 
-# Text search
-agentcrumbs query --since 24h --match "connection refused"
+# Query across all apps
+agentcrumbs query --since 5m --all-apps
 ```

docs/content/docs/cli/tail.mdx

@@ -19,6 +19,8 @@ agentcrumbs tail
 | `--tag <tag>` | Filter by tag |
 | `--match <text>` | Filter by content |
 | `--session <id>` | Filter by session ID |
+| `--app <name>` | Scope to a specific app (default: auto-detect from package.json) |
+| `--all-apps` | Show crumbs from all apps |
 | `--json` | JSON output (for piping to jq, etc.) |
 
 ### Examples
@@ -37,6 +39,12 @@ agentcrumbs tail --match "userId:123"
 # Filter by session
 agentcrumbs tail --session a1b2c3
 
+# Scope to a specific app
+agentcrumbs tail --app my-project
+
+# Show crumbs from all apps
+agentcrumbs tail --all-apps
+
 # JSON output for piping
 agentcrumbs tail --json | jq '.data.userId'
 ```

docs/content/docs/config/env-var.mdx

@@ -43,16 +43,35 @@ AGENTCRUMBS='{"ns":"*","port":9999}'
 
 # JSON output format (instead of pretty)
 AGENTCRUMBS='{"ns":"*","format":"json"}'
+
+# Explicit app name
+AGENTCRUMBS='{"app":"my-project","ns":"*"}'
 ```
 
 ## Config schema
 
 | Field | Type | Default | Description |
 | --- | --- | --- | --- |
+| `app` | `string` | (auto-detect) | App name. Defaults to nearest `package.json` name |
 | `ns` | `string` | (required) | Namespace filter pattern |
 | `port` | `number` | `8374` | Collector HTTP port |
 | `format` | `"pretty"` \| `"json"` | `"pretty"` | Output format for stderr |
 
+## App name
+
+Every crumb is stamped with an app name. This keeps crumbs from different projects separate.
+
+The app name is resolved in this order:
+1. `app` field in the JSON config
+2. `AGENTCRUMBS_APP` environment variable
+3. Auto-detected from the nearest `package.json` name field (walking up from `cwd`)
+4. Fallback: `"unknown"`
+
+```bash
+# Override via dedicated env var
+AGENTCRUMBS_APP=my-project AGENTCRUMBS=1 node app.js
+```
+
 ## Namespace patterns
 
 - `*` matches everything

docs/content/docs/crumb-format.mdx

@@ -9,6 +9,7 @@ Each crumb is a JSON object. When stored, they are written as JSONL (one JSON ob
 
 ```json
 {
+  "app": "my-project",
   "ts": "2026-03-07T10:00:00.123Z",
   "ns": "auth-service",
   "msg": "user logged in",
@@ -28,6 +29,7 @@ Each crumb is a JSON object. When stored, they are written as JSONL (one JSON ob
 
 | Field | Type | Required | Description |
 | --- | --- | --- | --- |
+| `app` | `string` | Yes | App name (auto-detected from `package.json` or explicit config) |
 | `ts` | `string` | Yes | ISO 8601 timestamp |
 | `ns` | `string` | Yes | Namespace |
 | `msg` | `string` | Yes | Message |
@@ -57,4 +59,4 @@ Each crumb is a JSON object. When stored, they are written as JSONL (one JSON ob
 
 ## Storage
 
-Crumbs are stored in `~/.agentcrumbs/crumbs.jsonl` by default. One JSON object per line, no trailing comma, no wrapping array.
+Crumbs are stored per-app at `~/.agentcrumbs/<app>/crumbs.jsonl`. One JSON object per line, no trailing comma, no wrapping array. The app name is auto-detected from the nearest `package.json` by default.

docs/content/docs/guides/cross-language.mdx

@@ -13,6 +13,7 @@ Send a `POST` request to `http://localhost:8374/crumb` with a JSON body matching
 
 | Field | Type | Description |
 | --- | --- | --- |
+| `app` | `string` | App name (used for per-app storage routing) |
 | `ts` | `string` | ISO 8601 timestamp |
 | `ns` | `string` | Namespace |
 | `msg` | `string` | Message |
@@ -38,7 +39,7 @@ Send a `POST` request to `http://localhost:8374/crumb` with a JSON body matching
 ```bash
 curl -X POST http://localhost:8374/crumb \
   -H "Content-Type: application/json" \
-  -d '{"ts":"2026-01-01T00:00:00Z","ns":"shell","msg":"hello","type":"crumb","dt":0,"pid":1}'
+  -d '{"app":"my-app","ts":"2026-01-01T00:00:00Z","ns":"shell","msg":"hello","type":"crumb","dt":0,"pid":1}'
 ```
 
 ### Python
@@ -50,6 +51,7 @@ from datetime import datetime
 def crumb(ns, msg, data=None):
     try:
         requests.post("http://localhost:8374/crumb", json={
+            "app": "my-app",
             "ts": datetime.utcnow().isoformat() + "Z",
             "ns": ns,
             "msg": msg,
@@ -69,6 +71,7 @@ crumb("python-service", "processing started", {"items": 42})
 ```go
 func crumb(ns, msg string, data any) {
     body, _ := json.Marshal(map[string]any{
+        "app":  "my-app",
         "ts":   time.Now().UTC().Format(time.RFC3339Nano),
         "ns":   ns,
         "msg":  msg,
@@ -86,6 +89,7 @@ func crumb(ns, msg string, data any) {
 ```rust
 fn crumb(ns: &str, msg: &str) {
     let body = serde_json::json!({
+        "app": "my-app",
         "ts": chrono::Utc::now().to_rfc3339(),
         "ns": ns,
         "msg": msg,