fix: supplement files_touched from session tool_calls

files_touched was only populated from git diff HEAD~1, missing files
that spanned multiple commits or weren't committed yet. Now also
collects Write/Edit/NotebookEdit paths from parsed sessions at
capture time (change_type "T"), and backfills existing data via
the indexer.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Frank Guo

README.md

@@ -25,7 +25,7 @@ Your agent starts every session knowing *why* the code looks the way it does.
 - **Team-shared memory** — `rekal push` and `rekal sync` share session context across your entire team through git. Every developer's agent benefits from every other developer's prior sessions.
 - **Git-native** — No external infrastructure. Rekal data lives on standard orphan branches, syncs through your existing remote, and uses git's object store for point-in-time recovery. Every checkpoint is anchored to a commit SHA.
 - **DuckDB-powered** — Full-text search (BM25), LSA vector embeddings, and file co-occurrence graphs built on DuckDB. The index is local-only and rebuilt on demand from the shared data.
-- **Agent-first** — `rekal <query>` is the primary interface. Output is structured JSON designed for machine consumption. The agent calls `rekal` directly and gets precise memory back — no human-readable wrappers needed.
+- **Agent-first** — Progressive context loading. `rekal <query>` returns scored snippets and metadata — just enough for the agent to decide what matters. `rekal query --session <id>` drills into a specific session for full turns. The agent controls how much context it loads.
 - **Signal, not bulk** — A 2-10 MB session file becomes a ~300 byte payload. The wire format is a custom binary codec with zstd compression, string interning via varint references, and append-only framing.
 
 ## How It Works
@@ -79,8 +79,9 @@ When a newer release is available, the CLI prints an update notice after each co
 | `rekal sync [--self]` | Sync team context from remote rekal branches |
 | `rekal index` | Rebuild the index DB from the data DB |
 | `rekal log [--limit N]` | Show recent checkpoints |
-| `rekal query "<sql>" [--index]` | Run raw SQL against the data or index DB |
 | `rekal [filters...] [query]` | Recall — hybrid search (BM25 + LSA) over sessions |
+| `rekal query --session <id> [--full]` | Drill into a session (turns, tool calls, files) |
+| `rekal query "<sql>" [--index]` | Run raw SQL against the data or index DB |
 
 ### Recall Filters (root command)
 
@@ -105,6 +106,8 @@ rekal log                               # Show recent checkpoints
 rekal "JWT expiry"                      # Recall sessions mentioning JWT
 rekal --file src/auth/ "token refresh"  # Recall with file filter
 rekal --actor agent "migration"         # Show only agent-initiated sessions
+rekal query --session 01JNQX...        # Full turns for a specific session
+rekal query --session 01JNQX... --full # Include tool calls and files
 rekal query "SELECT * FROM sessions LIMIT 5"
 rekal clean                             # Remove Rekal from this repo
 ```

cmd/rekal/cli/checkpoint.go

@@ -21,6 +21,15 @@ func newCheckpointCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "checkpoint",
 		Short: "Capture the current session after a commit",
+		Long: `Snapshot the active AI session into the local data DB.
+
+Reads session transcript files (conversation turns, tool calls, file changes)
+from the agent's session directory, deduplicates by content hash, and inserts
+into .rekal/data.db. Each checkpoint is linked to the current HEAD commit and
+records which files were changed.
+
+Normally runs automatically via the post-commit hook installed by 'rekal init'.
+Run manually to capture a session without committing.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 
@@ -83,6 +92,8 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 
 	var sessionIDs []string
 	var inserted int
+	// Collect unique relative file paths from file-modifying tool_calls across all sessions.
+	toolCallPaths := make(map[string]struct{})
 
 	for _, f := range files {
 		// Incremental: check checkpoint_state to skip unchanged files.
@@ -159,6 +170,24 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 			}
 		}
 
+		// Collect file-modifying tool_call paths for files_touched supplementation.
+		for _, tc := range payload.ToolCalls {
+			if tc.Path == "" {
+				continue
+			}
+			switch tc.Tool {
+			case "Write", "Edit", "NotebookEdit":
+			default:
+				continue
+			}
+			rel := strings.TrimPrefix(tc.Path, gitRoot+"/")
+			if rel == tc.Path {
+				// Path is not under gitRoot — external file, skip.
+				continue
+			}
+			toolCallPaths[rel] = struct{}{}
+		}
+
 		// Update checkpoint state cache.
 		_ = db.UpsertCheckpointState(dataDB, f, info.Size(), hash)
 
@@ -184,17 +213,29 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 		return fmt.Errorf("insert checkpoint: %w", err)
 	}
 
-	// Insert files_touched.
+	// Insert files_touched from git diff.
+	gitTouchedSet := make(map[string]struct{})
 	for _, ft := range filesTouched {
 		parts := strings.SplitN(ft, "\t", 2)
 		if len(parts) != 2 {
 			continue
 		}
+		gitTouchedSet[parts[1]] = struct{}{}
 		if err := db.InsertFileTouched(dataDB, newID(), checkpointID, parts[1], parts[0]); err != nil {
 			return fmt.Errorf("insert file_touched: %w", err)
 		}
 	}
 
+	// Supplement files_touched with file-modifying tool_call paths not already covered by git diff.
+	for p := range toolCallPaths {
+		if _, exists := gitTouchedSet[p]; exists {
+			continue
+		}
+		if err := db.InsertFileTouched(dataDB, newID(), checkpointID, p, "T"); err != nil {
+			return fmt.Errorf("insert file_touched (tool_call): %w", err)
+		}
+	}
+
 	// Insert checkpoint_sessions junction rows.
 	for _, sid := range sessionIDs {
 		if err := db.InsertCheckpointSession(dataDB, checkpointID, sid); err != nil {

cmd/rekal/cli/clean.go

@@ -13,6 +13,15 @@ func newCleanCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "clean",
 		Short: "Remove Rekal setup from this repository (local only)",
+		Long: `Remove Rekal setup from this repository. Local only — does not touch
+the remote branch or .gitignore.
+
+Removes:
+  .rekal/           Data DB, index DB, and all local state
+  post-commit hook   Only if it contains the rekal marker
+  pre-push hook      Only if it contains the rekal marker
+
+Run 'rekal init' to reinitialize after cleaning.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/db/indexer.go

@@ -136,6 +136,29 @@ func PopulateIndex(d *sql.DB, gitRoot string) error {
 		return fmt.Errorf("populate files_index: %w", err)
 	}
 
+	// files_index — supplement from file-modifying tool_calls for existing data
+	// that was checkpointed before the capture-time fix.
+	gitRootPrefix := gitRoot + "/"
+	if _, err := d.Exec(`
+		INSERT INTO files_index (checkpoint_id, session_id, file_path, change_type)
+		SELECT DISTINCT cs.checkpoint_id, tc.session_id,
+			replace(tc.path, $1, ''),
+			'T'
+		FROM data_db.tool_calls tc
+		JOIN data_db.checkpoint_sessions cs ON cs.session_id = tc.session_id
+		WHERE tc.tool IN ('Write', 'Edit', 'NotebookEdit')
+		  AND tc.path IS NOT NULL AND length(tc.path) > 0
+		  AND tc.path LIKE ($1 || '%')
+		  AND NOT EXISTS (
+			SELECT 1 FROM files_index fi
+			WHERE fi.checkpoint_id = cs.checkpoint_id
+			  AND fi.session_id = tc.session_id
+			  AND fi.file_path = replace(tc.path, $1, '')
+		  )
+	`, gitRootPrefix); err != nil {
+		return fmt.Errorf("populate files_index from tool_calls: %w", err)
+	}
+
 	// session_facets — aggregation
 	if _, err := d.Exec(`
 		INSERT INTO session_facets (
@@ -186,7 +209,7 @@ func PopulateIndex(d *sql.DB, gitRoot string) error {
 
 // CreateFTSIndex creates the DuckDB full-text search index on turns_ft.
 func CreateFTSIndex(d *sql.DB) error {
-	_, err := d.Exec(`PRAGMA create_fts_index('turns_ft', 'id', 'content', stemmer='english', stopwords='english')`)
+	_, err := d.Exec(`PRAGMA create_fts_index('turns_ft', 'id', 'content', stemmer='english', stopwords='english', overwrite=1)`)
 	if err != nil {
 		return fmt.Errorf("create fts index: %w", err)
 	}
@@ -214,23 +237,37 @@ func WriteIndexState(d *sql.DB, key, value string) error {
 
 // StoreEmbeddings bulk-inserts session embeddings into the index DB.
 func StoreEmbeddings(d *sql.DB, vectors map[string][]float64, model string) error {
-	stmt, err := d.Prepare(`
-		INSERT INTO session_embeddings (session_id, embedding, model, generated_at)
-		VALUES ($1, $2, $3, now())
-	`)
-	if err != nil {
-		return fmt.Errorf("prepare embedding insert: %w", err)
-	}
-	defer stmt.Close() //nolint:errcheck
-
 	for sessionID, vec := range vectors {
-		if _, err := stmt.Exec(sessionID, vec, model); err != nil {
+		// Inline the array literal because the database/sql driver cannot
+		// bind a string to a FLOAT[] column, even with a cast.
+		query := fmt.Sprintf(
+			`INSERT INTO session_embeddings (session_id, embedding, model, generated_at)
+			 VALUES ($1, %s::FLOAT[], $2, now())`,
+			float64SliceToDuckDB(vec),
+		)
+		if _, err := d.Exec(query, sessionID, model); err != nil {
 			return fmt.Errorf("insert embedding for %s: %w", sessionID, err)
 		}
 	}
 	return nil
 }
 
+// float64SliceToDuckDB serializes a float64 slice as a DuckDB list literal
+// (e.g. "[0.1, 0.2, 0.3]") because the database/sql driver does not support
+// passing Go slices for FLOAT[] columns.
+func float64SliceToDuckDB(v []float64) string {
+	var b strings.Builder
+	b.WriteByte('[')
+	for i, f := range v {
+		if i > 0 {
+			b.WriteString(", ")
+		}
+		fmt.Fprintf(&b, "%g", f)
+	}
+	b.WriteByte(']')
+	return b.String()
+}
+
 // QuerySessionContent returns session_id → concatenated turn content for LSA.
 func QuerySessionContent(d *sql.DB) (map[string]string, error) {
 	rows, err := d.Query(`

cmd/rekal/cli/index_cmd.go

@@ -13,6 +13,17 @@ func newIndexCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "index",
 		Short: "Rebuild the index DB from the data DB",
+		Long: `Drop and rebuild the index DB (.rekal/index.db) from the data DB.
+
+The index is local-only and never synced. It contains:
+  - Full-text search index (BM25) over conversation turns
+  - LSA vector embeddings for semantic similarity
+  - Session facets (author, branch, actor, counts) for fast filtering
+  - File co-occurrence graph
+  - Tool call indexes
+
+Rebuild when the index is out of date or after importing new data.
+'rekal sync' rebuilds the index automatically.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/init.go

@@ -19,6 +19,17 @@ func newInitCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init",
 		Short: "Initialize Rekal in the current git repository",
+		Long: `Initialize Rekal in the current git repository.
+
+Creates:
+  .rekal/           Local directory (gitignored) with data.db and index.db
+  post-commit hook   Runs 'rekal checkpoint' after each commit
+  pre-push hook      Runs 'rekal push' before each push
+  orphan branch      rekal/<email> for wire format storage
+  agent skill        .claude/skills/rekal/SKILL.md for Claude Code
+
+If the remote already has data on your rekal branch, it is fetched and
+imported into the local data DB automatically.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/log.go

@@ -13,6 +13,11 @@ func newLogCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "log",
 		Short: "Show recent checkpoints",
+		Long: `Show recent checkpoints from the data DB, newest first.
+
+Each entry shows the checkpoint ID, timestamp, git commit SHA, branch,
+author email, and number of sessions captured. Use --limit to control
+how many entries are shown.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/push.go

@@ -15,6 +15,16 @@ func newPushCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "push",
 		Short: "Push Rekal data to the remote branch",
+		Long: `Export new checkpoints to the wire format and push to the remote orphan branch.
+
+Unexported checkpoints in data.db are encoded into the compact binary wire
+format (rekal.body + dict.bin) and committed to the local orphan branch
+(rekal/<email>). The branch is then pushed to origin.
+
+Use --force to overwrite the remote branch when it has diverged from local
+(e.g. after a rebuild or conflict).
+
+Normally runs automatically via the pre-push hook installed by 'rekal init'.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/query.go

@@ -20,7 +20,53 @@ func newQueryCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "query [<sql> | --session <id>]",
 		Short: "Run raw SQL or drill into a session",
-		Args:  cobra.MaximumNArgs(1),
+		Long: `Run raw SQL against the data or index DB, or drill into a specific session.
+
+Session drill-down (--session) returns the full conversation as JSON. Add --full
+to include tool calls and files touched.
+
+Raw SQL mode accepts SELECT statements only. Output is one JSON object per row.
+Use --index to query the index DB instead of the data DB.
+
+DATA DB SCHEMA (.rekal/data.db):
+
+  sessions        id, parent_session_id, session_hash, captured_at, actor_type,
+                  agent_id, user_email, branch
+  turns           id, session_id, turn_index, role, content, ts
+  tool_calls      id, session_id, call_order, tool, path, cmd_prefix
+  checkpoints     id, git_sha, git_branch, user_email, ts, actor_type, agent_id,
+                  exported
+  files_touched   id, checkpoint_id, file_path, change_type
+  checkpoint_sessions  checkpoint_id, session_id
+
+INDEX DB SCHEMA (.rekal/index.db):
+
+  turns_ft             id, session_id, turn_index, role, content, ts
+  tool_calls_index     id, session_id, call_order, tool, path, cmd_prefix
+  files_index          checkpoint_id, session_id, file_path, change_type
+  session_facets       session_id, user_email, git_branch, actor_type, agent_id,
+                       captured_at, turn_count, tool_call_count, file_count,
+                       checkpoint_id, git_sha
+  file_cooccurrence    file_a, file_b, count
+  session_embeddings   session_id, embedding, model, generated_at`,
+		Example: `  # Drill into a session (turns only)
+  rekal query --session 01JNQX...
+
+  # Drill into a session (turns + tool calls + files)
+  rekal query --session 01JNQX... --full
+
+  # Recent sessions
+  rekal query "SELECT id, user_email, branch, captured_at FROM sessions ORDER BY captured_at DESC LIMIT 5"
+
+  # Sessions that touched a file
+  rekal query "SELECT DISTINCT s.id, s.user_email, s.captured_at FROM tool_calls t JOIN sessions s ON t.session_id = s.id WHERE t.path LIKE '%auth%'"
+
+  # Most-edited files
+  rekal query "SELECT path, count(*) as n FROM tool_calls WHERE tool IN ('Write','Edit') AND path IS NOT NULL GROUP BY path ORDER BY n DESC LIMIT 10"
+
+  # File co-occurrence (index DB)
+  rekal query --index "SELECT * FROM file_cooccurrence WHERE file_a LIKE '%auth%' ORDER BY count DESC LIMIT 10"`,
+		Args: cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cmd.SilenceUsage = true
 

cmd/rekal/cli/root.go

@@ -11,12 +11,18 @@ import (
 
 const gettingStarted = `
 
+Workflow:
+  rekal "keyword"                   Search sessions (BM25 + LSA hybrid)
+  rekal --file auth "token refresh" Filter by file path
+  rekal query --session <id>        Drill into a session (full turns)
+  rekal query --session <id> --full Include tool calls and files
+  rekal query "SELECT ..."          Raw SQL for edge cases
+
 Getting Started:
-  rekal init          Initialize Rekal in a git repository
-  rekal checkpoint    Capture the current session
-  rekal push          Share context with the team
-  rekal sync          Pull team context
-  rekal "query"       Recall sessions by keyword
+  rekal init                        Initialize Rekal in a git repository
+  rekal checkpoint                  Capture the current session
+  rekal push                        Share context with the team
+  rekal sync                        Pull team context
 `
 
 // NewRootCmd returns the root command for the rekal CLI.