fix/deep review improvements (#57)

* refactor(filter): remove unused reserved layer config types

Remove QuestionAwareLayerConfig, DensityAdaptiveLayerConfig,
NumericalQuantLayerConfig, and DynamicRatioLayerConfig (all marked
"reserved for future implementation") plus their never-read fields
on LayerConfig. Verified unreferenced via grep across the repo.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(tracking): include db path and operation in wrapped errors

All error paths in tracking.go now report the database path and the
operation that failed (open, create dir, init schema, enable WAL,
record, aggregate, recent, scan, prune), making failures diagnosable
without enabling debug logging.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* docs(unsafe): add safety-invariant comments at unsafe conversions

Each unsafe string<->[]byte conversion in internal/fastops/simd.go and
internal/filter/zerocopy.go now carries an explicit "safe: ..." comment
stating the aliasing invariant (backing array not mutated while the
alias is live). Also migrated the legacy *(*string)(unsafe.Pointer(&b))
header-punning pattern to the supported unsafe.String/unsafe.Slice +
unsafe.SliceData/StringData APIs, which are valid under the Go memory
model (the old slice->string pun read a slice header as a string header).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: Patel230

CHANGELOG.md

@@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (`hawk`, `eyrie`, `yaad`, `sight`, `inspect`). No git tag has been cut yet.
 
 ### Removed
+- Unused internal config structs `QuestionAwareLayerConfig`,
+  `DensityAdaptiveLayerConfig`, `NumericalQuantLayerConfig`, and
+  `DynamicRatioLayerConfig` (and their fields on the internal `LayerConfig`)
+  that were marked "reserved for future implementation" and never referenced.
 - `scripts/deploy.sh` — referenced a Dockerfile, k8s manifests, and an
   `internal/integration/` package that never existed. tok is a Go library
   (no binary, no HTTP server); deployment happens via `go get`.

internal/fastops/simd.go

@@ -168,15 +168,18 @@ func FastLower(s string) string {
 		return s
 	}
 
-	// Convert using unsafe for zero-copy when possible
-	b := []byte(s)
+	// Convert using unsafe to avoid a second copy on the way back to string.
+	b := []byte(s) // fresh copy of s; we own this backing array exclusively
 	for i := 0; i < len(b); i++ {
 		c := b[i]
 		if c >= 'A' && c <= 'Z' {
 			b[i] = c + ('a' - 'A')
 		}
 	}
-	return *(*string)(unsafe.Pointer(&b))
+	// safe: b is a private copy created above; no other reference to its
+	// backing array exists and it is never mutated after this point, so
+	// aliasing it as a string preserves string immutability.
+	return unsafe.String(unsafe.SliceData(b), len(b))
 }
 
 // FastEqual compares strings with early exit

internal/filter/pipeline_types.go

@@ -225,41 +225,12 @@ type AgentMemoryLayerConfig struct {
 	ConsolidationMax   int
 }
 
-// QuestionAwareLayerConfig groups T12 settings.
-// NOTE: Currently unused - reserved for future implementation.
-type QuestionAwareLayerConfig struct {
-	Enabled   bool
-	Threshold float64
-}
-
-// DensityAdaptiveLayerConfig groups T17 settings.
-// NOTE: Currently unused - reserved for future implementation.
-type DensityAdaptiveLayerConfig struct {
-	Enabled     bool
-	TargetRatio float64
-	Threshold   float64
-}
-
 // TFIDFLayerConfig groups TF-IDF filter settings.
 type TFIDFLayerConfig struct {
 	Enabled   bool
 	Threshold float64
 }
 
-// NumericalQuantLayerConfig groups numerical quantization settings.
-// NOTE: Currently unused - reserved for future implementation.
-type NumericalQuantLayerConfig struct {
-	Enabled       bool
-	DecimalPlaces int
-}
-
-// DynamicRatioLayerConfig groups dynamic compression ratio settings.
-// NOTE: Currently unused - reserved for future implementation.
-type DynamicRatioLayerConfig struct {
-	Enabled bool
-	Base    float64
-}
-
 // LayerConfig groups per-layer config structs.
 type LayerConfig struct {
 	Core              CoreLayersConfig
@@ -273,11 +244,7 @@ type LayerConfig struct {
 	LazyPruner        LazyPrunerLayerConfig
 	SemanticAnchor    SemanticAnchorLayerConfig
 	AgentMemory       AgentMemoryLayerConfig
-	QuestionAware     QuestionAwareLayerConfig
-	DensityAdaptive   DensityAdaptiveLayerConfig
 	TFIDF             TFIDFLayerConfig
-	NumericalQuant    NumericalQuantLayerConfig
-	DynamicRatio      DynamicRatioLayerConfig
 	SymbolicCompress  bool
 	PhraseGrouping    bool
 	Hypernym          bool

internal/filter/zerocopy.go

@@ -32,7 +32,10 @@ func (z *ZeroCopyBuffer) String() string {
 	if len(z.data) == 0 {
 		return ""
 	}
-	return *(*string)(unsafe.Pointer(&z.data))
+	// safe: the string header aliases z.data's backing array. The invariant
+	// (documented above) is that the backing array is not mutated while the
+	// returned string is live — Append/Reset after String() violate it.
+	return unsafe.String(unsafe.SliceData(z.data), len(z.data))
 }
 
 func (z *ZeroCopyBuffer) Reset() {
@@ -51,7 +54,11 @@ func StringToBytes(s string) []byte {
 	if len(s) == 0 {
 		return nil
 	}
-	return *(*[]byte)(unsafe.Pointer(&s))
+	// safe: the slice aliases the string's backing array (len == cap == len(s)).
+	// The invariant is that the backing array is never mutated through the
+	// returned slice — callers treat it as read-only, preserving string
+	// immutability.
+	return unsafe.Slice(unsafe.StringData(s), len(s))
 }
 
 // BytesToString converts []byte to string without allocation.
@@ -62,5 +69,8 @@ func BytesToString(b []byte) string {
 	if len(b) == 0 {
 		return ""
 	}
-	return *(*string)(unsafe.Pointer(&b))
+	// safe: the string header aliases b's backing array. The invariant is that
+	// the backing array is not mutated (through b or any other alias) while the
+	// returned string is live, preserving string immutability.
+	return unsafe.String(unsafe.SliceData(b), len(b))
 }

internal/tracking/tracking.go

@@ -44,6 +44,7 @@ type Event struct {
 // Tracker is the public type. Construct with New() or NewAt(path).
 type Tracker struct {
 	db        *sql.DB
+	path      string
 	mu        sync.Mutex
 	retenDays int
 	closed    bool
@@ -77,16 +78,16 @@ func NewAt(ctx context.Context, path string) (*Tracker, error) {
 		return nil, errors.New("tracking: empty path")
 	}
 	if err := mkdirAll(filepath.Dir(path), 0o700); err != nil {
-		return nil, fmt.Errorf("tracking: create dir: %w", err)
+		return nil, fmt.Errorf("tracking: create dir %q: %w", filepath.Dir(path), err)
 	}
 	db, err := sql.Open("sqlite", path)
 	if err != nil {
-		return nil, fmt.Errorf("tracking: open db: %w", err)
+		return nil, fmt.Errorf("tracking: open db at %q: %w", path, err)
 	}
 	// SQLite is single-writer; use a per-connection mutex to serialize
 	// writes without serializing reads.
 	db.SetMaxOpenConns(1)
-	t := &Tracker{db: db, retenDays: 90}
+	t := &Tracker{db: db, path: path, retenDays: 90}
 	if err := t.initSchema(ctx); err != nil {
 		_ = db.Close()
 		return nil, err
@@ -127,7 +128,7 @@ func (t *Tracker) initSchema(ctx context.Context) error {
 		CREATE INDEX IF NOT EXISTS idx_events_command ON events(command);
 	`)
 	if err != nil {
-		return fmt.Errorf("tracking: init schema: %w", err)
+		return fmt.Errorf("tracking: init schema at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -137,7 +138,7 @@ func (t *Tracker) initSchema(ctx context.Context) error {
 func (t *Tracker) enableWAL(ctx context.Context) error {
 	_, err := t.db.ExecContext(ctx, `PRAGMA journal_mode=WAL;`)
 	if err != nil {
-		return fmt.Errorf("tracking: enable WAL: %w", err)
+		return fmt.Errorf("tracking: enable WAL at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -166,7 +167,7 @@ func (t *Tracker) Record(ctx context.Context, ev Event) error {
 		ev.Mode, ev.Tier, ev.Model,
 	)
 	if err != nil {
-		return fmt.Errorf("tracking: record: %w", err)
+		return fmt.Errorf("tracking: record event in db at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -209,7 +210,7 @@ func (t *Tracker) Aggregate(ctx context.Context, days int) (Aggregate, error) {
 		WHERE ts >= ?
 	`, since)
 	if err := row.Scan(&agg.EventCount, &agg.TotalBytesSaved, &agg.AvgSavingsPct); err != nil {
-		return agg, fmt.Errorf("tracking: aggregate: %w", err)
+		return agg, fmt.Errorf("tracking: aggregate query on db at %q: %w", t.path, err)
 	}
 	agg.PeriodStart = time.Unix(since, 0)
 	agg.PeriodEnd = time.Now()
@@ -235,7 +236,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) {
 		LIMIT ?
 	`, n)
 	if err != nil {
-		return nil, fmt.Errorf("tracking: recent: %w", err)
+		return nil, fmt.Errorf("tracking: recent query on db at %q: %w", t.path, err)
 	}
 	defer func() { _ = rows.Close() }()
 	var out []Event
@@ -246,7 +247,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) {
 			&ev.OriginalBytes, &ev.CompressedBytes,
 			&ev.OriginalTokens, &ev.CompressedTokens,
 			&ev.Mode, &ev.Tier, &ev.Model); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("tracking: scan event row from db at %q: %w", t.path, err)
 		}
 		ev.Timestamp = time.Unix(ts, 0)
 		if ev.OriginalBytes > 0 {
@@ -266,7 +267,7 @@ func (t *Tracker) Prune(ctx context.Context) (int64, error) {
 	cutoff := time.Now().Add(-time.Duration(t.retenDays) * 24 * time.Hour).Unix()
 	res, err := t.db.ExecContext(ctx, `DELETE FROM events WHERE ts < ?`, cutoff)
 	if err != nil {
-		return 0, fmt.Errorf("tracking: prune: %w", err)
+		return 0, fmt.Errorf("tracking: prune db at %q: %w", t.path, err)
 	}
 	return res.RowsAffected()
 }