extended: fail-closed cap enforcement when no evictable atoms remain

- Evictor.SelectForEviction now returns (ids, freed, ok) so callers know
  whether the requested bytes could be covered by non-pinned atoms.
- enforceCap uses per-atom effective size (chunk + atoms.json share +
  amortized vector cost) instead of raw store+index disk size, avoiding
  double-counting and making fail-closed checks reliable.
- After eviction, if the evictor reports it cannot free enough, AddAtom
  returns an error and the atom is not written.
- Add ExtendedMemory.Close() and atomVectorIndex.Wait() so tests can
  drain background compaction goroutines before TempDir cleanup.

Author: jkyberneees

internal/memory/extended/eviction.go

@@ -15,8 +15,10 @@ type sizedAtom struct {
 // Evictor selects atoms for eviction when the store approaches its size cap.
 type Evictor interface {
 	// SelectForEviction returns atom IDs to remove to free at least needBytes.
-	// sizedAtoms provides the actual disk size for each atom.
-	SelectForEviction(sizedAtoms []sizedAtom, needBytes int64) []string
+	// sizedAtoms provides the actual disk size for each atom. freed reports the
+	// number of bytes that removing ids would release, and ok is true when the
+	// requested needBytes can be covered by non-pinned atoms.
+	SelectForEviction(sizedAtoms []sizedAtom, needBytes int64) (ids []string, freed int64, ok bool)
 }
 
 // newEvictor builds the eviction strategy selected by cfg.EvictionPolicy.
@@ -36,9 +38,9 @@ type retentionDecayEvictor struct {
 }
 
 // SelectForEviction returns IDs to remove. Pinned atoms are never selected.
-func (e *retentionDecayEvictor) SelectForEviction(sizedAtoms []sizedAtom, needBytes int64) []string {
+func (e *retentionDecayEvictor) SelectForEviction(sizedAtoms []sizedAtom, needBytes int64) (ids []string, freed int64, ok bool) {
 	if needBytes <= 0 {
-		return nil
+		return nil, 0, true
 	}
 	scored := make([]struct {
 		sized sizedAtom
@@ -58,8 +60,6 @@ func (e *retentionDecayEvictor) SelectForEviction(sizedAtoms []sizedAtom, needBy
 		return scored[i].score < scored[j].score
 	})
 
-	var freed int64
-	var ids []string
 	for _, s := range scored {
 		if freed >= needBytes {
 			break
@@ -72,8 +72,9 @@ func (e *retentionDecayEvictor) SelectForEviction(sizedAtoms []sizedAtom, needBy
 	}
 	if freed < needBytes {
 		log.Printf("extended memory: eviction freed only %d of %d requested bytes", freed, needBytes)
+		return ids, freed, false
 	}
-	return ids
+	return ids, freed, true
 }
 
 // buildSizedAtoms resolves the actual on-disk size for each atom. If the store

internal/memory/extended/extended_memory.go

@@ -33,6 +33,8 @@ type ExtendedMemory struct {
 
 	// testCapBytes overrides cfg.MaxSizeMB in tests. 0 means use cfg.
 	testCapBytes int64
+
+	closeOnce sync.Once
 }
 
 // New creates an ExtendedMemory instance rooted at dir.
@@ -285,37 +287,36 @@ func (em *ExtendedMemory) enforceCap(ctx context.Context, newBytes int64) error
 		log.Printf("extended memory: evicted %d expired quarantined atom(s)", removed)
 	}
 
-	storeSize, err := em.store.Size()
-	if err != nil {
-		log.Printf("extended memory: store size failed: %v", err)
-		storeSize = 0
-	}
 	quarantineSize, err := em.quarantine.Size()
 	if err != nil {
 		log.Printf("extended memory: quarantine size failed: %v", err)
 		quarantineSize = 0
 	}
-	indexSize := em.index.Size()
-	total := storeSize + quarantineSize + indexSize + newBytes
 
-	if total <= maxBytes {
-		return nil
-	}
-
-	need := total - maxBytes + 4096 // headroom
 	atoms, err := em.store.List()
 	if err != nil {
 		log.Printf("extended memory: list atoms for eviction failed: %v", err)
-		return nil
+		return fmt.Errorf("extended memory: list atoms: %w", err)
 	}
 	sized := buildSizedAtoms(em.store, atoms)
 	// Include amortized vector cost in each atom's footprint.
 	for i := range sized {
 		sized[i].size += vectorCost(len(atoms))
 	}
 
+	var existingEffective int64
+	for _, s := range sized {
+		existingEffective += s.size
+	}
+	total := existingEffective + quarantineSize + newBytes
+
+	if total <= maxBytes {
+		return nil
+	}
+
+	need := total - maxBytes + 4096 // headroom
 	before := len(atoms)
-	ids := em.evictor.SelectForEviction(sized, need)
+	ids, _, ok := em.evictor.SelectForEviction(sized, need)
 	for _, id := range ids {
 		_ = em.store.Remove(id)
 		em.index.markDirty()
@@ -327,6 +328,10 @@ func (em *ExtendedMemory) enforceCap(ctx context.Context, newBytes int64) error
 			em.index.Compact()
 		}
 	}
+
+	if !ok {
+		return fmt.Errorf("extended memory: cannot free %s; all atoms are pinned or no evictable atoms exist", sizeLabel(need))
+	}
 	return nil
 }
 
@@ -362,6 +367,18 @@ func (em *ExtendedMemory) Compact() {
 	em.index.Compact()
 }
 
+// Close waits for background operations to finish. It is safe to call
+// multiple times.
+func (em *ExtendedMemory) Close() error {
+	if em == nil {
+		return nil
+	}
+	em.closeOnce.Do(func() {
+		em.index.Wait()
+	})
+	return nil
+}
+
 // Size returns the current on-disk size of the Extended Memory store.
 func (em *ExtendedMemory) Size() int64 {
 	if em == nil {

internal/memory/extended/index.go

@@ -37,6 +37,7 @@ type vectorMeta struct {
 // recall. It rebuilds from disk when dirty and caches the result.
 type atomVectorIndex struct {
 	mu     sync.RWMutex
+	wg     sync.WaitGroup
 	dir    string
 	store  *vector.Store
 	emb    textEmbedder
@@ -203,12 +204,22 @@ func (vi *atomVectorIndex) Compact() {
 	vi.dirty = true
 	vi.dirtySeq++
 	vi.mu.Unlock()
+	vi.wg.Add(1)
 	go func() {
+		defer vi.wg.Done()
 		vi.ensureFresh()
 		log.Printf("extended memory: vector index compacted")
 	}()
 }
 
+// Wait blocks until in-flight background compaction goroutines finish.
+func (vi *atomVectorIndex) Wait() {
+	if vi == nil {
+		return
+	}
+	vi.wg.Wait()
+}
+
 // tryLoadLocked attempts to load persisted state. Caller must hold vi.mu.
 func (vi *atomVectorIndex) tryLoadLocked() bool {
 	fp := vi.emb.Fingerprint()