fixing memor

Author: Daniel

go/cmd/engine/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/ForestHubAI/edge-agents/go/engine/backend"
 	"github.com/ForestHubAI/edge-agents/go/engine/build"
 	"github.com/ForestHubAI/edge-agents/go/engine/driver"
-	"github.com/ForestHubAI/edge-agents/go/engine/local"
 	"github.com/ForestHubAI/edge-agents/go/engine/memory"
 	"github.com/ForestHubAI/edge-agents/go/engine/websearch"
 	"github.com/ForestHubAI/edge-agents/go/llmproxy"
@@ -92,18 +91,17 @@ func main() {
 		logging.Logger.Fatal().Err(err).Msg("initialising driver registry")
 	}
 
-	// Memory subsystem: backed by the configured local dir, syncs through
-	// either the backend (cloud mode) or a filesystem-backed local store
-	// rooted at the same dir (standalone mode — declared memory survives
-	// engine restarts without a backend). Restore is invoked on every Build
-	// (deploy or initial), so no eager call here.
-	var memoryStore engine.MemoryStore
+	// Memory subsystem: the Manager owns durable local storage rooted at
+	// cfg.MemoryDir (declared memory survives engine restarts with no
+	// backend). The backend, when configured, is an optional remote mirror —
+	// it hydrates an empty local copy on a cold start and receives best-effort
+	// pushes; nil means local-only. Restore is invoked on every Build (deploy
+	// or initial), so no eager call here.
+	var memorySync engine.MemorySync
 	if backendClient != nil {
-		memoryStore = backendClient
-	} else {
-		memoryStore = local.NewMemoryStore(cfg.MemoryDir)
+		memorySync = backendClient
 	}
-	memoryManager := memory.NewManager(cfg.MemoryDir, memoryStore)
+	memoryManager := memory.NewManager(cfg.MemoryDir, memorySync)
 
 	// Optional web search provider. Built eagerly so a bad provider name fails
 	// fatal at boot; absent api key leaves it nil and any WebSearchTool node

go/docs/ports.md

@@ -0,0 +1,139 @@
+# Engine Ports & Implementations
+
+The engine never depends on a concrete external service. Everything it needs
+from "outside" is reached through an **interface — a port — declared in
+`engine/port.go`**. Concrete adapters satisfy those interfaces and map to
+their own wire forms privately, so the engine core compiles and runs without
+knowing whether it's talking to the fh-backend, a local file, or nothing at
+all.
+
+There are two sources of implementation:
+
+- **The fh-backend adapter** (`engine/backend`) — one HTTP client that
+  satisfies every port, used when `FH_BACKEND_URL` is configured.
+- **Built-in / standalone behavior** — what happens with no backend. For some
+  ports that's a real local implementation; for others it's "the port is
+  nil and the engine does without."
+
+`cmd/engine/main.go` is the only place that decides which adapter fills each
+seam, based on whether a backend client was constructed.
+
+## The matrix
+
+| Port | Methods | Required? | No backend (standalone) | fh-backend adapter |
+|------|---------|-----------|-------------------------|--------------------|
+| `LlmClient` | `Chat` | Required for agent nodes | Local providers via `llmproxy` (direct API keys) | Backend-routed provider fallback |
+| `Retriever` | `QueryRAG` | Required **only if** a retrieval node is deployed | **nil** → build rejects any Retriever node | Forwards to `/rag/query` |
+| `Supervisor` | `Register`, `Heartbeat` | Optional | **nil** → no registration/heartbeat | POSTs `/agents/bootCallback`, `/agents/heartbeat` |
+| `MemorySync` | `Hydrate`, `Push` | Optional (mirror only) | **nil** → local-only memory | HTTP `GET`/`PUT /agents/memory` |
+
+Two capabilities deliberately are **not** ports:
+
+- **Local memory persistence** — owned unconditionally by
+  `engine/memory.Manager`. The device always has a durable local copy; see
+  [Memory](#memorysync-optional-remote-mirror) below.
+- **Logging** — stderr is unconditional and the engine already depends on the
+  `logging` package. Optional log shipping to the backend is a `logging`
+  `HTTPWriter` wired by `main`, not a port.
+
+## LlmClient — required for agent nodes
+
+`Chat` is the chat-completion seam. The implementation is
+`llmproxy.Client`, which dispatches by model id across configured providers.
+
+- **Standalone:** providers configured with direct API keys
+  (`anthropic`/`openai`/`gemini`/`mistral`/`selfhosted`). This is the primary
+  path, not a fallback.
+- **Backend:** any provider the backend exposes but the engine has no local
+  key for is registered as a backend-routed stand-in, resolved by model id
+  exactly as if it were local.
+- **Missing:** an `AgentNode` with a nil `LlmClient` **fails the build**
+  (`engine/build/graph.go`). There is no silent default.
+
+## Retriever — required only when used
+
+`QueryRAG` is the RAG seam. There is **no built-in standalone implementation
+yet**.
+
+- **Standalone:** `Retriever` is nil. A workflow that declares a Retriever
+  node **fails to deploy** with a clear error, mirroring the WebSearch and
+  Agent build checks. A workflow with no retrieval node deploys and runs
+  fine. (This replaced a silent no-op that returned empty results.)
+- **Backend:** forwards to `/rag/query`.
+- **Planned:** a standalone pgvector-backed adapter (query embedding via
+  `llmproxy` + similarity search + ingestion) will live in its own package
+  (e.g. `engine/rag/pgvector`), not bundled with the trivial seams.
+
+## Supervisor — optional outbound callbacks
+
+`Supervisor` abstracts whoever receives this agent's callbacks: the
+registration sent at boot plus the periodic liveness heartbeat. It is a
+purely outbound seam — deploys/commands arrive the other way, through the
+engine's HTTP server.
+
+- **Standalone:** nil. With no one to report to, the engine simply doesn't
+  register or heartbeat. This is correct, not degraded — pull-based health
+  endpoints are the standalone observability story, not a fake heartbeat.
+- **Backend:** `backend.Client` POSTs `/agents/bootCallback` and
+  `/agents/heartbeat`. The retry/heartbeat loops live in
+  `engine/lifecycle.go`.
+
+## MemorySync — optional remote mirror
+
+Memory is **local-first (edge-primary)**. `engine/memory.Manager` owns a
+durable directory of `<uid>.json` records and is the source of truth: it
+reads them at boot and writes through on every mutation. `MemorySync` is
+*only* the optional remote mirror.
+
+- **`Hydrate`** — pulls the agent's accumulated content. Called by the
+  Manager **only on a cold start** (empty local directory) to seed a fresh
+  copy.
+- **`Push`** — mirrors each local write. **Best-effort**: the local write is
+  the truth, so a push failure is logged and the agent keeps working.
+- **Standalone:** nil. Memory is purely local and durable across restarts
+  (mount a persistent volume to survive container remounts).
+
+### Reconciliation on Restore
+
+`Restore(ctx, declared)` is called on every build with the memory files
+declared by the workflow. Content precedence is **local → cold-start mirror →
+declared seed**:
+
+1. An existing **local** copy wins — this preserves the agent's accumulated
+   edits across redeploys.
+2. Otherwise, on a cold start with a mirror, the **hydrated** content seeds
+   the file.
+3. Otherwise the workflow's declared `MemoryFile.Content` is used.
+
+So **`MemoryFile.Content` is initial content only — it never overwrites an
+existing file.** Declared *metadata* (label, description, size cap) is always
+authoritative; only content is preserved.
+
+> **Durability boundary:** with `Push` but no reverse path, the backend holds
+> only what the engine has pushed; runtime edits are durable on the local
+> volume. There is no backend → device or device → device reconciliation yet.
+> Adding a push-back path is additive and doesn't disturb this design.
+
+## Wiring (cmd/engine/main.go)
+
+```
+FH_BACKEND_URL set?
+├─ yes → backend.Client satisfies LlmClient (fallback), Retriever,
+│         Supervisor, and MemorySync.
+└─ no  → LlmClient: local providers only
+          Retriever:  nil (retrieval nodes fail the build)
+          Supervisor: nil (no register/heartbeat)
+          MemorySync: nil (local-only memory)
+```
+
+## Adding an adapter
+
+Group by **shared dependency**, not by port:
+
+- An adapter that talks to the fh-backend belongs in `engine/backend` (it
+  shares the one HTTP client).
+- A heavy standalone implementation (its own driver, ingestion, etc.) gets
+  its **own package** — e.g. a pgvector `Retriever` under `engine/rag`.
+  Don't bundle it with trivial seams.
+- Edit `port.go` only to add or change a seam; never widen a port with a
+  method a single adapter needs but the engine core doesn't call.

go/engine/backend/memory.go

@@ -12,23 +12,23 @@ type memoryUpsertBody struct {
 	Content string `json:"content"`
 }
 
-// Snapshot pulls every memory file owned by the calling agent. Called
-// once at engine boot to populate the local working copy.
-func (c *Client) Snapshot(ctx context.Context) ([]workflow.MemoryFile, error) {
+// Hydrate pulls every memory file owned by the calling agent. The Manager
+// calls this on a cold start to seed an empty local working copy.
+func (c *Client) Hydrate(ctx context.Context) ([]workflow.MemoryFile, error) {
 	var out []workflow.MemoryFile
 	if err := c.http.Do(ctx, http.MethodGet, "/agents/memory", nil, nil, &out); err != nil {
-		return nil, fmt.Errorf("backend memory snapshot: %w", err)
+		return nil, fmt.Errorf("backend memory hydrate: %w", err)
 	}
 	return out, nil
 }
 
-// Upsert pushes new content for the memory file identified by uid.
-// The engine calls this synchronously after every successful local write.
+// Push mirrors new content for the memory file identified by uid to the
+// backend. The Manager calls this best-effort after every local write.
 // The backend rejects unknown uids (404) and oversized payloads (413).
-func (c *Client) Upsert(ctx context.Context, uid, content string) error {
+func (c *Client) Push(ctx context.Context, uid, content string) error {
 	body := memoryUpsertBody{Content: content}
 	if err := c.http.Do(ctx, http.MethodPut, "/agents/memory/"+uid, nil, body, nil); err != nil {
-		return fmt.Errorf("backend memory upsert: %w", err)
+		return fmt.Errorf("backend memory push: %w", err)
 	}
 	return nil
 }

go/engine/backend/memory_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestSnapshot_Success(t *testing.T) {
+func TestHydrate_Success(t *testing.T) {
 	var (
 		gotKey    string
 		gotMethod string
@@ -34,7 +34,7 @@ func TestSnapshot_Success(t *testing.T) {
 	defer srv.Close()
 
 	c := NewClient(srv.URL, "secret")
-	got, err := c.Snapshot(context.Background())
+	got, err := c.Hydrate(context.Background())
 	require.NoError(t, err)
 	assert.Equal(t, http.MethodGet, gotMethod)
 	assert.Equal(t, "/agents/memory", gotPath)
@@ -45,19 +45,19 @@ func TestSnapshot_Success(t *testing.T) {
 	assert.Equal(t, "uid-log", got[1].Id)
 }
 
-func TestSnapshot_BackendError(t *testing.T) {
+func TestHydrate_BackendError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)
 	}))
 	defer srv.Close()
 
 	c := NewClient(srv.URL, "secret")
-	_, err := c.Snapshot(context.Background())
+	_, err := c.Hydrate(context.Background())
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "500")
 }
 
-func TestUpsert_Success(t *testing.T) {
+func TestPush_Success(t *testing.T) {
 	var (
 		gotKey    string
 		gotMethod string
@@ -74,21 +74,21 @@ func TestUpsert_Success(t *testing.T) {
 	defer srv.Close()
 
 	c := NewClient(srv.URL, "secret")
-	require.NoError(t, c.Upsert(context.Background(), "uid-notes", "hello world"))
+	require.NoError(t, c.Push(context.Background(), "uid-notes", "hello world"))
 	assert.Equal(t, http.MethodPut, gotMethod)
 	assert.Equal(t, "/agents/memory/uid-notes", gotPath)
 	assert.Equal(t, "secret", gotKey)
 	assert.JSONEq(t, `{"content":"hello world"}`, string(gotBody))
 }
 
-func TestUpsert_BackendError(t *testing.T) {
+func TestPush_BackendError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusNotFound)
 	}))
 	defer srv.Close()
 
 	c := NewClient(srv.URL, "secret")
-	err := c.Upsert(context.Background(), "uid-missing", "x")
+	err := c.Push(context.Background(), "uid-missing", "x")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "404")
 }

go/engine/build/build.go

@@ -28,7 +28,11 @@ type Builder struct {
 // current deploy).
 func (b *Builder) Build(ctx context.Context, wf *workflow.Workflow, dm engine.DeploymentMapping, ext *engine.ExternalResources) (*engine.Runner, error) {
 	if b.Memory != nil {
-		if err := b.Memory.Restore(ctx); err != nil {
+		declared, err := declaredMemoryFiles(wf)
+		if err != nil {
+			return nil, fmt.Errorf("memory: reading declared files: %w", err)
+		}
+		if err := b.Memory.Restore(ctx, declared); err != nil {
 			return nil, fmt.Errorf("refreshing memory: %w", err)
 		}
 	}
@@ -44,6 +48,31 @@ func (b *Builder) Build(ctx context.Context, wf *workflow.Workflow, dm engine.De
 	return runner, nil
 }
 
+// declaredMemoryFiles extracts the MemoryFile declarations from a workflow,
+// skipping other memory kinds (e.g. VectorDatabase, consumed by Retriever
+// nodes). These are the canonical set of files the memory Manager restores.
+func declaredMemoryFiles(wf *workflow.Workflow) ([]workflow.MemoryFile, error) {
+	if wf.Memory == nil {
+		return nil, nil
+	}
+	var out []workflow.MemoryFile
+	for i, m := range *wf.Memory {
+		disc, err := m.Discriminator()
+		if err != nil {
+			return nil, fmt.Errorf("memory[%d]: %w", i, err)
+		}
+		if disc != string(workflow.MemoryFileTypeMemoryFile) {
+			continue
+		}
+		mf, err := m.AsMemoryFile()
+		if err != nil {
+			return nil, fmt.Errorf("memory[%d]: %w", i, err)
+		}
+		out = append(out, mf)
+	}
+	return out, nil
+}
+
 // buildContext holds the inputs shared across every graph build.
 type buildContext struct {
 	ctx       context.Context