Add governed memory candidates

Author: nshkrdotcom

core/context_abi/lib/outer_brain/context_abi/context_packet.ex

@@ -49,6 +49,7 @@ defmodule OuterBrain.ContextABI.ContextPacket do
          {:ok, system_instruction_ref} <-
            Validator.required_string(attrs, :system_instruction_ref),
          {:ok, memory_refs} <- Validator.string_list(attrs, :memory_refs),
+         :ok <- reject_unpromoted_memory_refs(memory_refs),
          {:ok, budget_ref} <- Validator.required_string(attrs, :budget_ref),
          {:ok, model_class_allowlist} <- Validator.string_list(attrs, :model_class_allowlist),
          :ok <- require_nonempty(model_class_allowlist, :model_class_allowlist),
@@ -126,4 +127,20 @@ defmodule OuterBrain.ContextABI.ContextPacket do
   end
 
   defp require_nonempty(_values, _field), do: :ok
+
+  defp reject_unpromoted_memory_refs(memory_refs) do
+    case Enum.find(memory_refs, &unpromoted_memory_ref?/1) do
+      nil ->
+        :ok
+
+      ref ->
+        Validator.failure(:outer_brain, "outer_brain.context.unpromoted_memory_candidate.v1",
+          safe_message: "unpromoted memory candidates cannot enter production context packets",
+          evidence_refs: [ref]
+        )
+    end
+  end
+
+  defp unpromoted_memory_ref?("memory-candidate://" <> _suffix), do: true
+  defp unpromoted_memory_ref?(ref), do: String.contains?(ref, "/candidate/")
 end

core/context_abi/test/outer_brain/context_abi_test.exs

@@ -61,6 +61,16 @@ defmodule OuterBrain.ContextABITest do
     assert nested_failure.reason_code == "outer_brain.context.raw_payload_rejected.v1"
   end
 
+  test "compiler rejects unpromoted memory candidate refs in production packets" do
+    assert {:error, %Failure{} = failure} =
+             compile_request()
+             |> Map.put(:memory_refs, ["memory-candidate://tenant-a/unpromoted/a"])
+             |> ContextABI.compile()
+
+    assert failure.reason_code == "outer_brain.context.unpromoted_memory_candidate.v1"
+    assert failure.evidence_refs == ["memory-candidate://tenant-a/unpromoted/a"]
+  end
+
   test "context units validate vocabulary and reject raw metadata payloads" do
     assert {:ok, %ContextUnit{} = unit} =
              ContextUnit.new(%{

core/memory_contracts/lib/outer_brain/memory_contracts.ex

@@ -12,6 +12,7 @@ defmodule OuterBrain.MemoryContracts do
     ContextBudgetDecision,
     ContextBudgetRef,
     MemoryAccessReason,
+    MemoryCandidate,
     MemoryEvidenceRef,
     MemoryQueryIntent,
     MemoryRedactionPolicy,
@@ -39,6 +40,9 @@ defmodule OuterBrain.MemoryContracts do
   @spec budget_exhaustion_reasons() :: [atom()]
   def budget_exhaustion_reasons, do: Vocabulary.budget_exhaustion_reasons()
 
+  @spec memory_candidate_statuses() :: [atom()]
+  def memory_candidate_statuses, do: Vocabulary.memory_candidate_statuses()
+
   @spec scope_key(map() | MemoryScopeKey.t()) :: {:ok, MemoryScopeKey.t()} | error()
   def scope_key(attrs), do: MemoryScopeKey.new(attrs)
 
@@ -69,6 +73,9 @@ defmodule OuterBrain.MemoryContracts do
   @spec query_intent(map() | MemoryQueryIntent.t()) :: {:ok, MemoryQueryIntent.t()} | error()
   def query_intent(attrs), do: MemoryQueryIntent.new(attrs)
 
+  @spec memory_candidate(map() | MemoryCandidate.t()) :: {:ok, MemoryCandidate.t()} | error()
+  def memory_candidate(attrs), do: MemoryCandidate.new(attrs)
+
   @spec fetch_value(map(), atom()) :: term()
   def fetch_value(attrs, field), do: Validator.fetch_value(attrs, field)
 end

core/memory_contracts/lib/outer_brain/memory_contracts/memory_candidate.ex

@@ -0,0 +1,124 @@
+defmodule OuterBrain.MemoryContracts.MemoryCandidate do
+  @moduledoc """
+  Ref-only memory candidate contract for governed promotion.
+
+  A candidate is not production memory. It records the memory/evidence refs and
+  eval/Citadel facts required before a promoted memory ref can participate in a
+  production Context ABI packet.
+  """
+
+  alias OuterBrain.MemoryContracts.{MemoryEvidenceRef, MemoryRef, Validator, Vocabulary}
+
+  @enforce_keys [
+    :candidate_ref,
+    :tenant_ref,
+    :memory_ref,
+    :evidence_ref,
+    :eval_evidence_refs,
+    :authority_ref,
+    :trace_ref,
+    :redaction_policy_ref,
+    :status
+  ]
+  defstruct [:promotion_ref, :rollback_ref | @enforce_keys]
+
+  @type status :: :candidate | :promoted | :rolled_back
+
+  @type t :: %__MODULE__{
+          candidate_ref: String.t(),
+          tenant_ref: String.t(),
+          memory_ref: MemoryRef.t(),
+          evidence_ref: MemoryEvidenceRef.t(),
+          eval_evidence_refs: [String.t()],
+          authority_ref: String.t(),
+          trace_ref: String.t(),
+          redaction_policy_ref: String.t(),
+          status: status(),
+          promotion_ref: String.t() | nil,
+          rollback_ref: String.t() | nil
+        }
+
+  @spec new(map() | t()) :: {:ok, t()} | {:error, term()}
+  def new(%__MODULE__{} = candidate), do: {:ok, candidate}
+
+  def new(attrs) when is_map(attrs) do
+    with :ok <- Validator.reject_raw_payload(attrs),
+         {:ok, candidate_ref} <- required_string(attrs, :candidate_ref),
+         {:ok, tenant_ref} <- required_string(attrs, :tenant_ref),
+         {:ok, memory_ref} <- attrs |> Validator.fetch_value(:memory_ref) |> MemoryRef.new(),
+         {:ok, evidence_ref} <-
+           attrs |> Validator.fetch_value(:evidence_ref) |> MemoryEvidenceRef.new(),
+         {:ok, eval_evidence_refs} <- required_strings(attrs, :eval_evidence_refs),
+         {:ok, authority_ref} <- required_string(attrs, :authority_ref),
+         {:ok, trace_ref} <- required_string(attrs, :trace_ref),
+         {:ok, redaction_policy_ref} <- required_string(attrs, :redaction_policy_ref),
+         {:ok, status} <- candidate_status(attrs),
+         :ok <- status_refs_present(attrs, status) do
+      {:ok,
+       %__MODULE__{
+         candidate_ref: candidate_ref,
+         tenant_ref: tenant_ref,
+         memory_ref: memory_ref,
+         evidence_ref: evidence_ref,
+         eval_evidence_refs: eval_evidence_refs,
+         authority_ref: authority_ref,
+         trace_ref: trace_ref,
+         redaction_policy_ref: redaction_policy_ref,
+         status: status,
+         promotion_ref: optional_string(attrs, :promotion_ref),
+         rollback_ref: optional_string(attrs, :rollback_ref)
+       }}
+    end
+  end
+
+  def new(_attrs), do: {:error, :invalid_memory_candidate}
+
+  defp candidate_status(attrs) do
+    status = Validator.fetch_value(attrs, :status) || :candidate
+
+    if status in Vocabulary.memory_candidate_statuses() do
+      {:ok, status}
+    else
+      {:error, {:invalid_candidate_status, status}}
+    end
+  end
+
+  defp status_refs_present(attrs, :promoted) do
+    case required_string(attrs, :promotion_ref) do
+      {:ok, _ref} -> :ok
+      {:error, _reason} -> {:error, {:missing_candidate_ref, :promotion_ref}}
+    end
+  end
+
+  defp status_refs_present(attrs, :rolled_back) do
+    case required_string(attrs, :rollback_ref) do
+      {:ok, _ref} -> :ok
+      {:error, _reason} -> {:error, {:missing_candidate_ref, :rollback_ref}}
+    end
+  end
+
+  defp status_refs_present(_attrs, :candidate), do: :ok
+
+  defp required_string(attrs, field) do
+    case Validator.required_string(attrs, field) do
+      {:ok, value} -> {:ok, value}
+      {:error, _reason} -> {:error, {:missing_candidate_ref, field}}
+    end
+  end
+
+  defp optional_string(attrs, field), do: Validator.optional_string(attrs, field)
+
+  defp required_strings(attrs, field) do
+    case Validator.fetch_value(attrs, field) do
+      values when is_list(values) and values != [] ->
+        if Enum.all?(values, &(is_binary(&1) and String.trim(&1) != "")) do
+          {:ok, Enum.uniq(values)}
+        else
+          {:error, {:missing_candidate_ref, field}}
+        end
+
+      _other ->
+        {:error, {:missing_candidate_ref, field}}
+    end
+  end
+end

core/memory_contracts/lib/outer_brain/memory_contracts/vocabulary.ex

@@ -5,13 +5,19 @@ defmodule OuterBrain.MemoryContracts.Vocabulary do
     :body,
     :raw_body,
     :content,
+    :memory_body,
+    :model_output,
     :raw_content,
+    :raw_memory,
     :payload,
     :raw_payload,
     "body",
     "raw_body",
     "content",
+    "memory_body",
+    "model_output",
     "raw_content",
+    "raw_memory",
     "payload",
     "raw_payload"
   ]
@@ -60,6 +66,8 @@ defmodule OuterBrain.MemoryContracts.Vocabulary do
     :operator_override_denied
   ]
 
+  @memory_candidate_statuses [:candidate, :promoted, :rolled_back]
+
   @spec raw_payload_keys() :: [atom() | String.t()]
   def raw_payload_keys, do: @raw_payload_keys
 
@@ -80,4 +88,7 @@ defmodule OuterBrain.MemoryContracts.Vocabulary do
 
   @spec budget_exhaustion_reasons() :: [atom()]
   def budget_exhaustion_reasons, do: @budget_reasons
+
+  @spec memory_candidate_statuses() :: [atom()]
+  def memory_candidate_statuses, do: @memory_candidate_statuses
 end

core/memory_contracts/test/outer_brain/memory_contracts_test.exs

@@ -136,6 +136,34 @@ defmodule OuterBrain.MemoryContractsTest do
              |> MemoryContracts.query_intent()
   end
 
+  test "memory candidates are ref-only and require bounded eval evidence" do
+    assert {:ok, candidate} = MemoryContracts.memory_candidate(valid_memory_candidate())
+
+    assert candidate.status == :candidate
+    assert candidate.memory_ref.memory_id == "mem-1"
+    assert candidate.evidence_ref.memory_id == "mem-1"
+    assert candidate.eval_evidence_refs == ["eval://memory/a"]
+
+    assert {:error, {:missing_candidate_ref, :eval_evidence_refs}} =
+             valid_memory_candidate()
+             |> Map.put(:eval_evidence_refs, [])
+             |> MemoryContracts.memory_candidate()
+
+    assert {:error, {:raw_memory_body_forbidden, :memory_body}} =
+             valid_memory_candidate()
+             |> Map.put(:memory_body, "raw")
+             |> MemoryContracts.memory_candidate()
+  end
+
+  test "candidate status vocabulary distinguishes candidate, promoted, and rolled back" do
+    assert MemoryContracts.memory_candidate_statuses() == [:candidate, :promoted, :rolled_back]
+
+    assert {:error, {:invalid_candidate_status, :unknown}} =
+             valid_memory_candidate()
+             |> Map.put(:status, :unknown)
+             |> MemoryContracts.memory_candidate()
+  end
+
   defp valid_write_intent do
     %{
       tenant_ref: "tenant://a",
@@ -210,4 +238,18 @@ defmodule OuterBrain.MemoryContractsTest do
       trace_ref: "trace://a"
     }
   end
+
+  defp valid_memory_candidate do
+    %{
+      candidate_ref: "memory-candidate://tenant-a/a",
+      tenant_ref: "tenant://a",
+      memory_ref: valid_memory_ref(),
+      evidence_ref: valid_evidence_ref(),
+      eval_evidence_refs: ["eval://memory/a"],
+      authority_ref: "authority://citadel/memory/a",
+      trace_ref: "trace://a",
+      redaction_policy_ref: "policy://redact",
+      status: :candidate
+    }
+  end
 end

core/memory_engine/lib/outer_brain/memory_engine.ex

@@ -170,6 +170,61 @@ defmodule OuterBrain.MemoryEngine do
     end
   end
 
+  @spec candidate_from_evidence(map()) ::
+          {:ok, MemoryContracts.MemoryCandidate.t()} | {:error, term()}
+  def candidate_from_evidence(attrs) when is_map(attrs) do
+    attrs
+    |> Map.put_new(:status, :candidate)
+    |> MemoryContracts.memory_candidate()
+  end
+
+  @spec promote_candidate(MemoryContracts.MemoryCandidate.t() | map(), map()) ::
+          {:ok, MemoryContracts.MemoryCandidate.t()} | {:error, term()}
+  def promote_candidate(candidate_or_attrs, attrs) when is_map(attrs) do
+    with {:ok, candidate} <- MemoryContracts.memory_candidate(candidate_or_attrs),
+         {:ok, promotion_ref} <- required_string(attrs, :promotion_ref),
+         {:ok, citadel_authority_ref} <- required_string(attrs, :citadel_authority_ref),
+         {:ok, eval_refs} <- non_empty_strings(attrs, :eval_evidence_refs) do
+      MemoryContracts.memory_candidate(%{
+        candidate
+        | status: :promoted,
+          promotion_ref: promotion_ref,
+          authority_ref: citadel_authority_ref,
+          eval_evidence_refs: eval_refs
+      })
+    else
+      {:error, {:missing_ref, :eval_evidence_refs}} ->
+        {:error, :memory_candidate_eval_gate_required}
+
+      {:error, {:missing_ref, :citadel_authority_ref}} ->
+        {:error, :memory_candidate_citadel_gate_required}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  @spec rollback_candidate(MemoryContracts.MemoryCandidate.t() | map(), map()) ::
+          {:ok, MemoryContracts.MemoryCandidate.t()} | {:error, term()}
+  def rollback_candidate(candidate_or_attrs, attrs) when is_map(attrs) do
+    with {:ok, candidate} <- MemoryContracts.memory_candidate(candidate_or_attrs),
+         {:ok, rollback_ref} <- required_string(attrs, :rollback_ref),
+         {:ok, citadel_authority_ref} <- required_string(attrs, :citadel_authority_ref) do
+      MemoryContracts.memory_candidate(%{
+        candidate
+        | status: :rolled_back,
+          rollback_ref: rollback_ref,
+          authority_ref: citadel_authority_ref
+      })
+    else
+      {:error, {:missing_ref, :citadel_authority_ref}} ->
+        {:error, :memory_candidate_citadel_gate_required}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
   defp evidence(raw_body, max_export_bytes) when byte_size(raw_body) > max_export_bytes do
     {:ok, sha256(raw_body), "body_oversize_replaced_by_hash_ref"}
   end
@@ -244,4 +299,25 @@ defmodule OuterBrain.MemoryEngine do
   defp scope_tier(%MemoryContracts.MemoryScopeKey{run_ref: nil, agent_ref: nil}), do: :semantic
   defp scope_tier(%MemoryContracts.MemoryScopeKey{skill_ref: nil}), do: :episodic
   defp scope_tier(%MemoryContracts.MemoryScopeKey{}), do: :working
+
+  defp required_string(attrs, field) do
+    case Map.get(attrs, field) || Map.get(attrs, Atom.to_string(field)) do
+      value when is_binary(value) and value != "" -> {:ok, value}
+      _other -> {:error, {:missing_ref, field}}
+    end
+  end
+
+  defp non_empty_strings(attrs, field) do
+    case Map.get(attrs, field) || Map.get(attrs, Atom.to_string(field)) do
+      values when is_list(values) and values != [] ->
+        if Enum.all?(values, &(is_binary(&1) and String.trim(&1) != "")) do
+          {:ok, Enum.uniq(values)}
+        else
+          {:error, {:missing_ref, field}}
+        end
+
+      _other ->
+        {:error, {:missing_ref, field}}
+    end
+  end
 end