Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions .github/workflows/fork-pre-pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
name: Fork Pre-PR Check

# 在 fork 仓库推送 feat/* 分支时自动运行,
# 提 PR 到上游前提前发现 clippy / 编译问题。
on:
push:
branches:
- "feat/**"
- "fix/**"
- "chore/**"

env:
CARGO_TERM_COLOR: always
SQLX_OFFLINE: "true"
# 无需真实 DB / Embedding key,仅做静态检查
DATABASE_URL: mysql://root:111@localhost:6001/memoria_test

jobs:
check-and-clippy:
name: Check & Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- uses: dtolnay/rust-toolchain@1.85.0
with:
components: clippy

- uses: Swatinem/rust-cache@v2
with:
workspaces: memoria
shared-key: fork-pre-pr

- name: Check & Clippy
run: cd memoria && cargo check && cargo clippy -- -D warnings
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ If you're an AI agent helping a user set up Memoria:
3. **Run `memoria init -i`** in the user's project directory
4. **Tell user to restart** their AI tool, then **verify** with `memory_retrieve("test")`

> **Self-Hosted only:** Configure embedding BEFORE first MCP server start — dimension is locked into schema.
> **Self-Hosted only:** Embedding dimension defaults to auto-infer (`EMBEDDING_DIM=0`): Memoria probes the embedding service on startup. Set `EMBEDDING_DIM` explicitly if the service may be unavailable at boot. The dimension is locked into the database schema on first run.

---

Expand Down
65 changes: 65 additions & 0 deletions memoria/crates/memoria-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,9 @@ async fn cmd_serve(db_url: Option<String>, port: u16, master_key: String) -> Res

validate_embedding_config(&cfg)?;
bootstrap_runtime_topology(&mut cfg).await?;
if cfg.embedding_dim == 0 && cfg.has_embedding() {
cfg.embedding_dim = probe_embedding_dim(&cfg).await?;
}
let redacted_db_url = redact_url(&cfg.db_url);
let redacted_shared_db_url = redact_url(&cfg.shared_db_url);

Expand All @@ -541,7 +544,9 @@ async fn cmd_serve(db_url: Option<String>, port: u16, master_key: String) -> Res
multi_db = cfg.multi_db,
port = port,
instance_id = %cfg.instance_id,
embedding_dim = cfg.embedding_dim,
has_llm = cfg.has_llm(),
has_embedding = cfg.has_embedding(),
embedding_provider = %cfg.embedding_provider,
governance_plugin_binding = %cfg.governance_plugin_binding,
"Starting Memoria API server"
Expand Down Expand Up @@ -571,6 +576,7 @@ async fn cmd_serve(db_url: Option<String>, port: u16, master_key: String) -> Res
"shared_db_merged_pool",
);
store.migrate_shared().await?;
store.check_embedding_dim_compat().await?;
store.set_db_router(router.clone());
let git = Arc::new(GitForDataService::new(
shared_pool,
Expand All @@ -582,6 +588,7 @@ async fn cmd_serve(db_url: Option<String>, port: u16, master_key: String) -> Res
SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim, cfg.instance_id.clone())
.await?;
store.migrate().await?;
store.check_embedding_dim_compat().await?;
let pool = connect_git_pool(&cfg.db_url, false).await?;
let git_db_name = parse_db_name(&cfg.db_url).unwrap_or_else(|| cfg.db_name.clone());
let git = Arc::new(GitForDataService::new(pool, git_db_name));
Expand Down Expand Up @@ -738,11 +745,17 @@ async fn cmd_mcp(
let redacted_db_url = redact_url(&cfg.db_url);
let redacted_shared_db_url = redact_url(&cfg.shared_db_url);

// Auto-infer embedding dimension when EMBEDDING_DIM=0 (or unset).
if cfg.embedding_dim == 0 && cfg.has_embedding() {
cfg.embedding_dim = probe_embedding_dim(&cfg).await?;
}

tracing::info!(
db_url = %redacted_db_url,
shared_db_url = %redacted_shared_db_url,
multi_db = cfg.multi_db,
embedding_provider = %cfg.embedding_provider,
embedding_dim = cfg.embedding_dim,
has_llm = cfg.has_llm(),
governance_plugin_binding = %cfg.governance_plugin_binding,
user = %cfg.user,
Expand Down Expand Up @@ -773,6 +786,7 @@ async fn cmd_mcp(
"shared_db_merged_pool",
);
store.migrate_shared().await?;
store.check_embedding_dim_compat().await?;
store.set_db_router(router.clone());
let git = Arc::new(GitForDataService::new(
shared_pool,
Expand All @@ -784,6 +798,7 @@ async fn cmd_mcp(
SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim, cfg.instance_id.clone())
.await?;
store.migrate().await?;
store.check_embedding_dim_compat().await?;
let pool = connect_git_pool(&cfg.db_url, false).await?;
let git_db_name = parse_db_name(&cfg.db_url).unwrap_or_else(|| cfg.db_name.clone());
let git = Arc::new(GitForDataService::new(pool, git_db_name));
Expand Down Expand Up @@ -1256,6 +1271,56 @@ fn cmd_plugin_dev_keygen(dir: &Path) -> Result<()> {

// ── Shared helpers ────────────────────────────────────────────────────────────

/// Probe the configured embedding service to determine the vector dimension.
///
/// Called when `EMBEDDING_DIM=0` (the default). Makes a single embedding
/// request with a short probe string and returns `vec.len()` as the
/// actual dimension, which is then used to create or validate the database
/// schema.
///
/// # Errors
/// Returns an error if the embedding service is unreachable or returns an
/// empty vector, with a suggestion to set `EMBEDDING_DIM` explicitly.
async fn probe_embedding_dim(cfg: &memoria_service::Config) -> Result<usize> {
use memoria_core::interfaces::EmbeddingProvider;
use memoria_embedding::HttpEmbedder;

// Build a temporary embedder with dim=0 (dim is not used by embed()).
let embedder = HttpEmbedder::new(
&cfg.embedding_base_url,
&cfg.embedding_api_key,
&cfg.embedding_model,
0,
);

tracing::info!(
model = %cfg.embedding_model,
base_url = %cfg.embedding_base_url,
"EMBEDDING_DIM=0: probing embedding service to auto-infer dimension"
);

let vec = embedder
.embed("dimension probe")
.await
.map_err(|e| anyhow::anyhow!(
"EMBEDDING_DIM=0 but the embedding probe failed: {e}. \
Set EMBEDDING_DIM explicitly (e.g. EMBEDDING_DIM=768 for \
nomic-embed-text, EMBEDDING_DIM=1024 for BAAI/bge-m3) or \
check that your embedding service is reachable."
))?;

if vec.is_empty() {
return Err(anyhow::anyhow!(
"EMBEDDING_DIM=0: embedding service returned an empty vector. \
Set EMBEDDING_DIM explicitly."
));
}

let dim = vec.len();
tracing::info!(embedding_dim = dim, "Auto-inferred embedding dimension");
Ok(dim)
}

fn build_embedder(
cfg: &memoria_service::Config,
) -> Option<Arc<dyn memoria_core::interfaces::EmbeddingProvider>> {
Expand Down
2 changes: 1 addition & 1 deletion memoria/crates/memoria-service/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ impl Config {
let embedding_dim = std::env::var("EMBEDDING_DIM")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(1024usize);
.unwrap_or(0usize); // 0 = auto-infer from embedding service at startup

let llm_api_key = std::env::var("LLM_API_KEY").ok().filter(|s| !s.is_empty());

Expand Down
46 changes: 46 additions & 0 deletions memoria/crates/memoria-storage/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2050,6 +2050,52 @@ impl SqlMemoryStore {
Ok(())
}

/// Check that the configured embedding dimension matches the dimension
/// already stored in the database schema.
///
/// If `mem_memories` already exists with a different dimension, returning
/// an error here is far better than silently failing on the first INSERT.
/// Called after `migrate()` so the table is guaranteed to exist.
///
/// # Errors
/// Returns [`MemoriaError::Internal`] when a mismatch is detected,
/// with a human-readable message explaining how to resolve it.
pub async fn check_embedding_dim_compat(&self) -> Result<(), MemoriaError> {
// Query the actual column type stored in the schema, e.g. "vecf32(768)"
let col_type: Option<String> = sqlx::query_scalar(
"SELECT column_type \
FROM information_schema.columns \
WHERE table_schema = DATABASE() \
AND table_name = 'mem_memories' \
AND column_name = 'embedding'",
)
.fetch_optional(&self.pool)
.await
.map_err(db_err)?;

if let Some(ct) = col_type {
// Parse "vecf32(768)" → 768
if let Ok(schema_dim) = ct
.trim_start_matches("vecf32(")
.trim_end_matches(')')
.parse::<usize>()
{
if schema_dim != self.embedding_dim {
return Err(MemoriaError::Internal(format!(
"Embedding dimension mismatch: the database schema has \
{}d vectors but Memoria is configured for {}d. \
To fix: either set EMBEDDING_DIM={} to match the \
existing schema, or drop the database (data loss) and \
restart to rebuild with the new dimension.",
schema_dim, self.embedding_dim, schema_dim
)));
}
}
}

Ok(())
}

// ── Audit log ─────────────────────────────────────────────────────────────

/// Create a safety snapshot before destructive operations. Best-effort.
Expand Down
2 changes: 1 addition & 1 deletion skills/deployment/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Services: API on `:8100`, MatrixOne on `:6001`. Verify: `curl http://localhost:8
| `MEMORIA_EMBEDDING_API_KEY` | — | Required if provider is `openai` (single-backend) |
| `MEMORIA_EMBEDDING_BASE_URL` | — | Custom endpoint, OpenAI-compatible (single-backend) |
| `MEMORIA_EMBEDDING_ENDPOINTS` | — | JSON array for multi-backend round-robin. When set, supersedes `BASE_URL`/`API_KEY`. Format: `[{"url":"https://api1.example.com/v1","api_key":"sk-1"},{"url":"https://api2.example.com/v1","api_key":"sk-2"}]`. All endpoints must serve the same model. Requests rotate round-robin; failed/rate-limited endpoints are skipped automatically. |
| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension |
| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension. `0` = auto-infer: Memoria probes the embedding service on startup and uses the returned vector length. Set explicitly (e.g. `768`, `1024`) to skip the probe or when the embedding service may be unavailable at boot time. |

### Distributed

Expand Down