diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index eaaceb8b..e94417b2 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -17225,6 +17225,7 @@ mod tests { labels: vec![state.to_string()], }, remote: SearchRemoteState::default(), + match_context: None, score: 0, } } diff --git a/crates/loc-cli/src/commands.rs b/crates/loc-cli/src/commands.rs index c5c2d800..88f409e9 100644 --- a/crates/loc-cli/src/commands.rs +++ b/crates/loc-cli/src/commands.rs @@ -6404,6 +6404,9 @@ fn print_search_report(report: &SearchReport) { result.mount_id, result.connector, result.remote_id ); println!(" path: {}", result.absolute_path); + if let Some(context) = &result.match_context { + println!(" match: {}: {}", context.field, context.text); + } if !result.safety.agent_readable { println!(" safety: {}", result.safety.labels.join(", ")); } @@ -10388,6 +10391,7 @@ mod tests { labels: vec!["ready".to_string()], }, remote: SearchRemoteState::default(), + match_context: None, score: 100, }], }; diff --git a/crates/loc-cli/src/search.rs b/crates/loc-cli/src/search.rs index 91ec519d..54d5ce45 100644 --- a/crates/loc-cli/src/search.rs +++ b/crates/loc-cli/src/search.rs @@ -10,8 +10,8 @@ use std::path::{Path, PathBuf}; use locality_core::model::{EntityKind, HydrationState, RemoteId}; use locality_store::{ ConnectionRecord, ConnectionRepository, EntityRecord, EntityRepository, EntitySearchCandidate, - EntitySearchRepository, MountConfig, MountRepository, RemoteObservationRecord, - RemoteObservationRepository, StoreError, + EntitySearchDocument, EntitySearchRepository, MountConfig, MountRepository, + RemoteObservationRecord, RemoteObservationRepository, StoreError, }; use serde::Serialize; @@ -58,10 +58,18 @@ pub struct SearchResult { pub state: String, pub safety: SearchSafety, pub remote: SearchRemoteState, + #[serde(skip_serializing_if = "Option::is_none")] + pub match_context: Option, #[serde(skip_serializing)] pub score: i64, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct SearchMatchContext { + pub field: String, + pub text: String, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct SearchSafety { pub agent_readable: bool, @@ -148,29 +156,31 @@ where { let access_root = mount_access_root(&mount); - let mut mount_matches = if let Some(candidates) = store + let mount_matches = if let Some(candidates) = store .list_entity_search_candidates(&mount.mount_id, &query, notion_id.as_deref()) .map_err(SearchError::Store)? { - search_entity_candidates( + let mut indexed_matches = search_entity_candidates( &mount, &access_root, &candidates, &query, notion_id.as_deref(), - ) + ); + if indexed_matches.len() < options.limit { + let fallback_matches = search_all_indexed_entities( + store, + &mount, + &access_root, + &query, + notion_id.as_deref(), + )?; + merge_search_results(&mut indexed_matches, fallback_matches); + } + indexed_matches } else { search_all_indexed_entities(store, &mount, &access_root, &query, notion_id.as_deref())? }; - if mount_matches.len() < options.limit { - mount_matches = search_all_indexed_entities( - store, - &mount, - &access_root, - &query, - notion_id.as_deref(), - )?; - } let has_exact_mount_entity_match = has_exact_entity_match(&mount_matches, notion_id.as_deref()); if mount.remote_root_id.as_ref().is_some_and(|remote_id| { @@ -197,6 +207,7 @@ where labels: vec!["workspace".to_string(), "metadata_only".to_string()], }, remote: SearchRemoteState::default(), + match_context: None, score: 120_000, }); } @@ -266,6 +277,16 @@ fn has_exact_entity_match(results: &[SearchResult], notion_id: Option<&str>) -> }) } +fn merge_search_results(results: &mut Vec, additional: Vec) { + for result in additional { + if !results.iter().any(|existing| { + existing.mount_id == result.mount_id && existing.remote_id == result.remote_id + }) { + results.push(result); + } + } +} + fn search_all_indexed_entities( store: &S, mount: &MountConfig, @@ -371,7 +392,7 @@ pub fn search_indexed_entities( .iter() .filter_map(|entity| { let observation = observations.get(&entity.remote_id); - let score = indexed_entity_score(entity, observation, query, notion_id)?; + let matched = indexed_entity_match(entity, observation, None, query, notion_id)?; let remote = remote_state(entity, observation); let result_path = located_entity_path(entity); let state = search_state(&entity.hydration, &remote).to_string(); @@ -388,7 +409,8 @@ pub fn search_indexed_entities( state, safety: search_safety(&entity.hydration, &remote), remote, - score, + match_context: matched.context, + score: matched.score, }) }) .collect() @@ -409,6 +431,7 @@ pub fn search_entity_candidates( access_root, &candidate.entity, candidate.observation.as_ref(), + candidate.search_document.as_ref(), query, notion_id, ) @@ -421,10 +444,11 @@ fn search_entity_result( access_root: &Path, entity: &EntityRecord, observation: Option<&RemoteObservationRecord>, + search_document: Option<&EntitySearchDocument>, query: &str, notion_id: Option<&str>, ) -> Option { - let score = indexed_entity_score(entity, observation, query, notion_id)?; + let matched = indexed_entity_match(entity, observation, search_document, query, notion_id)?; let remote = remote_state(entity, observation); let result_path = located_entity_path(entity); let state = search_state(&entity.hydration, &remote).to_string(); @@ -441,37 +465,37 @@ fn search_entity_result( state, safety: search_safety(&entity.hydration, &remote), remote, - score, + match_context: matched.context, + score: matched.score, }) } -fn indexed_entity_score( +fn indexed_entity_match( entity: &EntityRecord, observation: Option<&RemoteObservationRecord>, + search_document: Option<&EntitySearchDocument>, query: &str, notion_id: Option<&str>, -) -> Option { +) -> Option { if let Some(notion_id) = notion_id { - if compact_notion_id(&entity.remote_id.0) == notion_id - || compact_path_id(&entity.path) == notion_id - { - return Some(100_000); + if compact_notion_id(&entity.remote_id.0) == notion_id { + return Some(SearchMatch::new( + 100_000, + Some(match_context("remote_id", &entity.remote_id.0, "", &[])), + )); + } + if compact_path_id(&entity.path) == notion_id { + let path = locality_platform::logical_path_display(&entity.path); + return Some(SearchMatch::new( + 100_000, + Some(match_context("path", &path, "", &[])), + )); } return None; } let normalized_query = normalize_search_text(query); let phrase = normalized_query.trim(); - - let title = normalize_search_text(&entity.title); - let path = normalize_search_text(&entity.path.to_string_lossy()); - let observed_title = observation - .map(|observation| normalize_search_text(&observation.title)) - .unwrap_or_default(); - let observed_path = observation - .map(|observation| normalize_search_text(&observation.projected_path.to_string_lossy())) - .unwrap_or_default(); - let haystack = format!("{title} {path} {observed_title} {observed_path}"); let tokens = phrase .split_whitespace() .filter(|token| search_token_allowed(token)) @@ -480,62 +504,305 @@ fn indexed_entity_score( return None; } - if title == phrase { - return Some(90_000); - } - if observed_title == phrase { - return Some(88_000); - } - if path == phrase { - return Some(86_000); - } - if observed_path == phrase { - return Some(84_000); + search_fields(entity, observation, search_document) + .into_iter() + .filter_map(|field| score_search_field(field, phrase, &tokens)) + .max_by(|left, right| { + left.score + .cmp(&right.score) + .then_with(|| left.context_field().cmp(right.context_field())) + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct SearchMatch { + score: i64, + context: Option, +} + +impl SearchMatch { + fn new(score: i64, context: Option) -> Self { + Self { score, context } } - if title.starts_with(phrase) { - return Some(82_000); + + fn context_field(&self) -> &str { + self.context + .as_ref() + .map(|context| context.field.as_str()) + .unwrap_or("") } - if observed_title.starts_with(phrase) { - return Some(80_000); +} + +#[derive(Clone, Copy, Debug)] +struct SearchFieldScores { + exact: i64, + prefix: i64, + contains: i64, + all_tokens: i64, + any_tokens: i64, +} + +#[derive(Clone, Debug)] +struct SearchField { + name: &'static str, + text: String, + scores: SearchFieldScores, +} + +const TITLE_FIELD: SearchFieldScores = SearchFieldScores { + exact: 90_000, + prefix: 82_000, + contains: 74_000, + all_tokens: 60_000, + any_tokens: 30_000, +}; +const REMOTE_ID_FIELD: SearchFieldScores = SearchFieldScores { + exact: 91_000, + prefix: 83_000, + contains: 75_000, + all_tokens: 62_000, + any_tokens: 32_000, +}; +const OBSERVED_TITLE_FIELD: SearchFieldScores = SearchFieldScores { + exact: 88_000, + prefix: 80_000, + contains: 72_000, + all_tokens: 59_000, + any_tokens: 29_000, +}; +const ALIAS_FIELD: SearchFieldScores = SearchFieldScores { + exact: 89_000, + prefix: 81_000, + contains: 73_000, + all_tokens: 61_000, + any_tokens: 31_000, +}; +const SOURCE_URL_FIELD: SearchFieldScores = SearchFieldScores { + exact: 87_000, + prefix: 79_000, + contains: 71_000, + all_tokens: 60_000, + any_tokens: 30_000, +}; +const PATH_FIELD: SearchFieldScores = SearchFieldScores { + exact: 86_000, + prefix: 78_000, + contains: 70_000, + all_tokens: 58_000, + any_tokens: 28_000, +}; +const OBSERVED_PATH_FIELD: SearchFieldScores = SearchFieldScores { + exact: 84_000, + prefix: 76_000, + contains: 68_000, + all_tokens: 57_000, + any_tokens: 27_000, +}; +const BREADCRUMB_FIELD: SearchFieldScores = SearchFieldScores { + exact: 76_000, + prefix: 68_000, + contains: 62_000, + all_tokens: 54_000, + any_tokens: 26_000, +}; +const METADATA_FIELD: SearchFieldScores = SearchFieldScores { + exact: 66_000, + prefix: 62_000, + contains: 56_000, + all_tokens: 42_000, + any_tokens: 22_000, +}; +const FRONTMATTER_FIELD: SearchFieldScores = SearchFieldScores { + exact: 54_000, + prefix: 50_000, + contains: 44_000, + all_tokens: 24_000, + any_tokens: 12_000, +}; +const BODY_FIELD: SearchFieldScores = SearchFieldScores { + exact: 28_000, + prefix: 26_000, + contains: 25_000, + all_tokens: 20_000, + any_tokens: 10_000, +}; + +fn search_fields( + entity: &EntityRecord, + observation: Option<&RemoteObservationRecord>, + search_document: Option<&EntitySearchDocument>, +) -> Vec { + let mut fields = Vec::new(); + push_search_field( + &mut fields, + "remote_id", + Some(entity.remote_id.0.as_str()), + REMOTE_ID_FIELD, + ); + push_search_field( + &mut fields, + "title", + Some(entity.title.as_str()), + TITLE_FIELD, + ); + push_search_field( + &mut fields, + "path", + Some(locality_platform::logical_path_display(&entity.path)), + PATH_FIELD, + ); + if let Some(observation) = observation { + push_search_field( + &mut fields, + "remote_title", + Some(observation.title.as_str()), + OBSERVED_TITLE_FIELD, + ); + push_search_field( + &mut fields, + "remote_path", + Some(locality_platform::logical_path_display( + &observation.projected_path, + )), + OBSERVED_PATH_FIELD, + ); + } + if let Some(document) = search_document { + push_search_field( + &mut fields, + "alias", + document.aliases.as_deref(), + ALIAS_FIELD, + ); + push_search_field( + &mut fields, + "source_url", + document.source_url.as_deref(), + SOURCE_URL_FIELD, + ); + push_search_field( + &mut fields, + "metadata", + document.metadata_text.as_deref(), + METADATA_FIELD, + ); + push_search_field( + &mut fields, + "breadcrumb", + document.breadcrumbs.as_deref(), + BREADCRUMB_FIELD, + ); + push_search_field( + &mut fields, + "frontmatter", + document.frontmatter.as_deref(), + FRONTMATTER_FIELD, + ); + push_search_field(&mut fields, "body", document.body.as_deref(), BODY_FIELD); + } + fields +} + +fn push_search_field( + fields: &mut Vec, + name: &'static str, + text: Option>, + scores: SearchFieldScores, +) { + let Some(text) = text else { + return; + }; + let text = text.into(); + if !text.trim().is_empty() { + fields.push(SearchField { name, text, scores }); } - if path.starts_with(phrase) { - return Some(78_000); +} + +fn score_search_field(field: SearchField, phrase: &str, tokens: &[&str]) -> Option { + let normalized = normalize_search_text(&field.text); + if normalized.is_empty() { + return None; } - if observed_path.starts_with(phrase) { - return Some(76_000); + let score = if normalized == phrase { + field.scores.exact + } else if normalized.starts_with(phrase) { + field.scores.prefix + } else if normalized.contains(phrase) { + field.scores.contains + } else { + let matched_tokens = tokens + .iter() + .filter(|token| normalized.contains(**token)) + .count(); + if matched_tokens == 0 { + return None; + } + if matched_tokens == tokens.len() { + field.scores.all_tokens + matched_tokens as i64 + } else { + field.scores.any_tokens + matched_tokens as i64 + } + }; + Some(SearchMatch::new( + score, + Some(match_context(field.name, &field.text, phrase, tokens)), + )) +} + +fn match_context( + field: impl Into, + text: &str, + phrase: &str, + tokens: &[&str], +) -> SearchMatchContext { + SearchMatchContext { + field: field.into(), + text: snippet_for_match(text, phrase, tokens), } - if title.contains(phrase) { - return Some(74_000); +} + +fn snippet_for_match(text: &str, phrase: &str, tokens: &[&str]) -> String { + let trimmed = text.trim(); + if trimmed.is_empty() { + return String::new(); } - if observed_title.contains(phrase) { - return Some(72_000); + let lower = trimmed.to_ascii_lowercase(); + let start = (!phrase.is_empty()) + .then(|| lower.find(phrase)) + .flatten() + .or_else(|| { + tokens + .iter() + .find_map(|token| lower.find(&token.to_ascii_lowercase())) + }) + .unwrap_or(0); + bounded_snippet(trimmed, start, 160) +} + +fn bounded_snippet(text: &str, match_start: usize, max_len: usize) -> String { + if text.len() <= max_len { + return text.split_whitespace().collect::>().join(" "); } - if path.contains(phrase) { - return Some(70_000); + + let half_window = max_len / 2; + let mut start = match_start.saturating_sub(half_window); + let mut end = start.saturating_add(max_len).min(text.len()); + while start > 0 && !text.is_char_boundary(start) { + start -= 1; } - if observed_path.contains(phrase) { - return Some(68_000); + while end > start && !text.is_char_boundary(end) { + end -= 1; } - - let matched_tokens = tokens - .iter() - .filter(|token| haystack.contains(**token)) - .count(); - if matched_tokens == 0 { - return None; + let snippet = text[start..end] + .split_whitespace() + .collect::>() + .join(" "); + match (start > 0, end < text.len()) { + (true, true) => format!("...{snippet}..."), + (true, false) => format!("...{snippet}"), + (false, true) => format!("{snippet}..."), + (false, false) => snippet, } - - let all_tokens_matched = matched_tokens == tokens.len(); - let title_bonus = tokens - .iter() - .filter(|token| title.contains(**token) || observed_title.contains(**token)) - .count() as i64 - * 500; - Some(if all_tokens_matched { - 60_000 + title_bonus + matched_tokens as i64 - } else { - 30_000 + title_bonus + matched_tokens as i64 - }) } fn search_token_allowed(token: &str) -> bool { diff --git a/crates/loc-cli/tests/search.rs b/crates/loc-cli/tests/search.rs index 6647f5e3..abbf139c 100644 --- a/crates/loc-cli/tests/search.rs +++ b/crates/loc-cli/tests/search.rs @@ -6,10 +6,11 @@ use std::time::{SystemTime, UNIX_EPOCH}; use loc_cli::search::{SearchError, SearchOptions, notion_id_from_url, run_search}; use locality_core::freshness::RemoteVersion; use locality_core::model::{EntityKind, HydrationState, MountId, RemoteId}; +use locality_core::shadow::ShadowDocument; use locality_store::{ ConnectionId, ConnectionRecord, ConnectionRepository, EntityRecord, EntityRepository, InMemoryStateStore, MountConfig, MountRepository, ProjectionMode, RemoteObservationRecord, - RemoteObservationRepository, SqliteStateStore, + RemoteObservationRepository, ShadowRepository, SqliteStateStore, }; #[test] @@ -209,6 +210,156 @@ fn search_uses_sqlite_candidate_index_without_changing_report() { ); } +#[test] +fn search_uses_hydrated_body_text_from_sqlite_index() { + let fixture = SearchFixture::new(); + let mut store = fixture.sqlite_store(); + fixture.seed_entities(&mut store); + let remote_id = RemoteId::new("37b3ac0ebb88802cbcf4d53c9cfc4972"); + store + .save_shadow( + &fixture.mount_id, + ShadowDocument::from_synced_body( + remote_id, + "# Initial Idea\n\nRetry budget planning for OAuth broker handoff.", + 1, + [RemoteId::new("heading-1"), RemoteId::new("paragraph-1")], + ) + .expect("shadow"), + ) + .expect("save shadow"); + + let report = run_search(&store, SearchOptions::new("broker handoff")).expect("body search"); + + assert_eq!(report.results.len(), 1); + assert_eq!(report.results[0].title, "Initial Idea"); + assert_eq!(report.results[0].path, "Product/Initial Idea/page.md"); + assert_eq!(report.results[0].state, "ready"); + assert_eq!( + report.results[0] + .match_context + .as_ref() + .map(|context| context.field.as_str()), + Some("body") + ); + assert!( + report.results[0] + .match_context + .as_ref() + .is_some_and(|context| context.text.contains("broker handoff")) + ); +} + +#[test] +fn search_uses_connector_metadata_from_sqlite_index() { + let fixture = SearchFixture::new(); + let mut store = fixture.sqlite_store(); + fixture.seed_entities(&mut store); + store + .save_remote_observation( + RemoteObservationRecord::new( + fixture.mount_id.clone(), + RemoteId::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + EntityKind::Page, + "Launch Plan", + "Engineering/Launch Plan/page.md", + "2026-06-16T00:00:00Z", + ) + .with_remote_version(RemoteVersion("remote-v2".to_string())) + .with_raw_metadata_json( + r#"{"loc_search":{"metadata_text":["customer escalation"],"aliases":["ENG-1"],"source_url":"https://linear.app/acme/issue/ENG-1/improve-sync"}}"#, + ), + ) + .expect("save observation"); + store + .save_shadow( + &fixture.mount_id, + ShadowDocument::from_synced_body( + RemoteId::new("37b3ac0ebb88802cbcf4d53c9cfc4972"), + "# Initial Idea\n\nENG-1 appears here only as body text.", + 1, + [RemoteId::new("heading-1"), RemoteId::new("paragraph-1")], + ) + .expect("shadow"), + ) + .expect("save body shadow"); + + let report = + run_search(&store, SearchOptions::new("customer escalation")).expect("metadata search"); + + assert_eq!(report.results.len(), 1); + assert_eq!(report.results[0].title, "Roadmap 2026"); + assert_eq!(report.results[0].state, "remote_update_available"); + assert_eq!( + report.results[0].remote.observed_title.as_deref(), + Some("Launch Plan") + ); + assert_eq!( + report.results[0] + .match_context + .as_ref() + .map(|context| context.field.as_str()), + Some("metadata") + ); + assert!( + report.results[0] + .match_context + .as_ref() + .is_some_and(|context| context.text.contains("customer escalation")) + ); + + let alias_report = run_search(&store, SearchOptions::new("ENG-1")).expect("alias search"); + + assert_eq!(alias_report.results.len(), 2); + assert_eq!(alias_report.results[0].title, "Roadmap 2026"); + assert_eq!( + alias_report.results[0] + .match_context + .as_ref() + .map(|context| context.field.as_str()), + Some("alias") + ); + assert_eq!(alias_report.results[1].title, "Initial Idea"); + assert_eq!( + alias_report.results[1] + .match_context + .as_ref() + .map(|context| context.field.as_str()), + Some("body") + ); +} + +#[test] +fn search_uses_generic_remote_ids_from_sqlite_index() { + let fixture = SearchFixture::new(); + let mut store = fixture.sqlite_store(); + store + .save_entity( + EntityRecord::new( + fixture.mount_id.clone(), + RemoteId::new("slack-C1234567"), + EntityKind::Page, + "Team Channel Recent", + "Slack/Team Channel/recent.md", + ) + .with_hydration(HydrationState::Hydrated), + ) + .expect("save generic remote id entity"); + + let report = + run_search(&store, SearchOptions::new("SLACK-C1234567")).expect("generic remote id search"); + + assert_eq!(report.results.len(), 1); + assert_eq!(report.results[0].title, "Team Channel Recent"); + assert_eq!( + report.results[0] + .match_context + .as_ref() + .map(|context| context.field.as_str()), + Some("remote_id") + ); +} + #[test] fn search_matches_numeric_short_title_terms() { let fixture = SearchFixture::new(); diff --git a/crates/locality-core/src/lib.rs b/crates/locality-core/src/lib.rs index 441c13a0..981502b4 100644 --- a/crates/locality-core/src/lib.rs +++ b/crates/locality-core/src/lib.rs @@ -23,6 +23,7 @@ pub mod planner; pub mod pull; pub mod push; pub mod readable_diff; +pub mod search; pub mod shadow; pub mod sync; pub mod undo; diff --git a/crates/locality-core/src/search.rs b/crates/locality-core/src/search.rs new file mode 100644 index 00000000..e363f1f6 --- /dev/null +++ b/crates/locality-core/src/search.rs @@ -0,0 +1,47 @@ +//! Connector-neutral search metadata. +//! +//! Connectors can include this payload under `loc_search` in persisted remote +//! observation metadata. The store treats it as rebuildable search-index input, +//! not as source-of-truth connector state. + +use serde::{Deserialize, Serialize}; + +pub const RAW_SEARCH_METADATA_KEY: &str = "loc_search"; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SearchMetadata { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub metadata_text: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub aliases: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, +} + +impl SearchMetadata { + pub fn is_empty(&self) -> bool { + self.metadata_text.is_empty() && self.aliases.is_empty() && self.source_url.is_none() + } + + pub fn push_metadata_text(&mut self, value: impl AsRef) { + push_non_empty(&mut self.metadata_text, value.as_ref()); + } + + pub fn push_alias(&mut self, value: impl AsRef) { + push_non_empty(&mut self.aliases, value.as_ref()); + } + + pub fn set_source_url(&mut self, value: impl Into) { + let value = value.into(); + if !value.trim().is_empty() { + self.source_url = Some(value); + } + } +} + +fn push_non_empty(values: &mut Vec, value: &str) { + let value = value.trim(); + if !value.is_empty() { + values.push(value.to_string()); + } +} diff --git a/crates/locality-gmail/src/connector.rs b/crates/locality-gmail/src/connector.rs index 9806f7a4..d16793f8 100644 --- a/crates/locality-gmail/src/connector.rs +++ b/crates/locality-gmail/src/connector.rs @@ -16,6 +16,7 @@ use locality_core::model::{ CanonicalDocument, EntityKind, HydrationState, MountId, RemoteId, TreeEntry, }; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind}; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::validation::ValidationIssue; use locality_core::{LocalityError, LocalityResult}; use serde::{Deserialize, Serialize}; @@ -291,9 +292,11 @@ impl Connector for GmailConnector { ) .with_parent(thread_remote_id(&mailbox, &thread_id)) .with_remote_version(RemoteVersion::new(remote_version(&message))) - .with_raw_metadata_json( - serde_json::to_string(&message).unwrap_or_else(|_| "{}".to_string()), - )); + .with_raw_metadata_json(gmail_message_metadata_json( + &message, + &mailbox, + Some(&thread_id), + ))); } if let Some((mailbox, thread_id)) = parse_thread_remote_id(&request.remote_id) { @@ -315,9 +318,7 @@ impl Connector for GmailConnector { ) .with_parent(RemoteId::new(mailbox_folder_id(&mailbox))) .with_remote_version(RemoteVersion::new(thread_remote_version(&thread))) - .with_raw_metadata_json( - serde_json::to_string(&thread).unwrap_or_else(|_| "{}".to_string()), - )); + .with_raw_metadata_json(gmail_thread_metadata_json(&thread, &mailbox))); } let message = self.api.get_message_metadata(request.remote_id.as_str())?; @@ -338,9 +339,7 @@ impl Connector for GmailConnector { ) .with_parent(RemoteId::new(parent_id)) .with_remote_version(RemoteVersion::new(remote_version(&message))) - .with_raw_metadata_json( - serde_json::to_string(&message).unwrap_or_else(|_| "{}".to_string()), - )) + .with_raw_metadata_json(gmail_message_metadata_json(&message, mailbox, None))) } fn fetch(&self, request: FetchRequest) -> LocalityResult { @@ -605,10 +604,125 @@ fn folder_observation( folder.title, ) .with_remote_version(RemoteVersion::new(format!("folder:{}", folder.title))) - .with_raw_metadata_json(format!( - r#"{{"kind":"gmail_folder","id":"{}","title":"{}"}}"#, - folder.id, folder.title - )) + .with_raw_metadata_json(gmail_folder_metadata_json(folder)) +} + +fn gmail_message_metadata_json( + message: &GmailMessage, + mailbox: &str, + thread_id: Option<&str>, +) -> String { + metadata_json( + message, + gmail_message_search_metadata(message, mailbox, thread_id), + ) +} + +fn gmail_thread_metadata_json(thread: &GmailThread, mailbox: &str) -> String { + metadata_json(thread, gmail_thread_search_metadata(thread, mailbox)) +} + +fn metadata_json(value: &T, search_metadata: SearchMetadata) -> String +where + T: Serialize, +{ + let mut value = serde_json::to_value(value).unwrap_or_else(|_| serde_json::json!({})); + if let serde_json::Value::Object(object) = &mut value + && !search_metadata.is_empty() + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn gmail_folder_metadata_json(folder: FolderSpec) -> String { + let mut search_metadata = SearchMetadata::default(); + search_metadata.push_metadata_text(folder.title); + search_metadata.push_metadata_text(folder.id); + search_metadata.push_alias(folder.id); + let mut value = serde_json::json!({ + "kind": "gmail_folder", + "id": folder.id, + "title": folder.title, + }); + if let serde_json::Value::Object(object) = &mut value + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + value.to_string() +} + +fn gmail_message_search_metadata( + message: &GmailMessage, + mailbox: &str, + thread_id: Option<&str>, +) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(mailbox); + push_gmail_message_search_values(&mut metadata, message); + metadata.push_alias(&message.id); + let source_thread_id = thread_id + .map(str::to_string) + .or_else(|| message.thread_id.clone()) + .unwrap_or_else(|| message.id.clone()); + metadata.push_alias(&source_thread_id); + metadata.set_source_url(gmail_source_url(&source_thread_id)); + metadata +} + +fn gmail_thread_search_metadata(thread: &GmailThread, mailbox: &str) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(mailbox); + metadata.push_metadata_text(&thread.id); + metadata.push_alias(&thread.id); + if let Some(history_id) = &thread.history_id { + metadata.push_metadata_text(history_id); + } + for message in &thread.messages { + push_gmail_message_search_values(&mut metadata, message); + metadata.push_alias(&message.id); + } + metadata.set_source_url(gmail_source_url(&thread.id)); + metadata +} + +fn push_gmail_message_search_values(metadata: &mut SearchMetadata, message: &GmailMessage) { + metadata.push_metadata_text(&message.id); + if let Some(thread_id) = &message.thread_id { + metadata.push_metadata_text(thread_id); + } + for label in &message.label_ids { + metadata.push_metadata_text(label); + } + if let Some(snippet) = &message.snippet { + metadata.push_metadata_text(snippet); + } + if let Some(internal_date) = &message.internal_date { + metadata.push_metadata_text(internal_date); + } + let headers = message.payload.as_ref().map(header_map).unwrap_or_default(); + for header in [ + "subject", + "from", + "to", + "cc", + "bcc", + "reply-to", + "sender", + "date", + "message-id", + "list-id", + ] { + if let Some(value) = headers.get(header) { + metadata.push_metadata_text(value); + } + } +} + +fn gmail_source_url(id: &str) -> String { + format!("https://mail.google.com/mail/u/0/#all/{id}") } fn list_label_entries( @@ -1104,6 +1218,7 @@ mod tests { use locality_core::model::{CanonicalDocument, EntityKind, MountId, RemoteId}; use locality_core::planner::{PropertyValue, PushOperation, PushPlan}; use locality_core::push::RemotePrecondition; + use locality_core::search::RAW_SEARCH_METADATA_KEY; use super::{GmailConfig, GmailConnector}; use crate::client::GmailApi; @@ -1475,6 +1590,21 @@ mod tests { std::path::PathBuf::from("inbox/1720900000000-hello-thread-inbox-1/page.md") ); assert!(observation.raw_metadata_json.contains("thread-inbox-1")); + let raw_metadata: serde_json::Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + serde_json::json!("https://mail.google.com/mail/u/0/#all/thread-inbox-1") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + serde_json::json!(["thread-inbox-1", "inbox-msg-1"]) + ); + let search_terms = raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text"); + assert!(search_terms.contains(&serde_json::json!("Ann "))); + assert!(search_terms.contains(&serde_json::json!("Hello"))); } #[test] diff --git a/crates/locality-google-calendar/src/connector.rs b/crates/locality-google-calendar/src/connector.rs index 82634b09..503e6390 100644 --- a/crates/locality-google-calendar/src/connector.rs +++ b/crates/locality-google-calendar/src/connector.rs @@ -16,6 +16,7 @@ use locality_core::model::{ CanonicalDocument, EntityKind, HydrationState, MountId, RemoteId, TreeEntry, }; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind}; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::validation::ValidationIssue; use locality_core::{LocalityError, LocalityResult}; use serde::{Deserialize, Serialize}; @@ -196,7 +197,7 @@ impl Connector for GoogleCalendarConnector { .with_parent(RemoteId::new(EVENTS_FOLDER_ID)) .with_remote_version(RemoteVersion::new(event.remote_version())) .deleted(deleted) - .with_raw_metadata_json(serde_json::to_string(&event).unwrap_or_else(|_| "{}".to_string()))) + .with_raw_metadata_json(event_metadata_json(&event))) } fn fetch(&self, request: FetchRequest) -> LocalityResult { @@ -408,10 +409,120 @@ fn folder_observation( PathBuf::from(folder.title), ) .with_remote_version(RemoteVersion::new(format!("folder:{}", folder.title))) - .with_raw_metadata_json(format!( - r#"{{"kind":"google_calendar_folder","id":"{}","title":"{}"}}"#, - folder.id, folder.title - )) + .with_raw_metadata_json(folder_metadata_json(folder)) +} + +fn event_metadata_json(event: &CalendarEvent) -> String { + let mut value = serde_json::to_value(event).unwrap_or_else(|_| serde_json::json!({})); + if let Value::Object(object) = &mut value { + let search_metadata = event_search_metadata(event); + if !search_metadata.is_empty() + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn folder_metadata_json(folder: FolderSpec) -> String { + let mut search_metadata = SearchMetadata::default(); + search_metadata.push_metadata_text(folder.title); + search_metadata.push_metadata_text(folder.id); + search_metadata.push_alias(folder.id); + let mut value = serde_json::json!({ + "kind": "google_calendar_folder", + "id": folder.id, + "title": folder.title, + }); + if let Value::Object(object) = &mut value + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + value.to_string() +} + +fn event_search_metadata(event: &CalendarEvent) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + if let Some(id) = &event.id { + metadata.push_metadata_text(id); + metadata.push_alias(id); + } + if let Some(summary) = &event.summary { + metadata.push_metadata_text(summary); + } + if let Some(location) = &event.location { + metadata.push_metadata_text(location); + } + if let Some(status) = &event.status { + metadata.push_metadata_text(status); + } + if let Some(event_type) = &event.event_type { + metadata.push_metadata_text(event_type); + } + if let Some(i_cal_uid) = &event.i_cal_uid { + metadata.push_metadata_text(i_cal_uid); + metadata.push_alias(i_cal_uid); + } + push_event_date_time_search_values(&mut metadata, event.start.as_ref()); + push_event_date_time_search_values(&mut metadata, event.end.as_ref()); + for recurrence in &event.recurrence { + metadata.push_metadata_text(recurrence); + } + if let Some(recurring_event_id) = &event.recurring_event_id { + metadata.push_metadata_text(recurring_event_id); + } + for attendee in &event.attendees { + if let Some(email) = &attendee.email { + metadata.push_metadata_text(email); + } + if let Some(display_name) = &attendee.display_name { + metadata.push_metadata_text(display_name); + } + if let Some(response_status) = &attendee.response_status { + metadata.push_metadata_text(response_status); + } + } + if let Some(creator) = &event.creator { + push_json_text_field(&mut metadata, creator, "email"); + push_json_text_field(&mut metadata, creator, "displayName"); + } + if let Some(organizer) = &event.organizer { + push_json_text_field(&mut metadata, organizer, "email"); + push_json_text_field(&mut metadata, organizer, "displayName"); + } + if let Some(hangout_link) = &event.hangout_link { + metadata.push_metadata_text(hangout_link); + } + if let Some(html_link) = &event.html_link { + metadata.set_source_url(html_link.clone()); + } + metadata +} + +fn push_event_date_time_search_values( + metadata: &mut SearchMetadata, + date_time: Option<&EventDateTime>, +) { + let Some(date_time) = date_time else { + return; + }; + if let Some(date) = &date_time.date { + metadata.push_metadata_text(date); + } + if let Some(value) = &date_time.date_time { + metadata.push_metadata_text(value); + } + if let Some(time_zone) = &date_time.time_zone { + metadata.push_metadata_text(time_zone); + } +} + +fn push_json_text_field(metadata: &mut SearchMetadata, value: &Value, field: &str) { + if let Some(text) = value.get(field).and_then(Value::as_str) { + metadata.push_metadata_text(text); + } } fn list_primary_event_entries( @@ -1026,6 +1137,7 @@ mod tests { use locality_core::model::{CanonicalDocument, EntityKind, MountId, RemoteId}; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind, PushPlan}; use locality_core::push::RemotePrecondition; + use locality_core::search::RAW_SEARCH_METADATA_KEY; use serde_json::json; use super::{GoogleCalendarConfig, GoogleCalendarConnector}; @@ -1162,6 +1274,22 @@ mod tests { std::path::PathBuf::from("events/20260720-100000-design-review-event-1.md") ); assert!(observation.raw_metadata_json.contains("\"event-1\"")); + let raw_metadata: serde_json::Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + json!("https://calendar.google.com/calendar/event?eid=event-1") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + json!(["event-1"]) + ); + assert!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text") + .contains(&json!("Room 12")) + ); let calls = api.calls.lock().expect("calls"); assert_eq!( diff --git a/crates/locality-google-docs/src/connector.rs b/crates/locality-google-docs/src/connector.rs index 0131aa86..946f6618 100644 --- a/crates/locality-google-docs/src/connector.rs +++ b/crates/locality-google-docs/src/connector.rs @@ -12,6 +12,7 @@ use locality_core::journal::JournalApplyEffect; use locality_core::model::{CanonicalDocument, EntityKind, HydrationState, RemoteId, TreeEntry}; use locality_core::path_projection::{page_container_path, page_document_path}; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind, PushPlan}; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::{LocalityError, LocalityResult}; use crate::client::{GoogleDocsApi, GoogleDriveApi, HttpGoogleApiClient}; @@ -263,7 +264,7 @@ impl Connector for GoogleDocsConnector { projected_path, ) .deleted(file.trashed) - .with_raw_metadata_json(serde_json::to_string(&file).unwrap_or_else(|_| "{}".to_string())); + .with_raw_metadata_json(drive_file_metadata_json(&file)); if let Some(parent) = file.parents.first() { observation = observation.with_parent(RemoteId::new(parent.clone())); } @@ -385,6 +386,52 @@ fn operation_targets_document(block_id: &RemoteId, remote_id: &RemoteId) -> bool .unwrap_or(false) } +fn drive_file_metadata_json(file: &DriveFile) -> String { + let mut value = serde_json::to_value(file).unwrap_or_else(|_| serde_json::json!({})); + if let serde_json::Value::Object(object) = &mut value { + let search_metadata = drive_file_search_metadata(file); + if !search_metadata.is_empty() + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn drive_file_search_metadata(file: &DriveFile) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(&file.id); + metadata.push_metadata_text(&file.name); + metadata.push_metadata_text(&file.mime_type); + metadata.push_alias(&file.id); + if file.is_google_doc() { + metadata.push_metadata_text("Google Docs"); + metadata.set_source_url(format!( + "https://docs.google.com/document/d/{}/edit", + file.id + )); + } else if file.is_folder() { + metadata.push_metadata_text("Google Drive folder"); + metadata.set_source_url(format!( + "https://drive.google.com/drive/folders/{}", + file.id + )); + } else { + metadata.set_source_url(format!("https://drive.google.com/file/d/{}/view", file.id)); + } + for parent in &file.parents { + metadata.push_metadata_text(parent); + } + if let Some(modified_time) = &file.modified_time { + metadata.push_metadata_text(modified_time); + } + if let Some(version) = &file.version { + metadata.push_metadata_text(version); + } + metadata +} + impl GoogleDocsConnector { fn workspace_folder_id(&self) -> LocalityResult<&RemoteId> { self.config.workspace_folder_id.as_ref().ok_or_else(|| { @@ -3064,6 +3111,7 @@ mod tests { use locality_core::model::{EntityKind, MountId, RemoteId}; use locality_core::planner::{PushOperation, PushPlan}; use locality_core::push::RemotePrecondition; + use locality_core::search::RAW_SEARCH_METADATA_KEY; use super::{ GoogleDocsConfig, GoogleDocsConnector, docs_block_text, docs_document_text_requests, @@ -3181,6 +3229,21 @@ mod tests { observation.remote_version.unwrap().as_str(), "drive:7:2026-06-25T10:00:00.000Z|docs:rev-1" ); + let raw_metadata: serde_json::Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + serde_json::json!("https://docs.google.com/document/d/doc-1/edit") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + serde_json::json!(["doc-1"]) + ); + let search_terms = raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text"); + assert!(search_terms.contains(&serde_json::json!("Launch Brief"))); + assert!(search_terms.contains(&serde_json::json!("Google Docs"))); } #[test] diff --git a/crates/locality-granola/src/connector.rs b/crates/locality-granola/src/connector.rs index 68163f78..b0166051 100644 --- a/crates/locality-granola/src/connector.rs +++ b/crates/locality-granola/src/connector.rs @@ -15,6 +15,7 @@ use locality_core::model::{ CanonicalDocument, EntityKind, HydrationState, MountId, RemoteId, TreeEntry, }; use locality_core::planner::PushOperationKind; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::{LocalityError, LocalityResult}; use crate::client::{GranolaApi, HttpGranolaApiClient}; @@ -187,7 +188,12 @@ impl Connector for GranolaConnector { let parent_path = PathBuf::from(meeting_directory_name(&GranolaNoteSummary::from(¬e))); let entry = content_entry(&request.mount_id, &parent_path, ¬e, kind); - return Ok(observation_from_entry(entry, Some(RemoteId::new(note.id)))); + let parent = RemoteId::new(note.id.clone()); + return Ok(observation_from_entry( + entry, + Some(parent), + Some(granola_note_metadata_json(¬e, Some(kind))), + )); } let note = self.api.get_note(request.remote_id.as_str(), false)?; let entry = meeting_entry( @@ -195,7 +201,11 @@ impl Connector for GranolaConnector { Path::new(""), &GranolaNoteSummary::from(¬e), ); - Ok(observation_from_entry(entry, None)) + Ok(observation_from_entry( + entry, + None, + Some(granola_note_metadata_json(¬e, None)), + )) } fn fetch(&self, request: FetchRequest) -> LocalityResult { @@ -303,7 +313,11 @@ fn content_entry( } } -fn observation_from_entry(entry: TreeEntry, parent: Option) -> RemoteObservation { +fn observation_from_entry( + entry: TreeEntry, + parent: Option, + raw_metadata_json: Option, +) -> RemoteObservation { let mut observation = RemoteObservation::new( entry.mount_id, entry.remote_id, @@ -317,9 +331,81 @@ fn observation_from_entry(entry: TreeEntry, parent: Option) -> RemoteO if let Some(version) = entry.remote_edited_at { observation = observation.with_remote_version(RemoteVersion::new(version)); } + if let Some(raw_metadata_json) = raw_metadata_json { + observation = observation.with_raw_metadata_json(raw_metadata_json); + } observation } +fn granola_note_metadata_json( + note: &GranolaNote, + content_kind: Option, +) -> String { + let mut value = serde_json::to_value(note).unwrap_or_else(|_| serde_json::json!({})); + if let serde_json::Value::Object(object) = &mut value { + let search_metadata = granola_note_search_metadata(note, content_kind); + if !search_metadata.is_empty() + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn granola_note_search_metadata( + note: &GranolaNote, + content_kind: Option, +) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(¬e.id); + metadata.push_alias(¬e.id); + metadata.push_metadata_text(note_title(note)); + metadata.push_metadata_text(¬e.object); + if let Some(kind) = content_kind { + metadata.push_metadata_text(kind.as_str()); + } + metadata.push_metadata_text(¬e.created_at); + metadata.push_metadata_text(¬e.updated_at); + metadata.push_metadata_text(¬e.owner.email); + if let Some(name) = ¬e.owner.name { + metadata.push_metadata_text(name); + } + for attendee in ¬e.attendees { + metadata.push_metadata_text(&attendee.email); + if let Some(name) = &attendee.name { + metadata.push_metadata_text(name); + } + } + for folder in ¬e.folder_membership { + metadata.push_metadata_text(&folder.id); + metadata.push_metadata_text(&folder.name); + } + if let Some(calendar_event) = ¬e.calendar_event { + if let Some(event_title) = &calendar_event.event_title { + metadata.push_metadata_text(event_title); + } + if let Some(organiser) = &calendar_event.organiser { + metadata.push_metadata_text(organiser); + } + if let Some(calendar_event_id) = &calendar_event.calendar_event_id { + metadata.push_metadata_text(calendar_event_id); + metadata.push_alias(calendar_event_id); + } + if let Some(start) = &calendar_event.scheduled_start_time { + metadata.push_metadata_text(start); + } + if let Some(end) = &calendar_event.scheduled_end_time { + metadata.push_metadata_text(end); + } + for invitee in &calendar_event.invitees { + metadata.push_metadata_text(&invitee.email); + } + } + metadata.set_source_url(note.web_url.clone()); + metadata +} + pub fn meeting_directory_name(note: &GranolaNoteSummary) -> String { let timestamp = DateTime::parse_from_rfc3339(¬e.created_at) .map(|value| { @@ -401,11 +487,17 @@ mod tests { use std::path::PathBuf; use std::sync::{Arc, Mutex}; - use locality_connector::{ChildContainer, Connector, EnumerateRequest, ListChildrenRequest}; - use locality_core::model::{EntityKind, MountId}; + use locality_connector::{ + ChildContainer, Connector, EnumerateRequest, ListChildrenRequest, ObserveRequest, + }; + use locality_core::model::{EntityKind, MountId, RemoteId}; + use locality_core::search::RAW_SEARCH_METADATA_KEY; use crate::client::GranolaApi; - use crate::dto::{GranolaNote, GranolaNoteList, GranolaNoteSummary, GranolaUser}; + use crate::dto::{ + GranolaCalendarEvent, GranolaFolder, GranolaNote, GranolaNoteList, GranolaNoteSummary, + GranolaUser, + }; use super::{GranolaConfig, GranolaConnector}; @@ -545,6 +637,36 @@ mod tests { ); } + #[test] + fn observe_adds_search_metadata_from_note_fields() { + let api = Arc::new(FakeApi::default().with_note(note())); + let connector = GranolaConnector::with_api(GranolaConfig::new("secret"), api); + + let observation = connector + .observe(ObserveRequest { + mount_id: MountId::new("granola-main"), + remote_id: RemoteId::new("not_1d3tmYTlCICgjy"), + }) + .expect("observe note"); + + let raw_metadata: serde_json::Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + serde_json::json!("https://app.granola.ai/notes/not_1d3tmYTlCICgjy") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + serde_json::json!(["not_1d3tmYTlCICgjy", "cal-1"]) + ); + let search_terms = raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text"); + assert!(search_terms.contains(&serde_json::json!("Customer call"))); + assert!(search_terms.contains(&serde_json::json!("Ada Lovelace"))); + assert!(search_terms.contains(&serde_json::json!("Deals"))); + } + #[derive(Debug, Default)] struct FakeApi { notes: Vec, @@ -560,6 +682,11 @@ mod tests { updated_after: Mutex::new(Vec::new()), } } + + fn with_note(self, note: GranolaNote) -> Self { + *self.note.lock().unwrap() = Some(note); + self + } } impl GranolaApi for FakeApi { @@ -609,4 +736,40 @@ mod tests { updated_at: created_at.to_string(), } } + + fn note() -> GranolaNote { + GranolaNote { + id: "not_1d3tmYTlCICgjy".to_string(), + object: "note".to_string(), + title: Some("Customer call".to_string()), + owner: GranolaUser { + name: Some("Ada Lovelace".to_string()), + email: "ada@example.com".to_string(), + }, + created_at: "2026-07-14T17:30:00Z".to_string(), + updated_at: "2026-07-14T18:00:00Z".to_string(), + web_url: "https://app.granola.ai/notes/not_1d3tmYTlCICgjy".to_string(), + calendar_event: Some(GranolaCalendarEvent { + event_title: Some("Customer discovery".to_string()), + invitees: Vec::new(), + organiser: Some("ada@example.com".to_string()), + calendar_event_id: Some("cal-1".to_string()), + scheduled_start_time: Some("2026-07-14T17:30:00Z".to_string()), + scheduled_end_time: Some("2026-07-14T18:00:00Z".to_string()), + }), + attendees: vec![GranolaUser { + name: Some("Grace Hopper".to_string()), + email: "grace@example.com".to_string(), + }], + folder_membership: vec![GranolaFolder { + id: "folder-1".to_string(), + object: "folder".to_string(), + name: "Deals".to_string(), + parent_folder_id: None, + }], + summary_text: "Summary".to_string(), + summary_markdown: Some("Summary".to_string()), + transcript: None, + } + } } diff --git a/crates/locality-linear/src/connector.rs b/crates/locality-linear/src/connector.rs index 89451a12..865c0bcb 100644 --- a/crates/locality-linear/src/connector.rs +++ b/crates/locality-linear/src/connector.rs @@ -15,6 +15,7 @@ use locality_core::model::{ CanonicalDocument, EntityKind, HydrationState, MountId, RemoteId, TreeEntry, }; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind}; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::{LocalityError, LocalityResult}; use crate::client::{HttpLinearApiClient, LinearApi}; @@ -562,7 +563,62 @@ fn observation_from_issue(mount_id: &MountId, issue: LinearIssue) -> RemoteObser ))) .with_remote_version(RemoteVersion::new(remote_version(&issue))) .deleted(issue.archived_at.is_some()) - .with_raw_metadata_json(serde_json::to_string(&issue).unwrap_or_else(|_| "{}".to_string())) + .with_raw_metadata_json(linear_issue_metadata_json(&issue)) +} + +fn linear_issue_metadata_json(issue: &LinearIssue) -> String { + let mut value = serde_json::to_value(issue).unwrap_or_else(|_| serde_json::json!({})); + if let serde_json::Value::Object(object) = &mut value { + let search_metadata = linear_issue_search_metadata(issue); + if !search_metadata.is_empty() { + if let Ok(search_value) = serde_json::to_value(search_metadata) { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + } + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn linear_issue_search_metadata(issue: &LinearIssue) -> SearchMetadata { + let mut metadata_text = Vec::new(); + push_search_value(&mut metadata_text, &issue.identifier); + push_search_value(&mut metadata_text, &issue.team.key); + push_search_value(&mut metadata_text, &issue.team.name); + push_search_value(&mut metadata_text, &issue.state.name); + if let Some(project) = &issue.project { + push_search_value(&mut metadata_text, &project.name); + } + if let Some(assignee) = &issue.assignee { + push_search_value(&mut metadata_text, &assignee.name); + if let Some(email) = &assignee.email { + push_search_value(&mut metadata_text, email); + } + } + for label in &issue.labels { + push_search_value(&mut metadata_text, &label.name); + } + if let Some(priority) = &issue.priority { + push_search_value(&mut metadata_text, &priority.label); + } + if let Some(due_date) = &issue.due_date { + push_search_value(&mut metadata_text, due_date); + } + + let mut aliases = Vec::new(); + push_search_value(&mut aliases, &issue.identifier); + + SearchMetadata { + metadata_text, + aliases, + source_url: (!issue.url.trim().is_empty()).then(|| issue.url.clone()), + } +} + +fn push_search_value(values: &mut Vec, value: &str) { + let value = value.trim(); + if !value.is_empty() { + values.push(value.to_string()); + } } fn list_linear_directory_children( diff --git a/crates/locality-linear/tests/connector.rs b/crates/locality-linear/tests/connector.rs index 972d17c2..4cfbf819 100644 --- a/crates/locality-linear/tests/connector.rs +++ b/crates/locality-linear/tests/connector.rs @@ -8,6 +8,7 @@ use locality_core::journal::{JournalApplyEffect, PushId, PushOperationId}; use locality_core::model::{EntityKind, MountId, RemoteId}; use locality_core::planner::{PropertyValue, PushOperation, PushOperationKind, PushPlan}; use locality_core::push::RemotePrecondition; +use locality_core::search::RAW_SEARCH_METADATA_KEY; use locality_linear::{ LinearApi, LinearConfig, LinearConnector, LinearIssue, LinearIssuePage, LinearIssuePriority, LinearIssueState, LinearIssueUpdateInput, LinearLabel, LinearProject, LinearTeam, LinearUser, @@ -178,6 +179,34 @@ fn observe_and_observe_batch_use_hierarchical_issue_path() { observed.parent_remote_id, Some(RemoteId::new("team-state:team-1:state-1")) ); + let raw_metadata = serde_json::from_str::(&observed.raw_metadata_json) + .expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + serde_json::json!("https://linear.app/acme/issue/ENG-1/improve-sync") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + serde_json::json!(["ENG-1"]) + ); + let search_terms = raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text array"); + for expected in [ + "ENG", + "Engineering", + "Todo", + "Launch", + "Ada", + "ada@example.com", + "Bug", + "High", + ] { + assert!( + search_terms.contains(&serde_json::json!(expected)), + "missing search term {expected}" + ); + } let batch = connector .observe_batch(BatchObserveRequest { diff --git a/crates/locality-notion/src/projection.rs b/crates/locality-notion/src/projection.rs index 15f14251..f57dde90 100644 --- a/crates/locality-notion/src/projection.rs +++ b/crates/locality-notion/src/projection.rs @@ -11,11 +11,14 @@ use locality_connector::ChildContainer; use locality_core::freshness::{RemoteObservation, RemoteVersion}; use locality_core::model::{EntityKind, HydrationState, MountId, RemoteId, TreeEntry}; use locality_core::path_projection::{page_container_path, page_document_path}; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::{LocalityError, LocalityResult}; use crate::client::NotionApi; -use crate::dto::{BlockDto, DatabaseDto, PageDto, ParentDto}; -use crate::render::{page_frontmatter, page_title}; +use crate::dto::{ + BlockDto, DatabaseDto, DateMentionDto, PageDto, PagePropertyDto, ParentDto, UserMentionDto, +}; +use crate::render::{page_frontmatter, page_title, rich_text_plain_text}; pub fn enumerate_root_page_tree( api: &dyn NotionApi, @@ -876,7 +879,7 @@ fn page_observation(mount_id: MountId, page: &PageDto) -> RemoteObservation { path, ) .deleted(page.archived || page.in_trash) - .with_raw_metadata_json(metadata_json(page)); + .with_raw_metadata_json(metadata_json(page, page_search_metadata(page))); if let Some(parent_id) = parent_remote_id(page.parent.as_ref()) { observation = observation.with_parent(parent_id); @@ -919,7 +922,7 @@ fn database_observation(mount_id: MountId, database: &DatabaseDto) -> RemoteObse path, ) .deleted(database.archived || database.in_trash) - .with_raw_metadata_json(metadata_json(database)); + .with_raw_metadata_json(metadata_json(database, database_search_metadata(database))); if let Some(parent_id) = parent_remote_id(database.parent.as_ref()) { observation = observation.with_parent(parent_id); @@ -942,15 +945,156 @@ fn parent_remote_id(parent: Option<&ParentDto>) -> Option { .map(|id| RemoteId::new(id.clone())) } -fn metadata_json(value: &T) -> String +fn metadata_json(value: &T, search_metadata: SearchMetadata) -> String where T: serde::Serialize, { - serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()) + let mut value = serde_json::to_value(value).unwrap_or_else(|_| serde_json::json!({})); + if let serde_json::Value::Object(object) = &mut value + && !search_metadata.is_empty() + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string()) +} + +fn page_search_metadata(page: &PageDto) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(page_title(page)); + metadata.push_metadata_text(&page.id); + metadata.push_alias(&page.id); + let compact_id = compact_notion_id(&page.id); + if compact_id != page.id { + metadata.push_alias(&compact_id); + } + metadata.set_source_url(notion_source_url(&page.id)); + for (name, property) in &page.properties { + metadata.push_metadata_text(name); + push_page_property_search_values(&mut metadata, property); + } + metadata +} + +fn database_search_metadata(database: &DatabaseDto) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + if let Some(title) = database_title(database) { + metadata.push_metadata_text(title); + } + metadata.push_metadata_text(&database.id); + metadata.push_alias(&database.id); + let compact_id = compact_notion_id(&database.id); + if compact_id != database.id { + metadata.push_alias(&compact_id); + } + metadata.set_source_url(notion_source_url(&database.id)); + for data_source in &database.data_sources { + metadata.push_metadata_text(&data_source.id); + if let Some(name) = &data_source.name { + metadata.push_metadata_text(name); + } + } + metadata +} + +fn push_page_property_search_values(metadata: &mut SearchMetadata, property: &PagePropertyDto) { + metadata.push_metadata_text(rich_text_plain_text(&property.title)); + metadata.push_metadata_text(rich_text_plain_text(&property.rich_text)); + if let Some(number) = &property.number { + metadata.push_metadata_text(number.to_string()); + } + if let Some(select) = &property.select { + metadata.push_metadata_text(&select.name); + } + for option in &property.multi_select { + metadata.push_metadata_text(&option.name); + } + if let Some(status) = &property.status { + metadata.push_metadata_text(&status.name); + } + if let Some(checked) = property.checkbox { + metadata.push_metadata_text(if checked { "checked" } else { "unchecked" }); + } + if let Some(date) = &property.date { + push_date_search_values(metadata, date); + } + if let Some(url) = &property.url { + metadata.push_metadata_text(url); + } + if let Some(email) = &property.email { + metadata.push_metadata_text(email); + } + if let Some(phone_number) = &property.phone_number { + metadata.push_metadata_text(phone_number); + } + for file in &property.files { + if let Some(name) = &file.name { + metadata.push_metadata_text(name); + } + } + for person in &property.people { + push_user_search_values(metadata, person); + } + for relation in &property.relation { + metadata.push_metadata_text(&relation.id); + } + if let Some(created_time) = &property.created_time { + metadata.push_metadata_text(created_time); + } + if let Some(last_edited_time) = &property.last_edited_time { + metadata.push_metadata_text(last_edited_time); + } + if let Some(created_by) = &property.created_by { + push_user_search_values(metadata, created_by); + } + if let Some(last_edited_by) = &property.last_edited_by { + push_user_search_values(metadata, last_edited_by); + } + if let Some(unique_id) = &property.unique_id { + match (&unique_id.prefix, unique_id.number) { + (Some(prefix), Some(number)) => { + metadata.push_metadata_text(format!("{prefix}{number}")) + } + (None, Some(number)) => metadata.push_metadata_text(number.to_string()), + _ => {} + } + } + if let Some(verification) = &property.verification { + if let Some(state) = &verification.state { + metadata.push_metadata_text(state); + } + if let Some(verified_by) = &verification.verified_by { + push_user_search_values(metadata, verified_by); + } + if let Some(date) = &verification.date { + push_date_search_values(metadata, date); + } + } +} + +fn push_user_search_values(metadata: &mut SearchMetadata, user: &UserMentionDto) { + metadata.push_metadata_text(&user.id); + if let Some(name) = &user.name { + metadata.push_metadata_text(name); + } +} + +fn push_date_search_values(metadata: &mut SearchMetadata, date: &DateMentionDto) { + metadata.push_metadata_text(&date.start); + if let Some(end) = &date.end { + metadata.push_metadata_text(end); + } + if let Some(time_zone) = &date.time_zone { + metadata.push_metadata_text(time_zone); + } +} + +fn notion_source_url(id: &str) -> String { + format!("https://www.notion.so/{}", compact_notion_id(id)) } fn database_title(database: &DatabaseDto) -> Option { - let title = crate::render::rich_text_plain_text(&database.title); + let title = rich_text_plain_text(&database.title); if title.trim().is_empty() { None } else { diff --git a/crates/locality-notion/tests/observe.rs b/crates/locality-notion/tests/observe.rs index fbfbd29d..b47c0153 100644 --- a/crates/locality-notion/tests/observe.rs +++ b/crates/locality-notion/tests/observe.rs @@ -6,6 +6,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use locality_connector::{Connector, ObserveRequest}; use locality_core::freshness::RemoteVersion; use locality_core::model::{EntityKind, MountId, RemoteId}; +use locality_core::search::RAW_SEARCH_METADATA_KEY; use locality_core::{LocalityError, LocalityResult}; use locality_notion::client::{HttpNotionApi, NotionApi}; use locality_notion::dto::{ @@ -52,6 +53,22 @@ fn notion_observe_page_reads_metadata_without_hydrating_blocks() { Some(RemoteVersion::new("2026-06-10T00:00:00.000Z")) ); assert!(!observation.deleted); + let raw_metadata: Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + json!("https://www.notion.so/ae1") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + json!(["page-1", "ae1"]) + ); + assert!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text") + .contains(&json!("Roadmap")) + ); assert_eq!(api.block_children_calls.load(Ordering::SeqCst), 0); } @@ -77,6 +94,16 @@ fn notion_observe_database_falls_back_to_database_metadata() { observation.remote_version, Some(RemoteVersion::new("2026-06-10T00:00:00.000Z")) ); + let raw_metadata: Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["source_url"], + json!("https://www.notion.so/daabae1") + ); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + json!(["database-1", "daabae1"]) + ); } #[test] diff --git a/crates/locality-slack/src/connector.rs b/crates/locality-slack/src/connector.rs index e35aead1..1c226d2e 100644 --- a/crates/locality-slack/src/connector.rs +++ b/crates/locality-slack/src/connector.rs @@ -14,6 +14,7 @@ use locality_core::model::{ CanonicalDocument, EntityKind, HydrationState, MountId, RemoteId, TreeEntry, }; use locality_core::planner::PushOperationKind; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::{LocalityError, LocalityResult}; use crate::client::{HttpSlackApiClient, SlackApi}; @@ -340,7 +341,8 @@ impl Connector for SlackConnector { fn observe(&self, request: ObserveRequest) -> LocalityResult { let remote_id_value = request.remote_id.as_str().to_string(); if remote_id_value == users_remote_id() { - let bundle = users_bundle(self.all_users()?); + let users = self.all_users()?; + let bundle = users_bundle(users.clone()); let version = slack_remote_version(&bundle)?; return Ok(RemoteObservation::new( request.mount_id, @@ -350,7 +352,7 @@ impl Connector for SlackConnector { "users.md", ) .with_remote_version(RemoteVersion::new(version)) - .with_raw_metadata_json(serde_json::json!({ "kind": "slack_users" }).to_string())); + .with_raw_metadata_json(slack_users_metadata_json(&users))); } if let Some(conversation_id) = parse_recent_remote_id(&remote_id_value) { @@ -372,6 +374,8 @@ impl Connector for SlackConnector { None, self.config.settings.slack.history_limit, )?; + let raw_metadata_json = + slack_recent_metadata_json(&conversation, &users_by_id, conversation_id); let bundle = recent_bundle(conversation, users, history.messages, BTreeMap::new()); let version = slack_remote_version(&bundle)?; return Ok(RemoteObservation::new( @@ -383,13 +387,7 @@ impl Connector for SlackConnector { ) .with_parent(conversation_entry.remote_id) .with_remote_version(RemoteVersion::new(version)) - .with_raw_metadata_json( - serde_json::json!({ - "kind": "slack_recent", - "conversation_id": conversation_id, - }) - .to_string(), - )); + .with_raw_metadata_json(raw_metadata_json)); } Err(LocalityError::Unsupported( @@ -487,6 +485,93 @@ fn recent_bundle( } } +fn slack_users_metadata_json(users: &[SlackUser]) -> String { + let mut search_metadata = SearchMetadata::default(); + search_metadata.push_metadata_text("users"); + search_metadata.push_alias(users_remote_id()); + for user in users { + push_slack_user_search_values(&mut search_metadata, user); + } + let mut value = serde_json::json!({ + "kind": "slack_users", + }); + if let serde_json::Value::Object(object) = &mut value + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + value.to_string() +} + +fn slack_recent_metadata_json( + conversation: &SlackConversation, + users: &BTreeMap, + conversation_id: &str, +) -> String { + let search_metadata = slack_conversation_search_metadata(conversation, users, conversation_id); + let mut value = serde_json::json!({ + "kind": "slack_recent", + "conversation_id": conversation_id, + }); + if let serde_json::Value::Object(object) = &mut value + && let Ok(search_value) = serde_json::to_value(search_metadata) + { + object.insert(RAW_SEARCH_METADATA_KEY.to_string(), search_value); + } + value.to_string() +} + +fn slack_conversation_search_metadata( + conversation: &SlackConversation, + users: &BTreeMap, + conversation_id: &str, +) -> SearchMetadata { + let mut metadata = SearchMetadata::default(); + metadata.push_metadata_text(conversation_id); + metadata.push_alias(conversation_id); + metadata.push_alias(recent_remote_id(conversation_id)); + metadata.push_metadata_text(conversation_title(conversation, users)); + metadata.push_metadata_text(conversation_type(conversation).root_folder()); + if let Some(name) = &conversation.name { + metadata.push_metadata_text(name); + } + if let Some(user_id) = &conversation.user { + metadata.push_metadata_text(user_id); + if let Some(user) = users.get(user_id) { + push_slack_user_search_values(&mut metadata, user); + } + } + if let Some(topic) = &conversation.topic + && let Some(value) = &topic.value + { + metadata.push_metadata_text(value); + } + if let Some(purpose) = &conversation.purpose + && let Some(value) = &purpose.value + { + metadata.push_metadata_text(value); + } + metadata +} + +fn push_slack_user_search_values(metadata: &mut SearchMetadata, user: &SlackUser) { + metadata.push_metadata_text(&user.id); + if let Some(name) = &user.name { + metadata.push_metadata_text(name); + } + if let Some(real_name) = &user.real_name { + metadata.push_metadata_text(real_name); + } + if let Some(profile) = &user.profile { + if let Some(real_name) = &profile.real_name { + metadata.push_metadata_text(real_name); + } + if let Some(display_name) = &profile.display_name { + metadata.push_metadata_text(display_name); + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] struct RootEntrySpec { remote_id: &'static str, @@ -988,6 +1073,17 @@ mod tests { }) .expect("observe recent"); assert_content_remote_version(observation.remote_version.expect("version").as_str()); + let raw_metadata: serde_json::Value = + serde_json::from_str(&observation.raw_metadata_json).expect("raw metadata json"); + assert_eq!( + raw_metadata[RAW_SEARCH_METADATA_KEY]["aliases"], + serde_json::json!(["C123", "slack-recent:C123"]) + ); + let search_terms = raw_metadata[RAW_SEARCH_METADATA_KEY]["metadata_text"] + .as_array() + .expect("metadata_text"); + assert!(search_terms.contains(&serde_json::json!("general"))); + assert!(search_terms.contains(&serde_json::json!("channels"))); } #[test] diff --git a/crates/locality-store/src/lib.rs b/crates/locality-store/src/lib.rs index 1f06b0cd..56c6505a 100644 --- a/crates/locality-store/src/lib.rs +++ b/crates/locality-store/src/lib.rs @@ -46,10 +46,11 @@ pub use records::{ }; pub use repository::{ AutoSaveRepository, ConnectionRepository, ConnectorProfileRepository, ConnectorStateRepository, - EntityRepository, EntitySearchCandidate, EntitySearchRepository, FreshnessStateRepository, - HydrationJobRepository, JournalRepository, MetadataDiscoveryJobRepository, - MountLiveModeRepository, MountRepository, RemoteObservationRepository, ShadowRepository, - VirtualMoveRepository, VirtualMoveTransition, VirtualMutationRepository, + EntityRepository, EntitySearchCandidate, EntitySearchDocument, EntitySearchRepository, + FreshnessStateRepository, HydrationJobRepository, JournalRepository, + MetadataDiscoveryJobRepository, MountLiveModeRepository, MountRepository, + RemoteObservationRepository, ShadowRepository, VirtualMoveRepository, VirtualMoveTransition, + VirtualMutationRepository, }; pub use reset::{ LocalStateResetCredentialError, LocalStateResetError, LocalStateResetStorageReport, diff --git a/crates/locality-store/src/repository.rs b/crates/locality-store/src/repository.rs index 164f971d..9f0ccc34 100644 --- a/crates/locality-store/src/repository.rs +++ b/crates/locality-store/src/repository.rs @@ -78,6 +78,21 @@ pub trait EntityRepository { pub struct EntitySearchCandidate { pub entity: EntityRecord, pub observation: Option, + pub search_document: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EntitySearchDocument { + pub title: Option, + pub path: Option, + pub observed_title: Option, + pub observed_path: Option, + pub frontmatter: Option, + pub body: Option, + pub metadata_text: Option, + pub breadcrumbs: Option, + pub aliases: Option, + pub source_url: Option, } pub trait EntitySearchRepository { diff --git a/crates/locality-store/src/sqlite.rs b/crates/locality-store/src/sqlite.rs index eca3eff3..3442daee 100644 --- a/crates/locality-store/src/sqlite.rs +++ b/crates/locality-store/src/sqlite.rs @@ -18,6 +18,7 @@ use locality_core::journal::{ use locality_core::model::{EntityKind, HydrationState, MountId, RemoteId}; use locality_core::planner::{PlanSummary, PushOperation, PushPlan}; use locality_core::readable_diff::ReadableDiffOutput; +use locality_core::search::{RAW_SEARCH_METADATA_KEY, SearchMetadata}; use locality_core::shadow::ShadowDocument; use rusqlite::{Connection, OpenFlags, OptionalExtension, TransactionBehavior, params}; use serde::Serialize; @@ -46,15 +47,17 @@ use crate::records::{ }; use crate::repository::{ AutoSaveRepository, ConnectionRepository, ConnectorProfileRepository, ConnectorStateRepository, - EntityRepository, EntitySearchCandidate, EntitySearchRepository, FreshnessStateRepository, - HydrationJobRepository, JournalRepository, MetadataDiscoveryJobRepository, - MountLiveModeRepository, MountRepository, RemoteObservationRepository, ShadowRepository, - VirtualMoveRepository, VirtualMoveTransition, VirtualMutationRepository, - validate_virtual_move_transition, virtual_move_content_changed, virtual_move_missing, + EntityRepository, EntitySearchCandidate, EntitySearchDocument, EntitySearchRepository, + FreshnessStateRepository, HydrationJobRepository, JournalRepository, + MetadataDiscoveryJobRepository, MountLiveModeRepository, MountRepository, + RemoteObservationRepository, ShadowRepository, VirtualMoveRepository, VirtualMoveTransition, + VirtualMutationRepository, validate_virtual_move_transition, virtual_move_content_changed, + virtual_move_missing, }; const DB_FILE: &str = "state.sqlite3"; -const SCHEMA_VERSION: i64 = 19; +const SCHEMA_VERSION: i64 = 20; +const ENTITY_SEARCH_COMPONENT_VERSION: i64 = 2; const JOURNALS_COMPONENT_VERSION: i64 = 3; const LINUX_FUSE_PROJECTION_LAYOUT_VERSION: i64 = 2; const WINDOWS_CLOUD_FILES_PROJECTION_LAYOUT_VERSION: i64 = 2; @@ -196,11 +199,11 @@ const CURRENT_COMPONENT_DEFINITIONS: &[StateComponentDefinition] = &[ StateComponentDefinition { component_id: "cache:entity_search", component_kind: "rebuildable_cache", - current_version: 1, - min_reader_version: 1, + current_version: ENTITY_SEARCH_COMPONENT_VERSION, + min_reader_version: ENTITY_SEARCH_COMPONENT_VERSION, required: false, rebuildable: true, - data_json: "{}", + data_json: "{\"index\":\"search_documents_fts\",\"legacy_index\":\"entity_search_fts\"}", }, ]; @@ -1562,10 +1565,15 @@ impl EntitySearchRepository for SqliteStateStore { }; let mut statement = connection.prepare( "SELECT remote_id - FROM entity_search_fts - WHERE entity_search_fts MATCH ?1 + FROM search_documents_fts + WHERE search_documents_fts MATCH ?1 AND mount_id = ?2 - ORDER BY bm25(entity_search_fts) + ORDER BY bm25( + search_documents_fts, + 0.0, 0.0, 0.0, 0.0, + 8.0, 6.0, 7.0, 5.0, + 2.0, 1.0, 4.0, 3.0, 9.0, 6.0 + ) LIMIT ?3", )?; let rows = statement.query_map( @@ -1579,9 +1587,11 @@ impl EntitySearchRepository for SqliteStateStore { for remote_id in remote_ids { let remote_id = RemoteId(remote_id); if let Some(entity) = self.get_entity(mount_id, &remote_id)? { + let search_document = search_document(&connection, mount_id, &remote_id)?; candidates.push(EntitySearchCandidate { entity, observation: self.get_remote_observation(mount_id, &remote_id)?, + search_document, }); } } @@ -1786,14 +1796,15 @@ impl ShadowRepository for SqliteStateStore { rendered_body = excluded.rendered_body, blocks_json = excluded.blocks_json", params![ - record.mount_id.0, - record.entity_id.0, - record.frontmatter, - record.body_hash, - record.rendered_body, + record.mount_id.0.as_str(), + record.entity_id.0.as_str(), + record.frontmatter.as_str(), + record.body_hash.as_str(), + record.rendered_body.as_str(), to_json(&record.blocks)?, ], )?; + upsert_entity_search_index(&connection, mount_id, &shadow.entity_id)?; Ok(()) } @@ -2532,6 +2543,7 @@ fn clear_mount_source_state(connection: &Connection, mount_id: &MountId) -> Stor "freshness_states", "journals", "entity_search_fts", + "search_documents_fts", ] { connection.execute( &format!("DELETE FROM {table} WHERE mount_id = ?1"), @@ -2737,6 +2749,7 @@ fn initialize_schema(connection: &Connection) -> StoreResult<()> { migrate_windows_cloud_files_projection_layout_to_v2(connection, false)?; migrate_journals_component_to_v3(connection)?; migrate_virtual_mutations_component_to_v3(connection)?; + migrate_entity_search_component_to_v2(connection)?; return Ok(()); } @@ -2974,6 +2987,23 @@ fn initialize_schema(connection: &Connection) -> StoreResult<()> { observed_path ); + CREATE VIRTUAL TABLE IF NOT EXISTS search_documents_fts USING fts5( + mount_id UNINDEXED, + remote_id UNINDEXED, + connector UNINDEXED, + kind UNINDEXED, + title, + path, + observed_title, + observed_path, + frontmatter, + body, + metadata_text, + breadcrumbs, + aliases, + source_url + ); + CREATE TABLE IF NOT EXISTS journals ( push_id TEXT PRIMARY KEY, mount_id TEXT NOT NULL, @@ -3244,6 +3274,14 @@ fn initialize_schema(connection: &Connection) -> StoreResult<()> { record_schema_migration(connection, user_version, SCHEMA_VERSION)?; } + if user_version < 20 { + create_entity_search_index(connection)?; + rebuild_entity_search_index(connection)?; + if user_version >= 13 { + record_schema_migration(connection, user_version, SCHEMA_VERSION)?; + } + } + if user_version < SCHEMA_VERSION { seed_default_notion_profile(connection)?; migrate_linux_fuse_projection_layout_to_v2(connection, user_version < 13)?; @@ -3339,6 +3377,13 @@ fn state_component_issue_allows_schema_migration( found, current: 3, } if component_id == "durable:virtual_mutations" && matches!(*found, 1 | 2) + ) || matches!( + issue, + StateCompatibilityIssue::OlderComponent { + component_id, + found: 1, + current: ENTITY_SEARCH_COMPONENT_VERSION, + } if component_id == "cache:entity_search" ) || matches!( issue, StateCompatibilityIssue::MissingComponent { component_id } @@ -4096,6 +4141,29 @@ fn migrate_journals_component_to_v3(connection: &Connection) -> StoreResult<()> migrate_state_component_to_current(connection, "durable:journals") } +fn migrate_entity_search_component_to_v2(connection: &Connection) -> StoreResult<()> { + create_state_management_tables(connection)?; + let component = connection + .query_row( + "SELECT version, min_reader_version + FROM state_components + WHERE component_id = 'cache:entity_search'", + [], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + ) + .optional()?; + if let Some((component_version, min_reader_version)) = component + && component_version >= ENTITY_SEARCH_COMPONENT_VERSION + && min_reader_version <= ENTITY_SEARCH_COMPONENT_VERSION + { + return Ok(()); + } + + create_entity_search_index(connection)?; + rebuild_entity_search_index(connection)?; + migrate_state_component_to_current(connection, "cache:entity_search") +} + fn migrate_state_component_to_current( connection: &Connection, component_id: &str, @@ -4738,6 +4806,23 @@ fn create_entity_search_index(connection: &Connection) -> StoreResult<()> { path, observed_title, observed_path + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS search_documents_fts USING fts5( + mount_id UNINDEXED, + remote_id UNINDEXED, + connector UNINDEXED, + kind UNINDEXED, + title, + path, + observed_title, + observed_path, + frontmatter, + body, + metadata_text, + breadcrumbs, + aliases, + source_url );", )?; Ok(()) @@ -4746,6 +4831,7 @@ fn create_entity_search_index(connection: &Connection) -> StoreResult<()> { fn rebuild_entity_search_index(connection: &Connection) -> StoreResult<()> { create_entity_search_index(connection)?; connection.execute("DELETE FROM entity_search_fts", [])?; + connection.execute("DELETE FROM search_documents_fts", [])?; let entity_ids = { let mut statement = connection.prepare("SELECT mount_id, remote_id FROM entities")?; @@ -4769,19 +4855,62 @@ fn upsert_entity_search_index( ) -> StoreResult<()> { delete_entity_search_index(connection, mount_id, remote_id)?; - let indexed: Option<(String, String, Option, Option)> = connection + let indexed: Option<( + String, + String, + String, + String, + Option, + Option, + Option, + Option, + Option, + )> = connection .query_row( - "SELECT e.title, e.path, o.title, o.projected_path + "SELECT m.connector, e.kind_json, e.title, e.path, + o.title, o.projected_path, s.frontmatter, s.rendered_body, + o.raw_metadata_json FROM entities e + INNER JOIN mounts m + ON m.mount_id = e.mount_id LEFT JOIN remote_observations o ON o.mount_id = e.mount_id AND o.remote_id = e.remote_id + LEFT JOIN shadows s + ON s.mount_id = e.mount_id AND s.entity_id = e.remote_id WHERE e.mount_id = ?1 AND e.remote_id = ?2", params![mount_id.0, remote_id.0], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + )) + }, ) .optional()?; - if let Some((title, path, observed_title, observed_path)) = indexed { + if let Some(( + connector, + kind, + title, + path, + observed_title, + observed_path, + frontmatter, + body, + raw_metadata_json, + )) = indexed + { + let breadcrumbs = search_breadcrumb_text(Path::new(&path)); + let search_metadata = search_metadata_from_raw_metadata_json(raw_metadata_json.as_deref()); + let metadata_text = join_search_metadata_values(&search_metadata.metadata_text); + let aliases = join_search_metadata_values(&search_metadata.aliases); connection.execute( "INSERT INTO entity_search_fts ( mount_id, @@ -4793,12 +4922,47 @@ fn upsert_entity_search_index( ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![ - mount_id.0, - remote_id.0, + mount_id.0.as_str(), + remote_id.0.as_str(), + title.as_str(), + path.as_str(), + observed_title.as_deref(), + observed_path.as_deref(), + ], + )?; + connection.execute( + "INSERT INTO search_documents_fts ( + mount_id, + remote_id, + connector, + kind, title, path, observed_title, observed_path, + frontmatter, + body, + metadata_text, + breadcrumbs, + aliases, + source_url + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + mount_id.0.as_str(), + remote_id.0.as_str(), + connector.as_str(), + kind.as_str(), + title.as_str(), + path.as_str(), + observed_title.as_deref(), + observed_path.as_deref(), + frontmatter.as_deref(), + body.as_deref(), + metadata_text.as_str(), + breadcrumbs.as_str(), + aliases.as_str(), + search_metadata.source_url.as_deref(), ], )?; } @@ -4816,9 +4980,111 @@ fn delete_entity_search_index( "DELETE FROM entity_search_fts WHERE mount_id = ?1 AND remote_id = ?2", params![mount_id.0, remote_id.0], )?; + connection.execute( + "DELETE FROM search_documents_fts WHERE mount_id = ?1 AND remote_id = ?2", + params![mount_id.0, remote_id.0], + )?; Ok(()) } +fn search_breadcrumb_text(path: &Path) -> String { + let mut components = path + .components() + .filter_map(|component| component.as_os_str().to_str()) + .map(str::trim) + .filter(|component| !component.is_empty()) + .collect::>(); + if components + .last() + .is_some_and(|component| component.eq_ignore_ascii_case("page.md")) + { + components.pop(); + } + components.join(" ") +} + +fn search_document( + connection: &Connection, + mount_id: &MountId, + remote_id: &RemoteId, +) -> StoreResult> { + let row = connection + .query_row( + "SELECT title, path, observed_title, observed_path, frontmatter, body, + metadata_text, breadcrumbs, aliases, source_url + FROM search_documents_fts + WHERE mount_id = ?1 AND remote_id = ?2 + LIMIT 1", + params![mount_id.0, remote_id.0], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + )) + }, + ) + .optional()?; + + Ok(row.map( + |( + title, + path, + observed_title, + observed_path, + frontmatter, + body, + metadata_text, + breadcrumbs, + aliases, + source_url, + )| { + EntitySearchDocument { + title, + path, + observed_title, + observed_path, + frontmatter, + body, + metadata_text, + breadcrumbs, + aliases, + source_url, + } + }, + )) +} + +fn search_metadata_from_raw_metadata_json(raw_metadata_json: Option<&str>) -> SearchMetadata { + let Some(raw_metadata_json) = raw_metadata_json else { + return SearchMetadata::default(); + }; + let Ok(value) = serde_json::from_str::(raw_metadata_json) else { + return SearchMetadata::default(); + }; + value + .get(RAW_SEARCH_METADATA_KEY) + .and_then(|value| serde_json::from_value::(value.clone()).ok()) + .unwrap_or_default() +} + +fn join_search_metadata_values(values: &[String]) -> String { + values + .iter() + .map(String::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .collect::>() + .join(" ") +} + fn discovery_entities( connection: &Connection, mount_id: &MountId, diff --git a/crates/locality-store/tests/sqlite.rs b/crates/locality-store/tests/sqlite.rs index b2ec5097..321ce328 100644 --- a/crates/locality-store/tests/sqlite.rs +++ b/crates/locality-store/tests/sqlite.rs @@ -49,7 +49,7 @@ fn sqlite_store_initializes_idempotently() { assert!(first.db_path.exists()); assert_eq!(first.db_path, second.db_path); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(journal_mode, "wal"); } @@ -75,8 +75,8 @@ fn sqlite_store_seeds_state_compatibility_components() { ( "cache:entity_search".to_string(), "rebuildable_cache".to_string(), - 1, - 1, + 2, + 2, 0, 1 ), @@ -96,7 +96,7 @@ fn sqlite_store_seeds_state_compatibility_components() { 1, 0 ), - ("core:schema".to_string(), "schema".to_string(), 19, 1, 1, 0), + ("core:schema".to_string(), "schema".to_string(), 20, 1, 1, 0), ( "durable:auto_save".to_string(), "durable_json".to_string(), @@ -398,12 +398,12 @@ fn sqlite_store_retires_removed_notion_workspace_roots_component() { } #[test] -fn sqlite_schema_snapshot_matches_v19_contract() { +fn sqlite_schema_snapshot_matches_v20_contract() { let fixture = SqliteFixture::new(); let store = fixture.open(); let connection = Connection::open(&store.db_path).expect("raw connection"); - assert_eq!(SqliteStateStore::current_schema_version(), 19); + assert_eq!(SqliteStateStore::current_schema_version(), 20); assert_eq!( schema_column_snapshot(&connection), "\ @@ -422,6 +422,7 @@ mount_live_modes: mount_id, enabled, state_json, last_reason, last_run_at, creat mounts: mount_id, connector, root, remote_root_id, read_only, projection_json, connection_id, settings_json projection_state: mount_id, projection, layout_version, min_reader_version, os_domain_id, root_item_id, repair_generation, state_json, updated_at remote_observations: mount_id, remote_id, kind_json, title, parent_remote_id, projected_path, remote_version_json, observed_at, deleted, raw_metadata_json +search_documents_fts: mount_id, remote_id, connector, kind, title, path, observed_title, observed_path, frontmatter, body, metadata_text, breadcrumbs, aliases, source_url shadows: mount_id, entity_id, frontmatter, body_hash, rendered_body, blocks_json state_components: component_id, component_kind, version, min_reader_version, required, rebuildable, data_json, updated_at state_migrations: migration_id, from_schema_version, to_schema_version, app_version, app_build_id, daemon_build_id, started_at, finished_at, status, error_json @@ -430,7 +431,7 @@ virtual_mutations: mount_id, local_id, mutation_kind_json, target_remote_id, par } #[test] -fn sqlite_store_migrates_v18_to_v19_without_discarding_pending_work() { +fn sqlite_store_migrates_v18_to_v20_without_discarding_pending_work() { let fixture = SqliteFixture::new(); let mut store = fixture.open(); store @@ -457,7 +458,7 @@ fn sqlite_store_migrates_v18_to_v19_without_discarding_pending_work() { let db_path = store.db_path.clone(); drop(store); - let connection = Connection::open(&db_path).expect("raw v19 connection"); + let connection = Connection::open(&db_path).expect("raw downgraded connection"); connection .execute_batch( "DROP TRIGGER discovery_projection_block_active_mount_delete; @@ -496,13 +497,13 @@ fn sqlite_store_migrates_v18_to_v19_without_discarding_pending_work() { let migration_count: i64 = connection .query_row( "SELECT COUNT(*) FROM state_migrations - WHERE migration_id = 'schema-18-to-19'", + WHERE migration_id = 'schema-18-to-20'", [], |row| row.get(0), ) .expect("migration history"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(component, ("durable_transaction".to_string(), 1, 1, 1, 0)); assert_eq!(migration_count, 1); assert!(sqlite_table_exists( @@ -547,7 +548,7 @@ fn sqlite_store_reports_v12_state_as_migratable_then_migrates() { before.issues, vec![StateCompatibilityIssue::OlderSchema { found: 12, - current: 19, + current: 20, }] ); @@ -558,13 +559,13 @@ fn sqlite_store_reports_v12_state_as_migratable_then_migrates() { .expect("user version"); let migration_count: i64 = connection .query_row( - "SELECT COUNT(*) FROM state_migrations WHERE migration_id = 'schema-12-to-19'", + "SELECT COUNT(*) FROM state_migrations WHERE migration_id = 'schema-12-to-20'", [], |row| row.get(0), ) .expect("migration row count"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(migration_count, 1); assert_eq!( store.get_mount(&fixture.mount_id).expect("get mount"), @@ -579,7 +580,7 @@ fn sqlite_store_reports_v12_state_as_migratable_then_migrates() { ); let after = - SqliteStateStore::inspect_compatibility(fixture.state_root.clone()).expect("inspect v19"); + SqliteStateStore::inspect_compatibility(fixture.state_root.clone()).expect("inspect v20"); assert_eq!(after.status, StateCompatibilityStatus::Ready); } @@ -599,7 +600,7 @@ fn sqlite_store_rejects_newer_schema_version() { error, StoreError::SchemaVersion { found: 999, - supported: 19, + supported: 20, } ); } @@ -622,7 +623,7 @@ fn sqlite_store_reports_newer_schema_as_needing_update() { report.issues, vec![StateCompatibilityIssue::NewerSchema { found: 999, - supported: 19, + supported: 20, }] ); } @@ -864,7 +865,7 @@ fn sqlite_store_migrates_journals_component_v2_to_v3_without_rewriting_rows() { .expect("journal"); assert_eq!((version, min_reader_version), (3, 3)); - assert_eq!(before_user_version, 19); + assert_eq!(before_user_version, 20); assert_eq!(after_user_version, before_user_version); assert_eq!(after_row, before_row); assert_eq!( @@ -921,7 +922,7 @@ fn sqlite_store_migrates_journals_component_v1_to_v3_at_current_schema() { .expect("user version"); assert_eq!(component, (3, 3)); - assert_eq!(before_user_version, 19); + assert_eq!(before_user_version, 20); assert_eq!(after_user_version, before_user_version); } @@ -1259,7 +1260,7 @@ fn sqlite_store_v13_valid_linux_fuse_v1_component_migrates_to_v2() { } #[test] -fn sqlite_store_v14_missing_live_mode_component_migrates_to_v19() { +fn sqlite_store_v14_missing_live_mode_component_migrates_to_v20() { let fixture = SqliteFixture::new(); let mount_point_root = fixture .state_root @@ -1306,7 +1307,7 @@ fn sqlite_store_v14_missing_live_mode_component_migrates_to_v19() { ) .expect("live mode component version"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert!(sqlite_table_exists(&connection, "mount_live_modes")); assert_eq!(component_version, 1); assert_eq!(query_state_migration_count(&connection), 1); @@ -1992,14 +1993,19 @@ fn entity_search_candidates_use_sqlite_index() { assert_eq!(numeric_matches[0].entity.remote_id, RemoteId::new("page-2")); store - .save_remote_observation(RemoteObservationRecord::new( - fixture.mount_id.clone(), - RemoteId::new("page-1"), - EntityKind::Page, - "Launch Plan", - "Planning/Launch Plan.md", - "2026-06-16T00:00:00Z", - )) + .save_remote_observation( + RemoteObservationRecord::new( + fixture.mount_id.clone(), + RemoteId::new("page-1"), + EntityKind::Page, + "Launch Plan", + "Planning/Launch Plan.md", + "2026-06-16T00:00:00Z", + ) + .with_raw_metadata_json( + r#"{"loc_search":{"metadata_text":["customer escalation"],"aliases":["ENG-1"],"source_url":"https://linear.app/acme/issue/ENG-1/improve-sync"}}"#, + ), + ) .expect("save observation"); let observed_matches = store .list_entity_search_candidates(&fixture.mount_id, "launch", None) @@ -2013,6 +2019,37 @@ fn entity_search_candidates_use_sqlite_index() { .map(|observation| observation.title.as_str()), Some("Launch Plan") ); + let metadata_matches = store + .list_entity_search_candidates(&fixture.mount_id, "customer escalation", None) + .expect("search connector metadata") + .expect("sqlite search"); + assert_eq!(metadata_matches.len(), 1); + assert_eq!( + metadata_matches[0].entity.remote_id, + RemoteId::new("page-1") + ); + assert!( + metadata_matches[0] + .search_document + .as_ref() + .and_then(|document| document.metadata_text.as_deref()) + .is_some_and(|text| text.contains("customer escalation")) + ); + let alias_matches = store + .list_entity_search_candidates(&fixture.mount_id, "ENG-1", None) + .expect("search connector alias") + .expect("sqlite search"); + assert_eq!(alias_matches.len(), 1); + assert_eq!(alias_matches[0].entity.remote_id, RemoteId::new("page-1")); + let source_url_matches = store + .list_entity_search_candidates(&fixture.mount_id, "improve sync", None) + .expect("search connector source url") + .expect("sqlite search"); + assert_eq!(source_url_matches.len(), 1); + assert_eq!( + source_url_matches[0].entity.remote_id, + RemoteId::new("page-1") + ); store .delete_remote_observation(&fixture.mount_id, &RemoteId::new("page-1")) @@ -2025,6 +2062,26 @@ fn entity_search_candidates_use_sqlite_index() { .is_empty() ); + store + .save_shadow( + &fixture.mount_id, + shadow_document("# Roadmap\n\nOAuth broker handoff checklist."), + ) + .expect("save searchable shadow"); + let body_matches = store + .list_entity_search_candidates(&fixture.mount_id, "broker handoff", None) + .expect("search body text") + .expect("sqlite search"); + assert_eq!(body_matches.len(), 1); + assert_eq!(body_matches[0].entity.remote_id, RemoteId::new("page-1")); + assert!( + body_matches[0] + .search_document + .as_ref() + .and_then(|document| document.body.as_deref()) + .is_some_and(|text| text.contains("OAuth broker handoff checklist.")) + ); + let id_matches = store .list_entity_search_candidates(&fixture.mount_id, "ignored", Some("page1")) .expect("search compact id") @@ -2055,6 +2112,14 @@ fn remounting_same_mount_id_to_different_connection_clears_source_scoped_state() ) .expect("save original mount"); seed_source_scoped_state(&mut store, &fixture.mount_id); + assert_eq!( + search_index_row_count(&store, &fixture.mount_id, "entity_search_fts"), + 1 + ); + assert_eq!( + search_index_row_count(&store, &fixture.mount_id, "search_documents_fts"), + 1 + ); store .save_mount( @@ -2133,6 +2198,14 @@ fn remounting_same_mount_id_to_different_connection_clears_source_scoped_state() .expect("search candidates"), Some(Vec::new()) ); + assert_eq!( + search_index_row_count(&reopened, &fixture.mount_id, "entity_search_fts"), + 0 + ); + assert_eq!( + search_index_row_count(&reopened, &fixture.mount_id, "search_documents_fts"), + 0 + ); } #[test] @@ -2673,7 +2746,7 @@ fn sqlite_store_migrates_v5_projection_and_connections_schema() { ) .expect("connections table"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(connection_column_count, 1); assert_eq!(projection_column_count, 1); assert_eq!(connection_table_count, 1); @@ -2748,7 +2821,7 @@ fn sqlite_store_migrates_v6_projection_schema_to_connections() { ) .expect("connections table"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(connection_column_count, 1); assert_eq!(connection_table_count, 1); assert_eq!( @@ -2833,7 +2906,7 @@ fn sqlite_store_migrates_v7_hydration_jobs_schema() { ) .expect("hydration_jobs table"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(hydration_jobs_table_count, 1); assert!( store @@ -2920,7 +2993,7 @@ fn sqlite_store_migrates_v8_connections_to_default_connector_profile() { ) .expect("profile_id column"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(profile_column_count, 1); let migrated_connection = store .get_connection(&ConnectionId::new("notion-work")) @@ -3033,14 +3106,16 @@ fn sqlite_store_migrates_v11_entity_search_index() { .expect("user version"); let search_table_count: i64 = connection .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE name = 'entity_search_fts'", + "SELECT COUNT(*) + FROM sqlite_master + WHERE name IN ('entity_search_fts', 'search_documents_fts')", [], |row| row.get(0), ) - .expect("entity search table"); + .expect("search tables"); - assert_eq!(user_version, 19); - assert_eq!(search_table_count, 1); + assert_eq!(user_version, 20); + assert_eq!(search_table_count, 2); let matches = store .list_entity_search_candidates(&fixture.mount_id, "launch", None) .expect("search migrated index") @@ -3514,7 +3589,7 @@ fn sqlite_store_migrates_v1_journals_with_empty_preimages() { .expect("get migrated journal") .expect("journal"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert!(entry.preimages.is_empty()); assert!(entry.apply_effects.is_empty()); } @@ -3585,7 +3660,7 @@ fn sqlite_store_migrates_v2_journals_with_empty_apply_effects() { .expect("get migrated journal") .expect("journal"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert!(entry.apply_effects.is_empty()); } @@ -3648,7 +3723,7 @@ fn sqlite_store_migrates_v16_journals_with_empty_edit_metadata() { .query_row( "SELECT COUNT(*) FROM state_migrations - WHERE migration_id = 'schema-16-to-19'", + WHERE migration_id = 'schema-16-to-20'", [], |row| row.get(0), ) @@ -3658,7 +3733,7 @@ fn sqlite_store_migrates_v16_journals_with_empty_edit_metadata() { .expect("get migrated journal") .expect("journal"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(journals_component_version, 3); assert_eq!( metadata_json, @@ -3726,7 +3801,7 @@ fn sqlite_store_migrates_v17_mounts_with_default_settings_json() { .query_row( "SELECT COUNT(*) FROM state_migrations - WHERE migration_id = 'schema-17-to-19'", + WHERE migration_id = 'schema-17-to-20'", [], |row| row.get(0), ) @@ -3741,7 +3816,7 @@ fn sqlite_store_migrates_v17_mounts_with_default_settings_json() { ) .expect("journal component metadata"); - assert_eq!(user_version, 19); + assert_eq!(user_version, 20); assert_eq!(settings_json, "{}"); assert_eq!(migration_count, 1); assert_eq!(journal_component, (3, 3)); @@ -3780,6 +3855,7 @@ fn schema_column_snapshot(connection: &Connection) -> String { WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT GLOB 'entity_search_fts_*' + AND name NOT GLOB 'search_documents_fts_*' ORDER BY name", ) .expect("prepare table snapshot query"); @@ -4471,6 +4547,21 @@ fn seed_source_scoped_state(store: &mut SqliteStateStore, mount_id: &MountId) { .expect("append journal"); } +fn search_index_row_count(store: &SqliteStateStore, mount_id: &MountId, table: &str) -> i64 { + assert!(matches!( + table, + "entity_search_fts" | "search_documents_fts" + )); + let connection = Connection::open(&store.db_path).expect("raw connection"); + connection + .query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE mount_id = ?1"), + params![mount_id.0.as_str()], + |row| row.get(0), + ) + .expect("search index row count") +} + fn apply_effects(push_id: &str) -> Vec { vec![JournalApplyEffect::UpdatedBlock { operation_id: PushOperationId(format!("{push_id}:0:update_block:paragraph-1")), diff --git a/docs-site/cli-reference.mdx b/docs-site/cli-reference.mdx index a2e4e0b4..85c188f7 100644 --- a/docs-site/cli-reference.mdx +++ b/docs-site/cli-reference.mdx @@ -239,16 +239,23 @@ loc pull ~/Locality/google-docs-main ### `loc search` -Search local mount metadata without contacting remote sources. +Search Locality's local SQLite search index. ```bash loc search [--connector ] [--limit ] [--all] ``` Use search for navigation, desktop typeahead, agent path lookup, and source URL -resolution. It reads local SQLite metadata only, so it is fast and safe but may -not know about the newest remote changes until a pull, daemon refresh, or -freshness check has run. +resolution. It reads local SQLite entity metadata, remote observations, derived +breadcrumb/path text, connector-provided search metadata such as source URLs and +aliases, and hydrated shadow frontmatter/body text. Ordinary search is +local-first, so it is fast and safe but may not know about the newest remote +changes until a pull, daemon refresh, or freshness check has run. Exact Notion URL +misses may trigger a targeted metadata refresh before re-running the local search. +Results are field-weighted so title, alias, source URL, and path hits outrank +broader metadata and body hits. Human output can include a compact `match:` line, +and JSON output can include stable optional `match_context` with the matched field +and best-effort snippet. Tools should tolerate absence and future field names. Key arguments: diff --git a/docs/cli.md b/docs/cli.md index 424ebcad..81c0bf2f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -184,10 +184,19 @@ references. ## Local Search -`loc search ` searches local mount metadata only. It reads SQLite mount, -entity, and remote-observation records; it does not call Notion or any other -remote connector. This makes search safe for desktop typeahead, large-workspace -navigation, and future agent/MCP surfaces. +`loc search ` searches Locality's local SQLite search index. It covers +mount/entity metadata, remote-observation metadata, derived breadcrumb/path +text, connector-provided search metadata such as source URLs and aliases, and +hydrated shadow frontmatter/body text. The shared search engine does not call +Notion or any other remote connector, which keeps ordinary title/body search safe +for desktop typeahead, large-workspace navigation, and future agent/MCP surfaces. +Results are field-weighted: exact title, alias, source URL, and path matches rank +ahead of broader metadata, frontmatter, and body matches. When an indexed field +explains the match, human output includes a compact `match:` line and JSON output +includes `match_context`. +The CLI has one targeted recovery path: an exact Notion URL or bare Notion ID +that misses locally may refresh Notion metadata through the daemon or connector, +then re-run the local search. Examples: @@ -199,8 +208,9 @@ loc search roadmap --connector notion --limit 5 --json ``` Human output lists title, entity kind, local state, projected path, mount, -connector, and remote id. Results that are not safe for direct agent reads also -print compact safety labels. JSON output is stable enough for tools: +connector, remote id, and match context when available. Results that are not safe +for direct agent reads also print compact safety labels. JSON output is stable +enough for tools: ```json { @@ -219,6 +229,10 @@ print compact safety labels. JSON output is stable enough for tools: "path": "Engineering/Roadmap 2026/page.md", "absolute_path": "/Users/alice/Locality/notion-main/Engineering/Roadmap 2026/page.md", "state": "ready", + "match_context": { + "field": "title", + "text": "Roadmap 2026" + }, "safety": { "agent_readable": true, "labels": ["ready"] @@ -235,11 +249,17 @@ print compact safety labels. JSON output is stable enough for tools: } ``` +`match_context` is a stable optional JSON object. `field` and `text` are stable +keys; tools must tolerate the object being absent and should allow new `field` +values as connectors add richer indexed metadata. The snippet text is +best-effort display context, not a durable document excerpt contract. + `state` is derived from local hydration plus the latest cheap remote observation: `online_only`, `ready`, `pending_changes`, `conflict`, -`remote_update_available`, `remote_deleted`, or `review_needed`. Because search -is local-only, run `loc pull`, `loc inspect`, or use the daemon freshness queue -when you need the newest remote facts. +`remote_update_available`, `remote_deleted`, or `review_needed`. Body matches +come only from content already stored locally in shadows. Run `loc pull`, +`loc inspect`, or use the daemon freshness queue when you need the newest remote +facts. `safety.agent_readable` is true only for clean hydrated results. Online-only, dirty, conflicted, stale, or remotely deleted results are still returned for diff --git a/docs/connector-sdk.md b/docs/connector-sdk.md index d18e9a76..d329184d 100644 --- a/docs/connector-sdk.md +++ b/docs/connector-sdk.md @@ -109,6 +109,31 @@ created by the target push must return an observation that reports that created entity as deleted. The host validates mount, entity, parent, path, deletion state, and path ownership before reconciling local files. +## Search Metadata + +Connectors may add connector-neutral search hints under the reserved +`loc_search` object inside `RemoteObservation.raw_metadata_json`. The host treats +this payload as rebuildable index input only. It does not infer identity, +parentage, projection paths, or push behavior from it. + +The supported shape is: + +```json +{ + "loc_search": { + "metadata_text": ["customer escalation", "Engineering", "Todo"], + "aliases": ["ENG-1"], + "source_url": "https://linear.app/acme/issue/ENG-1/improve-sync" + } +} +``` + +Use `metadata_text` for concise provider-specific fields users naturally search +for, such as issue identifiers, team/project/status names, labels, assignee +names, and due dates. Use `aliases` for stable short handles or alternate IDs. +Use `source_url` for the canonical provider URL. Do not include secrets, +credentials, opaque auth state, or large raw provider payloads solely for search. + ## v1 connector `locality-notion` is the first connector. It owns Notion-specific block mapping, database schema translation, OAuth/API behavior, and conversion between Notion payloads and the canonical Locality document model. diff --git a/docs/desktop-app.md b/docs/desktop-app.md index 08eb2896..c2c07527 100644 --- a/docs/desktop-app.md +++ b/docs/desktop-app.md @@ -292,12 +292,17 @@ The CLI locate command should perform the same locate operations as the desktop GUI and print only the resolved local filesystem path. Richer locate metadata can be exposed later through a separate JSON/reporting surface if needed. -Initial desktop implementation can search metadata already stored in SQLite: -remote ID, title, and projected path. The next step is a dedicated local search -index that also covers Markdown body text, frontmatter properties, breadcrumbs, -recently opened pages, and aliases from Notion mentions. That index should be -updated by daemon reconciliation and virtual filesystem mutations, not by reading -every file in the mounted folder. +Initial desktop implementation searches metadata already stored in SQLite: +remote ID, title, and projected path. The connector-neutral +`search_documents_fts` cache extends that to hydrated Markdown body text, +frontmatter, observed remote title/path metadata, and derived breadcrumb/path +text plus connector-provided aliases and source URLs. Search callers receive +structured indexed fields so desktop suggestions can rank titles, aliases, URLs, +paths, metadata, and body text differently and show a compact match snippet. Later +iterations can add recently opened pages and richer mention-derived names. The +index should be updated by daemon +reconciliation, hydration, shadow writes, and virtual filesystem mutations, not +by reading every file in the mounted folder. Main app and tray search surfaces should show ranked suggestions while the user types, including the projected path and state label. A user should be able to @@ -315,7 +320,7 @@ Required behavior: - Projection is metadata-first: directory listings come from SQLite and remain fast even when bodies are not hydrated. - Search is local-first: title/path results should appear instantly from the - entity index; body search can be eventually consistent. + local search index; hydrated body search can be eventually consistent. - Hydration is intent-driven: opening, searching, pinning, or agent-targeting a page should raise its priority above background sync. - Navigation preserves hierarchy: search results should show enough path context diff --git a/docs/locality-store.md b/docs/locality-store.md index 965747c2..d755e46e 100644 --- a/docs/locality-store.md +++ b/docs/locality-store.md @@ -49,6 +49,9 @@ mount-scoped JSON settings field used by connector-specific mount options. - SQLite migrates v18 rows to v19 by adding durable discovery-projection transactions without rewriting or discarding existing mount work. +- SQLite migrates v19 rows to v20 by creating and rebuilding + `search_documents_fts`, a connector-neutral search cache over entity metadata + plus connector search metadata and hydrated shadow frontmatter/body. - SQLite records component versions for durable subsystems so compatibility is decided from persisted state contracts instead of desktop build IDs. - SQLite enables WAL mode, a busy timeout, foreign keys, and `PRAGMA user_version` schema versioning. @@ -147,8 +150,18 @@ The first schema keeps high-value lookup fields relational and stores complex co `windows_cloud_files`), optional connection id, and connector-specific `settings_json`; - `entities`: mount id, remote id, kind, title, projected path, hydration, content hash, remote edit time; -- `entity_search_fts`: derived full-text index over entity titles/paths and - observed remote titles/paths. It is rebuildable and stores no secrets; +- `entity_search_fts`: legacy derived full-text index over entity titles/paths + and observed remote titles/paths. It is rebuildable and stores no credential + material; +- `search_documents_fts`: derived connector-neutral full-text index over entity + titles, projected paths, observed remote titles/paths, derived breadcrumbs, and + hydrated shadow frontmatter/body. It also indexes connector-provided + `loc_search.metadata_text`, `loc_search.aliases`, and `loc_search.source_url` + from remote observation raw metadata. It is rebuildable and stores no + credential material, but it can contain indexed user document content from + hydrated shadows and connector metadata. SQLite returns structured indexed + fields to callers so UI and CLI search can rank title, alias, URL, metadata, + and body matches differently; - `shadows`: mount id, entity id, body hash, rendered body, JSON shadow blocks; - `journals`: push id, mount id, JSON remote ids, JSON push plan, JSON preimage snapshots, JSON apply effects, JSON status; - `state_components`: current durable/rebuildable component versions, minimum @@ -182,7 +195,7 @@ Shadow blocks, journal plans, journal preimages, and journal apply effects are J - Unknown required components block older binaries. Unknown non-required rebuildable components are ignored by older binaries. -The SQLite test suite includes a v19 schema snapshot, old-DB migration fixtures, +The SQLite test suite includes a v20 schema snapshot, old-DB migration fixtures, newer-schema detection, newer-component detection, and unknown-component compatibility checks. A PR that changes durable state should update these tests as part of the same change.