feat(retrieval): support image search (#3093)

Add multimodal image vectorization and image query support across the server, SDKs, and CLI.

Author: qin-ctx

Cargo.lock

@@ -22,6 +22,7 @@ futures = "0.3"
 colored = "2.1"
 dirs = "5.0"
 anyhow = "1.0"
+base64 = "0.22"
 chrono = "0.4"
 mime_guess = "2.0"
 thiserror = "1.0"

crates/ov_cli/Cargo.toml

@@ -3,6 +3,8 @@ use serde_json::{Map, Value};
 use std::env;
 use std::path::Path;
 
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
+
 pub use crate::base_client::{BaseClient, FileUploader, TimeoutConfig};
 
 use crate::error::{Error, Result};
@@ -33,6 +35,32 @@ fn compact_request_body(body: &mut Value) {
     });
 }
 
+fn normalize_image_input(image: Option<String>) -> Result<Option<String>> {
+    let Some(value) = image else {
+        return Ok(None);
+    };
+    if value.starts_with("data:image/")
+        || value.starts_with("http://")
+        || value.starts_with("https://")
+        || value.starts_with("viking://")
+    {
+        return Ok(Some(value));
+    }
+
+    let path = Path::new(&value);
+    if path.is_file() {
+        let bytes = std::fs::read(path)?;
+        let mime = mime_guess::from_path(path).first_or_octet_stream();
+        return Ok(Some(format!(
+            "data:{};base64,{}",
+            mime,
+            BASE64_STANDARD.encode(bytes)
+        )));
+    }
+
+    Ok(Some(value))
+}
+
 #[derive(serde::Serialize)]
 pub struct SnapshotCommitReq {
     pub message: String,
@@ -475,6 +503,7 @@ impl HttpClient {
         &self,
         query: String,
         uri: String,
+        image: Option<String>,
         node_limit: i32,
         threshold: Option<f64>,
         since: Option<String>,
@@ -484,8 +513,10 @@ impl HttpClient {
         context_type: Option<Vec<String>>,
         tags: Option<Vec<String>>,
     ) -> Result<serde_json::Value> {
+        let image_url = normalize_image_input(image)?;
         let mut body = serde_json::json!({
             "query": query,
+            "image_url": image_url,
             "target_uri": uri,
             "limit": node_limit,
             "score_threshold": threshold,
@@ -504,6 +535,7 @@ impl HttpClient {
         &self,
         query: String,
         uri: String,
+        image: Option<String>,
         session_id: Option<String>,
         node_limit: i32,
         threshold: Option<f64>,
@@ -514,8 +546,10 @@ impl HttpClient {
         context_type: Option<Vec<String>>,
         tags: Option<Vec<String>>,
     ) -> Result<serde_json::Value> {
+        let image_url = normalize_image_input(image)?;
         let mut body = serde_json::json!({
             "query": query,
+            "image_url": image_url,
             "target_uri": uri,
             "session_id": session_id,
             "limit": node_limit,

crates/ov_cli/src/client.rs

@@ -116,6 +116,7 @@ pub async fn find(
     client: &HttpClient,
     query: &str,
     uri: &str,
+    image: Option<String>,
     node_limit: i32,
     threshold: Option<f64>,
     since: Option<&str>,
@@ -131,6 +132,7 @@ pub async fn find(
         .find(
             query.to_string(),
             uri.to_string(),
+            image,
             node_limit,
             threshold,
             since.map(|s| s.to_string()),
@@ -154,6 +156,7 @@ pub async fn search(
     client: &HttpClient,
     query: &str,
     uri: &str,
+    image: Option<String>,
     session_id: Option<String>,
     node_limit: i32,
     threshold: Option<f64>,
@@ -170,6 +173,7 @@ pub async fn search(
         .search(
             query.to_string(),
             uri.to_string(),
+            image,
             session_id,
             node_limit,
             threshold,

crates/ov_cli/src/commands/search.rs

@@ -1256,8 +1256,9 @@ pub async fn handle_get(uri: String, local_path: String, ctx: CliContext) -> Res
 }
 
 pub async fn handle_find(
-    query: String,
+    query: Option<String>,
     uri: String,
+    image: Option<String>,
     node_limit: i32,
     threshold: Option<f64>,
     after: Option<String>,
@@ -1267,7 +1268,16 @@ pub async fn handle_find(
     tags: Option<Vec<String>>,
     ctx: CliContext,
 ) -> Result<()> {
+    let query = query.unwrap_or_default();
+    if query.trim().is_empty() && image.is_none() {
+        return Err(Error::Client(
+            "Search query or --image must not be empty.".to_string(),
+        ));
+    }
     let mut params = vec![format!("--uri={}", uri), format!("-n {}", node_limit)];
+    if let Some(ref img) = image {
+        params.push(format!("--image {}", img));
+    }
     if let Some(t) = threshold {
         params.push(format!("--threshold {}", t));
     }
@@ -1294,6 +1304,7 @@ pub async fn handle_find(
         &client,
         &query,
         &uri,
+        image,
         node_limit,
         threshold,
         after.as_deref(),
@@ -1309,8 +1320,9 @@ pub async fn handle_find(
 }
 
 pub async fn handle_search(
-    query: String,
+    query: Option<String>,
     uri: String,
+    image: Option<String>,
     session_id: Option<String>,
     node_limit: i32,
     threshold: Option<f64>,
@@ -1321,7 +1333,16 @@ pub async fn handle_search(
     tags: Option<Vec<String>>,
     ctx: CliContext,
 ) -> Result<()> {
+    let query = query.unwrap_or_default();
+    if query.trim().is_empty() && image.is_none() {
+        return Err(Error::Client(
+            "Search query or --image must not be empty.".to_string(),
+        ));
+    }
     let mut params = vec![format!("--uri={}", uri), format!("-n {}", node_limit)];
+    if let Some(ref img) = image {
+        params.push(format!("--image {}", img));
+    }
     if let Some(s) = &session_id {
         params.push(format!("--session-id {}", s));
     }
@@ -1351,6 +1372,7 @@ pub async fn handle_search(
         &client,
         &query,
         &uri,
+        image,
         session_id,
         node_limit,
         threshold,

crates/ov_cli/src/handlers.rs

@@ -455,6 +455,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
                 label: "ov find \"auth flow\" -u viking://projects/acme -L 1,2",
                 description: "Search a subtree and include overview/file results.",
             },
+            HelpItem {
+                label: "ov find --image ./query.png -u viking://resources/images",
+                description: "Search by image with a local file or image URI.",
+            },
         ],
         next_steps: &[
             HelpItem {
@@ -2818,7 +2822,7 @@ mod tests {
         );
 
         assert!(rendered.contains("OpenViking v"));
-        assert!(rendered.contains("ov find [OPTIONS] <query>"));
+        assert!(rendered.contains("ov find [OPTIONS] [query]"));
         assert!(rendered.contains("Examples"));
         assert!(rendered.contains("Common options"));
         assert!(rendered.contains("Next"));
@@ -2881,7 +2885,7 @@ mod tests {
         );
 
         assert!(rendered.contains("OpenViking v"));
-        assert!(rendered.contains("ov find [OPTIONS] <query>"));
+        assert!(rendered.contains("ov find [OPTIONS] [query]"));
         assert!(rendered.contains("Usage:"));
     }
 

crates/ov_cli/src/help_ui.rs

@@ -580,7 +580,14 @@ enum Commands {
     Find {
         /// Search query
         #[arg(value_name = "query")]
-        query: String,
+        query: Option<String>,
+        /// Image query: local path, data URI, HTTP URL, or viking:// URI
+        #[arg(
+            long = "image",
+            value_name = "path|uri",
+            help_heading = "Common options"
+        )]
+        image: Option<String>,
         /// Target URI
         #[arg(
             short,
@@ -638,7 +645,14 @@ enum Commands {
     Search {
         /// Search query
         #[arg(value_name = "query")]
-        query: String,
+        query: Option<String>,
+        /// Image query: local path, data URI, HTTP URL, or viking:// URI
+        #[arg(
+            long = "image",
+            value_name = "path|uri",
+            help_heading = "Common options"
+        )]
+        image: Option<String>,
         /// Target URI
         #[arg(
             short,
@@ -3096,6 +3110,7 @@ async fn main() {
         Commands::Get { uri, local_path } => handlers::handle_get(uri, local_path, ctx).await,
         Commands::Find {
             query,
+            image,
             uri,
             node_limit,
             threshold,
@@ -3108,6 +3123,7 @@ async fn main() {
             handlers::handle_find(
                 query,
                 uri,
+                image,
                 node_limit,
                 threshold,
                 after,
@@ -3121,6 +3137,7 @@ async fn main() {
         }
         Commands::Search {
             query,
+            image,
             uri,
             session_id,
             node_limit,
@@ -3134,6 +3151,7 @@ async fn main() {
             handlers::handle_search(
                 query,
                 uri,
+                image,
                 session_id,
                 node_limit,
                 threshold,
@@ -3237,6 +3255,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn cli_parses_find_image_without_query() {
+        let cli = Cli::try_parse_from(["ov", "find", "--image", "cat.png"])
+            .expect("find image should parse");
+
+        match cli.command {
+            Commands::Find { query, image, .. } => {
+                assert_eq!(query, None);
+                assert_eq!(image.as_deref(), Some("cat.png"));
+            }
+            _ => panic!("expected find command"),
+        }
+    }
+
     #[test]
     fn cli_parses_admin_migrate_cleanup_flag() {
         let migrate = Cli::try_parse_from(["ov", "--sudo", "admin", "migrate"])
@@ -3275,6 +3307,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn cli_parses_search_image_with_query() {
+        let cli = Cli::try_parse_from(["ov", "search", "poster", "--image", "viking://x.png"])
+            .expect("search image should parse");
+
+        match cli.command {
+            Commands::Search { query, image, .. } => {
+                assert_eq!(query.as_deref(), Some("poster"));
+                assert_eq!(image.as_deref(), Some("viking://x.png"));
+            }
+            _ => panic!("expected search command"),
+        }
+    }
+
     #[test]
     fn cli_find_and_search_reject_removed_peer_id_flag() {
         assert!(Cli::try_parse_from(["ov", "find", "invoice", "--peer-id", "peer-a"]).is_err());