Restructure CLI into modular architecture with quality improvements

Reorganise the monolithic main.rs into a multi-module layout and add
comprehensive feature work and quality improvements:

Architecture:
- Split into src/commands/{search,account,locations,archive,login}.rs
- Add src/{config,error,params,jq,output}.rs as focused utility modules
- Expose library interface via src/lib.rs for integration tests
- Add [lib]/[[bin]] split in Cargo.toml

Features:
- search: pagination with --all-pages/--max-pages and cycle detection
- login: interactive authentication with saved config (~/.config/serpapi)
- locations: public endpoint, no auth required
- archive: retrieve cached search results by ID
- jq: client-side jq filtering via pure-Rust jaq library
- output: colored JSON in TTY, plain JSON in pipes (IsTerminal detection)
- params: key=value CLI argument parsing with fields/json_restrictor support

Security:
- Redact api_key= from network error messages printed to stderr
- Create config directory with 0o700 permissions (unix)
- Use rpassword for echo-free API key entry in login
- Atomic config writes (tmp file + rename); clean up temp on failure
- Re-inject api_key into every pagination URL; reject api_key= as param
- 30s tokio::time::timeout on all API calls

Performance:
- current_thread Tokio runtime (sequential CLI needs no thread pool)
- make_client created once per paginated search (reuses TLS connection)
- BTreeMap for canonical URL param key (no sort-buffer Vec)
- Single HashSet::insert for cycle detection (was contains + insert)
- params_to_hashmap takes Vec<Param> by value; jq::apply takes Value by value

Modernization:
- thiserror derive for CliError; remove manual Display/Error impls
- network_err() helper replaces 6 identical map_err closures
- API_KEY_PARAM constant; HashMap::from in make_client
- save_config returns CliError directly; login uses check_api_error
- let-else in jq.rs; std::iter; Login handled inline in match arm
- print_jq_value accepts &mut impl Write for testability

Dependencies:
- clap 4.5, tokio 1 (rt + time + macros), thiserror 2, rpassword 7
- jaq-core/interpret 1.5, jaq-std 1.6; remove dead direct reqwest dep
- Fat LTO + single CGU + symbol stripping in dist profile
- rust-version = 1.70 MSRV

Author: ilyazub

Cargo.lock

@@ -2,27 +2,59 @@
 name = "serpapi-cli"
 version = "0.1.0"
 edition = "2021"
+rust-version = "1.70"
 authors = ["Thomas Hurst <thomas@serpapi.com>"]
 license = "MIT"
+description = "Official CLI client for SerpApi — fast search scraping for humans and AI agents"
+repository = "https://github.com/serpapi/serpapi-cli"
+homepage = "https://serpapi.com"
+documentation = "https://github.com/serpapi/serpapi-cli#readme"
+readme = "README.md"
+keywords = ["serpapi", "search", "scraping", "cli", "google"]
+categories = ["command-line-utilities", "web-programming::http-client"]
+include = [
+    "src/**/*",
+    "tests/**/*",
+    "README.md",
+    "LICENSE",
+    "Cargo.toml",
+]
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "serpapi"
+path = "src/main.rs"
 
 [dependencies]
-clap = { version = "4.4.11", features = ["derive", "env"] }
-color-eyre = "0.6.2"
-colored_json = "4.1.0"
-html2text = { version = "0.6.0", features = ["ansi_colours"] }
-reqwest = "0.11.22"
-serde = { version = "1.0.193", features = ["derive"] }
-serde_json = "1.0.108"
-serde_json_path = "0.6.4"
-serpapi = { version = "0.1.0", git = "https://github.com/serpapi/serpapi-rust.git", branch="serpapi-cli" }
-termion = "2.0.3"
-textwrap = "0.16.0"
-tokio = { version = "1.35.0", features = ["full"] }
-zstd = "0.13.0"
+clap = { version = "4.5", features = ["derive", "env"] }
+colored_json = "4.1"
+dirs = "5"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serpapi = "1.1.0"
+thiserror = "2"
+rpassword = "7"
+tokio = { version = "1", features = ["rt", "time", "macros"] }
+toml = "0.8"
+url = "2"
+jaq-core = "1.5"
+jaq-interpret = "1.5"
+jaq-parse = "1.0"
+jaq-std = "1.6"
 
 [package.metadata.docs.rs]
-# Docs are identical across targets.
 targets = []
 
+
+[dev-dependencies]
+assert_cmd = "2"
+predicates = "3"
+
+# The profile that 'dist' will build with
+[profile.dist]
+inherits = "release"
+lto = true
+codegen-units = 1
+strip = "symbols"

Cargo.toml

@@ -0,0 +1,14 @@
+use crate::commands::{check_api_error, make_client, network_err};
+use crate::error::CliError;
+use serde_json::Value;
+use std::collections::HashMap;
+
+/// Fetch and return the account information for the given API key.
+pub async fn run(api_key: &str) -> Result<Value, CliError> {
+    let client = make_client(api_key)?;
+    let result = client
+        .account(HashMap::new())
+        .await
+        .map_err(network_err)?;
+    check_api_error(result)
+}

src/commands/account.rs

@@ -0,0 +1,13 @@
+use crate::commands::{check_api_error, make_client, network_err};
+use crate::error::CliError;
+use serde_json::Value;
+
+/// Retrieve a previously cached search result from the SerpApi archive by its ID.
+pub async fn run(id: &str, api_key: &str) -> Result<Value, CliError> {
+    let client = make_client(api_key)?;
+    let result = client
+        .search_archive(id)
+        .await
+        .map_err(network_err)?;
+    check_api_error(result)
+}

src/commands/archive.rs

@@ -0,0 +1,16 @@
+use crate::commands::{check_api_error, make_client, network_err};
+use crate::error::CliError;
+use crate::params::{self, Param};
+use serde_json::Value;
+
+/// Search the SerpApi locations index using the provided parameters.
+pub async fn run(params: Vec<Param>) -> Result<Value, CliError> {
+    let params_map = params::params_to_hashmap(params);
+    // Locations endpoint is public – no API key needed.
+    let client = make_client("")?;
+    let result = client
+        .location(params_map)
+        .await
+        .map_err(network_err)?;
+    check_api_error(result)
+}

src/commands/locations.rs

@@ -0,0 +1,31 @@
+use crate::commands::{check_api_error, make_client, network_err};
+use crate::config;
+use crate::error::CliError;
+use std::collections::HashMap;
+
+/// Prompt the user for their SerpApi API key, verify it, and persist it to the config file.
+pub async fn run() -> Result<(), CliError> {
+    let api_key = rpassword::prompt_password("Enter your SerpApi API key: ")
+        .map_err(|e| CliError::UsageError {
+            message: format!("Failed to read input: {e}"),
+        })?;
+    let api_key = api_key.trim();
+
+    let client = make_client(api_key)?;
+    let result = client
+        .account(HashMap::new())
+        .await
+        .map_err(network_err)?;
+
+    let result = check_api_error(result)?;
+    let email = result
+        .get("account_email")
+        .and_then(|v| v.as_str())
+        .unwrap_or("unknown");
+    config::save_config(api_key)?;
+    eprintln!(
+        "Logged in as {email}. API key saved to {:?}",
+        config::config_path()
+    );
+    Ok(())
+}

src/commands/login.rs

@@ -0,0 +1,42 @@
+use serde_json::Value;
+use serpapi::serpapi::Client;
+use std::collections::HashMap;
+
+use crate::error::CliError;
+
+pub mod search;
+pub mod account;
+pub mod locations;
+pub mod archive;
+pub mod login;
+
+/// The query-parameter name used to pass the SerpApi key to every request.
+pub(crate) const API_KEY_PARAM: &str = "api_key";
+
+/// Convert a `Box<dyn Error>` from the serpapi client into a [`CliError::NetworkError`].
+pub(crate) fn network_err(e: Box<dyn std::error::Error>) -> CliError {
+    CliError::NetworkError { message: e.to_string() }
+}
+
+/// Build a `serpapi::Client` authenticated with the given API key.
+pub fn make_client(api_key: &str) -> Result<Client, CliError> {
+    let params = HashMap::from([(API_KEY_PARAM.to_string(), api_key.to_string())]);
+    Client::new(params).map_err(|e: Box<dyn std::error::Error>| CliError::NetworkError {
+        message: e.to_string(),
+    })
+}
+
+/// Inspect a successful API response and surface any embedded error field
+/// as a `CliError::ApiError`, passing through all other values unchanged.
+pub fn check_api_error(result: Value) -> Result<Value, CliError> {
+    if let Value::Object(ref map) = result {
+        if let Some(error_val) = map.get("error") {
+            let message = error_val
+                .as_str()
+                .unwrap_or("Unknown API error")
+                .to_string();
+            return Err(CliError::ApiError { message });
+        }
+    }
+    Ok(result)
+}

src/commands/mod.rs

@@ -0,0 +1,143 @@
+use std::collections::{HashMap, HashSet};
+use std::time::Duration;
+use url::Url;
+use crate::commands::{check_api_error, make_client, network_err, API_KEY_PARAM};
+use crate::error::CliError;
+use crate::params::{self, Param};
+use serde_json::Value;
+
+/// Execute a SerpApi search, optionally accumulating all pages into a single result.
+pub async fn run(
+    params: Vec<Param>,
+    api_key: &str,
+    fields: Option<&str>,
+    all_pages: bool,
+    max_pages: Option<usize>,
+) -> Result<Value, CliError> {
+    let mut params_map = params::params_to_hashmap(params);
+    params::apply_fields(&mut params_map, fields);
+
+    if !all_pages {
+        let client = make_client(api_key)?;
+        let result = tokio::time::timeout(
+            Duration::from_secs(30),
+            client.search(params_map),
+        )
+        .await
+        .map_err(|_| CliError::NetworkError { message: "Request timed out after 30s".to_string() })?
+        .map_err(network_err)?;
+        return check_api_error(result);
+    }
+
+    // Ensure api_key is in the initial params map so it survives page transitions.
+    params_map.insert(API_KEY_PARAM.to_string(), api_key.to_string());
+
+    let client = make_client(api_key)?;
+    let mut current_params = params_map;
+    let mut accumulated: Option<Value> = None;
+    let mut seen: HashSet<String> = HashSet::new();
+    let mut pages_fetched: usize = 0;
+
+    loop {
+        let result = tokio::time::timeout(
+            Duration::from_secs(30),
+            client.search(current_params.clone()),
+        )
+        .await
+        .map_err(|_| CliError::NetworkError { message: "Request timed out after 30s".to_string() })?
+        .map_err(network_err)?;
+        let page = check_api_error(result)?;
+        pages_fetched += 1;
+
+        let next_url = page
+            .get("serpapi_pagination")
+            .and_then(|p| p.get("next"))
+            .and_then(|n| n.as_str())
+            .map(String::from);
+
+        match &mut accumulated {
+            None => accumulated = Some(page),
+            Some(acc) => {
+                if let (Value::Object(acc_map), Value::Object(page_map)) = (acc, &page) {
+                    for (key, val) in page_map {
+                        if let Value::Array(new_items) = val {
+                            // Intentionally only merge array fields across pages.
+                            // Scalar/object fields (search_metadata, pagination info, etc.)
+                            // are kept from the first page, as they describe the overall
+                            // query rather than per-page state.
+                            match acc_map.get_mut(key) {
+                                Some(Value::Array(existing)) => existing.extend(new_items.iter().cloned()),
+                                _ => { acc_map.insert(key.clone(), val.clone()); }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        match next_url {
+            None => break,
+            Some(url) => {
+                if max_pages.is_some_and(|limit| pages_fetched >= limit) {
+                    break;
+                }
+                let mut next_params = parse_next_params(&url)?;
+                next_params.insert(API_KEY_PARAM.to_string(), api_key.to_string());
+                let canonical = canonical_params_key(&next_params);
+                if !seen.insert(canonical) {
+                    break;
+                }
+                current_params = next_params;
+            }
+        }
+    }
+
+    // Strip the pagination metadata — it's misleading in a merged result.
+    if let Some(Value::Object(ref mut map)) = accumulated {
+        map.remove("serpapi_pagination");
+    }
+
+    accumulated.ok_or_else(|| CliError::ApiError {
+        message: "No results returned".to_string(),
+    })
+}
+
+pub(crate) fn parse_next_params(next_url: &str) -> Result<HashMap<String, String>, CliError> {
+    let parsed = Url::parse(next_url).map_err(|e| CliError::NetworkError {
+        message: format!("Invalid pagination URL: {e}"),
+    })?;
+    Ok(parsed
+        .query_pairs()
+        .map(|(k, v)| (k.into_owned(), v.into_owned()))
+        .collect())
+}
+
+/// Produce a stable, unambiguous key from query params regardless of original URL order.
+/// Values are percent-encoded so that `=` and `&` inside values cannot collide with
+/// the key=value and pair-joining separators.
+pub(crate) fn canonical_params_key(params: &HashMap<String, String>) -> String {
+    let sorted: std::collections::BTreeMap<_, _> = params.iter().collect();
+    url::form_urlencoded::Serializer::new(String::new())
+        .extend_pairs(sorted)
+        .finish()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_canonical_params_key_is_order_independent() {
+        let mut a = HashMap::new();
+        a.insert("q".to_string(), "test".to_string());
+        a.insert("start".to_string(), "0".to_string());
+        a.insert("api_key".to_string(), "abc".to_string());
+
+        let mut b = HashMap::new();
+        b.insert("start".to_string(), "0".to_string());
+        b.insert("api_key".to_string(), "abc".to_string());
+        b.insert("q".to_string(), "test".to_string());
+
+        assert_eq!(canonical_params_key(&a), canonical_params_key(&b));
+    }
+}