Merge pull request #816 from DataDog/alonam/scan_secrets_endpoint

[IDE-5654] Add a new endpoint for secrets scanning

Author: alonam

Cargo.lock

@@ -12,8 +12,10 @@ use rocket::{
     serde::json::{json, Json, Value},
     Build, Rocket, Shutdown, State,
 };
+use secrets::model::secret_result::SecretResult;
 use server::model::analysis_request::ServerRule;
 use server::model::analysis_response::AnalysisResponse;
+use server::model::secret_scan::{SecretScanRequest, SecretScanResponse};
 use server::model::{
     analysis_request::AnalysisRequest, tree_sitter_tree_request::TreeSitterRequest,
 };
@@ -151,6 +153,82 @@ async fn analyze(
     .unwrap()
 }
 
+fn process_secret_scan_request(request: SecretScanRequest) -> Result<Vec<SecretResult>, String> {
+    // Maximum number of rules per request to prevent excessive CPU usage.
+    const MAX_RULES_COUNT: usize = 1000;
+
+    if request.rules.is_empty() {
+        return Err("No rules provided".to_string());
+    }
+
+    if request.rules.len() > MAX_RULES_COUNT {
+        return Err(format!(
+            "Too many rules: {} exceeds maximum of {}",
+            request.rules.len(),
+            MAX_RULES_COUNT
+        ));
+    }
+
+    // Validate filename (prevent path traversal attacks)
+    if request.filename.contains("..") || request.filename.contains('\0') {
+        return Err("Invalid filename: path traversal detected".to_string());
+    }
+
+    // Deserialize rules from JSON
+    let rules: Vec<secrets::model::secret_rule::SecretRule> = request
+        .rules
+        .iter()
+        .map(|r| serde_json::from_value(r.clone()))
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| format!("Failed to parse rules: {}", e))?;
+
+    // Build the scanner with the provided rules
+    let scanner = secrets::scanner::build_sds_scanner(&rules, request.use_debug)?;
+
+    // Configure analysis options
+    let options = common::analysis_options::AnalysisOptions {
+        use_debug: request.use_debug,
+        ..Default::default()
+    };
+
+    // Perform the secret scan
+    let results = secrets::scanner::find_secrets(
+        &scanner,
+        &rules,
+        &request.filename,
+        &request.data,
+        &options,
+    );
+
+    Ok(results)
+}
+
+/// Scans source code for secrets using the provided detection rules.
+#[rocket::post("/scan-secrets", format = "application/json", data = "<request>")]
+async fn scan_secrets(span: TraceSpan, request: Json<SecretScanRequest>) -> Value {
+    let _entered = span.enter();
+
+    rocket::tokio::task::spawn_blocking(move || {
+        let request = request.into_inner();
+        let (rule_responses, errors) = match process_secret_scan_request(request) {
+            Ok(resp) => (resp, vec![]),
+            Err(err) => (vec![], vec![err]),
+        };
+
+        json!(SecretScanResponse {
+            rule_responses,
+            errors,
+        })
+    })
+    .await
+    .unwrap_or_else(|e| {
+        json!(SecretScanResponse {
+            rule_responses: vec![],
+            errors: vec![format!("Internal error: {e}")],
+        })
+    })
+}
+
 #[rocket::post("/get-treesitter-ast", format = "application/json", data = "<request>")]
 fn get_tree(span: TraceSpan, request: Json<TreeSitterRequest>) -> Value {
     let _entered = span.enter();
@@ -207,6 +285,7 @@ fn mount_endpoints(rocket: Rocket<Build>) -> Rocket<Build> {
             "/",
             rocket::routes![
                 analyze,
+                scan_secrets,
                 get_tree,
                 get_version,
                 get_revision,

crates/bins/src/bin/datadog_static_analyzer_server/endpoints.rs

@@ -267,7 +267,8 @@ pub fn secret_analysis(
     files_to_analyze: &[PathBuf],
 ) -> anyhow::Result<AnalysisResult<SecretResult>> {
     let secrets_rules = &config.secrets_rules;
-    let sds_scanner = build_sds_scanner(secrets_rules, config.use_debug);
+    let sds_scanner =
+        build_sds_scanner(secrets_rules, config.use_debug).map_err(|e| anyhow::anyhow!(e))?;
 
     let nb_secrets_files = files_to_analyze.len();
     let directory_path = Path::new(config.source_directory.as_str());

crates/bins/src/lib.rs

@@ -17,15 +17,15 @@ use std::sync::Arc;
 /// our API.
 ///
 /// Once the scanner is built, use scanner.scan() to find secrets.
-pub fn build_sds_scanner(rules: &[SecretRule], use_debug: bool) -> Scanner {
+pub fn build_sds_scanner(rules: &[SecretRule], use_debug: bool) -> Result<Scanner, String> {
     let sds_rules = rules
         .iter()
         .map(|r| r.convert_to_sds_ruleconfig(use_debug).into_dyn())
         .collect::<Vec<RootRuleConfig<Arc<dyn RuleConfig>>>>();
     Scanner::builder(&sds_rules)
         .with_return_matches(true)
         .build()
-        .expect("error when instantiating the scanner")
+        .map_err(|e| format!("Failed to build scanner: {e}"))
 }
 
 /// Find secrets in code. This is the main entrypoint for our SDS integration.
@@ -138,7 +138,7 @@ mod tests {
             match_validation: None,
             pattern_capture_groups: vec![],
         }];
-        let scanner = build_sds_scanner(rules.as_slice(), false);
+        let scanner = build_sds_scanner(rules.as_slice(), false).expect("error building scanner");
         let text = "FOO\nFOOBAR\nFOOBAZ\nCAT";
         let matches = find_secrets(
             &scanner,
@@ -184,7 +184,7 @@ mod tests {
             match_validation: None,
             pattern_capture_groups: vec![],
         }];
-        let scanner = build_sds_scanner(rules.as_slice(), false);
+        let scanner = build_sds_scanner(rules.as_slice(), false).expect("error building scanner");
         // Line 1: FOOBAR - should be found
         // Line 2: #no-dd-secrets
         // Line 3: FOOBAZ - should be ignored
@@ -223,7 +223,7 @@ mod tests {
             match_validation: None,
             pattern_capture_groups: vec![],
         }];
-        let scanner = build_sds_scanner(rules.as_slice(), false);
+        let scanner = build_sds_scanner(rules.as_slice(), false).expect("error building scanner");
         // Line 1: FOOBAR - should be found
         // Line 2: #no-dd-secrets
         // Line 3: FOOBAZ - should be ignored
@@ -269,7 +269,7 @@ mod tests {
             match_validation: None,
             pattern_capture_groups: vec![],
         }];
-        let scanner = build_sds_scanner(rules.as_slice(), false);
+        let scanner = build_sds_scanner(rules.as_slice(), false).expect("error building scanner");
         // Directive on line 1 means ignore entire file
         let text = "#no-dd-secrets\nFOOBAR\nFOOBAZ";
         let matches = find_secrets(

crates/secrets/src/scanner.rs

@@ -7,6 +7,7 @@ version.workspace = true
 # local
 common = { package = "common", path = "../common" }
 kernel = { package = "static-analysis-kernel", path = "../static-analysis-kernel" }
+secrets = { package = "secrets", path = "../secrets" }
 # workspace
 anyhow = { workspace = true }
 base64 = { workspace = true }

crates/static-analysis-server/Cargo.toml

@@ -1,5 +1,6 @@
 pub mod analysis_request;
 pub mod analysis_response;
+pub mod secret_scan;
 pub mod tree_sitter_tree_node;
 pub mod tree_sitter_tree_request;
 pub mod tree_sitter_tree_response;

crates/static-analysis-server/src/model.rs

@@ -0,0 +1,17 @@
+use secrets::model::secret_result::SecretResult;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SecretScanRequest<T = serde_json::Value> {
+    pub filename: String,
+    pub data: String,
+    pub rules: Vec<T>,
+    #[serde(default)]
+    pub use_debug: bool,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SecretScanResponse {
+    pub rule_responses: Vec<SecretResult>,
+    pub errors: Vec<String>,
+}