feat: add --base argument to compare local run to another run

Add --base cli argument and simplify polling code
1. If a base run id is provided, we now use the `compareRunsPaginated`
   resolver and display results from there
2. If a base run id is not provided or if the base run id is not correct
   (missing, executor mismatch...), we display the single mode results

This overall simplifies the polling code and gets rid of the artificial
"for_run" and "for_exec" distinctions.

Author: GuillaumeLagrange

src/api_client.rs

@@ -81,39 +81,12 @@ nest! {
 
 #[derive(Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
-pub struct FetchLocalRunReportVars {
+pub struct FetchLocalRunVars {
     pub owner: String,
     pub name: String,
     pub run_id: String,
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
-pub enum ReportConclusion {
-    AcknowledgedFailure,
-    Failure,
-    MissingBaseRun,
-    NoBenchmarks,
-    Success,
-}
-
-impl Display for ReportConclusion {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            ReportConclusion::AcknowledgedFailure => {
-                write!(f, "{}", style("Acknowledged Failure").yellow().bold())
-            }
-            ReportConclusion::Failure => write!(f, "{}", style("Failure").red().bold()),
-            ReportConclusion::MissingBaseRun => {
-                write!(f, "{}", style("Missing Base Run").yellow().bold())
-            }
-            ReportConclusion::NoBenchmarks => {
-                write!(f, "{}", style("No Benchmarks").yellow().bold())
-            }
-            ReportConclusion::Success => write!(f, "{}", style("Success").green().bold()),
-        }
-    }
-}
-
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub enum RunStatus {
@@ -123,18 +96,23 @@ pub enum RunStatus {
     Processing,
 }
 
+// Custom deserializer to convert string values to i64
+fn deserialize_i64_from_string<'de, D>(deserializer: D) -> Result<i64, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    use serde::de;
+    let s = String::deserialize(deserializer)?;
+    s.parse().map_err(de::Error::custom)
+}
+
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    pub struct FetchLocalRunReportRun {
+    pub struct FetchLocalRunRun {
         pub id: String,
         pub status: RunStatus,
         pub url: String,
-        pub head_reports: Vec<pub struct FetchLocalRunReportHeadReport {
-            pub id: String,
-            pub impact: Option<f64>,
-            pub conclusion: ReportConclusion,
-        }>,
         pub results: Vec<pub struct FetchLocalRunBenchmarkResult {
             pub value: f64,
             pub benchmark: pub struct FetchLocalRunBenchmark {
@@ -166,42 +144,113 @@ nest! {
     }
 }
 
-// Custom deserializer to convert string values to i64
-fn deserialize_i64_from_string<'de, D>(deserializer: D) -> Result<i64, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    use serde::de;
-    let s = String::deserialize(deserializer)?;
-    s.parse().map_err(de::Error::custom)
+nest! {
+    #[derive(Debug, Deserialize, Serialize)]*
+    #[serde(rename_all = "camelCase")]*
+    struct FetchLocalRunData {
+        repository: struct FetchLocalRunRepository {
+            run: FetchLocalRunRun,
+        }
+    }
+}
+
+pub struct FetchLocalRunResponse {
+    pub run: FetchLocalRunRun,
+}
+
+#[derive(Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CompareRunsVars {
+    pub owner: String,
+    pub name: String,
+    pub base_run_id: String,
+    pub head_run_id: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "PascalCase")]
+pub enum ResultComparisonCategory {
+    Acknowledged,
+    Archived,
+    Ignored,
+    Improvement,
+    New,
+    Regression,
+    Skipped,
+    Untouched,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+pub enum BenchmarkReportStatus {
+    Improvement,
+    Missing,
+    New,
+    NoChange,
+    Regression,
+}
+
+impl Display for BenchmarkReportStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BenchmarkReportStatus::Improvement => {
+                write!(f, "{}", style("Improvement").green().bold())
+            }
+            BenchmarkReportStatus::Missing => write!(f, "{}", style("Missing").yellow().bold()),
+            BenchmarkReportStatus::New => write!(f, "{}", style("New").cyan().bold()),
+            BenchmarkReportStatus::NoChange => write!(f, "{}", style("No Change").dim()),
+            BenchmarkReportStatus::Regression => write!(f, "{}", style("Regression").red().bold()),
+        }
+    }
 }
 
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    struct FetchLocalRunReportData {
-        repository: pub struct FetchLocalRunReportRepository {
-            settings: struct FetchLocalRunReportSettings {
-                allowed_regression: f64,
-            },
-            run: FetchLocalRunReportRun,
-        }
+    pub struct CompareRunsBenchmarkResult {
+        pub value: Option<f64>,
+        pub base_value: Option<f64>,
+        pub change: Option<f64>,
+        pub category: ResultComparisonCategory,
+        pub status: BenchmarkReportStatus,
+        pub benchmark: pub struct CompareRunsBenchmark {
+            pub name: String,
+            pub executor: ExecutorName,
+        },
+    }
+}
+
+nest! {
+    #[derive(Debug, Deserialize, Serialize)]*
+    #[serde(rename_all = "camelCase")]*
+    pub struct CompareRunsHeadRun {
+        pub id: String,
+        pub status: RunStatus,
     }
 }
 
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    struct FetchLocalExecReportData {
-        project: pub struct FetchLocalExecReportProject {
-            run: FetchLocalRunReportRun,
+    struct CompareRunsData {
+        repository: struct CompareRunsRepository {
+            paginated_compare_runs: pub struct CompareRunsComparison {
+                pub impact: Option<f64>,
+                pub url: String,
+                pub head_run: CompareRunsHeadRun,
+                pub result_comparisons: Vec<CompareRunsBenchmarkResult>,
+            },
         }
     }
 }
 
-pub struct FetchLocalRunReportResponse {
-    pub allowed_regression: f64,
-    pub run: FetchLocalRunReportRun,
+pub struct CompareRunsResponse {
+    pub comparison: CompareRunsComparison,
+}
+
+pub enum CompareRunsOutcome {
+    Success(CompareRunsResponse),
+    BaseRunNotFound,
+    ExecutorMismatch,
 }
 
 #[derive(Serialize, Clone)]
@@ -274,26 +323,47 @@ impl CodSpeedAPIClient {
         }
     }
 
-    pub async fn fetch_local_run_report(
-        &self,
-        vars: FetchLocalRunReportVars,
-    ) -> Result<FetchLocalRunReportResponse> {
+    pub async fn compare_runs(&self, vars: CompareRunsVars) -> Result<CompareRunsOutcome> {
         let response = self
             .gql_client
-            .query_with_vars_unwrap::<FetchLocalRunReportData, FetchLocalRunReportVars>(
-                include_str!("queries/FetchLocalRunReport.gql"),
-                vars.clone(),
+            .query_with_vars_unwrap::<CompareRunsData, CompareRunsVars>(
+                include_str!("queries/CompareRuns.gql"),
+                vars,
+            )
+            .await;
+        match response {
+            Ok(response) => Ok(CompareRunsOutcome::Success(CompareRunsResponse {
+                comparison: response.repository.paginated_compare_runs,
+            })),
+            Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
+                bail!("Your session has expired, please login again using `codspeed auth login`")
+            }
+            Err(err) if err.contains_error_code("RUN_NOT_FOUND") => {
+                Ok(CompareRunsOutcome::BaseRunNotFound)
+            }
+            Err(err) if err.contains_error_code("NOT_FOUND") => {
+                Ok(CompareRunsOutcome::ExecutorMismatch)
+            }
+            Err(err) => bail!("Failed to compare runs: {err:?}"),
+        }
+    }
+
+    pub async fn fetch_local_run(&self, vars: FetchLocalRunVars) -> Result<FetchLocalRunResponse> {
+        let response = self
+            .gql_client
+            .query_with_vars_unwrap::<FetchLocalRunData, FetchLocalRunVars>(
+                include_str!("queries/FetchLocalRun.gql"),
+                vars,
             )
             .await;
         match response {
-            Ok(response) => Ok(FetchLocalRunReportResponse {
-                allowed_regression: response.repository.settings.allowed_regression,
+            Ok(response) => Ok(FetchLocalRunResponse {
                 run: response.repository.run,
             }),
             Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
                 bail!("Your session has expired, please login again using `codspeed auth login`")
             }
-            Err(err) => bail!("Failed to fetch local run report: {err}"),
+            Err(err) => bail!("Failed to fetch local run: {err}"),
         }
     }
 

src/cli/exec/mod.rs

@@ -59,6 +59,7 @@ impl ExecArgs {
 fn build_orchestrator_config(
     args: ExecArgs,
     target: executor::BenchmarkTarget,
+    poll_results_options: PollResultsOptions,
 ) -> Result<OrchestratorConfig> {
     let modes = args.shared.resolve_modes()?;
     let raw_upload_url = args
@@ -90,7 +91,7 @@ fn build_orchestrator_config(
         allow_empty: args.shared.allow_empty,
         go_runner_version: args.shared.go_runner_version,
         show_full_output: args.shared.show_full_output,
-        poll_results_options: PollResultsOptions::for_exec(),
+        poll_results_options,
         extra_env: HashMap::new(),
     })
 }
@@ -103,12 +104,17 @@ pub async fn run(
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
     let merged_args = args.merge_with_project_config(project_config);
+    let base_run_id = merged_args.shared.base.clone();
     let target = executor::BenchmarkTarget::Exec {
         command: merged_args.command.clone(),
         name: merged_args.name.clone(),
         walltime_args: merged_args.walltime_args.clone(),
     };
-    let config = build_orchestrator_config(merged_args, target)?;
+    let config = build_orchestrator_config(
+        merged_args,
+        target,
+        PollResultsOptions::new(false, base_run_id),
+    )?;
 
     execute_config(config, api_client, codspeed_config, setup_cache_dir).await
 }