From c6baf0655556578ac844b8073b60ab19b8874e8e Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Wed, 24 Jun 2026 09:24:49 +0200 Subject: [PATCH 01/14] Database loading for segmentation objective --- .../Utils/PatientMetricsStructure.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/raidionicsval/Utils/PatientMetricsStructure.py b/raidionicsval/Utils/PatientMetricsStructure.py index 2f3fbe0..d685f9b 100644 --- a/raidionicsval/Utils/PatientMetricsStructure.py +++ b/raidionicsval/Utils/PatientMetricsStructure.py @@ -106,6 +106,80 @@ def init_from_file(self, study_folder: str): else: self.__init_from_file_classification(study_folder=study_folder) + def init_from_db(self, conn): + if self.objective == "segmentation": + self.__init_from_db_segmentation(conn) + else: + # @TODO. Add database-backed loading for classification objective + pass + + def __init_from_db_segmentation(self, conn): + """ + Loads segmentation metrics for this patient from SQLite. + Fetches per-class results first, then the macro-averaged total_results rows. + Returns early if no data exists yet (i.e. first run). + """ + # Load per-class metrics using ClassMetrics.init_from_df + for c in list(self._class_metrics.keys()): + table_name = f"class_{c}" + try: + rows_df = pd.read_sql_query( + f"SELECT * FROM [{table_name}] WHERE [Patient] = ? AND [Fold] = ?", + conn, + params=(str(self._patient_id), self._fold_number) + ) + self._class_metrics[c].init_from_df(rows_df) + except Exception: + return # Table does not exist yet on first run + + # Load macro-averaged results from total_results + rows_df = pd.read_sql_query( + "SELECT * FROM [total_results] WHERE [Patient] = ? AND [Fold] = ?", + conn, + params=(str(self._patient_id), self._fold_number) + ) + + if len(rows_df) == 0: + return + + self._patientwise_metrics = [] + self._pixelwise_metrics = [] + self._objectwise_metrics = [] + self._extra_metrics = [] + + upper_idx = SharedResources.getInstance().upper_default_metrics_index + extra_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + extra_metric_names.extend([f'PiW {m}', f'OW {m}']) + + for thr in rows_df["Threshold"].values: + thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] + thr_val = thr_results[2] + pixelwise_values = list(thr_results[3:7]) + patientwise_values = list(thr_results[7:10]) + objectwise_values = list(thr_results[10:upper_idx]) + extra_values = list(thr_results[upper_idx:]) + + # Pad missing extra metrics with None to match expected length + while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): + extra_values.append(None) + + extra_values_description = list(rows_df.columns[upper_idx:]) + for name in extra_metric_names: + if name not in extra_values_description: + extra_values_description.append(name) + + self._pixelwise_metrics.append([thr_val] + pixelwise_values) + self._patientwise_metrics.append([thr_val] + patientwise_values) + self._objectwise_metrics.append([thr_val] + objectwise_values) + + extra_values_cat = [[x, y] for x, y in zip(extra_values_description, extra_values)] + if extra_values_cat: + self._extra_metrics.append([thr_val] + extra_values_cat) + + if len(self._extra_metrics) == 0: + self._extra_metrics = None + def __init_from_file_segmentation(self, study_folder: str): all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv') @@ -358,6 +432,52 @@ def init_from_file(self, scores_filename: str) -> None: if len(self._extra_metrics) == 0: self._extra_metrics = None + def init_from_df(self, rows_df: pd.DataFrame) -> None: + """ + Initializes class-level metrics from a pre-fetched DataFrame of per-threshold rows. + """ + if len(rows_df) == 0: + return + + upper_idx = SharedResources.getInstance().upper_default_metrics_index + + self._patientwise_metrics = [] + self._pixelwise_metrics = [] + self._objectwise_metrics = [] + self._extra_metrics = [] + + extra_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + extra_metric_names.extend([f'PiW {m}', f'OW {m}']) + + for thr in np.unique(rows_df["Threshold"].values): + thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] + thr_val = thr_results[2] + pixelwise_values = list(thr_results[3:7]) + patientwise_values = list(thr_results[7:10]) + objectwise_values = list(thr_results[10:upper_idx]) + extra_values = list(thr_results[upper_idx:]) + + # Pad missing extra metrics with NaN to match expected length + while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): + extra_values.append(float('nan')) + + extra_values_description = list(rows_df.columns[upper_idx:]) + for name in extra_metric_names: + if name not in extra_values_description: + extra_values_description.append(name) + + self._pixelwise_metrics.append([thr_val] + pixelwise_values) + self._patientwise_metrics.append([thr_val] + patientwise_values) + self._objectwise_metrics.append([thr_val] + objectwise_values) + + extra_values_cat = [[x, y] for x, y in zip(extra_values_description, extra_values)] + if extra_values_cat: + self._extra_metrics.append([thr_val] + extra_values_cat) + + if len(self._extra_metrics) == 0: + self._extra_metrics = None + def is_complete(self): """ From dfd8f73dbc5b1f92860574862ef359a3e2157568 Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Thu, 25 Jun 2026 10:44:20 +0200 Subject: [PATCH 02/14] Fix early-exit logic --- .../Validation/extra_metrics_computation.py | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/raidionicsval/Validation/extra_metrics_computation.py b/raidionicsval/Validation/extra_metrics_computation.py index 9721f50..e6527a7 100644 --- a/raidionicsval/Validation/extra_metrics_computation.py +++ b/raidionicsval/Validation/extra_metrics_computation.py @@ -21,23 +21,22 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold, metrics: List[str] = []): extra_metrics_results = [] try: - if (patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) is not None and - patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] is not None): - metric_values = [x[1] for x in patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:]] - if 'objectwise' in SharedResources.getInstance().validation_metric_spaces and 'patientwise' in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - elif 'patientwise' not in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values[1::2]]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - elif 'objectwise' not in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values[0::2]]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - else: - metric_values = [None] * len(metrics) + existing = patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) + if existing is not None: + metric_values = [x[1] for x in existing[1:]] + + values_to_check = [] + if 'patientwise' in SharedResources.getInstance().validation_metric_spaces: + # PiW values are at even indices (0, 2, 4...) + values_to_check.extend(metric_values[0::2]) + if 'objectwise' in SharedResources.getInstance().validation_metric_spaces: + # OW values are at odd indices (1, 3, 5...) + values_to_check.extend(metric_values[1::2]) + + if values_to_check and all(v is not None and v == v for v in values_to_check): + return existing[1:] + + metric_values = [None] * len(metrics) # ground_truth_ni = nib.load(patient_object._ground_truth_filepaths[class_index]) # detection_ni = nib.load(patient_object._prediction_filepaths[class_index]) From 9268eee425d4baf4ba7ce325b10bb214b020462a Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Thu, 25 Jun 2026 10:54:09 +0200 Subject: [PATCH 03/14] Convert to float since SQLite returns numeric values as strings --- .../Utils/PatientMetricsStructure.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/raidionicsval/Utils/PatientMetricsStructure.py b/raidionicsval/Utils/PatientMetricsStructure.py index d685f9b..ec341a9 100644 --- a/raidionicsval/Utils/PatientMetricsStructure.py +++ b/raidionicsval/Utils/PatientMetricsStructure.py @@ -142,6 +142,13 @@ def __init_from_db_segmentation(self, conn): if len(rows_df) == 0: return + def to_float(v): + # Convert to float since SQLite returns numeric values as strings + try: + return float(v) if v is not None else None + except (ValueError, TypeError): + return None + self._patientwise_metrics = [] self._pixelwise_metrics = [] self._objectwise_metrics = [] @@ -154,11 +161,11 @@ def __init_from_db_segmentation(self, conn): for thr in rows_df["Threshold"].values: thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] - thr_val = thr_results[2] - pixelwise_values = list(thr_results[3:7]) - patientwise_values = list(thr_results[7:10]) - objectwise_values = list(thr_results[10:upper_idx]) - extra_values = list(thr_results[upper_idx:]) + thr_val = float(thr_results[2]) + pixelwise_values = [to_float(x) for x in thr_results[3:7]] + patientwise_values = [to_float(x) for x in thr_results[7:10]] + objectwise_values = [to_float(x) for x in thr_results[10:upper_idx]] + extra_values = [to_float(x) for x in thr_results[upper_idx:]] # Pad missing extra metrics with None to match expected length while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): @@ -441,6 +448,13 @@ def init_from_df(self, rows_df: pd.DataFrame) -> None: upper_idx = SharedResources.getInstance().upper_default_metrics_index + def to_float(v): + # Convert to float since SQLite returns numeric values as strings + try: + return float(v) if v is not None else None + except (ValueError, TypeError): + return None + self._patientwise_metrics = [] self._pixelwise_metrics = [] self._objectwise_metrics = [] @@ -452,11 +466,11 @@ def init_from_df(self, rows_df: pd.DataFrame) -> None: for thr in np.unique(rows_df["Threshold"].values): thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] - thr_val = thr_results[2] - pixelwise_values = list(thr_results[3:7]) - patientwise_values = list(thr_results[7:10]) - objectwise_values = list(thr_results[10:upper_idx]) - extra_values = list(thr_results[upper_idx:]) + thr_val = float(thr_results[2]) + pixelwise_values = [to_float(x) for x in thr_results[3:7]] + patientwise_values = [to_float(x) for x in thr_results[7:10]] + objectwise_values = [to_float(x) for x in thr_results[10:upper_idx]] + extra_values = [to_float(x) for x in thr_results[upper_idx:]] # Pad missing extra metrics with NaN to match expected length while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): From a7d2f292f7c7747d764618e98f10de9889aadff5 Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Thu, 25 Jun 2026 13:16:21 +0200 Subject: [PATCH 04/14] Refactor kfold_model_validation to use SQLite as primary data store --- .../Validation/kfold_model_validation.py | 260 ++++++++++++------ 1 file changed, 180 insertions(+), 80 deletions(-) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 00c75c0..f99dc9d 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -2,8 +2,9 @@ import itertools import logging import os.path -import time import traceback +import sqlite3 +import csv import pandas as pd from math import ceil @@ -54,13 +55,22 @@ def run(self): if len(SharedResources.getInstance().validation_metric_names) != 0: logging.info("Computing extra metrics for cohort.") self.__compute_extra_metrics(class_optimal=class_optimal) + logging.info("Re-exporting results to CSV with extra metrics...") + self.__sqlite_to_csv("total_results", self.dice_output_filename) + for c in SharedResources.getInstance().validation_class_names: + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) + + # Read all extra metric columns actually present in the CSV, not just the ones from the current config. + tmp = pd.read_csv(self.dice_output_filename) + all_extra_metric_names = [col for col in tmp.columns if col not in self.results_df_base_columns] + logging.info("Computing average metrics for the cohort.") # All - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='All') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='All') # Positive, based on given ground truth volume limit - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='Positive') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='Positive') # True positive, based on given detection_overlap_thresholds - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='TP') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='TP') def __compute_metrics(self): """ @@ -71,13 +81,12 @@ def __compute_metrics(self): :return: """ cross_validation_description_file = os.path.join(self.input_folder, 'cross_validation_folds.txt') - self.results_df = [] - self.class_results_df = {} self.dice_output_filename = os.path.join(self.output_folder, 'all_dice_scores.csv') self.class_dice_output_filenames = {} for c in SharedResources.getInstance().validation_class_names: self.class_dice_output_filenames[c] = os.path.join(self.output_folder, c + '_dice_scores.csv') - self.class_results_df[c] = [] + + # Define the column schema shared by all result tables self.results_df_base_columns = ['Fold', 'Patient', 'Threshold'] self.results_df_base_columns.extend(["PiW Dice", "PiW Recall", "PiW Precision", "PiW F1"]) # self.results_df_base_columns.extend(["PaW Dice", "PaW Recall", "PaW Precision", "PaW F1"]) @@ -85,40 +94,73 @@ def __compute_metrics(self): self.results_df_base_columns.extend(["OW Global Recall", "OW Global Precision", "OW Global F1", "OW Dice", "OW Dice (std)", "OW Recall", "OW Recall (std)", "OW Precision", "OW Precision (std)", "OW F1", "OW F1 (std)", '#GT', '#Det']) - # For each extra metric, adding a pixelwise (PiW) and objectwise (OW) version of it! - # extra_metrics = [] - # for m in SharedResources.getInstance().validation_metric_names: - # extra_metrics.extend([f'PiW {m}', f'OW {m}']) self.results_df_base_columns.extend(self.metric_names) - # self.results_df_base_columns.extend(SharedResources.getInstance().validation_metric_names) - if not os.path.exists(self.dice_output_filename): - self.results_df = pd.DataFrame(columns=self.results_df_base_columns) + # Connect to SQLite database + # WAL mode ensures crash-safe writes + self.db_path = os.path.join(self.output_folder, "results.db") + self.conn = sqlite3.connect(self.db_path) + self.conn.execute("PRAGMA journal_mode=WAL;") + cursor = self.conn.cursor() + + # Initialize or resume the total_results table + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='total_results'") + if cursor.fetchone() is None: + # First run: seed from existing CSV if available, otherwise start empty + if os.path.exists(self.dice_output_filename): + results_df = pd.read_csv(self.dice_output_filename) + if results_df.columns[0] != 'Fold': + results_df = pd.read_csv(self.dice_output_filename, index_col=0) + missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if + not x in list(results_df.columns)[1:]] + for m in missing_metrics: + results_df[m] = None + else: + results_df = pd.DataFrame(columns=self.results_df_base_columns) + + results_df.to_sql("total_results", self.conn, if_exists="replace", index=False) + cursor.execute("CREATE INDEX IF NOT EXISTS idx_total_search ON total_results ([Patient], [Fold], [Threshold]);") + self.conn.commit() else: - self.results_df = pd.read_csv(self.dice_output_filename) - if self.results_df.columns[0] != 'Fold': - self.results_df = pd.read_csv(self.dice_output_filename, index_col=0) - missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if - not x in list(self.results_df.columns)[1:]] - for m in missing_metrics: - self.results_df[m] = None + logging.info("Existing database found, resuming from SQL...") + if not os.path.exists(self.dice_output_filename): + self.__sqlite_to_csv("total_results", self.dice_output_filename) + # Initialize or resume per-class tables for c in SharedResources.getInstance().validation_class_names: - if not os.path.exists(self.class_dice_output_filenames[c]): - self.class_results_df[c] = pd.DataFrame(columns=self.results_df_base_columns) + table_name = f"class_{c}" + cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'") + + if cursor.fetchone() is None: + if not os.path.exists(self.class_dice_output_filenames[c]): + class_df = pd.DataFrame(columns=self.results_df_base_columns) + else: + class_df = pd.read_csv(self.class_dice_output_filenames[c]) + if class_df.columns[0] != 'Fold': + class_df = pd.read_csv(self.class_dice_output_filenames[c], index_col=0) + missing_metrics = [x for x in self.metric_names if + x not in list(class_df.columns)[1:]] + for m in missing_metrics: + class_df[m] = None + + class_df.to_sql(table_name, self.conn, if_exists="replace", index=False) + cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_class_{c} ON [{table_name}] ([Patient], [Fold], [Threshold]);") + self.conn.commit() else: - self.class_results_df[c] = pd.read_csv(self.class_dice_output_filenames[c]) - if self.class_results_df[c].columns[0] != 'Fold': - self.class_results_df[c] = pd.read_csv(self.class_dice_output_filenames[c], index_col=0) - missing_metrics = [x for x in self.metric_names if - not x in list(self.class_results_df[c].columns)[1:]] - for m in missing_metrics: - self.class_results_df[c][m] = None + logging.info(f"Existing table found for class {c}, resuming from SQL...") + if not os.path.exists(self.class_dice_output_filenames[c]): + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) - self.results_df['Patient'] = self.results_df.Patient.astype(str) - for c in SharedResources.getInstance().validation_class_names: - self.class_results_df[c]['Patient'] = self.class_results_df[c].Patient.astype(str) + # Add any missing extra metric columns to all tables (e.g. when new metrics are added after initial run) + for metric_name in self.metric_names: + for table in ["total_results"] + [f"class_{c}" for c in SharedResources.getInstance().validation_class_names]: + try: + cursor.execute(f"ALTER TABLE [{table}] ADD COLUMN [{metric_name}] REAL") + self.conn.commit() + except sqlite3.OperationalError: + pass # Column already exists + # Process each fold results_per_folds = [] for fold in range(0, self.fold_number): logging.info(f'\nProcessing fold {fold+1}/{self.fold_number}.\n') @@ -129,9 +171,15 @@ def __compute_metrics(self): results = self.__compute_metrics_for_fold(data_list=test_set, fold_number=fold) results_per_folds.append(results) + logging.info("Exporting results to CSV...") + self.__sqlite_to_csv("total_results", self.dice_output_filename) + for c in SharedResources.getInstance().validation_class_names: + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) + def __compute_metrics_for_fold(self, data_list, fold_number): if not os.path.exists(os.path.join(SharedResources.getInstance().validation_input_folder, "predictions", str(fold_number))): logging.warning(f"No predictions folder for fold {fold_number} -- Skipping!") + print(f"No predictions folder for fold {fold_number} -- Skipping!") return 0 for i, patient in enumerate(tqdm(data_list)): @@ -151,8 +199,9 @@ def __compute_metrics_for_fold(self, data_list, fold_number): # Placeholder for holding all metrics for the current patient patient_metrics = PatientMetrics(id=uid, patient_id=pid, fold_number=fold_number, class_names=SharedResources.getInstance().validation_class_names) - patient_metrics.init_from_file(self.output_folder) - + + # Load any previously computed metrics from the database + patient_metrics.init_from_db(self.conn) success = self.__identify_patient_files(patient_metrics, sub_folder_index, fold_number) self.patients_metrics[uid] = patient_metrics @@ -320,25 +369,22 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): uid = patient_metrics.patient_id classes = SharedResources.getInstance().validation_class_names nb_classes = len(classes) - patient_filenames = {} thr_range = np.arange(0.1, 1.1, 0.1) # Iterating over all classes, where independent files are expected for c in range(nb_classes): gt_filename, det_filename = patient_metrics.get_class_filenames(c) - # ground_truth_ni = nib.load(gt_filename) - # gt = ground_truth_ni.get_fdata() - # detection_ni = nib.load(det_filename) gt, _, gt_specs = open_image_file(gt_filename) detection, _, det_specs = open_image_file(det_filename) gt[gt >= 1] = 1 class_tp_threshold = SharedResources.getInstance().validation_true_positive_volume_thresholds[c] - # gt_volume = np.count_nonzero(gt) * np.prod(ground_truth_ni.header.get_zooms()) * 1e-3 gt_volume = np.count_nonzero(gt) * np.prod(det_specs[1]) * 1e-3 tp_state = True if gt_volume > class_tp_threshold else False extra = [np.round(gt_volume, 4), tp_state, det_specs[1]] pat_results = [] + + # Compute Dice scores across all thresholds, using multiprocessing if configured if SharedResources.getInstance().number_processes > 1: pool = multiprocessing.Pool(processes=SharedResources.getInstance().number_processes) pat_results = pool.map(separate_dice_computation, zip(thr_range, @@ -357,29 +403,37 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): pat_results.append(thr_res) patient_metrics.set_class_regular_metrics(classes[c], pat_results) - # Filling in the csv files on disk for faster resume - class_results_filename = self.class_dice_output_filenames[classes[c]] + + # Write per-class results to the database + table_name = f"class_{classes[c]}" + columns = list(self.results_df_base_columns) + for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.class_results_df[classes[c]].loc[ - (self.class_results_df[classes[c]]['Patient'] == uid) & (self.class_results_df[classes[c]]['Fold'] == fold_number) & ( - self.class_results_df[classes[c]]['Threshold'] == th)] - # ind_values = np.asarray(pat_results[ind]) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: + + cursor = self.conn.cursor() + query = f"SELECT * from [{table_name}] where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(query, (str(uid), fold_number, th)) + row = cursor.fetchone() + + if row is None: extra_metrics = [None] * 2 * len(SharedResources.getInstance().validation_metric_names) ind_values = np.asarray(pat_results[ind][0] + extra_metrics) - buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - columns=list(self.results_df_base_columns)) - # self.class_results_df[classes[c]] = self.class_results_df[classes[c]].append(buff_df, - # ignore_index=True) - self.class_results_df[classes[c]] = pd.concat([self.class_results_df[classes[c]], buff_df], - ignore_index=True) + + column_names_str = ", ".join([f"[{col}]" for col in columns]) + placeholders = ", ".join(["?"] * len(columns)) + + insert_query = f"INSERT INTO {table_name} ({column_names_str}) VALUES ({placeholders})" + cursor.execute(insert_query, tuple(ind_values)) + else: - ind_values = pat_results[ind][0] + list(self.class_results_df[classes[c]].loc[sub_df.index.values[0], :].values[len(pat_results[ind][0]):]) - self.class_results_df[classes[c]].loc[sub_df.index.values[0], :] = ind_values - self.class_results_df[classes[c]].to_csv(class_results_filename, index=False) + ind_values = pat_results[ind][0] + list(row[len(pat_results[ind][0]):]) + set_clause = ", ".join([f"[{col}] = ?" for col in columns]) + update_query = f"UPDATE {table_name} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(update_query, tuple(ind_values) + (uid, fold_number, th)) + + self.conn.commit() + # Should compute the class macro-average results if multiple classes class_averaged_results = None @@ -392,31 +446,54 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): final_pat_class_res = [pat_class_results[x] + pat_class_extra_metrics[x] for x in range(len(thr_range))] class_results.append(final_pat_class_res) class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0) + current_columns = pd.read_sql_query(f"SELECT * FROM {'total_results'} LIMIT 0", self.conn).columns.tolist() - # Filling in the csv files on disk for faster resume for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.results_df.loc[ - (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number) & ( - self.results_df['Threshold'] == th)] - # ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: - ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) - buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - columns=list(self.results_df_base_columns)) - # self.results_df = self.results_df.append(buff_df, ignore_index=True) - self.results_df = pd.concat([self.results_df, buff_df], ignore_index=True) - else: + + cursor = self.conn.cursor() + + query = f"SELECT * from {'total_results'} where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(query, (str(uid), fold_number, th)) + + row = cursor.fetchone() + + if row is None: ind_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) - self.results_df.loc[sub_df.index.values[0], :] = ind_values - self.results_df.to_csv(self.dice_output_filename, index=False) + # Pad with None if fewer values than columns (e.g. when extra metrics are missing) + if len(ind_values) < len(current_columns): + ind_values += [None] * (len(current_columns) - len(ind_values)) + + column_names_str = ", ".join([f"[{col}]" for col in current_columns]) + placeholders = ", ".join(["?"] * len(current_columns)) + insert_query = f"INSERT INTO {'total_results'} ({column_names_str}) VALUES ({placeholders})" + + cursor.execute(insert_query, tuple(ind_values)) + + else: + row_dict = dict(zip(current_columns, row)) + new_values = [fold_number, uid, th] + list(class_averaged_results[ind]) + + for i, val in enumerate(new_values): + if i < len(current_columns): + row_dict[current_columns[i]] = val + + ind_values = [row_dict.get(col, None) for col in current_columns] + set_clause = ", ".join([f"[{col}] = ?" for col in current_columns]) + + update_query = f"UPDATE {'total_results'} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(update_query, tuple(ind_values) + (str(uid), fold_number, np.round(th, 2))) + + self.conn.commit() + def __compute_extra_metrics(self, class_optimal: dict = {}): """ - + Computes additional metrics at the optimal threshold for each class. + Results are written to both the per-class table and total_results in SQLite. """ + original_processes = SharedResources.getInstance().number_processes + SharedResources.getInstance().number_processes = 1 classes = SharedResources.getInstance().validation_class_names for c in classes: optimal_values = class_optimal[c]['All'] @@ -424,16 +501,39 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): try: # Initializing/completing the list which will hold the extra metrics self.patients_metrics[p].setup_extra_metrics(self.metric_names) + class_name = classes[0] + cm = self.patients_metrics[p]._class_metrics[class_name] pat_metrics = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], SharedResources.getInstance().validation_metric_names) self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) - - # Filling in the overall dataframe and dumping results to csv after each patient + cursor = self.conn.cursor() for pm in pat_metrics: metric_name = pm[0] metric_value = pm[1] - self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value - self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) + + update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), optimal_values[1])) + + update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), optimal_values[1])) + except Exception as e: logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") - continue \ No newline at end of file + continue + self.conn.commit() + + SharedResources.getInstance().number_processes = original_processes + + def __sqlite_to_csv(self, table_name, csv_path): + """ + Exports a full SQLite table to a CSV file using a streaming cursor + to avoid loading the entire table into memory. + """ + cursor = self.conn.cursor() + cursor.execute(f"SELECT * FROM [{table_name}]") + headers = [description[0] for description in cursor.description] + + with open(csv_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(cursor) \ No newline at end of file From 9acb485a8d7ddb8013c605ee8941d60ea8cdd23f Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Thu, 25 Jun 2026 16:08:47 +0200 Subject: [PATCH 05/14] Fix early-exit to only check currently configured metrics, not historical ones --- .../Validation/extra_metrics_computation.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/raidionicsval/Validation/extra_metrics_computation.py b/raidionicsval/Validation/extra_metrics_computation.py index e6527a7..66da309 100644 --- a/raidionicsval/Validation/extra_metrics_computation.py +++ b/raidionicsval/Validation/extra_metrics_computation.py @@ -23,19 +23,19 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold try: existing = patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) if existing is not None: - metric_values = [x[1] for x in existing[1:]] + current_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + if 'patientwise' in SharedResources.getInstance().validation_metric_spaces: + current_metric_names.append(f'PiW {m}') + if 'objectwise' in SharedResources.getInstance().validation_metric_spaces: + current_metric_names.append(f'OW {m}') - values_to_check = [] - if 'patientwise' in SharedResources.getInstance().validation_metric_spaces: - # PiW values are at even indices (0, 2, 4...) - values_to_check.extend(metric_values[0::2]) - if 'objectwise' in SharedResources.getInstance().validation_metric_spaces: - # OW values are at odd indices (1, 3, 5...) - values_to_check.extend(metric_values[1::2]) + existing_dict = {x[0]: x[1] for x in existing[1:]} + values_to_check = [existing_dict.get(m) for m in current_metric_names if m in existing_dict] if values_to_check and all(v is not None and v == v for v in values_to_check): return existing[1:] - + metric_values = [None] * len(metrics) # ground_truth_ni = nib.load(patient_object._ground_truth_filepaths[class_index]) From e04946379a8a5b00e94c7122ce45907e991c6a9a Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Fri, 26 Jun 2026 09:23:20 +0200 Subject: [PATCH 06/14] Fix extra metric filtering by separating fixed base columns from configurable extra metrics Previously, results_df_base_columns included current extra metrics (e.g. VOI), causing them to be filtered out when building all_extra_metric_names and missing from the fold average CSV files. Introduced results_df_fixed_columns to represent only the non-configurable base columns, used for filtering. --- raidionicsval/Validation/kfold_model_validation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index f99dc9d..fa9ecaa 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -62,7 +62,7 @@ def run(self): # Read all extra metric columns actually present in the CSV, not just the ones from the current config. tmp = pd.read_csv(self.dice_output_filename) - all_extra_metric_names = [col for col in tmp.columns if col not in self.results_df_base_columns] + all_extra_metric_names = [col for col in tmp.columns if col not in self.results_df_fixed_columns] logging.info("Computing average metrics for the cohort.") # All @@ -87,14 +87,14 @@ def __compute_metrics(self): self.class_dice_output_filenames[c] = os.path.join(self.output_folder, c + '_dice_scores.csv') # Define the column schema shared by all result tables - self.results_df_base_columns = ['Fold', 'Patient', 'Threshold'] - self.results_df_base_columns.extend(["PiW Dice", "PiW Recall", "PiW Precision", "PiW F1"]) + self.results_df_fixed_columns = ['Fold', 'Patient', 'Threshold'] + self.results_df_fixed_columns.extend(["PiW Dice", "PiW Recall", "PiW Precision", "PiW F1"]) # self.results_df_base_columns.extend(["PaW Dice", "PaW Recall", "PaW Precision", "PaW F1"]) - self.results_df_base_columns.extend(["GT volume (ml)", "True Positive", "Detection volume (ml)"]) - self.results_df_base_columns.extend(["OW Global Recall", "OW Global Precision", "OW Global F1", "OW Dice", + self.results_df_fixed_columns.extend(["GT volume (ml)", "True Positive", "Detection volume (ml)"]) + self.results_df_fixed_columns.extend(["OW Global Recall", "OW Global Precision", "OW Global F1", "OW Dice", "OW Dice (std)", "OW Recall", "OW Recall (std)", "OW Precision", "OW Precision (std)", "OW F1", "OW F1 (std)", '#GT', '#Det']) - self.results_df_base_columns.extend(self.metric_names) + self.results_df_base_columns = self.results_df_fixed_columns + self.metric_names # Connect to SQLite database # WAL mode ensures crash-safe writes From 8dc3302caccbd647c50d64b6a483ca48b1b7e573 Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Mon, 29 Jun 2026 10:19:56 +0200 Subject: [PATCH 07/14] small bug, fixed typing --- raidionicsval/Utils/PatientMetricsStructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raidionicsval/Utils/PatientMetricsStructure.py b/raidionicsval/Utils/PatientMetricsStructure.py index ec341a9..d5b8663 100644 --- a/raidionicsval/Utils/PatientMetricsStructure.py +++ b/raidionicsval/Utils/PatientMetricsStructure.py @@ -394,7 +394,7 @@ def set_results(self, results): self._objectwise_metrics = [] for index in range(len(results)): thr_results = results[index][0] - thr_val = thr_results[2] + thr_val = float(thr_results[2]) pixelwise_values = thr_results[3:7] patientwise_values = thr_results[7:10] objectwise_values = thr_results[10:SharedResources.getInstance().upper_default_metrics_index] From 27af369f2f8124a9843f233f953f8f7c678a0352 Mon Sep 17 00:00:00 2001 From: Aurora Johansen Date: Mon, 29 Jun 2026 10:20:45 +0200 Subject: [PATCH 08/14] small fix --- .../Validation/kfold_model_validation.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index fa9ecaa..ac380d8 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -72,6 +72,8 @@ def run(self): # True positive, based on given detection_overlap_thresholds compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='TP') + self.conn.close() + def __compute_metrics(self): """ Generate the Dice scores (and default instance detection metrics) for all the patients and 10 probability @@ -111,7 +113,7 @@ def __compute_metrics(self): results_df = pd.read_csv(self.dice_output_filename) if results_df.columns[0] != 'Fold': results_df = pd.read_csv(self.dice_output_filename, index_col=0) - missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if + missing_metrics = [x for x in self.metric_names if not x in list(results_df.columns)[1:]] for m in missing_metrics: results_df[m] = None @@ -370,6 +372,7 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): classes = SharedResources.getInstance().validation_class_names nb_classes = len(classes) thr_range = np.arange(0.1, 1.1, 0.1) + cursor = self.conn.cursor() # Iterating over all classes, where independent files are expected for c in range(nb_classes): @@ -411,7 +414,6 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): for ind, th in enumerate(thr_range): th = np.round(th, 2) - cursor = self.conn.cursor() query = f"SELECT * from [{table_name}] where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" cursor.execute(query, (str(uid), fold_number, th)) row = cursor.fetchone() @@ -451,8 +453,6 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): for ind, th in enumerate(thr_range): th = np.round(th, 2) - cursor = self.conn.cursor() - query = f"SELECT * from {'total_results'} where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" cursor.execute(query, (str(uid), fold_number, th)) @@ -501,8 +501,6 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): try: # Initializing/completing the list which will hold the extra metrics self.patients_metrics[p].setup_extra_metrics(self.metric_names) - class_name = classes[0] - cm = self.patients_metrics[p]._class_metrics[class_name] pat_metrics = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], SharedResources.getInstance().validation_metric_names) self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) @@ -511,11 +509,13 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): metric_name = pm[0] metric_value = pm[1] + thr_to_match = float(np.round(optimal_values[1], 2)) + update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" - cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), optimal_values[1])) + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" - cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), optimal_values[1])) + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) except Exception as e: logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") From b8a32ceb106039b38993c73969d4a2fed26bb114 Mon Sep 17 00:00:00 2001 From: dbouget Date: Wed, 1 Jul 2026 13:53:33 +0000 Subject: [PATCH 09/14] Fixing formatting for sql writing, performing db writing for classification, fixing extra metrics computation [skip ci] --- .../Validation/extra_metrics_computation.py | 23 +++- .../instance_segmentation_validation.py | 3 + .../Validation/kfold_model_validation.py | 121 +++++++++++++----- .../kfold_model_validation_classification.py | 117 +++++++++++++++-- 4 files changed, 217 insertions(+), 47 deletions(-) diff --git a/raidionicsval/Validation/extra_metrics_computation.py b/raidionicsval/Validation/extra_metrics_computation.py index 66da309..96a0d03 100644 --- a/raidionicsval/Validation/extra_metrics_computation.py +++ b/raidionicsval/Validation/extra_metrics_computation.py @@ -8,7 +8,7 @@ import pandas as pd import nibabel as nib import numpy as np -from typing import List +from typing import List, Tuple # from medpy.metric.binary import hd95, volume_correlation, assd, ravd, obj_assd from sklearn.metrics import jaccard_score, normalized_mutual_info_score, roc_auc_score, cohen_kappa_score from ..Utils.resources import SharedResources @@ -18,7 +18,16 @@ from ..Validation.instance_segmentation_validation import * -def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold, metrics: List[str] = []): +def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold, metrics: List[str] = []) -> Tuple[List, bool]: + """ + + Parameters + ---------- + + Returns + ------- + The first element is the list with metric values, the second element is a boolean indicating if (re)computation was performed. + """ extra_metrics_results = [] try: existing = patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) @@ -34,7 +43,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold values_to_check = [existing_dict.get(m) for m in current_metric_names if m in existing_dict] if values_to_check and all(v is not None and v == v for v in values_to_check): - return existing[1:] + return existing[1:], False metric_values = [None] * len(metrics) @@ -198,7 +207,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold print('Global issue computing metrics for patient {}'.format(patient_object.unique_id)) print(traceback.format_exc()) - return extra_metrics_results + return extra_metrics_results, True def parallel_metric_computation(args): @@ -228,7 +237,7 @@ def parallel_metric_computation(args): gt_spacing=gt_extra[1], det_spacing=det_extra[1]) except Exception as e: - print('Computing {} gave an exception'.format(metric)) + logging.warning(f'Computing {metric} failed with {e}') pass return [metric, metric_value] @@ -324,7 +333,9 @@ def compute_specific_metric_value(metric, gt, detection, tp, tn, fp, fn, gt_spac if (tp + fp) != 0: metric_value = tp / (tp + fp) elif metric == 'AUC': - metric_value = roc_auc_score(gt.flatten(), detection.flatten()) + metric_value = math.inf + if len(np.unique(gt)) > 1: # ROC AUC score is not defined if only class is present in y_true + metric_value = roc_auc_score(gt.flatten(), detection.flatten()) elif metric == 'MCC': metric_value = math.inf num = (tp * tn) - (fp * fn) diff --git a/raidionicsval/Validation/instance_segmentation_validation.py b/raidionicsval/Validation/instance_segmentation_validation.py index 4e020e4..c98077c 100644 --- a/raidionicsval/Validation/instance_segmentation_validation.py +++ b/raidionicsval/Validation/instance_segmentation_validation.py @@ -221,6 +221,9 @@ def __pair_candidates(self, study_state=False): dice_matrix[g, d] = 0. cost_matrix = -dice_matrix gt_inds, pred_inds = linear_sum_assignment(cost_matrix) + # @TODO. The 0.1 threshold here should be linked to the config file parameter (detection_overlap_thresholds). + # Performing such decoupling would require either enforing a single threshold per run, or extending the results with an extra column + # and save all OW results per threshold. matches = [(i, j) for i, j in zip(gt_inds, pred_inds) if dice_matrix[i, j] > 0.1] for m in matches: self.matching_results.append([m[0] + 1, m[1] + 1, dice_matrix[m[0], m[1]]]) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index ac380d8..8cbf043 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -5,11 +5,11 @@ import traceback import sqlite3 import csv - import pandas as pd from math import ceil - +from functools import partial from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from ..Computation.dice_computation_instance import separate_dice_computation from ..Validation.instance_segmentation_validation import * @@ -206,14 +206,14 @@ def __compute_metrics_for_fold(self, data_list, fold_number): patient_metrics.init_from_db(self.conn) success = self.__identify_patient_files(patient_metrics, sub_folder_index, fold_number) self.patients_metrics[uid] = patient_metrics - + # Checking if values have already been computed for the current patient to skip it if so. if patient_metrics.is_complete(): continue if not success: print('Input files not found for patient {}\n'.format(uid)) continue - + self.__generate_dice_scores_for_patient(patient_metrics, fold_number) except Exception as e: print('Issue processing patient {}\n'.format(uid)) @@ -418,9 +418,10 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): cursor.execute(query, (str(uid), fold_number, th)) row = cursor.fetchone() + pat_res_tmp =[float(x) if isinstance(x, np.float32) else x for x in pat_results[ind][0]] if row is None: extra_metrics = [None] * 2 * len(SharedResources.getInstance().validation_metric_names) - ind_values = np.asarray(pat_results[ind][0] + extra_metrics) + ind_values = np.asarray(pat_res_tmp + extra_metrics) column_names_str = ", ".join([f"[{col}]" for col in columns]) placeholders = ", ".join(["?"] * len(columns)) @@ -429,7 +430,7 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): cursor.execute(insert_query, tuple(ind_values)) else: - ind_values = pat_results[ind][0] + list(row[len(pat_results[ind][0]):]) + ind_values = pat_res_tmp + list(row[len(pat_results[ind][0]):]) set_clause = ", ".join([f"[{col}] = ?" for col in columns]) update_query = f"UPDATE {table_name} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" cursor.execute(update_query, tuple(ind_values) + (uid, fold_number, th)) @@ -447,7 +448,7 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): if len(SharedResources.getInstance().validation_metric_names) != 0: final_pat_class_res = [pat_class_results[x] + pat_class_extra_metrics[x] for x in range(len(thr_range))] class_results.append(final_pat_class_res) - class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0) + class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0).astype(float) current_columns = pd.read_sql_query(f"SELECT * FROM {'total_results'} LIMIT 0", self.conn).columns.tolist() for ind, th in enumerate(thr_range): @@ -467,16 +468,14 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): column_names_str = ", ".join([f"[{col}]" for col in current_columns]) placeholders = ", ".join(["?"] * len(current_columns)) insert_query = f"INSERT INTO {'total_results'} ({column_names_str}) VALUES ({placeholders})" - cursor.execute(insert_query, tuple(ind_values)) - else: row_dict = dict(zip(current_columns, row)) new_values = [fold_number, uid, th] + list(class_averaged_results[ind]) for i, val in enumerate(new_values): if i < len(current_columns): - row_dict[current_columns[i]] = val + row_dict[current_columns[i]] = float(val) if isinstance(val, np.float32) else val ind_values = [row_dict.get(col, None) for col in current_columns] set_clause = ", ".join([f"[{col}] = ?" for col in current_columns]) @@ -486,43 +485,105 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): self.conn.commit() - def __compute_extra_metrics(self, class_optimal: dict = {}): """ Computes additional metrics at the optimal threshold for each class. Results are written to both the per-class table and total_results in SQLite. + @TODO. Would need to properly compute metrics average over the different classes to fill in the total_results table (or just skip it). """ - original_processes = SharedResources.getInstance().number_processes - SharedResources.getInstance().number_processes = 1 classes = SharedResources.getInstance().validation_class_names for c in classes: optimal_values = class_optimal[c]['All'] - for p in tqdm(self.patients_metrics): - try: - # Initializing/completing the list which will hold the extra metrics - self.patients_metrics[p].setup_extra_metrics(self.metric_names) - pat_metrics = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], - SharedResources.getInstance().validation_metric_names) - self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) - cursor = self.conn.cursor() + if len(SharedResources.getInstance().validation_metric_names) < 10: + batch_results =[] + dump = 0 + with ThreadPoolExecutor(max_workers=SharedResources.getInstance().number_processes) as executor: + futures = {executor.submit(partial(self.__patient_metrics_computation, c=c, classes=classes, + optimal_values=optimal_values), item): item for item in self.patients_metrics} + for future in tqdm(as_completed(futures), total=len(futures)): + _ = futures[future] + try: + results = future.result() + batch_results.append(results) + dump += 1 + if dump % SharedResources.getInstance().number_processes == 0: + self.__update_database(batch_results) + batch_results.clear() + except Exception as e: + continue + else: + original_processes = SharedResources.getInstance().number_processes + SharedResources.getInstance().number_processes = 10 + for p in tqdm(self.patients_metrics): + recomputation = False + try: + # Initializing/completing the list which will hold the extra metrics + self.patients_metrics[p].setup_extra_metrics(self.metric_names) + pat_metrics, recomputation = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], + SharedResources.getInstance().validation_metric_names) + if recomputation: + self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) + cursor = self.conn.cursor() + for pm in pat_metrics: + metric_name = pm[0] + metric_value = pm[1] + + thr_to_match = float(np.round(optimal_values[1], 2)) + + update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + + update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + except Exception as e: + logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") + continue + finally: + if recomputation: + self.conn.commit() + SharedResources.getInstance().number_processes = original_processes + + + def __patient_metrics_computation(self, p, c, classes, optimal_values): + recomputation = False + result = None + try: + # Initializing/completing the list which will hold the extra metrics + thr_to_match = float(np.round(optimal_values[1], 2)) + self.patients_metrics[p].setup_extra_metrics(self.metric_names) + pat_metrics, recomputation = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], + SharedResources.getInstance().validation_metric_names) + if recomputation: + self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) + + result = [p, c, thr_to_match, pat_metrics, recomputation] + except Exception as e: + logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") + finally: + return result + + def __update_database(self, batch_results): + try: + cursor = self.conn.cursor() + for results in batch_results: + p = results[0] + c = results[1] + thr_to_match = results[2] + pat_metrics = results[3] + recompute = results[4] + if recompute: for pm in pat_metrics: metric_name = pm[0] metric_value = pm[1] - thr_to_match = float(np.round(optimal_values[1], 2)) - update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) - - except Exception as e: - logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") - continue - self.conn.commit() - - SharedResources.getInstance().number_processes = original_processes + self.conn.commit() + except Exception as e: + logging.error(f"Commiting to the database after parallel metrics computation failed with {e}") def __sqlite_to_csv(self, table_name, csv_path): """ diff --git a/raidionicsval/Validation/kfold_model_validation_classification.py b/raidionicsval/Validation/kfold_model_validation_classification.py index aba1b7f..687ae88 100644 --- a/raidionicsval/Validation/kfold_model_validation_classification.py +++ b/raidionicsval/Validation/kfold_model_validation_classification.py @@ -1,15 +1,14 @@ import multiprocessing import itertools import os.path - import time - +import sqlite3 import numpy as np import pandas as pd from math import ceil - +import logging from tqdm import tqdm - +import csv from ..Computation.dice_computation_instance import separate_dice_computation from ..Validation.instance_segmentation_validation import * from ..Utils.resources import SharedResources @@ -40,15 +39,17 @@ def __init__(self): self.metric_names = [] self.metric_names.extend(SharedResources.getInstance().validation_metric_names) self.gt_files_suffix = SharedResources.getInstance().validation_gt_files_suffix - self.gt_master_file = "/home/dbouget/Data/Studies/SequenceClassifier/seqclassifier_patients_parameters.csv" # @TODO. Should be in the config file + self.gt_master_file = SharedResources.getInstance().studies_extra_parameters_filename # @TODO. Not ideal to point to a studies parameter for validation self.gt_master_file_df = pd.read_csv(self.gt_master_file) self.prediction_files_suffix = SharedResources.getInstance().validation_prediction_files_suffix self.patients_metrics = {} def run(self): self.__compute_metrics() - # if len(SharedResources.getInstance().validation_metric_names) != 0: - # self.__compute_extra_metrics(class_optimal=class_optimal) + #if len(SharedResources.getInstance().validation_metric_names) != 0: + # self.__compute_extra_metrics(class_optimal=class_optimal) + # logging.info("Re-exporting results to CSV with extra metrics...") + # self.__sqlite_to_csv("total_results", self.all_scores_output_filename) compute_fold_average(self.input_folder, metrics=self.metric_names) def __compute_metrics(self): @@ -64,6 +65,37 @@ def __compute_metrics(self): self.results_df_base_columns.extend(["Prediction", "GT"]) self.results_df_base_columns.extend(SharedResources.getInstance().validation_metric_names) + # Connect to SQLite database + # WAL mode ensures crash-safe writes + self.db_path = os.path.join(self.output_folder, "results.db") + self.conn = sqlite3.connect(self.db_path) + self.conn.execute("PRAGMA journal_mode=WAL;") + cursor = self.conn.cursor() + + # Initialize or resume the total_results table + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='total_results'") + if cursor.fetchone() is None: + # First run: seed from existing CSV if available, otherwise start empty + if os.path.exists(self.all_scores_output_filename): + results_df = pd.read_csv(self.all_scores_output_filename) + if results_df.columns[0] != 'Fold': + results_df = pd.read_csv(self.all_scores_output_filename, index_col=0) + missing_metrics = [x for x in self.metric_names if + not x in list(results_df.columns)[1:]] + for m in missing_metrics: + results_df[m] = None + else: + results_df = pd.DataFrame(columns=self.results_df_base_columns) + + results_df.to_sql("total_results", self.conn, if_exists="replace", index=False) + cursor.execute("CREATE INDEX IF NOT EXISTS idx_total_search ON total_results ([Patient], [Fold]);") + self.conn.commit() + else: + logging.info("Existing database found, resuming from SQL...") + if not os.path.exists(self.all_scores_output_filename): + self.__sqlite_to_csv("total_results", self.all_scores_output_filename) + + """ if not os.path.exists(self.all_scores_output_filename): self.results_df = pd.DataFrame(columns=self.results_df_base_columns) else: @@ -75,6 +107,16 @@ def __compute_metrics(self): for m in missing_metrics: self.results_df[m] = None self.results_df['Patient'] = self.results_df.Patient.astype(str) + """ + + # Add any missing extra metric columns to all tables (e.g. when new metrics are added after initial run) + for metric_name in self.metric_names: + for table in ["total_results"]: + try: + cursor.execute(f"ALTER TABLE [{table}] ADD COLUMN [{metric_name}] REAL") + self.conn.commit() + except sqlite3.OperationalError: + pass # Column already exists results_per_folds = [] for fold in range(0, self.fold_number): @@ -86,6 +128,9 @@ def __compute_metrics(self): results = self.__compute_metrics_for_fold(data_list=test_set, fold_number=fold) results_per_folds.append(results) + logging.info("Exporting results to CSV...") + self.__sqlite_to_csv("total_results", self.all_scores_output_filename) + def __compute_metrics_for_fold(self, data_list, fold_number): for i, patient in enumerate(tqdm(data_list)): uid = None @@ -165,10 +210,11 @@ def __identify_patient_files(self, patient_metrics, folder_index, fold_number, m # ts = os.path.basename(detection_filename).split('.')[0].split('_')[-3] # image_uid = uid + '_' + ts # gt_class = masterfile.loc[masterfile["Patient"] == image_uid]["Sequence"].values[0] - gt_class = os.path.basename(detection_filename).split('.')[0].split('_')[-4] - gt_result = np.zeros(len(classes)).astype('uint8') - gt_result[classes.index(gt_class)] = 1 - np.savetxt(gt_results_filename, gt_result, delimiter=";") + if False: #Only for sequence classification + gt_class = os.path.basename(detection_filename).split('.')[0].split('_')[-4] + gt_result = np.zeros(len(classes)).astype('uint8') + gt_result[classes.index(gt_class)] = 1 + np.savetxt(gt_results_filename, gt_result, delimiter=";") patient_filenames = [gt_results_filename, detection_filename] patient_metrics.set_patient_filenames(patient_filenames) @@ -179,6 +225,7 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): Compute the basic metrics for all classes of the current patient :return: """ + cursor = self.conn.cursor() uid = patient_metrics.patient_id classes = SharedResources.getInstance().validation_class_names nb_classes = len(classes) @@ -210,6 +257,39 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): # self.class_results_df[classes[c]].loc[sub_df.index.values[0], :] = ind_values # self.results_df.to_csv(self.all_scores_output_filename, index=False) + current_columns = pd.read_sql_query(f"SELECT * FROM {'total_results'} LIMIT 0", self.conn).columns.tolist() + query = f"SELECT * from {'total_results'} where [Patient] = ? AND [Fold] = ?" + cursor.execute(query, (str(uid), fold_number)) + + row = cursor.fetchone() + + if row is None: + ind_values = [fold_number, uid] + list(pat_results) + # Pad with None if fewer values than columns (e.g. when extra metrics are missing) + if len(ind_values) < len(current_columns): + ind_values += [None] * (len(current_columns) - len(ind_values)) + + column_names_str = ", ".join([f"[{col}]" for col in current_columns]) + placeholders = ", ".join(["?"] * len(current_columns)) + insert_query = f"INSERT INTO {'total_results'} ({column_names_str}) VALUES ({placeholders})" + cursor.execute(insert_query, tuple(ind_values)) + else: + row_dict = dict(zip(current_columns, row)) + new_values = [fold_number, uid] + list(pat_results) + + for i, val in enumerate(new_values): + if i < len(current_columns): + row_dict[current_columns[i]] = float(val) if isinstance(val, np.float32) else val + + ind_values = [row_dict.get(col, None) for col in current_columns] + set_clause = ", ".join([f"[{col}] = ?" for col in current_columns]) + + update_query = f"UPDATE {'total_results'} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ?" + cursor.execute(update_query, tuple(ind_values) + (str(uid), fold_number)) + + self.conn.commit() + + """ # Filling in the csv files on disk for faster resume sub_df = self.results_df.loc[ (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number)] @@ -222,6 +302,7 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): ind_values = [fold_number, uid] + list(pat_results) self.results_df.loc[sub_df.index.values[0], :] = ind_values self.results_df.to_csv(self.all_scores_output_filename, index=False) + """ def __compute_extra_metrics(self, class_optimal: dict = {}): """ @@ -244,3 +325,17 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): metric_value = pm[1] self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) + + def __sqlite_to_csv(self, table_name, csv_path): + """ + Exports a full SQLite table to a CSV file using a streaming cursor + to avoid loading the entire table into memory. + """ + cursor = self.conn.cursor() + cursor.execute(f"SELECT * FROM [{table_name}]") + headers = [description[0] for description in cursor.description] + + with open(csv_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(cursor) \ No newline at end of file From fe0ca8c629a6b2cb72144cbf6ba23150a99cb916 Mon Sep 17 00:00:00 2001 From: dbouget Date: Wed, 1 Jul 2026 15:02:04 +0000 Subject: [PATCH 10/14] Using pool execution for speeding up metrics computation [skip ci] --- .../Validation/kfold_model_validation.py | 66 +++++++++---------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 8cbf043..17e8dc8 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -496,24 +496,19 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): optimal_values = class_optimal[c]['All'] if len(SharedResources.getInstance().validation_metric_names) < 10: batch_results =[] - dump = 0 - with ThreadPoolExecutor(max_workers=SharedResources.getInstance().number_processes) as executor: - futures = {executor.submit(partial(self.__patient_metrics_computation, c=c, classes=classes, - optimal_values=optimal_values), item): item for item in self.patients_metrics} - for future in tqdm(as_completed(futures), total=len(futures)): - _ = futures[future] - try: - results = future.result() - batch_results.append(results) - dump += 1 - if dump % SharedResources.getInstance().number_processes == 0: + args_list = [(p, self.patients_metrics[p], c, classes, optimal_values, SharedResources.getInstance().validation_metric_names) for p in self.patients_metrics] + with ProcessPoolExecutor(max_workers=SharedResources.getInstance().number_processes) as executor: + for result in tqdm(executor.map(patient_metrics_computation_worker, args_list), total=len(args_list)): + if result is not None: + batch_results.append(result) + if len(batch_results) >= SharedResources.getInstance().number_processes: self.__update_database(batch_results) batch_results.clear() - except Exception as e: - continue + + if batch_results: # flush remainder + self.__update_database(batch_results) + batch_results.clear() else: - original_processes = SharedResources.getInstance().number_processes - SharedResources.getInstance().number_processes = 10 for p in tqdm(self.patients_metrics): recomputation = False try: @@ -541,26 +536,6 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): finally: if recomputation: self.conn.commit() - SharedResources.getInstance().number_processes = original_processes - - - def __patient_metrics_computation(self, p, c, classes, optimal_values): - recomputation = False - result = None - try: - # Initializing/completing the list which will hold the extra metrics - thr_to_match = float(np.round(optimal_values[1], 2)) - self.patients_metrics[p].setup_extra_metrics(self.metric_names) - pat_metrics, recomputation = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], - SharedResources.getInstance().validation_metric_names) - if recomputation: - self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) - - result = [p, c, thr_to_match, pat_metrics, recomputation] - except Exception as e: - logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") - finally: - return result def __update_database(self, batch_results): try: @@ -597,4 +572,23 @@ def __sqlite_to_csv(self, table_name, csv_path): with open(csv_path, 'w', newline='', encoding='utf-8') as f: writer = csv.writer(f) writer.writerow(headers) - writer.writerows(cursor) \ No newline at end of file + writer.writerows(cursor) + +def patient_metrics_computation_worker(args): + p, metrics, c, classes, optimal_values, metric_names = args + recomputation = False + result = None + try: + # Initializing/completing the list which will hold the extra metrics + thr_to_match = float(np.round(optimal_values[1], 2)) + metrics.setup_extra_metrics(metric_names) + pat_metrics, recomputation = compute_patient_extra_metrics(metrics, classes.index(c), optimal_values[1], + SharedResources.getInstance().validation_metric_names) + if recomputation: + metrics.set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) + + result = [p, c, thr_to_match, pat_metrics, recomputation] + except Exception as e: + logging.error(f"Computing extra metrics for patient {metrics.patient_id} failed with: {e}.\n{traceback.format_exc()}") + finally: + return result From 4787b83a81d8fe6c62295cd59292748ed1f73b0d Mon Sep 17 00:00:00 2001 From: dbouget Date: Fri, 3 Jul 2026 11:25:58 +0000 Subject: [PATCH 11/14] Updating the CIs [skip ci] --- .github/workflows/build.yml | 43 ++++++++++++++++--------------------- main.py | 2 ++ raidionicsval/__main__.py | 2 ++ 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a36dfc..23d82c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,11 +18,11 @@ jobs: CMD_BUILD: python -m build --wheel steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 with: - python-version: "3.9" + python-version: "3.11" - name: Install dependencies run: | @@ -33,7 +33,7 @@ jobs: run: ${{matrix.CMD_BUILD}} - name: Upload Python wheel - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: Python wheel path: ${{github.workspace}}/dist/raidionicsval-*.whl @@ -42,12 +42,12 @@ jobs: setup-test-data: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.9 + python-version: 3.11 - name: Download test resources working-directory: tests @@ -56,7 +56,7 @@ jobs: python -c "from download_resources import download_resources; download_resources('../test_data')" - name: Upload test resources - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-resources path: ./test_data @@ -106,14 +106,6 @@ jobs: python-version: "3.12" - os: windows-2025 python-version: "3.13" - - os: macos-13 - python-version: "3.9" - - os: macos-13 - python-version: "3.10" - - os: macos-13 - python-version: "3.11" - - os: macos-13 - python-version: "3.12" - os: macos-14 python-version: "3.10" - os: macos-14 @@ -130,15 +122,14 @@ jobs: python-version: "3.12" - os: macos-15 python-version: "3.13" - steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: "Python wheel" @@ -152,22 +143,24 @@ jobs: run: raidionicsval --help - name: Clone repo - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Download test resources - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: test-resources path: ./tests/unit_tests_results_dir - name: Integration tests + env: + MPLBACKEND: Agg run: | pip install pytest pytest-cov pytest-timeout requests - pytest -vvv --cov=raidionicsval ${{github.workspace}}/tests/generic_tests --cov-report=xml --timeout=1000 --log-cli-level=DEBUG + pytest -vvv --cov=raidionicsval ${{github.workspace}}/tests/generic_tests --cov-report=xml --timeout=1000 --log-cli-level=INFO - name: Upload coverage to Codecov - if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.9' }} - uses: codecov/codecov-action@v4 + if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} slug: dbouget/validation_metrics_computation diff --git a/main.py b/main.py index 1980e19..45b37ec 100644 --- a/main.py +++ b/main.py @@ -42,4 +42,6 @@ def main(argv): if __name__ == "__main__": + import matplotlib + matplotlib.use("Agg") main(sys.argv[1:]) diff --git a/raidionicsval/__main__.py b/raidionicsval/__main__.py index 82f54a4..ce179a2 100644 --- a/raidionicsval/__main__.py +++ b/raidionicsval/__main__.py @@ -41,5 +41,7 @@ def main(): if __name__ == "__main__": logging.info("Internal main call.\n") + import matplotlib + matplotlib.use("Agg") main() From 52f647db8e44ddc5d33b60362c60c4c72812c243 Mon Sep 17 00:00:00 2001 From: dbouget Date: Fri, 3 Jul 2026 11:55:34 +0000 Subject: [PATCH 12/14] Trying results investigation when tests failed on CI [skip ci] --- .github/workflows/build.yml | 9 +++++++++ tests/conftest.py | 7 ++++--- tests/generic_tests/test_validation_pipeline_advanced.py | 8 ++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23d82c3..dc53a6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -158,6 +158,15 @@ jobs: pip install pytest pytest-cov pytest-timeout requests pytest -vvv --cov=raidionicsval ${{github.workspace}}/tests/generic_tests --cov-report=xml --timeout=1000 --log-cli-level=INFO + - name: Upload integration test results + if: failure() + uses: actions/upload-artifact@v7 + with: + name: integration-test-results-${{ matrix.os }}-py${{ matrix.python-version }} + path: tests/unit_tests_results_dir/ + if-no-files-found: ignore + retention-days: 1 + - name: Upload coverage to Codecov if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }} uses: codecov/codecov-action@v6 diff --git a/tests/conftest.py b/tests/conftest.py index 17f9f65..b0fefc7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ def test_dir(): if not os.listdir(test_dir): download_resources(test_dir=test_dir) yield test_dir - logging.info(f"Removing the temporary directory for tests.") - if os.path.exists(test_dir): - shutil.rmtree(test_dir) \ No newline at end of file + if os.environ.get("GITHUB_ACTIONS") != "true": # Allowing to collect the folder content as an artifact if the test failed in the CI + logging.info(f"Removing the temporary directory for tests.") + if os.path.exists(test_dir): + shutil.rmtree(test_dir) \ No newline at end of file diff --git a/tests/generic_tests/test_validation_pipeline_advanced.py b/tests/generic_tests/test_validation_pipeline_advanced.py index 1e7cfde..ad14b7a 100644 --- a/tests/generic_tests/test_validation_pipeline_advanced.py +++ b/tests/generic_tests/test_validation_pipeline_advanced.py @@ -61,8 +61,8 @@ def test_validation_pipeline_metrics_space(test_dir): assert round(gt_df['OW Recall (Mean)'][0], 2) == round(results_df['OW Recall (Mean)'][0],2), "OW Recall (Mean) values do not match" except Exception as e: logging.error(f"Error during k-fold cross-validation unit test with: {e} \n {traceback.format_exc()}.\n") - if os.path.exists(output_folder): - shutil.rmtree(output_folder) + # if os.path.exists(output_folder): + # shutil.rmtree(output_folder) raise ValueError("Error during k-fold cross-validation unit test with.\n") logging.info("k-fold cross-validation unit test succeeded.\n") @@ -121,8 +121,8 @@ def test_validation_pipeline_extra_metrics(test_dir): assert round(gt_df['OW MI (Std)'][0], 2) == round(results_df['OW MI (Std)'][0],2), "OW MI (Std) values do not match" except Exception as e: logging.error(f"Error during test with: {e} \n {traceback.format_exc()}.\n") - if os.path.exists(output_folder): - shutil.rmtree(output_folder) + # if os.path.exists(output_folder): + # shutil.rmtree(output_folder) raise ValueError(f"{e}.\n") if os.path.exists(output_folder): From b8b6f3230392f6ac42886286d7a123ae5627cf36 Mon Sep 17 00:00:00 2001 From: dbouget Date: Fri, 3 Jul 2026 12:44:36 +0000 Subject: [PATCH 13/14] Improving Action debugging [skip ci] --- .github/workflows/build.yml | 5 +- .../Validation/extra_metrics_computation.py | 313 +++++++++--------- tests/download_resources.py | 2 +- 3 files changed, 161 insertions(+), 159 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc53a6f..6a5b031 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -163,7 +163,10 @@ jobs: uses: actions/upload-artifact@v7 with: name: integration-test-results-${{ matrix.os }}-py${{ matrix.python-version }} - path: tests/unit_tests_results_dir/ + path: | + tests/unit_tests_results_dir/** + !tests/unit_tests_results_dir/**/Inputs/** + !tests/unit_tests_results_dir/**/Predictions/** if-no-files-found: ignore retention-days: 1 diff --git a/raidionicsval/Validation/extra_metrics_computation.py b/raidionicsval/Validation/extra_metrics_computation.py index 96a0d03..746c9d7 100644 --- a/raidionicsval/Validation/extra_metrics_computation.py +++ b/raidionicsval/Validation/extra_metrics_computation.py @@ -99,8 +99,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold pool.close() pool.join() except Exception as e: - print("Issue computing metrics for patient {} in the multiprocessing loop.".format(patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f"Issue computing metrics for patient {patient_object.unique_id} in the multiprocessing loop. Collected {e}\n{traceback.format_exc()}") else: for metric in metrics: try: @@ -111,8 +110,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold det_spacing=det_input_specs[1]) extra_metrics_results.append([metric, metric_value]) except Exception as e: - print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f'Issue computing metric {metric} for patient {patient_object.unique_id}. Collected {e}\n{traceback.format_exc()}') extra_metrics_results = [[f'PiW {x[0]}', x[1]] for x in extra_metrics_results] if "objectwise" in SharedResources.getInstance().validation_metric_spaces: @@ -179,8 +177,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold det_spacing=det_input_specs[1]) instance_results.append([metric, metric_value]) except Exception as e: - print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f'Issue computing metric {metric} for patient {patient_object.unique_id}. Collected {e} \n {traceback.format_exc()}') all_instance_results.append(instance_results) if len(all_instance_results) != 0: @@ -245,160 +242,162 @@ def parallel_metric_computation(args): def compute_specific_metric_value(metric, gt, detection, tp, tn, fp, fn, gt_spacing, det_spacing): metric_value = None - if metric == 'VS': - metric_value = math.inf - den = (2 * tp) + fp + fn - if den != 0: - metric_value = 1 - ((abs(fn - fp)) / ((2 * tp) + fp + fn)) - elif metric == 'GCE': - if (tp + fn) != 0 and (tn + fp) != 0 and (tp + fp) != 0 and (tn + fn) != 0: - param11 = (fn * (fn + (2 * tp))) / (tp + fn) - param12 = (fp * (fp + (2 * tn))) / (tn + fp) - param21 = (fp * (fp + (2 * tp))) / (tp + fp) - param22 = (fn * (fn + (2 * tn))) / (tn + fn) - # metric_value = (1 / np.prod(gt_ni_header.get_data_shape()[0:3])) * min(param11 + param12, param21 + param22) - metric_value = (1 / np.prod(gt.shape)) * min(param11 + param12, param21 + param22) - else: + try: + if metric == 'VS': metric_value = math.inf - elif metric == 'MI': - metric_value = normalized_mutual_info_score(gt.flatten(), detection.flatten()) - elif metric == 'RI': - metric_value = 0. - a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) - b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) - d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) - num = a + b - den = a + b + c + d - if den != 0: - metric_value = num / den - elif metric == 'ARI': - metric_value = 0. - a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) - b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) - d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) - num = 2 * (a * d - b * c) - den = math.pow(c, 2) + math.pow(b, 2) + 2 * a * d + (a + d) * (c + b) - if den != 0: - metric_value = num / den - elif metric == 'VOI': - fn_tp = fn + tp - fp_tp = fp + tp - # total = np.prod(gt_ni_header.get_data_shape()[0:3]) - total = np.prod(gt.shape) - - if fn_tp == 0 or (fn_tp / total) == 1 or fp_tp == 0 or (fp_tp / total) == 1: + den = (2 * tp) + fp + fn + if den != 0: + metric_value = 1 - ((abs(fn - fp)) / ((2 * tp) + fp + fn)) + elif metric == 'GCE': + if (tp + fn) != 0 and (tn + fp) != 0 and (tp + fp) != 0 and (tn + fn) != 0: + param11 = (fn * (fn + (2 * tp))) / (tp + fn) + param12 = (fp * (fp + (2 * tn))) / (tn + fp) + param21 = (fp * (fp + (2 * tp))) / (tp + fp) + param22 = (fn * (fn + (2 * tn))) / (tn + fn) + # metric_value = (1 / np.prod(gt_ni_header.get_data_shape()[0:3])) * min(param11 + param12, param21 + param22) + metric_value = (1 / np.prod(gt.shape)) * min(param11 + param12, param21 + param22) + else: + metric_value = math.inf + elif metric == 'MI': + metric_value = normalized_mutual_info_score(gt.flatten(), detection.flatten()) + elif metric == 'RI': + metric_value = 0. + a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) + b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) + d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) + num = a + b + den = a + b + c + d + if den != 0: + metric_value = num / den + elif metric == 'ARI': + metric_value = 0. + a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) + b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) + d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) + num = 2 * (a * d - b * c) + den = math.pow(c, 2) + math.pow(b, 2) + 2 * a * d + (a + d) * (c + b) + if den != 0: + metric_value = num / den + elif metric == 'VOI': + fn_tp = fn + tp + fp_tp = fp + tp + # total = np.prod(gt_ni_header.get_data_shape()[0:3]) + total = np.prod(gt.shape) + + if fn_tp == 0 or (fn_tp / total) == 1 or fp_tp == 0 or (fp_tp / total) == 1: + metric_value = math.inf + else: + h1 = -((fn_tp / total) * math.log2(fn_tp / total) + (1 - fn_tp / total) * math.log2(1 - fn_tp / total)) + h2 = -((fp_tp / total) * math.log2(fp_tp / total) + (1 - fp_tp / total) * math.log2(1 - fp_tp / total)) + + p00 = 1 if tn == 0 else (tn / total) + p01 = 1 if fn == 0 else (fn / total) + p10 = 1 if fp == 0 else (fp / total) + p11 = 1 if tp == 0 else (tp / total) + + h12 = -((tn / total) * math.log2(p00) + (fn / total) * math.log2(p01) + (fp / total) * math.log2(p10) + (tp / total) * math.log2(p11)) + mi = h1 + h2 - h12 + metric_value = h1 + h2 - (2 * mi) + elif metric == 'Jaccard': + metric_value = jaccard_score(gt.flatten(), detection.flatten()) + elif metric == 'IOU': metric_value = math.inf - else: - h1 = -((fn_tp / total) * math.log2(fn_tp / total) + (1 - fn_tp / total) * math.log2(1 - fn_tp / total)) - h2 = -((fp_tp / total) * math.log2(fp_tp / total) + (1 - fp_tp / total) * math.log2(1 - fp_tp / total)) - - p00 = 1 if tn == 0 else (tn / total) - p01 = 1 if fn == 0 else (fn / total) - p10 = 1 if fp == 0 else (fp / total) - p11 = 1 if tp == 0 else (tp / total) - - h12 = -((tn / total) * math.log2(p00) + (fn / total) * math.log2(p01) + (fp / total) * math.log2(p10) + (tp / total) * math.log2(p11)) - mi = h1 + h2 - h12 - metric_value = h1 + h2 - (2 * mi) - elif metric == 'Jaccard': - metric_value = jaccard_score(gt.flatten(), detection.flatten()) - elif metric == 'IOU': - metric_value = math.inf - intersection = (gt == 1) & (detection == 1) - union = (gt == 1) | (detection == 1) - if np.count_nonzero(union) != 0: - metric_value = np.count_nonzero(intersection) / np.count_nonzero(union) - elif metric == 'TPR': - metric_value = math.inf - if (tp + fn) != 0: - metric_value = tp / (tp + fn) - elif metric == 'TNR': - metric_value = math.inf - if (tn + fp) != 0: - metric_value = tn / (tn + fp) - elif metric == 'FPR': - metric_value = math.inf - if (fp + tn) != 0: - metric_value = fp / (fp + tn) - elif metric == 'FNR': - metric_value = math.inf - if (fn + tp) != 0: - metric_value = fn / (fn + tp) - elif metric == 'PPV': - metric_value = math.inf - if (tp + fp) != 0: - metric_value = tp / (tp + fp) - elif metric == 'AUC': - metric_value = math.inf - if len(np.unique(gt)) > 1: # ROC AUC score is not defined if only class is present in y_true - metric_value = roc_auc_score(gt.flatten(), detection.flatten()) - elif metric == 'MCC': - metric_value = math.inf - num = (tp * tn) - (fp * fn) - den = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) - if den != 0: - metric_value = num / den - elif metric == 'CKS': - metric_value = cohen_kappa_score(gt.flatten(), detection.flatten()) - elif metric == 'HD95': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_hd95(detection, gt, voxelspacing=det_spacing, connectivity=1) - elif metric == 'ASSD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_assd(detection, gt, voxel_spacing=det_spacing) - elif metric == 'OASSD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_object_assd(detection, gt, voxel_spacing=det_spacing) - elif metric == 'RAVD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_ravd(detection, gt) - elif metric == 'VC': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value, pval = compute_volume_correlation(detection, gt) - elif metric == 'MahaD': - metric_value = math.inf - gt_n = np.count_nonzero(detection) - seg_n = np.count_nonzero(gt) - - if gt_n != 0 and seg_n != 0: - gt_indices = np.flip(np.where(gt == 1), axis=0) - gt_mean = gt_indices.mean(axis=1) - gt_cov = np.cov(gt_indices) - - seg_indices = np.flip(np.where(detection == 1), axis=0) - seg_mean = seg_indices.mean(axis=1) - seg_cov = np.cov(seg_indices) - - # calculate common covariance matrix - common_cov = (gt_n * gt_cov + seg_n * seg_cov) / (gt_n + seg_n) - common_cov_inv = np.linalg.inv(common_cov) - - mean = gt_mean - seg_mean - metric_value = math.sqrt(mean.dot(common_cov_inv).dot(mean.T)) - elif metric == 'ProbD': - # metric_value = -1. - metric_value = math.inf - gt_flat = gt.flatten().astype(np.int8) - det_flat = detection.flatten().astype(np.int8) - - probability_difference = np.absolute(gt_flat - det_flat).sum() - probability_joint = (gt_flat * det_flat).sum() - - if probability_joint != 0: - metric_value = probability_difference / (2. * probability_joint) - else: - logging.warning("Metric with name {} has not been implemented!".format(metric)) - return metric_value + intersection = (gt == 1) & (detection == 1) + union = (gt == 1) | (detection == 1) + if np.count_nonzero(union) != 0: + metric_value = np.count_nonzero(intersection) / np.count_nonzero(union) + elif metric == 'TPR': + metric_value = math.inf + if (tp + fn) != 0: + metric_value = tp / (tp + fn) + elif metric == 'TNR': + metric_value = math.inf + if (tn + fp) != 0: + metric_value = tn / (tn + fp) + elif metric == 'FPR': + metric_value = math.inf + if (fp + tn) != 0: + metric_value = fp / (fp + tn) + elif metric == 'FNR': + metric_value = math.inf + if (fn + tp) != 0: + metric_value = fn / (fn + tp) + elif metric == 'PPV': + metric_value = math.inf + if (tp + fp) != 0: + metric_value = tp / (tp + fp) + elif metric == 'AUC': + metric_value = math.inf + if len(np.unique(gt)) > 1: # ROC AUC score is not defined if only class is present in y_true + metric_value = roc_auc_score(gt.flatten(), detection.flatten()) + elif metric == 'MCC': + metric_value = math.inf + num = (tp * tn) - (fp * fn) + den = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + if den != 0: + metric_value = num / den + elif metric == 'CKS': + metric_value = cohen_kappa_score(gt.flatten(), detection.flatten()) + elif metric == 'HD95': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_hd95(detection, gt, voxelspacing=det_spacing, connectivity=1) + elif metric == 'ASSD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_assd(detection, gt, voxel_spacing=det_spacing) + elif metric == 'OASSD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_object_assd(detection, gt, voxel_spacing=det_spacing) + elif metric == 'RAVD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_ravd(detection, gt) + elif metric == 'VC': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value, pval = compute_volume_correlation(detection, gt) + elif metric == 'MahaD': + metric_value = math.inf + gt_n = np.count_nonzero(detection) + seg_n = np.count_nonzero(gt) + + if gt_n != 0 and seg_n != 0: + gt_indices = np.flip(np.where(gt == 1), axis=0) + gt_mean = gt_indices.mean(axis=1) + gt_cov = np.cov(gt_indices) + + seg_indices = np.flip(np.where(detection == 1), axis=0) + seg_mean = seg_indices.mean(axis=1) + seg_cov = np.cov(seg_indices) + + # calculate common covariance matrix + common_cov = (gt_n * gt_cov + seg_n * seg_cov) / (gt_n + seg_n) + common_cov_inv = np.linalg.inv(common_cov) + + mean = gt_mean - seg_mean + metric_value = math.sqrt(mean.dot(common_cov_inv).dot(mean.T)) + elif metric == 'ProbD': + # metric_value = -1. + metric_value = math.inf + gt_flat = gt.flatten().astype(np.int8) + det_flat = detection.flatten().astype(np.int8) + + probability_difference = np.absolute(gt_flat - det_flat).sum() + probability_joint = (gt_flat * det_flat).sum() + if probability_joint != 0: + metric_value = probability_difference / (2. * probability_joint) + else: + logging.warning("Metric with name {} has not been implemented!".format(metric)) + return metric_value + except Exception as e: + raise ValueError(f"Computing {metric} failed with {e}") from e def compute_overall_metrics_correlation(input_folder, output_folder, data=None, class_name=None, best_threshold=0.5, best_overlap=0.0, suffix=None): diff --git a/tests/download_resources.py b/tests/download_resources.py index 4677718..89fffb2 100644 --- a/tests/download_resources.py +++ b/tests/download_resources.py @@ -23,7 +23,7 @@ def download_resources(test_dir: str): f.write(chunk) with zipfile.ZipFile(archive_dl_dest, 'r') as zip_ref: zip_ref.extractall(dest_dir) - + os.remove(archive_dl_dest) except Exception as e: logging.error(f"Error during input data download with: {e} \n {traceback.format_exc()}\n") if os.path.exists(dest_dir): From 5a6db024d57365dd7a4e3de2845629f4922586e0 Mon Sep 17 00:00:00 2001 From: dbouget Date: Fri, 3 Jul 2026 13:22:25 +0000 Subject: [PATCH 14/14] Debugging [skip ci] --- raidionicsval/Validation/kfold_model_validation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 17e8dc8..1c561c2 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -494,7 +494,7 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): classes = SharedResources.getInstance().validation_class_names for c in classes: optimal_values = class_optimal[c]['All'] - if len(SharedResources.getInstance().validation_metric_names) < 10: + if len(SharedResources.getInstance().validation_metric_names) < 10 and SharedResources.getInstance().number_processes != 1: batch_results =[] args_list = [(p, self.patients_metrics[p], c, classes, optimal_values, SharedResources.getInstance().validation_metric_names) for p in self.patients_metrics] with ProcessPoolExecutor(max_workers=SharedResources.getInstance().number_processes) as executor: @@ -535,10 +535,14 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): continue finally: if recomputation: + logging.debug("Updating the database with extra metrics results patient-wise") self.conn.commit() + else: + logging.debug("Skipping extra metrics recomputation") def __update_database(self, batch_results): try: + logging.debug("Updating the database with extra metrics results batch-wise") cursor = self.conn.cursor() for results in batch_results: p = results[0]