diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a36dfc..6a5b031 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,11 +18,11 @@ jobs: CMD_BUILD: python -m build --wheel steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 with: - python-version: "3.9" + python-version: "3.11" - name: Install dependencies run: | @@ -33,7 +33,7 @@ jobs: run: ${{matrix.CMD_BUILD}} - name: Upload Python wheel - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: Python wheel path: ${{github.workspace}}/dist/raidionicsval-*.whl @@ -42,12 +42,12 @@ jobs: setup-test-data: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.9 + python-version: 3.11 - name: Download test resources working-directory: tests @@ -56,7 +56,7 @@ jobs: python -c "from download_resources import download_resources; download_resources('../test_data')" - name: Upload test resources - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-resources path: ./test_data @@ -106,14 +106,6 @@ jobs: python-version: "3.12" - os: windows-2025 python-version: "3.13" - - os: macos-13 - python-version: "3.9" - - os: macos-13 - python-version: "3.10" - - os: macos-13 - python-version: "3.11" - - os: macos-13 - python-version: "3.12" - os: macos-14 python-version: "3.10" - os: macos-14 @@ -130,15 +122,14 @@ jobs: python-version: "3.12" - os: macos-15 python-version: "3.13" - steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: "Python wheel" @@ -152,22 +143,36 @@ jobs: run: raidionicsval --help - name: Clone repo - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Download test resources - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: test-resources path: ./tests/unit_tests_results_dir - name: Integration tests + env: + MPLBACKEND: Agg run: | pip install pytest pytest-cov pytest-timeout requests - pytest -vvv --cov=raidionicsval ${{github.workspace}}/tests/generic_tests --cov-report=xml --timeout=1000 --log-cli-level=DEBUG + pytest -vvv --cov=raidionicsval ${{github.workspace}}/tests/generic_tests --cov-report=xml --timeout=1000 --log-cli-level=INFO + + - name: Upload integration test results + if: failure() + uses: actions/upload-artifact@v7 + with: + name: integration-test-results-${{ matrix.os }}-py${{ matrix.python-version }} + path: | + tests/unit_tests_results_dir/** + !tests/unit_tests_results_dir/**/Inputs/** + !tests/unit_tests_results_dir/**/Predictions/** + if-no-files-found: ignore + retention-days: 1 - name: Upload coverage to Codecov - if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.9' }} - uses: codecov/codecov-action@v4 + if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} slug: dbouget/validation_metrics_computation diff --git a/main.py b/main.py index 1980e19..45b37ec 100644 --- a/main.py +++ b/main.py @@ -42,4 +42,6 @@ def main(argv): if __name__ == "__main__": + import matplotlib + matplotlib.use("Agg") main(sys.argv[1:]) diff --git a/raidionicsval/Utils/PatientMetricsStructure.py b/raidionicsval/Utils/PatientMetricsStructure.py index 2f3fbe0..d5b8663 100644 --- a/raidionicsval/Utils/PatientMetricsStructure.py +++ b/raidionicsval/Utils/PatientMetricsStructure.py @@ -106,6 +106,87 @@ def init_from_file(self, study_folder: str): else: self.__init_from_file_classification(study_folder=study_folder) + def init_from_db(self, conn): + if self.objective == "segmentation": + self.__init_from_db_segmentation(conn) + else: + # @TODO. Add database-backed loading for classification objective + pass + + def __init_from_db_segmentation(self, conn): + """ + Loads segmentation metrics for this patient from SQLite. + Fetches per-class results first, then the macro-averaged total_results rows. + Returns early if no data exists yet (i.e. first run). + """ + # Load per-class metrics using ClassMetrics.init_from_df + for c in list(self._class_metrics.keys()): + table_name = f"class_{c}" + try: + rows_df = pd.read_sql_query( + f"SELECT * FROM [{table_name}] WHERE [Patient] = ? AND [Fold] = ?", + conn, + params=(str(self._patient_id), self._fold_number) + ) + self._class_metrics[c].init_from_df(rows_df) + except Exception: + return # Table does not exist yet on first run + + # Load macro-averaged results from total_results + rows_df = pd.read_sql_query( + "SELECT * FROM [total_results] WHERE [Patient] = ? AND [Fold] = ?", + conn, + params=(str(self._patient_id), self._fold_number) + ) + + if len(rows_df) == 0: + return + + def to_float(v): + # Convert to float since SQLite returns numeric values as strings + try: + return float(v) if v is not None else None + except (ValueError, TypeError): + return None + + self._patientwise_metrics = [] + self._pixelwise_metrics = [] + self._objectwise_metrics = [] + self._extra_metrics = [] + + upper_idx = SharedResources.getInstance().upper_default_metrics_index + extra_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + extra_metric_names.extend([f'PiW {m}', f'OW {m}']) + + for thr in rows_df["Threshold"].values: + thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] + thr_val = float(thr_results[2]) + pixelwise_values = [to_float(x) for x in thr_results[3:7]] + patientwise_values = [to_float(x) for x in thr_results[7:10]] + objectwise_values = [to_float(x) for x in thr_results[10:upper_idx]] + extra_values = [to_float(x) for x in thr_results[upper_idx:]] + + # Pad missing extra metrics with None to match expected length + while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): + extra_values.append(None) + + extra_values_description = list(rows_df.columns[upper_idx:]) + for name in extra_metric_names: + if name not in extra_values_description: + extra_values_description.append(name) + + self._pixelwise_metrics.append([thr_val] + pixelwise_values) + self._patientwise_metrics.append([thr_val] + patientwise_values) + self._objectwise_metrics.append([thr_val] + objectwise_values) + + extra_values_cat = [[x, y] for x, y in zip(extra_values_description, extra_values)] + if extra_values_cat: + self._extra_metrics.append([thr_val] + extra_values_cat) + + if len(self._extra_metrics) == 0: + self._extra_metrics = None + def __init_from_file_segmentation(self, study_folder: str): all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv') @@ -313,7 +394,7 @@ def set_results(self, results): self._objectwise_metrics = [] for index in range(len(results)): thr_results = results[index][0] - thr_val = thr_results[2] + thr_val = float(thr_results[2]) pixelwise_values = thr_results[3:7] patientwise_values = thr_results[7:10] objectwise_values = thr_results[10:SharedResources.getInstance().upper_default_metrics_index] @@ -358,6 +439,59 @@ def init_from_file(self, scores_filename: str) -> None: if len(self._extra_metrics) == 0: self._extra_metrics = None + def init_from_df(self, rows_df: pd.DataFrame) -> None: + """ + Initializes class-level metrics from a pre-fetched DataFrame of per-threshold rows. + """ + if len(rows_df) == 0: + return + + upper_idx = SharedResources.getInstance().upper_default_metrics_index + + def to_float(v): + # Convert to float since SQLite returns numeric values as strings + try: + return float(v) if v is not None else None + except (ValueError, TypeError): + return None + + self._patientwise_metrics = [] + self._pixelwise_metrics = [] + self._objectwise_metrics = [] + self._extra_metrics = [] + + extra_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + extra_metric_names.extend([f'PiW {m}', f'OW {m}']) + + for thr in np.unique(rows_df["Threshold"].values): + thr_results = rows_df.loc[rows_df["Threshold"] == thr].values[0] + thr_val = float(thr_results[2]) + pixelwise_values = [to_float(x) for x in thr_results[3:7]] + patientwise_values = [to_float(x) for x in thr_results[7:10]] + objectwise_values = [to_float(x) for x in thr_results[10:upper_idx]] + extra_values = [to_float(x) for x in thr_results[upper_idx:]] + + # Pad missing extra metrics with NaN to match expected length + while len(extra_values) < 2 * len(SharedResources.getInstance().validation_metric_names): + extra_values.append(float('nan')) + + extra_values_description = list(rows_df.columns[upper_idx:]) + for name in extra_metric_names: + if name not in extra_values_description: + extra_values_description.append(name) + + self._pixelwise_metrics.append([thr_val] + pixelwise_values) + self._patientwise_metrics.append([thr_val] + patientwise_values) + self._objectwise_metrics.append([thr_val] + objectwise_values) + + extra_values_cat = [[x, y] for x, y in zip(extra_values_description, extra_values)] + if extra_values_cat: + self._extra_metrics.append([thr_val] + extra_values_cat) + + if len(self._extra_metrics) == 0: + self._extra_metrics = None + def is_complete(self): """ diff --git a/raidionicsval/Validation/extra_metrics_computation.py b/raidionicsval/Validation/extra_metrics_computation.py index 9721f50..746c9d7 100644 --- a/raidionicsval/Validation/extra_metrics_computation.py +++ b/raidionicsval/Validation/extra_metrics_computation.py @@ -8,7 +8,7 @@ import pandas as pd import nibabel as nib import numpy as np -from typing import List +from typing import List, Tuple # from medpy.metric.binary import hd95, volume_correlation, assd, ravd, obj_assd from sklearn.metrics import jaccard_score, normalized_mutual_info_score, roc_auc_score, cohen_kappa_score from ..Utils.resources import SharedResources @@ -18,26 +18,34 @@ from ..Validation.instance_segmentation_validation import * -def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold, metrics: List[str] = []): +def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold, metrics: List[str] = []) -> Tuple[List, bool]: + """ + + Parameters + ---------- + + Returns + ------- + The first element is the list with metric values, the second element is a boolean indicating if (re)computation was performed. + """ extra_metrics_results = [] try: - if (patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) is not None and - patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] is not None): - metric_values = [x[1] for x in patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:]] - if 'objectwise' in SharedResources.getInstance().validation_metric_spaces and 'patientwise' in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - elif 'patientwise' not in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values[1::2]]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - elif 'objectwise' not in SharedResources.getInstance().validation_metric_spaces: - if False not in [x == x for x in metric_values[0::2]]: - # If all metric values have been computed, i.e., no nan or None etc... - return patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold)[1:] - else: - metric_values = [None] * len(metrics) + existing = patient_object.get_optimal_class_extra_metrics(class_index, optimal_threshold) + if existing is not None: + current_metric_names = [] + for m in SharedResources.getInstance().validation_metric_names: + if 'patientwise' in SharedResources.getInstance().validation_metric_spaces: + current_metric_names.append(f'PiW {m}') + if 'objectwise' in SharedResources.getInstance().validation_metric_spaces: + current_metric_names.append(f'OW {m}') + + existing_dict = {x[0]: x[1] for x in existing[1:]} + values_to_check = [existing_dict.get(m) for m in current_metric_names if m in existing_dict] + + if values_to_check and all(v is not None and v == v for v in values_to_check): + return existing[1:], False + + metric_values = [None] * len(metrics) # ground_truth_ni = nib.load(patient_object._ground_truth_filepaths[class_index]) # detection_ni = nib.load(patient_object._prediction_filepaths[class_index]) @@ -91,8 +99,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold pool.close() pool.join() except Exception as e: - print("Issue computing metrics for patient {} in the multiprocessing loop.".format(patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f"Issue computing metrics for patient {patient_object.unique_id} in the multiprocessing loop. Collected {e}\n{traceback.format_exc()}") else: for metric in metrics: try: @@ -103,8 +110,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold det_spacing=det_input_specs[1]) extra_metrics_results.append([metric, metric_value]) except Exception as e: - print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f'Issue computing metric {metric} for patient {patient_object.unique_id}. Collected {e}\n{traceback.format_exc()}') extra_metrics_results = [[f'PiW {x[0]}', x[1]] for x in extra_metrics_results] if "objectwise" in SharedResources.getInstance().validation_metric_spaces: @@ -171,8 +177,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold det_spacing=det_input_specs[1]) instance_results.append([metric, metric_value]) except Exception as e: - print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id)) - print(traceback.format_exc()) + logging.error(f'Issue computing metric {metric} for patient {patient_object.unique_id}. Collected {e} \n {traceback.format_exc()}') all_instance_results.append(instance_results) if len(all_instance_results) != 0: @@ -199,7 +204,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold print('Global issue computing metrics for patient {}'.format(patient_object.unique_id)) print(traceback.format_exc()) - return extra_metrics_results + return extra_metrics_results, True def parallel_metric_computation(args): @@ -229,7 +234,7 @@ def parallel_metric_computation(args): gt_spacing=gt_extra[1], det_spacing=det_extra[1]) except Exception as e: - print('Computing {} gave an exception'.format(metric)) + logging.warning(f'Computing {metric} failed with {e}') pass return [metric, metric_value] @@ -237,158 +242,162 @@ def parallel_metric_computation(args): def compute_specific_metric_value(metric, gt, detection, tp, tn, fp, fn, gt_spacing, det_spacing): metric_value = None - if metric == 'VS': - metric_value = math.inf - den = (2 * tp) + fp + fn - if den != 0: - metric_value = 1 - ((abs(fn - fp)) / ((2 * tp) + fp + fn)) - elif metric == 'GCE': - if (tp + fn) != 0 and (tn + fp) != 0 and (tp + fp) != 0 and (tn + fn) != 0: - param11 = (fn * (fn + (2 * tp))) / (tp + fn) - param12 = (fp * (fp + (2 * tn))) / (tn + fp) - param21 = (fp * (fp + (2 * tp))) / (tp + fp) - param22 = (fn * (fn + (2 * tn))) / (tn + fn) - # metric_value = (1 / np.prod(gt_ni_header.get_data_shape()[0:3])) * min(param11 + param12, param21 + param22) - metric_value = (1 / np.prod(gt.shape)) * min(param11 + param12, param21 + param22) - else: + try: + if metric == 'VS': metric_value = math.inf - elif metric == 'MI': - metric_value = normalized_mutual_info_score(gt.flatten(), detection.flatten()) - elif metric == 'RI': - metric_value = 0. - a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) - b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) - d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) - num = a + b - den = a + b + c + d - if den != 0: - metric_value = num / den - elif metric == 'ARI': - metric_value = 0. - a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) - b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) - # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) - d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) - num = 2 * (a * d - b * c) - den = math.pow(c, 2) + math.pow(b, 2) + 2 * a * d + (a + d) * (c + b) - if den != 0: - metric_value = num / den - elif metric == 'VOI': - fn_tp = fn + tp - fp_tp = fp + tp - # total = np.prod(gt_ni_header.get_data_shape()[0:3]) - total = np.prod(gt.shape) - - if fn_tp == 0 or (fn_tp / total) == 1 or fp_tp == 0 or (fp_tp / total) == 1: + den = (2 * tp) + fp + fn + if den != 0: + metric_value = 1 - ((abs(fn - fp)) / ((2 * tp) + fp + fn)) + elif metric == 'GCE': + if (tp + fn) != 0 and (tn + fp) != 0 and (tp + fp) != 0 and (tn + fn) != 0: + param11 = (fn * (fn + (2 * tp))) / (tp + fn) + param12 = (fp * (fp + (2 * tn))) / (tn + fp) + param21 = (fp * (fp + (2 * tp))) / (tp + fp) + param22 = (fn * (fn + (2 * tn))) / (tn + fn) + # metric_value = (1 / np.prod(gt_ni_header.get_data_shape()[0:3])) * min(param11 + param12, param21 + param22) + metric_value = (1 / np.prod(gt.shape)) * min(param11 + param12, param21 + param22) + else: + metric_value = math.inf + elif metric == 'MI': + metric_value = normalized_mutual_info_score(gt.flatten(), detection.flatten()) + elif metric == 'RI': + metric_value = 0. + a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) + b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) + d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) + num = a + b + den = a + b + c + d + if den != 0: + metric_value = num / den + elif metric == 'ARI': + metric_value = 0. + a = 0.5 * ((tp * (tp - 1)) + (fp * (fp - 1)) + (tn * (tn - 1)) + (fn * (fn - 1))) + b = 0.5 * ((math.pow(tp + fn, 2) + math.pow(tn + fp, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + c = 0.5 * ((math.pow(tp + fp, 2) + math.pow(tn + fn, 2)) - (math.pow(tp, 2) + math.pow(tn, 2) + math.pow(fp, 2) + math.pow(fn, 2))) + # d = np.prod(gt_ni_header.get_data_shape()[0:3]) * (np.prod(gt_ni_header.get_data_shape()[0:3]) - 1) / 2 - (a + b + c) + d = np.prod(gt.shape) * (np.prod(gt.shape) - 1) / 2 - (a + b + c) + num = 2 * (a * d - b * c) + den = math.pow(c, 2) + math.pow(b, 2) + 2 * a * d + (a + d) * (c + b) + if den != 0: + metric_value = num / den + elif metric == 'VOI': + fn_tp = fn + tp + fp_tp = fp + tp + # total = np.prod(gt_ni_header.get_data_shape()[0:3]) + total = np.prod(gt.shape) + + if fn_tp == 0 or (fn_tp / total) == 1 or fp_tp == 0 or (fp_tp / total) == 1: + metric_value = math.inf + else: + h1 = -((fn_tp / total) * math.log2(fn_tp / total) + (1 - fn_tp / total) * math.log2(1 - fn_tp / total)) + h2 = -((fp_tp / total) * math.log2(fp_tp / total) + (1 - fp_tp / total) * math.log2(1 - fp_tp / total)) + + p00 = 1 if tn == 0 else (tn / total) + p01 = 1 if fn == 0 else (fn / total) + p10 = 1 if fp == 0 else (fp / total) + p11 = 1 if tp == 0 else (tp / total) + + h12 = -((tn / total) * math.log2(p00) + (fn / total) * math.log2(p01) + (fp / total) * math.log2(p10) + (tp / total) * math.log2(p11)) + mi = h1 + h2 - h12 + metric_value = h1 + h2 - (2 * mi) + elif metric == 'Jaccard': + metric_value = jaccard_score(gt.flatten(), detection.flatten()) + elif metric == 'IOU': metric_value = math.inf - else: - h1 = -((fn_tp / total) * math.log2(fn_tp / total) + (1 - fn_tp / total) * math.log2(1 - fn_tp / total)) - h2 = -((fp_tp / total) * math.log2(fp_tp / total) + (1 - fp_tp / total) * math.log2(1 - fp_tp / total)) - - p00 = 1 if tn == 0 else (tn / total) - p01 = 1 if fn == 0 else (fn / total) - p10 = 1 if fp == 0 else (fp / total) - p11 = 1 if tp == 0 else (tp / total) - - h12 = -((tn / total) * math.log2(p00) + (fn / total) * math.log2(p01) + (fp / total) * math.log2(p10) + (tp / total) * math.log2(p11)) - mi = h1 + h2 - h12 - metric_value = h1 + h2 - (2 * mi) - elif metric == 'Jaccard': - metric_value = jaccard_score(gt.flatten(), detection.flatten()) - elif metric == 'IOU': - metric_value = math.inf - intersection = (gt == 1) & (detection == 1) - union = (gt == 1) | (detection == 1) - if np.count_nonzero(union) != 0: - metric_value = np.count_nonzero(intersection) / np.count_nonzero(union) - elif metric == 'TPR': - metric_value = math.inf - if (tp + fn) != 0: - metric_value = tp / (tp + fn) - elif metric == 'TNR': - metric_value = math.inf - if (tn + fp) != 0: - metric_value = tn / (tn + fp) - elif metric == 'FPR': - metric_value = math.inf - if (fp + tn) != 0: - metric_value = fp / (fp + tn) - elif metric == 'FNR': - metric_value = math.inf - if (fn + tp) != 0: - metric_value = fn / (fn + tp) - elif metric == 'PPV': - metric_value = math.inf - if (tp + fp) != 0: - metric_value = tp / (tp + fp) - elif metric == 'AUC': - metric_value = roc_auc_score(gt.flatten(), detection.flatten()) - elif metric == 'MCC': - metric_value = math.inf - num = (tp * tn) - (fp * fn) - den = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) - if den != 0: - metric_value = num / den - elif metric == 'CKS': - metric_value = cohen_kappa_score(gt.flatten(), detection.flatten()) - elif metric == 'HD95': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_hd95(detection, gt, voxelspacing=det_spacing, connectivity=1) - elif metric == 'ASSD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_assd(detection, gt, voxel_spacing=det_spacing) - elif metric == 'OASSD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_object_assd(detection, gt, voxel_spacing=det_spacing) - elif metric == 'RAVD': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value = compute_ravd(detection, gt) - elif metric == 'VC': - metric_value = math.inf - if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array - metric_value, pval = compute_volume_correlation(detection, gt) - elif metric == 'MahaD': - metric_value = math.inf - gt_n = np.count_nonzero(detection) - seg_n = np.count_nonzero(gt) - - if gt_n != 0 and seg_n != 0: - gt_indices = np.flip(np.where(gt == 1), axis=0) - gt_mean = gt_indices.mean(axis=1) - gt_cov = np.cov(gt_indices) - - seg_indices = np.flip(np.where(detection == 1), axis=0) - seg_mean = seg_indices.mean(axis=1) - seg_cov = np.cov(seg_indices) - - # calculate common covariance matrix - common_cov = (gt_n * gt_cov + seg_n * seg_cov) / (gt_n + seg_n) - common_cov_inv = np.linalg.inv(common_cov) - - mean = gt_mean - seg_mean - metric_value = math.sqrt(mean.dot(common_cov_inv).dot(mean.T)) - elif metric == 'ProbD': - # metric_value = -1. - metric_value = math.inf - gt_flat = gt.flatten().astype(np.int8) - det_flat = detection.flatten().astype(np.int8) - - probability_difference = np.absolute(gt_flat - det_flat).sum() - probability_joint = (gt_flat * det_flat).sum() - - if probability_joint != 0: - metric_value = probability_difference / (2. * probability_joint) - else: - logging.warning("Metric with name {} has not been implemented!".format(metric)) - return metric_value + intersection = (gt == 1) & (detection == 1) + union = (gt == 1) | (detection == 1) + if np.count_nonzero(union) != 0: + metric_value = np.count_nonzero(intersection) / np.count_nonzero(union) + elif metric == 'TPR': + metric_value = math.inf + if (tp + fn) != 0: + metric_value = tp / (tp + fn) + elif metric == 'TNR': + metric_value = math.inf + if (tn + fp) != 0: + metric_value = tn / (tn + fp) + elif metric == 'FPR': + metric_value = math.inf + if (fp + tn) != 0: + metric_value = fp / (fp + tn) + elif metric == 'FNR': + metric_value = math.inf + if (fn + tp) != 0: + metric_value = fn / (fn + tp) + elif metric == 'PPV': + metric_value = math.inf + if (tp + fp) != 0: + metric_value = tp / (tp + fp) + elif metric == 'AUC': + metric_value = math.inf + if len(np.unique(gt)) > 1: # ROC AUC score is not defined if only class is present in y_true + metric_value = roc_auc_score(gt.flatten(), detection.flatten()) + elif metric == 'MCC': + metric_value = math.inf + num = (tp * tn) - (fp * fn) + den = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + if den != 0: + metric_value = num / den + elif metric == 'CKS': + metric_value = cohen_kappa_score(gt.flatten(), detection.flatten()) + elif metric == 'HD95': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_hd95(detection, gt, voxelspacing=det_spacing, connectivity=1) + elif metric == 'ASSD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_assd(detection, gt, voxel_spacing=det_spacing) + elif metric == 'OASSD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_object_assd(detection, gt, voxel_spacing=det_spacing) + elif metric == 'RAVD': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value = compute_ravd(detection, gt) + elif metric == 'VC': + metric_value = math.inf + if np.max(gt) == 1 and np.max(detection) == 1: # Computation does not work if no binary object in the array + metric_value, pval = compute_volume_correlation(detection, gt) + elif metric == 'MahaD': + metric_value = math.inf + gt_n = np.count_nonzero(detection) + seg_n = np.count_nonzero(gt) + + if gt_n != 0 and seg_n != 0: + gt_indices = np.flip(np.where(gt == 1), axis=0) + gt_mean = gt_indices.mean(axis=1) + gt_cov = np.cov(gt_indices) + + seg_indices = np.flip(np.where(detection == 1), axis=0) + seg_mean = seg_indices.mean(axis=1) + seg_cov = np.cov(seg_indices) + + # calculate common covariance matrix + common_cov = (gt_n * gt_cov + seg_n * seg_cov) / (gt_n + seg_n) + common_cov_inv = np.linalg.inv(common_cov) + + mean = gt_mean - seg_mean + metric_value = math.sqrt(mean.dot(common_cov_inv).dot(mean.T)) + elif metric == 'ProbD': + # metric_value = -1. + metric_value = math.inf + gt_flat = gt.flatten().astype(np.int8) + det_flat = detection.flatten().astype(np.int8) + probability_difference = np.absolute(gt_flat - det_flat).sum() + probability_joint = (gt_flat * det_flat).sum() + + if probability_joint != 0: + metric_value = probability_difference / (2. * probability_joint) + else: + logging.warning("Metric with name {} has not been implemented!".format(metric)) + return metric_value + except Exception as e: + raise ValueError(f"Computing {metric} failed with {e}") from e def compute_overall_metrics_correlation(input_folder, output_folder, data=None, class_name=None, best_threshold=0.5, best_overlap=0.0, suffix=None): diff --git a/raidionicsval/Validation/instance_segmentation_validation.py b/raidionicsval/Validation/instance_segmentation_validation.py index 4e020e4..c98077c 100644 --- a/raidionicsval/Validation/instance_segmentation_validation.py +++ b/raidionicsval/Validation/instance_segmentation_validation.py @@ -221,6 +221,9 @@ def __pair_candidates(self, study_state=False): dice_matrix[g, d] = 0. cost_matrix = -dice_matrix gt_inds, pred_inds = linear_sum_assignment(cost_matrix) + # @TODO. The 0.1 threshold here should be linked to the config file parameter (detection_overlap_thresholds). + # Performing such decoupling would require either enforing a single threshold per run, or extending the results with an extra column + # and save all OW results per threshold. matches = [(i, j) for i, j in zip(gt_inds, pred_inds) if dice_matrix[i, j] > 0.1] for m in matches: self.matching_results.append([m[0] + 1, m[1] + 1, dice_matrix[m[0], m[1]]]) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 00c75c0..1c561c2 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -2,13 +2,14 @@ import itertools import logging import os.path -import time import traceback - +import sqlite3 +import csv import pandas as pd from math import ceil - +from functools import partial from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from ..Computation.dice_computation_instance import separate_dice_computation from ..Validation.instance_segmentation_validation import * @@ -54,13 +55,24 @@ def run(self): if len(SharedResources.getInstance().validation_metric_names) != 0: logging.info("Computing extra metrics for cohort.") self.__compute_extra_metrics(class_optimal=class_optimal) + logging.info("Re-exporting results to CSV with extra metrics...") + self.__sqlite_to_csv("total_results", self.dice_output_filename) + for c in SharedResources.getInstance().validation_class_names: + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) + + # Read all extra metric columns actually present in the CSV, not just the ones from the current config. + tmp = pd.read_csv(self.dice_output_filename) + all_extra_metric_names = [col for col in tmp.columns if col not in self.results_df_fixed_columns] + logging.info("Computing average metrics for the cohort.") # All - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='All') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='All') # Positive, based on given ground truth volume limit - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='Positive') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='Positive') # True positive, based on given detection_overlap_thresholds - compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=self.metric_names, condition='TP') + compute_fold_average(self.output_folder, class_optimal=class_optimal, metrics=all_extra_metric_names, condition='TP') + + self.conn.close() def __compute_metrics(self): """ @@ -71,54 +83,86 @@ def __compute_metrics(self): :return: """ cross_validation_description_file = os.path.join(self.input_folder, 'cross_validation_folds.txt') - self.results_df = [] - self.class_results_df = {} self.dice_output_filename = os.path.join(self.output_folder, 'all_dice_scores.csv') self.class_dice_output_filenames = {} for c in SharedResources.getInstance().validation_class_names: self.class_dice_output_filenames[c] = os.path.join(self.output_folder, c + '_dice_scores.csv') - self.class_results_df[c] = [] - self.results_df_base_columns = ['Fold', 'Patient', 'Threshold'] - self.results_df_base_columns.extend(["PiW Dice", "PiW Recall", "PiW Precision", "PiW F1"]) + + # Define the column schema shared by all result tables + self.results_df_fixed_columns = ['Fold', 'Patient', 'Threshold'] + self.results_df_fixed_columns.extend(["PiW Dice", "PiW Recall", "PiW Precision", "PiW F1"]) # self.results_df_base_columns.extend(["PaW Dice", "PaW Recall", "PaW Precision", "PaW F1"]) - self.results_df_base_columns.extend(["GT volume (ml)", "True Positive", "Detection volume (ml)"]) - self.results_df_base_columns.extend(["OW Global Recall", "OW Global Precision", "OW Global F1", "OW Dice", + self.results_df_fixed_columns.extend(["GT volume (ml)", "True Positive", "Detection volume (ml)"]) + self.results_df_fixed_columns.extend(["OW Global Recall", "OW Global Precision", "OW Global F1", "OW Dice", "OW Dice (std)", "OW Recall", "OW Recall (std)", "OW Precision", "OW Precision (std)", "OW F1", "OW F1 (std)", '#GT', '#Det']) - # For each extra metric, adding a pixelwise (PiW) and objectwise (OW) version of it! - # extra_metrics = [] - # for m in SharedResources.getInstance().validation_metric_names: - # extra_metrics.extend([f'PiW {m}', f'OW {m}']) - self.results_df_base_columns.extend(self.metric_names) - # self.results_df_base_columns.extend(SharedResources.getInstance().validation_metric_names) - - if not os.path.exists(self.dice_output_filename): - self.results_df = pd.DataFrame(columns=self.results_df_base_columns) + self.results_df_base_columns = self.results_df_fixed_columns + self.metric_names + + # Connect to SQLite database + # WAL mode ensures crash-safe writes + self.db_path = os.path.join(self.output_folder, "results.db") + self.conn = sqlite3.connect(self.db_path) + self.conn.execute("PRAGMA journal_mode=WAL;") + cursor = self.conn.cursor() + + # Initialize or resume the total_results table + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='total_results'") + if cursor.fetchone() is None: + # First run: seed from existing CSV if available, otherwise start empty + if os.path.exists(self.dice_output_filename): + results_df = pd.read_csv(self.dice_output_filename) + if results_df.columns[0] != 'Fold': + results_df = pd.read_csv(self.dice_output_filename, index_col=0) + missing_metrics = [x for x in self.metric_names if + not x in list(results_df.columns)[1:]] + for m in missing_metrics: + results_df[m] = None + else: + results_df = pd.DataFrame(columns=self.results_df_base_columns) + + results_df.to_sql("total_results", self.conn, if_exists="replace", index=False) + cursor.execute("CREATE INDEX IF NOT EXISTS idx_total_search ON total_results ([Patient], [Fold], [Threshold]);") + self.conn.commit() else: - self.results_df = pd.read_csv(self.dice_output_filename) - if self.results_df.columns[0] != 'Fold': - self.results_df = pd.read_csv(self.dice_output_filename, index_col=0) - missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if - not x in list(self.results_df.columns)[1:]] - for m in missing_metrics: - self.results_df[m] = None + logging.info("Existing database found, resuming from SQL...") + if not os.path.exists(self.dice_output_filename): + self.__sqlite_to_csv("total_results", self.dice_output_filename) + # Initialize or resume per-class tables for c in SharedResources.getInstance().validation_class_names: - if not os.path.exists(self.class_dice_output_filenames[c]): - self.class_results_df[c] = pd.DataFrame(columns=self.results_df_base_columns) + table_name = f"class_{c}" + cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'") + + if cursor.fetchone() is None: + if not os.path.exists(self.class_dice_output_filenames[c]): + class_df = pd.DataFrame(columns=self.results_df_base_columns) + else: + class_df = pd.read_csv(self.class_dice_output_filenames[c]) + if class_df.columns[0] != 'Fold': + class_df = pd.read_csv(self.class_dice_output_filenames[c], index_col=0) + missing_metrics = [x for x in self.metric_names if + x not in list(class_df.columns)[1:]] + for m in missing_metrics: + class_df[m] = None + + class_df.to_sql(table_name, self.conn, if_exists="replace", index=False) + cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_class_{c} ON [{table_name}] ([Patient], [Fold], [Threshold]);") + self.conn.commit() else: - self.class_results_df[c] = pd.read_csv(self.class_dice_output_filenames[c]) - if self.class_results_df[c].columns[0] != 'Fold': - self.class_results_df[c] = pd.read_csv(self.class_dice_output_filenames[c], index_col=0) - missing_metrics = [x for x in self.metric_names if - not x in list(self.class_results_df[c].columns)[1:]] - for m in missing_metrics: - self.class_results_df[c][m] = None + logging.info(f"Existing table found for class {c}, resuming from SQL...") + if not os.path.exists(self.class_dice_output_filenames[c]): + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) - self.results_df['Patient'] = self.results_df.Patient.astype(str) - for c in SharedResources.getInstance().validation_class_names: - self.class_results_df[c]['Patient'] = self.class_results_df[c].Patient.astype(str) + # Add any missing extra metric columns to all tables (e.g. when new metrics are added after initial run) + for metric_name in self.metric_names: + for table in ["total_results"] + [f"class_{c}" for c in SharedResources.getInstance().validation_class_names]: + try: + cursor.execute(f"ALTER TABLE [{table}] ADD COLUMN [{metric_name}] REAL") + self.conn.commit() + except sqlite3.OperationalError: + pass # Column already exists + # Process each fold results_per_folds = [] for fold in range(0, self.fold_number): logging.info(f'\nProcessing fold {fold+1}/{self.fold_number}.\n') @@ -129,9 +173,15 @@ def __compute_metrics(self): results = self.__compute_metrics_for_fold(data_list=test_set, fold_number=fold) results_per_folds.append(results) + logging.info("Exporting results to CSV...") + self.__sqlite_to_csv("total_results", self.dice_output_filename) + for c in SharedResources.getInstance().validation_class_names: + self.__sqlite_to_csv(f"class_{c}", self.class_dice_output_filenames[c]) + def __compute_metrics_for_fold(self, data_list, fold_number): if not os.path.exists(os.path.join(SharedResources.getInstance().validation_input_folder, "predictions", str(fold_number))): logging.warning(f"No predictions folder for fold {fold_number} -- Skipping!") + print(f"No predictions folder for fold {fold_number} -- Skipping!") return 0 for i, patient in enumerate(tqdm(data_list)): @@ -151,18 +201,19 @@ def __compute_metrics_for_fold(self, data_list, fold_number): # Placeholder for holding all metrics for the current patient patient_metrics = PatientMetrics(id=uid, patient_id=pid, fold_number=fold_number, class_names=SharedResources.getInstance().validation_class_names) - patient_metrics.init_from_file(self.output_folder) - + + # Load any previously computed metrics from the database + patient_metrics.init_from_db(self.conn) success = self.__identify_patient_files(patient_metrics, sub_folder_index, fold_number) self.patients_metrics[uid] = patient_metrics - + # Checking if values have already been computed for the current patient to skip it if so. if patient_metrics.is_complete(): continue if not success: print('Input files not found for patient {}\n'.format(uid)) continue - + self.__generate_dice_scores_for_patient(patient_metrics, fold_number) except Exception as e: print('Issue processing patient {}\n'.format(uid)) @@ -320,25 +371,23 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): uid = patient_metrics.patient_id classes = SharedResources.getInstance().validation_class_names nb_classes = len(classes) - patient_filenames = {} thr_range = np.arange(0.1, 1.1, 0.1) + cursor = self.conn.cursor() # Iterating over all classes, where independent files are expected for c in range(nb_classes): gt_filename, det_filename = patient_metrics.get_class_filenames(c) - # ground_truth_ni = nib.load(gt_filename) - # gt = ground_truth_ni.get_fdata() - # detection_ni = nib.load(det_filename) gt, _, gt_specs = open_image_file(gt_filename) detection, _, det_specs = open_image_file(det_filename) gt[gt >= 1] = 1 class_tp_threshold = SharedResources.getInstance().validation_true_positive_volume_thresholds[c] - # gt_volume = np.count_nonzero(gt) * np.prod(ground_truth_ni.header.get_zooms()) * 1e-3 gt_volume = np.count_nonzero(gt) * np.prod(det_specs[1]) * 1e-3 tp_state = True if gt_volume > class_tp_threshold else False extra = [np.round(gt_volume, 4), tp_state, det_specs[1]] pat_results = [] + + # Compute Dice scores across all thresholds, using multiprocessing if configured if SharedResources.getInstance().number_processes > 1: pool = multiprocessing.Pool(processes=SharedResources.getInstance().number_processes) pat_results = pool.map(separate_dice_computation, zip(thr_range, @@ -357,29 +406,37 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): pat_results.append(thr_res) patient_metrics.set_class_regular_metrics(classes[c], pat_results) - # Filling in the csv files on disk for faster resume - class_results_filename = self.class_dice_output_filenames[classes[c]] + + # Write per-class results to the database + table_name = f"class_{classes[c]}" + columns = list(self.results_df_base_columns) + for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.class_results_df[classes[c]].loc[ - (self.class_results_df[classes[c]]['Patient'] == uid) & (self.class_results_df[classes[c]]['Fold'] == fold_number) & ( - self.class_results_df[classes[c]]['Threshold'] == th)] - # ind_values = np.asarray(pat_results[ind]) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: + + query = f"SELECT * from [{table_name}] where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(query, (str(uid), fold_number, th)) + row = cursor.fetchone() + + pat_res_tmp =[float(x) if isinstance(x, np.float32) else x for x in pat_results[ind][0]] + if row is None: extra_metrics = [None] * 2 * len(SharedResources.getInstance().validation_metric_names) - ind_values = np.asarray(pat_results[ind][0] + extra_metrics) - buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - columns=list(self.results_df_base_columns)) - # self.class_results_df[classes[c]] = self.class_results_df[classes[c]].append(buff_df, - # ignore_index=True) - self.class_results_df[classes[c]] = pd.concat([self.class_results_df[classes[c]], buff_df], - ignore_index=True) + ind_values = np.asarray(pat_res_tmp + extra_metrics) + + column_names_str = ", ".join([f"[{col}]" for col in columns]) + placeholders = ", ".join(["?"] * len(columns)) + + insert_query = f"INSERT INTO {table_name} ({column_names_str}) VALUES ({placeholders})" + cursor.execute(insert_query, tuple(ind_values)) + else: - ind_values = pat_results[ind][0] + list(self.class_results_df[classes[c]].loc[sub_df.index.values[0], :].values[len(pat_results[ind][0]):]) - self.class_results_df[classes[c]].loc[sub_df.index.values[0], :] = ind_values - self.class_results_df[classes[c]].to_csv(class_results_filename, index=False) + ind_values = pat_res_tmp + list(row[len(pat_results[ind][0]):]) + set_clause = ", ".join([f"[{col}] = ?" for col in columns]) + update_query = f"UPDATE {table_name} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(update_query, tuple(ind_values) + (uid, fold_number, th)) + + self.conn.commit() + # Should compute the class macro-average results if multiple classes class_averaged_results = None @@ -391,49 +448,151 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): if len(SharedResources.getInstance().validation_metric_names) != 0: final_pat_class_res = [pat_class_results[x] + pat_class_extra_metrics[x] for x in range(len(thr_range))] class_results.append(final_pat_class_res) - class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0) + class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0).astype(float) + current_columns = pd.read_sql_query(f"SELECT * FROM {'total_results'} LIMIT 0", self.conn).columns.tolist() - # Filling in the csv files on disk for faster resume for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.results_df.loc[ - (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number) & ( - self.results_df['Threshold'] == th)] - # ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: - ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) - buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - columns=list(self.results_df_base_columns)) - # self.results_df = self.results_df.append(buff_df, ignore_index=True) - self.results_df = pd.concat([self.results_df, buff_df], ignore_index=True) - else: + + query = f"SELECT * from {'total_results'} where [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(query, (str(uid), fold_number, th)) + + row = cursor.fetchone() + + if row is None: ind_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) - self.results_df.loc[sub_df.index.values[0], :] = ind_values - self.results_df.to_csv(self.dice_output_filename, index=False) + # Pad with None if fewer values than columns (e.g. when extra metrics are missing) + if len(ind_values) < len(current_columns): + ind_values += [None] * (len(current_columns) - len(ind_values)) + + column_names_str = ", ".join([f"[{col}]" for col in current_columns]) + placeholders = ", ".join(["?"] * len(current_columns)) + insert_query = f"INSERT INTO {'total_results'} ({column_names_str}) VALUES ({placeholders})" + cursor.execute(insert_query, tuple(ind_values)) + else: + row_dict = dict(zip(current_columns, row)) + new_values = [fold_number, uid, th] + list(class_averaged_results[ind]) + + for i, val in enumerate(new_values): + if i < len(current_columns): + row_dict[current_columns[i]] = float(val) if isinstance(val, np.float32) else val + ind_values = [row_dict.get(col, None) for col in current_columns] + set_clause = ", ".join([f"[{col}] = ?" for col in current_columns]) + + update_query = f"UPDATE {'total_results'} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ? AND [Threshold] = ?" + cursor.execute(update_query, tuple(ind_values) + (str(uid), fold_number, np.round(th, 2))) + + self.conn.commit() + def __compute_extra_metrics(self, class_optimal: dict = {}): """ - + Computes additional metrics at the optimal threshold for each class. + Results are written to both the per-class table and total_results in SQLite. + @TODO. Would need to properly compute metrics average over the different classes to fill in the total_results table (or just skip it). """ classes = SharedResources.getInstance().validation_class_names for c in classes: optimal_values = class_optimal[c]['All'] - for p in tqdm(self.patients_metrics): - try: - # Initializing/completing the list which will hold the extra metrics - self.patients_metrics[p].setup_extra_metrics(self.metric_names) - pat_metrics = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], - SharedResources.getInstance().validation_metric_names) - self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) - - # Filling in the overall dataframe and dumping results to csv after each patient + if len(SharedResources.getInstance().validation_metric_names) < 10 and SharedResources.getInstance().number_processes != 1: + batch_results =[] + args_list = [(p, self.patients_metrics[p], c, classes, optimal_values, SharedResources.getInstance().validation_metric_names) for p in self.patients_metrics] + with ProcessPoolExecutor(max_workers=SharedResources.getInstance().number_processes) as executor: + for result in tqdm(executor.map(patient_metrics_computation_worker, args_list), total=len(args_list)): + if result is not None: + batch_results.append(result) + if len(batch_results) >= SharedResources.getInstance().number_processes: + self.__update_database(batch_results) + batch_results.clear() + + if batch_results: # flush remainder + self.__update_database(batch_results) + batch_results.clear() + else: + for p in tqdm(self.patients_metrics): + recomputation = False + try: + # Initializing/completing the list which will hold the extra metrics + self.patients_metrics[p].setup_extra_metrics(self.metric_names) + pat_metrics, recomputation = compute_patient_extra_metrics(self.patients_metrics[p], classes.index(c), optimal_values[1], + SharedResources.getInstance().validation_metric_names) + if recomputation: + self.patients_metrics[p].set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) + cursor = self.conn.cursor() + for pm in pat_metrics: + metric_name = pm[0] + metric_value = pm[1] + + thr_to_match = float(np.round(optimal_values[1], 2)) + + update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + + update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + except Exception as e: + logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") + continue + finally: + if recomputation: + logging.debug("Updating the database with extra metrics results patient-wise") + self.conn.commit() + else: + logging.debug("Skipping extra metrics recomputation") + + def __update_database(self, batch_results): + try: + logging.debug("Updating the database with extra metrics results batch-wise") + cursor = self.conn.cursor() + for results in batch_results: + p = results[0] + c = results[1] + thr_to_match = results[2] + pat_metrics = results[3] + recompute = results[4] + if recompute: for pm in pat_metrics: metric_name = pm[0] metric_value = pm[1] - self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value - self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) - except Exception as e: - logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") - continue \ No newline at end of file + + update_query = f"UPDATE [class_{c}] SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + + update_query = f"UPDATE total_results SET [{metric_name}] = ? WHERE [Patient] = ? AND [Threshold] = ?" + cursor.execute(update_query, (metric_value, str(self.patients_metrics[p].patient_id), thr_to_match)) + self.conn.commit() + except Exception as e: + logging.error(f"Commiting to the database after parallel metrics computation failed with {e}") + + def __sqlite_to_csv(self, table_name, csv_path): + """ + Exports a full SQLite table to a CSV file using a streaming cursor + to avoid loading the entire table into memory. + """ + cursor = self.conn.cursor() + cursor.execute(f"SELECT * FROM [{table_name}]") + headers = [description[0] for description in cursor.description] + + with open(csv_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(cursor) + +def patient_metrics_computation_worker(args): + p, metrics, c, classes, optimal_values, metric_names = args + recomputation = False + result = None + try: + # Initializing/completing the list which will hold the extra metrics + thr_to_match = float(np.round(optimal_values[1], 2)) + metrics.setup_extra_metrics(metric_names) + pat_metrics, recomputation = compute_patient_extra_metrics(metrics, classes.index(c), optimal_values[1], + SharedResources.getInstance().validation_metric_names) + if recomputation: + metrics.set_optimal_class_extra_metrics(classes.index(c), optimal_values[1], pat_metrics) + + result = [p, c, thr_to_match, pat_metrics, recomputation] + except Exception as e: + logging.error(f"Computing extra metrics for patient {metrics.patient_id} failed with: {e}.\n{traceback.format_exc()}") + finally: + return result diff --git a/raidionicsval/Validation/kfold_model_validation_classification.py b/raidionicsval/Validation/kfold_model_validation_classification.py index aba1b7f..687ae88 100644 --- a/raidionicsval/Validation/kfold_model_validation_classification.py +++ b/raidionicsval/Validation/kfold_model_validation_classification.py @@ -1,15 +1,14 @@ import multiprocessing import itertools import os.path - import time - +import sqlite3 import numpy as np import pandas as pd from math import ceil - +import logging from tqdm import tqdm - +import csv from ..Computation.dice_computation_instance import separate_dice_computation from ..Validation.instance_segmentation_validation import * from ..Utils.resources import SharedResources @@ -40,15 +39,17 @@ def __init__(self): self.metric_names = [] self.metric_names.extend(SharedResources.getInstance().validation_metric_names) self.gt_files_suffix = SharedResources.getInstance().validation_gt_files_suffix - self.gt_master_file = "/home/dbouget/Data/Studies/SequenceClassifier/seqclassifier_patients_parameters.csv" # @TODO. Should be in the config file + self.gt_master_file = SharedResources.getInstance().studies_extra_parameters_filename # @TODO. Not ideal to point to a studies parameter for validation self.gt_master_file_df = pd.read_csv(self.gt_master_file) self.prediction_files_suffix = SharedResources.getInstance().validation_prediction_files_suffix self.patients_metrics = {} def run(self): self.__compute_metrics() - # if len(SharedResources.getInstance().validation_metric_names) != 0: - # self.__compute_extra_metrics(class_optimal=class_optimal) + #if len(SharedResources.getInstance().validation_metric_names) != 0: + # self.__compute_extra_metrics(class_optimal=class_optimal) + # logging.info("Re-exporting results to CSV with extra metrics...") + # self.__sqlite_to_csv("total_results", self.all_scores_output_filename) compute_fold_average(self.input_folder, metrics=self.metric_names) def __compute_metrics(self): @@ -64,6 +65,37 @@ def __compute_metrics(self): self.results_df_base_columns.extend(["Prediction", "GT"]) self.results_df_base_columns.extend(SharedResources.getInstance().validation_metric_names) + # Connect to SQLite database + # WAL mode ensures crash-safe writes + self.db_path = os.path.join(self.output_folder, "results.db") + self.conn = sqlite3.connect(self.db_path) + self.conn.execute("PRAGMA journal_mode=WAL;") + cursor = self.conn.cursor() + + # Initialize or resume the total_results table + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='total_results'") + if cursor.fetchone() is None: + # First run: seed from existing CSV if available, otherwise start empty + if os.path.exists(self.all_scores_output_filename): + results_df = pd.read_csv(self.all_scores_output_filename) + if results_df.columns[0] != 'Fold': + results_df = pd.read_csv(self.all_scores_output_filename, index_col=0) + missing_metrics = [x for x in self.metric_names if + not x in list(results_df.columns)[1:]] + for m in missing_metrics: + results_df[m] = None + else: + results_df = pd.DataFrame(columns=self.results_df_base_columns) + + results_df.to_sql("total_results", self.conn, if_exists="replace", index=False) + cursor.execute("CREATE INDEX IF NOT EXISTS idx_total_search ON total_results ([Patient], [Fold]);") + self.conn.commit() + else: + logging.info("Existing database found, resuming from SQL...") + if not os.path.exists(self.all_scores_output_filename): + self.__sqlite_to_csv("total_results", self.all_scores_output_filename) + + """ if not os.path.exists(self.all_scores_output_filename): self.results_df = pd.DataFrame(columns=self.results_df_base_columns) else: @@ -75,6 +107,16 @@ def __compute_metrics(self): for m in missing_metrics: self.results_df[m] = None self.results_df['Patient'] = self.results_df.Patient.astype(str) + """ + + # Add any missing extra metric columns to all tables (e.g. when new metrics are added after initial run) + for metric_name in self.metric_names: + for table in ["total_results"]: + try: + cursor.execute(f"ALTER TABLE [{table}] ADD COLUMN [{metric_name}] REAL") + self.conn.commit() + except sqlite3.OperationalError: + pass # Column already exists results_per_folds = [] for fold in range(0, self.fold_number): @@ -86,6 +128,9 @@ def __compute_metrics(self): results = self.__compute_metrics_for_fold(data_list=test_set, fold_number=fold) results_per_folds.append(results) + logging.info("Exporting results to CSV...") + self.__sqlite_to_csv("total_results", self.all_scores_output_filename) + def __compute_metrics_for_fold(self, data_list, fold_number): for i, patient in enumerate(tqdm(data_list)): uid = None @@ -165,10 +210,11 @@ def __identify_patient_files(self, patient_metrics, folder_index, fold_number, m # ts = os.path.basename(detection_filename).split('.')[0].split('_')[-3] # image_uid = uid + '_' + ts # gt_class = masterfile.loc[masterfile["Patient"] == image_uid]["Sequence"].values[0] - gt_class = os.path.basename(detection_filename).split('.')[0].split('_')[-4] - gt_result = np.zeros(len(classes)).astype('uint8') - gt_result[classes.index(gt_class)] = 1 - np.savetxt(gt_results_filename, gt_result, delimiter=";") + if False: #Only for sequence classification + gt_class = os.path.basename(detection_filename).split('.')[0].split('_')[-4] + gt_result = np.zeros(len(classes)).astype('uint8') + gt_result[classes.index(gt_class)] = 1 + np.savetxt(gt_results_filename, gt_result, delimiter=";") patient_filenames = [gt_results_filename, detection_filename] patient_metrics.set_patient_filenames(patient_filenames) @@ -179,6 +225,7 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): Compute the basic metrics for all classes of the current patient :return: """ + cursor = self.conn.cursor() uid = patient_metrics.patient_id classes = SharedResources.getInstance().validation_class_names nb_classes = len(classes) @@ -210,6 +257,39 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): # self.class_results_df[classes[c]].loc[sub_df.index.values[0], :] = ind_values # self.results_df.to_csv(self.all_scores_output_filename, index=False) + current_columns = pd.read_sql_query(f"SELECT * FROM {'total_results'} LIMIT 0", self.conn).columns.tolist() + query = f"SELECT * from {'total_results'} where [Patient] = ? AND [Fold] = ?" + cursor.execute(query, (str(uid), fold_number)) + + row = cursor.fetchone() + + if row is None: + ind_values = [fold_number, uid] + list(pat_results) + # Pad with None if fewer values than columns (e.g. when extra metrics are missing) + if len(ind_values) < len(current_columns): + ind_values += [None] * (len(current_columns) - len(ind_values)) + + column_names_str = ", ".join([f"[{col}]" for col in current_columns]) + placeholders = ", ".join(["?"] * len(current_columns)) + insert_query = f"INSERT INTO {'total_results'} ({column_names_str}) VALUES ({placeholders})" + cursor.execute(insert_query, tuple(ind_values)) + else: + row_dict = dict(zip(current_columns, row)) + new_values = [fold_number, uid] + list(pat_results) + + for i, val in enumerate(new_values): + if i < len(current_columns): + row_dict[current_columns[i]] = float(val) if isinstance(val, np.float32) else val + + ind_values = [row_dict.get(col, None) for col in current_columns] + set_clause = ", ".join([f"[{col}] = ?" for col in current_columns]) + + update_query = f"UPDATE {'total_results'} SET {set_clause} WHERE [Patient] = ? AND [Fold] = ?" + cursor.execute(update_query, tuple(ind_values) + (str(uid), fold_number)) + + self.conn.commit() + + """ # Filling in the csv files on disk for faster resume sub_df = self.results_df.loc[ (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number)] @@ -222,6 +302,7 @@ def __generate_scores_for_patient(self, patient_metrics, fold_number): ind_values = [fold_number, uid] + list(pat_results) self.results_df.loc[sub_df.index.values[0], :] = ind_values self.results_df.to_csv(self.all_scores_output_filename, index=False) + """ def __compute_extra_metrics(self, class_optimal: dict = {}): """ @@ -244,3 +325,17 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): metric_value = pm[1] self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) + + def __sqlite_to_csv(self, table_name, csv_path): + """ + Exports a full SQLite table to a CSV file using a streaming cursor + to avoid loading the entire table into memory. + """ + cursor = self.conn.cursor() + cursor.execute(f"SELECT * FROM [{table_name}]") + headers = [description[0] for description in cursor.description] + + with open(csv_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(cursor) \ No newline at end of file diff --git a/raidionicsval/__main__.py b/raidionicsval/__main__.py index 82f54a4..ce179a2 100644 --- a/raidionicsval/__main__.py +++ b/raidionicsval/__main__.py @@ -41,5 +41,7 @@ def main(): if __name__ == "__main__": logging.info("Internal main call.\n") + import matplotlib + matplotlib.use("Agg") main() diff --git a/tests/conftest.py b/tests/conftest.py index 17f9f65..b0fefc7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ def test_dir(): if not os.listdir(test_dir): download_resources(test_dir=test_dir) yield test_dir - logging.info(f"Removing the temporary directory for tests.") - if os.path.exists(test_dir): - shutil.rmtree(test_dir) \ No newline at end of file + if os.environ.get("GITHUB_ACTIONS") != "true": # Allowing to collect the folder content as an artifact if the test failed in the CI + logging.info(f"Removing the temporary directory for tests.") + if os.path.exists(test_dir): + shutil.rmtree(test_dir) \ No newline at end of file diff --git a/tests/download_resources.py b/tests/download_resources.py index 4677718..89fffb2 100644 --- a/tests/download_resources.py +++ b/tests/download_resources.py @@ -23,7 +23,7 @@ def download_resources(test_dir: str): f.write(chunk) with zipfile.ZipFile(archive_dl_dest, 'r') as zip_ref: zip_ref.extractall(dest_dir) - + os.remove(archive_dl_dest) except Exception as e: logging.error(f"Error during input data download with: {e} \n {traceback.format_exc()}\n") if os.path.exists(dest_dir): diff --git a/tests/generic_tests/test_validation_pipeline_advanced.py b/tests/generic_tests/test_validation_pipeline_advanced.py index 1e7cfde..ad14b7a 100644 --- a/tests/generic_tests/test_validation_pipeline_advanced.py +++ b/tests/generic_tests/test_validation_pipeline_advanced.py @@ -61,8 +61,8 @@ def test_validation_pipeline_metrics_space(test_dir): assert round(gt_df['OW Recall (Mean)'][0], 2) == round(results_df['OW Recall (Mean)'][0],2), "OW Recall (Mean) values do not match" except Exception as e: logging.error(f"Error during k-fold cross-validation unit test with: {e} \n {traceback.format_exc()}.\n") - if os.path.exists(output_folder): - shutil.rmtree(output_folder) + # if os.path.exists(output_folder): + # shutil.rmtree(output_folder) raise ValueError("Error during k-fold cross-validation unit test with.\n") logging.info("k-fold cross-validation unit test succeeded.\n") @@ -121,8 +121,8 @@ def test_validation_pipeline_extra_metrics(test_dir): assert round(gt_df['OW MI (Std)'][0], 2) == round(results_df['OW MI (Std)'][0],2), "OW MI (Std) values do not match" except Exception as e: logging.error(f"Error during test with: {e} \n {traceback.format_exc()}.\n") - if os.path.exists(output_folder): - shutil.rmtree(output_folder) + # if os.path.exists(output_folder): + # shutil.rmtree(output_folder) raise ValueError(f"{e}.\n") if os.path.exists(output_folder):