trust fixes

Author: FerTV

nebula/addons/trustworthiness/trustworthiness.py

@@ -167,40 +167,40 @@ async def _generate_factsheet(self, trust_config, experiment_name):
         factsheet.populate_factsheet_post_train(experiment_name, self._start_time, self._end_time, class_counter)
         logging.info("[FER] factsheet post train done")
 
-        # data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json")
-        # with open(data_file_path, 'r') as data_file:
-        #     data = json.load(data_file)
+        data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json")
+        with open(data_file_path, 'r') as data_file:
+            data = json.load(data_file)
 
-        #     weights = {
-        #         "robustness": float(data["robustness_pillar"]),
-        #         "resilience_to_attacks": float(data["resilience_to_attacks"]),
-        #         "algorithm_robustness": float(data["algorithm_robustness"]),
-        #         "client_reliability": float(data["client_reliability"]),
-        #         "privacy": float(data["privacy_pillar"]),
-        #         "technique": float(data["technique"]),
-        #         "uncertainty": float(data["uncertainty"]),
-        #         "indistinguishability": float(data["indistinguishability"]),
-        #         "fairness": float(data["fairness_pillar"]),
-        #         "selection_fairness": float(data["selection_fairness"]),
-        #         "performance_fairness": float(data["performance_fairness"]),
-        #         "class_distribution": float(data["class_distribution"]),
-        #         "explainability": float(data["explainability_pillar"]),
-        #         "interpretability": float(data["interpretability"]),
-        #         "post_hoc_methods": float(data["post_hoc_methods"]),
-        #         "accountability": float(data["accountability_pillar"]),
-        #         "factsheet_completeness":  float(data["factsheet_completeness"]),
-        #         "architectural_soundness": float(data["architectural_soundness_pillar"]),
-        #         "client_management": float(data["client_management"]),
-        #         "optimization": float(data["optimization"]),
-        #         "sustainability": float(data["sustainability_pillar"]),
-        #         "energy_source": float(data["energy_source"]),
-        #         "hardware_efficiency": float(data["hardware_efficiency"]),
-        #         "federation_complexity": float(data["federation_complexity"])
-        #     }
+            weights = {
+                "robustness": float(data["robustness_pillar"]),
+                "resilience_to_attacks": float(data["resilience_to_attacks"]),
+                "algorithm_robustness": float(data["algorithm_robustness"]),
+                "client_reliability": float(data["client_reliability"]),
+                "privacy": float(data["privacy_pillar"]),
+                "technique": float(data["technique"]),
+                "uncertainty": float(data["uncertainty"]),
+                "indistinguishability": float(data["indistinguishability"]),
+                "fairness": float(data["fairness_pillar"]),
+                "selection_fairness": float(data["selection_fairness"]),
+                "performance_fairness": float(data["performance_fairness"]),
+                "class_distribution": float(data["class_distribution"]),
+                "explainability": float(data["explainability_pillar"]),
+                "interpretability": float(data["interpretability"]),
+                "post_hoc_methods": float(data["post_hoc_methods"]),
+                "accountability": float(data["accountability_pillar"]),
+                "factsheet_completeness":  float(data["factsheet_completeness"]),
+                "architectural_soundness": float(data["architectural_soundness_pillar"]),
+                "client_management": float(data["client_management"]),
+                "optimization": float(data["optimization"]),
+                "sustainability": float(data["sustainability_pillar"]),
+                "energy_source": float(data["energy_source"]),
+                "hardware_efficiency": float(data["hardware_efficiency"]),
+                "federation_complexity": float(data["federation_complexity"])
+            }
 
-        #     trust_metric_manager = TrustMetricManager(self._start_time)
-        #     trust_metric_manager.evaluate(experiment_name, weights, use_weights=True)
-        #     logging.info("[FER] evaluation done")
+            trust_metric_manager = TrustMetricManager(self._start_time)
+            # trust_metric_manager.evaluate(experiment_name, weights, use_weights=True)
+            logging.info("[FER] evaluation done")
 
     async def _process_test_metrics_event(self, tme: TestMetricsEvent):
         cur_loss, cur_acc = await tme.get_event_data()

nebula/addons/trustworthiness/utils.py

@@ -31,7 +31,7 @@ def count_class_samples(scenario_name, dataloaders_files, class_counter: Counter
     dataloaders = []
 
     if class_counter:
-        result = dict(class_counter)
+        result = {hashids.encode(int(class_id)): count for class_id, count in class_counter.items()}
     else:
         for file in dataloaders_files:
             with open(file, "rb") as f: