feat: add 90 and 95 percentiles

Author: iraedeus

experimental_env/analysis/analyze_summarizers/error_summarizer.py

@@ -44,16 +44,20 @@ def calculate(self, results: list[ExperimentDescription]) -> tuple:
         mean = np.mean(errors)
         std = np.std(errors)
         median = np.median(errors)
+        percentile_90 = np.percentile(errors, 90)
+        percentile_95 = np.percentile(errors, 95)
 
-        return float(mean), float(std), float(median)
+        return float(mean), float(std), float(median), float(percentile_90), float(percentile_95)
 
     def analyze_method(self, results: list[ExperimentDescription], method: str):
-        mean, deviation, median = self.calculate(results)
+        mean, deviation, median, percentile_90, percentile_95 = self.calculate(results)
 
         info_dict = {
             "mean": round_sig(mean, 3),
             "standart_deviation": round_sig(deviation, 3),
             "median": round_sig(median, 3),
+            "percentile_90": round_sig(percentile_90, 3),
+            "percentile_95": round_sig(percentile_95, 3),
         }
         yaml_path: Path = self._out_dir.joinpath("metric_info.yaml")
 
@@ -67,16 +71,20 @@ def compare_methods(
         method_1: str,
         method_2: str,
     ):
-        mean_1, deviation_1, median_1 = self.calculate(results_1)
-        mean_2, deviation_2, median_2 = self.calculate(results_2)
+        mean_1, deviation_1, median_1, percentile_90_1, percentile_95_1 = self.calculate(results_1)
+        mean_2, deviation_2, median_2, percentile_90_2, percentile_95_2 = self.calculate(results_2)
 
         info_dict = {
             f"{method_1}_mean": round_sig(mean_1, 3),
             f"{method_1}_standart_deviation": round_sig(deviation_1, 3),
             f"{method_1}_median": round_sig(median_1, 3),
+            f"{method_1}_percentile_90": round_sig(percentile_90_1, 3),
+            f"{method_1}_percentile_95": round_sig(percentile_95_1, 3),
             f"{method_2}_mean": round_sig(mean_2, 3),
             f"{method_2}_standart_deviation": round_sig(deviation_2, 3),
             f"{method_2}_median": round_sig(median_2, 3),
+            f"{method_2}_percentile_90": round_sig(percentile_90_2, 3),
+            f"{method_2}_percentile_95": round_sig(percentile_95_2, 3),
         }
         yaml_path: Path = self._out_dir.joinpath("metric_info.yaml")
 

script_stage_1.py

@@ -1,64 +1,24 @@
-from pathlib import Path
+"""The script implements the first step of the experiment"""
 
-import numpy as np
+from pathlib import Path
 
 from experimental_env.preparation.dataset_generator import (
-    ConcreteDatasetGenerator,
     RandomDatasetGenerator,
 )
 from mpest.models import ExponentialModel, GaussianModel, WeibullModelExp
 
-WORKING_DIR = Path("/home/danil/PycharmProjects/Projects/EM-algo-DT/experiment/stage_1")
+WORKING_DIR = Path(dir_stage_1)
 SAMPLES_SIZE = 1000
 
-np.random.seed(42)
-
-r_generator = RandomDatasetGenerator()
+r_generator = RandomDatasetGenerator(42)
 mixtures = [
     [ExponentialModel],
     [GaussianModel],
     [WeibullModelExp],
     [WeibullModelExp, GaussianModel],
     [ExponentialModel, GaussianModel],
     [WeibullModelExp, WeibullModelExp],
-    [ExponentialModel, ExponentialModel]
+    [ExponentialModel, ExponentialModel],
 ]
 for models in mixtures:
     r_generator.generate(SAMPLES_SIZE, models, Path(WORKING_DIR), exp_count=100)
-
-c_generator2 = ConcreteDatasetGenerator()
-models = [ExponentialModel]
-c_generator2.add_distribution(models[0], [1.0], 1.0)
-c_generator2.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5)
-
-c_generator3 = ConcreteDatasetGenerator()
-models = [GaussianModel]
-c_generator3.add_distribution(models[0], [0, 1.0], 1.0)
-c_generator3.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5)
-
-c_generator4 = ConcreteDatasetGenerator()
-models = [WeibullModelExp]
-c_generator4.add_distribution(models[0], [1.0, 1.0], 1.0)
-c_generator4.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5)
-
-c_generator5 = ConcreteDatasetGenerator()
-models = [WeibullModelExp]
-c_generator5.add_distribution(models[0], [1.0, 1.0], 1.0)
-c_generator5.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5)
-
-c_generator6 = ConcreteDatasetGenerator()
-models = [WeibullModelExp]
-c_generator6.add_distribution(models[0], [1.0, 0.5], 1.0)
-c_generator6.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5)
-
-c_generator7 = ConcreteDatasetGenerator()
-models = [GaussianModel, GaussianModel]
-c_generator7.add_distribution(models[0], [-1.0, 2.5], 0.3)
-c_generator7.add_distribution(models[1], [1.0, 0.5], 0.7)
-c_generator7.generate(SAMPLES_SIZE, Path(WORKING_DIR), 10)
-
-c_generator8 = ConcreteDatasetGenerator()
-models = [GaussianModel, GaussianModel]
-c_generator8.add_distribution(models[0], [0.0, 1.5], 0.6)
-c_generator8.add_distribution(models[1], [1.0, 1.0], 0.4)
-c_generator8.generate(SAMPLES_SIZE, Path(WORKING_DIR), 10)

script_stage_2.py

@@ -1,7 +1,6 @@
-import random
-from pathlib import Path
+"""The script implements the second step of the experiment"""
 
-import numpy as np
+from pathlib import Path
 
 from experimental_env.experiment.estimators import (
     LikelihoodEstimator,
@@ -10,27 +9,22 @@
 from experimental_env.experiment.experiment_executors.random_executor import (
     RandomExperimentExecutor,
 )
-from experimental_env.experiment.experiment_executors.standart_executor import (
-    StandartExperimentExecutor,
-)
 from experimental_env.preparation.dataset_parser import SamplesDatasetParser
 from mpest.em.breakpointers import StepCountBreakpointer
 from mpest.em.distribution_checkers import (
     FiniteChecker,
     PriorProbabilityThresholdChecker,
 )
 
-SOURCE_DIR = Path("/home/danil/PycharmProjects/Projects/EM-algo-DT/experiment/stage_1")
-WORKING_DIR = Path("/home/danil/PycharmProjects/Projects/EM-algo-DT/experiment/stage_2")
-
-random.seed(42)
+SOURCE_DIR = Path(dir_stage_1)
+WORKING_DIR = Path(dir_stage_2)
 
 # Parse stage 1
 parser = SamplesDatasetParser()
 datasets = parser.parse(SOURCE_DIR)
 
 # Execute stage 2
-executor = RandomExperimentExecutor(WORKING_DIR, 5)
+executor = RandomExperimentExecutor(WORKING_DIR, 5, 43)
 executor.execute(
     datasets,
     LMomentsEstimator(
@@ -39,7 +33,7 @@
     ),
 )
 
-executor = RandomExperimentExecutor(WORKING_DIR, 5)
+executor = RandomExperimentExecutor(WORKING_DIR, 5, 43)
 executor.execute(
     datasets,
     LikelihoodEstimator(

script_stage_3.py

@@ -1,3 +1,5 @@
+"""The script implements the third step of the experiment"""
+
 from pathlib import Path
 
 from experimental_env.analysis.analysis import Analysis
@@ -13,28 +15,15 @@
 from experimental_env.analysis.metrics import SquaredError
 from experimental_env.experiment.experiment_parser import ExperimentParser
 
-EXPERIMENT_DIR = "experiment"
-WORKING_DIR = Path(
-    f"/home/danil/PycharmProjects/Projects/EM-algo-DT/{EXPERIMENT_DIR}/stage_3"
-)
-
+WORKING_DIR = Path(dir_stage_2)
 
-# Compare results
-LMOMENTS_DIR = Path(
-    f"/home/danil/PycharmProjects/Projects/EM-algo-DT/{EXPERIMENT_DIR}/stage_2/LM-EM"
-)
-LIKELIHOOD_DIR = Path(
-    f"/home/danil/PycharmProjects/Projects/EM-algo-DT/{EXPERIMENT_DIR}/stage_2/MLE-EM"
-)
+LIKELIHOOD_DIR = Path(dir_EM_results)
+LMOMENTS_DIR = Path(dir_ELM_results)
 
 results_1 = ExperimentParser().parse(LMOMENTS_DIR)
 results_2 = ExperimentParser().parse(LIKELIHOOD_DIR)
 
 analyze_actions = [DensityPlot(), TimePlot(), ErrorConvergence(SquaredError())]
 analyze_summarizers = [ErrorSummarizer(SquaredError()), TimeSummarizer()]
 
-Analysis(WORKING_DIR, analyze_actions, analyze_summarizers).analyze(results_1, "LM-EM")
-Analysis(WORKING_DIR, analyze_actions, analyze_summarizers).analyze(results_2, "MLE-EM")
-Analysis(WORKING_DIR, analyze_actions, analyze_summarizers).compare(
-    results_1, results_2, "LM-EM", "MLE-EM"
-)
+Analysis(WORKING_DIR, analyze_actions, analyze_summarizers).compare(results_1, results_2, "ELM", "MLE-EM")