Update the testing pipeline

basicts/metrics/__init__.py

@@ -1,6 +1,8 @@
+from .cls_metrics import accuracy, f1_score, precision, recall
 from .corr import masked_corr
 from .mae import masked_mae
 from .mape import masked_mape
+from .metric_meter import AvgMeter, RMSEMeter
 from .mse import masked_mse
 from .r_square import masked_r2
 from .rmse import masked_rmse
@@ -15,17 +17,32 @@
             'WAPE': masked_wape,
             'SMAPE': masked_smape,
             'R2': masked_r2,
-            'CORR': masked_corr
+            'CORR': masked_corr,
+            "accuracy": accuracy,
+            "precision": precision,
+            "recall": recall,
+            "f1": f1_score
             }
 
+METRIC_METER = {
+    'RMSE': RMSEMeter,
+    'default': AvgMeter
+}
+
 __all__ = [
     'masked_mae',
     'masked_mse',
     'masked_rmse',
+    'incremental_masked_rmse',
     'masked_mape',
     'masked_wape',
     'masked_smape',
     'masked_r2',
     'masked_corr',
-    'ALL_METRICS'
+    'accuracy',
+    'precision',
+    'recall',
+    'f1_score',
+    'ALL_METRICS',
+    'METRIC_METER'
 ]

basicts/metrics/cls_metrics.py

@@ -0,0 +1,60 @@
+import torch
+
+
+def accuracy(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+    """
+    Calculate the accuracy of predictions.
+
+    Args:
+        pred (torch.Tensor): The predicted values as a tensor.
+        target (torch.Tensor): The ground truth values as a tensor with the same shape as `pred`.
+
+    Returns:
+        torch.Tensor: A scalar tensor representing the accuracy.
+    """
+    return (pred == target).float().mean()
+
+def precision(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+    """
+    Calculate the precision of predictions.
+
+    Args:
+        pred (torch.Tensor): The predicted values as a tensor.
+        target (torch.Tensor): The ground truth values as a tensor with the same shape as `pred`.
+
+    Returns:
+        torch.Tensor: A scalar tensor representing the precision.
+    """
+    true_positives = (pred == target).float().sum()
+    false_positives = (pred != target).float().sum()
+    return true_positives / (true_positives + false_positives)
+
+def recall(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+    """
+    Calculate the recall of predictions.
+
+    Args:
+        pred (torch.Tensor): The predicted values as a tensor.
+        target (torch.Tensor): The ground truth values as a tensor with the same shape as `pred`.
+
+    Returns:
+        torch.Tensor: A scalar tensor representing the recall.
+    """
+    true_positives = (pred == target).float().sum()
+    false_negatives = (pred != target).float().sum()
+    return true_positives / (true_positives + false_negatives)
+
+def f1_score(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+    """
+    Calculate the F1 score of predictions.
+
+    Args:
+        pred (torch.Tensor): The predicted values as a tensor.
+        target (torch.Tensor): The ground truth values as a tensor with the same shape as `pred`.
+
+    Returns:
+        torch.Tensor: A scalar tensor representing the F1 score.
+    """
+    precision_item = precision(pred, target)
+    recall_item = recall(pred, target)
+    return 2 * (precision_item * recall_item) / (precision_item + recall_item)

basicts/metrics/metric_meter.py

@@ -0,0 +1,76 @@
+class AvgMeter:
+    """Average meter.
+    """
+
+    def __init__(self):
+        self._sum: float = 0.
+        self._count: int = 0
+
+    def reset(self):
+        """Reset counter.
+        """
+
+        self._sum = 0.
+        self._count = 0
+
+    def update(self, value: float, n: int = 1):
+        """Update sum and count.
+
+        Args:
+            value (float): value.
+            n (int): number.
+        """
+
+        self._sum += value * n
+        self._count += n
+
+    @property
+    def value(self) -> float:
+        """Get average value.
+
+        Returns:
+            avg (float)
+        """
+
+        return self._sum / self._count if self._count != 0 else 0
+
+
+class RMSEMeter:
+    """
+    RMSE meter.
+    This meter maintains **MSE** and calculate **RMSE** in the post process.
+    """
+
+    def __init__(self):
+        self._mse: float = 0.
+        self._count: int = 0
+
+    def reset(self):
+        """Reset counter.
+        """
+
+        self._mse = 0.
+        self._count = 0
+
+    def update(self, value: float, n: int = 1):
+        """Update sum and count.
+
+        Args:
+            value (float): value.
+            n (int): number.
+        """
+
+        self._mse += value ** 2 * n
+        self._count += n
+
+    @property
+    def value(self) -> float:
+        """Get average value.
+
+        Returns:
+            avg (float)
+        """
+
+        mse = self._mse / self._count if self._count != 0 else 0
+
+        return mse ** 0.5

basicts/runners/base_epoch_runner.py

@@ -10,7 +10,6 @@
 from easytorch.core.checkpoint import (backup_last_ckpt, clear_ckpt, load_ckpt,
                                        save_ckpt)
 from easytorch.core.data_loader import build_data_loader, build_data_loader_ddp
-from easytorch.core.meter_pool import MeterPool
 from easytorch.device import to_device
 from easytorch.utils import (TimePredictor, get_local_rank, get_logger,
                              is_master, master_only, set_env)
@@ -22,7 +21,7 @@
 from torch.utils.tensorboard import SummaryWriter
 from tqdm import tqdm
 
-from ..utils import get_dataset_name
+from ..utils import MeterPool, get_dataset_name
 from . import optim
 
 
@@ -546,18 +545,24 @@ def test_pipeline(self, cfg: Optional[Dict] = None, train_epoch: Optional[int] =
         self.model.eval()
 
         # execute the test process
-        self.test(train_epoch=train_epoch, save_results=save_results, save_metrics=save_metrics)
+        self.test(train_epoch=train_epoch, save_metrics=save_metrics, save_results=save_results)
 
         test_end_time = time.time()
         self.update_epoch_meter('test/time', test_end_time - test_start_time)
 
         self.print_epoch_meters('test')
+
         if train_epoch is not None:
             self.plt_epoch_meters('test', train_epoch // self.test_interval)
 
+        if len(self.evaluation_horizons) > 0:
+            self.logger.info(f'Evaluation on horizons: {[h + 1 for h in self.evaluation_horizons]}.')
+            for i in self.evaluation_horizons:
+                self.print_epoch_meters(f'test @ horizon {i+1}')
+
         # logging here for intuitiveness
-        if save_results:
-            self.logger.info(f'Test results saved to {os.path.join(self.ckpt_save_dir, "test_results.npz")}.')
+        if self.save_results:
+            self.logger.info(f'Test results saved to {os.path.join(self.ckpt_save_dir, "test_results")}.')
         if save_metrics:
             self.logger.info(f'Test metrics saved to {os.path.join(self.ckpt_save_dir, "test_metrics.json")}.')
 
@@ -591,7 +596,7 @@ def inference_pipeline(self, cfg: Optional[Dict] = None, input_data: Union[str,
         result = self.inference(save_result_path=output_data_file_path)
 
         inference_end_time = time.time()
-        self.update_epoch_meter('inference/time', inference_end_time - inference_start_time)
+        self.update_epoch_meter('inference/time', 'inference', inference_end_time - inference_start_time)
 
         self.print_epoch_meters('inference')
 
@@ -649,7 +654,6 @@ def test(self, train_epoch: Optional[int] = None, save_metrics: bool = False, sa
         Args:
             train_epoch (int, optional): Current epoch during training. Defaults to None.
             save_metrics (bool, optional): Save the test metrics. Defaults to False.
-            save_results (bool, optional): Save the test results. Defaults to False.
 
         Raises:
             NotImplementedError: Must be implemented in a subclass.
@@ -919,7 +923,7 @@ def save_best_model(self, epoch: int, metric_name: str, greater_best: bool = Tru
                 `False` means lower value is best, such as `loss`. Defaults to True.
         """
 
-        metric = self.meter_pool.get_avg(metric_name)
+        metric = self.meter_pool.get_value(metric_name)
         best_metric = self.best_metrics.get(metric_name)
         if best_metric is None or (metric > best_metric if greater_best else metric < best_metric):
             self.best_metrics[metric_name] = metric