add evaluate for utsf models

baselines/ChronosBolt/config/chronos_base.py

@@ -16,10 +16,10 @@
 MODEL_ARCH = ChronosBolt
 
 context_length = 1024
-predict_length = 64 # ref: /workspace/S22/TSFM_LLaMA3/huggingface_ckpts/chronos-bolt-tiny/config.json
+predict_length = 64 # ref: chronos-bolt-base/config.json
 
 MODEL_PARAM = {
-    "model_id": "./baselines/ChronosBolt/ckpt/chronos-bolt-base",
+    "model_id": "baselines/ChronosBolt/ckpt/chronos-bolt-base",
     "from_pretrained": False,
     "device_map": "cpu",
 }
@@ -110,3 +110,8 @@
     'target_length': predict_length,
     'num_valid_samples': 1000
 })
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+})

baselines/ChronosBolt/evaluate_config/ETTh1/base.py

@@ -0,0 +1,104 @@
+import os
+import sys
+from easydict import EasyDict
+
+from basicts.data.simple_tsf_dataset import TimeSeriesForecastingDataset
+from basicts.scaler.z_score_scaler import ZScoreScaler
+from basicts.utils.serialization import get_regular_settings
+from basicts.metrics import masked_mae, masked_mse
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ...arch import ChronosBolt
+from ...data import BLASTDatasetWoMixUp
+from ...runner import ChronosRunner
+from ...loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+MODEL_ARCH = ChronosBolt
+
+CONTEXT_LENGTH = None
+PREDICTION_LENGTH = None # ref: chronos-bolt-tiny/config.json
+
+MODEL_PARAM = {
+    "model_id": "baselines/ChronosBolt/ckpt/chronos-bolt-base",
+    "from_pretrained": False,
+    "device_map": "cpu",
+}
+DATA_NAME = "ETTh1"
+
+NUM_ITERATIONS = None # 总轮数
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'Chronos-Bolt Base | Debug: Data'
+CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# Runner
+CFG.RUNNER = ChronosRunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = False
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+
+regular_settings = get_regular_settings(dataset_name=DATA_NAME)
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = TimeSeriesForecastingDataset
+CFG.DATASET.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_val_test_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'],
+    'input_len': CONTEXT_LENGTH,
+    'output_len': PREDICTION_LENGTH,
+    'overlap': True
+})
+CFG.TEST = EasyDict()
+CFG.TEST.DATA = EasyDict()
+CFG.TEST.DATA.BATCH_SIZE = 32
+CFG.TEST.DATA.SHUFFLE = False
+
+############################## TEST Scaler Configuration ##############################
+CFG.SCALER = EasyDict()
+# Scaler settings
+CFG.SCALER.TYPE = ZScoreScaler # Scaler class
+CFG.SCALER.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'][0],
+    'norm_each_channel': regular_settings['NORM_EACH_CHANNEL'],
+    'rescale': regular_settings['RESCALE'],
+})
+
+############################## Metrics Configuration ##############################
+
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({
+                                'MAE': masked_mae,
+                                'MSE': masked_mse,
+                            })
+CFG.METRICS.NULL_VAL = regular_settings['NULL_VAL'] # Null value in the data
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+})

baselines/ChronosBolt/evaluate_config/ETTh1/small.py

@@ -0,0 +1,104 @@
+import os
+import sys
+from easydict import EasyDict
+
+from basicts.data.simple_tsf_dataset import TimeSeriesForecastingDataset
+from basicts.scaler.z_score_scaler import ZScoreScaler
+from basicts.utils.serialization import get_regular_settings
+from basicts.metrics import masked_mae, masked_mse
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ...arch import ChronosBolt
+from ...data import BLASTDatasetWoMixUp
+from ...runner import ChronosRunner
+from ...loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+MODEL_ARCH = ChronosBolt
+
+CONTEXT_LENGTH = None
+PREDICTION_LENGTH = None # ref: chronos-bolt-tiny/config.json
+
+MODEL_PARAM = {
+    "model_id": "baselines/ChronosBolt/ckpt/chronos-bolt-small",
+    "from_pretrained": False,
+    "device_map": "cpu",
+}
+DATA_NAME = "ETTh1"
+
+NUM_ITERATIONS = None # 总轮数
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'Chronos-Bolt Base | Debug: Data'
+CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# Runner
+CFG.RUNNER = ChronosRunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = False
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+
+regular_settings = get_regular_settings(dataset_name=DATA_NAME)
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = TimeSeriesForecastingDataset
+CFG.DATASET.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_val_test_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'],
+    'input_len': CONTEXT_LENGTH,
+    'output_len': PREDICTION_LENGTH,
+    'overlap': True
+})
+CFG.TEST = EasyDict()
+CFG.TEST.DATA = EasyDict()
+CFG.TEST.DATA.BATCH_SIZE = 32
+CFG.TEST.DATA.SHUFFLE = False
+
+############################## TEST Scaler Configuration ##############################
+CFG.SCALER = EasyDict()
+# Scaler settings
+CFG.SCALER.TYPE = ZScoreScaler # Scaler class
+CFG.SCALER.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'][0],
+    'norm_each_channel': regular_settings['NORM_EACH_CHANNEL'],
+    'rescale': regular_settings['RESCALE'],
+})
+
+############################## Metrics Configuration ##############################
+
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({
+                                'MAE': masked_mae,
+                                'MSE': masked_mse,
+                            })
+CFG.METRICS.NULL_VAL = regular_settings['NULL_VAL'] # Null value in the data
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+})

baselines/ChronosBolt/evaluate_config/ETTh2/base.py

@@ -0,0 +1,104 @@
+import os
+import sys
+from easydict import EasyDict
+
+from basicts.data.simple_tsf_dataset import TimeSeriesForecastingDataset
+from basicts.scaler.z_score_scaler import ZScoreScaler
+from basicts.utils.serialization import get_regular_settings
+from basicts.metrics import masked_mae, masked_mse
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ...arch import ChronosBolt
+from ...data import BLASTDatasetWoMixUp
+from ...runner import ChronosRunner
+from ...loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+MODEL_ARCH = ChronosBolt
+
+CONTEXT_LENGTH = None
+PREDICTION_LENGTH = None # ref: chronos-bolt-tiny/config.json
+
+MODEL_PARAM = {
+    "model_id": "baselines/ChronosBolt/ckpt/chronos-bolt-base",
+    "from_pretrained": False,
+    "device_map": "cpu",
+}
+DATA_NAME = "ETTh2"
+
+NUM_ITERATIONS = None # 总轮数
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'Chronos-Bolt Base | Debug: Data'
+CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# CFG.GPU_NUM = 8 # Number of GPUs to use (0 for CPU mode)
+# Runner
+CFG.RUNNER = ChronosRunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = False
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+
+regular_settings = get_regular_settings(dataset_name=DATA_NAME)
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = TimeSeriesForecastingDataset
+CFG.DATASET.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_val_test_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'],
+    'input_len': CONTEXT_LENGTH,
+    'output_len': PREDICTION_LENGTH,
+    'overlap': True
+})
+CFG.TEST = EasyDict()
+CFG.TEST.DATA = EasyDict()
+CFG.TEST.DATA.BATCH_SIZE = 32
+CFG.TEST.DATA.SHUFFLE = False
+
+############################## TEST Scaler Configuration ##############################
+CFG.SCALER = EasyDict()
+# Scaler settings
+CFG.SCALER.TYPE = ZScoreScaler # Scaler class
+CFG.SCALER.PARAM = EasyDict({
+    'dataset_name': DATA_NAME,
+    'train_ratio': regular_settings['TRAIN_VAL_TEST_RATIO'][0],
+    'norm_each_channel': regular_settings['NORM_EACH_CHANNEL'],
+    'rescale': regular_settings['RESCALE'],
+})
+
+############################## Metrics Configuration ##############################
+
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({
+                                'MAE': masked_mae,
+                                'MSE': masked_mse,
+                            })
+CFG.METRICS.NULL_VAL = regular_settings['NULL_VAL'] # Null value in the data
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+})