Skip to content

Commit dc52979

Browse files
ASuduASudu
committed
chronosbolt leaderboard updated
1 parent 940d66e commit dc52979

2 files changed

Lines changed: 158 additions & 138 deletions

File tree

‎leaderboard.py‎

Lines changed: 129 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -75,136 +75,137 @@ def calc_pred_and_context_len(freq):
7575

7676

7777
if __name__ == "__main__":
78-
model_name = MODEL_NAMES[1]
79-
80-
# create csv file for leaderboard if not already created
81-
csv_path = f"leaderboard/{model_name}.csv"
82-
if not os.path.exists(csv_path):
83-
print(f"Creating leaderboard csv file: {csv_path}")
84-
df = pd.DataFrame(columns=["dataset", "size_in_MB", "eval_time", "mse", "mae", "mase", "mape", "rmse", "nrmse", "smape", "msis", "nd", "mwsq", "crps"])
85-
df.to_csv(csv_path, index=False)
86-
87-
# Load model config
88-
if model_name == "timesfm":
89-
arg_path = "config/timesfm.json"
90-
args = load_args(arg_path)
91-
elif model_name == "moment":
92-
arg_path = "config/moment_forecast.json"
93-
args = load_args(arg_path)
94-
elif model_name == "chronos":
95-
arg_path = "config/chronos.json"
96-
args = load_args(arg_path)
97-
elif model_name == "ttm":
98-
arg_path = "config/tinytimemixer.json"
99-
args = load_args(arg_path)
100-
elif model_name == "moirai":
101-
arg_path = "config/moirai.json"
102-
args = load_args(arg_path)
103-
104-
for fname, freq, fs in filesizes:
105-
print(f"Evaluating {fname} ({freq})")
106-
# Adjust the context and prediction length based on the frequency
107-
# pred_len, context_len = calc_pred_and_context_len(freq)
108-
pred_len, context_len = 96, 512
109-
if model_name == "timesfm":
110-
args["config"]["horizon_len"] = pred_len
111-
args["config"]["context_len"] = context_len
112-
elif model_name == "moment":
113-
args["config"]["forecast_horizon"] = pred_len
114-
elif model_name == "ttm":
115-
args["config"]["horizon_len"] = pred_len
116-
args["config"]["context_len"] = context_len
117-
elif model_name == "moirai":
118-
args["config"]["horizon_len"] = pred_len
119-
args["config"]["context_len"] = context_len
120-
121-
# Set the dataset path
122-
if len(NAMES.get(fname)) == 1:
123-
dataset_path = f"data/gifteval/{fname}/data.csv"
124-
else:
125-
dataset_path = f"data/gifteval/{fname}/{freq}/data.csv"
126-
127-
# Initialize the model and dataset
78+
79+
for model_name in MODEL_NAMES[4:]:
80+
print(f"Evaluating model: {model_name}")
81+
# create csv file for leaderboard if not already created
82+
csv_path = f"leaderboard/{model_name}.csv"
83+
if not os.path.exists(csv_path):
84+
print(f"Creating leaderboard csv file: {csv_path}")
85+
df = pd.DataFrame(columns=["dataset", "size_in_MB", "eval_time", "mse", "mae", "mase", "mape", "rmse", "nrmse", "smape", "msis", "nd", "mwsq", "crps"])
86+
df.to_csv(csv_path, index=False)
87+
88+
# Load model config
12889
if model_name == "timesfm":
129-
model = TimesfmModel(**args)
130-
dataset = TimesfmDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=args["config"]["context_len"], horizon_len=args["config"]["horizon_len"], boundaries=(-1, -1, -1), batchsize=64)
131-
start = time.time()
132-
metrics = model.evaluate(dataset)
133-
end = time.time()
134-
print(f"Size of dataset: {fs:.2f} MB")
135-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
136-
90+
arg_path = "config/timesfm.json"
91+
args = load_args(arg_path)
13792
elif model_name == "moment":
138-
model = MomentModel(**args)
139-
args["config"]["task_name"] = "forecasting"
140-
train_dataset = MomentDataset(datetime_col='timestamp', path=dataset_path, mode='train', horizon_len=args["config"]["forecast_horizon"], normalize=False)
141-
dataset = MomentDataset(datetime_col='timestamp', path=dataset_path, mode='test', horizon_len=args["config"]["forecast_horizon"], normalize=False)
142-
finetuned_model = model.finetune(train_dataset, task_name="forecasting")
143-
start = time.time()
144-
metrics = model.evaluate(dataset, task_name="forecasting")
145-
end = time.time()
146-
print(f"Size of dataset: {fs:.2f} MB")
147-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
148-
print(metrics)
149-
93+
arg_path = "config/moment_forecast.json"
94+
args = load_args(arg_path)
15095
elif model_name == "chronos":
151-
model = ChronosModel(**args)
152-
dataset_config = load_args("config/chronos_dataset.json")
153-
dataset_config["context_length"] = context_len
154-
dataset_config["prediction_length"] = pred_len
155-
dataset = ChronosDataset(datetime_col='timestamp', path=dataset_path, mode='test', config=dataset_config, batch_size=4)
156-
start = time.time()
157-
metrics = model.evaluate(dataset, horizon_len=dataset_config["prediction_length"], quantile_levels=[0.1, 0.5, 0.9])
158-
end = time.time()
159-
print(f"Size of dataset: {fs:.2f} MB")
160-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
161-
162-
elif model_name == "chronosbolt":
163-
repo = "amazon/chronos-bolt-small"
164-
model = ChronosBoltModel(repo=repo)
165-
dataset = ChronosBoltDataset(datetime_col='timestamp', path=dataset_path, mode='test', batch_size=8, context_len=context_len, horizon_len=pred_len)
166-
start = time.time()
167-
metrics = model.evaluate(dataset, horizon_len=pred_len, quantile_levels=[0.1, 0.5, 0.9])
168-
end = time.time()
169-
print(f"Size of dataset: {fs:.2f} MB")
170-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
171-
96+
arg_path = "config/chronos.json"
97+
args = load_args(arg_path)
17298
elif model_name == "ttm":
173-
model = TinyTimeMixerModel(**args)
174-
dataset = TinyTimeMixerDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=context_len, horizon_len=pred_len)
175-
start = time.time()
176-
metrics = model.evaluate(dataset)
177-
end = time.time()
178-
print(f"Size of dataset: {fs:.2f} MB")
179-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
180-
99+
arg_path = "config/tinytimemixer.json"
100+
args = load_args(arg_path)
181101
elif model_name == "moirai":
182-
model = MoiraiTSModel(**args)
183-
dataset = MoiraiDataset(name=fname,datetime_col='timestamp', freq=freq,
184-
path=dataset_path, mode='test', context_len=context_len, horizon_len=pred_len)
185-
186-
start = time.time()
187-
metrics = model.evaluate(dataset,leaderboard=True)
188-
end = time.time()
189-
print(f"Size of dataset: {fs:.2f} MB")
190-
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
191-
192-
print("Evaluation done!")
193-
194-
eval_time = end - start
195-
unit = "s"
196-
if eval_time > 1000: # convert to minutes
197-
eval_time = eval_time / 60
198-
unit = "m"
199-
200-
201-
df = pd.read_csv(csv_path)
202-
if fname in df["dataset"].values:
203-
df.loc[df["dataset"] == fname, "size_in_MB"] = round(fs,2)
204-
df.loc[df["dataset"] == fname, "eval_time"] = str(round(eval_time,2)) + unit
205-
df.loc[df["dataset"] == fname, list(metrics.keys())] = list(metrics.values())
206-
else:
207-
new_row = pd.DataFrame([{**{"dataset": fname, "size_in_MB":round(fs,2), "eval_time":str(round(eval_time,2)) + unit}, **metrics}])
208-
df = pd.concat([df, new_row], ignore_index=True)
209-
210-
df.to_csv(csv_path, index=False)
102+
arg_path = "config/moirai.json"
103+
args = load_args(arg_path)
104+
105+
for fname, freq, fs in filesizes:
106+
print(f"Evaluating {fname} ({freq})")
107+
# Adjust the context and prediction length based on the frequency
108+
# pred_len, context_len = calc_pred_and_context_len(freq)
109+
pred_len, context_len = 96, 512
110+
if model_name == "timesfm":
111+
args["config"]["horizon_len"] = pred_len
112+
args["config"]["context_len"] = context_len
113+
elif model_name == "moment":
114+
args["config"]["forecast_horizon"] = pred_len
115+
elif model_name == "ttm":
116+
args["config"]["horizon_len"] = pred_len
117+
args["config"]["context_len"] = context_len
118+
elif model_name == "moirai":
119+
args["config"]["horizon_len"] = pred_len
120+
args["config"]["context_len"] = context_len
121+
122+
# Set the dataset path
123+
if len(NAMES.get(fname)) == 1:
124+
dataset_path = f"data/gifteval/{fname}/data.csv"
125+
else:
126+
dataset_path = f"data/gifteval/{fname}/{freq}/data.csv"
127+
128+
# Initialize the model and dataset
129+
if model_name == "timesfm":
130+
model = TimesfmModel(**args)
131+
dataset = TimesfmDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=args["config"]["context_len"], horizon_len=args["config"]["horizon_len"], boundaries=(-1, -1, -1), batchsize=64)
132+
start = time.time()
133+
metrics = model.evaluate(dataset)
134+
end = time.time()
135+
print(f"Size of dataset: {fs:.2f} MB")
136+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
137+
138+
elif model_name == "moment":
139+
model = MomentModel(**args)
140+
args["config"]["task_name"] = "forecasting"
141+
train_dataset = MomentDataset(datetime_col='timestamp', path=dataset_path, mode='train', horizon_len=args["config"]["forecast_horizon"], normalize=False)
142+
dataset = MomentDataset(datetime_col='timestamp', path=dataset_path, mode='test', horizon_len=args["config"]["forecast_horizon"], normalize=False)
143+
finetuned_model = model.finetune(train_dataset, task_name="forecasting")
144+
start = time.time()
145+
metrics = model.evaluate(dataset, task_name="forecasting")
146+
end = time.time()
147+
print(f"Size of dataset: {fs:.2f} MB")
148+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
149+
print(metrics)
150+
151+
elif model_name == "chronos":
152+
model = ChronosModel(**args)
153+
dataset_config = load_args("config/chronos_dataset.json")
154+
dataset_config["context_length"] = context_len
155+
dataset_config["prediction_length"] = pred_len
156+
dataset = ChronosDataset(datetime_col='timestamp', path=dataset_path, mode='test', config=dataset_config, batch_size=4)
157+
start = time.time()
158+
metrics = model.evaluate(dataset, horizon_len=dataset_config["prediction_length"], quantile_levels=[0.1, 0.5, 0.9])
159+
end = time.time()
160+
print(f"Size of dataset: {fs:.2f} MB")
161+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
162+
163+
elif model_name == "chronosbolt":
164+
repo = "amazon/chronos-bolt-small"
165+
model = ChronosBoltModel(repo=repo)
166+
dataset = ChronosBoltDataset(datetime_col='timestamp', path=dataset_path, mode='test', batch_size=8, context_len=context_len, horizon_len=pred_len)
167+
start = time.time()
168+
metrics = model.evaluate(dataset, horizon_len=pred_len, quantile_levels=[0.1, 0.5, 0.9])
169+
end = time.time()
170+
print(f"Size of dataset: {fs:.2f} MB")
171+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
172+
173+
elif model_name == "ttm":
174+
model = TinyTimeMixerModel(**args)
175+
dataset = TinyTimeMixerDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=context_len, horizon_len=pred_len)
176+
start = time.time()
177+
metrics = model.evaluate(dataset)
178+
end = time.time()
179+
print(f"Size of dataset: {fs:.2f} MB")
180+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
181+
182+
elif model_name == "moirai":
183+
model = MoiraiTSModel(**args)
184+
dataset = MoiraiDataset(name=fname,datetime_col='timestamp', freq=freq,
185+
path=dataset_path, mode='test', context_len=context_len, horizon_len=pred_len)
186+
187+
start = time.time()
188+
metrics = model.evaluate(dataset,leaderboard=True)
189+
end = time.time()
190+
print(f"Size of dataset: {fs:.2f} MB")
191+
print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
192+
193+
print("Evaluation done!")
194+
195+
eval_time = end - start
196+
unit = "s"
197+
if eval_time > 1000: # convert to minutes
198+
eval_time = eval_time / 60
199+
unit = "m"
200+
201+
202+
df = pd.read_csv(csv_path)
203+
if fname in df["dataset"].values:
204+
df.loc[df["dataset"] == fname, "size_in_MB"] = round(fs,2)
205+
df.loc[df["dataset"] == fname, "eval_time"] = str(round(eval_time,2)) + unit
206+
df.loc[df["dataset"] == fname, list(metrics.keys())] = list(metrics.values())
207+
else:
208+
new_row = pd.DataFrame([{**{"dataset": fname, "size_in_MB":round(fs,2), "eval_time":str(round(eval_time,2)) + unit}, **metrics}])
209+
df = pd.concat([df, new_row], ignore_index=True)
210+
211+
df.to_csv(csv_path, index=False)

0 commit comments

Comments
 (0)