Skip to content

Commit 44ff48c

Browse files
ChernovAndreyChernovAndrey
committed
updated notebok
1 parent 80f3f8b commit 44ff48c

1 file changed

Lines changed: 129 additions & 48 deletions

File tree

‎examples/model_comparison_simple.ipynb‎

Lines changed: 129 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -10,32 +10,40 @@
1010
]
1111
},
1212
{
13+
"cell_type": "code",
1314
"metadata": {
1415
"ExecuteTime": {
15-
"end_time": "2025-11-05T17:50:00.017080Z",
16-
"start_time": "2025-11-05T17:50:00.011339Z"
16+
"end_time": "2025-11-05T18:01:16.424417Z",
17+
"start_time": "2025-11-05T18:01:16.421605Z"
1718
}
1819
},
19-
"cell_type": "code",
2020
"source": [
2121
"import os\n",
22+
"\n",
2223
"# provide FAIM API KEY here\n",
2324
"FAIM_API_KEY = os.environ['FAIM_API_KEY']"
2425
],
2526
"outputs": [],
26-
"execution_count": 24
27+
"execution_count": 1
2728
},
2829
{
2930
"cell_type": "code",
3031
"metadata": {
3132
"ExecuteTime": {
32-
"end_time": "2025-11-05T17:50:00.034908Z",
33-
"start_time": "2025-11-05T17:50:00.031883Z"
33+
"end_time": "2025-11-05T18:01:17.160610Z",
34+
"start_time": "2025-11-05T18:01:16.575528Z"
3435
}
3536
},
36-
"source": "import matplotlib.pyplot as plt\nimport numpy as np\nfrom generate_data import generate_linear_trend_series\n\nfrom faim_sdk import Chronos2ForecastRequest, FlowStateForecastRequest, ForecastClient, TiRexForecastRequest\nfrom faim_sdk.eval import crps_from_quantiles, mae, mse",
37+
"source": [
38+
"import matplotlib.pyplot as plt\n",
39+
"import numpy as np\n",
40+
"from generate_data import generate_linear_trend_series\n",
41+
"\n",
42+
"from faim_sdk import Chronos2ForecastRequest, FlowStateForecastRequest, ForecastClient, TiRexForecastRequest\n",
43+
"from faim_sdk.eval import mae, mse"
44+
],
3745
"outputs": [],
38-
"execution_count": 25
46+
"execution_count": 2
3947
},
4048
{
4149
"cell_type": "markdown",
@@ -46,8 +54,8 @@
4654
"cell_type": "code",
4755
"metadata": {
4856
"ExecuteTime": {
49-
"end_time": "2025-11-05T17:50:00.090111Z",
50-
"start_time": "2025-11-05T17:50:00.086690Z"
57+
"end_time": "2025-11-05T18:01:17.175867Z",
58+
"start_time": "2025-11-05T18:01:17.171578Z"
5159
}
5260
},
5361
"source": [
@@ -73,7 +81,7 @@
7381
]
7482
}
7583
],
76-
"execution_count": 26
84+
"execution_count": 3
7785
},
7886
{
7987
"cell_type": "markdown",
@@ -86,8 +94,8 @@
8694
"cell_type": "code",
8795
"metadata": {
8896
"ExecuteTime": {
89-
"end_time": "2025-11-05T17:50:03.098380Z",
90-
"start_time": "2025-11-05T17:50:00.121843Z"
97+
"end_time": "2025-11-05T18:01:20.572932Z",
98+
"start_time": "2025-11-05T18:01:17.194514Z"
9199
}
92100
},
93101
"source": [
@@ -115,7 +123,7 @@
115123
]
116124
}
117125
],
118-
"execution_count": 27
126+
"execution_count": 4
119127
},
120128
{
121129
"cell_type": "markdown",
@@ -128,11 +136,25 @@
128136
"cell_type": "code",
129137
"metadata": {
130138
"ExecuteTime": {
131-
"end_time": "2025-11-05T17:50:03.136415Z",
132-
"start_time": "2025-11-05T17:50:03.133247Z"
139+
"end_time": "2025-11-05T18:01:20.602274Z",
140+
"start_time": "2025-11-05T18:01:20.599145Z"
133141
}
134142
},
135-
"source": "# Calculate metrics for each model\nmodels = {\n \"FlowState\": flowstate_response.point,\n \"Chronos2\": chronos2_response.point,\n \"TiRex\": tirex_response.point\n}\n\nprint(\"Point Forecast Metrics:\")\nprint(\"-\" * 50)\nfor name, pred in models.items():\n mse_score = mse(test_data, pred)\n mae_score = mae(test_data, pred)\n print(f\"{name:12s} - MSE: {mse_score:.4f}, MAE: {mae_score:.4f}\")",
143+
"source": [
144+
"# Calculate metrics for each model\n",
145+
"models = {\n",
146+
" \"FlowState\": flowstate_response.point,\n",
147+
" \"Chronos2\": chronos2_response.point,\n",
148+
" \"TiRex\": tirex_response.point\n",
149+
"}\n",
150+
"\n",
151+
"print(\"Point Forecast Metrics:\")\n",
152+
"print(\"-\" * 50)\n",
153+
"for name, pred in models.items():\n",
154+
" mse_score = mse(test_data, pred)\n",
155+
" mae_score = mae(test_data, pred)\n",
156+
" print(f\"{name:12s} - MSE: {mse_score:.4f}, MAE: {mae_score:.4f}\")"
157+
],
136158
"outputs": [
137159
{
138160
"name": "stdout",
@@ -146,7 +168,7 @@
146168
]
147169
}
148170
],
149-
"execution_count": 28
171+
"execution_count": 5
150172
},
151173
{
152174
"cell_type": "markdown",
@@ -159,11 +181,37 @@
159181
"cell_type": "code",
160182
"metadata": {
161183
"ExecuteTime": {
162-
"end_time": "2025-11-05T17:50:03.269751Z",
163-
"start_time": "2025-11-05T17:50:03.180507Z"
184+
"end_time": "2025-11-05T18:01:20.728973Z",
185+
"start_time": "2025-11-05T18:01:20.626677Z"
164186
}
165187
},
166-
"source": "# Plot all forecasts together\nfig, ax = plt.subplots(figsize=(12, 6))\n\n# Plot only last 'horizon' points of training data\ntrain_context_start = max(0, train_size - horizon)\ntrain_context_indices = np.arange(train_context_start, train_size)\nax.plot(train_context_indices, train_data[0, train_context_start:, 0], label=\"Training Data\", color=\"blue\", linewidth=2)\n\n# Plot test data\ntest_indices = np.arange(train_size, train_size + horizon)\nax.plot(test_indices, test_data[0, :, 0], label=\"Test Data\", color=\"black\", linewidth=2, linestyle=\"--\")\n\n# Plot forecasts\ncolors = {\"FlowState\": \"green\", \"Chronos2\": \"red\", \"TiRex\": \"orange\"}\nfor name, pred in models.items():\n ax.plot(test_indices, pred[0, :, 0], label=f\"{name} Forecast\", color=colors[name], linewidth=1.5, alpha=0.8)\n\nax.axvline(x=train_size, color=\"gray\", linestyle=\":\", linewidth=1, alpha=0.7)\nax.set_xlabel(\"Time\")\nax.set_ylabel(\"Value\")\nax.set_title(\"Point Forecast Comparison\")\nax.legend(loc=\"best\")\nax.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.show()",
188+
"source": [
189+
"# Plot all forecasts together\n",
190+
"fig, ax = plt.subplots(figsize=(12, 6))\n",
191+
"\n",
192+
"# Plot only last 'horizon' points of training data\n",
193+
"train_context_start = max(0, train_size - horizon)\n",
194+
"train_context_indices = np.arange(train_context_start, train_size)\n",
195+
"ax.plot(train_context_indices, train_data[0, train_context_start:, 0], label=\"Training Data\", color=\"blue\", linewidth=2)\n",
196+
"\n",
197+
"# Plot test data\n",
198+
"test_indices = np.arange(train_size, train_size + horizon)\n",
199+
"ax.plot(test_indices, test_data[0, :, 0], label=\"Test Data\", color=\"black\", linewidth=2, linestyle=\"--\")\n",
200+
"\n",
201+
"# Plot forecasts\n",
202+
"colors = {\"FlowState\": \"green\", \"Chronos2\": \"red\", \"TiRex\": \"orange\"}\n",
203+
"for name, pred in models.items():\n",
204+
" ax.plot(test_indices, pred[0, :, 0], label=f\"{name} Forecast\", color=colors[name], linewidth=1.5, alpha=0.8)\n",
205+
"\n",
206+
"ax.axvline(x=train_size, color=\"gray\", linestyle=\":\", linewidth=1, alpha=0.7)\n",
207+
"ax.set_xlabel(\"Time\")\n",
208+
"ax.set_ylabel(\"Value\")\n",
209+
"ax.set_title(\"Point Forecast Comparison\")\n",
210+
"ax.legend(loc=\"best\")\n",
211+
"ax.grid(True, alpha=0.3)\n",
212+
"plt.tight_layout()\n",
213+
"plt.show()"
214+
],
167215
"outputs": [
168216
{
169217
"data": {
@@ -176,7 +224,7 @@
176224
"output_type": "display_data"
177225
}
178226
],
179-
"execution_count": 29
227+
"execution_count": 6
180228
},
181229
{
182230
"cell_type": "markdown",
@@ -187,29 +235,11 @@
187235
"cell_type": "code",
188236
"metadata": {
189237
"ExecuteTime": {
190-
"end_time": "2025-11-05T17:50:06.070629Z",
191-
"start_time": "2025-11-05T17:50:03.297232Z"
238+
"end_time": "2025-11-05T18:01:23.556084Z",
239+
"start_time": "2025-11-05T18:01:20.748416Z"
192240
}
193241
},
194-
"source": [
195-
"# Generate quantile forecasts from all three models\n",
196-
"quantile_levels = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
197-
"\n",
198-
"flowstate_q_request = FlowStateForecastRequest(\n",
199-
" x=train_data, horizon=horizon, prediction_type=\"quantile\", output_type=\"quantiles\"\n",
200-
")\n",
201-
"flowstate_q_response = client.forecast(flowstate_q_request)\n",
202-
"\n",
203-
"chronos2_q_request = Chronos2ForecastRequest(\n",
204-
" x=train_data, horizon=horizon, output_type=\"quantiles\", quantiles=quantile_levels\n",
205-
")\n",
206-
"chronos2_q_response = client.forecast(chronos2_q_request)\n",
207-
"\n",
208-
"tirex_q_request = TiRexForecastRequest(x=train_data, horizon=horizon, output_type=\"quantiles\")\n",
209-
"tirex_q_response = client.forecast(tirex_q_request)\n",
210-
"\n",
211-
"print(\"Quantile forecasts generated successfully\")"
212-
],
242+
"source": "# Generate quantile forecasts from all three models\nquantile_levels = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n\nflowstate_q_request = FlowStateForecastRequest(\n x=train_data, horizon=horizon, prediction_type=\"quantile\", output_type=\"quantiles\"\n)\nflowstate_q_response = client.forecast(flowstate_q_request)\n\nchronos2_q_request = Chronos2ForecastRequest(\n x=train_data, horizon=horizon, output_type=\"quantiles\", quantiles=quantile_levels\n)\nchronos2_q_response = client.forecast(chronos2_q_request)\n\ntirex_q_request = TiRexForecastRequest(x=train_data, horizon=horizon, output_type=\"quantiles\")\ntirex_q_response = client.forecast(tirex_q_request)\n\n# Create dictionary of quantile models for visualization\nquantile_models = {\n \"FlowState\": flowstate_q_response.quantiles,\n \"Chronos2\": chronos2_q_response.quantiles,\n \"TiRex\": tirex_q_response.quantiles\n}\n\nprint(\"Quantile forecasts generated successfully\")",
213243
"outputs": [
214244
{
215245
"name": "stdout",
@@ -219,7 +249,7 @@
219249
]
220250
}
221251
],
222-
"execution_count": 30
252+
"execution_count": 7
223253
},
224254
{
225255
"cell_type": "markdown",
@@ -232,11 +262,62 @@
232262
"cell_type": "code",
233263
"metadata": {
234264
"ExecuteTime": {
235-
"end_time": "2025-11-05T17:50:06.293765Z",
236-
"start_time": "2025-11-05T17:50:06.084541Z"
265+
"end_time": "2025-11-05T18:01:23.853901Z",
266+
"start_time": "2025-11-05T18:01:23.589279Z"
237267
}
238268
},
239-
"source": "# Plot quantile forecasts for all models\nfig, axes = plt.subplots(3, 1, figsize=(12, 12))\n\n# Plot only last 'horizon' points of training data\ntrain_context_start = max(0, train_size - horizon)\ntrain_context_indices = np.arange(train_context_start, train_size)\n\nfor idx, (name, quantiles) in enumerate(quantile_models.items()):\n ax = axes[idx]\n \n # Plot training data (last horizon points)\n ax.plot(train_context_indices, train_data[0, train_context_start:, 0], label=\"Training Data\", color=\"blue\", linewidth=2)\n \n # Plot test data\n ax.plot(test_indices, test_data[0, :, 0], label=\"Test Data\", color=\"black\", linewidth=2, linestyle=\"--\")\n \n # Plot median (index 2 for 0.5 quantile)\n median_idx = 2\n ax.plot(test_indices, quantiles[0, :, median_idx, 0], label=\"Median Forecast\", color=colors[name], linewidth=2)\n \n # Plot prediction intervals\n # 80% interval (0.1 to 0.9)\n ax.fill_between(\n test_indices,\n quantiles[0, :, 0, 0], # 0.1 quantile\n quantiles[0, :, 4, 0], # 0.9 quantile\n alpha=0.2,\n color=colors[name],\n label=\"80% Interval\"\n )\n \n # 50% interval (0.25 to 0.75)\n ax.fill_between(\n test_indices,\n quantiles[0, :, 1, 0], # 0.25 quantile\n quantiles[0, :, 3, 0], # 0.75 quantile\n alpha=0.3,\n color=colors[name],\n label=\"50% Interval\"\n )\n \n ax.axvline(x=train_size, color=\"gray\", linestyle=\":\", linewidth=1, alpha=0.7)\n ax.set_xlabel(\"Time\")\n ax.set_ylabel(\"Value\")\n ax.set_title(f\"{name} Quantile Forecast\")\n ax.legend(loc=\"best\")\n ax.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()",
269+
"source": [
270+
"# Plot quantile forecasts for all models\n",
271+
"fig, axes = plt.subplots(3, 1, figsize=(12, 12))\n",
272+
"\n",
273+
"# Plot only last 'horizon' points of training data\n",
274+
"train_context_start = max(0, train_size - horizon)\n",
275+
"train_context_indices = np.arange(train_context_start, train_size)\n",
276+
"\n",
277+
"for idx, (name, quantiles) in enumerate(quantile_models.items()):\n",
278+
" ax = axes[idx]\n",
279+
" \n",
280+
" # Plot training data (last horizon points)\n",
281+
" ax.plot(train_context_indices, train_data[0, train_context_start:, 0], label=\"Training Data\", color=\"blue\", linewidth=2)\n",
282+
" \n",
283+
" # Plot test data\n",
284+
" ax.plot(test_indices, test_data[0, :, 0], label=\"Test Data\", color=\"black\", linewidth=2, linestyle=\"--\")\n",
285+
" \n",
286+
" # Plot median (index 2 for 0.5 quantile)\n",
287+
" median_idx = 2\n",
288+
" ax.plot(test_indices, quantiles[0, :, median_idx, 0], label=\"Median Forecast\", color=colors[name], linewidth=2)\n",
289+
" \n",
290+
" # Plot prediction intervals\n",
291+
" # 80% interval (0.1 to 0.9)\n",
292+
" ax.fill_between(\n",
293+
" test_indices,\n",
294+
" quantiles[0, :, 0, 0], # 0.1 quantile\n",
295+
" quantiles[0, :, 4, 0], # 0.9 quantile\n",
296+
" alpha=0.2,\n",
297+
" color=colors[name],\n",
298+
" label=\"80% Interval\"\n",
299+
" )\n",
300+
" \n",
301+
" # 50% interval (0.25 to 0.75)\n",
302+
" ax.fill_between(\n",
303+
" test_indices,\n",
304+
" quantiles[0, :, 1, 0], # 0.25 quantile\n",
305+
" quantiles[0, :, 3, 0], # 0.75 quantile\n",
306+
" alpha=0.3,\n",
307+
" color=colors[name],\n",
308+
" label=\"50% Interval\"\n",
309+
" )\n",
310+
" \n",
311+
" ax.axvline(x=train_size, color=\"gray\", linestyle=\":\", linewidth=1, alpha=0.7)\n",
312+
" ax.set_xlabel(\"Time\")\n",
313+
" ax.set_ylabel(\"Value\")\n",
314+
" ax.set_title(f\"{name} Quantile Forecast\")\n",
315+
" ax.legend(loc=\"best\")\n",
316+
" ax.grid(True, alpha=0.3)\n",
317+
"\n",
318+
"plt.tight_layout()\n",
319+
"plt.show()"
320+
],
240321
"outputs": [
241322
{
242323
"data": {
@@ -249,7 +330,7 @@
249330
"output_type": "display_data"
250331
}
251332
],
252-
"execution_count": 31
333+
"execution_count": 8
253334
},
254335
{
255336
"cell_type": "markdown",

0 commit comments

Comments
 (0)