Merge branch 'main' into Sudarshan

Author: Risto0211

.gitignore

@@ -25,7 +25,8 @@ htmlcov
 .mypy_cache
 .pyre/
 .pytype/
-data/
 *.png
 .vscode/
 
+
+/data

download_data.py

@@ -7,36 +7,36 @@
     if not os.path.exists(save_dir):
         os.makedirs(save_dir)
     dataset_names = [
-    # "weather",
-    # "tourism_yearly",
-    # "tourism_quarterly",
-    # "tourism_monthly",
-    # "cif_2016",
-    # "london_smart_meters",
-    # "australian_electricity_demand",
-    # "wind_farms_minutely",
-    # "bitcoin",
-    # "pedestrian_counts",
-    # "vehicle_trips",
-    # "kdd_cup_2018",
-    # "nn5_daily",
-    # "nn5_weekly",
-    # "kaggle_web_traffic",
-    # "kaggle_web_traffic_weekly",
-    # "solar_10_minutes",
-    # "solar_weekly",
-    # "car_parts",
-    # "fred_md",
-    # "traffic_hourly",
-    # "traffic_weekly",
-    # "hospital",
-    # "covid_deaths",
-    # "sunspot",
-    # "saugeenday",
-    # "us_births",
-    # "solar_4_seconds",
-    # "wind_4_seconds",
-    # "rideshare",
+    "weather",
+    "tourism_yearly",
+    "tourism_quarterly",
+    "tourism_monthly",
+    "cif_2016",
+    "london_smart_meters",
+    "australian_electricity_demand",
+    "wind_farms_minutely",
+    "bitcoin",
+    "pedestrian_counts",
+    "vehicle_trips",
+    "kdd_cup_2018",
+    "nn5_daily",
+    "nn5_weekly",
+    "kaggle_web_traffic",
+    "kaggle_web_traffic_weekly",
+    "solar_10_minutes",
+    "solar_weekly",
+    "car_parts",
+    "fred_md",
+    "traffic_hourly",
+    "traffic_weekly",
+    "hospital",
+    "covid_deaths",
+    "sunspot",
+    "saugeenday",
+    "us_births",
+    "solar_4_seconds",
+    "wind_4_seconds",
+    "rideshare",
     "oikolab_weather",
     "temperature_rain"
 ]

example/lptm.ipynb

@@ -36,6 +36,76 @@
 print(f"Time taken to load datasets: {end-start:.2f} seconds")
 
 MODEL_NAMES = ["moirai", "chronos", "chronosbolt", "timesfm", "moment", "ttm"]
+MONASH_NAMES = {
+    # "weather": "1D",
+    "tourism_yearly": ["1YE"],
+    "tourism_quarterly": ["1Q"],
+    "tourism_monthly": ["1M"],
+    "cif_2016": ["1M"],
+    # "london_smart_meters": ["30min"],
+    "australian_electricity_demand": ["30min"],
+    # "wind_farms_minutely": ["1min"],
+    "bitcoin": ["1D"],
+    "pedestrian_counts": ["1h"],
+    "vehicle_trips": ["1D"],
+    "kdd_cup_2018": ["1H"],
+    "nn5_daily": ["1D"],
+    "nn5_weekly": ["1W"],
+    # "kaggle_web_traffic": ["1D"],
+    # "kaggle_web_traffic_weekly": ["1W"],
+    "solar_10_minutes": ["10min"],
+    "solar_weekly": ["1W"],
+    "car_parts": ["1M"],
+    "fred_md": ["1M"],
+    "traffic_hourly": ["1h"],
+    "traffic_weekly": ["1W"],
+    "hospital": ["1M"],
+    "covid_deaths": ["1D"],
+    "sunspot": ["1D"],
+    "saugeenday": ["1D"],
+    "us_births": ["1D"],
+    "solar_4_seconds": ["4s"],
+    "wind_4_seconds": ["4s"],
+    "rideshare": ["1h"],
+    "oikolab_weather": ["1h"],
+    "temperature_rain": ["1D"]
+}
+
+MONASH_SETTINGS = {
+    # "weather": 30,
+    "tourism_yearly": 4,
+    "tourism_quarterly": 8,
+    "tourism_monthly": 24,
+    "cif_2016": 12,
+    # "london_smart_meters": 60,
+    "australian_electricity_demand": 60,
+    # "wind_farms_minutely": 60,
+    "bitcoin": 30,
+    "pedestrian_counts": 48,
+    "vehicle_trips": 30,
+    "kdd_cup_2018": 48,
+    "nn5_daily": 56,
+    "nn5_weekly": 8,
+    # "kaggle_web_traffic": 59,
+    # "kaggle_web_traffic_weekly": 8,
+    "solar_10_minutes": 60,
+    "solar_weekly": 5,
+    "car_parts": 12,
+    "fred_md": 12,
+    "traffic_hourly": 48,
+    "traffic_weekly": 8,
+    "hospital": 12,
+    "covid_deaths": 30,
+    "sunspot": 30,
+    "saugeenday": 30,
+    "us_births": 30,
+    "solar_4_seconds": 60,
+    "wind_4_seconds": 60,
+    "rideshare": 48,
+    "oikolab_weather": 48,
+    "temperature_rain": 30
+}
+
 MODEL_CONTEXT_LEN = {
     "timesfm": 32,
     "moment": 512,
@@ -105,8 +175,11 @@ def calc_pred_and_context_len(freq):
         for fname, freq, fs in filesizes:
             print(f"Evaluating {fname} ({freq})")
             # Adjust the context and prediction length based on the frequency
+
             # pred_len, context_len = calc_pred_and_context_len(freq)
             pred_len, context_len = 96, 512
+            if msh:
+                pred_len, context_len = MONASH_SETTINGS[dataset_name], 512
             if model_name == "timesfm":
                 args["config"]["horizon_len"] = pred_len
                 args["config"]["context_len"] = context_len
@@ -126,6 +199,14 @@ def calc_pred_and_context_len(freq):
                 dataset_path = f"data/gifteval/{fname}/{freq}/data.csv"
 
             # Initialize the model and dataset
+            if msh:
+                dataset_path = f"data/monash/{dataset_name}/test/data.csv"
+            else:
+                if len(freqs) == 1:
+                    dataset_path = f"data/gifteval/{dataset_name}/data.csv"
+                else:
+                    dataset_path = f"data/gifteval/{dataset_name}/{freq}/data.csv"
+            print(f"Creating leaderboard for dataset: {dataset_name}, context_len: {context_len}, horizon_len: {pred_len}")
             if model_name == "timesfm":
                 model = TimesfmModel(**args)
                 dataset = TimesfmDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=args["config"]["context_len"], horizon_len=args["config"]["horizon_len"], boundaries=(-1, -1, -1), batchsize=64)

example/lptm_zero.ipynb

@@ -0,0 +1,28 @@
+dataset,mse,mae,mase,mape,rmse,nrmse,smape,msis,nd,mwsq,crps
+tourism_yearly,444141692.32991457,1091.6817688190906,0.041673069482334,258942.9412618748,21074.669447702247,0.0056080123981317,0.069903645118077,2315085360.881909,0.0937644062905105,239.86975188013847,105468655.5279068
+tourism_quarterly,6928432844.955573,16058.179070454917,0.219953558391302,17033866.447995476,83237.20829626359,0.0118343129663926,0.7357593638009957,0.129624555852842,0.3376971205486341,216650372.01311347,57348306032441.02
+tourism_monthly,190129684.77504933,3006.765480087757,0.5812615013938789,0.5539094888596078,13788.752110870995,0.0548260521285198,1.1077688946932025,0.0580983604960161,0.9999999966719292,0.0003618518892388,32.20209788417588
+cif_2016,1.203214870564485e+16,10224691.385480355,0.4199903410326607,1021039116.791298,109691151.44643553,0.0739733070305259,0.8060840675419945,2.0952376901855407,0.8399768110936794,11067973489.49984,2.174084312096058e+18
+australian_electricity_demand,3341478.908718931,712.8013315638235,,0.2392915110667949,1827.9712548940508,0.2657256314340223,0.4784888274983774,0.0547413870139505,0.9999999860293306,0.0001166485832802,0.6991631823939618
+bitcoin,1027181666879.6758,163108.16262782528,,0.1986968013352183,1013499.7123234302,0.1522775090971237,0.3972998371276115,0.0010149408681403,0.9999999999389824,0.0209032186294409,186509.0281783463
+pedestrian_counts,228594.34047321105,128.47884119836914,0.503464522496511,0.1560662013498633,478.115404973748,0.0734432264576522,0.312031576953403,0.0253257396070908,0.9999999225750844,1.8699487607833263e-05,0.0459507402418486
+vehicle_trips,12944.928988510794,32.11006947344806,0.5454305395333896,0.3289663930775494,113.77578384045874,0.1142327135523409,0.6578524140559953,0.0401714920146534,0.999999689474565,3.976037404105965e-06,0.0023768851609593
+kdd_cup_2018,5859.462274087244,20.07007816671078,0.6065589496924628,0.220538504946128,76.54712453180227,0.1040749469626822,0.4415058613042274,0.0260086316122621,1.0000364849647414,0.0003742988248793,0.0023274558891799
+nn5_daily,84.31101334863708,3.8252922590923766,1.546564363951125,0.2199299254310309,9.182102882708136,0.1372937272737188,0.4397646271359306,0.0864441601113176,0.9999973973145824,5.936201735498581e-07,1.6354225978457626e-05
+nn5_weekly,1434.0553996533272,25.91377713704548,0.3999215499897716,0.2169826998834618,37.86892393049118,0.0969497218088367,0.2384182640276755,0.1038714168011784,0.2471726169922481,1659.5134230779124,147781.99617711874
+solar_10_minutes,0.1193782517528535,0.0210894843515833,0.5453868444274433,0.0253123334121342,0.3455115797666607,0.0193563801794318,0.0505052039691868,2108.9487197905933,0.9995295901274224,8.345428103678382e-08,4.64501025476016e-08
+solar_weekly,3115173.9774587457,1268.2185252844597,0.3964157272080421,0.2898165957313559,1764.9855459631235,0.0987733284632026,0.2322388023965503,0.14352721870811,0.3609709695986244,-5750399.354954169,14351019438.737406
+car_parts,1.4482230689164717,0.4312706358651241,0.6238069437176991,3591.236062680824,1.2034214012208988,0.0300855275091405,0.9228552893912162,0.101706622670986,1.0377270347805376,0.1260305670088572,0.4300970526348686
+fred_md,596678701.7317729,4747.458188851477,0.418512827719631,0.6901471294216209,24427.00762950249,0.0792932358470027,1.2341052375211934,0.0547806777312514,0.835420873265261,9464211.357160904,271734124990.43045
+traffic_hourly,0.0012408267518655,0.0164771175073004,1.920375330283732,0.2803240677344528,0.0352253708549039,0.0923556562620379,0.5605629458709293,0.1283124850031318,0.9993953964097638,5.6946830577454864e-08,2.426979569144636e-10
+traffic_weekly,4.330103429838062,1.5275869707844083,0.3660947153271351,0.1544419116201267,2.0808900571241296,0.0588545095973592,0.1725321287321687,0.0851924259707548,0.1630502056889859,11.097874348338795,35.70920819671376
+hospital,538650.1377824282,219.3589574775862,0.5111717857336138,0.8134917999289581,733.9278832299725,0.0632641912419042,1.4303838519944216,0.0738407343066312,0.8331902825793209,19364.327364448287,31426275.1677896
+covid_deaths,10885.932603545672,4.382604233492268,0.4113709230009012,0.0620728957504436,104.33567272771892,0.035260450278849,0.1240379439938584,1.4129466990756654,0.9999977335047028,6.094549571428515e-07,0.0019350583597334
+sunspot,1908.999997991608,15.400000054947062,,0.1731458620114036,43.69210452692349,0.2275630325588185,0.3461901848530289,0.0662351601844637,0.9999993542177612,2.699789169486293e-06,0.0004650855344745
+saugeenday,269.8163317717818,11.023333342256688,,0.4708444976961905,16.426086928169525,0.4874208324617566,0.9416232138456884,0.327101782529343,0.9999990936437032,1.432314843488929e-06,4.855334875272051e-05
+us_births,43719532.26602842,4511.600000008923,,0.4708447023482694,6612.074732338438,0.6386009972911365,0.9416236259946013,0.3768951160167986,0.9999999977854692,0.0005369385937169,7.587445059005216
+solar_4_seconds,6.135621587873876e-15,7.747566570515118e-08,,0.0077475665705151,7.833020865460449e-08,0.0078330208654604,0.0153734036898451,0.0077475665705151,0.0077475665705151,7.747565615178496e-08,6.135620772494481e-15
+wind_4_seconds,69.02933271778653,4.013333375021395,,0.2392913829873439,8.308389297438254,0.4830456085454749,0.4784885690779666,0.2333332000982206,0.9999975186992214,7.204877227383475e-07,1.4554829158972376e-05
+rideshare,41.59257181027639,1.9854899661244407,0.9003329006690424,0.2536560772838738,6.449230326967427,0.07010032202135,0.5072268674056887,0.0379857954952194,0.999994981093036,3.290119220325788e-07,7.992024858382082e-06
+oikolab_weather,376969134.2603373,3722.07442711643,,0.2737455091364411,19415.692989443804,0.1906107690066239,0.5474063116953172,-0.0154761834873608,0.9999999973222178,0.0005518700247492,74.02871341393308
+temperature_rain,97.59024449532855,2.484052272779186,0.6232494721809301,0.2042947760327026,9.878777479796199,0.056311145724182256,0.40849688722974814,0.025763328950111283,0.9999959928198491,3.7952170635389444e-07,1.8271202610847874e-05

example/timesfm.ipynb

@@ -0,0 +1,28 @@
+dataset,mse,mae,mase,mape,rmse,nrmse,smape,msis,nd,mwsq,crps
+tourism_yearly,0.1193213462829589,0.0552518963813781,0.0502919629216194,-0.0192584041506052,0.3454292118549347,0.0170845361689556,0.0596492551267147,0.0432840399444103,-0.0332806138839186,,
+tourism_quarterly,15.174468994140623,2.131948232650757,0.4453669488430023,0.6742493510246277,3.895442008972168,0.0364306505544808,0.6830410361289978,0.100502148270607,0.5155130011882703,,
+tourism_monthly,,,,,,,,,,,
+cif_2016,3.051345109939575,0.770009458065033,0.1008551716804504,2.7060635089874268,1.746809959411621,0.0052530776336364,0.5942800641059875,0.4276498556137085,-0.3124985771275554,,
+australian_electricity_demand,,,,,,,,,,,
+bitcoin,,,,,,,,,,,
+pedestrian_counts,,,,,,,,,,,
+vehicle_trips,,,,,,,,,,,
+kdd_cup_2018,,,,,,,,,,,
+nn5_daily,,,,,,,,,,,
+nn5_weekly,7.142962455749512,1.8851556777954104,0.9576101303100586,-0.4964344501495361,2.6726322174072266,0.0984423137013404,1.3164684772491455,0.1382441818714141,-15.28806140511234,,
+solar_10_minutes,,,,,,,,,,,
+solar_weekly,1.0168001651763916,0.7820338010787964,1.376811146736145,-0.7340742945671082,1.0083651542663574,0.4434716977863531,1.3281521797180176,0.4218072891235351,-1.0606733235349932,,
+car_parts,3.754984617233277,0.7155598402023315,0.6451520919799805,4147.560546875,1.9377782344818115,0.0139375998713791,0.7457103133201599,0.1027182564139366,16.873369196029433,,
+fred_md,2.542620658874512,1.1022943258285522,0.3314592540264129,0.6176068782806396,1.594559669494629,0.0840454737283687,0.7133917212486267,0.1085511520504951,0.6246864106058683,,
+traffic_hourly,,,,,,,,,,,
+traffic_weekly,5.216869831085205,0.8759592771530151,0.4404235184192657,-1.4892243146896362,2.2840468883514404,0.0377834815133501,0.7395275831222534,0.1350718736648559,0.947835748847314,,
+hospital,3.000790596008301,1.285870909690857,0.8030640482902527,120.14913177490234,1.7322790622711182,0.0711396958176925,1.35271418094635,0.2003196626901626,4.766237518108584,,
+covid_deaths,,,,,,,,,,,
+sunspot,,,,,,,,,,,
+saugeenday,,,,,,,,,,,
+us_births,,,,,,,,,,,
+solar_4_seconds,,,,,,,,,,,
+wind_4_seconds,,,,,,,,,,,
+rideshare,,,,,,,,,,,
+oikolab_weather,,,,,,,,,,,
+temperature_rain,,,,,,,,,,,

example/tinytimemixer.ipynb

@@ -0,0 +1,28 @@
+dataset,mse,mae,mase,mape,rmse,nrmse,smape,msis,nd,mwsq,crps
+tourism_yearly,2361702.5,36.06318664550781,3606318.0,3606318.0,1536.783203125,153678320.3125,1.4851635694503784,3606318.0,3606318.664550781,-7780.55712890625,316451173.62888974
+tourism_quarterly,133073.890625,11.017659187316896,1101766.25,1101766.25,364.7929382324219,36479293.82324219,1.7359308004379272,1101766.25,1101765.9187316897,-3167.559814453125,9380487.746344984
+tourism_monthly,8602214.0,83.11030578613281,0.4208029210567474,246588.640625,2932.953125,0.0077529820907281,1.8755922317504885,30134452.0,0.4558699002701215,35345.91796875,3721804582.7465873
+cif_2016,4054.138916015625,12.505019187927246,0.1814535409212112,20152.20703125,63.672119140625,0.0231935676381596,1.714262843132019,0.0330332480370998,0.204135231840822,404.7507019042969,274699.42295985995
+australian_electricity_demand,150264.421875,232.8675231933593,0.3018090426921844,57704.55859375,387.6395568847656,0.0295914764548414,0.0868552923202514,0.0322753563523292,0.0669677612136246,-9590.701171875,524073785.36750144
+bitcoin,,,,,,,,,,,
+pedestrian_counts,27354.072265625,48.37232971191406,0.4173867702484131,23420.462890625,165.3906707763672,0.0103505019508643,1.1788418292999268,0.0315233282744884,0.1650090210907146,754.9451904296875,7847077.815458994
+vehicle_trips,181.2378387451172,0.5425539612770081,54255.40234375,54255.40234375,13.4624605178833,1346246.05178833,1.9117441177368164,54255.40234375,54255.3961277008,-2.324742794036865,1099.301869087636
+kdd_cup_2018,2242.1318359375,19.861251831054688,0.802638590335846,367465.21875,47.35115432739258,0.0143749709118527,0.8389987945556641,0.1068498790264129,0.6370928594170072,103.09420013427734,69039.25441101419
+nn5_daily,35.77730941772461,3.885481595993042,1.6794143915176392,16414.0,5.981413841247559,0.0623306355287951,0.2313048988580703,0.0763084515929222,0.2016468535974389,4.786635875701904,657.0211086576586
+nn5_weekly,89.06739044189453,1.0670193433761597,0.1561520546674728,143.2692108154297,9.437552452087402,0.0293910798458386,1.6629701852798462,0.0270845741033554,0.1691647435775213,-1.9155505895614624,1144.141507496138
+solar_10_minutes,26.70118904113769,2.4146080017089844,0.1946574002504348,17161.712890625,5.167319297790527,0.058125068929674,1.3448920249938965,0.0854992121458053,0.3819715876199319,3.173281192779541,169.20667857149823
+solar_weekly,78236.7578125,42.5200424194336,0.1622558981180191,142.00363159179688,279.7083435058594,0.0181469127403297,1.5168342590332031,0.0203413348644971,0.1802843047310721,15530.861328125,32472244.27291968
+car_parts,0.0023693332914263,0.0032368141692131,2.325936794281006,224.52639770507807,0.0486757978796958,0.0048675749203946,1.820556044578552,332.6125183105469,3.0938389132825828,0.0002686674706637,0.0001817149828545
+fred_md,337362592.0,1292.2945556640625,1.1678502559661863,26.98746681213379,18367.43359375,0.005784504681377,0.098615176975727,0.0782435983419418,0.0311425562484908,-106201.6875,363984351646.9304
+traffic_hourly,0.000455149973277,0.0080386660993099,0.387826532125473,21.04779624938965,0.0213342439383268,0.0294667816919213,0.1653950363397598,0.0385803952813148,0.1418321666411633,0.0036569747608155,0.0001586150792923
+traffic_weekly,0.0091974884271621,0.0065262168645858,0.0998282879590988,148.9840545654297,0.0959035381674766,0.0049818700666002,1.7232398986816406,12732.7841796875,0.1089020227146023,0.0025558837223798,0.0113577489246558
+hospital,306.1357116699219,0.6331541538238525,0.1295970529317855,191.8525848388672,17.496734619140625,0.0023567799832398,1.7972227334976196,862228.25,0.1555164023384381,3.4607455730438232,4272.47144303022
+covid_deaths,119150.6484375,13.035115242004396,0.1232983171939849,151.50901794433594,345.1820373535156,0.0064081616843543,1.8622676134109497,17.030107498168945,0.1438479938902019,-31.99930191040039,156593.61040537566
+sunspot,2275.333251953125,32.238399505615234,0.787270724773407,306006.84375,47.70045471191406,0.0903417685766976,0.8078023791313171,0.1073573678731918,0.4086495293865697,433.9431762695313,164093.2649063609
+saugeenday,1305.9161376953125,15.696843147277832,0.6664905548095703,0.4421868026256561,36.1374626159668,0.0566684355614381,0.4329459965229034,0.0825363099575042,0.5160752917687316,206.78466796875,33607.82410239941
+us_births,151821.234375,265.7854919433594,0.2479991912841797,0.0276550203561782,389.6424560546875,0.0630897758134374,0.0274470150470733,0.0591710805892944,0.0274304356335499,140754.0,1462221219.9125562
+solar_4_seconds,21.87527084350586,1.3645970821380615,0.5447545051574707,2320.71484375,4.677100658416748,0.0401467833214499,1.0651005506515503,0.0118864914402365,0.0418262921876244,-1.593536138534546,725.113524009159
+wind_4_seconds,23.43195533752441,1.7629423141479492,0.6767939925193787,1002.0794067382812,4.840656280517578,0.0370080718463087,0.1096571534872055,0.016058275476098,0.0435277317578082,2.227166652679444,927.9487571490704
+rideshare,46.68171691894531,0.3123773336410522,0.7474644184112549,7209.89306640625,6.832401752471924,0.0767685503908586,1.8551300764083865,0.2038945853710174,0.7291043059541679,0.1454752832651138,50.14438478675226
+oikolab_weather,81064.8515625,61.23777770996094,0.551532506942749,45665.16015625,284.7189025878906,0.002753571675954,0.3464515209197998,0.0006154436268843,0.0047957091499386,-133933.125,805123431.8426093
+temperature_rain,172.225341796875,5.509518623352051,2.190863847732544,51034.7421875,13.123465538024902,0.022516410521192846,1.5997639894485474,0.06347574293613434,0.8267697936668237,16.08089828491211,999.5042235450454