WIP on data fetching and feature engineering

Author: ogrisel

.gitignore

@@ -87,3 +87,6 @@ content/notebooks
 
 # jupyterlite build
 jupyterlite/.jupyterlite.doit.db
+
+# requests cache for weather data download
+.cache.sqlite

content/datasets/Total Load - Day Ahead _ Actual_202001010000-202101010000.csv

@@ -16,8 +16,8 @@
     "time",
     pl.DataFrame().with_columns(
         pl.datetime_range(
-            pl.datetime(2023, 1, 1, hour=0),
-            pl.datetime(2024, 12, 31, hour=23),
+            pl.datetime(2021, 1, 1, hour=0),
+            pl.datetime(2025, 6, 30, hour=23),
             time_zone="UTC",
             interval="1h",
         ).alias("time"),
@@ -75,17 +75,17 @@
 
 # %%
 some_city_weather_raw = skrub.var(
-    "some_city_weather_raw",
-    pl.read_csv("../datasets/open-meteo-48.12N1.65W44m.csv", skip_rows=3),
+    "paris_weather_raw",
+    pl.read_parquet("../datasets/weather_paris.parquet"),
+).with_columns(
+    [
+        pl.col("time").dt.cast_time_unit("us"),  # Ensure time column has the same type
+    ]
 )
 some_city_weather_raw
 
 # %%
-some_city_weather = some_city_weather_raw.with_columns(
-    [
-        pl.col("time").str.to_datetime("%Y-%m-%dT%H:%M", time_zone="UTC"),
-    ]
-).rename(lambda x: x if x == "time" else x + " some_city")
+some_city_weather = some_city_weather_raw.rename(lambda x: x if x == "time" else x + " some_city")
 time.join(some_city_weather, on="time", how="left")
 
 # %%

content/datasets/weather_lyon.parquet

@@ -0,0 +1,104 @@
+# %%
+%pip install -q openmeteo-requests retry-requests requests-cache ipyleaflet
+
+# %%
+from pathlib import Path
+from ipyleaflet import Map, Marker
+import openmeteo_requests
+
+import pandas as pd
+import requests_cache
+from retry_requests import retry
+
+# %%
+# List of 10 median to large urban areas with their GPS coordinates to cover
+# the most populous areas in France.
+cities = [
+    {"name": "Paris", "latitude": 48.8566, "longitude": 2.3522},
+    {"name": "Lyon", "latitude": 45.7640, "longitude": 4.8357},
+    # {"name": "Marseille", "latitude": 43.2965, "longitude": 5.3698},
+    # {"name": "Toulouse", "latitude": 43.6047, "longitude": 1.4442},
+    # {"name": "Lille", "latitude": 50.6292, "longitude": 3.0573},
+    # {"name": "Limoges", "latitude": 45.8336, "longitude": 1.2616},
+    # {"name": "Nantes", "latitude": 47.2184, "longitude": -1.5536},
+    # {"name": "Strasbourg", "latitude": 48.5734, "longitude": 7.7521},
+    # {"name": "Brest", "latitude": 48.3904, "longitude": -4.4861},
+    # {"name": "Bayonne", "latitude": 43.4833, "longitude": -1.4667},
+]
+
+map_center = [46.6034, 1.8883]  # Approximate center of France
+m = Map(center=map_center, zoom=6)
+for city in cities:
+    marker = Marker(location=(city["latitude"], city["longitude"]), title=city["name"])
+    m.add_layer(marker)
+m
+
+
+# %%
+
+def download_weather_data(city):
+    cache_session = requests_cache.CachedSession(".cache", expire_after=3600)
+    retry_session = retry(cache_session, retries=5, backoff_factor=0.2)
+    openmeteo = openmeteo_requests.Client(session=retry_session)
+
+    # Make sure all required weather variables are listed here. The order of
+    # variables in hourly or daily is important to assign them correctly below.
+    url = "https://historical-forecast-api.open-meteo.com/v1/forecast"
+    params = {
+        "latitude": city["latitude"],
+        "longitude": city["longitude"],
+        "start_date": "2021-01-01",
+        "end_date": "2025-05-31",
+        "hourly": [
+            "temperature_2m",
+            "precipitation",
+            "wind_speed_10m",
+            "cloud_cover",
+            "soil_moisture_1_to_3cm",
+            "relative_humidity_2m",
+        ],
+        "timezone": "GMT",  # Use GMT to ease temporal joins.
+    }
+    response = openmeteo.weather_api(url, params=params)[0]
+
+    # Process hourly data. The order of variables needs to be the same as requested.
+    hourly = response.Hourly()
+    hourly_temperature_2m = hourly.Variables(0).ValuesAsNumpy()
+    hourly_precipitation = hourly.Variables(1).ValuesAsNumpy()
+    hourly_wind_speed_10m = hourly.Variables(2).ValuesAsNumpy()
+    hourly_cloud_cover = hourly.Variables(3).ValuesAsNumpy()
+    hourly_soil_moisture_1_to_3cm = hourly.Variables(4).ValuesAsNumpy()
+    hourly_relative_humidity_2m = hourly.Variables(5).ValuesAsNumpy()
+
+    hourly_data = {
+        "time": pd.date_range(
+            start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
+            end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
+            freq=pd.Timedelta(seconds=hourly.Interval()),
+            inclusive="left",
+        )
+    }
+
+    hourly_data["temperature_2m"] = hourly_temperature_2m
+    hourly_data["precipitation"] = hourly_precipitation
+    hourly_data["wind_speed_10m"] = hourly_wind_speed_10m
+    hourly_data["cloud_cover"] = hourly_cloud_cover
+    hourly_data["soil_moisture_1_to_3cm"] = hourly_soil_moisture_1_to_3cm
+    hourly_data["relative_humidity_2m"] = hourly_relative_humidity_2m
+    return pd.DataFrame(data=hourly_data)
+
+
+# %%
+datasets_folder = Path("../datasets")
+for city in cities:
+    filepath = datasets_folder / f"weather_{city['name'].lower()}.parquet"
+    if filepath.exists():
+        print(f"Weather data for {city['name']} already exists at {filepath}.")
+        continue
+
+    print(f"Downloading weather data for {city['name']}...")
+    df = download_weather_data(city)
+    df.to_parquet(filepath, index=False)
+    print(f"Weather data for {city['name']} saved to {filepath}.")
+
+# %%