Add pandas 3.0 support (#294)

* Fix pandas 3.0 incompatibilities in GEO, BEYONDCOAL and MASTR loaders

- clean_name: astype(str) preserves NA under the pandas 3 string dtype,
  so unidecode received float NaN for unnamed plants (GEO, reduced
  retrieval). fillna("") before the conversion.
- BEYONDCOAL: the future.no_silent_downcasting option was removed in
  pandas 3 (its behaviour is now the default); requesting it raises.
  Guard the option_context behind a version check.
- MASTR: the chunked low-memory parser crashes with usecols on
  pandas 3 (IndexError in the DtypeWarning column lookup); read with
  low_memory=False. Postleitzahl astype(str) preserves NA, breaking the
  final astype(int); fillna("0") in between, identical outcome on
  pandas 2 where NaN already became the string "nan" -> "000".

All previously failing non-Duke tests pass under pandas 3.0.3
(test_data, test_plots, test_aggregate[GEO/BEYONDCOAL/MASTR],
test_reduced_retrieval); behaviour unchanged under pandas 2.x.

* Complete pandas 3.0 support: lift pin to <4, fix Duke/aggregate/matching paths

Relax the dependency pin from pandas<3 to pandas<4 and fix the four
remaining breakages that surface once pandas 3 is installed (these sit
behind the Java-gated Duke aggregate/matching tests, so they are not
visible without a JDK on PATH):

- duke.add_geoposition_for_duke: astype(str) preserves NA on pandas 3,
  so ','.join hit a float NaN. Stringify per item (map(str, s)) so NaN
  becomes 'nan', keeping the 'nan,nan' -> NaN cleanup valid.
- cleaning.aggregate_units: second future.no_silent_downcasting
  option_context (the twin of the BEYONDCOAL one) version-gated; drop
  the deprecated copy= kwarg from infer_objects (no-op under CoW).
- matching.best_matches: groupby(...).apply with as_index=False
  re-inserts the grouping column on pandas 3, absent from the per-group
  Series -> KeyError. Rewrote with a vectorised idxmax (behaviour-
  identical, faster, no apply).

Full test suite (incl. Duke aggregate + matching) passes under both
pandas 2.x and 3.0.3.

* Remove inline review comments from pandas 3 fixes

Per maintainer review on #294: drop the explanatory comments added for
review. Code changes are unchanged.

* Extract pandas-3 check into PANDAS_V3 constant

Addresses the review on #294: replace the inline
parse_version(pd.__version__).major < 3 check with a named PANDAS_V3
constant defined once in core.py, used by both cleaning.aggregate_units
and data.BEYONDCOAL. Behaviour unchanged; full test suite passes (48)
under pandas 3.0.3.

Author: MaykThewessen

powerplantmatching/cleaning.py

@@ -6,6 +6,7 @@
 Functions for vertically cleaning a dataset.
 """
 
+import contextlib
 import logging
 
 import networkx as nx
@@ -14,7 +15,7 @@
 import unidecode
 from deprecation import deprecated
 
-from .core import get_config, get_obj_if_Acc
+from .core import PANDAS_V3, get_config, get_obj_if_Acc
 from .duke import duke
 from .utils import get_name, set_column_name
 
@@ -74,7 +75,7 @@ def clean_name(df, config=None):
     if config is None:
         config = get_config()
 
-    name = df.Name.astype(str).copy().apply(unidecode.unidecode)
+    name = df.Name.fillna("").astype(str).copy().apply(unidecode.unidecode)
 
     roman_to_arabic = {
         "I": "1",
@@ -513,9 +514,13 @@ def aggregate_units(
     df = cliques(df, duplicates)
     df = df.groupby("grouped").agg(props_for_groups)
 
-    # Downcasting in replace is deprecated
-    with pd.option_context("future.no_silent_downcasting", True):
-        df[str_cols] = df[str_cols].replace("", pd.NA).infer_objects(copy=False)
+    no_downcast_ctx = (
+        contextlib.nullcontext()
+        if PANDAS_V3
+        else pd.option_context("future.no_silent_downcasting", True)
+    )
+    with no_downcast_ctx:
+        df[str_cols] = df[str_cols].replace("", pd.NA).infer_objects()
 
     df = (
         df.assign(

powerplantmatching/core.py

@@ -6,6 +6,13 @@
 from os import environ, makedirs
 from os.path import abspath, dirname, exists, expanduser, isdir, join
 
+import pandas as pd
+from packaging.version import parse as parse_version
+
+# pandas >= 3.0 changed several defaults (NA-preserving string dtype,
+# no-silent-downcasting) and removed knobs the loaders previously gated on.
+PANDAS_V3 = parse_version(pd.__version__).major >= 3
+
 # for the writable data directory (i.e. the one where new data goes), follow
 # the XDG guidelines found at
 # https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html

powerplantmatching/data.py

@@ -6,6 +6,7 @@
 Collection of power plant data bases and statistical data
 """
 
+import contextlib
 import json
 import logging
 import os
@@ -24,7 +25,7 @@
     gather_set_info,
     gather_specifications,
 )
-from .core import _package_data, get_config
+from .core import PANDAS_V3, _package_data, get_config
 from .heuristics import PLZ_to_LatLon_map, scale_to_net_capacities
 from .utils import (
     config_filter,
@@ -93,7 +94,12 @@ def BEYONDCOAL(raw=False, update=False, config=None):
         "heat": "CHP",
     }
 
-    with pd.option_context("future.no_silent_downcasting", True):
+    no_downcast_ctx = (
+        contextlib.nullcontext()
+        if PANDAS_V3
+        else pd.option_context("future.no_silent_downcasting", True)
+    )
+    with no_downcast_ctx:
         phaseout_col = "Covered by country phase-out? [if yes: country phase-out year]"
         date_out = (
             df["(Announced) Retirement year"]
@@ -2398,7 +2404,7 @@ def MASTR(
                     )
                     usecols = available_columns.intersection(target_columns)
                     df = (
-                        pd.read_csv(file.open(name), usecols=usecols)
+                        pd.read_csv(file.open(name), usecols=usecols, low_memory=False)
                         .assign(Filesuffix=fueltype)
                         .query("Nettonennleistung >= @THRESHOLD_KW")
                     )
@@ -2434,7 +2440,10 @@ def MASTR(
 
     PLZ_map = PLZ_to_LatLon_map()
     df.Postleitzahl = (
-        df.Postleitzahl.astype(str).str.replace(r"[^0-9]", "0", regex=True).astype(int)
+        df.Postleitzahl.astype(str)
+        .fillna("0")
+        .str.replace(r"[^0-9]", "0", regex=True)
+        .astype(int)
     )
     df["PLZ_lat"] = df.Postleitzahl.map(PLZ_map.lat)
     df["PLZ_lon"] = df.Postleitzahl.map(PLZ_map.lon)

powerplantmatching/duke.py

@@ -25,8 +25,7 @@ def add_geoposition_for_duke(df):
     if not df.loc[:, ["lat", "lon"]].isnull().all().all():
         return df.assign(
             Geoposition=df[["lat", "lon"]]
-            .astype(str)
-            .apply(lambda s: ",".join(s), axis=1)
+            .apply(lambda s: ",".join(map(str, s)), axis=1)
             .replace("nan,nan", np.nan)
         )
     else:

powerplantmatching/matching.py

@@ -34,9 +34,9 @@ def best_matches(links):
     if links.empty:
         return pd.DataFrame(columns=labels)
     else:
-        return links.groupby(links.iloc[:, 1], as_index=False, sort=False).apply(
-            lambda x: x.loc[x.scores.astype(float).idxmax(), labels]
-        )
+        scores = links["scores"].astype(float)
+        best_idx = scores.groupby(links.iloc[:, 1], sort=False).idxmax()
+        return links.loc[best_idx, labels].reset_index(drop=True)
 
 
 def compare_two_datasets(dfs, labels, country_wise=True, config=None, **dukeargs):

pyproject.toml

@@ -33,7 +33,7 @@ requires-python = ">=3.10"
 dependencies = [
     "numpy",
     "scipy",
-    "pandas>=0.24.0,<3",
+    "pandas>=0.24.0,<4",
     "networkx>=1.10",
     "pycountry",
     "country_converter",