Update pandas version upper bound to support python 3.14 (#39056)

* Update pandas version upper bound to support python 3.14

* Fix the failed tests causing by updating pandas

* Skip the test as the pandas bug is unfixed at 2.3.3

* Refactor get_dummies NaN column lookup to use global static constant

Author: shunping

sdks/python/apache_beam/dataframe/frames.py

@@ -63,6 +63,10 @@
 # Get major, minor version
 PD_VERSION = tuple(map(int, pd.__version__.split('.')[0:2]))
 
+# Find the name pandas uses for the NaN column in get_dummies().
+# Older pandas versions (<2.3.0) use 'nan', whereas newer versions (>=2.3.0) use 'NaN'.
+_DUMMY_NAN_COLUMN = 'NaN' if PD_VERSION >= (2, 3) else 'nan'
+
 
 def populate_not_implemented(pd_type):
   def wrapper(deferred_type):
@@ -5054,7 +5058,8 @@ def get_dummies(self, **kwargs):
     # the data includes NaNs, which is not valid to be casted as a Category,
     # but nevertheless would be broadcasted as a column in get_dummies()
     columns = sorted(set().union(*split_cats))
-    columns = columns + ['nan'] if 'nan' not in columns else columns
+    if _DUMMY_NAN_COLUMN not in columns:
+      columns = columns + [_DUMMY_NAN_COLUMN]
 
     proxy = pd.DataFrame(columns=columns).astype(int)
 

sdks/python/apache_beam/dataframe/frames_test.py

@@ -37,6 +37,8 @@
 
 # Get major, minor version
 PD_VERSION = tuple(map(int, pd.__version__.split('.')[0:2]))
+# Get major, minor, patch version
+PD_FULL_VERSION = tuple(int(x) for x in re.findall(r'\d+', pd.__version__)[0:3])
 
 GROUPBY_DF = pd.DataFrame({
     'group': ['a' if i % 5 == 0 or i % 3 == 0 else 'b' for i in range(100)],
@@ -1437,7 +1439,7 @@ def test_unstack_pandas_example2(self):
     self._run_test(lambda s: s.unstack(level=0), s)
 
   @unittest.skipIf(
-      sys.version_info >= (3, 12) and PD_VERSION < (2, 3),
+      sys.version_info >= (3, 12) and PD_FULL_VERSION < (2, 3, 4),
       'https://github.com/pandas-dev/pandas/issues/58604')
   def test_unstack_pandas_example3(self):
     index = self._unstack_get_categorical_index()

sdks/python/setup.py

@@ -162,7 +162,7 @@ def cythonize(*args, **kwargs):
 # https://github.com/pandas-dev/pandas/issues/45725
 # must update the below "docs" and "test" for extras_require
 dataframe_dependency = [
-    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3',
+    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.4',
 ]
 
 milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']
@@ -485,7 +485,7 @@ def get_portability_package_data():
               'docstring-parser>=0.15,<1.0',
               'docutils>=0.18.1',
               'markdown',
-              'pandas<2.3.0',
+              'pandas<2.4.0',
               'openai',
               'virtualenv-clone>=0.5,<1.0',
           ],
@@ -496,7 +496,7 @@ def get_portability_package_data():
               'jinja2>=3.0,<3.2',
               'joblib>=1.0.1',
               'mock>=1.0.1,<6.0.0',
-              'pandas<2.3.0',
+              'pandas<2.4.0',
               'parameterized>=0.7.1,<0.10.0',
               'pydot>=1.2.0,<2',
               'pyhamcrest>=1.9,!=1.10.0,<3.0.0',