Add support for pandas 2.0 (apache#28636)

* Add support for pandas 2.0

* Fix pyarrow tests for pandas 2 compat

* Fix tox.ini maybe

sdks/python/apache_beam/examples/dataframe/flight_delays.py

@@ -44,7 +44,7 @@ def get_mean_delay_at_top_airports(airline_df):
   top_airports = total.nlargest(10, keep='all').dropna()
   at_top_airports = airline_df['arrival_airport'].isin(
       top_airports.index.values)
-  return airline_df[at_top_airports].mean()
+  return airline_df[at_top_airports].mean(numeric_only=True)
 
 
 def input_date(date):

sdks/python/setup.py

@@ -166,7 +166,7 @@ def cythonize(*args, **kwargs):
 # Exclude 1.5.0 and 1.5.1 because of
 # https://github.com/pandas-dev/pandas/issues/45725
 dataframe_dependency = [
-    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"',
+    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.1;python_version>="3.8"',
 ]
 
 def find_by_ext(root_dir, ext):
@@ -337,14 +337,14 @@ def get_portability_package_data():
               # Pinning docutils as a workaround for Sphinx issue:
               # https://github.com/sphinx-doc/sphinx/issues/9727
               'docutils==0.17.1',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
           ],
           'test': [
               'docstring-parser>=0.15,<1.0',
               'freezegun>=0.3.12',
               'joblib>=1.0.1',
               'mock>=1.0.1,<6.0.0',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
               'parameterized>=0.7.1,<0.10.0',
               'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
               'pyyaml>=3.12,<7.0.0',

sdks/python/test-suites/tox/py38/build.gradle

@@ -89,6 +89,10 @@ toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
 test.dependsOn "testPy38pandas-15"
 preCommitPyCoverage.dependsOn "testPy38pandas-15"
 
+toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}"
+test.dependsOn "testPy38pandas-20"
+preCommitPyCoverage.dependsOn "testPy38pandas-20"
+
 // Create a test task for each minor version of pytorch
 toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
 test.dependsOn "testPy38pytorch-19"

sdks/python/tox.ini

@@ -266,12 +266,18 @@ extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
 
+
 [testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
 deps =
+  # Pandas 2 minimum for pyarrow is 7
   3: pyarrow>=3,<4
+  3: pandas<2
   4: pyarrow>=4,<5
+  4: pandas<2
   5: pyarrow>=5,<6
+  5: pandas<2
   6: pyarrow>=6,<7
+  6: pandas<2
   7: pyarrow>=7,<8
   8: pyarrow>=8,<9
   9: pyarrow>=9,<10
@@ -296,11 +302,13 @@ commands =
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
 
 
-[testenv:py{38,39,310,311}-pandas-{14,15}]
+[testenv:py{38,39,310,311}-pandas-{14,15,20}]
 deps =
   14: pandas>=1.4.3,<1.5.0
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
+  20: pandas>=2.0.0,<2.1.0
+  20: pyarrow>=7
 commands =
   # Log pandas and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"