ARROW-255 Update test config and address test failures (#240)

* get test suite running

* undo docs changes

* address warning

* fix utc now handling

* fix utc now handling

* bump to 3.9+

* update changelog

* lint

* lint

* lint

* fix test

.github/workflows/release-python.yml

@@ -32,15 +32,11 @@ jobs:
         - [ubuntu-20.04, manylinux_aarch64]
         - [macos-14, macosx_*]
         - [windows-2019, win_amd64]
-        python: ["cp38", "cp39", "cp310", "cp311", "cp312"]
+        python: ["cp39", "cp310", "cp311", "cp312"]
         exclude:
-          - buildplat: [macos-14, macosx_*]
-            python: "cp38"
           - buildplat: [macos-14, macosx_*]
             python: "cp39"
         include:
-          - buildplat: [macos-12, macosx_*]
-            python: "cp38"
           - buildplat: [macos-12, macosx_*]
             python: "cp39"
 
@@ -75,13 +71,6 @@ jobs:
       - name: Install cibuildwheel
         run: python -m pip install "cibuildwheel>=2.4,<3"
 
-      - name: Build MacOS Py38 Wheel
-        if: ${{ matrix.python == 'cp38' && matrix.buildplat[0] == 'macos-11' }}
-        env:
-          CIBW_BUILD: cp38-macosx_x86_64
-          MACOSX_DEPLOYMENT_TARGET: "10.14"
-        run: python -m cibuildwheel --output-dir wheelhouse
-
       - name: Build MacOS Py39 Wheels
         if: ${{ matrix.python == 'cp39' && matrix.buildplat[0] == 'macos-11' }}
         env:
@@ -114,7 +103,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           # Build sdist on lowest supported Python
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install tox
         run: |

.github/workflows/test-python.yml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     name: CPython ${{ matrix.python-version }}-${{ matrix.os }}
     steps:

bindings/python/CHANGELOG.md

@@ -3,6 +3,10 @@
 
 ---
 
+# Changes in Version 1.6.0
+
+- Drop support for Python 3.8.
+
 # Changes in Version 1.5.2
 
 - Fix support for PyMongo 4.9.

bindings/python/CONTRIBUTING.md

@@ -13,7 +13,7 @@ be of interest or that has already been addressed.
 
 ## Supported Interpreters
 
-PyMongoArrow supports CPython 3.7+ and PyPy3.8+. Language features not
+PyMongoArrow supports CPython 3.9+ and PyPy3.9+. Language features not
 supported by all interpreters can not be used.
 
 ## Style Guide

bindings/python/pymongoarrow/pandas_types.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 # Pandas Extension Types
+from __future__ import annotations
 
 import numbers
 import re
-from typing import Type, Union
 
 import numpy as np
 import pandas as pd
@@ -38,7 +38,7 @@ class PandasBSONDtype(ExtensionDtype):
     def name(self) -> str:
         return f"bson_{self.__class__.__name__}"
 
-    def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]) -> ExtensionArray:
+    def __from_arrow__(self, array: pa.Array | pa.ChunkedArray) -> ExtensionArray:
         chunks = [array] if isinstance(array, pa.Array) else array.chunks
 
         arr_type = self.construct_array_type()
@@ -204,7 +204,7 @@ def name(self) -> str:
         return f"bson_{self.type.__name__}[{self.subtype}]"
 
     @classmethod
-    def construct_array_type(cls) -> Type["PandasBinaryArray"]:
+    def construct_array_type(cls) -> type[PandasBinaryArray]:
         return PandasBinaryArray
 
     @classmethod
@@ -242,7 +242,7 @@ class PandasObjectId(PandasBSONDtype):
     type = ObjectId
 
     @classmethod
-    def construct_array_type(cls) -> Type["PandasObjectIdArray"]:
+    def construct_array_type(cls) -> type[PandasObjectIdArray]:
         return PandasObjectIdArray
 
 
@@ -266,7 +266,7 @@ class PandasDecimal128(PandasBSONDtype):
     type = Decimal128
 
     @classmethod
-    def construct_array_type(cls) -> Type["PandasDecimal128Array"]:
+    def construct_array_type(cls) -> type[PandasDecimal128Array]:
         return PandasDecimal128Array
 
 
@@ -290,7 +290,7 @@ class PandasCode(PandasBSONDtype):
     type = Code
 
     @classmethod
-    def construct_array_type(cls) -> Type["PandasCodeArray"]:
+    def construct_array_type(cls) -> type[PandasCodeArray]:
         return PandasCodeArray
 
 

bindings/python/pyproject.toml

@@ -23,7 +23,6 @@ classifiers = [
     "Operating System :: POSIX",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -32,7 +31,7 @@ classifiers = [
     "Topic :: Database",
 ]
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     # Must be kept in sync with "build_sytem.requires" above.
     "pyarrow >=17.0,<17.1",
@@ -95,9 +94,8 @@ archs = "x86_64 arm64"
 [tool.pytest.ini_options]
 minversion = "7"
 addopts = ["-ra", "--strict-config", "--strict-markers", "--durations=5", "--junitxml=xunit-results/TEST-results.xml"]
-testpaths = ["test", "test/pandas_types"]
+testpaths = ["test"]
 log_cli_level = "INFO"
-norecursedirs = ["test/*"]
 faulthandler_timeout = 1500
 xfail_strict = true
 filterwarnings = [

bindings/python/test/test_arrow.py

@@ -74,6 +74,7 @@ def setUpClass(cls):
         cls.coll = cls.client.pymongoarrow_test.get_collection(
             "test", write_concern=WriteConcern(w="majority")
         )
+        cls.addClassCleanup(cls.client.close)
 
     def setUp(self):
         self.coll.drop()
@@ -113,6 +114,14 @@ def test_find_simple(self):
         self.assertEqual(find_cmd.command_name, "find")
         self.assertEqual(find_cmd.command["projection"], {"_id": True, "data": True})
 
+    def test_find_repeat_type(self):
+        expected = Table.from_pydict(
+            {"_id": [1, 2, 3, 4], "data": [10, 20, 30, None]},
+            ArrowSchema([("_id", int32()), ("data", int32())]),
+        )
+        table = self.run_find({}, schema=Schema({"_id": int32(), "data": int32()}))
+        self.assertEqual(table, expected)
+
     def test_find_with_projection(self):
         expected = Table.from_pydict(
             {"_id": [4, 3], "data": [None, 60]},
@@ -266,18 +275,19 @@ def test_pymongo_error(self):
             },
             ArrowSchema(schema),
         )
-
+        client = MongoClient(
+            host="somedomainthatdoesntexist.org",
+            port=123456789,
+            serverSelectionTimeoutMS=10,
+        )
         with self.assertRaises(ArrowWriteError) as exc:
             write(
-                MongoClient(
-                    host="somedomainthatdoesntexist.org",
-                    port=123456789,
-                    serverSelectionTimeoutMS=10,
-                ).pymongoarrow_test.get_collection(
+                client.pymongoarrow_test.get_collection(
                     "test", write_concern=WriteConcern(w="majority")
                 ),
                 data,
             )
+        client.close()
         self.assertEqual(
             exc.exception.details.keys(),
             {"nInserted", "writeConcernErrors", "writeErrors"},
@@ -840,6 +850,7 @@ def setUpClass(cls):
         cls.client = client_context.get_client(
             event_listeners=[cls.getmore_listener, cls.cmd_listener]
         )
+        cls.addClassCleanup(cls.client.close)
 
     def test_find_decimal128(self):
         oids = list(ObjectId() for i in range(4))

bindings/python/test/test_builders.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import calendar
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, timezone
 from unittest import TestCase
 
 from bson import Binary, Code, Decimal128, ObjectId
@@ -86,7 +86,7 @@ def test_simple(self):
         self.maxDiff = None
 
         builder = DatetimeBuilder(dtype=timestamp("ms"))
-        datetimes = [datetime.utcnow() + timedelta(days=k * 100) for k in range(5)]
+        datetimes = [datetime.now(timezone.utc) + timedelta(days=k * 100) for k in range(5)]
         builder.append(self._datetime_to_millis(datetimes[0]))
         builder.append_values([self._datetime_to_millis(k) for k in datetimes[1:]])
         builder.append_null()
@@ -97,7 +97,9 @@ def test_simple(self):
         self.assertEqual(len(arr), len(datetimes) + 1)
         for actual, expected in zip(arr, datetimes + [None]):
             if actual.is_valid:
-                self.assertEqual(actual.as_py(), self._millis_only(expected))
+                self.assertEqual(
+                    actual.as_py().timetuple(), self._millis_only(expected).timetuple()
+                )
             else:
                 self.assertIsNone(expected)
         self.assertEqual(arr.type, timestamp("ms"))

bindings/python/test/test_datetime.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import unittest
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from test import client_context
 
 import pytz
@@ -36,14 +36,17 @@ def setUp(self):
         self.coll.drop()
         self.coll.insert_many(
             [
-                {"_id": 1, "data": datetime.utcnow() + timedelta(milliseconds=10)},
-                {"_id": 2, "data": datetime.utcnow() + timedelta(milliseconds=25)},
+                {"_id": 1, "data": datetime.now(timezone.utc) + timedelta(milliseconds=10)},
+                {"_id": 2, "data": datetime.now(timezone.utc) + timedelta(milliseconds=25)},
             ]
         )
         self.expected_times = []
         for doc in self.coll.find({}, sort=[("_id", ASCENDING)]):
             self.expected_times.append(doc["data"])
 
+    def tearDown(self):
+        self.client.close()
+
     def test_context_creation_fails_with_unsupported_granularity(self):
         unsupported_granularities = ["s", "us", "ns"]
         for g in unsupported_granularities:

bindings/python/test/test_numpy.py

@@ -42,6 +42,10 @@ def setUpClass(cls):
         )
         cls.schema = {}
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def assert_numpy_equal(self, actual, expected):
         self.assertIsInstance(actual, dict)
         for field in expected:
@@ -321,7 +325,7 @@ def table_from_dict(self, d, schema=None):
         out = {}
         for k, v in d.items():
             if any(isinstance(x, int) for x in v) and None in v:
-                out[k] = np.array(v, dtype=np.float_)
+                out[k] = np.array(v, dtype=np.float64)
             else:
                 out[k] = np.array(v, dtype=np.dtype(type(v[0])))  # Infer
         return out

bindings/python/test/test_pandas.py

@@ -47,6 +47,10 @@ def setUpClass(cls):
             event_listeners=[cls.getmore_listener, cls.cmd_listener]
         )
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
 
 class TestExplicitPandasApi(PandasTestBase):
     @classmethod

bindings/python/test/test_polars.py

@@ -50,6 +50,7 @@ def setUpClass(cls):
             event_listeners=[cls.getmore_listener, cls.cmd_listener],
             uuidRepresentation="standard",
         )
+        cls.addClassCleanup(cls.client.close)
 
 
 class TestExplicitPolarsApi(PolarsTestBase):
@@ -182,11 +183,11 @@ def test_polars_types(self):
             "Boolean": pl.Series([True, False, None]),
         }
 
-        df_in = pl.DataFrame._from_dict(data=data, schema=pl_schema)
+        df_in = pl.from_dict(data=data, schema=pl_schema)
         self.coll.drop()
         write(self.coll, df_in)
         df_out = find_polars_all(self.coll, {}, schema=Schema(pa_schema))
-        pl.testing.assert_frame_equal(df_in, df_out.drop("_id"))
+        pl.testing.assert_frame_equal(df_in, df_out)
 
     def test_extension_types_fail(self):
         """Confirm failure on ExtensionTypes for Polars.DataFrame.from_arrow"""
@@ -264,10 +265,9 @@ def test_exceptions_for_unsupported_polar_types(self):
         class MyObject:
             pass
 
-        with self.assertRaises(pl.PolarsPanicError) as exc:
+        with self.assertRaises(pl.exceptions.PanicException) as exc:
             df_in = pl.DataFrame(data=[MyObject()] * 2)
             write(self.coll, df_in)
-        self.assertTrue("not implemented" in exc.exception.args[0])
 
     def test_polars_binary_type(self):
         """Demonstrates that binary data is not yet supported. TODO [ARROW-214]
@@ -393,5 +393,5 @@ def test_bson_types(self):
             try:
                 dfpl = pl.from_arrow(table.drop("_id"))
                 assert dfpl["value"].dtype == data_type["ptype"]
-            except pl.ComputeError:
+            except pl.exceptions.ComputeError:
                 assert isinstance(table["value"].type, pa.ExtensionType)

bindings/python/test/test_pymongoarrow.py

@@ -26,6 +26,10 @@ def setUpClass(cls):
             raise unittest.SkipTest("cannot connect to MongoDB")
         cls.client = client_context.get_client()
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.close()
+
     def test_version(self):
         self.assertIsNotNone(__version__)
         self.assertIsInstance(__version__, str)