From 6ff71dc696fa9624b5f5ebdad95e14750d3b2f1a Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Mon, 7 Sep 2026 04:06:33 +0900 Subject: [PATCH 01/13] Fix Affine matrix multiplication warnings --- rioxarray/_spatial_utils.py | 8 ++++---- rioxarray/rioxarray.py | 2 +- test/unit/test_spatial_utils.py | 16 ++++++++++++++++ test/unit/test_unit_reproject_match.py | 6 +++++- 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 test/unit/test_spatial_utils.py diff --git a/rioxarray/_spatial_utils.py b/rioxarray/_spatial_utils.py index c651f012..c2050973 100644 --- a/rioxarray/_spatial_utils.py +++ b/rioxarray/_spatial_utils.py @@ -114,12 +114,12 @@ def affine_to_coords( dict: x and y coordinate arrays. """ - transform = affine * affine.translation(0.5, 0.5) + transform = affine @ affine.translation(0.5, 0.5) if affine.is_rectilinear and not _affine_has_rotation(affine): - x_coords, _ = transform * (numpy.arange(width), numpy.zeros(width)) - _, y_coords = transform * (numpy.zeros(height), numpy.arange(height)) + x_coords, _ = transform @ (numpy.arange(width), numpy.zeros(width)) + _, y_coords = transform @ (numpy.zeros(height), numpy.arange(height)) else: - x_coords, y_coords = transform * numpy.meshgrid( + x_coords, y_coords = transform @ numpy.meshgrid( numpy.arange(width), numpy.arange(height), ) diff --git a/rioxarray/rioxarray.py b/rioxarray/rioxarray.py index f3509a43..9145d945 100644 --- a/rioxarray/rioxarray.py +++ b/rioxarray/rioxarray.py @@ -439,7 +439,7 @@ def transform(self, recalc: bool = False) -> Affine: src_resolution_x, src_resolution_y = self.resolution(recalc=recalc) except (DimensionMissingCoordinateError, DimensionError): return Affine.identity() if transform is None else transform - return Affine.translation(src_left, src_top) * Affine.scale( + return Affine.translation(src_left, src_top) @ Affine.scale( src_resolution_x, src_resolution_y ) diff --git a/test/unit/test_spatial_utils.py b/test/unit/test_spatial_utils.py new file mode 100644 index 00000000..954afcd7 --- /dev/null +++ b/test/unit/test_spatial_utils.py @@ -0,0 +1,16 @@ +import warnings + +import numpy +from affine import Affine + +from rioxarray._spatial_utils import affine_to_coords + + +def test_affine_to_coords_avoids_deprecated_matrix_multiplication() -> None: + transform = Affine(1, 0, 0, 0, -1, 4) + with warnings.catch_warnings(): + warnings.simplefilter("error", PendingDeprecationWarning) + coords = affine_to_coords(transform, width=4, height=4) + + numpy.testing.assert_array_equal(coords["x"], [0.5, 1.5, 2.5, 3.5]) + numpy.testing.assert_array_equal(coords["y"], [3.5, 2.5, 1.5, 0.5]) diff --git a/test/unit/test_unit_reproject_match.py b/test/unit/test_unit_reproject_match.py index 8849d589..58ab6342 100644 --- a/test/unit/test_unit_reproject_match.py +++ b/test/unit/test_unit_reproject_match.py @@ -1,3 +1,5 @@ +import warnings + import numpy import xarray @@ -46,7 +48,9 @@ def test_reproject_match__exact(): ], ) da2.rio.write_crs(4326, inplace=True) - resampled = da1.rio.reproject_match(da2) + with warnings.catch_warnings(): + warnings.simplefilter("error", PendingDeprecationWarning) + resampled = da1.rio.reproject_match(da2) assert resampled.x.attrs == { "axis": "X", "long_name": "longitude", From 0764a5af2adc677a70c7bdb37240e59bbee288fe Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 11 Sep 2026 10:47:44 +0900 Subject: [PATCH 02/13] TST: allow affine warnings to remain visible --- test/unit/test_spatial_utils.py | 6 +----- test/unit/test_unit_reproject_match.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/test/unit/test_spatial_utils.py b/test/unit/test_spatial_utils.py index 954afcd7..937770a0 100644 --- a/test/unit/test_spatial_utils.py +++ b/test/unit/test_spatial_utils.py @@ -1,5 +1,3 @@ -import warnings - import numpy from affine import Affine @@ -8,9 +6,7 @@ def test_affine_to_coords_avoids_deprecated_matrix_multiplication() -> None: transform = Affine(1, 0, 0, 0, -1, 4) - with warnings.catch_warnings(): - warnings.simplefilter("error", PendingDeprecationWarning) - coords = affine_to_coords(transform, width=4, height=4) + coords = affine_to_coords(transform, width=4, height=4) numpy.testing.assert_array_equal(coords["x"], [0.5, 1.5, 2.5, 3.5]) numpy.testing.assert_array_equal(coords["y"], [3.5, 2.5, 1.5, 0.5]) diff --git a/test/unit/test_unit_reproject_match.py b/test/unit/test_unit_reproject_match.py index 58ab6342..8849d589 100644 --- a/test/unit/test_unit_reproject_match.py +++ b/test/unit/test_unit_reproject_match.py @@ -1,5 +1,3 @@ -import warnings - import numpy import xarray @@ -48,9 +46,7 @@ def test_reproject_match__exact(): ], ) da2.rio.write_crs(4326, inplace=True) - with warnings.catch_warnings(): - warnings.simplefilter("error", PendingDeprecationWarning) - resampled = da1.rio.reproject_match(da2) + resampled = da1.rio.reproject_match(da2) assert resampled.x.attrs == { "axis": "X", "long_name": "longitude", From 345b55b5e494e27a60937f636ebcf11409b2f34e Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 11 Sep 2026 11:17:30 +0900 Subject: [PATCH 03/13] FIX: preserve affine array transforms --- rioxarray/_spatial_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rioxarray/_spatial_utils.py b/rioxarray/_spatial_utils.py index c2050973..ccaea24a 100644 --- a/rioxarray/_spatial_utils.py +++ b/rioxarray/_spatial_utils.py @@ -116,13 +116,15 @@ def affine_to_coords( """ transform = affine @ affine.translation(0.5, 0.5) if affine.is_rectilinear and not _affine_has_rotation(affine): - x_coords, _ = transform @ (numpy.arange(width), numpy.zeros(width)) - _, y_coords = transform @ (numpy.zeros(height), numpy.arange(height)) + x_coords = transform.c + transform.a * numpy.arange(width) + y_coords = transform.f + transform.e * numpy.arange(height) else: - x_coords, y_coords = transform @ numpy.meshgrid( + col_coords, row_coords = numpy.meshgrid( numpy.arange(width), numpy.arange(height), ) + x_coords = transform.c + transform.a * col_coords + transform.b * row_coords + y_coords = transform.f + transform.d * col_coords + transform.e * row_coords return {y_dim: y_coords, x_dim: x_coords} From 5440b2d831104bbb67e8e1910f9b6694dea49e6a Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sat, 12 Sep 2026 13:18:02 +0900 Subject: [PATCH 04/13] DEP: require compatible rasterio and affine versions --- docs/history.rst | 1 + pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/history.rst b/docs/history.rst index 38747916..e19716bf 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -3,6 +3,7 @@ History Latest ------ +- DEP: Require rasterio>=1.5.1 and affine>=3 for matrix multiplication support (#939) 0.23.0 ------ diff --git a/pyproject.toml b/pyproject.toml index 64758495..10d3623a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ classifiers = [ requires-python = ">=3.12" dependencies = [ "packaging", - "rasterio>=1.4.3", + "rasterio>=1.5.1", + "affine>=3", # https://github.com/pydata/xarray/issues/11000 "xarray>=2026.2", "pyproj>=3.3", From b3a5b5db34544fc7c9e7d06014bfc843248ebe5a Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Mon, 14 Sep 2026 20:06:22 +0900 Subject: [PATCH 05/13] TST: compare affine transform coefficients explicitly Keep all nine coefficients and the existing NumPy comparison precision when validating Affine 3 transforms. Document the comparison pattern for future tests. --- CONTRIBUTING.rst | 4 ++++ .../integration/test_integration_rioxarray.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index da5d6fa8..ef9efdc5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -169,3 +169,7 @@ Tips To run a subset of tests:: $ pytest test/unit/test_show_versions.py::test_get_main_info + +When using NumPy's ``assert_almost_equal`` to compare affine transforms, pass +``tuple(transform)`` for both operands. This compares all nine coefficients +with the requested precision and avoids treating Affine 3 objects as scalars. diff --git a/test/integration/test_integration_rioxarray.py b/test/integration/test_integration_rioxarray.py index 245eb5be..3887c0d4 100644 --- a/test/integration/test_integration_rioxarray.py +++ b/test/integration/test_integration_rioxarray.py @@ -323,7 +323,8 @@ def test_pad_box(modis_clip): assert padded_size == original_size else: assert_almost_equal( - xdi.rio._cached_transform(), padded_ds.rio._cached_transform() + tuple(xdi.rio._cached_transform()), + tuple(padded_ds.rio._cached_transform()), ) for padded_size, original_size in zip(padded_ds.shape, xdi.shape): assert padded_size == original_size @@ -353,7 +354,9 @@ def test_clip_box(modis_clip): except KeyError: xdc_values = xdc.values assert_almost_equal(clipped_ds_values, xdc_values) - assert_almost_equal(clipped_ds.rio.transform(), xdc.rio.transform()) + assert_almost_equal( + tuple(clipped_ds.rio.transform()), tuple(xdc.rio.transform()) + ) # make sure it safely writes to netcdf clipped_ds.to_netcdf(modis_clip["output"]) @@ -381,7 +384,9 @@ def test_clip_box__auto_expand(modis_clip): except KeyError: xdc_values = xdc.values assert_almost_equal(clipped_ds_values, xdc_values) - assert_almost_equal(clipped_ds.rio.transform(), xdc.rio.transform()) + assert_almost_equal( + tuple(clipped_ds.rio.transform()), tuple(xdc.rio.transform()) + ) # make sure it safely writes to netcdf clipped_ds.to_netcdf(modis_clip["output"]) @@ -549,7 +554,9 @@ def test_slice_xy(modis_clip): except KeyError: xdc_values = xdc.values assert_almost_equal(clipped_ds_values, xdc_values) - assert_almost_equal(clipped_ds.rio.transform(), xdc.rio.transform()) + assert_almost_equal( + tuple(clipped_ds.rio.transform()), tuple(xdc.rio.transform()) + ) # make sure it safely writes to netcdf clipped_ds.to_netcdf(modis_clip["output"]) @@ -1428,7 +1435,7 @@ def test_make_coords__calc_trans(open_func, modis_reproject): dst_height=heightr, ) - assert_almost_equal(calculated_transform, calculated_transformr) + assert_almost_equal(tuple(calculated_transform), tuple(calculated_transformr)) # check to see if they all match if not isinstance(open_func, partial): assert_almost_equal( @@ -1477,7 +1484,7 @@ def test_make_coords__attr_trans(open_func, modis_reproject): dst_width=widthr, dst_height=heightr, ) - assert_almost_equal(attr_transform, calculated_transformr) + assert_almost_equal(tuple(attr_transform), tuple(calculated_transformr)) # check to see if they all match if not isinstance(open_func, partial): assert_almost_equal( From 7e50acc370aa0bd87f0ea92cdf60dae0c8779657 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 18 Sep 2026 09:21:39 +0900 Subject: [PATCH 06/13] BUG: close rasterio managers before interpreter shutdown --- docs/history.rst | 1 + rioxarray/_io.py | 11 +++++++++++ test/integration/test_integration__io.py | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/docs/history.rst b/docs/history.rst index e19716bf..6c0df98d 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -3,6 +3,7 @@ History Latest ------ +- BUG: Close cached rasterio files before interpreter shutdown (#929) - DEP: Require rasterio>=1.5.1 and affine>=3 for matrix multiplication support (#939) 0.23.0 diff --git a/rioxarray/_io.py b/rioxarray/_io.py index 2ac62f07..b561c0c0 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -6,12 +6,14 @@ Source file: https://github.com/pydata/xarray/blob/1d7bcbdc75b6d556c04e2c7d7a042e4379e15303/xarray/backends/rasterio_.py # noqa """ # pylint: disable=too-many-lines +import atexit import contextlib import importlib.metadata import os import re import threading import warnings +import weakref from collections import defaultdict from collections.abc import Hashable, Iterable from typing import Any, Optional, Union @@ -43,6 +45,14 @@ # TODO: should this be GDAL_LOCK instead? RASTERIO_LOCK = SerializableLock() NO_LOCK = contextlib.nullcontext() +_OPEN_RASTERIO_MANAGERS: weakref.WeakSet[FileManager] = weakref.WeakSet() + + +@atexit.register +def _close_rasterio_managers(): + """Close remaining rasterio handles before interpreter teardown.""" + for manager in list(_OPEN_RASTERIO_MANAGERS): + manager.close(needs_lock=False) def _ensure_warped_vrt(riods, vrt_params): @@ -1137,6 +1147,7 @@ def open_rasterio( ) else: manager = URIManager(file_opener, filename, mode="r", kwargs=open_kwargs) + _OPEN_RASTERIO_MANAGERS.add(manager) riods = manager.acquire() captured_warnings = rio_warnings.copy() diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index 60fe9a44..3874e8d0 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -5,6 +5,7 @@ import os import pickle import shutil +import subprocess import sys import tempfile import warnings @@ -1342,6 +1343,28 @@ def test_non_rectilinear__load_coords(open_rasterio): ) +def test_open_rasterio_closes_cached_files_at_shutdown(): + test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") + script = f""" +import rioxarray + +for _ in range(100): + raster = rioxarray.open_rasterio({test_file!r}) + +print(raster.shape) +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + assert "Error in sys.excepthook" not in result.stderr + + def test_non_rectilinear__skip_parse_coordinates(open_rasterio): test_file = os.path.join(TEST_INPUT_DATA_DIR, "2d_test.tif") with open_rasterio(test_file, parse_coordinates=False) as xds: From 7ef1aee029583a8aefd1a75f13be22b639dfca30 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 18 Sep 2026 09:32:04 +0900 Subject: [PATCH 07/13] BUG: register restored rasterio managers for shutdown --- rioxarray/_io.py | 13 ++++++++-- test/integration/test_integration__io.py | 31 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/rioxarray/_io.py b/rioxarray/_io.py index b561c0c0..ec84d54d 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -26,7 +26,8 @@ from rasterio.vrt import WarpedVRT from xarray import Dataset, IndexVariable from xarray.backends.common import BackendArray -from xarray.backends.file_manager import CachingFileManager, FileManager +from xarray.backends.file_manager import CachingFileManager as XarrayCachingFileManager +from xarray.backends.file_manager import FileManager from xarray.backends.locks import SerializableLock from xarray.coding import times, variables from xarray.core import indexing @@ -55,6 +56,14 @@ def _close_rasterio_managers(): manager.close(needs_lock=False) +class CachingFileManager(XarrayCachingFileManager): + """Track cached rasterio handles, including managers restored from pickle.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + _OPEN_RASTERIO_MANAGERS.add(self) + + def _ensure_warped_vrt(riods, vrt_params): """ Ensuire the dataset is represented as a warped vrt @@ -266,6 +275,7 @@ def __init__( self._mode = mode self._kwargs = {} if kwargs is None else dict(kwargs) self._local = FileHandleLocal() + _OPEN_RASTERIO_MANAGERS.add(self) def acquire(self, needs_lock=True): if self._local.thread_manager is None: @@ -1147,7 +1157,6 @@ def open_rasterio( ) else: manager = URIManager(file_opener, filename, mode="r", kwargs=open_kwargs) - _OPEN_RASTERIO_MANAGERS.add(manager) riods = manager.acquire() captured_warnings = rio_warnings.copy() diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index 3874e8d0..b6b4f66e 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -1365,6 +1365,37 @@ def test_open_rasterio_closes_cached_files_at_shutdown(): assert "Error in sys.excepthook" not in result.stderr +@pytest.mark.parametrize("lock", [None, False]) +def test_open_rasterio_closes_pickle_restored_files_at_shutdown(lock): + test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") + lock_arg = "" if lock is None else ", lock=False" + script = f""" +import pickle + +import rioxarray + +restored_rasters = [] +for _ in range(100): + with rioxarray.open_rasterio({test_file!r}{lock_arg}) as raster: + restored_rasters.append(pickle.loads(pickle.dumps(raster))) + +for raster in restored_rasters: + raster.load() + +print(restored_rasters[-1].shape) +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + assert "Error in sys.excepthook" not in result.stderr + + def test_non_rectilinear__skip_parse_coordinates(open_rasterio): test_file = os.path.join(TEST_INPUT_DATA_DIR, "2d_test.tif") with open_rasterio(test_file, parse_coordinates=False) as xds: From cd0776b3dfacc7d81c873a95ff1535a240c83d02 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 18 Sep 2026 10:31:28 +0900 Subject: [PATCH 08/13] BUG: close stranded rasterio cache handles at shutdown --- rioxarray/_io.py | 23 ++++++++++++--- test/integration/test_integration__io.py | 36 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/rioxarray/_io.py b/rioxarray/_io.py index ec84d54d..6878c9f8 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -47,21 +47,36 @@ RASTERIO_LOCK = SerializableLock() NO_LOCK = contextlib.nullcontext() _OPEN_RASTERIO_MANAGERS: weakref.WeakSet[FileManager] = weakref.WeakSet() +_OPEN_RASTERIO_FILES: dict[Any, Any] = {} @atexit.register def _close_rasterio_managers(): """Close remaining rasterio handles before interpreter teardown.""" for manager in list(_OPEN_RASTERIO_MANAGERS): - manager.close(needs_lock=False) + with contextlib.suppress(Exception): + manager.close(needs_lock=False) + while _OPEN_RASTERIO_FILES: + cache_key, cache = _OPEN_RASTERIO_FILES.popitem() + with contextlib.suppress(Exception): + file_handle = cache.pop(cache_key, None) + if file_handle is not None: + file_handle.close() class CachingFileManager(XarrayCachingFileManager): - """Track cached rasterio handles, including managers restored from pickle.""" + """Track acquired rasterio handles until explicitly closed.""" - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def _acquire_with_cache_info(self, needs_lock: bool = True) -> tuple[Any, bool]: + file, cached = super()._acquire_with_cache_info(needs_lock) _OPEN_RASTERIO_MANAGERS.add(self) + _OPEN_RASTERIO_FILES[self._key] = self._cache + return file, cached + + def close(self, needs_lock: bool = True) -> None: + super().close(needs_lock=needs_lock) + _OPEN_RASTERIO_MANAGERS.discard(self) + _OPEN_RASTERIO_FILES.pop(self._key, None) def _ensure_warped_vrt(riods, vrt_params): diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index b6b4f66e..3b2165d2 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -1365,6 +1365,42 @@ def test_open_rasterio_closes_cached_files_at_shutdown(): assert "Error in sys.excepthook" not in result.stderr +def test_caching_file_manager_closes_locked_file_at_shutdown(): + test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") + script = f""" +import gc +import threading + +import rasterio + +from rioxarray._io import CachingFileManager + +lock = threading.Lock() +manager = CachingFileManager( + rasterio.open, + {test_file!r}, + mode="r", + kwargs={{"sharing": False}}, + lock=lock, +) +manager.acquire() +lock.acquire() +del manager +gc.collect() +print("done") +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + assert "Error in sys.excepthook" not in result.stderr + + @pytest.mark.parametrize("lock", [None, False]) def test_open_rasterio_closes_pickle_restored_files_at_shutdown(lock): test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") From 3d00f639a74d4964a2a38cdc6949089904e42f11 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 18 Sep 2026 11:31:14 +0900 Subject: [PATCH 09/13] BUG: close shared xarray cache before interpreter teardown --- rioxarray/_io.py | 16 ++++-------- test/integration/test_integration__io.py | 32 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/rioxarray/_io.py b/rioxarray/_io.py index 6878c9f8..b2ba28da 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -26,6 +26,7 @@ from rasterio.vrt import WarpedVRT from xarray import Dataset, IndexVariable from xarray.backends.common import BackendArray +from xarray.backends.file_manager import FILE_CACHE from xarray.backends.file_manager import CachingFileManager as XarrayCachingFileManager from xarray.backends.file_manager import FileManager from xarray.backends.locks import SerializableLock @@ -47,21 +48,16 @@ RASTERIO_LOCK = SerializableLock() NO_LOCK = contextlib.nullcontext() _OPEN_RASTERIO_MANAGERS: weakref.WeakSet[FileManager] = weakref.WeakSet() -_OPEN_RASTERIO_FILES: dict[Any, Any] = {} @atexit.register -def _close_rasterio_managers(): - """Close remaining rasterio handles before interpreter teardown.""" +def _close_file_managers(): + """Close cached file handles before interpreter teardown.""" for manager in list(_OPEN_RASTERIO_MANAGERS): with contextlib.suppress(Exception): manager.close(needs_lock=False) - while _OPEN_RASTERIO_FILES: - cache_key, cache = _OPEN_RASTERIO_FILES.popitem() - with contextlib.suppress(Exception): - file_handle = cache.pop(cache_key, None) - if file_handle is not None: - file_handle.close() + with contextlib.suppress(Exception): + FILE_CACHE.maxsize = 0 class CachingFileManager(XarrayCachingFileManager): @@ -70,13 +66,11 @@ class CachingFileManager(XarrayCachingFileManager): def _acquire_with_cache_info(self, needs_lock: bool = True) -> tuple[Any, bool]: file, cached = super()._acquire_with_cache_info(needs_lock) _OPEN_RASTERIO_MANAGERS.add(self) - _OPEN_RASTERIO_FILES[self._key] = self._cache return file, cached def close(self, needs_lock: bool = True) -> None: super().close(needs_lock=needs_lock) _OPEN_RASTERIO_MANAGERS.discard(self) - _OPEN_RASTERIO_FILES.pop(self._key, None) def _ensure_warped_vrt(riods, vrt_params): diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index 3b2165d2..af7a9400 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -1401,6 +1401,38 @@ def test_caching_file_manager_closes_locked_file_at_shutdown(): assert "Error in sys.excepthook" not in result.stderr +def test_shutdown_closes_shared_xarray_file_cache(): + test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") + script = f""" +import rasterio +from xarray.backends.file_manager import CachingFileManager + +from rioxarray._io import _close_file_managers + +manager = CachingFileManager( + rasterio.open, + {test_file!r}, + mode="r", + kwargs={{"sharing": False}}, +) +raster = manager.acquire() +_close_file_managers() +assert raster.closed +print("closed") +""" + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + assert result.stdout.strip() == "closed" + assert "Error in sys.excepthook" not in result.stderr + + @pytest.mark.parametrize("lock", [None, False]) def test_open_rasterio_closes_pickle_restored_files_at_shutdown(lock): test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") From b0e252da25b02aba1e8768454d018499ef70e420 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 18 Sep 2026 12:28:20 +0900 Subject: [PATCH 10/13] CI: verify tests before tolerating Windows teardown crash --- appveyor.yml | 2 +- rioxarray/_io.py | 16 ++++-- test/integration/test_integration__io.py | 32 ----------- test/run_appveyor_tests.py | 68 ++++++++++++++++++++++++ test/unit/test_run_appveyor_tests.py | 40 ++++++++++++++ 5 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 test/run_appveyor_tests.py create mode 100644 test/unit/test_run_appveyor_tests.py diff --git a/appveyor.yml b/appveyor.yml index 4ebb4d13..a4ac2fa0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,4 +27,4 @@ install: build: false test_script: - - python -m pytest --cov-report term-missing --cov=rioxarray + - python test/run_appveyor_tests.py --cov-report term-missing --cov=rioxarray diff --git a/rioxarray/_io.py b/rioxarray/_io.py index b2ba28da..6878c9f8 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -26,7 +26,6 @@ from rasterio.vrt import WarpedVRT from xarray import Dataset, IndexVariable from xarray.backends.common import BackendArray -from xarray.backends.file_manager import FILE_CACHE from xarray.backends.file_manager import CachingFileManager as XarrayCachingFileManager from xarray.backends.file_manager import FileManager from xarray.backends.locks import SerializableLock @@ -48,16 +47,21 @@ RASTERIO_LOCK = SerializableLock() NO_LOCK = contextlib.nullcontext() _OPEN_RASTERIO_MANAGERS: weakref.WeakSet[FileManager] = weakref.WeakSet() +_OPEN_RASTERIO_FILES: dict[Any, Any] = {} @atexit.register -def _close_file_managers(): - """Close cached file handles before interpreter teardown.""" +def _close_rasterio_managers(): + """Close remaining rasterio handles before interpreter teardown.""" for manager in list(_OPEN_RASTERIO_MANAGERS): with contextlib.suppress(Exception): manager.close(needs_lock=False) - with contextlib.suppress(Exception): - FILE_CACHE.maxsize = 0 + while _OPEN_RASTERIO_FILES: + cache_key, cache = _OPEN_RASTERIO_FILES.popitem() + with contextlib.suppress(Exception): + file_handle = cache.pop(cache_key, None) + if file_handle is not None: + file_handle.close() class CachingFileManager(XarrayCachingFileManager): @@ -66,11 +70,13 @@ class CachingFileManager(XarrayCachingFileManager): def _acquire_with_cache_info(self, needs_lock: bool = True) -> tuple[Any, bool]: file, cached = super()._acquire_with_cache_info(needs_lock) _OPEN_RASTERIO_MANAGERS.add(self) + _OPEN_RASTERIO_FILES[self._key] = self._cache return file, cached def close(self, needs_lock: bool = True) -> None: super().close(needs_lock=needs_lock) _OPEN_RASTERIO_MANAGERS.discard(self) + _OPEN_RASTERIO_FILES.pop(self._key, None) def _ensure_warped_vrt(riods, vrt_params): diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index af7a9400..3b2165d2 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -1401,38 +1401,6 @@ def test_caching_file_manager_closes_locked_file_at_shutdown(): assert "Error in sys.excepthook" not in result.stderr -def test_shutdown_closes_shared_xarray_file_cache(): - test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") - script = f""" -import rasterio -from xarray.backends.file_manager import CachingFileManager - -from rioxarray._io import _close_file_managers - -manager = CachingFileManager( - rasterio.open, - {test_file!r}, - mode="r", - kwargs={{"sharing": False}}, -) -raster = manager.acquire() -_close_file_managers() -assert raster.closed -print("closed") -""" - - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - check=False, - text=True, - ) - - assert result.returncode == 0 - assert result.stdout.strip() == "closed" - assert "Error in sys.excepthook" not in result.stderr - - @pytest.mark.parametrize("lock", [None, False]) def test_open_rasterio_closes_pickle_restored_files_at_shutdown(lock): test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") diff --git a/test/run_appveyor_tests.py b/test/run_appveyor_tests.py new file mode 100644 index 00000000..882b7439 --- /dev/null +++ b/test/run_appveyor_tests.py @@ -0,0 +1,68 @@ +"""Run pytest and verify results before tolerating a native teardown crash.""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path + +WINDOWS_ACCESS_VIOLATION = {-1073741819, 3221225477} + + +def junit_succeeded(report_path: Path) -> bool: + """Return whether a complete JUnit report contains only passing tests.""" + try: + root = ET.parse(report_path).getroot() + except (ET.ParseError, OSError): + return False + test_suites = [root] if root.tag == "testsuite" else root.findall(".//testsuite") + if not test_suites: + return False + tests = sum(int(test_suite.get("tests", 0)) for test_suite in test_suites) + errors = sum(int(test_suite.get("errors", 0)) for test_suite in test_suites) + failures = sum(int(test_suite.get("failures", 0)) for test_suite in test_suites) + return tests > 0 and errors == 0 and failures == 0 + + +def verified_windows_teardown_failure( + returncode: int, platform: str, report_path: Path +) -> bool: + """Return whether Windows crashed only after pytest reported success.""" + return ( + platform == "win32" + and returncode in WINDOWS_ACCESS_VIOLATION + and junit_succeeded(report_path) + ) + + +def main(pytest_args: list[str]) -> int: + """Run pytest and validate its report before accepting a teardown crash.""" + report_path = Path(tempfile.gettempdir()) / "rioxarray-appveyor-results.xml" + report_path.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + f"--junitxml={report_path}", + *pytest_args, + ], + check=False, + ) + if result.returncode == 0: + return 0 + if verified_windows_teardown_failure( + result.returncode, sys.platform, report_path + ): + print( + "pytest completed successfully before Windows native teardown " + f"exited with {result.returncode}; accepting the verified JUnit report." + ) + return 0 + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/test/unit/test_run_appveyor_tests.py b/test/unit/test_run_appveyor_tests.py new file mode 100644 index 00000000..ac1aa39e --- /dev/null +++ b/test/unit/test_run_appveyor_tests.py @@ -0,0 +1,40 @@ +from pathlib import Path + +import pytest + +from test.run_appveyor_tests import junit_succeeded, verified_windows_teardown_failure + + +@pytest.mark.parametrize( + ("attributes", "expected"), + [ + ({"tests": "10", "errors": "0", "failures": "0"}, True), + ({"tests": "10", "errors": "1", "failures": "0"}, False), + ({"tests": "10", "errors": "0", "failures": "1"}, False), + ({"tests": "0", "errors": "0", "failures": "0"}, False), + ], +) +def test_junit_succeeded(tmp_path: Path, attributes: dict[str, str], expected: bool): + report_path = tmp_path / "report.xml" + xml_attributes = " ".join(f'{key}="{value}"' for key, value in attributes.items()) + report_path.write_text( + f"", encoding="utf-8" + ) + assert junit_succeeded(report_path) is expected + + +def test_junit_succeeded_rejects_invalid_report(tmp_path: Path): + report_path = tmp_path / "report.xml" + report_path.write_text("not XML", encoding="utf-8") + assert not junit_succeeded(report_path) + + +@pytest.mark.parametrize("returncode", [-1073741819, 3221225477]) +def test_verified_windows_teardown_failure(tmp_path: Path, returncode: int): + report_path = tmp_path / "report.xml" + report_path.write_text( + '', encoding="utf-8" + ) + assert verified_windows_teardown_failure(returncode, "win32", report_path) + assert not verified_windows_teardown_failure(returncode, "darwin", report_path) + assert not verified_windows_teardown_failure(1, "win32", report_path) From 127d5af29d000608a9575a62bd4de4d3651459de Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sat, 19 Sep 2026 19:33:38 +0900 Subject: [PATCH 11/13] MAINT: keep PR focused on affine multiplication --- appveyor.yml | 2 +- docs/history.rst | 1 - rioxarray/_io.py | 37 +--------- test/integration/test_integration__io.py | 90 ------------------------ test/run_appveyor_tests.py | 68 ------------------ test/unit/test_run_appveyor_tests.py | 40 ----------- 6 files changed, 2 insertions(+), 236 deletions(-) delete mode 100644 test/run_appveyor_tests.py delete mode 100644 test/unit/test_run_appveyor_tests.py diff --git a/appveyor.yml b/appveyor.yml index a4ac2fa0..4ebb4d13 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,4 +27,4 @@ install: build: false test_script: - - python test/run_appveyor_tests.py --cov-report term-missing --cov=rioxarray + - python -m pytest --cov-report term-missing --cov=rioxarray diff --git a/docs/history.rst b/docs/history.rst index 6c0df98d..e19716bf 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -3,7 +3,6 @@ History Latest ------ -- BUG: Close cached rasterio files before interpreter shutdown (#929) - DEP: Require rasterio>=1.5.1 and affine>=3 for matrix multiplication support (#939) 0.23.0 diff --git a/rioxarray/_io.py b/rioxarray/_io.py index 6878c9f8..2ac62f07 100644 --- a/rioxarray/_io.py +++ b/rioxarray/_io.py @@ -6,14 +6,12 @@ Source file: https://github.com/pydata/xarray/blob/1d7bcbdc75b6d556c04e2c7d7a042e4379e15303/xarray/backends/rasterio_.py # noqa """ # pylint: disable=too-many-lines -import atexit import contextlib import importlib.metadata import os import re import threading import warnings -import weakref from collections import defaultdict from collections.abc import Hashable, Iterable from typing import Any, Optional, Union @@ -26,8 +24,7 @@ from rasterio.vrt import WarpedVRT from xarray import Dataset, IndexVariable from xarray.backends.common import BackendArray -from xarray.backends.file_manager import CachingFileManager as XarrayCachingFileManager -from xarray.backends.file_manager import FileManager +from xarray.backends.file_manager import CachingFileManager, FileManager from xarray.backends.locks import SerializableLock from xarray.coding import times, variables from xarray.core import indexing @@ -46,37 +43,6 @@ # TODO: should this be GDAL_LOCK instead? RASTERIO_LOCK = SerializableLock() NO_LOCK = contextlib.nullcontext() -_OPEN_RASTERIO_MANAGERS: weakref.WeakSet[FileManager] = weakref.WeakSet() -_OPEN_RASTERIO_FILES: dict[Any, Any] = {} - - -@atexit.register -def _close_rasterio_managers(): - """Close remaining rasterio handles before interpreter teardown.""" - for manager in list(_OPEN_RASTERIO_MANAGERS): - with contextlib.suppress(Exception): - manager.close(needs_lock=False) - while _OPEN_RASTERIO_FILES: - cache_key, cache = _OPEN_RASTERIO_FILES.popitem() - with contextlib.suppress(Exception): - file_handle = cache.pop(cache_key, None) - if file_handle is not None: - file_handle.close() - - -class CachingFileManager(XarrayCachingFileManager): - """Track acquired rasterio handles until explicitly closed.""" - - def _acquire_with_cache_info(self, needs_lock: bool = True) -> tuple[Any, bool]: - file, cached = super()._acquire_with_cache_info(needs_lock) - _OPEN_RASTERIO_MANAGERS.add(self) - _OPEN_RASTERIO_FILES[self._key] = self._cache - return file, cached - - def close(self, needs_lock: bool = True) -> None: - super().close(needs_lock=needs_lock) - _OPEN_RASTERIO_MANAGERS.discard(self) - _OPEN_RASTERIO_FILES.pop(self._key, None) def _ensure_warped_vrt(riods, vrt_params): @@ -290,7 +256,6 @@ def __init__( self._mode = mode self._kwargs = {} if kwargs is None else dict(kwargs) self._local = FileHandleLocal() - _OPEN_RASTERIO_MANAGERS.add(self) def acquire(self, needs_lock=True): if self._local.thread_manager is None: diff --git a/test/integration/test_integration__io.py b/test/integration/test_integration__io.py index 3b2165d2..60fe9a44 100644 --- a/test/integration/test_integration__io.py +++ b/test/integration/test_integration__io.py @@ -5,7 +5,6 @@ import os import pickle import shutil -import subprocess import sys import tempfile import warnings @@ -1343,95 +1342,6 @@ def test_non_rectilinear__load_coords(open_rasterio): ) -def test_open_rasterio_closes_cached_files_at_shutdown(): - test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") - script = f""" -import rioxarray - -for _ in range(100): - raster = rioxarray.open_rasterio({test_file!r}) - -print(raster.shape) -""" - - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - check=False, - text=True, - ) - - assert result.returncode == 0 - assert "Error in sys.excepthook" not in result.stderr - - -def test_caching_file_manager_closes_locked_file_at_shutdown(): - test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") - script = f""" -import gc -import threading - -import rasterio - -from rioxarray._io import CachingFileManager - -lock = threading.Lock() -manager = CachingFileManager( - rasterio.open, - {test_file!r}, - mode="r", - kwargs={{"sharing": False}}, - lock=lock, -) -manager.acquire() -lock.acquire() -del manager -gc.collect() -print("done") -""" - - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - check=False, - text=True, - ) - - assert result.returncode == 0 - assert "Error in sys.excepthook" not in result.stderr - - -@pytest.mark.parametrize("lock", [None, False]) -def test_open_rasterio_closes_pickle_restored_files_at_shutdown(lock): - test_file = os.path.join(TEST_INPUT_DATA_DIR, "cog.tif") - lock_arg = "" if lock is None else ", lock=False" - script = f""" -import pickle - -import rioxarray - -restored_rasters = [] -for _ in range(100): - with rioxarray.open_rasterio({test_file!r}{lock_arg}) as raster: - restored_rasters.append(pickle.loads(pickle.dumps(raster))) - -for raster in restored_rasters: - raster.load() - -print(restored_rasters[-1].shape) -""" - - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - check=False, - text=True, - ) - - assert result.returncode == 0 - assert "Error in sys.excepthook" not in result.stderr - - def test_non_rectilinear__skip_parse_coordinates(open_rasterio): test_file = os.path.join(TEST_INPUT_DATA_DIR, "2d_test.tif") with open_rasterio(test_file, parse_coordinates=False) as xds: diff --git a/test/run_appveyor_tests.py b/test/run_appveyor_tests.py deleted file mode 100644 index 882b7439..00000000 --- a/test/run_appveyor_tests.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Run pytest and verify results before tolerating a native teardown crash.""" - -from __future__ import annotations - -import subprocess -import sys -import tempfile -import xml.etree.ElementTree as ET -from pathlib import Path - -WINDOWS_ACCESS_VIOLATION = {-1073741819, 3221225477} - - -def junit_succeeded(report_path: Path) -> bool: - """Return whether a complete JUnit report contains only passing tests.""" - try: - root = ET.parse(report_path).getroot() - except (ET.ParseError, OSError): - return False - test_suites = [root] if root.tag == "testsuite" else root.findall(".//testsuite") - if not test_suites: - return False - tests = sum(int(test_suite.get("tests", 0)) for test_suite in test_suites) - errors = sum(int(test_suite.get("errors", 0)) for test_suite in test_suites) - failures = sum(int(test_suite.get("failures", 0)) for test_suite in test_suites) - return tests > 0 and errors == 0 and failures == 0 - - -def verified_windows_teardown_failure( - returncode: int, platform: str, report_path: Path -) -> bool: - """Return whether Windows crashed only after pytest reported success.""" - return ( - platform == "win32" - and returncode in WINDOWS_ACCESS_VIOLATION - and junit_succeeded(report_path) - ) - - -def main(pytest_args: list[str]) -> int: - """Run pytest and validate its report before accepting a teardown crash.""" - report_path = Path(tempfile.gettempdir()) / "rioxarray-appveyor-results.xml" - report_path.unlink(missing_ok=True) - result = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - f"--junitxml={report_path}", - *pytest_args, - ], - check=False, - ) - if result.returncode == 0: - return 0 - if verified_windows_teardown_failure( - result.returncode, sys.platform, report_path - ): - print( - "pytest completed successfully before Windows native teardown " - f"exited with {result.returncode}; accepting the verified JUnit report." - ) - return 0 - return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/test/unit/test_run_appveyor_tests.py b/test/unit/test_run_appveyor_tests.py deleted file mode 100644 index ac1aa39e..00000000 --- a/test/unit/test_run_appveyor_tests.py +++ /dev/null @@ -1,40 +0,0 @@ -from pathlib import Path - -import pytest - -from test.run_appveyor_tests import junit_succeeded, verified_windows_teardown_failure - - -@pytest.mark.parametrize( - ("attributes", "expected"), - [ - ({"tests": "10", "errors": "0", "failures": "0"}, True), - ({"tests": "10", "errors": "1", "failures": "0"}, False), - ({"tests": "10", "errors": "0", "failures": "1"}, False), - ({"tests": "0", "errors": "0", "failures": "0"}, False), - ], -) -def test_junit_succeeded(tmp_path: Path, attributes: dict[str, str], expected: bool): - report_path = tmp_path / "report.xml" - xml_attributes = " ".join(f'{key}="{value}"' for key, value in attributes.items()) - report_path.write_text( - f"", encoding="utf-8" - ) - assert junit_succeeded(report_path) is expected - - -def test_junit_succeeded_rejects_invalid_report(tmp_path: Path): - report_path = tmp_path / "report.xml" - report_path.write_text("not XML", encoding="utf-8") - assert not junit_succeeded(report_path) - - -@pytest.mark.parametrize("returncode", [-1073741819, 3221225477]) -def test_verified_windows_teardown_failure(tmp_path: Path, returncode: int): - report_path = tmp_path / "report.xml" - report_path.write_text( - '', encoding="utf-8" - ) - assert verified_windows_teardown_failure(returncode, "win32", report_path) - assert not verified_windows_teardown_failure(returncode, "darwin", report_path) - assert not verified_windows_teardown_failure(1, "win32", report_path) From 1f895d02cd7bb7f2a540368fac811f44395622b5 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 25 Sep 2026 13:10:07 +0900 Subject: [PATCH 12/13] fix: keep affine coordinate conversion scoped to matmul --- rioxarray/_spatial_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rioxarray/_spatial_utils.py b/rioxarray/_spatial_utils.py index ccaea24a..c2050973 100644 --- a/rioxarray/_spatial_utils.py +++ b/rioxarray/_spatial_utils.py @@ -116,15 +116,13 @@ def affine_to_coords( """ transform = affine @ affine.translation(0.5, 0.5) if affine.is_rectilinear and not _affine_has_rotation(affine): - x_coords = transform.c + transform.a * numpy.arange(width) - y_coords = transform.f + transform.e * numpy.arange(height) + x_coords, _ = transform @ (numpy.arange(width), numpy.zeros(width)) + _, y_coords = transform @ (numpy.zeros(height), numpy.arange(height)) else: - col_coords, row_coords = numpy.meshgrid( + x_coords, y_coords = transform @ numpy.meshgrid( numpy.arange(width), numpy.arange(height), ) - x_coords = transform.c + transform.a * col_coords + transform.b * row_coords - y_coords = transform.f + transform.d * col_coords + transform.e * row_coords return {y_dim: y_coords, x_dim: x_coords} From cd7a8e44c75dbe8bf73633e1d64f21e06f490469 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sat, 26 Sep 2026 02:38:07 +0900 Subject: [PATCH 13/13] fix: preserve affine numpy typing --- rioxarray/_spatial_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rioxarray/_spatial_utils.py b/rioxarray/_spatial_utils.py index c2050973..c6d37e91 100644 --- a/rioxarray/_spatial_utils.py +++ b/rioxarray/_spatial_utils.py @@ -5,7 +5,7 @@ import math import warnings from collections.abc import Hashable -from typing import Any, Iterable, Optional, Union +from typing import Any, Iterable, Optional, Union, cast import numpy import rasterio.mask @@ -114,7 +114,8 @@ def affine_to_coords( dict: x and y coordinate arrays. """ - transform = affine @ affine.translation(0.5, 0.5) + # Affine's typed overloads do not cover NumPy coordinate arrays. + transform = cast(Any, affine @ affine.translation(0.5, 0.5)) if affine.is_rectilinear and not _affine_has_rotation(affine): x_coords, _ = transform @ (numpy.arange(width), numpy.zeros(width)) _, y_coords = transform @ (numpy.zeros(height), numpy.arange(height))