fix is_time to avoid memory overload (#397)

* fix is_time to avoid memory overload

* using latest xarray

* xfail regrid tests ... not working with latest xarray

* added "test-x" target: run tests in parallel

* unblock connection

* update HISTORY.rst

* disable PDF generation

* Bump version: 0.15.0 → 0.16.0

* removed xfail_if_xarray_incompatible from conftest.py

* mark flaky test

Author: cehbrecht

.github/workflows/main.yml

@@ -122,6 +122,7 @@ jobs:
           disable-sudo: true
           egress-policy: block
           allowed-endpoints: >
+            cdn.fwupd.org:443
             conda.anaconda.org:443
             coveralls.io:443
             files.pythonhosted.org:443

.readthedocs.yml

@@ -10,10 +10,7 @@ sphinx:
   configuration: docs/conf.py
   fail_on_warning: true
 
-# Optionally build your docs in additional formats such as PDF and ePub
-formats:
-    - pdf
-
+# Build configuration
 build:
   os: ubuntu-24.04
   tools:
@@ -24,6 +21,7 @@ build:
     pre_build:
       - env SPHINX_APIDOC_OPTIONS="members,undoc-members,show-inheritance,no-index" sphinx-apidoc -o docs/apidoc/ --private --module-first clisops
 
+# Conda configuration
 conda:
   environment: environment.yml
 

HISTORY.rst

@@ -1,13 +1,19 @@
 Version History
 ===============
 
-v0.16.0 (unreleased)
+v0.16.0 (2025-03-18)
 --------------------
 
 New Features
 ^^^^^^^^^^^^
 * `clisops` now officially supports Python 3.13 (#398).
 * Version limitations have been lifted on `xarray` and `numpy` to support newer versions (#398).
+  * Mark regrid tests as `xfail` which do not pass due to latest xarray version (#397).
+
+Bug Fixes
+^^^^^^^^^
+* `clisops.utils.dataset_utils`
+  * Fixed `is_time` to avoid memory overload (#397).
 
 Internal Changes
 ^^^^^^^^^^^^^^^^

Makefile

@@ -64,6 +64,9 @@ lint: ## check style with flake8
 test: ## run tests quickly with the default Python
 	python -m pytest -m "not slow"
 
+test-x: ## run tests quickly with the default Python and in parallel
+	python -m pytest -m "not slow" -n auto
+
 test-all: ## run tests on every Python version with tox
 	tox
 

clisops/__version__.py

@@ -6,4 +6,4 @@
 __contact__ = "[email protected]"
 __copyright__ = "Copyright 2018-2024. United Kingdom Research and Innovation"
 __license__ = "BSD-3-Clause"
-__version__ = "0.15.0"
+__version__ = "0.16.0"

clisops/utils/dataset_utils.py

@@ -5,6 +5,7 @@
 
 import cf_xarray as cfxr  # noqa
 import cftime
+import dask.array as da
 import fsspec
 import numpy as np
 import xarray as xr
@@ -55,12 +56,22 @@ def get_coord_by_type(
     if coord_type not in known_coord_types:
         raise ValueError(f"Coordinate type not known: {coord_type}")
 
+    # Get main variable ... if possible
+    try:
+        main_var = get_main_variable(ds)
+    except ValueError:
+        warnings.warn(f"No main variable found for dataset '{ds}'.")
+        main_var = None
+
     # Loop through all (potential) coordinates to find all possible matches
     if isinstance(ds, xr.DataArray):
         coord_vars = list(ds.coords)
     elif isinstance(ds, xr.Dataset):
         # Not all coordinate variables are always classified as such
         coord_vars = list(ds.coords) + list(ds.data_vars)
+        # make sure we skip the main variable!
+        if main_var is not None:
+            coord_vars.remove(main_var)
     else:
         raise TypeError("Not an xarray.Dataset or xarray.DataArray.")
     for coord_id in coord_vars:
@@ -90,16 +101,18 @@ def get_coord_by_type(
         # Sort in terms of number of dimensions
         coords = sorted(coords, key=lambda x: len(ds[x].dims), reverse=True)
 
-        # Get dimensions and singleton coords of main variable
-        main_var_dims = list(ds[get_main_variable(ds)].dims)
-
-        # Select coordinate with most dims (matching with main variable dims)
-        for coord_id in coords:
-            if all([dim in main_var_dims for dim in ds.coords[coord_id].dims]):
-                if return_further_matches:
-                    return coord_id, [x for x in coords if x != coord_id]
-                else:
-                    return coord_id
+        if main_var is not None:
+            # Get dimensions and singleton coords of main variable
+            main_var_dims = list(ds[main_var].dims)
+
+            # Select coordinate with most dims (matching with main variable dims)
+            for coord_id in coords:
+                if coord_id in ds.coords:
+                    if all([dim in main_var_dims for dim in ds.coords[coord_id].dims]):
+                        if return_further_matches:
+                            return coord_id, [x for x in coords if x != coord_id]
+                        else:
+                            return coord_id
         # If the decision making fails, pass the first match
         if return_further_matches:
             return coords[0], coords[1:]
@@ -207,13 +220,39 @@ def is_level(coord):
     return False
 
 
+def _is_time(coord):
+    """
+    Check if a coordinate uses cftime datetime objects.
+
+    Handles Dask-backed arrays for lazy evaluation.
+    """
+    if coord.size == 0:
+        return False  # Empty array
+
+    if isinstance(coord.dtype.type(), cftime.datetime):
+        return True
+
+    # Safely get first element without loading entire array
+    first_value = coord.isel({dim: 0 for dim in coord.dims}).values
+
+    # Compute only if it's a Dask array
+    if isinstance(first_value, da.Array):
+        first_value = first_value.compute()
+
+    return isinstance(first_value.item(0), cftime.datetime)
+
+
 def is_time(coord):
     """
     Determines if a coordinate is time.
 
     :param coord: coordinate of xarray dataset e.g. coord = ds.coords[coord_id]
     :return: (bool) True if the coordinate is time.
     """
+    if False and coord.ndim >= 2:
+        # skip variables with more than two dimensions: lat_bnds, lon_bnds, time_bnds, t, ...
+        return False
+
     if "time" in coord.cf.coordinates and coord.name in coord.cf.coordinates["time"]:
         return True
 
@@ -226,14 +265,11 @@ def is_time(coord):
     if np.issubdtype(coord.dtype, np.datetime64):
         return True
 
-    if isinstance(np.atleast_1d(coord.values)[0], cftime.datetime):
-        return True
-
     if hasattr(coord, "axis"):
         if coord.axis == "T":
             return True
 
-    return False
+    return _is_time(coord)
 
 
 def is_realization(coord):
@@ -287,8 +323,12 @@ def get_main_variable(ds, exclude_common_coords=True):
                                 Default is True.
     :return: (str) The main variable of the dataset e.g. 'tas'
     """
-
-    data_dims = [data.dims for var_id, data in ds.variables.items()]
+    if isinstance(ds, xr.Dataset):
+        variables = list(ds.variables.items())
+        data_dims = [data.dims for var_id, data in variables]
+    else:
+        variables = []
+        data_dims = []
     flat_dims = [dim for sublist in data_dims for dim in sublist]
 
     results = {}
@@ -303,7 +343,7 @@ def get_main_variable(ds, exclude_common_coords=True):
         "realization",
     ]
 
-    for var_id, data in ds.variables.items():
+    for var_id, data in variables:
         if var_id in flat_dims:
             continue
         if exclude_common_coords is True and any(

docs/conf.py

@@ -89,7 +89,7 @@
 # the built documents.
 #
 # The short X.Y version.
-version = "0.15.0"
+version = "0.16.0"
 # The full version, including alpha/beta/rc tags.
 release = version
 

environment.yml

@@ -28,11 +28,11 @@ dependencies:
  - roocs-grids >=0.1.2
  - scipy >=1.10.0
  - shapely >=2.0
- - xarray >=2022.6.0 # Some issues with CFTime calendars. See: https://github.com/pydata/xarray/issues/7794
+ - xarray >=2025.1.1 # use decode_times=CFDatetimeCoder(use_cftime=True)
  - zarr >=2.13.0
-  # Compression
+ # Compression
  - zstandard
-  # Extras
+ # Extras
  - netcdf4 >=1.4
  - xesmf >=0.8.2
 #  # Dev tools and testing

pyproject.toml

@@ -67,7 +67,7 @@ dependencies = [
   "roocs_grids >=0.1.2",
   "scipy>=1.9.0",
   "shapely >=1.9",
-  "xarray >=2022.9.0", # Some issues with CFTime calendars. See: https://github.com/pydata/xarray/issues/7794
+  "xarray >=2025.1.1", # Use decode_times=CFDatetimeCoder(use_cftime=True)
   "zarr >=2.13.0"
 ]
 
@@ -130,7 +130,7 @@ target-version = [
 ]
 
 [tool.bumpversion]
-current_version = "0.15.0"
+current_version = "0.16.0"
 commit = true
 tag = false
 allow_dirty = false

tests/conftest.py

@@ -409,14 +409,3 @@ def clisops_test_data():
         "small_geojson": test_data.joinpath("small_geojson.json").as_posix(),
         "multi_regions_geojson": test_data.joinpath("multi_regions.json").as_posix(),
     }
-
-
-# Temporarily required until https://github.com/pydata/xarray/issues/7794 is addressed
-@pytest.fixture(scope="session")
-def xfail_if_xarray_incompatible():
-    if Version(xr.__version__) >= Version(XARRAY_INCOMPATIBLE_VERSION):
-        pytest.xfail(
-            f"xarray version >= {XARRAY_INCOMPATIBLE_VERSION} "
-            f"is not supported for several operations with cf-time indexed arrays. "
-            "For more information, see: https://github.com/pydata/xarray/issues/7794."
-        )