Merge branch 'v3' into upath

Author: dstansby

.github/workflows/hypothesis.yaml

@@ -0,0 +1,84 @@
+name: Slow Hypothesis CI
+on:
+  push:
+    branches:
+      - "main"
+      - "v3"
+  pull_request:
+    branches:
+      - "main"
+      - "v3"
+    types: [opened, reopened, synchronize, labeled]
+  schedule:
+    - cron: "0 0 * * *" # Daily “At 00:00” UTC
+  workflow_dispatch: # allows you to trigger manually
+
+env:
+  FORCE_COLOR: 3
+
+jobs:
+
+  hypothesis:
+    name: Slow Hypothesis Tests
+    runs-on: "ubuntu-latest"
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      matrix:
+        python-version: ['3.11']
+        numpy-version: ['1.26']
+        dependency-set: ["optional"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install Hatch
+      run: |
+        python -m pip install --upgrade pip
+        pip install hatch
+    - name: Set Up Hatch Env
+      run: |
+        hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
+        hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
+    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+    - name: Restore cached hypothesis directory
+      id: restore-hypothesis-cache
+      uses: actions/cache/restore@v4
+      with:
+        path: .hypothesis/
+        key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+        restore-keys: |
+          cache-hypothesis-
+
+    - name: Run slow Hypothesis tests
+      if: success()
+      id: status
+      run: |
+        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis
+
+    # explicitly save the cache so it gets updated, also do this even if it fails.
+    - name: Save cached hypothesis directory
+      id: save-hypothesis-cache
+      if: always() && steps.status.outcome != 'skipped'
+      uses: actions/cache/save@v4
+      with:
+        path: .hypothesis/
+        key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+
+    - name: Generate and publish the report
+      if: |
+        failure()
+        && steps.status.outcome == 'failure'
+        && github.event_name == 'schedule'
+        && github.repository_owner == 'zarr-developers'
+      uses: xarray-contrib/issue-from-pytest-log@v1
+      with:
+        log-path: output-${{ matrix.python-version }}-log.jsonl
+        issue-title: "Nightly Hypothesis tests failed"
+        issue-label: "topic-hypothesis"

.gitignore

@@ -78,5 +78,8 @@ src/zarr/_version.py
 #test_sync*
 data/*
 src/fixture/
+fixture/
 
 .DS_Store
+tests/.hypothesis
+.hypothesis/

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.5.4'
+    rev: 'v0.5.7'
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -22,7 +22,7 @@ repos:
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.0
+    rev: v1.11.1
     hooks:
       - id: mypy
         files: src

docs/_static/custom.css

@@ -117,7 +117,3 @@ html[data-theme=dark] .sd-card .sd-card-footer {
 html[data-theme=dark] h1 {
   color: var(--pst-color-primary);
 }
-
-html[data-theme=dark] h3 {
-  color: #0a6774;
-}

pyproject.toml

@@ -29,10 +29,8 @@ dependencies = [
     'numcodecs>=0.10.0',
     'fsspec>2024',
     'crc32c',
-    'zstandard',
     'typing_extensions',
     'donfig',
-    'pytest'
 ]
 dynamic = [
   "version",
@@ -55,6 +53,22 @@ license = {text = "MIT License"}
 keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 
 [project.optional-dependencies]
+test = [
+    "coverage",
+    "pytest",
+    "pytest-cov",
+    "msgpack",
+    "lmdb",
+    "s3fs",
+    "pytest-asyncio",
+    "moto[s3]",
+    "flask-cors",
+    "flask",
+    "requests",
+    "mypy",
+    "hypothesis"
+]
+
 jupyter = [
     'notebook',
     'ipytree>=0.2.2',
@@ -70,8 +84,8 @@ docs = [
     'pydata-sphinx-theme',
     'numpydoc',
     'numcodecs[msgpack]',
-    "msgpack",
-    "lmdb",
+    'msgpack',
+    'lmdb',
 ]
 extra = [
     'msgpack',
@@ -109,21 +123,7 @@ dependencies = [
     "numpy~={matrix:numpy}",
     "universal_pathlib"
 ]
-extra-dependencies = [
-    "coverage",
-    "pytest",
-    "pytest-cov",
-    "msgpack",
-    "lmdb",
-    "s3fs",
-    "pytest-asyncio",
-    "moto[s3]",
-    "flask-cors",
-    "flask",
-    "requests",
-    "mypy"
-]
-features = ["extra"]
+features = ["test", "extra"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.10", "3.11", "3.12"]
@@ -140,6 +140,7 @@ run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests"
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
+run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py"
 list-env = "pip list"
 
 [tool.hatch.envs.docs]
@@ -182,9 +183,11 @@ extend-select = [
     "UP",  # pyupgrade
     "RSE",
     "RUF",
+    "TRY", # tryceratops
 ]
 ignore = [
     "RUF005",
+    "TRY003",
 ]
 
 [tool.mypy]

src/zarr/__init__.py

@@ -26,9 +26,9 @@
     zeros,
     zeros_like,
 )
-from zarr.array import Array, AsyncArray
-from zarr.config import config
-from zarr.group import AsyncGroup, Group
+from zarr.core.array import Array, AsyncArray
+from zarr.core.config import config
+from zarr.core.group import AsyncGroup, Group
 
 # in case setuptools scm screw up and find version to be 0.0.0
 assert not __version__.startswith("0.0.0")

src/zarr/abc/codec.py

@@ -8,17 +8,28 @@
 
 from zarr.abc.metadata import Metadata
 from zarr.abc.store import ByteGetter, ByteSetter
-from zarr.buffer import Buffer, NDBuffer
-from zarr.chunk_grids import ChunkGrid
-from zarr.common import ChunkCoords, concurrent_map
-from zarr.config import config
+from zarr.core.buffer import Buffer, NDBuffer
+from zarr.core.chunk_grids import ChunkGrid
+from zarr.core.common import ChunkCoords, concurrent_map
+from zarr.core.config import config
 
 if TYPE_CHECKING:
     from typing_extensions import Self
 
-    from zarr.array_spec import ArraySpec
-    from zarr.common import JSON
-    from zarr.indexing import SelectorTuple
+    from zarr.core.array_spec import ArraySpec
+    from zarr.core.common import JSON
+    from zarr.core.indexing import SelectorTuple
+
+__all__ = [
+    "CodecInput",
+    "CodecOutput",
+    "ArrayArrayCodec",
+    "ArrayBytesCodec",
+    "BytesBytesCodec",
+    "ArrayBytesCodecPartialDecodeMixin",
+    "ArrayBytesCodecPartialEncodeMixin",
+    "CodecPipeline",
+]
 
 CodecInput = TypeVar("CodecInput", bound=NDBuffer | Buffer)
 CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer)
@@ -112,7 +123,7 @@ async def decode(
         -------
         Iterable[CodecInput | None]
         """
-        return await batching_helper(self._decode_single, chunks_and_specs)
+        return await _batching_helper(self._decode_single, chunks_and_specs)
 
     async def _encode_single(
         self, chunk_data: CodecInput, chunk_spec: ArraySpec
@@ -135,7 +146,7 @@ async def encode(
         -------
         Iterable[CodecOutput | None]
         """
-        return await batching_helper(self._encode_single, chunks_and_specs)
+        return await _batching_helper(self._encode_single, chunks_and_specs)
 
 
 class ArrayArrayCodec(_Codec[NDBuffer, NDBuffer]):
@@ -401,18 +412,18 @@ def from_dict(cls, data: Iterable[JSON | Codec]) -> Self:
         return cls(**data)
 
 
-async def batching_helper(
+async def _batching_helper(
     func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]],
     batch_info: Iterable[tuple[CodecInput | None, ArraySpec]],
 ) -> list[CodecOutput | None]:
     return await concurrent_map(
         list(batch_info),
-        noop_for_none(func),
+        _noop_for_none(func),
         config.get("async.concurrency"),
     )
 
 
-def noop_for_none(
+def _noop_for_none(
     func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]],
 ) -> Callable[[CodecInput | None, ArraySpec], Awaitable[CodecOutput | None]]:
     async def wrap(chunk: CodecInput | None, chunk_spec: ArraySpec) -> CodecOutput | None:

src/zarr/abc/metadata.py

@@ -8,7 +8,9 @@
 
 from dataclasses import dataclass, fields
 
-from zarr.common import JSON
+from zarr.core.common import JSON
+
+__all__ = ["Metadata"]
 
 
 @dataclass(frozen=True)

src/zarr/abc/store.py

@@ -4,8 +4,10 @@
 
 from typing_extensions import Self
 
-from zarr.buffer import Buffer, BufferPrototype
-from zarr.common import AccessModeLiteral, BytesLike
+from zarr.core.buffer import Buffer, BufferPrototype
+from zarr.core.common import AccessModeLiteral, BytesLike
+
+__all__ = ["Store", "AccessMode", "ByteGetter", "ByteSetter", "set_or_delete"]
 
 
 class AccessMode(NamedTuple):

src/zarr/api/asynchronous.py

@@ -9,17 +9,45 @@
 import numpy.typing as npt
 
 from zarr.abc.codec import Codec
-from zarr.array import Array, AsyncArray
-from zarr.buffer import NDArrayLike
-from zarr.chunk_key_encodings import ChunkKeyEncoding
-from zarr.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
-from zarr.group import AsyncGroup
-from zarr.metadata import ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.array import Array, AsyncArray
+from zarr.core.buffer import NDArrayLike
+from zarr.core.chunk_key_encodings import ChunkKeyEncoding
+from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
+from zarr.core.group import AsyncGroup
+from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
 from zarr.store import (
     StoreLike,
     make_store_path,
 )
 
+__all__ = [
+    "consolidate_metadata",
+    "copy",
+    "copy_all",
+    "copy_store",
+    "load",
+    "open",
+    "open_consolidated",
+    "save",
+    "save_array",
+    "save_group",
+    "tree",
+    "array",
+    "group",
+    "open_group",
+    "create",
+    "empty",
+    "empty_like",
+    "full",
+    "full_like",
+    "ones",
+    "ones_like",
+    "open_array",
+    "open_like",
+    "zeros",
+    "zeros_like",
+]
+
 # TODO: this type could use some more thought, noqa to avoid "Variable "asynchronous.ArrayLike" is not valid as a type"
 ArrayLike = Union[AsyncArray | Array | npt.NDArray[Any]]  # noqa
 PathLike = str
@@ -66,7 +94,7 @@ def _like_args(a: ArrayLike, kwargs: dict[str, Any]) -> dict[str, Any]:
         if isinstance(a.metadata, ArrayV3Metadata):
             new["codecs"] = a.metadata.codecs
         else:
-            raise ValueError(f"Unsupported zarr format: {a.metadata.zarr_format}")
+            raise TypeError(f"Unsupported zarr format: {a.metadata.zarr_format}")
     else:
         # TODO: set default values compressor/codecs
         # to do this, we may need to evaluate if this is a v2 or v3 array
@@ -862,7 +890,7 @@ async def open_array(
 
     try:
         return await AsyncArray.open(store_path, zarr_format=zarr_format)
-    except FileNotFoundError as e:
+    except FileNotFoundError:
         if store_path.store.mode.create:
             return await create(
                 store=store_path,
@@ -871,7 +899,7 @@ async def open_array(
                 overwrite=store_path.store.mode.overwrite,
                 **kwargs,
             )
-        raise e
+        raise
 
 
 async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AsyncArray: