Merge remote-tracking branch 'upstream/main' into pyest-xdist

Author: hugovk

.ci/install.sh

@@ -66,7 +66,7 @@ if [[ $(uname) != CYGWIN* ]]; then
     pushd depends && ./install_raqm.sh && popd
 
     # libavif
-    pushd depends && CMAKE_POLICY_VERSION_MINIMUM=3.5 ./install_libavif.sh && popd
+    pushd depends && ./install_libavif.sh && popd
 
     # extra test images
     pushd depends && ./install_extra_test_images.sh && popd

.ci/requirements-mypy.txt

@@ -4,6 +4,7 @@ IceSpringPySideStubs-PySide6
 ipython
 numpy
 packaging
+pyarrow-stubs
 pytest
 sphinx
 types-atheris

.github/workflows/test-valgrind-memory.yml

@@ -0,0 +1,60 @@
+name: Test Valgrind Memory Leaks
+
+# like the Docker tests, but running valgrind only on *.c/*.h changes.
+
+# this is very expensive. Only run on the pull request.
+on:
+  # push:
+  #   branches:
+  #     - "**"
+  #   paths:
+  #     - ".github/workflows/test-valgrind.yml"
+  #     - "**.c"
+  #     - "**.h"
+  pull_request:
+    paths:
+      - ".github/workflows/test-valgrind.yml"
+      - "**.c"
+      - "**.h"
+      - "depends/docker-test-valgrind-memory.sh"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        docker: [
+          ubuntu-22.04-jammy-amd64-valgrind,
+        ]
+        dockerTag: [main]
+
+    name: ${{ matrix.docker }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+
+    - name: Build system information
+      run: python3 .github/workflows/system-info.py
+
+    - name: Docker pull
+      run: |
+        docker pull pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }}
+
+    - name: Build and Run Valgrind
+      run: |
+        # The Pillow user in the docker container is UID 1001
+        sudo chown -R 1001 $GITHUB_WORKSPACE
+        docker run --name pillow_container -e "PILLOW_VALGRIND_TEST=true" -v $GITHUB_WORKSPACE:/Pillow pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }} /Pillow/depends/docker-test-valgrind-memory.sh
+        sudo chown -R runner $GITHUB_WORKSPACE

.github/workflows/test-windows.yml

@@ -31,16 +31,15 @@ env:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
+    runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["pypy3.11", "pypy3.10", "3.10", "3.11", "3.12", "3.13", "3.14"]
         architecture: ["x64"]
-        os: ["windows-latest"]
         include:
             # Test the oldest Python on 32-bit
-            - { python-version: "3.9", architecture: "x86", os: "windows-2019" }
+            - { python-version: "3.9", architecture: "x86" }
 
     timeout-minutes: 45
 

Makefile

@@ -97,13 +97,27 @@ test:
 	python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest
 	python3 -m pytest -qq
 
+.PHONY: test-p
+test-p:
+	python3 -c "import xdist" > /dev/null 2>&1 || python3 -m pip install pytest-xdist
+	python3 -m pytest -qq -n auto
+
+
 .PHONY: valgrind
 valgrind:
 	python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind
-	PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \
+	PILLOW_VALGRIND_TEST=true PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \
             --log-file=/tmp/valgrind-output \
             python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output
 
+.PHONY: valgrind-leak
+valgrind-leak:
+	python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind
+	PILLOW_VALGRIND_TEST=true PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \
+	    --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \
+            --log-file=/tmp/valgrind-output \
+            python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output
+
 .PHONY: readme
 readme:
 	python3 -c "import markdown2" > /dev/null 2>&1 || python3 -m pip install markdown2

Tests/helper.py

@@ -161,6 +161,12 @@ def assert_tuple_approx_equal(
             pytest.fail(msg + ": " + repr(actuals) + " != " + repr(targets))
 
 
+def timeout_unless_slower_valgrind(timeout: float) -> pytest.MarkDecorator:
+    if "PILLOW_VALGRIND_TEST" in os.environ:
+        return pytest.mark.pil_noop_mark()
+    return pytest.mark.timeout(timeout)
+
+
 def skip_unless_feature(feature: str) -> pytest.MarkDecorator:
     reason = f"{feature} not available"
     return pytest.mark.skipif(not features.check(feature), reason=reason)

Tests/oss-fuzz/python.supp

@@ -14,3 +14,23 @@
    fun:_TIFFReadEncodedTileAndAllocBuffer
    ...
 }
+
+{
+   <python_alloc_possible_leak>
+   Memcheck:Leak
+   match-leak-kinds: all
+   fun:malloc
+   fun:_PyMem_RawMalloc
+   fun:PyObject_Malloc
+   ...
+}
+
+{
+   <python_realloc_possible_leak>
+   Memcheck:Leak
+   match-leak-kinds: all
+   fun:malloc
+   fun:_PyMem_RawRealloc
+   fun:PyMem_Realloc
+   ...
+}

Tests/test_file_eps.py

@@ -15,6 +15,7 @@
     is_win32,
     mark_if_feature_version,
     skip_unless_feature,
+    timeout_unless_slower_valgrind,
 )
 
 HAS_GHOSTSCRIPT = EpsImagePlugin.has_ghostscript()
@@ -398,7 +399,7 @@ def test_emptyline() -> None:
     assert image.format == "EPS"
 
 
-@pytest.mark.timeout(timeout=5)
+@timeout_unless_slower_valgrind(5)
 @pytest.mark.parametrize(
     "test_file",
     ["Tests/images/eps/timeout-d675703545fee17acab56e5fec644c19979175de.eps"],

Tests/test_file_fli.py

@@ -7,7 +7,12 @@
 
 from PIL import FliImagePlugin, Image, ImageFile
 
-from .helper import assert_image_equal, assert_image_equal_tofile, is_pypy
+from .helper import (
+    assert_image_equal,
+    assert_image_equal_tofile,
+    is_pypy,
+    timeout_unless_slower_valgrind,
+)
 
 # created as an export of a palette image from Gimp2.6
 # save as...-> hopper.fli, default options.
@@ -189,7 +194,7 @@ def test_seek() -> None:
         "Tests/images/timeout-bff0a9dc7243a8e6ede2408d2ffa6a9964698b87.fli",
     ],
 )
-@pytest.mark.timeout(timeout=3)
+@timeout_unless_slower_valgrind(3)
 def test_timeouts(test_file: str) -> None:
     with open(test_file, "rb") as f:
         with Image.open(f) as im:

Tests/test_file_jpeg.py

@@ -32,6 +32,7 @@
     is_win32,
     mark_if_feature_version,
     skip_unless_feature,
+    timeout_unless_slower_valgrind,
 )
 
 ElementTree: ModuleType | None
@@ -1033,7 +1034,7 @@ def test_save_xmp(self, tmp_path: Path) -> None:
         with pytest.raises(ValueError):
             im.save(f, xmp=b"1" * 65505)
 
-    @pytest.mark.timeout(timeout=1)
+    @timeout_unless_slower_valgrind(1)
     def test_eof(self, monkeypatch: pytest.MonkeyPatch) -> None:
         # Even though this decoder never says that it is finished
         # the image should still end when there is no new data