BUG: to_parquet (pyarrow) opens local path twice

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mkviatkovskii

doc/source/whatsnew/v3.1.0.rst

@@ -360,6 +360,7 @@ I/O
 - Fixed bug in :meth:`HDFStore.select` where passing ``where`` as a list of conditions referencing caller-scope variables failed on Python 3.12+ due to :pep:`709` inlining list comprehension stack frames (:issue:`64881`)
 - Storing a :class:`DataFrame` or :class:`Series` with a :class:`MultiIndex` level named ``'index'`` via :meth:`HDFStore.put` or :meth:`HDFStore.append` with ``format='table'`` now raises a clear ``ValueError`` instead of an opaque reshape error (:issue:`6208`)
 - Writing a :class:`DataFrame` with ``format='table'`` and a column named ``'index'`` as a ``data_columns`` entry (including ``data_columns=True``) now raises a clear ``ValueError`` instead of an opaque reshape error (:issue:`41437`)
+- Fixed bug in :meth:`DataFrame.to_parquet` (``pyarrow`` engine) where a local file path was opened twice, once by pandas and again by pyarrow, wasting a syscall and silently truncating output to 0 bytes on filesystems that finalize a file's contents on close (:issue:`65810`)
 
 Period
 ^^^^^^

pandas/io/parquet.py

@@ -33,6 +33,7 @@
 from pandas.io._util import arrow_table_to_pandas
 from pandas.io.common import (
     IOHandles,
+    check_parent_directory,
     get_handle,
     is_fsspec_url,
     is_url,
@@ -142,24 +143,26 @@ def _get_path_or_handle(
         and isinstance(path_or_handle, str)
         and not os.path.isdir(path_or_handle)
     ):
-        # use get_handle only when we are very certain that it is not a directory
-        # fsspec resources can also point to directories
-        # this branch is used for example when reading from non-fsspec URLs
-        handles = get_handle(
-            path_or_handle, mode, is_text=False, storage_options=storage_options
-        )
-        fs = None
-        path_or_handle = handles.handle
-        if hasattr(path_or_handle, "name") and isinstance(
-            path_or_handle.name, (str, bytes)
-        ):
-            # Unwrap the Python file handle back to a string path so that
-            # PyArrow can use memory-mapped and multithreaded C++ I/O
-            # instead of going through the Python I/O layer. GH#47702
-            if isinstance(path_or_handle.name, bytes):
-                path_or_handle = path_or_handle.name.decode()
-            else:
-                path_or_handle = path_or_handle.name
+        if is_url(path_or_handle):
+            # pyarrow cannot read non-fsspec URLs (e.g. http/https), so let
+            # get_handle download them into a buffer for pyarrow to consume.
+            handles = get_handle(
+                path_or_handle, mode, is_text=False, storage_options=storage_options
+            )
+            path_or_handle = handles.handle
+        else:
+            # Local path: hand the string to pyarrow so it can use memory-mapped,
+            # multithreaded C++ I/O rather than the Python I/O layer (GH#47702).
+            # Do not open it via get_handle as well: pyarrow opens the path
+            # itself, so going through get_handle too would open the file twice.
+            # That wastes a syscall on POSIX and, on filesystems that finalize a
+            # file's contents on close, lets the empty pandas-side descriptor
+            # close last and clobber pyarrow's data to 0 bytes. get_handle would
+            # also expand "~" and check the parent directory on write, so
+            # reproduce both below to keep behavior unchanged.
+            path_or_handle = os.path.expanduser(path_or_handle)
+            if "w" in mode or "a" in mode or "x" in mode:
+                check_parent_directory(path_or_handle)
     return path_or_handle, handles, fs
 
 

pandas/tests/io/test_parquet.py

@@ -1255,6 +1255,65 @@ def test_maps_as_pydicts(self, pa, temp_file):
             read_kwargs={"to_pandas_kwargs": {"maps_as_pydicts": "strict"}},
         )
 
+    def test_to_parquet_local_path_does_not_call_get_handle(
+        self, pa, temp_file, monkeypatch
+    ):
+        # GH#65810 local paths are handed to pyarrow directly; get_handle used
+        # to open them only to unwrap the name back to a string, opening the
+        # path a second time and truncating output to 0 bytes on filesystems
+        # that finalize contents on close
+        def fail(*args, **kwargs):
+            pytest.fail("get_handle should not be called for a local path")
+
+        monkeypatch.setattr("pandas.io.parquet.get_handle", fail)
+
+        df = pd.DataFrame({"a": [1, 2, 3]})
+        df.to_parquet(temp_file, engine=pa)
+
+    def test_to_parquet_local_path_opens_destination_once(
+        self, pa, temp_file, monkeypatch
+    ):
+        # GH#65810 pandas must not open the destination itself; pyarrow opens it
+        # via C++ (bypassing builtins.open), so no Python-level open is expected
+        opens = []
+        real_open = open
+        target = os.fspath(temp_file)
+
+        def spy_open(file, *args, **kwargs):
+            if os.fspath(file) == target:
+                opens.append(file)
+            return real_open(file, *args, **kwargs)
+
+        monkeypatch.setattr("builtins.open", spy_open)
+
+        df = pd.DataFrame({"a": [1, 2, 3]})
+        df.to_parquet(temp_file, engine=pa)
+        assert opens == []
+
+    def test_to_parquet_missing_parent_directory_raises(self, pa, tmp_path):
+        # GH#65810 skipping get_handle must keep its parent-directory check
+        df = pd.DataFrame({"a": [1, 2, 3]})
+        path = tmp_path / "missing" / "out.parquet"
+        msg = "Cannot save file into a non-existent directory"
+        with pytest.raises(OSError, match=msg):
+            df.to_parquet(path, engine=pa)
+
+    def test_read_parquet_url_still_uses_get_handle(self, pa, monkeypatch):
+        # GH#65810 only local paths skip get_handle; non-fsspec URLs must still
+        # be routed through it because pyarrow cannot fetch them
+        url = "http://example.com/nonexistent.parquet"
+        calls = []
+
+        def stub(path, mode, **kwargs):
+            calls.append((path, mode))
+            raise ValueError("reached get_handle")
+
+        monkeypatch.setattr("pandas.io.parquet.get_handle", stub)
+
+        with pytest.raises(ValueError, match="reached get_handle"):
+            read_parquet(url, engine=pa)
+        assert calls == [(url, "rb")]
+
 
 class TestParquetFastParquet(Base):
     def test_basic(self, fp, df_full, request, temp_file):