fix: support editable installs of Cython pxd files (#516)

Editable installs of packages containing Cython currently do not
properly support downstream Cython packages finding the pxd files from
the editable install. The issue is that editable installs are managed
using a meta path finder, and [Cython does not appear to support that
for pxd files](cython/cython#5855). However,
Cython does support pth files adding include paths directly, so this PR
solves the pxd search issue by adding the necessary paths directly to
the pth file that currently adds the meta path finder.

Resolves #442

---------

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Henry Schreiner <[email protected]>

Author: 3 people

src/scikit_build_core/build/wheel.py

@@ -29,7 +29,7 @@
 from .generate import generate_file_contents
 
 if TYPE_CHECKING:
-    from collections.abc import Sequence
+    from collections.abc import Iterable, Sequence
 
     from ..settings.skbuild_model import ScikitBuildSettings
 
@@ -50,6 +50,7 @@ def _make_editable(
     reload_dir: Path | None,
     settings: ScikitBuildSettings,
     wheel: WheelWriter,
+    packages: Iterable[str],
 ) -> None:
     modules = mapping_to_modules(mapping, libdir)
     installed = libdir_to_installed(libdir)
@@ -68,9 +69,14 @@ def _make_editable(
         f"_{name}_editable.py",
         editable_txt.encode(),
     )
+    # Support Cython by adding the source directory directly to the path.
+    # This is necessary because Cython does not support sys.meta_path for
+    # cimports (as of 3.0.5).
+    import_strings = [f"import _{name}_editable", *packages, ""]
+    pth_import_paths = "\n".join(import_strings)
     wheel.writestr(
         f"_{name}_editable.pth",
-        f"import _{name}_editable\n".encode(),
+        pth_import_paths.encode(),
     )
 
 
@@ -341,6 +347,7 @@ def _build_wheel_impl(
         ) as wheel:
             wheel.build(wheel_dirs)
 
+            str_pkgs = (str(Path.cwd().joinpath(p).parent.resolve()) for p in packages)
             if editable and settings.editable.mode == "redirect":
                 reload_dir = build_dir.resolve() if settings.build_dir else None
 
@@ -353,15 +360,13 @@ def _build_wheel_impl(
                     settings=settings,
                     wheel=wheel,
                     name=normalized_name,
+                    packages=str_pkgs,
                 )
             elif editable and settings.editable.mode == "inplace":
                 if not packages:
                     msg = "Editable inplace mode requires at least one package"
                     raise AssertionError(msg)
 
-                str_pkgs = (
-                    str(Path.cwd().joinpath(p).parent.resolve()) for p in packages
-                )
                 wheel.writestr(
                     f"_{normalized_name}_editable.pth",
                     "\n".join(str_pkgs).encode(),

tests/conftest.py

@@ -50,6 +50,7 @@ def pep518_wheelhouse(tmp_path_factory: pytest.TempPathFactory) -> Path:
     )
     packages = [
         "build",
+        "cython",
         "hatchling",
         "pip>=23",
         "pybind11",

tests/packages/cython_pxd_editable/pkg1/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15)
+project(${SKBUILD_PROJECT_NAME} LANGUAGES C)
+
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module
+  REQUIRED)
+
+find_program(CYTHON "cython")
+
+add_custom_command(
+  OUTPUT src/pkg1/one.c
+  DEPENDS src/pkg1/one.pyx
+  VERBATIM
+  COMMAND "${CYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/src/pkg1/one.pyx"
+          --output-file "${CMAKE_CURRENT_BINARY_DIR}/src/pkg1/one.c")
+
+python_add_library(one MODULE "${CMAKE_CURRENT_BINARY_DIR}/src/pkg1/one.c"
+                   WITH_SOABI)
+
+install(TARGETS one DESTINATION pkg1/)

tests/packages/cython_pxd_editable/pkg1/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cython>=3.0.0",
+    "scikit-build-core",
+]
+
+[project]
+name = "pkg1"
+version = "1.0.0"
+
+[tool.scikit-build]
+wheel.packages = ["src/pkg1"]

tests/packages/cython_pxd_editable/pkg1/src/pkg1/__init__.py

@@ -0,0 +1 @@
+cdef int one()

tests/packages/cython_pxd_editable/pkg1/src/pkg1/one.pxd

@@ -0,0 +1,2 @@
+cdef int one():
+    return 1

tests/packages/cython_pxd_editable/pkg1/src/pkg1/one.pyx

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15)
+project(${SKBUILD_PROJECT_NAME} LANGUAGES C)
+
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module
+  REQUIRED)
+
+find_program(CYTHON "cython")
+
+add_custom_command(
+  OUTPUT src/pkg2/two.c
+  DEPENDS src/pkg2/two.pyx
+  VERBATIM
+  COMMAND "${CYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/src/pkg2/two.pyx"
+          --output-file "${CMAKE_CURRENT_BINARY_DIR}/src/pkg2/two.c")
+
+python_add_library(two MODULE "${CMAKE_CURRENT_BINARY_DIR}/src/pkg2/two.c"
+                   WITH_SOABI)
+
+install(TARGETS two DESTINATION pkg2/)

tests/packages/cython_pxd_editable/pkg2/CMakeLists.txt

@@ -0,0 +1,13 @@
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cython>=3.0.0",
+    "scikit-build-core",
+]
+
+[project]
+name = "pkg2"
+version = "1.0.0"
+
+[tool.scikit-build]
+wheel.packages = ["src/pkg2"]