Allow file: for dependencies and optional-dependencies in pyproject.toml

Author: akx

changelog.d/3255.change.rst

@@ -0,0 +1 @@
+Enabled using ``file:`` for dependencies and optional-dependencies in pyproject.toml -- by :user:`akx`

docs/userguide/pyproject_config.rst

@@ -181,16 +181,28 @@ In the ``dynamic`` table, the ``attr`` directive [#directives]_ will read an
 attribute from the given module [#attr]_, while ``file`` will read the contents
 of all given files and concatenate them in a single string.
 
-================= =================== =========================
-Key               Directive           Notes
-================= =================== =========================
-``version``       ``attr``, ``file``
-``readme``        ``file``
-``description``   ``file``            One-line text
-``classifiers``   ``file``            Multi-line text with one classifier per line
-``entry-points``  ``file``            INI format following :doc:`PyPUG:specifications/entry-points`
-                                      (``console_scripts`` and ``gui_scripts`` can be included)
-================= =================== =========================
+=================          =================== =========================
+Key                        Directive           Notes
+=================          =================== =========================
+``version``                ``attr``, ``file``
+``readme``                 ``file``
+``description``            ``file``            One-line text
+``classifiers``            ``file``            Multi-line text with one classifier per line
+``entry-points``           ``file``            INI format following :doc:`PyPUG:specifications/entry-points`
+                                               (``console_scripts`` and ``gui_scripts`` can be included)
+``dependencies``           ``file``            ``requirements.txt`` format (``#`` comments and blank lines excluded)
+``optional-dependencies``  ``file``            ``requirements.txt`` format per group (``#`` comments and blank lines excluded)
+========================== =================== =========================
+
+Supporting ``file`` for dependencies is meant for a convenience for packaging
+applications with possibly strictly versioned dependencies.
+
+Library packagers are discouraged from using overly strict (or "locked")
+dependency versions in their ``dependencies`` and ``optional-dependencies``.
+
+Currently, when specifying ``optional-dependencies`` dynamically, all of the groups
+must be specified dynamically; one can not specify some of them statically and
+some of them dynamically.
 
 ----
 

setuptools/config/_validate_pyproject/fastjsonschema_validations.py

@@ -266,6 +266,8 @@ def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, st
             "scripts",
             "gui-scripts",
             "classifiers",
+            "dependencies",
+            "optional-dependencies",
         )
         # `_obtain` functions are assumed to raise appropriate exceptions/warnings.
         obtained_dynamic = {
@@ -278,6 +280,8 @@ def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, st
             version=self._obtain_version(dist, package_dir),
             readme=self._obtain_readme(dist),
             classifiers=self._obtain_classifiers(dist),
+            dependencies=self._obtain_dependencies(dist),
+            optional_dependencies=self._obtain_optional_dependencies(dist),
         )
         # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value
         # might have already been set by setup.py/extensions, so avoid overwriting.
@@ -294,18 +298,25 @@ def _ensure_previously_set(self, dist: "Distribution", field: str):
             )
             raise OptionError(msg)
 
+    def _expand_directive(
+        self, specifier: str, directive, package_dir: Mapping[str, str]
+    ):
+        with _ignore_errors(self.ignore_option_errors):
+            root_dir = self.root_dir
+            if "file" in directive:
+                return _expand.read_files(directive["file"], root_dir)
+            if "attr" in directive:
+                return _expand.read_attr(directive["attr"], package_dir, root_dir)
+            raise ValueError(f"invalid `{specifier}`: {directive!r}")
+        return None
+
     def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]):
         if field in self.dynamic_cfg:
-            directive = self.dynamic_cfg[field]
-            with _ignore_errors(self.ignore_option_errors):
-                root_dir = self.root_dir
-                if "file" in directive:
-                    return _expand.read_files(directive["file"], root_dir)
-                if "attr" in directive:
-                    return _expand.read_attr(directive["attr"], package_dir, root_dir)
-                msg = f"invalid `tool.setuptools.dynamic.{field}`: {directive!r}"
-                raise ValueError(msg)
-            return None
+            return self._expand_directive(
+                f"tool.setuptools.dynamic.{field}",
+                self.dynamic_cfg[field],
+                package_dir,
+            )
         self._ensure_previously_set(dist, field)
         return None
 
@@ -365,6 +376,38 @@ def _obtain_classifiers(self, dist: "Distribution"):
                 return value.splitlines()
         return None
 
+    def _obtain_dependencies(self, dist: "Distribution"):
+        if "dependencies" in self.dynamic:
+            value = self._obtain(dist, "dependencies", {})
+            if value:
+                return _parse_requirements_list(value)
+        return None
+
+    def _obtain_optional_dependencies(self, dist: "Distribution"):
+        if "optional-dependencies" not in self.dynamic:
+            return None
+        if "optional-dependencies" in self.dynamic_cfg:
+            optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]
+            assert isinstance(optional_dependencies_map, dict)
+            return {
+                group: _parse_requirements_list(self._expand_directive(
+                    f"tool.setuptools.dynamic.optional-dependencies.{group}",
+                    directive,
+                    {},
+                ))
+                for group, directive in optional_dependencies_map.items()
+            }
+        self._ensure_previously_set(dist, "optional-dependencies")
+        return None
+
+
+def _parse_requirements_list(value):
+    return [
+        line
+        for line in value.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+
 
 @contextmanager
 def _ignore_errors(ignore_option_errors: bool):

setuptools/config/pyprojecttoml.py

@@ -0,0 +1,82 @@
+import pytest
+
+from setuptools.config.pyprojecttoml import apply_configuration
+from setuptools.dist import Distribution
+from setuptools.tests.textwrap import DALS
+
+
+def test_dynamic_dependencies(tmp_path):
+    (tmp_path / "requirements.txt").write_text("six\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["dependencies"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+
+    [tool.setuptools.dynamic.dependencies]
+    file = ["requirements.txt"]
+    """))
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject)
+    assert dist.install_requires == ["six"]
+
+
+def test_dynamic_optional_dependencies(tmp_path):
+    (tmp_path / "requirements-docs.txt").write_text("sphinx\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["optional-dependencies"]
+
+    [tool.setuptools.dynamic.optional-dependencies.docs]
+    file = ["requirements-docs.txt"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+    """))
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject)
+    assert dist.extras_require == {"docs": ["sphinx"]}
+
+
+def test_mixed_dynamic_optional_dependencies(tmp_path):
+    """
+    Test that if PEP 621 was loosened to allow mixing of dynamic and static
+    configurations in the case of fields containing sub-fields (groups),
+    things would work out.
+    """
+    (tmp_path / "requirements-images.txt").write_text("pillow~=42.0\n  # comment\n")
+    pyproject = (tmp_path / "pyproject.toml")
+    pyproject.write_text(DALS("""
+    [project]
+    name = "myproj"
+    version = "1.0"
+    dynamic = ["optional-dependencies"]
+
+    [project.optional-dependencies]
+    docs = ["sphinx"]
+
+    [tool.setuptools.dynamic.optional-dependencies.images]
+    file = ["requirements-images.txt"]
+
+    [build-system]
+    requires = ["setuptools", "wheel"]
+    build-backend = "setuptools.build_meta"
+    """))
+    # Test that the mix-and-match doesn't currently validate.
+    with pytest.raises(ValueError, match="project.optional-dependencies"):
+        apply_configuration(Distribution(), pyproject)
+
+    # Explicitly disable the validation and try again, to see that the mix-and-match
+    # result would be correct.
+    dist = Distribution()
+    dist = apply_configuration(dist, pyproject, ignore_option_errors=True)
+    assert dist.extras_require == {"docs": ["sphinx"], "images": ["pillow~=42.0"]}