Merge pull request #13859 from notatallshaw/conflict-priority-simple

Add conflict driven priority reordering when backtracking

Author: notatallshaw

docs/html/topics/more-dependency-resolution.md

@@ -161,10 +161,15 @@ Pip's current implementation of the provider implements
 * If Requires-Python is present only consider that
 * If there are causes of resolution conflict (backtrack causes) then
     only consider them until there are no longer any resolution conflicts
+* If any identifiers have appeared unresolved in backtrack causes at
+    least 5 times, only consider those so they get pinned before other
+    packages pick a version
 
 Pip's current implementation of the provider implements `get_preference`
 for known requirements with the following preferences in the following order:
 
+* Any requirement that has appeared in repeated conflicts (see
+    ``narrow_requirement_selection`` above).
 * Any requirement that is "direct", e.g., points to an explicit URL.
 * Any requirement that is "pinned", i.e., contains the operator ``===``
     or ``==`` without a wildcard.

news/13859.feature.rst

@@ -0,0 +1 @@
+Speed up dependency resolution when there are complex conflicts.

src/pip/_internal/resolution/resolvelib/provider.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import math
+from collections import defaultdict
 from collections.abc import Iterable, Iterator, Mapping, Sequence
 from functools import cache
 from typing import (
@@ -27,6 +28,8 @@
 else:
     _ProviderBase = AbstractProvider
 
+_CONFLICT_PRIORITY_THRESHOLD = 5
+
 # Notes on the relationship between the provider, the factory, and the
 # candidate and requirement classes.
 #
@@ -99,6 +102,8 @@ def __init__(
         self._ignore_dependencies = ignore_dependencies
         self._upgrade_strategy = upgrade_strategy
         self._user_requested = user_requested
+        self._conflict_counts: defaultdict[str, int] = defaultdict(int)
+        self._conflict_promoted: set[str] = set()
 
     @property
     def constraints(self) -> dict[str, Constraint]:
@@ -130,29 +135,42 @@ def narrow_requirement_selection(
               Further, the current backtrack causes likely need to be resolved
               before other requirements as a resolution can't be found while
               there is a conflict.
+            * Identifiers that repeatedly appear as not-yet-pinned in conflicts
+              get promoted so they are resolved earlier. This lets their
+              constraints take effect before other packages pick a version.
         """
         backtrack_identifiers = set()
         for info in backtrack_causes:
-            backtrack_identifiers.add(info.requirement.name)
+            names = [info.requirement.name]
             if info.parent is not None:
-                backtrack_identifiers.add(info.parent.name)
+                names.append(info.parent.name)
+            for name in names:
+                backtrack_identifiers.add(name)
+                if name not in resolutions:
+                    self._conflict_counts[name] += 1
+                    if self._conflict_counts[name] >= _CONFLICT_PRIORITY_THRESHOLD:
+                        self._conflict_promoted.add(name)
 
         current_backtrack_causes = []
+        promoted = []
         for identifier in identifiers:
-            # Requires-Python has only one candidate and the check is basically
-            # free, so we always do it first to avoid needless work if it fails.
-            # This skips calling get_preference() for all other identifiers.
             if identifier == REQUIRES_PYTHON_IDENTIFIER:
                 return [identifier]
 
-            # Check if this identifier is a backtrack cause
             if identifier in backtrack_identifiers:
                 current_backtrack_causes.append(identifier)
                 continue
 
+            if identifier in self._conflict_promoted:
+                promoted.append(identifier)
+                continue
+
         if current_backtrack_causes:
             return current_backtrack_causes
 
+        if promoted:
+            return promoted
+
         return identifiers
 
     def get_preference(
@@ -223,7 +241,10 @@ def get_preference(
         unfree = bool(operators)
         requested_order = self._user_requested.get(identifier, math.inf)
 
+        conflict_promoted = identifier in self._conflict_promoted
+
         return (
+            not conflict_promoted,
             not direct,
             not pinned,
             not upper_bounded,

tests/unit/resolution_resolvelib/test_provider.py

@@ -12,7 +12,10 @@
 from pip._internal.resolution.resolvelib.base import Candidate
 from pip._internal.resolution.resolvelib.candidates import REQUIRES_PYTHON_IDENTIFIER
 from pip._internal.resolution.resolvelib.factory import Factory
-from pip._internal.resolution.resolvelib.provider import PipProvider
+from pip._internal.resolution.resolvelib.provider import (
+    _CONFLICT_PRIORITY_THRESHOLD,
+    PipProvider,
+)
 from pip._internal.resolution.resolvelib.requirements import (
     ExplicitRequirement,
     SpecifierRequirement,
@@ -60,55 +63,55 @@ def build_explicit_req_info(
             {"pinned-package": [build_req_info("pinned-package==1.0")]},
             [],
             {},
-            (True, False, True, math.inf, False, "pinned-package"),
+            (True, True, False, True, math.inf, False, "pinned-package"),
         ),
         # Star-specified package, i.e. with "*"
         (
             "star-specified-package",
             {"star-specified-package": [build_req_info("star-specified-package==1.*")]},
             [],
             {},
-            (True, True, False, math.inf, False, "star-specified-package"),
+            (True, True, True, False, math.inf, False, "star-specified-package"),
         ),
         # Package that caused backtracking
         (
             "backtrack-package",
             {"backtrack-package": [build_req_info("backtrack-package")]},
             [build_req_info("backtrack-package")],
             {},
-            (True, True, True, math.inf, True, "backtrack-package"),
+            (True, True, True, True, math.inf, True, "backtrack-package"),
         ),
         # Root package requested by user
         (
             "root-package",
             {"root-package": [build_req_info("root-package")]},
             [],
             {"root-package": 1},
-            (True, True, True, 1, True, "root-package"),
+            (True, True, True, True, 1, True, "root-package"),
         ),
         # Unfree package (with specifier operator)
         (
             "unfree-package",
             {"unfree-package": [build_req_info("unfree-package!=1")]},
             [],
             {},
-            (True, True, True, math.inf, False, "unfree-package"),
+            (True, True, True, True, math.inf, False, "unfree-package"),
         ),
         # Free package (no operator)
         (
             "free-package",
             {"free-package": [build_req_info("free-package")]},
             [],
             {},
-            (True, True, True, math.inf, True, "free-package"),
+            (True, True, True, True, math.inf, True, "free-package"),
         ),
         # Test case for "direct" preference (explicit URL)
         (
             "direct-package",
             {"direct-package": [build_explicit_req_info("direct-package")]},
             [],
             {},
-            (False, True, True, math.inf, True, "direct-package"),
+            (True, False, True, True, math.inf, True, "direct-package"),
         ),
         # Upper bounded with <= operator
         (
@@ -120,15 +123,15 @@ def build_explicit_req_info(
             },
             [],
             {},
-            (True, True, False, math.inf, False, "upper-bound-lte-package"),
+            (True, True, True, False, math.inf, False, "upper-bound-lte-package"),
         ),
         # Upper bounded with < operator
         (
             "upper-bound-lt-package",
             {"upper-bound-lt-package": [build_req_info("upper-bound-lt-package<2.0")]},
             [],
             {},
-            (True, True, False, math.inf, False, "upper-bound-lt-package"),
+            (True, True, True, False, math.inf, False, "upper-bound-lt-package"),
         ),
         # Upper bounded with ~= operator
         (
@@ -140,15 +143,23 @@ def build_explicit_req_info(
             },
             [],
             {},
-            (True, True, False, math.inf, False, "upper-bound-compatible-package"),
+            (
+                True,
+                True,
+                True,
+                False,
+                math.inf,
+                False,
+                "upper-bound-compatible-package",
+            ),
         ),
         # Not upper bounded, using only >= operator
         (
             "lower-bound-package",
             {"lower-bound-package": [build_req_info("lower-bound-package>=1.0")]},
             [],
             {},
-            (True, True, True, math.inf, False, "lower-bound-package"),
+            (True, True, True, True, math.inf, False, "lower-bound-package"),
         ),
     ],
 )
@@ -225,7 +236,8 @@ def test_narrow_requirement_selection(
     """Test that narrow_requirement_selection correctly prioritizes identifiers:
     1. REQUIRES_PYTHON_IDENTIFIER (if present)
     2. Backtrack causes (if present)
-    3. All other identifiers (as-is)
+    3. Conflict-promoted identifiers (if present)
+    4. All other identifiers (as-is)
     """
     provider = PipProvider(
         factory=factory,
@@ -240,3 +252,41 @@ def test_narrow_requirement_selection(
     )
 
     assert list(result) == expected, f"Expected {expected}, got {list(result)}"
+
+
+def test_conflict_promotion_after_threshold(provider: PipProvider) -> None:
+    """Repeated unresolved backtrack causes get promoted after the threshold."""
+    narrow = provider.narrow_requirement_selection
+    cause = [build_req_info("conflict-pkg")]
+
+    # Below threshold: no promotion, all identifiers returned.
+    for i in range(1, _CONFLICT_PRIORITY_THRESHOLD):
+        result = list(narrow(["other-pkg"], {}, {}, {}, cause))
+        assert result == ["other-pkg"], f"Unexpected promotion at call {i}"
+
+    # At threshold: conflict-pkg is a backtrack cause so it wins on that basis.
+    result = list(narrow(["other-pkg", "conflict-pkg"], {}, {}, {}, cause))
+    assert result == ["conflict-pkg"]
+
+    # Without active backtrack causes, the promoted package is still preferred.
+    result = list(narrow(["other-pkg", "conflict-pkg"], {}, {}, {}, []))
+    assert result == ["conflict-pkg"]
+
+    # Backtrack causes still win over promoted-only packages.
+    other_cause = [build_req_info("other-pkg")]
+    result = list(narrow(["other-pkg", "conflict-pkg"], {}, {}, {}, other_cause))
+    assert result == ["other-pkg"]
+
+
+def test_conflict_promoted_get_preference(provider: PipProvider) -> None:
+    """Promoted packages sort before non-promoted in get_preference."""
+    provider._conflict_promoted.add("promoted-pkg")
+
+    info = {
+        "promoted-pkg": [build_req_info("promoted-pkg")],
+        "normal-pkg": [build_req_info("normal-pkg")],
+    }
+    pref = provider.get_preference("promoted-pkg", {}, {}, info, [])
+    pref_other = provider.get_preference("normal-pkg", {}, {}, info, [])
+
+    assert pref < pref_other