Turn PipDeprecationWarning into errors when running tests (#13947)

So these don't go undetected in our test suite.

Author: danielhollas

pyproject.toml

@@ -330,6 +330,9 @@ markers = [
     "git: VCS: git",
     "search: tests for 'pip search'",
 ]
+filterwarnings = [
+  'error::pip._internal.utils.deprecation.PipDeprecationWarning:',
+]
 
 ######################################################################################
 # coverage

src/pip/_internal/utils/deprecation.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import warnings
 from typing import Any, TextIO
 
@@ -48,7 +49,11 @@ def _showwarning(
 
 def install_warning_logger() -> None:
     # Enable our Deprecation Warnings
-    warnings.simplefilter("default", PipDeprecationWarning, append=True)
+    # If we're running pip test suite, promote the PipDeprecationWarning into errors.
+    if os.environ.get("_PIP_TEST_ENV", None):
+        warnings.simplefilter("error", PipDeprecationWarning)
+    else:
+        warnings.simplefilter("default", PipDeprecationWarning, append=True)
 
     global _original_showwarning
 

tests/functional/test_install.py

@@ -347,6 +347,8 @@ def _patched_get_environment(lib_locations):
             )
         """))
 
+    # Make sure PipDeprecationWarnings don't turn into errors
+    script.environ["_PIP_TEST_ENV"] = ""
     result = script.run(
         "python", str(runner), str(wheel_path.parent), expect_stderr=True
     )

tests/functional/test_install_reqs.py

@@ -629,6 +629,8 @@ def test_double_install_spurious_hash_mismatch(
 def test_install_with_extras_from_constraints(
     script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant
 ) -> None:
+    # Make sure PipDeprecationWarnings don't turn into errors
+    script.environ["_PIP_TEST_ENV"] = ""
     to_install = data.packages.joinpath("LocalExtras")
     file = script.temporary_file(
         "constraints.txt", f"LocalExtras[bar] @ {to_install.as_uri()}"
@@ -696,6 +698,8 @@ def test_install_with_extras_and_url_constraint(
 def test_install_with_extras_joined(
     script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant
 ) -> None:
+    # Make sure PipDeprecationWarnings don't turn into errors
+    script.environ["_PIP_TEST_ENV"] = ""
     to_install = data.packages.joinpath("LocalExtras")
     file = script.temporary_file(
         "constraints.txt", f"LocalExtras[bar] @ {to_install.as_uri()}"
@@ -762,6 +766,8 @@ def test_install_distribution_union_with_constraints(
     data: TestData,
     resolver_variant: ResolverVariant,
 ) -> None:
+    # Make sure PipDeprecationWarnings don't turn into errors
+    script.environ["_PIP_TEST_ENV"] = ""
     to_install = data.packages.joinpath("LocalExtras")
     script.scratch_path.joinpath("constraints.txt").write_text(f"{to_install}[bar]")
     result = script.pip_install_local(

tests/functional/test_new_resolver.py

@@ -733,6 +733,8 @@ def test_new_resolver_constraint_no_specifier(script: PipTestEnvironment) -> Non
 def test_new_resolver_constraint_reject_invalid(
     script: PipTestEnvironment, constraint: str, error: str
 ) -> None:
+    # Make sure PipDeprecationWarnings don't turn into errors
+    script.environ["_PIP_TEST_ENV"] = ""
     create_basic_wheel_for_package(script, "pkg", "1.0")
     constraints_file = script.scratch_path / "constraints.txt"
     constraints_file.write_text(constraint)

tests/functional/test_warning.py

@@ -25,15 +25,28 @@ def warnings_demo(tmpdir: Path) -> Path:
 def test_deprecation_warnings_are_correct(
     script: PipTestEnvironment, warnings_demo: Path
 ) -> None:
+    script.environ["_PIP_TEST_ENV"] = ""
     result = script.run("python", os.fspath(warnings_demo), expect_stderr=True)
     expected = "WARNING:pip._internal.deprecations:DEPRECATION: deprecated!\n"
     assert result.stderr == expected
 
 
+def test_deprecation_warnings_turn_into_errors_in_tests(
+    script: PipTestEnvironment, warnings_demo: Path
+) -> None:
+    result = script.run(
+        "python", os.fspath(warnings_demo), expect_error=True, expect_stderr=True
+    )
+    assert result.stderr.startswith("Traceback")
+    expected = "utils.deprecation.PipDeprecationWarning: DEPRECATION: deprecated!\n"
+    assert result.stderr.endswith(expected)
+
+
 def test_deprecation_warnings_can_be_silenced(
     script: PipTestEnvironment, warnings_demo: Path
 ) -> None:
     script.environ["PYTHONWARNINGS"] = "ignore"
+    script.environ["_PIP_TEST_ENV"] = ""
     result = script.run("python", os.fspath(warnings_demo))
     assert result.stderr == ""
 

tests/lib/__init__.py

@@ -537,6 +537,8 @@ def __init__(
         environ["PYTHONDONTWRITEBYTECODE"] = "1"
         # Make sure we get UTF-8 on output, even on Windows...
         environ["PYTHONIOENCODING"] = "UTF-8"
+        # Custom env flag so pip knows it's running in test environment
+        environ["_PIP_TEST_ENV"] = "1"
 
         # Whether all pip invocations should expect stderr
         # (useful for Python version deprecation)