Vendor pypa/packaging into micropip

.github/workflows/main.yml

@@ -28,6 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:
@@ -88,6 +90,7 @@ jobs:
       - uses: actions/checkout@cbb722410c2e876e24abbe8de2cc27693e501dcb # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
 
       - id: check-integration-test-trigger
         name: Check integration test trigger
@@ -110,6 +113,8 @@ jobs:
     environment: PyPi-deploy
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.github/workflows/remote_package_index_test.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.gitmodules

@@ -0,0 +1,5 @@
+# d8e3b31b734926ebbcaff654279f6855a73e052f, for 24.2 release
+# https://github.com/pypa/packaging/releases/tag/24.2
+[submodule "micropip/_vendored/packaging"]
+	path = micropip/_vendored/packaging
+	url = https://github.com/pypa/packaging/

CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (the ones that starts with `../` or `./`)
   [#174](https://github.com/pyodide/micropip/pull/174)
 
+### Added
+
+- `micropip` now vendors `pypa/packaging` for better reliability.
+  [#178](https://github.com/pyodide/micropip/pull/178)
+
 ## [0.8.0] - 2024/12/15
 
 ### Added

micropip/_compat/_compat_not_in_pyodide.py

@@ -14,6 +14,7 @@
 class CompatibilityNotInPyodide(CompatibilityLayer):
 
     # Vendored from packaging
+    # TODO: use packaging APIs here instead?
     _canonicalize_regex = re.compile(r"[-_.]+")
 
     class HttpStatusError(Exception):

micropip/_utils.py

@@ -4,14 +4,19 @@
 from pathlib import Path
 from sysconfig import get_config_var, get_platform
 
-from packaging.requirements import Requirement
-from packaging.tags import Tag
-from packaging.tags import sys_tags as sys_tags_orig
-from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
-from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig
-from packaging.version import InvalidVersion, Version
-
 from ._compat import REPODATA_PACKAGES
+from ._vendored.packaging.requirements import Requirement
+from ._vendored.packaging.tags import Tag
+from ._vendored.packaging.tags import sys_tags as sys_tags_orig
+from ._vendored.packaging.utils import (
+    BuildTag,
+    InvalidWheelFilename,
+    canonicalize_name,
+)
+from ._vendored.packaging.utils import (
+    parse_wheel_filename as parse_wheel_filename_orig,
+)
+from ._vendored.packaging.version import InvalidVersion, Version
 
 
 def get_dist_info(dist: Distribution) -> Path:

micropip/_vendored/__init__.py

@@ -0,0 +1,108 @@
+# micropip/_vendored/__init__.py
+
+# This is a proxy file that redirects imports from micropip._vendored.packaging
+# to the actual packaging API present in the packaging/src/packaging directory
+# next to this file.
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+PACKAGING_PATH = Path(__file__).parent / "packaging" / "src" / "packaging"
+
+
+def _create_module(name, package_path=None):
+    """
+    Creates a module object for the given name and makes it available both under
+    micropip._vendored.packaging and packaging namespaces.
+
+    Args:
+        name: The name of the module (without the full package path)
+        package_path: Optional path to the module file, if different from default location
+    """
+    vendored_name = f"micropip._vendored.packaging.{name}"
+    direct_name = f"packaging.{name}"
+
+    # If the module is already in sys.modules, return it, and we
+    # add it to sys.modules under both names before executing it.
+    if vendored_name in sys.modules:
+        return sys.modules[vendored_name]
+
+    module = types.ModuleType(vendored_name)
+    module.__package__ = "micropip._vendored.packaging"
+
+    sys.modules[vendored_name] = module
+    sys.modules[direct_name] = module
+
+    if package_path is None:
+        module_path = PACKAGING_PATH / f"{name}.py"
+    else:
+        module_path = package_path
+
+    if module_path.exists():
+        spec = importlib.util.spec_from_file_location(
+            vendored_name, module_path, submodule_search_locations=[str(PACKAGING_PATH)]
+        )
+        module.__spec__ = spec
+        module.__file__ = str(module_path)
+        loader = spec.loader
+        loader.exec_module(module)
+
+    return module
+
+
+####################################################
+
+packaging_vendored = types.ModuleType("micropip._vendored.packaging")
+packaging_direct = types.ModuleType(
+    "packaging"
+)  # this is where we redirect the imports.
+
+packaging_vendored.__path__ = [str(PACKAGING_PATH)]
+packaging_vendored.__package__ = "micropip._vendored"
+packaging_direct.__path__ = [str(PACKAGING_PATH)]
+packaging_direct.__package__ = ""
+
+sys.modules["micropip._vendored.packaging"] = packaging_vendored
+sys.modules["packaging"] = packaging_direct
+
+####################################################
+
+# 1. First, handle any packages: these are directories with __init__.py.
+# 2. Then, we load all the internal modules
+# 3. Finally, we'll load all the regular modules (whatever is in the
+# public API)
+#
+# While rudimentary, this order is important because the internal modules
+# may depend on subpackages, and regular modules may depend on internal ones.
+#
+# For example, the metadata.py module imports from licenses, requirements,
+# specifiers, and utils.
+#
+# Similarly, tags.py needs _manylinux and _musllinux to be available.
+
+
+for path in PACKAGING_PATH.glob("*/__init__.py"):
+    package_name = path.parent.name
+    module = _create_module(f"{package_name}/__init__", path)
+    setattr(packaging_vendored, package_name, module)
+    setattr(packaging_direct, package_name, module)
+
+internal_modules = [path.stem for path in PACKAGING_PATH.glob("_*.py")]
+for name in internal_modules:
+    module = _create_module(name)
+    setattr(packaging_vendored, name, module)
+    setattr(packaging_direct, name, module)
+
+
+for path in PACKAGING_PATH.glob("*.py"):
+    if path.stem == "__init__" or path.stem.startswith("_"):
+        continue
+
+    module = _create_module(path.stem)
+    setattr(packaging_vendored, path.stem, module)
+    setattr(packaging_direct, path.stem, module)
+
+
+globals()["packaging"] = packaging_vendored

micropip/externals/mousebender/simple.py

@@ -7,13 +7,12 @@
 import warnings
 from typing import Any, Dict, List, Optional, Union, Literal, TypeAlias, TypedDict
 
-import packaging.utils
+import micropip._vendored.packaging.utils as packaging_utils
 
 
 ACCEPT_JSON_V1 = "application/vnd.pypi.simple.v1+json"
 
 
-
 class UnsupportedAPIVersion(Exception):
     """The major version of an API response is not supported."""
 
@@ -92,15 +91,15 @@ class ProjectDetails_1_0(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`691`)."""
 
     meta: _Meta_1_0
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_0]
 
 
 class ProjectDetails_1_1(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`700`)."""
 
     meta: _Meta_1_1
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_1]
     # PEP 700
     versions: List[str]
@@ -235,6 +234,6 @@ def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0:
         files.append(details)
     return {
         "meta": {"api-version": "1.0"},
-        "name": packaging.utils.canonicalize_name(name),
+        "name": packaging_utils.canonicalize_name(name),
         "files": files,
     }

micropip/freeze.py

@@ -4,9 +4,8 @@
 from copy import deepcopy
 from typing import Any
 
-from packaging.utils import canonicalize_name
-
 from ._utils import fix_package_dependencies
+from ._vendored.packaging.utils import canonicalize_name
 
 
 def freeze_lockfile(

micropip/install.py

@@ -4,9 +4,8 @@
 from pathlib import Path
 from typing import Any
 
-from packaging.markers import default_environment
-
 from ._compat import loadPackage, to_js
+from ._vendored.packaging.markers import default_environment
 from .constants import FAQ_URLS
 from .logging import setup_logging
 from .transaction import Transaction

micropip/metadata.py

@@ -7,8 +7,8 @@
 from collections.abc import Iterable
 from pathlib import Path
 
-from packaging.requirements import Requirement
-from packaging.utils import canonicalize_name
+from ._vendored.packaging.requirements import Requirement
+from ._vendored.packaging.utils import canonicalize_name
 
 
 def safe_name(name):

micropip/package.py

@@ -3,7 +3,7 @@
 from dataclasses import astuple, dataclass
 from typing import Any
 
-from packaging.utils import canonicalize_name
+from ._vendored.packaging.utils import canonicalize_name
 
 __all__ = ["PackageDict"]
 

micropip/package_index.py

@@ -9,11 +9,10 @@
 from typing import Any
 from urllib.parse import urljoin, urlparse, urlunparse
 
-from packaging.utils import InvalidWheelFilename
-from packaging.version import InvalidVersion, Version
-
 from ._compat import HttpStatusError, fetch_string_and_headers
 from ._utils import is_package_compatible, parse_version
+from ._vendored.packaging.utils import InvalidWheelFilename
+from ._vendored.packaging.version import InvalidVersion, Version
 from .externals.mousebender.simple import from_project_details_html
 from .types import DistributionMetadata
 from .wheelinfo import WheelInfo

micropip/transaction.py

@@ -6,12 +6,11 @@
 from importlib.metadata import PackageNotFoundError
 from urllib.parse import urlparse
 
-from packaging.requirements import Requirement
-from packaging.utils import canonicalize_name
-
 from . import package_index
 from ._compat import REPODATA_PACKAGES
 from ._utils import best_compatible_tag_index, check_compatible
+from ._vendored.packaging.requirements import Requirement
+from ._vendored.packaging.utils import canonicalize_name
 from .constants import FAQ_URLS
 from .package import PackageMetadata
 from .package_index import ProjectInfo

micropip/wheelinfo.py

@@ -7,17 +7,16 @@
 from typing import Any, Literal
 from urllib.parse import ParseResult, urlparse
 
-from packaging.requirements import Requirement
-from packaging.tags import Tag
-from packaging.version import Version
-
 from ._compat import (
     fetch_bytes,
     get_dynlibs,
     loadDynlibsFromPackage,
     loadedPackages,
 )
 from ._utils import parse_wheel_filename
+from ._vendored.packaging.requirements import Requirement
+from ._vendored.packaging.tags import Tag
+from ._vendored.packaging.version import Version
 from .metadata import Metadata, safe_name, wheel_dist_info_dir
 from .types import DistributionMetadata
 

pyproject.toml

@@ -13,7 +13,8 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dynamic = ["version"]
-dependencies = ["packaging>=23.0"]
+dependencies = []
+
 [project.optional-dependencies]
 test = [
   "pytest-httpserver",
@@ -64,7 +65,12 @@ known-first-party = [
 ]
 
 [tool.mypy]
+exclude = ["micropip/_vendored/"]
 python_version = "3.12"
 show_error_codes = true
 warn_unreachable = true
 ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "micropip._vendored.*"
+warn_unreachable = false

tests/conftest.py

@@ -10,10 +10,11 @@
 from typing import Any
 
 import pytest
-from packaging.utils import parse_wheel_filename
 from pytest_httpserver import HTTPServer
 from pytest_pyodide import spawn_web_server
 
+from micropip._vendored.packaging.utils import parse_wheel_filename
+
 
 def pytest_addoption(parser):
     parser.addoption(

tests/test_install.py

@@ -1,9 +1,9 @@
 import pytest
 from conftest import mock_fetch_cls
-from packaging.utils import parse_wheel_filename
 from pytest_pyodide import run_in_pyodide
 
 import micropip
+from micropip._vendored.packaging.utils import parse_wheel_filename
 
 
 def test_install_custom_url(selenium_standalone_micropip, wheel_catalog):