Vendor pypa/packaging into micropip (#178)

Author: agriyakhetarpal

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        pyodide-version: ['0.27.0a2']
+        pyodide-version: ["0.27.0a2"]
         test-config: [
           # FIXME: recent version of chrome gets timeout
           {runner: selenium, runtime: chrome, runtime-version: "125" },
@@ -28,6 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:
@@ -88,6 +90,7 @@ jobs:
       - uses: actions/checkout@cbb722410c2e876e24abbe8de2cc27693e501dcb # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
 
       - id: check-integration-test-trigger
         name: Check integration test trigger
@@ -110,15 +113,25 @@ jobs:
     environment: PyPi-deploy
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:
           python-version: 3.12
+
+      # IMPORTANT: always build sdist, and then the wheel from
+      # the sdist (like it is currently done here). This is
+      # because we want to ensure that no extra files get
+      # copied, which can be the case with building in-tree.
+      # The MANIFEST.in file ensures that the sdist doesn't
+      # contain any unnecessary files.
       - name: Install requirements and build wheel
         shell: bash -l {0}
         run: |
           python -m pip install build twine
           python -m build .
+
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/remote_package_index_test.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.gitmodules

@@ -0,0 +1,5 @@
+# d8e3b31b734926ebbcaff654279f6855a73e052f, for 24.2 release
+# https://github.com/pypa/packaging/releases/tag/24.2
+[submodule "micropip/_vendored/packaging"]
+	path = micropip/_vendored/packaging
+	url = https://github.com/pypa/packaging/

CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (the ones that starts with `../` or `./`)
   [#174](https://github.com/pyodide/micropip/pull/174)
 
+### Added
+
+- `micropip` now vendors `pypa/packaging` for better reliability.
+  [#178](https://github.com/pyodide/micropip/pull/178)
+
 ## [0.8.0] - 2024/12/15
 
 ### Added

MANIFEST.in

@@ -0,0 +1,16 @@
+exclude micropip/_vendored/packaging/.github/*
+exclude micropip/_vendored/packaging/docs/*
+exclude micropip/_vendored/packaging/tasks/*
+exclude micropip/_vendored/packaging/tests/*
+exclude micropip/_vendored/packaging/.pre-commit-config.yaml
+exclude micropip/_vendored/packaging/.readthedocs.yml
+exclude micropip/_vendored/packaging/CHANGELOG.rst
+exclude micropip/_vendored/packaging/CONTRIBUTING.rst
+exclude micropip/_vendored/packaging/noxfile.py
+exclude micropip/_vendored/packaging/pyproject.toml
+
+include micropip/_vendored/packaging/LICENSE
+include micropip/_vendored/packaging/LICENSE.APACHE
+include micropip/_vendored/packaging/LICENSE.BSD
+
+include micropip/_vendored/packaging/README.rst

micropip/_compat/_compat_not_in_pyodide.py

@@ -14,6 +14,7 @@
 class CompatibilityNotInPyodide(CompatibilityLayer):
 
     # Vendored from packaging
+    # TODO: use packaging APIs here instead?
     _canonicalize_regex = re.compile(r"[-_.]+")
 
     class HttpStatusError(Exception):

micropip/_utils.py

@@ -4,14 +4,19 @@
 from pathlib import Path
 from sysconfig import get_config_var, get_platform
 
-from packaging.requirements import Requirement
-from packaging.tags import Tag
-from packaging.tags import sys_tags as sys_tags_orig
-from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
-from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig
-from packaging.version import InvalidVersion, Version
-
 from ._compat import REPODATA_PACKAGES
+from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.tags import Tag
+from ._vendored.packaging.src.packaging.tags import sys_tags as sys_tags_orig
+from ._vendored.packaging.src.packaging.utils import (
+    BuildTag,
+    InvalidWheelFilename,
+    canonicalize_name,
+)
+from ._vendored.packaging.src.packaging.utils import (
+    parse_wheel_filename as parse_wheel_filename_orig,
+)
+from ._vendored.packaging.src.packaging.version import InvalidVersion, Version
 
 
 def get_dist_info(dist: Distribution) -> Path:

micropip/_vendored/packaging

@@ -7,13 +7,12 @@
 import warnings
 from typing import Any, Dict, List, Optional, Union, Literal, TypeAlias, TypedDict
 
-import packaging.utils
+import micropip._vendored.packaging.src.packaging.utils as packaging_utils
 
 
 ACCEPT_JSON_V1 = "application/vnd.pypi.simple.v1+json"
 
 
-
 class UnsupportedAPIVersion(Exception):
     """The major version of an API response is not supported."""
 
@@ -92,15 +91,15 @@ class ProjectDetails_1_0(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`691`)."""
 
     meta: _Meta_1_0
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_0]
 
 
 class ProjectDetails_1_1(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`700`)."""
 
     meta: _Meta_1_1
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_1]
     # PEP 700
     versions: List[str]
@@ -235,6 +234,6 @@ def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0:
         files.append(details)
     return {
         "meta": {"api-version": "1.0"},
-        "name": packaging.utils.canonicalize_name(name),
+        "name": packaging_utils.canonicalize_name(name),
         "files": files,
     }

micropip/externals/mousebender/simple.py

@@ -4,9 +4,8 @@
 from copy import deepcopy
 from typing import Any
 
-from packaging.utils import canonicalize_name
-
 from ._utils import fix_package_dependencies
+from ._vendored.packaging.src.packaging.utils import canonicalize_name
 
 
 def freeze_lockfile(