Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
525d0d7
added support for creating mock packages
joemarshall Nov 17, 2022
111046f
Merge branch 'main' of github.com:pyodide/micropip into add_package_mock
joemarshall Nov 17, 2022
799560d
reverted typo change
joemarshall Nov 17, 2022
f66370c
removed unneeded imports
joemarshall Nov 17, 2022
43fa3f3
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 17, 2022
16739f5
remembered to run pre-commit hooks
joemarshall Nov 17, 2022
0a6471a
added test
joemarshall Nov 17, 2022
5e5b0d9
removed debug print
joemarshall Nov 17, 2022
0eb9803
merged and linted
joemarshall Nov 17, 2022
20b74f0
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 17, 2022
c76a8e6
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 17, 2022
3418848
dist-info metadata for easier package finding etc.
joemarshall Nov 23, 2022
ea8c424
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 23, 2022
6628396
added support for temporary modules
joemarshall Nov 23, 2022
9cb4ac2
default to non-persistent mocking
joemarshall Nov 23, 2022
d6aab95
in memory sub packages fixes
joemarshall Nov 23, 2022
005efec
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 23, 2022
a12cd86
test in pyodide with correct (local) version of micropip
joemarshall Nov 23, 2022
10c7f84
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 23, 2022
5d96d84
fixed missing package in test
joemarshall Nov 23, 2022
786e5e8
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 23, 2022
35539ef
dedent in memory modules
joemarshall Nov 23, 2022
107004d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 23, 2022
ac8bc37
simpler std output capture
joemarshall Nov 23, 2022
98b9192
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 23, 2022
e83cb59
invalidate importlib cache for pyodide
joemarshall Nov 23, 2022
4d719c2
testing different order to see what is broken
joemarshall Nov 23, 2022
fd037ae
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 23, 2022
3b77d88
fixed tests for pyodide
joemarshall Nov 23, 2022
782b6af
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 23, 2022
d34ff58
review fixes
joemarshall Nov 24, 2022
5e8f5e6
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 24, 2022
2c34df0
added changelog
joemarshall Nov 24, 2022
9c16b28
Merge branch 'add_package_mock' of https://github.com/joemarshall/mic…
joemarshall Nov 24, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions micropip/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from ._micropip import _list as list
from ._micropip import freeze, install
from ._micropip import add_mock_package, freeze, install

try:
from ._version import __version__
except ImportError:
pass

__all__ = ["install", "list", "freeze", "__version__"]
__all__ = ["install", "list", "freeze", "add_mock_package", "__version__"]
72 changes: 72 additions & 0 deletions micropip/_micropip.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import hashlib
import importlib
import json
import os
import site
import warnings
from asyncio import gather
from dataclasses import dataclass, field
Expand All @@ -10,6 +12,7 @@
from importlib.metadata import version as importlib_version
from pathlib import Path
from sysconfig import get_platform
from textwrap import dedent
from typing import IO, Any
from urllib.parse import ParseResult, urlparse
from zipfile import ZipFile
Expand Down Expand Up @@ -692,3 +695,72 @@ def _list():
source_ = pkg_source
packages[name] = PackageMetadata(name=name, version=version, source=source_)
return packages


def add_mock_package(name, version, *, modules=None):

@ryanking13 ryanking13 Nov 23, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please add type hints here?

"""
Add a mock version of a package to the package dictionary.

This means that if it is a dependency, it is skipped on install.

By default a single empty module is installed with the same
name as the package. You can alternatively give one or more modules to make a
set of named modules.

The modules parameter is a dictionary mapping module name to module text.

e.g.
``
{
mylovely_module:'''
def module_method(an_argument):
print("This becomes a module level argument")
module_value = "this value becomes a module level variable"

print("This is run on import of module")
'''
}


``

Parameters

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is an indentation issue in this docstring, but we can fix it once we start actually building docs for it.

----------
name : ``str ``

Package name to add

modules : ``Optional(dict) ``
Comment thread
joemarshall marked this conversation as resolved.

Dictionary of module_name:string pairs.
The string contains the source of the mock module or is blank for
an empty module.
"""
if modules is None:
# make a single mock module with this name
modules = {name: ""}
# make empty mock modules with the requested names in user site packages
site_packages = Path(site.getusersitepackages())
metadata_file = site_packages / (name + "-" + version + ".egg-info")
Comment thread
joemarshall marked this conversation as resolved.
Outdated
with open(metadata_file, "w") as mf:

mf.write(
f"""Metadata-Version: 1.1
Name: {name}
Version: {version}
Summary: {name} mock module generated by micropip
Author-email: {name}@micro.pip.non-working-fake-host
"""
)
for n, s in modules.items():
mf.write(f"Provides:{n}\n")

if s is None:
s = ""
s = dedent(s)
path_parts = n.split(".")
dir_path = Path(site_packages, *path_parts)
os.makedirs(dir_path, exist_ok=True)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should raise error if a directory exists, because it would mean that the package is already installed somehow.

init_file = dir_path / "__init__.py"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we actually need to create these physical files?

I was thinking we could maybe directly initialize sys.modules with the mapping from modules: dict[str, ModuleType | str]? That way it's also easy to use MagicMock as a module or any other custom object defined at runtime to make a given dependency work.

Though yes, the API to create modules dynamically via importlib.util.module_from_spec or by inheriting from ModuleType isn't very straightforward.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe making another custom Finder/Loader would be a good way to go. The API for that isn't too bad.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I actually wrote it initially using mock objects and putting them into sys.modules, but I reverted to writing them out to files. I did it like that because it is really simple and reliable, things like file paths etc. just work. It also works nicely in terms of loading modules, in that init code is only run on actual import. Oh and in the case that you're running somewhere with persistent file system, your mock modules continue to exist on reload of python.

It's a pretty minor change - you just create a module and exec the code into it in the loader, I can put that back in if you reckon the possibility of using mock objects makes sense.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually I think rather than take an object it needs to take a callable which takes a module object and does whatever needs to be done on it to make it. I.e. the equivalent of loader.exec_module

That way you can mock module initialisation correctly.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One further thought on this - at some point I want to implement some kind of mega wheel option, to bundle a set of wheels and all deps into a single wheel with no deps. For that use case keeping persistent mock modules would be nice. If it's okay I'd like to keep that as an option in micropip?

It would also be nice for any point that pyodide is run in a persistent file system.

I'm thinking that a flag "persistent" could be added (which only works for string modules) and then the behaviour with finder and loader could be used in the case that the flag isn't set.

with open(init_file, "w") as f:
f.write(s)

@ryanking13 ryanking13 Nov 23, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should keep the list of mocked modules, so it can be accessed via e.g. micropip.mock_modules

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added list_mock_modules instead - which uses module installer metadata to list modules it has installed

42 changes: 42 additions & 0 deletions tests/test_micropip.py
Original file line number Diff line number Diff line change
Expand Up @@ -934,3 +934,45 @@ def test_check_compatible(mock_platform, interp, abi, arch, ctx):
wheel_name = f"{pkg}-{interp}-{abi}-{arch}.whl"
with ctx:
WheelInfo.from_url(wheel_name).check_compatible()


def test_add_mock_package(monkeypatch, capsys):
import site
from importlib.metadata import version as importlib_version

from micropip._micropip import add_mock_package

with TemporaryDirectory() as tmpdirname:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a tmp_path fixture for this purpose.


def _getusersitepackages():
return tmpdirname

monkeypatch.setattr(site, "getusersitepackages", _getusersitepackages)
monkeypatch.setattr(sys, "path", [tmpdirname])
add_mock_package("test_1", "1.0.0")
add_mock_package(
"test_2",
"1.2.0",
modules={
"t1": "print('hi from t1')",
"t2": """
def fn():
print("Hello from fn")
""",
},
)
import t1

dir(t1)
import t2

dir(t2)
import test_1

dir(test_1)

t2.fn()
assert importlib_version("test_2") == "1.2.0"
captured = capsys.readouterr()
assert captured.out.find("hi from t1") != -1
assert captured.out.find("Hello from fn") != -1