From ea7eced6be9967731d1623827f1336fa182825ea Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 13 Mar 2023 21:44:31 -0400 Subject: [PATCH 1/9] ENH: Implement load_requirement --- lazy_loader/__init__.py | 70 ++++++++++++++++++++++++++++++++++++----- pyproject.toml | 4 +++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index 7eade11..000efd8 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -13,6 +13,11 @@ import types import warnings +try: + import importlib_metadata +except ImportError: + import importlib.metadata as importlib_metadata + __all__ = ["attach", "load", "attach_stub"] @@ -99,17 +104,18 @@ def __dir__(): class DelayedImportErrorModule(types.ModuleType): - def __init__(self, frame_data, *args, **kwargs): + def __init__(self, frame_data, *args, message=None, **kwargs): self.__frame_data = frame_data + self.__message = message or f"No module named '{frame_data['spec']}'" super().__init__(*args, **kwargs) def __getattr__(self, x): - if x in ("__class__", "__file__", "__frame_data"): + if x in ("__class__", "__file__", "__frame_data", "__message"): super().__getattr__(x) else: fd = self.__frame_data raise ModuleNotFoundError( - f"No module named '{fd['spec']}'\n\n" + f"{self.__message}\n\n" "This error is lazily reported, having originally occured in\n" f' File {fd["filename"]}, line {fd["lineno"]}, in {fd["function"]}\n\n' f'----> {"".join(fd["code_context"] or "").strip()}' @@ -185,20 +191,33 @@ def myfunc(): warnings.warn(msg, RuntimeWarning) spec = importlib.util.find_spec(fullname) + return _module_from_spec( + spec, + fullname, + f"No module named '{fullname}'", + error_on_import, + ) + + +def _module_from_spec(spec, fullname, failure_message, error_on_import): + """Return lazy module, DelayedImportErrorModule, or raise error""" if spec is None: if error_on_import: - raise ModuleNotFoundError(f"No module named '{fullname}'") + raise ModuleNotFoundError(failure_message) else: try: - parent = inspect.stack()[1] + parent = inspect.stack()[2] frame_data = { - "spec": fullname, "filename": parent.filename, "lineno": parent.lineno, "function": parent.function, "code_context": parent.code_context, } - return DelayedImportErrorModule(frame_data, "DelayedImportErrorModule") + return DelayedImportErrorModule( + frame_data, + "DelayedImportErrorModule", + message=failure_message, + ) finally: del parent @@ -269,3 +288,40 @@ def attach_stub(package_name: str, filename: str): visitor = _StubVisitor() visitor.visit(stub_node) return attach(package_name, visitor._submodules, visitor._submod_attrs) + + +def load_requirement(requirement, fullname=None, error_on_import=False): + # Old style lazy loading to avoid polluting sys.modules + import packaging.requirements + + req = packaging.requirements.Requirement(requirement) + + if fullname is None: + fullname = req.name + + not_found_msg = f"No module named '{fullname}'" + + module = sys.modules.get(fullname) + have_mod = module is not None + if not have_mod: + spec = importlib.util.find_spec(fullname) + have_mod = spec is not None + + if have_mod and req.specifier: + # Note: req.name is the distribution name, not the module name + try: + version = importlib_metadata.version(req.name) + except importlib_metadata.PackageNotFoundError as e: + raise ValueError( + f"Found module '{fullname}' but cannot test requirement '{req}'. " + "Requirements must match distribution name, not module name." + ) from e + have_mod = any(req.specifier.filter((version,))) + if not have_mod: + spec = None + not_found_msg = f"No distribution can be found matching '{req}'" + + if have_mod and module is not None: + return module, have_mod + + return _module_from_spec(spec, fullname, not_found_msg, error_on_import), have_mod diff --git a/pyproject.toml b/pyproject.toml index af0b3f5..19ae42c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,10 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] description = "Makes it easy to load subpackages and functions on demand." +dependencies = [ + "packaging", + "importlib_metadata; python_version < '3.9'", +] [project.optional-dependencies] test = ["pytest >= 7.4", "pytest-cov >= 4.1"] From da8154d8ea4854270ff185467bc7b50b37b79553 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 14 Mar 2023 13:14:35 -0400 Subject: [PATCH 2/9] RF: Rewrite load_requirement as argument to load, add have_module() function --- lazy_loader/__init__.py | 138 ++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 77 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index 000efd8..89da6aa 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -104,9 +104,9 @@ def __dir__(): class DelayedImportErrorModule(types.ModuleType): - def __init__(self, frame_data, *args, message=None, **kwargs): + def __init__(self, frame_data, *args, message, **kwargs): self.__frame_data = frame_data - self.__message = message or f"No module named '{frame_data['spec']}'" + self.__message = message super().__init__(*args, **kwargs) def __getattr__(self, x): @@ -122,7 +122,7 @@ def __getattr__(self, x): ) -def load(fullname, error_on_import=False): +def load(fullname, *, require=None, error_on_import=False): """Return a lazily imported proxy for a module. We often see the following pattern:: @@ -177,10 +177,12 @@ def myfunc(): Actual loading of the module occurs upon first attribute request. """ - try: - return sys.modules[fullname] - except KeyError: - pass + module = sys.modules.get(fullname) + have_module = module is not None + + # Most common, short-circuit + if have_module and require is None: + return module if "." in fullname: msg = ( @@ -190,46 +192,65 @@ def myfunc(): ) warnings.warn(msg, RuntimeWarning) - spec = importlib.util.find_spec(fullname) - return _module_from_spec( - spec, - fullname, - f"No module named '{fullname}'", - error_on_import, - ) + spec = None + if not have_module: + spec = importlib.util.find_spec(fullname) + have_module = spec is not None + + if not have_module: + not_found_message = f"No module named '{fullname}'" + elif require is not None: + # Old style lazy loading to avoid polluting sys.modules + import packaging.requirements + + req = packaging.requirements.Requirement(require) + try: + have_module = req.specifier.contains( + importlib_metadata.version(req.name), + prereleases=True, + ) + except importlib_metadata.PackageNotFoundError as e: + raise ValueError( + f"Found module '{fullname}' but cannot test requirement '{require}'. " + "Requirements must match distribution name, not module name." + ) from e + if not have_module: + not_found_message = f"No distribution can be found matching '{require}'" -def _module_from_spec(spec, fullname, failure_message, error_on_import): - """Return lazy module, DelayedImportErrorModule, or raise error""" - if spec is None: + if not have_module: if error_on_import: - raise ModuleNotFoundError(failure_message) - else: - try: - parent = inspect.stack()[2] - frame_data = { - "filename": parent.filename, - "lineno": parent.lineno, - "function": parent.function, - "code_context": parent.code_context, - } - return DelayedImportErrorModule( - frame_data, - "DelayedImportErrorModule", - message=failure_message, - ) - finally: - del parent - - module = importlib.util.module_from_spec(spec) - sys.modules[fullname] = module - - loader = importlib.util.LazyLoader(spec.loader) - loader.exec_module(module) + raise ModuleNotFoundError(not_found_message) + try: + parent = inspect.stack()[1] + frame_data = { + "filename": parent.filename, + "lineno": parent.lineno, + "function": parent.function, + "code_context": parent.code_context, + } + return DelayedImportErrorModule( + frame_data, + "DelayedImportErrorModule", + message=not_found_message, + ) + finally: + del parent + + if spec is not None: + module = importlib.util.module_from_spec(spec) + sys.modules[fullname] = module + + loader = importlib.util.LazyLoader(spec.loader) + loader.exec_module(module) return module +def have_module(module_like: types.ModuleType) -> bool: + return not isinstance(module_like, DelayedImportErrorModule) + + class _StubVisitor(ast.NodeVisitor): """AST visitor to parse a stub file for submodules and submod_attrs.""" @@ -288,40 +309,3 @@ def attach_stub(package_name: str, filename: str): visitor = _StubVisitor() visitor.visit(stub_node) return attach(package_name, visitor._submodules, visitor._submod_attrs) - - -def load_requirement(requirement, fullname=None, error_on_import=False): - # Old style lazy loading to avoid polluting sys.modules - import packaging.requirements - - req = packaging.requirements.Requirement(requirement) - - if fullname is None: - fullname = req.name - - not_found_msg = f"No module named '{fullname}'" - - module = sys.modules.get(fullname) - have_mod = module is not None - if not have_mod: - spec = importlib.util.find_spec(fullname) - have_mod = spec is not None - - if have_mod and req.specifier: - # Note: req.name is the distribution name, not the module name - try: - version = importlib_metadata.version(req.name) - except importlib_metadata.PackageNotFoundError as e: - raise ValueError( - f"Found module '{fullname}' but cannot test requirement '{req}'. " - "Requirements must match distribution name, not module name." - ) from e - have_mod = any(req.specifier.filter((version,))) - if not have_mod: - spec = None - not_found_msg = f"No distribution can be found matching '{req}'" - - if have_mod and module is not None: - return module, have_mod - - return _module_from_spec(spec, fullname, not_found_msg, error_on_import), have_mod From 94e64653c8914aa697b76d1a2b9d3d66fb7630c0 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 14 Mar 2023 15:58:22 -0400 Subject: [PATCH 3/9] TEST: Test load(..., require=...) keyword arg and have_module() func --- lazy_loader/tests/test_lazy_loader.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lazy_loader/tests/test_lazy_loader.py b/lazy_loader/tests/test_lazy_loader.py index 83fb9ed..c8a4ee6 100644 --- a/lazy_loader/tests/test_lazy_loader.py +++ b/lazy_loader/tests/test_lazy_loader.py @@ -1,11 +1,19 @@ import importlib import sys import types +from unittest import mock import pytest import lazy_loader as lazy +try: + import importlib_metadata # noqa + + have_importlib_metadata = True +except ImportError: + have_importlib_metadata = False + def test_lazy_import_basics(): math = lazy.load("math") @@ -149,3 +157,32 @@ def test_stub_loading_errors(tmp_path): with pytest.raises(ValueError, match="Cannot load imports from non-existent stub"): lazy.attach_stub("name", "not a file") + + +def test_require_kwarg(): + dot = "_" if have_importlib_metadata else "." + # Test with a module that definitely exists, behavior hinges on requirement + with mock.patch(f"importlib{dot}metadata.version") as version: + version.return_value = "1.0.0" + math = lazy.load("math", require="somepkg >= 2.0") + assert isinstance(math, lazy.DelayedImportErrorModule) + + math = lazy.load("math", require="somepkg >= 1.0") + assert math.sin(math.pi) == pytest.approx(0, 1e-6) + + # We can fail even after a successful import + math = lazy.load("math", require="somepkg >= 2.0") + assert isinstance(math, lazy.DelayedImportErrorModule) + + # When a module can be loaded but the version can't be checked, + # raise a ValueError + with pytest.raises(ValueError): + lazy.load("math", require="somepkg >= 1.0") + + +def test_have_module(): + math = lazy.load("math") + anything_not_real = lazy.load("anything_not_real") + + assert lazy.have_module(math) + assert not lazy.have_module(anything_not_real) From add4bfe1e703491f0400a28b52928dc5a59dffdb Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 22 Mar 2023 11:33:27 -0400 Subject: [PATCH 4/9] ENH: Delay loading of less likely modules Using python -X importtime -c "import lazy_loader": Before ------ import time: self [us] | cumulative | imported package [...] import time: 131 | 22995 | lazy_loader After ----- import time: self [us] | cumulative | imported package [...] import time: 115 | 4248 | lazy_loader --- lazy_loader/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index 89da6aa..e313618 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -7,17 +7,11 @@ import ast import importlib import importlib.util -import inspect import os import sys import types import warnings -try: - import importlib_metadata -except ImportError: - import importlib.metadata as importlib_metadata - __all__ = ["attach", "load", "attach_stub"] @@ -200,9 +194,13 @@ def myfunc(): if not have_module: not_found_message = f"No module named '{fullname}'" elif require is not None: - # Old style lazy loading to avoid polluting sys.modules import packaging.requirements + try: + import importlib_metadata + except ImportError: + import importlib.metadata as importlib_metadata + req = packaging.requirements.Requirement(require) try: have_module = req.specifier.contains( @@ -221,6 +219,8 @@ def myfunc(): if not have_module: if error_on_import: raise ModuleNotFoundError(not_found_message) + import inspect + try: parent = inspect.stack()[1] frame_data = { From bba26037a40adcca7dfae8e3b1118e9f053abb5b Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 16 Aug 2023 22:21:21 -0400 Subject: [PATCH 5/9] RF: Split requirement check into function, prefer importlib.metadata --- lazy_loader/__init__.py | 49 +++++++++++++++++++-------- lazy_loader/tests/test_lazy_loader.py | 10 ++---- pyproject.toml | 2 +- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index e313618..7053685 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -194,27 +194,15 @@ def myfunc(): if not have_module: not_found_message = f"No module named '{fullname}'" elif require is not None: - import packaging.requirements - - try: - import importlib_metadata - except ImportError: - import importlib.metadata as importlib_metadata - - req = packaging.requirements.Requirement(require) try: - have_module = req.specifier.contains( - importlib_metadata.version(req.name), - prereleases=True, - ) - except importlib_metadata.PackageNotFoundError as e: + have_module = _check_requirement(require) + except ModuleNotFoundError as e: raise ValueError( f"Found module '{fullname}' but cannot test requirement '{require}'. " "Requirements must match distribution name, not module name." ) from e - if not have_module: - not_found_message = f"No distribution can be found matching '{require}'" + not_found_message = f"No distribution can be found matching '{require}'" if not have_module: if error_on_import: @@ -247,6 +235,37 @@ def myfunc(): return module +def _check_requirement(require: str) -> bool: + """Verify that a package requirement is satisfied + + If the package is required, a ``ModuleNotFoundError`` is raised + by ``importlib.metadata``. + + Parameters + ---------- + require : str + A dependency requirement as defined in PEP-508 + + Returns + ------- + satisfied : bool + True if the installed version of the dependency matches + the specified version, False otherwise. + """ + import packaging.requirements + + try: + import importlib.metadata as importlib_metadata + except ImportError: # PY37 + import importlib_metadata + + req = packaging.requirements.Requirement(require) + return req.specifier.contains( + importlib_metadata.version(req.name), + prereleases=True, + ) + + def have_module(module_like: types.ModuleType) -> bool: return not isinstance(module_like, DelayedImportErrorModule) diff --git a/lazy_loader/tests/test_lazy_loader.py b/lazy_loader/tests/test_lazy_loader.py index c8a4ee6..f8948c4 100644 --- a/lazy_loader/tests/test_lazy_loader.py +++ b/lazy_loader/tests/test_lazy_loader.py @@ -7,13 +7,6 @@ import lazy_loader as lazy -try: - import importlib_metadata # noqa - - have_importlib_metadata = True -except ImportError: - have_importlib_metadata = False - def test_lazy_import_basics(): math = lazy.load("math") @@ -160,7 +153,8 @@ def test_stub_loading_errors(tmp_path): def test_require_kwarg(): - dot = "_" if have_importlib_metadata else "." + have_importlib_metadata = importlib.util.find_spec("importlib.metadata") is not None + dot = "." if have_importlib_metadata else "_" # Test with a module that definitely exists, behavior hinges on requirement with mock.patch(f"importlib{dot}metadata.version") as version: version.return_value = "1.0.0" diff --git a/pyproject.toml b/pyproject.toml index 19ae42c..6c13203 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ description = "Makes it easy to load subpackages and functions on demand." dependencies = [ "packaging", - "importlib_metadata; python_version < '3.9'", + "importlib_metadata; python_version < '3.8'", ] [project.optional-dependencies] From a557fb44c678cd75d93afddda2c50680f1d9dc71 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 16 Aug 2023 22:27:04 -0400 Subject: [PATCH 6/9] Remove have_module (out-of-scope) --- lazy_loader/__init__.py | 4 ---- lazy_loader/tests/test_lazy_loader.py | 8 -------- 2 files changed, 12 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index 7053685..55a1fe3 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -266,10 +266,6 @@ def _check_requirement(require: str) -> bool: ) -def have_module(module_like: types.ModuleType) -> bool: - return not isinstance(module_like, DelayedImportErrorModule) - - class _StubVisitor(ast.NodeVisitor): """AST visitor to parse a stub file for submodules and submod_attrs.""" diff --git a/lazy_loader/tests/test_lazy_loader.py b/lazy_loader/tests/test_lazy_loader.py index f8948c4..a7c166c 100644 --- a/lazy_loader/tests/test_lazy_loader.py +++ b/lazy_loader/tests/test_lazy_loader.py @@ -172,11 +172,3 @@ def test_require_kwarg(): # raise a ValueError with pytest.raises(ValueError): lazy.load("math", require="somepkg >= 1.0") - - -def test_have_module(): - math = lazy.load("math") - anything_not_real = lazy.load("anything_not_real") - - assert lazy.have_module(math) - assert not lazy.have_module(anything_not_real) From ba248f9acc4f8b38074f89ff5f712a1306a51d88 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 16 Aug 2023 22:58:39 -0400 Subject: [PATCH 7/9] DOC: Update docstring and README --- README.md | 18 +++++++++++++++++- lazy_loader/__init__.py | 8 ++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 946797c..377de70 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,22 @@ discouraged._ You can ask `lazy.load` to raise import errors as soon as it is called: -``` +```python linalg = lazy.load('scipy.linalg', error_on_import=True) ``` + +#### Optional requirements + +One use for lazy loading is for loading optional dependencies, with +errors only arising when optional functionality is accessed. If optional +functionality depends on a specific version, a version requirement can +be set: + +```python +np = lazy.load("numpy", require="numpy >=1.24") +``` + +In this case, if `numpy` is installed, but the version is less than 1.24, +the `np` module returned will raise an error on attribute access. Using +this feature is not all-or-nothing: One module may rely on one version of +numpy, while another module may not set any requirement. diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index 55a1fe3..f0f4672 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -160,6 +160,14 @@ def myfunc(): sp = lazy.load('scipy') # import scipy as sp + require : str + A dependency requirement as defined in PEP-508. For example:: + + "numpy >=1.24" + + If defined, the proxy module will raise an error if the installed + version does not satisfy the requirement. + error_on_import : bool Whether to postpone raising import errors until the module is accessed. If set to `True`, import errors are raised as soon as `load` is called. From 178820333c0760ce56a74118b8579c15498aeb2f Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 16 Aug 2023 23:07:00 -0400 Subject: [PATCH 8/9] DOC: Note discrepancy between distribution and import names --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 377de70..42a2f6f 100644 --- a/README.md +++ b/README.md @@ -134,3 +134,10 @@ In this case, if `numpy` is installed, but the version is less than 1.24, the `np` module returned will raise an error on attribute access. Using this feature is not all-or-nothing: One module may rely on one version of numpy, while another module may not set any requirement. + +_Note that the requirement must use the package [distribution name][] instead +of the module [import name][]. For example, the `pyyaml` distribution provides +the `yaml` module for import._ + +[distribution name]: https://packaging.python.org/en/latest/glossary/#term-Distribution-Package +[import name]: https://packaging.python.org/en/latest/glossary/#term-Import-Package From 15a1d1a266d1d2e04dd4d433610fb487acce929e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 26 Jan 2024 14:26:11 -0500 Subject: [PATCH 9/9] Update README.md Co-authored-by: Dan Schult --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42a2f6f..6600c8c 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ linalg = lazy.load('scipy.linalg', error_on_import=True) #### Optional requirements One use for lazy loading is for loading optional dependencies, with -errors only arising when optional functionality is accessed. If optional +`ImportErrors` only arising when optional functionality is accessed. If optional functionality depends on a specific version, a version requirement can be set: