From 9e4d3652951a36ea23927bd299562a202428f5d7 Mon Sep 17 00:00:00 2001 From: Dan Schult Date: Mon, 19 Jun 2023 23:10:44 -0400 Subject: [PATCH 1/4] Add warning and change docs to discourage subpackages with load() --- lazy_loader/__init__.py | 38 ++++++++++++++++++++------- lazy_loader/tests/test_lazy_loader.py | 11 ++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index ed6f7dd..fa7097c 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -11,6 +11,7 @@ import os import sys import types +import warnings __all__ = ["attach", "load", "attach_stub"] @@ -121,26 +122,37 @@ def load(fullname, error_on_import=False): We often see the following pattern:: def myfunc(): - from numpy import linalg as la - la.norm(...) + import numpy as np + np.norm(...) .... - This is to prevent a module, in this case `numpy`, from being - imported at function definition time, since that can be slow. + Putting the import inside the function prevents, in this case, + `numpy`, from being imported at function definition time. + That saves time if `myfunc` ends up not being called. - This function provides a proxy module that, upon access, imports + This `load` function returns a proxy module that, upon access, imports the actual module. So the idiom equivalent to the above example is:: - la = lazy.load("numpy.linalg") + np = lazy.load("numpy") def myfunc(): - la.norm(...) + np.norm(...) .... The initial import time is fast because the actual import is delayed until the first attribute is requested. The overall import time may decrease as well for users that don't make use of large portions - of the library. + of your library. + + Warning + ------- + + While lazily loading subpackages technically works, it causes the + package (that contains the subpackage) to be eagerly loaded even + if the package is already lazily loaded. + So, you probably shouldn't use subpackages with this `load` feature. + Instead you should encourage the package maintainers to use the + lazy_loader `attach` feature to make their subpackages lazily load. Parameters ---------- @@ -148,7 +160,7 @@ def myfunc(): The full name of the module or submodule to import. For example:: sp = lazy.load('scipy') # import scipy as sp - spla = lazy.load('scipy.linalg') # import scipy.linalg as spla + error_on_import : bool Whether to postpone raising import errors until the module is accessed. If set to `True`, import errors are raised as soon as `load` is called. @@ -165,6 +177,14 @@ def myfunc(): except KeyError: pass + if "." in fullname: + msg = ( + "subpackages can technically be lazily loaded, but it causes the " + "package to be eagerly loaded even if it is already lazily loaded." + "So, you probably shouldn't use subpackages with this lazy feature." + ) + warnings.warn(msg, RuntimeWarning) + spec = importlib.util.find_spec(fullname) if spec is None: if error_on_import: diff --git a/lazy_loader/tests/test_lazy_loader.py b/lazy_loader/tests/test_lazy_loader.py index 1c8dbd3..017b619 100644 --- a/lazy_loader/tests/test_lazy_loader.py +++ b/lazy_loader/tests/test_lazy_loader.py @@ -1,3 +1,4 @@ +import importlib import sys import types @@ -27,6 +28,16 @@ def test_lazy_import_basics(): pass +def test_lazy_import_subpackages(): + with pytest.warns(RuntimeWarning): + ctextpad = lazy.load("curses.textpad") + assert "curses" in sys.modules + assert type(sys.modules["curses"]) == type(pytest) + assert isinstance(ctextpad, importlib.util._LazyModule) + assert "curses.textpad" in sys.modules + assert sys.modules["curses.textpad"] == ctextpad + + def test_lazy_import_impact_on_sys_modules(): math = lazy.load("math") anything_not_real = lazy.load("anything_not_real") From 42ac558bffd7581d1b17c196f5b8fa27d3a9abaa Mon Sep 17 00:00:00 2001 From: Dan Schult Date: Tue, 20 Jun 2023 07:34:10 -0400 Subject: [PATCH 2/4] Update test to use builtin lib stable over python versions --- lazy_loader/tests/test_lazy_loader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lazy_loader/tests/test_lazy_loader.py b/lazy_loader/tests/test_lazy_loader.py index 017b619..04e9013 100644 --- a/lazy_loader/tests/test_lazy_loader.py +++ b/lazy_loader/tests/test_lazy_loader.py @@ -30,12 +30,12 @@ def test_lazy_import_basics(): def test_lazy_import_subpackages(): with pytest.warns(RuntimeWarning): - ctextpad = lazy.load("curses.textpad") - assert "curses" in sys.modules - assert type(sys.modules["curses"]) == type(pytest) - assert isinstance(ctextpad, importlib.util._LazyModule) - assert "curses.textpad" in sys.modules - assert sys.modules["curses.textpad"] == ctextpad + hp = lazy.load("html.parser") + assert "html" in sys.modules + assert type(sys.modules["html"]) == type(pytest) + assert isinstance(hp, importlib.util._LazyModule) + assert "html.parser" in sys.modules + assert sys.modules["html.parser"] == hp def test_lazy_import_impact_on_sys_modules(): From f29c65ddc5cb62f7511f3fc65b957259a7d9090e Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Tue, 27 Jun 2023 16:15:37 -0700 Subject: [PATCH 3/4] Minor edits --- lazy_loader/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lazy_loader/__init__.py b/lazy_loader/__init__.py index fa7097c..b45430b 100644 --- a/lazy_loader/__init__.py +++ b/lazy_loader/__init__.py @@ -146,13 +146,12 @@ def myfunc(): Warning ------- - - While lazily loading subpackages technically works, it causes the + While lazily loading *sub*packages technically works, it causes the package (that contains the subpackage) to be eagerly loaded even if the package is already lazily loaded. So, you probably shouldn't use subpackages with this `load` feature. Instead you should encourage the package maintainers to use the - lazy_loader `attach` feature to make their subpackages lazily load. + `lazy_loader.attach` to make their subpackages load lazily. Parameters ---------- From 3f9830246adcfd99063db6e7b5d8992f6b42c769 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Tue, 27 Jun 2023 16:20:27 -0700 Subject: [PATCH 4/4] Also update README --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ade8b34..946797c 100644 --- a/README.md +++ b/README.md @@ -104,10 +104,16 @@ internal imports. Use `lazy.load` to lazily import external libraries: ```python -linalg = lazy.load('scipy.linalg') # `linalg` will only be loaded when accessed +sp = lazy.load('scipy') # `sp` will only be loaded when accessed +sp.linalg.norm(...) ``` -You can also ask `lazy.load` to raise import errors as soon as it is called: +_Note that lazily importing *sub*packages, +i.e. `load('scipy.linalg')` will cause the package containing the +subpackage to be imported immediately; thus, this usage is +discouraged._ + +You can ask `lazy.load` to raise import errors as soon as it is called: ``` linalg = lazy.load('scipy.linalg', error_on_import=True)