Skip to content

Commit c40b26d

Browse files
committed
gh-117378: Fix multiprocessing forkserver preload sys.path inheritance.
`sys.path` was not properly being sent from the parent process when launching the multiprocessing forkserver process to preload imports. This bug has been there since the forkserver start method was introduced in Python ~3.4. It was always _supposed_ to inherit `sys.path` the same way the spawn method does. Observable behavior change: A `''` value in `sys.path` will now be replaced in the forkserver's `sys.path` with an absolute pathname `os.path.abspath(os.getcwd())` saved at the time that `multiprocessing` was imported in the parent process as it already was when using the spawn start method. A workaround for the bug this fixes was to set PYTHONPATH in the environment before the forkserver process was started.
1 parent dbb6e22 commit c40b26d

File tree

2 files changed

+59
-0
lines changed

2 files changed

+59
-0
lines changed

Lib/multiprocessing/forkserver.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
174174
spawn.import_main_path(main_path)
175175
finally:
176176
del process.current_process()._inheriting
177+
if sys_path is not None:
178+
sys.path[:] = sys_path
177179
for modname in preload:
178180
try:
179181
__import__(modname)
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import os
2+
import sys
3+
import unittest
4+
from unittest import mock
5+
6+
from multiprocessing import forkserver
7+
8+
9+
class TestForkserverMain(unittest.TestCase):
10+
11+
def setUp(self):
12+
self._orig_sys_path = list(sys.path)
13+
14+
def tearDown(self):
15+
sys.path[:] = self._orig_sys_path
16+
17+
@mock.patch("multiprocessing.process.current_process")
18+
@mock.patch("multiprocessing.spawn.import_main_path")
19+
@mock.patch("multiprocessing.util._close_stdin")
20+
def test_preload_kwargs(
21+
self,
22+
mock_close_stdin,
23+
mock_import_main_path,
24+
mock_current_process,
25+
):
26+
# Very much a whitebox test of the first stanza of main before
27+
# we start diddling with file descriptors and pipes.
28+
mock_close_stdin.side_effect = RuntimeError("stop test")
29+
self.assertNotIn(
30+
"colorsys",
31+
sys.modules.keys(),
32+
msg="Thie test requires a module that has not yet been imported.",
33+
)
34+
35+
with self.assertRaisesRegex(RuntimeError, "stop test"):
36+
forkserver.main(None, None, ["sys", "colorsys"])
37+
mock_current_process.assert_not_called()
38+
mock_import_main_path.assert_not_called()
39+
self.assertIn("colorsys", sys.modules.keys())
40+
self.assertEqual(sys.path, self._orig_sys_path) # unmodified
41+
42+
del sys.modules["colorsys"] # unimport
43+
fake_path = os.path.dirname(__file__)
44+
with self.assertRaisesRegex(RuntimeError, "stop test"):
45+
forkserver.main(None, None, ["sys", "colorsys"], sys_path=[fake_path])
46+
self.assertEqual(
47+
sys.path, [fake_path], msg="sys.path should have been overridden"
48+
)
49+
self.assertNotIn(
50+
"colorsys",
51+
sys.modules.keys(),
52+
msg="import of colorsys should have failed with unusual sys.path",
53+
)
54+
55+
56+
if __name__ == "__main__":
57+
unittest.main()

0 commit comments

Comments
 (0)