gh-116167: Allow disabling the GIL with PYTHON_GIL=0 or -X gil=0 (#116338)

In free-threaded builds, running with `PYTHON_GIL=0` will now disable the
GIL. Follow-up issues track work to re-enable the GIL when loading an
incompatible extension, and to disable the GIL by default.

In order to support re-enabling the GIL at runtime, all GIL-related data
structures are initialized as usual, and disabling the GIL simply sets a flag
that causes `take_gil()` and `drop_gil()` to return early.

Author: swtaarrs

Doc/using/cmdline.rst

@@ -559,6 +559,9 @@ Miscellaneous options
      :mod:`__main__`. This can be used to execute code early during Python
      initialization. Python needs to be :ref:`built in debug mode <debug-build>`
      for this option to exist.  See also :envvar:`PYTHON_PRESITE`.
+   * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled,
+     respectively. Only available in builds configured with
+     :option:`--disable-gil`. See also :envvar:`PYTHON_GIL`.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -601,6 +604,9 @@ Miscellaneous options
    .. versionchanged:: 3.13
       Added the ``-X cpu_count`` and ``-X presite`` options.
 
+   .. versionchanged:: 3.13
+      Added the ``-X gil`` option.
+
 .. _using-on-controlling-color:
 
 Controlling color
@@ -1138,6 +1144,18 @@ conflict.
 
    .. versionadded:: 3.13
 
+.. envvar:: PYTHON_GIL
+
+   If this variable is set to ``1``, the global interpreter lock (GIL) will be
+   forced on. Setting it to ``0`` forces the GIL off.
+
+   See also the :option:`-X gil <-X>` command-line option, which takes
+   precedence over this variable.
+
+   Needs Python configured with the :option:`--disable-gil` build option.
+
+   .. versionadded:: 3.13
+
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~
 

Include/cpython/initconfig.h

@@ -181,6 +181,9 @@ typedef struct PyConfig {
     int int_max_str_digits;
 
     int cpu_count;
+#ifdef Py_GIL_DISABLED
+    int enable_gil;
+#endif
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_gil.h

@@ -20,6 +20,11 @@ extern "C" {
 #define FORCE_SWITCHING
 
 struct _gil_runtime_state {
+#ifdef Py_GIL_DISABLED
+    /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime
+       if, for example, a module that requires the GIL is loaded. */
+    int enabled;
+#endif
     /* microseconds (the Python API uses seconds, though) */
     unsigned long interval;
     /* Last PyThreadState holding / having held the GIL. This helps us

Include/internal/pycore_initconfig.h

@@ -153,6 +153,18 @@ typedef enum {
     _PyConfig_INIT_ISOLATED = 3
 } _PyConfigInitEnum;
 
+typedef enum {
+    /* For now, this means the GIL is enabled.
+
+       gh-116329: This will eventually change to "the GIL is disabled but can
+       be reenabled by loading an incompatible extension module." */
+    _PyConfig_GIL_DEFAULT = -1,
+
+    /* The GIL has been forced off or on, and will not be affected by module loading. */
+    _PyConfig_GIL_DISABLE = 0,
+    _PyConfig_GIL_ENABLE = 1,
+} _PyConfigGILEnum;
+
 // Export for '_testembed' program
 PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config);
 

Lib/subprocess.py

@@ -350,7 +350,7 @@ def _args_from_interpreter_flags():
     if dev_mode:
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
-                'frozen_modules', 'showrefcount', 'utf8'):
+                'frozen_modules', 'showrefcount', 'utf8', 'gil'):
         if opt in xoptions:
             value = xoptions[opt]
             if value is True:

Lib/test/_test_embed_set_config.py

@@ -9,6 +9,7 @@
 import os
 import sys
 import unittest
+from test import support
 from test.support import MS_WINDOWS
 
 
@@ -211,6 +212,19 @@ def test_flags(self):
         self.set_config(use_hash_seed=1, hash_seed=123)
         self.assertEqual(sys.flags.hash_randomization, 1)
 
+        if support.Py_GIL_DISABLED:
+            self.set_config(enable_gil=-1)
+            self.assertEqual(sys.flags.gil, None)
+            self.set_config(enable_gil=0)
+            self.assertEqual(sys.flags.gil, 0)
+            self.set_config(enable_gil=1)
+            self.assertEqual(sys.flags.gil, 1)
+        else:
+            # Builds without Py_GIL_DISABLED don't have
+            # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for
+            # consistency.
+            self.assertEqual(sys.flags.gil, 1)
+
     def test_options(self):
         self.check(warnoptions=[])
         self.check(warnoptions=["default", "ignore"])

Lib/test/test_cmd_line.py

@@ -869,6 +869,39 @@ def test_pythondevmode_env(self):
         self.assertEqual(proc.stdout.rstrip(), 'True')
         self.assertEqual(proc.returncode, 0, proc)
 
+    @unittest.skipUnless(support.Py_GIL_DISABLED,
+                         "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds")
+    def test_python_gil(self):
+        cases = [
+            # (env, opt, expected, msg)
+            (None, None, 'None', "no options set"),
+            ('0', None, '0', "PYTHON_GIL=0"),
+            ('1', None, '1', "PYTHON_GIL=1"),
+            ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"),
+            (None, '0', '0', "-X gil=0"),
+            (None, '1', '1', "-X gil=1"),
+        ]
+
+        code = "import sys; print(sys.flags.gil)"
+        environ = dict(os.environ)
+
+        for env, opt, expected, msg in cases:
+            with self.subTest(msg, env=env, opt=opt):
+                environ.pop('PYTHON_GIL', None)
+                if env is not None:
+                    environ['PYTHON_GIL'] = env
+                extra_args = []
+                if opt is not None:
+                    extra_args = ['-X', f'gil={opt}']
+
+                proc = subprocess.run([sys.executable, *extra_args, '-c', code],
+                                      stdout=subprocess.PIPE,
+                                      stderr=subprocess.PIPE,
+                                      text=True, env=environ)
+                self.assertEqual(proc.returncode, 0, proc)
+                self.assertEqual(proc.stdout.rstrip(), expected)
+                self.assertEqual(proc.stderr, '')
+
     @unittest.skipUnless(sys.platform == 'win32',
                          'bpo-32457 only applies on Windows')
     def test_argv0_normalization(self):

Lib/test/test_embed.py

@@ -523,6 +523,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         CONFIG_COMPAT['_pystats'] = 0
     if support.Py_DEBUG:
         CONFIG_COMPAT['run_presite'] = None
+    if support.Py_GIL_DISABLED:
+        CONFIG_COMPAT['enable_gil'] = -1
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
             'legacy_windows_stdio': 0,

Misc/python.man

@@ -607,6 +607,10 @@ output. Setting it to 0 deactivates this behavior.
 .IP PYTHON_HISTORY
 This environment variable can be used to set the location of a history file
 (on Unix, it is \fI~/.python_history\fP by default).
+.IP PYTHON_GIL
+If this variable is set to 1, the global interpreter lock (GIL) will be forced
+on. Setting it to 0 forces the GIL off. Only available in builds configured
+with \fB--disable-gil\fP.
 .SS Debug-mode variables
 Setting these variables only has an effect in a debug build of Python, that is,
 if Python was configured with the

Python/ceval_gil.c

@@ -219,6 +219,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate)
     // XXX assert(tstate == NULL || !tstate->_status.cleared);
 
     struct _gil_runtime_state *gil = ceval->gil;
+#ifdef Py_GIL_DISABLED
+    if (!gil->enabled) {
+        return;
+    }
+#endif
     if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) {
         Py_FatalError("drop_gil: GIL is not locked");
     }
@@ -294,6 +299,11 @@ take_gil(PyThreadState *tstate)
     assert(_PyThreadState_CheckConsistency(tstate));
     PyInterpreterState *interp = tstate->interp;
     struct _gil_runtime_state *gil = interp->ceval.gil;
+#ifdef Py_GIL_DISABLED
+    if (!gil->enabled) {
+        return;
+    }
+#endif
 
     /* Check that _PyEval_InitThreads() was called to create the lock */
     assert(gil_created(gil));
@@ -440,6 +450,11 @@ static void
 init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)
 {
     assert(!gil_created(gil));
+#ifdef Py_GIL_DISABLED
+    // gh-116329: Once it is safe to do so, change this condition to
+    // (enable_gil == _PyConfig_GIL_ENABLE), so the GIL is disabled by default.
+    gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil != _PyConfig_GIL_DISABLE;
+#endif
     create_gil(gil);
     assert(gil_created(gil));
     interp->ceval.gil = gil;