Merge branch 'main' into run-test-suite-with-warnings-as-error

graingert · web-flow · commit e7af412e2d03 · 2025-01-16T02:23:58.000Z
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
@@ -380,13 +380,15 @@ ABC hierarchy::
 
 .. class:: ResourceLoader
 
+   *Superseded by TraversableResources*
+
     An abstract base class for a :term:`loader` which implements the optional
     :pep:`302` protocol for loading arbitrary resources from the storage
     back-end.
 
     .. deprecated:: 3.7
        This ABC is deprecated in favour of supporting resource loading
-       through :class:`importlib.resources.abc.ResourceReader`.
+       through :class:`importlib.resources.abc.TraversableResources`.
 
     .. abstractmethod:: get_data(path)
 
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
@@ -45,12 +45,13 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
  * the per-object lock.
  */
 #ifdef Py_GIL_DISABLED
-#  define _PyGC_BITS_TRACKED        (1)     // Tracked by the GC
-#  define _PyGC_BITS_FINALIZED      (2)     // tp_finalize was called
-#  define _PyGC_BITS_UNREACHABLE    (4)
-#  define _PyGC_BITS_FROZEN         (8)
-#  define _PyGC_BITS_SHARED         (16)
-#  define _PyGC_BITS_DEFERRED       (64)    // Use deferred reference counting
+#  define _PyGC_BITS_TRACKED        (1<<0)     // Tracked by the GC
+#  define _PyGC_BITS_FINALIZED      (1<<1)     // tp_finalize was called
+#  define _PyGC_BITS_UNREACHABLE    (1<<2)
+#  define _PyGC_BITS_FROZEN         (1<<3)
+#  define _PyGC_BITS_SHARED         (1<<4)
+#  define _PyGC_BITS_ALIVE          (1<<5)    // Reachable from a known root.
+#  define _PyGC_BITS_DEFERRED       (1<<6)    // Use deferred reference counting
 #endif
 
 #ifdef Py_GIL_DISABLED
@@ -330,6 +331,9 @@ struct _gc_runtime_state {
        collections, and are awaiting to undergo a full collection for
        the first time. */
     Py_ssize_t long_lived_pending;
+
+    /* True if gc.freeze() has been used. */
+    int freeze_active;
 #endif
 };
 
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -299,12 +299,6 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra);
 extern int _PyType_CheckConsistency(PyTypeObject *type);
 extern int _PyDict_CheckConsistency(PyObject *mp, int check_content);
 
-/* Update the Python traceback of an object. This function must be called
-   when a memory block is reused from a free list.
-
-   Internal function called by _Py_NewReference(). */
-extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*);
-
 // Fast inlined version of PyType_HasFeature()
 static inline int
 _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h
@@ -25,7 +25,7 @@ struct _PyTraceMalloc_Config {
     } initialized;
 
     /* Is tracemalloc tracing memory allocations?
-       Variable protected by the GIL */
+       Variable protected by the TABLES_LOCK(). */
     int tracing;
 
     /* limit of the number of frames in a traceback, 1 by default.
@@ -85,14 +85,14 @@ struct _tracemalloc_runtime_state {
     size_t peak_traced_memory;
     /* Hash table used as a set to intern filenames:
        PyObject* => PyObject*.
-       Protected by the GIL */
+       Protected by the TABLES_LOCK(). */
     _Py_hashtable_t *filenames;
     /* Buffer to store a new traceback in traceback_new().
-       Protected by the GIL. */
+       Protected by the TABLES_LOCK(). */
     struct tracemalloc_traceback *traceback;
     /* Hash table used as a set to intern tracebacks:
        traceback_t* => traceback_t*
-       Protected by the GIL */
+       Protected by the TABLES_LOCK(). */
     _Py_hashtable_t *tracebacks;
     /* pointer (void*) => trace (trace_t*).
        Protected by TABLES_LOCK(). */
@@ -144,7 +144,7 @@ extern PyObject* _PyTraceMalloc_GetTraces(void);
 extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj);
 
 /* Initialize tracemalloc */
-extern int _PyTraceMalloc_Init(void);
+extern PyStatus _PyTraceMalloc_Init(void);
 
 /* Start tracemalloc */
 extern int _PyTraceMalloc_Start(int max_nframe);
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
@@ -74,7 +74,7 @@ def __init__(self):
         import warnings
         warnings.warn('importlib.abc.ResourceLoader is deprecated in '
                       'favour of supporting resource loading through '
-                      'importlib.resources.abc.ResourceReader.',
+                      'importlib.resources.abc.TraversableResources.',
                       DeprecationWarning, stacklevel=2)
         super().__init__()
 
diff --git a/Lib/opcode.py b/Lib/opcode.py
@@ -17,8 +17,9 @@
 EXTENDED_ARG = opmap['EXTENDED_ARG']
 
 opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
-for op, i in opmap.items():
-    opname[i] = op
+for m in (opmap, _specialized_opmap):
+    for op, i in m.items():
+        opname[i] = op
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=')
 
diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py
@@ -38,6 +38,13 @@ def test_is_valid(self):
         opcodes = [dis.opmap[opname] for opname in names]
         self.check_bool_function_result(_opcode.is_valid, opcodes, True)
 
+    def test_opmaps(self):
+        def check_roundtrip(name, map):
+            return self.assertEqual(opcode.opname[map[name]], name)
+
+        check_roundtrip('BINARY_OP', opcode.opmap)
+        check_roundtrip('BINARY_OP_ADD_INT', opcode._specialized_opmap)
+
     def test_oplists(self):
         def check_function(self, func, expected):
             for op in [-10, 520]:
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
@@ -659,7 +659,8 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self):
                 stderr.splitlines()[-3:],
                 [   b'    foo = """\\q"""',
                     b'          ^^^^^^^^',
-                    b'SyntaxError: invalid escape sequence \'\\q\''
+                    b'SyntaxError: "\\q" is an invalid escape sequence. '
+                    b'Did you mean "\\\\q"? A raw string is also an option.'
                 ],
             )
 
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
@@ -282,7 +282,7 @@ def test_warning(self):
         # Test that the warning is only returned once.
         with warnings_helper.check_warnings(
                 ('"is" with \'str\' literal', SyntaxWarning),
-                ("invalid escape sequence", SyntaxWarning),
+                ('"\\\\e" is an invalid escape sequence', SyntaxWarning),
                 ) as w:
             compile_command(r"'\e' is 0")
             self.assertEqual(len(w.warnings), 2)
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
@@ -999,12 +999,14 @@ def test_boundaries(self):
     def test_widths(self):
         long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT',
                             'INSTRUMENTED_CALL_FUNCTION_EX'])
-        for opcode, opname in enumerate(dis.opname):
+        for op, opname in enumerate(dis.opname):
             if opname in long_opcodes or opname.startswith("INSTRUMENTED"):
                 continue
+            if opname in opcode._specialized_opmap:
+                continue
             with self.subTest(opname=opname):
                 width = dis._OPNAME_WIDTH
-                if opcode in dis.hasarg:
+                if op in dis.hasarg:
                     width += 1 + dis._OPARG_WIDTH
                 self.assertLessEqual(len(opname), width)
 
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
@@ -116,7 +116,9 @@ def test_eval_str_invalid_escape(self):
             warnings.simplefilter('always', category=SyntaxWarning)
             eval("'''\n\\z'''")
         self.assertEqual(len(w), 1)
-        self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'")
+        self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. '
+                         r'Such sequences will not work in the future. '
+                         r'Did you mean "\\z"? A raw string is also an option.')
         self.assertEqual(w[0].filename, '<string>')
         self.assertEqual(w[0].lineno, 1)
 
@@ -126,7 +128,8 @@ def test_eval_str_invalid_escape(self):
                 eval("'''\n\\z'''")
             exc = cm.exception
         self.assertEqual(w, [])
-        self.assertEqual(exc.msg, r"invalid escape sequence '\z'")
+        self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
+                         r'Did you mean "\\z"? A raw string is also an option.')
         self.assertEqual(exc.filename, '<string>')
         self.assertEqual(exc.lineno, 1)
         self.assertEqual(exc.offset, 1)
@@ -153,7 +156,9 @@ def test_eval_str_invalid_octal_escape(self):
             eval("'''\n\\407'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(str(w[0].message),
-                         r"invalid octal escape sequence '\407'")
+                         r'"\407" is an invalid octal escape sequence. '
+                         r'Such sequences will not work in the future. '
+                         r'Did you mean "\\407"? A raw string is also an option.')
         self.assertEqual(w[0].filename, '<string>')
         self.assertEqual(w[0].lineno, 1)
 
@@ -163,7 +168,8 @@ def test_eval_str_invalid_octal_escape(self):
                 eval("'''\n\\407'''")
             exc = cm.exception
         self.assertEqual(w, [])
-        self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'")
+        self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
+                                 r'Did you mean "\\407"? A raw string is also an option.')
         self.assertEqual(exc.filename, '<string>')
         self.assertEqual(exc.lineno, 1)
         self.assertEqual(exc.offset, 1)
@@ -205,7 +211,9 @@ def test_eval_bytes_invalid_escape(self):
             warnings.simplefilter('always', category=SyntaxWarning)
             eval("b'''\n\\z'''")
         self.assertEqual(len(w), 1)
-        self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'")
+        self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. '
+                         r'Such sequences will not work in the future. '
+                         r'Did you mean "\\z"? A raw string is also an option.')
         self.assertEqual(w[0].filename, '<string>')
         self.assertEqual(w[0].lineno, 1)
 
@@ -215,7 +223,8 @@ def test_eval_bytes_invalid_escape(self):
                 eval("b'''\n\\z'''")
             exc = cm.exception
         self.assertEqual(w, [])
-        self.assertEqual(exc.msg, r"invalid escape sequence '\z'")
+        self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. '
+                         r'Did you mean "\\z"? A raw string is also an option.')
         self.assertEqual(exc.filename, '<string>')
         self.assertEqual(exc.lineno, 1)
 
@@ -228,8 +237,9 @@ def test_eval_bytes_invalid_octal_escape(self):
             warnings.simplefilter('always', category=SyntaxWarning)
             eval("b'''\n\\407'''")
         self.assertEqual(len(w), 1)
-        self.assertEqual(str(w[0].message),
-                         r"invalid octal escape sequence '\407'")
+        self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. '
+                         r'Such sequences will not work in the future. '
+                         r'Did you mean "\\407"? A raw string is also an option.')
         self.assertEqual(w[0].filename, '<string>')
         self.assertEqual(w[0].lineno, 1)
 
@@ -239,7 +249,8 @@ def test_eval_bytes_invalid_octal_escape(self):
                 eval("b'''\n\\407'''")
             exc = cm.exception
         self.assertEqual(w, [])
-        self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'")
+        self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. '
+                         r'Did you mean "\\407"? A raw string is also an option.')
         self.assertEqual(exc.filename, '<string>')
         self.assertEqual(exc.lineno, 1)
 
diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py
@@ -651,7 +651,9 @@ def test_multiquote_joined_string(self):
 
     def test_backslash_in_format_spec(self):
         import re
-        msg = re.escape("invalid escape sequence '\\ '")
+        msg = re.escape('"\\ " is an invalid escape sequence. '
+                        'Such sequences will not work in the future. '
+                        'Did you mean "\\\\ "? A raw string is also an option.')
         with self.assertWarnsRegex(SyntaxWarning, msg):
             self.check_ast_roundtrip("""f"{x:\\ }" """)
         self.check_ast_roundtrip("""f"{x:\\n}" """)
diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst
@@ -941,7 +941,7 @@ result from ``entry_points()`` as deprecated.
 
 ..
 
-.. gh: 47383
+.. gh-issue: 47383
 .. date: 2021-04-08-19-32-26
 .. nonce: YI1hdL
 .. section: Library
diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst
@@ -570,7 +570,7 @@ planned). Patch by Alex Waygood.
 
 ..
 
-.. gh: 78157
+.. gh-issue: 78157
 .. date: 2022-05-05-20-40-45
 .. nonce: IA_9na
 .. section: Library
@@ -1289,7 +1289,7 @@ Deprecate the chunk module.
 
 ..
 
-.. gh: 91498
+.. gh-issue: 91498
 .. date: 2022-04-10-08-39-44
 .. nonce: 8oII92
 .. section: Library
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst
@@ -0,0 +1 @@
+Improved the ``SyntaxWarning`` message for invalid escape sequences to clarify that such sequences will raise a ``SyntaxError`` in future Python releases. The new message also suggests a potential fix, i.e., ``Did you mean "\\e"?``.
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst
@@ -0,0 +1,6 @@
+Add a marking phase to the free-threaded GC.  This is similar to what was
+done in GH-126491.  Since the free-threaded GC does not have generations and
+is not incremental, the marking phase looks for all objects reachable from
+known roots.  The roots are objects known to not be garbage, like the module
+dictionary for :mod:`sys`.  For most programs, this marking phase should
+make the GC a bit faster since typically less work is done per object.
diff --git a/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst b/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst
@@ -0,0 +1 @@
+Add specialized opcodes to ``opcode.opname``.
diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c
@@ -215,18 +215,14 @@ static struct PyModuleDef module_def = {
 PyMODINIT_FUNC
 PyInit__tracemalloc(void)
 {
-    PyObject *m;
-    m = PyModule_Create(&module_def);
-    if (m == NULL)
+    PyObject *mod = PyModule_Create(&module_def);
+    if (mod == NULL) {
         return NULL;
+    }
+
 #ifdef Py_GIL_DISABLED
-    PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED);
+    PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
 #endif
 
-    if (_PyTraceMalloc_Init() < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
+    return mod;
 }
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
@@ -1205,7 +1205,8 @@ PyObject *PyBytes_DecodeEscape(const char *s,
         unsigned char c = *first_invalid_escape;
         if ('4' <= c && c <= '7') {
             if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
-                                 "invalid octal escape sequence '\\%.3s'",
+                                 "b\"\\%.3s\" is an invalid octal escape sequence. "
+                                 "Such sequences will not work in the future. ",
                                  first_invalid_escape) < 0)
             {
                 Py_DECREF(result);
@@ -1214,7 +1215,8 @@ PyObject *PyBytes_DecodeEscape(const char *s,
         }
         else {
             if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
-                                 "invalid escape sequence '\\%c'",
+                                 "b\"\\%c\" is an invalid escape sequence. "
+                                 "Such sequences will not work in the future. ",
                                  c) < 0)
             {
                 Py_DECREF(result);
@@ -1223,7 +1225,6 @@ PyObject *PyBytes_DecodeEscape(const char *s,
         }
     }
     return result;
-
 }
 /* -------------------------------------------------------------------- */
 /* object api */
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
@@ -6853,7 +6853,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
         unsigned char c = *first_invalid_escape;
         if ('4' <= c && c <= '7') {
             if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
-                                 "invalid octal escape sequence '\\%.3s'",
+                                 "\"\\%.3s\" is an invalid octal escape sequence. "
+                                 "Such sequences will not work in the future. ",
                                  first_invalid_escape) < 0)
             {
                 Py_DECREF(result);
@@ -6862,7 +6863,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
         }
         else {
             if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
-                                 "invalid escape sequence '\\%c'",
+                                 "\"\\%c\" is an invalid escape sequence. "
+                                 "Such sequences will not work in the future. ",
                                  c) < 0)
             {
                 Py_DECREF(result);
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c

Original file line number	Diff line number	Diff line change
`@@ -659,7 +659,8 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self):`
`659`	`659`	`stderr.splitlines()[-3:],`
`660`	`660`	`[ b' foo = """\\q"""',`
`661`	`661`	`b' ^^^^^^^^',`
`662`		`- b'SyntaxError: invalid escape sequence \'\\q\''`
	`662`	`+ b'SyntaxError: "\\q" is an invalid escape sequence. '`
	`663`	`+ b'Did you mean "\\\\q"? A raw string is also an option.'`
`663`	`664`	`],`
`664`	`665`	`)`
`665`	`666`