From ddacdb7d177b4a8bf2415f4fefee756bb8f94e31 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 23 Sep 2024 23:40:27 -0700 Subject: [PATCH 1/6] gh-119180: Add VALUE_WITH_FAKE_GLOBALS format to annotationlib --- Doc/library/annotationlib.rst | 11 +++++++++++ Lib/annotationlib.py | 7 +++++-- Lib/test/test_annotationlib.py | 14 +++++++++++++- Lib/typing.py | 15 ++++++++++----- Python/codegen.c | 10 +++++++++- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 1e72c5421674bc..868672e267fa09 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -144,6 +144,17 @@ Classes The exact values of these strings may change in future versions of Python. + .. attribute:: VALUE_WITH_FAKE_GLOBALS + :value: 4 + + Special value used to signal that an annotate function is being + evaluated in a special environment with fake globals. When passed this + value, annotate functions should either return the same value as for + the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` + to signal that they do not support execution in this environment. + This format is only used internally and should not be passed to + the functions in this module. + .. versionadded:: 3.14 .. class:: ForwardRef diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 0a67742a2b3081..bf16ebbf9e69a1 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -20,6 +20,7 @@ class Format(enum.IntEnum): VALUE = 1 FORWARDREF = 2 SOURCE = 3 + VALUE_WITH_FAKE_GLOBALS = 4 _Union = None @@ -459,6 +460,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): on the generated ForwardRef objects. """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") try: return annotate(format) except NotImplementedError: @@ -492,7 +495,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__, ) - annos = func(Format.VALUE) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) if _is_evaluate: return annos if isinstance(annos, str) else repr(annos) return { @@ -552,7 +555,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__, ) - result = func(Format.VALUE) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) for obj in globals.stringifiers: obj.__class__ = ForwardRef return result diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dd8ceb55a411fb..4bc87d24855d24 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -382,6 +382,18 @@ def f2(a: undefined): annotationlib.get_annotations(f1, format=0) with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=42) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + annotationlib.get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): annotationlib.get_annotations(f1, format=4) def test_custom_object_with_annotations(self): @@ -840,7 +852,7 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): def evaluate(format, exc=NotImplementedError): - if format != 1: + if format != 1 and format != 4: raise exc return undefined diff --git a/Lib/typing.py b/Lib/typing.py index 9377e771d60f4b..bacac8f3e4396d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2945,10 +2945,13 @@ def _make_eager_annotate(types): checked_types = {key: _type_check(val, f"field {key} annotation must be a type") for key, val in types.items()} def annotate(format): - if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): - return checked_types - else: - return _convert_to_source(types) + match format: + case annotationlib.Format.VALUE | annotationlib.Format.FORWARDREF: + return checked_types + case annotationlib.Format.SOURCE: + return _convert_to_source(types) + case _: + raise NotImplementedError(format) return annotate @@ -3242,8 +3245,10 @@ def __annotate__(format): } elif format == annotationlib.Format.SOURCE: own = _convert_to_source(own_annotations) - else: + elif format in (annotationlib.Format.FORWARDREF, annotationlib.Format.VALUE): own = own_checked_annotations + else: + raise NotImplementedError(format) annos.update(own) return annos diff --git a/Python/codegen.c b/Python/codegen.c index 0305f4299aec56..19340e3f205219 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -655,13 +655,21 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); - // if .format != 1: raise NotImplementedError + // if .format != 1 and .format != 4: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + + ADDOP_I(c, loc, LOAD_FAST, 0); + PyObject *four = PyLong_FromLong(4); + assert(four != NULL); + ADDOP_LOAD_CONST(c, loc, four); + ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, body); From ab0508901d77007930934af4647b37407bec657e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 12:22:11 -0700 Subject: [PATCH 2/6] Change the values --- Lib/annotationlib.py | 6 +++--- Lib/test/test_annotationlib.py | 22 +++++++++++----------- Lib/test/test_type_annotations.py | 4 ++-- Python/codegen.c | 14 ++++---------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index bf16ebbf9e69a1..cdb7ad91f18bee 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -18,9 +18,9 @@ class Format(enum.IntEnum): VALUE = 1 - FORWARDREF = 2 - SOURCE = 3 - VALUE_WITH_FAKE_GLOBALS = 4 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + SOURCE = 4 _Union = None diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 4bc87d24855d24..a7deea7608fef1 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -29,11 +29,14 @@ def test_enum(self): self.assertEqual(annotationlib.Format.VALUE.value, 1) self.assertEqual(annotationlib.Format.VALUE, 1) - self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) - self.assertEqual(annotationlib.Format.FORWARDREF, 2) + self.assertEqual(annotationlib.Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(annotationlib.Format.VALUE_WITH_FAKE_GLOBALS, 2) - self.assertEqual(annotationlib.Format.SOURCE.value, 3) - self.assertEqual(annotationlib.Format.SOURCE, 3) + self.assertEqual(annotationlib.Format.FORWARDREF.value, 3) + self.assertEqual(annotationlib.Format.FORWARDREF, 3) + + self.assertEqual(annotationlib.Format.SOURCE.value, 4) + self.assertEqual(annotationlib.Format.SOURCE, 4) class TestForwardRefFormat(unittest.TestCase): @@ -370,16 +373,13 @@ def f2(a: undefined): annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), {"a": fwd}, ) - self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) + self.assertEqual(annotationlib.get_annotations(f2, format=3), {"a": fwd}) self.assertEqual( annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), {"a": "int"}, ) - self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) - - with self.assertRaises(ValueError): - annotationlib.get_annotations(f1, format=0) + self.assertEqual(annotationlib.get_annotations(f1, format=4), {"a": "int"}) with self.assertRaises(ValueError): annotationlib.get_annotations(f1, format=42) @@ -394,7 +394,7 @@ def f2(a: undefined): ValueError, r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", ): - annotationlib.get_annotations(f1, format=4) + annotationlib.get_annotations(f1, format=2) def test_custom_object_with_annotations(self): class C: @@ -852,7 +852,7 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): def evaluate(format, exc=NotImplementedError): - if format != 1 and format != 4: + if format > 2: raise exc return undefined diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 91082e6b23c04b..da87d88163e568 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -316,7 +316,7 @@ def test_module(self): ns = run_code("x: undefined = 1") anno = ns["__annotate__"] with self.assertRaises(NotImplementedError): - anno(2) + anno(3) with self.assertRaises(NameError): anno(1) @@ -376,7 +376,7 @@ class X: annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.SOURCE) - with self.assertRaises(NotImplementedError): + with self.assertRaises(TypeError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Python/codegen.c b/Python/codegen.c index bf29995b9d2121..0fb599c145fc87 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -655,21 +655,15 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); - // if .format != 1 and .format != 4: raise NotImplementedError + // if .format > 2: raise NotImplementedError _Py_DECLARE_STR(format, ".format"); + PyObject *two = PyLong_FromLong(2); ADDOP_I(c, loc, LOAD_FAST, 0); - ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); - ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + ADDOP_LOAD_CONST(c, loc, two); + ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); - ADDOP_I(c, loc, LOAD_FAST, 0); - PyObject *four = PyLong_FromLong(4); - assert(four != NULL); - ADDOP_LOAD_CONST(c, loc, four); - ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); - ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); - ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, body); From 3ba132a9ae3e945ffb1826d72475f67132178c0c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 12:52:57 -0700 Subject: [PATCH 3/6] fix constevaluator --- Objects/typevarobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index d3656155fae330..9d61d943c69e9d 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // SOURCE + if (format == 4) { // SOURCE PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; From 2012bb477d017ce3bab648da14b960c57ab956e0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 17:00:55 -0700 Subject: [PATCH 4/6] fix tests --- Lib/annotationlib.py | 2 ++ Lib/test/test_annotationlib.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index d9db44ff0d7ca5..7849a7f0b54da2 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -697,6 +697,8 @@ def get_annotations( # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) return ann + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") case _: raise ValueError(f"Unsupported format {format!r}") diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index debb2289954977..1549a25a612692 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -467,6 +467,8 @@ def foo(a: int, b: str): foo.__annotations__ = {"a": "foo", "b": "str"} for format in annotationlib.Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -778,6 +780,8 @@ def __annotations__(self): wa = WeirdAnnotations() for format in Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue with ( self.subTest(format=format), self.assertRaisesRegex( From 4081f8f64a01332f34323db12678a14071cfbedf Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 29 Sep 2024 06:04:16 -0700 Subject: [PATCH 5/6] Add C enum for format --- Include/internal/pycore_object.h | 7 +++++++ Objects/typevarobject.c | 4 ++-- Python/codegen.c | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 80b588815bc9cf..7df23911920640 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -879,6 +879,13 @@ PyAPI_DATA(int) _Py_SwappedOp[]; extern void _Py_GetConstant_Init(void); +enum _PyAnnotateFormat { + _Py_ANNOTATE_FORMAT_VALUE = 1, + _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2, + _Py_ANNOTATE_FORMAT_FORWARDREF = 3, + _Py_ANNOTATE_FORMAT_STRING = 4, +}; + #ifdef __cplusplus } #endif diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index c2691463594c44..bfb889ddccabac 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1,6 +1,6 @@ // TypeVar, TypeVarTuple, and ParamSpec #include "Python.h" -#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK, PyAnnotateFormat #include "pycore_typevarobject.h" #include "pycore_unionobject.h" // _Py_union_type_or @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 4) { // STRING + if (format == _Py_ANNOTATE_FORMAT_STRING) { PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; diff --git a/Python/codegen.c b/Python/codegen.c index 0fb599c145fc87..6cca6ef939c007 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -24,6 +24,7 @@ #include "pycore_instruction_sequence.h" // _PyInstructionSequence_NewLabel() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_object.h" // _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject @@ -655,9 +656,8 @@ codegen_setup_annotations_scope(compiler *c, location loc, codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, key, loc.lineno, NULL, &umd)); - // if .format > 2: raise NotImplementedError - _Py_DECLARE_STR(format, ".format"); - PyObject *two = PyLong_FromLong(2); + // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + PyObject *two = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); ADDOP_I(c, loc, LOAD_FAST, 0); ADDOP_LOAD_CONST(c, loc, two); ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); From 718205f02eb42043ef27430cfdda056b04353309 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 26 Nov 2024 07:16:58 -0800 Subject: [PATCH 6/6] Rename poorly named variable --- Python/codegen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 6e4fef58f7bd1c..a5e550cf8c947e 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -674,11 +674,11 @@ codegen_setup_annotations_scope(compiler *c, location loc, key, loc.lineno, NULL, &umd)); // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError - PyObject *two = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); + PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); _Py_DECLARE_STR(format, ".format"); ADDOP_I(c, loc, LOAD_FAST, 0); - ADDOP_LOAD_CONST(c, loc, two); + ADDOP_LOAD_CONST(c, loc, value_with_fake_globals); ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);