From e2645635077754d4b37151ae8bcab37b8602f97c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 25 Feb 2024 12:12:59 +0000 Subject: [PATCH 01/12] gh-115775: Use static analysis to estimate how many items a class dict can contain --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_compile.py | 58 +++++++++++++++++++ ...-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst | 3 + Python/compile.c | 52 +++++++++++++++++ 7 files changed, 119 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d2287687181450..9eab24e4b75818 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -623,6 +623,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__enter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__expected_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fb9ec44d3f52aa..c1e7e460be276b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -112,6 +112,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__enter__) STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) + STRUCT_FOR_ID(__expected_attributes__) STRUCT_FOR_ID(__file__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 658bf8030f661d..ef84c96fe7b049 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -621,6 +621,7 @@ extern "C" { INIT_ID(__enter__), \ INIT_ID(__eq__), \ INIT_ID(__exit__), \ + INIT_ID(__expected_attributes__), \ INIT_ID(__file__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d72353d56eae60..9c394e9d27dbf3 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -177,6 +177,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__exit__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__expected_attributes__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__file__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d3e69bfedccd07..a4c85994f1104e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1960,6 +1960,64 @@ def test_load_super_attr(self): ) +class TestExpectedAttributes(unittest.TestCase): + + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + + self.assertIsInstance(C.__expected_attributes__, tuple) + self.assertEqual(sorted(C.__expected_attributes__), ['a', 'b']) + + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__expected_attributes__), ['u', 'v', 'x', 'y', 'z']) + + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__expected_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__expected_attributes__), ['y', 'z']) + + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__expected_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__expected_attributes__), ['y', 'z']) + + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object # stays within reasonable bounds (see issue #21523 for an example diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst new file mode 100644 index 00000000000000..5565d1936a4813 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst @@ -0,0 +1,3 @@ +Compiler populate the new ``__expected_attributes__`` field on a class with +the names of attributes of this class which are accessed through self.X from +any function in its body. diff --git a/Python/compile.c b/Python/compile.c index 3291d31a5cc8ed..83a32612449d40 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -359,6 +359,7 @@ struct compiler_unit { int u_scope_type; PyObject *u_private; /* for private name mangling */ + PyObject *u_expected_attributes; /* for class: attributes accessed via self.X */ instr_sequence u_instr_sequence; /* codegen output */ @@ -690,9 +691,27 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); + Py_CLEAR(u->u_expected_attributes); PyMem_Free(u); } +static struct compiler_unit * +get_class_compiler_unit(struct compiler *c) +{ + Py_ssize_t stack_size = PyList_GET_SIZE(c->c_stack); + assert(stack_size >= 1); + for (Py_ssize_t i = stack_size - 1; i >= 0; i--) { + PyObject *capsule = PyList_GET_ITEM(c->c_stack, i); + struct compiler_unit *u = (struct compiler_unit *)PyCapsule_GetPointer( + capsule, CAPSULE_NAME); + assert(u); + if (u->u_scope_type == COMPILER_SCOPE_CLASS) { + return u; + } + } + return NULL; +} + static int compiler_set_qualname(struct compiler *c) { @@ -1336,6 +1355,16 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_private = NULL; + if (scope_type == COMPILER_SCOPE_CLASS) { + u->u_expected_attributes = PySet_New(0); + if (!u->u_expected_attributes) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_expected_attributes = NULL; + } /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -2517,6 +2546,17 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } + assert(c->u->u_expected_attributes); + PyObject *expected_attributes = PySequence_Tuple(c->u->u_expected_attributes); + if (expected_attributes == NULL) { + compiler_exit_scope(c); + return ERROR; + } + ADDOP_LOAD_CONST(c, NO_LOCATION, expected_attributes); + if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__expected_attributes__), Store) < 0) { + compiler_exit_scope(c); + return ERROR; + } /* The following code is artificial */ /* Set __classdictcell__ if necessary */ if (c->u->u_ste->ste_needs_classdict) { @@ -2657,6 +2697,7 @@ compiler_class(struct compiler *c, stmt_ty s) s->v.ClassDef.keywords)); PyCodeObject *co = optimize_and_assemble(c, 0); + compiler_exit_scope(c); if (co == NULL) { return ERROR; @@ -6246,6 +6287,17 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP(c, loc, NOP); return SUCCESS; } + if (e->v.Attribute.value->kind == Name_kind && + _PyUnicode_EqualToASCIIString(e->v.Attribute.value->v.Name.id, "self")) + { + struct compiler_unit *class_u = get_class_compiler_unit(c); + if (class_u != NULL) { + assert(class_u->u_scope_type == COMPILER_SCOPE_CLASS); + assert(class_u->u_expected_attributes); + RETURN_IF_ERROR( + PySet_Add(class_u->u_expected_attributes, e->v.Attribute.attr)); + } + } VISIT(c, expr, e->v.Attribute.value); loc = LOC(e); loc = update_start_location_to_match_attr(c, loc, e); From b48d320c854dd4f4a86b50273a2d6aa4f93921d5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 26 Feb 2024 20:01:14 +0000 Subject: [PATCH 02/12] whitespace --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 83a32612449d40..0903a4e5e213d0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2697,7 +2697,7 @@ compiler_class(struct compiler *c, stmt_ty s) s->v.ClassDef.keywords)); PyCodeObject *co = optimize_and_assemble(c, 0); - + compiler_exit_scope(c); if (co == NULL) { return ERROR; From cd989ad2c2fdc7c477bdb98ebd7215a276c6f051 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 26 Feb 2024 23:21:20 +0000 Subject: [PATCH 03/12] fix failing tests --- Lib/enum.py | 3 ++- Lib/pydoc.py | 3 ++- Lib/test/test_descr.py | 10 +++++----- Lib/test/test_io.py | 2 +- Lib/test/test_metaclass.py | 8 +++++--- Lib/typing.py | 2 +- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 5c5e711f9b078f..afbe8507ae7c6b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -2018,7 +2018,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__expected_attributes__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 08fd7aba7c9472..b3d310006e36af 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -313,7 +313,8 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__expected_attributes__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 5404d8d3b99d5d..a8a1b9f161e0ce 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5079,8 +5079,8 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__expected_attributes__', + '__module__', '__weakref__', 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5089,7 +5089,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 5) + self.assertEqual(len(values), 6) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5099,8 +5099,8 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__expected_attributes__', + '__module__', '__weakref__', 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5491c0575dbd3f..dc9924e2a50da3 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1160,7 +1160,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__expected_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 36e8ab4cda3dad..90d4ffb901b8e5 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -167,6 +167,7 @@ d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 + d['__expected_attributes__'] = () >>> Use a metaclass that doesn't derive from type. @@ -182,12 +183,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -208,8 +209,9 @@ d['a'] = 1 d['a'] = 2 d['b'] = 3 + d['__expected_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/typing.py b/Lib/typing.py index 533b64062834d2..b97ac19cadc674 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1717,7 +1717,7 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', + '__match_args__', '__expected_attributes__', }) # These special attributes will be not collected as protocol members. From 6af0e7f21a058c6e390b0ec184457a2fe2bfe4f6 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 17:07:02 +0000 Subject: [PATCH 04/12] fix refleak --- Python/compile.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/compile.c b/Python/compile.c index 0903a4e5e213d0..b5ac1e0c1ea9e8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2553,6 +2553,7 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) return ERROR; } ADDOP_LOAD_CONST(c, NO_LOCATION, expected_attributes); + Py_CLEAR(expected_attributes); if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__expected_attributes__), Store) < 0) { compiler_exit_scope(c); return ERROR; From add1804d7d7c5924b345f6a983252afcbf9b294e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 27 Feb 2024 14:03:56 +0000 Subject: [PATCH 05/12] pass keys_size to _PyDict_NewKeysForClass --- Include/internal/pycore_dict.h | 2 +- Objects/dictobject.c | 6 +++--- Objects/typeobject.c | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index ef59960dbab071..520862223cf677 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -81,7 +81,7 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -extern PyDictKeysObject *_PyDict_NewKeysForClass(void); +extern PyDictKeysObject *_PyDict_NewKeysForClass(Py_ssize_t keys_size); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 536746ca41eed5..b2de52444142d0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -6522,18 +6522,18 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) /* Returns NULL if cannot allocate a new PyDictKeysObject, but does not set an error */ PyDictKeysObject * -_PyDict_NewKeysForClass(void) +_PyDict_NewKeysForClass(Py_ssize_t keys_size) { PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = new_keys_object( - interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + interp, estimate_log2_keysize(keys_size), 1); if (keys == NULL) { PyErr_Clear(); } else { assert(keys->dk_nentries == 0); /* Set to max size+1 as it will shrink by one before each new object */ - keys->dk_usable = SHARED_KEYS_MAX_SIZE; + keys->dk_usable = keys_size; keys->dk_kind = DICT_KEYS_SPLIT; } return keys; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 82822784aaf407..b798e01e746c7b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7760,6 +7760,12 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) return 0; } +static Py_ssize_t +expected_number_of_shared_keys(PyTypeObject *type) +{ + return SHARED_KEYS_MAX_SIZE; +} + static int type_ready_managed_dict(PyTypeObject *type) { @@ -7775,7 +7781,7 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(); + et->ht_cached_keys = _PyDict_NewKeysForClass(expected_number_of_shared_keys(type)); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; From 1bb4ad6c0f35679fbf41b53d9b0f0c54d41a4d01 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 27 Feb 2024 15:17:56 +0000 Subject: [PATCH 06/12] simple implementation of _PyType_SharedKeysMaxSize (no inheritance lookup) --- Include/internal/pycore_dict.h | 2 +- Include/internal/pycore_object.h | 3 +++ Objects/dictobject.c | 6 +++--- Objects/typeobject.c | 17 +++++++++++++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 520862223cf677..3d5144bdfc3475 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -256,7 +256,7 @@ PyAPI_FUNC(PyObject *)_PyDict_FromItems( static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { - assert(ix < SHARED_KEYS_MAX_SIZE); + // assert(ix < SHARED_KEYS_MAX_SIZE); uint8_t *size_ptr = ((uint8_t *)values)-2; int size = *size_ptr; assert(size+2 < DICT_VALUES_SIZE(values)); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 13fe543133f11e..6d73b4602fc6f1 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -237,6 +237,7 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); extern int _PyType_CheckConsistency(PyTypeObject *type); extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); + /* Update the Python traceback of an object. This function must be called when a memory block is reused from a free list. @@ -255,6 +256,8 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); +extern Py_ssize_t _PyType_SharedKeysMaxSize(PyTypeObject *type); + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b2de52444142d0..19619ab27abcc3 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -623,7 +623,7 @@ static PyDictKeysObject empty_keys_struct = { static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { - assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + // assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); assert(i < (((char *)mp->ma_values)[-2])); return ((char *)mp->ma_values)[-3-i]; } @@ -671,7 +671,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) } else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); - CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + // CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); } if (check_content) { @@ -1615,7 +1615,7 @@ insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name, Py_hash_t hash ep->me_key = Py_NewRef(name); split_keys_entry_added(keys); } - assert (ix < SHARED_KEYS_MAX_SIZE); + // assert (ix < SHARED_KEYS_MAX_SIZE); return ix; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b798e01e746c7b..f5cedf9ae8526d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7760,9 +7760,18 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) return 0; } -static Py_ssize_t -expected_number_of_shared_keys(PyTypeObject *type) -{ +Py_ssize_t +_PyType_SharedKeysMaxSize(PyTypeObject *type) +{ + if (PyObject_HasAttr((PyObject*)type, &_Py_ID(__expected_attributes__))) { + PyObject *attrs = PyObject_GetAttr((PyObject*)type, &_Py_ID(__expected_attributes__)); + if (attrs != NULL && PyTuple_Check(attrs)) { + Py_ssize_t num_keys = PyTuple_GET_SIZE(attrs); + if (num_keys > SHARED_KEYS_MAX_SIZE) { + return 1 + num_keys; + } + } + } return SHARED_KEYS_MAX_SIZE; } @@ -7781,7 +7790,7 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(expected_number_of_shared_keys(type)); + et->ht_cached_keys = _PyDict_NewKeysForClass(_PyType_SharedKeysMaxSize(type)); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; From 4f277d8b18f6e9f2f6b05d8417cb1aa6a95ffd23 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 17:07:45 +0000 Subject: [PATCH 07/12] Revert "simple implementation of _PyType_SharedKeysMaxSize (no inheritance lookup)" This reverts commit 1bb4ad6c0f35679fbf41b53d9b0f0c54d41a4d01. --- Include/internal/pycore_dict.h | 2 +- Include/internal/pycore_object.h | 3 --- Objects/dictobject.c | 6 +++--- Objects/typeobject.c | 17 ++++------------- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 3d5144bdfc3475..520862223cf677 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -256,7 +256,7 @@ PyAPI_FUNC(PyObject *)_PyDict_FromItems( static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { - // assert(ix < SHARED_KEYS_MAX_SIZE); + assert(ix < SHARED_KEYS_MAX_SIZE); uint8_t *size_ptr = ((uint8_t *)values)-2; int size = *size_ptr; assert(size+2 < DICT_VALUES_SIZE(values)); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 6d73b4602fc6f1..13fe543133f11e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -237,7 +237,6 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); extern int _PyType_CheckConsistency(PyTypeObject *type); extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); - /* Update the Python traceback of an object. This function must be called when a memory block is reused from a free list. @@ -256,8 +255,6 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); -extern Py_ssize_t _PyType_SharedKeysMaxSize(PyTypeObject *type); - /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 19619ab27abcc3..b2de52444142d0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -623,7 +623,7 @@ static PyDictKeysObject empty_keys_struct = { static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { - // assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); assert(i < (((char *)mp->ma_values)[-2])); return ((char *)mp->ma_values)[-3-i]; } @@ -671,7 +671,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) } else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); - // CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); } if (check_content) { @@ -1615,7 +1615,7 @@ insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name, Py_hash_t hash ep->me_key = Py_NewRef(name); split_keys_entry_added(keys); } - // assert (ix < SHARED_KEYS_MAX_SIZE); + assert (ix < SHARED_KEYS_MAX_SIZE); return ix; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f5cedf9ae8526d..b798e01e746c7b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7760,18 +7760,9 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) return 0; } -Py_ssize_t -_PyType_SharedKeysMaxSize(PyTypeObject *type) -{ - if (PyObject_HasAttr((PyObject*)type, &_Py_ID(__expected_attributes__))) { - PyObject *attrs = PyObject_GetAttr((PyObject*)type, &_Py_ID(__expected_attributes__)); - if (attrs != NULL && PyTuple_Check(attrs)) { - Py_ssize_t num_keys = PyTuple_GET_SIZE(attrs); - if (num_keys > SHARED_KEYS_MAX_SIZE) { - return 1 + num_keys; - } - } - } +static Py_ssize_t +expected_number_of_shared_keys(PyTypeObject *type) +{ return SHARED_KEYS_MAX_SIZE; } @@ -7790,7 +7781,7 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(_PyType_SharedKeysMaxSize(type)); + et->ht_cached_keys = _PyDict_NewKeysForClass(expected_number_of_shared_keys(type)); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; From 010c64fcb6d13874bb52982e37745585920a0aac Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 17:08:05 +0000 Subject: [PATCH 08/12] Revert "pass keys_size to _PyDict_NewKeysForClass" This reverts commit add1804d7d7c5924b345f6a983252afcbf9b294e. --- Include/internal/pycore_dict.h | 2 +- Objects/dictobject.c | 6 +++--- Objects/typeobject.c | 8 +------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 520862223cf677..ef59960dbab071 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -81,7 +81,7 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -extern PyDictKeysObject *_PyDict_NewKeysForClass(Py_ssize_t keys_size); +extern PyDictKeysObject *_PyDict_NewKeysForClass(void); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b2de52444142d0..536746ca41eed5 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -6522,18 +6522,18 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) /* Returns NULL if cannot allocate a new PyDictKeysObject, but does not set an error */ PyDictKeysObject * -_PyDict_NewKeysForClass(Py_ssize_t keys_size) +_PyDict_NewKeysForClass(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = new_keys_object( - interp, estimate_log2_keysize(keys_size), 1); + interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } else { assert(keys->dk_nentries == 0); /* Set to max size+1 as it will shrink by one before each new object */ - keys->dk_usable = keys_size; + keys->dk_usable = SHARED_KEYS_MAX_SIZE; keys->dk_kind = DICT_KEYS_SPLIT; } return keys; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b798e01e746c7b..82822784aaf407 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7760,12 +7760,6 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) return 0; } -static Py_ssize_t -expected_number_of_shared_keys(PyTypeObject *type) -{ - return SHARED_KEYS_MAX_SIZE; -} - static int type_ready_managed_dict(PyTypeObject *type) { @@ -7781,7 +7775,7 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(expected_number_of_shared_keys(type)); + et->ht_cached_keys = _PyDict_NewKeysForClass(); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; From 67957a6315f4a3beab8b09dfb45a17111f1f74ab Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 18:10:39 +0000 Subject: [PATCH 09/12] rename expected_attributes --> static_attributes --- .../pycore_global_objects_fini_generated.h | 2 +- Include/internal/pycore_global_strings.h | 2 +- .../internal/pycore_runtime_init_generated.h | 2 +- .../internal/pycore_unicodeobject_generated.h | 6 ++-- Lib/enum.py | 2 +- Lib/pydoc.py | 2 +- Lib/test/test_compile.py | 14 +++++----- Lib/test/test_descr.py | 4 +-- Lib/test/test_io.py | 2 +- Lib/test/test_metaclass.py | 10 +++---- Lib/typing.py | 2 +- ...-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst | 2 +- Python/compile.c | 28 +++++++++---------- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9eab24e4b75818..9aa34f5927dea8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -623,7 +623,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__enter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__expected_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); @@ -725,6 +724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slotnames__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slots__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__spec__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__static_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__str__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__sub__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasscheck__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index c1e7e460be276b..9a0d42f6f12a1e 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -112,7 +112,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__enter__) STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) - STRUCT_FOR_ID(__expected_attributes__) STRUCT_FOR_ID(__file__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) @@ -214,6 +213,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__slotnames__) STRUCT_FOR_ID(__slots__) STRUCT_FOR_ID(__spec__) + STRUCT_FOR_ID(__static_attributes__) STRUCT_FOR_ID(__str__) STRUCT_FOR_ID(__sub__) STRUCT_FOR_ID(__subclasscheck__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ef84c96fe7b049..d75f0f88656128 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -621,7 +621,6 @@ extern "C" { INIT_ID(__enter__), \ INIT_ID(__eq__), \ INIT_ID(__exit__), \ - INIT_ID(__expected_attributes__), \ INIT_ID(__file__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ @@ -723,6 +722,7 @@ extern "C" { INIT_ID(__slotnames__), \ INIT_ID(__slots__), \ INIT_ID(__spec__), \ + INIT_ID(__static_attributes__), \ INIT_ID(__str__), \ INIT_ID(__sub__), \ INIT_ID(__subclasscheck__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 9c394e9d27dbf3..7f67e67f571eae 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -177,9 +177,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__exit__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__expected_attributes__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__file__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -483,6 +480,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__spec__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__static_attributes__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__str__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/enum.py b/Lib/enum.py index afbe8507ae7c6b..2a135e1b1f1826 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -2019,7 +2019,7 @@ def _test_simple_enum(checked_enum, simple_enum): ) for key in set(checked_keys + simple_keys): if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', - '__expected_attributes__'): + '__static_attributes__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b3d310006e36af..d9cf03fb4ffd2a 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -314,7 +314,7 @@ def visiblename(name, all=None, obj=None): '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__', - '__expected_attributes__'}: + '__static_attributes__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index a4c85994f1104e..6e3f6fa12fb531 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1967,8 +1967,8 @@ class C: def f(self): self.a = self.b = 42 - self.assertIsInstance(C.__expected_attributes__, tuple) - self.assertEqual(sorted(C.__expected_attributes__), ['a', 'b']) + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) def test_nested_function(self): class C: @@ -1987,7 +1987,7 @@ def h(self, a): obj.self = 8 - self.assertEqual(sorted(C.__expected_attributes__), ['u', 'v', 'x', 'y', 'z']) + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) def test_nested_class(self): class C: @@ -2000,8 +2000,8 @@ def g(self): self.y = 42 self.z = 42 - self.assertEqual(sorted(C.__expected_attributes__), ['x', 'y']) - self.assertEqual(sorted(C.D.__expected_attributes__), ['y', 'z']) + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) def test_subclass(self): class C: @@ -2014,8 +2014,8 @@ def g(self): self.y = 42 self.z = 42 - self.assertEqual(sorted(C.__expected_attributes__), ['x', 'y']) - self.assertEqual(sorted(D.__expected_attributes__), ['y', 'z']) + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) class TestExpressionStackSize(unittest.TestCase): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index a8a1b9f161e0ce..6df87232d9b605 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5079,7 +5079,7 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__expected_attributes__', + self.assertEqual(keys, ['__dict__', '__doc__', '__static_attributes__', '__module__', '__weakref__', 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), @@ -5099,7 +5099,7 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__expected_attributes__', + self.assertEqual(keys, ['__dict__', '__doc__', '__static_attributes__', '__module__', '__weakref__', 'meth']) def test_dict_type_with_metaclass(self): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index dc9924e2a50da3..4ea1ef15c0661d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1160,7 +1160,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__', '__expected_attributes__')) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 90d4ffb901b8e5..b74ad3f676f803 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -167,7 +167,7 @@ d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 - d['__expected_attributes__'] = () + d['__static_attributes__'] = () >>> Use a metaclass that doesn't derive from type. @@ -183,12 +183,12 @@ ... b = 24 ... meta: C () - ns: [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -209,9 +209,9 @@ d['a'] = 1 d['a'] = 2 d['b'] = 3 - d['__expected_attributes__'] = () + d['__static_attributes__'] = () meta: C () - ns: [('__expected_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/typing.py b/Lib/typing.py index b97ac19cadc674..581d187235dc7e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1717,7 +1717,7 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', '__expected_attributes__', + '__match_args__', '__static_attributes__', }) # These special attributes will be not collected as protocol members. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst index 5565d1936a4813..78bef746b67d85 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst @@ -1,3 +1,3 @@ -Compiler populate the new ``__expected_attributes__`` field on a class with +Compiler populates the new ``__static_attributes__`` field on a class with the names of attributes of this class which are accessed through self.X from any function in its body. diff --git a/Python/compile.c b/Python/compile.c index b5ac1e0c1ea9e8..4770a0c39fa747 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -358,8 +358,8 @@ struct compiler_unit { int u_scope_type; - PyObject *u_private; /* for private name mangling */ - PyObject *u_expected_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_private; /* for private name mangling */ + PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ instr_sequence u_instr_sequence; /* codegen output */ @@ -691,7 +691,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); - Py_CLEAR(u->u_expected_attributes); + Py_CLEAR(u->u_static_attributes); PyMem_Free(u); } @@ -1356,14 +1356,14 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_private = NULL; if (scope_type == COMPILER_SCOPE_CLASS) { - u->u_expected_attributes = PySet_New(0); - if (!u->u_expected_attributes) { + u->u_static_attributes = PySet_New(0); + if (!u->u_static_attributes) { compiler_unit_free(u); return ERROR; } } else { - u->u_expected_attributes = NULL; + u->u_static_attributes = NULL; } /* Push the old compiler_unit on the stack. */ @@ -2546,15 +2546,15 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } - assert(c->u->u_expected_attributes); - PyObject *expected_attributes = PySequence_Tuple(c->u->u_expected_attributes); - if (expected_attributes == NULL) { + assert(c->u->u_static_attributes); + PyObject *static_attributes = PySequence_Tuple(c->u->u_static_attributes); + if (static_attributes == NULL) { compiler_exit_scope(c); return ERROR; } - ADDOP_LOAD_CONST(c, NO_LOCATION, expected_attributes); - Py_CLEAR(expected_attributes); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__expected_attributes__), Store) < 0) { + ADDOP_LOAD_CONST(c, NO_LOCATION, static_attributes); + Py_CLEAR(static_attributes); + if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store) < 0) { compiler_exit_scope(c); return ERROR; } @@ -6294,9 +6294,9 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) struct compiler_unit *class_u = get_class_compiler_unit(c); if (class_u != NULL) { assert(class_u->u_scope_type == COMPILER_SCOPE_CLASS); - assert(class_u->u_expected_attributes); + assert(class_u->u_static_attributes); RETURN_IF_ERROR( - PySet_Add(class_u->u_expected_attributes, e->v.Attribute.attr)); + PySet_Add(class_u->u_static_attributes, e->v.Attribute.attr)); } } VISIT(c, expr, e->v.Attribute.value); From 5f1e601b3b06fa4b20af6a6c8c8a4681c52c4806 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 19:35:43 +0000 Subject: [PATCH 10/12] remove assert --- Python/compile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 4770a0c39fa747..e9507e47dac8fe 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -699,7 +699,6 @@ static struct compiler_unit * get_class_compiler_unit(struct compiler *c) { Py_ssize_t stack_size = PyList_GET_SIZE(c->c_stack); - assert(stack_size >= 1); for (Py_ssize_t i = stack_size - 1; i >= 0; i--) { PyObject *capsule = PyList_GET_ITEM(c->c_stack, i); struct compiler_unit *u = (struct compiler_unit *)PyCapsule_GetPointer( From 79309ab29683ecc953766fde5ef1033cd786d9c8 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 23 Mar 2024 19:42:20 +0000 Subject: [PATCH 11/12] fix test --- Lib/test/test_descr.py | 10 ++++++---- Lib/test/test_metaclass.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 6df87232d9b605..097ca38e0b1ed8 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5079,8 +5079,9 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__static_attributes__', - '__module__', '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + '__static_attributes__', '__weakref__', + 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5099,8 +5100,9 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__static_attributes__', - '__module__', '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + '__static_attributes__', '__weakref__', + 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index b74ad3f676f803..70f9c5d9400bf6 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -183,12 +183,12 @@ ... b = 24 ... meta: C () - ns: [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -211,7 +211,7 @@ d['b'] = 3 d['__static_attributes__'] = () meta: C () - ns: [('__static_attributes__', ()), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> From 692c34606338c4a7248aabcd9cbb4a52fdc9f606 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:36:50 +0000 Subject: [PATCH 12/12] add comment --- Lib/test/test_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 6e3f6fa12fb531..9d5f721806a884 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1975,7 +1975,7 @@ class C: def f(self): self.x = 1 self.y = 2 - self.x = 3 + self.x = 3 # check deduplication def g(self, obj): self.y = 4