From 1d101afb6e87d58a6b1b0d45db28a74059bf26e0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 13 Aug 2024 17:49:21 +0100 Subject: [PATCH 01/10] Extract eval breaker check from CALL_KW --- Include/internal/pycore_code.h | 3 + Include/internal/pycore_magic_number.h | 1 + Include/internal/pycore_opcode_metadata.h | 30 +- Include/internal/pycore_uop_ids.h | 290 +++++++-------- Include/internal/pycore_uop_metadata.h | 24 ++ Include/opcode_ids.h | 109 +++--- Lib/_opcode_metadata.py | 114 +++--- Python/bytecodes.c | 147 +++++++- Python/executor_cases.c.h | 178 ++++++++- Python/generated_cases.c.h | 421 ++++++++++++++++++---- Python/opcode_targets.h | 6 +- Python/optimizer_cases.c.h | 45 ++- Python/specialize.c | 67 ++++ 13 files changed, 1104 insertions(+), 331 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 67aeab24db2347..878a16faca1902 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -156,6 +156,7 @@ typedef struct { } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { _Py_BackoffCounter counter; @@ -335,6 +336,8 @@ extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); +extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 0af6e3f3cd4c92..d0519a7b7d30af 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -257,6 +257,7 @@ Known values: Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) Python 3.14a1 3604 (Do not duplicate test at end of while statements) Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) + Python 3.14a1 3606 (Specialize CALL_KW) Python 3.15 will start with 3650 diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c056ff13c418db..9d6907414c1cf5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -105,6 +105,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_KW: return 3 + oparg; + case CALL_KW_BOUND_METHOD: + return 3 + oparg; + case CALL_KW_NON_PY: + return 3 + oparg; + case CALL_KW_PY: + return 3 + oparg; case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: @@ -552,6 +558,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW: return 1; + case CALL_KW_BOUND_METHOD: + return 0; + case CALL_KW_NON_PY: + return 1; + case CALL_KW_PY: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: @@ -1023,7 +1035,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1250,6 +1265,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, 0, 0 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, 0, 0 }, { _CALL_KW_NON_PY, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1431,6 +1449,9 @@ const char *_PyOpcode_OpName[264] = { [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_NON_PY] = "CALL_KW_NON_PY", + [CALL_KW_PY] = "CALL_KW_PY", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", @@ -1637,6 +1658,7 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, + [CALL_KW] = 3, [BINARY_OP] = 1, }; #endif @@ -1680,6 +1702,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, + [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_NON_PY] = CALL_KW, + [CALL_KW_PY] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, @@ -1892,9 +1917,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 223: \ - case 224: \ - case 225: \ case 226: \ case 227: \ case 228: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 27d7f96863fa8c..5a4358dd2aee29 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -43,41 +43,44 @@ extern "C" { #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW CALL_KW +#define _CALL_KW_NON_PY 318 #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 318 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 319 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 320 -#define _CALL_METHOD_DESCRIPTOR_O 321 -#define _CALL_NON_PY_GENERAL 322 -#define _CALL_STR_1 323 -#define _CALL_TUPLE_1 324 +#define _CALL_METHOD_DESCRIPTOR_FAST 319 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 320 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 321 +#define _CALL_METHOD_DESCRIPTOR_O 322 +#define _CALL_NON_PY_GENERAL 323 +#define _CALL_STR_1 324 +#define _CALL_TUPLE_1 325 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 325 -#define _CHECK_ATTR_METHOD_LAZY_DICT 326 -#define _CHECK_ATTR_MODULE 327 -#define _CHECK_ATTR_WITH_HINT 328 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 329 +#define _CHECK_ATTR_CLASS 326 +#define _CHECK_ATTR_METHOD_LAZY_DICT 327 +#define _CHECK_ATTR_MODULE 328 +#define _CHECK_ATTR_WITH_HINT 329 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 330 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 330 -#define _CHECK_FUNCTION_EXACT_ARGS 331 -#define _CHECK_FUNCTION_VERSION 332 -#define _CHECK_IS_NOT_PY_CALLABLE 333 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 334 -#define _CHECK_METHOD_VERSION 335 -#define _CHECK_PEP_523 336 -#define _CHECK_PERIODIC 337 -#define _CHECK_STACK_SPACE 338 -#define _CHECK_STACK_SPACE_OPERAND 339 -#define _CHECK_VALIDITY 340 -#define _CHECK_VALIDITY_AND_SET_IP 341 -#define _COMPARE_OP 342 -#define _COMPARE_OP_FLOAT 343 -#define _COMPARE_OP_INT 344 -#define _COMPARE_OP_STR 345 -#define _CONTAINS_OP 346 +#define _CHECK_FUNCTION 331 +#define _CHECK_FUNCTION_EXACT_ARGS 332 +#define _CHECK_FUNCTION_VERSION 333 +#define _CHECK_FUNCTION_VERSION_KW 334 +#define _CHECK_IS_NOT_PY_CALLABLE 335 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 336 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 +#define _CHECK_METHOD_VERSION 338 +#define _CHECK_METHOD_VERSION_KW 339 +#define _CHECK_PEP_523 340 +#define _CHECK_PERIODIC 341 +#define _CHECK_STACK_SPACE 342 +#define _CHECK_STACK_SPACE_OPERAND 343 +#define _CHECK_VALIDITY 344 +#define _CHECK_VALIDITY_AND_SET_IP 345 +#define _COMPARE_OP 346 +#define _COMPARE_OP_FLOAT 347 +#define _COMPARE_OP_INT 348 +#define _COMPARE_OP_STR 349 +#define _CONTAINS_OP 350 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -89,56 +92,58 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 347 +#define _DEOPT 351 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 348 -#define _DYNAMIC_EXIT 349 +#define _DO_CALL 352 +#define _DO_CALL_KW 353 +#define _DYNAMIC_EXIT 354 #define _END_SEND END_SEND -#define _ERROR_POP_N 350 +#define _ERROR_POP_N 355 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 351 -#define _FATAL_ERROR 352 +#define _EXPAND_METHOD 356 +#define _EXPAND_METHOD_KW 357 +#define _FATAL_ERROR 358 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 353 -#define _FOR_ITER_GEN_FRAME 354 -#define _FOR_ITER_TIER_TWO 355 +#define _FOR_ITER 359 +#define _FOR_ITER_GEN_FRAME 360 +#define _FOR_ITER_TIER_TWO 361 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 356 -#define _GUARD_BOTH_INT 357 -#define _GUARD_BOTH_UNICODE 358 -#define _GUARD_BUILTINS_VERSION 359 -#define _GUARD_DORV_NO_DICT 360 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 361 -#define _GUARD_GLOBALS_VERSION 362 -#define _GUARD_IS_FALSE_POP 363 -#define _GUARD_IS_NONE_POP 364 -#define _GUARD_IS_NOT_NONE_POP 365 -#define _GUARD_IS_TRUE_POP 366 -#define _GUARD_KEYS_VERSION 367 -#define _GUARD_NOS_FLOAT 368 -#define _GUARD_NOS_INT 369 -#define _GUARD_NOT_EXHAUSTED_LIST 370 -#define _GUARD_NOT_EXHAUSTED_RANGE 371 -#define _GUARD_NOT_EXHAUSTED_TUPLE 372 -#define _GUARD_TOS_FLOAT 373 -#define _GUARD_TOS_INT 374 -#define _GUARD_TYPE_VERSION 375 +#define _GUARD_BOTH_FLOAT 362 +#define _GUARD_BOTH_INT 363 +#define _GUARD_BOTH_UNICODE 364 +#define _GUARD_BUILTINS_VERSION 365 +#define _GUARD_DORV_NO_DICT 366 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 367 +#define _GUARD_GLOBALS_VERSION 368 +#define _GUARD_IS_FALSE_POP 369 +#define _GUARD_IS_NONE_POP 370 +#define _GUARD_IS_NOT_NONE_POP 371 +#define _GUARD_IS_TRUE_POP 372 +#define _GUARD_KEYS_VERSION 373 +#define _GUARD_NOS_FLOAT 374 +#define _GUARD_NOS_INT 375 +#define _GUARD_NOT_EXHAUSTED_LIST 376 +#define _GUARD_NOT_EXHAUSTED_RANGE 377 +#define _GUARD_NOT_EXHAUSTED_TUPLE 378 +#define _GUARD_TOS_FLOAT 379 +#define _GUARD_TOS_INT 380 +#define _GUARD_TYPE_VERSION 381 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 376 -#define _INIT_CALL_PY_EXACT_ARGS 377 -#define _INIT_CALL_PY_EXACT_ARGS_0 378 -#define _INIT_CALL_PY_EXACT_ARGS_1 379 -#define _INIT_CALL_PY_EXACT_ARGS_2 380 -#define _INIT_CALL_PY_EXACT_ARGS_3 381 -#define _INIT_CALL_PY_EXACT_ARGS_4 382 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 382 +#define _INIT_CALL_PY_EXACT_ARGS 383 +#define _INIT_CALL_PY_EXACT_ARGS_0 384 +#define _INIT_CALL_PY_EXACT_ARGS_1 385 +#define _INIT_CALL_PY_EXACT_ARGS_2 386 +#define _INIT_CALL_PY_EXACT_ARGS_3 387 +#define _INIT_CALL_PY_EXACT_ARGS_4 388 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -152,65 +157,65 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE #define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _INTERNAL_INCREMENT_OPT_COUNTER 383 -#define _IS_NONE 384 +#define _INTERNAL_INCREMENT_OPT_COUNTER 389 +#define _IS_NONE 390 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 385 -#define _ITER_CHECK_RANGE 386 -#define _ITER_CHECK_TUPLE 387 -#define _ITER_JUMP_LIST 388 -#define _ITER_JUMP_RANGE 389 -#define _ITER_JUMP_TUPLE 390 -#define _ITER_NEXT_LIST 391 -#define _ITER_NEXT_RANGE 392 -#define _ITER_NEXT_TUPLE 393 -#define _JUMP_TO_TOP 394 +#define _ITER_CHECK_LIST 391 +#define _ITER_CHECK_RANGE 392 +#define _ITER_CHECK_TUPLE 393 +#define _ITER_JUMP_LIST 394 +#define _ITER_JUMP_RANGE 395 +#define _ITER_JUMP_TUPLE 396 +#define _ITER_NEXT_LIST 397 +#define _ITER_NEXT_RANGE 398 +#define _ITER_NEXT_TUPLE 399 +#define _JUMP_TO_TOP 400 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 395 -#define _LOAD_ATTR_CLASS 396 -#define _LOAD_ATTR_CLASS_0 397 -#define _LOAD_ATTR_CLASS_1 398 +#define _LOAD_ATTR 401 +#define _LOAD_ATTR_CLASS 402 +#define _LOAD_ATTR_CLASS_0 403 +#define _LOAD_ATTR_CLASS_1 404 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 399 -#define _LOAD_ATTR_INSTANCE_VALUE_0 400 -#define _LOAD_ATTR_INSTANCE_VALUE_1 401 -#define _LOAD_ATTR_METHOD_LAZY_DICT 402 -#define _LOAD_ATTR_METHOD_NO_DICT 403 -#define _LOAD_ATTR_METHOD_WITH_VALUES 404 -#define _LOAD_ATTR_MODULE 405 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 406 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 407 -#define _LOAD_ATTR_PROPERTY_FRAME 408 -#define _LOAD_ATTR_SLOT 409 -#define _LOAD_ATTR_SLOT_0 410 -#define _LOAD_ATTR_SLOT_1 411 -#define _LOAD_ATTR_WITH_HINT 412 +#define _LOAD_ATTR_INSTANCE_VALUE 405 +#define _LOAD_ATTR_INSTANCE_VALUE_0 406 +#define _LOAD_ATTR_INSTANCE_VALUE_1 407 +#define _LOAD_ATTR_METHOD_LAZY_DICT 408 +#define _LOAD_ATTR_METHOD_NO_DICT 409 +#define _LOAD_ATTR_METHOD_WITH_VALUES 410 +#define _LOAD_ATTR_MODULE 411 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 412 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 413 +#define _LOAD_ATTR_PROPERTY_FRAME 414 +#define _LOAD_ATTR_SLOT 415 +#define _LOAD_ATTR_SLOT_0 416 +#define _LOAD_ATTR_SLOT_1 417 +#define _LOAD_ATTR_WITH_HINT 418 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 413 -#define _LOAD_CONST_INLINE_BORROW 414 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 415 -#define _LOAD_CONST_INLINE_WITH_NULL 416 +#define _LOAD_CONST_INLINE 419 +#define _LOAD_CONST_INLINE_BORROW 420 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 421 +#define _LOAD_CONST_INLINE_WITH_NULL 422 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 417 -#define _LOAD_FAST_0 418 -#define _LOAD_FAST_1 419 -#define _LOAD_FAST_2 420 -#define _LOAD_FAST_3 421 -#define _LOAD_FAST_4 422 -#define _LOAD_FAST_5 423 -#define _LOAD_FAST_6 424 -#define _LOAD_FAST_7 425 +#define _LOAD_FAST 423 +#define _LOAD_FAST_0 424 +#define _LOAD_FAST_1 425 +#define _LOAD_FAST_2 426 +#define _LOAD_FAST_3 427 +#define _LOAD_FAST_4 428 +#define _LOAD_FAST_5 429 +#define _LOAD_FAST_6 430 +#define _LOAD_FAST_7 431 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 426 -#define _LOAD_GLOBAL_BUILTINS 427 -#define _LOAD_GLOBAL_MODULE 428 +#define _LOAD_GLOBAL 432 +#define _LOAD_GLOBAL_BUILTINS 433 +#define _LOAD_GLOBAL_MODULE 434 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -223,55 +228,56 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 429 -#define _MONITOR_CALL 430 +#define _MAYBE_EXPAND_METHOD 435 +#define _MONITOR_CALL 436 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 431 -#define _POP_JUMP_IF_TRUE 432 +#define _POP_JUMP_IF_FALSE 437 +#define _POP_JUMP_IF_TRUE 438 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 433 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 439 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 434 +#define _PUSH_FRAME 440 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 435 -#define _REPLACE_WITH_TRUE 436 +#define _PY_FRAME_GENERAL 441 +#define _PY_FRAME_KW 442 +#define _REPLACE_WITH_TRUE 443 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 437 -#define _SEND 438 -#define _SEND_GEN_FRAME 439 +#define _SAVE_RETURN_OFFSET 444 +#define _SEND 445 +#define _SEND_GEN_FRAME 446 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 440 -#define _STORE_ATTR 441 -#define _STORE_ATTR_INSTANCE_VALUE 442 -#define _STORE_ATTR_SLOT 443 -#define _STORE_ATTR_WITH_HINT 444 +#define _START_EXECUTOR 447 +#define _STORE_ATTR 448 +#define _STORE_ATTR_INSTANCE_VALUE 449 +#define _STORE_ATTR_SLOT 450 +#define _STORE_ATTR_WITH_HINT 451 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 445 -#define _STORE_FAST_0 446 -#define _STORE_FAST_1 447 -#define _STORE_FAST_2 448 -#define _STORE_FAST_3 449 -#define _STORE_FAST_4 450 -#define _STORE_FAST_5 451 -#define _STORE_FAST_6 452 -#define _STORE_FAST_7 453 +#define _STORE_FAST 452 +#define _STORE_FAST_0 453 +#define _STORE_FAST_1 454 +#define _STORE_FAST_2 455 +#define _STORE_FAST_3 456 +#define _STORE_FAST_4 457 +#define _STORE_FAST_5 458 +#define _STORE_FAST_6 459 +#define _STORE_FAST_7 460 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 454 +#define _STORE_SUBSCR 461 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 455 -#define _TO_BOOL 456 +#define _TIER2_RESUME_CHECK 462 +#define _TO_BOOL 463 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -281,13 +287,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 457 +#define _UNPACK_SEQUENCE 464 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 457 +#define MAX_UOP_ID 464 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f5c666454dcbef..d1cd2c54f8a270 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -236,6 +236,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -309,6 +315,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_LEN] = "_CALL_LEN", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", @@ -329,9 +336,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", @@ -363,6 +373,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", @@ -478,6 +489,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_KW] = "_PY_FRAME_KW", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", @@ -968,6 +980,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; + case _PY_FRAME_KW: + return 3 + oparg; + case _CHECK_FUNCTION_VERSION_KW: + return 3 + oparg; + case _CHECK_METHOD_VERSION_KW: + return 3 + oparg; + case _EXPAND_METHOD_KW: + return 3 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE_KW: + return 3 + oparg; + case _CALL_KW_NON_PY: + return 3 + oparg; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 3353e8011bb3ff..71c3c54881a4b0 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -147,59 +147,62 @@ extern "C" { #define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 #define CALL_BUILTIN_O 168 #define CALL_ISINSTANCE 169 -#define CALL_LEN 170 -#define CALL_LIST_APPEND 171 -#define CALL_METHOD_DESCRIPTOR_FAST 172 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 -#define CALL_METHOD_DESCRIPTOR_NOARGS 174 -#define CALL_METHOD_DESCRIPTOR_O 175 -#define CALL_NON_PY_GENERAL 176 -#define CALL_PY_EXACT_ARGS 177 -#define CALL_PY_GENERAL 178 -#define CALL_STR_1 179 -#define CALL_TUPLE_1 180 -#define CALL_TYPE_1 181 -#define COMPARE_OP_FLOAT 182 -#define COMPARE_OP_INT 183 -#define COMPARE_OP_STR 184 -#define CONTAINS_OP_DICT 185 -#define CONTAINS_OP_SET 186 -#define FOR_ITER_GEN 187 -#define FOR_ITER_LIST 188 -#define FOR_ITER_RANGE 189 -#define FOR_ITER_TUPLE 190 -#define LOAD_ATTR_CLASS 191 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 -#define LOAD_ATTR_INSTANCE_VALUE 193 -#define LOAD_ATTR_METHOD_LAZY_DICT 194 -#define LOAD_ATTR_METHOD_NO_DICT 195 -#define LOAD_ATTR_METHOD_WITH_VALUES 196 -#define LOAD_ATTR_MODULE 197 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 -#define LOAD_ATTR_PROPERTY 200 -#define LOAD_ATTR_SLOT 201 -#define LOAD_ATTR_WITH_HINT 202 -#define LOAD_GLOBAL_BUILTIN 203 -#define LOAD_GLOBAL_MODULE 204 -#define LOAD_SUPER_ATTR_ATTR 205 -#define LOAD_SUPER_ATTR_METHOD 206 -#define RESUME_CHECK 207 -#define SEND_GEN 208 -#define STORE_ATTR_INSTANCE_VALUE 209 -#define STORE_ATTR_SLOT 210 -#define STORE_ATTR_WITH_HINT 211 -#define STORE_SUBSCR_DICT 212 -#define STORE_SUBSCR_LIST_INT 213 -#define TO_BOOL_ALWAYS_TRUE 214 -#define TO_BOOL_BOOL 215 -#define TO_BOOL_INT 216 -#define TO_BOOL_LIST 217 -#define TO_BOOL_NONE 218 -#define TO_BOOL_STR 219 -#define UNPACK_SEQUENCE_LIST 220 -#define UNPACK_SEQUENCE_TUPLE 221 -#define UNPACK_SEQUENCE_TWO_TUPLE 222 +#define CALL_KW_BOUND_METHOD 170 +#define CALL_KW_NON_PY 171 +#define CALL_KW_PY 172 +#define CALL_LEN 173 +#define CALL_LIST_APPEND 174 +#define CALL_METHOD_DESCRIPTOR_FAST 175 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 176 +#define CALL_METHOD_DESCRIPTOR_NOARGS 177 +#define CALL_METHOD_DESCRIPTOR_O 178 +#define CALL_NON_PY_GENERAL 179 +#define CALL_PY_EXACT_ARGS 180 +#define CALL_PY_GENERAL 181 +#define CALL_STR_1 182 +#define CALL_TUPLE_1 183 +#define CALL_TYPE_1 184 +#define COMPARE_OP_FLOAT 185 +#define COMPARE_OP_INT 186 +#define COMPARE_OP_STR 187 +#define CONTAINS_OP_DICT 188 +#define CONTAINS_OP_SET 189 +#define FOR_ITER_GEN 190 +#define FOR_ITER_LIST 191 +#define FOR_ITER_RANGE 192 +#define FOR_ITER_TUPLE 193 +#define LOAD_ATTR_CLASS 194 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 195 +#define LOAD_ATTR_INSTANCE_VALUE 196 +#define LOAD_ATTR_METHOD_LAZY_DICT 197 +#define LOAD_ATTR_METHOD_NO_DICT 198 +#define LOAD_ATTR_METHOD_WITH_VALUES 199 +#define LOAD_ATTR_MODULE 200 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 201 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 202 +#define LOAD_ATTR_PROPERTY 203 +#define LOAD_ATTR_SLOT 204 +#define LOAD_ATTR_WITH_HINT 205 +#define LOAD_GLOBAL_BUILTIN 206 +#define LOAD_GLOBAL_MODULE 207 +#define LOAD_SUPER_ATTR_ATTR 208 +#define LOAD_SUPER_ATTR_METHOD 209 +#define RESUME_CHECK 210 +#define SEND_GEN 211 +#define STORE_ATTR_INSTANCE_VALUE 212 +#define STORE_ATTR_SLOT 213 +#define STORE_ATTR_WITH_HINT 214 +#define STORE_SUBSCR_DICT 215 +#define STORE_SUBSCR_LIST_INT 216 +#define TO_BOOL_ALWAYS_TRUE 217 +#define TO_BOOL_BOOL 218 +#define TO_BOOL_INT 219 +#define TO_BOOL_LIST 220 +#define TO_BOOL_NONE 221 +#define TO_BOOL_STR 222 +#define UNPACK_SEQUENCE_LIST 223 +#define UNPACK_SEQUENCE_TUPLE 224 +#define UNPACK_SEQUENCE_TWO_TUPLE 225 #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 7c559f6190fc85..a13bef5d4d3fa1 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -107,6 +107,11 @@ "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ], } _specialized_opmap = { @@ -131,59 +136,62 @@ 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, 'CALL_BUILTIN_O': 168, 'CALL_ISINSTANCE': 169, - 'CALL_LEN': 170, - 'CALL_LIST_APPEND': 171, - 'CALL_METHOD_DESCRIPTOR_FAST': 172, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, - 'CALL_METHOD_DESCRIPTOR_O': 175, - 'CALL_NON_PY_GENERAL': 176, - 'CALL_PY_EXACT_ARGS': 177, - 'CALL_PY_GENERAL': 178, - 'CALL_STR_1': 179, - 'CALL_TUPLE_1': 180, - 'CALL_TYPE_1': 181, - 'COMPARE_OP_FLOAT': 182, - 'COMPARE_OP_INT': 183, - 'COMPARE_OP_STR': 184, - 'CONTAINS_OP_DICT': 185, - 'CONTAINS_OP_SET': 186, - 'FOR_ITER_GEN': 187, - 'FOR_ITER_LIST': 188, - 'FOR_ITER_RANGE': 189, - 'FOR_ITER_TUPLE': 190, - 'LOAD_ATTR_CLASS': 191, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, - 'LOAD_ATTR_INSTANCE_VALUE': 193, - 'LOAD_ATTR_METHOD_LAZY_DICT': 194, - 'LOAD_ATTR_METHOD_NO_DICT': 195, - 'LOAD_ATTR_METHOD_WITH_VALUES': 196, - 'LOAD_ATTR_MODULE': 197, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, - 'LOAD_ATTR_PROPERTY': 200, - 'LOAD_ATTR_SLOT': 201, - 'LOAD_ATTR_WITH_HINT': 202, - 'LOAD_GLOBAL_BUILTIN': 203, - 'LOAD_GLOBAL_MODULE': 204, - 'LOAD_SUPER_ATTR_ATTR': 205, - 'LOAD_SUPER_ATTR_METHOD': 206, - 'RESUME_CHECK': 207, - 'SEND_GEN': 208, - 'STORE_ATTR_INSTANCE_VALUE': 209, - 'STORE_ATTR_SLOT': 210, - 'STORE_ATTR_WITH_HINT': 211, - 'STORE_SUBSCR_DICT': 212, - 'STORE_SUBSCR_LIST_INT': 213, - 'TO_BOOL_ALWAYS_TRUE': 214, - 'TO_BOOL_BOOL': 215, - 'TO_BOOL_INT': 216, - 'TO_BOOL_LIST': 217, - 'TO_BOOL_NONE': 218, - 'TO_BOOL_STR': 219, - 'UNPACK_SEQUENCE_LIST': 220, - 'UNPACK_SEQUENCE_TUPLE': 221, - 'UNPACK_SEQUENCE_TWO_TUPLE': 222, + 'CALL_KW_BOUND_METHOD': 170, + 'CALL_KW_NON_PY': 171, + 'CALL_KW_PY': 172, + 'CALL_LEN': 173, + 'CALL_LIST_APPEND': 174, + 'CALL_METHOD_DESCRIPTOR_FAST': 175, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 176, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 177, + 'CALL_METHOD_DESCRIPTOR_O': 178, + 'CALL_NON_PY_GENERAL': 179, + 'CALL_PY_EXACT_ARGS': 180, + 'CALL_PY_GENERAL': 181, + 'CALL_STR_1': 182, + 'CALL_TUPLE_1': 183, + 'CALL_TYPE_1': 184, + 'COMPARE_OP_FLOAT': 185, + 'COMPARE_OP_INT': 186, + 'COMPARE_OP_STR': 187, + 'CONTAINS_OP_DICT': 188, + 'CONTAINS_OP_SET': 189, + 'FOR_ITER_GEN': 190, + 'FOR_ITER_LIST': 191, + 'FOR_ITER_RANGE': 192, + 'FOR_ITER_TUPLE': 193, + 'LOAD_ATTR_CLASS': 194, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 195, + 'LOAD_ATTR_INSTANCE_VALUE': 196, + 'LOAD_ATTR_METHOD_LAZY_DICT': 197, + 'LOAD_ATTR_METHOD_NO_DICT': 198, + 'LOAD_ATTR_METHOD_WITH_VALUES': 199, + 'LOAD_ATTR_MODULE': 200, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 201, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 202, + 'LOAD_ATTR_PROPERTY': 203, + 'LOAD_ATTR_SLOT': 204, + 'LOAD_ATTR_WITH_HINT': 205, + 'LOAD_GLOBAL_BUILTIN': 206, + 'LOAD_GLOBAL_MODULE': 207, + 'LOAD_SUPER_ATTR_ATTR': 208, + 'LOAD_SUPER_ATTR_METHOD': 209, + 'RESUME_CHECK': 210, + 'SEND_GEN': 211, + 'STORE_ATTR_INSTANCE_VALUE': 212, + 'STORE_ATTR_SLOT': 213, + 'STORE_ATTR_WITH_HINT': 214, + 'STORE_SUBSCR_DICT': 215, + 'STORE_SUBSCR_LIST_INT': 216, + 'TO_BOOL_ALWAYS_TRUE': 217, + 'TO_BOOL_BOOL': 218, + 'TO_BOOL_INT': 219, + 'TO_BOOL_LIST': 220, + 'TO_BOOL_NONE': 221, + 'TO_BOOL_STR': 222, + 'UNPACK_SEQUENCE_LIST': 223, + 'UNPACK_SEQUENCE_TUPLE': 224, + 'UNPACK_SEQUENCE_TWO_TUPLE': 225, } opmap = { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ffa53bb5e4b7c2..7c150a93373655 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3999,6 +3999,14 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_FAST + _CHECK_PERIODIC; + // Cache layout: counter/1, func_version/2 + // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! + family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { + CALL_KW_BOUND_METHOD, + CALL_KW_PY, + CALL_KW_NON_PY, + }; + inst(INSTRUMENTED_CALL_KW, ( -- )) { int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; @@ -4009,10 +4017,11 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); ERROR_IF(err, error); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } - inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -4097,6 +4106,142 @@ dummy_func( CHECK_EVAL_BREAKER(); } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null, unused[oparg], kwnames -- callable, self_or_null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_KW_PY) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION_KW + + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], kwnames -- callable, null, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(!PyStackRef_IsNull(null)); + } + + op(_EXPAND_METHOD_KW, (callable, null, unused[oparg], kwnames -- method, self, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + } + + macro(CALL_KW_BOUND_METHOD) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, args[oparg], kwnames -- callable, self_or_null, args[oparg], kwnames)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + + macro(CALL_KW) = + _SPECIALIZE_CALL_KW + + unused/2 + + _DO_CALL_KW; + + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], kwnames -- callable, unused, unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); + } + + + op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + PyStackRef_AsPyObjectBorrow(kwnames)); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + macro(CALL_KW_NON_PY) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE_KW + + _CALL_KW_NON_PY + + _CHECK_PERIODIC; + + inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0bccaf992fb010..7ac1584893e58d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4657,7 +4657,183 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _PY_FRAME_KW: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION_KW: { + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD_KW: { + _PyStackRef kwnames; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); + PyStackRef_CLOSE(callable); + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_KW_NON_PY: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + PyStackRef_AsPyObjectBorrow(kwnames)); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ef00f6f55a3bcf..b37b4e6a29fa72 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1654,113 +1654,308 @@ TARGET(CALL_KW) { frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; + _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; + // _SPECIALIZE_CALL_KW + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _DO_CALL_KW kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o + ); + PyStackRef_CLOSE(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef kwnames; + _PyStackRef method; + _PyStackRef self; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null_o != NULL) { - args--; - total_args++; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null), CALL_KW); } - if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); + // _EXPAND_METHOD_KW + kwnames = stack_pointer[-1]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-2 - oparg] = self; + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-3 - oparg] = method; + assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // flush + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, positional_args, kwnames_o + args, total_args, NULL ); PyStackRef_CLOSE(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef kwnames; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + callable = stack_pointer[-3 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + PyStackRef_AsPyObjectBorrow(kwnames)); PyStackRef_CLOSE(kwnames); - if (true) { + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; + // _CHECK_PERIODIC + { } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1768,6 +1963,85 @@ DISPATCH(); } + TARGET(CALL_KW_PY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + callable = stack_pointer[-3 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null_o != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3765,6 +4039,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index db92b0262efe76..ebc7d30e2ccc34 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -169,6 +169,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_BUILTIN_O, &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, @@ -232,9 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3f4080d164506e..93fe1d772b5a19 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1966,7 +1966,50 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + + case _PY_FRAME_KW: { + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + _Py_UopsSymbol *kwnames; + method = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + kwnames = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = method; + stack_pointer[-2 - oparg] = self; + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ diff --git a/Python/specialize.c b/Python/specialize.c index 4a227381a8b1d7..c18fd0bb9403e5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1904,6 +1904,33 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return 0; } + +static int +specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, + bool bound_method) +{ + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + PyCodeObject *code = (PyCodeObject *)func->func_code; + int kind = function_kind(code); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); + return -1; + } + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + return -1; + } + int version = _PyFunction_GetVersionForCurrentState(func); + if (version == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; + } + write_u32(cache->func_version, version); + instr->op.code = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + return 0; +} + static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { @@ -1999,6 +2026,46 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } +void +_Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + int fail = -1; +// if (PyFunction_Check(callable)) { +// fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); +// } +// else if (PyMethod_Check(callable)) { +// PyObject *func = ((PyMethodObject *)callable)->im_func; +// if (PyFunction_Check(func)) { +// fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); +// } +// else { +// SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); +// fail = -1; +// } +// } +// else { +// instr->op.code = CALL_KW_NON_PY; +// fail = 0; +// } + if (fail) { + STAT_INC(CALL, failure); + assert(!PyErr_Occurred()); + instr->op.code = CALL; + cache->counter = adaptive_counter_backoff(cache->counter); + } + else { + STAT_INC(CALL, success); + assert(!PyErr_Occurred()); + cache->counter = adaptive_counter_cooldown(); + } +} + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) From 4d5910e05bad249dce0b612489e472d1c8b35b61 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 09:08:30 +0100 Subject: [PATCH 02/10] Add inline cache to CALL_KW --- Include/internal/pycore_magic_number.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Lib/test/test_dis.py | 3 +++ Python/bytecodes.c | 7 +++---- Python/generated_cases.c.h | 10 +++++++--- Python/specialize.c | 6 +++--- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index d0519a7b7d30af..095eb0f8a89b79 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -270,7 +270,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3605 +#define PYC_MAGIC_NUMBER 3606 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9d6907414c1cf5..78f5612ba2def5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1095,7 +1095,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 80f66c168bab60..9ca7fe16283d84 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -257,6 +257,9 @@ def wrap_func_w_kwargs(): LOAD_CONST 3 (5) LOAD_CONST 4 (('c',)) CALL_KW 3 + CACHE + CACHE + CACHE POP_TOP RETURN_CONST 0 (None) """ % (wrap_func_w_kwargs.__code__.co_firstlineno, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7c150a93373655..d53fa26bce5105 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4000,14 +4000,13 @@ dummy_func( _CHECK_PERIODIC; // Cache layout: counter/1, func_version/2 - // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { CALL_KW_BOUND_METHOD, CALL_KW_PY, CALL_KW_NON_PY, }; - inst(INSTRUMENTED_CALL_KW, ( -- )) { + inst(INSTRUMENTED_CALL_KW, (counter/1, version/2 -- )) { int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); @@ -4063,8 +4062,8 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b37b4e6a29fa72..c7abc5104b8a50 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1725,8 +1725,8 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; + assert(next_instr - this_instr == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL_KW; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -4028,8 +4028,12 @@ TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); diff --git a/Python/specialize.c b/Python/specialize.c index c18fd0bb9403e5..b40d04cad02f4d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2032,8 +2032,8 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); assert(ENABLE_SPECIALIZATION); - assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); - assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); + assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail = -1; // if (PyFunction_Check(callable)) { @@ -2056,7 +2056,7 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - instr->op.code = CALL; + instr->op.code = CALL_KW; cache->counter = adaptive_counter_backoff(cache->counter); } else { From 6ed3368fa0eb604003e1306b96a5100f02b79f32 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 10:02:26 +0100 Subject: [PATCH 03/10] Add CALL_KW_PY specialization --- Python/bytecodes.c | 4 +++- Python/executor_cases.c.h | 4 +++- Python/generated_cases.c.h | 8 ++++++-- Python/specialize.c | 6 +++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d53fa26bce5105..53e4b7213c5bae 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4115,12 +4115,14 @@ dummy_func( args--; total_args++; } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, positional_args, kwnames_o ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7ac1584893e58d..94fb05130e3a59 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4678,12 +4678,14 @@ args--; total_args++; } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, positional_args, kwnames_o ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c7abc5104b8a50..9a7fc5bc2a4eb9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1844,12 +1844,14 @@ args--; total_args++; } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, positional_args, kwnames_o ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -2000,12 +2002,14 @@ args--; total_args++; } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, - args, total_args, NULL + args, positional_args, kwnames_o ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, diff --git a/Python/specialize.c b/Python/specialize.c index b40d04cad02f4d..20316c2d407339 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2036,9 +2036,9 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail = -1; -// if (PyFunction_Check(callable)) { -// fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); -// } + if (PyFunction_Check(callable)) { + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); + } // else if (PyMethod_Check(callable)) { // PyObject *func = ((PyMethodObject *)callable)->im_func; // if (PyFunction_Check(func)) { From 464c6dcc5b4bb3c430dea9a5e9bb2d2194e57259 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 10:06:28 +0100 Subject: [PATCH 04/10] Add CALL_KW_BOUND_METHOD specialization --- Python/specialize.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 20316c2d407339..d5597ec7147704 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2039,16 +2039,16 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) if (PyFunction_Check(callable)) { fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); } -// else if (PyMethod_Check(callable)) { -// PyObject *func = ((PyMethodObject *)callable)->im_func; -// if (PyFunction_Check(func)) { -// fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); -// } -// else { -// SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); -// fail = -1; -// } -// } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); + } + else { + SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } + } // else { // instr->op.code = CALL_KW_NON_PY; // fail = 0; From 343a54c5f00c072616a03e530face6095e4edd0e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 10:20:34 +0100 Subject: [PATCH 05/10] Add CALL_KW_NON_PY specialization --- Python/bytecodes.c | 8 +++++--- Python/executor_cases.c.h | 6 ++++-- Python/generated_cases.c.h | 6 ++++-- Python/specialize.c | 10 +++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 53e4b7213c5bae..3dee3ccc4b088c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4221,10 +4221,12 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - PyStackRef_AsPyObjectBorrow(kwnames)); + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4235,6 +4237,7 @@ dummy_func( ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } + macro(CALL_KW_NON_PY) = unused/1 + // Skip over the counter unused/2 + @@ -4242,7 +4245,6 @@ dummy_func( _CALL_KW_NON_PY + _CHECK_PERIODIC; - inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 94fb05130e3a59..167d5506c78417 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4818,10 +4818,12 @@ PyStackRef_CLOSE(kwnames); if (true) JUMP_TO_ERROR(); } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - PyStackRef_AsPyObjectBorrow(kwnames)); + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9a7fc5bc2a4eb9..8b3b0900aa0d3e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1937,10 +1937,12 @@ goto error; } } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - PyStackRef_AsPyObjectBorrow(kwnames)); + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/specialize.c b/Python/specialize.c index d5597ec7147704..6a6a6d778954be 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2035,7 +2035,7 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); _PyCallCache *cache = (_PyCallCache *)(instr + 1); - int fail = -1; + int fail; if (PyFunction_Check(callable)) { fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); } @@ -2049,10 +2049,10 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = -1; } } -// else { -// instr->op.code = CALL_KW_NON_PY; -// fail = 0; -// } + else { + instr->op.code = CALL_KW_NON_PY; + fail = 0; + } if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); From 6b9055c6e84817f4d15d6b3d5eeff3eddeb8ee04 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 10:42:25 +0100 Subject: [PATCH 06/10] Handle _PY_FRAME_KW in tier 2 optimizer --- Python/optimizer.c | 2 +- Python/optimizer_bytecodes.c | 9 +++++++++ Python/optimizer_cases.c.h | 17 +++++++++++++++-- Python/specialize.c | 6 +++--- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index dbd5467f0d4653..9198e410627dd4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -807,7 +807,7 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } - assert(_PyOpcode_Deopt[opcode] == CALL); + assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 010733e6ac221a..97e4c642225786 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -627,6 +627,15 @@ dummy_func(void) { ctx->done = true; } + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame*)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + } + op(_RETURN_VALUE, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 93fe1d772b5a19..d36eb50629f400 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1969,8 +1969,21 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _PyInterpreterFrame *new_frame; - new_frame = sym_new_not_null(ctx); + _Py_UopsSymbol *kwnames; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/specialize.c b/Python/specialize.c index 6a6a6d778954be..4fec5a36fc91c7 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2049,9 +2049,9 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = -1; } } - else { - instr->op.code = CALL_KW_NON_PY; - fail = 0; + else { + instr->op.code = CALL_KW_NON_PY; + fail = 0; } if (fail) { STAT_INC(CALL, failure); From e9ca1f9911b3bb962838b9ebfb7f2f5d9eac0459 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 11:39:07 +0100 Subject: [PATCH 07/10] Add news item --- .../2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst new file mode 100644 index 00000000000000..6e45a6c17a9d36 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst @@ -0,0 +1,5 @@ +Add three specializations for :opcode:`CALL_KW`: + +* :opcode:!`CALL_KW_PY` for calls to Python functions +* :opcode:!`CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:!`CALL_KW_NON_PY` for all other calls From e71448fb3fd8add28f834e1dc33a617f5972f9af Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Aug 2024 11:41:45 +0100 Subject: [PATCH 08/10] Fix rst formatting --- .../2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst index 6e45a6c17a9d36..4a3a094a6fc074 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-14-11-38-56.gh-issue-118093.3BywDP.rst @@ -1,5 +1,5 @@ Add three specializations for :opcode:`CALL_KW`: -* :opcode:!`CALL_KW_PY` for calls to Python functions -* :opcode:!`CALL_KW_BOUND_METHOD` for calls to bound methods -* :opcode:!`CALL_KW_NON_PY` for all other calls +* :opcode:`!CALL_KW_PY` for calls to Python functions +* :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:`!CALL_KW_NON_PY` for all other calls From 56e4e7a9d3cb6d69a5cd4621e62011db8b646927 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 16 Aug 2024 14:27:33 +0100 Subject: [PATCH 09/10] Add CALL_KW cache format to opcode.py --- Lib/opcode.py | 4 ++++ Lib/test/test_dis.py | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 2698609cd5636d..974f4d35e2a524 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -85,6 +85,10 @@ "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 9ca7fe16283d84..80f66c168bab60 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -257,9 +257,6 @@ def wrap_func_w_kwargs(): LOAD_CONST 3 (5) LOAD_CONST 4 (('c',)) CALL_KW 3 - CACHE - CACHE - CACHE POP_TOP RETURN_CONST 0 (None) """ % (wrap_func_w_kwargs.__code__.co_firstlineno, From 8b504ae81979dfc8ad0a316def3aaffb960b29b1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 16 Aug 2024 15:40:16 +0100 Subject: [PATCH 10/10] Update Include/internal/pycore_code.h Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Include/internal/pycore_code.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 878a16faca1902..2e76e9f64d4aec 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -337,7 +337,7 @@ extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); + int nargs); extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs,