diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index c93d6fc3d237a2..a4bea0b107ecad 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -248,12 +248,12 @@ def run_cases_test(self, input: str, expected: str): ) with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions + labels # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) @@ -1639,6 +1639,61 @@ def test_kill_in_wrong_order(self): with self.assertRaises(SyntaxError): self.run_cases_test(input, "") + def test_complex_label(self): + input = """ + label(my_label) { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + my_label: + { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + self.run_cases_test(input, output) + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_3; + } + """ + + output = """ + my_label_1: + { + // Comment + do_thing1(); + goto my_label_2; + } + + my_label_2: + { + // Comment + do_thing2(); + goto my_label_3; + } + """ class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8bed29ca2c8fc7..61d0f1d8f09cbb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -53,6 +53,7 @@ #define super(name) static int SUPER_##name #define family(name, ...) static int family_##name #define pseudo(name) static int pseudo_##name +#define label(name) name: /* Annotations */ #define guard @@ -103,7 +104,6 @@ dummy_func( PyObject *codeobj; PyObject *cond; PyObject *descr; - _PyInterpreterFrame entry_frame; PyObject *exc; PyObject *exit; PyObject *fget; @@ -5192,6 +5192,125 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + label(pop_4_error) { + STACK_SHRINK(1); + goto pop_3_error; + } + + label(pop_3_error) { + STACK_SHRINK(1); + goto pop_2_error; + } + + label(pop_2_error) { + STACK_SHRINK(1); + goto pop_1_error; + } + + label(pop_1_error) { + STACK_SHRINK(1); + goto error; + } + + label(error) { + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + label(exception_unwind) { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + + label(exit_unwind) { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + label(resume_with_error) { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index 58a54467f06bb0..0f263134bb7930 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -898,143 +898,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); - { - /* Start instructions */ -#if !USE_COMPUTED_GOTOS - dispatch_opcode: - switch (opcode) -#endif - { - #include "generated_cases.c.h" -#if USE_COMPUTED_GOTOS - _unknown_opcode: -#else - EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode -#endif - /* Tell C compilers not to hold the opcode variable in the loop. - next_instr points the current instruction without TARGET(). */ - opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; - - } /* End instructions */ - - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != &entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == &entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - - #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d0ab667a8bc8ab..3fab4c77b3d050 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,13 @@ #endif #define TIER_ONE 1 +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { + /* BEGIN INSTRUCTIONS */ + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; @@ -8516,4 +8523,153 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } + + /* END INSTRUCTIONS */ +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + /* BEGIN LABELS */ + + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + DISPATCH(); + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + resume_with_error: + { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + +/* END LABELS */ #undef TIER_ONE diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 1f15c3bb9c88af..8a1b2f07d96a75 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -255,6 +255,12 @@ def is_super(self) -> bool: return False +@dataclass +class Label: + name: str + body: list[lexer.Token] + + @dataclass class PseudoInstruction: name: str @@ -288,6 +294,7 @@ class Analysis: uops: dict[str, Uop] families: dict[str, Family] pseudos: dict[str, PseudoInstruction] + labels: dict[str, Label] opmap: dict[str, int] have_arg: int min_instrumented: int @@ -949,6 +956,13 @@ def add_pseudo( ) +def add_label( + label: parser.LabelDef, + labels: dict[str, Label], +) -> None: + labels[label.name] = Label(label.name, label.block.tokens) + + def assign_opcodes( instructions: dict[str, Instruction], families: dict[str, Family], @@ -1067,6 +1081,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: uops: dict[str, Uop] = {} families: dict[str, Family] = {} pseudos: dict[str, PseudoInstruction] = {} + labels: dict[str, Label] = {} for node in forest: match node: case parser.InstDef(name): @@ -1081,6 +1096,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: pass case parser.Pseudo(): pass + case parser.LabelDef(): + pass case _: assert False for node in forest: @@ -1092,6 +1109,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: add_family(node, instructions, families) case parser.Pseudo(): add_pseudo(node, instructions, pseudos) + case parser.LabelDef(): + add_label(node, labels) case _: pass for uop in uops.values(): @@ -1117,7 +1136,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: families["BINARY_OP"].members.append(inst) opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) return Analysis( - instructions, uops, families, pseudos, opmap, first_arg, min_instrumented + instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented ) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 303a1c02705a3b..473414e6070356 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -213,6 +213,9 @@ def choice(*opts: str) -> str: # A macro in the DSL MACRO = "MACRO" kwds.append(MACRO) +# A label in the DSL +LABEL = "LABEL" +kwds.append(LABEL) keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index db672ad5501f15..68bbb88719e682 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -3,6 +3,7 @@ Macro, Pseudo, Family, + LabelDef, Parser, Context, CacheEffect, diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index b50bb627b6c41c..151421b27983a5 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -149,8 +149,13 @@ class Pseudo(Node): targets: list[str] # opcodes this can be replaced by as_sequence: bool +@dataclass +class LabelDef(Node): + name: str + block: Block -AstNode = InstDef | Macro | Pseudo | Family + +AstNode = InstDef | Macro | Pseudo | Family | LabelDef class Parser(PLexer): @@ -164,6 +169,18 @@ def definition(self) -> AstNode | None: return pseudo if inst := self.inst_def(): return inst + if label := self.label_def(): + return label + return None + + @contextual + def label_def(self) -> LabelDef | None: + if self.expect(lx.LABEL): + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.RPAREN): + if block := self.block(): + return LabelDef(tkn.text, block) return None @contextual diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index f4409da3d83ccd..dee0f805b5540a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -32,6 +32,10 @@ FOOTER = "#undef TIER_ONE\n" +INSTRUCTION_START_MARKER = "/* BEGIN INSTRUCTIONS */" +INSTRUCTION_END_MARKER = "/* END INSTRUCTIONS */" +LABEL_START_MARKER = "/* BEGIN LABELS */" +LABEL_END_MARKER = "/* END LABELS */" def declare_variable(var: StackItem, out: CWriter) -> None: @@ -130,13 +134,64 @@ def generate_tier1( ) -> None: write_header(__file__, filenames, outfile) outfile.write( - """ + f""" #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 + +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + {{ + {INSTRUCTION_START_MARKER} """ ) + generate_tier1_cases(analysis, outfile, lines) + outfile.write(f""" + {INSTRUCTION_END_MARKER} +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + }} + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + {LABEL_START_MARKER} +""") + generate_tier1_labels(analysis, outfile, lines) + outfile.write(f"{LABEL_END_MARKER}\n") + outfile.write(FOOTER) + +def generate_tier1_labels( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, label in analysis.labels.items(): + out.emit(f"{name}:\n") + for tkn in label.body: + out.emit(tkn) + out.emit("\n") + out.emit("\n") + +def generate_tier1_cases( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: out = CWriter(outfile, 2, lines) emitter = Emitter(out) out.emit("\n") @@ -182,7 +237,6 @@ def generate_tier1( out.start_line() out.emit("}") out.emit("\n") - outfile.write(FOOTER) arg_parser = argparse.ArgumentParser(