Universal global regs 3

.github/workflows/push.yml

@@ -123,6 +123,7 @@ jobs:
           configurationParameters: >-
             --${{ matrix.debug && 'enable' || 'disable' }}-debug
             --${{ matrix.zts && 'enable' || 'disable' }}-zts
+            --enable-universal-ip
             ${{ matrix.asan && 'CFLAGS="-fsanitize=undefined,address -DZEND_TRACK_ARENA_ALLOC" LDFLAGS="-fsanitize=undefined,address" CC=clang-16 CXX=clang++-16 --disable-opcache-jit' || '' }}
           skipSlow: ${{ matrix.asan }}
       - name: make
@@ -249,7 +250,8 @@ jobs:
             --with-mysqli=mysqlnd \
             --with-openssl \
             --with-pdo-sqlite \
-            --with-valgrind
+            --with-valgrind \
+            --enable-universal-ip
       - name: make
         run: make -j$(/usr/bin/nproc) >/dev/null
       - name: make install

Zend/Zend.m4

@@ -385,11 +385,60 @@ int emu(const opcode_handler_t *ip, void *fp) {
     ZEND_GCC_GLOBAL_REGS=no
   ])
 fi
+
 if test "$ZEND_GCC_GLOBAL_REGS" = "yes"; then
   AC_DEFINE([HAVE_GCC_GLOBAL_REGS], 1, [Define if the target system has support for global register variables])
 fi
 AC_MSG_RESULT($ZEND_GCC_GLOBAL_REGS)
 
+AC_ARG_ENABLE([universal-ip],
+  [AS_HELP_STRING([--enable-universal-ip],
+    [whether to use a GCC global register variables for IP universally])],
+  [ZEND_UNIVERSAL_IP=$enableval],
+  [ZEND_UNIVERSAL_IP=no])
+
+if test "$ZEND_UNIVERSAL_IP" = "yes"; then
+  if test "$ZEND_GCC_GLOBAL_REGS" != "yes"; then
+    AC_MSG_ERROR(May not enable universal IP without --enable-universal-ip)
+  fi
+
+  AC_RUN_IFELSE(
+    [AC_LANG_SOURCE([[
+#include <stdio.h>
+
+#if defined(__GNUC__)
+# define ZEND_GCC_VERSION (__GNUC__ * 1000 + __GNUC_MINOR__)
+#else
+# define ZEND_GCC_VERSION 0
+#endif
+#if defined(__GNUC__) && ZEND_GCC_VERSION >= 4008 && defined(i386)
+# define ZEND_VM_IP_GLOBAL_REG "edi"
+#elif defined(__GNUC__) && ZEND_GCC_VERSION >= 4008 && defined(__x86_64__)
+# define ZEND_VM_IP_GLOBAL_REG "r15"
+#elif defined(__GNUC__) && ZEND_GCC_VERSION >= 4008 && defined(__powerpc64__)
+# define ZEND_VM_IP_GLOBAL_REG "r29"
+#elif defined(__IBMC__) && ZEND_GCC_VERSION >= 4002 && defined(__powerpc64__)
+# define ZEND_VM_IP_GLOBAL_REG "r29"
+#elif defined(__GNUC__) && ZEND_GCC_VERSION >= 4008 && defined(__aarch64__)
+# define ZEND_VM_IP_GLOBAL_REG "x28"
+#elif defined(__GNUC__) && ZEND_GCC_VERSION >= 4008 && defined(__riscv) && __riscv_xlen == 64
+# define ZEND_VM_IP_GLOBAL_REG "x19"
+#else
+# error "global register variables are not supported"
+#endif
+
+int main(void) {
+  fprintf(stderr, "%s\n", ZEND_VM_IP_GLOBAL_REG);
+  return 0;
+}
+    ]])],
+    [zend_ip_reg=$(./conftest$EXEEXT 2>&1)],
+    [AC_MSG_ERROR(Unexpected, guarded above)])
+
+  AC_DEFINE([ZEND_UNIVERSAL_IP], 1, [Define whether to use global register variables universally])
+  CFLAGS="$CFLAGS -ffixed-$zend_ip_reg"
+fi
+
 dnl Check whether __cpuid_count is available.
 AC_CACHE_CHECK(whether __cpuid_count is available, ac_cv_cpuid_count_available, [
 AC_LINK_IFELSE([AC_LANG_PROGRAM([[

Zend/zend_exceptions.c

@@ -29,6 +29,7 @@
 #include "zend_smart_str.h"
 #include "zend_exceptions_arginfo.h"
 #include "zend_observer.h"
+#include "zend_universal_ip.h"
 
 ZEND_API zend_class_entry *zend_ce_throwable;
 ZEND_API zend_class_entry *zend_ce_exception;
@@ -164,7 +165,69 @@ static zend_always_inline bool is_handle_exception_set(void) {
 	return !execute_data
 		|| !execute_data->func
 		|| !ZEND_USER_CODE(execute_data->func->common.type)
+#ifdef ZEND_UNIVERSAL_IP
+		|| zend_universal_ip->opcode == ZEND_HANDLE_EXCEPTION;
+#else
 		|| execute_data->opline->opcode == ZEND_HANDLE_EXCEPTION;
+#endif
+}
+
+#ifdef ZEND_UNIVERSAL_IP
+static void zend_copy_exception_consts(uint32_t opnum)
+{
+	const zend_op *orig_op = &EG(opline_before_exception)[opnum];
+	zend_op *exception_op = &EG(exception_op)[opnum];
+
+	if (orig_op->op1_type == IS_CONST) {
+		zval *op1 = &EG(exception_consts)[opnum * 2];
+		ZVAL_COPY_VALUE(op1, RT_CONSTANT(orig_op, orig_op->op1));
+		exception_op->op1.constant = (char *)op1 - (char *)exception_op;
+	}
+	if (orig_op->op2_type == IS_CONST) {
+		zval *op2 = &EG(exception_consts)[opnum * 2 + 1];
+		ZVAL_COPY_VALUE(op2, RT_CONSTANT(orig_op, orig_op->op2));
+		exception_op->op2.constant = (char *)op2 - (char *)exception_op;
+	}
+}
+
+static void zend_copy_exception_ops(void)
+{
+	const zend_op *orig_op = EG(opline_before_exception);
+	zend_op *exception_op = &EG(exception_op)[0];
+	bool has_opdata = orig_op[1].opcode == ZEND_OP_DATA;
+
+	/* The handler may still access opline. Make sure operands keep working. */
+	const void *handler = exception_op->handler;
+	*exception_op = *orig_op;
+	exception_op->opcode = ZEND_HANDLE_EXCEPTION;
+	exception_op->handler = handler;
+	zend_copy_exception_consts(/* opnum */ 0);
+	if (has_opdata) {
+		exception_op[1] = orig_op[1];
+		exception_op[1].opcode = ZEND_OP_DATA;
+		zend_copy_exception_consts(/* opnum */ 1);
+	} else {
+		exception_op[1] = exception_op[2];
+	}
+}
+#endif
+
+ZEND_API void zend_rethrow_exception(zend_execute_data *execute_data)
+{
+#ifdef ZEND_UNIVERSAL_IP
+	if (zend_universal_ip->opcode != ZEND_HANDLE_EXCEPTION) {
+		EG(opline_before_exception) = zend_universal_ip;
+	}
+#else
+	if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
+		EG(opline_before_exception) = EX(opline);
+	}
+#endif
+	EX(opline) = EG(exception_op);
+#ifdef ZEND_UNIVERSAL_IP
+	zend_universal_ip = EG(exception_op);
+	zend_copy_exception_ops();
+#endif
 }
 
 ZEND_API ZEND_COLD void zend_throw_exception_internal(zend_object *exception) /* {{{ */
@@ -223,8 +286,16 @@ ZEND_API ZEND_COLD void zend_throw_exception_internal(zend_object *exception) /*
 		/* no need to rethrow the exception */
 		return;
 	}
+#ifdef ZEND_UNIVERSAL_IP
+	EG(opline_before_exception) = zend_universal_ip;
+#else
 	EG(opline_before_exception) = EG(current_execute_data)->opline;
+#endif
 	EG(current_execute_data)->opline = EG(exception_op);
+#ifdef ZEND_UNIVERSAL_IP
+	zend_universal_ip = EG(exception_op);
+	zend_copy_exception_ops();
+#endif
 }
 /* }}} */
 
@@ -244,6 +315,9 @@ ZEND_API void zend_clear_exception(void) /* {{{ */
 	OBJ_RELEASE(exception);
 	if (EG(current_execute_data)) {
 		EG(current_execute_data)->opline = EG(opline_before_exception);
+		if (ZEND_USER_CODE(EG(current_execute_data)->func->type)) {
+			LOAD_CURRENT_OPLINE();
+		}
 	}
 #if ZEND_DEBUG
 	EG(opline_before_exception) = NULL;
@@ -1017,15 +1091,23 @@ ZEND_API ZEND_COLD void zend_throw_unwind_exit(void)
 {
 	ZEND_ASSERT(!EG(exception));
 	EG(exception) = zend_create_unwind_exit();
+#ifdef ZEND_UNIVERSAL_IP
+	EG(opline_before_exception) = zend_universal_ip;
+#else
 	EG(opline_before_exception) = EG(current_execute_data)->opline;
+#endif
 	EG(current_execute_data)->opline = EG(exception_op);
 }
 
 ZEND_API ZEND_COLD void zend_throw_graceful_exit(void)
 {
 	ZEND_ASSERT(!EG(exception));
 	EG(exception) = zend_create_graceful_exit();
+#ifdef ZEND_UNIVERSAL_IP
+	EG(opline_before_exception) = zend_universal_ip;
+#else
 	EG(opline_before_exception) = EG(current_execute_data)->opline;
+#endif
 	EG(current_execute_data)->opline = EG(exception_op);
 }
 

Zend/zend_exceptions.h

@@ -80,13 +80,7 @@ ZEND_API bool zend_is_graceful_exit(const zend_object *ex);
 
 #include "zend_globals.h"
 
-static zend_always_inline void zend_rethrow_exception(zend_execute_data *execute_data)
-{
-	if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
-		EG(opline_before_exception) = EX(opline);
-		EX(opline) = EG(exception_op);
-	}
-}
+ZEND_API void zend_rethrow_exception(zend_execute_data *execute_data);
 
 END_EXTERN_C()
 

Zend/zend_execute.c

@@ -106,7 +106,19 @@
 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
   register const zend_op* volatile opline __asm__(ZEND_VM_IP_GLOBAL_REG);
 # pragma GCC diagnostic warning "-Wvolatile-register-var"
+#endif
+
+#ifdef ZEND_UNIVERSAL_IP
+# define ZEND_CURRENT_OPLINE opline
+# define LOAD_CURRENT_OPLINE() \
+	do { \
+		if (EG(current_execute_data)) { \
+			opline = EG(current_execute_data)->opline; \
+		} \
+	} while (0)
 #else
+# define ZEND_CURRENT_OPLINE EX(opline)
+# define LOAD_CURRENT_OPLINE()
 #endif
 
 #define _CONST_CODE  0
@@ -279,12 +291,12 @@ static zend_never_inline ZEND_COLD zval* zval_undefined_cv(uint32_t var EXECUTE_
 
 static zend_never_inline ZEND_COLD zval* ZEND_FASTCALL _zval_undefined_op1(EXECUTE_DATA_D)
 {
-	return zval_undefined_cv(EX(opline)->op1.var EXECUTE_DATA_CC);
+	return zval_undefined_cv(ZEND_CURRENT_OPLINE->op1.var EXECUTE_DATA_CC);
 }
 
 static zend_never_inline ZEND_COLD zval* ZEND_FASTCALL _zval_undefined_op2(EXECUTE_DATA_D)
 {
-	return zval_undefined_cv(EX(opline)->op2.var EXECUTE_DATA_CC);
+	return zval_undefined_cv(ZEND_CURRENT_OPLINE->op2.var EXECUTE_DATA_CC);
 }
 
 #define ZVAL_UNDEFINED_OP1() _zval_undefined_op1(EXECUTE_DATA_C)
@@ -1680,8 +1692,10 @@ static zend_never_inline zend_long zend_check_string_offset(zval *dim, int type
 ZEND_API ZEND_COLD void zend_wrong_string_offset_error(void)
 {
 	const char *msg = NULL;
+#ifndef ZEND_UNIVERSAL_IP
 	const zend_execute_data *execute_data = EG(current_execute_data);
 	const zend_op *opline = execute_data->opline;
+#endif
 
 	if (UNEXPECTED(EG(exception) != NULL)) {
 		return;
@@ -5134,18 +5148,29 @@ zval * ZEND_FASTCALL zend_handle_named_arg(
 	return arg;
 }
 
-static zend_execute_data *start_fake_frame(zend_execute_data *call, const zend_op *opline) {
+static zend_execute_data *start_fake_frame(zend_execute_data *call, const zend_op *op) {
 	zend_execute_data *old_prev_execute_data = call->prev_execute_data;
 	call->prev_execute_data = EG(current_execute_data);
-	call->opline = opline;
+	call->opline = op;
 	EG(current_execute_data) = call;
+#ifdef ZEND_UNIVERSAL_IP
+	if (call->prev_execute_data) {
+		call->prev_execute_data->opline = opline;
+	}
+	opline = op;
+#endif
 	return old_prev_execute_data;
 }
 
 static void end_fake_frame(zend_execute_data *call, zend_execute_data *old_prev_execute_data) {
 	zend_execute_data *prev_execute_data = call->prev_execute_data;
 	EG(current_execute_data) = prev_execute_data;
 	call->prev_execute_data = old_prev_execute_data;
+#ifdef ZEND_UNIVERSAL_IP
+	if (prev_execute_data) {
+		opline = prev_execute_data->opline;
+	}
+#endif
 	if (UNEXPECTED(EG(exception)) && ZEND_USER_CODE(prev_execute_data->func->common.type)) {
 		zend_rethrow_exception(prev_execute_data);
 	}
@@ -5306,7 +5331,7 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
 #define ZEND_VM_NEXT_OPCODE_EX(check_exception, skip) \
 	CHECK_SYMBOL_TABLES() \
 	if (check_exception) { \
-		OPLINE = EX(opline) + (skip); \
+		OPLINE = ZEND_CURRENT_OPLINE + (skip); \
 	} else { \
 		ZEND_ASSERT(!EG(exception)); \
 		OPLINE = opline + (skip); \
@@ -5354,7 +5379,7 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
 	ZEND_VM_CONTINUE()
 #define ZEND_VM_SMART_BRANCH(_result, _check) do { \
 		if ((_check) && UNEXPECTED(EG(exception))) { \
-			OPLINE = EX(opline); \
+			OPLINE = ZEND_CURRENT_OPLINE; \
 		} else if (EXPECTED(opline->result_type == (IS_SMART_BRANCH_JMPZ|IS_TMP_VAR))) { \
 			if (_result) { \
 				ZEND_VM_SET_NEXT_OPCODE(opline + 2); \
@@ -5375,7 +5400,7 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
 	} while (0)
 #define ZEND_VM_SMART_BRANCH_JMPZ(_result, _check) do { \
 		if ((_check) && UNEXPECTED(EG(exception))) { \
-			OPLINE = EX(opline); \
+			OPLINE = ZEND_CURRENT_OPLINE; \
 		} else if (_result) { \
 			ZEND_VM_SET_NEXT_OPCODE(opline + 2); \
 		} else { \
@@ -5385,7 +5410,7 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
 	} while (0)
 #define ZEND_VM_SMART_BRANCH_JMPNZ(_result, _check) do { \
 		if ((_check) && UNEXPECTED(EG(exception))) { \
-			OPLINE = EX(opline); \
+			OPLINE = ZEND_CURRENT_OPLINE; \
 		} else if (!(_result)) { \
 			ZEND_VM_SET_NEXT_OPCODE(opline + 2); \
 		} else { \

Zend/zend_execute_API.c

@@ -38,6 +38,7 @@
 #include "zend_inheritance.h"
 #include "zend_observer.h"
 #include "zend_call_stack.h"
+#include "zend_universal_ip.h"
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
 #endif
@@ -644,15 +645,21 @@ ZEND_API uint32_t zend_get_executed_lineno(void) /* {{{ */
 		ex = ex->prev_execute_data;
 	}
 	if (ex) {
-		if (!ex->opline) {
+#ifdef ZEND_UNIVERSAL_IP
+		const zend_op *opline = ex == EG(current_execute_data) ? zend_universal_ip : ex->opline;
+#else
+		const zend_op *opline = ex->opline;
+#endif
+
+		if (!opline) {
 			/* Missing SAVE_OPLINE()? Falling back to first line of function */
 			return ex->func->op_array.opcodes[0].lineno;
 		}
-		if (EG(exception) && ex->opline->opcode == ZEND_HANDLE_EXCEPTION &&
-		    ex->opline->lineno == 0 && EG(opline_before_exception)) {
+		if (EG(exception) && opline->opcode == ZEND_HANDLE_EXCEPTION &&
+		    opline->lineno == 0 && EG(opline_before_exception)) {
 			return EG(opline_before_exception)->lineno;
 		}
-		return ex->opline->lineno;
+		return opline->lineno;
 	} else {
 		return 0;
 	}
@@ -810,6 +817,7 @@ zend_result zend_call_function(zend_fcall_info *fci, zend_fcall_info_cache *fci_
 
 	call = zend_vm_stack_push_call_frame(call_info,
 		func, fci->param_count, object_or_called_scope);
+	SAVE_CURRENT_OPLINE();
 
 	if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_DEPRECATED)) {
 		zend_deprecated_function(func);