embed EG()/CG() into _tsrm_ls_cache

Author: henderkes

TSRM/TSRM.h

@@ -177,17 +177,18 @@ TSRM_API bool tsrm_is_managed_thread(void);
 #define TSRMG_BULK_STATIC(id, type)	((type) (*((void ***) TSRMLS_CACHE))[TSRM_UNSHUFFLE_RSRC_ID(id)])
 #define TSRMG_FAST_STATIC(offset, type, element)	(TSRMG_FAST_BULK_STATIC(offset, type)->element)
 #define TSRMG_FAST_BULK_STATIC(offset, type)	((type) (((char*) TSRMLS_CACHE)+(offset)))
+struct _zend_tsrm_ls_cache;
 #ifdef __cplusplus
-#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; }
-#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE; }
+#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS struct _zend_tsrm_ls_cache _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; }
+#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS struct _zend_tsrm_ls_cache _tsrm_ls_cache; }
 #else
-#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR;
-#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE;
+#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS struct _zend_tsrm_ls_cache _tsrm_ls_cache TSRM_TLS_MODEL_ATTR;
+#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS struct _zend_tsrm_ls_cache _tsrm_ls_cache;
 #endif
-#define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR = NULL;
-#define TSRMLS_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE = NULL;
+#define TSRMLS_MAIN_CACHE_DEFINE()
+#define TSRMLS_CACHE_DEFINE()
 #define TSRMLS_CACHE_UPDATE() TSRMLS_CACHE = tsrm_get_ls_cache()
-#define TSRMLS_CACHE _tsrm_ls_cache
+#define TSRMLS_CACHE (*(void **) &_tsrm_ls_cache)
 
 #ifdef __cplusplus
 }

Zend/zend.c

@@ -52,18 +52,16 @@ static bool startup_done = false;
 #ifdef ZTS
 ZEND_API int compiler_globals_id;
 ZEND_API int executor_globals_id;
-ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_executor_globals executor_globals_tls;
-ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_compiler_globals compiler_globals_tls;
+ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache = {0};
 /* ts_allocate_tls_id takes a callback so each thread resolves its own block.
  * A plain &..._tls would capture only the registering thread's address. */
-static void *executor_globals_tls_addr(void) { return &executor_globals_tls; }
-static void *compiler_globals_tls_addr(void) { return &compiler_globals_tls; }
+static void *executor_globals_tls_addr(void) { return &_tsrm_ls_cache.eg; }
+static void *compiler_globals_tls_addr(void) { return &_tsrm_ls_cache.cg; }
 static HashTable *global_function_table = NULL;
 static HashTable *global_class_table = NULL;
 static HashTable *global_constants_table = NULL;
 static HashTable *global_auto_globals_table = NULL;
 static HashTable *global_persistent_list = NULL;
-TSRMLS_MAIN_CACHE_DEFINE()
 # define GLOBAL_FUNCTION_TABLE		global_function_table
 # define GLOBAL_CLASS_TABLE			global_class_table
 # define GLOBAL_CONSTANTS_TABLE		global_constants_table
@@ -805,6 +803,7 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{
 
 static void executor_globals_ctor(zend_executor_globals *executor_globals) /* {{{ */
 {
+	_tsrm_ls_cache.self = &_tsrm_ls_cache;
 	zend_startup_constants();
 	zend_copy_constants(executor_globals->zend_constants, GLOBAL_CONSTANTS_TABLE);
 	zend_init_rsrc_plist();

Zend/zend_globals.h

@@ -327,6 +327,15 @@ struct _zend_executor_globals {
 	void *reserved[ZEND_MAX_RESERVED_RESOURCES];
 };
 
+#ifdef ZTS
+struct _zend_tsrm_ls_cache {
+	void *cache;
+	void *self;
+	zend_executor_globals eg;
+	zend_compiler_globals cg;
+};
+#endif
+
 #define EG_FLAGS_INITIAL				(0)
 #define EG_FLAGS_IN_SHUTDOWN			(1<<0)
 #define EG_FLAGS_OBJECT_STORE_NO_REUSE	(1<<1)

Zend/zend_globals_macros.h

@@ -38,11 +38,15 @@ typedef struct _zend_ini_scanner_globals zend_ini_scanner_globals;
 
 BEGIN_EXTERN_C()
 
+#ifdef ZTS
+typedef struct _zend_tsrm_ls_cache zend_tsrm_ls_cache;
+extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache;
+#endif
+
 /* Compiler */
 #ifdef ZTS
 # ifdef ZEND_TLS_DIRECT
-extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_compiler_globals compiler_globals_tls;
-#  define CG(v) (compiler_globals_tls.v)
+#  define CG(v) (_tsrm_ls_cache.cg.v)
 # else
 #  define CG(v) ZEND_TSRMG(compiler_globals_id, zend_compiler_globals *, v)
 # endif
@@ -56,8 +60,7 @@ ZEND_API int zendparse(void);
 /* Executor */
 #ifdef ZTS
 # ifdef ZEND_TLS_DIRECT
-extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_executor_globals executor_globals_tls;
-#  define EG(v) (executor_globals_tls.v)
+#  define EG(v) (_tsrm_ls_cache.eg.v)
 # else
 #  define EG(v) ZEND_TSRMG(executor_globals_id, zend_executor_globals *, v)
 # endif

ext/opcache/jit/ir/ir_aarch64.dasc

@@ -5868,8 +5868,6 @@ static void ir_emit_tls(ir_ctx *ctx, ir_ref def, ir_insn *insn)
 |		ldr Rx(reg), [Rx(reg), #insn->op3]
 ||	}
 ||# else
-||	/* op2 == 0 with no index requests the bare thread pointer (used to form
-||	 * &EG/&CG with an add); a real TLS var never sits at tprel offset 0. */
 ||	if (insn->op2 != 0 || insn->op3 != IR_NULL) {
 ||//???	IR_ASSERT(insn->op2 <= LDR_STR_PIMM64);
 |		ldr Rx(reg), [Rx(reg), #insn->op2]

ext/opcache/jit/zend_jit_ir.c

@@ -201,35 +201,21 @@ static uint32_t default_mflags = 0;
 static bool delayed_call_chain = false; // TODO: remove this var (use jit->delayed_call_level) ???
 
 #ifdef ZTS
-static size_t eg_tls_tcb_offset = 0;
-static size_t cg_tls_tcb_offset = 0;
-/* gottpoff yields the offset from the %fs-based thread pointer that ir_TLS(0)
- * loads. */
-# if defined(__ELF__) && defined(__x86_64__) && defined(__GNUC__) && !defined(TSRM_TLS_MODEL_DEFAULT)
-#  define ZEND_JIT_TLS_TCB_OFFSET(sym) __extension__({ \
-		size_t _off; \
-		__asm__ ("movq " #sym "@gottpoff(%%rip),%0" : "=r" (_off)); \
-		_off; \
-	})
-# elif defined(__ELF__) && defined(__aarch64__) && !defined(__APPLE__) && \
-		(defined(__GNUC__) || defined(__clang__))
-/* The TLS variable sits at a fixed offset from tpidr_el0 (the thread pointer
- * the JIT reads with mrs); compute it once on the main thread. Subtracting the
- * thread pointer is model-independent (works for both local- and initial-exec)
- * and matches tsrm_get_ls_cache_tcb_offset()'s tprel reasoning. */
-#  define ZEND_JIT_TLS_TCB_OFFSET(sym) __extension__({ \
-		char *_tp; \
-		__asm__ ("mrs %0, tpidr_el0" : "=r" (_tp)); \
-		(size_t)((char*)&(sym) - _tp); \
-	})
-# else
-#  define ZEND_JIT_TLS_TCB_OFFSET(sym) ((size_t)0)
-# endif
+static size_t tsrm_ls_cache_tcb_offset = 0;
+static size_t tsrm_tls_index = -1;
+static size_t tsrm_tls_offset = -1;
+
+# define EG_TLS_OFFSET(field) \
+	(tsrm_ls_cache_tcb_offset + offsetof(zend_tsrm_ls_cache, eg) + offsetof(zend_executor_globals, field))
+
+# define CG_TLS_OFFSET(field) \
+	(tsrm_ls_cache_tcb_offset + offsetof(zend_tsrm_ls_cache, cg) + offsetof(zend_compiler_globals, field))
 
 # define jit_EG(_field) \
-	ir_ADD_OFFSET(jit_EG_base(jit), offsetof(zend_executor_globals, _field))
+	ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
+
 # define jit_CG(_field) \
-	ir_ADD_OFFSET(jit_CG_base(jit), offsetof(zend_compiler_globals, _field))
+	ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
 
 #else
 
@@ -312,9 +298,7 @@ typedef struct _zend_jit_ctx {
 	uint32_t             delayed_call_level;
 	int                  b;           /* current basic block number or -1 */
 #ifdef ZTS
-	ir_ref               tp;     /* cached thread pointer for &EG/&CG */
-	ir_ref               eg_tls; /* cached base of __thread executor_globals_tls */
-	ir_ref               cg_tls; /* cached base of __thread compiler_globals_tls */
+	ir_ref               tls;
 #endif
 	ir_ref               fp;
 	ir_ref               poly_func_ref; /* restored from parent trace snapshot */
@@ -505,60 +489,41 @@ static const char* zend_reg_name(int8_t reg)
 /* IR helpers */
 
 #ifdef ZTS
-static void * ZEND_FASTCALL zend_jit_get_eg_tls(void)
-{
-	return &executor_globals_tls;
-}
-static void * ZEND_FASTCALL zend_jit_get_cg_tls(void)
+static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void)
 {
-	return &compiler_globals_tls;
+	return &_tsrm_ls_cache;
 }
 
-/* Walk the control chain back from the current point: reuse the cached ref if we
- * reach it (it still dominates here), but bail at a block start or a call, since
- * the cached value lives in a caller-saved register that a call would clobber. */
-static ir_ref jit_tls_reuse(zend_jit_ctx *jit, ir_ref cached)
+static ir_ref jit_TLS(zend_jit_ctx *jit)
 {
-	ir_ref ref = jit->ctx.control;
+	ZEND_ASSERT(jit->ctx.control);
+	if (jit->tls) {
+		/* Emit "TLS" once for basic block */
+		ir_insn *insn;
+		ir_ref ref = jit->ctx.control;
 
-	while (cached) {
-		if (ref == cached) {
-			return cached;
-		}
-		ir_insn *insn = &jit->ctx.ir_base[ref];
-		if (insn->op >= IR_START || insn->op == IR_CALL) {
-			break;
+		while (1) {
+			if (ref == jit->tls) {
+				return jit->tls;
+			}
+			insn = &jit->ctx.ir_base[ref];
+			if (insn->op >= IR_START || insn->op == IR_CALL) {
+				break;
+			}
+			ref = insn->op1;
 		}
-		ref = insn->op1;
 	}
-	return IR_UNUSED;
-}
 
-/* Thread pointer, cached per basic block, used to form &EG/&CG with an add. */
-static ir_ref jit_TP(zend_jit_ctx *jit)
-{
-	ZEND_ASSERT(jit->ctx.control);
-	if (!jit_tls_reuse(jit, jit->tp)) {
-		jit->tp = ir_TLS(0, IR_NULL);
+	if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) {
+		jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache));
+	} else if (tsrm_ls_cache_tcb_offset) {
+		jit->tls = ir_TLS(0, IR_NULL);
+	} else {
+		jit->tls = ir_TLS(tsrm_tls_index, tsrm_tls_offset + offsetof(zend_tsrm_ls_cache, self));
 	}
-	return jit->tp;
-}
 
-/* Used where the TCB offset is unknown: resolve the base via a cached call. */
-static ir_ref jit_GLOBALS_TLS_call(zend_jit_ctx *jit, ir_ref *cache, const void *fn)
-{
-	ZEND_ASSERT(jit->ctx.control);
-	if (!jit_tls_reuse(jit, *cache)) {
-		*cache = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(fn));
-	}
-	return *cache;
+	return jit->tls;
 }
-# define jit_EG_base(jit) (eg_tls_tcb_offset \
-	? ir_ADD_OFFSET(jit_TP(jit), eg_tls_tcb_offset) \
-	: jit_GLOBALS_TLS_call((jit), &(jit)->eg_tls, zend_jit_get_eg_tls))
-# define jit_CG_base(jit) (cg_tls_tcb_offset \
-	? ir_ADD_OFFSET(jit_TP(jit), cg_tls_tcb_offset) \
-	: jit_GLOBALS_TLS_call((jit), &(jit)->cg_tls, zend_jit_get_cg_tls))
 #endif
 
 static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
@@ -2855,9 +2820,7 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
 	delayed_call_chain = false;
 	jit->b = -1;
 #ifdef ZTS
-	jit->tp = IR_UNUSED;
-	jit->eg_tls = IR_UNUSED;
-	jit->cg_tls = IR_UNUSED;
+	jit->tls = IR_UNUSED;
 #endif
 	jit->fp = IR_UNUSED;
 	jit->poly_func_ref = IR_UNUSED;
@@ -3251,8 +3214,7 @@ static void zend_jit_setup_disasm(void)
 
 	REGISTER_DATA(CG(map_ptr_base));
 #else /* ZTS */
-	REGISTER_HELPER(zend_jit_get_eg_tls);
-	REGISTER_HELPER(zend_jit_get_cg_tls);
+	REGISTER_HELPER(zend_jit_get_tsrm_ls_cache);
 #endif
 #endif
 }
@@ -3463,8 +3425,15 @@ static void zend_jit_setup(bool reattached)
 #endif
 
 #ifdef ZTS
-	eg_tls_tcb_offset = ZEND_JIT_TLS_TCB_OFFSET(executor_globals_tls);
-	cg_tls_tcb_offset = ZEND_JIT_TLS_TCB_OFFSET(compiler_globals_tls);
+	zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets(
+		&tsrm_ls_cache_tcb_offset,
+		&tsrm_tls_index,
+		&tsrm_tls_offset
+	);
+	if (result == FAILURE) {
+		zend_accel_error(ACCEL_LOG_INFO,
+				"Could not get _tsrm_ls_cache offsets, will fallback to runtime resolution");
+	}
 #endif
 
 #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)