Mono provider: guest code entering the runtime is Running whatever state it called from

The crossing hooks were one stack: calling guest code pushed whether it
entered GC-safe, any entry from guest code popped it. Guest code that a
P/Invoke called runs inside the interpreter's own GC-safe region, so the
call pushed nothing, and when it called the embedding API back (Hollow
Knight: mono_gc_get_used_size, mono_runtime_invoke from inside an internal
call) the runtime ran in the Blocking state and asserted ("Cannot transition
thread from STATE_BLOCKING with DO_BLOCKING"). Now: a call into guest code
enters the region only when Running and its return leaves only what it
entered (the cookie is the caller's); an entry from guest code makes the
thread Running for its duration (enter_gc_unsafe, which aborts a blocking
state) and its return restores the state; a raise through guest frames
(gexp_call_guest_catching) drops the abandoned entries and puts the thread
back in the state it called from. embed passes.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01PeRutwFgZ66XpxjasKrPn1

tools/unity-dotnet/mono/host/gexmono-provider.c

@@ -138,40 +138,96 @@ dl_symbol (void *handle, const char *name, char **err, void *user_data)
  * A thread attached to the runtime runs guest code in GC Safe (blocking)
  * mode, so a collection does not wait for it to poll: gex stops it at its
  * next safepoint instead (sgen-stw.c, gexp_stop_guest_threads), and its
- * guest stack and registers are scanned conservatively (sgen-mono.c). Each
- * crossing into guest code pushes whether it entered the region. */
+ * guest stack and registers are scanned conservatively (sgen-mono.c).
+ *
+ * Calling guest code enters the region unless the thread is already in it
+ * (the interpreter's P/Invoke transition), and its return leaves only what
+ * the call entered. Entering from guest code makes the thread Running for
+ * the call's duration whatever state it was in -- guest code a P/Invoke
+ * called, calling the embedding API back (Unity does: mono_runtime_invoke,
+ * mono_gc_get_used_size), is in the interpreter's GC-safe region -- and the
+ * return to guest code puts it back; a thread that was Running in guest code
+ * (its first entry attached it) is left GC-safe. */
 
 #include <mono/utils/mono-threads.h>
 #include <mono/utils/mono-threads-api.h>
 
 #define GEXMONO_SAFE_MAX 256
-static __thread gpointer safe_cookie [GEXMONO_SAFE_MAX];
-static __thread int safe_depth;
+static __thread gpointer entry_cookie [GEXMONO_SAFE_MAX];
+static __thread int entry_depth;
 
-void
-gexp_hook_to_guest (void)
+void *
+gexp_hook_call_guest (void)
 {
-	gpointer cookie = NULL;
 	MonoThreadInfo *info = mono_thread_info_current_unchecked ();
 	if (info && mono_thread_info_current_state (info) == STATE_RUNNING) {
 		gpointer dummy;
-		cookie = mono_threads_enter_gc_safe_region_unbalanced (&dummy);
+		return mono_threads_enter_gc_safe_region_unbalanced (&dummy);
+	}
+	return NULL;
+}
+
+void
+gexp_hook_call_return (void *cookie)
+{
+	if (cookie) {
+		gpointer dummy;
+		mono_threads_exit_gc_safe_region_unbalanced (cookie, &dummy);
 	}
-	if (safe_depth < GEXMONO_SAFE_MAX)
-		safe_cookie [safe_depth] = cookie;
-	safe_depth++;
 }
 
 void
 gexp_hook_from_guest (void)
 {
-	if (!safe_depth)
-		return; /* the thread's first entry */
-	safe_depth--;
-	gpointer cookie = safe_depth < GEXMONO_SAFE_MAX ? safe_cookie [safe_depth] : NULL;
-	if (cookie) {
+	gpointer cookie = NULL;
+	MonoThreadInfo *info = mono_thread_info_current_unchecked ();
+	if (info && mono_thread_info_current_state (info) != STATE_RUNNING) {
 		gpointer dummy;
-		mono_threads_exit_gc_safe_region_unbalanced (cookie, &dummy);
+		cookie = mono_threads_enter_gc_unsafe_region_unbalanced (&dummy);
+	}
+	if (entry_depth < GEXMONO_SAFE_MAX)
+		entry_cookie [entry_depth] = cookie;
+	entry_depth++;
+}
+
+void
+gexp_hook_to_guest (void)
+{
+	gpointer cookie = NULL;
+	if (entry_depth > 0) {
+		entry_depth--;
+		cookie = entry_depth < GEXMONO_SAFE_MAX ? entry_cookie [entry_depth] : NULL;
+	}
+	gpointer dummy;
+	if (cookie) {
+		mono_threads_exit_gc_unsafe_region_unbalanced (cookie, &dummy);
+		return;
+	}
+	MonoThreadInfo *info = mono_thread_info_current_unchecked ();
+	if (info && mono_thread_info_current_state (info) == STATE_RUNNING)
+		mono_threads_enter_gc_safe_region_unbalanced (&dummy);
+}
+
+/* A raise that abandoned guest frames (gexp_call_guest_catching): the
+ * entries inside them never return, and the thread (Running: it raised) goes
+ * back to the state it called the guest in -- GC-safe when the interpreter's
+ * P/Invoke had entered the region, which it will leave itself. */
+int
+gexp_hook_mark (void)
+{
+	MonoThreadInfo *info = mono_thread_info_current_unchecked ();
+	int safe = info && mono_thread_info_current_state (info) != STATE_RUNNING;
+	return entry_depth * 2 + safe;
+}
+
+void
+gexp_hook_unwind (int mark)
+{
+	entry_depth = mark >> 1;
+	MonoThreadInfo *info = mono_thread_info_current_unchecked ();
+	if ((mark & 1) && info && mono_thread_info_current_state (info) == STATE_RUNNING) {
+		gpointer dummy;
+		mono_threads_enter_gc_safe_region_unbalanced (&dummy);
 	}
 }
 

tools/unity-dotnet/provider/gexprov64.c

@@ -18,28 +18,45 @@ __attribute__((import_module("gex"), import_name("provider_stop_guest"))) void g
 __attribute__((import_module("gex"), import_name("provider_start_guest"))) void gexp_host_start_guest(void);
 __attribute__((import_module("gex"), import_name("provider_guest_stack_end"))) uint64_t gexp_host_guest_stack_end(uint64_t sp);
 
-/* A runtime's hooks at the crossing (a GC-safe region while in guest code). */
+/* A runtime's hooks at the crossing (a GC-safe region while in guest code).
+ * Entering from guest code and returning to it (an adapter: gexp_enter,
+ * gexp_return) are one pair; calling guest code and its return
+ * (gexp_call_guest) the other, whose cookie the caller keeps. A raise that
+ * abandons guest frames resets the entry pairs to a mark. */
 static void no_hook(void) {}
+static void *no_call_hook(void) { return 0; }
+static void no_call_return(void *cookie) {}
+static int no_mark(void) { return 0; }
+static void no_unwind(int mark) {}
 __attribute__((weak, alias("no_hook"))) void gexp_hook_to_guest(void);
 __attribute__((weak, alias("no_hook"))) void gexp_hook_from_guest(void);
+__attribute__((weak, alias("no_call_hook"))) void *gexp_hook_call_guest(void);
+__attribute__((weak, alias("no_call_return"))) void gexp_hook_call_return(void *cookie);
+__attribute__((weak, alias("no_mark"))) int gexp_hook_mark(void);
+__attribute__((weak, alias("no_unwind"))) void gexp_hook_unwind(int mark);
 
 void gexp_attach_thread(void)
 {
 	gexp_host_thread_register(&gexp_in_guest, &gexp_stop);
 }
 
+static void in_guest(int v)
+{
+	__atomic_store_n(&gexp_in_guest, v, __ATOMIC_SEQ_CST);
+	if (v && __atomic_load_n(&gexp_stop, __ATOMIC_SEQ_CST))
+		gexp_host_park();
+}
+
 void gexp_from_guest(void)
 {
-	__atomic_store_n(&gexp_in_guest, 0, __ATOMIC_SEQ_CST);
+	in_guest(0);
 	gexp_hook_from_guest();
 }
 
 void gexp_to_guest(void)
 {
 	gexp_hook_to_guest();
-	__atomic_store_n(&gexp_in_guest, 1, __ATOMIC_SEQ_CST);
-	if (__atomic_load_n(&gexp_stop, __ATOMIC_SEQ_CST))
-		gexp_host_park();
+	in_guest(1);
 }
 
 void gexp_stop_guest_threads(void)
@@ -119,7 +136,8 @@ void gexp_call_guest(uint64_t fn, gexp_call *c)
 	for (unsigned i = 0; i < c->nxmm && i < 8; i++)
 		*(volatile uint64_t *)((uintptr_t)(uint32_t)ctx + GEXP_CTX_V + 16 * i) = c->xmm[i];
 	GEXP_X(ctx, 4) = sp;
-	gexp_to_guest();
+	void *cookie = gexp_hook_call_guest();
+	in_guest(1);
 	/* Run the guest until it returns to `ra`. A function body returns 1 when
 	 * its return went to the address its caller predicted; otherwise 0, with
 	 * PC where the guest continues (a longjmp inside the callee, a signal, a
@@ -139,7 +157,8 @@ void gexp_call_guest(uint64_t fn, gexp_call *c)
 	gexp_ctx = ctx;
 	gexp_depth = depth;
 	__gex_set_ctx((uint32_t)ctx);
-	gexp_from_guest();
+	in_guest(0);
+	gexp_hook_call_return(cookie);
 	GEXP_X(ctx, 4) = rsp0;
 	c->rax = GEXP_X(ctx, 0);
 	c->rdx = GEXP_X(ctx, 2);
@@ -177,6 +196,7 @@ int gexp_catching(void)
 uint64_t gexp_call_guest_catching(uint64_t fn, gexp_call *c)
 {
 	catching k = { catch_top, fn, c, 0, gexp_ctx, gexp_depth };
+	int mark = gexp_hook_mark();
 	for (int i = 0; i < 16; i++)
 		k.saved[i] = GEXP_X(k.ctx, i);
 	catch_top = &k;
@@ -191,6 +211,8 @@ uint64_t gexp_call_guest_catching(uint64_t fn, gexp_call *c)
 	gexp_ctx = k.ctx;
 	gexp_depth = k.depth;
 	__gex_set_ctx((uint32_t)k.ctx);
+	in_guest(0);
+	gexp_hook_unwind(mark);
 	if (code != GEXP_RAISE_CODE)
 		gexp_host_throw(code); /* someone else's: keep going */
 	return k.payload ? k.payload : 1;