Scheduler and memory fixes from the comparison with totk-pc-recomp; 3 recording threads at most

- core_kick matches one owner to each core waiter (best waiter first, the worst
  owner it may take), as totk-pc-recomp: asking every worse owner made all
  waiters interrupt the same owner while the others ran on.
- A checkpoint inside an SDK spin lock sets interrupt_flag, so the SDK's SVC
  0x36 hands the core over as it leaves the lock (before: the next checkpoint).
- nvnMemoryPoolInitialize/Finalize/MapVirtual and Read/WriteTexels give the
  virtual core up: pool setup ran up to many ms on the game's loader threads
  while its workers waited for that core.
- UnmapPhysicalMemory decommits each run of pages in one call (one call per
  4 KiB page before, under memory_lock); MapPhysicalMemory and SetHeapSize
  commit the new pages at once instead of one exception per 64 KiB on first
  touch (Windows).
- Recording threads: the logical processors less two, at most 3: with 4 on 6
  processors the player saw more hitches, 3 had the frame rate of 4.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/gfx/backend_vk.c

@@ -2949,13 +2949,14 @@ int vkb_mt_on(void) {
         int cpus = sched_getaffinity(0, sizeof set, &set) == 0 ? CPU_COUNT(&set) : (int)sysconf(_SC_NPROCESSORS_ONLN);
         if (cpus < 1) cpus = 4;
 #endif
-        // The logical processors less two, at most 4: the game's threads wait for the recording most of a frame, so
+        // The logical processors less two, at most 3: the game's threads wait for the recording most of a frame, so
         // the processors they leave are free. Measured (frame p50): 6 processors (Linux, RX 570, ~7700 draws a frame,
-        // 2026-11-05) 1 thread 36.0 ms, 2 24.2, 3 22.0, 4 20.2; the Steam Deck (8) submits 2-3 ms sooner with 4 than
-        // with 3 at the same draw counts; 8 processors on Windows (2026-11-03, a Depths forest, unlimited frames) 3
-        // threads 12.8-12.9 ms and 4 13.4-13.7, within the noise. 3/8 of the processors before (6: 2, 8: 3).
+        // 2026-11-05) 1 thread 36.0 ms, 2 24.2, 3 22.0, 4 20.2 - but with 4 the player saw clearly more hitches, 3 had
+        // the frame rate of 4 and the hitches of 2; 8 processors on Windows (2026-11-03, a Depths forest, unlimited
+        // frames) 3 threads 12.8-12.9 ms, 4 13.4-13.7; the Steam Deck (8) submits 2-3 ms sooner with 4 than with 3.
+        // 3/8 of the processors before (6: 2, 8: 3), then at most 4 (2026-11-05).
         const char *e = getenv("TOTK_VK_RECORD_THREADS");
-        n = e && *e ? atoi(e) : cpus - 2 < 4 ? cpus - 2 : 4;
+        n = e && *e ? atoi(e) : cpus - 2 < 3 ? cpus - 2 : 3;
         if (n > VKB_MAXREC) n = VKB_MAXREC;
         if (n < 1) n = 1;
         static const char *const diag[] = {"TOTK_VK_DRAWTRACE", "TOTK_VK_TRACE", "TOTK_SNAP", "TOTK_GPU_PROF", "TOTK_GPU_PASSPROF",

src/gfx/nvn/hle_nvn.c

@@ -325,6 +325,7 @@ static int cmdlog_env(void) { static int on = -1; if (on < 0) on = getenv("TOTK_
 static int poollog_env(void) { static int on = -1; if (on < 0) on = getenv("TOTK_POOL_LOG") != NULL; return on; }
 static int chainlog_env(void) { static int on = -1; if (on < 0) on = getenv("TOTK_CHAIN_LOG") != NULL; return on; }   // read once: EndRecording and Submit run hundreds of times a frame
 NVN(nvnMemoryPoolInitialize, NVNboolean, NVNmemoryPool *p, const NVNmemoryPoolBuilder *b) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     const pool_builder_t *s = (const pool_builder_t *)b; pool_t *o = (pool_t *)p;
     memset(o, 0, sizeof *o);
     o->magic = MAGIC('P', 'O', 'O', 'L'); o->flags = s->flags; o->mem = s->mem; o->size = s->size;
@@ -344,6 +345,7 @@ NVN(nvnMemoryPoolInitialize, NVNboolean, NVNmemoryPool *p, const NVNmemoryPoolBu
     return o->mem != NULL;
 }
 NVN(nvnMemoryPoolFinalize, void, NVNmemoryPool *p) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     pool_t *o = (pool_t *)p;
     // TOTK translation thread: deleted in queue order (x_poolrel), after the submissions queued before - the game waited
     // for the GPU work that used the pool before finalizing it, so those do not touch it; before 2026-10-28 the game thread
@@ -358,6 +360,7 @@ NVN(nvnMemoryPoolFlushMappedRange, void, const NVNmemoryPool *p, ptrdiff_t off,
 NVN(nvnMemoryPoolGetFlags, NVNmemoryPoolFlags, const NVNmemoryPool *p){return (NVNmemoryPoolFlags)((const pool_t *)p)->flags;}
 NVN(nvnMemoryPoolMap, void *, const NVNmemoryPool *p){const pool_t *v=(const void *)p;if(v->magic!=MAGIC('P','O','O','L'))return NULL;return (v->flags&NVN_MEMORY_POOL_FLAGS_CPU_NO_ACCESS)?NULL:v->mem;}
 NVN(nvnMemoryPoolMapVirtual, NVNboolean, NVNmemoryPool *p, int n, const NVNmappingRequest *r) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     return (NVNboolean)totk_vpool_map((pool_t *)p, n, (const totk_mapping_request *)(const void *)r);
 }
 NVN(nvnMemoryPoolGetBufferAddress, NVNbufferAddress, const NVNmemoryPool *p) { const pool_t *o = (const pool_t *)p; return (NVNbufferAddress)(o->va ? o->va : (uint64_t)(uintptr_t)o->mem); }
@@ -565,9 +568,11 @@ static void read_texels(const tex_t *o, const view_t *v, const NVNcopyRegion *r,
     x_read_texels(o, level, layer0, r, dst, (size_t)row, (size_t)img);
 }
 NVN(nvnTextureReadTexels, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r, void *dst) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     read_texels(CTX(t), (const view_t *)v, r, dst, 0, 0);
 }
 NVN(nvnTextureReadTexelsStrided, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r, void *dst, ptrdiff_t row, ptrdiff_t img) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     read_texels(CTX(t), (const view_t *)v, r, dst, row, img);
 }
 static void write_texels(const tex_t *o, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, ptrdiff_t row, ptrdiff_t img) {
@@ -582,9 +587,11 @@ static void write_texels(const tex_t *o, const view_t *v, const NVNcopyRegion *r
     if (mtl_write_texels_mem(o, level, layer0, r, src, (size_t)row, (size_t)img)) x_texwrite(o, level, layer0, r, src, (size_t)row, (size_t)img, wb, hb);
 }
 NVN(nvnTextureWriteTexels, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r, const void *src) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     write_texels(CTX(t), (const view_t *)v, r, src, 0, 0);
 }
 NVN(nvnTextureWriteTexelsStrided, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r, const void *src, ptrdiff_t row, ptrdiff_t img) {
+    NVN_BLOCKING();   // pool setup / texel copies run while the world streams in: without the core (as on the Switch, a driver call), not holding up the game's other threads
     write_texels(CTX(t), (const view_t *)v, r, src, row, img);
 }
 NVN(nvnTextureFlushTexels, void, const NVNtexture *t, const NVNtextureView *v, const NVNcopyRegion *r) { (void)t; (void)v; (void)r; }

src/host/svc.c

@@ -137,16 +137,28 @@ static uint64_t alias_map(int unmap, uint64_t addr, uint64_t size) {
     if (unmap) {
         for (unsigned i = 0; i < nstack_maps; i++)
             if (overlaps(addr, size, stack_maps[i].src, stack_maps[i].size)) { pthread_mutex_unlock(&memory_lock); return 0xdc01; }
+        // mapped again later: zero-filled like a fresh page. Each run of mapped pages in one call (a page a call, as
+        // before, was 8192 calls for 32 MiB, each a TLB flush across the CPUs, with memory_lock held)
         uint64_t released = 0;
-        for (unsigned i = 0; i < count; i++)
-            if (alias_pages[first + i]) {
-                alias_pages[first + i] = 0;
-                released += 4096;
-                decommit(addr + (uint64_t)i * 4096, 4096);   // mapped again later: zero-filled like a fresh page
-            }
+        for (unsigned i = 0; i < count;) {
+            if (!alias_pages[first + i]) { i++; continue; }
+            unsigned j = i;
+            while (j < count && alias_pages[first + j]) alias_pages[first + j++] = 0;
+            released += (uint64_t)(j - i) * 4096;
+            decommit(addr + (uint64_t)i * 4096, (uint64_t)(j - i) * 4096);
+            i = j;
+        }
         alias_bytes -= released;
     } else {
-        for (unsigned i = 0; i < count; i++) alias_pages[first + i] = 1;
+        // the new pages committed now, a run in one call: else each 64 KiB of them is committed by an exception on its
+        // first touch (runtime/platform/windows.c lazy_commit), thousands of times as the world streams in
+        for (unsigned i = 0; i < count;) {
+            if (alias_pages[first + i]) { i++; continue; }
+            unsigned j = i;
+            while (j < count && !alias_pages[first + j]) alias_pages[first + j++] = 1;
+            plat_commit(addr + (uint64_t)i * 4096, (uint64_t)(j - i) * 4096);
+            i = j;
+        }
         alias_bytes += fresh;
     }
     pthread_mutex_unlock(&memory_lock);
@@ -330,6 +342,7 @@ uint64_t runtime_svc(unsigned num, uint64_t *r, uint64_t pc) {
         uint64_t size = r[1];
         if (size > HEAP_CAP || size % 0x200000 || size + runtime_image_bytes() > APP_POOL) { r[0] = 0xd401; return r[0]; }
         if (size < heap_size) decommit(HEAP_START + size, heap_size - size);
+        else if (size > heap_size) plat_commit(HEAP_START + heap_size, size - heap_size);   // now, not by an exception per 64 KiB on first touch
         heap_size = size;
         r[0] = 0; r[1] = HEAP_START;
         return 0;

src/host/threads.c

@@ -125,6 +125,7 @@ typedef struct thr {
     uint64_t st_inv_ns, st_inv_max;   // core waits while a worse-priority thread held a core this one may use: total, longest
     // asynchronous preemption (below): this thread's host-call depth and Cpu (TLR), interruption state
     volatile int *depth; Cpu *cpu; int ap_ok, ap_pending; uint64_t ap_rip, ap_last;
+    struct thr *target;        // waiting for a core: the owner core_kick matched to it (one owner per waiter)
 #ifdef _WIN32
     HANDLE ap_h; CONTEXT *ap_ctx;
 #else
@@ -263,12 +264,36 @@ static void core_grant(int c) {   // holding G: give free core c to the best wai
     if (!b) return;
     *bp = b->cnext; b->cnext = NULL; b->cur_core = c; b->core_since = now_ns(); g_core_owner[c] = b; gc_signal(&b->cv);
 }
-// t waits for a core: the owners of the cores it may run on that count worse give theirs up at their next checkpoint
-// (holding G)
-static void core_kick(const thr_t *t) {
-    int tp = eff_prio(t);
-    for (int i = 0; i < NCORE; i++)
-        if (core_allowed(t, i) && g_core_owner[i] && eff_prio(g_core_owner[i]) > tp) g_core_owner[i]->preempt = 1;
+// Owners asked to give their cores up, one per waiter (holding G): the waiters best first, each claims the worst owner
+// worse than itself on a core it may run on that no better waiter claimed (a request already pending is reused), and
+// that owner gives its core up at its next checkpoint or when interrupted (asynchronous preemption, ap_try). Asking
+// every worse owner for every waiter (as before) made the waiters all interrupt the same owner while the others ran
+// on, and the cores several owners gave up for one waiter went to worse waiters, which asked again. As
+// totk-pc-recomp runtime/kernel.c core_kick (MIT). The argument is unused: every waiter is matched anew.
+static void core_kick(const thr_t *unused) {
+    (void)unused;
+    enum { NW = 64 };
+    thr_t *w[NW];
+    int wp[NW], op[NCORE], n = 0, claimed = 0;
+    for (int i = 0; i < NCORE; i++) op[i] = g_core_owner[i] ? eff_prio(g_core_owner[i]) : 0;
+    for (thr_t *p = g_core_wait; p && n < NW; p = p->cnext) {   // insertion sort: priority, then who came first
+        int i = n++, pp = eff_prio(p);
+        for (; i > 0 && (wp[i - 1] > pp || (wp[i - 1] == pp && w[i - 1]->cseq > p->cseq)); i--) w[i] = w[i - 1], wp[i] = wp[i - 1];
+        w[i] = p, wp[i] = pp;
+        p->target = NULL;
+    }
+    for (int k = 0; k < n && claimed != (1 << NCORE) - 1; k++) {
+        int v = -1;
+        for (int i = 0; i < NCORE; i++) {
+            thr_t *o = g_core_owner[i];
+            if ((claimed >> i & 1) || !core_allowed(w[k], i) || !o || op[i] <= wp[k]) continue;
+            if (v < 0 || op[i] > op[v] || (op[i] == op[v] && o->preempt && !g_core_owner[v]->preempt)) v = i;
+        }
+        if (v < 0) continue;
+        claimed |= 1 << v;
+        w[k]->target = g_core_owner[v];
+        g_core_owner[v]->preempt = 1;
+    }
 }
 static void core_put(thr_t *t) {   // holding G
     if (t->cur_core < 0) return;
@@ -562,18 +587,14 @@ __asm__(".text\n"
 #endif
 
 enum { AP_NONE, AP_RETRY, AP_SENT };
-// The owner a waiter t would interrupt: the worst one, flagged to give its core up, among those of cores t may use
-// that count worse than t (holding G).
-static thr_t *ap_target(const thr_t *t) {
-    int tp = eff_prio(t), wp = tp;
-    thr_t *o = NULL;
-    for (int i = 0; i < NCORE; i++) {
-        thr_t *h = g_core_owner[i];
-        if (!core_allowed(t, i) || !h || !h->preempt) continue;
-        int p = eff_prio(h);
-        if (p > wp) wp = p, o = h;
+// The owner a waiter t would interrupt: the one core_kick matched to it (holding G).
+static thr_t *ap_target(thr_t *t) {
+    thr_t *o = t->target;
+    if (!o || o->cur_core < 0 || !core_allowed(t, o->cur_core)) {   // stale (that owner moved on): match the waiters again
+        core_kick(t);
+        o = t->target;
     }
-    return o;
+    return o && o->cur_core >= 0 && o->preempt ? o : NULL;
 }
 // Interrupt o if it runs top-level game code (holding G; o is stopped only between ap_stop and ap_resume, during
 // which nothing here takes a lock, and it holds no host lock where it is moved).
@@ -687,9 +708,13 @@ void runtime_core_back(int *away) {
 void runtime_core_checkpoint(void) {
     thr_t *t = me_or_null(); if (!t || !t->preempt) return;
     Cpu *c = guest_cpu();
-    if (c && c->tpidrro && pi_on() && *(volatile uint16_t *)(uintptr_t)(c->tpidrro + 0x100)) return;
+    int off = c && c->tpidrro && pi_on() && *(volatile uint16_t *)(uintptr_t)(c->tpidrro + 0x100);
     g_lock();
-    if (t->cur_core >= 0 && core_better_waiter(t, t->cur_core)) { core_put(t); core_take(t); }   // core_put clears the request
+    int better = t->cur_core >= 0 && core_better_waiter(t, t->cur_core);
+    // in an SDK spin lock: interrupt_flag (TLR+0x102) makes the SDK call SVC 0x36 as it leaves, which hands the core over
+    // (runtime_ap_sync); before, the request waited for a later checkpoint
+    if (better && off) *(volatile uint16_t *)(uintptr_t)(c->tpidrro + 0x102) = 1, t->preempt = 0, g_ap_deferred++;
+    else if (better) { core_put(t); core_take(t); }   // core_put clears the request
     else t->preempt = 0;
     g_unlock();
 }