ssa: rename the blocks only the virtual root dominates

A block reached from two entries (the function's own and an interior
entry) has the virtual root as its only dominator. The renaming walk
started only at the entry blocks, so that block and everything it
dominates kept the memory form. A jump table ending there read its index
from the context and no longer resolved. In UNDERTALE.exe this hit the
GameMaker VM's two 103-case opcode switches at 0x5441af and 0x5441f9: they
resolved until an interior entry appeared, and were then dropped for
good. Those blocks are now roots of the walk too, starting from their
parameters.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_ea9f71488b6e448da93d6c2dabf4879c

crates/gex-ir/src/ssa.rs

@@ -292,10 +292,20 @@ pub fn to_ssa_sp(f: &Function, call: &dyn Fn(Callee) -> CallEffect, sp: GuestReg
     let mut end_state: Vec<Option<State>> = vec![None; n];
     let mut subst: HashMap<Value, Value> = HashMap::new();
 
-    // Iterative preorder over the dominator tree.
+    // Iterative preorder over the dominator tree: a tree per entry block,
+    // and one per reachable block whose only dominator is the virtual root
+    // above the entries (a join of paths from two entries, such as the code
+    // after an interior entry that the function's own entry also reaches).
+    // Such a block starts with nothing known but its parameters: it is in
+    // the dominance frontier of every entry, which defines every register.
+    // Left out, its tree kept its memory form, and a jump table ending in it
+    // read its index from the context and no longer resolved.
     let mut stack: Vec<BlockId> = entries.iter().rev().copied().collect();
-    // Also any reachable block with no idom that is not an entry (cannot
-    // happen, but keeps the walk total).
+    for &b in rpo.iter().rev() {
+        if idom[b.index()].is_none() && !is_entry[b.index()] {
+            stack.insert(0, b);
+        }
+    }
     while let Some(b) = stack.pop() {
         let mut st: State = match idom[b.index()] {
             None => vec![None; N_REGS],

crates/gex-ir/tests/func.rs

@@ -617,3 +617,55 @@ fn x87_arithmetic_and_compares_round_trip_and_run() {
     assert_eq!(o.regs[reg_index(x(1))], 0x3fff);
     assert_eq!(o.regs[reg_index(x(2))], 0);
 }
+
+/// Entry `x1 = 1`, interior entry `x1 = 2`, both into `x2 = x1 + 1`, then a
+/// block that reads x2 back and returns. The join's only dominator is the
+/// virtual root above the two entries.
+fn two_entries_join() -> Function {
+    let mut t = F::new(0x1000);
+    let b0 = t.block(0x1000);
+    let b1 = t.block(0x1010);
+    let b2 = t.block(0x1020);
+    let b3 = t.block(0x1030);
+    t.f.entries = vec![0x1010];
+    let one = t.def(b0, Expr::ConstI64(1));
+    t.write(b0, x(1), one);
+    t.term(b0, FTerm::Jump(Edge::to(b2)));
+    let two = t.def(b1, Expr::ConstI64(2));
+    t.write(b1, x(1), two);
+    t.term(b1, FTerm::Jump(Edge::to(b2)));
+    let v = t.read(b2, x(1));
+    let k = t.def(b2, Expr::ConstI64(1));
+    let w = t.def(b2, Expr::Bin { op: BinOp::Add, ty: Ty::I64, a: v, b: k });
+    t.write(b2, x(2), w);
+    t.term(b2, FTerm::Jump(Edge::to(b3)));
+    let r = t.read(b3, x(2));
+    t.write(b3, x(0), r);
+    let lr = t.read(b3, x(30));
+    t.term(b3, FTerm::Return { target: lr });
+    t.f
+}
+
+#[test]
+fn a_join_of_two_entries_is_renamed_too() {
+    let f = two_entries_join();
+    f.verify().unwrap();
+    let s = to_ssa(&f, &|_| CallEffect::UNKNOWN);
+    s.verify().unwrap_or_else(|e| panic!("{e}\n{}", text::print(&s)));
+    roundtrip(&s);
+    // x1 is a parameter of the join, and x2 is forwarded from its write:
+    // neither is read from the context after the entries.
+    let join = s.block_at(0x1020).unwrap();
+    let regs: Vec<GuestReg> = s.block(join).params.iter().map(|p| p.reg).collect();
+    assert!(regs.contains(&x(1)), "{}", text::print(&s));
+    let reads: Vec<GuestReg> = [0x1020, 0x1030]
+        .iter()
+        .flat_map(|&a| s.block(s.block_at(a).unwrap()).stmts.iter())
+        .filter_map(|st| match st {
+            Stmt::Def { expr: Expr::ReadReg(r), .. } => Some(*r),
+            _ => None,
+        })
+        .collect();
+    assert!(!reads.contains(&x(1)) && !reads.contains(&x(2)), "{reads:?}\n{}", text::print(&s));
+    same_run(&f, &s, &|_| CallEffect::UNKNOWN, GuestReg::Sp);
+}