Census: orders renders every function under each write order and compares what must not move

`census.py orders` runs `pddj` over every function of the given binaries with the natural order and
the three `placement_mutants` orders (`R2SLEIGH_WRITE_ORDER` = o1, o2, o3) and compares per
function: the ledger's columns, the names read as never assigned (from the proof line), the residual
sites by instruction address, cause, gap kind, callee and reason, the declared variables and links,
and the set of statement lines apart from control (labels, jumps, braces) and site numbers. It
fails when a verdict differs from natural's, or when an order changes the text of fewer functions
than `--min-live` (a knob that moves nothing proves nothing), and reports a function refused on one
side only. A difference that only the selection pass explains (`if (c) x = a; else x = b;` against
`x = c ? a : b;`, a structure stage that reads the layout) is counted apart, not as a move.

Measured over bash, ls-toybox and pigz (2,508 functions) with the mutant build of each commit of the
branch: the mutants alone (base behaviour) 0 differ under O1, 120 under O2 and 187 under O3 (the
skeptic's 119 and 183 by ledger alone); the spelling pass 0, 0, 0; the forward-edge rule 0, 0, 0. Text
changes in 64 functions under O1 (an exit is inlined only in a loop that has one), 338 under O2 and
722 under O3. Python tests for the fingerprint, the selection rule and the refusal parity.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01456qmqCGpCWJkrg6JUZNNM

scripts/census.py

@@ -19,6 +19,12 @@
                                                    peak RSS grow with a function's instructions
                                                    (ROADMAP LX's exit): `afl; pdd @ f` less `afl`,
                                                    under `perf stat` (Linux); fails above X
+  census.py orders --r2s R2S [--out DIR] [--reuse] [--min-live N] BINARY...
+                                                   pddj of every function under the write orders of a
+                                                   `--features placement_mutants` build (natural, then
+                                                   o1: no exit inlined, o2: exits reversed, o3: merges
+                                                   reversed); fails when a verdict differs from
+                                                   natural's (ROADMAP D58) or an order moves no text
   census.py heap [--top N] [INSTRUCTIONS=]PROFILE...
                                                    bytes live at each dhat profile's heap peak by
                                                    allocation site (a `--features dhat-heap` build
@@ -415,6 +421,146 @@ def heap(args):
         print(f"{growth} {cells}  {' <- '.join(frame[:70] for frame in site.split(' <- '))}")
 
 
+WRITE_ORDERS = ("natural", "o1", "o2", "o3")
+NEVER_ASSIGNED = re.compile(r"never assigned, read as residuals \(([^)]*)\)")
+RESIDUAL_CALL = re.compile(r"(r2sleigh_residual_\w+)\(\d+\)")
+TRAILING_COMMENT = re.compile(r"\s*/\*.*?\*/\s*$")
+LABEL = re.compile(r"^\w+:(\s*;)?$")
+CONTROL = ("goto ", "continue;", "break;", "if (", "} else", "else", "for (", "while (", "} while (", "do {",
+           "switch (", "case ", "default:", "/*")
+
+
+def never_assigned(code):
+    """The names the proof line says were read as residuals for want of a value."""
+    found = NEVER_ASSIGNED.search(code)
+    return sorted(name.strip() for name in found[1].split(",")) if found else []
+
+
+def leaf_statements(code):
+    """The set of statement lines of a rendering, apart from its control (a label, a jump, a brace) and
+    the numbering of its residual sites: what no block write order may change. A set, since a tail is
+    written once per jump to it."""
+    leaves = set()
+    for line in code.splitlines():
+        line = line.strip()
+        without_comment = TRAILING_COMMENT.sub("", line)
+        if not without_comment.endswith(";") or line.startswith(CONTROL) or LABEL.match(line):
+            continue
+        leaves.add(RESIDUAL_CALL.sub(r"\1(N)", line))
+    return sorted(leaves)
+
+
+def fingerprint(record):
+    """What a rendering says that no write order may change: the ledger's columns, the values read as
+    never assigned, the residual sites by (instruction address, cause, gap kind, callee, reason), the
+    names it declares and links, and its leaf statements. `None` for a function that was refused."""
+    if record.get("refused"):
+        return None
+    addrs = {line["line"]: sorted(line["addrs"]) for line in record["lines"]}
+    return {
+        "proof": record["proof"],
+        "never_assigned": never_assigned(record["code"]),
+        "sites": sorted(json.dumps([addrs.get(site["line"], []), site["cause"], site["gap"], site["callee"],
+                                    site["reason"], site["type"]]) for site in record["residuals"]),
+        "variables": sorted(json.dumps(variable, sort_keys=True) for variable in record["variables"]),
+        "links": sorted(json.dumps(link, sort_keys=True) for link in record["links"]),
+        "statements": leaf_statements(record["code"]),
+    }
+
+
+ASSIGNMENT = re.compile(r"^(\S+) = (.*);$")
+
+
+def selection_only(natural, mutant):
+    """Whether two leaf-statement sets differ only where the structurer's selection pass chose differently:
+    each variable they disagree on is assigned on both sides, and a conditional expression is one of the
+    assignments (`if (c) x = a; else x = b;` against `x = c ? a : b;`). That pass reads the layout, so it is
+    no verdict of D2's; any other difference is."""
+    only = [line for line in set(natural) ^ set(mutant)]
+    targets = collections.defaultdict(lambda: [0, 0, False])
+    for line in only:
+        found = ASSIGNMENT.match(line)
+        if not found:
+            return False
+        side = targets[found[1]]
+        side[0 if line in natural else 1] += 1
+        side[2] |= " ? " in found[2]
+    return all(n and m and ternary for n, m, ternary in targets.values())
+
+
+def verdict_moves(natural, mutant):
+    """The parts of two fingerprints that differ, and how the pair is refused (`None` where neither is).
+    A difference in the statements that only the selection pass explains is no move (`selection_only`)."""
+    if natural is None or mutant is None:
+        return ([], "refused only under the mutant" if natural is not None else
+                "refused only at natural" if mutant is not None else None)
+    moved = [part for part in natural if natural[part] != mutant[part]]
+    if "statements" in moved and selection_only(natural["statements"], mutant["statements"]):
+        moved.remove("statements")
+    return moved, None
+
+
+def order_answers(args, order, binary):
+    """`{address: (fingerprint or refusal, code hash)}` for every function of `binary` under `order`."""
+    sys.path.insert(0, str(ROOT / "tests" / "equiv"))
+    import r2s_batch
+
+    listing = r2s(binary, "aa; afl", args.r2s)
+    addresses = [int(line.split()[0], 16) for line in listing.splitlines()
+                 if line.startswith("0x") and not line.split()[3].startswith(("sym.imp", "sym._"))]
+    env = {**os.environ, "R2SLEIGH_WRITE_ORDER": order}
+    report = r2s_batch.run_batch(args.r2s, binary, addresses, function_timeout=120, startup_timeout=600, env=env)
+    answers = {}
+    for answer in report.answers:
+        if answer.ok:
+            record = answer.record
+            answers[hex(answer.address)] = {"fingerprint": fingerprint(record),
+                                            "text": hashlib.sha256(record["code"].encode()).hexdigest()}
+        else:
+            answers[hex(answer.address)] = {"fingerprint": None, "text": None, "cause": (answer.cause or "")[:120]}
+    return answers
+
+
+def orders(args):
+    """Every function under each write order, against natural: its verdicts must not move, and the order
+    must (a knob that moves no text proves nothing)."""
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    binaries = [Path(b) for b in args.binaries]
+    results = {}
+    for order in WRITE_ORDERS:
+        cached = out / f"{order}.json"
+        if args.reuse and cached.exists():
+            results[order] = json.loads(cached.read_text())
+            continue
+        with concurrent.futures.ThreadPoolExecutor(min(3, len(binaries))) as pool:
+            done = list(pool.map(lambda binary: order_answers(args, order, binary), binaries))
+        results[order] = {binary.name: answers for binary, answers in zip(binaries, done)}
+        cached.write_text(json.dumps(results[order]))
+    failed = False
+    total = sum(len(answers) for answers in results["natural"].values())
+    print(f"orders: {total} functions in {', '.join(b.name for b in binaries)}")
+    for order in WRITE_ORDERS[1:]:
+        differ, live, selected, parity, shown = 0, 0, 0, collections.Counter(), []
+        for name, natural in results["natural"].items():
+            for address, was in natural.items():
+                now = results[order][name].get(address, {"fingerprint": None, "text": None})
+                moved, refusal = verdict_moves(was["fingerprint"], now["fingerprint"])
+                live += was["text"] != now["text"]
+                if was["fingerprint"] and now["fingerprint"] and not moved:
+                    selected += was["fingerprint"]["statements"] != now["fingerprint"]["statements"]
+                if refusal:
+                    parity[refusal] += 1
+                if moved or refusal == "refused only under the mutant":
+                    differ += 1
+                    shown.append(f"  {name} {address}: {', '.join(moved) or refusal}")
+        print(f"{order}: {differ} differ, {live} change text, {selected} differ only in a selection, {dict(parity)}")
+        print("\n".join(shown[:args.show]))
+        failed |= differ > 0 or live < args.min_live
+    if failed:
+        sys.exit(1)
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     sub = parser.add_subparsers(dest="command", required=True)
@@ -452,6 +598,14 @@ def main():
     p.add_argument("--out")
     p.add_argument("binaries", nargs="+")
     p.set_defaults(func=exponent)
+    p = sub.add_parser("orders")
+    p.add_argument("--r2s", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--reuse", action="store_true", help="read an order's answers from OUT where it has them")
+    p.add_argument("--min-live", type=int, default=50, help="the fewest functions each order must change the text of")
+    p.add_argument("--show", type=int, default=20)
+    p.add_argument("binaries", nargs="+")
+    p.set_defaults(func=orders)
     p = sub.add_parser("heap")
     p.add_argument("--top", type=int, default=30)
     p.add_argument("profiles", nargs="+")

scripts/test_census.py

@@ -95,5 +95,92 @@ class HeapSites(unittest.TestCase):
         self.assertEqual(sites, {site: 320})
 
 
+# F1 of crates/r2engine/tests/engine/structure.rs under two write orders: the loop's merge `rax_2` has a gap for
+# its entry copy, and one order reads it as a variable in the exit after the loop.
+EXIT_INLINED = """
+    /* r2dec proof: 3 constructs are marked below; 36 source obligations: 20 rendered, 8 elided, 0 refused, 8 residual; 11 statements rendered; 1 never assigned, read as residuals (RAX_2) */
+    {
+        r2sleigh_residual_void(1); /* r2dec gap: ValuesNotRendered at 0x1009 (render::values) */
+        rdi_1 = arg0;
+        for (; ; ) {
+            if (c) {
+                return (uint64_t)r2sleigh_residual_u64(2);
+            } else {
+                rax_2 = (uint64_t)((uint64_t)r2sleigh_residual_u64(3) + (uint64_t)1U);
+                continue;
+            }
+        }
+        L1:
+        {
+            return (uint64_t)((uint64_t)rax_2 + (uint64_t)5U);
+        }
+    }
+"""
+EXIT_AFTER = EXIT_INLINED.replace("r2sleigh_residual_u64(2)", "rax_2").replace("20 rendered", "22 rendered")
+
+
+def record(code, **parts):
+    base = {"refused": None, "code": code, "proof": {"total": 36, "residual": 8}, "variables": [], "links": [],
+            "lines": [{"line": 5, "addrs": [4096]}],
+            "residuals": [{"site": 1, "type": "void", "cause": "gap", "gap": "ValuesNotRendered", "callee": None,
+                           "reason": None, "line": 5}]}
+    return {**base, **parts}
+
+
+class WriteOrders(unittest.TestCase):
+    def test_the_names_read_as_never_assigned_come_from_the_proof_line(self) -> None:
+        self.assertEqual(census.never_assigned(EXIT_INLINED), ["RAX_2"])
+        self.assertEqual(census.never_assigned(HIDDEN), [])
+
+    def test_statements_are_leaves_without_control_or_site_numbers(self) -> None:
+        leaves = census.leaf_statements(EXIT_INLINED)
+        self.assertIn("rdi_1 = arg0;", leaves)
+        self.assertIn("rax_2 = (uint64_t)((uint64_t)r2sleigh_residual_u64(N) + (uint64_t)1U);", leaves)
+        self.assertIn("r2sleigh_residual_void(N); /* r2dec gap: ValuesNotRendered at 0x1009 (render::values) */", leaves)
+        self.assertFalse([leaf for leaf in leaves if leaf.startswith(("for", "if", "continue", "L1"))])
+        self.assertEqual(census.leaf_statements("L17: ;\nL4:\n    x = 1;\n"), ["x = 1;"])
+
+    def test_a_layout_that_moves_only_control_moves_no_verdict(self) -> None:
+        moved = EXIT_INLINED.replace("continue;", "goto L0;").replace("L1:", "L1: L2:")
+        self.assertEqual(census.verdict_moves(census.fingerprint(record(EXIT_INLINED)),
+                                              census.fingerprint(record(moved))), ([], None))
+
+    def test_a_read_that_turns_from_a_residual_into_a_variable_is_a_moved_verdict(self) -> None:
+        moves, refusal = census.verdict_moves(census.fingerprint(record(EXIT_INLINED)),
+                                              census.fingerprint(record(EXIT_AFTER)))
+        self.assertEqual((moves, refusal), (["statements"], None))
+
+    def test_the_ledger_and_the_residual_sites_are_verdicts(self) -> None:
+        natural = census.fingerprint(record(EXIT_INLINED))
+        columns = census.fingerprint(record(EXIT_INLINED, proof={"total": 36, "residual": 6}))
+        sites = census.fingerprint(record(EXIT_INLINED, lines=[{"line": 5, "addrs": [4100]}]))
+        self.assertEqual(census.verdict_moves(natural, columns), (["proof"], None))
+        self.assertEqual(census.verdict_moves(natural, sites), (["sites"], None))
+
+    def test_a_selection_the_structurer_chose_differently_is_no_move(self) -> None:
+        arms = EXIT_INLINED.replace("rdi_1 = arg0;", "x_1 = 1;\n        x_1 = 2;")
+        selected = EXIT_INLINED.replace("rdi_1 = arg0;", "x_1 = c ? 1 : 2;")
+        natural, mutant = census.fingerprint(record(arms)), census.fingerprint(record(selected))
+        self.assertEqual(census.verdict_moves(natural, mutant), ([], None))
+        self.assertNotEqual(natural["statements"], mutant["statements"])
+
+    def test_a_statement_that_is_no_selection_is_a_move(self) -> None:
+        other = EXIT_INLINED.replace("rdi_1 = arg0;", "x_1 = 1;\n        y_1 = 2;")
+        lone = EXIT_INLINED.replace("rdi_1 = arg0;", "x_1 = c ? 1 : 2;")
+        moves = census.verdict_moves(census.fingerprint(record(other)), census.fingerprint(record(lone)))
+        self.assertEqual(moves, (["statements"], None))
+        # A target assigned on one side only is no selection either.
+        gone = census.fingerprint(record(EXIT_INLINED.replace("rdi_1 = arg0;", "")))
+        self.assertEqual(census.verdict_moves(census.fingerprint(record(EXIT_INLINED)), gone)[0], ["statements"])
+
+    def test_a_function_refused_under_one_side_only_is_reported_with_the_side(self) -> None:
+        refused = {"refused": {"kind": "rendering", "reason": "control certificate"}}
+        natural = census.fingerprint(record(EXIT_INLINED))
+        self.assertIsNone(census.fingerprint(refused))
+        self.assertEqual(census.verdict_moves(natural, None), ([], "refused only under the mutant"))
+        self.assertEqual(census.verdict_moves(None, natural), ([], "refused only at natural"))
+        self.assertEqual(census.verdict_moves(None, None), ([], None))
+
+
 if __name__ == "__main__":
     unittest.main()