diff --git a/doc/dae-todo.md b/doc/dae-todo.md index af166a8..e02c495 100644 --- a/doc/dae-todo.md +++ b/doc/dae-todo.md @@ -15,7 +15,7 @@ cluster layouts next, then metadata/value restoration and profile detection. - Use unsigned LEB128 references through Dart 2.17. - Use the `ReadRefId` +128 encoding from Dart 2.18 onward. - Commit: `Add Versioned Snapshot Reference Decoding` -- [ ] **R2F-DAE-003 — Handle versioned ObjectPool entry bits** +- [x] **R2F-DAE-003 — Handle versioned ObjectPool entry bits** - Model low-seven-bit, swapped low-seven-bit, and modern bitfield eras. - Commit: `Handle Versioned ObjectPool Entry Bits` - [ ] **R2F-DAE-004 — Parse versioned Code cluster layouts** diff --git a/doc/learn.md b/doc/learn.md index 6607ad2..b4f8e07 100644 --- a/doc/learn.md +++ b/doc/learn.md @@ -1394,3 +1394,16 @@ All cluster reference reads must use the snapshot profile's reference codec. For example, Dart 2.17 encodes reference 300 as `2c 82`; interpreting those bytes with the newer codec produces a different reference and can silently misalign subsequent object fields. + +## ObjectPool Entries Have Three Bit-Packing Eras + +The ObjectPool cluster itself does not need separate parser implementations. +r2flutter's normalized pool-entry model remains useful, but the byte-to-model +translation must be selected independently from other legacy cluster rules. + +Dart 2.10 through 3.1 put the entry type in bits 0-6 and patchability in bit 7; +type 0 is a tagged-object reference and type 1 an immediate. Dart 3.2 retains +that bit width but swaps those two type values. Dart 3.3 and later use the +modern layout: a four-bit type, bit 4 patchability, and bits 5-7 snapshot +behavior. A broad Dart-2-versus-Dart-3 check therefore misdecodes 3.0-3.2 and +may skip a serialized payload, shifting every following pool entry. diff --git a/include/r2flutter/dart_version.h b/include/r2flutter/dart_version.h index 17076d9..e048ecf 100644 --- a/include/r2flutter/dart_version.h +++ b/include/r2flutter/dart_version.h @@ -37,6 +37,14 @@ typedef enum { DART_REF_ENCODING_UNSIGNED } DartRefEncoding; +// ObjectPool entry type/patch/behavior bit packing changed independently of +// the surrounding cluster layout. +typedef enum { + DART_POOL_ENCODING_MODERN = 0, // type[0..3], patch[4], behavior[5..7] + DART_POOL_ENCODING_LOW7, // type[0..6], patch[7], tagged=0, immediate=1 + DART_POOL_ENCODING_LOW7_SWAPPED // type[0..6], patch[7], immediate=0, tagged=1 +} DartPoolEncoding; + // How the Dart version/layout for a snapshot was determined. Reported so users // can tell whether decoding rests on an exact match or a best-effort guess. typedef enum { @@ -59,6 +67,7 @@ typedef struct { int header_fields; DartHeaderStyle header_style; DartRefEncoding ref_encoding; + DartPoolEncoding pool_encoding; } DartVerLayout; // Lookup Dart version from a snapshot hash (MD5) diff --git a/scripts/update-dart-version b/scripts/update-dart-version index 7642e28..c005bd0 100755 --- a/scripts/update-dart-version +++ b/scripts/update-dart-version @@ -65,6 +65,12 @@ REF_ENCODING_NAMES = { "unsigned": "DART_REF_ENCODING_UNSIGNED", } +POOL_ENCODING_NAMES = { + "modern": "DART_POOL_ENCODING_MODERN", + "low7": "DART_POOL_ENCODING_LOW7", + "low7_swapped": "DART_POOL_ENCODING_LOW7_SWAPPED", +} + TAG_STYLE_VALUES = {v: k for k, v in TAG_STYLE_NAMES.items()} CID_KIND_TO_CLASS_NAME = [ @@ -605,6 +611,15 @@ def ref_encoding_for_version(version): return REF_ENCODING_NAMES["ref_id128"] +def pool_encoding_for_version(version): + ver_tuple = parse_version(version) + if ver_tuple < (3, 2): + return POOL_ENCODING_NAMES["low7"] + if ver_tuple < (3, 3): + return POOL_ENCODING_NAMES["low7_swapped"] + return POOL_ENCODING_NAMES["modern"] + + def profile_diff(existing, inferred): diffs = [] for key in ("cws", "heap_object_tag", "max_alignment", "it_cap", "tag_style", "header_fields"): @@ -720,10 +735,11 @@ def gen_profile_entries(offsets): tag_name = TAG_STYLE_NAMES.get(v["tag_style"], "0") header_style = header_style_for_version(ver) ref_encoding = ref_encoding_for_version(ver) + pool_encoding = pool_encoding_for_version(ver) lines.append(f'\t{{ "", "{ver}", {v["cws"]}, {v["heap_object_tag"]}, ' f'{v["max_alignment"]}, {v["it_cap"]}, {tag_name}, ' - f'{v["header_fields"]}, {header_style}, {ref_encoding} }}, // Dart {ver}') - lines.append("\t{ { 0 }, NULL, 0, 0, 0, 0, 0, 0, 0, 0 }") + f'{v["header_fields"]}, {header_style}, {ref_encoding}, {pool_encoding} }}, // Dart {ver}') + lines.append("\t{ { 0 }, NULL, 0, 0, 0, 0, 0, 0, 0, 0, 0 }") lines.append("};") return "\n".join(lines) diff --git a/src/lib/dart_pool_modern.c b/src/lib/dart_pool_modern.c index 15f5c89..c74d73a 100644 --- a/src/lib/dart_pool_modern.c +++ b/src/lib/dart_pool_modern.c @@ -377,8 +377,8 @@ typedef struct { int typed_data_internal_limit; int typed_data_stride; // Pre-3.0 snapshots (Dart 2.x) use an older serialization format for a - // handful of clusters and for the object pool entry bits. The exact byte - // layout of those readers is gated on this flag; everything else is shared. + // handful of clusters. Their readers are gated on this flag; everything + // else is shared. bool legacy_format; // every CID this layout defines, indexed by kind; -1 when the kind is absent int by_kind[DART_CID_KIND_COUNT]; @@ -1740,27 +1740,21 @@ static bool modern_read_pool_entry(ClusterStream *s, DartCtx *ctx, ut64 index, M if (!cs_read_u8 (s, &entry->bits)) { return false; } - if (ctx && ctx->layout && modern_layout_is_legacy (ctx->layout)) { - // Dart 2.x ObjectPoolBuilderEntry packs the entry type into TypeBits [0,7) - // and the patchability into bit 7. The serializer emits payloads by - // EntryType: kTaggedObject=0 (a ref), kImmediate=1 (a tagged word), and - // kNativeFunction=2 / kSwitchableCallMissEntryPoint=3 / - // kMegamorphicCallEntryPoint=4 (no payload). Downstream code keys off an - // internal convention -- 0 immediate, 1 tagged_object, 2 payload-less -- - // so remap here. There is no separate "behavior" field in this format. + DartPoolEncoding encoding = ctx && ctx->layout? ctx->layout->pool_encoding: DART_POOL_ENCODING_MODERN; + if (encoding == DART_POOL_ENCODING_LOW7 || encoding == DART_POOL_ENCODING_LOW7_SWAPPED) { + // Dart 2.10-3.2 packs the type into bits [0,7) and patchability into + // bit 7, without a behavior field. Dart 3.2 swaps the values assigned to + // immediate and tagged entries. Remap both eras to the internal modern + // convention: 0 immediate, 1 tagged_object, 2 payload-less native. const ut8 raw_type = entry->bits & 0x7f; entry->patch = (entry->bits >> 7) & 1; entry->behavior = 0; - switch (raw_type) { - case 0: - entry->type = 1; // kTaggedObject -> tagged_object - break; - case 1: - entry->type = 0; // kImmediate -> immediate - break; - default: + if (raw_type > 1) { entry->type = 2; // native / switchable / megamorphic: no payload - break; + } else if (encoding == DART_POOL_ENCODING_LOW7_SWAPPED) { + entry->type = raw_type; + } else { + entry->type = raw_type ^ 1; } } else { entry->type = entry->bits & 0x0f; diff --git a/src/lib/dart_version.c b/src/lib/dart_version.c index a990389..e079476 100644 --- a/src/lib/dart_version.c +++ b/src/lib/dart_version.c @@ -130,31 +130,31 @@ static const struct { // Range-start version profiles. CID tables are generated in dart_cid.c. static const DartVerLayout version_profiles[] = { - { "", "2.10.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 4, DART_HEADER_STYLE_210, DART_REF_ENCODING_UNSIGNED }, // Dart 2.10.0 - { "", "2.12.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 5, DART_HEADER_STYLE_212, DART_REF_ENCODING_UNSIGNED }, // Dart 2.12.0 - { "", "2.13.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 5, DART_HEADER_STYLE_212, DART_REF_ENCODING_UNSIGNED }, // Dart 2.13.0 - { "", "2.14.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_214, DART_REF_ENCODING_UNSIGNED }, // Dart 2.14.0 - { "", "2.15.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_214, DART_REF_ENCODING_UNSIGNED }, // Dart 2.15.0 - { "", "2.16.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 6, DART_HEADER_STYLE_216, DART_REF_ENCODING_UNSIGNED }, // Dart 2.16.0 - { "", "2.17.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 6, DART_HEADER_STYLE_216, DART_REF_ENCODING_UNSIGNED }, // Dart 2.17.0 - { "", "2.18.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 2.18.0 - { "", "2.18.2", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 2.18.2 - { "", "2.18.3", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 2.18.3 - { "", "2.19.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 2.19.0 - { "", "3.0.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.0.0 - { "", "3.1.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.1.0 - { "", "3.2.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.2.0 - { "", "3.3.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.3.0 - { "", "3.4.0", 8, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.4.0 - { "", "3.5.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.5.0 - { "", "3.6.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.6.0 - { "", "3.7.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.7.0 - { "", "3.8.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.8.0 - { "", "3.9.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.9.0 - { "", "3.10.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.10.0 - { "", "3.11.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.11.0 - { "", "3.12.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128 }, // Dart 3.12.0 - { { 0 }, NULL, 0, 0, 0, 0, 0, 0, 0, 0 } + { "", "2.10.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 4, DART_HEADER_STYLE_210, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.10.0 + { "", "2.12.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 5, DART_HEADER_STYLE_212, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.12.0 + { "", "2.13.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_INT32, 5, DART_HEADER_STYLE_212, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.13.0 + { "", "2.14.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_214, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.14.0 + { "", "2.15.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_214, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.15.0 + { "", "2.16.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 6, DART_HEADER_STYLE_216, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.16.0 + { "", "2.17.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 6, DART_HEADER_STYLE_216, DART_REF_ENCODING_UNSIGNED, DART_POOL_ENCODING_LOW7 }, // Dart 2.17.0 + { "", "2.18.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 2.18.0 + { "", "2.18.2", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 2.18.2 + { "", "2.18.3", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 2.18.3 + { "", "2.19.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 2.19.0 + { "", "3.0.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 3.0.0 + { "", "3.1.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7 }, // Dart 3.1.0 + { "", "3.2.0", 8, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_LOW7_SWAPPED }, // Dart 3.2.0 + { "", "3.3.0", 4, 1, 16, 20000, DART_TAG_STYLE_CID_SHIFT1, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.3.0 + { "", "3.4.0", 8, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.4.0 + { "", "3.5.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.5.0 + { "", "3.6.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.6.0 + { "", "3.7.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.7.0 + { "", "3.8.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.8.0 + { "", "3.9.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.9.0 + { "", "3.10.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.10.0 + { "", "3.11.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.11.0 + { "", "3.12.0", 4, 1, 16, 20000, DART_TAG_STYLE_OBJECT_HEADER, 5, DART_HEADER_STYLE_MODERN, DART_REF_ENCODING_REF_ID128, DART_POOL_ENCODING_MODERN }, // Dart 3.12.0 + { { 0 }, NULL, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; const char *dart_version_from_hash(const char *hash) { diff --git a/test/db/cmd/object-pool-entry-bits b/test/db/cmd/object-pool-entry-bits new file mode 100644 index 0000000..7846753 --- /dev/null +++ b/test/db/cmd/object-pool-entry-bits @@ -0,0 +1,14 @@ +NAME=versioned-object-pool-entry-bits +FILE=- +CMDS=</dev/null | python3 -c 'import sys; line=next(x for x in sys.stdin if "bits=" in x); print("3.0", *(x for x in line.split() if x.startswith(("bits=", "type=", "patch=", "behavior=", "ref="))))' +!!python3 -c 'import sys; open(sys.argv[1], "wb").write(bytes.fromhex(sys.argv[2]))' /tmp/r2flutter-pool-320.bin 50414421f5f5dcdc61000000000000000300000000000000396436333632323636623738653431393866373834646131613238666438326270726f647563742061726d3634206e6f2d636f6d707265737365642d706f696e74657273008283818589ec8181818102ac +!!../bin/r2flutter -q -HHH -l 1 /tmp/r2flutter-pool-320.bin 2>/dev/null | python3 -c 'import sys; line=next(x for x in sys.stdin if "bits=" in x); print("3.2", *(x for x in line.split() if x.startswith(("bits=", "type=", "patch=", "behavior=", "ref="))))' +!!rm -f /tmp/r2flutter-pool-300.bin /tmp/r2flutter-pool-320.bin +EOF +EXPECT=<