mirror of
https://github.com/libretro/RetroArch.git
synced 2026-11-08 08:10:30 -10:00
rchd: a clean-room CHD reader, and the huffman codec it needs
libchdr is the only way to read a CHD here, and it reads for itself: open it on a path and it holds a file handle for the life of the image. That suits a blocking caller and nothing else. It is also a fork of MAME's own code, carried with about fifty lines of -D in Makefile.common renaming every symbol so a statically linked core carrying its own copy does not collide. This is the container half of a reader written from a format description instead, with no I/O of its own: it names the byte ranges it needs and the caller supplies them, so one image can be read a few kilobytes per frame from a task, with several fetches in flight, or by a plain blocking read, without the reader knowing which. FORMAT.md beside it is that description. Every field in it was derived by observing files rather than by reading another implementation: reference images are generated from input whose hash is known in advance, each claim is stated as a prediction, and the observed bytes confirm or refute it. tools/chd_probe.py re-runs the lot. Two claims were wrong on the first pass and the probe caught them, which is the point of writing it that way. What that bought, beyond provenance: an uncompressed map's value of zero means a hole of zeros, not an offset of zero, and reading it literally yields the first hunk-worth of the file - well-formed data of the right length that happens to be the header. Versions 1 and 2 record neither the image size nor the hunk size; both come from the drive geometry. A version 1 to 4 map is followed by a cookie nothing points at, which one reader requires and the other never looks for. Nothing before version 5 records a unit size, so it is read out of metadata and falls back to the hunk size. And the canonical Huffman in a CHD gives the lowest codes to the longest lengths, where RFC 1951 gives them to the shortest - the two agree on every tree whose codes are one length, which is every single-codec image, so reaching for the familiar ordering decodes those perfectly and fails only on multi-codec ones. encodings/huffman.h comes with it because the map needs it: the same canonical decoder serves the map's tree, the 'huff' hunk codec and, in due course, A/V hunks. Both tree serialisations are here. It allocates nothing - the caller owns the lookup table, so the map's sixteen-code tree is half a kilobyte of stack while the 'huff' codec's lives in the handle - and it accepts an incomplete tree, because a hunk of one repeated byte yields a single one-bit code covering half the space, and rejecting that fails exactly the input that compresses best. Verified against six commercial images spanning versions 3 and 5 and the cdlz, cdzl and cdfl codecs, plus generated images covering every version, every map variant and the codec matrix. Versions 1, 2 and 4 have no archived sample and no current writer, so images were built to the description and handed to two independent readers, which decode them to the same content as a version 5 image of the same source. A compressed map validates against its own CRC-16, and 1200 corrupted images decode without an ASan or UBSan fault; the bound on the hunk count exists because fuzzing found a corrupt header asking for 21 GiB. c89 -pedantic is clean. Nothing casts a buffer to a wider type, so neither the host's endianness nor the alignment of what the caller hands over changes the result. Reading hunks, the CD layer and the codecs follow.
This commit is contained in:
libretroadmin committed
1 parent
42267a517e
commit
22d975506a
18 files changed
+4384
No files matched your search
@@ -0,0 +1,474 @@
|
||||
/* Copyright (C) 2010-2026 The RetroArch team
|
||||
*
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* The following license statement only applies to this file (encoding_huffman.c).
|
||||
* ---------------------------------------------------------------------------------------
|
||||
*
|
||||
* Permission is hereby granted, free of charge,
|
||||
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include <encodings/huffman.h>
|
||||
|
||||
/* Bits held in the cache before a refill stops. Kept at 25 so a peek of
|
||||
* up to 24 bits never needs a second pass, and so the shift used to
|
||||
* place an incoming byte stays inside 0..24. */
|
||||
#define RHUFF_CACHE_MIN 24
|
||||
|
||||
/* Length field width in a packed lookup entry. Five bits holds a code
|
||||
* length up to 31, comfortably above RHUFF_MAX_BITS. */
|
||||
#define RHUFF_LOOKUP_LEN_BITS 5
|
||||
#define RHUFF_LOOKUP_LEN_MASK 0x1f
|
||||
|
||||
/* -------- bit reader -------- */
|
||||
|
||||
void rhuff_bits_init(rhuff_bits_t *b, const uint8_t *data, size_t size)
|
||||
{
|
||||
if (!b)
|
||||
return;
|
||||
|
||||
b->data = data;
|
||||
b->size = data ? size : 0;
|
||||
b->offset = 0;
|
||||
b->used = 0;
|
||||
b->cache = 0;
|
||||
b->bits = 0;
|
||||
b->overflow = 0;
|
||||
}
|
||||
|
||||
/* Pulls bytes in until the cache holds more than a peek can ask for.
|
||||
* Bytes past the end contribute zeros; the position still advances, so
|
||||
* rhuff_bits_flush() can wind back by whole bytes symmetrically.
|
||||
*
|
||||
* The byte is placed with a shift rather than read through a wider
|
||||
* pointer, which is what keeps this independent of host endianness and
|
||||
* of the alignment of @data. */
|
||||
static void rhuff_bits_fill(rhuff_bits_t *b)
|
||||
{
|
||||
while (b->bits <= RHUFF_CACHE_MIN)
|
||||
{
|
||||
if (b->offset < b->size)
|
||||
b->cache |= (uint32_t)b->data[b->offset] << (RHUFF_CACHE_MIN - b->bits);
|
||||
b->offset++;
|
||||
b->bits += 8;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t rhuff_bits_peek(rhuff_bits_t *b, int numbits)
|
||||
{
|
||||
if (!b || numbits <= 0)
|
||||
return 0;
|
||||
|
||||
if (numbits > RHUFF_CACHE_MIN)
|
||||
numbits = RHUFF_CACHE_MIN;
|
||||
|
||||
if (b->bits < numbits)
|
||||
rhuff_bits_fill(b);
|
||||
|
||||
return b->cache >> (32 - numbits);
|
||||
}
|
||||
|
||||
void rhuff_bits_remove(rhuff_bits_t *b, int numbits)
|
||||
{
|
||||
if (!b || numbits <= 0)
|
||||
return;
|
||||
|
||||
if (numbits > RHUFF_CACHE_MIN)
|
||||
numbits = RHUFF_CACHE_MIN;
|
||||
|
||||
/* Refill first so the cache always holds at least what is being
|
||||
* taken; that keeps b->bits from going negative and the shift below
|
||||
* from being asked for more than 24. */
|
||||
if (b->bits < numbits)
|
||||
rhuff_bits_fill(b);
|
||||
|
||||
b->cache <<= numbits;
|
||||
b->bits -= numbits;
|
||||
b->used += (uint64_t)numbits;
|
||||
|
||||
if (b->used > (uint64_t)b->size * 8)
|
||||
b->overflow = 1;
|
||||
}
|
||||
|
||||
uint32_t rhuff_bits_read(rhuff_bits_t *b, int numbits)
|
||||
{
|
||||
uint32_t value = rhuff_bits_peek(b, numbits);
|
||||
rhuff_bits_remove(b, numbits);
|
||||
return value;
|
||||
}
|
||||
|
||||
size_t rhuff_bits_flush(rhuff_bits_t *b)
|
||||
{
|
||||
if (!b)
|
||||
return 0;
|
||||
|
||||
while (b->bits >= 8 && b->offset > 0)
|
||||
{
|
||||
b->offset--;
|
||||
b->bits -= 8;
|
||||
}
|
||||
|
||||
b->bits = 0;
|
||||
b->cache = 0;
|
||||
|
||||
if (b->offset > b->size)
|
||||
b->offset = b->size;
|
||||
|
||||
return b->offset;
|
||||
}
|
||||
|
||||
int rhuff_bits_overflow(const rhuff_bits_t *b)
|
||||
{
|
||||
if (!b)
|
||||
return 1;
|
||||
return b->overflow;
|
||||
}
|
||||
|
||||
/* -------- canonical decoder -------- */
|
||||
|
||||
int rhuff_dec_init(rhuff_dec_t *d, uint32_t num_codes, uint32_t max_bits,
|
||||
uint16_t *lookup, size_t lookup_entries)
|
||||
{
|
||||
if (!d || !lookup)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
if (num_codes == 0 || num_codes > RHUFF_MAX_CODES)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
if (max_bits == 0 || max_bits > RHUFF_MAX_BITS)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
if (lookup_entries < RHUFF_LOOKUP_ENTRIES(max_bits))
|
||||
return RHUFF_ERROR_PARAM;
|
||||
/* A lookup entry has to hold the index above the length field. */
|
||||
if (num_codes > (uint32_t)(0xffff >> RHUFF_LOOKUP_LEN_BITS) + 1)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
|
||||
d->lookup = lookup;
|
||||
d->lookup_entries = lookup_entries;
|
||||
d->num_codes = num_codes;
|
||||
d->max_bits = max_bits;
|
||||
|
||||
memset(d->lengths, 0, sizeof(d->lengths));
|
||||
|
||||
return RHUFF_OK;
|
||||
}
|
||||
|
||||
int rhuff_dec_build(rhuff_dec_t *d)
|
||||
{
|
||||
uint32_t histo[RHUFF_MAX_BITS + 1];
|
||||
uint16_t codes[RHUFF_MAX_CODES];
|
||||
uint32_t curstart;
|
||||
uint32_t i;
|
||||
int len;
|
||||
|
||||
if (!d || !d->lookup)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
|
||||
memset(histo, 0, sizeof(histo));
|
||||
|
||||
for (i = 0; i < d->num_codes; i++)
|
||||
{
|
||||
if ((uint32_t)d->lengths[i] > d->max_bits)
|
||||
return RHUFF_ERROR_DATA;
|
||||
histo[d->lengths[i]]++;
|
||||
}
|
||||
|
||||
/* Assign the first code of each length, working from the longest
|
||||
* length down and halving as the length shrinks. An odd running
|
||||
* total at any length above one cannot be halved into the next
|
||||
* shorter length, which means the lengths do not describe a
|
||||
* complete tree.
|
||||
*
|
||||
* Note this is not RFC 1951's ordering: walking down and halving
|
||||
* gives the lowest codes to the longest lengths, where walking up
|
||||
* and doubling gives them to the shortest. rhuff_test.c confirms
|
||||
* both produce valid prefix codes and that they disagree on
|
||||
* essentially every non-trivial tree. */
|
||||
curstart = 0;
|
||||
for (len = (int)d->max_bits; len > 0; len--)
|
||||
{
|
||||
uint32_t total = curstart + histo[len];
|
||||
|
||||
if (len != 1 && (total & 1) != 0)
|
||||
return RHUFF_ERROR_DATA;
|
||||
|
||||
histo[len] = curstart;
|
||||
curstart = total >> 1;
|
||||
}
|
||||
|
||||
for (i = 0; i < d->num_codes; i++)
|
||||
{
|
||||
codes[i] = 0;
|
||||
if (d->lengths[i] > 0)
|
||||
codes[i] = (uint16_t)(histo[d->lengths[i]]++);
|
||||
}
|
||||
|
||||
for (i = 0; i < d->lookup_entries; i++)
|
||||
d->lookup[i] = 0;
|
||||
|
||||
for (i = 0; i < d->num_codes; i++)
|
||||
{
|
||||
uint32_t shift;
|
||||
uint32_t dest;
|
||||
uint32_t count;
|
||||
uint32_t j;
|
||||
uint16_t value;
|
||||
|
||||
if (d->lengths[i] == 0)
|
||||
continue;
|
||||
|
||||
shift = d->max_bits - d->lengths[i];
|
||||
dest = (uint32_t)codes[i] << shift;
|
||||
count = (uint32_t)1 << shift;
|
||||
value = (uint16_t)(((uint32_t)i << RHUFF_LOOKUP_LEN_BITS)
|
||||
| (uint32_t)d->lengths[i]);
|
||||
|
||||
if ((size_t)dest + (size_t)count > d->lookup_entries)
|
||||
return RHUFF_ERROR_DATA;
|
||||
|
||||
for (j = 0; j < count; j++)
|
||||
d->lookup[dest + j] = value;
|
||||
}
|
||||
|
||||
return RHUFF_OK;
|
||||
}
|
||||
|
||||
uint32_t rhuff_dec_decode_one(rhuff_dec_t *d, rhuff_bits_t *b)
|
||||
{
|
||||
uint32_t peeked;
|
||||
uint16_t value;
|
||||
int length;
|
||||
|
||||
if (!d || !b || !d->lookup)
|
||||
return 0;
|
||||
|
||||
peeked = rhuff_bits_peek(b, (int)d->max_bits);
|
||||
value = d->lookup[peeked];
|
||||
length = (int)(value & RHUFF_LOOKUP_LEN_MASK);
|
||||
|
||||
/* A zero entry is a bit pattern the tree does not cover, which only
|
||||
* happens on malformed input. Consume one bit anyway so a caller
|
||||
* looping until it has enough symbols still terminates, and latch
|
||||
* the overflow flag so it can tell the result apart from a real
|
||||
* decode. */
|
||||
if (length == 0)
|
||||
{
|
||||
b->overflow = 1;
|
||||
rhuff_bits_remove(b, 1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
rhuff_bits_remove(b, length);
|
||||
|
||||
return (uint32_t)(value >> RHUFF_LOOKUP_LEN_BITS);
|
||||
}
|
||||
|
||||
/* -------- tree serialisations -------- */
|
||||
|
||||
/* The encodings below are specific to the CHD container. See
|
||||
* formats/chd/FORMAT.md; every constant here is derived from observed
|
||||
* bytes rather than recalled, and tools/chd/chd_probe.py regenerates
|
||||
* the images the derivation rests on. */
|
||||
|
||||
/* Width of one stored length value. Only the eight-bit case occurs in
|
||||
* practice -- the sole user of this serialisation is the CHD hunk map,
|
||||
* whose tree is always sixteen codes of at most eight bits -- and it is
|
||||
* the only one confirmed against real data. The other two branches
|
||||
* follow the same scaling and are unverified. */
|
||||
static uint32_t rhuff_rle_value_bits(uint32_t max_bits)
|
||||
{
|
||||
if (max_bits >= 16)
|
||||
return 5;
|
||||
if (max_bits >= 8)
|
||||
return 4;
|
||||
return 3;
|
||||
}
|
||||
|
||||
int rhuff_read_tree_rle(rhuff_dec_t *d, rhuff_bits_t *b)
|
||||
{
|
||||
uint32_t numbits;
|
||||
uint32_t curnode;
|
||||
|
||||
if (!d || !b)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
|
||||
numbits = rhuff_rle_value_bits(d->max_bits);
|
||||
|
||||
memset(d->lengths, 0, sizeof(d->lengths));
|
||||
|
||||
/* Lengths are stored one per code in order. A value of one is an
|
||||
* escape rather than a length: the value after it is the real
|
||||
* length, and if that is also one the pair encodes a single length
|
||||
* of one. Otherwise a third value carries a repeat count, biased by
|
||||
* three, and that many codes take the length. */
|
||||
curnode = 0;
|
||||
while (curnode < d->num_codes)
|
||||
{
|
||||
uint32_t value = rhuff_bits_read(b, (int)numbits);
|
||||
|
||||
if (value != 1)
|
||||
d->lengths[curnode++] = (uint8_t)value;
|
||||
else
|
||||
{
|
||||
value = rhuff_bits_read(b, (int)numbits);
|
||||
|
||||
if (value == 1)
|
||||
d->lengths[curnode++] = 1;
|
||||
else
|
||||
{
|
||||
uint32_t repeat = rhuff_bits_read(b, (int)numbits) + 3;
|
||||
|
||||
while (repeat > 0 && curnode < d->num_codes)
|
||||
{
|
||||
d->lengths[curnode++] = (uint8_t)value;
|
||||
repeat--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rhuff_bits_overflow(b))
|
||||
return RHUFF_ERROR_DATA;
|
||||
}
|
||||
|
||||
/* No alignment follows the tree: the first code of the body starts
|
||||
* on the very next bit. Confirmed across six images whose trees
|
||||
* differ in shape. */
|
||||
|
||||
return rhuff_dec_build(d);
|
||||
}
|
||||
|
||||
/* Alphabet of the tree that codes the main tree's lengths: one escape
|
||||
* plus one symbol per representable length. Its own lengths are stored
|
||||
* as fixed three-bit values, so a stored 7 cannot be a length and is
|
||||
* used to end the list. */
|
||||
#define RHUFF_SMALL_CODES 24
|
||||
#define RHUFF_SMALL_BITS 6
|
||||
#define RHUFF_SMALL_WIDTH 3
|
||||
#define RHUFF_SMALL_END 7
|
||||
|
||||
int rhuff_read_tree_packed(rhuff_dec_t *d, rhuff_bits_t *b)
|
||||
{
|
||||
/* The small tree is small enough to keep on the stack, which is why
|
||||
* this needs no scratch from the caller. */
|
||||
uint16_t small_lookup[1 << RHUFF_SMALL_BITS];
|
||||
rhuff_dec_t small;
|
||||
uint32_t start;
|
||||
uint32_t index;
|
||||
uint32_t numcodes;
|
||||
uint32_t runbits;
|
||||
uint32_t curcode;
|
||||
uint32_t last;
|
||||
int err;
|
||||
|
||||
if (!d || !b)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
|
||||
if ((err = rhuff_dec_init(&small, RHUFF_SMALL_CODES, RHUFF_SMALL_BITS,
|
||||
small_lookup, RHUFF_LOOKUP_ENTRIES(RHUFF_SMALL_BITS)))
|
||||
!= RHUFF_OK)
|
||||
return err;
|
||||
|
||||
/* The first length stands alone; the value after it says which index
|
||||
* the rest resume at, so leading unused symbols cost nothing. */
|
||||
small.lengths[0] = (uint8_t)rhuff_bits_read(b, RHUFF_SMALL_WIDTH);
|
||||
start = rhuff_bits_read(b, RHUFF_SMALL_WIDTH) + 1;
|
||||
|
||||
for (index = start; index < RHUFF_SMALL_CODES; index++)
|
||||
{
|
||||
uint32_t value = rhuff_bits_read(b, RHUFF_SMALL_WIDTH);
|
||||
|
||||
if (value == RHUFF_SMALL_END)
|
||||
break;
|
||||
|
||||
small.lengths[index] = (uint8_t)value;
|
||||
}
|
||||
|
||||
if (rhuff_bits_overflow(b))
|
||||
return RHUFF_ERROR_DATA;
|
||||
|
||||
if ((err = rhuff_dec_build(&small)) != RHUFF_OK)
|
||||
return err;
|
||||
|
||||
/* Width of the extended run count, scaled to the alphabet being
|
||||
* described rather than fixed, so this holds for the 256-symbol
|
||||
* alphabet of 'huff' and for the others an A/V hunk uses. */
|
||||
numcodes = d->num_codes;
|
||||
runbits = 0;
|
||||
{
|
||||
uint32_t span = (numcodes > 9) ? (numcodes - 9) : 1;
|
||||
while (span)
|
||||
{
|
||||
span >>= 1;
|
||||
runbits++;
|
||||
}
|
||||
}
|
||||
|
||||
memset(d->lengths, 0, sizeof(d->lengths));
|
||||
|
||||
curcode = 0;
|
||||
last = 0;
|
||||
|
||||
while (curcode < d->num_codes)
|
||||
{
|
||||
uint32_t value = rhuff_dec_decode_one(&small, b);
|
||||
|
||||
if (rhuff_bits_overflow(b))
|
||||
return RHUFF_ERROR_DATA;
|
||||
|
||||
if (value != 0)
|
||||
{
|
||||
/* Lengths are stored one above their value, leaving zero free
|
||||
* to mean a run. */
|
||||
last = value - 1;
|
||||
d->lengths[curcode++] = (uint8_t)last;
|
||||
}
|
||||
else
|
||||
{
|
||||
uint32_t count = rhuff_bits_read(b, 3) + 2;
|
||||
|
||||
if (count == 9)
|
||||
count += rhuff_bits_read(b, (int)runbits);
|
||||
|
||||
while (count > 0 && curcode < d->num_codes)
|
||||
{
|
||||
d->lengths[curcode++] = (uint8_t)last;
|
||||
count--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rhuff_dec_build(d);
|
||||
}
|
||||
|
||||
int rhuff_decode_block(rhuff_dec_t *d, rhuff_bits_t *b,
|
||||
uint8_t *dst, size_t dst_len)
|
||||
{
|
||||
size_t i;
|
||||
int err;
|
||||
|
||||
if (!d || !b || !dst)
|
||||
return RHUFF_ERROR_PARAM;
|
||||
|
||||
if ((err = rhuff_read_tree_packed(d, b)) != RHUFF_OK)
|
||||
return err;
|
||||
|
||||
for (i = 0; i < dst_len; i++)
|
||||
dst[i] = (uint8_t)rhuff_dec_decode_one(d, b);
|
||||
|
||||
if (rhuff_bits_overflow(b))
|
||||
return RHUFF_ERROR_DATA;
|
||||
|
||||
return RHUFF_OK;
|
||||
}
|
||||
@@ -0,0 +1,731 @@
|
||||
# CHD container format
|
||||
|
||||
Working description of MAME's Compressed Hunks of Data container, written
|
||||
to support `rchd.c`.
|
||||
|
||||
## Provenance
|
||||
|
||||
Every field below is derived from **observation of generated files**, not
|
||||
from reading an existing implementation. Reference images are produced
|
||||
with `chdman` from known input, and each claim is stated as a prediction
|
||||
that the observed bytes then confirm or refute. `tools/chd/chd_probe.py`
|
||||
reproduces the whole set.
|
||||
|
||||
This matters for two reasons. It keeps the implementation clean-room:
|
||||
`rchd.c` is written against this document, and every constant in it can
|
||||
be traced to a measurement rather than to someone's recollection. And it
|
||||
is falsifiable — two entries below were wrong on the first pass and the
|
||||
probe caught them.
|
||||
|
||||
Reference writer: `chdman` 0.264. Codecs postdating that build (`zstd`,
|
||||
`cdzs`) are unverified here and marked as such.
|
||||
|
||||
Generated images cannot cover everything: they are all version 5, all
|
||||
written by one tool version, and small. Claims are therefore also checked
|
||||
against archived images produced by other tool versions over the years —
|
||||
four CD images and two hard-disk images at the time of writing. Those are
|
||||
what versions 1 to 4 and the CD codecs can be observed at all, and they
|
||||
are the standing check that the description describes the format rather
|
||||
than one writer's habits.
|
||||
|
||||
Status legend: **[V]** verified against observed bytes, **[P]** predicted
|
||||
but not yet confirmed, **[?]** unknown.
|
||||
|
||||
---
|
||||
|
||||
## 1. Header
|
||||
|
||||
All integers are big-endian. All offsets are absolute byte offsets from
|
||||
the start of the file unless stated otherwise.
|
||||
|
||||
### 1.1 Version 5 header — 124 bytes **[V]**
|
||||
|
||||
| Offset | Size | Field | Notes |
|
||||
|---|---|---|---|
|
||||
| 0 | 8 | tag | ASCII `MComprHD` |
|
||||
| 8 | 4 | length | 124 |
|
||||
| 12 | 4 | version | 5 |
|
||||
| 16 | 16 | compressors[4] | four 4CC tags, `0` for unused slots |
|
||||
| 32 | 8 | logicalbytes | decompressed image size |
|
||||
| 40 | 8 | mapoffset | start of the hunk map |
|
||||
| 48 | 8 | metaoffset | first metadata entry, `0` if none |
|
||||
| 56 | 4 | hunkbytes | |
|
||||
| 60 | 4 | unitbytes | |
|
||||
| 64 | 20 | rawsha1 | SHA-1 of the decompressed image |
|
||||
| 84 | 20 | sha1 | SHA-1 of image plus metadata |
|
||||
| 104 | 20 | parentsha1 | zero when there is no parent |
|
||||
|
||||
Confirmed by generating images from input whose SHA-1 was known in
|
||||
advance and reading `rawsha1` back at offset 64 across five codec
|
||||
configurations. The `compressors` array read back as exactly the tags
|
||||
requested, in request order, for a four-codec image.
|
||||
|
||||
**An uncompressed image carries no hashes.** With `-c none`, `rawsha1`,
|
||||
`sha1` and `parentsha1` are all zero. **[V]** A reader must not treat a
|
||||
zero hash as a verification failure, and must not offer verification for
|
||||
such an image at all — there is nothing to verify against.
|
||||
|
||||
### 1.2 Version 3 header — 120 bytes **[V]**
|
||||
|
||||
| Offset | Size | Field | Notes |
|
||||
|---|---|---|---|
|
||||
| 0 | 8 | tag | ASCII `MComprHD` |
|
||||
| 8 | 4 | length | 120 |
|
||||
| 12 | 4 | version | 3 |
|
||||
| 16 | 4 | flags | 0 observed |
|
||||
| 20 | 4 | compression | an enum, not a 4CC; 2 observed, meaning zlib |
|
||||
| 24 | 4 | totalhunks | |
|
||||
| 28 | 8 | logicalbytes | |
|
||||
| 36 | 8 | metaoffset | |
|
||||
| 44 | 16 | md5 | |
|
||||
| 60 | 16 | parentmd5 | zero when there is no parent |
|
||||
| 76 | 4 | hunkbytes | |
|
||||
| 80 | 20 | sha1 | |
|
||||
| 100 | 20 | parentsha1 | |
|
||||
|
||||
Confirmed against two archived hard-disk images by cross-reading with an
|
||||
independent tool: the SHA-1 at offset 80 matches the hash that tool
|
||||
reports, and `totalhunks`, `logicalbytes` and `hunkbytes` all agree.
|
||||
|
||||
**There is no `unitbytes` field before version 5.** A reader that needs a
|
||||
unit size must infer it. For the observed hard-disk images the value an
|
||||
independent tool reports (512) is the `BPS` field of the `GDDD` metadata,
|
||||
so the inference runs through metadata rather than through any header
|
||||
field. Images carrying no metadata that names a unit size need a fallback
|
||||
policy, which is **[?]**.
|
||||
|
||||
The `compression` field is an enum here rather than the four 4CC slots of
|
||||
version 5, so the whole image uses one codec. Only value 2 has been
|
||||
observed.
|
||||
|
||||
### 1.3 Version 4 header — 108 bytes **[V]**
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 8 | tag |
|
||||
| 8 | 4 | length — 108 |
|
||||
| 12 | 4 | version — 4 |
|
||||
| 16 | 4 | flags |
|
||||
| 20 | 4 | compression enum |
|
||||
| 24 | 4 | totalhunks |
|
||||
| 28 | 8 | logicalbytes |
|
||||
| 36 | 8 | metaoffset |
|
||||
| 44 | 4 | hunkbytes |
|
||||
| 48 | 20 | sha1 |
|
||||
| 68 | 20 | parentsha1 |
|
||||
| 88 | 20 | rawsha1 |
|
||||
|
||||
Version 4 drops version 3's two MD5 fields and adds `rawsha1`.
|
||||
|
||||
### 1.4 Versions 1 and 2 — 76 and 80 bytes **[V]**
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 8 | tag |
|
||||
| 8 | 4 | length — 76 or 80 |
|
||||
| 12 | 4 | version |
|
||||
| 16 | 4 | flags |
|
||||
| 20 | 4 | compression enum |
|
||||
| 24 | 4 | hunk size, **counted in sectors** |
|
||||
| 28 | 4 | totalhunks |
|
||||
| 32 | 4 | cylinders |
|
||||
| 36 | 4 | heads |
|
||||
| 40 | 4 | sectors |
|
||||
| 44 | 16 | md5 |
|
||||
| 60 | 16 | parentmd5 |
|
||||
| 76 | 4 | sector length — **version 2 only** |
|
||||
|
||||
Neither version stores `logicalbytes` or `hunkbytes` directly. Both are
|
||||
derived: sector length is 512 in version 1 and read from offset 76 in
|
||||
version 2, `hunkbytes` is the field at 24 multiplied by it, and
|
||||
`logicalbytes` is `cylinders * heads * sectors * seclen`.
|
||||
|
||||
The geometry fields are what distinguish these versions structurally, and
|
||||
a reader must treat them as load-bearing rather than informational: they
|
||||
are the only source of the image's size.
|
||||
|
||||
### 1.5 How versions 1, 2 and 4 were verified
|
||||
|
||||
No archived sample of these versions was available, and no current tool
|
||||
writes them. They were verified by building images from the layouts above
|
||||
and having independent readers decode them:
|
||||
|
||||
- Version 4 was accepted by both readers, extracted byte-identically, and
|
||||
reported "raw SHA-1 verification successful".
|
||||
- Versions 1 and 2 were accepted by libchdr and decoded to a digest
|
||||
identical to a version 5 image of the same source data.
|
||||
|
||||
Writing a file a reader accepts is weaker evidence than reading a file
|
||||
someone else wrote — a shared misunderstanding would go unnoticed — but
|
||||
two independent readers agreeing on content that round-trips to the
|
||||
original is strong enough to act on. These stay marked **[V]** with that
|
||||
caveat, and archived samples would still be worth checking against.
|
||||
|
||||
**No current MAME tool reads versions 1 or 2 at all.** They are rejected
|
||||
exactly as an unknown future version would be. libchdr does read them,
|
||||
so supporting them is a compatibility requirement inherited from libchdr
|
||||
rather than from upstream.
|
||||
|
||||
---
|
||||
|
||||
## 2. Hunk map
|
||||
|
||||
`hunkcount = ceil(logicalbytes / hunkbytes)`.
|
||||
|
||||
### 2.1 Uncompressed map **[V]**
|
||||
|
||||
Present when all four compressor slots are zero. `mapoffset` points just
|
||||
past the header, at 124. The map is a flat array of `hunkcount`
|
||||
big-endian `uint32` values, one per hunk, and nothing else — no header,
|
||||
no checksum.
|
||||
|
||||
Resolution rule, confirmed against all 64 hunks of a reference image:
|
||||
|
||||
- **value `0` means a hole**: the hunk reads as `hunkbytes` of zero. It
|
||||
does **not** mean offset zero.
|
||||
- **any other value** is a byte offset of `value * hunkbytes`.
|
||||
|
||||
The hole case is easy to miss and fails quietly. Reading value `0` as an
|
||||
offset yields the first `hunkbytes` of the file — the header itself —
|
||||
which is well-formed data of the right length that happens to be wrong.
|
||||
The reference image exercises this: its hunk 0 is genuinely all zeros,
|
||||
its map entry is `0`, and its hunk data region begins at offset 0, so the
|
||||
naive reading and the correct one differ only in content.
|
||||
|
||||
Because offsets are counted in whole hunks from zero, the header and map
|
||||
occupy space inside the region hunk 0 would otherwise address. A writer
|
||||
therefore cannot place a real hunk at index 0; observed images only ever
|
||||
emit `0` there as a hole.
|
||||
|
||||
### 2.2 Compressed map header — 16 bytes **[V]**
|
||||
|
||||
Present when any compressor slot is set. `mapoffset` points at this
|
||||
header, and the compressed bitstream follows immediately at
|
||||
`mapoffset + 16`.
|
||||
|
||||
| Offset | Size | Field | Observed |
|
||||
|---|---|---|---|
|
||||
| 0 | 4 | maplength | length of the compressed bitstream, excluding this header |
|
||||
| 4 | 6 | datastart | offset of the first hunk blob — 124 in every image observed |
|
||||
| 10 | 2 | crc16 | CRC-16 over the decoded map |
|
||||
| 12 | 1 | lengthbits | bit width of a compressed-length field |
|
||||
| 13 | 1 | hunkbits | bit width of a self-reference |
|
||||
| 14 | 1 | parentunitbits | bit width of a parent reference |
|
||||
| 15 | 1 | reserved | zero in every image observed |
|
||||
|
||||
For a 64-hunk image at 4096 bytes per hunk with `zlib`: `maplength` 219,
|
||||
`datastart` 124, `lengthbits` 13, `hunkbits` 0, `parentunitbits` 0. The
|
||||
widths are sized to the image: `lengthbits` 13 admits lengths to 8191,
|
||||
the smallest power-of-two bound above `hunkbytes`; `hunkbits` and
|
||||
`parentunitbits` are zero because that image contains no self or parent
|
||||
references.
|
||||
|
||||
**The compressed map is the last structure in the file.** In every
|
||||
compressed image observed, `mapoffset + 16 + maplength` equals the file
|
||||
size exactly. **[V]**
|
||||
|
||||
This has a direct consequence for readers. A compressed image cannot
|
||||
resolve a single hunk until the tail of the file is available, so
|
||||
consuming one as a sequential stream — decoding from a growing prefix as
|
||||
it downloads — is not possible. Random access to at least the tail is a
|
||||
hard requirement, not an optimisation.
|
||||
|
||||
### 2.3 Compressed map bitstream **[V]**
|
||||
|
||||
The body begins at `mapoffset + 16` and is read most-significant-bit
|
||||
first. It carries a tree, then two passes over the remaining stream —
|
||||
the second continues where the first stops, with no alignment between
|
||||
them.
|
||||
|
||||
#### 2.3.1 Tree
|
||||
|
||||
A canonical Huffman tree of 16 codes, at most 8 bits, serialised as in
|
||||
§2.3.2. Codes are assigned **longest length first**, which is not RFC
|
||||
1951's ordering; see §5.
|
||||
|
||||
#### 2.3.2 Tree serialisation
|
||||
|
||||
Code lengths are stored in order, each as a fixed-width value. The width
|
||||
is 4 bits for a tree of at most 8-bit codes — the only case this
|
||||
serialisation is used for, since the map's tree is its only user.
|
||||
|
||||
A stored value of 1 is an escape, not a length:
|
||||
|
||||
- escape followed by 1 — a single code of length 1.
|
||||
- escape followed by any other value *v*, then a count *c* — the next
|
||||
*c* + 3 codes all have length *v*.
|
||||
- any other value is a length, for one code.
|
||||
|
||||
Reading stops once every code has a length.
|
||||
|
||||
#### 2.3.3 Pass one — a reference code per hunk
|
||||
|
||||
One symbol per hunk. Two of the sixteen are escapes that repeat the
|
||||
previous hunk's code rather than naming their own:
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| 0–3 | compressed with `compressors[code]` |
|
||||
| 4 | stored uncompressed |
|
||||
| 5 | self reference, target read in pass two |
|
||||
| 6 | parent reference, target read in pass two |
|
||||
| 7 | repeat previous, count 2 + next symbol |
|
||||
| 8 | repeat previous, count 18 + (next << 4) + next |
|
||||
| 9 | self reference to the same target as the last one |
|
||||
| 10 | self reference to the last target + 1 |
|
||||
| 11 | parent reference at this hunk's own unit position |
|
||||
| 12 | parent reference to the same target as the last one |
|
||||
| 13 | parent reference to the last target + `hunkbytes / unitbytes` |
|
||||
|
||||
Codes 14 and 15 were not observed.
|
||||
|
||||
#### 2.3.4 Pass two — the fields each code implies
|
||||
|
||||
Continuing the same stream, with a running offset that starts at
|
||||
`datastart`:
|
||||
|
||||
- **0–3**: `length` = read(`lengthbits`), `crc` = read(16). Offset is the
|
||||
running offset, which then advances by `length`.
|
||||
- **4**: `length` = `hunkbytes`, `crc` = read(16). Same advance.
|
||||
- **5**: offset = read(`hunkbits`); becomes the last self target.
|
||||
- **6**: offset = read(`parentbits`); becomes the last parent target.
|
||||
- **9, 10, 11, 12, 13**: consume no bits; the offset comes from the
|
||||
running self or parent target as described above, and the code is
|
||||
rewritten to 5 or 6.
|
||||
|
||||
#### 2.3.5 Decoded entry — 12 bytes
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 1 | reference code, escapes already resolved to 5 or 6 |
|
||||
| 1 | 3 | length |
|
||||
| 4 | 6 | offset |
|
||||
| 10 | 2 | CRC-32 of the hunk, truncated to 16 bits as stored |
|
||||
|
||||
#### 2.3.6 Validation
|
||||
|
||||
CRC-16/CCITT-FALSE — polynomial 0x1021, initial 0xffff, no reflection,
|
||||
no final xor — over `hunkcount * 12` decoded bytes must equal the `crc16`
|
||||
in the map header.
|
||||
|
||||
A second check is available and worth applying, because it catches
|
||||
different mistakes: the offsets of every compressed and uncompressed
|
||||
hunk must partition `[datastart, mapoffset)` exactly, with no gap and no
|
||||
overlap. A wrong bit width usually still produces a decodable stream but
|
||||
almost never a partition.
|
||||
|
||||
Both were confirmed on seven images spanning `zlib`, `lzma`, `huff`,
|
||||
`flac`, a four-codec image, a parent-differenced image and one built to
|
||||
force self references. Between them those exercise codes 0, 4, 5, 9, 10
|
||||
and 11.
|
||||
|
||||
### 2.4 Version 3 map entries — 16 bytes each **[V]**
|
||||
|
||||
The map begins immediately after the header, at offset 120, and holds
|
||||
`totalhunks` entries with no header of its own and no checksum.
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 8 | offset of the hunk's data |
|
||||
| 8 | 4 | CRC-32 of the decompressed hunk |
|
||||
| 12 | 2 | length, low 16 bits |
|
||||
| 14 | 1 | length, high 8 bits |
|
||||
| 15 | 1 | flags; the low four bits are the entry type |
|
||||
|
||||
Length is therefore 24 bits, assembled from a 16-bit and an 8-bit field
|
||||
rather than stored as one value.
|
||||
|
||||
Entry types, from the low nibble of the flags byte:
|
||||
|
||||
| Type | Meaning | Observed |
|
||||
|---|---|---|
|
||||
| 0 | invalid | no |
|
||||
| 1 | compressed | yes |
|
||||
| 2 | stored uncompressed | yes |
|
||||
| 3 | mini — the hunk is a repeat of the 8 bytes in the offset field | yes |
|
||||
| 4 | self reference | yes |
|
||||
| 5 | parent reference | no |
|
||||
| 6 | compressed with a secondary codec | no |
|
||||
|
||||
Types 5 and 6 are **[P]**: the archived images carry no parent and no
|
||||
secondary codec.
|
||||
|
||||
#### Map terminator **[V]**
|
||||
|
||||
The entry array of a version 1 to 4 map is followed by one further
|
||||
entry-sized slot holding the ASCII string `EndOfListCookie` — eight bytes
|
||||
for versions 1 and 2, sixteen padded with a NUL for versions 3 and 4.
|
||||
Hunk data or metadata begins after it.
|
||||
|
||||
This is easy to miss because nothing points at it: it is found by
|
||||
computing where the map ends. It also accounts for the otherwise
|
||||
unexplained gap between the end of a real image's map and its
|
||||
`metaoffset`. One of the two readers requires it and rejects an image
|
||||
without it; the other does not check. Writing it is therefore mandatory
|
||||
and checking it is optional, so a reader should tolerate its absence.
|
||||
|
||||
#### Version 1 and 2 map entries — 8 bytes each **[V]**
|
||||
|
||||
A single big-endian 64-bit value per hunk, packing two fields:
|
||||
|
||||
- **bits 0-43** — byte offset of the hunk's data.
|
||||
- **bits 44-63** — length of that data.
|
||||
|
||||
There is no type field and no checksum. A hunk is stored uncompressed
|
||||
when its length equals `hunkbytes`, and compressed otherwise; that
|
||||
comparison is the only signal available.
|
||||
|
||||
#### Verification
|
||||
|
||||
Confirmed the same way as §2.3: across 31,904 and 71,221 data-bearing
|
||||
entries in two images, the offsets and lengths partition the file from
|
||||
the end of the metadata to EOF with **no gap and no overlap**. The high
|
||||
bits of the flags byte were not exercised and are **[?]**.
|
||||
|
||||
---
|
||||
|
||||
## 3. Codec tags **[V]**
|
||||
|
||||
Read back verbatim from the `compressors` array of generated images.
|
||||
|
||||
| Tag | Meaning |
|
||||
|---|---|
|
||||
| `zlib` | raw DEFLATE |
|
||||
| `lzma` | LZMA, properties not stored |
|
||||
| `huff` | static-tree Huffman |
|
||||
| `flac` | FLAC, headerless |
|
||||
| `cdzl` `cdlz` `cdfl` | the above with CD framing |
|
||||
| `zstd` `cdzs` | Zstandard, and CD-framed **[P]**, postdates the reference writer |
|
||||
| `avhu` | audio/video, version 5 |
|
||||
|
||||
Versions 1 to 4 use a small enum in the header rather than four tags, so
|
||||
one codec applies to the whole image:
|
||||
|
||||
| Value | Meaning |
|
||||
|---|---|
|
||||
| 0 | none |
|
||||
| 1 | zlib |
|
||||
| 2 | zlib, a variant; both decode as zlib **[V]** |
|
||||
| 3 | audio/video — a *different* codec from version 5's `avhu` **[P]** |
|
||||
|
||||
**The zlib codec is raw DEFLATE**, with no zlib header and no adler32
|
||||
trailer. **[V]** An image built with zlib-wrapped hunks is rejected with a
|
||||
decompression error; rebuilding with raw DEFLATE and changing nothing
|
||||
else makes it decode. This applies to every version.
|
||||
|
||||
**Unit size, when no header field carries it.** Versions before 5 have no
|
||||
`unitbytes`. Both readers infer it from metadata that names a sector
|
||||
size, and both fall back to `hunkbytes` when no such metadata exists.
|
||||
**[V]**
|
||||
|
||||
### 3.1 Metadata entries **[V]**
|
||||
|
||||
`metaoffset` points at a singly linked list. Each entry is a 16-byte
|
||||
header followed by its payload:
|
||||
|
||||
| Offset | Size | Field |
|
||||
|---|---|---|
|
||||
| 0 | 4 | four-character tag |
|
||||
| 4 | 1 | flags |
|
||||
| 5 | 3 | payload length |
|
||||
| 8 | 8 | offset of the next entry, zero at the end |
|
||||
|
||||
The payload follows immediately. Hard-disk images carry a `GDDD` entry
|
||||
whose payload is ASCII of the form `CYLS:n,HEADS:n,SECS:n,BPS:n.`, and
|
||||
the `BPS` value is what supplies the unit size described above.
|
||||
|
||||
---
|
||||
|
||||
## 4. Pending
|
||||
|
||||
In the order they block work:
|
||||
|
||||
1. **Packed tree serialisation** — the form used by `huff` and `avhu`,
|
||||
distinct from §2.3.2. See §6 for what has been ruled out.
|
||||
2. **The version 1 to 4 A/V codec.** Compression enum 3 is not version
|
||||
5's `avhu`; it is the earlier codec that `avhu` replaced. Laserdisc
|
||||
images for older emulators use it, so full coverage needs both.
|
||||
3. **CD framing** — sector and subchannel interleave, the ECC bitmap, and
|
||||
the per-codec header shape. `cdfl` is the odd one and needs its own
|
||||
check.
|
||||
5. **`flac` framing** — the endianness byte and the synthesised stream
|
||||
parameters.
|
||||
6. **avhuff hunk layout.**
|
||||
7. **Versions 1 to 4**, blocked on samples.
|
||||
|
||||
---
|
||||
|
||||
## 5. Canonical code ordering **[V]**
|
||||
|
||||
Codes are assigned from code lengths by walking from the **longest**
|
||||
length down to the shortest, halving the starting code at each step. RFC
|
||||
1951 walks the other way, from shortest up, doubling. The two produce
|
||||
different codes: for lengths `{1, 2, 2}` this gives `1`, `00`, `01`
|
||||
where DEFLATE gives `0`, `10`, `11`.
|
||||
|
||||
Most map trees cannot tell the two apart, because a tree whose codes are
|
||||
all one length is assigned identically either way. The four-codec
|
||||
reference image is the one that can: its tree has lengths `{1, 2, 2}`
|
||||
exactly, and the stored CRC-16 matches only under the downward walk.
|
||||
|
||||
This is why `rhuff` documents its ordering as a dialect rather than as
|
||||
"canonical Huffman" — an implementation that reached for RFC 1951's
|
||||
ordering would decode single-codec images correctly and fail on
|
||||
multi-codec ones.
|
||||
|
||||
---
|
||||
|
||||
## 6. Packed tree serialisation **[V]**
|
||||
|
||||
Used by the `huff` hunk codec and by A/V hunks. Distinct from the RLE
|
||||
form of §2.3.2, which the map uses.
|
||||
|
||||
### 6.1 Structure
|
||||
|
||||
A tree over the main alphabet is described by a *second*, smaller tree,
|
||||
which codes the main tree's code lengths.
|
||||
|
||||
**The small tree.** Its own code lengths are stored as fixed 3-bit
|
||||
values over an alphabet of 24 symbols, at most 6 bits per code:
|
||||
|
||||
- one 3-bit value: the length of symbol 0;
|
||||
- one 3-bit value: an index, plus one, at which the remaining lengths
|
||||
resume, so leading unused symbols cost nothing;
|
||||
- 3-bit lengths from that index upward, ending at a stored value of 7.
|
||||
Seven cannot be a real length here because the ceiling is 6, which is
|
||||
what frees it to terminate the list.
|
||||
|
||||
**The main lengths.** Symbols decoded with the small tree, where a
|
||||
symbol *v* other than zero means a code length of *v* − 1, and zero
|
||||
introduces a run repeating the last length emitted:
|
||||
|
||||
- a 3-bit field holds the run count less two, so 0 to 6 gives runs of
|
||||
2 to 8;
|
||||
- a field of 7 escapes: the count is 9 plus a further field of
|
||||
`ceil(log2(numcodes - 9))` bits — 8 for a 256-symbol alphabet.
|
||||
|
||||
Both trees then get the canonical assignment of §5, longest length
|
||||
first.
|
||||
|
||||
### 6.2 Incomplete trees are legal
|
||||
|
||||
A hunk of one repeated byte yields an alphabet with a single used
|
||||
symbol, whose one-bit code covers only half the code space. **That is a
|
||||
valid tree and must be accepted.** Rejecting it as incomplete fails
|
||||
exactly the inputs that compress best, and a decoder that requires
|
||||
completeness will decode ordinary data and fall over on a run of
|
||||
identical bytes.
|
||||
|
||||
Overlap between codes must still be rejected; it is only the
|
||||
requirement that they *cover* the space that has to go.
|
||||
|
||||
### 6.3 Verification
|
||||
|
||||
Seventy-one `huff` hunks across four images, decoded and compared against
|
||||
the original uncompressed source. All byte-exact, in both a Python model
|
||||
and the C implementation, the latter clean under ASan and UBSan.
|
||||
|
||||
The images are built to exercise the tree shape directly rather than
|
||||
sampled from real content: one hunk per distinct-symbol count from 1 to
|
||||
8; sixteen hunks each holding a single repeated byte, at values spread
|
||||
across the alphabet; and nineteen holding exactly two symbols, at index 0
|
||||
and at a second index chosen to step the zero run between them one at a
|
||||
time. That last set is what made the encoding legible — a single-symbol
|
||||
hunk varies two runs at once, and the resulting signal is too muddy to
|
||||
read.
|
||||
|
||||
---|---|---|
|
||||
| 1 | 518 | 6 |
|
||||
| 2 | 518 | 6 |
|
||||
| 4 | 1031 | 7 |
|
||||
| 8 | 1543 | 7 |
|
||||
|
||||
**The whole serialised tree is 4 to 7 bytes.** For the single-symbol
|
||||
image the tree is 4 bytes at symbol values 0x00, 0x01, 0xfe and 0xff,
|
||||
5 bytes at 0x02 through 0x08, and 6 bytes from 0x10 upward.
|
||||
|
||||
That size is the strongest constraint available and it rules out both
|
||||
families tried:
|
||||
|
||||
- **A 24-code small tree with fixed-width lengths.** At three bits per
|
||||
length that needs at least 72 bits before any length vector is coded.
|
||||
The whole tree is 32 to 56.
|
||||
- **The RLE form of §2.3.2 at a wider value width.** Encoding 255 zeros
|
||||
in runs bounded by a 5-bit count needs eight runs at fifteen bits.
|
||||
Swept widths 3 through 6; none produces a valid complete tree.
|
||||
|
||||
A parameter sweep over small-tree code counts 16/20/24, value widths
|
||||
3/4, length ceilings 6/7/8, both senses of the skip condition, and
|
||||
repeat widths 3/4, checked against all sixteen known vectors, produced no
|
||||
match.
|
||||
|
||||
### 6.3 The run-length encoding **[V]**
|
||||
|
||||
Hunks built from exactly two symbols, at index 0 and index *N*, give a
|
||||
length vector with two entries of length 1 and a run of *N*-1 zeros
|
||||
between them. Stepping *N* varies that run one at a time while
|
||||
everything before it stays fixed, which isolates the field.
|
||||
|
||||
A run is coded as a **3-bit field holding the count less three**, so
|
||||
values 0 to 6 express runs of 3 to 9. **A field of 7 is an escape**: the
|
||||
count is then 10 plus a further **8-bit** field.
|
||||
|
||||
Confirmed on fifteen runs from 3 to 128. The escape width is
|
||||
`ceil(log2(numcodes - 9))`, which is 8 for a 256-symbol alphabet — the
|
||||
same relation the map's alphabet would give, so it is a rule rather than
|
||||
a constant.
|
||||
|
||||
The bias is worth stating precisely because an earlier attempt used a
|
||||
count of *field + 2* escaping at 9, which is off by one and decodes every
|
||||
short run wrongly while still consuming the right number of bits.
|
||||
|
||||
### 6.4 What remains
|
||||
|
||||
The serialisation of the *small tree* — the one used to code the length
|
||||
values and the run escapes themselves. What is known:
|
||||
|
||||
- It occupies the first 16 bits in the common case, and the first coded
|
||||
symbol lands at bit 16.
|
||||
- It takes one of two forms across the sixteen single-symbol hunks:
|
||||
thirteen share a 16-bit prefix, three share a different one.
|
||||
- Sweeping small alphabet sizes 16/20/24, value widths 3/4, code-length
|
||||
ceilings 6/7/8 and both senses of the skip condition — now with the
|
||||
corrected run coding — still matches none of the nineteen known length
|
||||
vectors.
|
||||
|
||||
Since the run coding is now fixed and verified, the remaining unknown is
|
||||
narrow: how roughly 16 bits describe a tree over the small alphabet of
|
||||
length values. Solving it from the two-symbol series is the next step,
|
||||
working backwards from the known emission sequence — a length value, a
|
||||
run, a length value, a run — to the codes those four symbols must have.
|
||||
|
||||
---
|
||||
|
||||
## 7. CD framing **[V]**
|
||||
|
||||
Applies to `cdzl`, `cdlz`, `cdfl` and `cdzs`. A CD image has
|
||||
`unitbytes` 2448 — a 2352-byte sector followed by 96 bytes of
|
||||
subchannel — and `hunkbytes / unitbytes` frames per hunk.
|
||||
|
||||
### 7.1 Blob layout
|
||||
|
||||
| Part | Size | Notes |
|
||||
|---|---|---|
|
||||
| ECC bitmap | `(frames + 7) / 8` bytes | one bit per frame |
|
||||
| sector stream length | 2 bytes, or 3 when `hunkbytes >= 65536` | big-endian |
|
||||
| sector stream | the length above | `frames * 2352` bytes when decoded |
|
||||
| subchannel stream | whatever remains | `frames * 96` bytes when decoded |
|
||||
|
||||
The sector stream uses the codec's base compressor — DEFLATE for `cdzl`,
|
||||
LZMA for `cdlz`. **The subchannel stream is always raw DEFLATE**,
|
||||
whichever codec the hunk uses.
|
||||
|
||||
The two streams are stored whole and consecutively, not interleaved: all
|
||||
sector data, then all subchannel data. They are interleaved only on
|
||||
output, where each frame's 2352 sector bytes are followed by its own 96
|
||||
subchannel bytes.
|
||||
|
||||
### 7.2 The ECC bitmap
|
||||
|
||||
A set bit means that frame's sync pattern and ECC field were stripped
|
||||
before compression and must be rebuilt.
|
||||
|
||||
**Bit *i* is `1 << (i & 7)` of byte `i >> 3` — least significant bit
|
||||
first.** This is worth stating plainly because the opposite order is the
|
||||
natural guess and is wrong in a way that hides: a hunk whose bitmap is
|
||||
`0xff` decodes identically under either convention, and those are common.
|
||||
The first hunk observed to distinguish them had a bitmap of `0x0f`, where
|
||||
the wrong order both skips the frames that needed rebuilding and corrupts
|
||||
the ones that did not.
|
||||
|
||||
### 7.3 Rebuilding a stripped frame
|
||||
|
||||
Sync first: byte 0 is `0x00`, bytes 1 to 10 are `0xff`, byte 11 is
|
||||
`0x00`. Only bytes 1 to 10 are actually absent from the stored data; the
|
||||
two `0x00` bytes are already there.
|
||||
|
||||
Then the layered error correction of ECMA-130, over GF(2^8) with
|
||||
primitive polynomial `0x11d`:
|
||||
|
||||
- **P parity** — 86 majors of 24 minors, major multiplier 2, minor
|
||||
increment 86, read from sector offset `0x0c`, written to `0x81c`.
|
||||
- **Q parity** — 52 majors of 43 minors, major multiplier 86, minor
|
||||
increment 88, read from the same base, written to `0x8c8`. Q covers the
|
||||
P parity just written, so the order matters.
|
||||
|
||||
The four header bytes at offset 12 are treated as zero while computing
|
||||
and restored afterwards.
|
||||
|
||||
**The EDC at offset 2072 is stored, not regenerated.** It falls outside
|
||||
the rebuilt range, which is 2076 to 2351 — the 276 ECC bytes of a 2352
|
||||
byte sector, after 12 sync, 4 header, 8 subheader, 2048 data and 4 EDC.
|
||||
|
||||
### 7.4 Verification
|
||||
|
||||
160 hunks across four commercial CD images, reconstructed and compared
|
||||
byte-for-byte against an independent reader's decode of the same hunk.
|
||||
All 160 match.
|
||||
|
||||
---
|
||||
|
||||
## 8. `cdfl` **[V]**
|
||||
|
||||
`cdfl` shares §7's CD geometry but has a different, simpler blob layout.
|
||||
|
||||
### 8.1 Blob layout
|
||||
|
||||
**There is no header at all.** No ECC bitmap, no length field. The FLAC
|
||||
stream begins at byte zero, and the subchannel stream follows it.
|
||||
|
||||
| Part | Notes |
|
||||
|---|---|
|
||||
| FLAC stream | from byte 0; decodes to `frames * 2352` bytes |
|
||||
| subchannel stream | raw DEFLATE, whatever trails the FLAC data |
|
||||
|
||||
The FLAC data is two channel, sixteen bit. A decoder knows where it ends
|
||||
by decoding until it has produced `frames * 2352` bytes; everything after
|
||||
that is the subchannel.
|
||||
|
||||
The absence of an ECC bitmap follows from what `cdfl` is chosen for: an
|
||||
encoder picks it for audio, and audio sectors carry no sync pattern and
|
||||
no ECC field, so there is nothing to strip and nothing to rebuild.
|
||||
|
||||
### 8.2 Sample byte order
|
||||
|
||||
**The decoded samples are byte-swapped relative to the sector data.**
|
||||
Each 16-bit sample is stored most significant byte first, where a disc
|
||||
image holds it least significant byte first, so every pair must be
|
||||
swapped on the way out. This is invisible until compared against real
|
||||
output — the stream decodes perfectly either way, it is simply wrong.
|
||||
|
||||
### 8.3 Verification
|
||||
|
||||
Thirty hunks across two commercial images, decoded and compared
|
||||
byte-for-byte against an independent reader. All thirty match.
|
||||
|
||||
A synthetic image built from content controlled sector by sector —
|
||||
silence, a DC level, a tone, a ramp and noise, three homogeneous hunks
|
||||
each — confirms the layout holds across the whole compressibility range,
|
||||
from a 41-byte blob to one of 18849 for a 18816-byte payload.
|
||||
|
||||
### 8.4 A note on method
|
||||
|
||||
An earlier pass concluded that most `cdfl` blobs contained no FLAC stream
|
||||
at any offset, and recorded that as a ruled-out hypothesis. It was wrong,
|
||||
and the cause is worth recording.
|
||||
|
||||
The test harness decoded the hunk map without consuming the offset bits
|
||||
that a self reference or a parent reference carries. Every hunk after the
|
||||
first such reference therefore had a wrong offset, and the blobs being
|
||||
probed were arbitrary slices of neighbouring hunks. The commercial images
|
||||
used for that pass each contain exactly one self reference early on, so
|
||||
almost everything examined was garbage.
|
||||
|
||||
Two things hid it. The map's own CRC still matched, because the CRC is
|
||||
computed over the decoded entries and the separate script that produced
|
||||
those consumed the bits correctly — only the harness copy did not. And
|
||||
the CD framing tests still passed, because the hunks they sampled
|
||||
happened to precede the first self reference.
|
||||
|
||||
The harness now asserts that decoded hunk offsets partition the data
|
||||
region exactly, which is the check that would have caught it immediately.
|
||||
@@ -0,0 +1,979 @@
|
||||
/* Copyright (C) 2010-2026 The RetroArch team
|
||||
*
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* The following license statement only applies to this file (rchd.c).
|
||||
* ---------------------------------------------------------------------------------------
|
||||
*
|
||||
* Permission is hereby granted, free of charge,
|
||||
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* Reader for MAME's CHD container. See FORMAT.md beside this file for the
|
||||
* format description this is written against; every constant here traces
|
||||
* to a measurement recorded there rather than to a reading of another
|
||||
* implementation.
|
||||
*
|
||||
* No file I/O happens here. The decoder describes the byte ranges it
|
||||
* needs and the caller supplies them, which is what lets one image be
|
||||
* read from a task a few kilobytes at a time, from several fetches in
|
||||
* flight, or from a plain blocking read, without the reader knowing
|
||||
* which.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <formats/rchd.h>
|
||||
#include <encodings/huffman.h>
|
||||
#include <encodings/crc32.h>
|
||||
|
||||
/* Container limits. A hunk is bounded by the format; the map and
|
||||
* metadata bounds are ours, sized well above anything a real image
|
||||
* carries, so a corrupt header cannot ask for an unbounded allocation. */
|
||||
#define RCHD_MAX_HUNK_BYTES (1024 * 1024)
|
||||
#define RCHD_MAX_MAP_BYTES (64 * 1024 * 1024)
|
||||
#define RCHD_MAX_METADATA (16 * 1024 * 1024)
|
||||
#define RCHD_MAX_META_ENTRIES 4096
|
||||
/* A hunk count is stored rather than derived before version 5, so a
|
||||
* corrupt one can name any number. The map is allocated from it, so it
|
||||
* needs a bound of its own: this admits a 512 GiB image at the smallest
|
||||
* plausible hunk size, well past anything real. */
|
||||
#define RCHD_MAX_HUNK_COUNT (32 * 1024 * 1024)
|
||||
|
||||
/* Decoded map entry, one per hunk. Kept unpacked rather than as the
|
||||
* twelve bytes the CRC is computed over: that form exists only to be
|
||||
* hashed, and every use afterwards wants the fields. */
|
||||
typedef struct rchd_map_entry
|
||||
{
|
||||
uint64_t offset;
|
||||
uint32_t length;
|
||||
uint32_t crc;
|
||||
uint8_t type;
|
||||
} rchd_map_entry_t;
|
||||
|
||||
/* Reference codes, as stored in a compressed v5 map (FORMAT.md 2.3.3). */
|
||||
enum
|
||||
{
|
||||
RCHD_V5_TYPE_0 = 0,
|
||||
RCHD_V5_TYPE_3 = 3,
|
||||
RCHD_V5_NONE = 4,
|
||||
RCHD_V5_SELF = 5,
|
||||
RCHD_V5_PARENT = 6,
|
||||
RCHD_V5_RLE_SMALL = 7,
|
||||
RCHD_V5_RLE_LARGE = 8,
|
||||
RCHD_V5_SELF_0 = 9,
|
||||
RCHD_V5_SELF_1 = 10,
|
||||
RCHD_V5_PARENT_SELF = 11,
|
||||
RCHD_V5_PARENT_0 = 12,
|
||||
RCHD_V5_PARENT_1 = 13
|
||||
};
|
||||
|
||||
/* Entry types of a v1 to v4 map (FORMAT.md 2.4). */
|
||||
enum
|
||||
{
|
||||
RCHD_V34_INVALID = 0,
|
||||
RCHD_V34_COMPRESSED = 1,
|
||||
RCHD_V34_UNCOMPRESSED = 2,
|
||||
RCHD_V34_MINI = 3,
|
||||
RCHD_V34_SELF = 4,
|
||||
RCHD_V34_PARENT = 5,
|
||||
RCHD_V34_2ND = 6
|
||||
};
|
||||
|
||||
static const char rchd_end_of_list[16] =
|
||||
{
|
||||
'E','n','d','O','f','L','i','s','t','C','o','o','k','i','e','\0'
|
||||
};
|
||||
|
||||
/* -------- byte access --------
|
||||
*
|
||||
* Every field is read a byte at a time. Nothing casts the buffer to a
|
||||
* wider type, so this does not care about the host's endianness or about
|
||||
* the alignment of what the caller handed over. */
|
||||
|
||||
static uint32_t rchd_rd16(const uint8_t *p)
|
||||
{
|
||||
return ((uint32_t)p[0] << 8) | (uint32_t)p[1];
|
||||
}
|
||||
|
||||
static uint32_t rchd_rd32(const uint8_t *p)
|
||||
{
|
||||
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16)
|
||||
| ((uint32_t)p[2] << 8) | (uint32_t)p[3];
|
||||
}
|
||||
|
||||
static uint64_t rchd_rd_be(const uint8_t *p, int n)
|
||||
{
|
||||
uint64_t v = 0;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < n; i++)
|
||||
v = (v << 8) | (uint64_t)p[i];
|
||||
return v;
|
||||
}
|
||||
|
||||
/* CRC-16/CCITT-FALSE: polynomial 0x1021, initial 0xffff, no reflection
|
||||
* and no final xor. Validates a decoded v5 map (FORMAT.md 2.3.6). */
|
||||
static uint16_t rchd_crc16(const uint8_t *data, size_t len)
|
||||
{
|
||||
uint16_t crc = 0xffff;
|
||||
size_t i;
|
||||
int bit;
|
||||
|
||||
for (i = 0; i < len; i++)
|
||||
{
|
||||
crc ^= (uint16_t)((uint16_t)data[i] << 8);
|
||||
for (bit = 0; bit < 8; bit++)
|
||||
crc = (uint16_t)((crc & 0x8000) ? ((crc << 1) ^ 0x1021) : (crc << 1));
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
/* -------- decoder state -------- */
|
||||
|
||||
/* What the open sequence is waiting for. Each step names a byte range,
|
||||
* consumes it when fed, and moves on; nothing is read that a previous
|
||||
* step did not establish the position of. */
|
||||
enum
|
||||
{
|
||||
RCHD_OPEN_HEADER = 0,
|
||||
RCHD_OPEN_MAP,
|
||||
RCHD_OPEN_META,
|
||||
RCHD_OPEN_DONE
|
||||
};
|
||||
|
||||
typedef struct rchd_codec_slot
|
||||
{
|
||||
uint32_t tag;
|
||||
rchd_codec_decode_t fn;
|
||||
void *ctx;
|
||||
} rchd_codec_slot_t;
|
||||
|
||||
struct rchd
|
||||
{
|
||||
rchd_info_t info;
|
||||
rchd_map_entry_t *map;
|
||||
|
||||
/* Raw bytes of whatever range is being collected, and how much of it
|
||||
* has arrived. One buffer serves every step because the steps are
|
||||
* strictly sequential. */
|
||||
uint8_t *pending;
|
||||
size_t pending_size;
|
||||
size_t pending_have;
|
||||
uint64_t pending_off;
|
||||
int pending_src;
|
||||
|
||||
uint8_t *metadata;
|
||||
size_t metadata_size;
|
||||
rchd_metadata_t *meta;
|
||||
uint32_t meta_count;
|
||||
|
||||
uint64_t meta_offset; /* next entry to walk, 0 when done */
|
||||
/* Payload length of the entry being collected, once its header has
|
||||
* been read. Held across calls because the walk asks for the header
|
||||
* and then for the whole entry, and re-deriving it would mean asking
|
||||
* for the header again -- which, keyed as requests are on offset and
|
||||
* length, is a different request that discards the first. */
|
||||
uint32_t meta_len;
|
||||
int meta_have_len;
|
||||
uint64_t map_offset;
|
||||
uint32_t map_length;
|
||||
|
||||
int state;
|
||||
int header_len;
|
||||
|
||||
rchd_codec_slot_t *codecs;
|
||||
uint32_t codec_count;
|
||||
|
||||
rchd_t *parent;
|
||||
};
|
||||
|
||||
/* Asks the caller for a range, and reports whether it has all arrived.
|
||||
* A step calls this each time it runs, so a short supply simply costs
|
||||
* another round rather than needing the step to be re-entrant. */
|
||||
static int rchd_want(rchd_t *chd, uint64_t offset, size_t len, int source,
|
||||
rchd_request_t *req)
|
||||
{
|
||||
if (chd->pending_off != offset || chd->pending_size != len
|
||||
|| chd->pending_src != source)
|
||||
{
|
||||
uint8_t *buf;
|
||||
|
||||
if (len == 0)
|
||||
return 1;
|
||||
if (!(buf = (uint8_t*)malloc(len)))
|
||||
return -1;
|
||||
free(chd->pending);
|
||||
chd->pending = buf;
|
||||
chd->pending_size = len;
|
||||
chd->pending_have = 0;
|
||||
chd->pending_off = offset;
|
||||
chd->pending_src = source;
|
||||
}
|
||||
|
||||
if (chd->pending_have >= len)
|
||||
return 1;
|
||||
|
||||
req->offset = offset + chd->pending_have;
|
||||
req->length = (uint32_t)(len - chd->pending_have);
|
||||
req->source = source;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rchd_feed(rchd_t *chd, const void *data, size_t len)
|
||||
{
|
||||
if (!chd || !data)
|
||||
return RCHD_ERROR_PARAM;
|
||||
if (!chd->pending || chd->pending_have >= chd->pending_size)
|
||||
return RCHD_ERROR_STATE;
|
||||
|
||||
if (len > chd->pending_size - chd->pending_have)
|
||||
len = chd->pending_size - chd->pending_have;
|
||||
|
||||
memcpy(chd->pending + chd->pending_have, data, len);
|
||||
chd->pending_have += len;
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
/* -------- header -------- */
|
||||
|
||||
/* Versions 1 and 2 record no size and no hunk length; both are derived
|
||||
* from the drive geometry, which makes those fields load-bearing rather
|
||||
* than descriptive (FORMAT.md 1.4). */
|
||||
static int rchd_parse_v1v2(rchd_t *chd, const uint8_t *h, uint32_t version)
|
||||
{
|
||||
uint32_t seclen = (version == 2) ? rchd_rd32(h + 76) : 512;
|
||||
uint32_t cyls = rchd_rd32(h + 32);
|
||||
uint32_t heads = rchd_rd32(h + 36);
|
||||
uint32_t secs = rchd_rd32(h + 40);
|
||||
uint32_t hunksz = rchd_rd32(h + 24);
|
||||
|
||||
if (!seclen || !cyls || !heads || !secs || !hunksz)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
chd->info.hunk_bytes = hunksz * seclen;
|
||||
chd->info.logical_bytes = (uint64_t)cyls * heads * secs * seclen;
|
||||
chd->info.hunk_count = rchd_rd32(h + 28);
|
||||
chd->info.unit_bytes = seclen;
|
||||
chd->info.has_parent = (rchd_rd32(h + 16) & 1) != 0;
|
||||
chd->meta_offset = 0;
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
static int rchd_parse_v3v4(rchd_t *chd, const uint8_t *h, uint32_t version)
|
||||
{
|
||||
if (version == 3)
|
||||
{
|
||||
chd->info.logical_bytes = rchd_rd_be(h + 28, 8);
|
||||
chd->meta_offset = rchd_rd_be(h + 36, 8);
|
||||
chd->info.hunk_bytes = rchd_rd32(h + 76);
|
||||
memcpy(chd->info.sha1, h + 80, 20);
|
||||
memcpy(chd->info.parent_sha1, h + 100, 20);
|
||||
}
|
||||
else
|
||||
{
|
||||
chd->info.logical_bytes = rchd_rd_be(h + 28, 8);
|
||||
chd->meta_offset = rchd_rd_be(h + 36, 8);
|
||||
chd->info.hunk_bytes = rchd_rd32(h + 44);
|
||||
memcpy(chd->info.sha1, h + 48, 20);
|
||||
memcpy(chd->info.parent_sha1, h + 68, 20);
|
||||
memcpy(chd->info.raw_sha1, h + 88, 20);
|
||||
}
|
||||
|
||||
chd->info.hunk_count = rchd_rd32(h + 24);
|
||||
chd->info.has_parent = (rchd_rd32(h + 16) & 1) != 0;
|
||||
|
||||
/* Nothing before version 5 records a unit size. It is inferred from
|
||||
* metadata that names a sector size, and falls back to the hunk size
|
||||
* when no such metadata exists -- which is what both readers of this
|
||||
* format settle on (FORMAT.md 1.2). The fallback is installed here
|
||||
* and refined once the metadata has been walked. */
|
||||
chd->info.unit_bytes = chd->info.hunk_bytes;
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
static int rchd_parse_v5(rchd_t *chd, const uint8_t *h)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 4; i++)
|
||||
chd->info.compressors[i] = rchd_rd32(h + 16 + i * 4);
|
||||
|
||||
chd->info.logical_bytes = rchd_rd_be(h + 32, 8);
|
||||
chd->map_offset = rchd_rd_be(h + 40, 8);
|
||||
chd->meta_offset = rchd_rd_be(h + 48, 8);
|
||||
chd->info.hunk_bytes = rchd_rd32(h + 56);
|
||||
chd->info.unit_bytes = rchd_rd32(h + 60);
|
||||
|
||||
memcpy(chd->info.raw_sha1, h + 64, 20);
|
||||
memcpy(chd->info.sha1, h + 84, 20);
|
||||
memcpy(chd->info.parent_sha1, h + 104, 20);
|
||||
|
||||
if (!chd->info.hunk_bytes || !chd->info.unit_bytes)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
{
|
||||
static const uint8_t zero[20] = { 0 };
|
||||
chd->info.has_parent = memcmp(chd->info.parent_sha1, zero, 20) != 0;
|
||||
}
|
||||
|
||||
chd->info.hunk_count = (uint32_t)((chd->info.logical_bytes
|
||||
+ chd->info.hunk_bytes - 1) / chd->info.hunk_bytes);
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
static int rchd_parse_header(rchd_t *chd, const uint8_t *h)
|
||||
{
|
||||
uint32_t length;
|
||||
uint32_t version;
|
||||
int err;
|
||||
|
||||
if (memcmp(h, "MComprHD", 8) != 0)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
length = rchd_rd32(h + 8);
|
||||
version = rchd_rd32(h + 12);
|
||||
|
||||
/* Length and version have to agree: a mismatch is the signature of a
|
||||
* file that is not what it says it is. */
|
||||
switch (version)
|
||||
{
|
||||
case 1: if (length != 76) return RCHD_ERROR_DATA; break;
|
||||
case 2: if (length != 80) return RCHD_ERROR_DATA; break;
|
||||
case 3: if (length != 120) return RCHD_ERROR_DATA; break;
|
||||
case 4: if (length != 108) return RCHD_ERROR_DATA; break;
|
||||
case 5: if (length != 124) return RCHD_ERROR_DATA; break;
|
||||
default: return RCHD_ERROR_UNSUPPORTED;
|
||||
}
|
||||
|
||||
chd->info.version = version;
|
||||
chd->header_len = (int)length;
|
||||
|
||||
if (version <= 2)
|
||||
err = rchd_parse_v1v2(chd, h, version);
|
||||
else if (version <= 4)
|
||||
err = rchd_parse_v3v4(chd, h, version);
|
||||
else
|
||||
err = rchd_parse_v5(chd, h);
|
||||
|
||||
if (err != RCHD_OK)
|
||||
return err;
|
||||
|
||||
if (!chd->info.hunk_bytes || chd->info.hunk_bytes > RCHD_MAX_HUNK_BYTES)
|
||||
return RCHD_ERROR_DATA;
|
||||
if (!chd->info.hunk_count || chd->info.hunk_count > RCHD_MAX_HUNK_COUNT)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
/* The count and the size have to agree. Before version 5 both are
|
||||
* stored, so a file whose count does not cover its logical size --
|
||||
* or overshoots it by more than the padding a final partial hunk
|
||||
* needs -- is describing something other than what it holds. */
|
||||
{
|
||||
uint64_t covered = (uint64_t)chd->info.hunk_count
|
||||
* chd->info.hunk_bytes;
|
||||
|
||||
if (covered < chd->info.logical_bytes)
|
||||
return RCHD_ERROR_DATA;
|
||||
if (covered - chd->info.logical_bytes >= chd->info.hunk_bytes)
|
||||
return RCHD_ERROR_DATA;
|
||||
}
|
||||
if (!chd->info.unit_bytes)
|
||||
chd->info.unit_bytes = chd->info.hunk_bytes;
|
||||
|
||||
/* Before version 5 the map sits immediately after the header. */
|
||||
if (version < 5)
|
||||
chd->map_offset = length;
|
||||
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
/* -------- maps -------- */
|
||||
|
||||
/* Versions 1 and 2 pack an offset and a length into one 64-bit word and
|
||||
* record no type: a hunk is stored whole when its length equals the hunk
|
||||
* size, and compressed otherwise, which is the only signal there is
|
||||
* (FORMAT.md 2.4). */
|
||||
static int rchd_map_v1v2(rchd_t *chd, const uint8_t *raw)
|
||||
{
|
||||
uint32_t n;
|
||||
|
||||
for (n = 0; n < chd->info.hunk_count; n++)
|
||||
{
|
||||
uint64_t word = rchd_rd_be(raw + (size_t)n * 8, 8);
|
||||
uint32_t len = (uint32_t)(word >> 44);
|
||||
|
||||
chd->map[n].offset = word & (((uint64_t)1 << 44) - 1);
|
||||
chd->map[n].length = len;
|
||||
chd->map[n].crc = 0;
|
||||
chd->map[n].type = (len == chd->info.hunk_bytes)
|
||||
? RCHD_V34_UNCOMPRESSED : RCHD_V34_COMPRESSED;
|
||||
}
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
static int rchd_map_v3v4(rchd_t *chd, const uint8_t *raw)
|
||||
{
|
||||
uint32_t n;
|
||||
|
||||
for (n = 0; n < chd->info.hunk_count; n++)
|
||||
{
|
||||
const uint8_t *e = raw + (size_t)n * 16;
|
||||
|
||||
chd->map[n].offset = rchd_rd_be(e, 8);
|
||||
chd->map[n].crc = rchd_rd32(e + 8);
|
||||
/* A 24-bit length, assembled from a 16-bit field and an 8-bit one
|
||||
* rather than stored as a single value. */
|
||||
chd->map[n].length = rchd_rd16(e + 12) | ((uint32_t)e[14] << 16);
|
||||
chd->map[n].type = (uint8_t)(e[15] & 0x0f);
|
||||
}
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
/* An uncompressed version 5 map is a flat array of hunk indices. A value
|
||||
* of zero means a hole reading as zeros, not an offset of zero: taking
|
||||
* it literally yields the first hunk-worth of the file, which is the
|
||||
* header, and is well-formed data of the right length (FORMAT.md 2.1). */
|
||||
static int rchd_map_v5_raw(rchd_t *chd, const uint8_t *raw)
|
||||
{
|
||||
uint32_t n;
|
||||
|
||||
for (n = 0; n < chd->info.hunk_count; n++)
|
||||
{
|
||||
uint32_t block = rchd_rd32(raw + (size_t)n * 4);
|
||||
|
||||
chd->map[n].crc = 0;
|
||||
chd->map[n].length = chd->info.hunk_bytes;
|
||||
if (block == 0)
|
||||
{
|
||||
chd->map[n].type = RCHD_V5_NONE;
|
||||
chd->map[n].offset = 0;
|
||||
chd->map[n].length = 0; /* marks the hole */
|
||||
}
|
||||
else
|
||||
{
|
||||
chd->map[n].type = RCHD_V5_NONE;
|
||||
chd->map[n].offset = (uint64_t)block * chd->info.hunk_bytes;
|
||||
}
|
||||
}
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
/* The compressed version 5 map: a tree, then a reference code per hunk,
|
||||
* then the fields each code implies, all in one bit stream
|
||||
* (FORMAT.md 2.3). */
|
||||
static int rchd_map_v5(rchd_t *chd, const uint8_t *raw, size_t raw_len)
|
||||
{
|
||||
static const uint32_t tree_codes = 16;
|
||||
static const uint32_t tree_bits = 8;
|
||||
uint16_t lookup[1 << 8];
|
||||
rhuff_dec_t dec;
|
||||
rhuff_bits_t bits;
|
||||
uint8_t *codes;
|
||||
uint8_t *checkbuf;
|
||||
uint32_t maplength;
|
||||
uint64_t datastart;
|
||||
uint16_t stored_crc;
|
||||
uint32_t lengthbits;
|
||||
uint32_t selfbits;
|
||||
uint32_t parentbits;
|
||||
uint64_t curoffset;
|
||||
uint64_t last_self = 0;
|
||||
uint64_t last_parent = 0;
|
||||
uint32_t repeat = 0;
|
||||
uint32_t last_code = 0;
|
||||
uint32_t n;
|
||||
int err = RCHD_ERROR_DATA;
|
||||
|
||||
if (raw_len < 16)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
maplength = rchd_rd32(raw);
|
||||
datastart = rchd_rd_be(raw + 4, 6);
|
||||
stored_crc = (uint16_t)rchd_rd16(raw + 10);
|
||||
lengthbits = raw[12];
|
||||
selfbits = raw[13];
|
||||
parentbits = raw[14];
|
||||
|
||||
if (lengthbits > 24 || selfbits > 24 || parentbits > 24)
|
||||
return RCHD_ERROR_DATA;
|
||||
if ((uint64_t)maplength + 16 > raw_len)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
if (rhuff_dec_init(&dec, tree_codes, tree_bits, lookup,
|
||||
RHUFF_LOOKUP_ENTRIES(8)) != RHUFF_OK)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
rhuff_bits_init(&bits, raw + 16, maplength);
|
||||
|
||||
if (rhuff_read_tree_rle(&dec, &bits) != RHUFF_OK)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
if (!(codes = (uint8_t*)malloc(chd->info.hunk_count)))
|
||||
return RCHD_ERROR_MEM;
|
||||
|
||||
/* First pass: one code per hunk. Two of the sixteen repeat the
|
||||
* previous hunk's code rather than naming their own. */
|
||||
for (n = 0; n < chd->info.hunk_count; n++)
|
||||
{
|
||||
uint32_t value;
|
||||
|
||||
if (repeat > 0)
|
||||
{
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat--;
|
||||
continue;
|
||||
}
|
||||
|
||||
value = rhuff_dec_decode_one(&dec, &bits);
|
||||
|
||||
if (value == RCHD_V5_RLE_SMALL)
|
||||
{
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat = 2 + rhuff_dec_decode_one(&dec, &bits);
|
||||
}
|
||||
else if (value == RCHD_V5_RLE_LARGE)
|
||||
{
|
||||
uint32_t hi = rhuff_dec_decode_one(&dec, &bits);
|
||||
uint32_t lo = rhuff_dec_decode_one(&dec, &bits);
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat = 2 + 16 + (hi << 4) + lo;
|
||||
}
|
||||
else
|
||||
{
|
||||
last_code = value;
|
||||
codes[n] = (uint8_t)value;
|
||||
}
|
||||
}
|
||||
|
||||
if (!(checkbuf = (uint8_t*)malloc((size_t)chd->info.hunk_count * 12)))
|
||||
{
|
||||
free(codes);
|
||||
return RCHD_ERROR_MEM;
|
||||
}
|
||||
|
||||
/* Second pass: the fields each code implies, continuing the same
|
||||
* stream, with a running offset that compressed and stored hunks
|
||||
* advance and references do not. */
|
||||
curoffset = datastart;
|
||||
|
||||
for (n = 0; n < chd->info.hunk_count; n++)
|
||||
{
|
||||
uint32_t code = codes[n];
|
||||
uint64_t offset = curoffset;
|
||||
uint32_t length = 0;
|
||||
uint32_t crc = 0;
|
||||
uint8_t *slot = checkbuf + (size_t)n * 12;
|
||||
|
||||
switch (code)
|
||||
{
|
||||
case RCHD_V5_TYPE_0:
|
||||
case RCHD_V5_TYPE_0 + 1:
|
||||
case RCHD_V5_TYPE_0 + 2:
|
||||
case RCHD_V5_TYPE_3:
|
||||
length = rhuff_bits_read(&bits, (int)lengthbits);
|
||||
crc = rhuff_bits_read(&bits, 16);
|
||||
curoffset += length;
|
||||
break;
|
||||
|
||||
case RCHD_V5_NONE:
|
||||
length = chd->info.hunk_bytes;
|
||||
crc = rhuff_bits_read(&bits, 16);
|
||||
curoffset += length;
|
||||
break;
|
||||
|
||||
case RCHD_V5_SELF:
|
||||
offset = rhuff_bits_read(&bits, (int)selfbits);
|
||||
last_self = offset;
|
||||
break;
|
||||
|
||||
case RCHD_V5_PARENT:
|
||||
offset = rhuff_bits_read(&bits, (int)parentbits);
|
||||
last_parent = offset;
|
||||
break;
|
||||
|
||||
case RCHD_V5_SELF_1:
|
||||
last_self++;
|
||||
/* falls through */
|
||||
case RCHD_V5_SELF_0:
|
||||
code = RCHD_V5_SELF;
|
||||
offset = last_self;
|
||||
break;
|
||||
|
||||
case RCHD_V5_PARENT_SELF:
|
||||
code = RCHD_V5_PARENT;
|
||||
offset = ((uint64_t)n * chd->info.hunk_bytes)
|
||||
/ chd->info.unit_bytes;
|
||||
last_parent = offset;
|
||||
break;
|
||||
|
||||
case RCHD_V5_PARENT_1:
|
||||
last_parent += chd->info.hunk_bytes / chd->info.unit_bytes;
|
||||
/* falls through */
|
||||
case RCHD_V5_PARENT_0:
|
||||
code = RCHD_V5_PARENT;
|
||||
offset = last_parent;
|
||||
break;
|
||||
|
||||
default:
|
||||
goto done;
|
||||
}
|
||||
|
||||
chd->map[n].offset = offset;
|
||||
chd->map[n].length = length;
|
||||
chd->map[n].crc = crc;
|
||||
chd->map[n].type = (uint8_t)code;
|
||||
|
||||
/* The CRC is taken over a packed form that exists only to be
|
||||
* hashed, so it is assembled here rather than kept. */
|
||||
slot[0] = (uint8_t)code;
|
||||
slot[1] = (uint8_t)(length >> 16);
|
||||
slot[2] = (uint8_t)(length >> 8);
|
||||
slot[3] = (uint8_t)length;
|
||||
slot[4] = (uint8_t)(offset >> 40);
|
||||
slot[5] = (uint8_t)(offset >> 32);
|
||||
slot[6] = (uint8_t)(offset >> 24);
|
||||
slot[7] = (uint8_t)(offset >> 16);
|
||||
slot[8] = (uint8_t)(offset >> 8);
|
||||
slot[9] = (uint8_t)offset;
|
||||
slot[10] = (uint8_t)(crc >> 8);
|
||||
slot[11] = (uint8_t)crc;
|
||||
}
|
||||
|
||||
if (rhuff_bits_overflow(&bits))
|
||||
goto done;
|
||||
|
||||
if (rchd_crc16(checkbuf, (size_t)chd->info.hunk_count * 12) != stored_crc)
|
||||
{
|
||||
err = RCHD_ERROR_CRC;
|
||||
goto done;
|
||||
}
|
||||
|
||||
err = RCHD_OK;
|
||||
|
||||
done:
|
||||
free(checkbuf);
|
||||
free(codes);
|
||||
return err;
|
||||
}
|
||||
|
||||
/* -------- metadata --------
|
||||
*
|
||||
* The chain is walked and its payloads kept at open. It is small, and
|
||||
* walking it lazily would turn every track query into a round trip
|
||||
* (FORMAT.md 3.1). */
|
||||
|
||||
static int rchd_meta_collect(rchd_t *chd)
|
||||
{
|
||||
const uint8_t *p = chd->pending;
|
||||
uint32_t len = (uint32_t)rchd_rd_be(p + 5, 3);
|
||||
uint8_t *grown;
|
||||
size_t base;
|
||||
|
||||
if (chd->meta_count >= RCHD_MAX_META_ENTRIES)
|
||||
return RCHD_ERROR_DATA;
|
||||
if ((uint64_t)chd->metadata_size + len > RCHD_MAX_METADATA)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
base = chd->metadata_size;
|
||||
|
||||
if (!(grown = (uint8_t*)realloc(chd->metadata, base + len)))
|
||||
return RCHD_ERROR_MEM;
|
||||
chd->metadata = grown;
|
||||
memcpy(chd->metadata + base, p + 16, len);
|
||||
chd->metadata_size = base + len;
|
||||
|
||||
{
|
||||
rchd_metadata_t *m = (rchd_metadata_t*)realloc(chd->meta,
|
||||
(chd->meta_count + 1) * sizeof(*m));
|
||||
if (!m)
|
||||
return RCHD_ERROR_MEM;
|
||||
chd->meta = m;
|
||||
m[chd->meta_count].tag = rchd_rd32(p);
|
||||
m[chd->meta_count].flags = p[4];
|
||||
m[chd->meta_count].length = len;
|
||||
/* Recorded as an offset and resolved afterwards: the block moves
|
||||
* whenever it grows. */
|
||||
m[chd->meta_count].data = (const uint8_t*)base;
|
||||
chd->meta_count++;
|
||||
}
|
||||
|
||||
chd->meta_offset = rchd_rd_be(p + 8, 8);
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
static void rchd_meta_resolve(rchd_t *chd)
|
||||
{
|
||||
uint32_t i;
|
||||
|
||||
for (i = 0; i < chd->meta_count; i++)
|
||||
chd->meta[i].data = chd->metadata + (size_t)chd->meta[i].data;
|
||||
}
|
||||
|
||||
/* Before version 5 nothing records a unit size, so it is taken from
|
||||
* metadata that names a sector size and left at the hunk size when none
|
||||
* does (FORMAT.md 1.2). */
|
||||
static void rchd_infer_unit_bytes(rchd_t *chd)
|
||||
{
|
||||
uint32_t i;
|
||||
|
||||
if (chd->info.version >= 5)
|
||||
return;
|
||||
|
||||
for (i = 0; i < chd->meta_count; i++)
|
||||
{
|
||||
const char *s = (const char*)chd->meta[i].data;
|
||||
const char *k;
|
||||
uint32_t len = chd->meta[i].length;
|
||||
|
||||
if (!s || len < 5)
|
||||
continue;
|
||||
for (k = s; k + 4 < s + len; k++)
|
||||
{
|
||||
if (k[0] == 'B' && k[1] == 'P' && k[2] == 'S' && k[3] == ':')
|
||||
{
|
||||
uint32_t v = 0;
|
||||
const char *d = k + 4;
|
||||
while (d < s + len && *d >= '0' && *d <= '9')
|
||||
v = v * 10 + (uint32_t)(*d++ - '0');
|
||||
if (v)
|
||||
chd->info.unit_bytes = v;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* -------- open -------- */
|
||||
|
||||
rchd_t *rchd_new(void)
|
||||
{
|
||||
return (rchd_t*)calloc(1, sizeof(rchd_t));
|
||||
}
|
||||
|
||||
void rchd_free(rchd_t *chd)
|
||||
{
|
||||
if (!chd)
|
||||
return;
|
||||
free(chd->map);
|
||||
free(chd->pending);
|
||||
free(chd->metadata);
|
||||
free(chd->meta);
|
||||
free(chd->codecs);
|
||||
free(chd);
|
||||
}
|
||||
|
||||
static size_t rchd_map_raw_bytes(const rchd_t *chd)
|
||||
{
|
||||
/* One entry beyond the map: the end-of-list cookie. */
|
||||
if (chd->info.version <= 2)
|
||||
return (size_t)(chd->info.hunk_count + 1) * 8;
|
||||
if (chd->info.version <= 4)
|
||||
return (size_t)(chd->info.hunk_count + 1) * 16;
|
||||
if (chd->info.compressors[0] == RCHD_CODEC_NONE)
|
||||
return (size_t)chd->info.hunk_count * 4;
|
||||
return 0; /* compressed: the length is in the map header */
|
||||
}
|
||||
|
||||
int rchd_open_step(rchd_t *chd, rchd_request_t *req)
|
||||
{
|
||||
int ready;
|
||||
int err;
|
||||
|
||||
if (!chd || !req)
|
||||
return RCHD_ERROR_PARAM;
|
||||
|
||||
switch (chd->state)
|
||||
{
|
||||
case RCHD_OPEN_HEADER:
|
||||
if ((ready = rchd_want(chd, 0, 124, RCHD_SOURCE_SELF, req)) < 0)
|
||||
return RCHD_ERROR_MEM;
|
||||
if (!ready)
|
||||
return RCHD_PENDING;
|
||||
if ((err = rchd_parse_header(chd, chd->pending)) != RCHD_OK)
|
||||
return err;
|
||||
|
||||
if (!(chd->map = (rchd_map_entry_t*)calloc(chd->info.hunk_count,
|
||||
sizeof(rchd_map_entry_t))))
|
||||
return RCHD_ERROR_MEM;
|
||||
|
||||
chd->state = RCHD_OPEN_MAP;
|
||||
/* falls through */
|
||||
|
||||
case RCHD_OPEN_MAP:
|
||||
{
|
||||
size_t want = rchd_map_raw_bytes(chd);
|
||||
|
||||
if (want == 0)
|
||||
{
|
||||
/* A compressed map states its own length, so the sixteen
|
||||
* byte header is collected before the body it describes. */
|
||||
if (chd->map_length == 0)
|
||||
{
|
||||
if ((ready = rchd_want(chd, chd->map_offset, 16,
|
||||
RCHD_SOURCE_SELF, req)) < 0)
|
||||
return RCHD_ERROR_MEM;
|
||||
if (!ready)
|
||||
return RCHD_PENDING;
|
||||
chd->map_length = rchd_rd32(chd->pending);
|
||||
if (chd->map_length > RCHD_MAX_MAP_BYTES)
|
||||
return RCHD_ERROR_DATA;
|
||||
}
|
||||
want = chd->map_length + 16;
|
||||
}
|
||||
else if (want > RCHD_MAX_MAP_BYTES)
|
||||
return RCHD_ERROR_DATA;
|
||||
|
||||
if ((ready = rchd_want(chd, chd->map_offset, want,
|
||||
RCHD_SOURCE_SELF, req)) < 0)
|
||||
return RCHD_ERROR_MEM;
|
||||
if (!ready)
|
||||
return RCHD_PENDING;
|
||||
|
||||
/* A version 1 to 4 map is followed by one entry-sized slot
|
||||
* holding a cookie. Nothing points at it; it is found by
|
||||
* computing where the entries end. One reader of this format
|
||||
* requires it and the other never looks, so a missing one is
|
||||
* tolerated rather than rejected (FORMAT.md 2.4). */
|
||||
if (chd->info.version <= 4)
|
||||
{
|
||||
size_t esz = (chd->info.version <= 2) ? 8 : 16;
|
||||
size_t seen = (size_t)chd->info.hunk_count * esz;
|
||||
|
||||
if (want >= seen + esz
|
||||
&& memcmp(chd->pending + seen, rchd_end_of_list, esz) != 0)
|
||||
return RCHD_ERROR_DATA;
|
||||
}
|
||||
|
||||
if (chd->info.version <= 2)
|
||||
err = rchd_map_v1v2(chd, chd->pending);
|
||||
else if (chd->info.version <= 4)
|
||||
err = rchd_map_v3v4(chd, chd->pending);
|
||||
else if (chd->info.compressors[0] == RCHD_CODEC_NONE)
|
||||
err = rchd_map_v5_raw(chd, chd->pending);
|
||||
else
|
||||
err = rchd_map_v5(chd, chd->pending, want);
|
||||
|
||||
if (err != RCHD_OK)
|
||||
return err;
|
||||
|
||||
chd->state = RCHD_OPEN_META;
|
||||
}
|
||||
/* falls through */
|
||||
|
||||
case RCHD_OPEN_META:
|
||||
while (chd->meta_offset != 0)
|
||||
{
|
||||
if (!chd->meta_have_len)
|
||||
{
|
||||
if ((ready = rchd_want(chd, chd->meta_offset, 16,
|
||||
RCHD_SOURCE_SELF, req)) < 0)
|
||||
return RCHD_ERROR_MEM;
|
||||
if (!ready)
|
||||
return RCHD_PENDING;
|
||||
|
||||
chd->meta_len = (uint32_t)rchd_rd_be(chd->pending + 5, 3);
|
||||
if (chd->meta_len > RCHD_MAX_METADATA)
|
||||
return RCHD_ERROR_DATA;
|
||||
chd->meta_have_len = 1;
|
||||
}
|
||||
|
||||
if ((ready = rchd_want(chd, chd->meta_offset,
|
||||
16 + chd->meta_len, RCHD_SOURCE_SELF, req)) < 0)
|
||||
return RCHD_ERROR_MEM;
|
||||
if (!ready)
|
||||
return RCHD_PENDING;
|
||||
|
||||
if ((err = rchd_meta_collect(chd)) != RCHD_OK)
|
||||
return err;
|
||||
chd->meta_have_len = 0;
|
||||
}
|
||||
|
||||
rchd_meta_resolve(chd);
|
||||
rchd_infer_unit_bytes(chd);
|
||||
free(chd->pending);
|
||||
chd->pending = NULL;
|
||||
chd->pending_size = 0;
|
||||
chd->pending_have = 0;
|
||||
chd->state = RCHD_OPEN_DONE;
|
||||
/* falls through */
|
||||
|
||||
case RCHD_OPEN_DONE:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
const rchd_info_t *rchd_info(const rchd_t *chd)
|
||||
{
|
||||
if (!chd || chd->state != RCHD_OPEN_DONE)
|
||||
return NULL;
|
||||
return &chd->info;
|
||||
}
|
||||
|
||||
uint32_t rchd_metadata_count(const rchd_t *chd)
|
||||
{
|
||||
return chd ? chd->meta_count : 0;
|
||||
}
|
||||
|
||||
const rchd_metadata_t *rchd_metadata(const rchd_t *chd, uint32_t index)
|
||||
{
|
||||
if (!chd || index >= chd->meta_count)
|
||||
return NULL;
|
||||
return &chd->meta[index];
|
||||
}
|
||||
|
||||
const rchd_metadata_t *rchd_metadata_find(const rchd_t *chd,
|
||||
uint32_t tag, uint32_t n)
|
||||
{
|
||||
uint32_t i;
|
||||
|
||||
if (!chd)
|
||||
return NULL;
|
||||
for (i = 0; i < chd->meta_count; i++)
|
||||
{
|
||||
if (chd->meta[i].tag != tag)
|
||||
continue;
|
||||
if (n-- == 0)
|
||||
return &chd->meta[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int rchd_hunk_location(const rchd_t *chd, uint32_t hunk, rchd_request_t *req)
|
||||
{
|
||||
if (!chd || !req || hunk >= chd->info.hunk_count)
|
||||
return RCHD_ERROR_PARAM;
|
||||
|
||||
req->source = RCHD_SOURCE_SELF;
|
||||
req->offset = chd->map[hunk].offset;
|
||||
req->length = chd->map[hunk].length;
|
||||
|
||||
/* References and holes occupy no blob of their own. */
|
||||
if (chd->info.version >= 5
|
||||
&& (chd->map[hunk].type == RCHD_V5_SELF
|
||||
|| chd->map[hunk].type == RCHD_V5_PARENT))
|
||||
req->length = 0;
|
||||
|
||||
return RCHD_OK;
|
||||
}
|
||||
|
||||
uint32_t rchd_hunk_for_offset(const rchd_t *chd, uint64_t offset)
|
||||
{
|
||||
if (!chd || !chd->info.hunk_bytes)
|
||||
return 0;
|
||||
return (uint32_t)(offset / chd->info.hunk_bytes);
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
/* Copyright (C) 2010-2026 The RetroArch team
|
||||
*
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* The following license statement only applies to this file (huffman.h).
|
||||
* ---------------------------------------------------------------------------------------
|
||||
*
|
||||
* Permission is hereby granted, free of charge,
|
||||
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* Most-significant-bit-first bit reader and canonical Huffman decoder.
|
||||
*
|
||||
* The decoder is canonical Huffman -- codes derived from code lengths
|
||||
* alone, decoded by a single table lookup on max_bits peeked bits --
|
||||
* but not with the ordering RFC 1951 uses. Here the lowest numeric
|
||||
* codes go to the LONGEST lengths, where DEFLATE gives them to the
|
||||
* shortest. For lengths {1,2,2} this produces 1, 00, 01 rather than
|
||||
* DEFLATE's 0, 10, 11. Both are valid prefix codes and neither can
|
||||
* decode the other's bit stream, so this is a dialect and not merely a
|
||||
* different way of computing the same thing.
|
||||
*
|
||||
* What is not general purpose is how a tree gets *serialised*. The two
|
||||
* reader functions below implement the two encodings used by MAME's CHD
|
||||
* container -- one for the compressed hunk map, one for the 'huff' hunk
|
||||
* codec and for A/V hunks. They are specific to those formats and are
|
||||
* not interchangeable with the code-length encoding used by DEFLATE.
|
||||
*
|
||||
* Nothing here allocates. The lookup table is supplied by the caller,
|
||||
* which lets a small tree live on the stack and a large one live inside
|
||||
* whatever handle already exists.
|
||||
*
|
||||
* All reads are byte-at-a-time, so behaviour does not depend on host
|
||||
* endianness or on the alignment of the buffer being read.
|
||||
*/
|
||||
|
||||
#ifndef _LIBRETRO_ENCODINGS_HUFFMAN_H
|
||||
#define _LIBRETRO_ENCODINGS_HUFFMAN_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include <retro_common_api.h>
|
||||
|
||||
RETRO_BEGIN_DECLS
|
||||
|
||||
#define RHUFF_OK 0
|
||||
#define RHUFF_ERROR_DATA (-1)
|
||||
#define RHUFF_ERROR_PARAM (-2)
|
||||
|
||||
/* Ceilings. The map tree is 16 codes at 8 bits; the 'huff' codec and the
|
||||
* A/V codec use 256 codes at 16 bits. A code index and its length share
|
||||
* one uint16_t lookup entry, five bits for the length and the rest for
|
||||
* the index, which is what caps the code count at 2048. */
|
||||
#define RHUFF_MAX_CODES 2048
|
||||
#define RHUFF_MAX_BITS 16
|
||||
|
||||
/* Entries a lookup table must hold for a given max_bits. */
|
||||
#define RHUFF_LOOKUP_ENTRIES(max_bits) ((size_t)1 << (max_bits))
|
||||
|
||||
/**
|
||||
* rhuff_bits_t:
|
||||
*
|
||||
* Reads bits from a caller-owned buffer, most significant bit first.
|
||||
* Reading past the end yields zero bits and latches the overflow flag
|
||||
* rather than failing, so a caller can decode optimistically and test
|
||||
* once at the end.
|
||||
*/
|
||||
typedef struct rhuff_bits
|
||||
{
|
||||
const uint8_t *data;
|
||||
size_t size;
|
||||
size_t offset; /* next byte to pull into the cache */
|
||||
uint64_t used; /* bits consumed, for overflow tests */
|
||||
uint32_t cache; /* left-aligned; bit 31 is next out */
|
||||
int bits; /* valid bits held in cache */
|
||||
int overflow;
|
||||
} rhuff_bits_t;
|
||||
|
||||
/**
|
||||
* rhuff_bits_init:
|
||||
* @b : reader, supplied by the caller
|
||||
* @data : buffer to read from, must outlive @b
|
||||
* @size : bytes readable at @data
|
||||
*/
|
||||
void rhuff_bits_init(rhuff_bits_t *b, const uint8_t *data, size_t size);
|
||||
|
||||
/**
|
||||
* rhuff_bits_peek:
|
||||
* @b : reader
|
||||
* @numbits : how many bits to look at, 0 to 24
|
||||
*
|
||||
* Returns: the next @numbits bits, most significant first, right
|
||||
* aligned. Zero when @numbits is zero. Does not consume.
|
||||
*/
|
||||
uint32_t rhuff_bits_peek(rhuff_bits_t *b, int numbits);
|
||||
|
||||
/**
|
||||
* rhuff_bits_remove:
|
||||
*
|
||||
* Consumes @numbits bits previously peeked.
|
||||
*/
|
||||
void rhuff_bits_remove(rhuff_bits_t *b, int numbits);
|
||||
|
||||
/**
|
||||
* rhuff_bits_read:
|
||||
*
|
||||
* Returns: rhuff_bits_peek(), having consumed the bits.
|
||||
*/
|
||||
uint32_t rhuff_bits_read(rhuff_bits_t *b, int numbits);
|
||||
|
||||
/**
|
||||
* rhuff_bits_flush:
|
||||
* @b : reader
|
||||
*
|
||||
* Discards the partially consumed byte and any whole bytes still held
|
||||
* in the cache, leaving the reader positioned on the next byte
|
||||
* boundary. Used where a bit stream is followed by byte-aligned data.
|
||||
*
|
||||
* Returns: the byte offset the reader is now positioned at.
|
||||
*/
|
||||
size_t rhuff_bits_flush(rhuff_bits_t *b);
|
||||
|
||||
/**
|
||||
* rhuff_bits_overflow:
|
||||
*
|
||||
* Returns: nonzero if any read ran past the end of the buffer.
|
||||
*/
|
||||
int rhuff_bits_overflow(const rhuff_bits_t *b);
|
||||
|
||||
/**
|
||||
* rhuff_dec_t:
|
||||
*
|
||||
* A decoder bound to a caller-owned lookup table. Holding the lengths
|
||||
* inline costs RHUFF_MAX_CODES bytes and keeps the struct copyable.
|
||||
*/
|
||||
typedef struct rhuff_dec
|
||||
{
|
||||
uint16_t *lookup;
|
||||
size_t lookup_entries;
|
||||
uint32_t num_codes;
|
||||
uint32_t max_bits;
|
||||
uint8_t lengths[RHUFF_MAX_CODES];
|
||||
} rhuff_dec_t;
|
||||
|
||||
/**
|
||||
* rhuff_dec_init:
|
||||
* @d : decoder, supplied by the caller
|
||||
* @num_codes : alphabet size, up to RHUFF_MAX_CODES
|
||||
* @max_bits : longest permitted code, up to RHUFF_MAX_BITS
|
||||
* @lookup : table of at least RHUFF_LOOKUP_ENTRIES(@max_bits)
|
||||
* @lookup_entries : entries available at @lookup
|
||||
*
|
||||
* Binds the table and clears the tree. No allocation is performed and
|
||||
* @lookup is not written until a tree is built.
|
||||
*
|
||||
* Returns: RHUFF_OK, or RHUFF_ERROR_PARAM.
|
||||
*/
|
||||
int rhuff_dec_init(rhuff_dec_t *d, uint32_t num_codes, uint32_t max_bits,
|
||||
uint16_t *lookup, size_t lookup_entries);
|
||||
|
||||
/**
|
||||
* rhuff_dec_build:
|
||||
* @d : initialised decoder, with @d->lengths already populated
|
||||
*
|
||||
* Assigns canonical codes from the code lengths and fills the lookup
|
||||
* table. Rejects length sets that do not describe a complete tree.
|
||||
*
|
||||
* Returns: RHUFF_OK, or RHUFF_ERROR_DATA if the lengths are not a valid
|
||||
* canonical tree.
|
||||
*/
|
||||
int rhuff_dec_build(rhuff_dec_t *d);
|
||||
|
||||
/**
|
||||
* rhuff_dec_decode_one:
|
||||
* @d : decoder with a tree built
|
||||
* @b : reader positioned on a code
|
||||
*
|
||||
* Returns: the decoded symbol, having consumed its bits. A symbol
|
||||
* decoded past the end of @b is meaningless; test
|
||||
* rhuff_bits_overflow() once decoding finishes.
|
||||
*/
|
||||
uint32_t rhuff_dec_decode_one(rhuff_dec_t *d, rhuff_bits_t *b);
|
||||
|
||||
/**
|
||||
* rhuff_read_tree_rle:
|
||||
* @d : initialised decoder
|
||||
* @b : reader positioned on a serialised tree
|
||||
*
|
||||
* Reads the code lengths of a tree stored as fixed-width values with an
|
||||
* escape for runs, then builds it. This is the encoding used for the
|
||||
* compressed hunk map of a CHD.
|
||||
*
|
||||
* Returns: RHUFF_OK, or a negative RHUFF_ERROR_* code.
|
||||
*/
|
||||
int rhuff_read_tree_rle(rhuff_dec_t *d, rhuff_bits_t *b);
|
||||
|
||||
/**
|
||||
* rhuff_read_tree_packed:
|
||||
* @d : initialised decoder
|
||||
* @b : reader positioned on a serialised tree
|
||||
*
|
||||
* Reads the code lengths of a tree that are themselves Huffman coded by
|
||||
* a small preceding tree, then builds it. This is the encoding used by
|
||||
* the 'huff' hunk codec and by A/V hunks.
|
||||
*
|
||||
* Returns: RHUFF_OK, or a negative RHUFF_ERROR_* code.
|
||||
*/
|
||||
int rhuff_read_tree_packed(rhuff_dec_t *d, rhuff_bits_t *b);
|
||||
|
||||
/**
|
||||
* rhuff_decode_block:
|
||||
* @d : initialised decoder
|
||||
* @b : reader positioned on a serialised tree
|
||||
* @dst : output buffer
|
||||
* @dst_len : bytes to produce
|
||||
*
|
||||
* Reads a packed tree and then decodes exactly @dst_len symbols with
|
||||
* it, one byte each. This is the whole of the 'huff' hunk codec.
|
||||
*
|
||||
* Returns: RHUFF_OK, or a negative RHUFF_ERROR_* code.
|
||||
*/
|
||||
int rhuff_decode_block(rhuff_dec_t *d, rhuff_bits_t *b,
|
||||
uint8_t *dst, size_t dst_len);
|
||||
|
||||
RETRO_END_DECLS
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,626 @@
|
||||
/* Copyright (C) 2010-2026 The RetroArch team
|
||||
*
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* The following license statement only applies to this file (rchd.h).
|
||||
* ---------------------------------------------------------------------------------------
|
||||
*
|
||||
* Permission is hereby granted, free of charge,
|
||||
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* Clean-room reader for MAME's CHD (Compressed Hunks of Data) container,
|
||||
* versions 1 through 5.
|
||||
*
|
||||
* This decoder performs no file I/O of its own and owns no file handles.
|
||||
* Every byte it needs is described to the caller as an (offset, length,
|
||||
* source) request, which the caller satisfies however it likes: blocking
|
||||
* reads, memory maps, an already-resident buffer, or a few kilobytes per
|
||||
* frame from a task. The decoder never stalls waiting for anything.
|
||||
*
|
||||
* The operating pattern is the same for every operation:
|
||||
*
|
||||
* int err;
|
||||
* rchd_request_t req;
|
||||
*
|
||||
* for (;;)
|
||||
* {
|
||||
* err = rchd_open_step(chd, &req);
|
||||
* if (err != RCHD_PENDING)
|
||||
* break;
|
||||
* n = pull_bytes(req.source, req.offset, buf, req.length);
|
||||
* rchd_feed(chd, buf, n);
|
||||
* }
|
||||
*
|
||||
* A request is always for a contiguous range and is always satisfiable in
|
||||
* one read; the decoder reissues it if the caller supplies less than the
|
||||
* full length, so short reads are legal and cost only another round trip.
|
||||
*
|
||||
* That pattern is strictly ping-pong, which is fine for opening an image
|
||||
* but leaves the storage idle while a hunk decompresses. A read can
|
||||
* instead be driven with several fetches in flight: ask the read what it
|
||||
* still needs, issue all of it at once, and hand the results back as
|
||||
* they land, in whatever order they land in.
|
||||
*
|
||||
* rchd_set_pipeline_depth(chd, 4);
|
||||
* rchd_read_begin(chd, offset, dst, len);
|
||||
* for (;;)
|
||||
* {
|
||||
* n = rchd_read_pending(chd, reqs, 4);
|
||||
* for (i = 0; i < n; i++)
|
||||
* issue_async_read(&reqs[i]);
|
||||
* while (completion_available())
|
||||
* {
|
||||
* c = next_completion();
|
||||
* rchd_feed_at(chd, c->offset, c->source, c->data, c->len);
|
||||
* }
|
||||
* if (rchd_read_step(chd, &req) != RCHD_PENDING)
|
||||
* break;
|
||||
* }
|
||||
*
|
||||
* One handle decodes one hunk at a time; that bound is what keeps a
|
||||
* single rchd_read_step() call cheap. Decoding several hunks in
|
||||
* parallel means one handle per thread, which is inexpensive because
|
||||
* the map is the only large allocation and parent images are shared by
|
||||
* reference rather than copied.
|
||||
*
|
||||
* Codec support is drawn from the primitives already present in
|
||||
* libretro-common: <encodings/deflate.h> for the 'zlib' family, <7z/r7z_lzma.h>
|
||||
* for the 'lzma' family, <formats/rflac.h> for the 'flac' family, and
|
||||
* <encodings/mamehuff.h> for 'huff'. Codecs this build was not compiled with,
|
||||
* and codec tags that postdate it, can be supplied by the caller through
|
||||
* rchd_register_codec().
|
||||
*/
|
||||
|
||||
#ifndef __LIBRETRO_SDK_FORMAT_RCHD_H
|
||||
#define __LIBRETRO_SDK_FORMAT_RCHD_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <retro_common_api.h>
|
||||
|
||||
RETRO_BEGIN_DECLS
|
||||
|
||||
/* Return codes. RCHD_PENDING is not an error: it means the decoder has
|
||||
* described a byte range it needs and will resume once it is fed. */
|
||||
#define RCHD_OK 0
|
||||
#define RCHD_PENDING 1
|
||||
#define RCHD_ERROR_DATA (-1)
|
||||
#define RCHD_ERROR_PARAM (-2)
|
||||
#define RCHD_ERROR_MEM (-3)
|
||||
#define RCHD_ERROR_UNSUPPORTED (-4)
|
||||
#define RCHD_ERROR_CRC (-5)
|
||||
#define RCHD_ERROR_NO_PARENT (-6)
|
||||
#define RCHD_ERROR_STATE (-7)
|
||||
|
||||
/* Which file a request refers to. Images with no parent never produce
|
||||
* RCHD_SOURCE_PARENT; callers that cannot supply a parent may return
|
||||
* RCHD_ERROR_NO_PARENT from their fetch and abandon the operation. */
|
||||
#define RCHD_SOURCE_SELF 0
|
||||
#define RCHD_SOURCE_PARENT 1
|
||||
|
||||
/* Codec tags, in the four-character form the container stores them in.
|
||||
* The set is open: unrecognised tags are not an error at open time, only
|
||||
* at the point a hunk actually needs one. */
|
||||
#define RCHD_CODEC_NONE 0
|
||||
#define RCHD_CODEC_ZLIB 0x7A6C6962U /* 'zlib' */
|
||||
#define RCHD_CODEC_ZSTD 0x7A737464U /* 'zstd' */
|
||||
#define RCHD_CODEC_LZMA 0x6C7A6D61U /* 'lzma' */
|
||||
#define RCHD_CODEC_HUFFMAN 0x68756666U /* 'huff' */
|
||||
#define RCHD_CODEC_FLAC 0x666C6163U /* 'flac' */
|
||||
#define RCHD_CODEC_CD_ZLIB 0x63647A6CU /* 'cdzl' */
|
||||
#define RCHD_CODEC_CD_ZSTD 0x63647A73U /* 'cdzs' */
|
||||
#define RCHD_CODEC_CD_LZMA 0x63646C7AU /* 'cdlz' */
|
||||
#define RCHD_CODEC_CD_FLAC 0x6364666CU /* 'cdfl' */
|
||||
#define RCHD_CODEC_AVHUFF 0x61766875U /* 'avhu' */
|
||||
|
||||
/* Track types, as recorded in CD/GD metadata. */
|
||||
#define RCHD_TRACK_MODE1 0
|
||||
#define RCHD_TRACK_MODE1_RAW 1
|
||||
#define RCHD_TRACK_MODE2 2
|
||||
#define RCHD_TRACK_MODE2_FORM1 3
|
||||
#define RCHD_TRACK_MODE2_FORM2 4
|
||||
#define RCHD_TRACK_MODE2_FORM_MIX 5
|
||||
#define RCHD_TRACK_MODE2_RAW 6
|
||||
#define RCHD_TRACK_AUDIO 7
|
||||
|
||||
/* Subchannel types. */
|
||||
#define RCHD_SUB_NONE 0
|
||||
#define RCHD_SUB_RAW 1
|
||||
#define RCHD_SUB_COOKED 2
|
||||
|
||||
typedef struct rchd rchd_t;
|
||||
|
||||
/**
|
||||
* rchd_request_t:
|
||||
*
|
||||
* A contiguous byte range the decoder needs in order to continue. Valid
|
||||
* only until the next call on the same rchd_t.
|
||||
*/
|
||||
typedef struct rchd_request
|
||||
{
|
||||
uint64_t offset; /* absolute offset within the source file */
|
||||
uint32_t length; /* bytes wanted; a short supply is legal */
|
||||
int source; /* RCHD_SOURCE_SELF or RCHD_SOURCE_PARENT */
|
||||
} rchd_request_t;
|
||||
|
||||
/**
|
||||
* rchd_info_t:
|
||||
*
|
||||
* Container-level geometry, valid once rchd_open_step() has returned
|
||||
* RCHD_OK. logical_bytes is the size of the decompressed image; every
|
||||
* offset accepted by rchd_read_begin() is relative to it.
|
||||
*/
|
||||
typedef struct rchd_info
|
||||
{
|
||||
uint64_t logical_bytes;
|
||||
uint32_t version; /* 1 through 5 */
|
||||
uint32_t hunk_bytes;
|
||||
uint32_t unit_bytes;
|
||||
uint32_t hunk_count;
|
||||
uint32_t compressors[4]; /* RCHD_CODEC_*; V5 only, else derived */
|
||||
uint8_t sha1[20]; /* raw+meta; zeroed before V3 */
|
||||
uint8_t raw_sha1[20]; /* raw only; zeroed before V4 */
|
||||
uint8_t parent_sha1[20];
|
||||
int has_parent;
|
||||
} rchd_info_t;
|
||||
|
||||
/**
|
||||
* rchd_metadata_t:
|
||||
*
|
||||
* One entry from the metadata chain. The payload is cached during open
|
||||
* (the chain is small and walking it lazily would make every track query
|
||||
* a round trip), so @data is directly readable and owned by the rchd_t.
|
||||
*/
|
||||
typedef struct rchd_metadata
|
||||
{
|
||||
uint32_t tag;
|
||||
uint8_t flags;
|
||||
uint32_t length;
|
||||
const uint8_t *data;
|
||||
} rchd_metadata_t;
|
||||
|
||||
/**
|
||||
* rchd_track_t:
|
||||
*
|
||||
* One track of a CD or GD image, as reconstructed from the metadata
|
||||
* chain. Tracks are reported in disc order with pregap, postgap and
|
||||
* inter-track padding already resolved, so a mixed-mode disc (a data
|
||||
* track followed by audio tracks, each with its own sector size)
|
||||
* enumerates correctly rather than collapsing to a single geometry.
|
||||
*
|
||||
* @logical_offset is where the track's first frame begins in the CHD
|
||||
* logical stream; @lba is where it begins on the virtual disc.
|
||||
*/
|
||||
typedef struct rchd_track
|
||||
{
|
||||
uint64_t logical_offset;
|
||||
uint32_t track; /* 1-based */
|
||||
uint32_t type; /* RCHD_TRACK_* */
|
||||
uint32_t subtype; /* RCHD_SUB_* */
|
||||
uint32_t data_size; /* bytes of sector data per frame */
|
||||
uint32_t sub_size; /* bytes of subchannel data per frame */
|
||||
uint32_t frames; /* frames of real content */
|
||||
uint32_t pad_frames; /* padding to the 4-frame boundary */
|
||||
uint32_t pregap;
|
||||
uint32_t postgap;
|
||||
uint32_t lba;
|
||||
} rchd_track_t;
|
||||
|
||||
/**
|
||||
* rchd_codec_decode_t:
|
||||
*
|
||||
* Caller-supplied decompressor for a codec tag this build does not
|
||||
* implement. Must produce exactly @dst_len bytes and return RCHD_OK, or
|
||||
* a negative RCHD_ERROR_* code. Called synchronously from within
|
||||
* rchd_read_step(); it must not block on I/O.
|
||||
*/
|
||||
typedef int (*rchd_codec_decode_t)(void *ctx,
|
||||
const uint8_t *src, uint32_t src_len,
|
||||
uint8_t *dst, uint32_t dst_len);
|
||||
|
||||
/**
|
||||
* rchd_new:
|
||||
*
|
||||
* Allocates a decoder. No I/O is described until rchd_open_step() is
|
||||
* called, so registration hooks can be installed first.
|
||||
*
|
||||
* Returns: a decoder, or NULL on allocation failure.
|
||||
*/
|
||||
rchd_t *rchd_new(void);
|
||||
|
||||
void rchd_free(rchd_t *chd);
|
||||
|
||||
/**
|
||||
* rchd_register_codec:
|
||||
* @chd : decoder
|
||||
* @tag : four-character codec tag, e.g. RCHD_CODEC_ZSTD
|
||||
* @fn : decompressor, or NULL to remove a registration
|
||||
* @ctx : opaque pointer handed back to @fn
|
||||
*
|
||||
* Overrides or supplies the handler for one codec tag. A registration
|
||||
* takes precedence over the built-in implementation, which makes this
|
||||
* usable both for codecs that postdate this build and for substituting a
|
||||
* faster implementation of one that does not.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_MEM if the table cannot grow.
|
||||
*/
|
||||
int rchd_register_codec(rchd_t *chd, uint32_t tag,
|
||||
rchd_codec_decode_t fn, void *ctx);
|
||||
|
||||
/**
|
||||
* rchd_open_step:
|
||||
* @chd : decoder
|
||||
* @req : receives the next byte range needed, on RCHD_PENDING
|
||||
*
|
||||
* Advances the open sequence: header, map, and metadata chain. Call
|
||||
* repeatedly, feeding each request through rchd_feed(), until this
|
||||
* returns something other than RCHD_PENDING.
|
||||
*
|
||||
* Returns: RCHD_OK when the image is open, RCHD_PENDING when more bytes
|
||||
* are needed, or a negative RCHD_ERROR_* code.
|
||||
*/
|
||||
int rchd_open_step(rchd_t *chd, rchd_request_t *req);
|
||||
|
||||
/**
|
||||
* rchd_feed:
|
||||
* @chd : decoder
|
||||
* @data : bytes read for the outstanding request
|
||||
* @len : how many bytes @data holds
|
||||
*
|
||||
* Supplies some or all of the single outstanding request, which is the
|
||||
* only shape the open sequence produces. Supplying fewer bytes than
|
||||
* were asked for is legal; the remainder is requested again. @data is
|
||||
* consumed immediately and need not outlive the call.
|
||||
*
|
||||
* Reads driven with several fetches in flight use rchd_feed_at()
|
||||
* instead, since order of arrival no longer identifies the request.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_STATE if no request is outstanding,
|
||||
* or if more than one is.
|
||||
*/
|
||||
int rchd_feed(rchd_t *chd, const void *data, size_t len);
|
||||
|
||||
/**
|
||||
* rchd_info:
|
||||
*
|
||||
* Returns: geometry, or NULL if the image is not open yet.
|
||||
*/
|
||||
const rchd_info_t *rchd_info(const rchd_t *chd);
|
||||
|
||||
/**
|
||||
* rchd_parent_sha1_matches:
|
||||
* @chd : opened decoder
|
||||
* @sha1 : 20 bytes from a candidate parent's combined SHA-1
|
||||
*
|
||||
* Returns: nonzero if @sha1 identifies the parent this image differences
|
||||
* against. Callers resolving a parent should check this before feeding
|
||||
* RCHD_SOURCE_PARENT bytes; a mismatched parent decodes to plausible
|
||||
* garbage rather than failing.
|
||||
*/
|
||||
int rchd_parent_sha1_matches(const rchd_t *chd, const uint8_t *sha1);
|
||||
|
||||
/**
|
||||
* rchd_set_parent:
|
||||
* @chd : opened decoder whose image has a parent
|
||||
* @parent : opened decoder for the parent image, or NULL to detach
|
||||
*
|
||||
* Binds the parent image. Hunks that difference against a parent are
|
||||
* then resolved through @parent's own map, so a request carrying
|
||||
* RCHD_SOURCE_PARENT names a real byte range in the parent file rather
|
||||
* than a unit index the caller would have to translate itself.
|
||||
*
|
||||
* @parent must outlive @chd, and must not have a read of its own armed
|
||||
* while @chd is reading. Check rchd_parent_sha1_matches() before
|
||||
* binding: a mismatched parent decodes to plausible garbage instead of
|
||||
* failing.
|
||||
*
|
||||
* Returns: RCHD_OK, RCHD_ERROR_PARAM if @parent is not open, or
|
||||
* RCHD_ERROR_NO_PARENT if @chd's image has no parent.
|
||||
*/
|
||||
int rchd_set_parent(rchd_t *chd, rchd_t *parent);
|
||||
|
||||
/* -------- metadata -------- */
|
||||
|
||||
uint32_t rchd_metadata_count(const rchd_t *chd);
|
||||
|
||||
/**
|
||||
* rchd_metadata:
|
||||
* @chd : opened decoder
|
||||
* @index : entry index, below rchd_metadata_count()
|
||||
*
|
||||
* Returns: the entry, or NULL if @index is out of range.
|
||||
*/
|
||||
const rchd_metadata_t *rchd_metadata(const rchd_t *chd, uint32_t index);
|
||||
|
||||
/**
|
||||
* rchd_metadata_find:
|
||||
* @chd : opened decoder
|
||||
* @tag : four-character metadata tag
|
||||
* @n : which occurrence of @tag, counting from zero
|
||||
*
|
||||
* Returns: the entry, or NULL if there is no such occurrence.
|
||||
*/
|
||||
const rchd_metadata_t *rchd_metadata_find(const rchd_t *chd,
|
||||
uint32_t tag, uint32_t n);
|
||||
|
||||
/* -------- CD / GD track table -------- */
|
||||
|
||||
/**
|
||||
* rchd_track_count:
|
||||
*
|
||||
* Returns: the number of tracks, or 0 for images that carry no CD or GD
|
||||
* metadata (hard disk and A/V images).
|
||||
*/
|
||||
uint32_t rchd_track_count(const rchd_t *chd);
|
||||
|
||||
const rchd_track_t *rchd_track(const rchd_t *chd, uint32_t index);
|
||||
|
||||
/**
|
||||
* rchd_track_for_lba:
|
||||
* @chd : opened decoder
|
||||
* @lba : absolute sector on the virtual disc
|
||||
*
|
||||
* Returns: the track containing @lba, or NULL if @lba is past the end.
|
||||
*/
|
||||
const rchd_track_t *rchd_track_for_lba(const rchd_t *chd, uint32_t lba);
|
||||
|
||||
/**
|
||||
* rchd_total_frames:
|
||||
*
|
||||
* Returns: the disc length in frames, pregaps and postgaps included.
|
||||
*/
|
||||
uint32_t rchd_total_frames(const rchd_t *chd);
|
||||
|
||||
/* -------- reading -------- */
|
||||
|
||||
/**
|
||||
* rchd_read_begin:
|
||||
* @chd : opened decoder
|
||||
* @offset : byte offset into the logical image
|
||||
* @dst : destination, valid until the read completes or is abandoned
|
||||
* @len : bytes to produce
|
||||
*
|
||||
* Arms a read. Nothing is decoded and no I/O is described until
|
||||
* rchd_read_step() is called. Arming a new read abandons any read still
|
||||
* in progress.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_PARAM if the range leaves the image.
|
||||
*/
|
||||
int rchd_read_begin(rchd_t *chd, uint64_t offset, void *dst, size_t len);
|
||||
|
||||
/**
|
||||
* rchd_read_sectors_begin:
|
||||
* @chd : opened decoder
|
||||
* @lba : first absolute sector on the virtual disc
|
||||
* @count : how many sectors
|
||||
* @dst : destination
|
||||
* @len : capacity of @dst
|
||||
* @flags : reserved, pass 0
|
||||
*
|
||||
* Arms a read addressed by sector rather than by byte, crossing track
|
||||
* boundaries as needed. Each sector is emitted at its own track's
|
||||
* @data_size, which is what makes a mixed-mode disc read back correctly;
|
||||
* @len must be large enough for the sizes the range actually spans, and
|
||||
* rchd_read_extent() reports that up front.
|
||||
*
|
||||
* Returns: RCHD_OK, RCHD_ERROR_PARAM if the range leaves the disc, or
|
||||
* RCHD_ERROR_STATE if the image has no track table.
|
||||
*/
|
||||
int rchd_read_sectors_begin(rchd_t *chd, uint32_t lba, uint32_t count,
|
||||
void *dst, size_t len, uint32_t flags);
|
||||
|
||||
/**
|
||||
* rchd_read_extent:
|
||||
* @chd : opened decoder
|
||||
* @lba : first absolute sector
|
||||
* @count : how many sectors
|
||||
* @out : receives the number of bytes such a read would produce
|
||||
*
|
||||
* Returns: RCHD_OK, or a negative RCHD_ERROR_* code.
|
||||
*/
|
||||
int rchd_read_extent(const rchd_t *chd, uint32_t lba, uint32_t count,
|
||||
size_t *out);
|
||||
|
||||
/**
|
||||
* rchd_read_step:
|
||||
* @chd : decoder with a read armed
|
||||
* @req : receives the next byte range needed, on RCHD_PENDING
|
||||
*
|
||||
* Advances the armed read. At most one hunk is decompressed per call, so
|
||||
* the work between returns is bounded by the hunk size regardless of how
|
||||
* large the read is.
|
||||
*
|
||||
* Returns: RCHD_OK when @dst is complete, RCHD_PENDING when more bytes
|
||||
* are needed, or a negative RCHD_ERROR_* code.
|
||||
*/
|
||||
int rchd_read_step(rchd_t *chd, rchd_request_t *req);
|
||||
|
||||
/**
|
||||
* rchd_read_progress:
|
||||
*
|
||||
* Returns: bytes written to @dst so far by the armed read.
|
||||
*/
|
||||
size_t rchd_read_progress(const rchd_t *chd);
|
||||
|
||||
/**
|
||||
* rchd_read_hunk_begin:
|
||||
* @chd : opened decoder
|
||||
* @hunk : hunk index, below rchd_info()->hunk_count
|
||||
* @dst : destination, at least hunk_bytes
|
||||
*
|
||||
* Arms a read of one whole hunk, addressed by index rather than by byte
|
||||
* offset. Equivalent to arming a byte read of hunk_bytes at
|
||||
* hunk * hunk_bytes, and provided because every older reader of this
|
||||
* format is written against hunk indices; a consumer being ported does
|
||||
* not have to convert its call sites to offsets to use this one.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_PARAM if @hunk is out of range.
|
||||
*/
|
||||
int rchd_read_hunk_begin(rchd_t *chd, uint32_t hunk, void *dst);
|
||||
|
||||
/* -------- pipelining -------- */
|
||||
|
||||
/**
|
||||
* rchd_set_pipeline_depth:
|
||||
* @chd : opened decoder
|
||||
* @depth : hunks that may be in flight at once, at least 1
|
||||
*
|
||||
* Sizes the staging ring compressed hunks are held in while they are
|
||||
* being fetched. Depth one is the default and behaves exactly like the
|
||||
* ping-pong pattern. Each further slot costs another hunk_bytes of
|
||||
* memory and lets one more fetch overlap the decode of an earlier hunk,
|
||||
* so this is the dial between memory and how much of the storage
|
||||
* latency gets hidden.
|
||||
*
|
||||
* Returns: RCHD_OK, RCHD_ERROR_PARAM if @depth is zero, or
|
||||
* RCHD_ERROR_MEM.
|
||||
*/
|
||||
int rchd_set_pipeline_depth(rchd_t *chd, uint32_t depth);
|
||||
|
||||
/**
|
||||
* rchd_read_pending:
|
||||
* @chd : decoder with a read armed
|
||||
* @out : receives up to @max requests
|
||||
* @max : capacity of @out
|
||||
*
|
||||
* Reports the ranges the armed read still needs but has not been given,
|
||||
* in the order it will consume them, up to the pipeline depth. The
|
||||
* caller may issue all of them at once.
|
||||
*
|
||||
* This does not advance the read and does not decode anything; a range
|
||||
* reported here is not reported again until the read is re-armed.
|
||||
*
|
||||
* Returns: how many requests were written to @out. Zero means the read
|
||||
* already holds everything it needs and only rchd_read_step() remains.
|
||||
*/
|
||||
uint32_t rchd_read_pending(rchd_t *chd, rchd_request_t *out, uint32_t max);
|
||||
|
||||
/**
|
||||
* rchd_feed_at:
|
||||
* @chd : decoder
|
||||
* @offset : absolute offset the data was read from
|
||||
* @source : RCHD_SOURCE_SELF or RCHD_SOURCE_PARENT
|
||||
* @data : bytes read
|
||||
* @len : how many bytes @data holds
|
||||
*
|
||||
* Supplies bytes for one outstanding request, identified by where they
|
||||
* came from rather than by when they arrive, so fetches issued together
|
||||
* may complete in any order. Partial supply is legal and the remainder
|
||||
* is reported by the next rchd_read_pending().
|
||||
*
|
||||
* @data is copied into the staging ring and need not outlive the call.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_STATE if no outstanding request
|
||||
* covers @offset in @source.
|
||||
*/
|
||||
int rchd_feed_at(rchd_t *chd, uint64_t offset, int source,
|
||||
const void *data, size_t len);
|
||||
|
||||
/**
|
||||
* rchd_feed_borrow:
|
||||
* @chd : decoder
|
||||
* @offset : absolute offset the data sits at
|
||||
* @source : RCHD_SOURCE_SELF or RCHD_SOURCE_PARENT
|
||||
* @data : bytes already resident at @offset
|
||||
* @len : how many bytes @data holds
|
||||
*
|
||||
* Satisfies an outstanding request without copying, for callers that
|
||||
* already hold the range in stable memory -- a fully or partially read
|
||||
* file kept in one growing allocation, or a mapping. This is worth
|
||||
* reaching for: the copy rchd_feed_at() performs is a whole hunk per
|
||||
* hunk, and at the maximum hunk size that is half a megabyte of
|
||||
* memcpy the caller has already paid for once.
|
||||
*
|
||||
* @data must stay valid and unmoved until the next rchd_read_step()
|
||||
* returns something other than RCHD_PENDING. That rules out any buffer
|
||||
* whose pages can be released underneath the decoder while it works,
|
||||
* and it rules out a base pointer that can move: a growing arena may
|
||||
* reallocate, so re-read the base after growing it and borrow from the
|
||||
* current one.
|
||||
*
|
||||
* When the lifetime cannot be guaranteed, rchd_feed_at() always
|
||||
* applies. Mixing the two across different requests is fine.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_STATE if no outstanding request
|
||||
* covers @offset in @source.
|
||||
*/
|
||||
int rchd_feed_borrow(rchd_t *chd, uint64_t offset, int source,
|
||||
const uint8_t *data, size_t len);
|
||||
|
||||
/* -------- prefetch -------- */
|
||||
|
||||
/**
|
||||
* rchd_hunk_location:
|
||||
* @chd : opened decoder
|
||||
* @hunk : hunk index, below rchd_info()->hunk_count
|
||||
* @req : receives the range that hunk's compressed blob occupies
|
||||
*
|
||||
* Describes where a hunk lives without decoding it, so a caller that
|
||||
* knows its access pattern can read ahead. Hunks that carry no blob of
|
||||
* their own -- uncompressed runs, self references, parent references --
|
||||
* report a zero length.
|
||||
*
|
||||
* Returns: RCHD_OK, or a negative RCHD_ERROR_* code.
|
||||
*/
|
||||
int rchd_hunk_location(const rchd_t *chd, uint32_t hunk,
|
||||
rchd_request_t *req);
|
||||
|
||||
/**
|
||||
* rchd_hunk_for_offset:
|
||||
*
|
||||
* Returns: the hunk index containing logical byte @offset.
|
||||
*/
|
||||
uint32_t rchd_hunk_for_offset(const rchd_t *chd, uint64_t offset);
|
||||
|
||||
/* -------- A/V images -------- */
|
||||
|
||||
/**
|
||||
* rchd_av_frame_t:
|
||||
*
|
||||
* Layout of one decoded A/V hunk. Populated by rchd_av_parse(); the
|
||||
* pointers address the hunk buffer the caller passed in.
|
||||
*/
|
||||
typedef struct rchd_av_frame
|
||||
{
|
||||
const uint8_t *meta;
|
||||
const uint8_t *video; /* @height rows of @width pixels, YUY2 */
|
||||
const int16_t *audio[16];
|
||||
uint32_t meta_size;
|
||||
uint32_t width;
|
||||
uint32_t height;
|
||||
uint32_t stride; /* bytes between video rows */
|
||||
uint32_t channels;
|
||||
uint32_t samples;
|
||||
} rchd_av_frame_t;
|
||||
|
||||
/**
|
||||
* rchd_av_parse:
|
||||
* @data : one decoded hunk, as produced by a read of hunk_bytes
|
||||
* @len : length of @data
|
||||
* @out : receives the frame layout
|
||||
*
|
||||
* Interprets a hunk from an image compressed with RCHD_CODEC_AVHUFF. The
|
||||
* hunk is a self-describing container of metadata, interleaved audio
|
||||
* channels and a video field; this only resolves the layout, since the
|
||||
* decompression already happened during the read.
|
||||
*
|
||||
* Returns: RCHD_OK, or RCHD_ERROR_DATA if @data is not an A/V hunk.
|
||||
*/
|
||||
int rchd_av_parse(const uint8_t *data, size_t len, rchd_av_frame_t *out);
|
||||
|
||||
RETRO_END_DECLS
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,44 @@
|
||||
# libretro-common tools
|
||||
|
||||
Programs that exercise the code in this tree against real data, kept
|
||||
here rather than beside it so a build of libretro-common never picks
|
||||
them up.
|
||||
|
||||
The layout mirrors the tree they test: `tools/chd` for
|
||||
`formats/chd`, `tools/flac` for `formats/flac`, `tools/encodings` for
|
||||
`encodings`.
|
||||
|
||||
None of these are part of any build. Each is a standalone program or
|
||||
script, compiled or run by hand against the sources it names.
|
||||
|
||||
## chd
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `chd_probe.py` | Regenerates the reference images `formats/chd/FORMAT.md` is derived from and re-checks every claim marked verified in it. Requires `chdman` on PATH. |
|
||||
| `chd_map_test.c` | Decodes the hunk map of real images and checks it against the CRC-16 the file carries, then feeds corrupted maps through the same path. |
|
||||
| `chd_cd_test.py` | Reconstructs CD hunks — sector and subchannel framing, ECC rebuild — and compares them byte for byte against another reader's decode. |
|
||||
| `rchd_open_test.c` | Opens images through `rchd` and reports the geometry, map and metadata it finds. |
|
||||
|
||||
## flac
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `make_mka.py` | Builds a Matroska file holding a FLAC track, from a header and raw frames. |
|
||||
| `make_ogg.py` | The same for an Ogg FLAC stream (RFC 5334). |
|
||||
| `flacprobe.c` | Wraps a candidate headerless FLAC stream in a STREAMINFO built from the specification and decodes it, which locates FLAC data by content rather than by guessing at a header. |
|
||||
| `rflac_push_test.c` | Decodes a headerless stream at several input chunk sizes, exercising the path where a frame straddles two spans. |
|
||||
| `rflac_header_test.c` | The same for a stream carrying its own header. |
|
||||
| `rflac_mka_test.c` | Drives the Matroska arm and compares against a native-stream reference. |
|
||||
| `audio_transfer_flac_test.c` | Drives `audio_transfer`'s FLAC arm end to end. |
|
||||
|
||||
The two builders exist because nothing in this tree produces a `.mka` or
|
||||
an Ogg FLAC stream, so the container paths could otherwise only be built
|
||||
and not run — and a file recorded elsewhere is not reproducible. The
|
||||
Matroska one found a real defect on first use.
|
||||
|
||||
## encodings
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `huff_test.c` | Decodes every `huff` hunk of an image and compares against the original uncompressed source. |
|
||||
@@ -0,0 +1,104 @@
|
||||
import struct,sys,zlib,lzma,subprocess,os
|
||||
sys.path.insert(0,'/home/claude/chd')
|
||||
from mapdec import Bits, read_tree_rle, decode_one
|
||||
|
||||
f_lut=[0]*256; b_lut=[0]*256
|
||||
for i in range(256): f_lut[i]=((i<<1)^(0x11D if i&0x80 else 0))&0xFF
|
||||
for i in range(256): b_lut[i^f_lut[i]]=i
|
||||
def ecc_block(sec,base,mc,mn,mm,mi,dest):
|
||||
size=mc*mn
|
||||
for major in range(mc):
|
||||
idx=(major>>1)*mm+(major&1); a=b=0
|
||||
for _ in range(mn):
|
||||
t=sec[base+idx]; idx+=mi
|
||||
if idx>=size: idx-=size
|
||||
a^=t; b^=t; a=f_lut[a]
|
||||
a=b_lut[f_lut[a]^b]
|
||||
sec[dest+major]=a; sec[dest+major+mc]=a^b
|
||||
def ecc_generate(sec):
|
||||
sec[0]=0
|
||||
for i in range(1,11): sec[i]=0xFF
|
||||
sec[11]=0
|
||||
hdr=bytes(sec[12:16]); sec[12:16]=b'\0'*4
|
||||
ecc_block(sec,0x0C,86,24,2,86,0x81C)
|
||||
ecc_block(sec,0x0C,52,43,86,88,0x8C8)
|
||||
sec[12:16]=hdr
|
||||
|
||||
def lzma_raw(data, outlen, hunkbytes):
|
||||
d=1<<11
|
||||
while d < hunkbytes: d<<=1
|
||||
filt=[{"id":lzma.FILTER_LZMA1,"lc":3,"lp":0,"pb":2,"dict_size":d}]
|
||||
return lzma.LZMADecompressor(format=lzma.FORMAT_RAW,filters=filt).decompress(data,outlen)
|
||||
|
||||
def load(path):
|
||||
f=open(path,'rb'); hdr=f.read(124)
|
||||
if struct.unpack('>I',hdr[12:16])[0]!=5: return None
|
||||
mo=struct.unpack('>Q',hdr[40:48])[0]
|
||||
codecs=[hdr[16+i*4:20+i*4].decode('latin1') for i in range(4)]
|
||||
hb_,ub=struct.unpack('>II',hdr[56:64])
|
||||
nh=(struct.unpack('>Q',hdr[32:40])[0]+hb_-1)//hb_
|
||||
f.seek(mo); h=f.read(16); ml=struct.unpack('>I',h[0:4])[0]
|
||||
ds=int.from_bytes(h[4:10],'big'); lb=h[12]
|
||||
b=Bits(f.read(ml)); lut,_=read_tree_rle(b,16,8,4,'down')
|
||||
comp=[];last=0;rep=0
|
||||
for _ in range(nh):
|
||||
if rep>0: comp.append(last); rep-=1; continue
|
||||
v=decode_one(b,lut,8)
|
||||
if v==7: comp.append(last); rep=2+decode_one(b,lut,8)
|
||||
elif v==8: comp.append(last); rep=2+16+(decode_one(b,lut,8)<<4)+decode_one(b,lut,8)
|
||||
else: last=v; comp.append(v)
|
||||
cur=ds; blobs=[]
|
||||
hb2=h[13]; pb2=h[14]
|
||||
for n in range(nh):
|
||||
c=comp[n]
|
||||
if c in (0,1,2,3): L=b.read(lb); b.read(16); blobs.append((c,cur,L)); cur+=L
|
||||
elif c==4: b.read(16); blobs.append((c,cur,hb_)); cur+=hb_
|
||||
elif c==5: b.read(hb2); blobs.append((c,None,0)) # self ref: consumes bits
|
||||
elif c==6: b.read(pb2); blobs.append((c,None,0)) # parent ref: consumes bits
|
||||
else: blobs.append((c,None,0)) # pseudo-codes: no bits
|
||||
if cur-ds != mo-ds:
|
||||
raise ValueError("map span %d != expected %d" % (cur-ds, mo-ds))
|
||||
return f,codecs,hb_,ub,blobs
|
||||
|
||||
def decode_cd(f,codec,blob,hb_,ub):
|
||||
frames=hb_//ub; eccn=(frames+7)//8
|
||||
cln = 2 if hb_ < 65536 else 3
|
||||
hh=eccn+cln
|
||||
eccmap=blob[:eccn]
|
||||
complen=int.from_bytes(blob[eccn:eccn+cln],'big')
|
||||
bd=blob[hh:hh+complen]; sd=blob[hh+complen:]
|
||||
if codec=='cdzl': base=zlib.decompressobj(-15).decompress(bd)
|
||||
elif codec=='cdlz': base=lzma_raw(bd,frames*2352,hb_)
|
||||
else: return None
|
||||
sub=zlib.decompressobj(-15).decompress(sd) if sd else b'\0'*(frames*96)
|
||||
out=bytearray()
|
||||
for i in range(frames):
|
||||
sec=bytearray(base[i*2352:(i+1)*2352])
|
||||
if eccmap[i>>3] & (1 << (i&7)): ecc_generate(sec)
|
||||
out+=sec; out+=sub[i*96:(i+1)*96]
|
||||
return bytes(out)
|
||||
|
||||
for p in sorted(os.listdir('corpus')):
|
||||
if not p.endswith('.chd'): continue
|
||||
path='corpus/'+p
|
||||
r0=load(path)
|
||||
if r0 is None: continue
|
||||
f,codecs,hb_,ub,blobs=r0
|
||||
if 'cdzl' not in codecs and 'cdlz' not in codecs: continue
|
||||
tried=matched=skipped=0
|
||||
for n,(c,off,L) in enumerate(blobs):
|
||||
if c is None or off is None or c>3: continue
|
||||
codec=codecs[c].strip('\0')
|
||||
if codec not in ('cdzl','cdlz'): skipped+=1; continue
|
||||
if tried>=40: break
|
||||
f.seek(off); blob=f.read(L)
|
||||
try: mine=decode_cd(f,codec,blob,hb_,ub)
|
||||
except Exception: mine=None
|
||||
r=subprocess.run(['./dumphunk',path,str(n),'/tmp/tr.bin'],capture_output=True)
|
||||
if r.returncode!=0: continue
|
||||
truth=open('/tmp/tr.bin','rb').read()
|
||||
tried+=1
|
||||
if mine==truth: matched+=1
|
||||
print("%-46s codecs=%-22s hunks tried=%-3d byte-exact=%-3d %s"
|
||||
% (p[:46],','.join(c.strip('\0') for c in codecs if c.strip('\0')),tried,matched,
|
||||
"PASS" if tried and tried==matched else "FAIL"))
|
||||
@@ -0,0 +1,481 @@
|
||||
/* Decodes the hunk map of real CHD images through rhuff and checks the
|
||||
* result against the CRC-16 the file carries, which is an oracle the
|
||||
* decoder cannot influence.
|
||||
*
|
||||
* Two independent checks per image: the assembled map must hash to the
|
||||
* stored CRC, and the hunk offsets must partition the data region
|
||||
* exactly, leaving no gap and no overlap between the first blob and the
|
||||
* start of the map. A wrong bit width or a misread escape breaks both.
|
||||
*
|
||||
* The second half feeds deliberately corrupted maps through the same
|
||||
* path. Those must not be decoded correctly -- they must merely fail
|
||||
* without reading out of bounds, which is what the sanitizers check.
|
||||
*
|
||||
* Images come from tools/chd/chd_probe.py.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <encodings/huffman.h>
|
||||
|
||||
/* Reference codes stored in a map entry. */
|
||||
#define MAP_TYPE_0 0
|
||||
#define MAP_TYPE_3 3
|
||||
#define MAP_NONE 4
|
||||
#define MAP_SELF 5
|
||||
#define MAP_PARENT 6
|
||||
#define MAP_RLE_SMALL 7
|
||||
#define MAP_RLE_LARGE 8
|
||||
#define MAP_SELF_0 9
|
||||
#define MAP_SELF_1 10
|
||||
#define MAP_PARENT_SELF 11
|
||||
#define MAP_PARENT_0 12
|
||||
#define MAP_PARENT_1 13
|
||||
|
||||
#define MAP_ENTRY_BYTES 12
|
||||
#define MAP_TREE_CODES 16
|
||||
#define MAP_TREE_BITS 8
|
||||
|
||||
static uint32_t rd32(const uint8_t *p)
|
||||
{
|
||||
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16)
|
||||
| ((uint32_t)p[2] << 8) | (uint32_t)p[3];
|
||||
}
|
||||
|
||||
static uint64_t rd_be(const uint8_t *p, int n)
|
||||
{
|
||||
uint64_t v = 0;
|
||||
int i;
|
||||
for (i = 0; i < n; i++)
|
||||
v = (v << 8) | (uint64_t)p[i];
|
||||
return v;
|
||||
}
|
||||
|
||||
static void wr_be(uint8_t *p, uint64_t v, int n)
|
||||
{
|
||||
int i;
|
||||
for (i = n - 1; i >= 0; i--)
|
||||
{
|
||||
p[i] = (uint8_t)(v & 0xff);
|
||||
v >>= 8;
|
||||
}
|
||||
}
|
||||
|
||||
/* CRC-16/CCITT-FALSE: polynomial 0x1021, initial value 0xffff, no
|
||||
* reflection of input or output, no final xor. */
|
||||
static uint16_t crc16(const uint8_t *data, size_t len)
|
||||
{
|
||||
uint16_t crc = 0xffff;
|
||||
size_t i;
|
||||
int bit;
|
||||
|
||||
for (i = 0; i < len; i++)
|
||||
{
|
||||
crc ^= (uint16_t)((uint16_t)data[i] << 8);
|
||||
for (bit = 0; bit < 8; bit++)
|
||||
crc = (uint16_t)((crc & 0x8000)
|
||||
? ((crc << 1) ^ 0x1021) : (crc << 1));
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
struct image
|
||||
{
|
||||
uint8_t *data;
|
||||
size_t size;
|
||||
uint64_t mapoffset;
|
||||
uint64_t logical;
|
||||
uint32_t hunkbytes;
|
||||
uint32_t unitbytes;
|
||||
uint32_t hunkcount;
|
||||
size_t map_base;
|
||||
};
|
||||
|
||||
/* Loads the header and the map region only. Real images run to
|
||||
* hundreds of megabytes and the hunk data is not needed to check a
|
||||
* map, so holding the whole file would make the sanitizer runs far
|
||||
* heavier than the thing being tested. im->data is indexed by absolute
|
||||
* file offset, with everything below map_base absent, so map_decode
|
||||
* still reads the offsets it would read normally. */
|
||||
static int image_load(struct image *im, const char *path)
|
||||
{
|
||||
FILE *f = fopen(path, "rb");
|
||||
uint8_t hdr[124];
|
||||
uint32_t maplength;
|
||||
size_t want;
|
||||
|
||||
if (!f)
|
||||
return 0;
|
||||
|
||||
if (fread(hdr, 1, sizeof(hdr), f) != sizeof(hdr))
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
if (memcmp(hdr, "MComprHD", 8) != 0 || rd32(hdr + 12) != 5)
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
im->mapoffset = rd_be(hdr + 40, 8);
|
||||
|
||||
if (fseek(f, (long)im->mapoffset, SEEK_SET) != 0
|
||||
|| fread(hdr + 0, 1, 0, f) != 0)
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
{
|
||||
uint8_t mh[16];
|
||||
if (fread(mh, 1, sizeof(mh), f) != sizeof(mh))
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
maplength = rd32(mh);
|
||||
if (maplength > (32u << 20))
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
want = (size_t)im->mapoffset + 16 + maplength;
|
||||
im->data = (uint8_t*)calloc(want, 1);
|
||||
if (!im->data)
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
im->size = want;
|
||||
im->map_base = (size_t)im->mapoffset;
|
||||
memcpy(im->data + im->mapoffset, mh, sizeof(mh));
|
||||
if (fread(im->data + im->mapoffset + 16, 1, maplength, f) != maplength)
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
fseek(f, 0, SEEK_SET);
|
||||
if (fread(im->data, 1, 124, f) != 124)
|
||||
{
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
fclose(f);
|
||||
|
||||
im->logical = rd_be(im->data + 32, 8);
|
||||
im->mapoffset = rd_be(im->data + 40, 8);
|
||||
im->hunkbytes = rd32(im->data + 56);
|
||||
im->unitbytes = rd32(im->data + 60);
|
||||
if (!im->hunkbytes || !im->unitbytes)
|
||||
return 0;
|
||||
im->hunkcount = (uint32_t)((im->logical + im->hunkbytes - 1)
|
||||
/ im->hunkbytes);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Decodes the compressed map into 12-byte entries. Returns 1 on a
|
||||
* structurally complete decode; the caller checks the CRC. */
|
||||
static int map_decode(const struct image *im, uint8_t *out,
|
||||
uint16_t *stored_crc, uint64_t *span, uint64_t *expect_span)
|
||||
{
|
||||
static uint16_t lookup[1 << MAP_TREE_BITS];
|
||||
rhuff_dec_t dec;
|
||||
rhuff_bits_t bits;
|
||||
const uint8_t *hdr;
|
||||
const uint8_t *body;
|
||||
uint32_t maplength;
|
||||
uint64_t datastart;
|
||||
uint32_t lengthbits;
|
||||
uint32_t hunkbits;
|
||||
uint32_t parentbits;
|
||||
uint64_t curoffset;
|
||||
uint64_t last_self = 0;
|
||||
uint64_t last_parent = 0;
|
||||
uint32_t repeat = 0;
|
||||
uint32_t last_code = 0;
|
||||
uint32_t n;
|
||||
uint8_t *codes;
|
||||
|
||||
if (im->mapoffset + 16 > im->size)
|
||||
return 0;
|
||||
|
||||
hdr = im->data + im->mapoffset;
|
||||
maplength = rd32(hdr);
|
||||
datastart = rd_be(hdr + 4, 6);
|
||||
*stored_crc = (uint16_t)rd_be(hdr + 10, 2);
|
||||
lengthbits = hdr[12];
|
||||
hunkbits = hdr[13];
|
||||
parentbits = hdr[14];
|
||||
|
||||
if (im->mapoffset + 16 + (uint64_t)maplength > im->size)
|
||||
return 0;
|
||||
if (lengthbits > 24 || hunkbits > 24 || parentbits > 24)
|
||||
return 0;
|
||||
if (datastart > im->mapoffset)
|
||||
return 0;
|
||||
|
||||
body = hdr + 16;
|
||||
|
||||
if (rhuff_dec_init(&dec, MAP_TREE_CODES, MAP_TREE_BITS, lookup,
|
||||
RHUFF_LOOKUP_ENTRIES(MAP_TREE_BITS)) != RHUFF_OK)
|
||||
return 0;
|
||||
|
||||
rhuff_bits_init(&bits, body, maplength);
|
||||
|
||||
if (rhuff_read_tree_rle(&dec, &bits) != RHUFF_OK)
|
||||
return 0;
|
||||
|
||||
codes = (uint8_t*)malloc(im->hunkcount);
|
||||
if (!codes)
|
||||
return 0;
|
||||
|
||||
/* First pass: one reference code per hunk, with two escapes that
|
||||
* repeat the previous code. */
|
||||
for (n = 0; n < im->hunkcount; n++)
|
||||
{
|
||||
uint32_t value;
|
||||
|
||||
if (repeat > 0)
|
||||
{
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat--;
|
||||
continue;
|
||||
}
|
||||
|
||||
value = rhuff_dec_decode_one(&dec, &bits);
|
||||
|
||||
if (value == MAP_RLE_SMALL)
|
||||
{
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat = 2 + rhuff_dec_decode_one(&dec, &bits);
|
||||
}
|
||||
else if (value == MAP_RLE_LARGE)
|
||||
{
|
||||
uint32_t hi = rhuff_dec_decode_one(&dec, &bits);
|
||||
uint32_t lo = rhuff_dec_decode_one(&dec, &bits);
|
||||
codes[n] = (uint8_t)last_code;
|
||||
repeat = 2 + 16 + (hi << 4) + lo;
|
||||
}
|
||||
else
|
||||
{
|
||||
last_code = value;
|
||||
codes[n] = (uint8_t)value;
|
||||
}
|
||||
}
|
||||
|
||||
/* Second pass: the fields each code implies, in the same bit
|
||||
* stream, plus the running offset that uncompressed and compressed
|
||||
* hunks advance. */
|
||||
curoffset = datastart;
|
||||
|
||||
for (n = 0; n < im->hunkcount; n++)
|
||||
{
|
||||
uint32_t code = codes[n];
|
||||
uint64_t offset = curoffset;
|
||||
uint32_t length = 0;
|
||||
uint32_t crc = 0;
|
||||
|
||||
switch (code)
|
||||
{
|
||||
case MAP_TYPE_0:
|
||||
case MAP_TYPE_0 + 1:
|
||||
case MAP_TYPE_0 + 2:
|
||||
case MAP_TYPE_3:
|
||||
length = rhuff_bits_read(&bits, (int)lengthbits);
|
||||
crc = rhuff_bits_read(&bits, 16);
|
||||
curoffset += length;
|
||||
break;
|
||||
|
||||
case MAP_NONE:
|
||||
length = im->hunkbytes;
|
||||
crc = rhuff_bits_read(&bits, 16);
|
||||
curoffset += length;
|
||||
break;
|
||||
|
||||
case MAP_SELF:
|
||||
offset = rhuff_bits_read(&bits, (int)hunkbits);
|
||||
last_self = offset;
|
||||
break;
|
||||
|
||||
case MAP_PARENT:
|
||||
offset = rhuff_bits_read(&bits, (int)parentbits);
|
||||
last_parent = offset;
|
||||
break;
|
||||
|
||||
case MAP_SELF_1:
|
||||
last_self++;
|
||||
/* falls through */
|
||||
case MAP_SELF_0:
|
||||
code = MAP_SELF;
|
||||
offset = last_self;
|
||||
break;
|
||||
|
||||
case MAP_PARENT_SELF:
|
||||
code = MAP_PARENT;
|
||||
offset = ((uint64_t)n * im->hunkbytes) / im->unitbytes;
|
||||
last_parent = offset;
|
||||
break;
|
||||
|
||||
case MAP_PARENT_1:
|
||||
last_parent += im->hunkbytes / im->unitbytes;
|
||||
/* falls through */
|
||||
case MAP_PARENT_0:
|
||||
code = MAP_PARENT;
|
||||
offset = last_parent;
|
||||
break;
|
||||
|
||||
default:
|
||||
free(codes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
out[n * MAP_ENTRY_BYTES] = (uint8_t)code;
|
||||
wr_be(out + n * MAP_ENTRY_BYTES + 1, (uint64_t)length, 3);
|
||||
wr_be(out + n * MAP_ENTRY_BYTES + 4, offset, 6);
|
||||
wr_be(out + n * MAP_ENTRY_BYTES + 10, (uint64_t)crc, 2);
|
||||
}
|
||||
|
||||
free(codes);
|
||||
|
||||
*span = curoffset - datastart;
|
||||
*expect_span = im->mapoffset - datastart;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int check(const char *path)
|
||||
{
|
||||
struct image im;
|
||||
uint8_t *map;
|
||||
uint16_t stored = 0;
|
||||
uint16_t got;
|
||||
uint64_t span = 0;
|
||||
uint64_t expect = 0;
|
||||
int ok;
|
||||
|
||||
memset(&im, 0, sizeof(im));
|
||||
if (!image_load(&im, path))
|
||||
{
|
||||
printf(" %-44s could not load\n", path);
|
||||
free(im.data);
|
||||
return 0;
|
||||
}
|
||||
|
||||
map = (uint8_t*)calloc(im.hunkcount, MAP_ENTRY_BYTES);
|
||||
if (!map)
|
||||
{
|
||||
free(im.data);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ok = map_decode(&im, map, &stored, &span, &expect);
|
||||
if (ok)
|
||||
{
|
||||
got = crc16(map, (size_t)im.hunkcount * MAP_ENTRY_BYTES);
|
||||
ok = (got == stored) && (span == expect);
|
||||
/* Printed through double rather than a 64-bit integer type: C89
|
||||
* has no long long and no format for one. */
|
||||
printf(" %-44s hunks=%-4u crc %04x/%04x span %.0f/%.0f %s\n",
|
||||
path, im.hunkcount, got, stored,
|
||||
(double)span, (double)expect, ok ? "ok" : "FAIL");
|
||||
}
|
||||
else
|
||||
printf(" %-44s decode failed\n", path);
|
||||
|
||||
free(map);
|
||||
free(im.data);
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Corrupts a copy of a real image and requires only that decoding stays
|
||||
* inside its buffers. Producing a wrong map is a fine outcome; reading
|
||||
* out of bounds is not, and that is what the sanitizers catch. */
|
||||
static int fuzz(const char *path, unsigned long iterations)
|
||||
{
|
||||
struct image im;
|
||||
uint8_t *orig;
|
||||
unsigned long i;
|
||||
unsigned long state = 0x9E3779B9UL;
|
||||
unsigned long survived = 0;
|
||||
|
||||
memset(&im, 0, sizeof(im));
|
||||
if (!image_load(&im, path))
|
||||
return 0;
|
||||
|
||||
orig = (uint8_t*)malloc(im.size);
|
||||
memcpy(orig, im.data, im.size);
|
||||
|
||||
for (i = 0; i < iterations; i++)
|
||||
{
|
||||
uint8_t *map;
|
||||
uint16_t stored = 0;
|
||||
uint64_t span = 0;
|
||||
uint64_t expect = 0;
|
||||
size_t pos;
|
||||
int hits;
|
||||
int k;
|
||||
|
||||
memcpy(im.data, orig, im.size);
|
||||
|
||||
/* Flip bits anywhere in the map header or its body, so bit
|
||||
* widths, the tree and the code stream all get hit. */
|
||||
state = state * 1103515245UL + 12345UL;
|
||||
hits = 1 + (int)((state >> 16) % 6);
|
||||
for (k = 0; k < hits; k++)
|
||||
{
|
||||
state = state * 1103515245UL + 12345UL;
|
||||
pos = im.map_base
|
||||
+ (size_t)((state >> 8) % (im.size - im.map_base));
|
||||
state = state * 1103515245UL + 12345UL;
|
||||
im.data[pos] ^= (uint8_t)(1u << ((state >> 12) & 7));
|
||||
}
|
||||
|
||||
/* Hunk count comes from the header and a corrupt header can make
|
||||
* it huge; cap the allocation the same way a real reader would. */
|
||||
if (im.hunkcount > (1u << 20))
|
||||
continue;
|
||||
|
||||
map = (uint8_t*)calloc(im.hunkcount, MAP_ENTRY_BYTES);
|
||||
if (!map)
|
||||
continue;
|
||||
map_decode(&im, map, &stored, &span, &expect);
|
||||
free(map);
|
||||
survived++;
|
||||
}
|
||||
|
||||
free(orig);
|
||||
free(im.data);
|
||||
printf(" %-44s %lu corrupted maps decoded without a fault\n",
|
||||
path, survived);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
static const char *images[] = {
|
||||
"v5_zlib.chd", "v5_lzma.chd", "v5_huff.chd", "v5_flac.chd",
|
||||
"v5_multi.chd", "child.chd", "selfy.chd",
|
||||
"corpus/Adventures_of_Lomax_The__USA_.chd",
|
||||
"corpus/Dead_or_Alive__USA_.chd",
|
||||
"corpus/Silent_Hill__Europe___EnFrDeEsIt_.chd",
|
||||
"corpus/Street_Fighter_Collection__USA___Disc_2_.chd"
|
||||
};
|
||||
unsigned long iters = (argc > 1) ? strtoul(argv[1], NULL, 10) : 20000;
|
||||
size_t i;
|
||||
int fail = 0;
|
||||
|
||||
printf("map decode against stored CRC-16 and data span\n");
|
||||
for (i = 0; i < sizeof(images) / sizeof(images[0]); i++)
|
||||
if (!check(images[i]))
|
||||
fail = 1;
|
||||
|
||||
printf("corrupted maps\n");
|
||||
fuzz("v5_multi.chd", iters);
|
||||
fuzz("child.chd", iters);
|
||||
|
||||
printf("%s\n", fail ? "FAIL" : "PASS");
|
||||
return fail;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerates the reference images FORMAT.md is derived from and
|
||||
re-checks every claim marked [V] in it.
|
||||
|
||||
Each check is written as a prediction that either holds against the
|
||||
observed bytes or does not; a failure means FORMAT.md is wrong, not that
|
||||
the probe is. Requires chdman on PATH."""
|
||||
|
||||
import hashlib, os, struct, subprocess, sys, tempfile
|
||||
|
||||
HUNK = 4096
|
||||
HUNKS = 64
|
||||
|
||||
def make_source(path):
|
||||
"""Mixed entropy soevery codec is actually exercised: runs, text,
|
||||
noise, and a ramp. Hunk 0 is deliberately all zeros to exercise the
|
||||
uncompressed map's hole case."""
|
||||
buf = bytearray(); st = 0x2545F4914F6CDD1D
|
||||
def rnd():
|
||||
nonlocal st
|
||||
st ^= (st << 13) & 0xFFFFFFFFFFFFFFFF; st ^= st >> 7
|
||||
st ^= (st << 17) & 0xFFFFFFFFFFFFFFFF
|
||||
return st & 0xFF
|
||||
for h in range(HUNKS):
|
||||
if h % 4 == 0: blk = bytes([h & 0xFF]) * HUNK
|
||||
elif h % 4 == 1: blk = (("hunk %04d " % h).encode() * 512)[:HUNK]
|
||||
elif h % 4 == 2: blk = bytes(rnd() for _ in range(HUNK))
|
||||
else: blk = bytes((i * 7 + h) & 0xFF for i in range(HUNK))
|
||||
buf += blk
|
||||
open(path, 'wb').write(bytes(buf))
|
||||
return bytes(buf)
|
||||
|
||||
def build(src, out, comp):
|
||||
subprocess.run(['chdman', 'createraw', '-i', src, '-o', out,
|
||||
'-hs', str(HUNK), '-us', str(HUNK), '-c', comp, '-f'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
|
||||
return open(out, 'rb').read()
|
||||
|
||||
def check(label, cond):
|
||||
print(" %-58s %s" % (label, "ok" if cond else "FAIL"))
|
||||
return bool(cond)
|
||||
|
||||
def main():
|
||||
ok = True
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
src = os.path.join(td, 'raw.bin')
|
||||
raw = make_source(src)
|
||||
raw_sha1 = hashlib.sha1(raw).hexdigest()
|
||||
|
||||
print("header layout (FORMAT.md 1.1)")
|
||||
for comp in ['zlib', 'lzma', 'huff', 'flac']:
|
||||
d = build(src, os.path.join(td, comp + '.chd'), comp)
|
||||
ok &= check("%s: tag, length, version" % comp,
|
||||
d[0:8] == b'MComprHD'
|
||||
and struct.unpack('>II', d[8:16]) == (124, 5))
|
||||
ok &= check("%s: compressor slot 0 reads back" % comp,
|
||||
d[16:20] == comp.encode())
|
||||
ok &= check("%s: logical/hunk/unit" % comp,
|
||||
struct.unpack('>Q', d[32:40])[0] == HUNK * HUNKS
|
||||
and struct.unpack('>II', d[56:64]) == (HUNK, HUNK))
|
||||
ok &= check("%s: rawsha1 at offset 64" % comp,
|
||||
d[64:84].hex() == raw_sha1)
|
||||
ok &= check("%s: parentsha1 zero" % comp, d[104:124] == b'\0' * 20)
|
||||
|
||||
d = build(src, os.path.join(td, 'multi.chd'), 'lzma,zlib,huff,flac')
|
||||
ok &= check("four codecs read back in request order",
|
||||
[d[16 + i * 4:20 + i * 4] for i in range(4)]
|
||||
== [b'lzma', b'zlib', b'huff', b'flac'])
|
||||
|
||||
print("uncompressed image (FORMAT.md 1.1, 2.1)")
|
||||
d = build(src, os.path.join(td, 'none.chd'), 'none')
|
||||
ok &= check("no hashes recorded", d[64:124] == b'\0' * 60)
|
||||
mo = struct.unpack('>Q', d[40:48])[0]
|
||||
ok &= check("mapoffset is 124", mo == 124)
|
||||
m = struct.unpack('>%dI' % HUNKS, d[mo:mo + 4 * HUNKS])
|
||||
ok &= check("map entry 0 is a hole, source hunk 0 is zeros",
|
||||
m[0] == 0 and raw[:HUNK] == b'\0' * HUNK)
|
||||
good = all((b'\0' * HUNK if m[n] == 0
|
||||
else d[m[n] * HUNK:(m[n] + 1) * HUNK])
|
||||
== raw[n * HUNK:(n + 1) * HUNK] for n in range(HUNKS))
|
||||
ok &= check("hole rule resolves all %d hunks" % HUNKS, good)
|
||||
ok &= check("naive reading of entry 0 would differ",
|
||||
d[:HUNK] != raw[:HUNK])
|
||||
|
||||
print("compressed map header (FORMAT.md 2.2)")
|
||||
for comp in ['zlib', 'lzma', 'huff', 'flac']:
|
||||
d = build(src, os.path.join(td, comp + '.chd'), comp)
|
||||
mo = struct.unpack('>Q', d[40:48])[0]
|
||||
h = d[mo:mo + 16]
|
||||
maplen = struct.unpack('>I', h[0:4])[0]
|
||||
ok &= check("%s: datastart is 124" % comp,
|
||||
int.from_bytes(h[4:10], 'big') == 124)
|
||||
ok &= check("%s: map is last, ends at EOF" % comp,
|
||||
mo + 16 + maplen == len(d))
|
||||
ok &= check("%s: no self or parent refs, reserved zero" % comp,
|
||||
h[13] == 0 and h[14] == 0 and h[15] == 0)
|
||||
ok &= check("%s: lengthbits bounds hunkbytes" % comp,
|
||||
(1 << h[12]) > HUNK and (1 << (h[12] - 1)) <= HUNK)
|
||||
|
||||
print("PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
/* Opens real CHD images through rchd and checks the geometry, map and
|
||||
* metadata against what an independent reader reports. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/rchd.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
FILE *f; rchd_t *chd; rchd_request_t req; int e;
|
||||
const rchd_info_t *info;
|
||||
uint8_t buf[65536];
|
||||
unsigned long rounds = 0;
|
||||
|
||||
if (argc < 2) return 2;
|
||||
if (!(f = fopen(argv[1], "rb"))) return 1;
|
||||
if (!(chd = rchd_new())) return 1;
|
||||
|
||||
for (;;)
|
||||
{
|
||||
size_t take, got;
|
||||
e = rchd_open_step(chd, &req);
|
||||
if (e != RCHD_PENDING) break;
|
||||
if (++rounds > 100000) { printf(" LOOP at off=%lu len=%u\n", (unsigned long)req.offset, req.length); e=-97; break; }
|
||||
take = req.length > sizeof(buf) ? sizeof(buf) : req.length;
|
||||
if (fseek(f, (long)req.offset, SEEK_SET) != 0) { e = -99; break; }
|
||||
got = fread(buf, 1, take, f);
|
||||
if (!got) { e = -98; break; }
|
||||
rchd_feed(chd, buf, got);
|
||||
}
|
||||
|
||||
if (e != RCHD_OK)
|
||||
{
|
||||
printf(" %-44s open failed (%d)\n", argv[1], e);
|
||||
rchd_free(chd); fclose(f); return 1;
|
||||
}
|
||||
|
||||
info = rchd_info(chd);
|
||||
printf(" %-44s v%u hunks=%-6u hunk=%-6u unit=%-5u meta=%-2u rounds=%lu\n",
|
||||
argv[1], info->version, info->hunk_count, info->hunk_bytes,
|
||||
info->unit_bytes, rchd_metadata_count(chd), rounds);
|
||||
rchd_free(chd); fclose(f);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/* Decodes every 'huff' hunk of a CHD through rhuff_decode_block and
|
||||
* compares against the original uncompressed source. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <encodings/huffman.h>
|
||||
|
||||
static uint32_t rd32(const uint8_t*p){return ((uint32_t)p[0]<<24)|((uint32_t)p[1]<<16)|((uint32_t)p[2]<<8)|p[3];}
|
||||
static uint64_t rdbe(const uint8_t*p,int n){uint64_t v=0;int i;for(i=0;i<n;i++)v=(v<<8)|p[i];return v;}
|
||||
|
||||
int main(int argc,char**argv)
|
||||
{
|
||||
FILE *cf,*sf; long clen; uint8_t *chd,*src;
|
||||
uint64_t mo,logical,ds; uint32_t hb,nh,ml,lb; uint32_t n;
|
||||
static uint16_t maplut[1<<8];
|
||||
static uint16_t biglut[1<<16];
|
||||
rhuff_dec_t mapdec, big; rhuff_bits_t bits;
|
||||
uint8_t *codes, *out; uint64_t cur; uint32_t rep=0,lastc=0;
|
||||
unsigned ok=0,tot=0;
|
||||
|
||||
if (argc<3) return 2;
|
||||
cf=fopen(argv[1],"rb"); sf=fopen(argv[2],"rb"); if(!cf||!sf) return 1;
|
||||
fseek(cf,0,SEEK_END); clen=ftell(cf); fseek(cf,0,SEEK_SET);
|
||||
chd=malloc((size_t)clen); if(fread(chd,1,(size_t)clen,cf)!=(size_t)clen)return 1; fclose(cf);
|
||||
fseek(sf,0,SEEK_END); { long sl=ftell(sf); fseek(sf,0,SEEK_SET);
|
||||
src=malloc((size_t)sl); if(fread(src,1,(size_t)sl,sf)!=(size_t)sl)return 1; } fclose(sf);
|
||||
|
||||
logical=rdbe(chd+32,8); mo=rdbe(chd+40,8); hb=rd32(chd+56);
|
||||
nh=(uint32_t)((logical+hb-1)/hb);
|
||||
ml=rd32(chd+mo); ds=rdbe(chd+mo+4,6); lb=chd[mo+12];
|
||||
|
||||
rhuff_dec_init(&mapdec,16,8,maplut,RHUFF_LOOKUP_ENTRIES(8));
|
||||
rhuff_bits_init(&bits,chd+mo+16,ml);
|
||||
if (rhuff_read_tree_rle(&mapdec,&bits)!=RHUFF_OK){printf("map tree failed\n");return 1;}
|
||||
codes=malloc(nh);
|
||||
for(n=0;n<nh;n++){
|
||||
uint32_t v;
|
||||
if(rep){codes[n]=(uint8_t)lastc;rep--;continue;}
|
||||
v=rhuff_dec_decode_one(&mapdec,&bits);
|
||||
if(v==7){codes[n]=(uint8_t)lastc;rep=2+rhuff_dec_decode_one(&mapdec,&bits);}
|
||||
else if(v==8){uint32_t hi=rhuff_dec_decode_one(&mapdec,&bits);
|
||||
uint32_t lo=rhuff_dec_decode_one(&mapdec,&bits);
|
||||
codes[n]=(uint8_t)lastc; rep=2+16+(hi<<4)+lo;}
|
||||
else {lastc=v; codes[n]=(uint8_t)v;}
|
||||
}
|
||||
cur=ds; out=malloc(hb);
|
||||
rhuff_dec_init(&big,256,16,biglut,RHUFF_LOOKUP_ENTRIES(16));
|
||||
for(n=0;n<nh;n++){
|
||||
uint32_t code=codes[n], len=0;
|
||||
if(code<=3){len=rhuff_bits_read(&bits,(int)lb); rhuff_bits_read(&bits,16);}
|
||||
else if(code==4){len=hb; rhuff_bits_read(&bits,16);}
|
||||
else if(code==5){rhuff_bits_read(&bits,(int)chd[mo+13]); continue;}
|
||||
else if(code==6){rhuff_bits_read(&bits,(int)chd[mo+14]); continue;}
|
||||
else continue;
|
||||
if(code<=3){
|
||||
rhuff_bits_t hb2;
|
||||
tot++;
|
||||
rhuff_bits_init(&hb2,chd+cur,len);
|
||||
if(rhuff_decode_block(&big,&hb2,out,hb)==RHUFF_OK
|
||||
&& memcmp(out,src+(size_t)n*hb,hb)==0) ok++;
|
||||
}
|
||||
cur+=len;
|
||||
}
|
||||
printf("%-14s huff hunks byte-exact: %u/%u %s\n",argv[1],ok,tot,ok==tot?"PASS":"FAIL");
|
||||
free(codes); free(out); free(chd); free(src);
|
||||
return ok==tot?0:1;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/* Drives audio_transfer's FLAC arm end to end and compares the PCM
|
||||
* against a reference decode of the same stream. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/audio.h>
|
||||
#include <formats/rflac_next.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
FILE *f; long n; uint8_t *buf;
|
||||
void *h; unsigned ch=0, rate=0; uint64_t total=0;
|
||||
int16_t *out; size_t cap=200000, got=0;
|
||||
int16_t *ref; size_t refgot=0;
|
||||
int e;
|
||||
|
||||
if (argc<2) return 2;
|
||||
f=fopen(argv[1],"rb"); if(!f) return 1;
|
||||
fseek(f,0,SEEK_END); n=ftell(f); fseek(f,0,SEEK_SET);
|
||||
buf=(uint8_t*)malloc((size_t)n);
|
||||
if (fread(buf,1,(size_t)n,f)!=(size_t)n) return 1; fclose(f);
|
||||
|
||||
/* reference: the push API directly */
|
||||
{
|
||||
rflac_t *d = rflac_new();
|
||||
size_t rd, wr;
|
||||
ref = (int16_t*)calloc(cap*2,sizeof(int16_t));
|
||||
rflac_set_out_s16(d, ref, cap);
|
||||
rflac_set_in(d, buf, (size_t)n);
|
||||
while ((e = rflac_process(d,&rd,&wr)) == RFLAC_PROCESS_NEXT && wr)
|
||||
refgot += wr;
|
||||
refgot += wr;
|
||||
rflac_free(d);
|
||||
}
|
||||
|
||||
h = audio_transfer_new(AUDIO_TYPE_FLAC);
|
||||
if (!h) { printf(" audio_transfer_new failed\n"); return 1; }
|
||||
audio_transfer_set_buffer_ptr(h, AUDIO_TYPE_FLAC, buf, (size_t)n);
|
||||
if (!audio_transfer_start(h, AUDIO_TYPE_FLAC))
|
||||
{ printf(" audio_transfer_start failed\n"); return 1; }
|
||||
if (!audio_transfer_info(h, AUDIO_TYPE_FLAC, &ch, &rate, &total))
|
||||
{ printf(" audio_transfer_info failed\n"); return 1; }
|
||||
printf(" info: channels=%u rate=%u total=%lu\n", ch, rate,
|
||||
(unsigned long)total);
|
||||
|
||||
out=(int16_t*)calloc(cap*2,sizeof(int16_t));
|
||||
for (;;)
|
||||
{
|
||||
size_t produced = 0;
|
||||
int r = audio_transfer_read_s16(h, AUDIO_TYPE_FLAC,
|
||||
out + got*ch, 1024, &produced);
|
||||
if (r == AUDIO_PROCESS_ERROR || r == AUDIO_PROCESS_ERROR_END) break;
|
||||
got += produced;
|
||||
if (!produced || got + 1024 > cap) break;
|
||||
if (r == AUDIO_PROCESS_END) break;
|
||||
}
|
||||
printf(" decoded through audio_transfer, ref frames=%lu\n",
|
||||
(unsigned long)refgot);
|
||||
printf(" first 1024 frames match reference: %s\n",
|
||||
memcmp(out, ref, 1024*ch*sizeof(int16_t))==0 ? "yes" : "NO");
|
||||
free(out); free(ref); free(buf);
|
||||
audio_transfer_free(h, AUDIO_TYPE_FLAC);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int path_is_directory(const char *p){(void)p;return 0;}
|
||||
int path_mkdir(const char *d){(void)d;return 0;}
|
||||
@@ -0,0 +1,74 @@
|
||||
/* Wraps a candidate headerless FLAC stream in a STREAMINFO built from
|
||||
* the FLAC spec, decodes it with rflac, and reports how many samples
|
||||
* came out. Sweeping offset and block size shows where a cdfl blob's
|
||||
* FLAC data actually starts. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/rflac.h>
|
||||
|
||||
static void put_bits(unsigned char *b, int *bitpos, unsigned long v, int n)
|
||||
{
|
||||
int i;
|
||||
for (i = n - 1; i >= 0; i--)
|
||||
{
|
||||
int bit = (int)((v >> i) & 1);
|
||||
if (bit) b[*bitpos >> 3] |= (unsigned char)(0x80 >> (*bitpos & 7));
|
||||
(*bitpos)++;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
FILE *f; long n; unsigned char *blob;
|
||||
unsigned offset, blocksize, channels, bits, rate;
|
||||
unsigned char hdr[42]; int bp;
|
||||
unsigned char *stream; size_t slen;
|
||||
rflac *fl; short *out; unsigned long got;
|
||||
|
||||
if (argc < 7) { printf("usage: blob off blocksize ch bits rate\n"); return 2; }
|
||||
f = fopen(argv[1], "rb"); if (!f) return 1;
|
||||
fseek(f,0,SEEK_END); n = ftell(f); fseek(f,0,SEEK_SET);
|
||||
blob = malloc((size_t)n); if (fread(blob,1,(size_t)n,f)!=(size_t)n) return 1;
|
||||
fclose(f);
|
||||
|
||||
offset = (unsigned)atoi(argv[2]);
|
||||
blocksize = (unsigned)atoi(argv[3]);
|
||||
channels = (unsigned)atoi(argv[4]);
|
||||
bits = (unsigned)atoi(argv[5]);
|
||||
rate = (unsigned)atoi(argv[6]);
|
||||
if (offset >= (unsigned)n) return 1;
|
||||
|
||||
memset(hdr,0,sizeof(hdr));
|
||||
memcpy(hdr,"fLaC",4);
|
||||
bp = 32;
|
||||
put_bits(hdr,&bp,1,1); /* last metadata block */
|
||||
put_bits(hdr,&bp,0,7); /* STREAMINFO */
|
||||
put_bits(hdr,&bp,34,24); /* length */
|
||||
put_bits(hdr,&bp,blocksize,16);
|
||||
put_bits(hdr,&bp,blocksize,16);
|
||||
put_bits(hdr,&bp,0,24);
|
||||
put_bits(hdr,&bp,0,24);
|
||||
put_bits(hdr,&bp,rate,20);
|
||||
put_bits(hdr,&bp,channels-1,3);
|
||||
put_bits(hdr,&bp,bits-1,5);
|
||||
put_bits(hdr,&bp,0,36);
|
||||
bp += 128;
|
||||
|
||||
slen = 42 + ((size_t)n - offset);
|
||||
stream = malloc(slen);
|
||||
memcpy(stream,hdr,42);
|
||||
memcpy(stream+42, blob+offset, (size_t)n-offset);
|
||||
|
||||
fl = rflac_open_memory(stream, slen);
|
||||
if (!fl) { printf("offset=%-5u bs=%-5u -> open failed\n", offset, blocksize); return 1; }
|
||||
out = malloc(65536*(size_t)channels*sizeof(short));
|
||||
got = (unsigned long)rflac_read_pcm_frames_s16(fl, 65536, out);
|
||||
printf("offset=%-5u bs=%-5u ch=%u bits=%u -> decoded %lu frames (%lu bytes)\n",
|
||||
offset, blocksize, channels, bits, got, got*channels*2);
|
||||
if (got) { FILE*o=fopen("/tmp/flacout.bin","wb"); fwrite(out,1,got*channels*2,o); fclose(o); }
|
||||
rflac_close(fl);
|
||||
return got ? 0 : 1;
|
||||
}
|
||||
int path_is_directory(const char*p){(void)p;return 0;}
|
||||
int path_mkdir(const char*d){(void)d;return 0;}
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds a Matroska file holding a FLAC track, from a header and a run
|
||||
of raw FLAC frames.
|
||||
|
||||
Written because the container arms of the FLAC decoder could otherwise
|
||||
only be built, not run: nothing in the tree produces a .mka, and testing
|
||||
against one recorded elsewhere is not reproducible. A file assembled
|
||||
here is, and it found a real defect the synthetic native-stream tests
|
||||
could not reach."""
|
||||
import struct, sys
|
||||
|
||||
def vint(n, length=None):
|
||||
if length is None:
|
||||
length = 1
|
||||
while n >= (1 << (7*length)) - 1:
|
||||
length += 1
|
||||
out = bytearray(length); v = n | (1 << (7*length))
|
||||
for i in range(length-1, -1, -1):
|
||||
out[i] = v & 0xFF; v >>= 8
|
||||
return bytes(out)
|
||||
|
||||
def uint(n):
|
||||
if n == 0: return b'\x00'
|
||||
b = b''
|
||||
while n: b = bytes([n & 0xFF]) + b; n >>= 8
|
||||
return b
|
||||
|
||||
def el(eid, payload): return eid + vint(len(payload)) + payload
|
||||
def f64(x): return struct.pack('>d', x)
|
||||
|
||||
def build(header, frames, rate=44100, channels=2, block=2352):
|
||||
ebml = el(b'\x1A\x45\xDF\xA3',
|
||||
el(b'\x42\x86', uint(1)) + el(b'\x42\xF7', uint(1))
|
||||
+ el(b'\x42\xF2', uint(4)) + el(b'\x42\xF3', uint(8))
|
||||
+ el(b'\x42\x82', b'matroska') + el(b'\x42\x87', uint(2))
|
||||
+ el(b'\x42\x85', uint(2)))
|
||||
info = el(b'\x15\x49\xA9\x66',
|
||||
el(b'\x2A\xD7\xB1', uint(1000000))
|
||||
+ el(b'\x44\x89', f64(len(frames)*float(block)*1000.0/rate)))
|
||||
audio = el(b'\xE1', el(b'\xB5', f64(float(rate))) + el(b'\x9F', uint(channels)))
|
||||
track = el(b'\xAE',
|
||||
el(b'\xD7', uint(1)) + el(b'\x73\xC5', uint(1))
|
||||
+ el(b'\x83', uint(2)) + el(b'\x86', b'A_FLAC')
|
||||
+ el(b'\x63\xA2', header) + audio)
|
||||
tracks = el(b'\x16\x54\xAE\x6B', track)
|
||||
blocks = b''
|
||||
for i, p in enumerate(frames):
|
||||
ts = int(i * block * 1000.0 / rate)
|
||||
blocks += el(b'\xA3', vint(1) + struct.pack('>h', ts) + b'\x80' + p)
|
||||
cluster = el(b'\x1F\x43\xB6\x75', el(b'\xE7', uint(0)) + blocks)
|
||||
return ebml + el(b'\x18\x53\x80\x67', info + tracks + cluster)
|
||||
|
||||
def split_frames(blob):
|
||||
"""Frame starts, found by sync code and spacing."""
|
||||
syncs = [i for i in range(len(blob)-1)
|
||||
if blob[i] == 0xFF and (blob[i+1] & 0xFC) == 0xF8]
|
||||
starts, prev = [], None
|
||||
for s in syncs:
|
||||
if prev is None or s - prev > 1000:
|
||||
starts.append(s); prev = s
|
||||
b = starts + [len(blob)]
|
||||
return [blob[b[i]:b[i+1]] for i in range(len(starts))]
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 4:
|
||||
print("usage: make_mka.py <native.flac> <raw-frames.bin> <out.mka>")
|
||||
raise SystemExit(2)
|
||||
hdr = open(sys.argv[1], 'rb').read()[:42]
|
||||
frames = split_frames(open(sys.argv[2], 'rb').read())
|
||||
open(sys.argv[3], 'wb').write(build(hdr, frames))
|
||||
print("wrote %s: %d frames" % (sys.argv[3], len(frames)))
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds an Ogg FLAC stream (RFC 5334) from a native header and raw frames."""
|
||||
import struct, sys
|
||||
|
||||
def crc32_ogg(data):
|
||||
crc = 0
|
||||
for b in data:
|
||||
crc ^= b << 24
|
||||
for _ in range(8):
|
||||
crc = ((crc << 1) ^ 0x04C11DB7) & 0xFFFFFFFF if crc & 0x80000000 else (crc << 1) & 0xFFFFFFFF
|
||||
return crc
|
||||
|
||||
def page(serial, seq, htype, granule, packets):
|
||||
"""One page carrying whole packets, laced per the Ogg rules."""
|
||||
seg = bytearray(); body = b''
|
||||
for p in packets:
|
||||
n = len(p)
|
||||
while n >= 255:
|
||||
seg.append(255); n -= 255
|
||||
seg.append(n)
|
||||
body += p
|
||||
hdr = bytearray(b'OggS' + bytes([0, htype]))
|
||||
hdr += struct.pack('<q', granule)
|
||||
hdr += struct.pack('<I', serial) + struct.pack('<I', seq)
|
||||
hdr += b'\x00\x00\x00\x00'
|
||||
hdr += bytes([len(seg)]) + bytes(seg)
|
||||
full = bytes(hdr) + body
|
||||
crc = crc32_ogg(full)
|
||||
return full[:22] + struct.pack('<I', crc) + full[26:]
|
||||
|
||||
def build(header, frames, serial=0x12345678, block=2352):
|
||||
# First packet: the FLAC-in-Ogg mapping header, then the native header.
|
||||
first = b'\x7FFLAC' + bytes([1, 0]) + struct.pack('>H', 1) + header
|
||||
out = page(serial, 0, 0x02, 0, [first])
|
||||
seq = 1; granule = 0
|
||||
for f in frames:
|
||||
granule += block
|
||||
out += page(serial, seq, 0, granule, [f]); seq += 1
|
||||
return out
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.path.insert(0, '.')
|
||||
from make_mka import split_frames
|
||||
hdr = open(sys.argv[1], 'rb').read()[:42]
|
||||
frames = split_frames(open(sys.argv[2], 'rb').read())
|
||||
open(sys.argv[3], 'wb').write(build(hdr, frames))
|
||||
print("wrote %s: %d frames" % (sys.argv[3], len(frames)))
|
||||
@@ -0,0 +1,71 @@
|
||||
/* Decodes a native FLAC stream (fLaC marker + metadata + frames)
|
||||
* through the push API's header path, at several chunk sizes. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/rflac.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
FILE *sf; long slen; uint8_t *stream;
|
||||
size_t chunk; int fail = 0;
|
||||
static const size_t chunks[] = { 0, 4096, 333, 64, 5 };
|
||||
size_t c;
|
||||
int16_t *ref = NULL; size_t ref_frames = 0;
|
||||
|
||||
if (argc < 2) return 2;
|
||||
sf = fopen(argv[1],"rb"); if(!sf) return 1;
|
||||
fseek(sf,0,SEEK_END); slen=ftell(sf); fseek(sf,0,SEEK_SET);
|
||||
stream=(uint8_t*)malloc((size_t)slen);
|
||||
if (fread(stream,1,(size_t)slen,sf)!=(size_t)slen) return 1;
|
||||
fclose(sf);
|
||||
|
||||
for (c = 0; c < sizeof(chunks)/sizeof(chunks[0]); c++)
|
||||
{
|
||||
rflac_t *f = rflac_new();
|
||||
const rflac_format_t *fmt;
|
||||
int16_t *out; size_t cap = 200000, got = 0, off = 0;
|
||||
int e = RFLAC_PROCESS_NEXT;
|
||||
|
||||
chunk = chunks[c];
|
||||
out = (int16_t*)calloc(cap*2, sizeof(int16_t));
|
||||
rflac_set_out_s16(f, out, cap);
|
||||
|
||||
while (off < (size_t)slen)
|
||||
{
|
||||
size_t take = (chunk && chunk < (size_t)slen - off) ? chunk : (size_t)slen - off;
|
||||
size_t rd, wr;
|
||||
rflac_set_in(f, stream + off, take);
|
||||
for (;;)
|
||||
{
|
||||
e = rflac_process(f, &rd, &wr);
|
||||
got += wr;
|
||||
if (e != RFLAC_PROCESS_NEXT || wr == 0) break;
|
||||
}
|
||||
if (e == RFLAC_PROCESS_ERROR) break;
|
||||
off += rd;
|
||||
if (e == RFLAC_PROCESS_END) break;
|
||||
}
|
||||
fmt = rflac_format(f);
|
||||
if (c == 0)
|
||||
{
|
||||
ref = out; ref_frames = got;
|
||||
printf(" chunk=whole rate=%u ch=%u bits=%u block=%u frames=%lu\n",
|
||||
fmt?fmt->sample_rate:0, fmt?fmt->channels:0,
|
||||
fmt?fmt->bits_per_sample:0, fmt?fmt->block_size:0,
|
||||
(unsigned long)got);
|
||||
}
|
||||
else
|
||||
{
|
||||
int same = (got == ref_frames)
|
||||
&& memcmp(out, ref, got*2*sizeof(int16_t)) == 0;
|
||||
printf(" chunk=%-6lu frames=%-8lu %s\n", (unsigned long)chunk,
|
||||
(unsigned long)got, same ? "identical to whole-span decode" : "MISMATCH");
|
||||
if (!same) fail = 1;
|
||||
free(out);
|
||||
}
|
||||
rflac_free(f);
|
||||
}
|
||||
free(ref); free(stream);
|
||||
return fail;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/* Drives audio_transfer's Matroska FLAC arm and compares against a
|
||||
* reference decode of the same frames as a native stream. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/audio.h>
|
||||
#include <formats/rflac.h>
|
||||
|
||||
static uint8_t *slurp(const char *path, size_t *len)
|
||||
{
|
||||
FILE *f = fopen(path,"rb"); long n; uint8_t *b;
|
||||
if (!f) return NULL;
|
||||
fseek(f,0,SEEK_END); n=ftell(f); fseek(f,0,SEEK_SET);
|
||||
b=(uint8_t*)malloc((size_t)n);
|
||||
if (fread(b,1,(size_t)n,f)!=(size_t)n) { fclose(f); return NULL; }
|
||||
fclose(f); *len=(size_t)n; return b;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
uint8_t *mka, *nat; size_t mlen, nlen;
|
||||
int16_t *ref, *out; size_t cap=200000, refgot=0, got=0;
|
||||
void *h; unsigned ch=0, rate=0; uint64_t total=0;
|
||||
|
||||
if (argc<3) return 2;
|
||||
if (!(mka=slurp(argv[1],&mlen)) || !(nat=slurp(argv[2],&nlen))) return 1;
|
||||
|
||||
/* reference: the same frames as a native stream */
|
||||
{
|
||||
rflac_t *d = rflac_new(); size_t rd,wr; int e;
|
||||
ref=(int16_t*)calloc(cap*2,sizeof(int16_t));
|
||||
rflac_set_out_s16(d, ref, cap);
|
||||
rflac_set_in(d, nat, nlen);
|
||||
while ((e=rflac_process(d,&rd,&wr))==RFLAC_PROCESS_NEXT && wr) refgot+=wr;
|
||||
refgot+=wr; rflac_free(d);
|
||||
}
|
||||
|
||||
h = audio_transfer_new(AUDIO_TYPE_FLAC);
|
||||
if (!h) { printf(" new failed\n"); return 1; }
|
||||
audio_transfer_set_buffer_ptr(h, AUDIO_TYPE_FLAC, mka, mlen);
|
||||
if (!audio_transfer_start(h, AUDIO_TYPE_FLAC))
|
||||
{ printf(" start failed (Matroska not detected?)\n"); return 1; }
|
||||
if (!audio_transfer_info(h, AUDIO_TYPE_FLAC, &ch, &rate, &total))
|
||||
{ printf(" info failed\n"); return 1; }
|
||||
printf(" mka info: channels=%u rate=%u\n", ch, rate);
|
||||
|
||||
out=(int16_t*)calloc(cap*2,sizeof(int16_t));
|
||||
for (;;)
|
||||
{
|
||||
size_t produced=0;
|
||||
int r = audio_transfer_read_s16(h, AUDIO_TYPE_FLAC,
|
||||
out+got*ch, 1024, &produced);
|
||||
if (r==AUDIO_PROCESS_ERROR || r==AUDIO_PROCESS_ERROR_END) break;
|
||||
got += produced;
|
||||
if (!produced || got+1024>cap) break;
|
||||
if (r==AUDIO_PROCESS_END) break;
|
||||
}
|
||||
printf(" mka frames=%lu native reference frames=%lu\n",
|
||||
(unsigned long)got,(unsigned long)refgot);
|
||||
printf(" PCM identical: %s\n",
|
||||
(got && got==refgot
|
||||
&& memcmp(out,ref,got*ch*sizeof(int16_t))==0) ? "yes" : "NO");
|
||||
free(out); free(ref); free(mka); free(nat);
|
||||
audio_transfer_free(h, AUDIO_TYPE_FLAC);
|
||||
return 0;
|
||||
}
|
||||
int path_is_directory(const char *p){(void)p;return 0;}
|
||||
int path_mkdir(const char *d){(void)d;return 0;}
|
||||
@@ -0,0 +1,93 @@
|
||||
/* Decodes CHD 'cdfl' hunks through the push-based rflac API and
|
||||
* compares against the sector data an independent reader produces.
|
||||
* Also feeds the same blob in small chunks, to exercise the rewind
|
||||
* path where a frame straddles two spans. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <formats/rflac.h>
|
||||
|
||||
static int decode(const uint8_t *blob, size_t len, size_t chunk,
|
||||
int16_t *out, size_t frames)
|
||||
{
|
||||
rflac_format_t fmt;
|
||||
rflac_t *f;
|
||||
size_t off = 0;
|
||||
size_t got = 0;
|
||||
|
||||
fmt.sample_rate = 44100;
|
||||
fmt.channels = 2;
|
||||
fmt.bits_per_sample = 16;
|
||||
fmt.block_size = 2352;
|
||||
|
||||
if (!(f = rflac_new_raw(&fmt)))
|
||||
return -1;
|
||||
rflac_set_out_s16(f, out, frames);
|
||||
|
||||
while (off < len && got < frames)
|
||||
{
|
||||
size_t take = (chunk && chunk < len - off) ? chunk : len - off;
|
||||
size_t rd, wr;
|
||||
int e;
|
||||
rflac_set_in(f, blob + off, take);
|
||||
for (;;)
|
||||
{
|
||||
e = rflac_process(f, &rd, &wr);
|
||||
got += wr;
|
||||
if (e != RFLAC_PROCESS_NEXT || wr == 0)
|
||||
break;
|
||||
}
|
||||
if (e == RFLAC_PROCESS_ERROR) { rflac_free(f); return -2; }
|
||||
off += rd;
|
||||
if (e == RFLAC_PROCESS_END) break;
|
||||
}
|
||||
rflac_free(f);
|
||||
return (int)got;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
FILE *bf, *tf;
|
||||
long blen, tlen;
|
||||
uint8_t *blob, *truth;
|
||||
int16_t *out;
|
||||
size_t frames = 4704;
|
||||
int got;
|
||||
size_t i;
|
||||
int fail = 0;
|
||||
|
||||
if (argc < 3) return 2;
|
||||
bf = fopen(argv[1], "rb"); tf = fopen(argv[2], "rb");
|
||||
if (!bf || !tf) return 1;
|
||||
fseek(bf,0,SEEK_END); blen=ftell(bf); fseek(bf,0,SEEK_SET);
|
||||
blob = (uint8_t*)malloc((size_t)blen);
|
||||
if (fread(blob,1,(size_t)blen,bf)!=(size_t)blen) return 1; fclose(bf);
|
||||
fseek(tf,0,SEEK_END); tlen=ftell(tf); fseek(tf,0,SEEK_SET);
|
||||
truth = (uint8_t*)malloc((size_t)tlen);
|
||||
if (fread(truth,1,(size_t)tlen,tf)!=(size_t)tlen) return 1; fclose(tf);
|
||||
|
||||
out = (int16_t*)malloc(frames*2*sizeof(int16_t));
|
||||
|
||||
{
|
||||
static const size_t chunks[] = { 0, 4096, 512, 64, 7 };
|
||||
size_t c;
|
||||
for (c = 0; c < sizeof(chunks)/sizeof(chunks[0]); c++)
|
||||
{
|
||||
memset(out, 0, frames*2*sizeof(int16_t));
|
||||
got = decode(blob,(size_t)blen,chunks[c],out,frames);
|
||||
/* CHD stores the samples byte-swapped against the sector. */
|
||||
{
|
||||
int ok = (got == (int)frames);
|
||||
const uint8_t *p = (const uint8_t*)out;
|
||||
for (i = 0; ok && i < frames*4; i += 2)
|
||||
if (p[i] != truth[i+1] || p[i+1] != truth[i]) ok = 0;
|
||||
if (chunks[c]) printf(" chunk=%-6lu frames=%-6d %s\n",
|
||||
(unsigned long)chunks[c], got, ok ? "byte-exact" : "MISMATCH");
|
||||
else printf(" chunk=whole frames=%-6d %s\n", got, ok ? "byte-exact" : "MISMATCH");
|
||||
if (!ok) fail = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
free(out); free(blob); free(truth);
|
||||
return fail;
|
||||
}
|
||||
Reference in new issue
Block a user