libretro/RetroArch
1
0 Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2026-11-08 02:08:22 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

inflate: hoist hot state, 64-bit bit buffer, memcpy window commit

Browse Source 
The clean-room inflate is what HAVE_ZLIB=0 builds get, and it was well
behind zlib on the same input.  Four changes, in descending order of
what they were worth:

1. The fast inner loop kept the bit buffer, both cursors and the table
   pointers in the state struct.  The store to s->out[] may alias the
   struct as far as the compiler can tell, so every one of those fields
   round-tripped through memory once per symbol.  Hoisting them into
   locals for the duration of the loop and writing back on exit is worth
   more than the other three put together.  The rare paths that call
   back into rinf_decode sync the locals down and reload after.

2. bitbuf widens from uint32_t to uint64_t and the fast loop refills
   once per iteration to 57+ bits.  A whole length/distance group is at
   most 48 bits - litlen 15, length extra 5, distance 15, distance
   extra 13 - so the three intermediate refills the old code did per
   match are gone.  The loop guard already established that 8 input
   bytes are available and the refill consumes at most 7, so no new
   bounds check is needed.

3. The distance symbol went through rinf_decode, a real call carrying
   its own refill loop.  With 2 in place that refill is dead, so the
   fast-table lookup is inlined and only long codes fall back to the
   shared decoder.

4. rinf_window_commit copied up to 32 KB into the ring one byte at a
   time, once per process() call.  rpng feeds 32 KB slices, so that
   byte loop was re-copying the entire image alongside the inflate that
   produced it.  Two memcpys now, wrapping at most once.

One-shot inflate against zlib 1.3 on the same IDAT payload, gcc 13.3
-O3, best of 5 (ratio = in-house / zlib, lower is better):

                              before   after
  art_1920x1080_rgb_paeth      1.03x   0.82x
  gray1_1280x720_sub           1.18x   0.83x
  wall_3840x2160_rgba_paeth    1.30x   1.07x
  icon_256x256_rgba_paeth      1.36x   1.08x
  photo_1920x1080_rgba_paeth   1.31x   1.09x
  plt8_1280x720_sub            1.48x   1.22x

End to end, rpng decode in a HAVE_ZLIB=0 build against libpng 1.6.43
(which uses real zlib), best of 7:

                              before   after   libpng   after vs libpng
  art_1920x1080_rgb_paeth       37.5    30.8     46.3       1.50x
  wall_3840x2160_rgba_paeth    401.7   360.0    446.5       1.24x
  wall_3840x2160_rgb_paeth     333.8   303.9    348.1       1.15x
  rgb16_960x540_paeth           36.1    30.7     38.8       1.27x
  photo_1920x1080_rgba_paeth    98.3    89.2    112.2       1.26x
  icon_256x256_rgba_paeth        3.03    2.67     3.58      1.34x
  plt8_1280x720_sub             11.7    10.7     12.2       1.14x
  gray8_1280x720_paeth          15.7    14.7     16.5       1.12x

Still behind on the poorly-compressible end of the corpus: the
photo_1920x1080_* files, whose streams are nearly all literals, land at
0.71x-0.92x of libpng in a HAVE_ZLIB=0 build.  One-shot they are only
1.04x-1.20x off zlib, so most of that is per-call overhead rather than
the decode loop - stubbing the adler32 fold accounts for 2-4ms of a
~20ms gap over 255 calls and the rest is not yet attributed.  Left
alone rather than guessed at; HAVE_ZLIB=1 builds are unaffected either
way, since they never reach this backend.

Validation, all under -fsanitize=address,undefined:
- 9750 chunked round-trips against a zlib reference across the 27-file
  and 1625-file corpora, with randomized input and output slice sizes
  down to one byte each, so the decoder suspends and resumes mid-symbol,
  mid-match, and across back-references reaching into flushed output.
- 39000 truncated and bit-flipped streams, none of which crashed or
  overran.
- 200 deflate->inflate self round-trips over mixed-entropy input.
- Full rpng decode of both corpora with HAVE_ZLIB=0, byte-exact against
  libpng on all 1652 images.
C89 verified on x86-64 and aarch64.
This commit is contained in:
LibretroAdmin authored and libretroadmin committed 2026-07-27 03:51:23 +02:00
1 parent f8e8b00353
commit bbf299b3b8
1 file changed
+188 -111
Download Patch File Download Diff File Expand all files Collapse all files
libretro-common/encodings/encoding_deflate.c
+188 -111
View File
@@ -323,7 +323,8 @@ struct rinflate
size_t out_pos;
/* bit buffer (LSB-first per DEFLATE) */
uint32_t bitbuf;
uint64_t bitbuf; /* 64-bit: one refill covers a whole
* length/distance symbol group */
int bitcnt;
/* 32KB sliding window for back-references */
@@ -382,7 +383,7 @@ static int rinf_need(struct rinflate *s, int n)
{
if (s->in_pos >= s->in_size)
return 0;
s->bitbuf |= (uint32_t)s->in[s->in_pos++] << s->bitcnt;
s->bitbuf |= (uint64_t)s->in[s->in_pos++] << s->bitcnt;
s->bitcnt += 8;
}
return 1;
@@ -478,11 +479,11 @@ static int rinf_decode(struct rinflate *s, struct rinf_huff *h)
uint32_t rev, cur;
uint16_t f;
/* Refill the bit buffer as full as it will go (up to 24 bits kept) in a
* single pass, so the common case needs no further input reads. */
while (s->bitcnt <= 24 && s->in_pos < s->in_size)
/* Refill the bit buffer as full as it will go in a single pass, so
* the common case needs no further input reads. */
while (s->bitcnt <= 56 && s->in_pos < s->in_size)
{
s->bitbuf |= (uint32_t)s->in[s->in_pos++] << s->bitcnt;
s->bitbuf |= (uint64_t)s->in[s->in_pos++] << s->bitcnt;
s->bitcnt += 8;
}
@@ -598,13 +599,23 @@ static void rinf_window_commit(struct rinflate *s)
}
else
src = s->out;
/* append n bytes into the ring at wnext */
/* Append n bytes into the ring at wnext, in at most two memcpys.
* n is bounded by the ring size above, so the copy wraps at most
* once. This runs once per process() call over the whole slice -
* a byte loop here costs as much as the inflate that produced the
* data when the caller feeds small slices (rpng uses 32 KB). */
{
size_t i;
for (i = 0; i < n; i++)
size_t first = 32768 - s->wnext;
if (first > n)
first = n;
memcpy(s->window + s->wnext, src, first);
s->wnext += first;
if (s->wnext == 32768)
s->wnext = 0;
if (first < n)
{
s->window[s->wnext++] = src[i];
if (s->wnext == 32768) s->wnext = 0;
memcpy(s->window, src + first, n - first);
s->wnext = n - first;
}
}
s->whave += (uint32_t)n;
@@ -849,123 +860,189 @@ int rinflate_process(void *data, size_t *read, size_t *wrote)
* + 13 extra ~ 48 bits) and enough output room for the longest
* match (258 bytes) plus a literal. We bail to the careful path
* as soon as either margin gets tight. */
while (!s->copy_active
&& s->ld_step == 0
&& s->in_pos + 8 <= s->in_size
&& s->out_pos + 258 + 1 <= s->out_size)
/* Hoisted state. Every s-> access in here would otherwise
* be a reload: the compiler cannot prove that the store to
* s->out[] does not alias the struct itself, so the bit
* buffer, the cursors and the table pointers all round-trip
* through memory once per symbol. Keeping them in locals
* and writing back on exit is worth more than any of the
* decode changes. */
if (!s->copy_active && s->ld_step == 0)
{
int sym;
/* refill: guaranteed input available */
while (s->bitcnt <= 24)
uint64_t bitbuf = s->bitbuf;
int bitcnt = s->bitcnt;
size_t in_pos = s->in_pos;
size_t out_pos = s->out_pos;
const uint8_t *in = s->in;
uint8_t *out = s->out;
const uint16_t *lfast = s->lencode.fast;
const uint16_t *dfast = s->distcode.fast;
int done_fast = 0;
while (in_pos + 8 <= s->in_size
&& out_pos + 258 + 1 <= s->out_size)
{
s->bitbuf |= (uint32_t)s->in[s->in_pos++] << s->bitcnt;
s->bitcnt += 8;
}
{
uint16_t f = s->lencode.fast[s->bitbuf & ((1 << RINF_FAST_BITS) - 1)];
if (f)
int sym;
/* One refill per iteration. 57+ buffered bits cover a
* whole length/distance group - litlen (15) + length
* extra (5) + distance (15) + distance extra (13) = 48
* - so nothing below needs to touch the input again.
* The loop guard has already established that 8 input
* bytes are available, and this consumes at most 7. */
while (bitcnt <= 56)
{
int l = f & 15;
s->bitbuf >>= l;
s->bitcnt -= l;
sym = f >> 4;
bitbuf |= (uint64_t)in[in_pos++] << bitcnt;
bitcnt += 8;
}
else
{
sym = rinf_decode(s, &s->lencode);
if (sym < -1)
goto error;
/* Shouldn't happen given margin */
if (sym == -1)
break;
}
}
if (sym < 256)
{
s->out[s->out_pos++] = (uint8_t)sym;
continue;
}
if (sym == 256)
{
s->phase = s->bfinal
? (s->wrapped ? RINF_ADLER : RINF_DONE)
: RINF_BLOCK_HDR;
goto block_done;
}
{
int li = sym - 257, dsym, ei;
uint32_t length, dist;
if (li >= 29) { s->error = 1; goto error; }
ei = rinf_len_extra[li];
if (s->bitcnt < ei)
{
while (s->bitcnt <= 24 && s->in_pos < s->in_size)
{ s->bitbuf |= (uint32_t)s->in[s->in_pos++] << s->bitcnt; s->bitcnt += 8; }
}
length = rinf_len_base[li] + (ei ? rinf_getbits(s, ei) : 0);
dsym = rinf_decode(s, &s->distcode);
if (dsym < 0)
{
if (dsym == -1)
break;
goto error;
}
if (dsym >= 30)
{
s->error = 1;
goto error;
}
ei = rinf_dist_extra[dsym];
if (s->bitcnt < ei)
{
while (s->bitcnt <= 24 && s->in_pos < s->in_size)
uint16_t f = lfast[bitbuf & ((1 << RINF_FAST_BITS) - 1)];
if (f)
{
s->bitbuf |= (uint32_t)s->in[s->in_pos++] << s->bitcnt;
s->bitcnt += 8;
int l = f & 15;
bitbuf >>= l;
bitcnt -= l;
sym = f >> 4;
}
}
dist = rinf_dist_base[dsym] + (ei ? rinf_getbits(s, ei) : 0);
if (dist > s->out_pos + s->whave)
{
s->error = 1;
goto error;
}
/* copy the match */
if (dist <= s->out_pos)
{
uint8_t *dst = s->out + s->out_pos;
const uint8_t *srcp = dst - dist;
if (dist >= length)
memcpy(dst, srcp, length);
else if (dist == 1) /* run of a single byte */
memset(dst, srcp[0], length);
else
{
/* overlapping run: grow the copied region by
* doubling so we memcpy progressively larger blocks
* instead of one byte at a time */
uint32_t done = dist;
memcpy(dst, srcp, dist);
while (done < length)
/* Long code: hand back to the shared decoder. */
s->bitbuf = bitbuf;
s->bitcnt = bitcnt;
s->in_pos = in_pos;
sym = rinf_decode(s, &s->lencode);
bitbuf = s->bitbuf;
bitcnt = s->bitcnt;
in_pos = s->in_pos;
if (sym < -1)
{
uint32_t chunk = done;
if (chunk > length - done)
chunk = length - done;
memcpy(dst + done, dst, chunk);
done += chunk;
s->out_pos = out_pos;
goto error;
}
if (sym == -1)
break;
}
s->out_pos += length;
}
else
if (sym < 256)
{
/* rare: reaches into prior-call output; use slow path */
s->copy_len = length;
s->copy_dist = dist;
s->copy_active = 1;
out[out_pos++] = (uint8_t)sym;
continue;
}
if (sym == 256)
{
s->phase = s->bfinal
? (s->wrapped ? RINF_ADLER : RINF_DONE)
: RINF_BLOCK_HDR;
done_fast = 1;
break;
}
{
int li = sym - 257, dsym, ei;
uint32_t length, dist;
if (li >= 29)
{
s->error = 1;
s->out_pos = out_pos;
goto error;
}
ei = rinf_len_extra[li];
length = rinf_len_base[li];
if (ei)
{
length += (uint32_t)(bitbuf & ((1u << ei) - 1));
bitbuf >>= ei;
bitcnt -= ei;
}
{
uint16_t fd = dfast[bitbuf
& ((1 << RINF_FAST_BITS) - 1)];
if (fd)
{
int dl = fd & 15;
bitbuf >>= dl;
bitcnt -= dl;
dsym = fd >> 4;
}
else
{
s->bitbuf = bitbuf;
s->bitcnt = bitcnt;
s->in_pos = in_pos;
dsym = rinf_decode(s, &s->distcode);
bitbuf = s->bitbuf;
bitcnt = s->bitcnt;
in_pos = s->in_pos;
if (dsym < 0)
{
if (dsym == -1)
break;
s->out_pos = out_pos;
goto error;
}
}
}
if (dsym >= 30)
{
s->error = 1;
s->out_pos = out_pos;
goto error;
}
ei = rinf_dist_extra[dsym];
dist = rinf_dist_base[dsym];
if (ei)
{
dist += (uint32_t)(bitbuf & ((1u << ei) - 1));
bitbuf >>= ei;
bitcnt -= ei;
}
if (dist > out_pos + s->whave)
{
s->error = 1;
s->out_pos = out_pos;
goto error;
}
if (dist <= out_pos)
{
uint8_t *dst = out + out_pos;
const uint8_t *srcp = dst - dist;
if (dist >= length)
memcpy(dst, srcp, length);
else if (dist == 1) /* run of a single byte */
memset(dst, srcp[0], length);
else
{
/* overlapping run: grow the copied region by
* doubling so we memcpy progressively larger
* blocks instead of one byte at a time */
uint32_t cdone = dist;
memcpy(dst, srcp, dist);
while (cdone < length)
{
uint32_t chunk = cdone;
if (chunk > length - cdone)
chunk = length - cdone;
memcpy(dst + cdone, dst, chunk);
cdone += chunk;
}
}
out_pos += length;
}
else
{
/* rare: reaches into prior-call output */
s->copy_len = length;
s->copy_dist = dist;
s->copy_active = 1;
break;
}
}
}
s->bitbuf = bitbuf;
s->bitcnt = bitcnt;
s->in_pos = in_pos;
s->out_pos = out_pos;
if (done_fast)
goto block_done;
}
for (;;)