intfstream: crc_step hashes a mapped file in place; one scratch buffer per stream

intfstream_crc_step() malloc'd 256 KB, read into it, hashed it and
freed it - per step.  256 KB is above glibc's mmap threshold, so
every quarter megabyte of a scanned disc image was a fresh mmap(),
64 zero-filled page faults, a read() copy, and a munmap(): the
allocator and the fault handler doing more work than the CRC.

Two changes.  A file the VFS has mapped whole - opened with
RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS, borrowed through
filestream_get_mapped_ptr() - is now hashed straight out of the
mapping, position advanced by seek: no scratch, no copy.  Whole file
or nothing, so file offsets are map offsets.  Everything else reads
into a scratch buffer allocated on the first step and kept with the
stream (freed in intfstream_close()), so a multi-gigabyte image
costs one allocation rather than thousands.

Checked against a reference CRC over a 3 MB pattern: whole file and
a bounded span, mapped and buffered, all agree; intfstream_get_crc()
too.  core_crc_slice_test and core_backup_io_test pass under ASan.
This commit is contained in:
LibretroAdmin authored and libretroadmin committed 2026-09-11 03:41:33 +02:00
1 parent 9b7733990e
commit ac368f47a0
1 file changed
+52 -11
+52 -11
View File
@@ -68,6 +68,10 @@ struct intfstream_internal
uint64_t pos; /* logical read position */
uint64_t size;
} buffered;
/* Scratch for intfstream_crc_step() on streams whose bytes are not
* addressable: allocated on the first step, kept for the rest,
* freed with the stream. It used to malloc 256 KB per step. */
uint8_t *crc_scratch;
enum intfstream_type type;
};
@@ -233,6 +237,12 @@ int intfstream_close(intfstream_internal_t *intf)
if (!intf)
return -1;
if (intf->crc_scratch)
{
free(intf->crc_scratch);
intf->crc_scratch = NULL;
}
switch (intf->type)
{
case INTFSTREAM_FILE:
@@ -279,6 +289,7 @@ void *intfstream_init(intfstream_info_t *info)
intf->type = info->type;
memset(&intf->buffered, 0, sizeof(intf->buffered));
intf->crc_scratch = NULL;
intf->file.fp = NULL;
intf->memory.fp = NULL;
#ifdef HAVE_CHD
@@ -879,32 +890,62 @@ int64_t intfstream_crc_step(intfstream_internal_t *intf,
uint32_t *accumulator, size_t max_bytes)
{
int64_t data_read;
uint8_t *buffer;
/* 256 KB reads instead of a 4 KB stack buffer: CRCing a scanned
/* 256 KB per step instead of a 4 KB stack buffer: CRCing a scanned
* multi-gigabyte disc image at 4 KB a call is a quarter of a
* million reads per gigabyte, and the call overhead dominates
* the checksum. */
size_t buffer_len = 256 * 1024;
size_t step_len = 256 * 1024;
if (!intf || !accumulator)
return -1;
if (max_bytes < buffer_len)
buffer_len = max_bytes;
if (!buffer_len)
if (max_bytes < step_len)
step_len = max_bytes;
if (!step_len)
return 0;
if (!(buffer = (uint8_t*)malloc(buffer_len)))
/* A file the VFS has mapped whole (opened with
* RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS) is hashed in place:
* no scratch, no copy, the page cache read straight into the CRC.
* Whole file or nothing, so the offsets below are file offsets. */
if (intf->type == INTFSTREAM_FILE && intf->file.fp)
{
int64_t map_len = 0;
const uint8_t *map = filestream_get_mapped_ptr(intf->file.fp, &map_len);
if (map && map_len == filestream_get_size(intf->file.fp))
{
int64_t pos = filestream_tell(intf->file.fp);
if (pos < 0)
return -1;
if (pos >= map_len)
return 0;
data_read = map_len - pos;
if (data_read > (int64_t)step_len)
data_read = (int64_t)step_len;
*accumulator = encoding_crc32(*accumulator, map + pos,
(size_t)data_read);
if (filestream_seek(intf->file.fp, pos + data_read,
RETRO_VFS_SEEK_POSITION_START) < 0)
return -1;
return data_read;
}
}
/* Everything else: read into a scratch buffer kept for the life
* of the stream. Allocating 256 KB per step, as this did, was a
* malloc above glibc's mmap threshold - a fresh mmap() and 64
* zero-filled page faults, then a munmap() - for every quarter
* megabyte hashed. */
if (!intf->crc_scratch
&& !(intf->crc_scratch = (uint8_t*)malloc(256 * 1024)))
return -1;
data_read = intfstream_read(intf, buffer, buffer_len);
data_read = intfstream_read(intf, intf->crc_scratch, step_len);
if (data_read > 0)
*accumulator = encoding_crc32(*accumulator, buffer,
*accumulator = encoding_crc32(*accumulator, intf->crc_scratch,
(size_t)data_read);
free(buffer);
return data_read;
}