mirror of
https://github.com/borgbackup/borg.git
synced 2026-10-12 02:11:36 +00:00
crc32_clmul: only feed full aligned blocks to the folding code
The folding implementation loads full 16 byte vectors from 16 byte aligned addresses, but it was also used for the bytes in front of the first alignment boundary and for inputs shorter than 16 bytes. That caused two defects: - For inputs of 4..15 bytes it loaded 16 bytes starting at src, reading up to 12 bytes past the end of the buffer. Where those bytes fall into an unmapped page, this segfaults. repository.py _read() calls crc32(memoryview(header)[4:]), which is a 5 byte view, so this was on the hot path for every segment entry read and is the cause of the intermittent NetBSD test suite segfaults tracked in #5922. fixes #10149 - In the alignment prologue XOR_INITIAL() xored the 32 bit initial value into the low 4 bytes of the vector, but partial_fold() then kept only algn_diff bytes of it. For algn_diff of 1..3 the upper bytes of the initial value were silently discarded, giving a wrong checksum. fixes #10150 Hand both cases to the table driven implementation instead, so the folding code only ever sees full aligned blocks and the initial value is always folded into a complete 16 byte vector. This is the structure zlib-ng uses as well, see crc32_copy_small() in their arch/x86/crc32_pclmulqdq_tpl.h. It is also faster for the sizes borg actually uses. Measured on an amd64 Zen 3 machine, ns per call: len before after 5 9.45 5.00 9 9.60 7.90 16 7.42 6.70 41 11.68 9.28 4096 248.88 246.30 Only 12..15 byte inputs get slower (9.6 -> 13.8 ns), and borg passes 5 and 37 byte headers. Tests: test_crc32 only swept start offsets 0..3. CPython's bytes payloads are 16 byte aligned, so those cover alignments 0, 15, 14 and 13 and can never reach the broken 1..3 - widen the sweep to 0..15. Add three more tests, all comparing against zlib.crc32: - test_crc32_random_slices takes memoryview slices of a 1 MiB buffer at random, unaligned start and end offsets, which gives large slices with an unaligned head at every 16 byte alignment. - test_crc32_short_slices walks every end offset from start to start+50 for 10 random starts, covering the empty input, the scalar path, the short path and the folding path at arbitrary alignments. - test_crc32_clmul_no_overread puts the buffer flush against a PROT_NONE guard page. It runs in a subprocess, so a regression fails the test rather than killing the test runner with a SIGSEGV. All four fail against the unpatched implementation and pass with this change; they add about 0.4s to the test suite. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Thomas Waldmann
co-authored by
Claude Opus 5
Claude Opus 5
parent
04d1a53ec9
commit
38b6dcea15
@@ -354,36 +354,33 @@ crc32_clmul(const uint8_t *src, long len, uint32_t initial_crc)
|
||||
uint32_t crc;
|
||||
__m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
|
||||
|
||||
if (len < 16) {
|
||||
if (len == 0)
|
||||
return initial_crc;
|
||||
if (len < 4) {
|
||||
/*
|
||||
* no idea how to do this for <4 bytes, delegate to classic impl.
|
||||
*/
|
||||
uint32_t crc = ~initial_crc;
|
||||
switch (len) {
|
||||
case 3: crc = (crc >> 8) ^ Crc32Lookup[0][(crc & 0xFF) ^ *src++]; // fallthrough
|
||||
case 2: crc = (crc >> 8) ^ Crc32Lookup[0][(crc & 0xFF) ^ *src++]; // fallthrough
|
||||
case 1: crc = (crc >> 8) ^ Crc32Lookup[0][(crc & 0xFF) ^ *src++];
|
||||
}
|
||||
return ~crc;
|
||||
}
|
||||
xmm_crc_part = _mm_loadu_si128((__m128i *)src);
|
||||
XOR_INITIAL(xmm_crc_part);
|
||||
goto partial;
|
||||
}
|
||||
if (len == 0)
|
||||
return initial_crc;
|
||||
|
||||
/* this alignment computation would be wrong for len<16 handled above */
|
||||
/*
|
||||
* The folding code below loads full 16 byte vectors from 16 byte aligned
|
||||
* addresses. Hand both short inputs and the bytes in front of the first
|
||||
* alignment boundary to the table driven implementation, so that the
|
||||
* vector code only ever sees full aligned blocks:
|
||||
*
|
||||
* - loading a 16 byte vector from a shorter buffer reads up to 12 bytes
|
||||
* past its end, which faults when the buffer ends flush against the
|
||||
* last mapped page (#10149)
|
||||
* - folding fewer than 4 bytes ahead of the aligned part would discard
|
||||
* the upper bytes of initial_crc, giving a wrong result (#10150)
|
||||
*
|
||||
* This mirrors what zlib-ng does, see crc32_copy_small() there.
|
||||
*/
|
||||
algn_diff = (0 - (uintptr_t)src) & 0xF;
|
||||
if (len < (long)(algn_diff + 16))
|
||||
algn_diff = (unsigned long)len; /* too short to fold at all */
|
||||
if (algn_diff) {
|
||||
xmm_crc_part = _mm_loadu_si128((__m128i *)src);
|
||||
XOR_INITIAL(xmm_crc_part);
|
||||
|
||||
initial_crc = crc32_slice_by_8(src, (size_t)algn_diff, initial_crc);
|
||||
src += algn_diff;
|
||||
len -= algn_diff;
|
||||
|
||||
partial_fold(algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, &xmm_crc_part);
|
||||
if (len == 0)
|
||||
return initial_crc;
|
||||
xmm_initial = _mm_cvtsi32_si128(initial_crc);
|
||||
}
|
||||
|
||||
while ((len -= 64) >= 0) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
@@ -17,7 +20,10 @@ def test_crc32(implementation):
|
||||
data = os.urandom(300)
|
||||
mv = memoryview(data)
|
||||
initial_crc = 0x12345678
|
||||
for start in range(0, 4): # 4B / int32 alignment, head processing
|
||||
# start 0..15 covers every 16B alignment the folding implementation can see.
|
||||
# Only sweeping 0..3 misses the alignments that used to truncate the initial
|
||||
# value (#10150), because CPython's bytes payloads are 16B aligned.
|
||||
for start in range(0, 16):
|
||||
for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
|
||||
31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
|
||||
63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
|
||||
@@ -27,6 +33,99 @@ def test_crc32(implementation):
|
||||
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc)
|
||||
|
||||
|
||||
|
||||
# Big enough that random start/end offsets land on every 16B alignment and
|
||||
# produce large slices, i.e. the folding path with an unaligned head.
|
||||
CRC32_BUFFER_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
@pytest.mark.parametrize('implementation', crc32_implementations)
|
||||
def test_crc32_random_slices(implementation):
|
||||
# Random, deliberately unaligned start *and* end offsets. Slices of a bytes
|
||||
# object would be copies at a fresh 16B aligned address, so take memoryview
|
||||
# slices - those keep the offset and therefore the alignment.
|
||||
data = os.urandom(CRC32_BUFFER_SIZE)
|
||||
mv = memoryview(data)
|
||||
for _ in range(200):
|
||||
a, b = random.randrange(CRC32_BUFFER_SIZE), random.randrange(CRC32_BUFFER_SIZE)
|
||||
start, end = min(a, b), max(a, b)
|
||||
d = mv[start:end]
|
||||
for initial_crc in (0, 0x12345678, 0xffffffff):
|
||||
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc), \
|
||||
'start=%d end=%d initial_crc=%#x' % (start, end, initial_crc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('implementation', crc32_implementations)
|
||||
def test_crc32_short_slices(implementation):
|
||||
# Random start, but every end from start to start+142: walks the empty input,
|
||||
# the too-short-to-fold inputs and the folding path through every main loop /
|
||||
# epilogue fold / partial tail combination, each at whatever alignment the
|
||||
# random start happens to have.
|
||||
data = os.urandom(CRC32_BUFFER_SIZE)
|
||||
mv = memoryview(data)
|
||||
for _ in range(10):
|
||||
start = random.randrange(CRC32_BUFFER_SIZE - 143)
|
||||
# 142 = 15 (max unaligned prefix) + 64 (one full fold_4 block)
|
||||
# + 48 (largest epilogue fold) + 15 (max partial tail)
|
||||
for end in range(start, start + 143):
|
||||
d = mv[start:end]
|
||||
for initial_crc in (0, 0x12345678, 0xffffffff):
|
||||
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc), \
|
||||
'start=%d end=%d initial_crc=%#x' % (start, end, initial_crc)
|
||||
|
||||
# Runs in a subprocess: reading past the end of the buffer is a SIGSEGV, which
|
||||
# would take the whole test runner down with it.
|
||||
OVERREAD_CHECK = """
|
||||
import ctypes, ctypes.util, mmap, sys, zlib
|
||||
from borg.algorithms.checksums import crc32_clmul
|
||||
|
||||
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
|
||||
libc.mmap.restype = ctypes.c_void_p
|
||||
libc.mmap.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int,
|
||||
ctypes.c_int, ctypes.c_long]
|
||||
libc.mprotect.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int]
|
||||
|
||||
PAGE = mmap.PAGESIZE
|
||||
PROT_NONE, PROT_READ, PROT_WRITE = 0, 1, 2
|
||||
MAP_PRIVATE = 0x02
|
||||
MMAP_FAILED = 2 ** (8 * ctypes.sizeof(ctypes.c_void_p)) - 1
|
||||
|
||||
# MAP_ANONYMOUS is 0x20 on Linux and 0x1000 on the BSDs and macOS; mmap() fails
|
||||
# with the wrong one, so just try both.
|
||||
for MAP_ANON in (0x20, 0x1000):
|
||||
addr = libc.mmap(None, 2 * PAGE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0)
|
||||
if addr not in (0, MMAP_FAILED):
|
||||
break
|
||||
else:
|
||||
sys.exit(0) # can not build the trap here, do not fail the test over it
|
||||
|
||||
# make the page behind our buffer unreadable, so an over-read faults
|
||||
if libc.mprotect(ctypes.c_void_p(addr + PAGE), PAGE, PROT_NONE) != 0:
|
||||
sys.exit(0)
|
||||
|
||||
for length in range(1, 16):
|
||||
data = bytes(bytearray(range(length)))
|
||||
# place the data flush against the end of the readable page
|
||||
buf = (ctypes.c_char * length).from_address(addr + PAGE - length)
|
||||
buf[:] = data
|
||||
view = memoryview(buf)
|
||||
try:
|
||||
assert crc32_clmul(view, 0x12345678) == zlib.crc32(data, 0x12345678)
|
||||
finally:
|
||||
view.release()
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.skipif(not checksums.have_clmul, reason='needs CLMUL support')
|
||||
@pytest.mark.skipif(sys.platform == 'win32', reason='needs POSIX mmap/mprotect')
|
||||
def test_crc32_clmul_no_overread():
|
||||
# crc32_clmul used to load 16 bytes unconditionally for inputs of 4..15 bytes,
|
||||
# reading up to 12 bytes past the end of the buffer. That is a SIGSEGV whenever
|
||||
# the buffer ends flush against the last mapped page, see #10149.
|
||||
rc = subprocess.call([sys.executable, '-c', OVERREAD_CHECK])
|
||||
assert rc == 0, 'crc32_clmul read past the end of the buffer (subprocess rc=%r)' % rc
|
||||
|
||||
|
||||
def test_xxh64():
|
||||
assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'
|
||||
assert bin_to_hex(checksums.xxh64(b'test')) == '4fdcca5ddb678139'
|
||||
|
||||
Reference in New Issue
Block a user