CI: haiku: preload a mmap shim against the file cache corruption

Haiku r1beta6 can silently store stale pages when a write() gets its data
from a memory mapped file, see https://dev.haiku-os.org/ticket/20392 .
That is what broke rust builds there (corrupt .rlib archives, rustc ICEs).

scripts/haiku_mmap_pread_shim.c replaces private mappings of regular files
by anonymous memory filled via pread(). The haiku job builds it and preloads
it into everything it runs.

With that, the previous workarounds are removed: the pip install retry loop,
the serialized rust build and the relocated TMPDIR / cargo target dir.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

.github/workflows/ci.yml

@@ -1066,6 +1066,14 @@ jobs:
               pkgman install -y lz4_devel openssl3_devel
               pkgman install -y rust_bin
 
+              # haiku r1beta6 corrupts data: a write() from a memory mapped file can silently
+              # store stale pages, see https://dev.haiku-os.org/ticket/20392. That breaks rust
+              # builds (rustc copies mapped object files into its .rlib archives). The shim
+              # replaces private file mappings by anonymous memory filled via pread(), we
+              # preload it into everything we run here.
+              gcc -O2 -Wall -shared -fPIC -o /boot/home/haiku_mmap_pread_shim.so scripts/haiku_mmap_pread_shim.c
+              export LD_PRELOAD=/boot/home/haiku_mmap_pread_shim.so
+
               python3 -m ensurepip --upgrade
               python3 -m pip install --upgrade pip wheel
               python3 -m venv .venv
@@ -1075,30 +1083,8 @@ jobs:
               export BORG_LIBLZ4_PREFIX=/system/develop
               export BORG_OPENSSL_PREFIX=/system/develop
 
-              # this VM corrupts data every now and then: rustc crashed on crate metadata it
-              # had written itself ("assertion failed: bytes[len] == STR_SENTINEL") and pip
-              # found sha256 mismatches reading big wheels back from its own cache. Build
-              # outside /boot/system/cache/tmp (haiku's /tmp) and run one rustc at a time,
-              # in case that corruption comes from disk or memory pressure.
-              export TMPDIR=/boot/home/borg-tmp
-              mkdir -p "$TMPDIR"
-              export CARGO_TARGET_DIR="$TMPDIR/cargo-target"
-              export CARGO_BUILD_JOBS=1
-
-              # retry a pip install that hit a corrupted file, dropping the cached (possibly
-              # corrupted) files first, so one of them does not fail the whole job - and does
-              # not end up in the actions cache, failing every following run, too.
-              pip_install() {
-                for attempt in 1 2 3; do
-                  pip install "$@" && return 0
-                  echo "*** pip install failed (attempt $attempt), purging the pip cache and retrying ***"
-                  pip cache purge || true
-                done
-                return 1
-              }
-
-              pip_install -r requirements.d/development.lock.txt
-              pip_install -e .
+              pip install -r requirements.d/development.lock.txt
+              pip install -e .
 
               # troubles with either tox or pytest xdist, so we run pytest manually:
               pytest -v -n auto -rs --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml --benchmark-skip -k "not remote and not socket"

scripts/haiku_mmap_pread_shim.c

@@ -0,0 +1,104 @@
+/*
+ * haiku_mmap_pread_shim: LD_PRELOAD workaround for the Haiku file cache bug
+ * https://dev.haiku-os.org/ticket/20392 (also see
+ * https://github.com/cross-platform-actions/haiku-builder/issues/4).
+ *
+ * A write() whose source buffer is a file mapping can store stale pages in the
+ * destination file. This shim replaces private mappings of regular files by
+ * anonymous memory filled with pread(), so such a write() reads from ordinary
+ * memory instead of from file cache pages.
+ *
+ * Not replaced (passed through to the real mmap()):
+ *   - MAP_SHARED mappings (writes must reach the file),
+ *   - anonymous mappings,
+ *   - executable mappings,
+ *   - anything that is not a regular file.
+ *
+ * Differences to a real private file mapping: the whole range is read at
+ * mmap() time and uses memory of its own, and later changes of the file are
+ * not visible in the mapping.
+ *
+ * Build:  gcc -O2 -Wall -shared -fPIC -o haiku_mmap_pread_shim.so haiku_mmap_pread_shim.c
+ * Use:    LD_PRELOAD=/path/to/haiku_mmap_pread_shim.so <command>
+ * Debug:  HAIKU_MMAP_PREAD_SHIM_DEBUG=1 prints one line per replaced mapping.
+ */
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+typedef void *(*mmap_func)(void *, size_t, int, int, int, off_t);
+
+static mmap_func real_mmap;
+static int debug = -1;
+
+void *
+mmap(void *address, size_t length, int protection, int flags, int fd,
+	off_t offset)
+{
+	if (real_mmap == NULL)
+		real_mmap = (mmap_func)dlsym(RTLD_NEXT, "mmap");
+	if (real_mmap == NULL) {
+		errno = ENOSYS;
+		return MAP_FAILED;
+	}
+
+	struct stat st;
+	if (fd < 0 || length == 0 || offset < 0
+		|| (flags & (MAP_SHARED | MAP_ANONYMOUS)) != 0
+		|| (flags & MAP_PRIVATE) == 0
+		|| (protection & PROT_EXEC) != 0
+		|| fstat(fd, &st) != 0 || !S_ISREG(st.st_mode)) {
+		return real_mmap(address, length, protection, flags, fd, offset);
+	}
+
+	void *memory = real_mmap(address, length, PROT_READ | PROT_WRITE,
+		flags | MAP_ANONYMOUS, -1, 0);
+	if (memory == MAP_FAILED)
+		return MAP_FAILED;
+
+	// Fill the memory from the file. Whatever is beyond the end of the file
+	// stays zero, like in a real mapping of the last file page.
+	size_t done = 0;
+	while (done < length) {
+		ssize_t bytesRead = pread(fd, (char *)memory + done, length - done,
+			offset + (off_t)done);
+		if (bytesRead < 0) {
+			if (errno == EINTR)
+				continue;
+			int error = errno;
+			munmap(memory, length);
+			errno = error;
+			return MAP_FAILED;
+		}
+		if (bytesRead == 0)
+			break;
+		done += (size_t)bytesRead;
+	}
+
+	if ((protection & PROT_WRITE) == 0
+		&& mprotect(memory, length, protection) != 0) {
+		int error = errno;
+		munmap(memory, length);
+		errno = error;
+		return MAP_FAILED;
+	}
+
+	if (debug < 0)
+		debug = getenv("HAIKU_MMAP_PREAD_SHIM_DEBUG") != NULL;
+	if (debug) {
+		char line[128];
+		int size = snprintf(line, sizeof(line),
+			"haiku_mmap_pread_shim: fd %d offset %lld length %zu -> %p\n", fd,
+			(long long)offset, length, memory);
+		if (size > 0)
+			write(STDERR_FILENO, line, (size_t)size);
+	}
+
+	return memory;
+}