commit f41bfcd5c72ca30cce6498e912d7513481f55a1f Author: Rui Ueyama Date: Sun Aug 30 22:44:58 2026 +0900 Import current mold Rust port C++ reference: mold fbaaa75f9fb5ad16ce6971e31c5c31b6f09b5d71 Benchmark protocol: 32 threads; one C++ warmup followed by five interleaved measured runs per linker; median reported. Every linker invocation was preceded by sync and followed by sync; sleep 1. Workload C++ (s) Rust (s) Rust vs C++ blender-debug 0.821 0.770 -6.21% blender-release 0.187 0.165 -11.76% chrome-debug-arm64 1.714 1.815 +5.89% chrome-debug-x86_64 1.602 1.643 +2.56% chrome-release-arm64 0.713 0.751 +5.33% chrome-release-x86_64 0.608 0.636 +4.61% clang-debug 1.303 1.139 -12.59% clang-release 0.097 0.097 +0.00% clickhouse-debug 0.917 0.906 -1.20% clickhouse-release 0.379 0.386 +1.85% firefox-debug-arm64 0.918 0.935 +1.85% firefox-debug 0.787 0.802 +1.91% firefox-release 0.200 0.206 +3.00% godot-debug 0.445 0.418 -6.07% godot-release 0.066 0.075 +13.64% libmergedlo.so-debug 0.424 0.439 +3.54% libmergedlo.so-release 0.184 0.171 -7.07% pytorch-debug 0.794 0.736 -7.30% pytorch-release 0.142 0.136 -4.23% tensorflow-debug 3.043 2.808 -7.72% tensorflow-release 0.657 0.610 -7.15% Validation: cargo build --release; ./run-tests.sh --all (pass 5360, skip 636, fail 0); cargo clippy --release --all-targets. diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..1560b9ba5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/target +/out +/ld +/mold-wrapper.so +/mold +# Byproducts of tests that write into the working directory +/exe +/sde-cet-checker-out.txt diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..1a0377fbf --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,575 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mold" +version = "2.42.0" +dependencies = [ + "blake3", + "bstr", + "cpp_demangle", + "crc32fast", + "flate2", + "hashbrown", + "libc", + "memchr", + "memmap2", + "rayon", + "rustc-demangle", + "rustc-hash", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "mold-cli" +version = "2.42.0" +dependencies = [ + "mold", + "mold-mimalloc", + "mold-target-arm32", + "mold-target-arm32be", + "mold-target-arm64", + "mold-target-arm64be", + "mold-target-i386", + "mold-target-loongarch32", + "mold-target-loongarch64", + "mold-target-m68k", + "mold-target-ppc32", + "mold-target-ppc64v1", + "mold-target-ppc64v2", + "mold-target-riscv32", + "mold-target-riscv32be", + "mold-target-riscv64", + "mold-target-riscv64be", + "mold-target-s390x", + "mold-target-sh4", + "mold-target-sh4be", + "mold-target-sparc64", + "mold-target-x86_64", +] + +[[package]] +name = "mold-mimalloc" +version = "2.42.0" + +[[package]] +name = "mold-target-arm32" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-arm32be" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-arm64" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-arm64be" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-i386" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-loongarch32" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-loongarch64" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-m68k" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-ppc32" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-ppc64v1" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-ppc64v2" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-riscv32" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-riscv32be" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-riscv64" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-riscv64be" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-s390x" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-sh4" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-sh4be" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-sparc64" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-target-x86_64" +version = "2.42.0" +dependencies = [ + "mold", +] + +[[package]] +name = "mold-test-runner" +version = "2.42.0" +dependencies = [ + "libc", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..b5d95df2a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "mold" +version = "2.42.0" +edition = "2021" +description = "A high-performance ELF linker, rewritten in Rust" +license = "MIT" + +# The linker is generic over the target. Instantiating it for all twenty +# targets in one crate keeps the compiler on a single core for minutes, so +# each target is instantiated in a crate of its own under targets/, and the +# executable in cli/ dispatches to them. +[workspace] +members = ["cli", "mimalloc", "targets/*", "test-runner"] +default-members = ["cli", "test-runner"] + +[dependencies] +blake3 = "1.8.7" +bstr = "1" +cpp_demangle = "0.4" +crc32fast = "1" +flate2 = "1" +hashbrown = "0.17.1" +libc = "0.2" +memchr = "2.8.3" +memmap2 = "0.9" +rayon = "1" +rustc-demangle = "0.1" +rustc-hash = "2" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +zstd = "0.13" + +[profile.release] +debug = 1 +codegen-units = 16 + +[profile.dev] +opt-level = 1 diff --git a/README.md b/README.md new file mode 100644 index 000000000..411b65007 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# mold, in Rust + +A port of the [mold](https://github.com/rui314/mold) linker (version +2.42.0) to Rust. It is a drop-in replacement for `ld`: it takes the same +command line, produces byte-for-byte the same kind of output, and passes +mold's own test suite. + +## Building + +``` +cargo build --release +``` + +The binary is `target/release/mold`. It answers to `ld` as well, so a +symlink named `ld` in a directory passed to the compiler with `-B` makes +GCC and Clang use it. The build also compiles three C parts: the vendored +mimalloc in `csrc/mimalloc` (the same copy the C++ mold links, used as the +global allocator), `mold-wrapper.so`, the preload library behind +`mold -run`, and the variadic adapter the LTO plugin API needs. + +The linker is generic over the target, and instantiating it for all +twenty targets in one crate keeps the compiler on a single core for +minutes. So the crate is a workspace: `mold` is the generic library, each +`targets/` crate instantiates it for one target, and `cli` is the +executable, with a feature per target. Building the executable alone for +one target is much faster: + +``` +cargo build --release -p mold-cli --no-default-features --features x86_64 +``` + +## Testing + +The tests are mold's own shell scripts, copied unchanged into `test/`. +`run-tests.sh` runs them in parallel and prints a summary: + +``` +./run-tests.sh # the host's tests +./run-tests.sh -j 8 tls- # only tests matching a pattern +TRIPLE=aarch64-linux-gnu ./run-tests.sh # another target, with a cross + # compiler and qemu-user +``` + +Logs go to `out/test/results//`. As in mold's CMake setup, a +target runs the generic tests plus the ones prefixed with its +architecture; tests whose prerequisites are missing skip themselves. + +## Targets + +x86-64, i386, ARM64 (both byte orders), ARM32 (little-endian and BE8), +RISC-V (32- and 64-bit, both byte orders), PowerPC (32-bit, and 64-bit +ELFv1 and ELFv2), s390x, SPARC64, m68k, SH-4 (both byte orders) and +LoongArch (32- and 64-bit). Every target for which a cross toolchain was +available passes its full suite: x86-64, i386, ARM64, ARM32, RISC-V 64, +PowerPC 32/64/64LE, s390x, SPARC64, m68k and SH-4. + +## Layout + +| Module | What it holds | +| --- | --- | +| `driver` | The link in order: reading inputs, resolving symbols, laying out the output, writing it | +| `args`, `linker_script` | Command line and script parsing | +| `input_files`, `input_sections`, `symbol` | Object files, shared libraries, their sections and symbols | +| `passes`, `gc_sections`, `icf`, `relax`, `thunks` | The link's passes, from symbol resolution to range extension thunks | +| `chunks` | Everything that ends up in the output: output sections and the synthesized ones (`.got`, `.plt`, `.dynamic`, `.eh_frame`, ...) | +| `arch` | One module per target: relocation scanning and application, PLT stubs, thunks, relaxation | +| `lto`, `gdb_index`, `output_file`, `elf`, `util` | The LTO plugin bridge, `.gdb_index` generation, output files, the ELF format and helpers | diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..74127d9c0 --- /dev/null +++ b/build.rs @@ -0,0 +1,54 @@ +//! Builds the two small C parts of mold. +//! +//! `mold-wrapper.so` is the preload library behind `mold -run`. It +//! interposes the exec family, including the C-variadic `execl` functions +//! that stable Rust cannot define, so it is a C file. It is placed next to +//! the `mold` executable, where `-run` looks for it. +//! +//! `lto-message.c` adapts the LTO plugin's printf-like diagnostics +//! callback, which is C-variadic as well, and is linked into mold. + +use std::path::PathBuf; +use std::process::Command; + +fn main() { + println!("cargo:rerun-if-changed=csrc/mold-wrapper.c"); + println!("cargo:rerun-if-changed=csrc/lto-message.c"); + + let cc = std::env::var("CC").unwrap_or_else(|_| "cc".to_string()); + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + + let object = out_dir.join("lto-message.o"); + let status = Command::new(&cc) + .args(["-c", "-O2", "-fPIC", "-o"]) + .arg(&object) + .arg("csrc/lto-message.c") + .status(); + if !matches!(status, Ok(s) if s.success()) { + panic!("could not compile csrc/lto-message.c with {cc}"); + } + let archive = out_dir.join("libltomessage.a"); + let status = Command::new("ar") + .arg("rcs") + .arg(&archive) + .arg(&object) + .status(); + if !matches!(status, Ok(s) if s.success()) { + panic!("could not create {}", archive.display()); + } + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-lib=static=ltomessage"); + + // OUT_DIR is //build/-/out. + let profile_dir = out_dir.ancestors().nth(3).unwrap().to_path_buf(); + let wrapper = profile_dir.join("mold-wrapper.so"); + let status = Command::new(&cc) + .args(["-shared", "-fPIC", "-O2", "-o"]) + .arg(&wrapper) + .arg("csrc/mold-wrapper.c") + .arg("-ldl") + .status(); + if !matches!(status, Ok(s) if s.success()) { + println!("cargo:warning=could not build mold-wrapper.so; `mold -run` will not work"); + } +} diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 000000000..d6e62299b --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "mold-cli" +version = "2.42.0" +edition = "2021" +description = "The mold executable" +license = "MIT" + +[[bin]] +name = "mold" +path = "src/main.rs" + +[dependencies] +mold-mimalloc = { path = "../mimalloc" } +mold = { path = ".." } +mold-target-x86_64 = { path = "../targets/x86_64", optional = true } +mold-target-i386 = { path = "../targets/i386", optional = true } +mold-target-arm32 = { path = "../targets/arm32", optional = true } +mold-target-arm32be = { path = "../targets/arm32be", optional = true } +mold-target-arm64 = { path = "../targets/arm64", optional = true } +mold-target-arm64be = { path = "../targets/arm64be", optional = true } +mold-target-riscv64 = { path = "../targets/riscv64", optional = true } +mold-target-riscv64be = { path = "../targets/riscv64be", optional = true } +mold-target-riscv32 = { path = "../targets/riscv32", optional = true } +mold-target-riscv32be = { path = "../targets/riscv32be", optional = true } +mold-target-ppc64v2 = { path = "../targets/ppc64v2", optional = true } +mold-target-ppc32 = { path = "../targets/ppc32", optional = true } +mold-target-ppc64v1 = { path = "../targets/ppc64v1", optional = true } +mold-target-s390x = { path = "../targets/s390x", optional = true } +mold-target-sparc64 = { path = "../targets/sparc64", optional = true } +mold-target-m68k = { path = "../targets/m68k", optional = true } +mold-target-sh4 = { path = "../targets/sh4", optional = true } +mold-target-sh4be = { path = "../targets/sh4be", optional = true } +mold-target-loongarch64 = { path = "../targets/loongarch64", optional = true } +mold-target-loongarch32 = { path = "../targets/loongarch32", optional = true } + +# One feature per target. All are built by default; a build for one +# target alone (`--no-default-features --features x86_64`) is much faster. +[features] +default = [ + "x86_64", "i386", "arm32", "arm32be", "arm64", "arm64be", "riscv64", "riscv64be", "riscv32", "riscv32be", + "ppc64v2", "ppc32", "ppc64v1", "s390x", "sparc64", "m68k", "sh4", "sh4be", "loongarch64", "loongarch32", +] +x86_64 = ["dep:mold-target-x86_64"] +i386 = ["dep:mold-target-i386"] +arm32 = ["dep:mold-target-arm32"] +arm32be = ["dep:mold-target-arm32be"] +arm64 = ["dep:mold-target-arm64"] +arm64be = ["dep:mold-target-arm64be"] +riscv64 = ["dep:mold-target-riscv64"] +riscv64be = ["dep:mold-target-riscv64be"] +riscv32 = ["dep:mold-target-riscv32"] +riscv32be = ["dep:mold-target-riscv32be"] +ppc64v2 = ["dep:mold-target-ppc64v2"] +ppc32 = ["dep:mold-target-ppc32"] +ppc64v1 = ["dep:mold-target-ppc64v1"] +s390x = ["dep:mold-target-s390x"] +sparc64 = ["dep:mold-target-sparc64"] +m68k = ["dep:mold-target-m68k"] +sh4 = ["dep:mold-target-sh4"] +sh4be = ["dep:mold-target-sh4be"] +loongarch64 = ["dep:mold-target-loongarch64"] +loongarch32 = ["dep:mold-target-loongarch32"] diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 000000000..7342a79a5 --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,65 @@ +//! The `mold` executable: runs the linker instantiated for the target the +//! inputs are for. Each target is instantiated in a crate of its own so +//! that the compiler can build them in parallel, and a feature per target +//! decides which of them are built in. + +use mold::diagnostics::Diagnostics; + +#[global_allocator] +static GLOBAL: mold_mimalloc::MiMalloc = mold_mimalloc::MiMalloc; + +/// Links for a target, or reports the target the inputs are actually for. +fn link_for_target(target: &str, cmdline: &[String], diag: &Diagnostics) -> Result { + match target { + #[cfg(feature = "x86_64")] + "x86_64" => mold_target_x86_64::link(cmdline, diag), + #[cfg(feature = "i386")] + "i386" => mold_target_i386::link(cmdline, diag), + #[cfg(feature = "arm32")] + "arm32" => mold_target_arm32::link(cmdline, diag), + #[cfg(feature = "arm32be")] + "arm32be" => mold_target_arm32be::link(cmdline, diag), + #[cfg(feature = "arm64")] + "arm64" => mold_target_arm64::link(cmdline, diag), + #[cfg(feature = "arm64be")] + "arm64be" => mold_target_arm64be::link(cmdline, diag), + #[cfg(feature = "riscv64")] + "riscv64" => mold_target_riscv64::link(cmdline, diag), + #[cfg(feature = "riscv64be")] + "riscv64be" => mold_target_riscv64be::link(cmdline, diag), + #[cfg(feature = "riscv32")] + "riscv32" => mold_target_riscv32::link(cmdline, diag), + #[cfg(feature = "riscv32be")] + "riscv32be" => mold_target_riscv32be::link(cmdline, diag), + #[cfg(feature = "ppc64v2")] + "ppc64v2" => mold_target_ppc64v2::link(cmdline, diag), + #[cfg(feature = "ppc32")] + "ppc32" => mold_target_ppc32::link(cmdline, diag), + #[cfg(feature = "ppc64v1")] + "ppc64v1" => mold_target_ppc64v1::link(cmdline, diag), + #[cfg(feature = "s390x")] + "s390x" => mold_target_s390x::link(cmdline, diag), + #[cfg(feature = "sparc64")] + "sparc64" => mold_target_sparc64::link(cmdline, diag), + #[cfg(feature = "m68k")] + "m68k" => mold_target_m68k::link(cmdline, diag), + #[cfg(feature = "sh4")] + "sh4" => mold_target_sh4::link(cmdline, diag), + #[cfg(feature = "sh4be")] + "sh4be" => mold_target_sh4be::link(cmdline, diag), + #[cfg(feature = "loongarch64")] + "loongarch64" => mold_target_loongarch64::link(cmdline, diag), + #[cfg(feature = "loongarch32")] + "loongarch32" => mold_target_loongarch32::link(cmdline, diag), + _ => { + eprintln!("mold: unsupported target: {target}; rebuild mold with the appropriate target support"); + std::process::exit(1); + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let status = mold::driver::main(args, link_for_target); + std::process::exit(status); +} diff --git a/csrc/lto-message.c b/csrc/lto-message.c new file mode 100644 index 000000000..cd9090267 --- /dev/null +++ b/csrc/lto-message.c @@ -0,0 +1,18 @@ +// The linker plugin reports diagnostics through a printf-like callback. +// C-variadic functions can't be defined in stable Rust, so this adapter +// formats the message and hands it to Rust. + +#include +#include + +void mold_lto_report(int level, const char *msg); + +int mold_lto_message(int level, const char *fmt, ...) { + char buf[1000]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + mold_lto_report(level, buf); + return 0; +} diff --git a/csrc/mimalloc/LICENSE b/csrc/mimalloc/LICENSE new file mode 100644 index 000000000..53315ebee --- /dev/null +++ b/csrc/mimalloc/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/csrc/mimalloc/include/mimalloc-new-delete.h b/csrc/mimalloc/include/mimalloc-new-delete.h new file mode 100644 index 000000000..aaf185bb1 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc-new-delete.h @@ -0,0 +1,66 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_NEW_DELETE_H +#define MIMALLOC_NEW_DELETE_H + +// ---------------------------------------------------------------------------- +// This header provides convenient overrides for the new and +// delete operations in C++. +// +// This header should be included in only one source file! +// +// On Windows, or when linking dynamically with mimalloc, these +// can be more performant than the standard new-delete operations. +// See +// --------------------------------------------------------------------------- +#if defined(__cplusplus) + #include + #include + + #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_) + // stay consistent with VCRT definitions + #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n) + #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n) + #else + #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict + #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict + #endif + + void operator delete(void* p) noexcept { mi_free(p); }; + void operator delete[](void* p) noexcept { mi_free(p); }; + + void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); } + void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); } + + mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } + mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } + + mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } + mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); }; + void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); }; + #endif + + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + + void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + #endif +#endif + +#endif // MIMALLOC_NEW_DELETE_H diff --git a/csrc/mimalloc/include/mimalloc-override.h b/csrc/mimalloc/include/mimalloc-override.h new file mode 100644 index 000000000..46e054528 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc-override.h @@ -0,0 +1,68 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_OVERRIDE_H +#define MIMALLOC_OVERRIDE_H + +/* ---------------------------------------------------------------------------- +This header can be used to statically redirect malloc/free and new/delete +to the mimalloc variants. This can be useful if one can include this file on +each source file in a project (but be careful when using external code to +not accidentally mix pointers from different allocators). +-----------------------------------------------------------------------------*/ + +#include + +// Standard C allocation +#define malloc(n) mi_malloc(n) +#define calloc(c,n) mi_calloc(c,n) +#define realloc(p,n) mi_realloc(p,n) +#define free(p) mi_free(p) + +#define strdup(s) mi_strdup(s) +#define strndup(s,n) mi_strndup(s,n) +#define realpath(f,n) mi_realpath(f,n) + +// Microsoft extensions +#define _expand(p,n) mi_expand(p,n) +#define _msize(p) mi_usable_size(p) +#define _recalloc(p,c,n) mi_recalloc(p,c,n) + +#define _strdup(s) mi_strdup(s) +#define _strndup(s,n) mi_strndup(s,n) +#define _wcsdup(s) mi_wcsdup(s) +#define _mbsdup(s) mi_mbsdup(s) +#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm) +#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm) + +// Various Posix and Unix variants +#define reallocf(p,n) mi_reallocf(p,n) +#define malloc_size(p) mi_usable_size(p) +#define malloc_usable_size(p) mi_usable_size(p) +#define malloc_good_size(n) mi_malloc_good_size(n) +#define cfree(p) mi_free(p) + +#define valloc(n) mi_valloc(n) +#define pvalloc(n) mi_pvalloc(n) +#define reallocarray(p,c,n) mi_reallocarray(p,c,n) +#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n) +#define memalign(a,n) mi_memalign(a,n) +#define aligned_alloc(a,n) mi_aligned_alloc(a,n) +#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) +#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n) + +// Microsoft aligned variants +#define _aligned_malloc(n,a) mi_malloc_aligned(n,a) +#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a) +#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a) +#define _aligned_msize(p,a,o) mi_usable_size(p) +#define _aligned_free(p) mi_free(p) +#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o) +#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o) +#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o) + +#endif // MIMALLOC_OVERRIDE_H diff --git a/csrc/mimalloc/include/mimalloc-stats.h b/csrc/mimalloc/include/mimalloc-stats.h new file mode 100644 index 000000000..f8be41aa6 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc-stats.h @@ -0,0 +1,171 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2024-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_STATS_H +#define MIMALLOC_STATS_H + +#include +#include // memset +#include // int64_t + +#define MI_STAT_VERSION 5 // increased on every backward incompatible change + +// alignment for atomic fields +#if defined(_MSC_VER) +#define mi_decl_align(a) __declspec(align(a)) +#elif defined(__GNUC__) +#define mi_decl_align(a) __attribute__((aligned(a))) +#elif __cplusplus >= 201103L +#define mi_decl_align(a) alignas(a) +#else +#define mi_decl_align(a) _Alignas(a) +#endif + + +// count allocation over time +typedef struct mi_stat_count_s { + int64_t total; // total allocated + int64_t peak; // peak allocation + int64_t current; // current allocation +} mi_stat_count_t; + +// counters only increase +typedef struct mi_stat_counter_s { + int64_t total; // total count +} mi_stat_counter_t; + +#define MI_STAT_FIELDS() \ + MI_STAT_COUNT(pages) /* count of mimalloc pages */ \ + MI_STAT_COUNT(reserved) /* reserved memory bytes */ \ + MI_STAT_COUNT(committed) /* committed bytes */ \ + MI_STAT_COUNTER(reset) /* reset bytes */ \ + MI_STAT_COUNTER(purged) /* purged bytes */ \ + MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \ + MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \ + MI_STAT_COUNT(threads) /* number of threads */ \ + MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ + MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \ + \ + MI_STAT_COUNTER(mmap_calls) \ + MI_STAT_COUNTER(commit_calls) \ + MI_STAT_COUNTER(reset_calls) \ + MI_STAT_COUNTER(purge_calls) \ + MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \ + MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \ + MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \ + \ + /* internal statistics */ \ + MI_STAT_COUNTER(arena_rollback_count) \ + MI_STAT_COUNTER(arena_purges) \ + MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \ + MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \ + MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \ + MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \ + /* only on v1 and v2 */ \ + MI_STAT_COUNT(segments) \ + MI_STAT_COUNT(segments_abandoned) \ + MI_STAT_COUNT(segments_cache) \ + MI_STAT_COUNT(_segments_reserved) \ + /* only on v3 */ \ + MI_STAT_COUNT(heaps) \ + MI_STAT_COUNT(theaps) \ + MI_STAT_COUNTER(pages_reclaim_on_alloc) \ + MI_STAT_COUNTER(pages_reclaim_on_free) \ + MI_STAT_COUNTER(pages_reabandon_full) \ + MI_STAT_COUNTER(pages_unabandon_busy_wait) \ + MI_STAT_COUNTER(heaps_delete_wait) + +// Size bins for chunks +typedef enum mi_chunkbin_e { + MI_CBIN_SMALL, // slice_count == 1 + MI_CBIN_OTHER, // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS + MI_CBIN_MEDIUM, // slice_count == 8 + MI_CBIN_LARGE, // slice_count == MI_SIZE_BITS (only used if MI_ENABLE_LARGE_PAGES is 1) + MI_CBIN_HUGE, // slice_count > MI_BCHUNK_BITS + MI_CBIN_NONE, // no bin assigned yet (the chunk is completely free) + MI_CBIN_COUNT +} mi_chunkbin_t; + + +// Define the statistics structure +#define MI_BIN_HUGE (73U) // see types.h +#define MI_STAT_COUNT(stat) mi_stat_count_t stat; +#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat; + +typedef struct mi_stats_s +{ + size_t size; // size of the mi_stats_t structure + size_t version; + + mi_decl_align(8) MI_STAT_FIELDS() + + // future extension + mi_stat_count_t _stat_reserved[4]; + mi_stat_counter_t _stat_counter_reserved[4]; + + // size segregated statistics + mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin + mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin + mi_stat_count_t chunk_bins[MI_CBIN_COUNT]; // chunks per page sizes +} mi_stats_t; + +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER + +// Initialization +static inline void mi_stats_header_init(mi_stats_t* stats) { + stats->size = sizeof(*stats); + stats->version = MI_STAT_VERSION; +} +static inline void mi_stats_init(mi_stats_t* stats) { + memset(stats,0,sizeof(*stats)); + mi_stats_header_init(stats); +} + +#define mi_stats_t_decl(name) mi_stats_t name; mi_stats_init(&name); + +// Exported definitions +#ifdef __cplusplus +extern "C" { +#endif + +// stats from a heap +mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept; +mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL +mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept; + +// stats from a theap +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + +// stats from a subprocess and its heaps aggregated +mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; +mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL +mi_decl_export void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept; +// print subprocess and all its heap stats segregated +mi_decl_export void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept; + +// stats aggregated for the current subprocess and all its heaps. +mi_decl_export bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept; +mi_decl_export char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL +mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; + +// add the stats of the heap to the subprocess and clear the heap stats +mi_decl_export void mi_heap_stats_merge_to_subproc(mi_heap_t* heap); + +// stats from the subprocess without aggregating its heaps +mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; + +mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL +mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept; + +#ifdef __cplusplus +} +#endif + +#endif // MIMALLOC_STATS_H diff --git a/csrc/mimalloc/include/mimalloc.h b/csrc/mimalloc/include/mimalloc.h new file mode 100644 index 000000000..229e39484 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc.h @@ -0,0 +1,730 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_H +#define MIMALLOC_H + +#define MI_MALLOC_VERSION 30500 // major + 2 digits minor + 2 digits patch + +// ------------------------------------------------------ +// Compiler specific attributes +// ------------------------------------------------------ + +#ifdef __cplusplus + #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 + #define mi_attr_noexcept noexcept + #else + #define mi_attr_noexcept throw() + #endif +#else + #define mi_attr_noexcept +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201703) + #define mi_decl_nodiscard [[nodiscard]] +#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl + #define mi_decl_nodiscard __attribute__((warn_unused_result)) +#elif defined(_HAS_NODISCARD) + #define mi_decl_nodiscard _NODISCARD +#elif (_MSC_VER >= 1700) + #define mi_decl_nodiscard _Check_return_ +#else + #define mi_decl_nodiscard +#endif + +#if defined(_MSC_VER) || defined(__MINGW32__) + #if !defined(MI_SHARED_LIB) + #define mi_decl_export + #elif defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __declspec(dllexport) + #else + #define mi_decl_export __declspec(dllimport) + #endif + #if defined(__MINGW32__) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #else + #if (_MSC_VER >= 1900) && !defined(__EDG__) + #define mi_decl_restrict __declspec(allocator) __declspec(restrict) + #else + #define mi_decl_restrict __declspec(restrict) + #endif + #define mi_attr_malloc + #endif + #define mi_cdecl __cdecl + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) + #elif defined(__INTEL_COMPILER) + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) + #else + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) __attribute__((alloc_align(p))) + #endif +#else + #define mi_cdecl + #define mi_decl_export + #define mi_decl_restrict + #define mi_attr_malloc + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#endif + +// ------------------------------------------------------ +// Includes +// ------------------------------------------------------ + +#include // size_t, wchar_t +#include // bool + +#ifdef __cplusplus +extern "C" { +#endif + +// ------------------------------------------------------ +// Standard malloc interface +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_export void mi_free(void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept; + +// ------------------------------------------------------ +// Extended allocation functions +// ------------------------------------------------------ +#define MI_SMALL_WSIZE_MAX 128 +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; + +// `mi_free_size` can be more efficient (as at calls mi_free_small internally). +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; + +// `mi_free_small` is for special applications like language runtimes. +// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` +mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; + +// ------------------------------------------------------------------------------------- +// Aligned allocation +// Note that `alignment` always follows `size` for consistency with unaligned +// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. +// ------------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); + + +// ------------------------------------------------------ +// Typed allocation, the type is always the first parameter +// ------------------------------------------------------ + +#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp))) +#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) +#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) +#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) +#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) +#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp))) + +#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) +#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) +#define mi_heap_calloc_tp(tp,hp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) +#define mi_heap_mallocn_tp(tp,hp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) +#define mi_heap_reallocn_tp(tp,hp,p,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp))) +#define mi_heap_recalloc_tp(tp,hp,p,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp))) + + +// ------------------------------------------------------ +// Internals +// See also `mimalloc-stats.h` for statistics +// ------------------------------------------------------ + +typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); +mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); +mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_error_fun)(int err, void* arg); +mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); + +mi_decl_export void mi_collect(bool force) mi_attr_noexcept; +mi_decl_export int mi_version(void); +mi_decl_export void mi_options_print(void) mi_attr_noexcept; +mi_decl_export void mi_process_info_print(void) mi_attr_noexcept; +mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; +mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; +mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, + size_t* current_rss, size_t* peak_rss, + size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; + + + +// Generally do not use the following as these are usually called automatically +mi_decl_export void mi_process_init(void) mi_attr_noexcept; +mi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept; +mi_decl_export void mi_thread_init(void) mi_attr_noexcept; +mi_decl_export void mi_thread_done(void) mi_attr_noexcept; +mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool + + +// ----------------------------------------------------------------- +// Return allocated block size (if the return value is not NULL) +// ----------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + + +// ------------------------------------------------------------------------------------- +// Heaps: first-class. Can allocate from any thread (and be free'd from any thread) +// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages) +// ------------------------------------------------------------------------------------- + +struct mi_heap_s; +typedef struct mi_heap_s mi_heap_t; + +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); +mi_decl_export void mi_heap_delete(mi_heap_t* heap); // move live blocks to the main heap +mi_decl_export void mi_heap_destroy(mi_heap_t* heap); // free all live blocks +mi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node); +mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force); + +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void); +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p); +mi_decl_nodiscard mi_decl_export bool mi_heap_contains(const mi_heap_t* heap, const void* p); +mi_decl_nodiscard mi_decl_export bool mi_any_heap_contains(const void* p); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); + + +// -------------------------------------------------------------------------------- +// Zero initialized re-allocation. +// Only valid on memory that was originally allocated with zero initialization too. +// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc. +// see +// -------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4); + + + +// ------------------------------------------------------ +// Visiting pages and individual blocks in a heap. +// ------------------------------------------------------ + +// An area of heap space contains blocks of a single size. +typedef struct mi_heap_area_s { + void* blocks; // start of the area containing theap blocks + size_t reserved; // bytes reserved for this area (virtual) + size_t committed; // current available bytes for this area + size_t used; // number of allocated blocks + size_t block_size; // size in bytes of each block + size_t full_block_size; // size in bytes of a full block including padding and metadata. + void* reserved1; // internal +} mi_heap_area_t; + +typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); + +mi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); +mi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + + +// ------------------------------------------------------ +// Arena memory management +// Arena's are larger memory area's provided by the OS or user +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; + +mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; +mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; + +mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept; + +mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; +mi_decl_export void mi_arenas_print(void) mi_attr_noexcept; +mi_decl_export size_t mi_arena_min_alignment(void); +mi_decl_export size_t mi_arena_min_size(void); +mi_decl_export size_t mi_arena_max_object_size(void); + +typedef void* mi_arena_id_t; +mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); +mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export bool mi_arena_contains(mi_arena_id_t arena_id, const void* p); + +// Create a heap that only allocates in the specified arena +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); + + +// ------------------------------------------------------ +// Subprocesses +// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them). +// Used for example for separate interpreters in one process. +// ------------------------------------------------------ + +typedef struct { void* _mi_subproc_id; } mi_subproc_id_t; // abstract type +mi_decl_export mi_subproc_id_t mi_subproc_main(void); +mi_decl_export mi_subproc_id_t mi_subproc_current(void); +mi_decl_export mi_subproc_id_t mi_subproc_new(void); +mi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc); +mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet) + +typedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg); +mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg); + + +// ------------------------------------------------------------------------------------- +// A "theap" is a thread-local heap. This API is only provided for special circumstances like runtimes +// that already have a thread-local context and can store the theap there for (slightly) faster allocations. +// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from +// that theap (instead of the main (t)heap). +// Theaps are first-class, but can only allocate from the same thread that created it. +// Allocation through a `theap` may be a tiny bit faster than using plain malloc +// (as we don't need to lookup the thread local variable). +// ------------------------------------------------------------------------------------- + +struct mi_theap_s; +typedef struct mi_theap_s mi_theap_t; + +mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap); +mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap); +mi_decl_export mi_theap_t* mi_theap_get_default(void); +mi_decl_export void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); + +// ------------------------------------------------------ +// Fast constant size allocations. +// ------------------------------------------------------ + +static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } +} +static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } +} +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +} +static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } +} + +// ------------------------------------------------------ +// Experimental +// ------------------------------------------------------ + +// Experimental: objects followed by a guard page. +// Setting the sample rate on a specific theap can be used to test parts of the program more +// specifically (in combination with `mi_theap_set_default`). +// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object. +// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages. +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed); +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max); + +// very experimental +typedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg); +mi_decl_export bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, + mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept; + +//mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size); +//mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id); +//mi_decl_export bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena); +//mi_decl_export void mi_theap_unload(mi_theap_t* theap); + +// unsafe: assumes the page belonging to `p` is only accessed by the calling thread. +mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept; + +// ------------------------------------------------------ +// Deprecated +// ------------------------------------------------------ + +mi_decl_export bool mi_check_owned(const void* p); + +mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; +mi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + +mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; +mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept; + +mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; +mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; +mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL + +mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; // not deprecated but declared in `mimalloc-stats.h` now. + + +// ------------------------------------------------------ +// Options +// ------------------------------------------------------ + +typedef enum mi_option_e { + // stable options + mi_option_show_errors, // print error messages + mi_option_show_stats, // print statistics on termination + mi_option_verbose, // print verbose messages + // advanced options + mi_option_deprecated_eager_commit, + mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) + mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit) + mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit. + mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup + mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node + mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) + mi_option_deprecated_segment_cache, + mi_option_deprecated_page_reset, + mi_option_deprecated_abandoned_page_purge, + mi_option_deprecated_segment_reset, + mi_option_deprecated_eager_commit_delay, + mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) + mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. + mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) + mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100) + mi_option_max_errors, // issue at most N error messages + mi_option_max_warnings, // issue at most N warning messages + mi_option_deprecated_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%) + mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe + mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`) + mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10) + mi_option_deprecated_purge_extend_delay, + mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) + mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) + mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0) + mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0) + mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) + mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls + mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always + mi_option_page_full_retain, // retain N full (small) pages per size class (=2) + mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) + mi_option_max_vabits, // max user space virtual address bits to consider (=48) + mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) + mi_option_page_commit_on_demand, // commit page memory on-demand + mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited)) + mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16) + mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. + mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. + mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). + mi_option_arena_is_numa_local, // experimental + _mi_option_last, + // legacy option names + mi_option_large_os_pages = mi_option_allow_large_os_pages, + mi_option_eager_region_commit = mi_option_arena_eager_commit, + mi_option_reset_decommits = mi_option_purge_decommits, + mi_option_reset_delay = mi_option_purge_delay, + mi_option_limit_os_alloc = mi_option_disallow_os_alloc +} mi_option_t; + + +mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); +mi_decl_export void mi_option_enable(mi_option_t option); +mi_decl_export void mi_option_disable(mi_option_t option); +mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); +mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); + +mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); +mi_decl_export void mi_option_set(mi_option_t option, long value); +mi_decl_export void mi_option_set_default(mi_option_t option, long value); + + +// ------------------------------------------------------------------------------------------------------- +// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. +// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) +// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing. +// ------------------------------------------------------------------------------------------------------- + +mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; + +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size); // mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); + +mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; + +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; +mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; + +// wide characters +mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; + +// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. +// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); + +#ifdef __cplusplus +} +#endif + +// --------------------------------------------------------------------------------------------- +// Implement the C++ std::allocator interface for use in STL containers. +// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally) +// --------------------------------------------------------------------------------------------- +#ifdef __cplusplus + +#include // std::size_t +#include // PTRDIFF_MAX +#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 +#include // std::true_type +#include // std::forward +#endif + +template struct _mi_stl_allocator_common { + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type& reference; + typedef value_type const& const_reference; + typedef value_type* pointer; + typedef value_type const* const_pointer; + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } + template void destroy(U* p) mi_attr_noexcept { p->~U(); } + #else + void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } + void destroy(pointer p) { p->~value_type(); } + #endif + + size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +}; + +template struct mi_stl_allocator : public _mi_stl_allocator_common { + using typename _mi_stl_allocator_common::size_type; + using typename _mi_stl_allocator_common::value_type; + using typename _mi_stl_allocator_common::pointer; + template struct rebind { typedef mi_stl_allocator other; }; + + mi_stl_allocator() mi_attr_noexcept = default; + mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; + template mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept { } + mi_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_new_n(count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_new_n(count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::true_type; + #endif +}; + +template bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; } +template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } + + +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 +#define MI_HAS_HEAP_STL_ALLOCATOR 1 + +#include // std::shared_ptr + +// Common base class for STL allocators in a specific theap +template struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common { + using typename _mi_stl_allocator_common::size_type; + using typename _mi_stl_allocator_common::value_type; + using typename _mi_stl_allocator_common::pointer; + + _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */ + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::false_type; + #endif + + void collect(bool force) { mi_heap_collect(this->heap.get(), force); } + template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); } + +protected: + std::shared_ptr heap; + template friend struct _mi_heap_stl_allocator_common; + + _mi_heap_stl_allocator_common() { + mi_heap_t* hp = mi_heap_new(); + this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ + } + _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + +private: + static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } + static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } +}; + +// STL allocator allocation in a specific heap +template struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common { + using typename _mi_heap_stl_allocator_common::size_type; + mi_heap_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is deleted when the destructor is called + mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } + + mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + template struct rebind { typedef mi_heap_stl_allocator other; }; +}; + +template bool operator==(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); } +template bool operator!=(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); } + + +// STL allocator allocation in a specific heap, where `free` does nothing and +// the heap is destroyed in one go on destruction -- use with care! +template struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common { + using typename _mi_heap_stl_allocator_common::size_type; + mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is destroyed when the destructor is called + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } + + mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ } + template struct rebind { typedef mi_heap_destroy_stl_allocator other; }; +}; + +template bool operator==(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); } +template bool operator!=(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); } + +#endif // C++11 + +#endif // __cplusplus + +#endif diff --git a/csrc/mimalloc/include/mimalloc/atomic.h b/csrc/mimalloc/include/mimalloc/atomic.h new file mode 100644 index 000000000..ffde71d0c --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/atomic.h @@ -0,0 +1,560 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2024 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_ATOMIC_H +#define MI_ATOMIC_H + +// include windows.h or pthreads.h +#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#endif +#if MI_TLS_MODEL_PTHREADS || (!defined(_WIN32) && !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) +#define MI_USE_PTHREADS 1 +#include +#endif + +// -------------------------------------------------------------------------------------------- +// Atomics +// We need to be portable between C, C++, and MSVC. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. +// This is why we try to use only `uintptr_t` and `*` as atomic types. +// To gain better insight in the range of used atomics, we use explicitly named memory order operations +// instead of passing the memory order as a parameter. +// ----------------------------------------------------------------------------------------------- + +#if defined(__cplusplus) +// Use C++ atomics +#include +#define _Atomic(tp) std::atomic +#define mi_atomic(name) std::atomic_##name +#define mi_memory_order(name) std::memory_order_##name +#if (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#elif defined(_MSC_VER) +// Use MSVC C wrapper for C11 atomics +#define _Atomic(tp) tp +#define MI_ATOMIC_VAR_INIT(x) x +#define mi_atomic(name) mi_atomic_##name +#define mi_memory_order(name) mi_memory_order_##name +#else +// Use C11 atomics +#include +#define mi_atomic(name) atomic_##name +#define mi_memory_order(name) memory_order_##name +#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#endif + +// Various defines for all used memory orders in mimalloc +#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) + +#define mi_atomic_cas_weak_relaxed(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed)) +#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) +#define mi_atomic_cas_strong_relaxed(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed)) +#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) + +#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_and_relaxed(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_or_relaxed(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) + +#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) +#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) +#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) +#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) + +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); + + +#if defined(__cplusplus) || !defined(_MSC_VER) + +// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) +// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well +#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p) +#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p) + +// In C++ we need to add casts to help resolve templates if NULL is passed +#if defined(__cplusplus) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) +#else +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) +#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) +#endif + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { + return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); +} +static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { + const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd); + if (add != 0) { + mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); + } +} +static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { + int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); + while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; +} + +// Used by timers +#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) + +#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d) +#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i) + + +#elif defined(_MSC_VER) + +// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. +// It is recommended to always compile as C++ when using MSVC. + +#include +#ifdef _WIN64 +typedef LONG64 msc_intptr_t; +#define MI_MSC_64(f) f##64 +#define MI_MSC_XX(f) f##64 +#else +typedef LONG msc_intptr_t; +#define MI_MSC_64(f) f +#define MI_MSC_XX(f) f##32 +#endif + +typedef enum mi_memory_order_e { + mi_memory_order_relaxed, + mi_memory_order_consume, + mi_memory_order_acquire, + mi_memory_order_release, + mi_memory_order_acq_rel, + mi_memory_order_seq_cst +} mi_memory_order; + +static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); +} +static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)); +} +static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + (void)(mo1); (void)(mo2); + const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected)); + if (read == *expected) { + return true; + } + else { + *expected = read; + return false; + } +} +static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2); +} +static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); +} + +static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { + (void)(mo); + // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc + #if defined(_M_IX86) || defined(_M_X64) + // on x86/x64 we have a strong memory model so any load is acquire + return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); + #elif defined(_M_ARM) || defined(_M_ARM64) + if (mo == mi_memory_order_relaxed) { + return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); + } + else if (mo <= mi_memory_order_acquire) { + return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p); + } + else { + const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); + __dmb(15); // _ARM(64)_BARRIER_SY + return u; + } + #else + #warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)" + return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); + #endif +} +static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + // assert(mo<=mi_memory_order_release); // others are not used by mimalloc + #if defined(_M_IX86) || defined(_M_X64) + MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x); + #elif defined(_M_ARM) || defined(_M_ARM64) + if (mo == mi_memory_order_relaxed) { + MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x); + } + else if (mo <= mi_memory_order_release) { + MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x); + } + else { + mi_atomic_exchange_explicit(p, x, mo); + } + #else + mi_atomic_exchange_explicit(p, x, mo); + #endif +} + +static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) { + (void)(mo); + // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc + #if defined(_M_IX86) || defined(_M_X64) + return __iso_volatile_load64((volatile const int64_t*)p); + #elif defined(_M_ARM) || defined(_M_ARM64) + if (mo == mi_memory_order_relaxed) { + return __iso_volatile_load64((volatile const int64_t*)p); + } + #if defined(_M_ARM64) + else if (mo <= mi_memory_order_acquire) { + return __ldar64((volatile const uintptr_t*)p); + } + #endif + else { + const int64_t i = __iso_volatile_load64((volatile const int64_t*)p); + __dmb(15); // _ARM(64)_BARRIER_SY + return i; + } + #else + #warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)" + return __iso_volatile_load64((volatile const int64_t*)p); + #endif +} + +static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) { + (void)(mo); + // assert(mo<=mi_memory_order_release); // others are not used by mimalloc + #if defined(_M_IX86) || defined(_M_X64) + __iso_volatile_store64((volatile int64_t*)p,x); + #elif defined(_M_ARM) || defined(_M_ARM64) + if (mo == mi_memory_order_relaxed) { + __iso_volatile_store64((volatile int64_t*)p,x); + } + #if defined(_M_ARM64) + else if (mo == mi_memory_order_release) { + __stlr64((volatile uint64_t*)p, (uint64_t)x); + } + #endif + else { + InterlockedExchange64(p, x); + } + #else + InterlockedExchange64(p, x); + #endif +} + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { + #ifdef _WIN64 + return (int64_t)mi_atomic_addi((int64_t*)p, add); + #elif defined(_M_ARM) + return _InterlockedExchangeAdd64(p, add); + #else + // x86 + int64_t current; + int64_t sum; + do { + current = __iso_volatile_load64((volatile const int64_t*)p); + sum = current + add; + } while (_InterlockedCompareExchange64(p, sum, current) != current); + return current; + #endif +} + +static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) { + const int64_t add = *padd; + if (add != 0) { + mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add); + } +} + +static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) { + int64_t current; + do { + current = *p; + } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); +} + +static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) { + mi_atomic_addi64_relaxed(p, i); +} + +static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) { + const int64_t read = _InterlockedCompareExchange64(p, des, *exp); + if (read == *exp) { + return true; + } + else { + *exp = read; + return false; + } +} + +// The pointer macros cast to `uintptr_t`. +#define mi_atomic_load_ptr_acquire(tp,p) ((tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))) +#define mi_atomic_load_ptr_relaxed(tp,p) ((tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) ((tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_release(tp,p,x) ((tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) ((tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) + +#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed)) + + +#endif + + +// Atomically add a signed value; returns the previous value. +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { + return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); +} + +// Atomically subtract a signed value; returns the previous value. +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { + return (intptr_t)mi_atomic_addi(p, -sub); +} + + +// ---------------------------------------------------------------------- +// Guard +// ---------------------------------------------------------------------- + +typedef _Atomic(uintptr_t) mi_atomic_guard_t; + +// Allows only one thread to execute at a time (without blocking anyone) +#define mi_atomic_guard(guard) \ + uintptr_t _mi_guard_expected = 0; \ + for(bool _mi_guard_once = true; \ + _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ + (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) + + +// ---------------------------------------------------------------------- +// Locks +// These should be light-weight in-process only locks. +// Only used for reserving arena's and to maintain the abandoned list. +// ---------------------------------------------------------------------- +#if _MSC_VER +#pragma warning(disable:26110) // unlock with holding lock +#endif + +#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) ) +#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) ) + + +#if defined(_WIN32) || defined(__CYGWIN__) + +typedef struct mi_lock_s { + SRWLOCK mutex; // slim reader-writer lock +} mi_lock_t; + +#define MI_LOCK_INITIALIZER { SRWLOCK_INIT } + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return TryAcquireSRWLockExclusive(&lock->mutex); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + AcquireSRWLockExclusive(&lock->mutex); +} +static inline void mi_lock_release(mi_lock_t* lock) { + ReleaseSRWLockExclusive(&lock->mutex); +} +static inline void mi_lock_init(mi_lock_t* lock) { + InitializeSRWLock(&lock->mutex); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#elif defined(MI_USE_PTHREADS) + +#include // memcpy +void _mi_error_message(int err, const char* fmt, ...); + +typedef struct mi_lock_s { + pthread_mutex_t mutex; +} mi_lock_t; + +#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return (pthread_mutex_trylock(&lock->mutex) == 0); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + const int err = pthread_mutex_lock(&lock->mutex); + if (err != 0) { + _mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err); + } +} +static inline void mi_lock_release(mi_lock_t* lock) { + pthread_mutex_unlock(&lock->mutex); +} +static inline void mi_lock_init(mi_lock_t* lock) { + if(lock==NULL) return; + // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize) + const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + memcpy(&lock->mutex,&mutex,sizeof(mutex)); +} +static inline void mi_lock_done(mi_lock_t* lock) { + pthread_mutex_destroy(&lock->mutex); +} + +#elif defined(__cplusplus) + +#include +#include +#include + +typedef struct mi_lock_s { + std::mutex mutex; +} mi_lock_t; + +#define MI_LOCK_INITIALIZER { } + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return lock->mutex.try_lock(); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + lock->mutex.lock(); +} +static inline void mi_lock_release(mi_lock_t* lock) { + lock->mutex.unlock(); +} +static inline void mi_lock_init(mi_lock_t* lock) { + new(&lock->mutex) std::mutex(); // in-place constructor +} +static inline void mi_lock_done(mi_lock_t* lock) { + lock->mutex.~mutex(); // in-place destructor +} + +#else + +// fall back to poor man's locks. +// this should only be the case in a single-threaded environment (like __wasi__) +#include +#ifndef EFAULT +#define EFAULT (14) +#endif +void _mi_error_message(int err, const char* fmt, ...); +void _mi_prim_thread_yield(void); + +typedef struct mi_lock_s { + _Atomic(uintptr_t) mutex; +} mi_lock_t; + +#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) } + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + size_t ticks = 0; + for (int i = 0; i < 10000; i++) { // for at most 10000 tries? + if (mi_lock_try_acquire(lock)) return; + _mi_prim_thread_yield(); + } + _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n"); +} +static inline void mi_lock_release(mi_lock_t* lock) { + mi_atomic_store_release(&lock->mutex, (uintptr_t)0); +} +static inline void mi_lock_init(mi_lock_t* lock) { + mi_lock_release(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#endif + + +typedef struct mi_atomic_once_s { + _Atomic(uintptr_t) tid; + mi_lock_t lock; +} mi_atomic_once_t; + +// Returns `true` only on the first invocation, signifying we can execute an action once. +// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action. +// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called. +bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c` +void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c` + +#define mi_atomic_do_once \ + static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \ + for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false)) + + +#endif // __MIMALLOC_ATOMIC_H diff --git a/csrc/mimalloc/include/mimalloc/bits.h b/csrc/mimalloc/include/mimalloc/bits.h new file mode 100644 index 000000000..c7ce294f9 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/bits.h @@ -0,0 +1,354 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2024 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc) +---------------------------------------------------------------------------- */ + +#pragma once +#ifndef MI_BITS_H +#define MI_BITS_H + +#include // size_t +#include // int64_t etc +#include // bool +#include // LONG_MAX + +// ------------------------------------------------------ +// Size of a pointer. +// We assume that `sizeof(void*)==sizeof(intptr_t)` +// and it holds for all platforms we know of. +// +// However, the C standard only requires that: +// p == (void*)((intptr_t)p)) +// but we also need: +// i == (intptr_t)((void*)i) +// or otherwise one might define an intptr_t type that is larger than a pointer... +// ------------------------------------------------------ + +#if INTPTR_MAX > INT64_MAX +# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#elif INTPTR_MAX == INT64_MAX +# define MI_INTPTR_SHIFT (3) +#elif INTPTR_MAX == INT32_MAX +# define MI_INTPTR_SHIFT (2) +#else +#error platform pointers must be 32, 64, or 128 bits +#endif + +#if (INTPTR_MAX) > LONG_MAX +# define MI_PU(x) x##ULL +#else +# define MI_PU(x) x##UL +#endif + +#if SIZE_MAX == UINT64_MAX +# define MI_SIZE_SHIFT (3) +typedef int64_t mi_ssize_t; +#elif SIZE_MAX == UINT32_MAX +# define MI_SIZE_SHIFT (2) +typedef int32_t mi_ssize_t; +#else +#error platform objects must be 32 or 64 bits in size +#endif + +#if (SIZE_MAX/2) > LONG_MAX +# define MI_ZU(x) x##ULL +#else +# define MI_ZU(x) x##UL +#endif + +#define MI_INTPTR_SIZE (1< +#elif MI_ARCH_ARM64 && MI_OPT_SIMD +#include +#endif +#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) +#include +#endif + +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // avx2 implies bmi2 +#define __BMI2__ 1 +#endif +#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__) || defined(__BMI__)) && !defined(__BMI1__) // bmi2 implies bmi1 +#define __BMI1__ 1 +#endif +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // avx2 implies lzcnt +#define __LZCNT__ 1 +#endif + +// Define big endian if needed +// #define MI_BIG_ENDIAN 1 + +// maximum virtual address bits in a user-space pointer +#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 +#define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS +#elif MI_ARCH_X64 +#define MI_MAX_VABITS (47) +#elif MI_INTPTR_SIZE > 4 +#define MI_MAX_VABITS (48) +#else +#define MI_MAX_VABITS (32) +#endif + +// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map +#if MI_MAX_VABITS <= 32 +#define MI_MIN_VABITS (32) +#elif MI_MAX_VABITS <= 42 +#define MI_MIN_VABITS MI_MAX_VABITS +#else +#define MI_MIN_VABITS (43) /* 8 TiB */ +#endif + +// use a flat page-map or a 2-level one +#ifndef MI_PAGE_MAP_FLAT +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP +#define MI_PAGE_MAP_FLAT 1 +#else +#define MI_PAGE_MAP_FLAT 0 +#endif +#endif + + +/* -------------------------------------------------------------------------------- + Builtin's +-------------------------------------------------------------------------------- */ + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#define mi_builtin(name) __builtin_##name +#define mi_has_builtin(name) __has_builtin(__builtin_##name) + +#if (LONG_MAX == INT32_MAX) +#define mi_builtin32(name) mi_builtin(name##l) +#define mi_has_builtin32(name) mi_has_builtin(name##l) +#else +#define mi_builtin32(name) mi_builtin(name) +#define mi_has_builtin32(name) mi_has_builtin(name) +#endif +#if (LONG_MAX == INT64_MAX) +#define mi_builtin64(name) mi_builtin(name##l) +#define mi_has_builtin64(name) mi_has_builtin(name##l) +#else +#define mi_builtin64(name) mi_builtin(name##ll) +#define mi_has_builtin64(name) mi_has_builtin(name##ll) +#endif + +#if (MI_SIZE_BITS == 32) +#define mi_builtinz(name) mi_builtin32(name) +#define mi_has_builtinz(name) mi_has_builtin32(name) +#define mi_msc_builtinz(name) name +#elif (MI_SIZE_BITS == 64) +#define mi_builtinz(name) mi_builtin64(name) +#define mi_has_builtinz(name) mi_has_builtin64(name) +#define mi_msc_builtinz(name) name##64 +#endif + +/* -------------------------------------------------------------------------------- + Popcount and count trailing/leading zero's +-------------------------------------------------------------------------------- */ + +size_t _mi_popcount_generic(size_t x); +extern bool _mi_cpu_has_popcnt; + +static inline size_t mi_popcount(size_t x) { + #if mi_has_builtinz(popcount) + return mi_builtinz(popcount)(x); + #elif defined(_MSC_VER) && (MI_ARCH_ARM64 || MI_ARCH_ARM32) + return mi_msc_builtinz(_CountOneBits)(x); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86) + if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); } + else { return _mi_popcount_generic(x); } // see issue #1291 + #elif MI_ARCH_X64 && defined(__BMI1__) + return (size_t)_mm_popcnt_u64(x); + #else + #define MI_HAS_FAST_POPCOUNT 0 + return (x<=1 ? x : _mi_popcount_generic(x)); + #endif +} + +#ifndef MI_HAS_FAST_POPCOUNT +#define MI_HAS_FAST_POPCOUNT 1 +#endif + + + +size_t _mi_clz_generic(size_t x); +size_t _mi_ctz_generic(size_t x); + +static inline size_t mi_ctz(size_t x) { + #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0 + size_t r; + __asm ("tzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc"); + return r; + #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) + return _tzcnt_u64(x); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) + unsigned long idx; + return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS); + #elif mi_has_builtinz(ctz) + return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS); + #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86) + size_t r = MI_SIZE_BITS; // bsf leaves destination unmodified if the argument is 0 (see ) + __asm ("bsf\t%1, %0" : "+r"(r) : "r"(x) : "cc"); + return r; + #elif MI_HAS_FAST_POPCOUNT + return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS); + #else + #define MI_HAS_FAST_BITSCAN 0 + return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS); + #endif +} + +static inline size_t mi_clz(size_t x) { + #if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0 + size_t r; + __asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc"); + return r; + #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) + return _lzcnt_u64(x); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) + unsigned long idx; + return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS); + #elif mi_has_builtinz(clz) + return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS); + #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86) + if (x==0) return MI_SIZE_BITS; + size_t r; + __asm ("bsr\t%1, %0" : "=r"(r) : "r"(x) : "cc"); + return (MI_SIZE_BITS - 1 - r); + #else + #define MI_HAS_FAST_BITSCAN 0 + return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS); + #endif +} + +#ifndef MI_HAS_FAST_BITSCAN +#define MI_HAS_FAST_BITSCAN 1 +#endif + +/* -------------------------------------------------------------------------------- + find trailing/leading zero (bit scan forward/reverse) +-------------------------------------------------------------------------------- */ + +// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's) +// return false if `x==0` (with `*idx` undefined) and true otherwise, +// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). +static inline bool mi_bsf(size_t x, size_t* idx) { + #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9) + // on x64 the carry flag is set on zero which gives better codegen + bool is_zero; + __asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" ); + return !is_zero; + #elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) + unsigned long i; + return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false); + #else + return (x!=0 ? (*idx = mi_ctz(x), true) : false); + #endif +} + +// Bit scan reverse: find the most significant bit that is set +// return false if `x==0` (with `*idx` undefined) and true otherwise, +// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). +static inline bool mi_bsr(size_t x, size_t* idx) { + #if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) + unsigned long i; + return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false); + #else + return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false); + #endif +} + + +/* -------------------------------------------------------------------------------- + rotate +-------------------------------------------------------------------------------- */ + +static inline size_t mi_rotr(size_t x, size_t r) { + #if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64) + return mi_builtin(rotateright64)(x,r); + #elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32) + return mi_builtin(rotateright32)(x,r); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64) + return _rotr64(x, (int)r); + #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32) + return _lrotr(x,(int)r); + #else + // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to + // avoid UB when `rshift==0`. See + const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1); + return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1)))); + #endif +} + +static inline size_t mi_rotl(size_t x, size_t r) { + #if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64) + return mi_builtin(rotateleft64)(x,r); + #elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32) + return mi_builtin(rotateleft32)(x,r); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64) + return _rotl64(x, (int)r); + #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32) + return _lrotl(x, (int)r); + #else + // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to + // avoid UB when `rshift==0`. See + const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1); + return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1)))); + #endif +} + +static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) { + #if mi_has_builtin(rotateleft32) + return mi_builtin(rotateleft32)(x,r); + #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32) + return _lrotl(x, (int)r); + #else + // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to + // avoid UB when `rshift==0`. See + const unsigned int rshift = (unsigned int)(r) & 31; + return ((x << rshift) | (x >> ((-rshift) & 31))); + #endif +} + + +#endif // MI_BITS_H diff --git a/csrc/mimalloc/include/mimalloc/internal.h b/csrc/mimalloc/include/mimalloc/internal.h new file mode 100644 index 000000000..67a08258b --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/internal.h @@ -0,0 +1,1503 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_H +#define MI_INTERNAL_H + +// -------------------------------------------------------------------------- +// This file contains the internal API's of mimalloc and various utility +// functions and macros. +// -------------------------------------------------------------------------- + +#include "types.h" +#include "track.h" +#include "bits.h" + + +// -------------------------------------------------------------------------- +// Compiler defines +// -------------------------------------------------------------------------- + +#if (MI_DEBUG>0) +#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) +#else +#define mi_trace_message(...) +#endif + +#define mi_decl_cache_align mi_decl_align(64) + +#if defined(_MSC_VER) +#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning +#define mi_decl_forceinline __forceinline +#define mi_decl_noinline __declspec(noinline) +#define mi_decl_thread __declspec(thread) +#define mi_decl_noreturn __declspec(noreturn) +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc +#if !MI_TRACK_ASAN +#define mi_decl_forceinline __attribute__((always_inline)) inline +#else +#define mi_decl_forceinline inline +#endif +#define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_thread __thread +#define mi_decl_noreturn __attribute__((noreturn)) +#define mi_decl_weak __attribute__((weak)) +#if defined(__MINGW32__) || defined(__CYGWIN__) +#define mi_decl_hidden +#else +#define mi_decl_hidden __attribute__((visibility("hidden"))) +#endif +#if (__GNUC__ >= 4) || defined(__clang__) +#define mi_decl_cold __attribute__((cold)) +#else +#define mi_decl_cold +#endif +#elif __cplusplus >= 201103L // c++11 +#define mi_decl_forceinline inline +#define mi_decl_noinline +#define mi_decl_thread thread_local +#define mi_decl_noreturn [[noreturn]] +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#else +#define mi_decl_forceinline inline +#define mi_decl_noinline +#define mi_decl_thread __thread // hope for the best :-) +#define mi_decl_noreturn +#define mi_decl_weak +#define mi_decl_hidden +#define mi_decl_cold +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] +#else +#define mi_unlikely(x) (x) +#define mi_likely(x) (x) +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc +#define mi_decl_maybe_unused __attribute__((unused)) +#elif __cplusplus >= 201703L // c++17 +#define mi_decl_maybe_unused [[maybe_unused]] +#else +#define mi_decl_maybe_unused +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#define mi_init_struct_zero { } +#else +#define mi_decl_externc +#define mi_init_struct_zero { 0 } +#endif + + +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + + +// -------------------------------------------------------------------------- +// Internal functions +// -------------------------------------------------------------------------- + + +// "libc.c" +#include +int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args); +int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied +bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); +char* _mi_strnstr(char* s, size_t max_len, const char* pat); +bool _mi_streq(const char* s, const char* t); +int _mi_getenv(const char* name, char* result, size_t result_size); +void _mi_detect_cpu_features(void); + +// "options.c" +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); +void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); +void _mi_raw_message(const char* fmt, ...); +void _mi_warning_message(const char* fmt, ...); +void _mi_verbose_message(const char* fmt, ...); +void _mi_trace_message(const char* fmt, ...); +void _mi_options_init(void); +void _mi_options_post_init(void); +long _mi_option_get_fast(mi_option_t option); +void _mi_error_message(int err, const char* fmt, ...); + +// random.c +void _mi_random_init(mi_random_ctx_t* ctx); +void _mi_random_init_weak(mi_random_ctx_t* ctx); +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); +uintptr_t _mi_random_next(mi_random_ctx_t* ctx); +uintptr_t _mi_theap_random_next(mi_theap_t* theap); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); +static inline uintptr_t _mi_random_shuffle(uintptr_t x); + +// prim-tls.c +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + +// subproc.c +mi_subproc_t* _mi_subproc_main_init(void); +void _mi_subproc_main_done(void); +mi_subproc_t* _mi_subproc_main(void); +bool _mi_subproc_is_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc(void); // current subproc of this thread +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_subprocs_unsafe_destroy_all(void); + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid ); +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid); +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p); + + +// init.c +mi_page_t* _mi_page_empty_get(void); +void _mi_auto_process_init(void); +void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; +bool _mi_is_redirected(void); +bool _mi_allocator_init(const char** message); +void _mi_allocator_done(void); +bool _mi_preloading(void); // true while the C runtime is not initialized yet +void _mi_thread_done(mi_theap_t* theap); +mi_theap_t* _mi_thread_init(void); +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); +bool _mi_is_empty_theap(const mi_theap_t* theap); + +// os.c +void _mi_os_init(void); // called from process init +void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); +void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid ); + +size_t _mi_os_page_size(void); +size_t _mi_os_guard_page_size(void); +size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); +bool _mi_os_has_virtual_reserve(void); +size_t _mi_os_virtual_address_bits(void); +size_t _mi_os_minimal_purge_size(void); + +bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); +void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); +mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); +mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); +mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); +bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); + +size_t _mi_os_secure_guard_page_size(void); +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); + +int _mi_os_numa_node(void); +int _mi_os_numa_node_count(void); + +void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_canuse_large_page(size_t size, size_t alignment); +size_t _mi_os_large_page_size(void); +void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); + +// threadlocal.c +#define mi_thread_local_key_fast ((mi_thread_local_t)1) + +mi_thread_local_t _mi_thread_local_create(void); +void _mi_thread_local_free( mi_thread_local_t key ); +bool _mi_thread_local_set( mi_thread_local_t key, void* val ); +void* _mi_thread_local_get( mi_thread_local_t key ); +void _mi_thread_locals_init(void); +void _mi_thread_locals_done(void); +void _mi_thread_locals_thread_done(void); + +// arena.c +mi_arena_id_t _mi_arena_id_none(void); +mi_arena_t* _mi_arena_from_id(mi_arena_id_t id); +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena); + +void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); +void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); +void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); +void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); + +mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); +void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap); +void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); +bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); + +// "page-map.c" +bool _mi_page_map_init(void); +mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); +void _mi_page_map_unregister(mi_page_t* page); +void _mi_page_map_unregister_range(void* start, size_t size); +mi_page_t* _mi_safe_ptr_page(const void* p); +void _mi_page_map_unsafe_destroy(void); + +// "page.c" +void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; + +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks +void _mi_page_unfull(mi_page_t* page); +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_deferred_free(mi_theap_t* theap, bool force); +void _mi_page_free_collect(mi_page_t* page, bool force); +void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); +bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); + +size_t _mi_page_stats_bin(const mi_page_t* page); // for stats +size_t _mi_bin_size(size_t bin); // for stats +size_t _mi_bin(size_t size); // for stats + +// "theap.c" +void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); +void _mi_theap_collect_retired(mi_theap_t* theap, bool force); +void _mi_theap_collect_abandon(mi_theap_t* theap); +bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); +void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); + +void _mi_heap_detach_theaps( mi_heap_t* heap ); +void _mi_tld_detach_theaps( mi_tld_t* tld ); +void _mi_theap_incref(mi_theap_t* theap); +void _mi_theap_decref(mi_theap_t* theap); +void _mi_theap_merge_stats(mi_theap_t* theap); + +// "heap.c" +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); +mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap +void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c" +void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c" +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main); +bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap); + +// "stats.c" +void _mi_stats_init(void); +void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from); + +mi_msecs_t _mi_clock_now(void); +mi_msecs_t _mi_clock_end(mi_msecs_t start); +mi_msecs_t _mi_clock_start(void); + +// "alloc.c" +void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); + +// "free.c" +void _mi_free_subproc_safe(void* p) mi_attr_noexcept; +void _mi_page_unguard_all(mi_page_t* page); +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; + +#if MI_DEBUG>1 +bool _mi_page_is_valid(mi_page_t* page); +#endif + + +// ------------------------------------------------------ +// Assertions +// ------------------------------------------------------ + +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept; +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + + +/* ----------------------------------------------------------- + Statistics (in `stats.c`) +----------------------------------------------------------- */ + +// add to stat keeping track of the peak +void __mi_stat_increase(mi_stat_count_t* stat, size_t amount); +void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount); +void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount); + +// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) +void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount); + +// counters can just be increased +void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); +void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); + +#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_decrease(heap,stat,amount) __mi_stat_decrease_mt( &(heap)->stats.stat, amount) +#define mi_heap_stat_adjust_increase(heap,stat,amnt) __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt) +#define mi_heap_stat_adjust_decrease(heap,stat,amnt) __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt) + +#define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount) +#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) + +#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) +#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) +#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) + +#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } +#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } + +/* ----------------------------------------------------------- + pthread thread locals +----------------------------------------------------------- */ + +#if MI_USE_PTHREADS + +#if defined(__APPLE__) && defined(__aarch64__) +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen +#else +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1)) +#endif + +#if defined(__linux__) && defined(__GLIBC__) +// pthread_getspecific returns NULL for invalid keys. +// see also: +#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1 +#endif + +mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init); + +static inline void* mi_pthread_key_get(pthread_key_t key) { + #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL + if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL; + #endif + return pthread_getspecific(key); +} + +static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) { + if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; } + else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); } + else return true; +} + +static inline void mi_pthread_key_delete(pthread_key_t* pkey) { + const pthread_key_t key = *pkey; + if (key!=MI_PTHREAD_KEY_INVALID) { + *pkey = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); + } +} +#endif + +/* ----------------------------------------------------------- + Options (exposed for the debugger) +----------------------------------------------------------- */ +typedef enum mi_option_init_e { + MI_OPTION_UNINIT, // not yet initialized + MI_OPTION_DEFAULTED, // not found in the environment, use default value + MI_OPTION_INITIALIZED // found in environment or set explicitly +} mi_option_init_t; + +typedef struct mi_option_desc_s { + long value; // the value + mi_option_init_t init; // is it initialized yet? (from the environment) + mi_option_t option; // for debugging: the option index should match the option + const char* name; // option name without `mimalloc_` prefix + const char* legacy_name; // potential legacy option name +} mi_option_desc_t; + + + +/* ----------------------------------------------------------- + Inlined definitions +----------------------------------------------------------- */ +#define MI_UNUSED(x) (void)(x) +#if (MI_DEBUG>1) +#define MI_UNUSED_RELEASE(x) +#else +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) +#endif + +#define MI_INIT4(x) x(),x(),x(),x() +#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) +#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) +#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) +#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) +#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) +#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) + +#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x() +#define MI_INIT5(x) MI_INIT4(x),x() +#define MI_INIT6(x) MI_INIT4(x),x(),x() + +#include +// initialize a local variable to zero; use memset as compilers optimize constant sized memset's +#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) + +// Is `x` a power of two? (0 is considered a power of two) +static inline bool _mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} + +// valid alignment values are as posix memalign: +static inline bool mi_alignment_is_valid(size_t alignment) { + return ((alignment!=0) && _mi_is_power_of_two(alignment)); +} + +// Is a pointer aligned? +static inline bool _mi_is_aligned(const void* p, size_t alignment) { + return (alignment==0 || ((uintptr_t)p % alignment) == 0); +} + +// Align upwards +static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + +// Align a pointer upwards +static inline void* _mi_align_up_ptr(const void* p, size_t alignment) { + return (void*)_mi_align_up((uintptr_t)p, alignment); +} + +// Align down +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz/alignment)*alignment); + } +} + +// Align a pointer downwards +static inline void* _mi_align_down_ptr(const void* p, size_t alignment) { + return (void*)_mi_align_down((uintptr_t)p, alignment); +} + +// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. +static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { + mi_assert_internal(divider != 0); + return (divider == 0 ? size : ((size + divider - 1) / divider)); +} + + +// clamp an integer +static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) { + if (sz < min) return min; + else if (sz > max) return max; + else return sz; +} + +// Is memory zero initialized? +static inline bool mi_mem_is_zero(const void* p, size_t size) { + for (size_t i = 0; i < size; i++) { + if (((uint8_t*)p)[i] != 0) return false; + } + return true; +} + +// Align a byte size to a size in _machine words_, +// i.e. byte size == `wsize*sizeof(void*)`. +static inline size_t _mi_wsize_from_size(size_t size) { + mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); + return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); +} + +// Overflow detecting multiply +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) +#include // UINT_MAX, ULONG_MAX +#if defined(_CLOCK_T) // for Illumos +#undef _CLOCK_T +#endif +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #if (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, (unsigned int *)total); + #elif (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); + #else + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); + #endif +} +#else /* __builtin_umul_overflow is unavailable */ +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + *total = count*size; + if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) { // did size and count fit both in the lower half bits of a size_t? + return false; + } + else { + return (size!=0 && (SIZE_MAX / size) < count); + } +} +#endif + +// Safe multiply `count*size` into `total`; return `true` on overflow. +static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { + if (count==1) { // quick check for the case where count is one (common for C++ allocators) + *total = size; + return false; + } + else if mi_likely(!mi_mul_overflow(count, size, total)) { + return false; + } + else { + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); + #endif + *total = SIZE_MAX; + return true; + } +} + + +/*---------------------------------------------------------------------------------------- + Heap functions +------------------------------------------------------------------------------------------- */ + +extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) +extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated + + +static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) { + return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap); +} + +static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) { + mi_heap_t* const heap = _mi_theap_heap_peek(theap); + mi_assert_internal(heap!=NULL); + return heap; +} + +static inline bool mi_theap_is_initialized(const mi_theap_t* theap) { + return (theap != NULL && _mi_theap_heap_peek(theap) != NULL); +} + +static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc); + return subproc; +} + +static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { + mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = _mi_wsize_from_size(size); + mi_assert_internal(idx < MI_PAGES_DIRECT); + return theap->pages_free_direct[idx]; +} + +static inline bool mi_theap_is_detached(mi_theap_t* theap) { + return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); +} + +static inline bool mi_theap_matches_thread(mi_theap_t* theap) { + const mi_threadid_t tid = _mi_thread_id(); + return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); +} + +/* ----------------------------------------------------------- + The page map maps addresses to `mi_page_t` pointers +----------------------------------------------------------- */ + +#if MI_PAGE_MAP_FLAT + +// flat page-map committed on demand, using one byte per slice (64 KiB). +// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) +// used by default on <= 40 bit virtual address spaces. +extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map; +extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; + +static inline size_t _mi_page_map_index(const void* p) { + return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT); +} + +static inline uint8_t _mi_page_map_at(size_t idx) { + return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx]; +} + +static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) { + const size_t idx = _mi_page_map_index(p); + const size_t ofs = _mi_page_map_at(idx); + if (valid != NULL) { *valid = (ofs != 0); } + return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT); +} + +static inline mi_page_t* _mi_checked_ptr_page(const void* p) { + #if MI_MIN_VABITS < MI_INTPTR_BITS + if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; + } + #endif + bool valid; + mi_page_t* const page = _mi_ptr_page_ex(p, &valid); + return (valid ? page : NULL); +} + +static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + return _mi_ptr_page_ex(p, NULL); +} + +#else + +// 2-level page map: +// double indirection, but low commit and low virtual reserve. +// +// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB. +// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well) +// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space +// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size. +#define MI_PAGE_MAP_SUB_SHIFT (13) +#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) +#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) + +typedef mi_page_t** mi_submap_t; +typedef struct mi_page_map_s { + _Atomic(size_t) committed_count; // currently committed entries + size_t reserved_size; // full reserved size (mi_page_map_t + submaps) + mi_memid_t memid; // provenance + mi_lock_t lock; // used when allocating new submaps + _Atomic(mi_submap_t) submaps[1]; +} mi_page_map_t; + +extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; + +static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { + const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); + if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; } + return (u / MI_PAGE_MAP_SUB_COUNT); +} + +static inline mi_page_map_t* _mi_page_map(void) { + return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); +} + +static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { + return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); +} + +static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + const mi_page_map_t* pmap = _mi_page_map(); + size_t sub_idx; + const size_t idx = _mi_page_map_index(p, &sub_idx); + return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL +} + +static inline mi_page_t* _mi_checked_ptr_page(const void* p) { + const mi_page_map_t* pmap = _mi_page_map(); + size_t sub_idx; + const size_t idx = _mi_page_map_index(p, &sub_idx); + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(idx >= committed_count) return NULL; + // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized + // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + // if mi_unlikely(idx >= committed_count) return NULL; + // } + // #endif + mi_submap_t const sub = _mi_page_map_at(pmap,idx); + if mi_unlikely(sub == NULL) return NULL; + return sub[sub_idx]; +} + +#endif + +#if MI_PAGE_META_IS_ALIGNED +// if the page meta data is aligned in front of pages we can find it efficiently +// without needing to go through the page map (for valid pointers). +static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); + return &page_metas[page_idx]; +} + +static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { + mi_page_t* const page = _mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return NULL; + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { + _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p); + return NULL; + } + #endif + return mi_atomic_load_acquire(&page->self); +} +#endif + +static inline mi_page_t* _mi_ptr_page(const void* p) { + mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); + #if MI_SECURE || MI_FREE_IS_CHECKED + return _mi_checked_ptr_page(p); + #elif MI_PAGE_META_IS_ALIGNED + return _mi_aligned_ptr_page(p); + #elif MI_DEBUG + return _mi_checked_ptr_page(p); + #else + return _mi_unchecked_ptr_page(p); + #endif +} + + +// Get the block size of a page +static inline size_t mi_page_block_size(const mi_page_t* page) { + mi_assert_internal(page->block_size > 0); + return page->block_size; +} + +// Page start +static inline uint8_t* mi_page_start(const mi_page_t* page) { + // multiplication must be done in `size_t` + return (uint8_t*)page + page->page_offset; +} + +static inline size_t mi_page_size(const mi_page_t* page) { + return mi_page_block_size(page) * page->reserved; +} + +static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) { + if (size) { *size = mi_page_size(page); } + return mi_page_start(page); +} + +static inline size_t mi_page_info_size(void) { + return _mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE); +} + +static inline bool mi_page_contains_address(const mi_page_t* page, const void* p) { + size_t psize; + uint8_t* start = mi_page_area(page, &psize); + return (start <= (uint8_t*)p && (uint8_t*)p < start + psize); +} + +static inline bool mi_page_is_in_arena(const mi_page_t* page) { + return (page->memid.memkind == MI_MEM_ARENA); +} + +static inline bool mi_page_is_singleton(const mi_page_t* page) { + return (page->reserved == 1); +} + +// Get the usable block size of a page without fixed padding. +// This may still include internal padding due to alignment and rounding up size classes. +static inline size_t mi_page_usable_block_size(const mi_page_t* page) { + return mi_page_block_size(page) - MI_PADDING_SIZE; +} + +static inline bool mi_page_meta_is_separated(const mi_page_t* page) { + #if MI_PAGE_META_IS_ALIGNED + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #elif MI_PAGE_META_IS_SEPARATED + // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED + return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED(page); + return false; + #endif +} + +static inline uint8_t* mi_page_slice_start(const mi_page_t* page) { + if (mi_page_meta_is_separated(page)) { + // page meta info is at a separate location (at `arena->pages`) + return (uint8_t*)_mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN); + } + else { + // page meta info is at the start of the page slices + return (uint8_t*)page; + } +} + +// This gives the offset relative to the start slice of a page. +static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) { + return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; +} + +// How much of the page is committed relative to the slice start? (or 0 if fully committed already) +static inline size_t mi_page_slice_committed(const mi_page_t* page) { + return ((size_t)page->slice_pcommitted * _mi_os_page_size()); +} + +// Currently committed part of a page +static inline size_t mi_page_committed(const mi_page_t* page) { + const size_t slice_committed = mi_page_slice_committed(page); + return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); +} + +// are all blocks in a page freed? +// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. +static inline bool mi_page_all_free(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->used == 0); +} + +// are there immediately available blocks, i.e. blocks available on the free list. +static inline bool mi_page_immediate_available(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->free != NULL); +} + + +// is the page not yet used up to its reserved space? +static inline bool mi_page_is_expandable(const mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(page->capacity <= page->reserved); + return (page->capacity < page->reserved); +} + + +static inline bool mi_page_is_full(const mi_page_t* page) { + const bool full = (page->reserved == page->used); + mi_assert_internal(!full || page->free == NULL); + return full; +} + +// is more than 7/8th of a page in use? +static inline bool mi_page_is_mostly_used(const mi_page_t* page) { + if (page==NULL) return true; + uint16_t frac = page->reserved / 8U; + return (page->reserved - page->used <= frac); +} + +// is more than (n-1)/n'th of a page in use? +static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) { + if (page==NULL) return true; + uint16_t frac = page->reserved / n; + return (page->reserved - page->used <= frac); +} + + +static inline bool mi_page_is_huge(const mi_page_t* page) { + return (mi_page_is_singleton(page) && + (page->block_size > MI_LARGE_MAX_OBJ_SIZE || + (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page))); +} + +static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) { + mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)]; + if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); } + return pq; +} + +static inline size_t mi_page_min_commit_size(void) { + const size_t psize = _mi_os_page_size(); + return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize); +} + +//----------------------------------------------------------- +// Page thread id and flags +//----------------------------------------------------------- + +// Thread id of thread that owns this page (with flags in the bottom 2 bits) +static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { + return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); +} + +// Plain thread id of the thread that owns this page +static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) { + return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK); +} + +static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) { + return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK); +} + +static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) { + mi_page_flags_t old; + if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); } + else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); } + return ((old & newflag) == newflag); +} + +static inline bool mi_page_is_in_full(const mi_page_t* page) { + return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0); +} + +static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { + const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE); + if (was_in_full != in_full) { + // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220) + mi_theap_t* const theap = page->theap; + mi_assert_internal(theap!=NULL); + if (theap != NULL) { + mi_assert_internal(page->capacity==page->reserved); + const size_t size = page->reserved * mi_page_block_size(page); + if (in_full) { theap->pages_full_size += size; } + else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; } + } + } +} + +static inline bool mi_page_has_interior_pointers(const mi_page_t* page) { + return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0); +} + +static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) { + mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS); +} + +static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { + // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy + page->theap = theap; + const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); + + // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`). + mi_threadid_t xtid_old = mi_page_xthread_id(page); + mi_threadid_t xtid; + do { + xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK); + } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid)); +} + +static inline bool mi_page_is_abandoned(const mi_page_t* page) { + // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free) + return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED); +} + +static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) { + return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED); +} + +static inline void mi_page_set_abandoned_mapped(mi_page_t* page) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED); +} + +static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { + mi_assert_internal(mi_page_is_abandoned_mapped(page)); + mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK); +} + + +static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { + mi_assert_internal(!mi_page_is_abandoned(page)); + mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty); + return page->theap; +} + +static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { + mi_assert_internal(!mi_page_is_abandoned(page)); + mi_assert_internal(page->theap != NULL); + return page->theap->tld; +} + + +static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { + mi_heap_t* heap = page->heap; + mi_assert_internal(heap != NULL); + return heap; +} + +static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) { + mi_heap_t* const heap = mi_page_heap(page); + return heap->subproc; +} + +static inline mi_heap_t* mi_arena_heap_main(const mi_arena_t* arena) { + return _mi_subproc_heap_main(arena->subproc); +} + +static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) { + return _mi_subproc_heap_main(heap->subproc); +} + +static inline bool _mi_is_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (mi_heap_get_heap_main(heap) == heap); +} + +static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (_mi_subproc_main()->heap_main == heap); +} + +//----------------------------------------------------------- +// Thread free list and ownership +//----------------------------------------------------------- + +// Thread free flag helpers +static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { + return (mi_block_t*)(tf & ~1); +} +static inline bool mi_tf_is_owned(mi_thread_free_t tf) { + return ((tf & 1) == 1); +} +static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) { + return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0)); +} + +// Thread free access +static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { + return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); +} + +// are there any available blocks? +static inline bool mi_page_has_any_available(const mi_page_t* page) { + mi_assert_internal(page != NULL && page->reserved > 0); + return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); +} + +// Owned? +static inline bool mi_page_is_owned(const mi_page_t* page) { + return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); +} + +// get ownership; returns true if the page was not owned before. +static inline bool mi_page_claim_ownership(mi_page_t* page) { + const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1); + return ((old&1)==0); +} + + +/* ------------------------------------------------------------------- + Guarded objects +------------------------------------------------------------------- */ +#if MI_GUARDED +// we always align guarded pointers in a block at an offset +// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones +#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) +#endif + +static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { +#if MI_GUARDED + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +#if MI_GUARDED +static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { + // this code is written to result in fast assembly as it is on the hot path for allocation + const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. + if mi_likely(count != 0) { + // no sample + theap->guarded_sample_count = count; + return false; + } + else { + // count == 0 + const size_t rate = theap->guarded_sample_rate; + if (rate == 0) { + return false; // don't write to an empty theap + } + else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_count = rate; // reset + return true; + } + else { + // failed size criteria, rewind count + theap->guarded_sample_count = 1; + return false; + } + } +} + +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +#endif + +/* ------------------------------------------------------------------- +Encoding/Decoding the free list next pointers + +This is to protect against buffer overflow exploits where the +free list is mutated. Many hardened allocators xor the next pointer `p` +with a secret key `k1`, as `p^k1`. This prevents overwriting with known +values but might be still too weak: if the attacker can guess +the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). +Moreover, if multiple blocks can be read as well, the attacker can +xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot +about the pointers (and subsequently `k1`). + +Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + next = (mi_block_t*)block->next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); + return next; +} + +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { + mi_track_mem_undefined(block,sizeof(mi_block_t)); + #if MI_ENCODE_FREELIST + block->next = mi_ptr_encode(null, next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + block->next = (mi_encoded_t)next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); +} + +mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c + +static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { + #if MI_ENCODE_FREELIST + mi_block_t* next = mi_block_nextx(page,block,page->keys); + // check for free list corruption: is `next` at least in the same page? + // todo: check if `next` is `page->block_size` aligned? + if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { + return _mi_block_next_is_corrupted(page,block,next); // returns NULL + } + return next; + #else + MI_UNUSED(page); + return mi_block_nextx(page,block,NULL); + #endif +} + +static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { + #if MI_ENCODE_FREELIST + mi_block_set_nextx(page,block,next, page->keys); + #else + MI_UNUSED(page); + mi_block_set_nextx(page,block,next,NULL); + #endif +} + + + +/* ----------------------------------------------------------- + arena blocks +----------------------------------------------------------- */ + +// Blocks needed for a given byte size +static inline size_t mi_slice_count_of_size(size_t size) { + return _mi_divide_up(size, MI_ARENA_SLICE_SIZE); +} + +// Byte size of a number of blocks +static inline size_t mi_size_of_slices(size_t bcount) { + return (bcount * MI_ARENA_SLICE_SIZE); +} + + +/* ----------------------------------------------------------- + memory id's +----------------------------------------------------------- */ + +static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) { + mi_memid_t memid; + _mi_memzero_var(memid); + memid.memkind = memkind; + return memid; +} + +static inline mi_memid_t _mi_memid_none(void) { + return _mi_memid_create(MI_MEM_NONE); +} + +static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) { + mi_memid_t memid = _mi_memid_create(MI_MEM_OS); + memid.mem.os.base = base; + memid.mem.os.size = size; + memid.initially_committed = committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_large; + return memid; +} + +static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) { + mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.is_pinned = true; + return memid; +} + +static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) { + mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.initially_zero = iszero; + memid.is_pinned = true; + return memid; +} + +static inline size_t _mi_memid_size(mi_memid_t memid) { + if (mi_memid_is_os(memid)) { + return memid.mem.os.size; + } + else if (memid.memkind == MI_MEM_ARENA) { + return mi_size_of_slices(memid.mem.arena.slice_count); + } + else if (memid.memkind == MI_MEM_MALLOC) { + return memid.mem.malloc.size; + } + else { + mi_assert_internal(mi_memid_needs_no_free(memid)); + return 0; + } +} + +// ------------------------------------------------------------------- +// Fast "random" shuffle +// ------------------------------------------------------------------- + +static inline uintptr_t _mi_random_shuffle(uintptr_t x) { + if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros +#if (MI_INTPTR_SIZE>=8) + // by Sebastiano Vigna, see: + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9UL; + x ^= x >> 27; + x *= 0x94d049bb133111ebUL; + x ^= x >> 31; +#elif (MI_INTPTR_SIZE==4) + // by Chris Wellons, see: + x ^= x >> 16; + x *= 0x7feb352dUL; + x ^= x >> 15; + x *= 0x846ca68bUL; + x ^= x >> 16; +#endif + return x; +} + + +// --------------------------------------------------------------------------------- +// Provide our own `_mi_memcpy/set` for potential performance optimizations. +// +// For now, only on x64/x86 we optimize to `rep movsb/stosb`. +// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support +// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. +// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. +// --------------------------------------------------------------------------------- + +#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) + +extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c +extern mi_decl_hidden size_t _mi_cpu_stosb_max; + +static inline void mi_rep_movsb(void* dst, const void* src, size_t n) { + #if defined(__GNUC__) + __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory"); + #else + __movsb((unsigned char*)dst, (const unsigned char*)src, n); + #endif +} + +static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { + #if defined(__GNUC__) + __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory"); + #else + __stosb((unsigned char*)dst, val, n); + #endif +} + +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) + mi_rep_movsb(dst, src, n); + } + else { + memcpy(dst, src, n); + } +} + +static inline void _mi_memset(void* dst, int val, size_t n) { + if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 + mi_rep_stosb(dst, (uint8_t)val, n); + } + else { + memset(dst, val, n); + } +} + +#else + +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + memcpy(dst, src, n); +} + +static inline void _mi_memset(void* dst, int val, size_t n) { + memset(dst, val, n); +} + +#endif + +// ------------------------------------------------------------------------------- +// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned +// This is used for example in `mi_realloc`. +// ------------------------------------------------------------------------------- + +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) + +// On GCC/CLang we provide a hint that the pointers are word aligned. +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); + _mi_memcpy(adst, asrc, n); +} + +static inline void _mi_memset_aligned(void* dst, int val, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + _mi_memset(adst, val, n); +} + +#else + +// Default fallback on `_mi_memcpy` +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + _mi_memcpy(dst, src, n); +} + +static inline void _mi_memset_aligned(void* dst, int val, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + _mi_memset(dst, val, n); +} + +#endif + +static inline void _mi_memzero(void* dst, size_t n) { + _mi_memset(dst, 0, n); +} + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + _mi_memset_aligned(dst, 0, n); +} + + + +#endif // MI_INTERNAL_H diff --git a/csrc/mimalloc/include/mimalloc/prim-tls.h b/csrc/mimalloc/include/mimalloc/prim-tls.h new file mode 100644 index 000000000..b0b5c137a --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/prim-tls.h @@ -0,0 +1,424 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_TLS_H +#define MIMALLOC_PRIM_TLS_H + +#include "types.h" +#include "internal.h" // mi_decl_hidden + +// -------------------------------------------------------------------------- +// We need fast access to both a unique thread id (in `free.c:mi_free`) and +// to a thread-local theap pointer (in `alloc.c:mi_malloc`). +// +// For performance, we tend to use specialized code for various platforms. +// This leads to quite a few ifdefs but it is just for performance and there +// is always a portable fallback (based on regular thread local variables). +// +// Windows : use NtCurrentTeB and TlsAlloc (MI_TLS_MODEL_WIN32) +// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL) +// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS) +// Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better? +// -------------------------------------------------------------------------- + +// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage +// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block +// static inline void mi_prim_tls_slot_set(size_t slot, void* value); + +static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread +static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap +static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api +static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap +static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap +static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached +static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) + + +// Default TLS model +#if !defined(MI_TLS_MODEL_LOCAL) && !defined(MI_TLS_MODEL_PTHREADS) && !defined(MI_TLS_MODEL_FIXED) && !defined(MI_TLS_MODEL_WIN32) +#if defined(_WIN32) +#define MI_TLS_MODEL_WIN32 1 +#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__) // and FreeBSD? +#define MI_TLS_MODEL_PTHREADS 1 +#else +#define MI_TLS_MODEL_LOCAL 1 +#endif +#endif + + +//------------------------------------------------------------------- +// Access to the thread pointer +//------------------------------------------------------------------- + +// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id +// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) +// Nevertheless, it seems needed on older graviton platforms (see issue #851). +// For now, we only enable this for specific platforms. +#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */ + #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ + && !defined(__CYGWIN__) \ + && !defined(MI_LIBC_MUSL) \ + && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ + #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ + || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ + || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv__))) + #define MI_USE_BUILTIN_THREAD_POINTER 1 + #endif + #endif +#endif + +#if defined(MI_PRIM_THREAD_POINTER) // potential user override +static inline void** mi_prim_thread_pointer(void) { + return MI_PRIM_THREAD_POINTER(); +} +#elif defined(_WIN32) +static inline void** mi_prim_thread_pointer(void) { + return (void**)NtCurrentTeb(); +} +#elif MI_USE_BUILTIN_THREAD_POINTER +static inline void** mi_prim_thread_pointer(void) { + return (void**)__builtin_thread_pointer(); +} +#elif defined(__GNUC__) && !defined(__CYGWIN__) + #if defined(__aarch64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) + __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + return tcb; + } + #elif defined(__riscv) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ ("mv %0, tp" : "=r" (tcb)); + return tcb; + } + #elif defined(__arm__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb)); + return tcb; + } + #elif defined(__i386__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS + return tcb; + } + #elif defined(__x86_64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) + __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS + #elif (MI_INTPTR_SIZE==4) + __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI + #else + __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS + #endif + return tcb; + } + #else + #define MI_NO_THREAD_POINTER (1) + #endif +#elif MI_USE_PTHREADS && defined(__APPLE__) +static inline void** mi_prim_thread_pointer(void) { + return (void**)pthread_self(); +} +#else +#define MI_NO_THREAD_POINTER (1) +#endif + +#if !MI_NO_THREAD_POINTER +#define MI_HAS_TLS_SLOT (1) +static inline void* mi_prim_tls_slot(size_t slot) { + #if defined(_WIN32) + #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) + return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs + #elif _M_IX86 && !defined(_M_ARM64EC) + return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs + #else + return mi_prim_thread_pointer()[slot]; + #endif + #else + return mi_prim_thread_pointer()[slot]; + #endif +} + +static inline void mi_prim_tls_slot_set(size_t slot, void* value) { + mi_prim_thread_pointer()[slot] = value; +} +#endif + +/* ---------------------------------------------------------------------------------------- + Get a unique thread-id for a thread +---------------------------------------------------------------------------------------- */ + +// Get a unique id for the current thread. +#if defined(MI_PRIM_THREAD_ID) +static inline mi_threadid_t __mi_prim_thread_id(void) { + return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) +} +#elif !MI_NO_THREAD_POINTER +static inline mi_threadid_t __mi_prim_thread_id(void) { + #if defined(__BIONIC__) + return (mi_threadid_t)mi_prim_tls_slot(1); + #else + return (mi_threadid_t)mi_prim_thread_pointer(); + #endif +} +#else +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). +extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper; +static inline mi_threadid_t __mi_prim_thread_id(void) { + return (uintptr_t)&__mi_thread_id_helper; +} +#endif + +static inline mi_threadid_t _mi_prim_thread_id(void) { + const mi_threadid_t tid = __mi_prim_thread_id(); + mi_assert_internal(tid > MI_THREADID_DETACHED); + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? + return tid; +} + + +/* ---------------------------------------------------------------------------------------- +Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`). + +This is inlined here as it is on the fast path for allocation functions. +We have 4 models: + +- MI_TLS_MODEL_LOCAL: use regular thread local (default on Linux, FreeBSD, etc) + On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a + thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures + that the storage will always be available and properly initialized (with an empty theap). + + On some platforms the underlying TLS implementation (or the loader) will call itself `malloc` + on a first access to a thread local and recurse in the MI_TLS_MODEL_LOCAL. + A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process + is initialized before accessing the thread-local. This is a check in the fast path though + so this should be avoided. + +- MI_TLS_MODEL_PTHREADS: use `pthread_getspecific`. (default on macOS and OpenBSD, maybe good for Android as well?) + Use pthread local storage. Can be as fast as thread locals on many platforms (like recent macOS). + +- MI_TLS_MODEL_FIXED: use a fixed slot in the TLS block. + This reserves an unused and fixed TLS slot. This is fast and avoids the problem + where the underlying TLS implementation (or the loader) will call itself `malloc` + on a first access to a thread local (and recurse in the MI_TLS_MODEL_LOCAL). + This goes wrong though if the OS or a library uses the same fixed slot, and also + prevents multiple instances of mimalloc in the same process. + +- MI_TLS_MODEL_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows) + We use TlsAlloc'd slot. First tries to use one of the "direct" first 64 slots which + are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots). + (If the allocated slot happens to always be under 64 for a particular program, + one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.) + +Each model should define `MI_THEAP_INITASNULL` to signify that the initial value +returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap). +This incurs an extra check in the fast path (but can often be combined in an existing check). +------------------------------------------------------------------------------------------- */ + +#if !defined(MI_TLS_RECURSE_GUARD) && MI_TLS_MODEL_LOCAL && defined(__APPLE__) +#define MI_TLS_RECURSE_GUARD 1 // macOS can allocate on thread-local initialization +#endif + +// Declared this way to optimize register spills and branches +mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void); + +static inline mi_theap_t* __mi_theap_empty(void) { + #if __GNUC__ + __asm(""); // prevent conditional load + return (mi_theap_t*)&_mi_theap_empty; + #else + return _mi_theap_empty_get(); + #endif +} + +#if MI_TLS_MODEL_LOCAL +// Thread local with an initial value (default on Linux). Very efficient. +extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from +extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap + +// defined in `init.c`; do not use these directly +extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? + +static inline mi_theap_t* _mi_theap_default(void) { + #if defined(MI_TLS_RECURSE_GUARD) + if mi_unlikely(!_mi_process_is_initialized) return _mi_theap_empty_get(); + #endif + return __mi_theap_default; +} + +static inline mi_theap_t* _mi_theap_cached(void) { + return __mi_theap_cached; +} + +#elif MI_TLS_MODEL_PTHREADS +// Dynamic pthread slots. This can be fast depending on the platform (default for macOS and OpenBSD) +// On some platforms (like macOS), the loader might allocate on thread local declarations which +// can be avoided with pthreads. +#define MI_THEAP_INITASNULL 1 + +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key; +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key; + +static inline mi_theap_t* _mi_theap_default(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); + #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT + // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc` + // todo: this is probably also the case on x64 and power pc? + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); + #else + return (mi_theap_t*)mi_pthread_key_get(key); + #endif +} + +static inline mi_theap_t* _mi_theap_cached(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); + #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); + #else + return (mi_theap_t*)mi_pthread_key_get(key); + #endif +} + +#elif MI_TLS_MODEL_WIN32 +// Dynamic TLS slots -- this is the default on Windows. +#define MI_THEAP_INITASNULL 1 + +// We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available) +// See for the offsets. +#if MI_SIZE_SIZE==4 +#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_INTPTR_SIZE) +#else +#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) +#endif + +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot; + +static inline mi_theap_t* _mi_theap_default(void) { + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_default_slot); + mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); + #if !MI_WIN_DIRECT_TLS + if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? + mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is actually the expansion slot entry + if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot)]; + } + } + #endif + return theap; +} + +static inline mi_theap_t* _mi_theap_cached(void) { + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_cached_slot); + mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); + #if !MI_WIN_DIRECT_TLS + if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? + mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry + if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot)]; + } + } + #endif + return theap; +} + +#elif MI_TLS_MODEL_FIXED +// Fixed TLS slot. Can be the fastest approach, but does not work if there are multiple instances of +// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be +// guaranteed to work in general. +#define MI_THEAP_INITASNULL 1 + +#if !MI_HAS_TLS_SLOT +#error this platform cannot support MI_TLS_MODEL_FIXED without defining mi_prim_tls_slot +#endif + +#if !defined(MI_TLS_MODEL_FIXED_DEFAULT) + #if defined(__APPLE__) && !defined(__POWERPC__) // macOS on arm64 or x64 + // we use the last two swift framework slots which seem unused. + // we may want to use slot 6 and 11 instead which are only used by Windows emulation. + // see for assigned slots + #define MI_TLS_MODEL_FIXED_DEFAULT 108 + #define MI_TLS_MODEL_FIXED_CACHED 109 + #elif defined(_WIN32) + // we use two seemingly unused fields in the Windows TEB. + // see + #define MI_TLS_MODEL_FIXED_DEFAULT 5 // arbitrary user pointer + #define MI_TLS_MODEL_FIXED_CACHED 7 // environment pointer (used by OS2) + #else + #error define the TLS model fixed slots (or change the TLS model away from MI_TLS_MODEL_FIXED) + #endif +#endif + +static inline mi_theap_t* _mi_theap_default(void) { + return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_DEFAULT); +} + +static inline mi_theap_t* _mi_theap_cached(void) { + return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_CACHED); +} + +#else +#error "no TLS model is defined for this platform?" +#endif + + +// Check if a thread is initialized (without using a thread-local if using fixed slots) +static inline bool _mi_thread_is_initialized(void) { + return mi_theap_is_initialized(_mi_theap_default()); +} + +// Get (and possible create) the theap belonging to a heap +// We cache the last accessed theap in `_mi_theap_cached` for better performance. +static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + return _mi_heap_theap_get_or_init(heap); +} + +// Get the theap belonging to a heap without creating it if it is not yet initialized. +static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // don't update the cache on a query + mi_assert_internal(theap==NULL || (!_mi_is_empty_theap(theap) && theap->heap==heap)); + return theap; +} + +// Find the associated theap or NULL if it does not exist (during shutdown) +// Should be fast as it is called in `free.c:mi_free_try_collect`. +static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { + mi_heap_t* const heap = mi_page_heap(page); + mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); + if (theap==NULL) return NULL; + if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation + mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap)); + // note: for pages allocated by a detached theap, the returned theap may not be detached + return theap; +} + +#endif // MI_PRIM_TLS_H diff --git a/csrc/mimalloc/include/mimalloc/prim.h b/csrc/mimalloc/include/mimalloc/prim.h new file mode 100644 index 000000000..82b2d9c3d --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/prim.h @@ -0,0 +1,136 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_H +#define MIMALLOC_PRIM_H + +#include "types.h" + +// -------------------------------------------------------------------------- +// This file specifies the primitive portability API. +// Each OS/host needs to implement these primitives, see `src/prim` +// for implementations on Window, macOS, WASI, and Linux/Unix. +// +// note: on all primitive functions, we always have result parameters != NULL, and: +// addr != NULL and page aligned +// size > 0 and page aligned +// the return value is an error code as an `int` where 0 is success +// -------------------------------------------------------------------------- + +// OS memory configuration +typedef struct mi_os_mem_config_s { + size_t page_size; // default to 4KiB + size_t large_page_size; // 0 if not supported, usually 2MiB + size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) + size_t physical_memory_in_kib; // physical memory size in KiB + size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) + bool has_overcommit; // can we reserve more memory than can be actually committed? + bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc) + bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) + bool has_transparent_huge_pages; // true if transparent huge pages are enabled (on Linux) +} mi_os_mem_config_t; + +// Initialize +void _mi_prim_mem_init( mi_os_mem_config_t* config ); + +// Free OS memory +int _mi_prim_free(void* addr, size_t size ); + +// Allocate OS memory. Return NULL on error. +// The `try_alignment` is just a hint and the returned pointer does not have to be aligned. +// If `commit` is false, the virtual memory range only needs to be reserved (with no access) +// which will later be committed explicitly using `_mi_prim_commit`. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored. +// pre: !commit => !allow_large +// try_alignment >= _mi_os_page_size() and a power of 2 +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); + +// Commit memory. Returns error code or 0 on success. +// For example, on Linux this would make the memory PROT_READ|PROT_WRITE. +// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows) +int _mi_prim_commit(void* addr, size_t size, bool* is_zero); + +// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true +// if the memory would need to be re-committed. For example, on Windows this is always true, +// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit. +// pre: needs_recommit != NULL +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); + +// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment. +// Returns error code or 0 on success. +int _mi_prim_reset(void* addr, size_t size); + +// Reuse memory. This is called for memory that is already committed but +// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false. +// Returns error code or 0 on success. On most platforms this is a no-op. +int _mi_prim_reuse(void* addr, size_t size); + +// Protect memory. Returns error code or 0 on success. +int _mi_prim_protect(void* addr, size_t size, bool protect); + +// Allocate huge (1GiB) pages possibly associated with a NUMA node. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// pre: size > 0 and a multiple of 1GiB. +// numa_node is either negative (don't care), or a numa node number. +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); + +// Return the current NUMA node +size_t _mi_prim_numa_node(void); + +// Return the number of logical NUMA nodes +size_t _mi_prim_numa_node_count(void); + +// Clock ticks +mi_msecs_t _mi_prim_clock_now(void); + +// Return process information (only for statistics) +typedef struct mi_process_info_s { + mi_msecs_t elapsed; + mi_msecs_t utime; + mi_msecs_t stime; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; +} mi_process_info_t; + +void _mi_prim_process_info(mi_process_info_t* pinfo); + +// Default stderr output. (only for warnings etc. with verbose enabled) +// msg != NULL && _mi_strlen(msg) > 0 +void _mi_prim_out_stderr( const char* msg ); + +// Get an environment variable. (only for options) +// name != NULL, result != NULL, result_size >= 64 +// Return 1 for success, 0 if not found, +// and -1 on error (for example, if `getenv` cannot be called yet during preloading). +int _mi_prim_getenv(const char* name, char* result, size_t result_size); + + +// Fill a buffer with strong randomness; return `false` on error or if +// there is no strong randomization available. +bool _mi_prim_random_buf(void* buf, size_t buf_len); + +// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination. +void _mi_prim_thread_init_auto_done(void); + +// Called on process exit and may take action to clean up resources associated with the thread auto done. +void _mi_prim_thread_done_auto_done(void); + +// Called when the default theap for a thread changes +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap); + +// Is this thread part of a thread pool? +bool _mi_prim_thread_is_in_threadpool(void); + +// Yield to other threads. Should be similar to `sleep(0)`. +// Is called only in rare situations and does not have to be lightning fast. +void _mi_prim_thread_yield(void); + +#endif // MI_PRIM_H diff --git a/csrc/mimalloc/include/mimalloc/track.h b/csrc/mimalloc/include/mimalloc/track.h new file mode 100644 index 000000000..753363366 --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/track.h @@ -0,0 +1,150 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_TRACK_H +#define MI_TRACK_H + +/* ------------------------------------------------------------------------------------------------------ +Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. +These can be defined for tracking allocation: + + #define mi_track_malloc_size(p,reqsize,size,zero) + #define mi_track_free_size(p,_size) + +The macros are set up such that the size passed to `mi_track_free_size` +always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). +The `reqsize` is what the user requested, and `size >= reqsize`. +The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, +or otherwise it is the usable block size which may be larger than the original request. +Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). +The `zero` parameter is `true` if the allocated block is zero initialized. + +Optional: + + #define mi_track_align(p,alignedp,offset,size) + #define mi_track_resize(p,oldsize,newsize) + #define mi_track_init() + +The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. +The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). +The `mi_track_resize` is currently unused but could be called on reallocations within a block. +`mi_track_init` is called at program start. + +The following macros are for tools like asan and valgrind to track whether memory is +defined, undefined, or not accessible at all: + + #define mi_track_mem_defined(p,size) + #define mi_track_mem_undefined(p,size) + #define mi_track_mem_noaccess(p,size) + +-------------------------------------------------------------------------------------------------------*/ + +#if MI_TRACK_VALGRIND +// valgrind tool + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on theap_destroy +#define MI_TRACK_TOOL "valgrind" + +#include +#include + +#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) +#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) +#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) +#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) +#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) +#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) + +#elif MI_TRACK_ASAN +// address sanitizer + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "asan" + +#include + +#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) + +#elif MI_TRACK_ETW +// windows event tracing + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 +#define MI_TRACK_TOOL "ETW" + +#include "../src/prim/windows/etw.h" + +#define mi_track_init() EventRegistermicrosoft_windows_mimalloc() +#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc() +#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) +#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) + +#else +// no tracking + +#define MI_TRACK_ENABLED 0 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "none" + +#define mi_track_malloc_size(p,reqsize,size,zero) +#define mi_track_free_size(p,_size) + +#endif + +// ------------------- +// Utility definitions + +#ifndef mi_track_resize +#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0) +#endif + +#ifndef mi_track_align +#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) +#endif + +#ifndef mi_track_init +#define mi_track_init() +#endif + +#ifndef mi_track_done +#define mi_track_done() +#endif + +#ifndef mi_track_mem_defined +#define mi_track_mem_defined(p,size) +#endif + +#ifndef mi_track_mem_undefined +#define mi_track_mem_undefined(p,size) +#endif + +#ifndef mi_track_mem_noaccess +#define mi_track_mem_noaccess(p,size) +#endif + + +#if MI_PADDING +#define mi_track_malloc(p,reqsize,zero) \ + do { if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)==(reqsize)); \ + mi_track_malloc_size(p,reqsize,reqsize,zero); \ + } } while(0) +#else +#define mi_track_malloc(p,reqsize,zero) \ + do { if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ + } } while(0) +#endif + +#endif // MI_TRACK_H diff --git a/csrc/mimalloc/include/mimalloc/types.h b/csrc/mimalloc/include/mimalloc/types.h new file mode 100644 index 000000000..bb00cdbdd --- /dev/null +++ b/csrc/mimalloc/include/mimalloc/types.h @@ -0,0 +1,804 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_TYPES_H +#define MI_TYPES_H + +// -------------------------------------------------------------------------- +// This file contains the main type definitions for mimalloc: +// mi_heap_t : all data for a heap; usually there is just one main default heap. +// mi_theap_t : a thread local heap belonging to a specific heap: +// maintains lists of thread-local heap pages that have free space. +// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from +// where objects of a single size are allocated. +// Note: we write "OS page" for OS memory pages while +// using plain "page" for mimalloc pages (`mi_page_t`). +// mi_arena_t : a large memory area where pages are allocated (process shared) +// mi_tld_t : thread local data +// mi_subproc_t : all heaps belong to a sub-process (usually just the main one) +// -------------------------------------------------------------------------- + + +#include +#include // ptrdiff_t +#include // uintptr_t, uint16_t, etc +#include // bool +#include // SIZE_MAX etc. +#include // error codes +#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations +#include "atomic.h" // _Atomic primitives + +// Minimal alignment necessary. On most platforms 16 bytes are needed +// due to SSE registers for example. This must be at least `sizeof(void*)` +#ifndef MI_MAX_ALIGN_SIZE +#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) +#endif + + +// ------------------------------------------------------ +// Variants +// ------------------------------------------------------ + +// Define NDEBUG in the release version to disable assertions. +// #define NDEBUG + +// Define MI_TRACK_ to enable tracking support +// #define MI_TRACK_VALGRIND 1 +// #define MI_TRACK_ASAN 1 +// #define MI_TRACK_ETW 1 + +// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STAT 1 + +// Define MI_SECURE to enable security mitigations +// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption +// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages +// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) +// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) +// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. + +#if !defined(MI_SECURE) +#define MI_SECURE 0 +#endif + +// Define MI_DEBUG for assertion and invariant checking +// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) +// #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) +// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON) +#if !defined(MI_DEBUG) +#if defined(MI_BUILD_RELEASE) || defined(NDEBUG) +#define MI_DEBUG 0 +#else +#define MI_DEBUG 2 +#endif +#endif + +// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking) +#ifndef MI_STAT +#if (MI_DEBUG>0) +#define MI_STAT 2 +#else +#define MI_STAT 0 +#endif +#endif + +// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) +#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL +#define MI_GUARDED 1 +#endif + +// Reserve extra padding at the end of each block to be more resilient against theap block overflows. +// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#define MI_PADDING 1 +#endif + +// Check for byte-precise buffer overflow? +#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1) +#define MI_PADDING_CHECK_BYTES 1 +#endif + + +// Encoded free lists allow detection of corrupted free lists +// and can detect buffer overflows, modify after free, and double `free`s. +#if (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_ENCODE_FREELIST 1 +#endif + +// Deprecated (double-free is now checked with padding) +// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// #define MI_CHECK_DOUBLE_FREE 1 +// #endif + +#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) +#define MI_PAGE_KEY_COUNT 2 +#else +#define MI_PAGE_KEY_COUNT 1 +#endif + +// Enable large pages for objects between 64KiB and 512KiB. +// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB +// are quite random which can lead to too many partially used large pages (but see issue #1104). +#ifndef MI_ENABLE_LARGE_PAGES +#define MI_ENABLE_LARGE_PAGES 1 +#endif + +// Place page meta info at the start of the page area or keep it separate? +// Separate keeps the page info at the arena start (default) which is more secure +// and reduces wasted space due to alignment and block sizes. +#if !defined(MI_PAGE_META_IS_SEPARATED) +#if MI_PAGE_MAP_FLAT +#define MI_PAGE_META_IS_SEPARATED 0 +#else +#define MI_PAGE_META_IS_SEPARATED 1 +#endif +#endif + +// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS +// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. +// This only works if valid pointers are passed to `mi_free` though. However, checked +// free `mi_cfree` can still use the page map to validate pointers. +#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP +#if MI_PAGE_META_IS_SEPARATED +#define MI_PAGE_META_IS_ALIGNED 1 +#define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE +#ifdef MI_PAGE_META_SMALL_IS_ALIGNED +#undef MI_PAGE_META_SMALL_IS_ALIGNED +#endif +#else +#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" +#endif +#endif + +// Deprecated (in favor of MI_PAGE_META_IS_ALIGNED): +// We can choose to only put page info of small pages at the start of the page area. +// This can be used to have a slightly faster `mi_free_small` function for specialized +// cases (like language runtime systems). +#if !MI_PAGE_META_IS_ALIGNED +#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#define MI_PAGE_META_SMALL_IS_ALIGNED 1 +#elif !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#endif +#endif + +// Configuration checks +#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE +#error "secure mode should use separated page infos" +#endif +#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE +#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" +#endif +#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT +#error "cannot have a flat page map with separated page infos" +#endif +#if MI_DEBUG && NDEBUG +#warning "mimalloc assertions enabled in a release build" +#endif + +// -------------------------------------------------------------- +// Sizes of internal data-structures +// (comments specify sizes on 64-bit, usually 32-bit is halved) +// -------------------------------------------------------------- + +// Main size parameter; determines max arena sizes and max arena object sizes etc. +#ifndef MI_ARENA_SLICE_SHIFT + #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility + #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT + #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB)) + #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages + #else + #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) + #endif +#endif +#if MI_ARENA_SLICE_SHIFT < 13 +#error Arena slices should be at least 8KiB +#endif + +#ifndef MI_BCHUNK_BITS_SHIFT + #if MI_ARENA_SLICE_SHIFT <= 13 // <= 8KiB + #define MI_BCHUNK_BITS_SHIFT (7) // 128 bits + #elif MI_ARENA_SLICE_SHIFT < 16 // <= 32KiB + #define MI_BCHUNK_BITS_SHIFT (8) // 256 bits + #else + #define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // 512 bits (or 256 on 32-bit) + #endif +#endif + +#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits +#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB +#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE) +#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) + +#define MI_ARENA_MIN_OBJ_SLICES (1) +#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit) + +#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE) +#define MI_ARENA_MAX_CHUNK_OBJ_SIZE (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE) + +#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024 +#error maximum object size may be too small to hold local thread data +#endif + +#define MI_SMALL_PAGE_SIZE MI_ARENA_MIN_OBJ_SIZE // 64 KiB +#define MI_MEDIUM_PAGE_SIZE (8*MI_SMALL_PAGE_SIZE) // 512 KiB (=byte in the bchunk bitmap) +#define MI_LARGE_PAGE_SIZE (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE) // 4 MiB (=word in the bchunk bitmap) + + +// Maximum number of size classes. (spaced exponentially in 12.5% increments) +#if MI_BIN_HUGE != 73U +#error "mimalloc internal: expecting 73 bins" +#endif +#define MI_BIN_FULL (MI_BIN_HUGE+1) +#define MI_BIN_COUNT (MI_BIN_FULL+1) + +// We never allocate more than PTRDIFF_MAX (see also ) +#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX + +// Minimal commit for a page on-demand commit +#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ + +#if MI_PAGE_META_IS_ALIGNED +#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS) +#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) +#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT +#else +#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN +#endif + +// ------------------------------------------------------ +// Arena's are large reserved areas of memory allocated from +// the OS that are managed by mimalloc to efficiently +// allocate MI_ARENA_SLICE_SIZE slices of memory for the +// mimalloc pages. +// ------------------------------------------------------ + +// A large memory arena where pages are allocated in. +typedef struct mi_arena_s mi_arena_t; // defined below + + +// ------------------------------------------------------ +// Heaps contain allocated blocks. Heaps are self-contained +// but share the (sub-process) memory in the arena's. +// ------------------------------------------------------ + +// A first-class heap. +typedef struct mi_heap_s mi_heap_t; // heaps + +// ------------------------------------------------------ +// We can have sub-processes that are fully separated +// from each other (for running multiple Python interpreters +// for example). A sub-process holds the memory arenas and heaps. +// ------------------------------------------------------ + +// A sub-process +typedef struct mi_subproc_s mi_subproc_t; + + +// --------------------------------------------------------------- +// a memory id tracks the provenance of arena/OS allocated memory +// --------------------------------------------------------------- + +// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated. +// The memid keeps track of this. +typedef enum mi_memkind_e { + MI_MEM_NONE, // not allocated (or static) + MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) + MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`)) + MI_MEM_OS, // allocated from the OS + MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory) + MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) + MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`) + MI_MEM_MALLOC // allocated with mi_malloc +} mi_memkind_t; + +static inline bool mi_memkind_is_os(mi_memkind_t memkind) { + return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP); +} + +static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) { + return (memkind <= MI_MEM_STATIC); +} + +typedef struct mi_meta_page_s mi_meta_page_t; + +typedef struct mi_memid_os_info { + void* base; // actual base address of the block (used for offset aligned allocations) + size_t size; // allocated full size + // size_t alignment; // alignment at allocation +} mi_memid_os_info_t; + +typedef struct mi_memid_arena_info { + mi_arena_t* arena; // arena that contains this memory + uint32_t slice_index; // slice index in the arena + uint32_t slice_count; // allocated slices +} mi_memid_arena_info_t; + +typedef struct mi_memid_malloc_info { + void* base; // returned pointer + size_t size; // allocated size +} mi_memid_malloc_info_t; + +typedef struct mi_memid_s { + union { + mi_memid_os_info_t os; // only used for MI_MEM_OS(_HUGE/_REMAP) + mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA + mi_memid_malloc_info_t malloc; // only used for MI_MEM_MALLOC + } mem; + mi_memkind_t memkind; + bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages) + bool initially_committed;// `true` if the memory was originally allocated as committed + bool initially_zero; // `true` if the memory was originally zero initialized +} mi_memid_t; + +#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } +#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) + +static inline bool mi_memid_is_os(mi_memid_t memid) { + return mi_memkind_is_os(memid.memkind); +} + +static inline bool mi_memid_needs_no_free(mi_memid_t memid) { + return mi_memkind_needs_no_free(memid.memkind); +} + +static inline mi_arena_t* mi_memid_arena(mi_memid_t memid) { + return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL); +} + + +// ------------------------------------------------------ +// Mimalloc pages contain allocated blocks +// ------------------------------------------------------ + +// The free lists use encoded next fields +// (Only actually encodes when MI_ENCODED_FREELIST is defined.) +typedef uintptr_t mi_encoded_t; + +// thread id's +typedef size_t mi_threadid_t; + +// free lists contain blocks +typedef struct mi_block_s { + mi_encoded_t next; +} mi_block_t; + + +// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`) +// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing) +// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing) +#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01) +#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02) +#define MI_PAGE_FLAG_MASK MI_ZU(0x03) +#define MI_PAGE_FLAG_BITS (2) +typedef size_t mi_page_flags_t; + +// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue), +// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped +// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse. +// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked) +#define MI_THREADID_ABANDONED MI_ZU(0) +#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) +#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) + +// Thread free list. +// Points to a list of blocks that are freed by other threads. +// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`). +// Ownership is required before we can read any non-atomic fields in the page. +// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. +typedef uintptr_t mi_thread_free_t; + +// Convenience +typedef size_t mi_used_t; + +// A page contains blocks of one specific size (`block_size`). +// Each page has three list of free blocks: +// `free` for blocks that can be allocated, +// `local_free` for freed blocks that are not yet available to `mi_malloc` +// `thread_free` for freed blocks by other threads +// The `local_free` and `thread_free` lists are migrated to the `free` list +// when it is exhausted. The separate `local_free` list is necessary to +// implement a monotonic heartbeat. The `thread_free` list is needed for +// avoiding atomic operations when allocating from the owning thread. +// +// `used - |thread_free|` == actual blocks that are in use (alive) +// `used - |thread_free| + |free| + |local_free| == capacity` +// +// We don't count "freed" (as |free|) but use only the `used` field to reduce +// the number of memory accesses in the `mi_page_all_free` function(s). +// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count. +// +// Notes: +// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1). +// Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically +// free an object and (re)claim ownership if the page was abandoned. +// - If a page is not part of a theap it is called "abandoned" (`theap==NULL`) -- in +// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that +// are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). +// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. +// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` + +typedef struct mi_page_s { + #if (MI_PAGE_META_IS_ALIGNED) + _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) + #endif + _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) + mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + mi_used_t used; // number of blocks in use (including blocks in `thread_free`) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + size_t block_size; // const: size available in each block (always `>0`) + size_t page_offset; // const: relative offset from the page to the start of the blocks + uint16_t capacity; // number of blocks committed + uint16_t reserved; // number of blocks reserved in memory + uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) + uint8_t retire_expire; // expiration count for retired blocks + bool free_is_zero; // `true` if the blocks in the free list are zero initialized + + // next cache line + _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) + mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) + mi_heap_t* heap; // const: the heap owning this page + + struct mi_page_s* next; // next page owned by the theap with the same `block_size` + struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` + mi_memid_t memid; // const: provenance of the page memory + + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align + #endif +} mi_page_t; + + +// ------------------------------------------------------ +// Object sizes +// ------------------------------------------------------ + +#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map. +#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b) +#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment) +#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses. +#define MI_PAGE_MAX_OVERALLOC_ALIGN MI_ARENA_SLICE_SIZE // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation + +// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes. +#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB +#if MI_ENABLE_LARGE_PAGES +#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB +#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB. note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` +#else +#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB +#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` +#endif +#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE) + +#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360) +#error "mimalloc internal: define more bins" +#endif + +// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See init.c for the size bins) +#if (MI_LARGE_MAX_OBJ_WSIZE <= 8192) // 64 KiB +#define MI_MAX_SINGLETON_BIN (48) +#elif (MI_LARGE_MAX_OBJ_WSIZE <= 32768) // 256KiB +#define MI_MAX_SINGLETON_BIN (56) +#elif (MI_LARGE_MAX_OBJ_WSIZE <= 65536) // 512KiB +#define MI_MAX_SINGLETON_BIN (60) +#else +#define MI_MAX_SINGLETON_BIN MI_BIN_HUGE +#endif + +// ------------------------------------------------------ +// Page kinds +// ------------------------------------------------------ + +typedef enum mi_page_kind_e { + MI_PAGE_SMALL, // small blocks go into 64KiB pages + MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages + MI_PAGE_LARGE, // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`) + MI_PAGE_SINGLETON // page containing a single block. + // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`. +} mi_page_kind_t; + + + +// ------------------------------------------------------ +// A "theap" is a thread local heap which owns pages. +// (making them thread-local avoids atomic operations) +// +// All theaps belong to a (non-thread-local) heap. +// A theap just owns a set of pages for allocation and +// can only be allocate/reallocate from the thread that created it. +// Freeing blocks can be done from any thread though. +// +// Per thread, there is always a default theap that belongs +// to the default heap. It is initialized to statically +// point initially to an empty theap to avoid initialization +// checks in the fast path. +// ------------------------------------------------------ + +// Thread local data +typedef struct mi_tld_s mi_tld_t; // defined below + +// Pages of a certain block size are held in a queue. +typedef struct mi_page_queue_s { + mi_page_t* first; + mi_page_t* last; + size_t count; + size_t block_size; +} mi_page_queue_t; + +// Random context +typedef struct mi_random_cxt_s { + uint32_t input[16]; + uint32_t output[16]; + int output_available; + bool weak; +} mi_random_ctx_t; + + +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows +#if MI_PADDING +typedef struct mi_padding_s { + uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) + uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) +} mi_padding_t; +#define MI_PADDING_SIZE (sizeof(mi_padding_t)) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#else +#define MI_PADDING_SIZE 0 +#define MI_PADDING_WSIZE 0 +#endif + +#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) + + +// A thread-local heap ("theap") owns a set of thread-local pages. +struct mi_theap_s { + // put in front for fast small allocations + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + + mi_tld_t* tld; // thread-local data + _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. + _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) + _Atomic(size_t) refcount; // reference count + + unsigned long long heartbeat; // monotonic heartbeat count + uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) + mi_random_ctx_t random; // random number context used for secure allocation + size_t page_count; // total number of pages in the `pages` queues. + size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) + size_t page_retired_max; // largest retired index into the `pages` array. + size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220) + long generic_count; // how often is `_mi_malloc_generic` called? + long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting? + + mi_theap_t* tnext; // list of theaps in this thread + mi_theap_t* tprev; + mi_theap_t* hnext; // list of theaps of the owning `heap` + mi_theap_t* hprev; + + long page_full_retain; // how many full pages can be retained per queue (before abandoning them) + bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages + bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + #if MI_GUARDED + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) + size_t guarded_sample_count; // current sample count (counting down to 0) + #endif + mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") + mi_memid_t memid; // provenance of the theap struct itself (meta or os) + mi_stats_t stats; // thread-local statistics +}; + + + + +// ------------------------------------------------------ +// Heaps contain allocated blocks. Heaps are self-contained +// but share the (sub-process) memory in the arena's. +// ------------------------------------------------------ + +// Keep track of all owned and abandoned pages in the arena's +struct mi_arena_pages_s; +typedef struct mi_arena_pages_s mi_arena_pages_t; + +#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`) + // 160 arenas is enough for ~2 TiB memory + +// A dynamic thread-local variable; 0 for an invalid thread-local +typedef size_t mi_thread_local_t; + +typedef struct mi_heap_s { + mi_subproc_t* subproc; // a heap belongs to a subprocess + size_t heap_seq; // unique sequence number for heaps in this subprocess + mi_heap_t* next; // list of heaps in this subprocess + mi_heap_t* prev; + mi_thread_local_t theap; // dynamic thread local for the thread-local theaps of this heap + + mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL) + int numa_node; // if >=0, prefer this numa node for allocations + + mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields) + mi_lock_t theaps_lock; // lock for the theaps list operations + + _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap + mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena + mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations) + + _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL) + mi_lock_t arena_pages_lock; // lock to update the arena_pages array + mi_memid_t memid; // provenance of the heap memory + mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap +} mi_heap_t; + + +// ------------------------------------------------------ +// Sub processes do not reclaim or visit pages from other sub processes. +// These are essentially the static variables of a process, and +// usually there is only one subprocess. This can be used for example +// by CPython to have separate interpreters within one process. +// Each thread can only belong to one subprocess +// (and needs to call `mi_subproc_add_current_thread` before any allocations). +// ------------------------------------------------------ + +struct mi_subproc_s { + size_t subproc_seq; // unique id for sub-processes + mi_subproc_t* next; // list of all sub-processes + mi_subproc_t* prev; + _Atomic(mi_meta_page_t*) meta_pages; // meta data pages + + _Atomic(size_t) arena_count; // current count of arena's + _Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process + mi_lock_t arena_reserve_lock; // lock to ensure arena's get reserved one at a time + mi_decl_align(8) // needed on some 32-bit platforms + _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged + + _Atomic(mi_heap_t*) heap_main; // main heap for this sub process + mi_heap_t* heaps; // heaps belonging to this sub-process + mi_lock_t heaps_lock; + + mi_theap_t* theap_meta; // detached theap for allocating meta-data + mi_lock_t theap_meta_lock; // all allocations in theap_meta need a lock + + _Atomic(size_t) thread_count; // current threads associated with this sub-process + _Atomic(size_t) thread_total_count; // total created threads associated with this sub-process + _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|) + _Atomic(size_t) heap_total_count; // total created heaps in this sub-process + + mi_memid_t memid; // provenance of this memory block (meta or static) + mi_subproc_t* parent; // subproc in which this one was allocated + mi_decl_align(8) // needed on some 32-bit platforms + mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed, + // and otherwise merged with heap stats when those are deleted +}; + + +// ------------------------------------------------------ +// Thread Local data +// ------------------------------------------------------ + +// Milliseconds as in `int64_t` to avoid overflows +typedef int64_t mi_msecs_t; + +// Thread local data +struct mi_tld_s { + mi_threadid_t thread_id; // thread id of this thread + size_t thread_seq; // thread sequence id (linear count of created threads) + int numa_node; // thread preferred numa node + mi_subproc_t* subproc; // sub-process this thread belongs to. + mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates) + mi_lock_t theaps_lock; // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`) + bool recurse; // true if deferred was called; used to prevent infinite recursion. + bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks) + mi_memid_t memid; // provenance of the tld memory itself (meta or OS) +}; + + +/* ---------------------------------------------------------------------------- + Arenas are fixed area's of OS memory from which we can allocate + large blocks (>= MI_ARENA_MIN_BLOCK_SIZE). + In contrast to the rest of mimalloc, the arenas are shared between + threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s). + + Arenas are also used to for huge OS page (1GiB) reservations or for reserving + OS memory upfront which can be improve performance or is sometimes needed + on embedded devices. We can also employ this with WASI or `sbrk` systems + to reserve large arenas upfront and be able to reuse the memory more effectively. +-----------------------------------------------------------------------------*/ + +#define MI_ARENA_BIN_COUNT (MI_MAX_SINGLETON_BIN+1) +#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit) +#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiB + +typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`) +typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`) + +struct mi_arena_pages_s { + mi_bitmap_t* pages; // all registered pages (abandoned and owned) + mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) + // followed by the bitmaps (whose siz`es depend on the arena size) +}; + + +// A memory arena +typedef struct mi_arena_s { + mi_memid_t memid; // provenance of the memory area + mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`) + size_t arena_idx; // index in the arenas array + void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.) + + size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) + size_t info_slices; // initial slices reserved for the arena bitmaps + int numa_node; // associated NUMA node + bool is_exclusive; // only allow allocations if specifically for this arena + mi_decl_align(8) // needed on some 32-bit platforms + _Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`. + mi_commit_fun_t* commit_fun; // custom commit/decommit memory + void* commit_fun_arg; // user argument for a custom commit function + + size_t total_size; // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0). + mi_arena_t* parent; // if this is a sub arena, this points to the first one in the memory area. + + mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes) + mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible) + mi_bitmap_t* slices_dirty; // is the slice potentially non-zero? + mi_bitmap_t* slices_purge; // slices that can be purged + mi_page_t* pages_meta; // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0` + mi_arena_pages_t pages_main; // arena page bitmaps for the main heap are allocated up front as well + + // followed by the bitmaps (whose sizes depend on the arena size) + // note: when adding bitmaps revise `mi_arena_info_slices_needed` +} mi_arena_t; + + + +/* ----------------------------------------------------------- + Error codes passed to `_mi_fatal_error` + All are recoverable but EFAULT is a serious error and aborts by default in secure mode. + For portability define undefined error codes using common Unix codes: + +----------------------------------------------------------- */ + +#ifndef EAGAIN // double free +#define EAGAIN (11) +#endif +#ifndef ENOMEM // out of memory +#define ENOMEM (12) +#endif +#ifndef EFAULT // corrupted free-list or meta-data +#define EFAULT (14) +#endif +#ifndef EINVAL // trying to free an invalid pointer +#define EINVAL (22) +#endif +#ifndef EOVERFLOW // count*size overflow +#define EOVERFLOW (75) +#endif +#ifndef ENOENT // environment variable not found +#define ENOENT (2) +#endif + + +/* ----------------------------------------------------------- + Debug constants +----------------------------------------------------------- */ + +#if !defined(MI_DEBUG_UNINIT) +#define MI_DEBUG_UNINIT (0xD0) +#endif +#if !defined(MI_DEBUG_FREED) +#define MI_DEBUG_FREED (0xDF) +#endif +#if !defined(MI_DEBUG_PADDING) +#define MI_DEBUG_PADDING (0xDE) +#endif + + +#endif // MI_TYPES_H diff --git a/csrc/mimalloc/readme.md b/csrc/mimalloc/readme.md new file mode 100644 index 000000000..a92903b27 --- /dev/null +++ b/csrc/mimalloc/readme.md @@ -0,0 +1,1024 @@ + +
+ v3: + v2: + v1: +   v3: +
+ +# mimalloc + +  + +mimalloc (pronounced "me-malloc") +is a general purpose allocator with excellent [performance](#performance) characteristics. +Initially developed by Daan Leijen for the runtime systems of the +[Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. + +Latest release : `v3.5.0` (2026-08-18) recommended. +Latest v2 release: `v2.5.0` (2026-08-18) stable. +Latest v1 release: `v1.15.0` (2026-08-18) legacy. + +mimalloc is a drop-in replacement for `malloc` and can be used in other programs +without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: +``` +> LD_PRELOAD=/usr/lib/libmimalloc.so myprogram +``` +It also includes a way to dynamically override the default allocator in [Windows](#override_on_windows). +Notable aspects of the design include: + +- __small and consistent__: the library is about 10k LOC using simple and + consistent data structures. This makes it very suitable + to integrate and adapt in other projects. For runtime systems it + provides hooks for a monotonic _heartbeat_ and deferred freeing (for + bounded worst-case times with reference counting). + Partly due to its simplicity, mimalloc has been ported to many systems (Windows, macOS, + Linux, WASM, various BSD's, Haiku, MUSL, etc) and has excellent support for dynamic overriding. + At the same time, it is an industrial strength allocator that runs (very) large scale + distributed services on thousands of machines with excellent worst case latencies. +- __free list sharding__: instead of one big free list (per size class) we have + many smaller lists per "mimalloc page" which reduces fragmentation and + increases locality -- + things that are allocated close in time get allocated close in memory. + (A mimalloc page contains blocks of one size class and is usually 64KiB on a 64-bit system). +- __free list multi-sharding__: the big idea! Not only do we shard the free list + per mimalloc page, but for each page we have multiple free lists. In particular, there + is one list for thread-local `free` operations, and another one for concurrent `free` + operations. Free-ing from another thread can now be a single CAS without needing + sophisticated coordination between threads. Since there will be + thousands of separate free lists, contention is naturally distributed over the heap, + and the chance of contending on a single location will be low -- this is quite + similar to randomized algorithms like skip lists where adding + a random oracle removes the need for a more complex algorithm. +- __eager page purging__: when a "page" becomes empty (with increased chance + due to free list sharding) the memory is marked to the OS as unused (reset or decommitted) + reducing (real) memory pressure and fragmentation, especially in long running + programs. +- __secure__: _mimalloc_ can be built in secure mode, adding guard pages, + randomized allocation, encrypted free lists, etc. to protect against various + heap vulnerabilities. The performance penalty is usually around 10% on average + over our benchmarks. +- __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. + A heap can be destroyed at once instead of deallocating each object separately. + New: v3 has true first-class heaps where one can allocate in a heap from any thread. +- __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation + times (_wcat_) (upto OS primitives), bounded space overhead (~0.2% meta-data, with low + internal fragmentation), and has no internal points of contention using only atomic operations. +- __fast__: In our benchmarks (see [below](#performance)), + _mimalloc_ outperforms other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc), + and often uses less memory. A nice property is that it does consistently well over a wide range + of benchmarks. There is also good huge OS page support for larger server programs. + +The [documentation](https://microsoft.github.io/mimalloc) gives a full overview of the API. +You can read more on the design of mimalloc in the [technical report](https://www.microsoft.com/en-us/research/publication/mimalloc-free-list-sharding-in-action) which also has detailed benchmark results. + +Enjoy! + +### Versions + +There are three maintained versions of mimalloc. These are mostly equal except for how the OS memory is handled. +New development is mostly on v3, while v1 and v2 are maintained with security and bug fixes. + +- __v3__: recommended: simplifies the lock-free design of previous versions and improves sharing of + memory between threads. On certain large workloads this version may use + (much) less memory. Also supports true first-class heaps (that can allocate from any thread) + and has more efficient heap-walking (for the CPython GC for example). + (release tags: `v3.x`, development branch `dev3`). +- __v2__: stable mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). + __Send PR's against this version if possible.__ + +### Releases + +* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` + using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for + faster load-acquire, increase retired page count from 1 to 3, faster double free + detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan + Andrade, #1346), faster pagemap lookup. + Other: fix numa sparse node count detection (#1365), add theap stats retrieval, + use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with + msvc (#1361), improved mingw-ucrt64 support. +* 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), + mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper + lock backoff for first-class heap deletion, improved riscv64 codegen. +* 2026-08-01, `v1.9.14`, `v2.4.4`, `v3.4.4`: various bug and security fixes through Opus 5 LLM audit (issue #1271, by @Zoxc). + (v3): use pthreads by default on macOS (issue #1333, issue #1327), fix glibc 2.44 crash (issue #1341), fix alignment check for realloc_aligned. (v1,v2,v3): Add initial mingw support, + set `errno` on allocation errors, improved double-free checks and size checks in secure mode, enable + build for universal windows platform and xbox (pr #1340), fix numa node detection when numa nodes are sparse. +* 2026-07-20, `v3.4.3`: revert TLS slots on macOS to 108/109 (issue #1333). +* 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). + Fix issue with using OS memory instead of arenas for > 4GiB memory usage (v3), fix concurrency bug in concurrent + heap destroy (v3), detect riscV virtual address bits at runtime, add riscV TLS support, reduce spinlock waits (v2), + use TLS slots 126/127 on macOS (v3), and other small fixes. All metadata is now separated from heap objects in v3. +* 2026-04-29, `v1.9.10`, `v2.3.2`, `v3.3.2`: various bug and security fixes through LLM audit (by @Zoxc). + Only increase minimal purge size automatically if allow_thp is set to 2. Enable large OS alignment + on all platforms (fixing OS large pages on Windows). Fix accounting of committed memory on Linux/macOS. + Update MSVC atomics implementation when using C mode. Upstream Emscripten fixes. Proper atomic do-once + implementation. +* 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to + @jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and + @Zoxc for his help in finding various issues. +* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases, + fix visiting of full pages during collection (performance), + fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode, + always use uncommitted areas between arenas (security), enable static overloading of `malloc` etc. on Windows with the + static CRT (by @Noxybot), fix TLS slot leak on Windows (v3), enable clean DLL load/unload with statically linked + mimalloc (v3), fix race in `mi_heap_destroy` (v3), by default put page meta info separate from allocated + objects (v3,security), fix C++ overrides for emscripten. Various bugs found by DeepTest include: + fix offset for `mi_heap_realloc_aligned`, fix `mi_(w)dupenv_s` buffer size, fix potential overflow in size options, + and error codes for `mi_reallocarr(ay)`. +* 2026-02-03, `v3.2.8` (rc3): Fix thread reinitialize issue on macOS. Fix SIMD codegen bug on older + GCC versions. Extend Windows TLS slot limit from 64 to 1088. Report commit statistics more precise. + Fixes issue in free-page search in arenas. +* 2026-01-15, `v1.9.7`, `v2.2.7`, `v3.2.7` (rc2): Fix zero initializing blocks that were OS allocated. + For v3 various bug and performance fixes. Fix Debian 32-bit compilation. +* 2026-01-08, `v1.9.6`, `v2.2.6`, `v3.2.6` (rc1): Important bug fixes. Many improvements to v3 including + true first-class heaps where one can allocate in heap from any thread, and track statistics per heap as well. + Added `MIMALLOC_ALLOW_THP` option. This is by default enabled except on Android. When THP is detected on v3, + mimalloc will set the `MIMALLOC_MINIMAL_PURGE_SIZE` to 2MiB to avoid breaking up potential THP huge pages. + v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. + Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning + allocated sizes on allocation and free. + +* [Older release notes](#older-release-notes) + +Special thanks to: + +* Sergiy Kuryata for his contributions on reducing memory commit -- especially on Windows with the Windows thread pool (now implemented in v3). +* [David Carlier](https://devnexen.blogspot.com/) (@devnexen) for his many contributions, and making + mimalloc work better on many less common operating systems, like Haiku, Dragonfly, etc. +* Mary Feofanova (@mary3000), Evgeniy Moiseenko, and Manuel Pöter (@mpoeter) for making mimalloc TSAN checkable, and finding + memory model bugs using the [genMC] model checker. +* Weipeng Liu (@pongba), Zhuowei Li, Junhua Wang, and Jakub Szymanski, for their early support of mimalloc and deployment + at large scale services, leading to many improvements in the mimalloc algorithms for large workloads. +* Jason Gibson (@jasongibson) for exhaustive testing on large scale workloads and server environments, and finding complex bugs + in (early versions of) `mimalloc`. +* Manuel Pöter (@mpoeter) and Sam Gross(@colesbury) for finding an ABA concurrency issue in abandoned segment reclamation. Sam also created the [no GIL](https://github.com/colesbury/nogil) Python fork which + uses mimalloc internally. + + +[genMC]: https://plv.mpi-sws.org/genmc/ + +### Usage + +mimalloc is used in various large scale low-latency services and programs, for example: + + + + + + + + +# Building + +## Windows + +Open `ide/vs2022/mimalloc.sln` in Visual Studio 2022 and build. +The `mimalloc-lib` project builds a static library (in `out/msvc-x64`), while the +`mimalloc-override-dll` project builds a DLL for overriding malloc +in the entire program. + +## Linux, macOS, BSD, etc. + +We use [`cmake`](https://cmake.org) as the build system: + +``` +> mkdir -p out/release +> cd out/release +> cmake ../.. +> make +``` +This builds the library as a shared (dynamic) +library (`.so` or `.dylib`), a static library (`.a`), and +as a single object file (`.o`). + +`> sudo make install` (install the library and header files in `/usr/local/lib` and `/usr/local/include`) + +You can build the debug version which does many internal checks and +maintains detailed statistics as: + +``` +> mkdir -p out/debug +> cd out/debug +> cmake -DCMAKE_BUILD_TYPE=Debug ../.. +> make +``` + +This will name the shared library as `libmimalloc-debug.so`. + +Finally, you can build a _secure_ version that uses guard pages, encrypted free lists, etc., as: + +``` +> mkdir -p out/secure +> cd out/secure +> cmake -DMI_SECURE=ON ../.. +> make +``` + +This will name the shared library as `libmimalloc-secure.so`. +Use `cmake ../.. -LH` to see all the available build options. + +The examples use the default compiler. If you like to use another, use: + +``` +> CC=clang CXX=clang++ cmake ../.. +``` + +## Cmake with Visual Studio + +You can also use cmake on Windows. Open a Visual Studio 2022 development prompt +and invoke `cmake` with the right [generator](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html) +and architecture, like: + +``` +> cmake ..\.. -G "Visual Studio 17 2022" -A x64 -DMI_OVERRIDE=ON +``` + +The cmake build type is specified when actually building, for example: + +``` +> cmake --build . --config=Release +``` + +You can also install the [LLVM toolset](https://learn.microsoft.com/en-us/cpp/build/clang-support-msbuild?view=msvc-170#install-1) +on Windows to build with the `clang-cl` compiler directly: + +``` +> cmake ../.. -G "Visual Studio 17 2022" -T ClangCl +``` + + +## Single Source + +You can also directly build the single `src/static.c` file as part of your project without +needing `cmake` at all. Make sure to also add the mimalloc `include` directory to the include path. + + +# Using the Library + +The preferred usage is including ``, linking with +the shared- or static library, and using the `mi_malloc` API exclusively for allocation. For example, +``` +> gcc -o myprogram -lmimalloc myfile.c +``` + +mimalloc uses only safe OS calls (`mmap` and `VirtualAlloc`) and can co-exist +with other allocators linked to the same program. +If you use `cmake`, you can simply use: +``` +find_package(mimalloc 1.8 REQUIRED) +``` +in your `CMakeLists.txt` to find a locally installed mimalloc. Then use either: +``` +target_link_libraries(myapp PUBLIC mimalloc) +``` +to link with the shared (dynamic) library, or: +``` +target_link_libraries(myapp PUBLIC mimalloc-static) +``` +to link with the static library. See `test\CMakeLists.txt` for an example. + +For best performance in C++ programs, it is also recommended to override the +global `new` and `delete` operators. For convenience, mimalloc provides +[`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) which does this for you -- just include it in a single(!) source file in your project. +In C++, mimalloc also provides the `mi_stl_allocator` struct which implements the `std::allocator` +interface. + +You can pass environment variables to print verbose messages (`MIMALLOC_VERBOSE=1`) +and statistics (`MIMALLOC_SHOW_STATS=1`) (in the debug version): +``` +> env MIMALLOC_SHOW_STATS=1 ./cfrac 175451865205073170563711388363 + +175451865205073170563711388363 = 374456281610909315237213 * 468551 + +subproc 0 + blocks peak total current block total# + bin S 4: 75.3 KiB 55.2 MiB 0 32 B 1.8 M ok + bin S 6: 31.0 KiB 180.4 KiB 0 48 B 3.8 K ok + bin S 8: 64 B 64 B 0 64 B 1 ok + bin S 9: 160 B 160 B 0 80 B 2 ok + bin S 17: 1.2 KiB 1.2 KiB 0 320 B 4 ok + bin S 21: 640 B 3.1 KiB 0 640 B 5 ok + bin S 33: 5.0 KiB 5.0 KiB 0 5.0 KiB 1 ok + + binned : 84.2 Ki 41.5 Mi 0 ok + huge : 0 0 0 ok + total : 84.2 KiB 41.5 MiB 0 + malloc req: 29.7 MiB + + pages peak total current block total# + touched : 152.8 KiB 152.8 KiB 152.8 KiB + pages : 8 14 0 ok + abandoned : 1 249 0 ok + reclaima : 0 + reclaimf : 249 + reabandon : 0 + waits : 0 + extended : 38 + retire : 35 + searches : 0.7 avg + + arenas peak total current block total# + reserved : 1.0 GiB 1.0 GiB 1.0 GiB + committed : 4.8 MiB 4.8 MiB 4.4 MiB + reset : 0 + purged : 385.5 Ki + arenas : 1 + rollback : 0 + mmaps : 3 + commits : 0 + resets : 1 + purges : 2 + guarded : 0 + heaps : 1 1 1 + + process peak total current block total# + threads : 1 1 1 + numa nodes: 1 + elapsed : 0.553 s + process : user: 0.557 s, system: 0.013 s, faults: 29, peak rss: 2.1 MiB, peak commit: 4.8 MiB +``` + +The above model of using the `mi_` prefixed API is not always possible +though in existing programs that already use the standard malloc interface, +and another option is to override the standard malloc interface +completely and redirect all calls to the _mimalloc_ library instead . + +## Environment Options + +You can set further options either programmatically (using [`mi_option_set`](https://microsoft.github.io/mimalloc/group__options.html)), or via environment variables: + +- `MIMALLOC_SHOW_STATS=1`: show statistics when the program terminates. +- `MIMALLOC_VERBOSE=1`: show verbose messages (including statistics). +- `MIMALLOC_SHOW_ERRORS=1`: show error and warning messages. + +Advanced options: + +- `MIMALLOC_ARENA_EAGER_COMMIT=2`: turns on eager commit for the large arenas (usually 1GiB) from which mimalloc + allocates segments and pages. Set this to 2 (default) to + only enable this on overcommit systems (e.g. Linux). Set this to 1 to enable explicitly on other systems + as well (like Windows or macOS) which may improve performance (as the whole arena is committed at once). + Note that eager commit only increases the commit but not the actual the peak resident set + (rss) so it is generally ok to enable this. +- `MIMALLOC_PURGE_DELAY=N`: the delay in `N` milli-seconds (by default `1000` in v3) after which mimalloc will purge + OS pages that are not in use. This signals to the OS that the underlying physical memory can be reused which + can reduce memory fragmentation especially in long running (server) programs. Setting `N` to `0` purges immediately when + a page becomes unused which can improve memory usage but also decreases performance. + Setting it to `-1` disables purging completely. +- `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, + `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused + memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual + address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). + +Further options for large workloads and services: + +- `MIMALLOC_ALLOW_THP=1`: By default always allow transparent huge pages (THP) on Linux systems. On Android only this is + by default off. When set to `0`, THP is disabled for the process that mimalloc runs in. If enabled, mimalloc also sets + the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when purging memory. +- `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected + at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than + the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA + nodes (but this can happen in any case as NUMA local allocation is always a best effort but not guaranteed). +- `MIMALLOC_ALLOW_LARGE_OS_PAGES=0`: Set to 1 to use large OS pages (2 or 4MiB) when available; for some workloads this can + significantly improve performance. However, large OS pages cannot be purged or shared with other processes so may lead + to increased memory usage in some cases. + Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs + to explicitly give permissions for large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes + the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that + can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead whenever possible). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where `N` is the number of 1GiB _huge_ OS pages. This reserves the huge pages at + startup and sometimes this can give a large (latency) performance improvement on big workloads. + Usually it is better to not use `MIMALLOC_ALLOW_LARGE_OS_PAGES=1` in combination with this setting. Just like large + OS pages, use with care as reserving + contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at + startup only once). + Note that we usually need to explicitly give permission for huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). + The huge pages are usually allocated evenly among NUMA nodes. + We can use `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N` where `N` is the numa node (starting at 0) to allocate all + the huge pages at a specific numa node instead. + +Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write +for all pages in the original process including the huge OS pages. When any memory is now written in that area, the +OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments. + +[linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 +[windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 + +## Secure Mode + +_mimalloc_ can be build in secure mode by using the `-DMI_SECURE=ON` flags in `cmake`. This build enables various mitigations +to make mimalloc more robust against exploits. In particular: + +- All internal mimalloc page meta-data is surrounded by guard pages (so a buffer overflow exploit cannot reach into the metadata). +- All free list pointers are + [encoded](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) + with per-page keys which is used both to prevent overwrites with a known pointer, as well as to detect heap corruption. +- Double free's are detected (and ignored). +- The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to + mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc + from the OS are also address randomized. +- If enabling `-DMI_SECURE_FULL=ON` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving + valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to + reaching the maximum VMA limit on Linux systems if the heap gets too large. + +As always, evaluate with care as part of an overall security strategy as all of the above are mitigations but not guarantees. + +## Debug Mode + +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`), +various checks are done at runtime to catch development errors. + +- Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. +- All objects have padding at the end to detect (byte precise) heap block overflows. +- Double free's, and freeing invalid heap pointers are detected. +- Corrupted free-lists and some forms of use-after-free are detected. + +## Guarded Mode + +_mimalloc_ can be build in guarded mode using the `-DMI_GUARDED=ON` flags in `cmake`. +This is `ON` by default when building a debug version of mimalloc. +Guarded mode enables placing OS guard pages behind certain object allocations to catch buffer overflows as they occur. +This can be invaluable to catch buffer-overflow bugs in large programs. However, it also means that any object +allocated with a guard page takes at least 8 KiB memory for the guard page and its alignment. As such, allocating +a guard page for every allocation may be too expensive both in terms of memory, and in terms of performance with +many system calls. Therefore, there are various environment variables (and options) to tune this: + +- `MIMALLOC_GUARDED_SAMPLE_RATE=N`: Set the sample rate to `N` (by default 0). This mode places a guard page + behind every `N` suitable object allocations (per thread). Since the performance in guarded mode without placing + guard pages is close to release mode, this can be used to enable guard pages even in production to catch latent + buffer overflow bugs. Set the sample rate to `1` to guard every object, and to `0` to place no guard pages at all. + +- `MIMALLOC_GUARDED_SAMPLE_SEED=N`: Start sampling at `N` (by default random). Can be used to reproduce a buffer + overflow if needed. + +- `MIMALLOC_GUARDED_MIN=N`, `MIMALLOC_GUARDED_MAX=N`: Minimal and maximal _rounded_ object sizes for which a guard + page is considered (`0` and `1GiB` respectively). If you suspect a buffer overflow occurs with an object of size + 141, set the minimum and maximum to `148` and the sample rate to `1` to have all of those guarded. + +- `MIMALLOC_GUARDED_PRECISE=1`: If we have an object of size 13, we would usually place it an aligned 16 bytes in + front of the guard page. Using `MIMALLOC_GUARDED_PRECISE` places it exactly 13 bytes before a page so that even + a 1 byte overflow is detected. This violates the C/C++ minimal alignment guarantees though so use with care. + + +# Overriding Standard Malloc + +Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. + +## Dynamic override + +This is the recommended way to override the standard malloc interface. + +### Dynamic Override on Linux, BSD + +On these ELF-based systems we preload the mimalloc shared +library so all calls to the standard `malloc` interface are +resolved to the _mimalloc_ library. +``` +> env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram +``` + +You can set extra environment variables to check that mimalloc is running, +like: +``` +> env MIMALLOC_VERBOSE=1 LD_PRELOAD=/usr/lib/libmimalloc.so myprogram +``` +or run with the debug version to get detailed statistics: +``` +> env MIMALLOC_SHOW_STATS=1 LD_PRELOAD=/usr/lib/libmimalloc-debug.so myprogram +``` + +### Dynamic Override on MacOS + +On macOS we can also preload the mimalloc shared +library so all calls to the standard `malloc` interface are +resolved to the _mimalloc_ library. +``` +> env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram +``` + +Note that certain security restrictions may apply when doing this from +the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). + + +### Dynamic Override on Windows + +We use a separate redirection DLL to override mimalloc on Windows +such that we redirect all malloc/free calls that go through the (dynamic) C runtime allocator, +including those from other DLL's or libraries. As it intercepts all allocation calls on a low level, +it can be used on large programs that include other 3rd party components. +There are four requirements to make the overriding work well: + +1. Use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). + +2. Link your program explicitly with the `mimalloc.dll.lib` export library for the `mimalloc.dll`. + (which must be compiled with `-DMI_OVERRIDE=ON`, which is the default though). + To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest + to insert some call to the mimalloc API in the `main` function, like `mi_version()` + (or use the `/include:mi_version` switch on the linker command, or + similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). + See the `mimalloc-test-override` project for an example on how to use this. + +3. The `mimalloc-redirect.dll` must be put in the same directory as the main + `mimalloc.dll` at runtime (as it is a dependency of that DLL). + The redirection DLL ensures that all calls to the C runtime malloc API get + redirected to mimalloc functions (which reside in `mimalloc.dll`). + +4. Ensure the `mimalloc.dll` comes as early as possible in the import + list of the final executable (so it can intercept all potential allocations). + You can use `minject -l ` to check this if needed. + +For best performance on Windows with C++, it is also recommended to also override +the `new`/`delete` operations (by including [`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) +a single(!) source file in your project). + +The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully +redirected. + +For different platforms than x64, you may need a specific [redirection dll](bin). +Furthermore, we cannot always re-link an executable or ensure `mimalloc.dll` comes +first in the import table. In such cases the [`minject`](bin) tool can be used +to patch the executable's import tables. + + +## Static override + +On Unix-like systems, you can also statically link with _mimalloc_ to override the standard +malloc interface. The recommended way is to link the final program with the +_mimalloc_ single object file (`mimalloc.o`). We use +an object file instead of a library file as linkers give preference to +that over archives to resolve symbols. To ensure that the standard +malloc interface resolves to the _mimalloc_ library, link it as the first +object file. For example: + +``` +> gcc -o myprogram mimalloc.o myfile1.c ... +``` + +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + +Note: recently we also enabled static overloading on Windows. In that case you need +to link with the static CRT _release_ runtime (`/MT`) and link with the static +`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). + +# Tools + +Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc +can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. +Moreover, it can be build to support Windows event tracing ([ETW]). +This has a small performance overhead but does allow detecting memory leaks and byte-precise +buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools. + +## Valgrind + +To build with [valgrind] support, use the `MI_TRACK_VALGRIND=ON` cmake option: + +``` +> cmake ../.. -DMI_TRACK_VALGRIND=ON +``` + +This can also be combined with secure mode or debug mode. +You can then run your programs directly under valgrind: + +``` +> valgrind +``` + +If you rely on overriding `malloc`/`free` by mimalloc (instead of using the `mi_malloc`/`mi_free` API directly), +you also need to tell `valgrind` to not intercept those calls itself, and use: + +``` +> MIMALLOC_SHOW_STATS=1 valgrind --soname-synonyms=somalloc=*mimalloc* -- +``` + +By setting the `MIMALLOC_SHOW_STATS` environment variable you can check that mimalloc is indeed +used and not the standard allocator. Even though the [Valgrind option][valgrind-soname] +is called `--soname-synonyms`, this also works when overriding with a static library or object file. +To dynamically override mimalloc using `LD_PRELOAD` together with `valgrind`, use: + +``` +> valgrind --trace-children=yes --soname-synonyms=somalloc=*mimalloc* /usr/bin/env LD_PRELOAD=/usr/lib/libmimalloc.so -- +``` + +See also the `test/test-wrong.c` file to test with `valgrind`. + +Valgrind support is in its initial development -- please report any issues. + +[Valgrind]: https://valgrind.org/ +[valgrind-soname]: https://valgrind.org/docs/manual/manual-core.html#opt.soname-synonyms + +## ASAN + +To build with the address sanitizer, use the `-DMI_TRACK_ASAN=ON` cmake option: + +``` +> cmake ../.. -DMI_TRACK_ASAN=ON +``` + +This can also be combined with secure mode or debug mode. +You can then run your programs as:' + +``` +> ASAN_OPTIONS=verbosity=1 +``` + +When you link a program with an address sanitizer build of mimalloc, you should +generally compile that program too with the address sanitizer enabled. +For example, assuming you build mimalloc in `out/debug`: + +``` +clang -g -o test-wrong -Iinclude test/test-wrong.c out/debug/libmimalloc-asan-debug.a -lpthread -fsanitize=address -fsanitize-recover=address +``` + +Since the address sanitizer redirects the standard allocation functions, on some platforms (macOSX for example) +it is required to compile mimalloc with `-DMI_OVERRIDE=OFF`. +Address sanitizer support is in its initial development -- please report any issues. + +[asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer + +## ETW + +Event tracing for Windows ([ETW]) provides a high performance way to capture all allocations though +mimalloc and analyze them later. To build with ETW support, use the `-DMI_TRACK_ETW=ON` cmake option. + +You can then capture an allocation trace using the Windows performance recorder (WPR), using the +`src/prim/windows/etw-mimalloc.wprp` profile. In an admin prompt, you can use: +``` +> wpr -start src\prim\windows\etw-mimalloc.wprp -filemode +> +> wpr -stop .etl +``` +and then open `.etl` in the Windows Performance Analyzer (WPA), or +use a tool like [TraceControl] that is specialized for analyzing mimalloc traces. + +[ETW]: https://learn.microsoft.com/en-us/windows-hardware/test/wpt/event-tracing-for-windows +[TraceControl]: https://github.com/xinglonghe/TraceControl + + +# Performance + +Last update: 2021-01-30 + +We tested _mimalloc_ against many other top allocators over a wide +range of benchmarks, ranging from various real world programs to +synthetic benchmarks that see how the allocator behaves under more +extreme circumstances. In our benchmark suite, _mimalloc_ outperforms other leading +allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc), and has a similar memory footprint. A nice property is that it +does consistently well over the wide range of benchmarks. + +General memory allocators are interesting as there exists no algorithm that is +optimal -- for a given allocator one can usually construct a workload +where it does not do so well. The goal is thus to find an allocation +strategy that performs well over a wide range of benchmarks without +suffering from (too much) underperformance in less common situations. + +As always, interpret these results with care since some benchmarks test synthetic +or uncommon situations that may never apply to your workloads. For example, most +allocators do not do well on `xmalloc-testN` but that includes even the best +industrial allocators like _jemalloc_ and _tcmalloc_ that are used in some of +the world's largest systems (like Chrome or FreeBSD). + +Also, the benchmarks here do not measure the behaviour on very large and long-running server workloads, +or worst-case latencies of allocation. Much work has gone into `mimalloc` to work well on such +workloads (for example, to reduce virtual memory fragmentation on long-running services) +but such optimizations are not always reflected in the current benchmark suite. + +We show here only an overview -- for +more specific details and further benchmarks we refer to the +[technical report](https://www.microsoft.com/en-us/research/publication/mimalloc-free-list-sharding-in-action). +The benchmark suite is automated and available separately +as [mimalloc-bench](https://github.com/daanx/mimalloc-bench). + + +## Benchmark Results on a 16-core AMD 5950x (Zen3) + +Testing on the 16-core AMD 5950x processor at 3.4Ghz (4.9Ghz boost), with +32GiB memory at 3600Mhz, running Ubuntu 20.04 with glibc 2.31 and GCC 9.3.0. + +We measure three versions of _mimalloc_: the main version `mi` (tag:v1.7.0), +the new v2.0 beta version as `xmi` (tag:v2.0.0), and the main version in secure mode as `smi` (tag:v1.7.0). + +The other allocators are +Google's [_tcmalloc_](https://github.com/gperftools/gperftools) (`tc`, tag:gperftools-2.8.1) used in Chrome, +Facebook's [_jemalloc_](https://github.com/jemalloc/jemalloc) (`je`, tag:5.2.1) by Jason Evans used in Firefox and FreeBSD, +the Intel thread building blocks [allocator](https://github.com/intel/tbb) (`tbb`, tag:v2020.3), +[rpmalloc](https://github.com/mjansson/rpmalloc) (`rp`,tag:1.4.1) by Mattias Jansson, +the original scalable [_Hoard_](https://github.com/emeryberger/Hoard) (git:d880f72) allocator by Emery Berger \[1], +the memory compacting [_Mesh_](https://github.com/plasma-umass/Mesh) (git:67ff31a) allocator by +Bobby Powers _et al_ \[8], +and finally the default system allocator (`glibc`, 2.31) (based on _PtMalloc2_). + + + + +Any benchmarks ending in `N` run on all 32 logical cores in parallel. +Results are averaged over 10 runs and reported relative +to mimalloc (where 1.2 means it took 1.2× longer to run). +The legend also contains the _overall relative score_ between the +allocators where 100 points is the maximum if an allocator is fastest on +all benchmarks. + +The single threaded _cfrac_ benchmark by Dave Barrett is an implementation of +continued fraction factorization which uses many small short-lived allocations. +All allocators do well on such common usage, where _mimalloc_ is just a tad +faster than _tcmalloc_ and +_jemalloc_. + +The _leanN_ program is interesting as a large realistic and +concurrent workload of the [Lean](https://github.com/leanprover/lean) +theorem prover compiling its own standard library, and there is a 13% +speedup over _tcmalloc_. This is +quite significant: if Lean spends 20% of its time in the +allocator that means that _mimalloc_ is 1.6× faster than _tcmalloc_ +here. (This is surprising as that is not measured in a pure +allocation benchmark like _alloc-test_. We conjecture that we see this +outsized improvement here because _mimalloc_ has better locality in +the allocation which improves performance for the *other* computations +in a program as well). + +The single threaded _redis_ benchmark again show that most allocators do well on such workloads. + +The _larsonN_ server benchmark by Larson and Krishnan \[2] allocates and frees between threads. They observed this +behavior (which they call _bleeding_) in actual server applications, and the benchmark simulates this. +Here, _mimalloc_ is quite a bit faster than _tcmalloc_ and _jemalloc_ probably due to the object migration between different threads. + +The _mstressN_ workload performs many allocations and re-allocations, +and migrates objects between threads (as in _larsonN_). However, it also +creates and destroys the _N_ worker threads a few times keeping some objects +alive beyond the life time of the allocating thread. We observed this +behavior in many larger server applications. + +The [_rptestN_](https://github.com/mjansson/rpmalloc-benchmark) benchmark +by Mattias Jansson is a allocator test originally designed +for _rpmalloc_, and tries to simulate realistic allocation patterns over +multiple threads. Here the differences between allocators become more apparent. + +The second benchmark set tests specific aspects of the allocators and +shows even more extreme differences between them. + +The _alloc-test_, by +[OLogN Technologies AG](http://ithare.com/testing-memory-allocators-ptmalloc2-tcmalloc-hoard-jemalloc-while-trying-to-simulate-real-world-loads/), is a very allocation intensive benchmark doing millions of +allocations in various size classes. The test is scaled such that when an +allocator performs almost identically on _alloc-test1_ as _alloc-testN_ it +means that it scales linearly. + +The _sh6bench_ and _sh8bench_ benchmarks are +developed by [MicroQuill](http://www.microquill.com/) as part of SmartHeap. +In _sh6bench_ _mimalloc_ does much +better than the others (more than 2.5× faster than _jemalloc_). +We cannot explain this well but believe it is +caused in part by the "reverse" free-ing pattern in _sh6bench_. +The _sh8bench_ is a variation with object migration +between threads; whereas _tcmalloc_ did well on _sh6bench_, the addition of object migration causes it to be 10× slower than before. + +The _xmalloc-testN_ benchmark by Lever and Boreham \[5] and Christian Eder, simulates an asymmetric workload where +some threads only allocate, and others only free -- they observed this pattern in +larger server applications. Here we see that +the _mimalloc_ technique of having non-contended sharded thread free +lists pays off as it outperforms others by a very large margin. Only _rpmalloc_, _tbb_, and _glibc_ also scale well on this benchmark. + +The _cache-scratch_ benchmark by Emery Berger \[1], and introduced with +the Hoard allocator to test for _passive-false_ sharing of cache lines. +With a single thread they all +perform the same, but when running with multiple threads the potential allocator +induced false sharing of the cache lines can cause large run-time differences. +Crundal \[6] describes in detail why the false cache line sharing occurs in the _tcmalloc_ design, and also discusses how this +can be avoided with some small implementation changes. +Only the _tbb_, _rpmalloc_ and _mesh_ allocators also avoid the +cache line sharing completely, while _Hoard_ and _glibc_ seem to mitigate +the effects. Kukanov and Voss \[7] describe in detail +how the design of _tbb_ avoids the false cache line sharing. + + +## On a 36-core Intel Xeon + +For completeness, here are the results on a big Amazon +[c5.18xlarge](https://aws.amazon.com/ec2/instance-types/#Compute_Optimized) instance +consisting of a 2×18-core Intel Xeon (Cascade Lake) at 3.4GHz (boost 3.5GHz) +with 144GiB ECC memory, running Ubuntu 20.04 with glibc 2.31, GCC 9.3.0, and +Clang 10.0.0. This time, the mimalloc allocators (mi, xmi, and smi) were +compiled with the Clang compiler instead of GCC. +The results are similar to the AMD results but it is interesting to +see the differences in the _larsonN_, _mstressN_, and _xmalloc-testN_ benchmarks. + + + + + +## Peak Working Set + +The following figure shows the peak working set (rss) of the allocators +on the benchmarks (on the c5.18xlarge instance). + + + + +Note that the _xmalloc-testN_ memory usage should be disregarded as it +allocates more the faster the program runs. Similarly, memory usage of +_larsonN_, _mstressN_, _rptestN_ and _sh8bench_ can vary depending on scheduling and +speed. Nevertheless, we hope to improve the memory usage on _mstressN_ +and _rptestN_ (just as _cfrac_, _larsonN_ and _sh8bench_ have a small working set which skews the results). + + + + +# References + +- \[1] Emery D. Berger, Kathryn S. McKinley, Robert D. Blumofe, and Paul R. Wilson. + _Hoard: A Scalable Memory Allocator for Multithreaded Applications_ + the Ninth International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS-IX). Cambridge, MA, November 2000. + [pdf](http://www.cs.utexas.edu/users/mckinley/papers/asplos-2000.pdf) + +- \[2] P. Larson and M. Krishnan. _Memory allocation for long-running server applications_. + In ISMM, Vancouver, B.C., Canada, 1998. [pdf](http://citeseer.ist.psu.edu/viewdoc/download?doi=10.1.1.45.1947&rep=rep1&type=pdf) + +- \[3] D. Grunwald, B. Zorn, and R. Henderson. + _Improving the cache locality of memory allocation_. In R. Cartwright, editor, + Proceedings of the Conference on Programming Language Design and Implementation, pages 177–186, New York, NY, USA, June 1993. [pdf](http://citeseer.ist.psu.edu/viewdoc/download?doi=10.1.1.43.6621&rep=rep1&type=pdf) + +- \[4] J. Barnes and P. Hut. _A hierarchical O(n*log(n)) force-calculation algorithm_. Nature, 324:446-449, 1986. + +- \[5] C. Lever, and D. Boreham. _Malloc() Performance in a Multithreaded Linux Environment._ + In USENIX Annual Technical Conference, Freenix Session. San Diego, CA. Jun. 2000. + Available at + +- \[6] Timothy Crundal. _Reducing Active-False Sharing in TCMalloc_. 2016. CS16S1 project at the Australian National University. [pdf](http://courses.cecs.anu.edu.au/courses/CSPROJECTS/16S1/Reports/Timothy_Crundal_Report.pdf) + +- \[7] Alexey Kukanov, and Michael J Voss. + _The Foundations for Scalable Multi-Core Software in Intel Threading Building Blocks._ + Intel Technology Journal 11 (4). 2007 + +- \[8] Bobby Powers, David Tench, Emery D. Berger, and Andrew McGregor. + _Mesh: Compacting Memory Management for C/C++_ + In Proceedings of the 40th ACM SIGPLAN Conference on Programming Language Design and Implementation (PLDI'19), June 2019, pages 333-–346. + + + +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + + +# Older Release Notes + +* 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory + was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose + for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows + fixed TLS offset, etc. +* 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: + fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, + fix execution on non BMI1 x64 systems. +* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. + Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. + Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. + Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 + for fast atomic operations. Add QNX support. +* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS + guard pages behind objects to catch buffer overflows as they occur. + Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for + thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; + consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; + add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. + +* 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches + from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. +* 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. +* 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds. + Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size + directly available (and new `block_size_shift` to improve aligned block free-ing). + New approach to collection of abandoned segments: When + a thread terminates the segments it owns are abandoned (containing still live objects) and these can be + reclaimed by other threads. We no longer use a list of abandoned segments but this is now done using bitmaps in arena's + which is more concurrent (and more aggressive). Abandoned memory can now also be reclaimed if a thread frees an object in + an abandoned page (which can be disabled using `mi_option_abandoned_reclaim_on_free`). The option `mi_option_max_segment_reclaim` + gives a maximum percentage of abandoned segments that can be reclaimed per try (=10%). + +* 2023-04-24, `v1.8.2`, `v2.1.2`: Fixes build issues on freeBSD, musl, and C17 (UE 5.1.1). Reduce code size/complexity + by removing regions and segment-cache's and only use arenas with improved memory purging -- this may improve memory + usage as well for larger services. Renamed options for consistency. Improved Valgrind and ASAN checking. + +* 2023-04-03, `v1.8.1`, `v2.1.1`: Fixes build issues on some platforms. + +* 2023-03-29, `v1.8.0`, `v2.1.0`: Improved support dynamic overriding on Windows 11. Improved tracing precision + with [asan](#asan) and [Valgrind](#valgrind), and added Windows event tracing [ETW](#ETW) (contributed by Xinglong He). Created an OS + abstraction layer to make it easier to port and separate platform dependent code (in `src/prim`). Fixed C++ STL compilation on older Microsoft C++ compilers, and various small bug fixes. + +* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [asan](#asan) and improved [Valgrind](#valgrind) support. + Support arbitrary large alignments (in particular for `std::pmr` pools). + Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). + Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). + Various small bug fixes. + +* 2022-11-03, `v1.7.7`, `v2.0.7`: Initial support for [Valgrind](#valgrind) for leak testing and heap block overflow + detection. Initial + support for attaching heaps to a specific memory area (only in v2). Fix `realloc` behavior for zero size blocks, remove restriction to integral multiple of the alignment in `alloc_align`, improved aligned allocation performance, reduced contention with many threads on few processors (thank you @dposluns!), vs2022 support, support `pkg-config`, . + +* 2022-04-14, `v1.7.6`, `v2.0.6`: fix fallback path for aligned OS allocation on Windows, improve Windows aligned allocation + even when compiling with older SDK's, fix dynamic overriding on macOS Monterey, fix MSVC C++ dynamic overriding, fix + warnings under Clang 14, improve performance if many OS threads are created and destroyed, fix statistics for large object + allocations, using MIMALLOC_VERBOSE=1 has no maximum on the number of error messages, various small fixes. + +* 2022-02-14, `v1.7.5`, `v2.0.5` (alpha): fix malloc override on + Windows 11, fix compilation with musl, potentially reduced + committed memory, add `bin/minject` for Windows, + improved wasm support, faster aligned allocation, + various small fixes. + +* 2021-11-14, `v1.7.3`, `v2.0.3` (beta): improved WASM support, improved macOS support and performance (including + M1), improved performance for v2 for large objects, Python integration improvements, more standard + installation directories, various small fixes. +* 2021-06-17, `v1.7.2`, `v2.0.2` (beta): support M1, better installation layout on Linux, fix + thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-windows 8, various small fixes. +* 2021-04-06, `v1.7.1`, `v2.0.1` (beta): fix bug in arena allocation for huge pages, improved aslr on large allocations, initial M1 support (still experimental). +* 2021-01-31, `v2.0.0`: beta release 2.0: new slice algorithm for managing internal mimalloc pages. +* 2021-01-31, `v1.7.0`: stable release 1.7: support explicit user provided memory regions, more precise statistics, + improve macOS overriding, initial support for Apple M1, improved DragonFly support, faster memcpy on Windows, various small fixes. + +* 2020-09-24, `v1.6.7`: stable release 1.6: using standard C atomics, passing tsan testing, improved + handling of failing to commit on Windows, add [`mi_process_info`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc.h#L156) api call. +* 2020-08-06, `v1.6.4`: stable release 1.6: improved error recovery in low-memory situations, + support for IllumOS and Haiku, NUMA support for Vista/XP, improved NUMA detection for AMD Ryzen, ubsan support. +* 2020-05-05, `v1.6.3`: stable release 1.6: improved behavior in out-of-memory situations, improved malloc zones on macOS, + build PIC static libraries by default, add option to abort on out-of-memory, line buffered statistics. +* 2020-04-20, `v1.6.2`: stable release 1.6: fix compilation on Android, MingW, Raspberry, and Conda, + stability fix for Windows 7, fix multiple mimalloc instances in one executable, fix `strnlen` overload, + fix aligned debug padding. +* 2020-02-17, `v1.6.1`: stable release 1.6: minor updates (build with clang-cl, fix alignment issue for small objects). +* 2020-02-09, `v1.6.0`: stable release 1.6: fixed potential memory leak, improved overriding + and thread local support on FreeBSD, NetBSD, DragonFly, and macOSX. New byte-precise + heap block overflow detection in debug mode (besides the double-free detection and free-list + corruption detection). Add `nodiscard` attribute to most allocation functions. + Enable `MIMALLOC_PAGE_RESET` by default. New reclamation strategy for abandoned heap pages + for better memory footprint. +* 2020-02-09, `v1.5.0`: stable release 1.5: improved free performance, small bug fixes. +* 2020-01-22, `v1.4.0`: stable release 1.4: improved performance for delayed OS page reset, +more eager concurrent free, addition of STL allocator, fixed potential memory leak. +* 2020-01-15, `v1.3.0`: stable release 1.3: bug fixes, improved randomness and [stronger +free list encoding](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) in secure mode. + +* 2019-12-22, `v1.2.2`: stable release 1.2: minor updates. +* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). Improved dynamic overriding on Windows. +* 2019-10-07, `v1.1.0`: stable release 1.1. +* 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. +* 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. diff --git a/csrc/mimalloc/src/alloc-aligned.c b/csrc/mimalloc/src/alloc-aligned.c new file mode 100644 index 000000000..111d63a8b --- /dev/null +++ b/csrc/mimalloc/src/alloc-aligned.c @@ -0,0 +1,462 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" // _mi_theap_default + +#include // memset + +// ------------------------------------------------------ +// Aligned Allocation +// ------------------------------------------------------ + +static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { + // certain blocks are always allocated at a certain natural alignment. + // (see also `arena.c:mi_arenas_page_alloc_fresh`). + mi_assert_internal(mi_alignment_is_valid(alignment)); + if (alignment > size) return false; + const size_t bsize = mi_good_size(size); + const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N + (alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0); // or multiple of N + if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size + return ok; +} + +#if MI_GUARDED +static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { + // use over allocation for guarded blocksl + #if MI_THEAP_INITASNULL + if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } + #endif + mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN); + if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) { + _mi_error_message(EOVERFLOW, "(guarded) aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); + return NULL; + } + const size_t oversize = size + alignment - 1; + void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage); + if (base==NULL) return NULL; + void* const p = _mi_align_up_ptr(base, alignment); + mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); + mi_assert_internal(mi_usable_size(p) >= size); + mi_assert_internal(_mi_is_aligned(p, alignment)); + return p; +} + +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { + #if MI_THEAP_INITASNULL + if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } + #endif + const size_t rate = theap->guarded_sample_rate; + // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` + if (rate != 0) { theap->guarded_sample_rate = 0; } + void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); + if (rate != 0) { theap->guarded_sample_rate = rate; } + return p; +} +#else +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { + return _mi_theap_malloc_zero(theap, size, zero, ppage); +} +#endif + +// Fallback aligned allocation that over-allocates -- split out for better codegen +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); + mi_assert_internal(mi_alignment_is_valid(alignment)); + + void* p; + size_t oversize; + mi_page_t* page; + if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // use OS allocation for large alignments and allocate inside a singleton page (not in an arena) + // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned + // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map). + if mi_unlikely(offset != 0) { + // todo: cannot support offset alignment for very large alignments yet + _mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); + return NULL; + } + oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); + // note: no guarded as alignment > 0 + p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + if (p == NULL) return NULL; + } + else { + // otherwise over-allocate + mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN); + mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow + oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that. + p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, &page); + if (p == NULL) return NULL; + } + mi_assert_internal(page == _mi_ptr_page(p)); + if (ppage!=NULL) { *ppage = page; } + + // .. and align within the allocation + const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` + const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask; + const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset); + mi_assert_internal(adjust < alignment); + void* aligned_p = (void*)((uintptr_t)p + adjust); + + // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`) + // and we no longer own it. We should be careful to only read constant fields in the page, + // or use safe atomic access as in `mi_page_set_has_interior_pointers`. + // (we can access the page though since the just allocated pointer keeps it alive) + if (aligned_p != p) { + mi_page_set_has_interior_pointers(page, true); + // if (usable!=NULL) { + // mi_assert_internal(*usable > adjust); + // if (*usable > adjust) { *usable = *usable - adjust; } + // mi_assert_internal(*usable >= size); + // } + #if MI_GUARDED + // set tag to aligned so mi_usable_size works with guard pages + if (adjust >= sizeof(mi_block_t)) { + mi_block_t* const block = (mi_block_t*)p; + block->next = MI_BLOCK_TAG_ALIGNED; + } + #endif + _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); + } + // todo: expand padding if overallocated ? + + mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size); + mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); + mi_assert_internal(mi_usable_size(aligned_p)>=size); + mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); + #if MI_DEBUG > 1 + mi_page_t* const apage = _mi_ptr_page(aligned_p); + void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == unalign_p); + #endif + + // now zero the block if needed + //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // // for the tracker, on huge aligned allocations only from the start of the large block is defined + // mi_track_mem_undefined(aligned_p, size); + // if (zero) { + // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); + // } + //} + + if (p != aligned_p) { + mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); + #if MI_GUARDED + mi_track_mem_defined(p, sizeof(mi_block_t)); + #endif + } + return aligned_p; +} + +// Generic primitive aligned allocation -- split out for better codegen +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert_internal(mi_alignment_is_valid(alignment)); + // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) + if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { + _mi_error_message(EINVAL, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); + return NULL; + } + + // use regular allocation if it is guaranteed to fit the alignment constraints. + // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist + // a page with the right block size, and if we always use the over-alloc fallback that would never happen. + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); + mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + if mi_likely(is_aligned_or_null) { + return p; + } + else { + // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. + mi_assert(false); + mi_free(p); + } + } + + // fall back to over-allocation + return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage); +} + + +static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, size_t alignment, size_t offset) { + _mi_error_message(EINVAL, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); + return NULL; +} + +// Primitive aligned allocation +static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) + return mi_error_bad_alignment(size, alignment, offset); + } + + #if MI_GUARDED + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { + return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); + } + #endif + + // try first if there happens to be a small block available with just the right alignment + // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already + // naturally aligned this can be often the case. + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); + if mi_likely(page->free != NULL) { + const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; + if mi_likely(is_aligned) + { + if (ppage!=NULL) { *ppage = page; } + void* p = _mi_page_malloc_zero(theap, page, padsize, zero); + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_track_malloc(p, size, zero); + return p; + } + } + } + } + + // fallback to generic aligned allocation + return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage); +} + + +// ------------------------------------------------------ +// Internal mi_theap_malloc_aligned / mi_malloc_aligned +// ------------------------------------------------------ + +static mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_malloc_aligned_at(theap, size, alignment, 0); +} + +static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL); +} + +mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_zalloc_aligned_at(theap, size, alignment, 0); +} + +static mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_zalloc_aligned_at(theap, total, alignment, offset); +} + +static mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0); +} + + +// ------------------------------------------------------ +// Aligned Allocation +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_aligned_at(_mi_theap_default(), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_zalloc_aligned_at(_mi_theap_default(), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_calloc_aligned_at(_mi_theap_default(), count, size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_calloc_aligned(_mi_theap_default(), count, size, alignment); +} + + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_malloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_malloc_aligned(_mi_heap_theap(heap), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_zalloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_zalloc_aligned(_mi_heap_theap(heap), size, alignment); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_calloc_aligned_at(_mi_heap_theap(heap), count, size, alignment, offset); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_calloc_aligned(_mi_heap_theap(heap), count, size, alignment); +} + + +// ------------------------------------------------------ +// Aligned re-allocation +// ------------------------------------------------------ + +static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { + mi_assert(mi_alignment_is_valid(alignment)); + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see ) + return mi_error_bad_alignment(newsize,alignment,offset); + } + if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); + if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); + const size_t size = mi_usable_size(p); + if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { + return p; // reallocation still fits, is aligned and not more than 50% waste + } + else { + // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) + mi_page_t* newpage; + void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); + if (newp != NULL) { + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763 + if (zero && usable > zero_start) { + // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start); + } + _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?) + mi_free(p); // only free if successful + } + return newp; + } +} + +static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { + mi_assert(alignment > 0); + if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero); + return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero); +} + +static void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false); +} + +static void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false); +} + +static void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true); +} + +static void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true); +} + +static void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(newcount, size, &total)) return NULL; + return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset); +} + +static void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(newcount, size, &total)) return NULL; + return mi_theap_rezalloc_aligned(theap, p, total, alignment); +} + + +mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset); +} + +mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment); +} + + +mi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset); +} + +mi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { + return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment); +} + +mi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { + return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset); +} + +mi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { + return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment); +} + + diff --git a/csrc/mimalloc/src/alloc-override.c b/csrc/mimalloc/src/alloc-override.c new file mode 100644 index 000000000..04da5f293 --- /dev/null +++ b/csrc/mimalloc/src/alloc-override.c @@ -0,0 +1,407 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#if !defined(MI_IN_ALLOC_C) +#error "this file should be included from 'alloc.c' (so aliases can work)" +#endif + + +#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL) + +#if defined(__APPLE__) +#include +mi_decl_externc void vfree(void* p); +mi_decl_externc size_t malloc_size(const void* p); +mi_decl_externc size_t malloc_good_size(size_t size); +#endif + +// helper definition for C override of C++ new +typedef void* mi_nothrow_t; + +// ------------------------------------------------------ +// Override system malloc +// ------------------------------------------------------ + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED + // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions + #if (defined(__GNUC__) && __GNUC__ >= 9) + #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); + #else + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); + #endif + #define MI_FORWARD1(fun,x) MI_FORWARD(fun) + #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) + #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) + #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) +#else + // otherwise use forwarding by calling our `mi_` function + #define MI_FORWARD1(fun,x) { return fun(x); } + #define MI_FORWARD2(fun,x,y) { return fun(x,y); } + #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } + #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD02(fun,x,y) { fun(x,y); } +#endif + +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for + // functions that are interposed (or the interposing does not work) + #define MI_OSX_IS_INTERPOSED + + mi_decl_externc size_t mi_malloc_size_checked(void *p) { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); + } + + // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` + // See: + struct mi_interpose_s { + const void* replacement; + const void* target; + }; + #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } + #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) + + #define MI_INTERPOSE_DECLS(name) __attribute__((used)) static struct mi_interpose_s name[] __attribute__((section("__DATA, __interpose"))) + + MI_INTERPOSE_DECLS(_mi_interposes) = + { + MI_INTERPOSE_MI(malloc), + MI_INTERPOSE_MI(calloc), + MI_INTERPOSE_MI(realloc), + MI_INTERPOSE_MI(strdup), + MI_INTERPOSE_MI(realpath), + MI_INTERPOSE_MI(posix_memalign), + MI_INTERPOSE_MI(reallocf), + MI_INTERPOSE_MI(valloc), + MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked), + MI_INTERPOSE_MI(malloc_good_size), + #ifdef MI_OSX_ZONE + // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely + MI_INTERPOSE_MI(free), + MI_INTERPOSE_FUN(vfree,mi_free), + #else + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), + #endif + }; + #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7) + MI_INTERPOSE_DECLS(_mi_interposes_10_7) = { MI_INTERPOSE_MI(strndup) }; + #endif + #if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15) + MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) }; + #endif + + #ifdef __cplusplus + extern "C" { + #endif + void _ZdlPv(void* p); // delete + void _ZdaPv(void* p); // delete[] + void _ZdlPvm(void* p, size_t n); // delete + void _ZdaPvm(void* p, size_t n); // delete[] + void* _Znwm(size_t n); // new + void* _Znam(size_t n); // new[] + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow + #ifdef __cplusplus + } + #endif + __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = + { + MI_INTERPOSE_FUN(_ZdlPv,mi_free), + MI_INTERPOSE_FUN(_ZdaPv,mi_free), + MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size), + MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size), + MI_INTERPOSE_FUN(_Znwm,mi_new), + MI_INTERPOSE_FUN(_Znam,mi_new), + MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow), + MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow), + }; + +#elif defined(_MSC_VER) + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE + void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { + return mi__expand(_Block, _Size); + } + _Check_return_ _ACRTIMP + size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT { + return mi_malloc_size(_Block); + } + _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE + size_t __cdecl _msize(_Pre_notnull_ void* _Block) { + return mi_malloc_size(_Block); + } + _ACRTIMP + void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) { + mi_free(_Block); + } + _ACRTIMP _CRT_HYBRIDPATCHABLE + void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) { + mi_free(_Block); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE + void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) { + return mi_malloc(_Size); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _malloc_base(_In_ size_t _Size) { + return mi_malloc(_Size); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Size) { + return mi_realloc(_Block, _Size); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE + void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { + return mi_realloc(_Block, _Size); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) { + return mi_calloc(_Count, _Size); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT + void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) { + return mi_calloc(_Count, _Size); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Count, _In_ size_t _Size) { + return mi_recalloc(_Block, _Count, _Size); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) { + return mi_recalloc(_Block, _Count, _Size); + } + _ACRTIMP + void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) { + mi_free(_Block); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) { + return mi_malloc_aligned(_Size, _Alignment); + } + _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) { + return mi_malloc_aligned_at(_Size, _Alignment, _Offset); + } + _Check_return_ _ACRTIMP + size_t __cdecl _aligned_msize(_Pre_notnull_ void* _Block, _In_ size_t _Alignment, _In_ size_t _Offset) { + MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) { + return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) { + return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) { + return mi_realloc_aligned(_Block, _Size, _Alignment); + } + _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT + void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) { + return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment); + } +#else + // On all other systems forward allocation primitives to our API + mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) + mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) + mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) + mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p) + // In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call) + // We only override if `strdup` is not a macro (as on some older libc's, see issue #885) + #if !defined(strdup) + mi_decl_export char* strdup(const char* str) MI_FORWARD1(mi_strdup, str) + #endif + #if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)) + mi_decl_export char* strndup(const char* str, size_t n) MI_FORWARD2(mi_strndup, str, n) + #endif +#endif + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +#pragma GCC visibility push(default) +#endif + +// ------------------------------------------------------ +// Override new/delete +// This is not really necessary as they usually call +// malloc/free anyway, but it improves performance. +// ------------------------------------------------------ +#ifdef __cplusplus + // ------------------------------------------------------ + // With a C++ compiler we override the new/delete operators. + // see + // ------------------------------------------------------ + #include + + #ifndef MI_OSX_IS_INTERPOSED + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) + + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + #endif + #endif + + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); } + + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + #endif + +#elif (defined(__GNUC__) || defined(__clang__)) + // ------------------------------------------------------ + // Override by defining the mangled C++ names of the operators (as + // used by GCC and CLang). + // See + // ------------------------------------------------------ + + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + + void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + + void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&) + void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&) + void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&) + void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&) + + #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + #elif (MI_INTPTR_SIZE==4) + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + #else + #error "define overloads for new/delete for this platform (just for performance, can be skipped)" + #endif +#endif // __cplusplus + +// ------------------------------------------------------ +// Further Posix & Unix functions definitions +// ------------------------------------------------------ + +#ifdef __cplusplus +extern "C" { +#endif + +// defined here instead alloc-posix so we can alias it +mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +#ifndef MI_OSX_IS_INTERPOSED + // Forward Posix/Unix calls as well + void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) + size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p) + #else + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p) + #endif + + // No forwarding here due to aliasing/name mangling issues + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + + // `aligned_alloc` is only available when __USE_ISOC11 is defined. + // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check + // for it if using glibc. + // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot + // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. + // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it + // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. + #if !defined(__GLIBC__) || __USE_ISOC11 + void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + #endif +#endif + +// no forwarding here due to aliasing/name mangling issues +void cfree(void* p) { mi_free(p); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +#if !defined(_WIN32) +void* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); } +#endif +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +// some systems define reallocarr so mark it as a weak symbol (#751) +mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } + +#if defined(__wasi__) + // forward __libc interface (see PR #667) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) + void __libc_free(void* p) MI_FORWARD0(mi_free, p) + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } + +#elif defined(__linux__) + // forward __libc interface (needed for glibc-based and musl-based Linux distributions) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) + + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } +#endif + +#ifdef __cplusplus +} +#endif + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) +#pragma GCC visibility pop +#endif + +#endif // MI_MALLOC_OVERRIDE diff --git a/csrc/mimalloc/src/alloc-posix.c b/csrc/mimalloc/src/alloc-posix.c new file mode 100644 index 000000000..a22524942 --- /dev/null +++ b/csrc/mimalloc/src/alloc-posix.c @@ -0,0 +1,184 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// ------------------------------------------------------------------------ +// mi prefixed publi definitions of various Posix, Unix, and C++ functions +// for convenience and used when overriding these functions. +// ------------------------------------------------------------------------ +#include "mimalloc.h" +#include "mimalloc/internal.h" + +// ------------------------------------------------------ +// Posix & Unix functions definitions +// ------------------------------------------------------ + +#include +#include // memset +#include // getenv + +#ifdef _MSC_VER +#pragma warning(disable:4996) // getenv _wgetenv +#endif + +#ifndef EINVAL +#define EINVAL 22 +#endif +#ifndef ENOMEM +#define ENOMEM 12 +#endif + +mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { + return mi_good_size(size); +} + +int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794) + // Note: The spec dictates we should not modify `*p` on an error. (issue#27) + // + if (p == NULL) return EINVAL; + // it is required that alignment is a power of 2 and a multiple of sizeof(void*) + if (alignment= SIZE_MAX - psize) return NULL; // overflow + size_t asize = _mi_align_up(size, psize); + return mi_malloc_aligned(asize, psize); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { + // C11 requires the size to be an integral multiple of the alignment, see . + // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check.. + /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment); + #endif + return NULL; + } + */ + // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned + void* p = mi_malloc_aligned(size, alignment); + mi_assert_internal(_mi_is_aligned(p,alignment)); + return p; +} + +mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD + size_t total; + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { + errno = EOVERFLOW; + return NULL; + } + void* newp = mi_realloc(p,total); + if (newp==NULL) { errno = ENOMEM; } + return newp; +} + +mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD + mi_assert(size != 0); + mi_assert(ptrp != NULL); + if (ptrp == NULL || size == 0) { + return (errno = EINVAL); + } + size_t total; + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { + return (errno = EOVERFLOW); + } + void** op = (void**)ptrp; + if (total == 0) { + mi_free(*op); + *op = NULL; + return 0; + } + else { + void* newp = mi_realloc(*op,total); + if (newp == NULL) { return (errno = ENOMEM); } + *op = newp; + return 0; + } +} + +void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft + void* res = mi_expand(p, newsize); + if (res == NULL) { errno = ENOMEM; } + return res; +} + +mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept { + if (s==NULL) return NULL; + size_t wlen; + for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1 + size_t size; + if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL; + wchar_t* p = (wchar_t*)mi_malloc(size); + if (p != NULL) { + _mi_memcpy(p,s,size); + } + return p; +} + +mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept { + return (unsigned char*)mi_strdup((const char*)s); +} + +int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { + if (size != NULL) *size = 0; + if (buf==NULL || name==NULL) return EINVAL; + char* p = getenv(name); + if (p==NULL) { + *buf = NULL; + } + else { + *buf = mi_strdup(p); + if (*buf==NULL) return ENOMEM; + if (size != NULL) { *size = _mi_strlen(p) + 1; } // cannot overflow as mi_strdup is limited to PTRDIFF_MAX + } + return 0; +} + +int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept { + if (size != NULL) *size = 0; + if (buf==NULL || name==NULL) return EINVAL; +#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP)) + // not supported + *buf = NULL; + return EINVAL; +#else + wchar_t* p = (wchar_t*)_wgetenv(name); + if (p==NULL) { + *buf = NULL; + } + else { + *buf = mi_wcsdup(p); + if (*buf==NULL) return ENOMEM; + if (size != NULL) { *size = wcslen(p) + 1; } // cannot overflow as wcsdup is limited to PTRDIFF_MAX + } + return 0; +#endif +} + +mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft + return mi_recalloc_aligned_at(p, newcount, size, alignment, offset); +} + +mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft + return mi_recalloc_aligned(p, newcount, size, alignment); +} diff --git a/csrc/mimalloc/src/alloc.c b/csrc/mimalloc/src/alloc.c new file mode 100644 index 000000000..bb98facbb --- /dev/null +++ b/csrc/mimalloc/src/alloc.c @@ -0,0 +1,976 @@ + +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim-tls.h" // _mi_prim_thread_id() + +#include // memset, strlen (for mi_strdup) +#include // malloc, abort + +#define MI_IN_ALLOC_C +#include "alloc-override.c" +#include "free.c" +#undef MI_IN_ALLOC_C + +// ------------------------------------------------------ +// Allocation +// ------------------------------------------------------ + +// Fast allocation in a page: just pop from the free list. +// Fall back to generic allocation only if the list is empty. +// Note: in release mode the (inlined) routine is about 7 instructions with a single test. +static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + if (page->block_size != 0) { // not the empty theap + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + } + + // check the free list + mi_block_t* const block = page->free; + const mi_used_t used = page->used; + #if defined(__GNUC__) + __asm("" : : : "memory"); // alway load used before the test + #endif + if (block == NULL) { + return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + } + mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); + if (ppage != NULL) { *ppage = page; }; + + // pop from the free list + mi_block_t* next = mi_block_next(page,block); + mi_track_mem_undefined(block,sizeof(*block)); + block->next = 0; // don't leak internal data + page->free = next; + page->used = used+1; + mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); + mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + + #if MI_DEBUG>3 + if (page->free_is_zero && size > sizeof(*block)) { + mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); + } + #endif + + // allow use of the block internally + // note: when tracking we need to avoid ever touching the MI_PADDING since + // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`) + const size_t bsize = mi_page_usable_block_size(page); + mi_track_mem_undefined(block, bsize); + + #if (MI_STAT>0) + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { + mi_theap_stat_increase(theap, malloc_normal, bsize); + #if (MI_STAT>1) + mi_theap_stat_counter_increase(theap, malloc_normal_count, 1); + const size_t bin = _mi_bin(bsize); + mi_theap_stat_increase(theap, malloc_bins[bin], 1); + mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE); + #endif + } + #endif + + // zero the block? note: we need to zero the full block size (issue #63) + if mi_likely(!zero) { + // #if MI_SECURE + // block->next = 0; // don't leak internal data + // #endif + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } + #endif + } + else { + if (!page->free_is_zero) { + _mi_memzero_aligned(block, bsize); + } + else { + block->next = 0; + mi_track_mem_defined(block, bsize); + } + } + + #if MI_PADDING // && !MI_TRACK_ENABLED + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); + ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + #if (MI_DEBUG>=2) + mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); + #endif + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = mi_ptr_encode_canary(page,block,page->keys); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK_BYTES + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif + #endif + + return block; +} + +// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. +extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { + return mi_page_malloc_zero(theap, page, size, zero, NULL); +} + +// main allocation primitives for small and generic allocation + +// internal small size allocation +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert(theap != NULL); + mi_assert(size <= MI_SMALL_SIZE_MAX); + #if MI_DEBUG + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local + #endif + #if (MI_PADDING || MI_GUARDED) + if mi_unlikely(size == 0) { size = sizeof(void*); } + #endif + #if MI_GUARDED + if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { + return _mi_theap_malloc_guarded(theap, size, zero, ppage); + } + #endif + + // get page in constant time, and allocate from it + mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); + mi_track_malloc(p,size,zero); + + #if MI_DEBUG>3 + if (p != NULL && zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} + +// internal generic allocation +static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept +{ + #if MI_GUARDED + #if MI_THEAP_INITASNULL + if (theap!=NULL) + #endif + if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { + return _mi_theap_malloc_guarded(theap, size, zero, ppage); + } + #endif + #if !MI_THEAP_INITASNULL + mi_assert(theap!=NULL); + #endif + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local + mi_assert((huge_alignment & 1)==0); + void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic + mi_track_malloc(p, size, zero); + + #if MI_DEBUG>3 + if (p != NULL && zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} + +// internal small allocation +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + #if MI_THEAP_INITASNULL + if (theap!=NULL) { + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + } + else { + return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall + } + #else + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + #endif +} + + +// allocate a small block +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero(theap, size, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { + return mi_theap_malloc_small(_mi_theap_default(), size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL); +} + +// The main internal allocation functions +static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + // fast path for small objects + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_assert_internal(huge_alignment == 0); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + } + else { + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); + } +} + +extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + // fast path for small objects + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) + #else + if mi_likely(size <= MI_SMALL_SIZE_MAX) + #endif + { + mi_assert_internal(huge_alignment == 0); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + } + else { + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); + } +} + +inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); +} + + +// Main allocation functions + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { + return mi_theap_malloc(_mi_theap_default(), size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); +} + + +// zero initialized small block +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero(theap, size, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); +} + + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return _mi_theap_malloc_zero(theap, size, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { + return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); +} + +mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count,size,&total)) return NULL; + return mi_theap_zalloc(theap,total); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { + return mi_theap_calloc(_mi_theap_default(),count,size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_zalloc(heap, total); +} + +static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { + mi_assert_internal(page == _mi_ptr_page(p)); + if (pblock_size!=NULL) { + if (p!=NULL) { *pblock_size = mi_page_block_size(page); } + } + return p; +} + +// Return usable size +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* pblock_size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count,size,&total)) return NULL; + return mi_uzalloc(total, pblock_size); +} + +// Uninitialized `calloc` +static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_malloc(theap, total); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { + return mi_theap_mallocn(_mi_theap_default(),count,size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_heap_malloc(heap, total); +} + + +// Expand (or shrink) in place (or fail) +void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { + #if MI_PADDING + // we do not shrink/expand with padding enabled + MI_UNUSED(p); MI_UNUSED(newsize); + return NULL; + #else + if (p == NULL) return NULL; + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); + const size_t size = _mi_page_usable_size(page,p); + if (newsize > size) return NULL; + return p; // it fits + #endif +} + +static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { + // if p == NULL then behave as malloc. + // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). + // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) + const mi_page_t* page; + size_t size; + if (p==NULL) { + page = NULL; + size = 0; + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } + } + else { + page = mi_validate_ptr_page(p,"mi_realloc"); + if mi_unlikely(page==NULL) { // invalid pointer + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } + if (pblock_size_post!=NULL) { *pblock_size_post = 0; } + return NULL; + } + size = _mi_page_usable_size(page,p); + if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); } + } + // check if we can reuse the existing block + if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) + mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0 + #if MI_THEAP_INITASNULL + if (theap!=NULL) + #endif + { + if (mi_page_heap(page)==_mi_theap_heap(theap)) { // and within the same heap + mi_assert_internal(p!=NULL); + // todo: do not track as the usable size is still the same in the free; adjust potential padding? + // mi_track_resize(p,size,newsize) + // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); } + return p; // reallocation still fits and not more than 50% waste + } + } + } + // note: we don't zero allocate upfront so we only zero initialize the expanded part + mi_page_t* newpage; // use block_size for zero-ing, issue #763 + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); + if mi_likely(newp != NULL) { + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); + if (zero && usable > zero_start) { + _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); + } + else if (newsize == 0) { + ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) + } + if mi_likely(p != NULL) { + mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking.. + _mi_memcpy_aligned(newp, p, copy_size); + mi_free(p); // only free the original pointer if successful + } + } + return newp; +} + +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL); +} + +mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { + // optimize p==NULL + if (p==NULL) { + return mi_theap_malloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, false); + } +} + +static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_realloc(theap, p, total); +} + + +// Reallocate but free `p` on errors +static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { + void* newp = mi_theap_realloc(theap, p, newsize); + if (newp==NULL && p!=NULL) mi_free(p); + return newp; +} + +mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { + // optimize p==NULL + if (p==NULL) { + return mi_theap_zalloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, true); + } +} + +static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count, size, &total)) return NULL; + return mi_theap_rezalloc(theap, p, total); +} + + +mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_realloc(_mi_theap_default(),p,newsize); +} + +mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_theap_reallocn(_mi_theap_default(),p,count,size); +} + +mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); +} + +// Reallocate but free `p` on errors +mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_reallocf(_mi_theap_default(),p,newsize); +} + +mi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_rezalloc(_mi_theap_default(), p, newsize); +} + +mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_theap_recalloc(_mi_theap_default(), p, count, size); +} + + +mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_realloc(_mi_heap_theap(heap), p, newsize); +} + +mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size); +} + +// Reallocate but free `p` on errors +mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_reallocf(_mi_heap_theap(heap), p, newsize); +} + +mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { + return mi_theap_rezalloc(_mi_heap_theap(heap), p, newsize); +} + +mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { + return mi_theap_recalloc(_mi_heap_theap(heap), p, count, size); +} + + + +// ------------------------------------------------------ +// strdup, strndup, and realpath +// ------------------------------------------------------ + +// `strdup` using mi_malloc +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept { + if (s == NULL) return NULL; + size_t len = _mi_strlen(s); + if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 + char* t = (char*)mi_theap_malloc(theap,len+1); + if (t == NULL) return NULL; + _mi_memcpy(t, s, len); + t[len] = 0; + return t; +} + +mi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { + return mi_theap_strdup(_mi_theap_default(), s); +} + +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { + return mi_theap_strdup(_mi_heap_theap(heap), s); +} + +// `strndup` using mi_malloc +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept { + if (s == NULL) return NULL; + const size_t len = _mi_strnlen(s,n); // len <= n + if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1 + char* t = (char*)mi_theap_malloc(theap, len+1); + if (t == NULL) return NULL; + _mi_memcpy(t, s, len); + t[len] = 0; + return t; +} + +mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { + return mi_theap_strndup(_mi_theap_default(),s,n); +} + +mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { + return mi_theap_strndup(_mi_heap_theap(heap), s, n); +} + +#ifndef __wasi__ +// `realpath` using mi_malloc +#ifdef _WIN32 +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { + // todo: use GetFullPathNameW to allow longer file names + if (fname==NULL || *fname==0) { + errno = EINVAL; + return NULL; + } + char buf[PATH_MAX]; + DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); + if (res == 0) { + DWORD err = GetLastError(); + switch (err) { + case ERROR_LOCK_VIOLATION: + case ERROR_SHARING_VIOLATION: + case ERROR_INVALID_ACCESS: errno = EACCES; break; + case ERROR_INVALID_HANDLE: + case ERROR_INVALID_FUNCTION: errno = EINVAL; break; + case ERROR_PATH_NOT_FOUND: errno = ENOTDIR; break; + case ERROR_FILE_NOT_FOUND: errno = ENOENT; break; + case ERROR_NOT_ENOUGH_MEMORY: errno = ENOMEM; break; + default: errno = EIO; + } + return NULL; + } + else if (res > PATH_MAX) { + errno = ENAMETOOLONG; + return NULL; + } + else if (resolved_name != NULL) { + return resolved_name; + } + else { + return mi_theap_strndup(theap, buf, PATH_MAX); + } +} + +#else + +#include // pathconf + +static size_t mi_path_max(void) { + static _Atomic(size_t) path_max = 0; + size_t pmax = mi_atomic_load_acquire(&path_max); + if (pmax == 0) { + long m = 0; + #ifdef _PC_PATH_MAX + m = pathconf("/",_PC_PATH_MAX); + #endif + if (m <= 0) pmax = 4096; // guess + else if (m < 256) pmax = 256; // at least 256 + else if (m > 64*1024) pmax = 64*1024; // at most 64 KiB + else pmax = m; + size_t expected = 0; + mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax); + } + return pmax; +} + +char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { + if (resolved_name != NULL) { + return realpath(fname,resolved_name); + } + else { + /* + char* rname = realpath(fname, NULL); + if (rname == NULL) return NULL; + char* result = mi_heap_strdup(heap, rname); + mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator + // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-( + return result; + */ + const size_t n = mi_path_max(); + char* const buf = (char*)mi_zalloc(n+1); + if (buf == NULL) { + errno = ENOMEM; + return NULL; + } + char* rname = realpath(fname,buf); + char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL` + mi_free(buf); + return result; + } +} +#endif + +mi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { + return mi_theap_realpath(_mi_theap_default(),fname,resolved_name); +} + +mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { + return mi_theap_realpath(_mi_heap_theap(heap), fname, resolved_name); +} +#endif + +/*------------------------------------------------------- +C++ new and new_aligned +The standard requires calling into `get_new_handler` and +throwing the bad_alloc exception on failure. If we compile +with a C++ compiler we can implement this precisely. If we +use a C compiler we cannot throw a `bad_alloc` exception +but we call `abort` instead (i.e. not returning). +Also, the standard requires calling the new handler until +it returns false, but we limit the total calls. +-------------------------------------------------------*/ +#define MI_TRY_NEW_MAX (4) + +#ifdef __cplusplus +#include +static bool mi_try_new_handler(bool nothrow) { + #if defined(_MSC_VER) || (__cplusplus >= 201103L) + std::new_handler h = std::get_new_handler(); + #else + std::new_handler h = std::set_new_handler(); + std::set_new_handler(h); + #endif + if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); + #if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled + if (!nothrow) { + throw std::bad_alloc(); + } + #else + MI_UNUSED(nothrow); + #endif + return false; + } + else if (!nothrow) { + h(); + return true; + } + else { + try { + h(); + } + catch(...) { // swallow std::bad_alloc + return false; // stop trying + } + return true; + } +} +#else +typedef void (*std_new_handler_t)(void); + +#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631 +std_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) { + return NULL; +} +static std_new_handler_t mi_get_new_handler(void) { + return _ZSt15get_new_handlerv(); +} +#else +// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`. +static std_new_handler_t mi_get_new_handler(void) { + return NULL; +} +#endif + +static bool mi_try_new_handler(bool nothrow) { + std_new_handler_t h = mi_get_new_handler(); + if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); + if (!nothrow) { + abort(); // cannot throw in plain C, use abort + } + return false; + } + else { + h(); + return true; + } +} +#endif + +static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { + void* p = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { + if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once + p = mi_theap_malloc(theap,size); + } + return p; +} + +static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { + return mi_theap_try_new(_mi_theap_default(), size, nothrow); +} + +static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) { + return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); +} + +mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { + void* p = mi_theap_malloc(theap,size); + if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { + return mi_theap_alloc_new(_mi_theap_default(), size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { + void* p = mi_heap_malloc(heap, size); + if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); + return p; +} + +mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { + size_t total; + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { + mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + return NULL; + } + else { + return mi_theap_alloc_new(theap,total); + } +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { + return mi_theap_alloc_new_n(_mi_theap_default(), count, size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { + return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { + void* p = mi_malloc(size); + if mi_unlikely(p == NULL) return mi_try_new(size, true); + return p; +} + +static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) { + void* p = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) { + if (!mi_alignment_is_valid(alignment)) return NULL; + p = mi_malloc_aligned(size,alignment); + } + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true); + return p; +} + +static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) { + void* q = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) { + if (newsize > MI_MAX_ALLOC_SIZE) return NULL; + q = mi_realloc(p,newsize); + } + return q; +} + +mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { + void* q = mi_realloc(p, newsize); + if (q == NULL) return mi_try_new_realloc(p,newsize); + return q; +} + +mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { + size_t total; + if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { + mi_try_new_handler(false); + return NULL; + } + else { + return mi_new_realloc(p, total); + } +} + +#if MI_GUARDED +// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` + +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { + // todo: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + // note: the alignment of the guard page relies on blocks being os_page_size aligned which + // is ensured in `mi_arena_page_alloc_fresh`. + mi_assert_internal(_mi_is_aligned(block, os_page_size)); + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { + const bool ok = _mi_os_protect(guard_page, os_page_size); + if mi_unlikely(!ok) { + _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); + } + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_PAGE_MAX_OVERALLOC_ALIGN; + } + uint8_t* const p = (uint8_t*)block + offset; + mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); + if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } + mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); + return p; +} + +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // allocate multiple of page size ending in a guard page + // ensure minimal alignment requirement? + if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow + _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); + return NULL; + } + const size_t os_page_size = _mi_os_page_size(); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); + if (block==NULL) return NULL; + size_t usable_size = 0; + void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) + } + + // stats + mi_track_malloc(p, usable_size, zero); + if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } + mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); + #if MI_STAT>1 + // adjust stats to only count the allocated size of the block (and not the guard page) + mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); + mi_theap_stat_increase(theap, malloc_requested, size); + #endif + #if MI_DEBUG>3 + if (zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} +#endif + +// ------------------------------------------------------ +// ensure explicit external inline definitions are emitted! +// ------------------------------------------------------ + +#ifdef __cplusplus +void* _mi_externs[] = { + (void*)&_mi_page_malloc_zero, + (void*)&_mi_theap_malloc_zero, + (void*)&_mi_theap_malloc_zero_ex, + (void*)&mi_theap_malloc, + (void*)&mi_theap_zalloc, + (void*)&mi_theap_malloc_small, + (void*)&mi_theap_zalloc_small, + (void*)&mi_theap_calloc, + (void*)&mi_malloc, + (void*)&mi_malloc_small, + (void*)&mi_zalloc, + (void*)&mi_zalloc_small, + (void*)&mi_heap_malloc, + (void*)&mi_heap_malloc_small, + (void*)&mi_malloc_aligned + // (void*)&mi_theap_alloc_new, + // (void*)&mi_theap_alloc_new_n +}; +#endif diff --git a/csrc/mimalloc/src/arena.c b/csrc/mimalloc/src/arena.c new file mode 100644 index 000000000..a09a309fe --- /dev/null +++ b/csrc/mimalloc/src/arena.c @@ -0,0 +1,2753 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +"Arenas" are fixed area's of OS memory from which we can allocate +large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB). +In contrast to the rest of mimalloc, the arenas are shared between +threads and need to be accessed using atomic operations. + +Arenas are also used to for huge OS page (1GiB) reservations or for reserving +OS memory upfront which can be improve performance or is sometimes needed +on embedded devices. We can also employ this with WASI or `sbrk` systems +to reserve large arenas upfront and be able to reuse the memory more effectively. + +The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" +#include "bitmap.h" + + +/* ----------------------------------------------------------- + Arena id's +----------------------------------------------------------- */ + +mi_arena_id_t _mi_arena_id_none(void) { + return NULL; +} + +mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) { + mi_arena_t* const arena = (mi_arena_t*)id; + mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's + return arena; +} + +mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) { + mi_assert_internal(arena==NULL || arena->parent==NULL); + return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena); +} + + +static bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) { + if (arena == req_arena) return true; // they match + if (arena == NULL) return false; + if (req_arena == NULL && !arena->is_exclusive) return true; // or the arena is not exclusive, and we didn't request a specific one + if (arena->parent != NULL && arena->parent == req_arena) return true; // sub-arena? (note that req_arena is never a sub arena) + return false; +} + +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) { + if (memid.memkind == MI_MEM_ARENA) { + return mi_arena_is_suitable(memid.mem.arena.arena, request_arena); + } + else { + return mi_arena_is_suitable(NULL, request_arena); + } +} + +size_t mi_arenas_get_count(mi_subproc_t* subproc) { + return mi_atomic_load_relaxed(&subproc->arena_count); +} + +mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) { + mi_assert_internal(idx < mi_arenas_get_count(subproc)); + return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]); +} + +static size_t mi_arena_info_slices(mi_arena_t* arena) { + return arena->info_slices; +} + +#if MI_DEBUG > 1 +static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) { + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx]; + return (page->memid.memkind == MI_MEM_ARENA && + page->memid.mem.arena.arena == arena && + arena_pages != NULL && + mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1)); +} +#endif + +size_t mi_arena_min_alignment(void) { + return MI_ARENA_ALIGNMENT; +} + +size_t mi_arena_min_size(void) { + return MI_ARENA_MIN_SIZE; +} + +// slices reserved for page meta info at the start of aligned chunks +static size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +// fixed limit for the maximum object size in an arena +static size_t mi_arena_max_fixed_object_size(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); + #else + return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info + #endif +} + +// Maximum object size allowed to be allocated in an arena +size_t mi_arena_max_object_size(void) { + size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); + max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE); + if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { + return MI_ARENA_MIN_OBJ_SIZE; + } + else if (max_size >= mi_arena_max_fixed_object_size()) { + return mi_arena_max_fixed_object_size(); + } + else { + return max_size; + } +} + +mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(subproc!=NULL); + if (arena != NULL && arena->commit_fun != NULL) { + return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); + } + else if (stat_already_committed > 0) { + return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); + } + else { + return _mi_os_commit(subproc, start, size, is_zero); + } +} + + + +/* ----------------------------------------------------------- + Util +----------------------------------------------------------- */ + + +// Size of an arena +static size_t mi_arena_size(mi_arena_t* arena) { + return mi_size_of_slices(arena->slice_count); +} + +// Start of the arena memory area +static uint8_t* mi_arena_start(mi_arena_t* arena) { + return ((uint8_t*)arena->start); +} + +// Start of a slice +uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { + mi_assert_internal(slice_index < arena->slice_count); + return (mi_arena_start(arena) + mi_size_of_slices(slice_index)); +} + +mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { + mi_assert_internal(slice_index < arena->slice_count); + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + return page; + #else + if (arena->pages_meta != NULL) { + mi_page_t* const page = &arena->pages_meta[slice_index]; + #if MI_PAGE_META_SMALL_IS_ALIGNED + // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) + if (page->block_size>0) return page; + #else + return page; + #endif + } + // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) + return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + #endif +} + +// Arena area +void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { + if (size != NULL) *size = 0; + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena == NULL) return NULL; + if (size != NULL) { + mi_assert_internal(mi_size_of_slices(arena->slice_count) <= arena->total_size); + *size = arena->total_size; + } + return mi_arena_start(arena); +} + + +// Create an arena memid +static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert_internal(slice_index < UINT32_MAX); + mi_assert_internal(slice_count < UINT32_MAX); + mi_assert_internal(slice_count > 0); + mi_assert_internal(slice_index < arena->slice_count); + mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA); + memid.mem.arena.arena = arena; + memid.mem.arena.slice_index = (uint32_t)slice_index; + memid.mem.arena.slice_count = (uint32_t)slice_count; + return memid; +} + +// get the arena and slice span +static mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) { + mi_assert_internal(memid.memkind == MI_MEM_ARENA); + mi_arena_t* arena = memid.mem.arena.arena; + if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; } + if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; } + return arena; +} + +static size_t mi_page_full_size(mi_page_t* page) { + if (page->memid.memkind == MI_MEM_ARENA) { + return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE; + } + else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) { + mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page); + const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base; + mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize); + return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize); + } + else { + return 0; + } +} + +/* ----------------------------------------------------------- + Arena Allocation +----------------------------------------------------------- */ + +static mi_decl_noinline void* mi_arena_try_alloc_at( + mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) +{ + mi_assert_internal(arena!=NULL); + mi_assert_internal(slice_count>0); + size_t slice_index; + if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL; + + // claimed it! + void* p = mi_arena_slice_start(arena, slice_index); + *memid = mi_memid_create_arena(arena, slice_index, slice_count); + memid->is_pinned = arena->memid.is_pinned; + + // set the dirty bits and track which slices become accessible + size_t touched_slices = slice_count; + if (arena->memid.initially_zero) { + size_t already_dirty = 0; + memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty); + mi_assert_internal(already_dirty <= touched_slices); + touched_slices -= already_dirty; + } + else { + // todo: properly count touched pages with a separate bitmap? + touched_slices = 0; + } + + // set commit state + if (commit) { + // commit requested, but the range may not be committed as a whole: ensure it is committed now + const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); + if (already_committed < slice_count) { + // not all committed, try to commit now + bool commit_zero = false; + if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) { + // if the commit fails, release ownership, and return NULL; + // note: this does not roll back dirty bits but that is ok. + mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + return NULL; + } + if (commit_zero) { + memid->initially_zero = true; + } + + // set the commit bits + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL); + + // committed + #if MI_DEBUG > 1 + if (memid->initially_zero) { + if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) { + _mi_error_message(EFAULT, "internal error: arena allocation was not zero-initialized!\n"); + memid->initially_zero = false; + } + } + #endif + } + else { + // already fully committed. + _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count)); + // if the OS has overcommit, and this is the first time we access these pages, then + // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) + if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) { + mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices)); + } + } + + mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + memid->initially_committed = true; + + // tool support + if (memid->initially_zero) { + mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE); + } + else { + mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE); + } + } + else { + // no need to commit, but check if it is already fully committed + memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); + if (!memid->initially_committed) { + // partly committed.. adjust stats + size_t already_committed_count = 0; + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count)); + } + } + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); } + if (commit) { mi_assert_internal(memid->initially_committed); } + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); + + return p; +} + + +static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id); + +// try to reserve a fresh arena space +static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id) +{ + const size_t arena_count = mi_arenas_get_count(subproc); + if (arena_count > (MI_MAX_ARENAS - 4)) return false; + + // calc reserve + size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve); + if (arena_reserve == 0) return false; + + if (!_mi_os_has_virtual_reserve()) { + arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example) + } + arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE); + + if (arena_count >= 1 && arena_count <= 128) { + // scale up the arena sizes exponentially every 8 entries + const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16); + size_t reserve = 0; + if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) { + arena_reserve = reserve; + } + } + + // try to accommodate the requested size for huge allocations + req_size = _mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info + if (arena_reserve < req_size) { + arena_reserve = req_size; + } + + // check arena bounds + const size_t min_reserve = MI_ARENA_MIN_SIZE; + const size_t max_reserve = MI_ARENA_MAX_SIZE; // 16 GiB + if (arena_reserve < min_reserve) { + arena_reserve = min_reserve; + } + else if (arena_reserve > max_reserve) { + arena_reserve = max_reserve; + } + + // should be able to at least handle the current allocation size + if (arena_reserve < req_size) return false; + + // commit eagerly? + bool arena_commit = false; + const bool overcommit = _mi_os_has_overcommit(); + if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); } + else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; } + + // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice + // is actually allocated for the first time it will be counted. + const bool adjust = (overcommit && arena_commit); + if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); } + // and try to reserve the arena + int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); + if (err != 0) { + if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back + // failed to allocate: try a smaller size arena as fallback? + const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit) + if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) { + // try again + if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); } + err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); + if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back + } + } + return (err==0); +} + + + + +/* ----------------------------------------------------------- + Arena iteration +----------------------------------------------------------- */ + +static inline bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) { + if (!allow_pinned && arena->memid.is_pinned) return false; + if (!mi_arena_is_suitable(arena, req_arena)) return false; + if (req_arena == NULL) { // if not specific, check numa affinity + const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node); + if (match_numa) { if (!numa_suitable) return false; } + else { if (numa_suitable) return false; } + } + return true; +} + +// determine the start of search; important to keep heaps and threads +// into their own memory regions to reduce contention. +static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) { + const size_t hseq = heap->heap_seq; + const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count); + if (arena_cycle <= 1) return 0; + if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs + + // spread heaps evenly among arena's, and then evenly for threads in their fraction + size_t start; + mi_assert_internal(arena_cycle <= 0x8FF); // prevent overflow on 32-bit + const size_t frac = (arena_cycle * 256) / hcount; // fraction in the arena_cycle; at most: arena_cycle * 0x100 + if (frac==0) { + // many heaps (> 256 per arena) + start = (hseq % arena_cycle); + } + else { + const size_t hspot = (hseq % hcount); + start = (frac * hspot) / 256; // (arena_cycle * (hseq % hcount)) / hcount + if (frac >= 512) { // at least 2 arena's per heap? + start = start + (tseq % (frac/256)); + } + } + mi_assert_internal(start < arena_cycle); + return start; +} + +#define mi_forall_arenas(heap, req_arena, tseq, name_arena) { \ + const size_t _arena_count = mi_arenas_get_count(heap->subproc); \ + const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \ + /* always start searching in the arena's below the max */ \ + const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \ + for (size_t _i = 0; _i < _arena_count; _i++) { \ + mi_arena_t* name_arena; \ + if (req_arena != NULL) { \ + name_arena = req_arena; /* if there is a specific req_arena, only search that one */\ + if (_i > 0) break; /* only once */ \ + } \ + else { \ + size_t _idx; \ + if (_i < _arena_cycle) { \ + _idx = _i + _start; \ + if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \ + } \ + else { \ + _idx = _i; /* remaining arena's after the cycle */ \ + } \ + name_arena = mi_arena_from_index(heap->subproc,_idx); \ + } \ + if (name_arena != NULL) \ + { + +#define mi_forall_arenas_end() \ + } \ + } \ + } + +#define mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, numa_node, allow_large, name_arena) \ + mi_forall_arenas(heap, req_arena,tseq,name_arena) { \ + if (mi_arena_is_suitable_ex(name_arena, req_arena, match_numa, numa_node, allow_large)) { \ + +#define mi_forall_suitable_arenas_end() \ + }} \ + mi_forall_arenas_end() + +/* ----------------------------------------------------------- + Arena allocation +----------------------------------------------------------- */ + +// allocate slices from the arenas +static mi_decl_noinline void* mi_arenas_try_find_free( + mi_heap_t* heap, size_t slice_count, size_t alignment, + bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + // mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE)); + mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); + if (alignment > MI_ARENA_SLICE_ALIGN) return NULL; + + // search arena's + mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, arena) + { + void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid); + if (p != NULL) return p; + } + mi_forall_suitable_arenas_end(); + if (numa_node < 0) return NULL; + + // search again but now regardless of preferred numa affinity + mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, arena) + { + void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid); + if (p != NULL) return p; + } + mi_forall_suitable_arenas_end(); + return NULL; +} + +// Allocate slices from the arena's -- potentially allocating a fresh arena +static mi_decl_noinline void* mi_arenas_try_alloc( + mi_heap_t* heap, + size_t slice_count, size_t alignment, + bool commit, bool allow_large, + mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES); + mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); + void* p; + + // not too large? + if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL; + + // try to find free slices in the arena's + p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + + // did we need a specific arena? + if (req_arena != NULL) return NULL; + + // don't create arena's while preloading (todo: or should we?) + // if (_mi_preloading()) return NULL; + + // don't create arena's if OS allocation is disallowed + if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; + + // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?) + mi_subproc_t* const subproc = heap->subproc; + const size_t arena_count = mi_arenas_get_count(subproc); + mi_lock(&subproc->arena_reserve_lock) { + if (arena_count == mi_arenas_get_count(subproc)) { + // we are the first to enter the lock, reserve a fresh arena + mi_arena_id_t arena_id = _mi_arena_id_none(); + mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id); + } + else { + // another thread already reserved a new arena + } + } + // try once more to allocate in the new arena + mi_assert_internal(req_arena == NULL); + p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + + return NULL; +} + +// Allocate from the OS (if allowed) +static void* mi_arena_os_alloc_aligned( + mi_subproc_t* subproc, + size_t size, size_t alignment, size_t align_offset, + bool commit, bool allow_large, + mi_arena_id_t req_arena_id, mi_memid_t* memid) +{ + // if we cannot use OS allocation, return NULL + if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) { + errno = ENOMEM; + return NULL; + } + + if (align_offset > 0) { + return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid); + } + else { + return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); + } +} + + +// Allocate large sized memory +void* _mi_arenas_alloc_aligned( mi_heap_t* heap, + size_t size, size_t alignment, size_t align_offset, + bool commit, bool allow_large, + mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + mi_assert_internal(memid != NULL); + mi_assert_internal(size > 0); + + // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data) + if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed? + size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() && // and not too small or too large + alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment + { + const size_t slice_count = mi_slice_count_of_size(size); + void* p = mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid); + if (p != NULL) return p; + } + + // fall back to the OS + void* p = mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid); + return p; +} + +void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid) +{ + return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid); +} + + + +/* ----------------------------------------------------------- + Arena page allocation +----------------------------------------------------------- */ + +// release ownership of a page. This may free the page if all blocks were concurrently +// freed in the meantime. Returns true if the page was freed. +static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + mi_thread_free_t tf_new; + mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); + do { + mi_assert_internal(mi_tf_is_owned(tf_old)); + while mi_unlikely(mi_tf_block(tf_old) != NULL) { + _mi_page_free_collect(page, false); // update used + if (mi_page_all_free(page)) { // it may become free just before unowning it + _mi_arenas_page_unabandon(page, current_theapx); + _mi_arenas_page_free(page, current_theapx); + return true; + } + tf_old = mi_atomic_load_relaxed(&page->xthread_free); + } + mi_assert_internal(mi_tf_block(tf_old)==NULL); + tf_new = mi_tf_create(NULL, false); + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); + return false; +} + + +static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) { + // found an abandoned page of the right size + mi_page_t* const page = mi_arena_page_at_slice(arena, slice_index); + // can we claim ownership? + if (!mi_page_claim_ownership(page)) { + // there was a concurrent free that reclaims this page .. + // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`, + // and wait for readers (us!) to finish. This is why it is very important to set the abandoned + // bit again (or otherwise the unabandon will never stop waiting). + *keep_abandoned = true; + return false; + } + else { + // yes, we can reclaim it, keep the abandoned map entry clear + *keep_abandoned = false; + return true; + } +} + +// allocate initial arena_pages from the main heap +static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena); + +static mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { + mi_assert_internal(arena!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); +} + +static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) { + // todo: maybe store the arena* directly in the page? + mi_assert_internal(mi_page_is_owned(page)); + mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count); + mi_assert_internal(arena != NULL); + if (parena_pages != NULL) { + mi_heap_t* heap = mi_page_heap(page); + mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); + mi_assert_internal(arena_pages != NULL); + mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index)); + *parena_pages = arena_pages; + } + return arena; +} + +static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) { + mi_assert_internal(arena!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert(arena->arena_idx < MI_MAX_ARENAS); + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages==NULL) { + mi_lock(&heap->arena_pages_lock) { + arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]); + if (arena_pages == NULL) { // still NULL? + if (_mi_is_heap_main(heap)) { + // the page info for the main heap is always allocated as part of an arena + arena_pages = &arena->pages_main; + } + else { + // always allocate the arena pages info from the main heap + // todo: allocate into the current arena? + arena_pages = mi_arena_pages_alloc(arena); + } + mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages); + } + } + } + if (_mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); } // can never fail + return arena_pages; +} + +static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size) +{ + mi_heap_t* const heap = _mi_theap_heap(theap); + const size_t tseq = theap->tld->thread_seq; + mi_arena_t* const req_arena = heap->exclusive_arena; + + MI_UNUSED(slice_count); + const size_t bin = _mi_bin(block_size); + if (bin >= MI_ARENA_BIN_COUNT) { + return NULL; // singleton page size + } + + // any abandoned in our size class? + mi_assert_internal(heap != NULL); + if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) { + return NULL; + } + + // search arena's + const bool allow_large = true; + const int any_numa = -1; + const bool match_numa = true; + mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena) + { + mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages != NULL) { + size_t slice_index; + mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; + + if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { + // found an abandoned page of the right size + // and claimed ownership. + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_heap_has_page(heap, arena, page)); + mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); + mi_theap_stat_decrease(theap, pages_abandoned, 1); + mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1); + + _mi_page_free_collect(page, false); // update `used` count + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_block_size(page) == block_size); + mi_assert_internal(!mi_page_is_full(page)); + return page; + } + } + } + mi_forall_suitable_arenas_end(); + return NULL; +} + +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { + MI_UNUSED(max_page_meta_count); + mi_assert_internal(parena_pages!=NULL); + + *parena_pages = NULL; + const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page + const size_t page_alignment = MI_ARENA_SLICE_ALIGN; + + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_tld_t* const tld = theap->tld; + mi_arena_t* const req_arena = heap->exclusive_arena; + const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node); + + // try to allocate from free space in arena's + uint8_t* start = NULL; + *memid = _mi_memid_none(); + const size_t alloc_size = mi_size_of_slices(slice_count); + if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's? + !os_align && // not large alignment + slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large + { + start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid); + if (start != NULL) { + mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena); + *parena_pages = arena_pages; + if (arena_pages==NULL) { + _mi_arenas_free(heap->subproc, start, mi_size_of_slices(slice_count), *memid); // roll back + start = NULL; + } + else { + // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count + // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count)); + mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index)); + // don't set yet: mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index); + } + } + } + + // otherwise fall back to the OS + if (start == NULL) { + #if MI_PAGE_META_IS_ALIGNED + size_t page_offset; // offset in the block for the page area + uint8_t* os_start; + if (block_alignment < MI_PAGE_META_ALIGNMENT) { + page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + else { + // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer + // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); + errno = EINVAL; + return NULL; + // page_offset = MI_PAGE_META_ALIGNMENT; + // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + if (os_start==NULL) return NULL; + // commit page info and page + const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; + const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + mi_assert_internal(min_page_meta < page_offset); + bool is_zero; + bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + if (ok && commit) { + ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + } + if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } + if (!is_zero && !memid->initially_zero) { + _mi_memzero_aligned(os_start,min_page_meta); + } + start = os_start + page_offset; + mi_assert_internal(_mi_is_aligned(start,block_alignment)); + mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); + mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); + mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta); + memid->initially_committed = true; // so we don't commit again + #else + if (os_align) { + // note: slice_count already includes the page + start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); + mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); + } + else { + start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); + } + #endif + } + + if (start == NULL) return NULL; + mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN)); + return start; +} + +#if !MI_PAGE_META_IS_ALIGNED +// Only used for non-separate pages +static size_t mi_page_block_start(size_t block_size, bool os_align) +{ + size_t offset; + #if MI_GUARDED + // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size + // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages) + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(MI_PAGE_ALIGN >= os_page_size); + if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) { + offset = _mi_align_up(mi_page_info_size(), os_page_size); + } + else + #endif + if (os_align) { + offset = MI_PAGE_ALIGN; + } + else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) + offset = _mi_align_up(mi_page_info_size(), block_size); + } + else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { + // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) + offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); + } + else { + // otherwise start after the info + offset = mi_page_info_size(); + } + return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); +} +#endif + +// Free a page without modifying page_bin stats +static void mi_arenas_page_free_prim(mi_page_t* page); + +static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { + #if MI_PAGE_META_IS_ALIGNED + if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { + // ensure the meta data is committed + if (memid_slice.memkind == MI_MEM_ARENA) { + mi_arena_t* const arena = memid_slice.mem.arena.arena; + uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); + mi_assert_internal(meta_slices >= mi_arena_start(arena)); + const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; + if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { + // try to commit all page meta slices now + const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // the following assertion does not hold in a concurrent setting.. + // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); + const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; + if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { + // if the commit fails return NULL + return NULL; + } + // set the commit bits + mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); + } + } + mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start); + return page_meta; + } + #else + if (memid_slice.memkind == MI_MEM_ARENA) { + MI_UNUSED(slice_start); + mi_arena_t* const arena = memid_slice.mem.arena.arena; + if (arena->pages_meta != NULL) { + mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); + return &arena->pages_meta[memid_slice.mem.arena.slice_index]; + } + } + #endif + return NULL; +} + +// Allocate a fresh page +static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) +{ + const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); + const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + + const size_t alloc_size = mi_size_of_slices(slice_count); + mi_memid_t memid = _mi_memid_none(); + mi_arena_pages_t* arena_pages = NULL; + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages); + if (!slice_start) return NULL; + + // guard page at the end of mimalloc page? + #if MI_SECURE>=5 + mi_assert(alloc_size > _mi_os_secure_guard_page_size()); + const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size(); + #else + const size_t page_noguard_size = alloc_size; + #endif + + // allocate the page meta info + mi_page_t* page = NULL; + bool page_meta_is_separate = false; + size_t block_start = 0; + + // allocate page meta info at the arena start? + mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); + if (page_meta!=NULL) { + mi_assert_internal(page_meta->block_size == 0); + #if MI_PAGE_META_SMALL_IS_ALIGNED + // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, + // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) + if (block_size > MI_SMALL_SIZE_MAX) + #endif + { + page = page_meta; + page_meta_is_separate = true; + block_start = 0; + #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) + #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ + #endif + if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { + block_start += block_size; + } + mi_assert_internal(page->block_size == 0); + _mi_memzero_aligned(page, sizeof(*page)); + } + } + if (page == NULL) { + #if MI_PAGE_META_IS_ALIGNED + // can only happen on failing to commit the page meta info + _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid); + return NULL; + #else + // put page meta info in front of the slice + page = (mi_page_t*)slice_start; + block_start = mi_page_block_start(block_size, os_align); + #endif + } + mi_assert_internal(block_start % MI_MAX_ALIGN_SIZE == 0); + + // commit first block? + size_t commit_size = 0; + if (!memid.initially_committed) { + commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size()); + if (commit_size > page_noguard_size) { commit_size = page_noguard_size; } + bool is_zero = false; + if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) { + _mi_arenas_free(_mi_theap_subproc(theap), slice_start, alloc_size, memid); + return NULL; + } + } + // now we can finish initalization and use `mi_arenas_free_page_prim` on error + + // zero initialize the page meta data + if (!memid.initially_zero && !page_meta_is_separate) { + _mi_memzero_aligned(page, sizeof(*page)); + } + + // set the guard page + #if MI_SECURE>=5 + if (memid.initially_committed) { + _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid); + } + #endif + + // claimed free slices: initialize the page partly + if (!memid.initially_zero && memid.initially_committed) { + mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); + } + else if (memid.initially_committed) { + mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); + } + #if MI_DEBUG > 1 + if (memid.initially_zero && memid.initially_committed) { + if (!mi_mem_is_zero(slice_start, page_noguard_size)) { + _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n"); + memid.initially_zero = false; + if (block_start > 0) { _mi_memzero_aligned(page, sizeof(*page)); } + } + } + #endif + const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size); + mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX); + + // initialize the page start + uint8_t* const start = slice_start + block_start; + mi_assert_internal(start > (uint8_t*)page); + page->page_offset = start - (uint8_t*)page; + + // initialize page meta-data + page->reserved = (uint16_t)reserved; + page->block_size = block_size; + page->memid = memid; + page->free_is_zero = memid.initially_zero; + + mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); + page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size()); + + page->heap = _mi_theap_heap(theap); + mi_page_set_theap(page,theap); + // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) + + #if MI_PAGE_META_IS_ALIGNED + mi_atomic_store_ptr_release(mi_page_t,&page->self,page); + if (page_meta_is_separate) { + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + for(size_t i = 1; i < max_page_meta_count; i++) { + mi_assert_internal(page[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page[i].self,page); + } + } + } + #if MI_DEBUG>1 + mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); + if (reserved>1) { + mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); + } + #endif + #endif + + mi_assert_internal(page->free==NULL); + mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); + mi_assert_internal(mi_page_slice_start(page) == slice_start); + mi_assert_internal(mi_page_size(page) <= page_noguard_size); + + + // now register in the arena_pages + if (arena_pages!=NULL) { + mi_assert_internal(memid.memkind == MI_MEM_ARENA); + mi_bitmap_set(arena_pages->pages, memid.mem.arena.slice_index); + } + + // and own it + mi_page_claim_ownership(page); + + // register in the page map + if mi_unlikely(!_mi_page_map_register(page)) { + mi_arenas_page_free_prim(page); + return NULL; + } + + // stats + mi_theap_stat_increase(theap, pages, 1); + mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1); + + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_block_size(page) == block_size); + // mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_page_is_owned(page)); + + return page; +} + +// Allocate a regular small/medium/large page. +static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size) +{ + // 1. look for an abandoned page + mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size); + if (page != NULL) { + return page; // return as abandoned + } + + // 2. find a free block, potentially allocating a new arena + const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); + const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages + (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed + (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); + page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); + if (page == NULL) return NULL; + + mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); + if (!_mi_page_init(theap, page)) { + _mi_arenas_page_free(page,theap); + return NULL; + } + + return page; +} + +// Allocate a page containing one block (very large, or with large alignment) +static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) +{ + #if MI_PAGE_META_IS_ALIGNED + const size_t info_size = 0; + #else + const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + #endif + #if MI_SECURE < 2 + const size_t slice_count = mi_slice_count_of_size(info_size + block_size); + #else + const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); + #endif + + mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */); + if (page == NULL) return NULL; + + mi_assert(page->reserved == 1); + if (!_mi_page_init(theap, page)) { + _mi_arenas_page_free(page,theap); + return NULL; + } + + return page; +} + + +mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { + mi_page_t* page; + // semi static assert: ensure that all non-singleton block size bins are covered. + mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT); + if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { + mi_assert_internal(_mi_is_power_of_two(block_alignment)); + page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); + } + else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) { + page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size); + } + else if (block_size <= MI_MEDIUM_MAX_OBJ_SIZE) { + page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size); + } + #if MI_ENABLE_LARGE_PAGES + else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) { + page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size); + } + #endif + else { + page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment); + } + if mi_unlikely(page == NULL) { + return NULL; + } + // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || _mi_is_aligned(mi_page_start(page), block_alignment)); + + return page; +} + +static void mi_arenas_page_free_prim(mi_page_t* page) { + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(page->next==NULL && page->prev==NULL); + + #if MI_DEBUG>1 + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + size_t bin = _mi_bin(mi_page_block_size(page)); + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); + mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1)); + // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may + // be (temporarily) not true if the free happens while trying to reclaim + // see `mi_arena_try_claim_abandoned` + } + #endif + + // unregister page + _mi_page_map_unregister(page); + + // recommit guard page at the end? + // we must do this since we may later allocate large spans over this page and cannot have a guard page in between + #if MI_SECURE >= 5 + if (!page->memid.is_pinned) { + _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); + } + #endif + + // and free + if (page->memid.memkind == MI_MEM_ARENA) { + mi_arena_pages_t* arena_pages; + size_t slice_index; + size_t slice_count; MI_UNUSED(slice_count); + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(arena_pages!=NULL); + mi_assert_internal(arena->subproc == mi_page_subproc(page)); + mi_bitmap_clear(arena_pages->pages, slice_index); + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { + // if committed on-demand, set the commit bits to account commit properly + mi_assert_internal(mi_page_full_size(page) >= slice_committed); + const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative + //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices)); + mi_assert_internal(slice_count >= total_slices); + if (total_slices > 0) { + mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL); + } + // any left over? + const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE; + if (extra > 0) { + // pretend it was decommitted already + mi_subproc_stat_decrease(arena->subproc, committed, extra); + } + } + else { + mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + } + } + if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking + _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); +} + +void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(page->next==NULL && page->prev==NULL); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + + mi_heap_t* const heap = mi_page_heap(page); + mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); + mi_theapx_stat_decrease(heap, current_theapx, pages, 1); + mi_arenas_page_free_prim(page); +} + +/* ----------------------------------------------------------- + Arena abandon +----------------------------------------------------------- */ + +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_all_free(page)); + mi_assert_internal(page->next==NULL && page->prev == NULL); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); + + // add to abandoned? + mi_heap_t* heap = mi_page_heap(page); + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + // make available for allocations + size_t bin = _mi_bin(mi_page_block_size(page)); + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + if (bin < MI_ARENA_BIN_COUNT) { // paranoia + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); + + mi_assert_internal(!mi_page_is_singleton(page)); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); + + mi_page_set_abandoned_mapped(page); + const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); + MI_UNUSED(was_clear); mi_assert_internal(was_clear); + mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); + return; + } + } + // otherwise, + // page is full (or a singleton), or the page is OS/externally allocated + // leave as is; it will be reclaimed when an object is free'd in the page + // but for non-arena pages, add to the subproc list so these can be visited + if (page->memid.memkind != MI_MEM_ARENA) { + mi_lock(&heap->os_abandoned_pages_lock) { + // push in front + page->prev = NULL; + page->next = heap->os_abandoned_pages; + if (page->next != NULL) { page->next->prev = page; } + heap->os_abandoned_pages = page; + } + } + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); +} + + +// this is called from `free.c:mi_free_try_collect_mt` only. +bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_is_abandoned_mapped(page)); + mi_assert_internal(!mi_page_is_full(page)); + mi_assert_internal(!mi_page_all_free(page)); + mi_assert_internal(!mi_page_is_singleton(page)); + if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { + return false; + } + else { + // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap + mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL + mi_heap_t* const heap = mi_page_heap(page); + // if (theapx==NULL) return false; + mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1); + mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh + _mi_arenas_page_abandon(page, theapx); + return true; + } +} + +// called from `mi_free` if trying to unabandon an abandoned page +void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); + + mi_heap_t* const heap = mi_page_heap(page); + if (mi_page_is_abandoned_mapped(page)) { + mi_assert_internal(page->memid.memkind==MI_MEM_ARENA); + // remove from the abandoned map + const size_t bin = _mi_bin(mi_page_block_size(page)); + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + size_t slice_index; + size_t slice_count; + mi_arena_pages_t* arena_pages; + mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + + // this busy waits until a concurrent reader (from alloc_abandoned) is done + mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); + mi_page_clear_abandoned_mapped(page); + mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); + } + else { + // page is full (or a singleton), page is OS allocated + // if not an arena page, remove from the subproc os pages list + if (page->memid.memkind != MI_MEM_ARENA) { + mi_lock(&heap->os_abandoned_pages_lock) { + if (page->prev != NULL) { page->prev->next = page->next; } + if (page->next != NULL) { page->next->prev = page->prev; } + if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; } + page->next = NULL; + page->prev = NULL; + } + } + } + mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); +} + + +/* ----------------------------------------------------------- + Arena free +----------------------------------------------------------- */ +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices); +static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); + +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { + if (p==NULL) return; + if (size==0) return; + + // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) + mi_track_mem_undefined(p, size); + + if (mi_memkind_is_os(memid.memkind)) { + // was a direct OS allocation, pass through + _mi_os_free(subproc, p, size, memid); + } + else if (memid.memkind == MI_MEM_ARENA) { + // allocated in an arena + size_t slice_count; + size_t slice_index; + mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count); + mi_assert_internal(arena!=NULL); + mi_assert_internal(arena->subproc == subproc); + mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0); + mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size); + mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p); + mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p); + // checks + if (arena == NULL) { + _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); + return; + } + mi_assert_internal(slice_index < arena->slice_count); + mi_assert_internal(slice_index >= mi_arena_info_slices(arena)); + if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) { + _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind); + return; + } + + // potentially decommit + if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed? + // (delay) purge the page + mi_arena_schedule_purge(arena, slice_index, slice_count); + } + + // and make it available to others again + bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + if (!all_inuse) { + _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count)); + return; + }; + } + else if (memid.memkind == MI_MEM_MALLOC) { + _mi_free_subproc_safe(p); + } + else { + // arena was none, external, or static; nothing to do + mi_assert_internal(mi_memid_needs_no_free(memid)); + } + + // try to purge expired decommits + // mi_arenas_try_purge(false, false, NULL); +} + +// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired +void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) { + mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq); +} + + +// Is a pointer contained in the given arena area? +static bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) { + return (arena != NULL && + mi_arena_start(arena) <= (const uint8_t*)p && + mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p); +} + +// Is a pointer inside any of our arenas? +static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) { + mi_subproc_t* subproc = _mi_subproc(); + const size_t max_arena = mi_arenas_get_count(subproc); + for (size_t i = 0; i < max_arena; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena != NULL) { + if (parent==NULL || arena==parent || arena->parent==parent) { + if (mi_arena_strictly_contains(arena, p)) { + return true; + } + } + } + } + return false; +} + +// // Is a pointer inside any of our arenas? +// bool _mi_arenas_contain(const void* p) { +// return mi_arenas_contain_ex(p, NULL); +// } + +// Is a pointer contained in the given arena area? +bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) { + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena==NULL) return false; + else if (mi_arena_strictly_contains(arena, p)) return true; + else return mi_arenas_contain_ex(p, arena); // maybe a subarena? +} + + +/* ----------------------------------------------------------- + Remove an arena. +----------------------------------------------------------- */ + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { + mi_assert_internal(subproc != NULL); + const size_t arena_count = mi_arenas_get_count(subproc); + for (size_t i = 0; i < arena_count; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena != NULL) { + // mi_lock_done(&arena->abandoned_visit_lock); + mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); + if (mi_memkind_is_os(arena->memid.memkind)) { + _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid); + } + } + } + // try to lower the max arena. + size_t expected = arena_count; + mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0); +} + + +// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` +// for dynamic libraries that are unloaded and need to release all their allocated memory. +void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) { + mi_arenas_unsafe_destroy(subproc); + // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */); // purge non-owned arenas +} + + +/* ----------------------------------------------------------- + Add an arena. +----------------------------------------------------------- */ + +static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id) +{ + mi_assert_internal(arena != NULL); + mi_assert_internal(arena->slice_count > 0); + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + + // try to find a NULL entry + mi_arena_t* expected; + size_t count = mi_arenas_get_count(subproc); + for( size_t i = 0; i < count; i++) { + if (mi_arena_from_index(subproc,i) == NULL) { + arena->arena_idx = i; + expected = NULL; + if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) { + // success + if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } + return true; + } + } + } + + // otherwise, try to allocate a fresh slot + while(countarena_count, &count, count+1)) { + arena->arena_idx = count; + expected = NULL; + if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) { + mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1); + if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); } + return true; + } + } + } + + // failed + arena->arena_idx = 0; + arena->subproc = NULL; + return false; +} + +static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { + if (slice_count == 0) slice_count = MI_BCHUNK_BITS; + mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); + const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE); + const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned + const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL); + const size_t size = base_size + bitmaps_size; + if (bitmap_base != NULL) *bitmap_base = base_size; + return size; +} + +static mi_arena_t* mi_arena_info(void* area) { + return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); +} + +static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { + if (slice_count == 0) slice_count = MI_BCHUNK_BITS; + mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); + const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned + const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED + const size_t pages_size = slice_count * sizeof(mi_page_t); + #else + const size_t pages_size = 0; + #endif + const size_t size = base_size + bitmaps_size + pages_size; + + const size_t os_page_size = _mi_os_page_size(); + const size_t info_size = _mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size(); + const size_t info_slices = mi_slice_count_of_size(info_size); + + if (bitmap_base != NULL) *bitmap_base = base_size; + return info_slices; +} + +static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) { + mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base); + *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */); + return bitmap; +} + +static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) { + mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base); + *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); + return bbitmap; +} + +static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { + const size_t slice_count = arena->slice_count; + size_t bitmap_base = 0; + const size_t size = mi_arena_pages_size(slice_count, &bitmap_base); + mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE); + if (arena_pages==NULL) return NULL; + uint8_t* base = (uint8_t*)arena_pages + bitmap_base; + mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE)); + arena_pages->pages = mi_arena_bitmap_init(slice_count, &base); + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + } + return arena_pages; +} + +static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, + size_t slice_count, mi_arena_t* parent, size_t total_size, + int numa_node, bool exclusive, + mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) +{ + mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT)); + mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); + + if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now + // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed) + _mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", mi_size_of_slices(slice_count)/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB); + return NULL; + } + + size_t bitmap_base; + const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base); + if (slice_count < info_slices+1) { + _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB); + return NULL; + } + // else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) { + // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); + // return NULL; + // } + + // commit & zero if needed + if (!memid.initially_committed) { + size_t commit_size = mi_size_of_slices(info_slices); + // leave a guard OS page decommitted at the end? + if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } + bool ok = false; + if (commit_fun != NULL) { + ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); + } + else { + ok = _mi_os_commit(subproc, start, commit_size, NULL); + } + if (!ok) { + _mi_warning_message("unable to commit meta-data for OS memory"); + return NULL; + } + } + else if (!memid.is_pinned) { + // if MI_SECURE, set a guard page at the end of the arena info + // todo: this does not respect the commit_fun as the memid is of external memory + _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid); + } + if (!memid.initially_zero) { + _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); + } + + // init + mi_arena_t* arena = mi_arena_info(start); + arena->start = start; + arena->subproc = subproc; + arena->memid = memid; + arena->is_exclusive = exclusive; + arena->slice_count = slice_count; + arena->info_slices = info_slices; + if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) { + arena->numa_node = _mi_os_numa_node(); + } + else { + arena->numa_node = numa_node; + } + arena->purge_expire = 0; + arena->commit_fun = commit_fun; + arena->commit_fun_arg = commit_fun_arg; + arena->parent = parent; + arena->total_size = total_size; + + // init bitmaps + uint8_t* base = mi_arena_start(arena) + bitmap_base; + arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base); + arena->slices_committed = mi_arena_bitmap_init(slice_count, &base); + arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base); + arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); + arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base); + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + } + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED + arena->pages_meta = (mi_page_t*)base; + base += (slice_count * sizeof(mi_page_t)); + #else + arena->pages_meta = NULL; + #endif + mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); + + // reserve our meta info (and reserve slices outside the memory area) + #if MI_PAGE_META_IS_ALIGNED + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { + // set all free slices (and skip the slices reserved for the page meta info) + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); + const size_t start_idx = (i + meta_slices); + size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; + if (start_idx < arena->slice_count) { + if (count + start_idx > arena->slice_count) { + count = arena->slice_count - start_idx; + mi_assert_internal(count > 0); + } + mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); + } + } + #else + mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); + #endif + if (memid.initially_committed) { + mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); + } + if (!memid.initially_zero) { + mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count); + } + + if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL; } + return arena; +} + +static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive, + mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept +{ + // checks + mi_assert(start!=NULL); + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + if (start==NULL) return false; + if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { + // we can align the start since the memid tracks the real base of the memory. + void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); + const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; + if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { + _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); + return false; + } + start = aligned_start; + size = size - diff; + } + + // allocate enough arena's to span the full memory area + // the first arena is the owner, the rest are "sub-arena" (with `parent` pointing to the first one) + size_t total_slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS); + size_t total_size = mi_size_of_slices(total_slice_count); + if (total_size < MI_ARENA_MIN_SIZE) { + _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB); + return false; + } + + mi_arena_t* parent = NULL; + do { + // counting down on the total_slice_count + size_t slice_count = total_slice_count; + if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now (with 64KiB slices) + slice_count = MI_BITMAP_MAX_BIT_COUNT; + } + + // initialize + mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent, + (parent==NULL ? total_size : 0), numa_node, exclusive, + memid, commit_fun, commit_fun_arg, + (parent==NULL ? arena_id : NULL)); + if (arena==NULL) { + // failed to initialize due to failing commit or too many arena's + if (parent==NULL) { + return false; + } + else { + // partial success, but failed to use the full area.. + // todo: roll-back in this case? that requires a lock on the arena's array though + mi_assert(mi_size_of_slices(total_slice_count) <= parent->total_size); + parent->total_size -= mi_size_of_slices(total_slice_count); + return true; + } + } + + // success + if (parent==NULL) { + parent = arena; + memid.memkind = MI_MEM_NONE; + } + mi_assert(slice_count <= total_slice_count); + total_slice_count -= slice_count; + start = (uint8_t*)start + mi_size_of_slices(slice_count); + } + while (total_slice_count > 0); + + return true; +} + +bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); + memid.mem.os.base = start; + memid.mem.os.size = size; + memid.initially_committed = is_committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_pinned; + return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id); +} + +bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept +{ + mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL); + memid.mem.os.base = start; + memid.mem.os.size = size; + memid.initially_committed = is_committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_pinned; + return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id); +} + + +// Reserve a range of regular OS memory +static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) { + if (arena_id != NULL) *arena_id = _mi_arena_id_none(); + if (size <= MI_MAX_ALLOC_SIZE) { + size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + } + if (size > MI_MAX_ALLOC_SIZE) { + _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size); + return ENOMEM; + } + mi_memid_t memid; + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); + if (start == NULL) return ENOMEM; + if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { + _mi_os_free_ex(subproc, start, size, commit, memid); + _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); + return ENOMEM; + } + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), memid.is_pinned ? " (in large os pages)" : ""); + // mi_debug_show_arenas(true, true, false); + + return 0; +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id); +} + +// Manage a range of regular OS memory +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL); +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { + return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL); +} + + +/* ----------------------------------------------------------- + Debugging +----------------------------------------------------------- */ + +// Return idx of the slice past the last used slice +static size_t mi_arena_used_slices(mi_arena_t* arena) { + size_t idx; + if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) { + return (idx + 1); + } + else { + return mi_arena_info_slices(arena); + } +} + +static size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) { + size_t bit_set_count = 0; + for (int bit = 0; bit < MI_BFIELD_BITS; bit++) { + bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); + if (is_set) bit_set_count++; + buf[*k] = (is_set ? 'x' : '.'); + *k = *k + 1; + } + return bit_set_count; +} + +typedef enum mi_ansi_color_e { + MI_BLACK = 30, + MI_MAROON, + MI_DARKGREEN, + MI_ORANGE, + MI_NAVY, + MI_PURPLE, + MI_TEAL, + MI_GRAY, + MI_DARKGRAY = 90, + MI_RED, + MI_GREEN, + MI_YELLOW, + MI_BLUE, + MI_MAGENTA, + MI_CYAN, + MI_WHITE +} mi_ansi_color_t; + +static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { + *k += _mi_snprintf(buf + *k, 32, "\x1B[%dm", (int)color); +} + +static int mi_page_commit_usage(mi_page_t* page) { + const size_t committed_size = mi_page_committed(page); + const size_t used_size = page->used * mi_page_block_size(page); + return (int)(used_size * 100 / committed_size); +} + +static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) { + size_t bit_set_count = 0; + long bit_of_page = *pbit_of_page; + mi_ansi_color_t color = *pcolor_of_page; + mi_ansi_color_t prev_color = MI_GRAY; + for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) { + // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0); + void* start = mi_arena_slice_start(arena, slice_index + bit); + mi_page_t* page = _mi_safe_ptr_page(start); + char c = ' '; + if (page!=NULL && start==mi_page_slice_start(page)) { + mi_assert_internal(bit_of_page <= 0); + bit_set_count++; + c = 'p'; + color = MI_GRAY; + if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; } + else if (mi_page_is_singleton(page)) { c = 's'; } + else if (mi_page_is_full(page)) { c = 'f'; } + if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); } + int commit_usage = mi_page_commit_usage(page); + if (commit_usage < 25) { color = MI_MAROON; } + else if (commit_usage < 50) { color = MI_ORANGE; } + else if (commit_usage < 75) { color = MI_TEAL; } + else color = MI_DARKGREEN; + bit_of_page = (long)page->memid.mem.arena.slice_count; + } + else { + c = '?'; + if (bit_of_page > 0) { c = '-'; } + // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } + else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } + #if MI_PAGE_META_IS_ALIGNED + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { + { c = 'i'; color = MI_GRAY; } + } + #endif + // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } + else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { + if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } + else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; } + else { c = '.'; color = MI_GRAY; } + } + if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; } + } + if (color != prev_color) { + mi_debug_color(buf, k, color); + prev_color = color; + } + buf[*k] = c; *k += 1; + } + mi_debug_color(buf, k, MI_GRAY); + *pbit_of_page = bit_of_page; + *pcolor_of_page = color; + return bit_set_count; +} + +static size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3, + size_t slice_count, size_t chunk_count, + mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) +{ + _mi_raw_message("\x1B[37m%s%s%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header1, header2, header3); + const size_t fields_per_line = (narrow ? 2 : 4); + const size_t used_slice_count = mi_arena_used_slices(arena); + size_t bit_count = 0; + size_t bit_set_count = 0; + long bit_of_page = 0; + mi_ansi_color_t color_of_page = MI_GRAY; + for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) { + char buf[5*MI_BCHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf)); + if (bit_count > used_slice_count && i+2 < chunk_count) { + const size_t diff = chunk_count - 1 - i; + bit_count += diff*MI_BCHUNK_BITS; + _mi_raw_message(" |\n"); + i = chunk_count-1; + } + + size_t k = 0; + + if (i<10) { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; } + else if (i<100) { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; } + else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); } + + char chunk_kind = ' '; + if (chunk_bins != NULL) { + switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) { + case MI_CBIN_SMALL: chunk_kind = 'S'; break; + case MI_CBIN_MEDIUM: chunk_kind = 'M'; break; + case MI_CBIN_LARGE: chunk_kind = 'L'; break; + case MI_CBIN_HUGE: chunk_kind = 'H'; break; + case MI_CBIN_OTHER: chunk_kind = 'X'; break; + default: chunk_kind = ' '; break; // suppress warning + // case MI_CBIN_NONE: chunk_kind = 'N'; break; + } + } + buf[k++] = chunk_kind; + buf[k++] = ' '; + + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + if (j > 0 && (j % fields_per_line) == 0) { + // buf[k++] = '\n'; _mi_memset(buf+k,' ',7); k += 7; + _mi_raw_message(" %s\n\x1B[37m", buf); + _mi_memzero(buf, sizeof(buf)); + _mi_memset(buf, ' ', 5); k = 5; + } + if (bit_count < slice_count) { + mi_bfield_t bfield = 0; + if (chunks!=NULL) { + bfield = chunks[i].bfields[j]; + } + if (invert) bfield = ~bfield; + size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page) + : mi_debug_show_bfield(bfield, buf, &k)); + if (invert) xcount = MI_BFIELD_BITS - xcount; + bit_set_count += xcount; + buf[k++] = ' '; + } + else { + _mi_memset(buf + k, 'o', MI_BFIELD_BITS); + k += MI_BFIELD_BITS; + } + bit_count += MI_BFIELD_BITS; + } + _mi_raw_message(" %s\n\x1B[37m", buf); + } + _mi_raw_message("\x1B[0m total pages: %zu\n", bit_set_count); + return bit_set_count; +} + +//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count, +// mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) { +// return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow); +//} + +static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept { + mi_subproc_t* subproc = heap->subproc; + size_t max_arenas = mi_arenas_get_count(subproc); + //size_t free_total = 0; + //size_t slice_total = 0; + //size_t abandoned_total = 0; + size_t page_total = 0; + for (size_t i = 0; i < max_arenas; i++) { + mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); + if (arena == NULL) continue; + mi_assert(arena->subproc == subproc); + // slice_total += arena->slice_count; + _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n", + (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB), + (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""), + arena->subproc->subproc_seq, arena->numa_node); + //if (show_inuse) { + // free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); + //} + //if (show_committed) { + // mi_debug_show_bitmap("committed slices", arena->slice_count, arena->slices_committed, false, NULL); + //} + // todo: abandoned slices + //if (show_purge) { + // purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL); + //} + if (show_pages) { + // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + // if (arena_pages != NULL) + { + const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)"; + const char* header2 = (narrow ? "\n " : " "); + const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)"; + page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count, + mi_bbitmap_chunk_count(arena->slices_free), NULL, + arena->slices_free->chunkmap_bins, false, arena, narrow); + } + } + } + // if (show_inuse) _mi_raw_message("total inuse slices : %zu\n", slice_total - free_total); + // if (show_abandoned) _mi_raw_message("total abandoned slices: %zu\n", abandoned_total); + if (show_pages) _mi_raw_message("total pages in arenas: %zu\n", page_total); +} + +void mi_debug_show_arenas(void) mi_attr_noexcept { + mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */); +} + +void mi_arenas_print(void) mi_attr_noexcept { + mi_debug_show_arenas(); +} + + +/* ----------------------------------------------------------- + Reserve a huge page arena. +----------------------------------------------------------- */ +// reserve at a specific numa node +int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { + if (arena_id != NULL) *arena_id = NULL; + if (pages==0) return 0; + if (numa_node < -1) numa_node = -1; + if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); + mi_subproc_t* subproc = _mi_subproc(); + size_t hsize = 0; + size_t pages_reserved = 0; + mi_memid_t memid; + void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); + if (p==NULL || pages_reserved==0) { + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); + return ENOMEM; + } + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); + + if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) { + _mi_os_free(subproc, p, hsize, memid); + return ENOMEM; + } + return 0; +} + +int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { + return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL); +} + +// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected) +int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept { + if (pages == 0) return 0; + + // pages per numa node + int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count()); + if (numa_count <= 0) { numa_count = 1; } + const size_t pages_per = pages / numa_count; + const size_t pages_mod = pages % numa_count; + const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50); + + // reserve evenly among numa nodes + for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) { + size_t node_pages = pages_per; // can be 0 + if ((size_t)numa_node < pages_mod) { node_pages++; } + int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per); + if (err) return err; + if (pages < node_pages) { + pages = 0; + } + else { + pages -= node_pages; + } + } + + return 0; +} + +int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { + MI_UNUSED(max_secs); + _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); + if (pages_reserved != NULL) *pages_reserved = 0; + int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); + if (err==0 && pages_reserved!=NULL) *pages_reserved = pages; + return err; +} + + + + + +/* ----------------------------------------------------------- + Arena purge +----------------------------------------------------------- */ + +static long mi_arena_purge_delay(void) { + // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay + const long delay = mi_option_get(mi_option_purge_delay); + const long mult = mi_option_get(mi_option_arena_purge_mult); + if (delay<0 || mult<0) { return -1; } + if (delay==0 || mult==0) { return 0; } + size_t total; + if (mi_mul_overflow((size_t)delay, (size_t)mult, &total)) { return delay; } + if (total > LONG_MAX) { return delay; } + return (long)total; +} + +// reset or decommit in an arena and update the commit bitmap +// assumes we own the area (i.e. slices_free is claimed by us) +// returns if the memory is no longer committed (versus reset which keeps the commit) +static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert_internal(!arena->memid.is_pinned); + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + + const size_t size = mi_size_of_slices(slice_count); + void* const p = mi_arena_slice_start(arena, slice_index); + //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); + size_t already_committed; + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..) + const bool all_committed = (already_committed == slice_count); + const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg); + + if (needs_recommit) { + // no longer committed + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + // we just counted in the purge to decommit all, but the some part was not committed so adjust that here + // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(slice_count - already_committed)); + } + else if (!all_committed) { + // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed) + mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); + // we adjust the commit count as parts will be re-committed + // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed)); + } + + return needs_recommit; +} + + +// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls. +// Note: assumes we (still) own the area as we may purge immediately +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + const long delay = mi_arena_purge_delay(); + if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return; // is purging allowed at all? + + mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); + if (delay == 0) { + // purge directly + mi_arena_purge(arena, slice_index, slice_count); + } + else { + // schedule purge + const mi_msecs_t expire = _mi_clock_now() + delay; + mi_msecs_t expire0 = 0; + if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) { + // expiration was not yet set + // maybe set the global arenas expire as well (if it wasn't set already) + mi_assert_internal(expire0==0); + mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire); + } + else { + // already an expiration was set + } + mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL); + } +} + +typedef struct mi_purge_visit_info_s { + mi_msecs_t now; + mi_msecs_t delay; + bool all_purged; + bool any_purged; +} mi_purge_visit_info_t; + +static bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) { + mi_assert(slice_count < MI_BCHUNK_BITS); + if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) { + // purge + bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted); + mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count)); + // and reset the free range + mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); + return true; + } + else { + // was allocated again already + return false; + } +} + +static bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg; + // try to purge: first claim the free blocks + if (mi_arena_try_purge_range(arena, slice_index, slice_count)) { + vinfo->any_purged = true; + vinfo->all_purged = true; + } + else if (slice_count > 1) + { + // failed to claim the full range, try per slice instead + for (size_t i = 0; i < slice_count; i++) { + const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1); + vinfo->any_purged = vinfo->any_purged || purged; + vinfo->all_purged = vinfo->all_purged && purged; + } + } + // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges + // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count); + return true; // continue +} + +// returns +// -1 = nothing was purged +// 0 = nothing was purged yet because have not yet reached the expire time +// 1 = some pages in the arena were purged +static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) +{ + // check pre-conditions + if (arena->memid.is_pinned) return -1; + + // expired yet? + mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); + if (expire==0) return -1; + if (!force && expire > now) return 0; + + // reset expire + mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); + mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1); + + // go through all purge info's (with max MI_BFIELD_BITS ranges at a time) + // this also clears those ranges atomically (so any newly freed blocks will get purged next + // time around) + mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/}; + + // we purge by at least `minslices` to not fragment transparent huge pages for example + const size_t minslices = mi_slice_count_of_size(_mi_os_minimal_purge_size()); + _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo); + + return (vinfo.any_purged ? 1 : -1); +} + + +static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq) +{ + // try purge can be called often so try to only run when needed + const long delay = mi_arena_purge_delay(); + if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled + + // check if any arena needs purging? + const mi_msecs_t now = _mi_clock_now(); + const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); + if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return; + + const size_t max_arena = mi_arenas_get_count(subproc); + if (max_arena == 0) return; + + // allow only one thread to purge at a time (todo: allow concurrent purging?) + static mi_atomic_guard_t purge_guard; + mi_atomic_guard(&purge_guard) + { + // increase global expire: at most one purge per delay cycle + if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); } + const size_t arena_start = tseq % max_arena; + size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1); + bool all_visited = true; + bool any_purged = false; + for (size_t _i = 0; _i < max_arena; _i++) { + size_t i = _i + arena_start; + if (i >= max_arena) { i -= max_arena; } + mi_arena_t* arena = mi_arena_from_index(subproc,i); + if (arena != NULL) { + const int purged = mi_arena_try_purge(arena, now, force); + if (purged >= 0) { // purged, or arena expire is not yet reached + any_purged = true; + if (purged >= 1) { // purged + if (max_purge_count <= 1) { + all_visited = false; + break; + } + max_purge_count--; + } + } + } + } + if (all_visited && !any_purged) { + mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0); + } + } +} + + +/* ----------------------------------------------------------- + Visit all pages and blocks in a heap +----------------------------------------------------------- */ + +typedef struct mi_heap_visit_info_s { + mi_heap_t* heap; + mi_block_visit_fun* visitor; + void* arg; + bool visit_blocks; +} mi_heap_visit_info_t; + +static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) { + mi_heap_area_t area; + _mi_heap_area_init(&area, page); + mi_assert_internal(vinfo->heap == mi_page_heap(page)); + if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) { + return false; + } + if (vinfo->visit_blocks) { + return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg); + } + else { + return true; + } +} + +static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(slice_count); + mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg; + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + return mi_heap_visit_page(page, vinfo); +} + +bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_assert(visitor!=NULL); + if (visitor==NULL) return false; + if (heap==NULL) { heap = mi_heap_main(); } + // visit all pages in a heap + // we don't have to claim because we assume we are the only thread running (with this heap). + // (but we could atomically claim as well by first doing abandoned_reclaim and afterwards reabandoning). + mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks }; + bool ok = true; + mi_forall_arenas(heap, NULL, 0, arena) { + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (ok && arena_pages != NULL) { + if (abandoned_only) { + for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) { + // todo: if we had a single abandoned page map as well, this can be faster. + if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) { + ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info); + } + } + } + else { + ok = _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info); + } + } + } + mi_forall_arenas_end(); + if (!ok) return false; + + // visit abandoned pages in OS allocated memory + // (technically we don't need the initial lock as we assume we are the only thread running in this subproc) + mi_page_t* page = NULL; + mi_lock(&heap->os_abandoned_pages_lock) { + page = heap->os_abandoned_pages; + } + while (ok && page != NULL) { + mi_page_t* next = page->next; // read upfront in case the visitor frees the page + ok = mi_heap_visit_page(page, &visit_info); + page = next; + } + + return ok; +} + +bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg); +} + +bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg); +} + + +typedef struct mi_heap_delete_visit_info_s { + mi_heap_t* heap_target; + mi_theap_t* theap_target; + mi_theap_t* theap; +} mi_heap_delete_visit_info_t; + +static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap); + mi_heap_delete_visit_info_t* info = (mi_heap_delete_visit_info_t*)arg; + mi_heap_t* heap_target = info->heap_target; + mi_theap_t* const theap = NULL; // info->theap; mi_assert_internal(_mi_theap_heap(theap) == heap); + mi_page_t* const page = (mi_page_t*)area->reserved1; + + mi_page_claim_ownership(page); // claim ownership + if (mi_page_is_abandoned(page)) { + _mi_arenas_page_unabandon(page,theap); + } + else { + page->next = page->prev = NULL; // yikes.. better not to try to access this from a thread later on.. + mi_page_set_theap(page,NULL); // set threadid to abandoned + } + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_page_is_owned(page)); + + if (page->used==0) { + // free the page + _mi_arenas_page_free(page, theap); + } + else if (heap_target==NULL) { + #if MI_GUARDED + _mi_page_unguard_all(page); // remove potential interior guard pages + #endif + // destroy the page + page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + _mi_arenas_page_free(page, theap); + } + else { + // move the page to `heap_target` as an abandoned page + // first remove it from the current heap + const size_t sbin = _mi_page_stats_bin(page); + mi_arena_t* arena = NULL; + size_t slice_index = 0; + if (page->memid.memkind == MI_MEM_ARENA) { + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index)); + mi_bitmap_clear(arena_pages->pages, slice_index); + } + else { + // os allocated + mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL); + } + if (theap != NULL) { + mi_theap_stat_decrease(theap, page_bins[sbin], 1); + mi_theap_stat_decrease(theap, pages, 1); + } + else { + mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1); + mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1); + } + mi_theap_t* theap_target = info->theap_target; + + // and then add it to the new target heap + if (arena != NULL) { + mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + if mi_unlikely(arena_pages_target==NULL) { + // if we cannot allocate this, we move it to the main heap instead (which does not require allocation) + heap_target = mi_arena_heap_main(arena); + theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld? + arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + mi_assert_internal(arena_pages_target!=NULL); + } + mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index)); + mi_bitmap_set(arena_pages_target->pages, slice_index); + } + page->heap = heap_target; + mi_theap_stat_increase(theap_target, page_bins[sbin], 1); + mi_theap_stat_increase(theap_target, pages, 1); + + // and abandon in the new heap + _mi_arenas_page_abandon(page,theap_target); + } + return true; +} + +static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { + mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL); + // mi_theap_t* const theap = _mi_heap_theap(heap); + mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL }; + _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info); + #if MI_DEBUG>1 + // no more arena pages? + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { + mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); + if (arena_pages!=NULL) { + mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages)); + } + } + // nor os abandoned pages? + mi_lock(&heap->os_abandoned_pages_lock) { + + mi_assert_internal(heap->os_abandoned_pages == NULL); + } + // nor arena abandoned pages? + for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0); + } + #endif +} + +void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) { + if (_mi_is_heap_main(heap_from)) return; + if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; } + mi_heap_delete_pages(heap_from, heap_to); +} + +void _mi_heap_destroy_pages(mi_heap_t* heap_from) { + if (_mi_is_heap_main(heap_from)) return; + mi_heap_delete_pages(heap_from, NULL); +} + +/* ----------------------------------------------------------- + Unloading and reloading an arena. +----------------------------------------------------------- */ +/* +static bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(arg); MI_UNUSED(slice_count); + mi_assert_internal(slice_count == 1); + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1)); + if (!_mi_page_map_register(page)) return false; // break + mi_assert_internal(_mi_ptr_page(page)==page); + return true; +} + +mi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) { + return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL); +} + +mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) { + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (arena==NULL) { + return false; + } + else if (!arena->is_exclusive) { + _mi_warning_message("cannot unload a non-exclusive arena (id %zu at %p)\n", arena_id, arena); + return false; + } + else if (arena->memid.memkind != MI_MEM_EXTERNAL) { + _mi_warning_message("can only unload managed arena's for external memory (id %zu at %p)\n", arena_id, arena); + return false; + } + + // find accessed size + const size_t asize = mi_size_of_slices(mi_arena_used_slices(arena)); + if (base != NULL) { *base = (void*)arena; } + if (full_size != NULL) { *full_size = arena->memid.mem.os.size; } + if (accessed_size != NULL) { *accessed_size = asize; } + + // adjust abandoned page count + mi_subproc_t* const subproc = arena->subproc; + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); + if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); } + } + + // unregister the pages + _mi_page_map_unregister_range(arena, asize); + + // set arena entry to NULL + const size_t count = mi_arenas_get_count(subproc); + for(size_t i = 0; i < count; i++) { + if (mi_arena_from_index(subproc, i) == arena) { + mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); + if (i + 1 == count) { // try adjust the count? + size_t expected = count; + mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1); + } + break; + } + } + return true; +} + +mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { + // assume the memory area is already containing the arena + if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } + if (start == NULL || size == 0) return false; + mi_arena_t* arena = (mi_arena_t*)start; + mi_memid_t memid = arena->memid; + if (memid.memkind != MI_MEM_EXTERNAL) { + _mi_warning_message("can only reload arena's from external memory (%p)\n", arena); + return false; + } + if (memid.mem.os.base != start) { + _mi_warning_message("the reloaded arena base address differs from the external memory (arena: %p, external: %p)\n", arena, start); + return false; + } + if (memid.mem.os.size != size) { + _mi_warning_message("the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\n", arena->memid.mem.os.size, size); + return false; + } + if (!arena->is_exclusive) { + _mi_warning_message("the reloaded arena is not exclusive\n"); + return false; + } + + // re-initialize + arena->is_exclusive = true; + arena->commit_fun = commit_fun; + arena->commit_fun_arg = commit_fun_arg; + arena->subproc = _mi_subproc(); + if (!mi_arenas_add(arena->subproc, arena, arena_id)) { + return false; + } + if (!mi_arena_pages_reregister(arena)) { + // todo: clear arena entry in the subproc? + return false; + } + + // adjust abandoned page count + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]); + if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); } + } + + return true; +} + +*/ diff --git a/csrc/mimalloc/src/bitmap.c b/csrc/mimalloc/src/bitmap.c new file mode 100644 index 000000000..01de1999a --- /dev/null +++ b/csrc/mimalloc/src/bitmap.c @@ -0,0 +1,2002 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically +---------------------------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/bits.h" +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "bitmap.h" + +#ifndef MI_OPT_SIMD +#define MI_OPT_SIMD 0 +#endif + +/* -------------------------------------------------------------------------------- + bfields +-------------------------------------------------------------------------------- */ + +static inline size_t mi_bfield_ctz(mi_bfield_t x) { + return mi_ctz(x); +} + +static inline size_t mi_bfield_clz(mi_bfield_t x) { + return mi_clz(x); +} + +static inline size_t mi_bfield_popcount(mi_bfield_t x) { + return mi_popcount(x); +} + +static inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) { + return (x & (x-1)); +} + +// find the least significant bit that is set (i.e. count trailing zero's) +// return false if `x==0` (with `*idx` undefined) and true otherwise, +// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). +static inline bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) { + return mi_bsf(x,idx); +} + +// find the most significant bit that is set. +// return false if `x==0` (with `*idx` undefined) and true otherwise, +// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). +static inline bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) { + return mi_bsr(x, idx); +} + + + +// find each set bit in a bit field `x` and clear it, until it becomes zero. +static inline bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) { + const bool found = mi_bfield_find_least_bit(*x, idx); + *x = mi_bfield_clear_least_bit(*x); + return found; +} + +static inline mi_bfield_t mi_bfield_zero(void) { + return 0; +} + +static inline mi_bfield_t mi_bfield_one(void) { + return 1; +} + +static inline mi_bfield_t mi_bfield_all_set(void) { + return ~((mi_bfield_t)0); +} + +// mask of `bit_count` bits set shifted to the left by `shiftl` +static inline mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) { + mi_assert_internal(bit_count > 0); + mi_assert_internal(bit_count + shiftl <= MI_BFIELD_BITS); + mi_assert_internal(shiftl < MI_BFIELD_BITS); + const mi_bfield_t mask0 = (bit_count < MI_BFIELD_BITS ? (mi_bfield_one() << bit_count)-1 : mi_bfield_all_set()); + return (mask0 << shiftl); +} + + +// ------- mi_bfield_atomic_set --------------------------------------- +// the `_set` functions return also the count of bits that were already set (for commit statistics) +// the `_clear` functions return also whether the new bfield is all clear or not (for the chunk_map) + +// Set a bit atomically. Returns `true` if the bit transitioned from 0 to 1 +static inline bool mi_bfield_atomic_set(_Atomic(mi_bfield_t)*b, size_t idx) { + mi_assert_internal(idx < MI_BFIELD_BITS); + const mi_bfield_t mask = mi_bfield_mask(1, idx);; + const mi_bfield_t old = mi_atomic_or_acq_rel(b, mask); + return ((old&mask) == 0); +} + +// Clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0. +// `all_clear` is set if the new bfield is zero. +static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) { + mi_assert_internal(idx < MI_BFIELD_BITS); + const mi_bfield_t mask = mi_bfield_mask(1, idx);; + mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask); + if (all_clear != NULL) { *all_clear = ((old&~mask)==0); } + return ((old&mask) == mask); +} + +// Clear a bit but only when/once it is set. This is used by concurrent free's while +// the page is abandoned and mapped. This can incur a busy wait :-( but it should +// be quite rare (and is accounted for in the stats) +static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomic(mi_bfield_t)*b, size_t idx) { + mi_assert_internal(idx < MI_BFIELD_BITS); + const mi_bfield_t mask = mi_bfield_mask(1, idx);; + mi_bfield_t old = mi_atomic_load_relaxed(b); + do { + if mi_unlikely((old&mask) == 0) { + old = mi_atomic_load_acquire(b); + if ((old&mask)==0) { + mi_subproc_stat_counter_increase(subproc, pages_unabandon_busy_wait, 1); + } + while ((old&mask)==0) { // busy wait + _mi_prim_thread_yield(); + old = mi_atomic_load_acquire(b); + } + } + } while (!mi_atomic_cas_weak_acq_rel(b,&old, (old&~mask))); + mi_assert_internal((old&mask)==mask); // we should only clear when it was set +} + +// Set a mask set of bits atomically, and return true of the mask bits transitioned from all 0's to 1's. +// `already_set` contains the count of bits that were already set (used when committing ranges to account +// statistics correctly). +static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) { + mi_assert_internal(mask != 0); + const mi_bfield_t old = mi_atomic_or_acq_rel(b,mask); + if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); } + return ((old&mask) == 0); +} + +// Clear a mask set of bits atomically, and return true of the mask bits transitioned from all 1's to 0's +// `all_clear` is set to `true` if the new bfield became zero. +static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) { + mi_assert_internal(mask != 0); + const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); + if (all_clear != NULL) { *all_clear = ((old&~mask)==0); } + return ((old&mask) == mask); +} + +static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already_set) { + const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_all_set()); + if (already_set!=NULL) { *already_set = mi_bfield_popcount(old); } + return (old==0); +} + + +// ------- mi_bfield_atomic_try_clear --------------------------------------- + +// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0 +// and false otherwise (leaving the bit field as is). Returns the acquired value back in `expect` (regardless of success). +// `all_clear` is set to `true` if the new bfield became zero. +// `did_temp_clear_bits` is set to `true` some bits were temporarily cleared. +static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t* previous, bool* did_temp_clear_bits) { + mi_assert_internal(mask != 0); + // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64) + // try to atomically clear the mask bits + mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); + if (previous!=NULL) { *previous = old; } + if mi_likely((old&mask)==mask) { + // all bits in the mask transitioned from 1 to 0 atomically + return true; + } + else { + // failed to transition all + if ((old&mask)!=0) { + // restore accidentally cleared ones + mi_atomic_or_acq_rel(b, old&mask); + if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; } + } + return false; + } +} + +// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success. +// Returns `true` if the bit transitioned from 1 to 0 +// and `false` otherwise (leaving the bfield `b` as-is). +// `all_clear` is set to true if the new bfield became zero (and false otherwise) +mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_Atomic(mi_bfield_t)* b, size_t idx, mi_bfield_t* previous) { + mi_assert_internal(idx < MI_BFIELD_BITS); + const mi_bfield_t mask = ((mi_bfield_t)1<bfields[i], idx); + if (already_set != NULL) { *already_set = (was_clear ? 0 : 1); } + return was_clear; +} + +// Set `0 < n <= MI_BFIELD_BITS`, and return true of the mask bits transitioned from all 0's to 1's. +// `already_set` contains the count of bits that were already set (used when committing ranges to account +// statistics correctly). +// Can cross over two bfields. +static inline bool mi_bchunk_setNX(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) { + mi_assert_internal(cidx < MI_BCHUNK_BITS); + mi_assert_internal(n > 0 && n <= MI_BFIELD_BITS); + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + if mi_likely(idx + n <= MI_BFIELD_BITS) { + // within one field + return mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(n,idx), already_set); + } + else { + // spanning two fields + const size_t m = MI_BFIELD_BITS - idx; // bits to clear in the first field + mi_assert_internal(m < n); + mi_assert_internal(i < MI_BCHUNK_FIELDS - 1); + mi_assert_internal(idx + m <= MI_BFIELD_BITS); + size_t already_set1; + const bool all_set1 = mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &already_set1); + mi_assert_internal(n - m > 0); + mi_assert_internal(n - m < MI_BFIELD_BITS); + size_t already_set2; + const bool all_set2 = mi_bfield_atomic_set_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &already_set2); + if (already_set != NULL) { *already_set = already_set1 + already_set2; } + return (all_set1 && all_set2); + } +} + +// Set a sequence of `n` bits within a chunk. +// Returns true if all bits transitioned from 0 to 1 (or 1 to 0). +mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* palready_set, bool* pmaybe_all_clear) { + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); + mi_assert_internal(n>0); + bool all_transition = true; + bool maybe_all_clear = true; + size_t total_already_set = 0; + size_t idx = cidx % MI_BFIELD_BITS; + size_t field = cidx / MI_BFIELD_BITS; + while (n > 0) { + size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field + if (m > n) { m = n; } + mi_assert_internal(idx + m <= MI_BFIELD_BITS); + mi_assert_internal(field < MI_BCHUNK_FIELDS); + const mi_bfield_t mask = mi_bfield_mask(m, idx); + size_t already_set = 0; + bool all_clear = false; + const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set) + : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear)); + mi_assert_internal(!set || ((transition && already_set == 0) || (!transition && already_set > 0))); + all_transition = all_transition && transition; + total_already_set += already_set; + maybe_all_clear = maybe_all_clear && all_clear; + // next field + field++; + idx = 0; + mi_assert_internal(m <= n); + n -= m; + } + if (palready_set!=NULL) { *palready_set = total_already_set; } + if (pmaybe_all_clear!=NULL) { *pmaybe_all_clear = maybe_all_clear; } + return all_transition; +} + +static inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) { + mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS); + if (n==1) return mi_bchunk_set(chunk, cidx, already_set); + // if (n==8 && (cidx%8) == 0) return mi_bchunk_set8(chunk, cidx, already_set); + // if (n==MI_BFIELD_BITS) return mi_bchunk_setX(chunk, cidx, already_set); + if (n<=MI_BFIELD_BITS) return mi_bchunk_setNX(chunk, cidx, n, already_set); + return mi_bchunk_xsetNC(MI_BIT_SET, chunk, cidx, n, already_set, NULL); +} + +// ------- mi_bchunk_clear --------------------------------------- + +static inline bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) { + mi_assert_internal(cidx < MI_BCHUNK_BITS); + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + return mi_bfield_atomic_clear(&chunk->bfields[i], idx, all_clear); +} + +static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) { + mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS); + if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear); + // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear); + // if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear); + // todo: implement mi_bchunk_xsetNX instead of setNX + return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear); +} + +// Check if a sequence of `n` bits within a chunk are all set/cleared. +// This can cross bfield's +mi_decl_noinline static size_t mi_bchunk_popcountNC(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) { + mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS); + size_t count = 0; + while (n > 0) { + size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field + if (m > n) { m = n; } + mi_assert_internal(idx + m <= MI_BFIELD_BITS); + mi_assert_internal(field_idx < MI_BCHUNK_FIELDS); + const size_t mask = mi_bfield_mask(m, idx); + count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask); + // next field + field_idx++; + idx = 0; + n -= m; + } + return count; +} + +// Count set bits a sequence of `n` bits. +static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) { + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); + mi_assert_internal(n>0); + if (n==0) return 0; + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); } + if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); } + return mi_bchunk_popcountNC(chunk, i, idx, n); +} + + +// ------- mi_bchunk_is_xset --------------------------------------- + +// Check if a sequence of `n` bits within a chunk are all set/cleared. +// This can cross bfield's +mi_decl_noinline static bool mi_bchunk_is_xsetNC(mi_xset_t set, const mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) { + mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS); + while (n > 0) { + size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field + if (m > n) { m = n; } + mi_assert_internal(idx + m <= MI_BFIELD_BITS); + mi_assert_internal(field_idx < MI_BCHUNK_FIELDS); + const size_t mask = mi_bfield_mask(m, idx); + if (!mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[field_idx], mask)) { + return false; + } + // next field + field_idx++; + idx = 0; + n -= m; + } + return true; +} + +// Check if a sequence of `n` bits within a chunk are all set/cleared. +static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) { + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); + mi_assert_internal(n>0); + if (n==0) return true; + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + if (n==1) { return mi_bfield_atomic_is_xset(set, &chunk->bfields[i], idx); } + if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[i], mi_bfield_mask(n, idx)); } + return mi_bchunk_is_xsetNC(set, chunk, i, idx, n); +} + + +// ------- mi_bchunk_try_clear --------------------------------------- + +// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary. +static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { + mi_assert_internal(cidx < MI_BCHUNK_BITS); + mi_assert_internal(n <= MI_BFIELD_BITS); + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + if mi_likely(idx + n <= MI_BFIELD_BITS) { + // within one field + return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear, did_temp_clear_bits); + } + else { + // spanning two fields (todo: use double-word atomic ops?) + const size_t m = MI_BFIELD_BITS - idx; // bits to clear in the first field + mi_assert_internal(m < n); + mi_assert_internal(i < MI_BCHUNK_FIELDS - 1); + bool field1_is_clear; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear, did_temp_clear_bits)) return false; + // try the second field as well + mi_assert_internal(n - m > 0); + mi_assert_internal(n - m < MI_BFIELD_BITS); + bool field2_is_clear; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear, did_temp_clear_bits)) { + // we failed to clear the second field, restore the first one + mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } + return false; + } + if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; } + return true; + } +} + +// Clear a full aligned bfield. +// static inline bool mi_bchunk_try_clearX(mi_bchunk_t* chunk, size_t cidx, bool* pmaybe_all_clear) { +// mi_assert_internal(cidx < MI_BCHUNK_BITS); +// mi_assert_internal((cidx%MI_BFIELD_BITS) == 0); +// const size_t i = cidx / MI_BFIELD_BITS; +// return mi_bfield_atomic_try_clearX(&chunk->bfields[i], pmaybe_all_clear); +// } + +// Try to atomically clear a sequence of `n` bits within a chunk. +// Returns true if all bits transitioned from 1 to 0, +// and false otherwise leaving all bit fields as is. +// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway.. +// `maybe_all_clear` is set to `true` if all the bfields involved become zero. +mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); + mi_assert_internal(n>0); + if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; } + if (n==0) return true; + + // first field + const size_t start_idx = cidx % MI_BFIELD_BITS; + const size_t start_field = cidx / MI_BFIELD_BITS; + size_t field = start_field; + size_t m = MI_BFIELD_BITS - start_idx; // m are the bits to clear in this field + if (m > n) { m = n; } + mi_assert_internal(start_idx + m <= MI_BFIELD_BITS); + mi_assert_internal(start_field < MI_BCHUNK_FIELDS); + const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx); + bool maybe_all_clear; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear, did_temp_clear_bits)) return false; + + // done? + mi_assert_internal(m <= n); + n -= m; + + // continue with mid fields and last field: if these fail we need to recover by unsetting previous fields + // mid fields? + while (n >= MI_BFIELD_BITS) { + field++; + mi_assert_internal(field < MI_BCHUNK_FIELDS); + bool field_is_clear; + if (!mi_bfield_atomic_try_clearX(&chunk->bfields[field], &field_is_clear)) goto restore; + maybe_all_clear = maybe_all_clear && field_is_clear; + n -= MI_BFIELD_BITS; + } + + // last field? + if (n > 0) { + mi_assert_internal(n < MI_BFIELD_BITS); + field++; + mi_assert_internal(field < MI_BCHUNK_FIELDS); + const mi_bfield_t mask_end = mi_bfield_mask(n, 0); + bool field_is_clear; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear, did_temp_clear_bits)) goto restore; + maybe_all_clear = maybe_all_clear && field_is_clear; + } + + if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = maybe_all_clear; } + return true; + +restore: + // `field` is the index of the field that failed to set atomically; we need to restore all previous fields + mi_assert_internal(field > start_field); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } + while( field > start_field) { + field--; + if (field == start_field) { + mi_bfield_atomic_set_mask(&chunk->bfields[field], mask_start, NULL); + } + else { + mi_bfield_atomic_setX(&chunk->bfields[field], NULL); // mid-field: set all bits again + } + } + return false; +} + + +static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) { + mi_assert_internal(n>0); + // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear); + if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); + return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); +} + + +// ------- mi_bchunk_try_find_and_clear --------------------------------------- + +#if MI_OPT_SIMD && defined(__AVX2__) +mi_decl_maybe_unused static inline __m256i mi_mm256_zero(void) { + return _mm256_setzero_si256(); +} +mi_decl_maybe_unused static inline __m256i mi_mm256_ones(void) { + return _mm256_set1_epi64x(~0); +} +mi_decl_maybe_unused static inline bool mi_mm256_is_ones(__m256i vec) { + return _mm256_testc_si256(vec, _mm256_cmpeq_epi32(vec, vec)); +} +mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) { + return _mm256_testz_si256(vec,vec); +} +#endif + + +static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) { + mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one + b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value + if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it + size_t bitidx = 0; + mi_bfield_find_least_bit(mask,&bitidx); + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; + mi_assert_internal(*pidx < MI_BCHUNK_BITS); + return true; + } + } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contention + return false; +} + +// Find least 1-bit in a chunk and try to clear it atomically +// set `*pidx` to the bit index (0 <= *pidx < MI_BCHUNK_BITS) on success. +// This is used to find free slices and abandoned pages and should be efficient. +// todo: try neon version +static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) { + #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries + const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); + const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF : 0) + const uint32_t mask = ~_mm256_movemask_epi8(vcmp); // mask of most significant bit of each byte (so each 8 bits are all set or clear) + // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared) + if (mask==0) return false; + mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24 + const size_t chunk_idx = _tzcnt_u32(mask) / 8; + if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true; + // try again + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif + } + #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries + size_t chunk_idx = 0; + #if 0 + // one vector at a time + __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); + if (mi_mm256_is_zero(vec)) { + chunk_idx += 4; + vec = _mm256_load_si256(((const __m256i*)chunk->bfields) + 1); + } + const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF : 0) + const uint32_t mask = ~_mm256_movemask_epi8(vcmp); // mask of most significant bit of each byte (so each 8 bits are all set or clear) + // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared) + if (mask==0) return false; + mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24 + chunk_idx += _tzcnt_u32(mask) / 8; + #else + // a cache line is 64b so we can just as well load all at the same time + const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); + const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1); + const __m256i cmpv = mi_mm256_zero(); + const __m256i vcmp1 = _mm256_cmpeq_epi64(vec1, cmpv); // (elem64 == 0 ? 0xFF : 0) + const __m256i vcmp2 = _mm256_cmpeq_epi64(vec2, cmpv); // (elem64 == 0 ? 0xFF : 0) + const uint32_t mask1 = ~_mm256_movemask_epi8(vcmp1); // mask of most significant bit of each byte (so each 8 bits are all set or clear) + const uint32_t mask2 = ~_mm256_movemask_epi8(vcmp2); // mask of most significant bit of each byte (so each 8 bits are all set or clear) + const uint64_t mask = ((uint64_t)mask2 << 32) | mask1; + // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared) + if (mask==0) return false; + mi_assert_internal((_tzcnt_u64(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , .. + chunk_idx = mi_ctz(mask) / 8; + #endif + if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true; + // try again + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif + } + #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64 + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries + // a cache line is 64b so we can just as well load all at the same time (?) + const uint64x2_t vzero1_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields)); // 2x64 bit is_zero + const uint64x2_t vzero1_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 2)); // 2x64 bit is_zero + const uint64x2_t vzero2_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 4)); // 2x64 bit is_zero + const uint64x2_t vzero2_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 6)); // 2x64 bit is_zero + const uint32x4_t vzero1 = vuzp1q_u32(vreinterpretq_u32_u64(vzero1_lo),vreinterpretq_u32_u64(vzero1_hi)); // unzip even elements: narrow to 4x32 bit is_zero () + const uint32x4_t vzero2 = vuzp1q_u32(vreinterpretq_u32_u64(vzero2_lo),vreinterpretq_u32_u64(vzero2_hi)); // unzip even elements: narrow to 4x32 bit is_zero () + const uint32x4_t vzero1x = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero1), 24)); // shift-right 2x32bit elem by 24: lo 16 bits contain the 2 lo bytes + const uint32x4_t vzero2x = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero2), 24)); + const uint16x8_t vzero12 = vreinterpretq_u16_u32(vuzp1q_u32(vzero1x,vzero2x)); // unzip even 32-bit elements into one vector + const uint8x8_t vzero = vmovn_u16(vzero12); // narrow the bottom 16-bits + const uint64_t mask = ~vget_lane_u64(vreinterpret_u64_u8(vzero), 0); // 1 byte for each bfield (0xFF => bfield has a bit set) + if (mask==0) return false; + mi_assert_internal((mi_ctz(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , .. + const size_t chunk_idx = mi_ctz(mask) / 8; + if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true; + // try again + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif + } + #else + for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { + if (mi_bchunk_try_find_and_clear_at(chunk, i, pidx)) return true; + } + #endif + return false; +} + +static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); + return mi_bchunk_try_find_and_clear(chunk, pidx); +} + +mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) { + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + // has_set8 has low bit in each byte set if the byte in x == 0xFF + const mi_bfield_t has_set8 = + ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F + (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 + >> 7; // shift high bit to low bit + size_t bitidx; + if (mi_bfield_find_least_bit(has_set8, &bitidx)) { // find least 1-bit + mi_assert_internal(bitidx <= (MI_BFIELD_BITS - 8)); + mi_assert_internal((bitidx%8)==0); + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(bfield,bitidx,&b,did_temp_clear_bits)) { // try to clear atomically (and update b) + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; + mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); + return true; + } + } + else { + return false; + } + } while(b!=0 && ++tries <= 4); // limit tries to reduce possible contention + return false; +} + +// find least aligned byte in a chunk with all bits set, and try unset it atomically +// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success. +// Used to find medium size pages in the free blocks. +// todo: try neon version +static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx, bool* did_temp_clear_bits) { + #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries + // since a cache-line is 64b, load all at once + const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); + const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1); + const __m256i cmpv = mi_mm256_ones(); + const __m256i vcmp1 = _mm256_cmpeq_epi8(vec1, cmpv); // (byte == ~0 ? 0xFF : 0) + const __m256i vcmp2 = _mm256_cmpeq_epi8(vec2, cmpv); // (byte == ~0 ? 0xFF : 0) + const uint32_t mask1 = _mm256_movemask_epi8(vcmp1); // mask of most significant bit of each byte + const uint32_t mask2 = _mm256_movemask_epi8(vcmp2); // mask of most significant bit of each byte + const uint64_t mask = ((uint64_t)mask2 << 32) | mask1; + // mask is inverted, so each bit is 0xFF iff the corresponding byte has a bit set (and thus can be cleared) + if (mask==0) return false; + const size_t bidx = _tzcnt_u64(mask); // byte-idx of the byte in the chunk + const size_t chunk_idx = bidx / 8; + const size_t idx = (bidx % 8)*8; + mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(&chunk->bfields[chunk_idx], idx, NULL, did_temp_clear_bits)) { // clear it atomically + *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; + mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); + return true; + } + // try again + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif + } + #else + for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { + if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx, did_temp_clear_bits)) return true; + } + #endif + return false; +} + +static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); + return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits); +} + + +// find a sequence of `n` bits in a chunk with `0 < n <= MI_BFIELD_BITS` with all bits set, +// and try to clear them atomically. +// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. +// will cross bfield boundaries. +mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + if (n == 0 || n > MI_BFIELD_BITS) return false; + const mi_bfield_t mask = mi_bfield_mask(n, 0); + // for all fields in the chunk + for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { + mi_bfield_t b0 = mi_atomic_load_relaxed(&chunk->bfields[i]); + mi_bfield_t b = b0; + size_t idx; + + // is there a range inside the field? + while (mi_bfield_find_least_bit(b, &idx)) { // find least 1-bit + if (idx + n > MI_BFIELD_BITS) break; // too short: maybe cross over, or continue with the next field + + const size_t bmask = mask<>idx == mask); + if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically + if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value + *pidx = (i*MI_BFIELD_BITS) + idx; + mi_assert_internal(*pidx < MI_BCHUNK_BITS); + mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); + return true; + } + else { + // if we failed to atomically commit, reload b and try again from the start + b = b0; // = mi_atomic_load_acquire(&chunk->bfields[i]); + } + } + else { + // advance by clearing the least run of ones, for example, with n>=4, idx=2: + // b = 1111 1101 1010 1100 + // .. + (1< 0) { + const size_t pre = mi_bfield_ctz(~mi_atomic_load_relaxed(&chunk->bfields[i+1])); + if (post + pre >= n) { + // it fits -- try to claim it atomically + const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post); + if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL, did_temp_clear_bits)) { + // we cleared all atomically + *pidx = cidx; + mi_assert_internal(*pidx < MI_BCHUNK_BITS); + mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); + return true; + } + } + } + } + } + return false; +} + +// find a sequence of `n` bits in a chunk with `n <= MI_BCHUNK_BITS` with all bits set, +// and try to clear them atomically. +// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. +// This can cross bfield boundaries. +static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk + + // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically + mi_assert_internal(n>0); + const size_t skip_count = (n-1)/MI_BFIELD_BITS; + size_t cidx; + for (size_t i = 0; i < MI_BCHUNK_FIELDS - skip_count; i++) + { + size_t m = n; // bits to go + + // first field + mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); + size_t ones = mi_bfield_clz(~b); + + cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - ones); // start index + if (ones >= m) { + // we found enough bits already! + m = 0; + } + else if (ones > 0) { + // keep scanning further fields until we have enough bits + m -= ones; + size_t j = 1; // field count from i + while (i+j < MI_BCHUNK_FIELDS) { + mi_assert_internal(m > 0); + b = mi_atomic_load_relaxed(&chunk->bfields[i+j]); + ones = mi_bfield_ctz(~b); + if (ones >= m) { + // we found enough bits + m = 0; + break; + } + else if (ones == MI_BFIELD_BITS) { + // not enough yet, proceed to the next field + j++; + m -= MI_BFIELD_BITS; + } + else { + // the range was not enough, start from scratch + i = i + j - 1; // no need to re-scan previous fields, except the last one (with clz this time) + mi_assert_internal(m>0); + break; + } + } + } + + // did we find a range? + if (m==0) { + if (mi_bchunk_try_clearN(chunk, cidx, n, NULL, did_temp_clear_bits)) { + // we cleared all atomically + *pidx = cidx; + mi_assert_internal(*pidx < MI_BCHUNK_BITS); + mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); + return true; + } + // note: if we fail for a small `n` on the first field, we don't rescan that field (as `i` is incremented) + } + // otherwise continue searching + } + return false; +} + + + +// ------- mi_bchunk_clear_once_set --------------------------------------- + +static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) { + mi_assert_internal(cidx < MI_BCHUNK_BITS); + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + mi_bfield_atomic_clear_once_set(subproc, &chunk->bfields[i], idx); +} + + +// ------- mi_bitmap_all_are_clear --------------------------------------- + + +// are all bits in a bitmap chunk clear? +static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) { + #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) + const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); + return mi_mm256_is_zero(vec); + #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) + // a 64b cache-line contains the entire chunk anyway so load both at once + const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); + const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1); + return (mi_mm256_is_zero(_mm256_or_si256(vec1,vec2))); + #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64 + const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields); + const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2); + const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4); + const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6); + const uint64x2_t v = vorrq_u64(vorrq_u64(v0,v1),vorrq_u64(v2,v3)); + return (vmaxvq_u32(vreinterpretq_u32_u64(v)) == 0); + #else + for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { + if (mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false; + } + return true; + #endif +} + +// are all bits in a bitmap chunk set? +static inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) { +#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256) + const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields); + return mi_mm256_is_ones(vec); +#elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) + // a 64b cache-line contains the entire chunk anyway so load both at once + const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); + const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1); + return (mi_mm256_is_ones(_mm256_and_si256(vec1, vec2))); +#elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64 + const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields); + const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2); + const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4); + const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6); + const uint64x2_t v = vandq_u64(vandq_u64(v0,v1),vandq_u64(v2,v3)); + return (vminvq_u32(vreinterpretq_u32_u64(v)) == 0xFFFFFFFFUL); +#else + for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { + if (~mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false; + } + return true; +#endif +} + + +static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { + for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { + i--; + mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); + size_t idx; + if (mi_bsr(b, &idx)) { + *pidx = (i*MI_BFIELD_BITS) + idx; + return true; + } + } + return false; +} + +static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { + for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { + i--; + mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); + size_t idx; + if (mi_bsr(~b, &idx)) { + *pidx = (i*MI_BFIELD_BITS) + idx; + return true; + } + } + return false; +} + +static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) { + size_t popcount = 0; + for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) { + const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); + popcount += mi_bfield_popcount(b); + } + return popcount; +} + + +/* -------------------------------------------------------------------------------- + bitmap chunkmap +-------------------------------------------------------------------------------- */ + +static void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); + mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); +} + +static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap)); + // check if the corresponding chunk is all clear + if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false; + // clear the chunkmap bit + mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL); + // .. but a concurrent set may have happened in between our all-clear test and the clearing of the + // bit in the mask. We check again to catch this situation. + if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) { + mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL); + return false; + } + return true; +} + + +/* -------------------------------------------------------------------------------- + bitmap +-------------------------------------------------------------------------------- */ + +size_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) { + mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); + bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS); + mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); + mi_assert_internal(bit_count > 0); + const size_t chunk_count = bit_count / MI_BCHUNK_BITS; + mi_assert_internal(chunk_count >= 1); + const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); + mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); + if (pchunk_count != NULL) { *pchunk_count = chunk_count; } + return size; +} + + +// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap +size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) { + size_t chunk_count; + const size_t size = mi_bitmap_size(bit_count, &chunk_count); + if (!already_zero) { + _mi_memzero_aligned(bitmap, size); + } + mi_atomic_store_release(&bitmap->chunk_count, chunk_count); + mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); + return size; +} + + +// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. +static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + + // start chunk and index + size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS); + + // first update the chunkmap + mi_bchunk_setN(cmap, chunk_idx, ccount, NULL); + + // first chunk + size_t m = MI_BCHUNK_BITS - cidx; + if (m > n) { m = n; } + mi_bchunk_setN(&chunks[chunk_idx], cidx, m, NULL); + + // n can be large so use memset for efficiency for all in-between chunks + chunk_idx++; + n -= m; + const size_t mid_chunks = n / MI_BCHUNK_BITS; + if (mid_chunks > 0) { + _mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE); + chunk_idx += mid_chunks; + n -= (mid_chunks * MI_BCHUNK_BITS); + } + + // last chunk + if (n > 0) { + mi_assert_internal(n < MI_BCHUNK_BITS); + mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL); + } +} + +// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. +void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap)); + mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n); +} + + + + +// ------- mi_bitmap_xset --------------------------------------- + +// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). +bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allclear = true; + size_t already_set = 0; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + size_t _already_set = 0; + were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear; + already_set += _already_set; + mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + if (palready_set != NULL) { *palready_set = already_set; } + return were_allclear; +} + +// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's. +bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allset = true; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + bool maybe_all_clear = false; + were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset; + if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); } + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return were_allset; +} + +// Count bits set in a range of `n` bits. +size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return 0; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + size_t popcount = 0; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m); + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return popcount; +} + + +// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) +bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_setN(bitmap, idx, 1, NULL); +} + +bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_clearN(bitmap, idx, 1); +} + + + +// ------- mi_bitmap_is_xset --------------------------------------- + +// Is a sequence of n bits already all set/cleared? +bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bitmap_max_bits(bitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool xset = true; + while (n > 0 && xset) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset; + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return xset; +} + +bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) { + return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap)); +} + +/* -------------------------------------------------------------------------------- + Iterate through a bfield +-------------------------------------------------------------------------------- */ + +// Cycle iteration through a bitfield. This is used to space out threads +// so there is less chance of contention. When searching for a free page we +// like to first search only the accessed part (so we reuse better). This +// high point is called the `cycle`. +// +// We then iterate through the bitfield as: +// first: [start, cycle> +// then : [0, start> +// then : [cycle, MI_BFIELD_BITS> +// +// The start is determined usually as `tseq % cycle` to have each thread +// start at a different spot. +// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?) +// - The `cycle_mask` is the part `[start, cycle>`. +#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \ + mi_assert_internal(start <= cycle); \ + mi_assert_internal(start < MI_BFIELD_BITS); \ + mi_assert_internal(cycle <= MI_BFIELD_BITS); \ + const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \ + size_t _bcount##SUF = mi_bfield_popcount(bfield); \ + mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\ + while(_bcount##SUF > 0) { \ + _bcount##SUF--;\ + if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \ + /* size_t name_idx; */ \ + const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \ + _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \ + mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \ + { \ + +#define mi_bfield_iterate_end(SUF) \ + } \ + } \ +} + + +#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \ + const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\ + mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF) + +#define mi_bfield_cycle_iterate_end(SUF) \ + mi_bfield_iterate_end(SUF); \ +} + + +/* -------------------------------------------------------------------------------- + mi_bitmap_find + (used to find free pages) +-------------------------------------------------------------------------------- */ + +typedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2); + +// Go through the bitmap and for every sequence of `n` set bits, call the visitor function. +// If it returns `true` stop the search. +static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2) +{ + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + // and for each chunkmap entry we iterate over its bits to find the chunks + const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t hi; + if (mi_bfield_find_highest_bit(cmap_entry, &hi)) { + size_t eidx = 0; + mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`) + { + mi_assert_internal(eidx <= MI_BFIELD_BITS); + const size_t chunk_idx = i*MI_BFIELD_BITS + eidx; + mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); + if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) { + return true; + } + } + mi_bfield_cycle_iterate_end(Y); + } + } + return false; +} + + +/* -------------------------------------------------------------------------------- + Bitmap: try_find_and_claim -- used to allocate abandoned pages + note: the compiler will fully inline the indirect function call +-------------------------------------------------------------------------------- */ + +typedef struct mi_claim_fun_data_s { + mi_arena_t* arena; +} mi_claim_fun_data_t; + +static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2) +{ + mi_assert_internal(n==1); MI_UNUSED(n); + mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1; + mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2; + size_t cidx; + if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) { + const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx; + mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap)); + bool keep_set = true; + if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) { + // success! + mi_assert_internal(!keep_set); + *pidx = slice_index; + return true; + } + else { + // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false) + if (keep_set) { + const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL); + mi_bitmap_chunkmap_set(bitmap, chunk_idx); + mi_assert_internal(wasclear); MI_UNUSED(wasclear); + } + } + } + else { + // we may find that all are cleared only on a second iteration but that is ok as + // the chunkmap is a conservative approximation. + mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); + } + return false; +} + +// Find a set bit in the bitmap and try to atomically clear it and claim it. +// (Used to find pages in the pages_abandoned bitmaps.) +mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, + mi_claim_fun_t* claim, mi_arena_t* arena ) +{ + mi_claim_fun_data_t claim_data = { arena }; + return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data); +} + + +bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) { + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = chunkmap_max; i > 0; ) { + i--; + mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + if (mi_bsr(cmap,&cmap_idx)) { + // from highest chunk to lowest (scan all in case the cmap entry was stale) + for (size_t j = cmap_idx+1; j>0; ) { + j--; + const size_t chunk_idx = (i*MI_BFIELD_BITS) + j; + size_t cidx; + if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) { + *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx; + return true; + } + } + } + } + return false; +} + +// Return count of all set bits in a bitmap. +size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) { + // for all chunkmap entries + size_t popcount = 0; + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // count bits in a chunk + popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]); + } + } + return popcount; +} + + + +// Clear a bit once it is set. +void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) { + mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); + const size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); + mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx); +} + + +// Visit all set bits in a bitmap. +// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? +bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { + // for all chunkmap entries + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for(size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]); + size_t bidx; + while (mi_bfield_foreach_bit(&b, &bidx)) { + const size_t idx = base_idx + bidx; + if (!visit(idx, 1, arena, arg)) return false; + } + } + } + } + return true; +} + +// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible. +// Also clear those ranges atomically. +// Used by purging to purge larger ranges when possible +// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? +bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { + // for all chunkmap entries + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); + #if MI_DEBUG > 1 + const size_t bpopcount = mi_popcount(b); + size_t rngcount = 0; + #endif + size_t bidx; + while (mi_bfield_find_least_bit(b, &bidx)) { + size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx + #if MI_DEBUG > 1 + rngcount += rng; + #endif + const size_t idx = base_idx + bidx; + mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS); + mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS); + mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap)); + // clear rng bits in b + b = b & ~mi_bfield_mask(rng, bidx); + if (!visit(idx, rng, arena, arg)) { + // break early: reset the non-visited bits + if (b!=0) { + mi_atomic_or_relaxed(&chunk->bfields[j], b); + } + return false; + } + } + mi_assert_internal(rngcount == bpopcount); + } + } + } + return true; +} + +// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible, +// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`) +// and clear those ranges atomically. +// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most. +// Used by purging to purge larger ranges when possible. With transparent huge pages we only +// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves. +bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) +{ + // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time) + if (rngslices<=1) { + return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg); + } + // mi_assert_internal(rngslices <= MI_BFIELD_BITS); + if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS; } // cap at MI_BFIELD_BITS at most + + // for all chunkmap entries + const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); + for (size_t i = 0; i < chunkmap_max; i++) { + mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + size_t cmap_idx; + // for each chunk (corresponding to a set bit in a chunkmap entry) + while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { + const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx; + // for each chunk field + mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { + const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); + mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear + mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them + size_t shift; + for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment + const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift); + if ((b & rngmask) == rngmask) { + const size_t idx = base_idx + shift; + if (!visit(idx, rngslices, arena, arg)) { + // break early: restore non-visited entries + mi_bfield_t notyet_visited = 0; + if (rngslices + shift < MI_BFIELD_BITS) { + notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices))); + } + mi_assert_internal((notyet_visited & skipped) == 0); + if ((notyet_visited | skipped) != 0) { + mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped); + } + return false; + } + } + else { + skipped = skipped | (b & rngmask); + } + } + if (shift < MI_BFIELD_BITS) { + // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`. + mi_assert_internal(MI_BFIELD_BITS % rngslices != 0); + skipped = skipped | (b & (~(mi_bfield_t)0 << shift)); + } + if (skipped != 0) { + // restore non-visited entries + mi_atomic_or_relaxed(&chunk->bfields[j], skipped); + } + } + } + } + return true; +} + + +/* -------------------------------------------------------------------------------- + binned bitmap's +-------------------------------------------------------------------------------- */ + + +size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) { + // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); + bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS); + mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); + mi_assert_internal(bit_count > 0); + const size_t chunk_count = bit_count / MI_BCHUNK_BITS; + mi_assert_internal(chunk_count >= 1); + const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE); + mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 ); + if (pchunk_count != NULL) { *pchunk_count = chunk_count; } + return size; +} + +// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap +size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) { + size_t chunk_count; + const size_t size = mi_bbitmap_size(bit_count, &chunk_count); + if (!already_zero) { + _mi_memzero_aligned(bbitmap, size); + } + mi_atomic_store_release(&bbitmap->chunk_count, chunk_count); + mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); + bbitmap->subproc = subproc; + return size; +} + +void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); + mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n); +} + +bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) { + // scan for highest zero bit in the bitmap + // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck + // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding! + const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap); + for(size_t i = chunk_count; i > 0; ) { + i--; + size_t cidx; + if (mi_bchunk_bsr_inv(&bbitmap->chunks[i], &cidx)) { + *idx = (i * MI_BCHUNK_BITS) + cidx; + return true; + } + } + return false; +} + + +/* -------------------------------------------------------------------------------- + binned bitmap used to track free slices +-------------------------------------------------------------------------------- */ + +// Assign a specific size bin to a chunk +static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) { + mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + if (ibin == bin) { + const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); + if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); } + } + else { + const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); + if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); } + } + } +} + +mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) { + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) { + return ibin; + } + } + return MI_CBIN_NONE; +} + +// Track the index of the highest chunk that is accessed. +static void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) { + size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); + if mi_unlikely(chunk_idx > oldmax) { + mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx); + } +} + +// Set a bit in the chunkmap +static void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) { + mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + if (check_all_set) { + if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) { + // all slices are free in this chunk: return back to the NONE bin + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE); + } + } + mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); + mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); +} + +static bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) { + mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + // check if the corresponding chunk is all clear + if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false; + // clear the chunkmap bit + mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL); + // .. but a concurrent set may have happened in between our all-clear test and the clearing of the + // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel) + if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) { + mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL); + return false; + } + mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx); + return true; +} + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_setN, try_clearN, and is_xsetN + (used to find free pages) +-------------------------------------------------------------------------------- */ + +// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). +bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bbitmap_max_bits(bbitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool were_allclear = true; + while (n > 0) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear; + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return were_allclear; +} + +// ------- mi_bbitmap_try_clearNC --------------------------------------- + +// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`. +bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(n<=MI_BCHUNK_BITS); + mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap)); + + const size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now) + mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); + if (cidx + n > MI_BCHUNK_BITS) return false; + bool maybe_all_clear = false; + bool did_temp_clear_bits = false; + const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits); + if (cleared && maybe_all_clear) { + mi_assert_internal(!did_temp_clear_bits); + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } else if (did_temp_clear_bits) { + // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + // note: we don't set the size class for an explicit try_clearN (only used by purging) + return cleared; +} + + + +// ------- mi_bbitmap_is_xset --------------------------------------- + +// Is a sequence of n bits already all set/cleared? +bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + const size_t maxbits = mi_bbitmap_max_bits(bbitmap); + mi_assert_internal(idx + n <= maxbits); + if (idx+n > maxbits) { // paranoia + if (idx >= maxbits) return false; + n = maxbits - idx; + } + + // iterate through the chunks + size_t chunk_idx = idx / MI_BCHUNK_BITS; + size_t cidx = idx % MI_BCHUNK_BITS; + bool xset = true; + while (n > 0 && xset) { + const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n); + xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset; + mi_assert_internal(m <= n); + n -= m; + cidx = 0; + chunk_idx++; + } + return xset; +} + + + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_find + (used to find free pages) +-------------------------------------------------------------------------------- */ + +typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits); + +// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function. +// If it returns `true` stop the search. +// +// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan) +// but also reduce fragmentation (through size bins). +static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) +{ + // we space out threads to reduce contention + const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS); + const size_t chunk_acc = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed); + const size_t cmap_acc = chunk_acc / MI_BFIELD_BITS; + const size_t cmap_acc_bits = 1 + (chunk_acc % MI_BFIELD_BITS); + + // create a mask over the chunkmap entries to iterate over them efficiently + mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS); + const mi_bfield_t cmap_mask = mi_bfield_mask(cmap_max_count,0); + const size_t cmap_cycle = cmap_acc+1; + const mi_chunkbin_t bbin = mi_chunkbin_of(n); + // visit each cmap entry + size_t cmap_idx = 0; + mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X) + { + // and for each chunkmap entry we iterate over its bits to find the chunks + const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]); + const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits); + if (cmap_entry == 0) { + continue; + } + + // get size bin masks + mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 }; + cmap_bins[MI_CBIN_NONE] = cmap_entry; + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { + const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]); + cmap_bins[ibin] = cmap_bin & cmap_entry; + cmap_bins[MI_CBIN_NONE] &= ~cmap_bin; // clear bits that are in an assigned size bin + } + + // consider only chunks for a particular size bin at a time + // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple + // iterations through all entries. + mi_assert_internal(bbin < MI_CBIN_NONE); + for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE; + // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation) + ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin))) + { + mi_assert_internal(ibin < MI_CBIN_COUNT); + const mi_bfield_t cmap_bin = cmap_bins[ibin]; + size_t eidx = 0; + mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y) + { + // assertion doesn't quite hold as the max_accessed may be out-of-date + // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE); + + // get the chunk + const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx; + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; + + size_t cidx; + bool did_temp_clear_bits = false; + if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) { + if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin + // this chunk is now reserved for the `bbin` size class + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin); + } + *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx; + mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); + return true; + } + else { + // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits) + // but that is ok as the chunkmap is a conservative approximation. + // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?` + // probably not as we already only call `try_clear` once we fail to clear any bits. + if (did_temp_clear_bits) { + // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + else { + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } + } + } + mi_bfield_cycle_iterate_end(Y); + } + } + mi_bfield_cycle_iterate_end(X); + return false; +} + +/* -------------------------------------------------------------------------------- + mi_bbitmap_try_find_and_clear -- used to find free pages + note: the compiler will fully inline the indirect function calls +-------------------------------------------------------------------------------- */ + +bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1); +} + +bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8); +} + +// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) { +// return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X); +// } + +bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + mi_assert_internal(n<=MI_BFIELD_BITS); + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX); +} + +bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + mi_assert_internal(n<=MI_BCHUNK_BITS); + return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC); +} + + +/* -------------------------------------------------------------------------------- + mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks +-------------------------------------------------------------------------------- */ + +// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks +static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) { + mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap)); + size_t m = n; // bits to go + size_t count = 0; // chunk count + while (m > 0) { + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; + if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) { + goto rollback; + } + m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS); + count++; + } + return true; + +rollback: + // we only need to reset chunks the we just fully cleared + while (count > 0) { + count--; + mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; + mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL); + // since we may race with clearing, we need to set the chunkmap conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); + } + return false; +} + +// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE` +// Since these are very large object allocations we always search from the start and only consider starting at the start +// of a chunk (for fragmentation and efficiency). +// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end +// Todo: This scans directly through the chunks -- we might want to consult the cmap as well? +bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + MI_UNUSED(tseq); + mi_assert(n > 0); if (n==0) { return false; } + + const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap); + const size_t chunk_req = _mi_divide_up(n, MI_BCHUNK_BITS); // minimal number of chunks needed + if (chunk_max < chunk_req) { return false; } + + // iterate through the chunks + size_t chunk_idx = 0; + while (chunk_idx <= chunk_max - chunk_req) + { + size_t count = 0; // chunk count + do { + mi_assert_internal(chunk_idx + count < chunk_max); + mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count]; + if (!mi_bchunk_all_are_set_relaxed(chunk)) { + break; + } + else { + count++; + } + } + while (count < chunk_req); + + // did we find a suitable range? + if (count == chunk_req) { + // now try to claim it! + if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) { + *pidx = (chunk_idx * MI_BCHUNK_BITS); + for (size_t i = 0; i < count; i++) { + mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE); + } + mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); + return true; + } + else { + // contended: we reset count to retry from the first + // (we still skip the first chunk to guarantee progress) + count = 0; + } + } + + // keep searching but skip the scanned range + chunk_idx += count+1; + } + return false; +} + + + + + diff --git a/csrc/mimalloc/src/bitmap.h b/csrc/mimalloc/src/bitmap.h new file mode 100644 index 000000000..ba8224b8a --- /dev/null +++ b/csrc/mimalloc/src/bitmap.h @@ -0,0 +1,344 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2024 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically +---------------------------------------------------------------------------- */ +#pragma once +#ifndef MI_BITMAP_H +#define MI_BITMAP_H + +/* -------------------------------------------------------------------------------- + Atomic bitmaps with release/acquire guarantees: + + `mi_bfield_t`: is a single machine word that can efficiently be bit counted (usually `size_t`) + each bit usually represents a single MI_ARENA_SLICE_SIZE in an arena (64 KiB). + We need 16K bits to represent a 1GiB arena. + + `mi_bchunk_t`: a chunk of bfield's of a total of MI_BCHUNK_BITS (= 512 on 64-bit, 256 on 32-bit) + allocations never span across chunks -- so MI_ARENA_MAX_OBJ_SIZE is the number + of bits in a chunk times the MI_ARENA_SLICE_SIZE (512 * 64KiB = 32 MiB). + These chunks are cache-aligned and we can use AVX2/AVX512/NEON/SVE/SVE2/etc. instructions + to scan for bits (perhaps) more efficiently. + + We allocate byte-sized ranges aligned to bytes in the bfield, and bfield-sized + ranges aligned to a bfield. + + Searching linearly through the chunks would be too slow (16K bits per GiB). + Instead we add a "chunkmap" to do a two-level search (more or less a btree of depth 2). + + `mi_bchunkmap_t` (== `mi_bchunk_t`): for each chunk we track if it has (potentially) any bit set. + The chunkmap has 1 bit per chunk that is set if the chunk potentially has a bit set. + This is used to avoid scanning every chunk. (and thus strictly an optimization) + It is conservative: it is fine to set a bit in the chunk map even if the chunk turns out + to have no bits set. It is also allowed to briefly have a clear bit even if the + chunk has bits set -- as long as we guarantee that the bit will be set later on; + (this allows us to set the chunkmap bit right after we set a bit in the corresponding chunk). + + However, when we clear a bit in a chunk, and the chunk is indeed all clear, we + cannot safely clear the bit corresponding to the chunk in the chunkmap since it + may race with another thread setting a bit in the same chunk. Therefore, when + clearing, we first test if a chunk is clear, then clear the chunkmap bit, and + then test again to catch any set bits that we may have missed. + + Since the chunkmap may thus be briefly out-of-sync, this means that we may sometimes + not find a free page even though it's there (but we accept this as we avoid taking + full locks). (Another way to do this is to use an epoch but we like to avoid that complexity + for now). + + `mi_bitmap_t`: a bitmap with N chunks. A bitmap has a chunkmap of MI_BCHUNK_BITS (512) + and thus has at most 512 chunks (=2^18 bits x 64 KiB slices = 16 GiB max arena size). + The minimum is 1 chunk which is a 32 MiB arena. + + For now, the implementation assumes MI_HAS_FAST_BITSCAN and uses trailing-zero-count + and pop-count (but we think it can be adapted work reasonably well on older hardware too) +--------------------------------------------------------------------------------------------- */ + +// A word-size bit field. +typedef size_t mi_bfield_t; + +#define MI_BFIELD_BITS_SHIFT (MI_SIZE_SHIFT+3) +#define MI_BFIELD_BITS (1 << MI_BFIELD_BITS_SHIFT) +#define MI_BFIELD_SIZE (MI_BFIELD_BITS/8) +#define MI_BFIELD_LO_BIT8 (((~(mi_bfield_t)0))/0xFF) // 0x01010101 .. +#define MI_BFIELD_HI_BIT8 (MI_BFIELD_LO_BIT8 << 7) // 0x80808080 .. + +#define MI_BCHUNK_SIZE (MI_BCHUNK_BITS / 8) +#define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit + + +// some compiler (msvc in C mode) cannot have expressions in the alignment attribute +#if MI_BCHUNK_SIZE==64 +#define mi_decl_bchunk_align mi_decl_align(64) +#elif MI_BCHUNK_SIZE==32 +#define mi_decl_bchunk_align mi_decl_align(32) +#else +#define mi_decl_bchunk_align mi_decl_align(MI_BCHUNK_SIZE) +#endif + + +// A bitmap chunk contains 512 bits on 64-bit (256 on 32-bit) +typedef mi_decl_bchunk_align struct mi_bchunk_s { + _Atomic(mi_bfield_t) bfields[MI_BCHUNK_FIELDS]; +} mi_bchunk_t; + + +// The chunkmap has one bit per corresponding chunk that is set if the chunk potentially has bits set. +// The chunkmap is itself a chunk. +typedef mi_bchunk_t mi_bchunkmap_t; + +#define MI_BCHUNKMAP_BITS MI_BCHUNK_BITS + +#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS) +#define MI_BITMAP_MIN_CHUNK_COUNT (1) +// #if MI_SIZE_BITS > 32 +// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map +// #else +#define MI_BITMAP_DEFAULT_CHUNK_COUNT (1) +// #endif +#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena +#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena +#define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena + + +// An atomic bitmap +typedef mi_decl_bchunk_align struct mi_bitmap_s { + _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS) + size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 1]; // suppress warning on msvc + mi_bchunkmap_t chunkmap; + mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT +} mi_bitmap_t; + + +static inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) { + return mi_atomic_load_relaxed(&((mi_bitmap_t*)bitmap)->chunk_count); +} + +static inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) { + return (mi_bitmap_chunk_count(bitmap) * MI_BCHUNK_BITS); +} + + + +/* -------------------------------------------------------------------------------- + Atomic bitmap operations +-------------------------------------------------------------------------------- */ + +// Many operations are generic over setting or clearing the bit sequence: we use `mi_xset_t` for this (true if setting, false if clearing) +typedef bool mi_xset_t; +#define MI_BIT_SET (true) +#define MI_BIT_CLEAR (false) + + +// Required size of a bitmap to represent `bit_count` bits. +size_t mi_bitmap_size(size_t bit_count, size_t* chunk_count); + +// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap. +size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero); + +// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks). +// Not atomic so only use if still local to a thread. +void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n); + + +// Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1 +bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx); + +// Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0 +bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx); + +// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's +// If `already_set` is not NULL, it is set to count of bits were already all set. +// (this is used for correct statistics if commiting over a partially committed area) +bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set); + +// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 1's to 0's +bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n); + + +// Is a sequence of n bits already all set/cleared? +bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n); + +// Is the bitmap completely clear? +bool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap); + +// Is a sequence of n bits already set? +// (Used to check if a memory range is already committed) +static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n); +} + +// Is a sequence of n bits already clear? +static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { + return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, idx, n); +} + +static inline bool mi_bitmap_is_set(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_is_setN(bitmap, idx, 1); +} + +static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_is_clearN(bitmap, idx, 1); +} + +// Called once a bit is cleared to see if the memory slice can be claimed. +typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set); + +// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true. +// If not claimed, continue on (potentially setting the bit again depending on `keep_set`). +// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`. +mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, + mi_claim_fun_t* claim, mi_arena_t* arena ); + + +// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set. +// This is used to delay free-ing a page that it at the same time being considered to be +// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`). +void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx); + + +// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit. +// Otherwise return `false` (and `*idx` is undefined). +// Used for unloading arena's +bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx); + +// Return count of all set bits in a bitmap. +size_t mi_bitmap_popcount(mi_bitmap_t* bitmap); + + +typedef bool (mi_forall_set_fun_t)(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg2); + +// Visit all set bits in a bitmap (`slice_count == 1`) +bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); + +// Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`) +// Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`) +bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); + +// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). +// This is used by purging to not break up transparent huge pages for example. +// Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`). +bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); + +// Count all set bits in given range in the bitmap. +size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n); + +/* ---------------------------------------------------------------------------- + Binned concurrent bitmap + Assigns a size class to each chunk such that small blocks don't cause too + much fragmentation since we keep chunks for larger blocks separate. +---------------------------------------------------------------------------- */ + +// mi_chunkbin_t is defined in mimalloc-stats.h + +static inline mi_chunkbin_t mi_chunkbin_inc(mi_chunkbin_t bbin) { + mi_assert_internal(bbin < MI_CBIN_COUNT); + return (mi_chunkbin_t)((int)bbin + 1); +} + +static inline mi_chunkbin_t mi_chunkbin_dec(mi_chunkbin_t bbin) { + mi_assert_internal(bbin > MI_CBIN_NONE); + return (mi_chunkbin_t)((int)bbin - 1); +} + +static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) { + if (slice_count==1) return MI_CBIN_SMALL; + if (slice_count==8) return MI_CBIN_MEDIUM; + #if MI_ENABLE_LARGE_PAGES + if (slice_count==MI_BFIELD_BITS) return MI_CBIN_LARGE; + #endif + if (slice_count > MI_BCHUNK_BITS) return MI_CBIN_HUGE; + return MI_CBIN_OTHER; +} + +// An atomic "binned" bitmap for the free slices where we keep chunks reserved for particular size classes +typedef mi_decl_bchunk_align struct mi_bbitmap_s { + _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS) + _Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set + mi_subproc_t* subproc; // constant, for stats + #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 3 + size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 3]; // suppress warning on msvc by aligning manually + #endif + mi_bchunkmap_t chunkmap; + mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE) + mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT +} mi_bbitmap_t; + + +static inline size_t mi_bbitmap_chunk_count(const mi_bbitmap_t* bbitmap) { + return mi_atomic_load_relaxed(&((mi_bbitmap_t*)bbitmap)->chunk_count); +} + +static inline size_t mi_bbitmap_max_bits(const mi_bbitmap_t* bbitmap) { + return (mi_bbitmap_chunk_count(bbitmap) * MI_BCHUNK_BITS); +} + +mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t chunk_idx); + +size_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count); + +// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear. +// Otherwise return `false` (and `*idx` is undefined). +// Used for debug output. +bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx); + +// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true +// returns the size of the bitmap. +size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero); + +// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks). +// Not atomic so only use if still local to a thread. +void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n); + + +// Set a sequence of `n` bits in the bbitmap; returns `true` if atomically transitioned from all 0's to 1's +bool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n); + + +// Is a sequence of n bits already all set/cleared? +bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n); + +// Is a sequence of n bits already set? +// (Used to check if a memory range is already committed) +static inline bool mi_bbitmap_is_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + return mi_bbitmap_is_xsetN(MI_BIT_SET, bbitmap, idx, n); +} + +// Is a sequence of n bits already clear? +static inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { + return mi_bbitmap_is_xsetN(MI_BIT_CLEAR, bbitmap, idx, n); +} + + +// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes. +// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`. +bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n); + + +// Specialized versions for common bit sequence sizes +bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 1-bit +bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits +// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS +bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS +bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS +bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS + +// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all. +// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`. +mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) { + if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx); // small pages + if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx); // medium pages + // if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages + if (n==0) return false; + if (n<=MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearNX(bbitmap, tseq, n, pidx); + if (n<=MI_BCHUNK_BITS) return mi_bbitmap_try_find_and_clearNC(bbitmap, tseq, n, pidx); + return mi_bbitmap_try_find_and_clearN_(bbitmap, tseq, n, pidx); +} + + +#endif // MI_BITMAP_H diff --git a/csrc/mimalloc/src/free.c b/csrc/mimalloc/src/free.c new file mode 100644 index 000000000..82ecf2601 --- /dev/null +++ b/csrc/mimalloc/src/free.c @@ -0,0 +1,817 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#if !defined(MI_IN_ALLOC_C) +#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)" +// add includes help an IDE +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" // _mi_prim_thread_id() +#endif + +// forward declarations +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); +static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); +// static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); + + +// ------------------------------------------------------ +// Free +// ------------------------------------------------------ + +// regular free of a (thread local) block pointer +// fast path written carefully to prevent spilling on the stack +static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) +{ + // checks + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; + // if mi_unlikely(!mi_check_is_double_free(page,block)) return; // checked with padding + + if (track_stats) { + mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + } + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); + _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); + #endif + + // actual free: push on the local free list + const mi_used_t used = page->used - 1; + mi_block_set_next(page, block, page->local_free); + page->used = used; + page->local_free = block; + if mi_unlikely(used==0) { + if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) + _mi_page_retire(page); + } + } + else if mi_unlikely(check_full && mi_page_is_in_full(page)) { + _mi_page_unfull(page); + } +} + +// Forward declaration for multi-threaded collect +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept; + +// Free a block multi-threaded +static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept +{ + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt + + // adjust stats (after padding check ) + mi_stat_free(page, block); // stat_free may access the padding + mi_track_free_size(block, usable_size); + + // _mi_padding_shrink(page, block, sizeof(mi_block_t)); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading + if (!was_guarded) { + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); + _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); + } + #endif + + // push atomically on the page thread free list + mi_thread_free_t tf_new; + mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); + do { + mi_block_set_next(page, block, mi_tf_block(tf_old)); + const bool new_owned = (allow_collect ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned + tf_new = mi_tf_create(block, new_owned); + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? + + // and atomically try to collect the page if it was abandoned + if (allow_collect) { + const bool is_owned_now = !mi_tf_is_owned(tf_old); + if (is_owned_now) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_free_try_collect_mt(page,block); + } + } +} + + +// Adjust a block that was allocated aligned, to the actual start of the block in the page. +// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the +// `page_woffset` and `block_size` fields; however these are constant and the page won't be +// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently. +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { + mi_assert_internal(page!=NULL && p!=NULL); + + const size_t diff = (uint8_t*)p - mi_page_start(page); + const size_t block_size = mi_page_block_size(page); + size_t adjust = diff & (block_size - 1); + if mi_unlikely(!_mi_is_power_of_two(block_size)) { + adjust = diff % block_size; + } + return (mi_block_t*)((uintptr_t)p - adjust); +} + +static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { + mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer + #if MI_SECURE > 0 + // in secure mode we always unalign to guard against free-ing interior pointers + return _mi_page_ptr_unalign(page,p); + #else + MI_UNUSED(page); + return (mi_block_t*)p; + #endif +} + +// forward declaration for a MI_GUARDED build +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration +static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { + if (mi_block_ptr_is_guarded(block, p)) { + mi_block_unguard(page, block, p); + return true; + } + else { + return false; + } +} +#else +static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); + return false; +} +#endif + + +// free a local pointer (page parameter comes first for better codegen) +static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { + mi_assert_internal(p!=NULL && page != NULL); + mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */); +} + +// free a pointer owned by another thread (page parameter comes first for better codegen) +static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { + mi_assert_internal(p!=NULL && page != NULL); + mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + const bool was_guarded = mi_block_check_unguard(page, block, p); + mi_free_block_mt(page, block, was_guarded, allow_collect); +} + +// generic free (for runtime integration) +void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { + if (is_local) mi_free_generic_local(page,p); + else mi_free_generic_mt(page,p,true); +} + + +// Get the page belonging to a pointer +// Does further checks in debug mode to see if this was a valid pointer. +static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, bool free_small, mi_page_t** ppage) +{ + MI_UNUSED_RELEASE(msg); + #if MI_DEBUG + if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { + _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); + return false; + } + #endif + + mi_page_t* page; + #if MI_PAGE_META_IS_ALIGNED + page = _mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return false; // p==NULL => page==NULL + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + + #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE + // for mi_free_small we can avoid a load-acquire + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } + #else + MI_UNUSED(free_small); + page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); + #endif + + mi_assert_internal(page!=NULL); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + #else + MI_UNUSED(free_small); + page = _mi_ptr_page(p); + if mi_unlikely(page==NULL) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } + #endif + *ppage = page; + return true; +} + +static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) { + mi_page_t* page; + return (mi_validate_ptr_page_nonnull(p,msg,false,&page) ? page : NULL); +} + +// Free a block +// Fast path written carefully to prevent register spilling on the stack +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +{ + mi_assert_internal(p!=NULL && page!=NULL); + if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + + const mi_threadid_t ptid = mi_page_xthread_id(page); + const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); + if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` + // thread-local, aligned, and not a full page + mi_block_t* const block = mi_validate_block_from_ptr(page,p); + mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); + } + else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` + // page is local, but is full or contains (inner) aligned blocks; use generic path + mi_free_generic_local(page, p); + } + // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) + else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` + // blocks are aligned (and not a full page); push on the thread_free list + mi_block_t* const block = mi_validate_block_from_ptr(page,p); + mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + } + else { + // page is full or contains (inner) aligned blocks; use generic multi-thread path + mi_free_generic_mt(page, p, allow_collect); + } +} + +void mi_free(void* p) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free",false,&page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } +} + +void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_ufree",false,&page)) { + mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); + } + else { + if (pblock_size!=NULL) { *pblock_size = 0; } + } +} + +void mi_free_small(void* p) mi_attr_noexcept { + #if MI_PAGE_META_SMALL_IS_ALIGNED + // deprecated: We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. + // If we keep page info in front of the page area for small objects, we can find the info + // just by aligning down the pointer instead of looking it up in the page map. + #if MI_GUARDED + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as MI_GUARDED is defined" + mi_free(p); + #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" + mi_free(p); + #else + mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); + mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); + mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); + mi_assert(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` + mi_free_nonnull(p, page, NULL, true); + #endif + #else + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true /* is_small? */,&page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } + #endif +} + +// void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { +// mi_free_nonnull(p, page, NULL, false /* allow collect? */); +// } + +// Free a pointer that is potentially allocated in a different sub-process +void _mi_free_subproc_safe(void* p) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"_mi_free_subproc_safe",false,&page)) { + mi_free_nonnull(p, page, NULL, false /* allow collect? */); + } +} + +// ------------------------------------------------------ +// Free variants +// ------------------------------------------------------ + +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + MI_UNUSED_RELEASE(size); + #if MI_DEBUG + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if mi_unlikely(size > usable) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + else if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_free_small(p); + } + else + #endif + { + mi_free(p); + } +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + +// checked free +bool mi_cfree(void* p) mi_attr_noexcept { + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + return true; + } + else { + return false; + } +} + + +// -------------------------------------------------------------------------------------------- +// `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page. +// 1. if the page becomes empty, free it +// 2. if it can be reclaimed, reclaim it in our theap +// 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages) +// -------------------------------------------------------------------------------------------- + +// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`) +static bool mi_abandoned_page_try_free(mi_page_t* page) +{ + if (!mi_page_all_free(page)) return false; + // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish) + _mi_arenas_page_unabandon(page,NULL); + _mi_arenas_page_free(page,NULL); // we can now free the page directly + return true; +} + +// Helper for mi_free_try_collect_mt: try if we can reabandon a previously abandoned mostly full page to be mapped +static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page) +{ + // if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations + // We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page + if (mi_page_is_mostly_used(page)) return false; // not too full + if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false; // and not already mapped (or unmappable) + + mi_assert(!mi_page_is_full(page)); + return _mi_arenas_page_try_reabandon_to_mapped(page); +} + +// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently +// freed in the meantime. Returns `true` if the page was freed. +// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`. +static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) { + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_all_free(page)); + // try to cas atomically the original free list (`mt_free`) back with the ownership cleared. + mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true); + mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false); + while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) { + mi_assert_internal(mi_tf_is_owned(tf_expect)); + // while the xthread_free list is not empty.. + while (mi_tf_block(tf_expect) != NULL) { + // if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used). + _mi_page_free_collect(page,false); // update used count + if (mi_abandoned_page_try_free(page)) return; + if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; + // otherwise continue un-owning + tf_expect = mi_atomic_load_relaxed(&page->xthread_free); + } + // and try again to release ownership + mi_assert_internal(mi_tf_block(tf_expect)==NULL); + tf_new = mi_tf_create(NULL, false); + } +} + +static inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) { + if (atmost < 0) return false; + mi_page_queue_t* const pq = mi_page_queue(theap,block_size); + mi_assert_internal(pq!=NULL); + return (pq->count <= (size_t)atmost); +} + +// Helper for mi_free_try_collect_mt: try to reclaim the page for ourselves +static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept +{ + // note: reclaiming can improve benchmarks like `larson` or `rbtree-ck` a lot even in the single-threaded case, + // since free-ing from an owned page avoids atomic operations. However, if we reclaim too eagerly in + // a multi-threaded scenario we may start to hold on to too much memory and reduce reuse among threads. + // If the current theap is where the page originally came from, we reclaim much more eagerly while + // 'cross-thread' reclaiming on free is by default off (and we only 'reclaim' these by finding the abandoned + // pages when we allocate a fresh page). + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_all_free(page)); + mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE); + mi_assert_internal(reclaim_on_free >= 0); + + // dont reclaim if we just have terminated this thread and we should + // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944) + if (!_mi_thread_is_initialized()) return false; + + // get our theap + mi_theap_t* const theap = _mi_page_associated_theap_peek(page); + if (theap==NULL || theap->tld==NULL || !theap->allow_page_reclaim) return false; // see issue #1289 + + // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance? + // set max_reclaim limit + long max_reclaim = 0; + if mi_likely(theap == page->theap) { // did this page originate from the current theap? (and thus allocated from this thread) + // originating theap + max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim); + } + else if (reclaim_on_free == 1 && // if cross-thread is allowed + !theap->tld->is_in_threadpool && // and we are not part of a threadpool + !mi_page_is_mostly_used(page) && // and the page is not too full + _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) { // and it fits our memory + // across threads + max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim); + } + + // are we within the reclaim limit? + if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) { + return false; + } + + // reclaim the page into this theap + // first remove it from the abandoned pages in the arena -- this might wait for any readers to finish + _mi_arenas_page_unabandon(page, theap); + _mi_theap_page_reclaim(theap, page); + mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1); + return true; +} + + +// We freed a block in an abandoned page (that was not owned). Try to collect +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept +{ + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mt_free != NULL); + // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses + + // we own the page now, and it is safe to collect the thread atomic free list + if (page->block_size <= MI_SMALL_SIZE_MAX) { + // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list + // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). + // however, if the page became completely free, the used count is guaranteed to be 0.) + mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. + _mi_page_free_collect_partly(page, mt_free); + } + else { + // for larger blocks we use the regular collect + _mi_page_free_collect(page,false /* no force */); + mt_free = NULL; // expected page->xthread_free value after collection + } + const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free); + #if MI_DEBUG > 1 + if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } + if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); } + #endif + + // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped + if (mi_abandoned_page_try_free(page)) return; + if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) { // early test for better codegen + if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return; + } + if (mi_abandoned_page_try_reabandon_to_mapped(page)) return; + + // otherwise unown the page again + mi_abandoned_page_unown_from_free(page, mt_free); +} + + +// ------------------------------------------------------ +// Usable size +// ------------------------------------------------------ + +// Bytes available in a block +static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + const size_t size = mi_page_usable_size_of(page, block, is_guarded); + mi_assert_internal((void*)p >= (void*)block); + const size_t adjust = (uint8_t*)p - (uint8_t*)block; + mi_assert_internal(adjust <= size); + const size_t aligned_size = (adjust <= size ? size - adjust : 0); // size can be zero if the padding is corrupted + return aligned_size; +} + +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { + if mi_unlikely(page==NULL) return 0; + mi_assert_internal(mi_validate_ptr_page(p,"_mi_page_usable_size") == page); + if mi_likely(!mi_page_has_interior_pointers(page)) { + const mi_block_t* block = mi_validate_block_from_ptr(page,p); + return mi_page_usable_size_of(page, block, false /* is guarded */); + } + else { + // split out to separate routine for improved code generation + return mi_page_usable_aligned_size_of(page, p); + } +} + +mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); + return _mi_page_usable_size(page,p); +} + + +// ------------------------------------------------------ +// Deprecated: double free is checked with padding now. +// Check for double free in secure and debug mode +// This is somewhat expensive so only enabled for secure mode 4 +// ------------------------------------------------------ + +#if MI_CHECK_DOUBLE_FREE +// linear check if the free list contains a specific element +static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { + const size_t max_count = page->capacity; // can never hold more blocks than the capacity + size_t count = 0; + while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterations + if (elem==list) return true; + list = mi_block_next(page, list); + count++; + } + if mi_unlikely(count > max_count) { + _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind); + } + return false; +} + +static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { + // The decoded value is in the same page (or NULL). + // Walk the free lists to verify positively if it is already freed + if (mi_list_contains(page, page->free, block, "free") || + mi_list_contains(page, page->local_free, block, "local free") || + mi_list_contains(page, mi_page_thread_free(page), block, "thread free")) + { + _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); + return true; + } + return false; +} + +// Used for double free checking to avoid checking free lists too frequently +static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { + mi_block_t* n = mi_block_nextx(page,block,page->keys); + return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? +} + +// check if `block` was free'd before +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL? + { + // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? + // (continue in separate function to improve code generation) + return mi_check_is_double_freex(page, block); + } + else return false; +} +#else +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(page); + MI_UNUSED(block); + return false; +} +#endif + + +// --------------------------------------------------------------------------- +// Check for theap block overflow by setting up padding at the end of the block +// --------------------------------------------------------------------------- + +#if MI_PADDING // && !MI_TRACK_ENABLED +static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) { + *bsize = mi_page_usable_block_size(page); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); + *delta = padding->delta; + const uint32_t canary = padding->canary; + const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); + if (double_free!=NULL) { + if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? + else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + } + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); + return ok; +} + +// Return the exact usable size of a block. +static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) { + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + return (bsize - _mi_os_page_size()); + } + else { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); + mi_assert_internal(ok); mi_assert_internal(delta <= bsize); + return (ok ? bsize - delta : 0); + } +} + +// When a non-thread-local block is freed, it becomes part of the thread delayed free +// list that is freed later by the owning theap. If the exact usable size is too small to +// contain the pointer for the delayed list, then shrink the padding (by decreasing delta) +// so it will later not trigger an overflow error in `mi_free_block`. +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); + mi_assert_internal(ok); + if (!ok || (bsize - delta) >= min_size) return; // usually already enough space + mi_assert_internal(bsize >= min_size); + if (bsize < min_size) return; // should never happen + size_t new_delta = (bsize - min_size); + mi_assert_internal(new_delta < bsize); + mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize); + mi_track_mem_defined(padding,sizeof(mi_padding_t)); + padding->delta = (uint32_t)new_delta; + mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); +} +#else +static inline size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) { + MI_UNUSED(block); + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + return (bsize - _mi_os_page_size()); + } + else { + return mi_page_usable_block_size(page); + } +} + +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size); +} +#endif + +#if MI_PADDING + +static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) { + size_t bsize; + size_t delta; + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free ); + *size = *wrong = bsize; + if (!ok) return false; + mi_assert_internal(bsize >= delta); + *size = bsize - delta; + #if MI_PADDING_CHECK_BYTES + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)block + bsize - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes + mi_track_mem_defined(fill, maxpad); + for (size_t i = 0; i < maxpad; i++) { + if (fill[i] != MI_DEBUG_PADDING) { + *wrong = bsize - delta + i; + ok = false; + break; + } + } + mi_track_mem_noaccess(fill, maxpad); + } + #endif + return ok; +} + +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + *usable_size = (bsize - _mi_os_page_size()); + return true; + } + else { + size_t wrong; + bool is_double_free; + if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) { + if (is_double_free) { + _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size); + } + else { + _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong ); + } + return false; + } + return true; + } +} + +#else + +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + *usable_size = mi_page_usable_size_of(page,block,is_guarded); + return true; +} + +#endif + +// only maintain stats for smaller objects if requested +#if (MI_STAT>0) +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(block); + mi_theap_t* theap = _mi_theap_default(); + mi_lock_t* lock = NULL; + mi_subproc_t* const subproc = mi_page_subproc(page); + mi_theap_t* const theap_meta = subproc->theap_meta; + if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) + // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page + (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { + theap = theap_meta; + lock = &subproc->theap_meta_lock; + mi_lock_acquire(lock); + } + + const size_t bsize = mi_page_usable_block_size(page); + // #if (MI_STAT>1) + // const size_t usize = mi_page_usable_size_of(page, block); + // mi_theap_stat_decrease(theap, malloc_requested, usize); + // #endif + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { + mi_theap_stat_decrease(theap, malloc_normal, bsize); + #if (MI_STAT > 1) + mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1); + #endif + } + else { + const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc + mi_theap_stat_decrease(theap, malloc_huge, bpsize); + } + + if mi_unlikely(lock!=NULL) { + mi_lock_release(lock); + } +} +#else +void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { + MI_UNUSED(page); MI_UNUSED(block); +} +#endif + + +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(p); + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_interior_pointers(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(!page->memid.is_pinned); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); +} + +// unguard a whole page (called from `mi_heap_destroy`) +void _mi_page_unguard_all(mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + uint8_t* const start = mi_page_start(page); + const size_t psize = mi_page_committed(page); + _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded +} +#else +void _mi_page_unguard_all(mi_page_t* page) { + MI_UNUSED(page); + // nothing to do +} +#endif diff --git a/csrc/mimalloc/src/heap.c b/csrc/mimalloc/src/heap.c new file mode 100644 index 000000000..3e7611b5e --- /dev/null +++ b/csrc/mimalloc/src/heap.c @@ -0,0 +1,304 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim-tls.h" // _mi_heap_theap + + +/* ----------------------------------------------------------- + Heap's +----------------------------------------------------------- */ + +mi_theap_t* mi_heap_theap(mi_heap_t* heap) { + return _mi_heap_theap(heap); // in prim.h +} + +void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) { + if (heap==NULL) { heap = mi_heap_main(); } + heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count()); +} + +void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) { + if (heap==NULL) { heap = mi_heap_main(); } + _mi_stats_merge_into(&heap->subproc->stats, &heap->stats); +} + +void mi_heap_stats_merge_to_main(mi_heap_t* heap) { + if (heap==NULL) return; + _mi_stats_merge_into(&mi_heap_get_heap_main(heap)->stats, &heap->stats); +} + +bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) { + mi_assert_internal((uintptr_t)theap == 1 || _mi_theap_heap(theap)==heap); + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(heap->theap != 0); + return _mi_thread_local_set(heap->theap,theap); +} + +// mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) { +// mi_theap_t* theap; +// mi_assert_internal(heap->theap != 0); +// if mi_likely(heap->theap!=0) { // paranoia +// theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); +// } +// else { +// _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap); +// return NULL; +// } +// mi_assert_internal(!_mi_is_empty_theap(theap)); +// mi_assert_internal(theap->heap == heap); // this goes wrong if using main heaps across subprocesses (as all share the same key) +// return theap; +// } + + +static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap) +{ + mi_heap_t* heap = (mi_heap_t*)const_heap; + mi_assert_internal(heap!=NULL); + + // initialize thread first in case this is the main heap + // (which may allocate the default theap already for the main heap) + if (!_mi_thread_is_initialized()) { + mi_thread_init(); + } + + // get the thread local theap + mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); + + // create a fresh theap? + if (theap==NULL) { + // allocate a fresh theap + theap = _mi_theap_create(heap, mi_theap_get_default()->tld); // sets the theap thread local + if (theap==NULL) { + _mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n"); + return NULL; + } + _mi_heap_theap_set(heap, theap); + mi_assert_internal(theap == (mi_theap_t*)_mi_thread_local_get(heap->theap)); + } + return theap; +} + + +// get (and possibly create) the theap belonging to a heap +mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) +{ + mi_assert_internal(heap->theap != 0); + mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); + if mi_unlikely(theap==NULL) { + theap = mi_heap_init_theap(heap); + if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic + } + _mi_theap_cached_set(theap); + return theap; +} + +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id) +{ + // init fields + heap->theap = theap_slot; + heap->subproc = subproc; + heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); + heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); + heap->numa_node = -1; // no initial affinity + mi_stats_header_init(&heap->stats); + mi_lock_init(&heap->theaps_lock); + mi_lock_init(&heap->os_abandoned_pages_lock); + mi_lock_init(&heap->arena_pages_lock); + + // push onto the subproc heaps + mi_lock(&heap->subproc->heaps_lock) { + mi_heap_t* head = heap->subproc->heaps; + heap->prev = NULL; + heap->next = head; + if (head!=NULL) { head->prev = heap; } + heap->subproc->heaps = heap; + } + mi_atomic_increment_relaxed(&subproc->heap_count); + mi_subproc_stat_increase(subproc, heaps, 1); + mi_assert_internal(_mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0); +} + +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap) { + mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL); + // heap data is allocated in the current subproc + mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main); + // todo: allocate heap data in the exclusive arena ? + mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) ); + if (heap==NULL) return NULL; + // reserve a thread local slot for this heap (see also issue #1230) + mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create()); + if (theap_slot == 0) { + _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n"); + mi_free(heap); + return NULL; + } + if (is_main_heap) { + mi_assert_internal(subproc->heap_main == NULL); + subproc->heap_main = heap; + } + _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id); + return heap; +} + +mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { + // `mi_heap_new` may be the very first mimalloc call in a process, in which case the + // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`. + mi_thread_init(); + return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false); +} + +mi_heap_t* mi_heap_new(void) { + return mi_heap_new_in_arena(0); +} + +// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency) +static void mi_heap_free_theaps(mi_heap_t* heap) { + // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), + // and we need to ensure we free theaps atomically. + + // We first detach our theaps list from any thread local lists + _mi_heap_detach_theaps(heap); + + // Now we can safely free the theaps + mi_lock(&heap->theaps_lock) { // paranoia + mi_theap_t* theap = heap->theaps; + heap->theaps = NULL; + while(theap != NULL) { + mi_theap_t* next = theap->hnext; + theap->hnext = NULL; + theap->hprev = NULL; + mi_assert_internal(theap->tld==NULL); + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // and free + _mi_theap_decref(theap); // a cached entry can still point to the theap + theap = next; + } + } +} + +// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) +static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { + mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap)); + + // free all arena pages infos + const bool is_main = _mi_is_heap_main(heap); + if (!is_main) { // pages for the main heap are pre-allocated in the arenas + mi_lock(&heap->arena_pages_lock) { + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { + mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); + if (arena_pages!=NULL) { + mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); + _mi_free_subproc_safe(arena_pages); + } + } + } + } + + // remove the heap from the subproc + if (!is_main) { + mi_heap_stats_merge_to_main(heap); + } + else { + _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + } + mi_atomic_decrement_relaxed(&heap->subproc->heap_count); + mi_subproc_stat_decrease(heap->subproc, heaps, 1); + mi_lock_maybe(&heap->subproc->heaps_lock, acquire_heaps_lock) { + if (heap->next!=NULL) { heap->next->prev = heap->prev; } + if (heap->prev!=NULL) { heap->prev->next = heap->next; } + else { heap->subproc->heaps = heap->next; } + } + + mi_lock_done(&heap->theaps_lock); + mi_lock_done(&heap->os_abandoned_pages_lock); + mi_lock_done(&heap->arena_pages_lock); + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + _mi_free_subproc_safe(heap); + } +} + +void mi_heap_delete(mi_heap_t* heap) { + if (heap==NULL) return; + mi_heap_t* heap_main = mi_heap_get_heap_main(heap); + if (heap == heap_main) { + _mi_warning_message("cannot delete the main heap\n"); + return; + } + mi_heap_free_theaps(heap); + _mi_heap_move_pages(heap, heap_main); + mi_heap_free(heap,true /* acquire subproc->heaps_lock */); +} + +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) { + if (heap==NULL) return; + mi_heap_free_theaps(heap); + _mi_heap_destroy_pages(heap); + // if (_mi_subproc_main()->heap_main == heap) { + // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + // } + // else + { + mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap? + } +} + +void mi_heap_destroy(mi_heap_t* heap) { + if (heap==NULL) return; + if (_mi_is_heap_main(heap)) { + _mi_warning_message("cannot destroy the main heap\n"); + return; + } + _mi_heap_force_destroy(heap,true /* acquire subproc->heaps_lock */); +} + +mi_heap_t* mi_heap_of(const void* p) { + mi_page_t* const page = _mi_safe_ptr_page(p); + if (page==NULL) return NULL; + return mi_page_heap(page); +} + +bool mi_any_heap_contains(const void* p) { + mi_page_t* const page = _mi_safe_ptr_page(p); + return (page!=NULL); +} + +bool mi_heap_contains(const mi_heap_t* heap, const void* p) { + if (heap==NULL) { heap = mi_heap_main(); } + return (heap==mi_heap_of(p)); +} + +// deprecated +bool mi_check_owned(const void* p) { + return mi_any_heap_contains(p); +} + +// unsafe heap utilization function for DragonFly (see issue #1258) +// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`. +// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`. +bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept { + if (p==NULL) return false; + const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer + if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false; + // If the page is the head of the queue, it is currently being used for + // allocations; we skip it to avoid immediate thrashing. + if (page->prev == NULL) return false; + + // match heap? + const mi_heap_t* const page_heap = mi_page_heap(page); + if (page_heap==NULL) return false; + if (heap!=NULL && page_heap!=heap) return false; + + // check utilization + if (page->capacity==0) return false; + if (perc_threshold>=100) return true; + return (perc_threshold >= ((100UL*page->used) / page->capacity)); +} diff --git a/csrc/mimalloc/src/init.c b/csrc/mimalloc/src/init.c new file mode 100644 index 000000000..4d85bc23a --- /dev/null +++ b/csrc/mimalloc/src/init.c @@ -0,0 +1,662 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" + +#include // memcpy, memset +#include // atexit + +// Empty page used to initialize the small free pages array +static const mi_page_t mi_page_empty = { + #if MI_PAGE_META_IS_ALIGNED + MI_ATOMIC_VAR_INIT(NULL), // self + #endif + MI_ATOMIC_VAR_INIT(0), // xthread_id + NULL, // free + 0, // used + NULL, // local_free + 0, // block_size + 0, // page_offset + 0, // capacity + 0, // reserved capacity + 0, // slice_pcommitted + 0, // retire_expire + false, // is_zero + MI_ATOMIC_VAR_INIT(0), // xthread_free + NULL, // theap + NULL, // heap + NULL, NULL, // next, prev + MI_MEMID_STATIC, // memid + #if (MI_PADDING || MI_ENCODE_FREELIST) + #if MI_PAGE_KEY_COUNT==2 + { 0, 0 }, // keys + #else + { 0 }, // key + #endif + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // { 0 }, // padding + #endif +}; + +#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) + +#if MI_SMALL_WSIZE_MAX == 128 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p) +#elif MI_SMALL_WSIZE_MAX == 256 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p) +#else +#error define initializer for direct pages +#endif + +#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#elif (MI_PADDING>0) +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#else +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } +#endif + + +// Empty page queues for every bin +#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t) } +#define MI_PAGE_QUEUES_EMPTY \ + { QNULL(1), \ + QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ + QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ + QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ + QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ + QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ + QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ + QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ + QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ + QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ + QNULL(MI_LARGE_MAX_OBJ_WSIZE + 1 /* 655360, Huge queue */), \ + QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ } + +#define MI_STAT_COUNT_NULL() {0,0,0} + +// Empty statistics +#define MI_STAT_COUNT(stat) {0,0,0}, +#define MI_STAT_COUNTER(stat) {0}, + +#define MI_STATS_FIELDS_NULL \ + MI_STAT_FIELDS() /* regular stat fields */ \ + { MI_INIT4(MI_STAT_COUNT_NULL) }, /* stat reserved */ \ + { { 0 }, { 0 }, { 0 }, { 0 } }, /* stat counter reserved */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* malloc_bins */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* page bins */ \ + { MI_INIT5(MI_STAT_COUNT_NULL) } /* chunk bins */ + +#define MI_STATS_NULL \ + { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL } + +// -------------------------------------------------------- +// Statically allocate an empty theap as the initial +// thread local value for the default theap, +// and statically allocate the backing theap for the main +// thread so it can function without doing any allocation +// itself (as accessing a thread local for the first time +// may lead to allocation itself on some platforms) +// -------------------------------------------------------- + +static mi_decl_cache_align mi_tld_t mi_tld_detached = { + MI_THREADID_DETACHED, // thread_id + 0, // thread_seq + 0, // default numa node + NULL, // subproc + NULL, // theaps list + MI_LOCK_INITIALIZER, // theaps lock + false, // recurse + false, // is_in_threadpool + MI_MEMID_STATIC // memid +}; + +mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + MI_SMALL_PAGES_EMPTY, // direct small pages + &mi_tld_detached, // tld + MI_ATOMIC_VAR_INIT(NULL), // heap + MI_ATOMIC_VAR_INIT(NULL), // subproc + MI_ATOMIC_VAR_INIT(1), // refcount + 0, // heartbeat + 0, // cookie + { {0}, {0}, 0, true }, // random + 0, // page count + MI_BIN_FULL, 0, // page retired min/max + 0, // pages_full_size + 0, 0, // generic count + NULL, NULL, // tnext, tprev + NULL, NULL, // hnext, hprev + 0, // full page retain + false, // allow reclaim + true, // allow abandon + true, // is_detached + #if MI_GUARDED + 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) + #endif + MI_PAGE_QUEUES_EMPTY, + MI_MEMID_STATIC, + MI_STATS_NULL, // stats +}; + +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER + + +// pre-allocate the process heap, and meta-data theap +static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero; + +// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats) +static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero; + +mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths +mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. + + +mi_page_t* _mi_page_empty_get(void) { + return (mi_page_t*)&mi_page_empty; +} + +mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) { + return (mi_theap_t*)&_mi_theap_empty; +} + +bool _mi_is_empty_theap(const mi_theap_t* theap) { + return (theap == &_mi_theap_empty); +} + +/* ----------------------------------------------------------- + Initialization + Note: on some platforms lock_init or just a thread local access + can cause allocation and induce recursion during initialization. +----------------------------------------------------------- */ + +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc); + +// Initialize main heap +static void mi_heap_main_init_once(void) { + mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC); + _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); + + // initialize the main subprocess + mi_subproc_t* subproc_main = _mi_subproc_main_init(); + + // detached tld for mi_theap_empty (and theap_meta) + mi_tld_detached.memid = memid_static; + mi_tld_init(&mi_tld_detached, 0, subproc_main); + + // main process heap + mi_process_heap_main.memid = memid_static; + mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main); + _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0); + + // detached theap for allocating meta-data (we can allocate on this without having an initialized thread) + mi_process_theap_meta.memid = memid_static; + _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); + mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads + mi_process_theap_meta.page_full_retain = 2; + subproc_main->theap_meta = &mi_process_theap_meta; + + // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` +} + +static void mi_heap_main_init(void) { + mi_atomic_do_once { + mi_heap_main_init_once(); + } +} + +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { + mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); + if mi_likely(heap!=NULL) { + return heap; + } + else if (_mi_subproc_is_main(subproc)) { + mi_heap_main_init(); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL); + return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); + } + else { + mi_assert_internal(false); + return &mi_process_heap_main; + } +} + +/* ----------------------------------------------------------- + Thread local data +----------------------------------------------------------- */ + +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) { + tld->subproc = subproc; + tld->theaps = NULL; + mi_lock_init(&tld->theaps_lock); + if (tld->thread_id == MI_THREADID_DETACHED) { + tld->numa_node = -1; + } + else { + tld->numa_node = _mi_os_numa_node(); + tld->thread_id = _mi_prim_thread_id(); + tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); + tld->thread_seq = tseq; + mi_atomic_increment_relaxed(&tld->subproc->thread_count); + } + return tld; +} + +// Allocate fresh tld +static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) { + mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate + const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count); + + mi_memid_t memid; + mi_tld_t* tld; + if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) { + tld = &mi_process_tld_main; + memid = _mi_memid_create_static(tld,sizeof(*tld)); + } + else { + tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid); + } + if (tld==NULL) { + _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n"); + return NULL; + } + tld->memid = memid; + return mi_tld_init(tld,tseq,subproc); +} + +mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { + if (tld==NULL) return; + mi_atomic_decrement_relaxed(&tld->subproc->thread_count); + tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id + // is reused by the OS after a thread has terminated. (see issue #1287) + mi_lock_done(&tld->theaps_lock); + _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld +} + + +/* ----------------------------------------------------------- + Thread Init +----------------------------------------------------------- */ + +#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) +static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) { + const mi_threadid_t tid = _mi_thread_id(); + mi_theap_t* thread_theap = NULL; + mi_lock(&heap->theaps_lock) { + for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) { + if (theap->tld->thread_id == tid) { + thread_theap = theap; + break; + } + } + } + return thread_theap; +} +#endif + +// Initialize thread +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) +{ + // ensure our process has started already + mi_process_init(); + + // if the theap_default is already set we have already initialized + mi_theap_t* theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) return theap; + + // initialize the default theap + // note: we cannot access thread-locals yet as that can cause (recursive) allocation + // (on macOS <= 14 for example where the loader allocates thread-local data on demand). + if (heap_main==NULL) { + heap_main = mi_heap_main(); + mi_assert_internal(heap_main == &mi_process_heap_main); + } + mi_assert_internal(heap_main!=NULL); + + #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) + theap = mi_heap_check_for_existing_theap(heap_main); // recursion check + #if !defined(MI_TLS_RECURSE_GUARD) + mi_assert_internal(theap==NULL); + #endif + #else + theap = NULL; + #endif + + if (theap==NULL) { + // allocated the tld + mi_tld_t* tld = mi_tld_create(heap_main->subproc); + if (tld==NULL) return NULL; // out-of-memory on tld allocation + // allocate and initialize the theap for the main heap + if (tld==&mi_process_tld_main) { + theap = &mi_process_theap_main; // initial theap is pre-allocated + theap->memid = _mi_memid_create_static(theap,sizeof(*theap)); + } + else { + theap = _mi_theap_alloc(heap_main,tld); // otherwise meta allocate + if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation + } + _mi_theap_init(theap,heap_main,tld); + } + + // now initialize the thread + _mi_theap_default_set(theap); + // and only then set the heap_theap field as that accesses thread locals + _mi_heap_theap_set(heap_main, theap); // todo: can fail! + + mi_assert_internal(mi_theap_is_initialized(theap)); + mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap)); + mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap); + + mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? + // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); + return theap; +} + +mi_theap_t* _mi_thread_init(void) { + return _mi_thread_init_with_heap(NULL); +} + +void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept { + _mi_thread_init(); +} + + + +/* ----------------------------------------------------------- + Theaps done +----------------------------------------------------------- */ + +// Free the thread local theaps +static void mi_thread_theaps_done(mi_tld_t* tld) +{ + // abandon the pages of all theaps in this thread + mi_lock(&tld->theaps_lock) { + mi_theap_t* theap = tld->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + // never destroy theaps; if a dll is linked statically with mimalloc, + // there may still be delete/free calls after the mi_fls_done is called. Issue #207 + _mi_theap_collect_abandon(theap); + mi_assert_internal(theap->page_count==0); + theap = next; + } + } + + // reset the thread local theaps + // note: do this after abandon as page->heap may be NULL and mi_heap_main should return the heap + // belonging to the right subprocess + _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty); + _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty); + + // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically. + // we first detach our theaps list from any heaps + _mi_tld_detach_theaps(tld); + + // no heaps point to our theaps anymore, free them + mi_lock(&tld->theaps_lock) { // paranoia + mi_theap_t* theap = tld->theaps; + tld->theaps = NULL; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); + theap->tld = NULL; + theap->tnext = NULL; + theap->tprev = NULL; + mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty + _mi_theap_decref(theap); + theap = next; + } + } + + mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete + mi_assert(!mi_theap_is_initialized(_mi_theap_default())); +} + + +// -------------------------------------------------------- +// Try to run `mi_thread_done()` automatically so any memory +// owned by the thread but not yet released can be abandoned +// and re-owned by another thread. +// +// 1. windows dynamic library: +// call from DllMain on DLL_THREAD_DETACH +// 2. windows static library: +// use special linker section to call a destructor when the thread is done +// 3. unix, pthreads: +// use a pthread key to call a destructor when a pthread is done +// +// In the last two cases we also need to call `mi_process_init` +// to set up the thread local keys. +// -------------------------------------------------------- + +// Set up hooks so `mi_thread_done` is called automatically +static void mi_process_setup_auto_thread_done(void) { + mi_atomic_do_once { + _mi_prim_thread_init_auto_done(); + } +} + +void mi_thread_done(void) mi_attr_noexcept { + _mi_thread_done(NULL); +} + +void _mi_thread_done(mi_theap_t* _theap_main) +{ + // NULL can be passed on some platforms + if (_theap_main==NULL) { + _theap_main = _mi_theap_default(); + } + + // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699) + if (!mi_theap_is_initialized(_theap_main)) { + return; + } + + // get the current tld + mi_tld_t* const tld = _theap_main->tld; + + // release dynamic thread_local's + _mi_thread_locals_thread_done(); + + // adjust stats + mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`? + + // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps... + if (tld->thread_id != _mi_prim_thread_id()) return; + + // delete the thread local theaps + mi_thread_theaps_done(tld); + + // free thread local data + mi_tld_free(tld); +} + +void mi_thread_set_in_threadpool(void) mi_attr_noexcept { + mi_theap_t* theap = mi_theap_get_default(); + theap->tld->is_in_threadpool = true; +} + + +// -------------------------------------------------------- +// Process init and done +// -------------------------------------------------------- + +static bool os_preloading = true; // true until this module is initialized + +// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. +bool mi_decl_noinline _mi_preloading(void) { + return os_preloading; +} + +// Returns true if mimalloc was redirected +mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { + return _mi_is_redirected(); +} + +// Called once by the process loader from `src/prim/prim.c` before `main` is called. +void _mi_auto_process_init(void) { + os_preloading = false; + + mi_process_init(); + mi_process_setup_auto_thread_done(); + + _mi_options_post_init(); // now we can print to stderr + if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); + + // show message from the redirector (if present) + const char* msg = NULL; + _mi_allocator_init(&msg); + if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { + _mi_fputs(NULL,NULL,NULL,msg); + } + + // reseed random + mi_theap_t* theap = _mi_theap_default(); + if (theap != NULL) { + _mi_random_reinit_if_weak(&theap->random); + mi_subproc_t* subproc = _mi_theap_subproc(theap); + if (subproc->theap_meta != NULL) { + mi_lock(&subproc->theap_meta_lock) { + _mi_random_reinit_if_weak(&subproc->theap_meta->random); + } + } + } +} + + +// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library) +static void mi_process_init_once(void) { + #if defined(__CYGWIN__) // we need to kickstart the cygwin runtime + __mi_thread_id_helper = NULL; + #endif + _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + + _mi_detect_cpu_features(); + _mi_options_init(); // read environment (if possible) + _mi_stats_init(); // start timer + _mi_os_init(); // primitive dependent + + mi_heap_main_init(); // before page_map_init so stats are working + _mi_page_map_init(); // todo: this could fail.. should we abort in that case? + mi_thread_init(); + + // the following can potentially allocate (on freeBSD for pthread keys) + _mi_tls_slots_init(); // pthread key create + _mi_thread_locals_init(); // pthread key create + _mi_process_is_initialized = true; + + #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) + // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. + // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup + // will not call _mi_thread_done on the (still executing) main thread. See issue #508. + _mi_prim_thread_associate_default_theap(NULL); + #endif + + // mi_stats_reset(); // only call stat reset *after* thread init (or the theap tld == NULL) + mi_track_init(); + if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { + size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); + int reserve_at = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX); + if (reserve_at != -1) { + mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); + } else { + mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + } + } + if (mi_option_is_enabled(mi_option_reserve_os_memory)) { + long ksize = mi_option_get(mi_option_reserve_os_memory); + if (ksize > 0) { + mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); + } + } + + #if MI_PAGE_META_IS_ALIGNED + mi_assert_internal((sizeof(mi_page_t)%MI_SIZE_SIZE) == 0); // or page->zoffset might not work + #endif +} + +// Initialize the process; called by thread_init or the process loader +void mi_process_init(void) mi_attr_noexcept { + mi_atomic_do_once { + mi_process_init_once(); + } +} + + +// Called when the process is done +static void mi_process_done_once(void) { + // only shutdown if we were initialized + if (!_mi_process_is_initialized) return; + // ensure we are called once + static bool process_done = false; + if (process_done) return; + process_done = true; + + // decref any cached theap + _mi_theap_cached_set(_mi_theap_empty_get()); + + // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread + _mi_prim_thread_done_auto_done(); + + #ifndef MI_SKIP_COLLECT_ON_EXIT + #if (MI_DEBUG || !defined(MI_SHARED_LIB)) + // free all memory if possible on process exit. This is not needed for a stand-alone process + // but should be done if mimalloc is statically linked into another shared library which + // is repeatedly loaded/unloaded, see issue #281. + mi_theap_collect(_mi_theap_default(), true /* force */); + #endif + #endif + + // done with tracking tools + mi_track_done(); + + // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free + // since after process_done there might still be other code running that calls `free` (like at_exit routines, + // or C-runtime termination code. + mi_subproc_t* subproc_main = _mi_subproc_main(); + if (mi_option_is_enabled(mi_option_destroy_on_exit)) { + _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map! + } + else { + // free dynamic thread locals (if used at all) + _mi_thread_locals_thread_done(); + _mi_thread_locals_done(); + if (subproc_main->heap_main != NULL) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + _mi_theap_merge_stats(subproc_main->theap_meta); + _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free + mi_heap_stats_merge_to_subproc(subproc_main->heap_main); + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek + } + } + } + + _mi_tls_slots_done(); + _mi_subproc_main_done(); + _mi_allocator_done(); + _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id); + os_preloading = true; // don't call the C runtime anymore +} + + +// Call when the process is done (cdecl as it is used with `at_exit` on some platforms) +void mi_cdecl mi_process_done(void) mi_attr_noexcept { + mi_atomic_do_once { + mi_process_done_once(); + } +} + +// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms) +void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept { + if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return; // allow disabling auto process done + mi_process_done(); +} diff --git a/csrc/mimalloc/src/libc.c b/csrc/mimalloc/src/libc.c new file mode 100644 index 000000000..17dd1d1dc --- /dev/null +++ b/csrc/mimalloc/src/libc.c @@ -0,0 +1,572 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// -------------------------------------------------------- +// This module defines various standard libc functions to reduce +// the dependency on libc, and also prevent errors caused +// by some libc implementations when called before `main` +// executes (due to malloc redirection) +// -------------------------------------------------------- + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // mi_prim_getenv + +char _mi_toupper(char c) { + if (c >= 'a' && c <= 'z') return (c - 'a' + 'A'); + else return c; +} + +int _mi_strnicmp(const char* s, const char* t, size_t n) { + mi_assert_internal(s!=NULL && t!=NULL); + if (n == 0) return 0; + for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { + if (_mi_toupper(*s) != _mi_toupper(*t)) break; + } + return (n == 0 ? 0 : *s - *t); +} + +bool _mi_streq(const char* s, const char* t) { + if (s==NULL && t==NULL) return true; + if (s==NULL || t==NULL) return false; + for (; *s != 0 && *t != 0; s++, t++) { + if (*s != *t) break; + } + return (*s == *t); +} + +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); + // copy until end of src, or when dest is (almost) full + while (*src != 0 && dest_size > 1) { + *dest++ = *src++; + dest_size--; + } + // always zero terminate + *dest = 0; + return (*src == 0); +} + +bool _mi_strlcat(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); + // find end of string in the dest buffer + while (*dest != 0 && dest_size > 1) { + dest++; + dest_size--; + } + // and catenate + return _mi_strlcpy(dest, src, dest_size); +} + +size_t _mi_strnlen(const char* s, size_t max_len) { + if (s==NULL) return 0; + size_t len = 0; + while(len < max_len && s[len] != 0) { len++; } + return len; +} + +size_t _mi_strlen(const char* s) { + return _mi_strnlen(s,PTRDIFF_MAX); +} + +char* _mi_strnstr(char* s, size_t max_len, const char* pat) { + if (s==NULL) return NULL; + if (pat==NULL) return s; + const size_t m = _mi_strnlen(s, max_len); + const size_t n = _mi_strlen(pat); + for (size_t start = 0; start + n <= m; start++) { + size_t i = 0; + while (i 0 ? 0 : (res == 0 ? ENOENT : EAGAIN)); +} +#endif + + +// -------------------------------------------------------- +// Define our own primitives for doing an action once +// -------------------------------------------------------- + +// Returns `true` only on the first invocation, signifying we can execute an action once. +// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action. +// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called. +bool _mi_atomic_once_enter(mi_atomic_once_t* once) { + const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid); + if mi_likely(once_tid == 1) { + return false; // already executed + } + const mi_threadid_t current_tid = _mi_thread_id(); + if (once_tid == current_tid) { + return false; // recursive invocation; don't block on ourselves + } + + mi_lock_acquire(&once->lock); + uintptr_t expected = 0; + if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well + return true; // should execute and release + } + else { + mi_lock_release(&once->lock); + return false; // already another thread entered and released + } +} + +void _mi_atomic_once_release(mi_atomic_once_t* once) { + if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia + mi_atomic_store_release(&once->tid,1); // done executing + mi_lock_release(&once->lock); + } +} + +#if MI_USE_PTHREADS +mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init) { + int err = pthread_key_create(pkey,destruct); + if mi_unlikely(err!=0) { + *pkey = MI_PTHREAD_KEY_INVALID; + _mi_error_message(ENOMEM,"unable to allocate a thread local variable (error %d)\n", err); + return false; + } + mi_assert_internal(*pkey != MI_PTHREAD_KEY_INVALID); + if (init!=NULL) { + pthread_setspecific(*pkey,init); + }; + mi_assert_internal(pthread_getspecific(*pkey)==init); + return true; +} +#endif + +// -------------------------------------------------------- +// Detect CPU features +// -------------------------------------------------------- +mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast +mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast +mi_decl_cache_align bool _mi_cpu_has_popcnt = false; + +#if (MI_ARCH_X64 || MI_ARCH_X86) +#if defined(__GNUC__) +// #include +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx) + // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect). + uint32_t eax, ebx, ecx, edx; + __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : ); + regs4[0] = eax; + regs4[1] = ebx; + regs4[2] = ecx; + regs4[3] = edx; + return true; +} + +#elif defined(_MSC_VER) +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel); + return true; +} +#else +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel); + return false; +} +#endif + +void _mi_detect_cpu_features(void) { + // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) + // FSRS for fast short rep stosb + bool amd = false; + bool fsrm = false; + // bool erms = false; + bool fsrs = false; + uint32_t cpu_info[4]; + if (mi_cpuid(cpu_info, 0, 0)) { + amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) + } + if (mi_cpuid(cpu_info, 7, 0)) { + fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see + // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see + } + if (mi_cpuid(cpu_info, 7, 1)) { + fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see + } + if (mi_cpuid(cpu_info, 1, 0)) { + _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see + } + + if (fsrm) { + _mi_cpu_movsb_max = 127; + } + if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 + _mi_cpu_stosb_max = 127; + } +} + +#else +void _mi_detect_cpu_features(void) { + #if MI_ARCH_ARM64 + _mi_cpu_has_popcnt = true; + #endif +} +#endif + + +// -------------------------------------------------------- +// Define our own limited `_mi_vsnprintf` and `_mi_snprintf` +// This is mostly to avoid calling these when libc is not yet +// initialized (and to reduce dependencies) +// +// format: d i, p x u, s +// prec: z l ll L +// width: 10 +// align-left: - +// fill: 0 +// plus: + +// -------------------------------------------------------- + +static void mi_outc(char c, char** out, char* end) { + char* p = *out; + if (p >= end) return; + *p = c; + *out = p + 1; +} + +static void mi_outs(const char* s, char** out, char* end) { + if (s == NULL) return; + char* p = *out; + while (*s != 0 && p < end) { + *p++ = *s++; + } + *out = p; +} + +static void mi_out_fill(char fill, size_t len, char** out, char* end) { + char* p = *out; + for (size_t i = 0; i < len && p < end; i++) { + *p++ = fill; + } + *out = p; +} + +static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) { + if (len == 0 || extra == 0) return; + if (start + len + extra >= end) return; + // move `len` characters to the right (in reverse since it can overlap) + for (size_t i = 1; i <= len; i++) { + start[len + extra - i] = start[len - i]; + } + // and fill the start + for (size_t i = 0; i < extra; i++) { + start[i] = fill; + } +} + + +static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end) +{ + if (x == 0 || base == 0 || base > 16) { + if (prefix != 0) { mi_outc(prefix, out, end); } + mi_outc('0',out,end); + } + else { + #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */ + char num[MI_MAX_OUT_DIGITS]; + int dcount = 0; + while(x>0 && dcount < MI_MAX_OUT_DIGITS) { + char digit = (char)(x % base); + num[dcount++] = (digit <= 9 ? '0' + digit : 'A' + digit - 10); + x = x / base; + } + if (dcount>=MI_MAX_OUT_DIGITS) return; // don't output anything? + if (prefix != 0) { + mi_outc(prefix, out, end); + } + while(dcount-- > 0) { + mi_outc(num[dcount], out, end); + } + } +} + + +#define MI_NEXTC() c = *in; if (c==0) break; in++; + +int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { + if (buf == NULL || bufsize == 0 || fmt == NULL) return 0; + buf[bufsize - 1] = 0; + char* const end = buf + (bufsize - 1); + const char* in = fmt; + char* out = buf; + while (true) { + if (out >= end) break; + char c; + MI_NEXTC(); + if (c != '%') { + if (c == '\\') { + MI_NEXTC(); + switch (c) { + case 'e': mi_outc('\x1B', &out, end); break; + case 't': mi_outc('\t', &out, end); break; + case 'n': mi_outc('\n', &out, end); break; + case 'r': mi_outc('\r', &out, end); break; + case '\\': mi_outc('\\', &out, end); break; + default: /* ignore */ break; + } + } + else if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t' || c=='\x1b') { // output visible ascii or standard control only + mi_outc(c, &out, end); + } + } + else { + MI_NEXTC(); + char fill = ' '; + size_t width = 0; + char numtype = 'd'; + char numplus = 0; + bool alignright = true; + if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); } + if (c == '-') { alignright = false; MI_NEXTC(); } + if (c == '0') { fill = '0'; MI_NEXTC(); } + if (c >= '1' && c <= '9') { + width = (c - '0'); MI_NEXTC(); + while (c >= '0' && c <= '9') { + if (width < SIZE_MAX/1024) { // no overflow + width = (10 * width) + (c - '0'); + } + MI_NEXTC(); + } + if (c == 0) break; // extra check due to while + } + if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); } + else if (c == 'l') { + numtype = c; MI_NEXTC(); + if (c == 'l') { numtype = 'L'; MI_NEXTC(); } + } + + char* start = out; + if (c == '%') { + mi_outc('%', &out, end); + } + else if (c == 's') { + // string + const char* s = va_arg(args, const char*); + mi_outs(s, &out, end); + } + else if (c == 'p' || c == 'x' || c == 'u') { + // unsigned + uintmax_t x = 0; + if (c == 'x' || c == 'u') { + if (numtype == 'z') x = va_arg(args, size_t); + else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t + else if (numtype == 'L') x = va_arg(args, unsigned long long); + else if (numtype == 'l') x = va_arg(args, unsigned long); + else x = va_arg(args, unsigned int); + } + else if (c == 'p') { + void* const p = va_arg(args, void*); + x = (uintptr_t)p; + mi_outs("0x", &out, end); + start = out; + width = (width >= 2 ? width - 2 : 0); + } + if (width == 0 && (c == 'x' || c == 'p')) { + if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); } + if (width == 0) { width = 2; } + if (alignright) { fill = '0'; } + } + mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end); + } + else if (c == 'i' || c == 'd') { + // signed + intmax_t x = 0; + if (numtype == 'z') x = va_arg(args, intptr_t ); + else if (numtype == 't') x = va_arg(args, ptrdiff_t); + else if (numtype == 'L') x = va_arg(args, long long); + else if (numtype == 'l') x = va_arg(args, long); + else x = va_arg(args, int); + char pre = 0; + if (x < 0) { + pre = '-'; + if (x > INTMAX_MIN) { x = -x; } + } + else if (numplus != 0) { + pre = numplus; + } + mi_out_num((uintmax_t)x, 10, pre, &out, end); + } + else if (c >= ' ' && c <= '~') { + // unknown format + mi_outc('%', &out, end); + mi_outc(c, &out, end); + } + + // fill & align + mi_assert_internal(out <= end); + mi_assert_internal(out >= start); + const size_t len = out - start; + if (len < width) { + mi_out_fill(fill, width - len, &out, end); + if (alignright && out <= end) { + mi_out_alignright(fill, start, len, width - len, end); + } + } + } + } + mi_assert_internal(out <= end); + *out = 0; + return (int)(out - buf); +} + +int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + const int written = _mi_vsnprintf(buf, buflen, fmt, args); + va_end(args); + return written; +} + +#undef MI_NEXTC + + +// -------------------------------------------------------- +// generic trailing and leading zero count, and popcount +// -------------------------------------------------------- + +#if !MI_HAS_FAST_BITSCAN + +static size_t mi_ctz_generic32(uint32_t x) { + // de Bruijn multiplication, see + static const uint8_t debruijn[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + if (x==0) return 32; + return debruijn[(uint32_t)((x & (~x + 1U)) * (uint32_t)(0x077CB531U)) >> 27]; +} + +static size_t mi_clz_generic32(uint32_t x) { + // de Bruijn multiplication, see + static const uint8_t debruijn[32] = { + 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, + 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0 + }; + if (x==0) return 32; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27]; +} + +size_t _mi_ctz_generic(size_t x) { + if (x==0) return MI_SIZE_BITS; + #if (MI_SIZE_BITS <= 32) + return mi_ctz_generic32((uint32_t)x); + #else + const uint32_t lo = (uint32_t)x; + if (lo != 0) { + return mi_ctz_generic32(lo); + } + else { + return (32 + mi_ctz_generic32((uint32_t)(x>>32))); + } + #endif +} + +size_t _mi_clz_generic(size_t x) { + if (x==0) return MI_SIZE_BITS; + #if (MI_SIZE_BITS <= 32) + return mi_clz_generic32((uint32_t)x); + #else + const uint32_t hi = (uint32_t)(x>>32); + if (hi != 0) { + return mi_clz_generic32(hi); + } + else { + return 32 + mi_clz_generic32((uint32_t)x); + } + #endif +} + +#endif // bit scan + + +#if MI_SIZE_SIZE == 4 +#define mi_mask_even_bits32 (0x55555555) +#define mi_mask_even_pairs32 (0x33333333) +#define mi_mask_even_nibbles32 (0x0F0F0F0F) + +// sum of all the bytes in `x` if it is guaranteed that the sum < 256! +static size_t mi_byte_sum32(uint32_t x) { + // perform `x * 0x01010101`: the highest byte contains the sum of all bytes. + x += (x << 8); + x += (x << 16); + return (size_t)(x >> 24); +} + +static size_t mi_popcount_generic32(uint32_t x) { + // first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10 + // in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair + // into the lower bit-pair: + x = x - ((x >> 1) & mi_mask_even_bits32); + // add the 2-bit pair results + x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32); + // add the 4-bit nibble results + x = (x + (x >> 4)) & mi_mask_even_nibbles32; + // each byte now has a count of its bits, we can sum them now: + return mi_byte_sum32(x); +} + +mi_decl_noinline size_t _mi_popcount_generic(size_t x) { + if (x<=1) return x; + if (~x==0) return MI_SIZE_BITS; + return mi_popcount_generic32(x); +} + +#else +#define mi_mask_even_bits64 (0x5555555555555555) +#define mi_mask_even_pairs64 (0x3333333333333333) +#define mi_mask_even_nibbles64 (0x0F0F0F0F0F0F0F0F) + +// sum of all the bytes in `x` if it is guaranteed that the sum < 256! +static size_t mi_byte_sum64(uint64_t x) { + x += (x << 8); + x += (x << 16); + x += (x << 32); + return (size_t)(x >> 56); +} + +static size_t mi_popcount_generic64(uint64_t x) { + x = x - ((x >> 1) & mi_mask_even_bits64); + x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64); + x = (x + (x >> 4)) & mi_mask_even_nibbles64; + return mi_byte_sum64(x); +} + +mi_decl_noinline size_t _mi_popcount_generic(size_t x) { + if (x<=1) return x; + if (~x==0) return MI_SIZE_BITS; + return mi_popcount_generic64(x); +} +#endif + diff --git a/csrc/mimalloc/src/options.c b/csrc/mimalloc/src/options.c new file mode 100644 index 000000000..a35661ece --- /dev/null +++ b/csrc/mimalloc/src/options.c @@ -0,0 +1,696 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // mi_prim_out_stderr + +#include // stdin/stdout +#include // abort + +static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit) +static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit) + +static void mi_add_stderr_output(void); + +int mi_version(void) { + return MI_MALLOC_VERSION; +} + + +// -------------------------------------------------------- +// Options +// These can be accessed by multiple threads and may be +// concurrently initialized, but an initializing data race +// is ok since they resolve to the same value. +// -------------------------------------------------------- + + +#define MI_OPTION(opt) mi_option_##opt, #opt, NULL +#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy + +// Some options can be set at build time for statically linked libraries +// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// +// This is useful if we cannot pass them as environment variables +// (and setting them programmatically would be too late) + +#ifndef MI_DEFAULT_VERBOSE +#define MI_DEFAULT_VERBOSE 0 +#endif + +#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT +#define MI_DEFAULT_ARENA_EAGER_COMMIT 2 +#endif + +// in KiB +#ifndef MI_DEFAULT_ARENA_RESERVE + #if (MI_INTPTR_SIZE>4) + #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L + #else + #define MI_DEFAULT_ARENA_RESERVE 128L*1024L + #endif +#endif + +#ifndef MI_DEFAULT_ARENA_MAX_OBJECT_SIZE +#define MI_DEFAULT_ARENA_MAX_OBJECT_SIZE ((MI_SIZE_BITS * MI_ARENA_MAX_CHUNK_OBJ_SIZE)/MI_KiB) /* 2 GiB (or 256 MiB on 32-bit), larger than this is alloc'd by the OS */ +#endif + +#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC +#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0 +#endif + +#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES +#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES +#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_OS_MEMORY +#define MI_DEFAULT_RESERVE_OS_MEMORY 0 +#endif + +#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE +#if MI_GUARDED && !MI_DEBUG +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000 +#else +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 +#endif +#endif + +#ifndef MI_DEFAULT_PAGEMAP_COMMIT +#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access +#define MI_DEFAULT_PAGEMAP_COMMIT 1 +#else +#define MI_DEFAULT_PAGEMAP_COMMIT 0 +#endif +#endif + +#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM +#define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited +#endif + +#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM +#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM 32 +#endif + +#ifndef MI_DEFAULT_ALLOW_THP +#if defined(__ANDROID__) +#define MI_DEFAULT_ALLOW_THP 0 +#else +#define MI_DEFAULT_ALLOW_THP 1 +#endif +#endif + +// Static options +static mi_option_desc_t mi_options[_mi_option_last] = +{ + // stable options +#if MI_DEBUG || defined(MI_SHOW_ERRORS) + { 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) }, +#else + { 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) }, +#endif + { 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) }, + { MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) }, + + // some of the following options are experimental and not all combinations are allowed. + { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) }, + { MI_DEFAULT_ARENA_EAGER_COMMIT, + MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, + MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, + MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + {-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N + { MI_DEFAULT_RESERVE_OS_MEMORY, + MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread + { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free + { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) }, + { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit) + { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) }, + { 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds + { 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. + { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) + { 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose + { 32, MI_OPTION_UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 32, MI_OPTION_UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output + { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. + { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) + { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. + { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) }, + { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects + { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects + { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + { MI_DEFAULT_GUARDED_SAMPLE_RATE, + MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) + { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)}, + { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls + { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps + { 2, MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues + { 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate + { 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits + { MI_DEFAULT_PAGEMAP_COMMIT, + MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? + { 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux)) + { MI_DEFAULT_PAGE_MAX_RECLAIM, + MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class + { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM, + MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class + { MI_DEFAULT_ALLOW_THP, + MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. + { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`). + { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, + MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). + { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation +}; + +static void mi_option_init(mi_option_desc_t* desc); + +static bool mi_option_has_size_in_kib(mi_option_t option) { + return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || + option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size); +} + +void _mi_options_init(void) { + // called on process load + for(int i = 0; i < _mi_option_last; i++ ) { + mi_option_t option = (mi_option_t)i; + long l = mi_option_get(option); MI_UNUSED(l); // initialize + } + mi_max_error_count = mi_option_get(mi_option_max_errors); + mi_max_warning_count = mi_option_get(mi_option_max_warnings); + #if MI_GUARDED + if (mi_option_get(mi_option_guarded_sample_rate) > 0) { + if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { + mi_option_disable(mi_option_allow_large_os_pages); + _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); + } + } + #endif +} + +// called at actual process load, it should be safe to print now +void _mi_options_post_init(void) { + mi_add_stderr_output(); // now it safe to use stderr for output + if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); } +} + +#define mi_stringifyx(str) #str // and stringify +#define mi_stringify(str) mi_stringifyx(str) // expand + +mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept +{ + // show version + const int vermajor = MI_MALLOC_VERSION/10000; + const int verminor = (MI_MALLOC_VERSION%10000)/100; + const int verpatch = (MI_MALLOC_VERSION%100); + _mi_fprintf(out, arg, "v%i.%i.%i%s%s\n", vermajor, verminor, verpatch, + #if defined(MI_CMAKE_BUILD_TYPE) + ", " mi_stringify(MI_CMAKE_BUILD_TYPE) + #else + "" + #endif + , + #if defined(MI_GIT_DESCRIBE) + ", git " mi_stringify(MI_GIT_DESCRIBE) + #else + "" + #endif + ); + + // show options + for (int i = 0; i < _mi_option_last; i++) { + mi_option_t option = (mi_option_t)i; + long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize + mi_option_desc_t* desc = &mi_options[option]; + _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : "")); + } + + // show build configuration + _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); + _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); + _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + #if MI_GUARDED + _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled"); + #endif + #if MI_TSAN + _mi_fprintf(out, arg, "thread santizer enabled\n"); + #endif + #if MI_PAGE_META_IS_ALIGNED + _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_FREE_IS_CHECKED + _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + #endif + #if MI_ENCODE_FREELIST + _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + #endif +} + +mi_decl_export void mi_options_print(void) mi_attr_noexcept { + mi_options_print_out(NULL, NULL); +} + +long _mi_option_get_fast(mi_option_t option) { + mi_assert(option >= 0 && option < _mi_option_last); + mi_option_desc_t* desc = &mi_options[option]; + mi_assert(desc->option == option); // index should match the option + //mi_assert(desc->init != MI_OPTION_UNINIT); + return desc->value; +} + + +mi_decl_nodiscard long mi_option_get(mi_option_t option) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return 0; + mi_option_desc_t* desc = &mi_options[option]; + mi_assert(desc->option == option); // index should match the option + if mi_unlikely(desc->init == MI_OPTION_UNINIT) { + mi_option_init(desc); + } + return desc->value; +} + +mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) { + long x = mi_option_get(option); + return (x < min ? min : (x > max ? max : x)); +} + +mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { + const long x = mi_option_get(option); + size_t size = (x < 0 ? 0 : (size_t)x); + if (mi_option_has_size_in_kib(option)) { + if (mi_mul_overflow(size, MI_KiB, &size)) { + size = MI_MAX_ALLOC_SIZE; + } + } + return size; +} + +void mi_option_set(mi_option_t option, long value) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return; + mi_option_desc_t* desc = &mi_options[option]; + mi_assert(desc->option == option); // index should match the option + desc->value = value; + desc->init = MI_OPTION_INITIALIZED; + // ensure min/max range; be careful to not recurse. + if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) { + mi_option_set(mi_option_guarded_max, value); + } + else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) { + mi_option_set(mi_option_guarded_min, value); + } +} + +void mi_option_set_default(mi_option_t option, long value) { + mi_assert(option >= 0 && option < _mi_option_last); + if (option < 0 || option >= _mi_option_last) return; + mi_option_desc_t* desc = &mi_options[option]; + if (desc->init != MI_OPTION_INITIALIZED) { + desc->value = value; + } +} + +mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) { + return (mi_option_get(option) != 0); +} + +void mi_option_set_enabled(mi_option_t option, bool enable) { + mi_option_set(option, (enable ? 1 : 0)); +} + +void mi_option_set_enabled_default(mi_option_t option, bool enable) { + mi_option_set_default(option, (enable ? 1 : 0)); +} + +void mi_option_enable(mi_option_t option) { + mi_option_set_enabled(option,true); +} + +void mi_option_disable(mi_option_t option) { + mi_option_set_enabled(option,false); +} + +static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { + MI_UNUSED(arg); + if (msg != NULL && msg[0] != 0) { + _mi_prim_out_stderr(msg); + } +} + +// Since an output function can be registered earliest in the `main` +// function we also buffer output that happens earlier. When +// an output function is registered it is called immediately with +// the output up to that point. +#ifndef MI_MAX_DELAY_OUTPUT +#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024)) +#endif +static char out_buf[MI_MAX_DELAY_OUTPUT+1]; +static _Atomic(size_t) out_len; +static mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER; + +static void mi_cdecl mi_out_buf(const char* msg, void* arg) { + MI_UNUSED(arg); + if (msg==NULL) return; + if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return; + size_t n = _mi_strlen(msg); + if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return; + // copy msg into the buffer + mi_lock(&out_buf_lock) { + const size_t start = mi_atomic_add_acq_rel(&out_len, n); + if (start < MI_MAX_DELAY_OUTPUT) { + // check bound + if (start+n >= MI_MAX_DELAY_OUTPUT) { + n = MI_MAX_DELAY_OUTPUT-start-1; + } + _mi_memcpy(&out_buf[start], msg, n); + } + } +} + +static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { + if (out==NULL) return; + // claim (if `no_more_buf == true`, no more output will be added after this point) + mi_lock(&out_buf_lock) { + size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); + // and output the current contents + if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; + out_buf[count] = 0; + out(out_buf,arg); + if (!no_more_buf) { + out_buf[count] = '\n'; // if continue with the buffer, insert a newline + } + } +} + + +// Once this module is loaded, switch to this routine +// which outputs to stderr and the delayed output buffer. +static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) { + mi_out_stderr(msg,arg); + mi_out_buf(msg,arg); +} + + + +// -------------------------------------------------------- +// Default output handler +// -------------------------------------------------------- + +// The program should only install a single output handler from a single thread +// since otherwise the argument and output function may not match. +static _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers) +static _Atomic(void*) mi_out_arg; // = NULL + +static mi_output_fun* mi_out_get_default(void** parg) { + mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default); + if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); } + return (out == NULL ? &mi_out_buf : out); +} + +void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer + mi_atomic_store_ptr_release(void,&mi_out_arg, arg); + if (out!=NULL) { mi_out_buf_flush(out,true,arg); } // output all the delayed output now +} + +// add stderr to the delayed output after the module is loaded +static void mi_add_stderr_output(void) { + mi_assert_internal(mi_out_default == NULL); + mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr + mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output + mi_atomic_store_ptr_release(void,&mi_out_arg,NULL); +} + +// -------------------------------------------------------- +// Messages, all end up calling `_mi_fputs`. +// -------------------------------------------------------- +static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings + +static mi_decl_noinline bool mi_recurse_enter_prim(void) { + return true; +} + +static mi_decl_noinline void mi_recurse_exit_prim(void) { + /* nothing */ +} + +static bool mi_recurse_enter(void) { + #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD) + if (_mi_preloading()) return false; + #endif + return mi_recurse_enter_prim(); +} + +static void mi_recurse_exit(void) { + #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD) + if (_mi_preloading()) return; + #endif + mi_recurse_exit_prim(); +} + +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) { + if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // todo: use mi_out_stderr for stderr? + if (!mi_recurse_enter()) return; + out = mi_out_get_default(&arg); + if (prefix != NULL) out(prefix, arg); + out(message, arg); + mi_recurse_exit(); + } + else { + if (prefix != NULL) out(prefix, arg); + out(message, arg); + } +} + +// Define our own limited `fprintf` that avoids memory allocation. +// We do this using `_mi_vsnprintf` with a limited buffer. +static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) { + char buf[992]; + if (fmt==NULL) return; + if (!mi_recurse_enter()) return; + _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args); + mi_recurse_exit(); + _mi_fputs(out,arg,prefix,buf); +} + +void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { + va_list args; + va_start(args,fmt); + mi_vfprintf(out,arg,NULL,fmt,args); + va_end(args); +} + +static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { + if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) { + char tprefix[64]; + _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id()); + mi_vfprintf(out, arg, tprefix, fmt, args); + } + else { + mi_vfprintf(out, arg, prefix, fmt, args); + } +} + +void _mi_raw_message(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + mi_vfprintf(NULL, NULL, NULL, fmt, args); + va_end(args); +} + +void _mi_trace_message(const char* fmt, ...) { + if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher + va_list args; + va_start(args, fmt); + mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args); + va_end(args); +} + +void _mi_verbose_message(const char* fmt, ...) { + if (!mi_option_is_enabled(mi_option_verbose)) return; + va_list args; + va_start(args,fmt); + mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args); + va_end(args); +} + +static void mi_show_error_message(const char* fmt, va_list args) { + if (!mi_option_is_enabled(mi_option_verbose)) { + if (!mi_option_is_enabled(mi_option_show_errors)) return; + if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return; + } + mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args); +} + +void _mi_warning_message(const char* fmt, ...) { + if (!mi_option_is_enabled(mi_option_verbose)) { + if (!mi_option_is_enabled(mi_option_show_errors)) return; + if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return; + } + va_list args; + va_start(args,fmt); + mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args); + va_end(args); +} + + +#if MI_DEBUG +mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept { + _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); + abort(); +} +#endif + +// -------------------------------------------------------- +// Errors +// -------------------------------------------------------- + +static mi_error_fun* volatile mi_error_handler; // = NULL +static _Atomic(void*) mi_error_arg; // = NULL + +static void mi_error_default(int err) { + MI_UNUSED(err); + #if (MI_DEBUG>0) + if (err==EFAULT) { + #ifdef _MSC_VER + __debugbreak(); + #endif + abort(); + } + #endif + #if (MI_SECURE>0) + if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) + abort(); + } + #endif + #if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW || err==EINVAL) { // abort on memory allocation fails in xmalloc mode + abort(); + } + #endif + if (errno==0) { + errno = (err==EINVAL ? EINVAL : ENOMEM /* compatibility */ ); + } +} + +void mi_register_error(mi_error_fun* fun, void* arg) { + mi_error_handler = fun; // can be NULL + mi_atomic_store_ptr_release(void,&mi_error_arg, arg); +} + +void _mi_error_message(int err, const char* fmt, ...) { + // show detailed error message + va_list args; + va_start(args, fmt); + mi_show_error_message(fmt, args); + va_end(args); + // and call the error handler which may abort (or return normally, potentially setting errno) + if (mi_error_handler != NULL) { + mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg)); + } + else { + mi_error_default(err); + } +} + +mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + return NULL; +} + +// -------------------------------------------------------- +// Initialize options by checking the environment +// -------------------------------------------------------- + +// TODO: implement ourselves to reduce dependencies on the C runtime +#include // strtol + +static void mi_option_init(mi_option_desc_t* desc) { + // Read option value from the environment + char s[64 + 1]; + char buf[64+1]; + _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + _mi_strlcat(buf, desc->name, sizeof(buf)); + int err = _mi_getenv(buf, s, sizeof(s)); + if (err==ENOENT && desc->legacy_name != NULL) { + _mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + _mi_strlcat(buf, desc->legacy_name, sizeof(buf)); + err = _mi_getenv(buf, s, sizeof(s)); + if (err==0) { + _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name); + } + } + + if (err==0) { + size_t len = _mi_strnlen(s, sizeof(buf) - 1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(s[i]); + } + buf[len] = 0; + if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) { + desc->value = 1; + desc->init = MI_OPTION_INITIALIZED; + } + else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) { + desc->value = 0; + desc->init = MI_OPTION_INITIALIZED; + } + else { + char* end = buf; + errno = 0; + long value = strtol(buf, &end, 10); + if (errno==0 && mi_option_has_size_in_kib(desc->option)) { + // this option is interpreted in KiB to prevent overflow of `long` for large allocations + // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) + size_t size = (value < 0 ? 0 : (size_t)value); + bool overflow = false; + if (*end == 'K') { end++; } + else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; } + else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; } + else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; } + else { size = (size + MI_KiB - 1) / MI_KiB; } + if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB + else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb + if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } + value = (size > LONG_MAX ? LONG_MAX : (long)size); + } + if (errno==0 && *end == 0) { + mi_option_set(desc->option, value); + } + else { + // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose. + desc->init = MI_OPTION_DEFAULTED; + if (desc->option == mi_option_verbose && desc->value == 0) { + // if the 'mimalloc_verbose' env var has a bogus value we'd never know + // (since the value defaults to 'off') so in that case briefly enable verbose + desc->value = 1; + _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); + desc->value = 0; + } + else { + _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); + } + } + } + mi_assert_internal(desc->init != MI_OPTION_UNINIT); + } + else if (err==ENOENT) { + desc->init = MI_OPTION_DEFAULTED; + } + // and on another error, keep unitialized to try again (can happen during preloading if getenv is not available) +} diff --git a/csrc/mimalloc/src/os.c b/csrc/mimalloc/src/os.c new file mode 100644 index 000000000..c74bc736a --- /dev/null +++ b/csrc/mimalloc/src/os.c @@ -0,0 +1,944 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" // _mi_theap_default for random + +/* ----------------------------------------------------------- + Initialization. +----------------------------------------------------------- */ +#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB +#if MI_INTPTR_SIZE < 8 +#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB +#else +#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB +#endif +#endif + +static mi_os_mem_config_t mi_os_mem_config = { + 4096, // page size + 0, // large page size (usually 2MiB) + 4096, // allocation granularity + MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB, + MI_MAX_VABITS, // in `bits.h` + true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) + false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + true, // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) + false // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages) +}; + +bool _mi_os_has_overcommit(void) { + return mi_os_mem_config.has_overcommit; +} + +bool _mi_os_has_virtual_reserve(void) { + return mi_os_mem_config.has_virtual_reserve; +} + + +// OS (small) page size +size_t _mi_os_page_size(void) { + return mi_os_mem_config.page_size; +} + +// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) +size_t _mi_os_large_page_size(void) { + return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size()); +} + +// minimal purge size. Can be larger than the page size if transparent huge pages are enabled. +size_t _mi_os_minimal_purge_size(void) { + size_t minsize = mi_option_get_size(mi_option_minimal_purge_size); + if (minsize != 0) { + return _mi_align_up(minsize, _mi_os_page_size()); + } + else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { + return _mi_os_large_page_size(); + } + else { + return _mi_os_page_size(); + } +} + +size_t _mi_os_guard_page_size(void) { + const size_t gsize = _mi_os_page_size(); + mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166 + return gsize; +} + +size_t _mi_os_virtual_address_bits(void) { + const size_t vbits = mi_os_mem_config.virtual_address_bits; + mi_assert(vbits <= MI_MAX_VABITS); + return vbits; +} + +bool _mi_os_canuse_large_page(size_t size, size_t alignment) { + // if we have access, check the size and alignment requirements + if (mi_os_mem_config.large_page_size == 0) return false; + return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); +} + +// round to a good OS allocation size (bounded by max 12.5% waste) +size_t _mi_os_good_alloc_size(size_t size) { + size_t align_size; + if (size < 512*MI_KiB) align_size = _mi_os_page_size(); + else if (size < 2*MI_MiB) align_size = 64*MI_KiB; + else if (size < 8*MI_MiB) align_size = 256*MI_KiB; + else if (size < 32*MI_MiB) align_size = 1*MI_MiB; + else align_size = 4*MI_MiB; + if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? + return _mi_align_up(size, align_size); +} + +void _mi_os_init(void) { + _mi_prim_mem_init(&mi_os_mem_config); +} + + +/* ----------------------------------------------------------- + Util +-------------------------------------------------------------- */ +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); +bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero); + +// On systems with enough virtual address bits, we can do efficient aligned allocation by using +// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address +// space (64TiB) we use this technique. (but see issue #939) +#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) + +// Return a `try_alignment` aligned address that is probably available. +// If this returns NULL, the OS will determine the address but on some OS's that may not be +// properly aligned which can be more costly as it needs to be adjusted afterwards. +// In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization; +// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses +// in the middle of the 2TiB - 6TiB address range (see issue #372)) + +#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) +#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) +{ + static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 + + // todo: perhaps only do alignment hints if THP is enabled? + if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; + if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space + + size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. + req_size += (try_alignment - 1); // ensure we can align in the requested size + req_size = _mi_align_up(req_size, _mi_os_large_page_size()); + #if (MI_SECURE>=1) + if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 + #endif + + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); + if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize + uintptr_t init = MI_HINT_BASE; + #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode + mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) + if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point + const uintptr_t r = _mi_theap_random_next(theap); + init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB + #endif + uintptr_t expected = hint + req_size; + mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); + hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + if (hint==0) return NULL; + } + const uintptr_t hint_align = _mi_align_up(hint,try_alignment); + mi_assert_internal(hint_align + sze < hint + req_size); + return (void*)hint_align; +} +#else +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { + MI_UNUSED(try_alignment); MI_UNUSED(size); + return NULL; +} +#endif + + +/* ----------------------------------------------------------- + Guard page allocation +----------------------------------------------------------- */ + +// In secure mode, return the size of a guard page, otherwise 0 +size_t _mi_os_secure_guard_page_size(void) { + #if MI_SECURE > 0 + return _mi_os_guard_page_size(); + #else + return 0; + #endif +} + +// In secure mode, try to decommit an area and output a warning if this fails. +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + if (addr == NULL) return true; + #if MI_SECURE > 0 + bool ok = false; + if (!memid.is_pinned) { + mi_arena_t* const arena = mi_memid_arena(memid); + if (arena != NULL && arena->commit_fun != NULL) { + ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); + } + else { + ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size()); + } + } + if (!ok) { + _mi_error_message(EINVAL, "secure level %d, but failed to commit guard page (at %p of size %zu)\n", MI_SECURE, addr, _mi_os_secure_guard_page_size()); + } + return ok; + #else + MI_UNUSED(subproc); MI_UNUSED(memid); + return true; + #endif +} + +// In secure mode, try to decommit an area and output a warning if this fails. +bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); +} + +// In secure mode, try to recommit an area +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + if (addr == NULL) return true; + #if MI_SECURE > 0 + if (!memid.is_pinned) { + mi_arena_t* const arena = mi_memid_arena(memid); + if (arena != NULL && arena->commit_fun != NULL) { + return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); + } + else { + return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL); + } + } + #else + MI_UNUSED(subproc); MI_UNUSED(memid); + #endif + return true; +} + +// In secure mode, try to recommit an area +bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + return _mi_os_secure_guard_page_reset_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); +} + + +/* ----------------------------------------------------------- + Free memory +-------------------------------------------------------------- */ + +static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size); + +static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) { + mi_assert_internal(subproc!=NULL); + mi_assert_internal((size % _mi_os_page_size()) == 0); + if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr) + int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041) + if (err != 0) { + _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); + } + if (adjust) { + if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); } + mi_subproc_stat_adjust_decrease(subproc, reserved, size); + } + else { + if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); } + mi_subproc_stat_decrease(subproc, reserved, size); + } +} + +void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) { + if (mi_memkind_is_os(memid.memkind)) { + size_t csize = memid.mem.os.size; + if (csize==0) { csize = _mi_os_good_alloc_size(size); } + mi_assert_internal(csize >= size); + size_t commit_size = (still_committed ? csize : 0); + void* base = addr; + // different base? (due to alignment) + if (memid.mem.os.base != base) { + mi_assert(memid.mem.os.base <= addr); + base = memid.mem.os.base; + const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base; + if (memid.mem.os.size==0) { + csize += diff; + } + if (still_committed) { + commit_size -= diff; // the (addr-base) part was already un-committed + } + } + // free it + if (memid.memkind == MI_MEM_OS_HUGE) { + mi_assert(memid.is_pinned); + mi_os_free_huge_os_pages(subproc, base, csize); + } + else { + mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false); + } + } + else { + // nothing to do + mi_assert(memid.memkind < MI_MEM_OS); + } +} + +void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { + _mi_os_free_ex(subproc, p, size, true, memid); +} + + +/* ----------------------------------------------------------- + Primitive allocation from the OS. +-------------------------------------------------------------- */ + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +// Also `hint_addr` is a hint and may be ignored. +static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(is_zero != NULL); + mi_assert_internal(is_large != NULL); + if (size == 0) return NULL; + if (!commit) { allow_large = false; } + if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning + + // try to align along large OS page size for larger allocations + const size_t large_page_size = mi_os_mem_config.large_page_size; + if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) { + try_alignment = large_page_size; + } + + *is_zero = false; + void* p = NULL; + int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p); + if (err != 0) { + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large); + } + + mi_subproc_stat_counter_increase(subproc, mmap_calls, 1); + if (p != NULL) { + mi_subproc_stat_increase(subproc, reserved, size); + if (commit) { + mi_subproc_stat_increase(subproc, committed, size); + // seems needed for asan (or `mimalloc-test-api` fails) + #ifdef MI_TRACK_ASAN + if (*is_zero) { mi_track_mem_defined(p,size); } + else { mi_track_mem_undefined(p,size); } + #endif + } + } + return p; +} + +static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { + return mi_os_prim_alloc_at(subproc, NULL /* hint addr */, size, try_alignment, commit, allow_large, is_large, is_zero); +} + + +// Primitive aligned allocation from the OS. +// This function guarantees the allocated memory is aligned. +static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) { + mi_assert_internal(memid!=NULL); + mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + *memid = _mi_memid_none(); + if (!commit) allow_large = false; + if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; + size = _mi_align_up(size, _mi_os_page_size()); + + #if MI_INTPTR_SIZE >= 8 + const bool try_direct_alloc = true; + #else + // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. + const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4); + #endif + + bool os_is_large = false; + bool os_is_zero = false; + void* os_base = NULL; + size_t os_size = size; + void* p = NULL; + if (try_direct_alloc) { + p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero); + } + + // aligned already? + if (p != NULL && _mi_is_aligned(p,alignment)) { + os_base = p; + } + else { + // if not aligned, free it, overallocate, and unmap around it + #if !MI_TRACK_ASAN + if (try_direct_alloc) { + _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); + } + #endif + if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); } + if (size >= (SIZE_MAX - alignment)) return NULL; // overflow + const size_t over_size = size + alignment; + + if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block + // over-allocate uncommitted (virtual) memory + p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero); + if (p == NULL) return NULL; + + // set p to the aligned part in the full region + // note: Windows VirtualFree needs the actual base pointer + // this is handled though by having the `base` field in the memid + os_base = p; // remember the base + os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free? + p = _mi_align_up_ptr(p, alignment); + + // explicitly commit only the aligned part + if (commit) { + if (!_mi_os_commit(subproc, p, size, NULL)) { + mi_os_prim_free(subproc, os_base, over_size, 0, true); + return NULL; + } + } + } + else { // mmap can free inside an allocation + // overallocate... + // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though. + p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero); + if (p == NULL) return NULL; + + // and selectively unmap parts around the over-allocated area. + void* const aligned_p = _mi_align_up_ptr(p, alignment); + const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; + const size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + const size_t post_size = over_size - pre_size - mid_size; + mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); + if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); } + if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); } + // we can return the aligned pointer on `mmap` systems + p = aligned_p; + os_base = aligned_p; // since we freed the pre part, `*base == p`. + os_size = mid_size; + } + } + + mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment)); + mi_assert_internal(os_base <= p && size <= os_size); + *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large); + return p; +} + + +/* ----------------------------------------------------------- + OS API: alloc and alloc_aligned +----------------------------------------------------------- */ + +void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) { + *memid = _mi_memid_none(); + if (size == 0) return NULL; + size = _mi_os_good_alloc_size(size); + bool os_is_large = false; + bool os_is_zero = false; + void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero); + if (p == NULL) return NULL; + + *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(memid->initially_committed); + return p; +} + +void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) +{ + MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings + *memid = _mi_memid_none(); + if (size == 0) return NULL; + size = _mi_os_good_alloc_size(size); + alignment = _mi_align_up(alignment, _mi_os_page_size()); + + void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid ); + if (p == NULL) return NULL; + + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(_mi_is_aligned(p,alignment)); + if (commit) { mi_assert_internal(memid->initially_committed); } + return p; +} + + +mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) { + if (p==NULL || size==0) return p; + // ensure committed + if (!memid->initially_committed) { + bool is_zero = false; + if (!_mi_os_commit(subproc, p, size, &is_zero)) { + _mi_os_free(subproc, p, size, *memid); + return NULL; + } + memid->initially_committed = true; + } + // ensure zero'd + if (memid->initially_zero) return p; + _mi_memzero_aligned(p,size); + memid->initially_zero = true; + return p; +} + +void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) { + void* p = _mi_os_alloc(subproc, size,memid); + return mi_os_ensure_zero(subproc, p, size, memid); +} + +/* ----------------------------------------------------------- + OS aligned allocation with an offset. This is used + for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc + page where the object can be aligned at an offset from the start of the segment. + As we may need to overallocate, we need to free such pointers using `mi_free_aligned` + to use the actual start of the memory region. +----------------------------------------------------------- */ + +void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) { + mi_assert(offset <= size); + mi_assert((alignment % _mi_os_page_size()) == 0); + *memid = _mi_memid_none(); + if (offset > size) return NULL; + if (offset == 0) { + // regular aligned allocation + return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); + } + else { + // overallocate to align at an offset + const size_t extra = _mi_align_up(offset, alignment) - offset; + if (size >= SIZE_MAX - extra) return NULL; // too large + const size_t oversize = size + extra; + void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid); + if (start == NULL) return NULL; + + void* const p = (uint8_t*)start + extra; + mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); + // decommit the overallocation at the start + // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well? + if (commit && extra >= _mi_os_page_size()) { + _mi_os_decommit(subproc, start, extra); + } + return p; + } +} + +/* ----------------------------------------------------------- + OS memory API: reset, commit, decommit, protect, unprotect. +----------------------------------------------------------- */ + +// OS page align within a given area, either conservative (pages inside the area only), +// or not (straddling pages outside the area is possible) +static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { + mi_assert(addr != NULL && size > 0); + if (newsize != NULL) *newsize = 0; + if (size == 0 || addr == NULL) return NULL; + + // page align conservatively within the range, or liberally straddling pages outside the range + void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size()) + : _mi_align_down_ptr(addr, _mi_os_page_size())); + void* end = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) + : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); + ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; + if (diff <= 0) return NULL; + + mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size)); + if (newsize != NULL) *newsize = (size_t)diff; + return start; +} + +static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) { + return mi_os_page_align_areax(true, addr, size, newsize); +} + +bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(size >= stat_already_committed); + if (is_zero != NULL) { *is_zero = false; } + mi_subproc_stat_counter_increase(subproc, commit_calls, 1); + + // page align range + size_t csize; + void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize); + if (csize == 0) return true; + + // commit + bool os_is_zero = false; + int err = _mi_prim_commit(start, csize, &os_is_zero); + if (err != 0) { + _mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + return false; + } + if (os_is_zero && is_zero != NULL) { + *is_zero = true; + mi_assert_expensive(mi_mem_is_zero(start, csize)); + } + // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails) + #ifdef MI_TRACK_ASAN + if (os_is_zero) { mi_track_mem_defined(start,csize); } + else { mi_track_mem_undefined(start,csize); } + #endif + mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit + return true; +} + +bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) { + return _mi_os_commit_ex(subproc, addr, size, is_zero, 0); +} + +static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) { + mi_assert_internal(needs_recommit!=NULL); + + // page align + size_t csize; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return true; + + // decommit + *needs_recommit = true; + int err = _mi_prim_decommit(start,csize,needs_recommit); + if (err != 0) { + _mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + } + else if (*needs_recommit) { + mi_subproc_stat_decrease(subproc, committed, stat_size); + } + mi_assert_internal(err == 0); + return (err == 0); +} + +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) { + bool needs_recommit; + return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size); +} + + +// Signal to the OS that the address range is no longer in use +// but may be used later again. This will release physical memory +// pages and reduce swapping while keeping the memory committed. +// We page align to a conservative area inside the range to reset. +bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) { + // page align conservatively within the range + size_t csize; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) + mi_subproc_stat_counter_increase(subproc, reset, csize); + mi_subproc_stat_counter_increase(subproc, reset_calls, 1); + + #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN + memset(start, 0, csize); // pretend it is eagerly reset + #endif + + int err = _mi_prim_reset(start, csize); + if (err != 0) { + _mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + } + return (err == 0); +} + + +void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) { + MI_UNUSED(subproc); + // page align conservatively within the range + size_t csize = 0; + void* const start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return; + const int err = _mi_prim_reuse(start, csize); + if (err != 0) { + _mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); + } +} + +// either resets or decommits memory, returns true if the memory needs +// to be recommitted if it is to be re-used later on. +bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg) +{ + if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? + mi_subproc_stat_counter_increase(subproc, purge_calls, 1); + mi_subproc_stat_counter_increase(subproc, purged, size); + + if (commit_fun != NULL) { + bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg); + return decommitted; // needs_recommit? + } + else if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? + !_mi_preloading()) // don't decommit during preloading (unsafe) + { + bool needs_recommit = true; + mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size); + return needs_recommit; + } + else { + if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory) + _mi_os_reset(subproc, p, size); + } + return false; // needs no recommit + } +} + +// either resets or decommits memory, returns true if the memory needs +// to be recommitted if it is to be re-used later on. +bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) { + return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL); +} + + +// Protect a region in memory to be not accessible. +static bool mi_os_protectx(void* addr, size_t size, bool protect) { + // page align conservatively within the range + size_t csize = 0; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return false; + /* + if (_mi_os_is_huge_reserved(addr)) { + _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); + } + */ + int err = _mi_prim_protect(start,csize,protect); + if (err != 0) { + _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); + } + return (err == 0); +} + +bool _mi_os_protect(void* addr, size_t size) { + return mi_os_protectx(addr, size, true); +} + +bool _mi_os_unprotect(void* addr, size_t size) { + return mi_os_protectx(addr, size, false); +} + + + +/* ---------------------------------------------------------------------------- +Support for allocating huge OS pages (1Gib) that are reserved up-front +and possibly associated with a specific NUMA node. (use `numa_node>=0`) +-----------------------------------------------------------------------------*/ +#define MI_HUGE_OS_PAGE_SIZE (MI_GiB) + + +#if (MI_INTPTR_SIZE >= 8) +// To ensure proper alignment, use our own area for huge OS pages +static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 + +// Claim an aligned address range for huge pages +static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { + if (total_size != NULL) *total_size = 0; + size_t size = 0; + if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) { + _mi_warning_message("too many huge pages requested: %zu\n", pages); + return NULL; + } + + uintptr_t start = 0; + uintptr_t end = 0; + uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start); + do { + start = huge_start; + if (start == 0) { + // Initialize the start address after the 32TiB area + start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint) + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode + mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) + if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness? + const uintptr_t r = _mi_theap_random_next(theap); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + } + else { + _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, (void*)start); + } + #endif + } + end = start + size; + } while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end)); + + if (total_size != NULL) *total_size = size; + return (uint8_t*)start; +} +#else +static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { + MI_UNUSED(pages); + if (total_size != NULL) *total_size = 0; + return NULL; +} +#endif + +// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages +void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) { + *memid = _mi_memid_none(); + if (psize != NULL) *psize = 0; + if (pages_reserved != NULL) *pages_reserved = 0; + size_t size = 0; + uint8_t* const start = mi_os_claim_huge_pages(pages, &size); + if (start == NULL) return NULL; // or 32-bit systems + + // Allocate one page at the time but try to place them contiguously + // We allocate one page at the time to be able to abort if it takes too long + // or to at least allocate as many as available on the system. + mi_msecs_t start_t = _mi_clock_start(); + size_t page = 0; + bool all_zero = true; + while (page < pages) { + // allocate a page + bool is_zero = false; + void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE); + void* p = NULL; + int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p); + if (!is_zero) { all_zero = false; } + if (err != 0) { + _mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); + break; + } + + // Did we succeed at a contiguous address? + if (p != addr) { + // no success, issue a warning and break + if (p != NULL) { + _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); + mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */); + } + break; + } + + // success, record it + page++; // increase before timeout check (see issue #711) + mi_subproc_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE); + mi_subproc_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE); + + // check for timeout + if (max_msecs > 0) { + mi_msecs_t elapsed = _mi_clock_end(start_t); + if (page >= 1) { + mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages); + if (estimate > 2*max_msecs) { // seems like we are going to timeout, break + elapsed = max_msecs + 1; + } + } + if (elapsed > max_msecs) { + _mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page); + break; + } + } + } + const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE; + mi_assert_internal(allocated <= size); + if (pages_reserved != NULL) { *pages_reserved = page; } + if (psize != NULL) { *psize = allocated; } + if (page != 0) { + mi_assert(start != NULL); + *memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */); + memid->memkind = MI_MEM_OS_HUGE; + mi_assert(memid->is_pinned); + #ifdef MI_TRACK_ASAN + if (all_zero) { mi_track_mem_defined(start,allocated); } + #endif + } + return (page == 0 ? NULL : start); +} + +// free every huge page in a range individually (as we allocated per page) +// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. +static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) { + if (p==NULL || size==0) return; + uint8_t* base = (uint8_t*)p; + while (size >= MI_HUGE_OS_PAGE_SIZE) { + mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */); + size -= MI_HUGE_OS_PAGE_SIZE; + base += MI_HUGE_OS_PAGE_SIZE; + } +} + + +/* ---------------------------------------------------------------------------- +Support NUMA aware allocation +-----------------------------------------------------------------------------*/ + +static _Atomic(size_t) mi_numa_node_count; // = 0 // cache the node count + +int _mi_os_numa_node_count(void) { + size_t count = mi_atomic_load_acquire(&mi_numa_node_count); + if mi_unlikely(count == 0) { + long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? + if (ncount > 0 && ncount < INT_MAX) { + count = (size_t)ncount; + } + else { + const size_t n = _mi_prim_numa_node_count(); // or detect dynamically + if (n == 0 || n > INT_MAX) { count = 1; } + else { count = n; } + } + mi_atomic_store_release(&mi_numa_node_count, count); // save it + if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); } + } + mi_assert_internal(count > 0 && count <= INT_MAX); + return (int)count; +} + +static int mi_os_numa_node_get(void) { + int numa_count = _mi_os_numa_node_count(); + if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 + // never more than the node count and >= 0 + const size_t n = _mi_prim_numa_node(); + int numa_node = (n < INT_MAX ? (int)n : 0); + if (numa_node >= numa_count) { numa_node = numa_node % numa_count; } + return numa_node; +} + +int _mi_os_numa_node(void) { + if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) { + return 0; + } + else { + return mi_os_numa_node_get(); + } +} + + +/* ---------------------------------------------------------------------------- + Public API +-----------------------------------------------------------------------------*/ +#if 0 +mi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) { + return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size); +} + +static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) { + mi_memid_t memid = _mi_memid_none(); + void* p = _mi_os_alloc_aligned(size, alignment, commit, allow_large, &memid); + if (p == NULL) return p; + if (is_committed != NULL) { *is_committed = memid.initially_committed; } + if (is_pinned != NULL) { *is_pinned = memid.is_pinned; } + if (base != NULL) { *base = memid.mem.os.base; } + if (full_size != NULL) { *full_size = memid.mem.os.size; } + if (!memid.initially_zero && memid.initially_committed) { + _mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size); + } + return p; +} + +mi_decl_export void* mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, void** base, size_t* full_size) { + return mi_os_alloc_aligned_ex(size, alignment, commit, false, NULL, NULL, base, full_size); +} + +mi_decl_export void* mi_os_alloc_aligned_allow_large(size_t size, size_t alignment, bool commit, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) { + return mi_os_alloc_aligned_ex(size, alignment, commit, true, is_committed, is_pinned, base, full_size); +} + +mi_decl_export void mi_os_free(void* p, size_t size) { + if (p==NULL || size == 0) return; + mi_memid_t memid = _mi_memid_create_os(p, size, true, false, false); + _mi_os_free(p, size, memid); +} + +mi_decl_export void mi_os_commit(void* p, size_t size) { + _mi_os_commit(p, size, NULL); +} + +mi_decl_export void mi_os_decommit(void* p, size_t size) { + _mi_os_decommit(p, size); +} +#endif diff --git a/csrc/mimalloc/src/page-map.c b/csrc/mimalloc/src/page-map.c new file mode 100644 index 000000000..949e37ac3 --- /dev/null +++ b/csrc/mimalloc/src/page-map.c @@ -0,0 +1,517 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2023-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "bitmap.h" + +static void mi_page_map_cannot_commit(void) { + _mi_warning_message("unable to commit the allocation page-map on-demand\n" ); +} + +#if MI_PAGE_MAP_FLAT + +// The page-map contains a byte for each 64kb slice in the address space. +// For an address `a` where `ofs = _mi_page_map[a >> 16]`: +// 0 = unused +// 1 = the slice at `a & ~0xFFFF` is a mimalloc page. +// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`. +// +// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map. +// A full 256 TiB address space (48 bit) needs a 4 GiB page map. +// A full 4 GiB address space (32 bit) needs only a 64 KiB page map. + +// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) +static uint8_t mi_page_map_empty[1] = { 1 }; // _mi_ptr_page(NULL) == NULL + +mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty; +mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL; +static mi_memid_t mi_page_map_memid; + +#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE +static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries + +mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count); + +bool _mi_page_map_init(void) { + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); + if (vbits == 0) { + vbits = _mi_os_virtual_address_bits(); + #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB + if (vbits >= 48) { vbits = 47; } + #endif + } + if (vbits < MI_ARENA_SLICE_SHIFT) { + vbits = MI_ARENA_SLICE_SHIFT; + } + if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr + vbits = MI_MIN_VABITS; + } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } + + // Allocate the page map and commit bits + mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); + const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT)); + const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems? + const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); + const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL)); + const size_t reserve_size = bitmap_size + page_map_size; + mi_subproc_t* const subproc = _mi_subproc_main(); + uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); + if (base==NULL) { + _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); + return false; + } + if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { + _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); + _mi_memzero_aligned(base, reserve_size); + } + if (bitmap_size > 0) { + mi_page_map_commit = (mi_bitmap_t*)base; + if (!_mi_os_commit(subproc, mi_page_map_commit, bitmap_size, NULL)) { + mi_page_map_cannot_commit(); + return false; + } + mi_bitmap_init(mi_page_map_commit, commit_bits, true); + } + mi_atomic_store_ptr_release(uint8_t,&_mi_page_map, base + bitmap_size); + + // commit the first part so NULL pointers get resolved without an access violation + if (!commit) { + if (!mi_page_map_ensure_committed(0, 1)) { + mi_page_map_cannot_commit(); + return false; + } + } + mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so _mi_ptr_page(NULL) == NULL + mi_assert_internal(_mi_ptr_page(NULL)==NULL); + return true; +} + +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) != NULL); + if (mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) == NULL) return; + _mi_os_free_ex(_mi_subproc_main(), mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid); + mi_atomic_store_ptr_release(uint8_t, &_mi_page_map, NULL); + mi_page_map_commit = NULL; + mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL); + mi_page_map_memid = _mi_memid_none(); +} + + +static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { + // is the page map area that contains the page address committed? + // we always set the commit bits so we can track what ranges are in-use. + // we only actually commit if the map wasn't committed fully already. + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + if (mi_page_map_commit != NULL) { + const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; + const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; + for (size_t i = commit_idx; i <= commit_idx_hi; i++) { // per bit to avoid crossing over bitmap chunks + if (mi_bitmap_is_clear(mi_page_map_commit, i)) { + // this may race, in which case we do multiple commits (which is ok) + bool is_zero; + uint8_t* const start = page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); + const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; + if (!_mi_os_commit(_mi_subproc_main(), start, size, &is_zero)) { + mi_page_map_cannot_commit(); + return false; + } + if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); } + mi_bitmap_set(mi_page_map_commit, i); + } + } + } + #if MI_DEBUG > 0 + page_map[idx] = 0; + page_map[idx+slice_count-1] = 0; + #endif + return true; +} + + +static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) { + size_t page_size; + *page_start = mi_page_area(page, &page_size); + if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer + *slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks + return _mi_page_map_index(page); +} + +bool _mi_page_map_register(mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access! + uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) { + if (!_mi_page_map_init()) return false; + page_map = mi_atomic_load_ptr_acquire(uint8_t,&_mi_page_map); + } + mi_assert(page_map!=NULL); + uint8_t* page_start; + size_t slice_count; + const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); + + if (!mi_page_map_ensure_committed(idx, slice_count)) { + return false; + } + + // set the offsets + for (size_t i = 0; i < slice_count; i++) { + mi_assert_internal(i < 128); + page_map[idx + i] = (uint8_t)(i+1); + } + return true; +} + +void _mi_page_map_unregister(mi_page_t* page) { + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + mi_assert_internal(page_map != NULL); + if (page_map == NULL) return; + // get index and count + uint8_t* page_start; + size_t slice_count; + const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); + // unset the offsets + _mi_memzero(page_map + idx, slice_count); +} + +void _mi_page_map_unregister_range(void* start, size_t size) { + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + mi_assert_internal(page_map!=NULL); + if (page_map == NULL) return; + const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); + const uintptr_t index = _mi_page_map_index(start); + // todo: scan the commit bits and clear only those ranges? + if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total; + return; + } + _mi_memzero(&page_map[index], slice_count); +} + + +mi_page_t* _mi_safe_ptr_page(const void* p) { + if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; + const uintptr_t idx = _mi_page_map_index(p); + if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL; + const uintptr_t ofs = _mi_page_map_at(idx); + if mi_unlikely(ofs == 0) return NULL; + return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT); +} + +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { + return (_mi_safe_ptr_page(p) != NULL); +} + +#else + +// A 2-level page map +#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) + +// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) +static mi_page_map_t mi_page_map_empty = { + MI_ATOMIC_VAR_INIT(1), + sizeof(mi_page_map_t), + MI_MEMID_STATIC, + MI_LOCK_INITIALIZER, + { MI_ATOMIC_VAR_INIT(NULL) } +}; + +mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty); + +static size_t mi_page_map_count_of_size(size_t size) { + return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t)); +} + +// static void* mi_page_map_addr_of_index(size_t idx) { +// return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE); +// } + +mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) { + const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); + if mi_unlikely(required_idx >= reserved_count) { + mi_page_map_cannot_commit(); + return false; + } + size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE ); + if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; } + const size_t commit_count = mi_page_map_count_of_size(commit_size); + mi_assert_internal(commit_count > required_idx); + mi_assert_internal(commit_count <= reserved_count); + // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is). + bool is_zero; + if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) { + mi_page_map_cannot_commit(); + return false; + } + mi_assert_internal(is_zero || pmap->memid.initially_zero); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + return true; +} + +mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { + mi_assert_internal(submap!=NULL && *submap==NULL); + if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) { + if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) { + if (!mi_page_map_commit_entries(pmap,idx)) return false; + mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); + } + } + *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); + return true; +} + +// initialize the page map +static bool mi_page_map_init_once(void) { + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); + if (vbits == 0) { + vbits = _mi_os_virtual_address_bits(); + #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB + if (vbits >= 48) { vbits = 47; } + #endif + } + if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) { + vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT; + } + if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr + vbits = MI_MIN_VABITS; + } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } + + // Allocate the page map and commit bits + mi_assert(MI_MAX_VABITS >= vbits); + mi_assert(MI_MIN_VABITS <= vbits); + const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t os_page_size = _mi_os_page_size(); + const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + const size_t extra_reserve_size = reserve_size + submap_size; + const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits + mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_memid_t memid; + mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); + if mi_unlikely(pmap==NULL) { + _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB); + return false; + } + + // commit + size_t commit_count; + if (memid.initially_committed) { + if (!memid.initially_zero) { + _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); + _mi_memzero_aligned(pmap, extra_reserve_size); + memid.initially_zero = true; + } + commit_count = mi_page_map_count_of_size(reserve_size); + } + else { + // commit first entries up to MI_MIN_VABITS entries + const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); + mi_assert_internal(min_commit_size <= reserve_size); + bool is_zero; + if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { + mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); + return false; + }; + mi_assert_internal(is_zero || memid.initially_zero); + commit_count = mi_page_map_count_of_size(min_commit_size); + mi_assert_internal(commit_count >= min_commit_count); + } + + // ensure there is a submap for the NULL address + mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already + if (!memid.initially_committed) { + if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087) + mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); + return false; + } + } + if (!memid.initially_zero) { // initialize low addresses with NULL + _mi_memzero_aligned(sub0, submap_size); + } + + // initialize the fields + pmap->memid = memid; + pmap->reserved_size = reserve_size; + mi_lock_init(&pmap->lock); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap); + mi_assert_internal(_mi_ptr_page(NULL)==NULL); + return true; +} + +bool _mi_page_map_init(void) { + bool ok = true; + mi_atomic_do_once { + ok = mi_page_map_init_once(); + } + return ok; +} + +void _mi_page_map_unsafe_destroy(void) { + mi_page_map_t* const pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); + if (pmap == NULL || pmap == &mi_page_map_empty) return; + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_lock_done(&pmap->lock); + for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) + // free all sub-maps + mi_submap_t sub = _mi_page_map_at(pmap,idx); + if (sub != NULL) { + mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); + _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL); + } + } + _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); +} + +mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) { + // sub map not yet allocated, alloc now + mi_submap_t sub = NULL; + mi_lock(&pmap->lock) + { + sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload + if (sub==NULL) // not yet allocated by another thread? + { + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_memid_t memid; + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); + if (sub==NULL) { + _mi_warning_message("internal error: unable to extend the page map\n"); + } + else { + mi_submap_t expect = NULL; + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) { + // another thread already allocated it.. free and continue + _mi_os_free(subproc, sub, submap_size, memid); + sub = expect; + } + } + } + } + return sub; +} + +mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { + mi_assert_internal(submap!=NULL && *submap==NULL); + mi_submap_t sub = NULL; + if (!mi_page_map_ensure_committed(pmap, idx, &sub)) { + return false; + } + if mi_unlikely(sub==NULL) { + sub = mi_page_map_alloc_submap_at(pmap, idx); + if (sub==NULL) return false; // unable to allocate the submap.. + } + mi_assert_internal(sub!=NULL); + *submap = sub; + return true; +} + +static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { + // is the page map area that contains the page address committed? + while (slice_count > 0) { + mi_submap_t sub = NULL; + if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) { + return false; + }; + mi_assert_internal(sub!=NULL); + // set the offsets for the page + while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) { + sub[sub_idx] = page; + slice_count--; + sub_idx++; + } + idx++; // potentially wrap around to the next idx + sub_idx = 0; + } + return true; +} + +static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { + if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) { + // failed to commit, call again to reset the page pointer if needed + if (page!=NULL) { + mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count); + } + return false; + } + return true; +} + +static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) { + size_t page_size; + uint8_t* page_start = mi_page_area(page, &page_size); + if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer + *slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks + return _mi_page_map_index(page_start, sub_idx); +} + +bool _mi_page_map_register(mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access! + if mi_unlikely(pmap == NULL) { + if (!_mi_page_map_init()) return false; + pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map); + } + mi_assert(pmap!=NULL); + size_t slice_count; + size_t sub_idx; + const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); + return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count); +} + +void _mi_page_map_unregister(mi_page_t* page) { + mi_assert_internal(_mi_page_map != NULL); + mi_assert_internal(page != NULL); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* const pmap = _mi_page_map(); + // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`) + if mi_unlikely(pmap == NULL) return; + // get index and count + size_t slice_count; + size_t sub_idx; + const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); + // unset the offsets + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); +} + +void _mi_page_map_unregister_range(void* start, size_t size) { + mi_page_map_t* const pmap = _mi_page_map(); + if mi_unlikely(pmap == NULL) return; + const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); + size_t sub_idx; + const uintptr_t idx = _mi_page_map_index(start, &sub_idx); + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? +} + +// Return NULL for invalid pointers +mi_page_t* _mi_safe_ptr_page(const void* p) { + return _mi_checked_ptr_page(p); +} + +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { + return (_mi_safe_ptr_page(p) != NULL); +} + +#endif diff --git a/csrc/mimalloc/src/page-queue.c b/csrc/mimalloc/src/page-queue.c new file mode 100644 index 000000000..c10c09257 --- /dev/null +++ b/csrc/mimalloc/src/page-queue.c @@ -0,0 +1,423 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + Definition of page queues for each block size +----------------------------------------------------------- */ + +#ifndef MI_IN_PAGE_C +#error "this file should be included from 'page.c'" +// include to help an IDE +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#endif + +/* ----------------------------------------------------------- + Minimal alignment in machine words (i.e. `sizeof(void*)`) +----------------------------------------------------------- */ + +#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) + #error "define alignment for more than 4x word size for this platform" +#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) + #define MI_ALIGN4W // 4 machine words minimal alignment +#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) + #define MI_ALIGN2W // 2 machine words minimal alignment +#else + // ok, default alignment is 1 word +#endif + + +/* ----------------------------------------------------------- + Queue query +----------------------------------------------------------- */ + + +static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t))); +} + +static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t)))); +} + +static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { + return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE); +} + +static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { + return pq->count; +} + +/* ----------------------------------------------------------- + Bins +----------------------------------------------------------- */ + +// Return the bin for a given field size. +// Returns MI_BIN_HUGE if the size is too large. +// We use `wsize` for the size in "machine word sizes", +// i.e. byte size == `wsize*sizeof(void*)`. +static size_t mi_bin(size_t size) { + size_t wsize = _mi_wsize_from_size(size); +#if defined(MI_ALIGN4W) + if mi_likely(wsize <= 4) { + return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes + } +#elif defined(MI_ALIGN2W) + if mi_likely(wsize <= 8) { + return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes + } +#else + if mi_likely(wsize <= 8) { + return (wsize == 0 ? 1 : wsize); + } +#endif + else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) { + return MI_BIN_HUGE; + } + else { + #if defined(MI_ALIGN4W) + if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes + #endif + wsize--; + // find the highest bit + const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize)); // note: wsize != 0 + // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). + // - adjust with 3 because we use do not round the first 8 sizes + // which each get an exact bin + const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3; + mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE); + return bin; + } +} + + + +/* ----------------------------------------------------------- + Queue of pages with free blocks +----------------------------------------------------------- */ + +size_t _mi_bin(size_t size) { + return mi_bin(size); +} + +size_t _mi_bin_size(size_t bin) { + mi_assert_internal(bin <= MI_BIN_HUGE); + return _mi_theap_empty.pages[bin].block_size; +} + +// Good size for allocation +mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept { + if (size <= MI_LARGE_MAX_OBJ_SIZE - MI_PADDING_SIZE) { + return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE)); + } + else if (size <= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { + return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size()); + } + else { + return size; + } +} + +#if (MI_DEBUG>1) +static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_page_t* list = queue->first; + while (list != NULL) { + mi_assert_internal(list->next == NULL || list->next->prev == list); + mi_assert_internal(list->prev == NULL || list->prev->next == list); + if (list == page) break; + list = list->next; + } + return (list == page); +} + +#endif + +#if (MI_DEBUG>1) +static bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) { + return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]); +} +#endif + +bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { + MI_UNUSED_RELEASE(theap); + if (pq==NULL) return false; + size_t count = 0; MI_UNUSED_RELEASE(count); + mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev); + for (mi_page_t* page = pq->first; page != NULL; page = page->next) { + mi_assert_internal(page->prev == prev); + if (mi_page_is_in_full(page)) { + mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); + } + else if (mi_page_is_huge(page)) { + mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); + } + else { + mi_assert_internal(mi_page_block_size(page) == pq->block_size); + } + mi_assert_internal(page->theap == theap); + if (page->next == NULL) { + mi_assert_internal(pq->last == page); + } + count++; + prev = page; + } + mi_assert_internal(pq->count == count); + return true; +} + +static size_t mi_page_bin(const mi_page_t* page) { + const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); + mi_assert_internal(bin <= MI_BIN_FULL); + return bin; +} + +// returns the page bin without using MI_BIN_FULL for statistics +size_t _mi_page_stats_bin(const mi_page_t* page) { + const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))); + mi_assert_internal(bin <= MI_BIN_HUGE); + return bin; +} + +static mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) { + mi_assert_internal(theap!=NULL); + const size_t bin = mi_page_bin(page); + mi_page_queue_t* pq = &theap->pages[bin]; + mi_assert_internal((mi_page_block_size(page) == pq->block_size) || + (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) || + (mi_page_is_in_full(page) && mi_page_queue_is_full(pq))); + return pq; +} + +static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { + mi_theap_t* theap = mi_page_theap(page); + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); + mi_assert_expensive(mi_page_queue_contains(pq, page)); + return pq; +} + +// The current small page array is for efficiency and for each +// small size (up to 256) it points directly to the page for that +// size without having to compute the bin. This means when the +// current free page queue is updated for a small bin, we need to update a +// range of entries in `_mi_page_small_free`. +static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) { + mi_assert_internal(mi_theap_contains_queue(theap,pq)); + const size_t size = pq->block_size; + if (size > MI_SMALL_SIZE_MAX) return; + + mi_page_t* page = pq->first; + if (pq->first == NULL) page = _mi_page_empty_get(); + + // find index in the right direct page array + const size_t idx = _mi_wsize_from_size(size); + mi_page_t** const pages_free = theap->pages_free_direct; + if (pages_free[idx] == page) return; // already set + + // find start slot + size_t start; + if (idx<=1) { + start = 0; + } + else { + // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped + mi_assert_internal(pq > &theap->pages[0]); // since idx > 1 + size_t bin = mi_bin(size); + const mi_page_queue_t* prev = pq - 1; + while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { + prev--; + } + start = 1 + _mi_wsize_from_size(prev->block_size); + if (start > idx) start = idx; + } + + // set size range to the right page + mi_assert(start <= idx); + for (size_t sz = start; sz <= idx; sz++) { + pages_free[sz] = page; + } +} + +/* +static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { + return (queue->first == NULL); +} +*/ + +static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_expensive(mi_page_queue_contains(queue, page)); + mi_assert_internal(queue->count >= 1); + mi_assert_internal(mi_page_block_size(page) == queue->block_size || + (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || + (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + mi_theap_t* theap = mi_page_theap(page); + if (page->prev != NULL) page->prev->next = page->next; + if (page->next != NULL) page->next->prev = page->prev; + if (page == queue->last) queue->last = page->prev; + if (page == queue->first) { + queue->first = page->next; + // update first + mi_assert_internal(mi_theap_contains_queue(theap, queue)); + mi_theap_queue_first_update(theap,queue); + } + theap->page_count--; + queue->count--; + page->next = NULL; + page->prev = NULL; + mi_page_set_in_full(page,false); +} + + +static void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(!mi_page_queue_contains(queue, page)); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); + #endif + mi_assert_internal(mi_page_block_size(page) == queue->block_size || + (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || + (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + + mi_page_set_in_full(page, mi_page_queue_is_full(queue)); + + page->next = queue->first; + page->prev = NULL; + if (queue->first != NULL) { + mi_assert_internal(queue->first->prev == NULL); + queue->first->prev = page; + queue->first = page; + } + else { + queue->first = queue->last = page; + } + queue->count++; + + // update direct + mi_theap_queue_first_update(theap, queue); + theap->page_count++; +} + +static void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(!mi_page_queue_contains(queue, page)); + + mi_assert_internal(mi_page_block_size(page) == queue->block_size || + (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || + (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + + mi_page_set_in_full(page, mi_page_queue_is_full(queue)); + + page->prev = queue->last; + page->next = NULL; + if (queue->last != NULL) { + mi_assert_internal(queue->last->next == NULL); + queue->last->next = page; + queue->last = page; + } + else { + queue->first = queue->last = page; + } + queue->count++; + + // update direct + if (queue->first == page) { + mi_theap_queue_first_update(theap, queue); + } + theap->page_count++; +} + +static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->first == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push(theap, queue, page); + mi_assert_internal(queue->first == page); +} + +static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(from->count >= 1); + mi_assert_expensive(mi_page_queue_contains(from, page)); + mi_assert_expensive(!mi_page_queue_contains(to, page)); + const size_t bsize = mi_page_block_size(page); + MI_UNUSED(bsize); + mi_assert_internal((bsize == to->block_size && bsize == from->block_size) || + (bsize == to->block_size && mi_page_queue_is_full(from)) || + (bsize == from->block_size && mi_page_queue_is_full(to)) || + (mi_page_is_huge(page) && mi_page_queue_is_huge(to)) || + (mi_page_is_huge(page) && mi_page_queue_is_full(to))); + + mi_theap_t* theap = mi_page_theap(page); + + // delete from `from` + if (page->prev != NULL) page->prev->next = page->next; + if (page->next != NULL) page->next->prev = page->prev; + if (page == from->last) from->last = page->prev; + if (page == from->first) { + from->first = page->next; + // update first + mi_assert_internal(mi_theap_contains_queue(theap, from)); + mi_theap_queue_first_update(theap, from); + } + from->count--; + + // insert into `to` + to->count++; + if (enqueue_at_end) { + // enqueue at the end + page->prev = to->last; + page->next = NULL; + if (to->last != NULL) { + mi_assert_internal(theap == mi_page_theap(to->last)); + to->last->next = page; + to->last = page; + } + else { + to->first = page; + to->last = page; + mi_theap_queue_first_update(theap, to); + } + } + else { + if (to->first != NULL) { + // enqueue at 2nd place + mi_assert_internal(theap == mi_page_theap(to->first)); + mi_page_t* next = to->first->next; + page->prev = to->first; + page->next = next; + to->first->next = page; + if (next != NULL) { + next->prev = page; + } + else { + to->last = page; + } + } + else { + // enqueue at the head (singleton list) + page->prev = NULL; + page->next = NULL; + to->first = page; + to->last = page; + mi_theap_queue_first_update(theap, to); + } + } + + mi_page_set_in_full(page, mi_page_queue_is_full(to)); +} + +static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { + mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page); +} + +static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { + // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`) + mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page); +} diff --git a/csrc/mimalloc/src/page.c b/csrc/mimalloc/src/page.c new file mode 100644 index 000000000..02ca72044 --- /dev/null +++ b/csrc/mimalloc/src/page.c @@ -0,0 +1,1117 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ----------------------------------------------------------- + The core of the allocator. Every segment contains + pages of a certain block size. The main function + exported is `mi_malloc_generic`. +----------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" + +/* ----------------------------------------------------------- + Definition of page queues for each block size +----------------------------------------------------------- */ + +#define MI_IN_PAGE_C +#include "page-queue.c" +#undef MI_IN_PAGE_C + + +/* ----------------------------------------------------------- + Page helpers +----------------------------------------------------------- */ + +// Index a block in a page +static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) { + MI_UNUSED(page); + mi_assert_internal(page != NULL); + mi_assert_internal(i <= page->reserved); + return (mi_block_t*)((uint8_t*)page_start + (i * block_size)); +} + +static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page); + +#if (MI_DEBUG>=3) +static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) { + mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page); + const uint8_t* slice_start = mi_page_slice_start(page); + mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN)); + size_t count = 0; + while (head != NULL) { + mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == _mi_ptr_page(head)); + count++; + head = mi_block_next(page, head); + } + return count; +} + +/* +// Start of the page available memory +static inline uint8_t* mi_page_area(const mi_page_t* page) { + return _mi_page_start(_mi_page_segment(page), page, NULL); +} +*/ + +static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { + size_t psize; + uint8_t* page_area = mi_page_area(page, &psize); + mi_block_t* start = (mi_block_t*)page_area; + mi_block_t* end = (mi_block_t*)(page_area + psize); + while(p != NULL) { + if (p < start || p >= end) return false; + p = mi_block_next(page, p); + } +#if MI_DEBUG>3 // generally too expensive to check this + if (page->free_is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); + } + } +#endif + return true; +} + +static bool mi_page_is_valid_init(mi_page_t* page) { + mi_assert_internal(mi_page_block_size(page) > 0); + mi_assert_internal(page->used <= page->capacity); + mi_assert_internal(page->capacity <= page->reserved); + + mi_assert_internal(page->heap!=NULL); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + + // const size_t bsize = mi_page_block_size(page); + // uint8_t* start = mi_page_start(page); + //mi_assert_internal(start + page->capacity*page->block_size == page->top); + + mi_assert_internal(mi_page_list_is_valid(page,page->free)); + mi_assert_internal(mi_page_list_is_valid(page,page->local_free)); + + #if MI_DEBUG>3 // generally too expensive to check this + if (page->free_is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); + } + } + #endif + + #if !MI_TRACK_ENABLED && !MI_TSAN + mi_block_t* tfree = mi_page_thread_free(page); + mi_assert_internal(mi_page_list_is_valid(page, tfree)); + //size_t tfree_count = mi_page_list_count(page, tfree); + //mi_assert_internal(tfree_count <= page->thread_freed + 1); + #endif + + size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); + mi_assert_internal(page->used + free_count == page->capacity); + + return true; +} + +extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? + +bool _mi_page_is_valid(mi_page_t* page) { + mi_assert_internal(mi_page_is_valid_init(page)); + #if MI_SECURE + mi_assert_internal(page->keys[0] != 0); + #endif + if (!mi_page_is_abandoned(page)) { + //mi_assert_internal(!_mi_process_is_initialized); + mi_assert_internal(page->heap!=NULL); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + { + mi_page_queue_t* pq = mi_page_queue_of(page); + mi_assert_internal(mi_page_queue_contains(pq, page)); + mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_is_huge(page) || mi_page_is_in_full(page)); + // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq)); + } + } + return true; +} +#endif + + +/* ----------------------------------------------------------- + Page collect the `local_free` and `thread_free` lists +----------------------------------------------------------- */ + +static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) +{ + if (head == NULL) return; + + // find the last block in the list -- also to get a proper use count (without data races) + size_t max_count = page->capacity; // cannot collect more than capacity + size_t count = 1; + mi_block_t* last = head; + mi_block_t* next; + while ((next = mi_block_next(page, last)) != NULL && count <= max_count) { + count++; + last = next; + } + + // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) + if mi_unlikely(count > max_count) { + _mi_error_message(EFAULT, "corrupted thread-free list (possibly due to a cross-thread double free)\n"); + return; // the thread-free items cannot be freed + } + // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) + else if mi_unlikely(count > page->used) { + _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n"); + return; // the thread-free items cannot be freed + } + + // and append the current local free list + mi_block_set_next(page, last, page->local_free); + page->local_free = head; + + // update counts now + mi_assert_internal(count <= UINT16_MAX); + mi_assert_internal(page->used >= (uint16_t)count); + page->used = page->used - (uint16_t)count; +} + +// Collect the local `thread_free` list using an atomic exchange. +static void mi_page_thread_free_collect(mi_page_t* page) +{ + // atomically capture the thread free list + mi_block_t* head; + mi_thread_free_t tfreex; + mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); + do { + head = mi_tf_block(tfree); + if mi_likely(head == NULL) return; // return if the list is empty + tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL + } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough? + mi_assert_internal(head != NULL); + + // and move it to the local list + mi_page_thread_collect_to_local(page, head); +} + +// returns `true` if after collection `mi_page_immediate_available` is true. +static inline bool mi_page_free_quick_collect(mi_page_t* page) { + if mi_likely(page->free != NULL) return true; + if (page->local_free == NULL) return false; + // move local_free to free + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + return true; +} + +void _mi_page_free_collect(mi_page_t* page, bool force) { + mi_assert_internal(page!=NULL); + + // collect the thread free list + mi_page_thread_free_collect(page); + + // and the local free list + if (page->local_free != NULL) { + if mi_likely(page->free == NULL) { + // usual case + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + } + else if (force) { + // append -- only on shutdown (force) as this is a linear operation + mi_block_t* tail = page->local_free; + mi_block_t* next; + while ((next = mi_block_next(page, tail)) != NULL) { + tail = next; + } + mi_block_set_next(page, tail, page->free); + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + } + } + + mi_assert_internal(!force || page->local_free == NULL); +} + +// Collect elements in the thread-free list starting at `head`. This is an optimized +// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// +// `head` must be in the `xthread_free` list. It will not collect `head` itself +// so the `used` count is not fully updated in general. However, if the `head` is +// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). +void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + if (head == NULL) return; + mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) + if (next != NULL) { + mi_block_set_next(page, head, NULL); + mi_page_thread_collect_to_local(page, next); + if (page->local_free != NULL && page->free == NULL) { + page->free = page->local_free; + page->local_free = NULL; + page->free_is_zero = false; + } + } + if (page->used == 1) { + // all elements are free'd since we skipped the `head` element itself + mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); + mi_assert_internal(mi_block_next(page,head) == NULL); + _mi_page_free_collect(page, false); // collect the final element + } +} + + +/* ----------------------------------------------------------- + Page fresh and retire +----------------------------------------------------------- */ + +// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page +void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) +{ + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + + mi_page_set_theap(page,theap); + _mi_page_free_collect(page, false); // ensure used count is up to date + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); + mi_page_queue_push_at_end(theap, pq, page); + mi_assert_expensive(_mi_page_is_valid(page)); +} + +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { + _mi_page_free_collect(page, false); // ensure used count is up to date + if (mi_page_all_free(page)) { + _mi_page_free(page, pq); + } + else { + mi_page_queue_remove(pq, page); + mi_theap_t* theap = page->theap; + mi_page_set_theap(page, NULL); + page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap + _mi_arenas_page_abandon(page, theap); + // _mi_arenas_collect(false, false, theap->tld); // allow purging + } +} + + +// allocate a fresh page from an arena +static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) { + #if !MI_HUGE_PAGE_ABANDON + mi_assert_internal(pq != NULL); + mi_assert_internal(mi_theap_contains_queue(theap, pq)); + mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size); + #endif + mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment); + if (page == NULL) { + // out-of-memory + return NULL; + } + if (mi_page_is_abandoned(page)) { + _mi_theap_page_reclaim(theap, page); + if (!mi_page_immediate_available(page)) { + if (mi_page_is_expandable(page)) { + if (!mi_page_extend_free(theap, page)) { + // cannot commit + _mi_page_abandon(page,pq); + return NULL; + }; + } + else { + mi_assert(false); // should not happen? + return NULL; + } + } + } + else if (pq != NULL) { + mi_page_queue_push(theap, pq, page); + } + mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size); + mi_assert_expensive(_mi_page_is_valid(page)); + return page; +} + +// Get a fresh page to use +static mi_page_t* mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) { + mi_assert_internal(mi_theap_contains_queue(theap, pq)); + mi_page_t* page = mi_page_fresh_alloc(theap, pq, pq->block_size, 0); + if (page==NULL) return NULL; + mi_assert_internal(pq->block_size==mi_page_block_size(page)); + mi_assert_internal(pq==mi_theap_page_queue_of(theap, page)); + return page; +} + + +/* ----------------------------------------------------------- + Unfull, abandon, free and retire +----------------------------------------------------------- */ + +// Move a page from the full list back to a regular list (called from thread-local mi_free) +void _mi_page_unfull(mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(mi_page_is_in_full(page)); + mi_assert_internal(!mi_page_theap(page)->allow_page_abandon); + if (!mi_page_is_in_full(page)) return; + + mi_theap_t* theap = mi_page_theap(page); + mi_page_queue_t* pqfull = &theap->pages[MI_BIN_FULL]; + mi_page_set_in_full(page, false); // to get the right queue + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); + mi_page_set_in_full(page, true); + mi_page_queue_enqueue_from_full(pq, pqfull, page); +} + +static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { + mi_assert_internal(pq == mi_page_queue_of(page)); + mi_assert_internal(!mi_page_immediate_available(page)); + mi_assert_internal(!mi_page_is_in_full(page)); + + mi_theap_t* theap = mi_page_theap(page); + if (theap->allow_page_abandon) { + // abandon full pages (this is the usual case in order to allow for sharing of memory between theaps) + _mi_page_abandon(page, pq); + } + else if (!mi_page_is_in_full(page)) { + // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed) + mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page); + _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set + } +} + + +// Free a page with no more free blocks +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(pq == mi_page_queue_of(page)); + mi_assert_internal(mi_page_all_free(page)); + // mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING); + + // no more aligned blocks in here + mi_page_set_has_interior_pointers(page, false); + + // remove from the page list + // (no need to do _mi_theap_delayed_free first as all blocks are already free) + mi_page_queue_remove(pq, page); + + // and free it + mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL); + mi_page_set_theap(page,NULL); + _mi_arenas_page_free(page, theap); + // _mi_arenas_collect(false, false, theap->tld); // allow purging +} + +#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */ +#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ + +// Retire a page with no more used blocks +// Important to not retire too quickly though as new +// allocations might coming. +// +// Note: called from `mi_free` and benchmarks often +// trigger this due to freeing everything and then +// allocating again so careful when changing this. +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { + mi_assert_internal(page != NULL); + mi_assert_expensive(_mi_page_is_valid(page)); + mi_assert_internal(mi_page_all_free(page)); + + if (page->retire_expire!=0) return; // already retired, just keep it retired + mi_page_set_has_interior_pointers(page, false); + + // don't retire too often.. + // (or we end up retiring and re-allocating most of the time) + // NOTE: refine this more: we should not retire if this + // is the only page left with free blocks. It is not clear + // how to check this efficiently though... + // for now, we don't retire if it is the only page left of this size class. + mi_page_queue_t* pq = mi_page_queue_of(page); + #if MI_RETIRE_CYCLES > 0 + const size_t bsize = mi_page_block_size(page); + if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue? + if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) { + mi_theap_t* theap = mi_page_theap(page); + mi_theap_stat_counter_increase(theap, pages_retire, 1); + page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); + mi_assert_internal(pq >= theap->pages); + const size_t index = pq - theap->pages; + mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE); + if (index < theap->page_retired_min) theap->page_retired_min = index; + if (index > theap->page_retired_max) theap->page_retired_max = index; + mi_assert_internal(mi_page_all_free(page)); + return; // don't free after all + } + } + #endif + _mi_page_free(page, pq); +} + + +static void mi_theap_collect_full_pages(mi_theap_t* theap) { + // note: normally full pages get immediately abandoned and the full queue is always empty + // this path is only used if abandoning is disabled due to a destroy-able theap or options + // set by the user. + mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL]; + for (mi_page_t* page = pq->first; page != NULL; ) { + mi_page_t* next = page->next; // get next in case we free the page + _mi_page_free_collect(page, false); // register concurrent free's + // no longer full? + if (!mi_page_is_full(page)) { + if (mi_page_all_free(page)) { + _mi_page_free(page, pq); + } + else { + _mi_page_unfull(page); + } + } + page = next; + } +} + +static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { + mi_assert_internal(page!=NULL && page->retire_expire!=0); + if (mi_page_all_free(page)) { + page->retire_expire--; + if (page->retire_expire == 0 || force) { + _mi_page_free(page, pq); + } + else { + // keep retired, update min/max + if (bin < *min) *min = bin; + if (bin > *max) *max = bin; + } + } + else { + page->retire_expire = 0; + } +} + +// free retired pages: we don't need to look at the entire queues +// since we only retire pages that are at the head position in a queue. +void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { + size_t min = MI_BIN_FULL; + size_t max = 0; + for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { + mi_page_queue_t* pq = &theap->pages[bin]; + mi_page_t* page = pq->first; + for (int i = 0; iretire_expire!=0; i++) { + mi_page_t* next = page->next; + mi_page_try_retire(pq,page,bin,force,&min,&max); + page = next; + } + } + theap->page_retired_min = min; + theap->page_retired_max = max; + if (!theap->allow_page_abandon) { + mi_theap_collect_full_pages(theap); + } +} + + + + +/* ----------------------------------------------------------- + Initialize the initial free list in a page. + In secure mode we initialize a randomized list by + alternating between slices. +----------------------------------------------------------- */ + +#define MI_MAX_SLICE_SHIFT (6) // at most 64 slices +#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT) +#define MI_MIN_SLICES (2) + +static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) { + #if (MI_SECURE < 2) + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + #endif + mi_assert_internal(page->capacity + extend <= page->reserved); + mi_assert_internal(bsize == mi_page_block_size(page)); + void* const page_area = mi_page_start(page); + + // initialize a randomized free list + // set up `slice_count` slices to alternate between + size_t shift = MI_MAX_SLICE_SHIFT; + while ((extend >> shift) == 0) { + shift--; + } + const size_t slice_count = (size_t)1U << shift; + const size_t slice_extend = extend / slice_count; + mi_assert_internal(slice_extend >= 1); + mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice + size_t counts[MI_MAX_SLICES]; // available objects in the slice + for (size_t i = 0; i < slice_count; i++) { + blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend); + counts[i] = slice_extend; + } + counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?) + + // and initialize the free list by randomly threading through them + // set up first element + const uintptr_t r = _mi_theap_random_next(theap); + size_t current = r % slice_count; + counts[current]--; + mi_block_t* const free_start = blocks[current]; + // and iterate through the rest; use `random_shuffle` for performance + uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + for (size_t i = 1; i < extend; i++) { + // call random_shuffle only every INTPTR_SIZE rounds + const size_t round = i%MI_INTPTR_SIZE; + if (round == 0) rnd = _mi_random_shuffle(rnd); + // select a random next slice index + size_t next = ((rnd >> 8*round) & (slice_count-1)); + while (counts[next]==0) { // ensure it still has space + next++; + if (next==slice_count) next = 0; + } + // and link the current block to it + counts[next]--; + mi_block_t* const block = blocks[current]; + blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block + mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` + current = next; + } + // prepend to the free list (usually NULL) + mi_block_set_next(page, blocks[current], page->free); // end of the list + page->free = free_start; +} + +static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend) +{ + #if (MI_SECURE < 2) + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + #endif + mi_assert_internal(page->capacity + extend <= page->reserved); + mi_assert_internal(bsize == mi_page_block_size(page)); + void* const page_area = mi_page_start(page); + + mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity); + + // initialize a sequential free list + mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1); + mi_block_t* block = start; + while(block <= last) { + mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize); + mi_block_set_next(page,block,next); + block = next; + } + // prepend to free list (usually `NULL`) + mi_block_set_next(page, last, page->free); + page->free = start; +} + +/* ----------------------------------------------------------- + Page initialize and extend the capacity +----------------------------------------------------------- */ + +#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well. +#if (MI_SECURE>=2) +#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many +#else +#define MI_MIN_EXTEND (1) +#endif + +// Extend the capacity (up to reserved) by initializing a free list +// We do at most `MI_MAX_EXTEND` to avoid touching too much memory +// Note: we also experimented with "bump" allocation on the first +// allocations but this did not speed up any benchmark (due to an +// extra test in malloc? or cache effects?) +static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { + mi_assert_expensive(mi_page_is_valid_init(page)); + #if (MI_SECURE < 2) + mi_assert(page->free == NULL); + mi_assert(page->local_free == NULL); + if (page->free != NULL) return true; + #endif + if (page->capacity >= page->reserved) return true; + + size_t page_size; + //uint8_t* page_start = + mi_page_area(page, &page_size); + #if MI_STAT>0 + mi_theap_stat_counter_increase(theap, pages_extended, 1); + #endif + + // calculate the extend count + const size_t bsize = mi_page_block_size(page); + size_t extend = (size_t)page->reserved - page->capacity; + mi_assert_internal(extend > 0); + + size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/bsize); + if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; } + mi_assert_internal(max_extend > 0); + + if (extend > max_extend) { + // ensure we don't touch memory beyond the page to reduce page commit. + // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%. + extend = max_extend; + } + + mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved); + mi_assert_internal(extend < (1UL<<16)); + + // commit on demand? + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { + // reduce extend if it commits more than an arena slice + if ((extend * bsize) > MI_ARENA_SLICE_SIZE) { + extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize); + } + // commit required size + const size_t needed_size = (page->capacity + extend)*bsize; + mi_assert_internal(needed_size <= page_size); + size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); + #if MI_SECURE>=5 + // the previous alignup could extend the commit into the guard page; re-adjust if needed + const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); + if (needed_commit > page_size_commit) { + needed_commit = page_size_commit; + } + #endif + if (needed_commit > slice_committed) { + mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0); + if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { + return false; + } + mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size()); + page->slice_pcommitted = (uint16_t)(needed_commit / _mi_os_page_size()); + } + } + + // and append the extend the free list + if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) { + mi_page_free_list_extend(page, bsize, extend ); + } + else { + mi_page_free_list_extend_secure(theap, page, bsize, extend); + } + // enable the new free list + page->capacity += (uint16_t)extend; + #if MI_STAT>0 + mi_theap_stat_increase(theap, page_committed, extend * bsize); + #endif + mi_assert_expensive(mi_page_is_valid_init(page)); + return true; +} + +// Initialize a fresh page (that is already partially initialized) +mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { + mi_assert(page != NULL); + mi_assert(theap!=NULL); + // page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap` + // mi_page_set_theap(page, theap); + + size_t page_size; + uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start); + mi_track_mem_noaccess(page_start,page_size); + mi_assert_internal(page_size / mi_page_block_size(page) < (1L<<16)); + mi_assert_internal(page->reserved > 0); + #if (MI_PADDING || MI_ENCODE_FREELIST) + page->keys[0] = _mi_theap_random_next(theap); + #if MI_PAGE_KEY_COUNT==2 + page->keys[1] = _mi_theap_random_next(theap); + #endif + #endif + #if MI_DEBUG>2 + if (page->memid.initially_zero) { + mi_track_mem_defined(mi_page_start(page), mi_page_committed(page)); + mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page))); + } + #endif + + mi_assert_internal(page->heap != NULL); + mi_assert_internal(page->heap == _mi_theap_heap(theap)); + mi_assert_internal(page->theap!=NULL); + mi_assert_internal(page->theap == mi_page_theap(page)); + mi_assert_internal(page->capacity == 0); + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->used == 0); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(page->xthread_free == 1); + mi_assert_internal(page->next == NULL); + mi_assert_internal(page->prev == NULL); + mi_assert_internal(page->retire_expire == 0); + mi_assert_internal(!mi_page_has_interior_pointers(page)); + #if (MI_PADDING || MI_ENCODE_FREELIST) + mi_assert_internal(page->keys[0] != 0); + #if MI_PAGE_KEY_COUNT==2 + mi_assert_internal(page->keys[1] != 0); + #endif + #endif + mi_assert_expensive(mi_page_is_valid_init(page)); + + // initialize an initial free list + if (!mi_page_extend_free(theap,page)) return false; + mi_assert(mi_page_immediate_available(page)); + return true; +} + + +/* ----------------------------------------------------------- + Find pages with free blocks +-------------------------------------------------------------*/ + +// Find a page with free blocks of `page->block_size`. +static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try) +{ + // search through the pages in "next fit" order + size_t count = 0; + long candidate_limit = 0; // we reset this on the first candidate to limit the search + long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages + mi_page_t* page_candidate = NULL; // a page with free space + mi_page_t* page = pq->first; + + while (page != NULL) + { + mi_page_t* next = page->next; // remember next (as this page can move to another queue) + count++; + candidate_limit--; + + // search up to N pages for a best candidate + + // is the local free list non-empty? + bool immediate_available = mi_page_immediate_available(page); + if (!immediate_available) { + // collect freed blocks by us and other threads to we get a proper use count + _mi_page_free_collect(page, false); + immediate_available = mi_page_immediate_available(page); + } + + // if the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + if (!immediate_available && !mi_page_is_expandable(page)) { + page_full_retain--; + if (page_full_retain < 0) { + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + mi_page_to_full(page, pq); + } + } + else { + // the page has free space, make it a candidate + // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages) + if (page_candidate == NULL) { + page_candidate = page; + candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates); + } + else if (mi_page_all_free(page_candidate)) { + _mi_page_free(page_candidate, pq); + page_candidate = page; + } + // prefer to reuse fuller pages (in the hope the less used page gets freed) + else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { + page_candidate = page; + } + // if we find a non-expandable candidate, or searched for N pages, return with the best candidate + if (immediate_available || candidate_limit <= 0) { + mi_assert_internal(page_candidate!=NULL); + break; + } + } + + #if 0 + // first-fit algorithm without candidates + // If the page contains free blocks, we are done + if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) { + break; // pick this one + } + + // If the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + mi_page_to_full(page, pq); + #endif + + page = next; + } // for each page + + mi_theap_stat_counter_increase(theap, page_searches, count); + mi_theap_stat_counter_increase(theap, page_searches_count, 1); + + // set the page to the best candidate + if (page_candidate != NULL) { + page = page_candidate; + } + if (page != NULL) { + if (!mi_page_immediate_available(page)) { + mi_assert_internal(mi_page_is_expandable(page)); + if (!mi_page_extend_free(theap, page)) { + page = NULL; // failed to extend + } + } + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + } + + if (page == NULL) { + _mi_theap_collect_retired(theap, false); // perhaps make a page available + page = mi_page_fresh(theap, pq); + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + if (page == NULL && first_try) { + // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again + page = mi_page_queue_find_free_ex(theap, pq, false); + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + } + } + else { + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + // move the page to the front of the queue + mi_page_queue_move_to_front(theap, pq, page); + page->retire_expire = 0; + // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this + } + mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + + + return page; +} + +// Look for a page with free blocks of `size` but don't try to search or allocate +static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) { + mi_assert_internal(!mi_page_queue_is_huge(pq)); + mi_page_t* page = pq->first; + if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { + // fast path + #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness + if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { + (void)mi_page_extend_free(theap, page); // ok if this fails + mi_assert_internal(mi_page_immediate_available(page)); + } + #else + MI_UNUSED(theap); + #endif + page->retire_expire = 0; + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; + } + else { + return NULL; + } +} + +// Find a page with free blocks of `size`. +static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) { + // mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(!mi_page_queue_is_huge(pq)); + + // check the first page: we even do this with candidate search or otherwise we re-search every time + mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq); + if (page==NULL) { + page = mi_page_queue_find_free_ex(theap, pq, true); + if (page==NULL) return NULL; + } + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; +} + +// Huge pages contain just one block, and the segment contains just that page. +// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) +// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. +static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) { + const size_t block_size = _mi_os_good_alloc_size(size); + // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); + #if MI_HUGE_PAGE_ABANDON + #error todo. + #else + // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1); // always in the huge queue regardless of the block size + mi_assert_internal(mi_page_queue_is_huge(pq)); + #endif + mi_page_t* page = mi_page_fresh_alloc(theap, pq, block_size, page_alignment); + if (page != NULL) { + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_page_is_huge(page)); + mi_assert_internal(mi_page_is_singleton(page)); + #if MI_HUGE_PAGE_ABANDON + mi_assert_internal(mi_page_is_abandoned(page)); + mi_page_set_theap(page, NULL); + #endif + mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); + mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); + } + return page; +} + +// Allocate a page +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size)); + mi_page_t* page; + // huge allocation? + if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) { + page = mi_huge_page_alloc(theap,size,huge_alignment,pq); + } + else { + // otherwise find a page with free blocks in our size segregated queues + #if MI_PADDING + mi_assert_internal(size >= MI_PADDING_SIZE); + #endif + page = mi_page_queue_find_free(theap,pq); + } + if (page==NULL) return NULL; + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; +} + + + +/* ----------------------------------------------------------- + Users can register a deferred free function called + when the `free` list is empty. Since the `local_free` + is separate this is deterministically called after + a certain number of allocations. +----------------------------------------------------------- */ + +// The program should only install a single deferred free handler before doing allocation. +static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) +static _Atomic(void*) deferred_arg; + +void _mi_deferred_free(mi_theap_t* theap, bool force) { + theap->heartbeat++; + mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); + if (fun != NULL && !theap->tld->recurse) { + theap->tld->recurse = true; + void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); + fun(force, theap->heartbeat, arg); + theap->tld->recurse = false; + } +} + +void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { + mi_atomic_store_ptr_release(void,&deferred_arg, arg); + mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); +} + + +/* ----------------------------------------------------------- + Admin +----------------------------------------------------------- */ + +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) +{ + if mi_unlikely(!mi_theap_is_initialized(theap)) { + if (theap==&_mi_theap_empty_wrong) { + // we were unable to allocate a theap for a first-class heap + return NULL; + } + // otherwise we initialize the thread and its default theap + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } + } + mi_assert_internal(mi_theap_is_initialized(theap)); + + // do administrative tasks every N generic mallocs + if mi_unlikely(theap->generic_count >= 1000) { + theap->generic_collect_count += theap->generic_count; + theap->generic_count = 0; + + // do a full theap collect every once in a while (10000 by default) + const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); + if (theap->generic_collect_count >= generic_collect) { + theap->generic_collect_count = 0; + mi_theap_collect(theap, false /* force? */); + } + else { + // otherwise we do a mini-collect + _mi_deferred_free(theap, false); // call potential deferred free routines + _mi_theap_collect_retired(theap, false); // free retired pages + } + } + return theap; +} + +/* ----------------------------------------------------------- + Generic allocation +----------------------------------------------------------- */ + +static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) +{ + // initialize if necessary + theap = mi_malloc_generic_admin(theap); + if (theap==NULL) return NULL; + + // find (or allocate) a page of the right size + mi_page_t* page = mi_find_page(theap, size, huge_alignment); + if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more + mi_theap_collect(theap, true /* force? */); + page = mi_find_page(theap, size, huge_alignment); + } + + if mi_unlikely(page == NULL) { // out of memory + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); + return NULL; + } + + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + if (ppage!=NULL) { *ppage = page; } + void* const p = _mi_page_malloc_zero(theap,page,size,zero); + mi_assert_internal(p != NULL); + + // move full pages to the full queue + if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { + mi_page_to_full(page, mi_page_queue_of(page)); + } + return p; +} + + +// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for +// very large requested alignments in which case we use a huge singleton page. +// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) +// to use 4 parameters which compiles better on msvc for the malloc fast path. +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +{ + #if !MI_THEAP_INITASNULL + mi_assert_internal(theap != NULL); + #endif + const bool zero = ((zero_huge_alignment & 1) != 0); + const size_t huge_alignment = (zero_huge_alignment & ~1); + mi_page_t* page = NULL; + + // fast path objects that fit in a small page + if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); + return _mi_page_malloc_zero(theap,page,size,zero); + } + } + } + // otherwise fallback + return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); +} diff --git a/csrc/mimalloc/src/prim/emscripten/prim.c b/csrc/mimalloc/src/prim/emscripten/prim.c new file mode 100644 index 000000000..992965d56 --- /dev/null +++ b/csrc/mimalloc/src/prim/emscripten/prim.c @@ -0,0 +1,263 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen, Alon Zakai +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" + +#include // sched_yield +#include // getentropy + +// Design +// ====== +// +// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top +// of sbrk. The reason for having three layers here is that we want mimalloc to +// be able to allocate and release system memory properly, the same way it would +// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited. +// Specifically, sbrk can only go up and down, and not "skip" over regions, and +// so we end up either never freeing memory to the system, or we can get stuck +// with holes. +// +// Atm wasm generally does *not* free memory back the system: once grown, we do +// not shrink back down (https://github.com/WebAssembly/design/issues/1397). +// However, that is expected to improve +// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md) +// and so we do not want to bake those limitations in here. +// +// Even without that issue, we want our system allocator to handle holes, that +// is, it should merge freed regions and allow allocating new content there of +// the full size, etc., so that we do not waste space. That means that the +// system allocator really does need to handle the general problem of allocating +// and freeing variable-sized chunks of memory in a random order, like malloc/ +// free do. And so it makes sense to layer mimalloc on top of such an +// implementation. +// +// emmalloc makes sense for the lower level because it is small and simple while +// still fully handling merging of holes etc. It is not the most efficient +// allocator, but our assumption is that mimalloc needs to be fast while the +// system allocator underneath it is called much less frequently. +// + +//--------------------------------------------- +// init +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config) { + config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB + config->alloc_granularity = 16; + config->has_overcommit = false; + config->has_partial_free = false; + config->has_virtual_reserve = false; +} + +extern void emmalloc_free(void*); + +int _mi_prim_free(void* addr, size_t size) { + emmalloc_free(addr); + return 0; +} + + +//--------------------------------------------- +// Allocation +//--------------------------------------------- + +extern void* emmalloc_memalign(size_t alignment, size_t size); + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); + *is_large = false; + // todo: Track the highest address ever seen; first uses of it are zeroes. + // That assumes no one else uses sbrk but us (they could go up, + // scribble, and then down), but we could assert on that perhaps. + *is_zero = false; + // emmalloc has a minimum alignment size. + #define MIN_EMMALLOC_ALIGN 8 + if (try_alignment < MIN_EMMALLOC_ALIGN) { + try_alignment = MIN_EMMALLOC_ALIGN; + } + void* p = emmalloc_memalign(try_alignment, size); + *addr = p; + if (p == 0) { + return ENOMEM; + } + return 0; +} + + +//--------------------------------------------- +// Commit/Reset +//--------------------------------------------- + +int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { + MI_UNUSED(addr); MI_UNUSED(size); + // See todo above. + *is_zero = false; + return 0; +} + +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { + MI_UNUSED(addr); MI_UNUSED(size); + *needs_recommit = false; + return 0; +} + +int _mi_prim_reset(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_reuse(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); + return 0; +} + + +//--------------------------------------------- +// Huge pages and NUMA nodes +//--------------------------------------------- + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *is_zero = true; + *addr = NULL; + return ENOSYS; +} + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +#include + +mi_msecs_t _mi_prim_clock_now(void) { + // todo: use a monotonic clock instead + return emscripten_date_now(); +} + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + // use defaults + MI_UNUSED(pinfo); +} + + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +#include + +void _mi_prim_out_stderr( const char* msg) { + emscripten_console_error(msg); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // For code size reasons, do not support environ customization for now. + MI_UNUSED(name); + MI_UNUSED(result); + MI_UNUSED(result_size); + return 0; // not found +} + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + int err = getentropy(buf, buf_len); + return !err; +} + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if defined(MI_USE_PTHREADS) + +// use pthread local storage keys to detect thread ending +// (and used with MI_TLS_PTHREADS for the default theap) +pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + +static void mi_pthread_done(void* value) { + if (value!=NULL) { + _mi_thread_done((mi_theap_t*)value); + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); + pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); +} + +void _mi_prim_thread_done_auto_done(void) { + pthread_key_t key = _mi_heap_default_key; + if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 + _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); + } +} + +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD + pthread_setspecific(_mi_heap_default_key, theap); + } +} + +#else + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); +} +#endif + +bool _mi_prim_thread_is_in_threadpool(void) { + return false; +} + +void _mi_prim_thread_yield(void) { + sched_yield(); +} diff --git a/csrc/mimalloc/src/prim/osx/alloc-override-zone.c b/csrc/mimalloc/src/prim/osx/alloc-override-zone.c new file mode 100644 index 000000000..48f979582 --- /dev/null +++ b/csrc/mimalloc/src/prim/osx/alloc-override-zone.c @@ -0,0 +1,492 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" // _mi_thread_is_initialized +#if defined(MI_MALLOC_OVERRIDE) + +#if !defined(__APPLE__) +#error "this file should only be included on macOS" +#endif + +/* ------------------------------------------------------ + Override system malloc on macOS + This is done through the malloc zone interface. + It seems to be most robust in combination with interposing + though or otherwise we may get zone errors as there are could + be allocations done by the time we take over the + zone. +------------------------------------------------------ */ + +#include +#include +#include // memset +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) +// only available from OSX 10.6 +extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); +#endif + +/* ------------------------------------------------------ + malloc zone members +------------------------------------------------------ */ + +static bool is_mimalloc_zone( malloc_zone_t* zone ); + +static size_t zone_size(malloc_zone_t* zone, const void* p) { + if (mi_any_heap_contains(p)) { + return mi_usable_size(p); + } + else if (!is_mimalloc_zone(zone)) { // can happen due to interpose + return zone->size(zone,p); + } + else { + return 0; + } +} + +static void* zone_malloc(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_malloc(size); +} + +static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { + MI_UNUSED(zone); + return mi_calloc(count, size); +} + +static void* zone_valloc(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_malloc_aligned(size, _mi_os_page_size()); +} + +static void zone_free(malloc_zone_t* zone, void* p) { + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. + if mi_unlikely(!mi_cfree(p)) { + if (!is_mimalloc_zone(zone)) { // can happen due to interpose + zone->free(zone,p); + } + } +} + +static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { + if (p == NULL || mi_any_heap_contains(p)) { + return mi_realloc(p, newsize); + } + else if (!is_mimalloc_zone(zone)) { // can happen due to interpose + return zone->realloc(zone,p,newsize); + } + else { + return NULL; + } +} + +static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { + MI_UNUSED(zone); + return mi_malloc_aligned(size,alignment); +} + +static void zone_destroy(malloc_zone_t* zone) { + if (!is_mimalloc_zone(zone)) { + zone->destroy(zone); + } +} + +static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { + unsigned i; + for (i = 0; i < count; i++) { + ps[i] = zone_malloc(zone, size); + if (ps[i] == NULL) break; + } + return i; +} + +static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { + for(size_t i = 0; i < count; i++) { + zone_free(zone, ps[i]); + ps[i] = NULL; + } +} + +static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); MI_UNUSED(size); + mi_collect(false); + return 0; +} + +static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { + MI_UNUSED(size); + zone_free(zone,p); +} + +static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); + return mi_is_in_heap_region(p); +} + +/* ------------------------------------------------------ + Introspection members +------------------------------------------------------ */ + +static kern_return_t intro_enumerator(task_t task, void* p, + unsigned type_mask, vm_address_t zone_address, + memory_reader_t reader, + vm_range_recorder_t recorder) +{ + // todo: enumerate all memory + MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); + MI_UNUSED(reader); MI_UNUSED(recorder); + return KERN_SUCCESS; +} + +static size_t intro_good_size(malloc_zone_t* zone, size_t size) { + MI_UNUSED(zone); + return mi_good_size(size); +} + +static boolean_t intro_check(malloc_zone_t* zone) { + MI_UNUSED(zone); + return true; +} + +static void intro_print(malloc_zone_t* zone, boolean_t verbose) { + MI_UNUSED(zone); MI_UNUSED(verbose); + mi_stats_print(NULL); +} + +static void intro_log(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); MI_UNUSED(p); + // todo? +} + +static void intro_force_lock(malloc_zone_t* zone) { + MI_UNUSED(zone); + // todo? +} + +static void intro_force_unlock(malloc_zone_t* zone) { + MI_UNUSED(zone); + // todo? +} + +static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { + MI_UNUSED(zone); + // todo... + stats->blocks_in_use = 0; + stats->size_in_use = 0; + stats->max_size_in_use = 0; + stats->size_allocated = 0; +} + +static boolean_t intro_zone_locked(malloc_zone_t* zone) { + MI_UNUSED(zone); + return false; +} + +// Required whenever the zone advertises version >= 9: macOS calls this from the +// atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps +// no zone-level locks that need reinitializing after fork, so a no-op is safe. +// Leaving it NULL makes the forked child jump to address 0 and crash in fork(). +static void intro_reinit_lock(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + + +/* ------------------------------------------------------ + At process start, override the default allocator +------------------------------------------------------ */ + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wc99-extensions" +#endif + +static malloc_introspection_t mi_introspect = { + .enumerator = &intro_enumerator, + .good_size = &intro_good_size, + .check = &intro_check, + .print = &intro_print, + .log = &intro_log, + .force_lock = &intro_force_lock, + .force_unlock = &intro_force_unlock, +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) + .statistics = &intro_statistics, + .zone_locked = &intro_zone_locked, +#endif +#if defined(MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_12) && !defined(__ppc__) + .reinit_lock = &intro_reinit_lock, +#endif +}; + +static malloc_zone_t mi_malloc_zone = { + // note: even with designators, the order is important for C++ compilation + //.reserved1 = NULL, + //.reserved2 = NULL, + .size = &zone_size, + .malloc = &zone_malloc, + .calloc = &zone_calloc, + .valloc = &zone_valloc, + .free = &zone_free, + .realloc = &zone_realloc, + .destroy = &zone_destroy, + .zone_name = "mimalloc", + .batch_malloc = &zone_batch_malloc, + .batch_free = &zone_batch_free, + .introspect = &mi_introspect, +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) + .version = 10, + #else + .version = 9, + #endif + // switch to version 9+ on OSX 10.6 to support memalign. + .memalign = &zone_memalign, + .free_definite_size = &zone_free_definite_size, + #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7) + .pressure_relief = &zone_pressure_relief, + #endif + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) + .claimed_address = &zone_claimed_address, + #endif +#else + .version = 4, +#endif +}; + +#ifdef __cplusplus +} +#endif + +static bool is_mimalloc_zone( malloc_zone_t* zone ) { + return (zone==NULL || zone==&mi_malloc_zone); +} + +#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) + +// ------------------------------------------------------ +// Override malloc_xxx and malloc_zone_xxx api's to use only +// our mimalloc zone. Since even the loader uses malloc +// on macOS, this ensures that all allocations go through +// mimalloc (as all calls are interposed). +// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, +// Here, we also override macOS specific API's like +// `malloc_zone_calloc` etc. see +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) { + mi_atomic_do_once { + malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see ) + } + return &mi_malloc_zone; +} + +mi_decl_externc int malloc_jumpstart(uintptr_t cookie); +mi_decl_externc void _malloc_fork_prepare(void); +mi_decl_externc void _malloc_fork_parent(void); +mi_decl_externc void _malloc_fork_child(void); + + +static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { + MI_UNUSED(size); MI_UNUSED(flags); + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_zone (void) { + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { + return mi_get_default_zone(); +} + +static void mi_malloc_destroy_zone(malloc_zone_t* zone) { + MI_UNUSED(zone); + // nothing. +} + +static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { + MI_UNUSED(task); MI_UNUSED(mr); + if (addresses != NULL) *addresses = NULL; + if (count != NULL) *count = 0; + return KERN_SUCCESS; +} + +static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { + return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); +} + +static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { + MI_UNUSED(zone); MI_UNUSED(name); +} + +static int mi_malloc_jumpstart(uintptr_t cookie) { + MI_UNUSED(cookie); + return 1; // or 0 for no error? +} + +static void mi__malloc_fork_prepare(void) { + // nothing +} +static void mi__malloc_fork_parent(void) { + // nothing +} +static void mi__malloc_fork_child(void) { + // nothing +} + +static void mi_malloc_printf(const char* fmt, ...) { + MI_UNUSED(fmt); +} + +static bool zone_check(malloc_zone_t* zone) { + MI_UNUSED(zone); + return true; +} + +static malloc_zone_t* zone_from_ptr(const void* p) { + MI_UNUSED(p); + return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL); +} + +static void zone_log(malloc_zone_t* zone, void* p) { + MI_UNUSED(zone); MI_UNUSED(p); +} + +static void zone_print(malloc_zone_t* zone, bool b) { + MI_UNUSED(zone); MI_UNUSED(b); +} + +static void zone_print_ptr_info(void* p) { + MI_UNUSED(p); +} + +static void zone_register(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + +static void zone_unregister(malloc_zone_t* zone) { + MI_UNUSED(zone); +} + +// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` +// See: +struct mi_interpose_s { + const void* replacement; + const void* target; +}; +#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } +#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) +#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) +__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = +{ + + MI_INTERPOSE_MI(malloc_create_zone), + MI_INTERPOSE_MI(malloc_default_purgeable_zone), + MI_INTERPOSE_MI(malloc_default_zone), + MI_INTERPOSE_MI(malloc_destroy_zone), + MI_INTERPOSE_MI(malloc_get_all_zones), + MI_INTERPOSE_MI(malloc_get_zone_name), + MI_INTERPOSE_MI(malloc_jumpstart), + MI_INTERPOSE_MI(malloc_printf), + MI_INTERPOSE_MI(malloc_set_zone_name), + MI_INTERPOSE_MI(_malloc_fork_child), + MI_INTERPOSE_MI(_malloc_fork_parent), + MI_INTERPOSE_MI(_malloc_fork_prepare), + + MI_INTERPOSE_ZONE(zone_batch_free), + MI_INTERPOSE_ZONE(zone_batch_malloc), + MI_INTERPOSE_ZONE(zone_calloc), + MI_INTERPOSE_ZONE(zone_check), + MI_INTERPOSE_ZONE(zone_free), + MI_INTERPOSE_ZONE(zone_from_ptr), + MI_INTERPOSE_ZONE(zone_log), + MI_INTERPOSE_ZONE(zone_malloc), + MI_INTERPOSE_ZONE(zone_memalign), + MI_INTERPOSE_ZONE(zone_print), + MI_INTERPOSE_ZONE(zone_print_ptr_info), + MI_INTERPOSE_ZONE(zone_realloc), + MI_INTERPOSE_ZONE(zone_register), + MI_INTERPOSE_ZONE(zone_unregister), + MI_INTERPOSE_ZONE(zone_valloc) +}; + + +#else + +// ------------------------------------------------------ +// hook into the zone api's without interposing +// This is the official way of adding an allocator but +// it seems less robust than using interpose. +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + // The first returned zone is the real default + malloc_zone_t** zones = NULL; + unsigned count = 0; + kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); + if (ret == KERN_SUCCESS && count > 0) { + return zones[0]; + } + else { + // fallback + return malloc_default_zone(); + } +} + +#if defined(__clang__) +__attribute__((constructor(101))) // highest priority +#else +__attribute__((constructor)) // priority level is not supported by gcc +#endif +__attribute__((used)) +static void _mi_macos_override_malloc(void) { + malloc_zone_t* purgeable_zone = NULL; + + #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) + // force the purgeable zone to exist to avoid strange bugs + if (malloc_default_purgeable_zone) { + purgeable_zone = malloc_default_purgeable_zone(); + } + #endif + + // Register our zone. + // thomcc: I think this is still needed to put us in the zone list. + malloc_zone_register(&mi_malloc_zone); + // Unregister the default zone, this makes our zone the new default + // as that was the last registered. + malloc_zone_t *default_zone = mi_get_default_zone(); + // thomcc: Unsure if the next test is *always* false or just false in the + // cases I've tried. I'm also unsure if the code inside is needed. at all + if (default_zone != &mi_malloc_zone) { + malloc_zone_unregister(default_zone); + + // Reregister the default zone so free and realloc in that zone keep working. + malloc_zone_register(default_zone); + } + + // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs + // earlier than the default zone. + if (purgeable_zone != NULL) { + malloc_zone_unregister(purgeable_zone); + malloc_zone_register(purgeable_zone); + } + +} +#endif // MI_OSX_INTERPOSE + +#endif // MI_MALLOC_OVERRIDE diff --git a/csrc/mimalloc/src/prim/osx/prim.c b/csrc/mimalloc/src/prim/osx/prim.c new file mode 100644 index 000000000..8a2f4e8aa --- /dev/null +++ b/csrc/mimalloc/src/prim/osx/prim.c @@ -0,0 +1,9 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// We use the unix/prim.c with the mmap API on macOSX +#include "../unix/prim.c" diff --git a/csrc/mimalloc/src/prim/prim-tls.c b/csrc/mimalloc/src/prim/prim-tls.c new file mode 100644 index 000000000..8c8b0c71f --- /dev/null +++ b/csrc/mimalloc/src/prim/prim-tls.c @@ -0,0 +1,252 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" + +// -------------------------------------------------------------------------- +// Implement fast access to the thread local storage for `_mi_theap_default()` +// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info +// on the TLS models. +// -------------------------------------------------------------------------- + +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + + +#if MI_TLS_MODEL_LOCAL +// the thread-local main theap for allocation +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty; +// the last used non-main theap +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty; +#endif + +mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL; + +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { + const mi_threadid_t tid = _mi_prim_thread_id(); + mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits + return tid; +} + +// ---------------------------------------------------------------------------- +// Setting the default and cached theap +// ---------------------------------------------------------------------------- + +#if MI_TLS_MODEL_WIN32 + +// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) +// See for the offsets. +#if MI_SIZE_SIZE==4 +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#else +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#endif +#define MI_TLS_DIRECT_SLOTS (64) +#define MI_TLS_EXPANSION_SLOTS (1024) + +#if !MI_WIN_DIRECT_TLS +// We initially use the last of the expansion slots as the default NULL. +// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc +// before we are initialized :-( (but this seems quite unlikely). +// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL) +#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) +#else +// With direct tls we need an initial NULL slot outside the expansion slots +#define MI_TLS_INITIAL_SLOT (5) // Arbitrary user pointer +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) // unused +#endif + +// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways) +#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer +#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation) + + +mi_decl_hidden mi_decl_cache_align _Atomic(size_t) _mi_theap_default_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); + +static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES; +static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES; + +static bool mi_win_tls_slot_alloc(_Atomic(size_t)* slot, _Atomic(size_t)* extended, DWORD* raw_index) { + // always write slot before extended due to concurrent readers + const DWORD index = TlsAlloc(); + *raw_index = index; + if (index==TLS_OUT_OF_INDEXES) { + mi_atomic_store_release(slot,MI_TLS_ERROR_SLOT); + mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT); + return false; + } + else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT); + if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) { + mi_prim_tls_slot_set(slot, value); + } + else { + mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS); + TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed + } +} + +#elif MI_TLS_MODEL_PTHREADS + +// only for pthreads for now +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); + +static void mi_theap_cached_key_destroy(void* theapv) { + mi_theap_t* theap = (mi_theap_t*)theapv; + if (theap!=NULL) { + _mi_theap_decref(theap); + } +} + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + pthread_key_t key; + if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); } + if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); } + } +} + +void _mi_tls_slots_done(void) { + pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_theap_default_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } + key = mi_atomic_exchange_relaxed(&_mi_theap_cached_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } +} + +#elif MI_TLS_MODEL_FIXED + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + mi_theap_t* theap = _mi_theap_default(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap); + } + theap = _mi_theap_cached(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap); + } + } +} + +void _mi_tls_slots_done(void) { + // nothing +} + + +#else + +void _mi_tls_slots_init(void) { + // nothing +} + +void _mi_tls_slots_done(void) { + // nothing +} + +#endif + +void _mi_theap_cached_set(mi_theap_t* theap) { + mi_theap_t* prev = _mi_theap_cached(); + if (prev==theap) return; + // set + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_cached = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_cached_slot), mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + // update refcounts (so cached theap memory keeps available until no longer cached) + _mi_theap_incref(theap); + _mi_theap_decref(prev); +} + +void _mi_theap_default_set(mi_theap_t* theap) { + mi_assert_internal(theap != NULL); + mi_assert_internal(theap->tld != NULL); + mi_assert_internal(mi_theap_matches_thread(theap)); + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_default = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_default_slot), mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + + // set theap main if needed + if (mi_theap_is_initialized(theap)) { + // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again) + _mi_prim_thread_associate_default_theap(theap); + } +} diff --git a/csrc/mimalloc/src/prim/prim.c b/csrc/mimalloc/src/prim/prim.c new file mode 100644 index 000000000..7f31c994d --- /dev/null +++ b/csrc/mimalloc/src/prim/prim.c @@ -0,0 +1,76 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Select the implementation of the primitives +// depending on the OS. + +#if defined(_WIN32) +#include "windows/prim.c" // VirtualAlloc (Windows) + +#elif defined(__APPLE__) +#include "osx/prim.c" // macOSX (actually defers to mmap in unix/prim.c) + +#elif defined(__wasi__) +#define MI_USE_SBRK +#include "wasi/prim.c" // memory-grow or sbrk (Wasm) + +#elif defined(__EMSCRIPTEN__) +#include "emscripten/prim.c" // emmalloc_*, + pthread support + +#else +#include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) + +#endif + +// Generic process initialization +#if !defined(MI_PRIM_HAS_PROCESS_ATTACH) +#if defined(__GNUC__) || defined(__clang__) + // gcc,clang: use the constructor/destructor attribute + // which for both seem to run before regular constructors/destructors + #if defined(__clang__) + #define mi_attr_constructor __attribute__((constructor(101))) + #define mi_attr_destructor __attribute__((destructor(101))) + #else + #define mi_attr_constructor __attribute__((constructor)) + #define mi_attr_destructor __attribute__((destructor)) + #endif + static void mi_attr_constructor mi_process_attach(void) { + _mi_auto_process_init(); + } + static void mi_attr_destructor mi_process_detach(void) { + _mi_auto_process_done(); + } +#elif defined(__cplusplus) + // C++: use static initialization to detect process start/end + // This is not guaranteed to be first/last but the best we can generally do? + struct mi_init_done_t { + mi_init_done_t() { + _mi_auto_process_init(); + } + ~mi_init_done_t() { + _mi_auto_process_done(); + } + }; + static mi_init_done_t mi_init_done; + #else + #pragma message("define a way to call _mi_auto_process_init/done on your platform") +#endif +#endif + +// Generic allocator init/done callback +#ifndef MI_PRIM_HAS_ALLOCATOR_INIT +bool _mi_is_redirected(void) { + return false; +} +bool _mi_allocator_init(const char** message) { + if (message != NULL) { *message = NULL; } + return true; +} +void _mi_allocator_done(void) { + // nothing to do +} +#endif diff --git a/csrc/mimalloc/src/prim/readme.md b/csrc/mimalloc/src/prim/readme.md new file mode 100644 index 000000000..02cb1b90c --- /dev/null +++ b/csrc/mimalloc/src/prim/readme.md @@ -0,0 +1,12 @@ +## Portability Primitives + +This is the portability layer where all primitives needed from the OS are defined. + +- `include/mimalloc/prim.h`: primitive portability API definition. +- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform + (and on macOS, `osx/prim.c` defers to `unix/prim.c`). + +- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc) +- `prim-tls.c`: TLS model implementation + +Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/csrc/mimalloc/src/prim/unix/prim.c b/csrc/mimalloc/src/prim/unix/prim.c new file mode 100644 index 000000000..bcb065b2d --- /dev/null +++ b/csrc/mimalloc/src/prim/unix/prim.c @@ -0,0 +1,1065 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined +#endif + +#if defined(__sun) +// illumos provides new mman.h api when any of these are defined +// otherwise the old api based on caddr_t which predates the void pointers one. +// stock solaris provides only the former, chose to atomically to discard those +// flags only here rather than project wide tough. +#undef _XOPEN_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" + +#include // mmap +#include // sysconf, sleep +#include // open, close, read, access +#include // getenv, arc4random_buf + +#if defined(__linux__) + #include + #include // THP disable, PR_SET_VMA + #include // sysinfo + #if defined(__GLIBC__) && !defined(PR_SET_VMA) + #include + #endif + #if defined(__GLIBC__) + #include // linux mmap flags + #else + #include + #endif + #if defined(__riscv) || defined(_M_RISCV) + #if defined(MI_HAS_SYS_HWPROBEH) + #include + #elif defined(MI_HAS_ASM_HWPROBEH) + #include + #endif + #endif +#elif defined(__APPLE__) + #include + #include + #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR) + #include // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc. + #endif + #if !defined(MAC_OS_X_VERSION_10_7) + #define MAC_OS_X_VERSION_10_7 1070 + #endif + #include +#elif defined(__FreeBSD__) || defined(__DragonFly__) + #include + #if __FreeBSD_version >= 1200000 + #include + #include + #endif + #include +#endif + +#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__) + #define MI_HAS_SYSCALL_H + #include +#endif + +#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED) // QNX +#define MADV_DONTNEED POSIX_MADV_DONTNEED +#endif +#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE) // QNX +#define MADV_FREE POSIX_MADV_FREE +#endif + +#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // todo: can we query the OS for this? + +//------------------------------------------------------------------------------------ +// Use syscalls for some primitives to allow for libraries that override open/read/close etc. +// and do allocation themselves; using syscalls prevents recursion when mimalloc is +// still initializing (issue #713) +// Declare inline to avoid unused function warnings. +//------------------------------------------------------------------------------------ + +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) + +static inline int mi_prim_open(const char* fpath, int open_flags) { + return syscall(SYS_open,fpath,open_flags,0); +} +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return syscall(SYS_read,fd,buf,bufsize); +} +static inline int mi_prim_close(int fd) { + return syscall(SYS_close,fd); +} +static inline int mi_prim_access(const char *fpath, int mode) { + return syscall(SYS_access,fpath,mode); +} + +#else + +static inline int mi_prim_open(const char* fpath, int open_flags) { + return open(fpath,open_flags); +} +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { + return read(fd,buf,bufsize); +} +static inline int mi_prim_close(int fd) { + return close(fd); +} +static inline int mi_prim_access(const char *fpath, int mode) { + return access(fpath,mode); +} + +#endif + + + +//--------------------------------------------- +// init +//--------------------------------------------- + +static bool unix_detect_overcommit(void) { + bool os_overcommit = true; + #if defined(__linux__) + int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd >= 0) { + char buf[32]; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } + } + #elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } + #else + // default: overcommit is true + #endif + return os_overcommit; +} + +static bool unix_detect_thp(void) { + bool thp_enabled = false; + #if defined(__linux__) + int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); + if (fd >= 0) { + char buf[64]; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + // + // between brackets is the current value, for example: always [madvise] never + if (nread >= 1) { + if (nread > 64) { nread = 64; } + thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL); + } + } + #endif + return thp_enabled; +} + +// try to detect the physical memory dynamically (if possible) +static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) { + #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS + MI_UNUSED(page_size); + int64_t physical_memory = 0; + size_t length = sizeof(int64_t); + #if defined(HW_PHYSMEM64) + int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + #else + int mib[2] = { CTL_HW, HW_MEMSIZE }; + #endif + const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0); + if (err==0 && physical_memory > 0) { + const int64_t phys_in_kib = physical_memory / MI_KiB; + if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) { + *physical_memory_in_kib = (size_t)phys_in_kib; + } + } + #elif defined(__linux__) + MI_UNUSED(page_size); + struct sysinfo info; _mi_memzero_var(info); + const int err = sysinfo(&info); + if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) { + if (info.mem_unit==MI_KiB) { + *physical_memory_in_kib = (size_t)info.totalram; + } + else { + size_t total = 0; + if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) { + *physical_memory_in_kib = (total / MI_KiB); + } + } + } + #elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100) + const long pphys = sysconf(_SC_PHYS_PAGES); + const size_t psize_in_kib = page_size / MI_KiB; + if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { + *physical_memory_in_kib = (size_t)pphys * psize_in_kib; + } + #endif +} + +// Detect the virtual address bits (currently Linux/RISC-V only) +static size_t unix_detect_virtual_address_bits(void) { + #if defined(__riscv) || defined(_M_RISCV) + #if defined(RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS) + struct riscv_hwprobe probe = { .key = RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS, }; + // Prefer the GNU libc interface if available, as it can also use the VDSO + #if defined(MI_HAS_SYS_HWPROBEH) + if (__riscv_hwprobe(&probe, 1, 0, NULL, 0) == 0) + #else + if (syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0) == 0) + #endif + { + if (probe.key != -1) { // If a key is unknown to the kernel, its key field will be cleared to -1. + return (MI_SIZE_BITS - mi_clz((uintptr_t)probe.value)); + } + } + #endif + // Fallback to checking /proc/cpuinfo for older kernels + const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY); + if (fd >= 0) { + char buf[2048]; + const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + if ((nread >= 1) && (nread <= (ssize_t)sizeof(buf))) { + if (_mi_strnstr(buf, nread, "sv39")) { return 39; } + else if (_mi_strnstr(buf, nread, "sv48")) { return 48; } + else if (_mi_strnstr(buf, nread, "sv57")) { return 57; } + } + } + #endif // riscv + // default + return MI_MAX_VABITS; +} + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) +{ + long psize = sysconf(_SC_PAGESIZE); + if (psize > 0 && (unsigned long)psize < SIZE_MAX) { + config->page_size = (size_t)psize; + config->alloc_granularity = (size_t)psize; + unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib); + } + config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE; + config->has_overcommit = unix_detect_overcommit(); + config->has_partial_free = true; // mmap can free in parts + config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE) + config->has_transparent_huge_pages = unix_detect_thp(); + config->virtual_address_bits = unix_detect_virtual_address_bits(); + + // disable transparent huge pages for this process? + #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE) + if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option + { + config->has_transparent_huge_pages = false; + if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { // -1 on error, 1 if already disabled + // Most likely since distros often come with always/madvise settings. + // Disabling only for mimalloc process rather than touching system wide settings + (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0); + } + } + #endif +} + + +//--------------------------------------------- +// free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + if (size==0) return 0; + bool err = (munmap(addr, size) == -1); + return (err ? errno : 0); +} + + +//--------------------------------------------- +// mmap +//--------------------------------------------- + +// return errno on failure +static int unix_madvise(void* addr, size_t size, int advice) { + #if defined(__sun) || defined(_AIX) + const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) + return (res==0 ? 0 : errno); + #elif defined(__QNX__) + return posix_madvise(addr, size, advice); // posix returns errno + #else + const int res = madvise(addr, size, advice); // linux returns -1 on failure and sets errno + return (res==0 ? 0 : errno); + #endif +} + +static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) { + void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */); + #if defined(__linux__) && defined(PR_SET_VMA) + if (p!=MAP_FAILED && p!=NULL) { + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc"); + } + #endif + return p; +} + +static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + MI_UNUSED(try_alignment); + void* p = NULL; + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = 0; + mi_bsr(try_alignment, &n); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + int err = errno; + _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); + } + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { + p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); // addr parameter is the required alignment + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif + #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + p = unix_mmap_prim(hint, size, protect_flags, flags, fd); + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? + int err = 0; + #else + int err = errno; + #endif + _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + } + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #endif + // regular mmap + p = unix_mmap_prim(addr, size, protect_flags, flags, fd); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; +} + +static int unix_mmap_fd(void) { + #if defined(VM_MAKE_TAG) + // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) + int os_tag = (int)mi_option_get(mi_option_os_tag); + if (os_tag < 100 || os_tag > 255) { os_tag = 254; } + return VM_MAKE_TAG(os_tag); + #else + return -1; + #endif +} + +#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) +#define MI_OS_HAS_HUGE_PAGES 1 +#endif + +static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { + #if !defined(MAP_ANONYMOUS) + #define MAP_ANONYMOUS MAP_ANON + #endif + #if !defined(MAP_NORESERVE) + #define MAP_NORESERVE 0 + #endif + void* p = NULL; + const int fd = unix_mmap_fd(); + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + if (_mi_os_has_overcommit()) { + flags |= MAP_NORESERVE; + } + #if defined(PROT_MAX) + protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD + #endif + // huge page allocation + #if MI_OS_HAS_HUGE_PAGES + if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) { + static _Atomic(size_t) large_page_try_ok; // = 0; + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // If the OS is not configured for large OS pages, or the user does not have + // enough permission, the `mmap` will always fail (but it might also fail for other reasons). + // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times + // to avoid too many failing calls to mmap. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux + int lfd = fd; + #ifdef MAP_ALIGNED_SUPER + lflags |= MAP_ALIGNED_SUPER; + #endif + #ifdef MAP_HUGETLB + lflags |= MAP_HUGETLB; + #endif + #ifdef MAP_HUGE_1GB + static _Atomic(size_t) mi_huge_1gib_pages_unavailable; + if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&mi_huge_1gib_pages_unavailable)==0)) { + lflags |= MAP_HUGE_1GB; + } + else + #endif + { + #ifdef MAP_HUGE_2MB + lflags |= MAP_HUGE_2MB; + #endif + } + #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB + lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; + #endif + if (large_only || lflags != flags) { + // try large OS page allocation + *is_large = true; + p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd); + #ifdef MAP_HUGE_1GB + if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) { + mi_atomic_store_relaxed(&mi_huge_1gib_pages_unavailable,1); // don't try huge 1GiB pages again + if (large_only) { + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); + } + lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); + p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd); + } + #endif + if (large_only) return p; + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations + } + } + } + } // huge pages + #else + MI_UNUSED(large_only); + #endif + // regular allocation + if (p == NULL) { + *is_large = false; + p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd); + #if !defined(MI_NO_THP) + if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; + #elif defined(__sun) + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = _mi_os_large_page_size(); + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + // *is_large = true; // possibly + } + #endif + } + #endif + } + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + *is_zero = true; + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : errno); +} + + +//--------------------------------------------- +// Commit/Reset +//--------------------------------------------- + +static void unix_mprotect_hint(int err) { + #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page + if (err == ENOMEM) { + _mi_warning_message("The next warning may be caused by a low memory map limit.\n" + " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" + " For example: sudo sysctl -w vm.max_map_count=262144\n"); + } + #else + MI_UNUSED(err); + #endif +} + + + + + +int _mi_prim_commit(void* start, size_t size, bool* is_zero) { + // commit: ensure we can access the area + // note: we may think that *is_zero can be true since the memory + // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but + // we sometimes call commit on a range with still partially committed + // memory and `mprotect` does not zero the range. + *is_zero = false; + int err = mprotect(start, size, (PROT_READ | PROT_WRITE)); + if (err != 0) { + err = errno; + unix_mprotect_hint(err); + } + return err; +} + +int _mi_prim_reuse(void* start, size_t size) { + MI_UNUSED(start); MI_UNUSED(size); + #if defined(__APPLE__) && defined(MADV_FREE_REUSE) + return unix_madvise(start, size, MADV_FREE_REUSE); + #endif + return 0; +} + +int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { + int err = 0; + #if 1 + #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) + err = unix_madvise(start, size, MADV_FREE_REUSABLE); + if (err) { err = unix_madvise(start, size, MADV_DONTNEED); } + #else + // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + err = unix_madvise(start, size, MADV_DONTNEED); + #endif + #if !MI_DEBUG && MI_SECURE<=2 + *needs_recommit = false; + #else + *needs_recommit = true; + mprotect(start, size, PROT_NONE); + #endif + #else + // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss) + *needs_recommit = true; + const int fd = unix_mmap_fd(); + void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); + if (p != start) { err = errno; } + #endif + return err; +} + +int _mi_prim_reset(void* start, size_t size) { + int err = 0; + + // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower) + #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + err = unix_madvise(start, size, MADV_FREE_REUSABLE); + if (err==0) return 0; + // fall through + #endif + + #if defined(MADV_FREE) + // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // will not reduce the `rss` stats in tools like `top` even though the memory is available + // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by + // default `MADV_DONTNEED` is used though. + static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); + int oadvice = (int)mi_atomic_load_relaxed(&advice); + while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ }; + if (err == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); + err = unix_madvise(start, size, MADV_DONTNEED); + } + #else + err = unix_madvise(start, size, MADV_DONTNEED); + #endif + return err; +} + +int _mi_prim_protect(void* start, size_t size, bool protect) { + int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); + if (err != 0) { err = errno; } + unix_mprotect_hint(err); + return err; +} + + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) + +#ifndef MPOL_PREFERRED +#define MPOL_PREFERRED 1 +#endif + +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind) +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags); +} +#else +static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { + MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); + return 0; +} +#endif + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + bool is_large = true; + *is_zero = true; + *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes + unsigned long numa_mask = (1UL << numa_node); + // todo: does `mbind` work correctly for huge OS pages? should we + // use `set_mempolicy` before calling mmap instead? + // see: + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + if (err != 0) { + err = errno; + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err); + } + } + return (*addr != NULL ? 0 : errno); +} + +#else + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *is_zero = false; + *addr = NULL; + return ENOMEM; +} + +#endif + +//--------------------------------------------- +// NUMA nodes +//--------------------------------------------- + +#if defined(__linux__) + +size_t _mi_prim_numa_node(void) { + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu) + unsigned int node = 0; + unsigned int ncpu = 0; + int err = syscall(SYS_getcpu, &ncpu, &node, NULL); + if (err != 0) return 0; + return node; + #else + return 0; + #endif +} + +size_t _mi_prim_numa_node_count(void) { + char buf[128]; + unsigned last_found = 0; + for(unsigned node = 1; node < 256; node++) { + // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) + _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node); + if (mi_prim_access(buf,R_OK) != 0) { + if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4 + } + else { last_found = node; } // highest found node + } + return last_found + 1; +} + +#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 + +size_t _mi_prim_numa_node(void) { + domainset_t dom; + size_t node; + int policy; + if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; + for (node = 0; node < MAXMEMDOM; node++) { + if (DOMAINSET_ISSET(node, &dom)) return node; + } + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ndomains = 0; + size_t len = sizeof(ndomains); + if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; + return ndomains; +} + +#elif defined(__DragonFly__) + +size_t _mi_prim_numa_node(void) { + // todo: DragonFly does not seem to provide any userland means to get this information. + return 0ul; +} + +size_t _mi_prim_numa_node_count(void) { + size_t ncpus = 0, nvirtcoresperphys = 0; + size_t len = sizeof(size_t); + if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; + if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; + return nvirtcoresperphys * ncpus; +} + +#else + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + +#endif + +// ---------------------------------------------------------------- +// Clock +// ---------------------------------------------------------------- + +#include + +// low resolution timer +static mi_msecs_t mi_prim_clock_now_lowres(void) { + const int64_t ticks = (int64_t)clock(); + #if !defined(CLOCKS_PER_SEC) + return ticks; + #else + if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { + return ticks; + } + else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) { + return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + } + else { + return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + } + #endif +} + +mi_msecs_t _mi_prim_clock_now(void) { + #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + #ifdef CLOCK_MONOTONIC + const clockid_t clockid = CLOCK_MONOTONIC; + #else + const clockid_t clockid = CLOCK_REALTIME; + #endif + struct timespec t; + if (clock_gettime(clockid,&t) == 0) { + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); + } + #endif + return mi_prim_clock_now_lowres(); +} + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) +#include +#include +#include + +#if defined(__APPLE__) +#include +#endif + +#if defined(__HAIKU__) +#include +#endif + +static mi_msecs_t timeval_secs(const struct timeval* tv) { + return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L); +} + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + struct rusage rusage; + if (getrusage(RUSAGE_SELF, &rusage) == 0) { + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); + #if !defined(__HAIKU__) + pinfo->page_faults = rusage.ru_majflt; + #endif + #if defined(__APPLE__) + pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes + #else + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB + #endif + } + + #if defined(__HAIKU__) + // Haiku does not have (yet?) a way to + // get these stats per process + thread_info tid; + if (get_thread_info(find_thread(0), &tid) == B_OK) { + area_info mem; + ssize_t c; + while (get_next_area_info(tid.team, &c, &mem) == B_OK) { + pinfo->peak_rss += mem.ram_size; + } + } + pinfo->page_faults = 0; + #elif defined(__APPLE__) + #ifdef MACH_TASK_BASIC_INFO + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #else + struct task_basic_info info; + mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #endif + #endif + // use defaults for commit +} + +#else + +#ifndef __wasi__ +// WebAssembly instances are not processes +#pragma message("define a way to get process info") +#endif + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + // use defaults + MI_UNUSED(pinfo); +} + +#endif + + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) +// On Posix systemsr use `environ` to access environment variables +// even before the C runtime is initialized. +#if defined(__APPLE__) && defined(__has_include) && __has_include() +#include +static char** mi_get_environ(void) { + return (*_NSGetEnviron()); +} +#else +extern char** environ; +static char** mi_get_environ(void) { + return environ; +} +#endif +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + if (name==NULL) return -1; + const size_t len = _mi_strlen(name); + if (len == 0) return -1; + char** env = mi_get_environ(); + if (env == NULL) return -1; + // compare up to 10000 entries + for (int i = 0; i < 10000 && env[i] != NULL; i++) { + const char* s = env[i]; + if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive + // found it + if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1; + return 1; // success + } + } + return 0; // not found +} +#else +// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return -1; // error, try again later + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found + if (!_mi_strlcpy(result, s, result_size)) return -1; + return 1; // success +} +#endif // !MI_USE_ENVIRON + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15) +#include +#include + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf + // may fail silently on macOS. See PR #390, and + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); +} + +#elif defined(__ANDROID__) || defined(__DragonFly__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__sun) || \ + (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7)) + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + arc4random_buf(buf, buf_len); + return true; +} + +#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__) // also for old apple versions < 10.7 (issue #829) + +#include +#include +#include + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h` + // and for the latter the actual `getrandom` call is not always defined. + // (see ) + // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed. + #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom) + #ifndef GRND_NONBLOCK + #define GRND_NONBLOCK (1) + #endif + static _Atomic(uintptr_t) no_getrandom; // = 0 + if (mi_atomic_load_acquire(&no_getrandom)==0) { + ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); + if (ret >= 0) return (buf_len == (size_t)ret); + if (errno != ENOSYS) return false; + mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom + } + #endif + int flags = O_RDONLY; + #if defined(O_CLOEXEC) + flags |= O_CLOEXEC; + #endif + int fd = mi_prim_open("/dev/urandom", flags); + if (fd < 0) return false; + size_t count = 0; + while(count < buf_len) { + ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count); + if (ret==0) { + break; + } + else if (ret<0) { + if (errno!=EAGAIN && errno!=EINTR) break; + } + else { + count += ret; + } + } + mi_prim_close(fd); + return (count==buf_len); +} + +#else + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} + +#endif + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +#if defined(MI_USE_PTHREADS) + +// use pthread local storage keys to detect thread ending +// (and used with MI_TLS_PTHREADS for the default theap) +pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + +static void mi_pthread_done(void* value) { + if (value!=NULL) { + _mi_thread_done((mi_theap_t*)value); + } +} + +void _mi_prim_thread_init_auto_done(void) { + mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); + pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); +} + +void _mi_prim_thread_done_auto_done(void) { + pthread_key_t key = _mi_heap_default_key; + if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 + _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); + } +} + +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD + pthread_setspecific(_mi_heap_default_key, theap); + } +} + + +#else + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); +} + +#endif + +bool _mi_prim_thread_is_in_threadpool(void) { + return false; +} + +void _mi_prim_thread_yield(void) { + sleep(0); +} diff --git a/csrc/mimalloc/src/prim/wasi/prim.c b/csrc/mimalloc/src/prim/wasi/prim.c new file mode 100644 index 000000000..2d69b7245 --- /dev/null +++ b/csrc/mimalloc/src/prim/wasi/prim.c @@ -0,0 +1,297 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" + +#include // fputs +#include // getenv +#include // sbrk, sleep + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { + config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB + config->alloc_granularity = 16; + config->has_overcommit = false; + config->has_partial_free = false; + config->has_virtual_reserve = false; +} + +//--------------------------------------------- +// Free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(addr); MI_UNUSED(size); + // wasi theap cannot be shrunk + return 0; +} + + +//--------------------------------------------- +// Allocation: sbrk or memory_grow +//--------------------------------------------- + +#if defined(MI_USE_SBRK) + static void* mi_memory_grow( size_t size ) { + void* p = sbrk(size); + if (p == (void*)(-1)) return NULL; + #if !defined(__wasi__) // on wasi this is always zero initialized already (?) + memset(p,0,size); + #endif + return p; + } +#elif defined(__wasi__) + static void* mi_memory_grow( size_t size ) { + size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) + : __builtin_wasm_memory_size(0)); + if (base == SIZE_MAX) return NULL; + return (void*)(base * _mi_os_page_size()); + } +#endif + +#if defined(MI_USE_PTHREADS) +static pthread_mutex_t mi_theap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + +static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { + void* p = NULL; + if (try_alignment <= 1) { + // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_theap_grow_mutex); + #endif + p = mi_memory_grow(size); + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_theap_grow_mutex); + #endif + } + else { + void* base = NULL; + size_t alloc_size = 0; + // to allocate aligned use a lock to try to avoid thread interaction + // between getting the current size and actual allocation + // (also, `sbrk` is not thread safe in general) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_theap_grow_mutex); + #endif + { + void* current = mi_memory_grow(0); // get current size + if (current != NULL) { + void* aligned_current = _mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space + alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); + base = mi_memory_grow(alloc_size); + } + } + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_theap_grow_mutex); + #endif + if (base != NULL) { + p = _mi_align_up_ptr(base, try_alignment); + if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { + // another thread used wasm_memory_grow/sbrk in-between and we do not have enough + // space after alignment. Give up (and waste the space as we cannot shrink :-( ) + // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) + p = NULL; + } + } + } + /* + if (p == NULL) { + _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); + errno = ENOMEM; + return NULL; + } + */ + mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + return p; +} + +// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); + *is_large = false; + *is_zero = false; + *addr = mi_prim_mem_grow(size, try_alignment); + return (*addr != NULL ? 0 : ENOMEM); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- + +int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { + MI_UNUSED(addr); MI_UNUSED(size); + *is_zero = false; + return 0; +} + +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { + MI_UNUSED(addr); MI_UNUSED(size); + *needs_recommit = false; + return 0; +} + +int _mi_prim_reset(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_reuse(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); + return 0; +} + + +//--------------------------------------------- +// Huge pages and NUMA nodes +//--------------------------------------------- + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); + *is_zero = true; + *addr = NULL; + return ENOSYS; +} + +size_t _mi_prim_numa_node(void) { + return 0; +} + +size_t _mi_prim_numa_node_count(void) { + return 1; +} + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +#include + +// low resolution timer +static mi_msecs_t mi_prim_clock_now_lowres(void) { + const int64_t ticks = (int64_t)clock(); + #if !defined(CLOCKS_PER_SEC) + return ticks; + #else + if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { + return ticks; + } + else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) { + return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + } + else { + return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); + } + #endif +} + +mi_msecs_t _mi_prim_clock_now(void) { + #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) + #ifdef CLOCK_MONOTONIC + const clockid_t clockid = CLOCK_MONOTONIC; + #else + const clockid_t clockid = CLOCK_REALTIME; + #endif + struct timespec t; + if (clock_gettime(clockid,&t) == 0) { + return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); + } + #endif + return mi_prim_clock_now_lowres(); +} + + +//---------------------------------------------------------------- +// Process info +//---------------------------------------------------------------- + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + // use defaults + MI_UNUSED(pinfo); +} + + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) { + fputs(msg,stderr); +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + // cannot call getenv() when still initializing the C runtime. + if (_mi_preloading()) return -1; // error, try again later + const char* s = getenv(name); + if (s == NULL) { + // we check the upper case name too. + char buf[64+1]; + size_t len = _mi_strnlen(name,sizeof(buf)-1); + for (size_t i = 0; i < len; i++) { + buf[i] = _mi_toupper(name[i]); + } + buf[len] = 0; + s = getenv(buf); + } + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found + _mi_strlcpy(result, s, result_size); + return 1; // found +} + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return false; +} + + +//---------------------------------------------------------------- +// Thread init/done +//---------------------------------------------------------------- + +void _mi_prim_thread_init_auto_done(void) { + // nothing +} + +void _mi_prim_thread_done_auto_done(void) { + // nothing +} + +void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); +} + +bool _mi_prim_thread_is_in_threadpool(void) { + return false; +} + +void _mi_prim_thread_yield(void) { + sleep(0); +} diff --git a/csrc/mimalloc/src/prim/windows/etw-mimalloc.wprp b/csrc/mimalloc/src/prim/windows/etw-mimalloc.wprp new file mode 100644 index 000000000..b00cd7adf --- /dev/null +++ b/csrc/mimalloc/src/prim/windows/etw-mimalloc.wprp @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/csrc/mimalloc/src/prim/windows/etw.h b/csrc/mimalloc/src/prim/windows/etw.h new file mode 100644 index 000000000..4e0a092a1 --- /dev/null +++ b/csrc/mimalloc/src/prim/windows/etw.h @@ -0,0 +1,905 @@ +//**********************************************************************` +//* This is an include file generated by Message Compiler. *` +//* *` +//* Copyright (c) Microsoft Corporation. All Rights Reserved. *` +//**********************************************************************` +#pragma once + +//***************************************************************************** +// +// Notes on the ETW event code generated by MC: +// +// - Structures and arrays of structures are treated as an opaque binary blob. +// The caller is responsible for packing the data for the structure into a +// single region of memory, with no padding between values. The macro will +// have an extra parameter for the length of the blob. +// - Arrays of nul-terminated strings must be packed by the caller into a +// single binary blob containing the correct number of strings, with a nul +// after each string. The size of the blob is specified in characters, and +// includes the final nul. +// - Arrays of SID are treated as a single binary blob. The caller is +// responsible for packing the SID values into a single region of memory with +// no padding. +// - The length attribute on the data element in the manifest is significant +// for values with intype win:UnicodeString, win:AnsiString, or win:Binary. +// The length attribute must be specified for win:Binary, and is optional for +// win:UnicodeString and win:AnsiString (if no length is given, the strings +// are assumed to be nul-terminated). For win:UnicodeString, the length is +// measured in characters, not bytes. +// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the +// length attribute applies to every value in the array, so every value in +// the array must have the same length. The values in the array are provided +// to the macro via a single pointer -- the caller is responsible for packing +// all of the values into a single region of memory with no padding between +// values. +// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary can be generated and collected on Vista or later. +// However, they may not decode properly without the Windows 10 2018 Fall +// Update. +// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary must be packed by the caller into a single region of +// memory. The format for each item is a UINT16 byte-count followed by that +// many bytes of data. When providing the array to the generated macro, you +// must provide the total size of the packed array data, including the UINT16 +// sizes for each item. In the case of win:CountedUnicodeString, the data +// size is specified in WCHAR (16-bit) units. In the case of +// win:CountedAnsiString and win:CountedBinary, the data size is specified in +// bytes. +// +//***************************************************************************** + +#include +#include +#include + +#ifndef ETW_INLINE + #ifdef _ETW_KM_ + // In kernel mode, save stack space by never inlining templates. + #define ETW_INLINE DECLSPEC_NOINLINE __inline + #else + // In user mode, save code size by inlining templates as appropriate. + #define ETW_INLINE __inline + #endif +#endif // ETW_INLINE + +#if defined(__cplusplus) +extern "C" { +#endif + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_USE_KERNEL_MODE_APIS macro: +// Controls whether the generated code uses kernel-mode or user-mode APIs. +// - Set to 0 to use Windows user-mode APIs such as EventRegister. +// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister. +// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h). +// Note that the APIs can also be overridden directly, e.g. by setting the +// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros. +// +#ifndef MCGEN_USE_KERNEL_MODE_APIS + #ifdef _ETW_KM_ + #define MCGEN_USE_KERNEL_MODE_APIS 1 + #else + #define MCGEN_USE_KERNEL_MODE_APIS 0 + #endif +#endif // MCGEN_USE_KERNEL_MODE_APIS + +// +// MCGEN_HAVE_EVENTSETINFORMATION macro: +// Controls how McGenEventSetInformation uses the EventSetInformation API. +// - Set to 0 to disable the use of EventSetInformation +// (McGenEventSetInformation will always return an error). +// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION. +// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress +// (user-mode) or MmGetSystemRoutineAddress (kernel-mode). +// Default is determined as follows: +// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else if the target OS version has EventSetInformation, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else set to 2 (i.e. try to dynamically locate EventSetInformation). +// Note that an McGenEventSetInformation function will only be generated if one +// or more provider in a manifest has provider traits. +// +#ifndef MCGEN_HAVE_EVENTSETINFORMATION + #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs, + #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EtwSetInformation" via MmGetSystemRoutineAddress. + #endif // else (using user-mode APIs) + #else // if target OS and SDK is Windows 8 or later, + #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED) + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EventSetInformation" via GetModuleHandleExW/GetProcAddress. + #endif + #endif +#endif // MCGEN_HAVE_EVENTSETINFORMATION + +// +// MCGEN Override Macros +// +// The following override macros may be defined before including this header +// to control the APIs used by this header: +// +// - MCGEN_EVENTREGISTER +// - MCGEN_EVENTUNREGISTER +// - MCGEN_EVENTSETINFORMATION +// - MCGEN_EVENTWRITETRANSFER +// +// If the the macro is undefined, the MC implementation will default to the +// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is +// undefined, the EventRegister[MyProviderName] macro will use EventRegister +// in user mode and will use EtwRegister in kernel mode. +// +// To prevent issues from conflicting definitions of these macros, the value +// of the override macro will be used as a suffix in certain internal function +// names. Because of this, the override macros must follow certain rules: +// +// - The macro must be defined before any MC-generated header is included and +// must not be undefined or redefined after any MC-generated header is +// included. Different translation units (i.e. different .c or .cpp files) +// may set the macros to different values, but within a translation unit +// (within a single .c or .cpp file), the macro must be set once and not +// changed. +// - The override must be an object-like macro, not a function-like macro +// (i.e. the override macro must not have a parameter list). +// - The override macro's value must be a simple identifier, i.e. must be +// something that starts with a letter or '_' and contains only letters, +// numbers, and '_' characters. +// - If the override macro's value is the name of a second object-like macro, +// the second object-like macro must follow the same rules. (The override +// macro's value can also be the name of a function-like macro, in which +// case the function-like macro does not need to follow the same rules.) +// +// For example, the following will cause compile errors: +// +// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon). +// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses). +// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro. +// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon). +// +// The following would be ok: +// +// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be "MyEventWriteFunction1". +// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2 +// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be "MyEventWriteFunction2". +// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d) +// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be "MY_FUNCTION_LIKE_MACRO". +// +#ifndef MCGEN_EVENTREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTREGISTER EtwRegister + #else + #define MCGEN_EVENTREGISTER EventRegister + #endif +#endif // MCGEN_EVENTREGISTER +#ifndef MCGEN_EVENTUNREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTUNREGISTER EtwUnregister + #else + #define MCGEN_EVENTUNREGISTER EventUnregister + #endif +#endif // MCGEN_EVENTUNREGISTER +#ifndef MCGEN_EVENTSETINFORMATION + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTSETINFORMATION EtwSetInformation + #else + #define MCGEN_EVENTSETINFORMATION EventSetInformation + #endif +#endif // MCGEN_EVENTSETINFORMATION +#ifndef MCGEN_EVENTWRITETRANSFER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer + #else + #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer + #endif +#endif // MCGEN_EVENTWRITETRANSFER + +// +// MCGEN_EVENT_ENABLED macro: +// Override to control how the EventWrite[EventName] macros determine whether +// an event is enabled. The default behavior is for EventWrite[EventName] to +// use the EventEnabled[EventName] macros. +// +#ifndef MCGEN_EVENT_ENABLED +#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName() +#endif + +// +// MCGEN_EVENT_ENABLED_FORCONTEXT macro: +// Override to control how the EventWrite[EventName]_ForContext macros +// determine whether an event is enabled. The default behavior is for +// EventWrite[EventName]_ForContext to use the +// EventEnabled[EventName]_ForContext macros. +// +#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT +#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext) +#endif + +// +// MCGEN_ENABLE_CHECK macro: +// Determines whether the specified event would be considered as enabled +// based on the state of the specified context. Slightly faster than calling +// McGenEventEnabled directly. +// +#ifndef MCGEN_ENABLE_CHECK +#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor)) +#endif + +#if !defined(MCGEN_TRACE_CONTEXT_DEF) +#define MCGEN_TRACE_CONTEXT_DEF +// This structure is for use by MC-generated code and should not be used directly. +typedef struct _MCGEN_TRACE_CONTEXT +{ + TRACEHANDLE RegistrationHandle; + TRACEHANDLE Logger; // Used as pointer to provider traits. + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + ULONG Flags; + ULONG IsEnabled; + UCHAR Level; + UCHAR Reserve; + USHORT EnableBitsCount; + PULONG EnableBitMask; + const ULONGLONG* EnableKeyWords; + const UCHAR* EnableLevel; +} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT; +#endif // MCGEN_TRACE_CONTEXT_DEF + +#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF) +#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF +// +// Determines whether an event with a given Level and Keyword would be +// considered as enabled based on the state of the specified context. +// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this +// function directly. +// +FORCEINLINE +BOOLEAN +McGenLevelKeywordEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ UCHAR Level, + _In_ ULONGLONG Keyword + ) +{ + // + // Check if the event Level is lower than the level at which + // the channel is enabled. + // If the event Level is 0 or the channel is enabled at level 0, + // all levels are enabled. + // + + if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0. + (EnableInfo->Level == 0)) { + + // + // Check if Keyword is enabled + // + + if ((Keyword == (ULONGLONG)0) || + ((Keyword & EnableInfo->MatchAnyKeyword) && + ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) { + return TRUE; + } + } + + return FALSE; +} +#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF + +#if !defined(MCGEN_EVENT_ENABLED_DEF) +#define MCGEN_EVENT_ENABLED_DEF +// +// Determines whether the specified event would be considered as enabled based +// on the state of the specified context. Note that you may want to use +// MCGEN_ENABLE_CHECK instead of calling this function directly. +// +FORCEINLINE +BOOLEAN +McGenEventEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ PCEVENT_DESCRIPTOR EventDescriptor + ) +{ + return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword); +} +#endif // MCGEN_EVENT_ENABLED_DEF + +#if !defined(MCGEN_CONTROL_CALLBACK) +#define MCGEN_CONTROL_CALLBACK + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +VOID +__stdcall +McGenControlCallbackV2( + _In_ LPCGUID SourceId, + _In_ ULONG ControlCode, + _In_ UCHAR Level, + _In_ ULONGLONG MatchAnyKeyword, + _In_ ULONGLONG MatchAllKeyword, + _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData, + _Inout_opt_ PVOID CallbackContext + ) +/*++ + +Routine Description: + + This is the notification callback for Windows Vista and later. + +Arguments: + + SourceId - The GUID that identifies the session that enabled the provider. + + ControlCode - The parameter indicates whether the provider + is being enabled or disabled. + + Level - The level at which the event is enabled. + + MatchAnyKeyword - The bitmask of keywords that the provider uses to + determine the category of events that it writes. + + MatchAllKeyword - This bitmask additionally restricts the category + of events that the provider writes. + + FilterData - The provider-defined data. + + CallbackContext - The context of the callback that is defined when the provider + called EtwRegister to register itself. + +Remarks: + + ETW calls this function to notify provider of enable/disable + +--*/ +{ + PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext; + ULONG Ix; +#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + UNREFERENCED_PARAMETER(SourceId); + UNREFERENCED_PARAMETER(FilterData); +#endif + + if (Ctx == NULL) { + return; + } + + switch (ControlCode) { + + case EVENT_CONTROL_CODE_ENABLE_PROVIDER: + Ctx->Level = Level; + Ctx->MatchAnyKeyword = MatchAnyKeyword; + Ctx->MatchAllKeyword = MatchAllKeyword; + Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER; + + for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) { + if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) { + Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32)); + } else { + Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32)); + } + } + break; + + case EVENT_CONTROL_CODE_DISABLE_PROVIDER: + Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER; + Ctx->Level = 0; + Ctx->MatchAnyKeyword = 0; + Ctx->MatchAllKeyword = 0; + if (Ctx->EnableBitsCount > 0) { +#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount + RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG)); + } + break; + + default: + break; + } + +#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + // + // Call user defined callback + // + MCGEN_PRIVATE_ENABLE_CALLBACK_V2( + SourceId, + ControlCode, + Level, + MatchAnyKeyword, + MatchAllKeyword, + FilterData, + CallbackContext + ); +#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + + return; +} + +#endif // MCGEN_CONTROL_CALLBACK + +#ifndef _mcgen_PENABLECALLBACK + #if MCGEN_USE_KERNEL_MODE_APIS + #define _mcgen_PENABLECALLBACK PETWENABLECALLBACK + #else + #define _mcgen_PENABLECALLBACK PENABLECALLBACK + #endif +#endif // _mcgen_PENABLECALLBACK + +#if !defined(_mcgen_PASTE2) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b) +#define _mcgen_PASTE2_imp(a, b) a##b +#endif // _mcgen_PASTE2 + +#if !defined(_mcgen_PASTE3) +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c) +#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c +#endif // _mcgen_PASTE3 + +// +// Macro validation +// + +// Validate MCGEN_EVENTREGISTER: + +// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER); + +// Trigger an error if MCGEN_EVENTREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER) + MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER); + +// Validate MCGEN_EVENTUNREGISTER: + +// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER); + +// Trigger an error if MCGEN_EVENTUNREGISTER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER) + MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro: +typedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER; +typedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER); + +// Validate MCGEN_EVENTSETINFORMATION: + +// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION); + +// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION) + MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers; + +// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro: +typedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION; +typedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION); + +// Validate MCGEN_EVENTWRITETRANSFER: + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier: +struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER); + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined: +typedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER) + MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;; + +// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro: +typedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER; +typedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER); + +#ifndef McGenEventWrite_def +#define McGenEventWrite_def + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventWrite( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_opt_ LPCGUID ActivityId, + _In_range_(1, 128) ULONG EventDataCount, + _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData + ) +{ + const USHORT UNALIGNED* Traits; + + // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId. + UNREFERENCED_PARAMETER(ActivityId); + + Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger; + + if (Traits == NULL) { + EventData[0].Ptr = 0; + EventData[0].Size = 0; + EventData[0].Reserved = 0; + } else { + EventData[0].Ptr = (ULONG_PTR)Traits; + EventData[0].Size = *Traits; + EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA + } + + return MCGEN_EVENTWRITETRANSFER( + Context->RegistrationHandle, + Descriptor, + ActivityId, + NULL, + EventDataCount, + EventData); +} +#endif // McGenEventWrite_def + +#if !defined(McGenEventRegisterUnregister) +#define McGenEventRegisterUnregister + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER) + +#pragma warning(push) +#pragma warning(disable:6103) +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventRegister( + _In_ LPCGUID ProviderId, + _In_opt_ _mcgen_PENABLECALLBACK EnableCallback, + _In_opt_ PVOID CallbackContext, + _Inout_ PREGHANDLE RegHandle + ) +/*++ + +Routine Description: + + This function registers the provider with ETW. + +Arguments: + + ProviderId - Provider ID to register with ETW. + + EnableCallback - Callback to be used. + + CallbackContext - Context for the callback. + + RegHandle - Pointer to registration handle. + +Remarks: + + Should not be called if the provider is already registered (i.e. should not + be called if *RegHandle != 0). Repeatedly registering a provider is a bug + and may indicate a race condition. However, for compatibility with previous + behavior, this function will return SUCCESS in this case. + +--*/ +{ + ULONG Error; + + if (*RegHandle != 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle); + } + + return Error; +} +#pragma warning(pop) + +// This macro is for use by MC-generated code and should not be used directly. +#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER) + +// This function is for use by MC-generated code and should not be used directly. +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventUnregister(_Inout_ PREGHANDLE RegHandle) +/*++ + +Routine Description: + + Unregister from ETW and set *RegHandle = 0. + +Arguments: + + RegHandle - the pointer to the provider registration handle + +Remarks: + + If provider has not been registered (i.e. if *RegHandle == 0), + return SUCCESS. It is safe to call McGenEventUnregister even if the + call to McGenEventRegister returned an error. + +--*/ +{ + ULONG Error; + + if(*RegHandle == 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTUNREGISTER(*RegHandle); + *RegHandle = (REGHANDLE)0; + } + + return Error; +} + +#endif // McGenEventRegisterUnregister + +#ifndef _mcgen_EVENT_BIT_SET + #if defined(_M_IX86) || defined(_M_X64) + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0) + #else // CPU type + // This macro is for use by MC-generated code and should not be used directly. + #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0) + #endif // CPU type +#endif // _mcgen_EVENT_BIT_SET + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// Provider "microsoft-windows-mimalloc" event count 2 +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779 +EXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}}; + +#ifndef ETW_MI_Provider_Traits +#define ETW_MI_Provider_Traits NULL +#endif // ETW_MI_Provider_Traits + +// +// Event Descriptors +// +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_ALLOC_value 0x64 +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0}; +#define ETW_MI_FREE_value 0x65 + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Event Enablement Bits +// These variables are for use by MC-generated code and should not be used directly. +// +EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1]; +EXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0}; +EXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4}; + +// +// Provider context +// +EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels}; + +// +// Provider REGHANDLE +// +#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle) + +// +// This macro is set to 0, indicating that the EventWrite[Name] macros do not +// have an Activity parameter. This is controlled by the -km and -um options. +// +#define ETW_MI_Provider_EventWriteActivity 0 + +// +// Register with ETW using the control GUID specified in the manifest. +// Invoke this macro during module initialization (i.e. program startup, +// DLL process attach, or driver load) to initialize the provider. +// Note that if this function returns an error, the error means that +// will not work, but no action needs to be taken -- even if EventRegister +// returns an error, it is generally safe to use EventWrite and +// EventUnregister macros (they will be no-ops if EventRegister failed). +// +#ifndef EventRegistermicrosoft_windows_mimalloc +#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Register with ETW using a specific control GUID (i.e. a GUID other than what +// is specified in the manifest). Advanced scenarios only. +// +#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc +#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle) +#endif + +// +// Unregister with ETW and close the provider. +// Invoke this macro during module shutdown (i.e. program exit, DLL process +// detach, or driver unload) to unregister the provider. +// Note that you MUST call EventUnregister before DLL or driver unload +// (not optional): failure to unregister a provider before DLL or driver unload +// will result in crashes. +// +#ifndef EventUnregistermicrosoft_windows_mimalloc +#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle) +#endif + +// +// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro: +// Define this macro to enable support for caller-allocated provider context. +// +#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Advanced scenarios: Caller-allocated provider context. +// Use when multiple differently-configured provider handles are needed, +// e.g. for container-aware drivers, one context per container. +// +// Usage: +// +// - Caller enables the feature before including this header, e.g. +// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1 +// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc)); +// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...); +// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext); +// - Caller frees memory, e.g. free(pContext); +// + +typedef struct tagMcGenContext_microsoft_windows_mimalloc { + // The fields of this structure are subject to change and should + // not be accessed directly. To access the provider's REGHANDLE, + // use microsoft_windows_mimallocHandle_ForContext(pContext). + MCGEN_TRACE_CONTEXT Context; + ULONG EnableBits[1]; +} McGenContext_microsoft_windows_mimalloc; + +#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext) +#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext) +#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle) + +// +// Provider REGHANDLE for caller-allocated context. +// +#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle) + +// This function is for use by MC-generated code and should not be used directly. +// Initialize and register the caller-allocated context. +__inline +ULONG __stdcall +_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)( + _In_ LPCGUID pProviderId, + _Out_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + RtlZeroMemory(pContext, sizeof(*pContext)); + pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits; + pContext->Context.EnableBitsCount = 1; + pContext->Context.EnableBitMask = pContext->EnableBits; + pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords; + pContext->Context.EnableLevel = microsoft_windows_mimallocLevels; + return McGenEventRegister( + pProviderId, + McGenControlCallbackV2, + &pContext->Context, + &pContext->Context.RegistrationHandle); +} + +// This function is for use by MC-generated code and should not be used directly. +// Trigger a compile error if called with the wrong parameter type. +FORCEINLINE +_Ret_ McGenContext_microsoft_windows_mimalloc* +_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext) +{ + return pContext; +} + +#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION + +// +// Enablement check macro for event "ETW_MI_ALLOC" +// +#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_ALLOC" +// +#define EventWriteETW_MI_ALLOC(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) +#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0 +#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +// +// Enablement check macro for event "ETW_MI_FREE" +// +#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0) +#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0) + +// +// Event write macros for event "ETW_MI_FREE" +// +#define EventWriteETW_MI_FREE(Address, Size) \ + MCGEN_EVENT_ENABLED(ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) +#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \ + MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \ + ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0 +#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \ + _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size) + +// This macro is for use by MC-generated code and should not be used directly. +#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER) + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Template Functions +// + +// +// Function for template "ETW_CUSTOM_HEAP_ALLOC_DATA" (and possibly others). +// This function is for use by MC-generated code and should not be used directly. +// +#ifndef McTemplateU0xx_def +#define McTemplateU0xx_def +ETW_INLINE +ULONG +_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_ const unsigned __int64 _Arg0, + _In_ const unsigned __int64 _Arg1 + ) +{ +#define McTemplateU0xx_ARGCOUNT 2 + + EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1]; + + EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) ); + + EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) ); + + return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData); +} +#endif // McTemplateU0xx_def + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +#if defined(__cplusplus) +} +#endif diff --git a/csrc/mimalloc/src/prim/windows/etw.man b/csrc/mimalloc/src/prim/windows/etw.man new file mode 100644 index 000000000..cfd1f8a9e Binary files /dev/null and b/csrc/mimalloc/src/prim/windows/etw.man differ diff --git a/csrc/mimalloc/src/prim/windows/prim.c b/csrc/mimalloc/src/prim/windows/prim.c new file mode 100644 index 000000000..0dfc42202 --- /dev/null +++ b/csrc/mimalloc/src/prim/windows/prim.c @@ -0,0 +1,1202 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// This file is included in `src/prim/prim.c` + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" +#include // fputs, stderr +#include // atexit + +// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle +#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) +#define MI_WIN_DESKTOP 1 +#endif + +//--------------------------------------------- +// Dynamically bind Windows API points for portability +//--------------------------------------------- + +#if defined(_MSC_VER) +#pragma warning(disable:4996) // don't use GetVersionExW +#endif + +static DWORD win_major_version = 6; +static DWORD win_minor_version = 0; + +// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. +// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) +// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB) +// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's. +typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E { + MiMemExtendedParameterInvalidType = 0, + MiMemExtendedParameterAddressRequirements, + MiMemExtendedParameterNumaNode, + MiMemExtendedParameterPartitionHandle, + MiMemExtendedParameterUserPhysicalHandle, + MiMemExtendedParameterAttributeFlags, + MiMemExtendedParameterMax +} MI_MEM_EXTENDED_PARAMETER_TYPE; + +typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S { + struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type; + union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg; +} MI_MEM_EXTENDED_PARAMETER; + +typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S { + PVOID LowestStartingAddress; + PVOID HighestEndingAddress; + SIZE_T Alignment; +} MI_MEM_ADDRESS_REQUIREMENTS; + +#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010 + +#include +typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); +typedef LONG (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); // avoid NTSTATUS as it is not defined on xbox (pr #1084) +static PVirtualAlloc2 pVirtualAlloc2 = NULL; +static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; + +// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7 (and GetNumaNodeProcessorMask is not supported on xbox) +typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER; + +typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber); +typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber); +typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask); +typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber); +typedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask); +typedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node); +static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL; +static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL; +static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL; +static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; +static PGetNumaNodeProcessorMask pGetNumaNodeProcessorMask = NULL; +static PGetNumaHighestNodeNumber pGetNumaHighestNodeNumber = NULL; + +// Not available on xbox +typedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID); +static PGetLargePageMinimum pGetLargePageMinimum = NULL; + +// Available after Windows XP +typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes ); +typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation); + + +// Load a library +static HMODULE mi_win_loadlibrary(const TCHAR* library) { + #if MI_WIN_DESKTOP + return LoadLibrary(library); + #else + return LoadPackagedLibrary(library, 0); + #endif +} + +// Get a library handle (and possibly load it) +static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) { + #if MI_WIN_DESKTOP + // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock) + HMODULE mod = GetModuleHandle(library); + if (mod!=NULL) { + *should_free = false; + return mod; + } + #endif + *should_free = true; + return mi_win_loadlibrary(library); +} + +static void mi_win_freelibrary(HMODULE mod, bool should_free) { + if (should_free) { + FreeLibrary(mod); + } +} + +//--------------------------------------------- +// Enable large page support dynamically (if possible) +//--------------------------------------------- + +static bool win_enable_large_os_pages_once(size_t* large_page_size) +{ + if (pGetLargePageMinimum==NULL) return false; // no large page support (xbox etc.) + + // Try to see if large OS pages are supported + // To use large pages on Windows, we first need access permission + // Set "Lock pages in memory" permission in the group policy editor + // + unsigned long err = 0; + HANDLE token = NULL; + BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + err = GetLastError(); + if (ok) { + TOKEN_PRIVILEGES tp; + ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + err = GetLastError(); + if (ok) { + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + err = GetLastError(); + if (ok) { + ok = (err == ERROR_SUCCESS); + if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) { + *large_page_size = (*pGetLargePageMinimum)(); + } + } + } + CloseHandle(token); + } + if (!ok) { + if (err == 0) { err = GetLastError(); } + _mi_warning_message("cannot enable large OS page support, error %lu\n", err); + } + return (ok!=0); +} + +static bool win_enable_large_os_pages(size_t* large_page_size) { + static size_t win_large_page_size = 0; + mi_atomic_do_once { + win_enable_large_os_pages_once(&win_large_page_size); + } + if (large_page_size != NULL) { *large_page_size = win_large_page_size; } + return (win_large_page_size > 0); +} + + +//--------------------------------------------- +// Initialize +//--------------------------------------------- + +static DWORD win_allocation_granularity = 64*MI_KiB; + +void _mi_prim_mem_init( mi_os_mem_config_t* config ) +{ + config->has_overcommit = false; + config->has_partial_free = false; + config->has_virtual_reserve = true; + + // get the page size + SYSTEM_INFO si; _mi_memzero_var(si); + GetSystemInfo(&si); + if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; } + if (si.dwAllocationGranularity > 0) { + config->alloc_granularity = si.dwAllocationGranularity; + win_allocation_granularity = si.dwAllocationGranularity; + } + // get virtual address bits + if ((uintptr_t)si.lpMaximumApplicationAddress > 0) { + const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress); + config->virtual_address_bits = vbits; + } + + // get the VirtualAlloc2 function + bool hDllFree; + HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree); + if (hDll != NULL) { + // use VirtualAlloc2FromApp if possible as it is available to Windows store apps + pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); + if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); + mi_win_freelibrary(hDll, hDllFree); + } + // NtAllocateVirtualMemoryEx is used for huge page allocation + hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree); + if (hDll != NULL) { + pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); + mi_win_freelibrary(hDll, hDllFree); + } + // Try to use Win7+ numa API + hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree); + if (hDll != NULL) { + pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); + pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); + pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx"); + pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode"); + pGetNumaNodeProcessorMask = (PGetNumaNodeProcessorMask)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMask"); + pGetNumaHighestNodeNumber = (PGetNumaHighestNodeNumber)(void (*)(void))GetProcAddress(hDll, "GetNumaHighestNodeNumber"); + pGetLargePageMinimum = (PGetLargePageMinimum)(void (*)(void))GetProcAddress(hDll, "GetLargePageMinimum"); + // Get physical memory (not available on XP, so check dynamically) + PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,"GetPhysicallyInstalledSystemMemory"); + if (pGetPhysicallyInstalledSystemMemory != NULL) { + ULONGLONG memInKiB = 0; + if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) { + if (memInKiB > 0 && memInKiB <= SIZE_MAX) { + config->physical_memory_in_kib = (size_t)memInKiB; + } + } + } + // Get Windows version + PGetVersionExW pGetVersionExW = (PGetVersionExW)(void (*)(void))GetProcAddress(hDll, "GetVersionExW"); + if (pGetVersionExW != NULL) { + OSVERSIONINFOW version; _mi_memzero_var(version); + version.dwOSVersionInfoSize = sizeof(version); + if ((*pGetVersionExW)(&version)) { + win_major_version = version.dwMajorVersion; + win_minor_version = version.dwMinorVersion; + } + } + mi_win_freelibrary(hDll, hDllFree); + } + // Enable large/huge OS page support? + if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { + win_enable_large_os_pages(&config->large_page_size); + } +} + + +//--------------------------------------------- +// Free +//--------------------------------------------- + +int _mi_prim_free(void* addr, size_t size ) { + MI_UNUSED(size); + DWORD errcode = 0; + bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + if (errcode == ERROR_INVALID_ADDRESS) { + // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside + // the memory region returned by VirtualAlloc; in that case we need to free using + // the start of the region. + MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info); + err = (VirtualQuery(addr, &info, sizeof(info)) == 0); + if (err) { errcode = GetLastError(); } + if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) { + errcode = 0; + err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); + if (err) { errcode = GetLastError(); } + } + } + return (int)errcode; +} + + +//--------------------------------------------- +// VirtualAlloc +//--------------------------------------------- + +static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { + #if (MI_INTPTR_SIZE >= 8) + // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations + if (addr == NULL) { + void* hint = _mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags); + // fall through on error + } + } + #endif + // on modern Windows try use VirtualAlloc2 for aligned allocation + if (addr == NULL && try_alignment > win_allocation_granularity && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; + reqs.Alignment = try_alignment; + MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; + param.Type.Type = MiMemExtendedParameterAddressRequirements; + param.Arg.Pointer = &reqs; + void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + if (p != NULL) return p; + _mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); + // fall through on error + } + // last resort + return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +} + +static bool win_is_out_of_memory_error(DWORD err) { + switch (err) { + case ERROR_COMMITMENT_MINIMUM: + case ERROR_COMMITMENT_LIMIT: + case ERROR_PAGEFILE_QUOTA: + case ERROR_NOT_ENOUGH_MEMORY: + return true; + default: + return false; + } +} + +static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { + long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds + if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to "true" + for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms) + void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags); + if (p != NULL) { + // success, return the address + return p; + } + else if (max_retry_msecs > 0 && (try_alignment <= 8*MI_MiB) && + (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 && + win_is_out_of_memory_error(GetLastError())) { + // if committing regular memory and being out-of-memory, + // keep trying for a bit in case memory frees up after all. See issue #894 + _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags); + long sleep_msecs = tries*40; // increasing waits + if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; } + max_retry_msecs -= sleep_msecs; + Sleep(sleep_msecs); + } + else { + // otherwise return with an error + break; + } + } + return NULL; +} + +static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { + mi_assert_internal(!(large_only && !allow_large)); + static _Atomic(size_t) large_page_try_ok; // = 0; + void* p = NULL; + // Try to allocate large OS pages (2MiB) if allowed or required. + if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages))) + && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) + { + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + if (!large_only && try_ok > 0) { + // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. + // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. + mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1); + } + else { + // large OS pages must always reserve and commit. + *is_large = true; + p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES); + if (large_only) return p; + // fall back to non-large page allocation on error (`p == NULL`). + if (p == NULL) { + mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations + } + } + } + // Fall back to regular page allocation + if (p == NULL) { + *is_large = ((flags&MEM_LARGE_PAGES) != 0); + p = win_virtual_alloc_prim(addr, size, try_alignment, flags); + } + //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); } + return p; +} + +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(commit || !allow_large); + mi_assert_internal(try_alignment > 0); + *is_zero = true; + int flags = MEM_RESERVE; + if (commit) { flags |= MEM_COMMIT; } + *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large); + return (*addr != NULL ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Commit/Reset/Protect +//--------------------------------------------- +#ifdef _MSC_VER +#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit) +#endif + +int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { + *is_zero = false; + /* + // zero'ing only happens on an initial commit... but checking upfront seems expensive.. + _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo); + if (VirtualQuery(addr, &meminfo, size) > 0) { + if ((meminfo.State & MEM_COMMIT) == 0) { + *is_zero = true; + } + } + */ + // commit + void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE); + if (p == NULL) return (int)GetLastError(); + return 0; +} + +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { + BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT); + *needs_recommit = true; // for safety, assume always decommitted even in the case of an error. + return (ok ? 0 : (int)GetLastError()); +} + +int _mi_prim_reset(void* addr, size_t size) { + void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); + mi_assert_internal(p == addr); + #if 0 + if (p != NULL) { + VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set + } + #endif + return (p != NULL ? 0 : (int)GetLastError()); +} + +int _mi_prim_reuse(void* addr, size_t size) { + MI_UNUSED(addr); MI_UNUSED(size); + return 0; +} + +int _mi_prim_protect(void* addr, size_t size, bool protect) { + DWORD oldprotect = 0; + BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); + return (ok ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Huge page allocation +//--------------------------------------------- + +static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node) +{ + const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; + + win_enable_large_os_pages(NULL); + + MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} }; + // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages + static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1); + if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) { + params[0].Type.Type = MiMemExtendedParameterAttributeFlags; + params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; + ULONG param_count = 1; + if (numa_node >= 0) { + param_count++; + params[1].Type.Type = MiMemExtendedParameterNumaNode; + params[1].Arg.ULong = (unsigned)numa_node; + } + SIZE_T psize = size; + void* base = hint_addr; + LONG err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count); + if (err == 0 && base != NULL) { + return base; + } + else { + // fall back to regular large pages + mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); + } + } + // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation + if (pVirtualAlloc2 != NULL && numa_node >= 0) { + params[0].Type.Type = MiMemExtendedParameterNumaNode; + params[0].Arg.ULong = (unsigned)numa_node; + return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1); + } + + // otherwise use regular virtual alloc on older windows + return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE); +} + +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { + *is_zero = true; + *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node); + return (*addr != NULL ? 0 : (int)GetLastError()); +} + + +//--------------------------------------------- +// Numa nodes +//--------------------------------------------- + +size_t _mi_prim_numa_node(void) { + USHORT numa_node = 0; + if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) { + // Extended API is supported + MI_PROCESSOR_NUMBER pnum; + (*pGetCurrentProcessorNumberEx)(&pnum); + USHORT nnode = 0; + BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode); + if (ok) { numa_node = nnode; } + } + else if (pGetNumaProcessorNode != NULL) { + // Vista or earlier, use older API that is limited to 64 processors. Issue #277 + DWORD pnum = GetCurrentProcessorNumber(); + UCHAR nnode = 0; + BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode); + if (ok) { numa_node = nnode; } + } + return numa_node; +} + +size_t _mi_prim_numa_node_count(void) { + ULONG numa_max = 0; + if (pGetNumaHighestNodeNumber!=NULL) { + (*pGetNumaHighestNodeNumber)(&numa_max); + } + // find the highest node number that has actual processors assigned to it. Issue #282 + while (numa_max > 0) { + if (pGetNumaNodeProcessorMaskEx != NULL) { + // Extended API is supported + GROUP_AFFINITY affinity; + if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) { + if (affinity.Mask != 0) break; // found the maximum non-empty node + } + } + else { + // Vista or earlier, use older API that is limited to 64 processors. + ULONGLONG mask; + if (pGetNumaNodeProcessorMask != NULL) { + if ((*pGetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) { + if (mask != 0) break; // found the maximum non-empty node + } + }; + } + // max node was invalid or had no processor assigned, try again + numa_max--; + } + return ((size_t)numa_max + 1); +} + + +//---------------------------------------------------------------- +// Clock +//---------------------------------------------------------------- + +static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { + static LARGE_INTEGER mfreq; // = 0 + if (mfreq.QuadPart == 0LL) { + LARGE_INTEGER f; + if (QueryPerformanceFrequency(&f)) { + mfreq.QuadPart = f.QuadPart/1000LL; + } + if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; + } + return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); +} + +mi_msecs_t _mi_prim_clock_now(void) { + LARGE_INTEGER t; + if (QueryPerformanceCounter(&t)) { + return mi_to_msecs(t); + } + else { + return 0; + } +} + + +//---------------------------------------------------------------- +// Process Info +//---------------------------------------------------------------- + +#include + +static mi_msecs_t filetime_msecs(const FILETIME* ftime) { + ULARGE_INTEGER i; + i.LowPart = ftime->dwLowDateTime; + i.HighPart = ftime->dwHighDateTime; + mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds + return msecs; +} + +typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD); +static PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL; + +void _mi_prim_process_info(mi_process_info_t* pinfo) +{ + FILETIME ct; + FILETIME ut; + FILETIME st; + FILETIME et; + if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) { + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); + } + + // load psapi on demand + mi_atomic_do_once{ + HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll")); + if (hDll != NULL) { + pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); + // mi_win_freelibrary(hDll, true); // don't free + } + } + + // get process info + if (pGetProcessMemoryInfo != NULL) { + PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); + if (pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) { + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; + } + } +} + +//---------------------------------------------------------------- +// Output +//---------------------------------------------------------------- + +void _mi_prim_out_stderr( const char* msg ) +{ + // on windows with redirection, the C runtime cannot handle locale dependent output + // after the main thread closes so we use direct console output. + if (!_mi_preloading()) { + static HANDLE hcon = INVALID_HANDLE_VALUE; + #if MI_WIN_DESKTOP + static bool hconIsConsole = false; + #endif + if (hcon == INVALID_HANDLE_VALUE) { + hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available + #if MI_WIN_DESKTOP + if (hcon==NULL) { + AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that + hcon = GetStdHandle(STD_ERROR_HANDLE); + } + CONSOLE_SCREEN_BUFFER_INFO sbi; + hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); + #endif + } + const size_t len = _mi_strlen(msg); + if (len > 0 && len < UINT32_MAX) { + DWORD written = 0; + if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) { + #if MI_WIN_DESKTOP + if (hconIsConsole) { + WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + } + else + #endif + { + // use direct write in case stderr was redirected + WriteFile(hcon, msg, (DWORD)len, &written, NULL); + } + } + // don't fall back to fputs or _cputs as the crt can have it locked + } + } +} + + +//---------------------------------------------------------------- +// Environment +//---------------------------------------------------------------- + +// On Windows use GetEnvironmentVariable instead of getenv to work +// reliably even when this is invoked before the C runtime is initialized. +// i.e. when `_mi_preloading() == true`. +// Note: on windows, environment names are not case sensitive. +int _mi_prim_getenv(const char* name, char* result, size_t result_size) { + result[0] = 0; + const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); + return (len < result_size ? (len > 0 ? 1 /* success */ : 0 /* not found */) : -1 /* error */); +} + + +//---------------------------------------------------------------- +// Random +//---------------------------------------------------------------- + +#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus) +// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using +// dynamic overriding, we observed it can raise an exception when compiled with C++, and +// sometimes deadlocks when also running under the VS debugger. +// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom. +// To be continued.. +#pragma comment (lib,"advapi32.lib") +#define RtlGenRandom SystemFunction036 +mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + return (RtlGenRandom(buf, (ULONG)buf_len) != 0); +} + +#else + +#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG +#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002 +#endif + +typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG); +static PBCryptGenRandom pBCryptGenRandom = NULL; + +bool _mi_prim_random_buf(void* buf, size_t buf_len) { + mi_assert(buf_len <= ULONG_MAX); + if (buf_len > ULONG_MAX) return false; + mi_atomic_do_once { + HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll")); + if (hDll != NULL) { + pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); + // mi_win_freelibrary(hDll); // don't free + } + } + if (pBCryptGenRandom == NULL) return false; + return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); +} + +#endif // MI_USE_RTLGENRANDOM + + +//---------------------------------------------------------------- +// Thread pool? +//---------------------------------------------------------------- + +bool _mi_prim_thread_is_in_threadpool(void) { +#if (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64) + if (win_major_version >= 6) { + // check if this thread belongs to a windows threadpool + // see: + struct _TEB* const teb = NtCurrentTeb(); + void* const pool_data = *((void**)((uint8_t*)teb + (MI_SIZE_BITS == 32 ? 0x0F90 : 0x1778))); + return (pool_data != NULL); + } +#endif + return false; +} + +void _mi_prim_thread_yield(void) { + SwitchToThread(); +} + +//---------------------------------------------------------------- +// Process & Thread Init/Done +//---------------------------------------------------------------- + +//static void mi_debug_out(const char* s) { +// HANDLE h = GetStdHandle(STD_ERROR_HANDLE); +// WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL); +//} + +static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(module); + if (reason==DLL_PROCESS_ATTACH) { + _mi_auto_process_init(); + } + else if (reason==DLL_PROCESS_DETACH) { + _mi_auto_process_done(); + } + else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) { + _mi_thread_done(NULL); + } +} + + +/* ----------------------------------------------------------------------- + Auto initialize and finalize mimalloc on process and thread start/end. + By default we use a combination of CRT init and TLS sections for + both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`). +------------------------------------------------------------------------- */ +#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS) + #if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER) + #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* needed for Intel ICX, see issue #1268 */ + #else + #define MI_WIN_INIT_USE_CRT_TLS 1 /* default */ + #endif +#endif + +#if defined(MI_WIN_INIT_USE_CRT_TLS) + #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #endif + + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) {} + void _mi_prim_thread_done_auto_done(void) {} + void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); + } + + static bool mi_module_is_dll(PVOID mod) { + if (mod==NULL) return false; + PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod; + PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew); + return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL); + } + + static bool mi_current_module_is_dll(void) { + HMODULE mod = NULL; + const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod); + return (ok && mi_module_is_dll(mod)); + } + + // Hook into CRT initialization and finalization. + static void mi_cdecl mi_crt_done(void) { + // mi_debug_out(mi_current_module_is_dll() ? "crt dll done\n" : "crt exe done\n"); + mi_win_main(NULL, DLL_PROCESS_DETACH, 0); + } + + static int mi_cdecl mi_crt_init(void) { + // mi_debug_out(mi_current_module_is_dll() ? "crt dll init\n" : "crt exe init\n"); + if (mi_current_module_is_dll()) { + // in a dll, atexit (crt_done) is called after tls process detach + atexit(&mi_crt_done); + } + return 0; + } + + // We also hook into the Windows loader TLS initialization and finalization. + static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_THREAD_ATTACH) { + //mi_debug_out("tls thread attach\n"); + mi_win_main(module, reason, reserved); + } + else if (reason == DLL_PROCESS_ATTACH) { + // mi_debug_out(mi_module_is_dll(module) ? "tls dll process attach\n" : "tls exe process attach\n"); + // tls process attach is always called before crt init + mi_win_main(module, reason, reserved); + } + } + + static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_THREAD_DETACH) { + //mi_debug_out("tls thread detach\n"); + mi_win_main(module, reason, reserved); + } + else if (reason == DLL_PROCESS_DETACH) { + // mi_debug_out(mi_module_is_dll(module) ? "tls dll process detach\n" : "tls exe process detach\n"); + if (!mi_module_is_dll(module)) { + // in an exe, tls process detach is called after atexit (mi_crt_done) + mi_win_main(module, reason, reserved); + } + } + } + + // Set up TLS callbacks in a statically linked library by using special data sections. + // See + // We use 2 entries to ensure we call attach events before constructors + // are called, and detach events after destructors are called. + // We also set up a CRT initialization callback. + #if defined(__cplusplus) + extern "C" { + #endif + + typedef int (mi_cdecl* mi_crt_callback_t)(void); + + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; + #pragma const_seg() + #pragma const_seg(".CRT$XLY") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; + #pragma const_seg() + #pragma const_seg(".CRT$XIB") + extern const mi_crt_callback_t _mi_crt_callback_init[]; + const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma const_seg() + #elif defined(_MSC_VER) // 32-bit + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") + #pragma data_seg(".CRT$XLB") + PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; + #pragma data_seg() + #pragma data_seg(".CRT$XLY") + PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; + #pragma data_seg() + #pragma data_seg(".CRT$XIB") + mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + __attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init; + #endif + + #if defined(__cplusplus) + } + #endif + +/* ---------------------------------------------------------------- + Here we use a combination of the raw DLL main and TLS sections for + both static and dynamic linkage. This is perhaps the most robust as + the raw DLL main ensure we can close mimalloc latest possible. + However, it can lead to link errors with on `_pRawDllMain` if other + libraries also try to hook into this (like Boost). +*/ +#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) {} + void _mi_prim_thread_done_auto_done(void) {} + void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); + } + + // If linked into a DLL module, this raw entry is called before the CRT attach and + // after the CRT detach through the CRT _pRawDllMain pointer. + static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) { + mi_win_main(module, reason, reserved); + return TRUE; + } + + // Set the value of the CRT _pRawDllMain pointer + #if defined(__cplusplus) + extern "C" + #endif + PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw; + + // We also hook into the Windows loader TLS initialization and finalization. + // If we are linked into an EXE module we rely on these as `mi_dll_main_raw` + // is not called (and otherwise we ignore the TLS callbacks by checking if we are in a DLL). + static bool mi_module_is_dll(PVOID mod) { + if (mod==NULL) return false; + PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod; + PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew); + return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL); + } + + static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) { + if (!mi_module_is_dll(module)) { + mi_win_main(module, reason, reserved); + } + } + } + + static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) { + if (!mi_module_is_dll(module)) { + mi_win_main(module, reason, reserved); + } + } + } + + // Set up TLS callbacks in a statically linked library by using special data sections. + // See + // We use 2 entries to ensure we call attach events before constructors + // are called, and detach events after destructors are called. + #if defined(__cplusplus) + extern "C" { + #endif + + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; + #pragma const_seg() + #pragma const_seg(".CRT$XLY") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; + #pragma const_seg() + #elif defined(_MSC_VER) // 32-bit + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma data_seg(".CRT$XLB") + PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; + #pragma data_seg() + #pragma data_seg(".CRT$XLY") + PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; + #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + #endif + + #if defined(__cplusplus) + } + #endif + +/* -------------------------------------------------------------------- + Legacy options: DllMain, TLS, and FLS + --------------------------------------------------------------------*/ +#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + + // Windows DLL: easy to hook into process_init and thread_done + BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { + mi_win_main((PVOID)inst,reason,reserved); + return TRUE; + } + + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); + } + +#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + + static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) { + mi_win_main(module, reason, reserved); + } + } + static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) { + mi_win_main(module, reason, reserved); + } + } + + // Set up TLS callbacks in a statically linked library by using special data sections. + // See + // We use 2 entries to ensure we call attach events before constructors + // are called, and detach events after destructors are called. + #if defined(__cplusplus) + extern "C" { + #endif + + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma const_seg() + #pragma const_seg(".CRT$XLY") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma const_seg() + #elif defined(_MSC_VER) // 32-bit + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma data_seg(".CRT$XLB") + PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma data_seg() + #pragma data_seg(".CRT$XLY") + PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + #endif + + #if defined(__cplusplus) + } + #endif + + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + MI_UNUSED(theap); + } + +#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api + + #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) + // MSVC: use data section magic for static libraries + // See + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + + static int mi_cdecl mi_crt_init(void) { + mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); + atexit(&_mi_auto_process_done); + return 0; + } + + #if defined(__cplusplus) + extern "C" { + #endif + typedef int (mi_cdecl* mi_crt_callback_t)(void); + #if defined(_WIN64) // 64-bit + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") + #pragma const_seg(".CRT$XIU") + extern const mi_crt_callback_t _mi_crt_callback_init[]; + const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma const_seg() + #else // 32-bit + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") + #pragma data_seg(".CRT$XIU") + mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma data_seg() + #endif + #if defined(__cplusplus) + } + #endif + #endif + + // use the fiber api for calling `_mi_thread_done`. + #include + #if (_WIN32_WINNT < 0x600) // before Windows Vista + WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); + WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); + WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); + WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); + #endif + + static DWORD mi_fls_key = (DWORD)(-1); + + static void NTAPI mi_fls_done(PVOID value) { + mi_theap_t* theap = (mi_theap_t*)value; + if (theap != NULL) { + _mi_thread_done(theap); + FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main theap, issue #672 + } + } + + void _mi_prim_thread_init_auto_done(void) { + mi_fls_key = FlsAlloc(&mi_fls_done); + } + + void _mi_prim_thread_done_auto_done(void) { + // call thread-done on all threads (except the main thread) to prevent + // dangling callback pointer if statically linked with a DLL; Issue #208 + FlsFree(mi_fls_key); + } + + void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { + mi_assert_internal(mi_fls_key != (DWORD)(-1)); + FlsSetValue(mi_fls_key, theap); + } +#else +#error "define windows process and thread auto initialization" +#endif + +// ---------------------------------------------------- +// Communicate with the redirection module on Windows +// ---------------------------------------------------- +#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) + #define MI_PRIM_HAS_ALLOCATOR_INIT 1 + + static bool mi_redirected = false; // true if malloc redirects to mi_malloc + + bool _mi_is_redirected(void) { + return mi_redirected; + } + + #ifdef __cplusplus + extern "C" { + #endif + mi_decl_export void _mi_redirect_entry(DWORD reason) { + // called on redirection; careful as this may be called before DllMain + if (reason == DLL_PROCESS_ATTACH) { + mi_redirected = true; + } + else if (reason == DLL_PROCESS_DETACH) { + mi_redirected = false; + } + else if (reason == DLL_THREAD_DETACH) { + // mi_debug_out("redirect thread detach\n"); + _mi_thread_done(NULL); + } + } + __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); + __declspec(dllimport) void mi_cdecl mi_allocator_done(void); + #ifdef __cplusplus + } + #endif + bool _mi_allocator_init(const char** message) { + return mi_allocator_init(message); + } + void _mi_allocator_done(void) { + mi_allocator_done(); + } +#endif + diff --git a/csrc/mimalloc/src/prim/windows/readme.md b/csrc/mimalloc/src/prim/windows/readme.md new file mode 100644 index 000000000..217c3d174 --- /dev/null +++ b/csrc/mimalloc/src/prim/windows/readme.md @@ -0,0 +1,17 @@ +## Primitives: + +- `prim.c` contains Windows primitives for OS allocation. + +## Event Tracing for Windows (ETW) + +- `etw.h` is generated from `etw.man` which contains the manifest for mimalloc events. + (100 is an allocation, 101 is for a free) + +- `etw-mimalloc.wprp` is a profile for the Windows Performance Recorder (WPR). + In an admin prompt, you can use: + ``` + > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode + > + > wpr -stop test.etl + ``` + and then open `test.etl` in the Windows Performance Analyzer (WPA). \ No newline at end of file diff --git a/csrc/mimalloc/src/random.c b/csrc/mimalloc/src/random.c new file mode 100644 index 000000000..464627acf --- /dev/null +++ b/csrc/mimalloc/src/random.c @@ -0,0 +1,260 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // _mi_prim_random_buf + +/* ---------------------------------------------------------------------------- +We use our own PRNG to keep predictable performance of random number generation +and to avoid implementations that use a lock. We only use the OS provided +random source to initialize the initial seeds. Since we do not need ultimate +performance but we do rely on the security (for secret cookies in secure mode) +we use a cryptographically secure generator (chacha20). +-----------------------------------------------------------------------------*/ + +#define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance? + + +/* ---------------------------------------------------------------------------- +Chacha20 implementation as the original algorithm with a 64-bit nonce +and counter: https://en.wikipedia.org/wiki/Salsa20 +The input matrix has sixteen 32-bit values: +Position 0 to 3: constant key +Position 4 to 11: the key +Position 12 to 13: the counter. +Position 14 to 15: the nonce. + +The implementation uses regular C code which compiles very well on modern compilers. +(gcc x64 has no register spills, and clang 6+ uses SSE instructions) +-----------------------------------------------------------------------------*/ + +static inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) { + x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 16); + x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 12); + x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 8); + x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 7); +} + +static void chacha_block(mi_random_ctx_t* ctx) +{ + // scramble into `x` + uint32_t x[16]; + for (size_t i = 0; i < 16; i++) { + x[i] = ctx->input[i]; + } + for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) { + qround(x, 0, 4, 8, 12); + qround(x, 1, 5, 9, 13); + qround(x, 2, 6, 10, 14); + qround(x, 3, 7, 11, 15); + qround(x, 0, 5, 10, 15); + qround(x, 1, 6, 11, 12); + qround(x, 2, 7, 8, 13); + qround(x, 3, 4, 9, 14); + } + + // add scrambled data to the initial state + for (size_t i = 0; i < 16; i++) { + ctx->output[i] = x[i] + ctx->input[i]; + } + ctx->output_available = 16; + + // increment the counter for the next round + ctx->input[12] += 1; + if (ctx->input[12] == 0) { + ctx->input[13] += 1; + if (ctx->input[13] == 0) { // and keep increasing into the nonce + ctx->input[14] += 1; + } + } +} + +static uint32_t chacha_next32(mi_random_ctx_t* ctx) { + if (ctx->output_available <= 0) { + chacha_block(ctx); + ctx->output_available = 16; // (assign again to suppress static analysis warning) + } + const uint32_t x = ctx->output[16 - ctx->output_available]; + ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out + ctx->output_available--; + return x; +} + +static inline uint32_t read32(const uint8_t* p, size_t idx32) { + const size_t i = 4*idx32; + return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24); +} + +static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce) +{ + // since we only use chacha for randomness (and not encryption) we + // do not _need_ to read 32-bit values as little endian but we do anyways + // just for being compatible :-) + ctx->output_available = 0; + _mi_memzero(ctx->output,sizeof(ctx->output)); + for (size_t i = 0; i < 4; i++) { + const uint8_t* sigma = (uint8_t*)"expand 32-byte k"; + ctx->input[i] = read32(sigma,i); + } + for (size_t i = 0; i < 8; i++) { + ctx->input[i + 4] = read32(key,i); + } + ctx->input[12] = 0; + ctx->input[13] = 0; + ctx->input[14] = (uint32_t)nonce; + ctx->input[15] = (uint32_t)(nonce >> 32); +} + +static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { + _mi_memzero(ctx_new, sizeof(*ctx_new)); + ctx_new->weak = ctx->weak; + _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + ctx_new->input[12] = 0; + ctx_new->input[13] = 0; + ctx_new->input[14] = (uint32_t)nonce; + ctx_new->input[15] = (uint32_t)(nonce >> 32); + mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces! + chacha_block(ctx_new); +} + + +/* ---------------------------------------------------------------------------- +Random interface +-----------------------------------------------------------------------------*/ + +#if MI_DEBUG>1 +static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { + return (ctx != NULL && ctx->input[0] != 0); +} +#endif + +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { + mi_assert_internal(mi_random_is_initialized(ctx)); + mi_assert_internal(ctx != ctx_new); + const uintptr_t nonce_rnd = _mi_random_next(ctx); + chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); +} + +uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { + mi_assert_internal(mi_random_is_initialized(ctx)); + uintptr_t r; + do { + #if MI_INTPTR_SIZE <= 4 + r = chacha_next32(ctx); + #elif MI_INTPTR_SIZE == 8 + r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); + #else + # error "define mi_random_next for this platform" + #endif + } while (r==0); + return r; +} + + +/* ---------------------------------------------------------------------------- +To initialize a fresh random context. +If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. +-----------------------------------------------------------------------------*/ + +uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { + uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random + x ^= _mi_prim_clock_now(); + // and do a few randomization steps + uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; + for (uintptr_t i = 0; i < max || x==0; i++, x++) { + x = _mi_random_shuffle(x); + } + mi_assert_internal(x != 0); + return x; +} + +static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { + uint8_t key[32]; + if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) { + // if we fail to get random data from the OS, we fall back to a + // weak random source based on the current time + #if !defined(__wasi__) + if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } + #endif + uintptr_t x = _mi_os_random_weak(0); + for (size_t i = 0; i < 32; i+=4, x++) { + x = _mi_random_shuffle(x); + key[i] = (uint8_t)(x); + key[i+1] = (uint8_t)(x>>8); + key[i+2] = (uint8_t)(x>>16); + key[i+3] = (uint8_t)(x>>24); + } + ctx->weak = true; + } + else { + ctx->weak = false; + } + chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); + _mi_memzero(key, sizeof(key)); +} + +void _mi_random_init(mi_random_ctx_t* ctx) { + mi_random_init_ex(ctx, false); +} + +void _mi_random_init_weak(mi_random_ctx_t * ctx) { + mi_random_init_ex(ctx, true); +} + +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) { + if (ctx->weak) { + _mi_random_init(ctx); + } +} + +/* -------------------------------------------------------- +test vectors from +----------------------------------------------------------- */ +/* +static bool array_equals(uint32_t* x, uint32_t* y, size_t n) { + for (size_t i = 0; i < n; i++) { + if (x[i] != y[i]) return false; + } + return true; +} +static void chacha_test(void) +{ + uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 }; + uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb }; + qround(x, 0, 1, 2, 3); + mi_assert_internal(array_equals(x, x_out, 4)); + + uint32_t y[16] = { + 0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a, + 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c, + 0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963, + 0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 }; + uint32_t y_out[16] = { + 0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a, + 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2, + 0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963, + 0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 }; + qround(y, 2, 7, 8, 13); + mi_assert_internal(array_equals(y, y_out, 16)); + + mi_random_ctx_t r = { + { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574, + 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c, + 0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c, + 0x00000001, 0x09000000, 0x4a000000, 0x00000000 }, + {0}, + 0 + }; + uint32_t r_out[16] = { + 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3, + 0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3, + 0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9, + 0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 }; + chacha_block(&r); + mi_assert_internal(array_equals(r.output, r_out, 16)); +} +*/ diff --git a/csrc/mimalloc/src/static.c b/csrc/mimalloc/src/static.c new file mode 100644 index 000000000..3554a4bc6 --- /dev/null +++ b/csrc/mimalloc/src/static.c @@ -0,0 +1,44 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#if defined(__sun) +// same remarks as os.c for the static's context. +#undef _XOPEN_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#include "mimalloc.h" +#include "mimalloc/internal.h" + +// For a static override we create a single object file +// containing the whole library. If it is linked first +// it will override all the standard library allocation +// functions (on Unix's). +#include "alloc.c" // includes alloc-override.c and free.c +#include "alloc-aligned.c" +#include "alloc-posix.c" +#include "arena.c" +#include "bitmap.c" +#include "heap.c" +#include "init.c" +#include "libc.c" +#include "options.c" +#include "os.c" +#include "page.c" // includes page-queue.c +#include "page-map.c" +#include "random.c" +#include "stats.c" +#include "subproc.c" +#include "theap.c" +#include "threadlocal.c" +#include "prim/prim.c" +#include "prim/prim-tls.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif diff --git a/csrc/mimalloc/src/stats.c b/csrc/mimalloc/src/stats.c new file mode 100644 index 000000000..9220bfec2 --- /dev/null +++ b/csrc/mimalloc/src/stats.c @@ -0,0 +1,847 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc-stats.h" +#include "mimalloc/internal.h" +#include "mimalloc/atomic.h" +#include "mimalloc/prim.h" // _mi_prim_clock_now, mi_process_info_t +#include "mimalloc/prim-tls.h" + +#include // memset + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +#pragma warning(disable:4204) // non-constant aggregate initializer +#endif + +/* ----------------------------------------------------------- + Statistics operations +----------------------------------------------------------- */ + +static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + // add atomically + int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); + mi_atomic_maxi64_relaxed(&stat->peak, current + amount); + if (amount > 0) { + mi_atomic_addi64_relaxed(&stat->total, amount); + } +} + +static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + // add thread local + stat->current += amount; + if (stat->current > stat->peak) { stat->peak = stat->current; } + if (amount > 0) { stat->total += amount; } +} + + +void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) { + mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); +} + +void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { + stat->total += amount; +} + +void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) { + mi_stat_update_mt(stat, (int64_t)amount); +} +void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) { + mi_stat_update(stat, (int64_t)amount); +} + +void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) { + mi_stat_update_mt(stat, -((int64_t)amount)); +} +void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { + mi_stat_update(stat, -((int64_t)amount)); +} + + +// Adjust stats to compensate; for example before committing a range, +// first adjust downwards with parts that were already committed so +// we avoid double counting. +static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + // adjust atomically + const size_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); + mi_atomic_addi64_relaxed(&stat->current, amount); + const size_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } +} + +static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + stat->current += amount; + if (stat->total==stat->peak) { stat->peak += amount; } + stat->total += amount; +} + +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust_mt(stat, (int64_t)amount); +} +void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust(stat, (int64_t)amount); +} +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust_mt(stat, -((int64_t)amount)); +} +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust(stat, -((int64_t)amount)); +} + + +// must be thread safe as it is called from stats_merge +static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) { + if (stat==src) return; + mi_atomic_void_addi64_relaxed(&stat->total, &src->total); + const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak); + const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current); + const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); + + // Global current plus thread peak approximates new global peak + // note: peak scores do not really work across threads. + // we used to just add them together but that often overestimates in practice. + // similarly, max does not seem to work well. The current approach + // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 + // for a longer description. + mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak); +} + +static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) { + if (stat==src) return; + mi_atomic_void_addi64_relaxed(&stat->total, &src->total); +} + +#define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat); +#define MI_STAT_COUNTER(stat) mi_stat_counter_add_mt(&stats->stat, &src->stat); + +// must be thread safe as it is called from stats_merge +static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { + if (stats==NULL || src==NULL || stats==src) return; + + // copy all fields + MI_STAT_FIELDS() + + #if MI_STAT>1 + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]); + } + #endif + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]); + } + for (size_t i = 0; i < MI_CBIN_COUNT; i++) { + mi_stat_count_add_mt(&stats->chunk_bins[i], &src->chunk_bins[i]); + } +} + +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER + +/* ----------------------------------------------------------- + Display statistics +----------------------------------------------------------- */ + +// unit > 0 : size in binary bytes +// unit == 0: count as decimal +// unit < 0 : count in binary +static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) { + char buf[32]; _mi_memzero_var(buf); + int len = 32; + const char* suffix = (unit <= 0 ? " " : "B"); + const int64_t base = (unit == 0 ? 1000 : 1024); + if (unit>0) n *= unit; + + const int64_t pos = (n < 0 ? -n : n); + if (pos < base) { + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + _mi_snprintf(buf, len, "%lld %-3s", (long long)n, (n==0 ? "" : suffix)); + } + } + else { + int64_t divider = base; + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } + const int64_t tens = (n / (divider/10)); + const long whole = (long)(tens/10); + const long frac1 = (long)(tens%10); + char unitdesc[8]; + _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); + } + if (limitwidth) { + _mi_fprintf(out, arg, "%12s", buf); + } + else { + _mi_fprintf(out, arg, "%s", buf); + } +} + + +static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { + mi_printf_amount(n,unit,out,arg,true); +} + +static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { + if (unit==1) _mi_fprintf(out, arg, "%12s"," "); + else mi_print_amount(n,0,out,arg); +} + +static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { + _mi_fprintf(out, arg," %-10s:", msg); + if (unit != 0) { + if (unit > 0) { + mi_print_amount(stat->peak, unit, out, arg); + mi_print_amount(stat->total, unit, out, arg); + // mi_print_amount(stat->freed, unit, out, arg); + mi_print_amount(stat->current, unit, out, arg); + mi_print_amount(unit, 1, out, arg); + mi_print_count(stat->total, unit, out, arg); + } + else { + mi_print_amount(stat->peak, -1, out, arg); + mi_print_amount(stat->total, -1, out, arg); + // mi_print_amount(stat->freed, -1, out, arg); + mi_print_amount(stat->current, -1, out, arg); + if (unit == -1) { + _mi_fprintf(out, arg, "%24s", ""); + } + else { + mi_print_amount(-unit, 1, out, arg); + mi_print_count((stat->total / -unit), 0, out, arg); + } + } + if (stat->current != 0) { + _mi_fprintf(out, arg, " "); + _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok)); + _mi_fprintf(out, arg, "\n"); + } + else { + _mi_fprintf(out, arg, " ok\n"); + } + } + else { + mi_print_amount(stat->peak, 0, out, arg); + mi_print_amount(stat->total, 0, out, arg); + mi_print_amount(stat->current, 0, out, arg); + _mi_fprintf(out, arg, "\n"); + } +} + +static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { + mi_stat_print_ex(stat, msg, unit, out, arg, NULL); +} + +#if MI_STAT>1 +static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { + _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, "%12s", " "); // no peak + mi_print_amount(stat->total, unit, out, arg); + _mi_fprintf(out, arg, "\n"); +} +#endif + +static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, " %-10s:", msg); + mi_print_amount(stat->total, 0, out, arg); + _mi_fprintf(out, arg, "\n"); +} + +static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, " %-10s:", msg); + mi_print_amount(stat->total, 1, out, arg); + _mi_fprintf(out, arg, "\n"); +} + +static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) { + const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); + const int64_t avg_whole = avg_tens/10; + const int64_t avg_frac1 = avg_tens%10; + _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); +} + + +static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { + _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n", + name, "peak ", "total ", "current ", "block ", "total# "); +} + +#if MI_STAT>1 +static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { + bool found = false; + char buf[64]; + for (size_t i = 0; i <= max; i++) { + if (bins[i].total > 0) { + found = true; + const size_t unit = _mi_bin_size((uint8_t)i); + const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? "S" : + (unit <= MI_MEDIUM_MAX_OBJ_SIZE ? "M" : + (unit <= MI_LARGE_MAX_OBJ_SIZE ? "L" : "H"))); + _mi_snprintf(buf, 64, "bin%2s %3lu", pagekind, (long)i); + mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg); + } + } + if (found) { + _mi_fprintf(out, arg, "\n"); + } + return found; +} +#endif + + + +//------------------------------------------------------------ +// Use an output wrapper for line-buffered output +// (which is nice when using loggers etc.) +//------------------------------------------------------------ +typedef struct buffered_s { + mi_output_fun* out; // original output function + void* arg; // and state + char* buf; // local buffer of at least size `count+1` + size_t used; // currently used chars `used <= count` + size_t count; // total chars available for output +} buffered_t; + +static void mi_buffered_flush(buffered_t* buf) { + buf->buf[buf->used] = 0; + _mi_fputs(buf->out, buf->arg, NULL, buf->buf); + buf->used = 0; +} + +static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { + buffered_t* buf = (buffered_t*)arg; + if (msg==NULL || buf==NULL) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (buf->used >= buf->count) mi_buffered_flush(buf); + mi_assert_internal(buf->used < buf->count); + buf->buf[buf->used++] = c; + if (c == '\n') mi_buffered_flush(buf); + } +} + +//------------------------------------------------------------ +// Print statistics +//------------------------------------------------------------ + +mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept +{ + size_t elapsed; + size_t user_time; + size_t sys_time; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; + mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); + _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); + _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", + user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); + mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); + if (peak_commit > 0) { + _mi_fprintf(out, arg, ", peak commit: "); + mi_printf_amount((int64_t)peak_commit, 1, out, arg, false); + } + _mi_fprintf(out, arg, "\n"); +} + +void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { + // wrap the output function to be line buffered + char buf[256]; _mi_memzero_var(buf); + buffered_t buffer = { out0, arg0, NULL, 0, 255 }; + buffer.buf = buf; + mi_output_fun* out = &mi_buffered_out; + void* arg = &buffer; + + // and print using that + _mi_fprintf(out, arg, "%s %zu\n", name, id); + + if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { + #if MI_STAT>1 + mi_print_header("blocks", out, arg); + mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); + #endif + #if MI_STAT + mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg); + mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg); + mi_stat_count_t total = { 0,0,0 }; + mi_stat_count_add_mt(&total, &stats->malloc_normal); + mi_stat_count_add_mt(&total, &stats->malloc_huge); + mi_stat_print_ex(&total, "total", 1, out, arg, ""); + #if MI_STAT>1 + mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); + #endif + _mi_fprintf(out, arg, "\n"); + #endif + } + + if (stats->pages.total != 0) { + mi_print_header("pages", out, arg); + mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); + // mi_stat_print(&stats->segments, "segments", -1, out, arg); + // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); + // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); + mi_stat_print(&stats->pages, "pages", 0, out, arg); + mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg); + mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); + mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg); + mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg); + mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); + mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); + mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); + mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); + _mi_fprintf(out, arg, "\n"); + } + + if (stats->arena_count.total > 0) { + mi_print_header("arenas", out, arg); + mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); + mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); + mi_stat_counter_print_size(&stats->reset, "reset", out, arg); + mi_stat_counter_print_size(&stats->purged, "purged", out, arg); + + mi_stat_counter_print(&stats->arena_count, "arenas", out, arg); + mi_stat_counter_print(&stats->arena_rollback_count, "rollback", out, arg); + mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); + mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); + mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); + mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); + mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); + mi_stat_print_ex(&stats->theaps, "theaps", 0, out, arg, ""); + mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, ""); + mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg); + _mi_fprintf(out, arg, "\n"); + } + + mi_print_header("process", out, arg); + mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); + _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + mi_process_info_print_out(out, arg); + + _mi_fprintf(out, arg, "\n"); +} + + +static mi_msecs_t mi_process_start; // = 0 + +// called on process init +void _mi_stats_init(void) { + if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); }; +} + +static void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) { + mi_assert_internal(to != NULL && from != NULL); + if (to == from) return; + mi_stats_add(to, from); +} + +void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) { + mi_assert_internal(to != NULL && from != NULL); + if (to == from) return; + mi_stats_add(to, from); + mi_stats_init(from); // zero field and keep the header +} + +static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept { + mi_stats_t* stats = &theap->stats; + mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats; + _mi_stats_merge_into( heap_stats, stats ); + return heap_stats; +} + +static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) { + if (heap==NULL) { heap = mi_heap_main(); } + mi_theap_t* theap = _mi_heap_theap_peek(heap); + if (theap==NULL) return &heap->stats; + else return mi_stats_merge_theap_to_heap(theap); +} + +static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) { + return &theap->stats; +} + +// deprecated +void mi_stats_reset(void) mi_attr_noexcept { + if (!mi_theap_is_initialized(_mi_theap_default())) return; + mi_heap_t* heap_main = mi_heap_main(); + mi_heap_get_stats(heap_main); + mi_heap_stats_merge_to_subproc(heap_main); +} + + +void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept { + if (heap==NULL) { heap = mi_heap_main(); } + _mi_stats_print("heap", heap->heap_seq, mi_heap_get_stats(heap), out, arg); +} + +typedef struct mi_heap_print_visit_info_s { + mi_output_fun* out; + void* out_arg; +} mi_heap_print_visit_info_t; + +static bool mi_cdecl mi_heap_print_visitor(mi_heap_t* heap, void* arg) { + mi_heap_print_visit_info_t* vinfo = (mi_heap_print_visit_info_t*)(arg); + mi_heap_stats_print_out(heap, vinfo->out, vinfo->out_arg); + return true; +} + + +// show each heap and then the subproc +void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return; + mi_heap_print_visit_info_t vinfo = { out, arg }; + mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo); + if (subproc->theap_meta!=NULL) { + _mi_stats_print("meta", subproc->subproc_seq, &subproc->theap_meta->stats, out, arg); + } + // if (subproc->theap_meta!=NULL) { _mi_stats_merge_into(&subproc->stats, &subproc->theap_meta->stats); } + _mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg); +} + + +// aggregate all stats from the heaps and subproc and print those +void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return; + mi_stats_t_decl(stats); + if (mi_subproc_stats_get(subproc_id, &stats)) { + _mi_stats_print("subproc", subproc->subproc_seq, &stats, out, arg); + } +} + +void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_subproc_stats_print_out(mi_subproc_current(),out, arg); +} + +// deprecated +void mi_stats_print(void* out) mi_attr_noexcept { + // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`) + mi_stats_print_out((mi_output_fun*)out, NULL); +} + +// deprecated +void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { + mi_theap_t* theap = _mi_theap_default(); + if (theap==NULL || !mi_theap_is_initialized(theap)) return; + _mi_stats_print("heap", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg); + mi_stats_merge_theap_to_heap(_mi_theap_default()); +} + + +// ---------------------------------------------------------------- +// Basic timer for convenience; use milli-seconds to avoid doubles +// ---------------------------------------------------------------- + +static mi_msecs_t mi_clock_diff; + +mi_msecs_t _mi_clock_now(void) { + return _mi_prim_clock_now(); +} + +mi_msecs_t _mi_clock_start(void) { + if (mi_clock_diff == 0.0) { + mi_msecs_t t0 = _mi_clock_now(); + mi_clock_diff = _mi_clock_now() - t0; + } + return _mi_clock_now(); +} + +mi_msecs_t _mi_clock_end(mi_msecs_t start) { + mi_msecs_t end = _mi_clock_now(); + return (end - start - mi_clock_diff); +} + + +// -------------------------------------------------------- +// Basic process statistics +// -------------------------------------------------------- + +mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept +{ + mi_process_info_t pinfo; + _mi_memzero_var(pinfo); + pinfo.elapsed = _mi_clock_end(mi_process_start); + { const mi_subproc_t* subproc = _mi_subproc_main(); + if (subproc!=NULL) { + const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)); + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)); + pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX)); + pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX)); + } + } + pinfo.current_rss = pinfo.current_commit; + pinfo.peak_rss = pinfo.peak_commit; + pinfo.utime = 0; + pinfo.stime = 0; + pinfo.page_faults = 0; + + _mi_prim_process_info(&pinfo); + + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); + if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); + if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); + if (current_rss!=NULL) *current_rss = pinfo.current_rss; + if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss; + if (current_commit!=NULL) *current_commit = pinfo.current_commit; + if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; + if (page_faults!=NULL) *page_faults = pinfo.page_faults; +} + +mi_decl_export void mi_process_info_print(void) mi_attr_noexcept { + mi_process_info_print_out(NULL, NULL); +} + + +// -------------------------------------------------------- +// Return statistics +// -------------------------------------------------------- + +size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept { + if (bin > MI_BIN_HUGE) return 0; + return _mi_bin_size(bin); +} + +static bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept { + if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false; + if (stats_from == NULL || stats_from->size != stats_to->size) return false; + _mi_memcpy(stats_to, stats_from, stats_to->size); + return true; +} + +bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept { + const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + return mi_stats_copy(stats,&subproc->stats); +} + +bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept { + return mi_stats_copy(stats, mi_heap_get_stats(heap)); +} + +bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { + return mi_stats_copy(stats, mi_theap_get_stats(theap)); +} + +static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { + mi_stats_t* stats = (mi_stats_t*)arg; + mi_stats_add_into(stats, mi_heap_get_stats(heap)); + return true; +} + +bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept { + if (stats==NULL) return false; + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc == NULL) return false; + if (!mi_stats_copy(stats, &subproc->stats)) return false; + mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats); + // hide meta data stats + // if (subproc->theap_meta!=NULL) { mi_stats_add_into(stats, &subproc->theap_meta->stats); } + return true; +} + +bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept { + return mi_subproc_stats_get(mi_subproc_current(), stats); +} + + +// -------------------------------------------------------- +// Statics in json format +// -------------------------------------------------------- + +typedef struct mi_json_buf_s { + char* buf; + size_t size; + size_t used; + bool can_realloc; +} mi_json_buf_t; + +static bool mi_json_buf_expand(mi_json_buf_t* hbuf) { + if (hbuf==NULL) return false; + if (hbuf->buf != NULL && hbuf->size>0) { + hbuf->buf[hbuf->size-1] = 0; + } + if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false; + const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size); + char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize); + if (newbuf == NULL) return false; + hbuf->buf = newbuf; + hbuf->size = newsize; + return true; +} + +static void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) { + if (msg==NULL || hbuf==NULL) return; + if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (hbuf->used + 1 >= hbuf->size) { + if (!mi_json_buf_expand(hbuf)) return; + } + mi_assert_internal(hbuf->used < hbuf->size); + hbuf->buf[hbuf->used++] = c; + } + mi_assert_internal(hbuf->used < hbuf->size); + hbuf->buf[hbuf->used] = 0; +} + +static void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) { + const size_t binsize = mi_stats_get_bin_size(bin); + const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE : + (binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE : + (binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0))); + char buf[128]; + _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : "")); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); +} + +static void mi_json_buf_print_count_cbin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) { + const char* cbin = " "; + switch(bin) { + case MI_CBIN_SMALL: cbin = "S"; break; + case MI_CBIN_MEDIUM: cbin = "M"; break; + case MI_CBIN_LARGE: cbin = "L"; break; + case MI_CBIN_HUGE: cbin = "H"; break; + case MI_CBIN_OTHER: cbin = "X"; break; + default: cbin = " "; break; + } + char buf[128]; + _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"bin\": \"%s\" }%s\n", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? "," : "")); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); +} + +static void mi_json_buf_print_count(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, bool add_comma) { + char buf[128]; + _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : "")); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); +} + +static void mi_json_buf_print_count_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_count_t* stat) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": ", name); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); + mi_json_buf_print_count(hbuf, "", stat, true); +} + +static void mi_json_buf_print_value(mi_json_buf_t* hbuf, const char* name, int64_t val) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": %lld,\n", name, val); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); +} + +static void mi_json_buf_print_size(mi_json_buf_t* hbuf, const char* name, size_t val, bool add_comma) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": %zu%s\n", name, val, (add_comma ? "," : "")); + buf[127] = 0; + mi_json_buf_print(hbuf, buf); +} + +static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_counter_t* stat) { + mi_json_buf_print_value(hbuf, name, stat->total); +} + +static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept { + if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL; + mi_json_buf_t hbuf = { NULL, 0, 0, true }; + if (output_size > 0 && output_buf != NULL) { + _mi_memzero(output_buf, output_size); + hbuf.buf = output_buf; + hbuf.size = output_size; + hbuf.can_realloc = false; + } + else { + if (!mi_json_buf_expand(&hbuf)) return NULL; + } + mi_json_buf_print(&hbuf, "{\n"); + mi_json_buf_print_value(&hbuf, "stat_version", MI_STAT_VERSION); + mi_json_buf_print_value(&hbuf, "mimalloc_version", MI_MALLOC_VERSION); + + // process info + mi_json_buf_print(&hbuf, " \"process\": {\n"); + size_t elapsed; + size_t user_time; + size_t sys_time; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; + mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); + mi_json_buf_print_size(&hbuf, "elapsed_msecs", elapsed, true); + mi_json_buf_print_size(&hbuf, "user_msecs", user_time, true); + mi_json_buf_print_size(&hbuf, "system_msecs", sys_time, true); + mi_json_buf_print_size(&hbuf, "page_faults", page_faults, true); + mi_json_buf_print_size(&hbuf, "rss_current", current_rss, true); + mi_json_buf_print_size(&hbuf, "rss_peak", peak_rss, true); + mi_json_buf_print_size(&hbuf, "commit_current", current_commit, true); + mi_json_buf_print_size(&hbuf, "commit_peak", peak_commit, false); + mi_json_buf_print(&hbuf, " },\n"); + + // statistics + #define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat); + #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); + + MI_STAT_FIELDS() + + #undef MI_STAT_COUNT + #undef MI_STAT_COUNTER + + // size bins + mi_json_buf_print(&hbuf, " \"malloc_bins\": [\n"); + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_json_buf_print_count_bin(&hbuf, " ", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE); + } + mi_json_buf_print(&hbuf, " ],\n"); + mi_json_buf_print(&hbuf, " \"page_bins\": [\n"); + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_json_buf_print_count_bin(&hbuf, " ", &stats->page_bins[i], i, i!=MI_BIN_HUGE); + } + mi_json_buf_print(&hbuf, " ],\n"); + mi_json_buf_print(&hbuf, " \"chunk_bins\": [\n"); + for (size_t i = 0; i < MI_CBIN_COUNT; i++) { + mi_json_buf_print_count_cbin(&hbuf, " ", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1); + } + mi_json_buf_print(&hbuf, " ]\n"); + mi_json_buf_print(&hbuf, "}\n"); + if (hbuf.used + 1 >= hbuf.size) { + // failed + if (hbuf.can_realloc) { mi_free(hbuf.buf); } + return NULL; + } + else { + return hbuf.buf; + } +} + +char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return NULL; + mi_stats_t_decl(stats); + if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL; + return mi_stats_get_json_from(&stats, buf_size, buf); +} + +char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept { + return mi_stats_get_json_from(mi_heap_get_stats(heap), buf_size, buf); +} + +char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept { + return mi_subproc_stats_get_json(mi_subproc_current(), buf_size, buf); +} + +char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept { + return mi_stats_get_json_from(stats, buf_size, buf); +} diff --git a/csrc/mimalloc/src/subproc.c b/csrc/mimalloc/src/subproc.c new file mode 100644 index 000000000..7cce9edd7 --- /dev/null +++ b/csrc/mimalloc/src/subproc.c @@ -0,0 +1,327 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +// pre-allocate the main subprocess structure. +static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero; +static mi_subproc_t* mi_subprocs = NULL; +static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER; + + +/* ----------------------------------------------------------- + Meta-data allocation + We allocate thread local data and theaps through a dedicated + theap `subproc.theap_meta` which uses a detached tld with + a detached thread id. The initial theap_meta is statically + allocated and can thus be used to allocate on an as yet + uninitialized thread or process. + We need to take a lock though to allocate safely on the + detached `theap_meta`. +----------------------------------------------------------- */ + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, size); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t aligned, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc_aligned(subproc->theap_meta, size, aligned); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which + // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358. + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, newsize); + } + if (p!=NULL) { + if (oldp!=NULL) { + const size_t oldsize = mi_usable_size(oldp); + const size_t copysize = (newsize < oldsize ? newsize : oldsize); + _mi_memcpy(p,oldp,copysize); + if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); } + else { mi_free(oldp); } + } + if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); } + } + else { + if (memid!=NULL) { *memid = _mi_memid_none(); } + } + return p; +} + +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { + if (p==NULL || mi_memid_needs_no_free(memid)) return; + if (memid.memkind == MI_MEM_MALLOC) { + mi_free(p); + } + else { + mi_assert_internal(subproc!=NULL); + _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid); + } +} + +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { + if (page==NULL) return false; + mi_theap_t* theap = page->theap; + return (theap != NULL && theap == subproc->theap_meta); +} + + +/* ----------------------------------------------------------- + Sub process helpers +----------------------------------------------------------- */ + +mi_subproc_t* _mi_subproc_main(void) { + return &mi_process_subproc_main; +} + +bool _mi_subproc_is_main(mi_subproc_t* subproc) { + return (subproc == &mi_process_subproc_main); +} + +mi_subproc_t* _mi_subproc(void) { + mi_theap_t* theap = _mi_theap_default(); + if (theap == NULL || theap->tld == NULL) { // see issue #1289 + return _mi_subproc_main(); + } + else { + return theap->tld->subproc; + } +} + +mi_heap_t* mi_heap_main(void) { + return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init +} + + +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { + return (mi_subproc_t*)(subproc_id._mi_subproc_id); +} + +mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) { + mi_subproc_id_t id = { subproc }; + return id; +} + +mi_subproc_id_t mi_subproc_main(void) { + return _mi_subproc_to_id(_mi_subproc_main()); +} + +mi_subproc_id_t mi_subproc_current(void) { + return _mi_subproc_to_id(_mi_subproc()); +} + + +/* ----------------------------------------------------------- + Sub process creation +----------------------------------------------------------- */ + + +static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) { + static _Atomic(size_t) subproc_total_count; + subproc->parent = parent; + subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count); + mi_stats_header_init(&subproc->stats); + mi_lock_init(&subproc->arena_reserve_lock); + mi_lock_init(&subproc->heaps_lock); + mi_lock_init(&subproc->theap_meta_lock); + mi_lock(&mi_subprocs_lock) { + // push on subproc list + subproc->next = mi_subprocs; + if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; } + mi_subprocs = subproc; + } + return subproc; +} + +mi_subproc_id_t mi_subproc_new(void) { + mi_thread_init(); + mi_subproc_t* const parent = _mi_subproc(); + mi_memid_t memid; + mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid); + if (subproc == NULL) { return _mi_subproc_to_id(NULL); } + subproc->memid = memid; + + mi_memid_t theap_memid; + mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid); + if (theap_meta==NULL) { + _mi_meta_free(parent, subproc, memid); + return _mi_subproc_to_id(NULL); + } + theap_meta->memid = memid; + + // init subproc + mi_subproc_init(subproc,parent); + + // init main heap + mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true); + if (heap_main==NULL) { + _mi_meta_free(parent, theap_meta, theap_meta->memid); + mi_subproc_destroy(_mi_subproc_to_id(subproc)); + return _mi_subproc_to_id(NULL); + } + mi_assert_internal(subproc->heap_main == heap_main); + + // init meta theap + mi_assert_internal(parent->theap_meta!=NULL); + mi_assert_internal(parent->theap_meta->tld!=NULL); + mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); + _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + subproc->theap_meta = theap_meta; + + return _mi_subproc_to_id(subproc); +} + + +/* ----------------------------------------------------------- + Sub process destruction +----------------------------------------------------------- */ + +// destroy all subproc resources including arena's, heap's etc. +static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock) +{ + if (subproc==NULL) return; + + // remove from the subproc list + mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) { + if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } + if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } + else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; } + } + + // destroy all subproc heaps + mi_lock(&subproc->heaps_lock) { + mi_heap_t* heap = subproc->heaps; + while (heap != NULL) { + mi_heap_t* next = heap->next; + if (heap!=subproc->heap_main) { _mi_heap_force_destroy(heap, false /* don't re-acquire the heaps_lock */); } + heap = next; + } + mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main); + if (subproc->heap_main!=NULL) { + _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key) + if (_mi_subproc_is_main(subproc)) { + _mi_thread_locals_done(); + } + _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap + } + } + + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + + if (!_mi_subproc_is_main(subproc)) { + // merge stats back into the main subproc + _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); + } + + // remove associated arenas + _mi_arenas_unsafe_destroy_all(subproc); + + // show stats of the main process (at process end) before releasing the heaps lock + if (_mi_subproc_is_main(subproc)) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + } + } + + // todo: should we refcount subprocesses? + mi_lock_done(&subproc->arena_reserve_lock); + mi_lock_done(&subproc->heaps_lock); + mi_lock_done(&subproc->theap_meta_lock); + _mi_meta_free( subproc->parent, subproc, subproc->memid); + if (_mi_subproc_is_main(subproc)) { + // for the main subproc, also release the global page map + _mi_page_map_unsafe_destroy(); + } +} + +void mi_subproc_destroy(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL || subproc==&mi_process_subproc_main) return; + mi_subproc_unsafe_destroy(subproc, true /* take lock */); +} + +void _mi_subprocs_unsafe_destroy_all(void) { + mi_lock(&mi_subprocs_lock) { + mi_subproc_t* subproc = mi_subprocs; + while (subproc!=NULL) { + mi_subproc_t* next = subproc->next; + if (subproc!=&mi_process_subproc_main) { + mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */); + } + subproc = next; + } + } + mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */); +} + + +/* ----------------------------------------------------------- + Sub process various +----------------------------------------------------------- */ + +void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + mi_assert_internal(subproc!=NULL); + if (subproc==NULL) return; + mi_assert_internal(subproc->heap_main!=NULL); + if (subproc->heap_main==NULL) return; + mi_theap_t* theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) { + if (theap->tld!=NULL && theap->tld->subproc != subproc) { + _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc); + } + return; + } + + // initialize this thread tld & theap + _mi_thread_init_with_heap(subproc->heap_main); +} + + +bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + bool ok = true; + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) { + ok = (*visitor)(heap, arg); + } + } + return ok; +} + + +mi_subproc_t* _mi_subproc_main_init(void) { + mi_lock_init(&mi_subprocs_lock); + mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t)); + mi_process_subproc_main.memid = memid; + mi_subproc_init(&mi_process_subproc_main,NULL); + return &mi_process_subproc_main; +} + +void _mi_subproc_main_done(void) { + mi_lock_done(&mi_subprocs_lock); +} + diff --git a/csrc/mimalloc/src/theap.c b/csrc/mimalloc/src/theap.c new file mode 100644 index 000000000..5d37893dd --- /dev/null +++ b/csrc/mimalloc/src/theap.c @@ -0,0 +1,724 @@ +/*---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim-tls.h" // _mi_theap_default + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +#pragma warning(disable:4204) // non-constant aggregate initializer +#endif + +/* ----------------------------------------------------------- + Helpers +----------------------------------------------------------- */ + +// return `true` if ok, `false` to break +typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2); + +// Visit all pages in a theap; returns `false` if break was called. +static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2) +{ + if (theap==NULL || theap->page_count==0) return true; + + // visit all pages + #if MI_DEBUG>1 + size_t total = theap->page_count; + size_t count = 0; + #endif + + const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1); + for (size_t i = 0; i <= max_bin; i++) { + mi_page_queue_t* pq = &theap->pages[i]; + mi_page_t* page = pq->first; + while(page != NULL) { + mi_page_t* next = page->next; // save next in case the page gets removed from the queue + mi_assert_internal(mi_page_theap(page) == theap); + #if MI_DEBUG>1 + count++; + #endif + if (!fn(theap, pq, page, arg1, arg2)) return false; + page = next; // and continue + } + } + mi_assert_internal(!include_full || count == total); + return true; +} + + +#if MI_DEBUG>=3 +static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(pq); + mi_assert_internal(mi_page_theap(page) == theap); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || theap == page_theap); + mi_assert_expensive(_mi_page_is_valid(page)); + return true; +} + +static bool mi_theap_is_valid(mi_theap_t* theap) { + mi_assert_internal(theap!=NULL); + mi_heap_t* const heap = _mi_theap_heap_peek(theap); + mi_assert_internal(heap != NULL); + mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); // don't use mi_heap_theap as that may re-initialize the thread + mi_assert_internal(heap_theap==NULL || heap_theap == theap); + mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL); + for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { + mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin])); + } + return true; +} +#endif + + + + +/* ----------------------------------------------------------- + "Collect" pages by migrating `local_free` and `thread_free` + lists and freeing empty pages. This is done when a thread + stops (and in that case abandons pages if there are still + blocks alive) +----------------------------------------------------------- */ + +typedef enum mi_collect_e { + MI_NORMAL, + MI_FORCE, + MI_ABANDON +} mi_collect_t; + + +static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { + MI_UNUSED(arg2); + MI_UNUSED(theap); + mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); + mi_collect_t collect = *((mi_collect_t*)arg_collect); + _mi_page_free_collect(page, collect >= MI_FORCE); + if (mi_page_all_free(page)) { + // no more used blocks, possibly free the page. + if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired + // note: this will potentially free retired pages as well. + _mi_page_free(page, pq); + } + } + else if (collect == MI_ABANDON) { + // still used blocks but the thread is done; abandon the page + _mi_page_abandon(page, pq); + } + return true; // don't break +} + +void _mi_theap_merge_stats(mi_theap_t* theap) { + mi_assert_internal(mi_theap_is_initialized(theap)); + mi_heap_t* const heap = _mi_theap_heap(theap); + _mi_stats_merge_into(&heap->stats, &theap->stats); +} + +static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) +{ + if (theap==NULL || !mi_theap_is_initialized(theap)) return; + mi_assert_expensive(mi_theap_is_valid(theap)); + + const bool force = (collect >= MI_FORCE); + _mi_deferred_free(theap, force); + + // python/cpython#112532: we may be called from a thread that is not the owner of the theap + // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id()); + + // collect retired pages (and full pages if theap->allow_page_abandon is false) + _mi_theap_collect_retired(theap, force); + + // collect all pages owned by this thread + mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220 + + // collect arenas (this is program wide so don't force purges on abandonment of threads) + //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1); + if (collect != MI_ABANDON) { + _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); + } + + // merge statistics + _mi_theap_merge_stats(theap); +} + +void _mi_theap_collect_abandon(mi_theap_t* theap) { + mi_theap_collect_ex(theap, MI_ABANDON); +} + +void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept { + mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL)); +} + +void mi_collect(bool force) mi_attr_noexcept { + // cannot really collect process wide, just a theap.. + mi_theap_collect(_mi_theap_default(), force); +} + +void mi_heap_collect(mi_heap_t* heap, bool force) { + // cannot really collect a heap, just a theap.. + mi_theap_collect(mi_heap_theap(heap), force); +} + +/* ----------------------------------------------------------- + Heap new +----------------------------------------------------------- */ + +mi_theap_t* mi_theap_get_default(void) { + mi_theap_t* theap = _mi_theap_default(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { + mi_thread_init(); + theap = _mi_theap_default(); + mi_assert_internal(mi_theap_is_initialized(theap)); + } + return theap; +} + +mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { + mi_theap_t* const previous = mi_theap_get_default(); + if (mi_theap_is_initialized(theap)) { + _mi_theap_default_set(theap); + } + return previous; +} + +#if MI_GUARDED +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = sample_rate; + theap->guarded_sample_count = sample_rate; // count down samples + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +static void mi_theap_guarded_init(mi_theap_t* theap) { + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); +} +#else +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); +} +static void mi_theap_guarded_init(mi_theap_t* theap) { + MI_UNUSED(theap); +} +#endif + +static void mi_theap_options_init(mi_theap_t* theap) { + theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); + theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); + theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); + theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); +} + +// todo: make order of parameters consistent (but would that break compat with CPython?) +void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) +{ + mi_assert_internal(theap!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert_internal(tld!=NULL); + mi_memid_t memid = theap->memid; + _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t)); + theap->memid = memid; + theap->tld = tld; // avoid reading the thread-local tld during initialization + mi_atomic_store_release(&theap->refcount,1); + mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc); + mi_assert_internal(theap->stats.size == sizeof(mi_stats_t)); + mi_theap_options_init(theap); + + if (theap->tld->is_in_threadpool) { + // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap. + // this is checked in `free.c:mi_free_try_collect_mt` + // .. but abandoning is good in this case: quarter the full page retain (possibly to 0) + // (so blocked threads do not hold on to too much memory) + if (theap->page_full_retain > 0) { + theap->page_full_retain = theap->page_full_retain / 4; + } + } + + // push on the thread local theaps list + mi_theap_t* head = NULL; + mi_random_ctx_t head_random; + mi_lock(&theap->tld->theaps_lock) { + head = theap->tld->theaps; + theap->tprev = NULL; + theap->tnext = head; + theap->tld->theaps = theap; + if (head!=NULL) { + head->tprev = theap; + head_random = head->random; + } + } + + // initialize random if heap==NULL + if (head==NULL) { // first theap of the first thread? + #if defined(_WIN32) && !defined(MI_SHARED_LIB) + if (tld->thread_seq==0) { + _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) + } + else + #endif + { + _mi_random_init(&theap->random); + } + } + else { + _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random + } + theap->cookie = _mi_theap_random_next(theap) | 1; + mi_theap_guarded_init(theap); // needs theap->random + if (!theap->is_detached) { + mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem + } + + // only now set the heap member as it is used to determine if a theap is initialized + mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap); + + // push on the heap's theap list + mi_lock(&heap->theaps_lock) { + head = heap->theaps; + theap->hprev = NULL; + theap->hnext = head; + if (head!=NULL) { head->hprev = theap; } + heap->theaps = theap; + } +} + +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) { + mi_assert_internal(tld!=NULL); + mi_assert_internal(heap!=NULL); + mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id); + // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init + + // allocate and initialize a theap + mi_memid_t memid; + mi_theap_t* theap; + + if (heap->exclusive_arena == NULL) { + theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid); + } + else { + // theaps associated with a specific arena are allocated in that arena + // note: takes up at least one slice which is quite wasteful... + const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE); + theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid); + } + if (theap==NULL) { + _mi_error_message(ENOMEM, "unable to allocate theap meta-data\n"); + return NULL; + } + + theap->memid = memid; + return theap; +} + +mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { + mi_theap_t* theap = _mi_theap_alloc(heap,tld); + if (theap == NULL) return NULL; + _mi_theap_init(theap, heap, tld); + return theap; +} + +uintptr_t _mi_theap_random_next(mi_theap_t* theap) { + return _mi_random_next(&theap->random); +} + +static void mi_theap_free_mem(mi_theap_t* theap) { + if (theap!=NULL) { + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + if (!theap->is_detached) { + mi_subproc_stat_decrease(subproc,theaps,1); + } + _mi_meta_free(subproc, theap, theap->memid); + } +} + +// we need to reference count theaps due to the _mi_theap_cached thread locals +void _mi_theap_incref(mi_theap_t* theap) { + if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) { + mi_atomic_increment_acq_rel(&theap->refcount); + } +} + +void _mi_theap_decref(mi_theap_t* theap) { + if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) { + if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) { + mi_theap_free_mem(theap); + } + } +} + +// Thread termination and heap delete/destroy might run concurrently +// and we need to ensure we free the memory correctly. A heap or tld +// will first "detach" its theaps so it has a list with theaps that are +// no longer shared, and only then free's the theaps in that list. +// To detach we need to hold both the `heap->theaps_lock` and the `tld->theaps_lock`. +// Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails +// we back-off, release the outer lock, and try again until we succeed. + +// Remove the theaps in this heap from any thread local tld lists. +void _mi_heap_detach_theaps( mi_heap_t* heap ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&heap->theaps_lock) { + mi_theap_t* theap = heap->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->hnext; + mi_tld_t* tld = theap->tld; + if (tld != NULL) { + if (mi_lock_try_acquire(&tld->theaps_lock)) { + // remove the theap from the tld theaps list + if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } + if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } + else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } + theap->tnext = theap->tprev = NULL; + theap->tld = NULL; + mi_lock_release(&tld->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } + } + if (!all_detached) { + mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); + } + } while (!all_detached); +} + +// Remove the theaps in this thread from the heaps that own them. +void _mi_tld_detach_theaps( mi_tld_t* tld ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&tld->theaps_lock) { + mi_theap_t* theap = tld->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration + if (heap != NULL) { + if (mi_lock_try_acquire(&heap->theaps_lock)) { + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // remove the theap from the heap list + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } + theap->hnext = theap->hprev = NULL; + // and set `heap` to NULL + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); + mi_lock_release(&heap->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } + } + if (!all_detached) { + mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); + } + } while (!all_detached); +} + + + +/* ----------------------------------------------------------- + Safe theap delete +----------------------------------------------------------- */ + +// Safe delete a theap without freeing any still allocated blocks in that theap. +// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) +// { +// mi_assert(theap != NULL); +// mi_assert(mi_theap_is_initialized(theap)); +// mi_assert_expensive(mi_theap_is_valid(theap)); +// if (theap==NULL || !mi_theap_is_initialized(theap)) return; + +// // abandon all pages +// _mi_theap_collect_abandon(theap); + +// mi_assert_internal(theap->page_count==0); +// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); +// } + + + +/* ----------------------------------------------------------- + Load/unload theaps +----------------------------------------------------------- */ +/* +void mi_theap_unload(mi_theap_t* theap) { + mi_assert(mi_theap_is_initialized(theap)); + mi_assert_expensive(mi_theap_is_valid(theap)); + if (theap==NULL || !mi_theap_is_initialized(theap)) return; + if (_mi_theap_heap(theap)->exclusive_arena == NULL) { + _mi_warning_message("cannot unload theaps that are not associated with an exclusive arena\n"); + return; + } + + // abandon all pages so all thread'id in the pages are cleared + _mi_theap_collect_abandon(theap); + mi_assert_internal(theap->page_count==0); + + // remove from theap list + mi_theap_free(theap, false); // but don't actually free the memory + + // disassociate from the current thread-local and static state + theap->tld = NULL; + return; +} + +bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) { + mi_assert(mi_theap_is_initialized(theap)); + if (theap==NULL || !mi_theap_is_initialized(theap)) return false; + if (_mi_theap_heap(theap)->exclusive_arena == NULL) { + _mi_warning_message("cannot reload theaps that were not associated with an exclusive arena\n"); + return false; + } + if (theap->tld != NULL) { + _mi_warning_message("cannot reload theaps that were not unloaded first\n"); + return false; + } + mi_arena_t* arena = _mi_arena_from_id(arena_id); + if (_mi_theap_heap(theap)->exclusive_arena != arena) { + _mi_warning_message("trying to reload a theap at a different arena address: %p vs %p\n", _mi_theap_heap(theap)->exclusive_arena, arena); + return false; + } + + mi_assert_internal(theap->page_count==0); + + // re-associate with the current thread-local and static state + theap->tld = mi_theap_get_default()->tld; + + // reinit direct pages (as we may be in a different process) + mi_assert_internal(theap->page_count == 0); + for (size_t i = 0; i < MI_PAGES_DIRECT; i++) { + theap->pages_free_direct[i] = _mi_page_empty_get(); + } + + // push on the thread local theaps list + theap->tnext = theap->tld->theaps; + theap->tld->theaps = theap; + return true; +} +*/ + + +/* ----------------------------------------------------------- + Visit all theap blocks and areas + Todo: enable visiting abandoned pages, and + enable visiting all blocks of all theaps across threads +----------------------------------------------------------- */ + +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); + area->reserved = page->reserved * bsize; + area->committed = page->capacity * bsize; + area->blocks = mi_page_start(page); + area->used = page->used; // number of blocks in use (#553) + area->block_size = ubsize; + area->full_block_size = bsize; + area->reserved1 = page; +} + +static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) { + mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); + *shift = MI_SIZE_BITS - mi_clz(divisor - 1); + *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1); +} + +static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) { + mi_assert_internal(n <= UINT32_MAX); + const uint64_t hi = ((uint64_t)n * magic) >> 32; + return (size_t)((hi + n) >> shift); +} + +bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) { + mi_assert(area != NULL); + if (area==NULL) return true; + mi_assert(page != NULL); + if (page == NULL) return true; + + _mi_page_free_collect(page,true); // collect both thread_delayed and local_free + mi_assert_internal(page->local_free == NULL); + if (page->used == 0) return true; + + size_t psize; + uint8_t* const pstart = mi_page_area(page, &psize); + mi_heap_t* const heap = mi_page_heap(page); + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); // without padding + + // optimize page with one block + if (page->capacity == 1) { + mi_assert_internal(page->used == 1 && page->free == NULL); + return visitor(heap, area, pstart, ubsize, arg); + } + mi_assert(bsize <= UINT32_MAX); + + // optimize full pages + if (page->used == page->capacity) { + uint8_t* block = pstart; + for (size_t i = 0; i < page->capacity; i++) { + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + return true; + } + + // create a bitmap of free blocks. + #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) + uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; + const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS); + memset(free_map, 0, bmapsize * sizeof(intptr_t)); + if (page->capacity % MI_INTPTR_BITS != 0) { + // mark left-over bits at the end as free + size_t shift = (page->capacity % MI_INTPTR_BITS); + uintptr_t mask = (UINTPTR_MAX << shift); + free_map[bmapsize - 1] = mask; + } + + // fast repeated division by the block size + uint64_t magic; + size_t shift; + mi_get_fast_divisor(bsize, &magic, &shift); + + #if MI_DEBUG>1 + size_t free_count = 0; + #endif + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) { + #if MI_DEBUG>1 + free_count++; + #endif + mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); + size_t offset = (uint8_t*)block - pstart; + mi_assert_internal(offset % bsize == 0); + mi_assert_internal(offset <= UINT32_MAX); + size_t blockidx = mi_fast_divide(offset, magic, shift); + mi_assert_internal(blockidx == offset / bsize); + mi_assert_internal(blockidx < MI_MAX_BLOCKS); + size_t bitidx = (blockidx / MI_INTPTR_BITS); + size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); + free_map[bitidx] |= ((uintptr_t)1 << bit); + } + mi_assert_internal(page->capacity == (free_count + page->used)); + + // walk through all blocks skipping the free ones + #if MI_DEBUG>1 + size_t used_count = 0; + #endif + uint8_t* block = pstart; + for (size_t i = 0; i < bmapsize; i++) { + if (free_map[i] == 0) { + // every block is in use + for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + #if MI_DEBUG>1 + used_count++; + #endif + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + } + else { + // visit the used blocks in the mask + uintptr_t m = ~free_map[i]; + while (m != 0) { + #if MI_DEBUG>1 + used_count++; + #endif + size_t bitidx = mi_ctz(m); + if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + m &= m - 1; // clear least significant bit + } + block += bsize * MI_INTPTR_BITS; + } + } + mi_assert_internal(page->used == used_count); + return true; +} + +// bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) { +// mi_heap_area_t area; +// _mi_heap_area_init(&area, page); +// return _mi_theap_area_visit_blocks(&area, page, visitor, arg); +// } + + +// Separate struct to keep `mi_page_t` out of the public interface +typedef struct mi_theap_area_ex_s { + mi_heap_area_t area; + mi_page_t* page; +} mi_theap_area_ex_t; + +typedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg); + +static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { + MI_UNUSED(theap); + MI_UNUSED(pq); + mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun; + mi_theap_area_ex_t xarea; + xarea.page = page; + _mi_heap_area_init(&xarea.area, page); + return fun(theap, &xarea, arg); +} + +// Visit all theap pages as areas +static bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) { + if (visitor == NULL) return false; + return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{ +} + +// Just to pass arguments +typedef struct mi_visit_blocks_args_s { + bool visit_blocks; + mi_block_visit_fun* visitor; + void* arg; +} mi_visit_blocks_args_t; + +static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) { + mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; + if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; + if (args->visit_blocks) { + return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); + } + else { + return true; + } +} + +// Visit all blocks in a theap +bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; + return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args); +} + diff --git a/csrc/mimalloc/src/threadlocal.c b/csrc/mimalloc/src/threadlocal.c new file mode 100644 index 000000000..355498cee --- /dev/null +++ b/csrc/mimalloc/src/threadlocal.c @@ -0,0 +1,316 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + Implement dynamic thread local variables (used by heap's for their theap's). + Unlike most OS native implementations there is no limit on the number + that can be allocated. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" + +/* ----------------------------------------------------------- + Each thread can have (a dynamically expanding) array of + thread-local values. Each slot has a value and a version. + The version is used to safely reuse slots. +----------------------------------------------------------- */ +typedef struct mi_tls_slot_s { + size_t version; + void* value; +} mi_tls_slot_t; + +typedef struct mi_thread_locals_s { + size_t count; + mi_memid_t memid; + mi_tls_slot_t slots[1]; +} mi_thread_locals_t; + +static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero; + + +/* ----------------------------------------------------------- + We have 2 thread local variable which we implement with either + a C thread local declaration or using pthread keys. + - mi_thread_locals: points to an array of thread locals for most keys + - mi_slot_fast: a single dedicated thread local for slightly faster access. (used for the main heap's theap) +----------------------------------------------------------- */ + +#if MI_TLS_MODEL_PTHREADS || defined(__APPLE__) // macOS has fast pthreads +// Use pthreads +#define mi_define_thread_local(tp,name,initval) \ + static pthread_key_t __##name##_key = MI_PTHREAD_KEY_INVALID; \ + static inline tp name##_peek(void) { return (tp)mi_pthread_key_get(__##name##_key); } \ + static inline tp name##_get(void) { tp result = name##_peek(); return (result!=NULL ? result : initval); } \ + static inline bool name##_set(tp val) { return mi_pthread_key_set(&__##name##_key,val); } \ + static inline void name##_delete(void) { mi_pthread_key_delete(&__##name##_key); } + +#else +// Direct thread locals +#define mi_define_thread_local(tp,name,initval) \ + static mi_decl_thread tp __##name = initval; \ + static inline tp name##_peek(void) { return __##name; } \ + static inline tp name##_get(void) { tp result = __##name; return (result!=NULL ? result : initval); } \ + static inline bool name##_set(tp val) { __##name = val; return true; } \ + static inline void name##_delete(void) { } +#endif + +mi_define_thread_local(mi_thread_locals_t*, mi_thread_locals, &mi_thread_locals_empty) +mi_define_thread_local(void*, mi_slot_fast, NULL) + + +/* ----------------------------------------------------------- + Each key consists of the slot index in the lower bits, + and its version it the top bits. When we get a value + the version must match or we return NULL. When we set + a value, we also set the version of the key. +----------------------------------------------------------- */ + +#if MI_SIZE_BITS >= 64 +#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) /* 16 bits for the index, 48 bits for the version */ +#elif MI_SIZE_BITS >= 32 +#define MI_TLS_IDX_BITS (12) /* 12 bits for index, 20 for the version? */ +#else +#error not enough bits for the version for thread locals +#endif +#define MI_TLS_IDX_MASK ((MI_ZU(1)<> MI_TLS_IDX_BITS); +} + +static mi_thread_local_t mi_key_create( size_t index, size_t version ) { + mi_assert_internal(version != 0 && version <= MI_TLS_VERSION_MAX); + mi_assert_internal(index <= MI_TLS_IDX_MAX); + const mi_thread_local_t key = ((version << MI_TLS_IDX_BITS) | index); + mi_assert_internal(key != 0); + return key; +} + + +// dynamically reallocate the thread local slots when needed +static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) { + mi_thread_locals_t* tls_old = mi_thread_locals_get(); + const size_t count_old = tls_old->count; + size_t count; + if (count_old==0) { + tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty + count = 16; // start with 16 slots + } + else if (count_old >= 1024) { + count = count_old + 1024; // at some point increase linearly + } + else { + count = 2*count_old; // and double initially + } + if (count <= least_idx) { + count = least_idx + 1; + } + if (count > MI_TLS_IDX_MAX) { return NULL; } // too large + // allocate as meta (for secure mode) + // we could also allocate on the main heap; this is recursion safe as that uses the fast local key + mi_memid_t memid = (tls_old==NULL ? _mi_memid_none() : tls_old->memid); + mi_thread_locals_t* tls = (mi_thread_locals_t*)_mi_meta_rezalloc(_mi_subproc(), tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t), &memid); + if mi_unlikely(tls==NULL) return NULL; + tls->memid = memid; + tls->count = count; + mi_thread_locals_set(tls); + return tls; +} + +static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) { + if (val==NULL) return true; + const size_t idx = mi_key_index(key); + mi_thread_locals_t* tls = mi_thread_locals_expand(idx); + if (tls==NULL) { + _mi_error_message(EFAULT,"unable to allocate thread local variables\n"); + return false; + } + mi_assert_internal(tls == mi_thread_locals_get()); + mi_assert_internal(idx < tls->count); + tls->slots[idx].value = val; + tls->slots[idx].version = mi_key_version(key); + return true; +} + +// set a tls slot; returns `true` if successful. +// Can return `false` if we could not reallocate the slots array. +static mi_decl_noinline bool mi_thread_local_set_regular( mi_thread_local_t key, void* val ) { + mi_thread_locals_t* tls = mi_thread_locals_get(); + mi_assert_internal(tls!=NULL); + mi_assert_internal(key!=0); + const size_t idx = mi_key_index(key); + if mi_likely(idx < tls->count) { + tls->slots[idx].value = val; + tls->slots[idx].version = mi_key_version(key); + return true; + } + else { + return mi_thread_local_set_expand( key, val ); // tailcall + } +} + +bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { + mi_assert_internal(key!=0); + if (key == mi_thread_local_key_fast) { + return mi_slot_fast_set(val); + } + else { + return mi_thread_local_set_regular(key,val); + } +} + +// get a tls slot value +static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { + mi_assert_internal(key!=0); + const mi_thread_locals_t* const tls = mi_thread_locals_peek(); + if mi_unlikely(tls==NULL) { + // this can happen if a thread local is accessed after the thread local has been freed + // from mi_thread_done or mi_process_done. + // todo: can we remove this check? now we can still call this from process done when stats are printed (which calls mi_heap_theap_peek) + return NULL; + } + const size_t idx = mi_key_index(key); + if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) { + return tls->slots[idx].value; + } + else { + return NULL; + } +} + +// get a thread local value +void* _mi_thread_local_get( mi_thread_local_t key ) { + mi_assert_internal(key!=0); + if mi_likely(key == mi_thread_local_key_fast) { + return mi_slot_fast_get(); + } + else { + return mi_thread_local_get_regular(key); + } +} + +void _mi_thread_locals_thread_done(void) { + mi_thread_locals_t* const tls = mi_thread_locals_peek(); + if (tls!=NULL && tls->count > 0) { + _mi_meta_free(_mi_subproc(), tls, tls->memid); + mi_thread_locals_set(NULL); + } + if (mi_slot_fast_peek() != NULL) { + mi_slot_fast_set(NULL); + } +} + +/* ----------------------------------------------------------- +Create and free fresh TLS key's +----------------------------------------------------------- */ +#include "bitmap.h" + +static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits +static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use) +static mi_memid_t mi_thread_locals_memid; // provenance of mi_thread_locals_free +static size_t mi_thread_locals_version; // version to be able to reuse slots safely + +void _mi_thread_locals_init(void) { + mi_lock_init(&mi_thread_locals_lock); +} + +void _mi_thread_locals_done(void) { + mi_lock(&mi_thread_locals_lock) { + mi_bitmap_t* const slots = mi_thread_locals_free; + if (slots!=NULL) { + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); + } + } + mi_lock_done(&mi_thread_locals_lock); + mi_thread_locals_delete(); + mi_slot_fast_delete(); +} + +// strange signature but allows us to reuse the arena code for claiming free pages +static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) { + MI_UNUSED(_slice_index); MI_UNUSED(_arena); + *keep_set = false; + return true; +} + +// When we claim a free slot, we increase the global version counter +// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it) +static mi_thread_local_t mi_thread_local_claim(void) { + size_t idx = 0; + if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) { + mi_thread_locals_version++; + if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; } /* wrap around the version */ + return mi_key_create( idx, mi_thread_locals_version); + } + else { + return 0; + } +} + +static bool mi_thread_local_create_expand(void) { + mi_bitmap_t* const slots = mi_thread_locals_free; + // 1024 bits at a time + const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots)); + const size_t newcount = 1024 + oldcount; + if (newcount > MI_TLS_IDX_MAX) { return false; } + const size_t newsize = mi_bitmap_size( newcount, NULL ); + // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE); + mi_memid_t memid; + mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess + mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE)); + if (newslots==NULL) { return false; } + if (slots!=NULL) { + // copy over the previous bitmap + const size_t oldsize = mi_bitmap_size(oldcount,NULL); + _mi_memcpy_aligned(newslots, slots, oldsize); + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); + } + mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */); + mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */ + mi_thread_locals_free = newslots; + mi_thread_locals_memid = memid; + return true; +} + + +// create a fresh key +mi_thread_local_t _mi_thread_local_create(void) { + mi_thread_local_t key = 0; + mi_lock(&mi_thread_locals_lock) { + key = mi_thread_local_claim(); + if (key==0) { + if (mi_thread_local_create_expand()) { + key = mi_thread_local_claim(); + } + } + } + mi_assert_internal(key!=0); + mi_assert_internal(key!=mi_thread_local_key_fast); + return key; +} + +// free a key +void _mi_thread_local_free(mi_thread_local_t key) { + if (key==0) return; + const size_t idx = mi_key_index(key); + mi_lock(&mi_thread_locals_lock) { + mi_bitmap_t* const slots = mi_thread_locals_free; + if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) { + mi_bitmap_set(slots,idx); + } + } +} + diff --git a/csrc/mold-wrapper.c b/csrc/mold-wrapper.c new file mode 100644 index 000000000..d63d4de10 --- /dev/null +++ b/csrc/mold-wrapper.c @@ -0,0 +1,146 @@ +#define _GNU_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#if __has_include() +# include +#endif + +extern char **environ; + +static char *get_mold_path() { + char *path = getenv("MOLD_PATH"); + if (path) + return path; + fprintf(stderr, "MOLD_PATH is not set\n"); + exit(1); +} + +static void debug_print(const char *fmt, ...) { + if (!getenv("MOLD_WRAPPER_DEBUG")) + return; + + va_list ap; + va_start(ap, fmt); + fprintf(stderr, "mold-wrapper.so: "); + vfprintf(stderr, fmt, ap); + fflush(stderr); + va_end(ap); +} + +static int count_args(va_list *ap) { + va_list aq; + va_copy(aq, *ap); + + int i = 0; + while (va_arg(aq, char *)) + i++; + va_end(aq); + return i; +} + +static void copy_args(char **argv, const char *arg0, va_list *ap) { + int i = 1; + char *arg; + while ((arg = va_arg(*ap, char *))) + argv[i++] = arg; + + ((const char **)argv)[0] = arg0; + ((const char **)argv)[i] = NULL; +} + +static bool is_ld(const char *path) { + const char *ptr = path + strlen(path); + while (path < ptr && ptr[-1] != '/') + ptr--; + + return !strcmp(ptr, "ld") || !strcmp(ptr, "ld.lld") || + !strcmp(ptr, "ld.gold") || !strcmp(ptr, "ld.bfd") || + !strcmp(ptr, "ld.mold"); +} + +int execvpe(const char *file, char *const *argv, char *const *envp) { + debug_print("execvpe %s\n", file); + + if (!strcmp(file, "ld") || is_ld(file)) + file = get_mold_path(); + + for (int i = 0; envp[i]; i++) + putenv(envp[i]); + + typeof(execvpe) *real = dlsym(RTLD_NEXT, "execvp"); + return real(file, argv, environ); +} + +int execve(const char *path, char *const *argv, char *const *envp) { + debug_print("execve %s\n", path); + if (is_ld(path)) + path = get_mold_path(); + typeof(execve) *real = dlsym(RTLD_NEXT, "execve"); + return real(path, argv, envp); +} + +int execl(const char *path, const char *arg0, ...) { + va_list ap; + va_start(ap, arg0); + char **argv = alloca((count_args(&ap) + 2) * sizeof(char *)); + copy_args(argv, arg0, &ap); + va_end(ap); + return execve(path, argv, environ); +} + +int execlp(const char *file, const char *arg0, ...) { + va_list ap; + va_start(ap, arg0); + char **argv = alloca((count_args(&ap) + 2) * sizeof(char *)); + copy_args(argv, arg0, &ap); + va_end(ap); + return execvpe(file, argv, environ); +} + +int execle(const char *path, const char *arg0, ...) { + va_list ap; + va_start(ap, arg0); + char **argv = alloca((count_args(&ap) + 2) * sizeof(char *)); + copy_args(argv, arg0, &ap); + char **env = va_arg(ap, char **); + va_end(ap); + return execve(path, argv, env); +} + +int execv(const char *path, char *const *argv) { + return execve(path, argv, environ); +} + +int execvp(const char *file, char *const *argv) { + return execvpe(file, argv, environ); +} + +int posix_spawn(pid_t *pid, const char *path, + const posix_spawn_file_actions_t *file_actions, + const posix_spawnattr_t *attrp, + char *const *argv, char *const *envp) { + debug_print("posix_spawn %s\n", path); + if (is_ld(path)) + path = get_mold_path(); + typeof(posix_spawn) *real = dlsym(RTLD_NEXT, "posix_spawn"); + return real(pid, path, file_actions, attrp, argv, envp); +} + +int posix_spawnp(pid_t *pid, const char *file, + const posix_spawn_file_actions_t *file_actions, + const posix_spawnattr_t *attrp, + char *const *argv, char *const *envp) { + debug_print("posix_spawnp %s\n", file); + if (is_ld(file)) + file = get_mold_path(); + typeof(posix_spawnp) *real = dlsym(RTLD_NEXT, "posix_spawnp"); + return real(pid, file, file_actions, attrp, argv, envp); +} diff --git a/mimalloc/Cargo.toml b/mimalloc/Cargo.toml new file mode 100644 index 000000000..84dc31330 --- /dev/null +++ b/mimalloc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "mold-mimalloc" +version = "2.42.0" +edition = "2021" +description = "The mimalloc allocator that mold links, built from the vendored source" +license = "MIT" +links = "mimalloc" diff --git a/mimalloc/build.rs b/mimalloc/build.rs new file mode 100644 index 000000000..90af78aff --- /dev/null +++ b/mimalloc/build.rs @@ -0,0 +1,48 @@ +//! Builds the vendored mimalloc (csrc/mimalloc, the same copy the C++ +//! mold links) into a static library, from its single-file build. + +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("../csrc/mimalloc"); + println!("cargo:rerun-if-changed={}", root.join("src").display()); + println!("cargo:rerun-if-changed={}", root.join("include").display()); + + let cc = std::env::var("CC").unwrap_or_else(|_| "cc".to_string()); + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let object = out_dir.join("mimalloc.o"); + + // MI_USE_ENVIRON=0 as in mold's build: the allocator's behavior does + // not depend on the environment. + let status = Command::new(&cc) + .args([ + "-c", + "-O3", + "-DNDEBUG", + "-DMI_USE_ENVIRON=0", + "-fPIC", + "-fvisibility=hidden", + "-I", + ]) + .arg(root.join("include")) + .arg("-o") + .arg(&object) + .arg(root.join("src/static.c")) + .status(); + if !matches!(status, Ok(s) if s.success()) { + panic!("could not compile csrc/mimalloc/src/static.c with {cc}"); + } + + let archive = out_dir.join("libmimalloc.a"); + let status = Command::new("ar") + .arg("rcs") + .arg(&archive) + .arg(&object) + .status(); + if !matches!(status, Ok(s) if s.success()) { + panic!("could not create {}", archive.display()); + } + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-lib=static=mimalloc"); +} diff --git a/mimalloc/src/lib.rs b/mimalloc/src/lib.rs new file mode 100644 index 000000000..839b283dc --- /dev/null +++ b/mimalloc/src/lib.rs @@ -0,0 +1,43 @@ +//! The mimalloc allocator as mold's global allocator, built from the same +//! vendored source as the C++ mold (see csrc/mimalloc). +//! +//! mold allocates heavily from many threads at once; mimalloc avoids the +//! heap-growth syscalls and lock contention that glibc's malloc suffers. + +use std::alloc::{GlobalAlloc, Layout}; +use std::ffi::c_void; + +extern "C" { + fn mi_malloc_aligned(size: usize, alignment: usize) -> *mut c_void; + fn mi_zalloc_aligned(size: usize, alignment: usize) -> *mut c_void; + fn mi_realloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void; + fn mi_free(p: *mut c_void); +} + +/// The allocator; declare it as the `#[global_allocator]`. +pub struct MiMalloc; + +// SAFETY: mimalloc's functions are thread-safe, and every allocation is +// requested and released with its own size and alignment as GlobalAlloc +// demands. +unsafe impl GlobalAlloc for MiMalloc { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + mi_malloc_aligned(layout.size(), layout.align()) as *mut u8 + } + + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + mi_zalloc_aligned(layout.size(), layout.align()) as *mut u8 + } + + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + mi_realloc_aligned(ptr as *mut c_void, new_size, layout.align()) as *mut u8 + } + + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + mi_free(ptr as *mut c_void); + } +} diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 000000000..3f96c5f1f --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Compatibility entry point for the Rust test runner. The test cases remain +# the unchanged upstream shell scripts under test/. + +set -e +cd "$(dirname "$0")" +exec cargo run --quiet --release -p mold-test-runner -- "$@" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..292fe499e --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/src/arch/arm32.rs b/src/arch/arm32.rs new file mode 100644 index 000000000..e3f0a606f --- /dev/null +++ b/src/arch/arm32.rs @@ -0,0 +1,815 @@ +//! ARM32, in either byte order. +//! +//! ARM processors support two instruction encodings: ARM, with 4-byte +//! instructions, and Thumb, with 2- or 4-byte ones, which was added later +//! for code density. The processor runs in one mode or the other and +//! switches with BX-family instructions, so calling a Thumb function from +//! ARM code needs such an instruction, sometimes in a linker-synthesized +//! interworking thunk. +//! +//! Since instructions are at least 2-byte aligned, the least significant +//! bit of a function address is free, and it is used to mark Thumb +//! functions: a function symbol with value 0x2001 is a Thumb function at +//! 0x2000, and so is a function pointer with that value. +//! +//! Relocations are of the REL type, and addends are packed into the +//! instruction fields they relocate. +//! +//! Big-endian ARM is rare but comes in two flavors: BE32, with big-endian +//! instructions and data, and BE8, the de facto standard since ARMv6, +//! with little-endian instructions and big-endian data. Only BE8 is +//! supported. Compilers nevertheless emit big-endian instructions into +//! object files compiled for big-endian mode, so relocations are applied +//! in the object files' byte order, and the code in input sections is +//! converted to little-endian at the very end, after it's copied to the +//! output; see [`swap_code_bytes`]. Linker-synthesized code is written +//! in little-endian form to begin with. +//! +//! https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst + +use std::marker::PhantomData; + +use crate::arch::{Arch, Family, ThunkLayout}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{ + check_tlsle, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection, SectionRef, +}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::thunks::Thunk; +use crate::util::{align_to, bit, bits, is_int, sign_extend}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Arm32Target(PhantomData); + +pub type Arm32 = Arm32Target; +pub type Arm32Be = Arm32Target; + +impl Layout for Arm32Target { + type Endian = End; + const IS_64: bool = false; + const IS_RELA: bool = false; +} + +fn b(val: u64, hi: u32, lo: u32) -> u32 { + bits(val, hi, lo) as u32 +} + +fn bt(val: u64, n: u32) -> u32 { + bit(val, n) as u32 +} + +/// The second halfword of a 32-bit Thumb instruction. +fn thm2(loc: &[u8]) -> u16 { + End::read_u16(&loc[2..]) +} + +fn write_arm_mov(loc: &mut [u8], val: u32) { + let imm12 = b(val as u64, 11, 0); + let imm4 = b(val as u64, 15, 12); + End::write_u32( + loc, + (End::read_u32(loc) & 0xfff0_f000) | (imm4 << 16) | imm12, + ); +} + +fn write_thm_b21(loc: &mut [u8], val: u32) { + let v = val as u64; + let (s, j2, j1) = (bt(v, 20), bt(v, 19), bt(v, 18)); + let imm6 = b(v, 17, 12); + let imm11 = b(v, 11, 1); + End::write_u16( + loc, + ((End::read_u16(loc) & 0b1111_1011_1100_0000) as u32 | (s << 10) | imm6) as u16, + ); + let second = + ((thm2::(loc) & 0b1101_0000_0000_0000) as u32 | (j1 << 13) | (j2 << 11) | imm11) + as u16; + End::write_u16(&mut loc[2..], second); +} + +fn write_thm_b25(loc: &mut [u8], val: u32) { + let v = val as u64; + let (s, i1, i2) = (bt(v, 24), bt(v, 23), bt(v, 22)); + let j1 = (i1 ^ 1) ^ s; + let j2 = (i2 ^ 1) ^ s; + let imm10 = b(v, 21, 12); + let imm11 = b(v, 11, 1); + End::write_u16( + loc, + ((End::read_u16(loc) & 0b1111_1000_0000_0000) as u32 | (s << 10) | imm10) as u16, + ); + let second = + ((thm2::(loc) & 0b1101_0000_0000_0000) as u32 | (j1 << 13) | (j2 << 11) | imm11) + as u16; + End::write_u16(&mut loc[2..], second); +} + +fn write_thm_mov(loc: &mut [u8], val: u32) { + let v = val as u64; + let imm4 = b(v, 15, 12); + let i = bt(v, 11); + let imm3 = b(v, 10, 8); + let imm8 = b(v, 7, 0); + End::write_u16( + loc, + ((End::read_u16(loc) & 0b1111_1011_1111_0000) as u32 | (i << 10) | imm4) as u16, + ); + let second = ((thm2::(loc) & 0b1000_1111_0000_0000) as u32 | (imm3 << 12) | imm8) as u16; + End::write_u16(&mut loc[2..], second); +} + +/// Sets the second halfword's bit that turns a Thumb BLX into a BL, or +/// clears it for the reverse. +fn set_thm_bl(loc: &mut [u8], is_bl: bool) { + let second = if is_bl { + thm2::(loc) | 0x1000 + } else { + thm2::(loc) & !0x1000 + }; + End::write_u16(&mut loc[2..], second); +} + +/// Whether a branch to the symbol lands in Thumb code. Only function +/// symbols carry the mode bit; a plain label doesn't switch modes. +fn is_thumb_func(ctx: &Context, sym: &Symbol) -> bool { + matches!(sym.ty(), STT_FUNC | STT_GNU_IFUNC) && sym.addr(ctx) & 1 != 0 +} + +/// Whether a branch to the symbol lands in ARM code. A PLT entry always +/// does. +fn is_arm_func(ctx: &Context, sym: &Symbol) -> bool { + sym.has_plt(&ctx.symbols) + || (matches!(sym.ty(), STT_FUNC | STT_GNU_IFUNC) && sym.addr(ctx) & 1 == 0) +} + +const ARM_NOP: u32 = 0xe320_f000; +const THM_NOP_W: u32 = 0x8000_f3af; + +const PLT_ENTRY: [u32; 4] = [ + 0xe59f_c004, // 1: ldr ip, 2f + 0xe08c_c00f, // add ip, ip, pc + 0xe59c_f000, // ldr pc, [ip] + 0x0000_0000, // 2: .word sym@GOT - 1b +]; + +/// Linker-synthesized code is little-endian whatever the data order. +fn write_code(buf: &mut [u8], words: &[u32]) { + for (i, &w) in words.iter().enumerate() { + LittleEndian::write_u32(&mut buf[i * 4..], w); + } +} + +/// What a mapping symbol marks the following bytes as: the width of +/// their instructions, or `None` for data. +fn mapping_symbol_kind(name: &[u8]) -> Option> { + let kind = |c: u8| name == [b'$', c] || (name.starts_with(&[b'$', c, b'.'])); + if kind(b'a') { + Some(Some(4)) + } else if kind(b't') { + Some(Some(2)) + } else if kind(b'd') { + Some(None) + } else { + None + } +} + +/// Converts the code in the output's input sections from the big-endian +/// form the compiler emitted to the little-endian form BE8 executes. +/// +/// A text section may mix 4-byte ARM instructions, 2-byte Thumb +/// instructions and data, which need 4-byte swaps, 2-byte swaps and no +/// change respectively. The mapping symbols `$a`, `$t` and `$d` mark +/// where each kind begins. +pub fn swap_code_bytes(ctx: &Context>, buf: &mut [u8]) { + for file in &ctx.objs { + let mut marks: Vec<(SectionRef, u64, Option)> = file + .base + .local_symbols() + .iter() + .map(|&id| &ctx.symbols[id]) + .filter_map(|sym| { + let kind = mapping_symbol_kind(sym.name())?; + let sec = sym.input_section()?; + let isec = ctx.section(sec); + (isec.is_alive() && isec.sh_flags & SHF_EXECINSTR as u64 != 0) + .then_some((sec, sym.value, kind)) + }) + .collect(); + marks.sort_by_key(|&(sec, offset, _)| (sec.shndx, offset)); + + for (i, &(sec, start, kind)) in marks.iter().enumerate() { + let Some(width) = kind else { continue }; + let isec = ctx.section(sec); + let end = match marks.get(i + 1) { + Some(&(next, offset, _)) if next == sec => offset, + _ => isec.sh_size, + }; + let base = ctx + .output_section(isec.output_section.expect("output section")) + .hdr + .shdr + .sh_offset + + isec.offset(); + for insn in buf[(base + start) as usize..(base + end) as usize].chunks_exact_mut(width) + { + insn.reverse(); + } + } + } +} + +impl Arch for Arm32Target { + const NAME: &'static str = if End::IS_LITTLE { "arm32" } else { "arm32be" }; + const FAMILY: Family = Family::Arm32; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_ARM; + const PLT_HDR_SIZE: u64 = 32; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 16; + const THUNK: Option = Some(ThunkLayout { + header_size: 16, + entry_size: 16, + }); + const TRAP: &'static [u8] = &[0xff, 0xde]; // udf + + const R_COPY: u32 = R_ARM_COPY; + const R_GLOB_DAT: u32 = R_ARM_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_ARM_JUMP_SLOT; + const R_ABS: u32 = R_ARM_ABS32; + const R_RELATIVE: u32 = R_ARM_RELATIVE; + const R_IRELATIVE: Option = Some(R_ARM_IRELATIVE); + const R_DTPOFF: u32 = R_ARM_TLS_DTPOFF32; + const R_TPOFF: u32 = R_ARM_TLS_TPOFF32; + const R_DTPMOD: u32 = R_ARM_TLS_DTPMOD32; + const R_TLSDESC: Option = Some(R_ARM_TLS_DESC); + const R_FUNCALL: &'static [u32] = &[ + R_ARM_JUMP24, + R_ARM_THM_JUMP24, + R_ARM_CALL, + R_ARM_THM_CALL, + R_ARM_PLT32, + ]; + + fn rel_to_string(r_type: u32) -> String { + arm32_rel_to_string(r_type) + } + + fn eflags(_ctx: &Context) -> u32 { + if End::IS_LITTLE { + EF_ARM_EABI_VER5 + } else { + EF_ARM_EABI_VER5 | EF_ARM_BE8 + } + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u32; 8] = [ + 0xe52d_e004, // push {lr} + 0xe59f_e004, // ldr lr, 2f + 0xe08f_e00e, // 1: add lr, pc, lr + 0xe5be_f008, // ldr pc, [lr, #8]! + 0x0000_0000, // 2: .word .got.plt - 1b - 8 + 0x0000_0000, // (padding) + 0x0000_0000, // (padding) + 0x0000_0000, // (padding) + ]; + write_code(buf, &INSN); + let gotplt = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr) + .wrapping_sub(16); + LittleEndian::write_u32(&mut buf[16..], gotplt as u32); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_code(buf, &PLT_ENTRY); + LittleEndian::write_u32( + &mut buf[12..], + sym.gotplt_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(12) as u32, + ); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_code(buf, &PLT_ENTRY); + LittleEndian::write_u32( + &mut buf[12..], + sym.got_pltgot_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(12) as u32, + ); + } + + fn apply_eh_reloc( + ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_ARM_ABS32 => End::write_u32(loc, val as u32), + R_ARM_REL32 => End::write_u32(loc, val.wrapping_sub(p) as u32), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_ARM_MOVW_ABS_NC | R_ARM_THM_MOVW_ABS_NC => scan_absrel(ctx, isec, sym, &rel), + R_ARM_THM_CALL | R_ARM_CALL | R_ARM_JUMP24 | R_ARM_PLT32 | R_ARM_THM_JUMP24 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_ARM_GOT_PREL | R_ARM_GOT_BREL | R_ARM_TARGET2 => sym.add_flags(NEEDS_GOT), + R_ARM_MOVT_PREL | R_ARM_THM_MOVT_PREL | R_ARM_PREL31 => { + scan_pcrel(ctx, isec, sym, &rel) + } + R_ARM_TLS_GD32 => sym.add_flags(NEEDS_TLSGD), + R_ARM_TLS_LDM32 => ctx + .needs_tlsld + .store(true, std::sync::atomic::Ordering::Relaxed), + R_ARM_TLS_IE32 => sym.add_flags(NEEDS_GOTTP), + R_ARM_TLS_CALL | R_ARM_THM_TLS_CALL => scan_tlsdesc(ctx, sym), + R_ARM_TLS_LE32 => check_tlsle(ctx, isec, sym, &rel), + R_ARM_ABS32 + | R_ARM_TARGET1 + | R_ARM_MOVT_ABS + | R_ARM_THM_MOVT_ABS + | R_ARM_REL32 + | R_ARM_BASE_PREL + | R_ARM_GOTOFF32 + | R_ARM_THM_JUMP8 + | R_ARM_THM_JUMP11 + | R_ARM_THM_JUMP19 + | R_ARM_MOVW_PREL_NC + | R_ARM_THM_MOVW_PREL_NC + | R_ARM_TLS_LDO32 + | R_ARM_V4BX + | R_ARM_TLS_GOTDESC => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let osec = &ctx.output_sections[isec.output_section.expect("output section").index()]; + + for (i, rel) in isec.rels::(file).iter().enumerate() { + if rel.r_type == R_NONE || rel.r_type == R_ARM_V4BX { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = isec.rel_addend::(&rel) as u64; + let p = isec.addr(ctx) + rel.r_offset; + let t = is_thumb_func(ctx, sym) as u64; + let got = ctx.got.hdr.shdr.sh_addr; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + // A thunk has two entry points: +0 for Thumb, +4 for ARM. + let thumb_thunk = || sym.thunk_addr(ctx, p); + let arm_thunk = || sym.thunk_addr(ctx, p) + 4; + // The TLSDESC trampoline is the header of the next thunk. + let tlsdesc_trampoline = || { + let i = osec.thunks.partition_point(|thunk| thunk.addr(osec) <= p); + osec.thunks.get(i).map_or_else( + || { + fatal!( + ctx, + "{}: no TLSDESC trampoline after the call", + isec.display(file) + ) + }, + |thunk| thunk.addr(osec), + ) + }; + let loc = &mut buf[off..]; + let write32 = |loc: &mut [u8], v: u32| End::write_u32(loc, v); + let write16 = |loc: &mut [u8], v: u16| End::write_u16(loc, v); + + match rel.r_type { + // Handled as absolute relocations by the output section. + R_ARM_ABS32 | R_ARM_TARGET1 => {} + R_ARM_REL32 => write32(loc, pcrel as u32), + R_ARM_THM_CALL => { + if sym.is_remaining_undef_weak() { + // A call to an undefined weak symbol falls through. + write32(loc, THM_NOP_W); + continue; + } + // THM_CALL refers to a BL or a BLX, which differ in one + // bit; BLX is for ARM targets. + let val1 = pcrel as i64; + let val2 = align_to(pcrel, 4) as i64; + let arm = is_arm_func(ctx, sym); + if !arm && is_int(val1, 25) { + set_thm_bl::(loc, true); + write_thm_b25::(loc, val1 as u32); + } else if arm && is_int(val2, 25) { + set_thm_bl::(loc, false); + write_thm_b25::(loc, val2 as u32); + } else { + set_thm_bl::(loc, true); + write_thm_b25::( + loc, + thumb_thunk().wrapping_add(a).wrapping_sub(p) as u32, + ); + } + } + R_ARM_BASE_PREL => write32(loc, got.wrapping_add(a).wrapping_sub(p) as u32), + R_ARM_GOTOFF32 => write32(loc, (sa | t).wrapping_sub(got) as u32), + R_ARM_GOT_PREL | R_ARM_TARGET2 => write32( + loc, + got.wrapping_add(g()).wrapping_add(a).wrapping_sub(p) as u32, + ), + R_ARM_GOT_BREL => write32(loc, g().wrapping_add(a) as u32), + R_ARM_CALL => { + if sym.is_remaining_undef_weak() { + write32(loc, ARM_NOP); + continue; + } + // Like THM_CALL, ARM_CALL refers to a BL or a BLX, which + // may need to be swapped for each other. + let insn = End::read_u32(loc); + let is_bl = insn & 0xff00_0000 == 0xeb00_0000; + let is_blx = insn & 0xfe00_0000 == 0xfa00_0000; + if !is_bl && !is_blx { + fatal!( + ctx, + "{}: R_ARM_CALL refers to neither BL nor BLX", + isec.display(file) + ); + } + if is_int(pcrel as i64, 26) { + if t != 0 { + write32(loc, 0xfa00_0000 | (bt(pcrel, 1) << 24) | b(pcrel, 25, 2)); + // BLX + } else { + write32(loc, 0xeb00_0000 | b(pcrel, 25, 2)); // BL + } + } else { + write32( + loc, + 0xeb00_0000 | b(arm_thunk().wrapping_add(a).wrapping_sub(p), 25, 2), + ); // BL + } + } + R_ARM_JUMP24 => { + if sym.is_remaining_undef_weak() { + write32(loc, ARM_NOP); + continue; + } + // B can't become BX in place, since BX takes a register + // rather than an immediate, so a mode switch goes + // through a thunk. + let mut val = pcrel; + if t != 0 || !is_int(val as i64, 26) { + val = arm_thunk().wrapping_add(a).wrapping_sub(p); + } + write32(loc, (End::read_u32(loc) & 0xff00_0000) | b(val, 25, 2)); + } + R_ARM_PLT32 => { + if sym.is_remaining_undef_weak() { + write32(loc, ARM_NOP); + } else { + let val = if t != 0 { arm_thunk() } else { s } + .wrapping_add(a) + .wrapping_sub(p); + write32(loc, (End::read_u32(loc) & 0xff00_0000) | b(val, 25, 2)); + } + } + R_ARM_THM_JUMP8 => { + check(pcrel as i64, -(1 << 8), 1 << 8); + write16(loc, (End::read_u16(loc) & 0xff00) | b(pcrel, 8, 1) as u16); + } + R_ARM_THM_JUMP11 => { + check(pcrel as i64, -(1 << 11), 1 << 11); + write16(loc, (End::read_u16(loc) & 0xf800) | b(pcrel, 11, 1) as u16); + } + R_ARM_THM_JUMP19 => { + check(pcrel as i64, -(1 << 20), 1 << 20); + write_thm_b21::(loc, pcrel as u32); + } + R_ARM_THM_JUMP24 => { + if sym.is_remaining_undef_weak() { + write32(loc, THM_NOP_W); + continue; + } + let mut val = pcrel; + if is_arm_func(ctx, sym) || !is_int(val as i64, 25) { + val = thumb_thunk().wrapping_add(a).wrapping_sub(p); + } + write_thm_b25::(loc, val as u32); + } + R_ARM_MOVW_PREL_NC => write_arm_mov::(loc, (sa | t).wrapping_sub(p) as u32), + R_ARM_MOVW_ABS_NC => write_arm_mov::(loc, (sa | t) as u32), + R_ARM_THM_MOVW_PREL_NC => { + write_thm_mov::(loc, (sa | t).wrapping_sub(p) as u32) + } + R_ARM_PREL31 => { + check(pcrel as i64, -(1 << 30), 1 << 30); + write32( + loc, + (End::read_u32(loc) & 0x8000_0000) | (pcrel as u32 & 0x7fff_ffff), + ); + } + R_ARM_THM_MOVW_ABS_NC => write_thm_mov::(loc, (sa | t) as u32), + R_ARM_MOVT_PREL => write_arm_mov::(loc, (pcrel >> 16) as u32), + R_ARM_THM_MOVT_PREL => write_thm_mov::(loc, (pcrel >> 16) as u32), + R_ARM_MOVT_ABS => write_arm_mov::(loc, (sa >> 16) as u32), + R_ARM_THM_MOVT_ABS => write_thm_mov::(loc, (sa >> 16) as u32), + R_ARM_TLS_GD32 => write32( + loc, + sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(p) as u32, + ), + R_ARM_TLS_LDM32 => write32( + loc, + ctx.got.tlsld_addr::().wrapping_add(a).wrapping_sub(p) as u32, + ), + R_ARM_TLS_LDO32 => write32(loc, sa.wrapping_sub(ctx.dtp_addr) as u32), + R_ARM_TLS_IE32 => write32( + loc, + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p) as u32, + ), + R_ARM_TLS_LE32 => write32(loc, sa.wrapping_sub(ctx.tp_addr) as u32), + // TLSDESC materializes a TP-relative address in r0: + // + // ldr r0, .L2 + // .L1: bl foo # R_ARM_TLS_CALL + // .L2: .word foo + . - .L1 # R_ARM_TLS_GOTDESC + // + // If the address is known at link time, the call becomes a + // nop and the word `foo(tpoff)`; if at load time, the call + // becomes `ldr r0, [pc, r0]` and the word + // `foo(gottpoff) + . - .L1`. + R_ARM_TLS_GOTDESC => { + // The addend is odd if the corresponding TLS_CALL is + // Thumb. + if sym.has_tlsdesc(&ctx.symbols) { + let adjust = if a & 1 != 0 { 6 } else { 4 }; + write32( + loc, + sym.tlsdesc_addr(ctx) + .wrapping_sub(p) + .wrapping_add(a) + .wrapping_sub(adjust) as u32, + ); + } else if sym.has_gottp(&ctx.symbols) { + let adjust = if a & 1 != 0 { 5 } else { 8 }; + write32( + loc, + sym.gottp_addr(ctx) + .wrapping_sub(p) + .wrapping_add(a) + .wrapping_sub(adjust) as u32, + ); + } else { + write32(loc, s.wrapping_sub(ctx.tp_addr) as u32); + } + } + R_ARM_TLS_CALL => { + if sym.has_tlsdesc(&ctx.symbols) { + write32( + loc, + 0xeb00_0000 + | b(tlsdesc_trampoline().wrapping_sub(p).wrapping_sub(8), 25, 2), + ); // bl 0 + } else if sym.has_gottp(&ctx.symbols) { + write32(loc, 0xe79f_0000); // ldr r0, [pc, r0] + } else { + write32(loc, ARM_NOP); + } + } + R_ARM_THM_TLS_CALL => { + if sym.has_tlsdesc(&ctx.symbols) { + let val = align_to(tlsdesc_trampoline().wrapping_sub(p).wrapping_sub(4), 4); + write_thm_b25::(loc, val as u32); + set_thm_bl::(loc, false); // BL -> BLX + } else if sym.has_gottp(&ctx.symbols) { + // `ldr r0, [pc, r0]` isn't encodable in Thumb; two + // instructions do the job. + write16(loc, 0x4478); // add r0, pc + write16(&mut loc[2..], 0x6800); // ldr r0, [r0] + } else { + write32(loc, THM_NOP_W); + } + } + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), isec.rel_addend::(&rel) as u64), + }; + let tombstone = isec.tombstone(ctx, sym, frag.map(|(f, _)| f)); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_ARM_ABS32 => End::write_u32(loc, tombstone.unwrap_or(s.wrapping_add(a)) as u32), + R_ARM_TLS_LDO32 => End::write_u32( + loc, + tombstone.unwrap_or(s.wrapping_add(a).wrapping_sub(ctx.dtp_addr)) as u32, + ), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + /// Thumb and ARM B instructions can't switch modes, so a branch to a + /// function of the other kind always goes through a thunk. + fn always_needs_thunk(ctx: &Context, sym: &Symbol, rel: &ElfRel) -> bool { + match rel.r_type { + R_ARM_JUMP24 | R_ARM_PLT32 => is_thumb_func(ctx, sym), + R_ARM_THM_JUMP24 => is_arm_func(ctx, sym), + _ => false, + } + } + + fn write_thunk(ctx: &Context, thunk: &Thunk, addr: u64, buf: &mut [u8]) { + // ARM32's TLSDESC is designed so that this common trampoline is + // factored out of object files to save space. Nobody provides it, + // so the linker synthesizes one per thunk. + const HDR: [u32; 4] = [ + 0xe08e_0000, // add r0, lr, r0 + 0xe590_1004, // ldr r1, [r0, #4] + 0xe12f_ff11, // bx r1 + 0xe320_f000, // nop + ]; + // A range extension and mode switch thunk, with two entry points: + // +0 for Thumb and +4 for ARM. + const ENTRY: [u8; 16] = [ + // .thumb + 0x78, 0x47, // bx pc # jumps to 1f + 0xc0, 0x46, // nop + // .arm + 0x00, 0xc0, 0x9f, 0xe5, // 1: ldr ip, 3f + 0x0f, 0xf0, 0x8c, 0xe0, // 2: add pc, ip, pc + 0x00, 0x00, 0x00, 0x00, // 3: .word sym - 2b + ]; + write_code(buf, &HDR); + for (i, &sym) in thunk.symbols.iter().enumerate() { + let s = ctx.symbols[sym].addr(ctx); + let p = addr + thunk.offsets[i]; + let entry = &mut buf[thunk.offsets[i] as usize..]; + entry[..16].copy_from_slice(&ENTRY); + LittleEndian::write_u32(&mut entry[12..], s.wrapping_sub(p).wrapping_sub(16) as u32); + } + } + + fn finish_output(ctx: &Context, buf: &mut [u8]) { + if !End::IS_LITTLE { + swap_code_bytes(ctx, buf); + } + } + + fn write_addend(loc: &mut [u8], val: i64, rel: &ElfRel) { + let v = val as u64; + match rel.r_type { + R_ARM_NONE => {} + R_ARM_ABS32 | R_ARM_REL32 | R_ARM_BASE_PREL | R_ARM_GOTOFF32 | R_ARM_GOT_PREL + | R_ARM_GOT_BREL | R_ARM_TLS_GD32 | R_ARM_TLS_LDM32 | R_ARM_TLS_LDO32 + | R_ARM_TLS_IE32 | R_ARM_TLS_LE32 | R_ARM_TLS_GOTDESC | R_ARM_TARGET1 + | R_ARM_TARGET2 => End::write_u32(loc, val as u32), + R_ARM_THM_JUMP8 => { + End::write_u16(loc, (End::read_u16(loc) & 0xff00) | b(v, 8, 1) as u16) + } + R_ARM_THM_JUMP11 => { + End::write_u16(loc, (End::read_u16(loc) & 0xf800) | b(v, 11, 1) as u16) + } + R_ARM_THM_CALL | R_ARM_THM_JUMP24 | R_ARM_THM_TLS_CALL => { + write_thm_b25::(loc, val as u32) + } + R_ARM_CALL | R_ARM_JUMP24 | R_ARM_PLT32 => { + End::write_u32(loc, (End::read_u32(loc) & 0xff00_0000) | b(v, 25, 2)) + } + R_ARM_MOVW_PREL_NC | R_ARM_MOVW_ABS_NC | R_ARM_MOVT_PREL | R_ARM_MOVT_ABS => { + write_arm_mov::(loc, val as u32) + } + R_ARM_PREL31 => End::write_u32( + loc, + (End::read_u32(loc) & 0x8000_0000) | (val as u32 & 0x7fff_ffff), + ), + R_ARM_THM_MOVW_PREL_NC + | R_ARM_THM_MOVW_ABS_NC + | R_ARM_THM_MOVT_PREL + | R_ARM_THM_MOVT_ABS => write_thm_mov::(loc, val as u32), + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + + fn get_addend(loc: &[u8], rel: &ElfRel) -> i64 { + let arm = || End::read_u32(loc) as u64; + let thm = |i: usize| End::read_u16(&loc[i * 2..]) as u64; + match rel.r_type { + R_ARM_ABS32 | R_ARM_REL32 | R_ARM_BASE_PREL | R_ARM_GOTOFF32 | R_ARM_GOT_PREL + | R_ARM_GOT_BREL | R_ARM_TLS_GD32 | R_ARM_TLS_LDM32 | R_ARM_TLS_LDO32 + | R_ARM_TLS_IE32 | R_ARM_TLS_LE32 | R_ARM_TLS_GOTDESC | R_ARM_TARGET1 + | R_ARM_TARGET2 => End::read_u32(loc) as i32 as i64, + R_ARM_THM_JUMP8 => sign_extend(thm(0), 8) << 1, + R_ARM_THM_JUMP11 => sign_extend(thm(0), 11) << 1, + R_ARM_THM_JUMP19 => { + // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/B--Branch- + let s = bit(thm(0), 10); + let j2 = bit(thm(1), 11); + let j1 = bit(thm(1), 13); + let imm6 = bits(thm(0), 5, 0); + let imm11 = bits(thm(1), 10, 0); + sign_extend( + (s << 20) | (j2 << 19) | (j1 << 18) | (imm6 << 12) | (imm11 << 1), + 21, + ) + } + R_ARM_THM_CALL | R_ARM_THM_JUMP24 | R_ARM_THM_TLS_CALL => { + // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/BL--BLX--immediate---Branch-with-Link-and-optional-Exchange--immediate-- + let s = bit(thm(0), 10); + let j1 = bit(thm(1), 13); + let j2 = bit(thm(1), 11); + let i1 = (j1 ^ s) ^ 1; + let i2 = (j2 ^ s) ^ 1; + let imm10 = bits(thm(0), 9, 0); + let imm11 = bits(thm(1), 10, 0); + sign_extend( + (s << 24) | (i1 << 23) | (i2 << 22) | (imm10 << 12) | (imm11 << 1), + 25, + ) + } + R_ARM_CALL | R_ARM_JUMP24 | R_ARM_PLT32 | R_ARM_TLS_CALL => sign_extend(arm(), 24) << 2, + R_ARM_MOVW_PREL_NC | R_ARM_MOVW_ABS_NC | R_ARM_MOVT_PREL | R_ARM_MOVT_ABS => { + // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/MOV--MOVS--immediate---Move--immediate-- + let imm4 = bits(arm(), 19, 16); + let imm12 = bits(arm(), 11, 0); + sign_extend((imm4 << 12) | imm12, 16) + } + R_ARM_PREL31 => sign_extend(arm(), 31), + R_ARM_THM_MOVW_PREL_NC + | R_ARM_THM_MOVW_ABS_NC + | R_ARM_THM_MOVT_PREL + | R_ARM_THM_MOVT_ABS => { + // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/MOVT--Move-Top- + let imm4 = bits(thm(0), 3, 0); + let i = bit(thm(0), 10); + let imm3 = bits(thm(1), 14, 12); + let imm8 = bits(thm(1), 7, 0); + sign_extend((imm4 << 12) | (i << 11) | (imm3 << 8) | imm8, 16) + } + _ => 0, + } + } +} diff --git a/src/arch/arm64.rs b/src/arch/arm64.rs new file mode 100644 index 000000000..667c4ab82 --- /dev/null +++ b/src/arch/arm64.rs @@ -0,0 +1,775 @@ +//! ARM64. +//! +//! ARM64's ELF psABI has nothing peculiar; it is a clean RISC instruction +//! set with PC-relative loads and stores. All instructions are 4 bytes, +//! and are little-endian even on big-endian targets, where only data is +//! byte-swapped. Branches reach ±128 MiB, so binaries with a larger +//! `.text` need range extension thunks. +//! +//! Unlike most targets, -fPIC code uses the TLSDESC model to access +//! thread-local variables by default rather than the less efficient GD +//! model, which needs `-mtls-dialect=trad`. GD is rare enough that GD to +//! LE relaxation isn't implemented. +//! +//! https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + +use std::marker::PhantomData; + +use crate::arch::{Arch, Family, ThunkLayout}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::thunks::Thunk; +use crate::util::{bits, is_int}; +use crate::{error, fatal}; + +/// ARM64, in either byte order. +#[derive(Clone, Copy, Debug, Default)] +pub struct Arm64Target(PhantomData); + +pub type Arm64 = Arm64Target; +pub type Arm64Be = Arm64Target; + +impl Layout for Arm64Target { + type Endian = End; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +/// Instructions are always little-endian. +fn insn(loc: &[u8]) -> u32 { + u32::from_le_bytes([loc[0], loc[1], loc[2], loc[3]]) +} + +fn write_insn(loc: &mut [u8], v: u32) { + loc[..4].copy_from_slice(&v.to_le_bytes()); +} + +fn or_insn(loc: &mut [u8], v: u32) { + write_insn(loc, insn(loc) | v); +} + +fn write_adrp(loc: &mut [u8], val: u64) { + or_insn( + loc, + ((bits(val, 13, 12) << 29) | (bits(val, 32, 14) << 5)) as u32, + ); +} + +fn write_adr(loc: &mut [u8], val: u64) { + or_insn( + loc, + ((bits(val, 1, 0) << 29) | (bits(val, 20, 2) << 5)) as u32, + ); +} + +/// Rewrites a MOV to MOVZ or MOVN, whichever represents `val`. +fn write_movn_movz(loc: &mut [u8], val: i64) { + let rd = insn(loc) & 0b0000_0000_0110_0000_0000_0000_0001_1111; + let imm = if val >= 0 { + 0xd280_0000 | (bits(val as u64, 15, 0) << 5) as u32 + } else { + 0x9280_0000 | (bits(!val as u64, 15, 0) << 5) as u32 + }; + write_insn(loc, rd | imm); +} + +fn page(val: u64) -> u64 { + val & !0xfff +} + +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions +fn is_adrp(loc: &[u8]) -> bool { + bits(insn(loc) as u64, 31, 24) & 0b1001_1111 == 0b1001_0000 +} + +fn is_ldr(loc: &[u8]) -> bool { + bits(insn(loc) as u64, 31, 20) & 0b1111_1111_1100 == 0b1111_1001_0100 +} + +fn is_add(loc: &[u8]) -> bool { + bits(insn(loc) as u64, 31, 20) & 0b1111_1111_1100 == 0b1001_0001_0000 +} + +const NOP: u32 = 0xd503_201f; + +impl Arm64Target { + /// Whether the ADRP+ADD pair at relocation `i` can become NOP+ADR, + /// which the psABI allows when the target is within ±1 MiB. + fn relaxes_adrp_add(ctx: &Context, isec: &InputSection, i: usize) -> bool { + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let rel = &rels.at(i); + if !matches!( + rel.r_type, + R_AARCH64_ADR_PREL_PG_HI21 | R_AARCH64_ADR_PREL_PG_HI21_NC + ) || !ctx.args.relax + { + return false; + } + let Some(rel2) = rels.get(i + 1) else { + return false; + }; + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if !sym.is_pcrel_linktime_const(ctx) { + return false; + } + let s = sym.addr(ctx); + let p = isec.addr(ctx) + rel.r_offset; + let val = s + .wrapping_add(rel.r_addend as u64) + .wrapping_sub(p) + .wrapping_sub(4) as i64; + let off = rel.r_offset as usize; + let loc = &isec.contents()[off..]; + is_int(val, 21) + && rel2.r_type == R_AARCH64_ADD_ABS_LO12_NC + && rel2.r_sym == rel.r_sym + && rel2.r_offset == rel.r_offset + 4 + && rel2.r_addend == rel.r_addend + && is_adrp(loc) + && is_add(&loc[4..]) + && bits(insn(loc) as u64, 4, 0) == bits(insn(&loc[4..]) as u64, 4, 0) + } +} + +impl Arch for Arm64Target { + const NAME: &'static str = if End::IS_LITTLE { "arm64" } else { "arm64be" }; + const FAMILY: Family = Family::Arm64; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_AARCH64; + const PLT_HDR_SIZE: u64 = 32; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 16; + const THUNK: Option = Some(ThunkLayout { + header_size: 0, + entry_size: 24, + }); + const SFRAME_ABI: Option = Some(if End::IS_LITTLE { + SFRAME_ABI_AARCH64_ENDIAN_LITTLE + } else { + SFRAME_ABI_AARCH64_ENDIAN_BIG + }); + const TRAP: &'static [u8] = &[0x00, 0x7d, 0x20, 0xd4]; // brk + + const R_COPY: u32 = R_AARCH64_COPY; + const R_GLOB_DAT: u32 = R_AARCH64_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_AARCH64_JUMP_SLOT; + const R_ABS: u32 = R_AARCH64_ABS64; + const R_RELATIVE: u32 = R_AARCH64_RELATIVE; + const R_IRELATIVE: Option = Some(R_AARCH64_IRELATIVE); + const R_DTPOFF: u32 = R_AARCH64_TLS_DTPREL64; + const R_TPOFF: u32 = R_AARCH64_TLS_TPREL64; + const R_DTPMOD: u32 = R_AARCH64_TLS_DTPMOD64; + const R_TLSDESC: Option = Some(R_AARCH64_TLSDESC); + const R_SFRAME: Option = Some(R_AARCH64_PREL64); + const R_FUNCALL: &'static [u32] = &[R_AARCH64_JUMP26, R_AARCH64_CALL26]; + + fn rel_to_string(r_type: u32) -> String { + arm64_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u32; 8] = [ + 0xa9bf_7bf0, // stp x16, x30, [sp,#-16]! + 0x9000_0010, // adrp x16, .got.plt[2] + 0xf940_0211, // ldr x17, [x16, .got.plt[2]] + 0x9100_0210, // add x16, x16, .got.plt[2] + 0xd61f_0220, // br x17 + 0xd420_7d00, // brk + 0xd420_7d00, // brk + 0xd420_7d00, // brk + ]; + for (i, &v) in INSN.iter().enumerate() { + write_insn(&mut buf[i * 4..], v); + } + let gotplt = ctx.gotplt.hdr.shdr.sh_addr + 16; + let plt = ctx.plt.hdr.shdr.sh_addr; + write_adrp(&mut buf[4..], page(gotplt).wrapping_sub(page(plt + 4))); + or_insn(&mut buf[8..], (bits(gotplt, 11, 3) << 10) as u32); + or_insn(&mut buf[12..], ((gotplt & 0xfff) << 10) as u32); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u32; 4] = [ + 0x9000_0010, // adrp x16, .got.plt[n] + 0xf940_0211, // ldr x17, [x16, .got.plt[n]] + 0x9100_0210, // add x16, x16, .got.plt[n] + 0xd61f_0220, // br x17 + ]; + for (i, &v) in INSN.iter().enumerate() { + write_insn(&mut buf[i * 4..], v); + } + let gotplt = sym.gotplt_addr(ctx); + let plt = sym.plt_addr(ctx); + write_adrp(buf, page(gotplt).wrapping_sub(page(plt))); + or_insn(&mut buf[4..], (bits(gotplt, 11, 3) << 10) as u32); + or_insn(&mut buf[8..], ((gotplt & 0xfff) << 10) as u32); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u32; 4] = [ + 0x9000_0010, // adrp x16, GOT[n] + 0xf940_0211, // ldr x17, [x16, GOT[n]] + 0xd61f_0220, // br x17 + 0xd420_7d00, // brk + ]; + for (i, &v) in INSN.iter().enumerate() { + write_insn(&mut buf[i * 4..], v); + } + let got = sym.got_pltgot_addr(ctx); + let plt = sym.plt_addr(ctx); + write_adrp(buf, page(got).wrapping_sub(page(plt))); + or_insn(&mut buf[4..], (bits(got, 11, 3) << 10) as u32); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + let check = |val: i64, lo: i64, hi: i64| eh_frame::check_range(ctx, isec, rel, val, lo, hi); + match rel.r_type { + R_NONE => {} + R_AARCH64_ABS64 => End::write_u64(loc, val), + R_AARCH64_PREL32 => { + check(val.wrapping_sub(p) as i64, -(1 << 31), 1 << 31); + End::write_u32(loc, val.wrapping_sub(p) as u32); + } + R_AARCH64_PREL64 => End::write_u64(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE || isec.record_undef_error(ctx, rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let loc = &isec.contents()[rel.r_offset as usize..]; + + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_AARCH64_MOVW_UABS_G3 => scan_absrel(ctx, isec, sym, rel), + R_AARCH64_ADR_GOT_PAGE => { + // An ADRP+LDR pair loading a symbol's address from the + // GOT can become ADRP+ADD if the address is a link-time + // constant, saving the memory load. The pair must be + // consecutive and use the same register. + let relaxable = ctx.args.relax + && sym.is_pcrel_linktime_const(ctx) + && rels.get(i).is_some_and(|rel2| { + rel2.r_type == R_AARCH64_LD64_GOT_LO12_NC + && rel2.r_offset == rel.r_offset + 4 + && rel2.r_sym == rel.r_sym + && rel.r_addend == 0 + && rel2.r_addend == 0 + && is_adrp(loc) + && is_ldr(&loc[4..]) + && { + let rd = bits(insn(loc) as u64, 4, 0); + let rn = bits(insn(&loc[4..]) as u64, 9, 5); + let rt = bits(insn(&loc[4..]) as u64, 4, 0); + rd == rn && rn == rt + } + }); + if relaxable { + i += 1; + } else { + sym.add_flags(NEEDS_GOT); + } + } + R_AARCH64_LD64_GOT_LO12_NC | R_AARCH64_LD64_GOTPAGE_LO15 | R_AARCH64_GOTPCREL32 => { + sym.add_flags(NEEDS_GOT) + } + R_AARCH64_CALL26 | R_AARCH64_JUMP26 | R_AARCH64_PLT32 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 | R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC => { + sym.add_flags(NEEDS_GOTTP) + } + R_AARCH64_ADR_PREL_PG_HI21 | R_AARCH64_ADR_PREL_PG_HI21_NC => { + scan_pcrel(ctx, isec, sym, rel) + } + R_AARCH64_TLSGD_ADR_PAGE21 => sym.add_flags(NEEDS_TLSGD), + R_AARCH64_TLSDESC_CALL => scan_tlsdesc(ctx, sym), + R_AARCH64_TLSLE_MOVW_TPREL_G2 + | R_AARCH64_TLSLE_ADD_TPREL_LO12 + | R_AARCH64_TLSLE_ADD_TPREL_LO12_NC => check_tlsle(ctx, isec, sym, rel), + R_AARCH64_ABS64 + | R_AARCH64_ADD_ABS_LO12_NC + | R_AARCH64_ADR_PREL_LO21 + | R_AARCH64_CONDBR19 + | R_AARCH64_LD_PREL_LO19 + | R_AARCH64_LDST16_ABS_LO12_NC + | R_AARCH64_LDST32_ABS_LO12_NC + | R_AARCH64_LDST64_ABS_LO12_NC + | R_AARCH64_LDST128_ABS_LO12_NC + | R_AARCH64_LDST8_ABS_LO12_NC + | R_AARCH64_MOVW_UABS_G0 + | R_AARCH64_MOVW_UABS_G0_NC + | R_AARCH64_MOVW_UABS_G1 + | R_AARCH64_MOVW_UABS_G1_NC + | R_AARCH64_MOVW_UABS_G2 + | R_AARCH64_MOVW_UABS_G2_NC + | R_AARCH64_MOVW_PREL_G0 + | R_AARCH64_MOVW_PREL_G0_NC + | R_AARCH64_MOVW_PREL_G1 + | R_AARCH64_MOVW_PREL_G1_NC + | R_AARCH64_MOVW_PREL_G2 + | R_AARCH64_MOVW_PREL_G2_NC + | R_AARCH64_MOVW_PREL_G3 + | R_AARCH64_PREL16 + | R_AARCH64_PREL32 + | R_AARCH64_PREL64 + | R_AARCH64_TLSGD_ADD_LO12_NC + | R_AARCH64_TLSLE_MOVW_TPREL_G0 + | R_AARCH64_TLSLE_MOVW_TPREL_G0_NC + | R_AARCH64_TLSLE_MOVW_TPREL_G1 + | R_AARCH64_TLSLE_MOVW_TPREL_G1_NC + | R_AARCH64_TLSLE_ADD_TPREL_HI12 + | R_AARCH64_TLSDESC_ADR_PAGE21 + | R_AARCH64_TLSDESC_LD64_LO12 + | R_AARCH64_TLSDESC_ADD_LO12 => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let got = ctx.got.hdr.shdr.sh_addr; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + let tprel = sa.wrapping_sub(ctx.tp_addr); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i - 1, val, lo, hi); + let loc = &mut buf[off..]; + + match rel.r_type { + // Handled as an absolute relocation by the output section. + R_AARCH64_ABS64 => {} + R_AARCH64_LDST8_ABS_LO12_NC | R_AARCH64_ADD_ABS_LO12_NC => { + or_insn(loc, (bits(sa, 11, 0) << 10) as u32) + } + R_AARCH64_LDST16_ABS_LO12_NC => or_insn(loc, (bits(sa, 11, 1) << 10) as u32), + R_AARCH64_LDST32_ABS_LO12_NC => or_insn(loc, (bits(sa, 11, 2) << 10) as u32), + R_AARCH64_LDST64_ABS_LO12_NC => or_insn(loc, (bits(sa, 11, 3) << 10) as u32), + R_AARCH64_LDST128_ABS_LO12_NC => or_insn(loc, (bits(sa, 11, 4) << 10) as u32), + R_AARCH64_MOVW_UABS_G0 => { + check(sa as i64, 0, 1 << 16); + or_insn(loc, (bits(sa, 15, 0) << 5) as u32); + } + R_AARCH64_MOVW_UABS_G0_NC => or_insn(loc, (bits(sa, 15, 0) << 5) as u32), + R_AARCH64_MOVW_UABS_G1 => { + check(sa as i64, 0, 1 << 32); + or_insn(loc, (bits(sa, 31, 16) << 5) as u32); + } + R_AARCH64_MOVW_UABS_G1_NC => or_insn(loc, (bits(sa, 31, 16) << 5) as u32), + R_AARCH64_MOVW_UABS_G2 => { + check(sa as i64, 0, 1 << 48); + or_insn(loc, (bits(sa, 47, 32) << 5) as u32); + } + R_AARCH64_MOVW_UABS_G2_NC => or_insn(loc, (bits(sa, 47, 32) << 5) as u32), + R_AARCH64_MOVW_UABS_G3 => or_insn(loc, (bits(sa, 63, 48) << 5) as u32), + R_AARCH64_MOVW_PREL_G0 => { + check(pcrel as i64, -(1 << 15), 1 << 15); + write_movn_movz(loc, pcrel as i64); + } + R_AARCH64_MOVW_PREL_G0_NC => or_insn(loc, (bits(pcrel, 15, 0) << 5) as u32), + R_AARCH64_MOVW_PREL_G1 => { + check(pcrel as i64, -(1 << 31), 1 << 31); + write_movn_movz(loc, (pcrel as i64) >> 16); + } + R_AARCH64_MOVW_PREL_G1_NC => or_insn(loc, (bits(pcrel, 31, 16) << 5) as u32), + R_AARCH64_MOVW_PREL_G2 => { + check(pcrel as i64, -(1 << 47), 1 << 47); + write_movn_movz(loc, (pcrel as i64) >> 32); + } + R_AARCH64_MOVW_PREL_G2_NC => or_insn(loc, (bits(pcrel, 47, 32) << 5) as u32), + R_AARCH64_MOVW_PREL_G3 => or_insn(loc, (bits(pcrel, 63, 48) << 5) as u32), + R_AARCH64_ADR_GOT_PAGE => { + if sym.has_got(&ctx.symbols) { + let val = page(g().wrapping_add(got).wrapping_add(a)).wrapping_sub(page(p)); + check(val as i64, -(1 << 32), 1 << 32); + write_adrp(loc, val); + } else { + // Relax the GOT-loading ADRP+LDR to ADRP+ADD. + let val = page(sa).wrapping_sub(page(p)); + check(val as i64, -(1 << 32), 1 << 32); + write_adrp(loc, val); + let reg = bits(insn(loc) as u64, 4, 0) as u32; + write_insn( + &mut loc[4..], + 0x9100_0000 | (reg << 5) | reg | (bits(sa, 11, 0) << 10) as u32, + ); + i += 1; + } + } + R_AARCH64_ADR_PREL_PG_HI21 | R_AARCH64_ADR_PREL_PG_HI21_NC => { + // ADRP+ADD may become NOP+ADR if the target is within + // ±1 MiB. + if Self::relaxes_adrp_add(ctx, isec, i - 1) { + let reg = bits(insn(loc) as u64, 4, 0) as u32; + write_insn(loc, NOP); + write_insn(&mut loc[4..], 0x1000_0000 | reg); + write_adr(&mut loc[4..], pcrel.wrapping_sub(4)); + i += 1; + } else { + let val = page(sa).wrapping_sub(page(p)); + if rel.r_type == R_AARCH64_ADR_PREL_PG_HI21 { + check(val as i64, -(1 << 32), 1 << 32); + } + write_adrp(loc, val); + } + } + R_AARCH64_ADR_PREL_LO21 => { + check(pcrel as i64, -(1 << 20), 1 << 20); + write_adr(loc, pcrel); + } + R_AARCH64_CALL26 | R_AARCH64_JUMP26 => { + if sym.is_remaining_undef_weak() { + // Calling an undefined weak symbol falls through. + write_insn(loc, NOP); + } else { + let mut val = pcrel; + if !is_int(val as i64, 28) { + val = sym.thunk_addr(ctx, p).wrapping_add(a).wrapping_sub(p); + } + or_insn(loc, bits(val, 27, 2) as u32); + } + } + R_AARCH64_PLT32 => { + check(pcrel as i64, -(1 << 31), 1 << 31); + End::write_u32(loc, pcrel as u32); + } + R_AARCH64_GOTPCREL32 => { + let val = g().wrapping_add(got).wrapping_add(a).wrapping_sub(p); + check(val as i64, -(1 << 31), 1 << 31); + End::write_u32(loc, val as u32); + } + R_AARCH64_CONDBR19 | R_AARCH64_LD_PREL_LO19 => { + check(pcrel as i64, -(1 << 20), 1 << 20); + or_insn(loc, (bits(pcrel, 20, 2) << 5) as u32); + } + R_AARCH64_PREL16 => { + check(pcrel as i64, -(1 << 15), 1 << 16); + End::write_u16(loc, pcrel as u16); + } + R_AARCH64_PREL32 => { + check(pcrel as i64, -(1 << 31), 1 << 32); + End::write_u32(loc, pcrel as u32); + } + R_AARCH64_PREL64 => End::write_u64(loc, pcrel), + R_AARCH64_LD64_GOT_LO12_NC => or_insn( + loc, + (bits(g().wrapping_add(got).wrapping_add(a), 11, 3) << 10) as u32, + ), + R_AARCH64_LD64_GOTPAGE_LO15 => { + let val = g() + .wrapping_add(got) + .wrapping_add(a) + .wrapping_sub(page(got)); + check(val as i64, 0, 1 << 15); + or_insn(loc, (bits(val, 14, 3) << 10) as u32); + } + R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 => { + let val = page(sym.gottp_addr(ctx).wrapping_add(a)).wrapping_sub(page(p)); + check(val as i64, -(1 << 32), 1 << 32); + write_adrp(loc, val); + } + R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC => or_insn( + loc, + (bits(sym.gottp_addr(ctx).wrapping_add(a), 11, 3) << 10) as u32, + ), + R_AARCH64_TLSLE_MOVW_TPREL_G0 => { + check(tprel as i64, -(1 << 15), 1 << 15); + write_movn_movz(loc, tprel as i64); + } + R_AARCH64_TLSLE_MOVW_TPREL_G0_NC => or_insn(loc, (bits(tprel, 15, 0) << 5) as u32), + R_AARCH64_TLSLE_MOVW_TPREL_G1 => { + check(tprel as i64, -(1 << 31), 1 << 31); + write_movn_movz(loc, (tprel as i64) >> 16); + } + R_AARCH64_TLSLE_MOVW_TPREL_G1_NC => or_insn(loc, (bits(tprel, 31, 16) << 5) as u32), + R_AARCH64_TLSLE_MOVW_TPREL_G2 => { + check(tprel as i64, -(1 << 47), 1 << 47); + write_movn_movz(loc, (tprel as i64) >> 32); + } + R_AARCH64_TLSLE_ADD_TPREL_HI12 => { + check(tprel as i64, 0, 1 << 24); + or_insn(loc, (bits(tprel, 23, 12) << 10) as u32); + } + R_AARCH64_TLSLE_ADD_TPREL_LO12 => { + check(tprel as i64, 0, 1 << 12); + or_insn(loc, (bits(tprel, 11, 0) << 10) as u32); + } + R_AARCH64_TLSLE_ADD_TPREL_LO12_NC => { + or_insn(loc, (bits(tprel, 11, 0) << 10) as u32) + } + R_AARCH64_TLSGD_ADR_PAGE21 => { + let val = page(sym.tlsgd_addr(ctx).wrapping_add(a)).wrapping_sub(page(p)); + check(val as i64, -(1 << 32), 1 << 32); + write_adrp(loc, val); + } + R_AARCH64_TLSGD_ADD_LO12_NC => or_insn( + loc, + (bits(sym.tlsgd_addr(ctx).wrapping_add(a), 11, 0) << 10) as u32, + ), + // TLSDESC materializes a TP-relative address in x0: + // + // adrp x0, 0 # R_AARCH64_TLSDESC_ADR_PAGE21 + // ldr x1, [x0] # R_AARCH64_TLSDESC_LD64_LO12 + // add x0, x0, #0 # R_AARCH64_TLSDESC_ADD_LO12 + // blr x1 # R_AARCH64_TLSDESC_CALL + // + // If the address is known at link time the sequence becomes + // two NOPs, `movz x0, hi, lsl #16` and `movk x0, lo`; if at + // load time, two NOPs, `adrp x0, gottprel` and `ldr x0`. + R_AARCH64_TLSDESC_ADR_PAGE21 => { + if sym.has_tlsdesc(&ctx.symbols) { + let val = page(sym.tlsdesc_addr(ctx).wrapping_add(a)).wrapping_sub(page(p)); + check(val as i64, -(1 << 32), 1 << 32); + write_adrp(loc, val); + } else { + write_insn(loc, NOP); + } + } + R_AARCH64_TLSDESC_LD64_LO12 => { + if sym.has_tlsdesc(&ctx.symbols) { + or_insn( + loc, + (bits(sym.tlsdesc_addr(ctx).wrapping_add(a), 11, 3) << 10) as u32, + ); + } else { + write_insn(loc, NOP); + } + } + R_AARCH64_TLSDESC_ADD_LO12 => { + if sym.has_tlsdesc(&ctx.symbols) { + or_insn( + loc, + (bits(sym.tlsdesc_addr(ctx).wrapping_add(a), 11, 0) << 10) as u32, + ); + } else if sym.has_gottp(&ctx.symbols) { + write_insn(loc, 0x9000_0000); // adrp x0, 0 + write_adrp( + loc, + page(sym.gottp_addr(ctx).wrapping_add(a)).wrapping_sub(page(p)), + ); + } else { + write_insn(loc, 0xd2a0_0000 | (bits(tprel, 32, 16) << 5) as u32); + // movz x0, 0, lsl #16 + } + } + R_AARCH64_TLSDESC_CALL => { + if sym.has_tlsdesc(&ctx.symbols) { + // Nothing to do. + } else if sym.has_gottp(&ctx.symbols) { + write_insn( + loc, + 0xf940_0000 + | (bits(sym.gottp_addr(ctx).wrapping_add(a), 11, 3) << 10) as u32, + ); // ldr x0, [x0, 0] + } else { + write_insn(loc, 0xf280_0000 | (bits(tprel, 15, 0) << 5) as u32); + // movk x0, 0 + } + } + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let off = rel.r_offset as usize; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let frag_ref = frag.map(|(f, _)| f); + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + let loc = &mut buf[off..]; + + match rel.r_type { + R_AARCH64_ABS64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => End::write_u64(loc, v), + None => End::write_u64(loc, s.wrapping_add(a)), + }, + R_AARCH64_ABS32 => { + check(s.wrapping_add(a) as i64, 0, 1 << 32); + End::write_u32(loc, s.wrapping_add(a) as u32); + } + R_AARCH64_TLS_DTPREL64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => End::write_u64(loc, v), + None => End::write_u64(loc, s.wrapping_add(a).wrapping_sub(ctx.dtp_addr)), + }, + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, i: usize) -> u32 { + if !isec.is_alloc() { + return rel.r_type; + } + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let follows_relaxed_got_load = || { + i > 0 + && rels.at(i - 1).r_type == R_AARCH64_ADR_GOT_PAGE + && rels.at(i - 1).r_sym == rel.r_sym + && !sym.has_got(&ctx.symbols) + }; + + match rel.r_type { + R_AARCH64_ADR_GOT_PAGE if !sym.has_got(&ctx.symbols) => R_AARCH64_ADR_PREL_PG_HI21, + R_AARCH64_LD64_GOT_LO12_NC if follows_relaxed_got_load() => R_AARCH64_ADD_ABS_LO12_NC, + R_AARCH64_ADR_PREL_PG_HI21 | R_AARCH64_ADR_PREL_PG_HI21_NC + if Self::relaxes_adrp_add(ctx, isec, i) => + { + R_NONE + } + R_AARCH64_ADD_ABS_LO12_NC if i > 0 && Self::relaxes_adrp_add(ctx, isec, i - 1) => { + R_AARCH64_ADR_PREL_LO21 + } + R_AARCH64_TLSDESC_ADR_PAGE21 | R_AARCH64_TLSDESC_LD64_LO12 + if !sym.has_tlsdesc(&ctx.symbols) => + { + R_NONE + } + R_AARCH64_TLSDESC_ADD_LO12 if !sym.has_tlsdesc(&ctx.symbols) => { + if sym.has_gottp(&ctx.symbols) { + R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 + } else { + R_AARCH64_TLSLE_MOVW_TPREL_G1 + } + } + R_AARCH64_TLSDESC_CALL if !sym.has_tlsdesc(&ctx.symbols) => { + if sym.has_gottp(&ctx.symbols) { + R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC + } else { + R_AARCH64_TLSLE_MOVW_TPREL_G0_NC + } + } + _ => rel.r_type, + } + } + + /// Thunk entries are 12 bytes when the target is within ±4 GiB and + /// 24 bytes otherwise. + fn thunk_offsets(ctx: &Context, thunk: &Thunk, addr: u64) -> Vec { + // The distance between S and P only shrinks during layout, but + // page(S) - page(P) may still grow by a page as addresses move, so + // a page of margin is kept: page(0x1200) - page(0x1000) is 0 while + // page(0x1100) - page(0xfff) is 0x1000 for a shorter distance. + let is_small = |prel: i64| is_int(prel + 0x1000, 33) && is_int(prel - 0x1000, 33); + let mut offsets = vec![0]; + let mut off = 0; + for &sym in &thunk.symbols { + let s = ctx.symbols[sym].addr(ctx); + let p = addr + off; + let prel = page(s).wrapping_sub(page(p)) as i64; + off += if is_small(prel) { 12 } else { 24 }; + offsets.push(off); + } + offsets + } + + fn write_thunk(ctx: &Context, thunk: &Thunk, addr: u64, buf: &mut [u8]) { + // A short thunk with a 33-bit displacement. + const SHORT: [u32; 3] = [ + 0x9000_0010, // adrp x16, 0 + 0x9100_0210, // add x16, x16 + 0xd61f_0200, // br x16 + ]; + // A long thunk with a 64-bit displacement. + const LONG: [u32; 6] = [ + 0x1000_0010, // adr x16, 0 + 0xd2a0_0011, // movz x17, 0, lsl #16 + 0xf2c0_0011, // movk x17, 0, lsl #32 + 0xf2e0_0011, // movk x17, 0, lsl #48 + 0x8b11_0210, // add x16, x16, x17 + 0xd61f_0200, // br x16 + ]; + + for (i, &sym) in thunk.symbols.iter().enumerate() { + let s = ctx.symbols[sym].addr(ctx); + let p = addr + thunk.offsets[i]; + let entry = &mut buf[thunk.offsets[i] as usize..thunk.offsets[i + 1] as usize]; + if entry.len() == 12 { + let prel = page(s).wrapping_sub(page(p)); + debug_assert!(is_int(prel as i64, 33)); + for (j, &v) in SHORT.iter().enumerate() { + write_insn(&mut entry[j * 4..], v); + } + write_adrp(entry, prel); + or_insn(&mut entry[4..], (bits(s, 11, 0) << 10) as u32); + } else { + let disp = s.wrapping_sub(p); + for (j, &v) in LONG.iter().enumerate() { + write_insn(&mut entry[j * 4..], v); + } + write_adr(entry, bits(disp, 15, 0)); + or_insn(&mut entry[4..], (bits(disp, 31, 16) << 5) as u32); + or_insn(&mut entry[8..], (bits(disp, 47, 32) << 5) as u32); + or_insn(&mut entry[12..], (bits(disp, 63, 48) << 5) as u32); + } + } + } +} diff --git a/src/arch/i386.rs b/src/arch/i386.rs new file mode 100644 index 000000000..e53828353 --- /dev/null +++ b/src/arch/i386.rs @@ -0,0 +1,611 @@ +//! i386. +//! +//! i386 has no PC-relative memory access, so position-independent code +//! first learns its own address by calling a thunk such as +//! `__x86.get_pc_thunk.bx`, which returns the address after the call in a +//! register, and then addresses variables relative to the GOT, whose +//! address it keeps in `%ebx`. PLT entries of position-independent code +//! use `%ebx` likewise; a position-dependent executable can't assume +//! `%ebx` holds anything, so it gets PLT entries with absolute addresses. +//! +//! Relocations are of the REL type: addends live in the relocated +//! locations rather than in the relocation entries. +//! +//! https://github.com/rui314/psabi/blob/main/i386.pdf + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct I386; + +impl Layout for I386 { + type Endian = LittleEndian; + const IS_64: bool = false; + const IS_RELA: bool = false; +} + +impl Arch for I386 { + const NAME: &'static str = "i386"; + const FAMILY: Family = Family::I386; + const PAGE_SIZE: u64 = 4096; + const E_MACHINE: u32 = EM_386; + const PLT_HDR_SIZE: u64 = 16; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 8; + const TRAP: &'static [u8] = &[0xcc]; // int3 + + const R_COPY: u32 = R_386_COPY; + const R_GLOB_DAT: u32 = R_386_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_386_JUMP_SLOT; + const R_ABS: u32 = R_386_32; + const R_RELATIVE: u32 = R_386_RELATIVE; + const R_IRELATIVE: Option = Some(R_386_IRELATIVE); + const R_DTPOFF: u32 = R_386_TLS_DTPOFF32; + const R_TPOFF: u32 = R_386_TLS_TPOFF; + const R_DTPMOD: u32 = R_386_TLS_DTPMOD32; + const R_TLSDESC: Option = Some(R_386_TLS_DESC); + const R_FUNCALL: &'static [u32] = &[R_386_PLT32]; + + fn rel_to_string(r_type: u32) -> String { + i386_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + let gotplt = ctx.gotplt.hdr.shdr.sh_addr; + if ctx.args.pic { + const INSN: [u8; 16] = [ + 0x51, // push %ecx + 0x8d, 0x8b, 0, 0, 0, 0, // lea GOTPLT+4(%ebx), %ecx + 0xff, 0x31, // push (%ecx) + 0xff, 0x61, 0x04, // jmp *0x4(%ecx) + 0xcc, 0xcc, 0xcc, 0xcc, // padding + ]; + buf[..16].copy_from_slice(&INSN); + write_u32( + &mut buf[3..], + (gotplt - ctx.got.hdr.shdr.sh_addr + 4) as u32, + ); + } else { + const INSN: [u8; 16] = [ + 0x51, // push %ecx + 0xb9, 0, 0, 0, 0, // mov GOTPLT+4, %ecx + 0xff, 0x31, // push (%ecx) + 0xff, 0x61, 0x04, // jmp *0x4(%ecx) + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // padding + ]; + buf[..16].copy_from_slice(&INSN); + write_u32(&mut buf[2..], (gotplt + 4) as u32); + } + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let reloc_offset = + sym.plt_idx(&ctx.symbols).unwrap() as u64 * ElfRel::size::() as u64; + if ctx.args.pic { + const INSN: [u8; 16] = [ + 0xb9, 0, 0, 0, 0, // mov $reloc_offset, %ecx + 0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx) + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // padding + ]; + buf[..16].copy_from_slice(&INSN); + write_u32(&mut buf[1..], reloc_offset as u32); + write_u32( + &mut buf[7..], + (sym.gotplt_addr(ctx) - ctx.got.hdr.shdr.sh_addr) as u32, + ); + } else { + const INSN: [u8; 16] = [ + 0xb9, 0, 0, 0, 0, // mov $reloc_offset, %ecx + 0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // padding + ]; + buf[..16].copy_from_slice(&INSN); + write_u32(&mut buf[1..], reloc_offset as u32); + write_u32(&mut buf[7..], sym.gotplt_addr(ctx) as u32); + } + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + if ctx.args.pic { + const INSN: [u8; 8] = [ + 0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx) + 0xcc, 0xcc, // padding + ]; + buf[..8].copy_from_slice(&INSN); + write_u32( + &mut buf[2..], + (sym.got_pltgot_addr(ctx) - ctx.got.hdr.shdr.sh_addr) as u32, + ); + } else { + const INSN: [u8; 8] = [ + 0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT + 0xcc, 0xcc, // padding + ]; + buf[..8].copy_from_slice(&INSN); + write_u32(&mut buf[2..], sym.got_pltgot_addr(ctx) as u32); + } + } + + fn apply_eh_reloc( + ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_386_32 => write_u32(loc, val as u32), + R_386_PC32 => write_u32(loc, val.wrapping_sub(p) as u32), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE || isec.record_undef_error(ctx, rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + if rel.r_type == R_386_TLS_GD || rel.r_type == R_386_TLS_LDM { + let next = rels.get(i).map(|r| r.r_type); + if !matches!( + next, + Some(R_386_PLT32 | R_386_PC32 | R_386_GOT32 | R_386_GOT32X) + ) { + fatal!( + ctx, + "{}: {} must be followed by PLT or GOT32", + isec.display(file), + rel.type_name::() + ); + } + } + + match rel.r_type { + R_386_8 | R_386_16 => scan_absrel(ctx, isec, sym, rel), + R_386_PC8 | R_386_PC16 | R_386_PC32 => scan_pcrel(ctx, isec, sym, rel), + R_386_GOT32 | R_386_GOTPC => sym.add_flags(NEEDS_GOT), + R_386_GOT32X => { + // GOT32X is relaxed even with --no-relax because static + // PIE doesn't work without it. + if !(sym.is_pcrel_linktime_const(ctx) + && relax_got32x(loc_before(isec, rel)) != 0) + { + sym.add_flags(NEEDS_GOT); + } + } + R_386_PLT32 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_386_TLS_GOTIE | R_386_TLS_IE => sym.add_flags(NEEDS_GOTTP), + R_386_TLS_GD => { + // Always relax with -static because libc.a doesn't + // contain __tls_get_addr. + if ctx.args.is_static || (ctx.args.relax && sym.is_tprel_linktime_const(ctx)) { + i += 1; + } else { + sym.add_flags(NEEDS_TLSGD); + } + } + R_386_TLS_LDM => { + if ctx.args.is_static || (ctx.args.relax && !ctx.args.shared) { + i += 1; + } else { + ctx.needs_tlsld + .store(true, std::sync::atomic::Ordering::Relaxed); + } + } + R_386_TLS_GOTDESC => scan_tlsdesc(ctx, sym), + R_386_TLS_LE => check_tlsle(ctx, isec, sym, rel), + R_386_32 | R_386_GOTOFF | R_386_TLS_LDO_32 | R_386_SIZE32 | R_386_TLS_DESC_CALL => { + } + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = isec.rel_addend::(rel) as u64; + let p = isec.addr(ctx) + rel.r_offset; + let got = ctx.got.hdr.shdr.sh_addr; + let g = || sym.got_addr(ctx).wrapping_sub(got); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i - 1, val, lo, hi); + + match rel.r_type { + R_386_8 => { + check(s.wrapping_add(a) as i64, 0, 1 << 8); + buf[off] = s.wrapping_add(a) as u8; + } + R_386_16 => { + check(s.wrapping_add(a) as i64, 0, 1 << 16); + write_u16(&mut buf[off..], s.wrapping_add(a) as u16); + } + // Handled as an absolute relocation by the output section. + R_386_32 => {} + R_386_PC8 => { + let v = s.wrapping_add(a).wrapping_sub(p); + check(v as i64, -(1 << 7), 1 << 7); + buf[off] = v as u8; + } + R_386_PC16 => { + let v = s.wrapping_add(a).wrapping_sub(p); + check(v as i64, -(1 << 15), 1 << 15); + write_u16(&mut buf[off..], v as u16); + } + R_386_PC32 | R_386_PLT32 => { + write_u32(&mut buf[off..], s.wrapping_add(a).wrapping_sub(p) as u32) + } + R_386_GOT32 => write_u32(&mut buf[off..], g().wrapping_add(a) as u32), + R_386_GOT32X => { + if sym.has_got(&ctx.symbols) { + write_u32(&mut buf[off..], g().wrapping_add(a) as u32); + } else { + let insn = relax_got32x(&buf[..off]); + debug_assert!(insn != 0); + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write_u32(&mut buf[off..], s.wrapping_add(a).wrapping_sub(got) as u32); + } + } + R_386_GOTOFF => { + write_u32(&mut buf[off..], s.wrapping_add(a).wrapping_sub(got) as u32) + } + R_386_GOTPC => { + write_u32(&mut buf[off..], got.wrapping_add(a).wrapping_sub(p) as u32) + } + R_386_TLS_GOTIE => write_u32( + &mut buf[off..], + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got) as u32, + ), + R_386_TLS_LE => write_u32( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.tp_addr) as u32, + ), + R_386_TLS_IE => { + write_u32(&mut buf[off..], sym.gottp_addr(ctx).wrapping_add(a) as u32) + } + R_386_TLS_GD => { + if sym.has_tlsgd(&ctx.symbols) { + write_u32( + &mut buf[off..], + sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got) as u32, + ); + } else { + let next = &rels.at(i); + i += 1; + relax_gd_to_le(buf, off, next, s.wrapping_sub(ctx.tp_addr)); + } + } + R_386_TLS_LDM => { + if ctx.got.has_tlsld() { + write_u32( + &mut buf[off..], + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got) as u32, + ); + } else { + let next = &rels.at(i); + i += 1; + relax_ld_to_le(buf, off, next, ctx.tp_addr.wrapping_sub(ctx.tls_begin)); + } + } + R_386_TLS_LDO_32 => write_u32( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.dtp_addr) as u32, + ), + R_386_SIZE32 => write_u32( + &mut buf[off..], + sym.esym(ctx).st_size.wrapping_add(a) as u32, + ), + R_386_TLS_GOTDESC => { + // TLSDESC materializes a TP-relative address in %eax: + // + // lea 0(%ebx), %eax # R_386_TLS_GOTDESC + // call *(%eax) # R_386_TLS_DESC_CALL + // + // If the address is known at link time it becomes + // `mov $foo@TPOFF, %eax; nop`, and if at load time + // `mov foo@GOTTPOFF(%ebx), %eax; nop`. LLVM may put a + // `mov %reg, %eax` between the two instructions, which + // is why the lea may target any register. + if sym.has_tlsdesc(&ctx.symbols) { + write_u32( + &mut buf[off..], + sym.tlsdesc_addr(ctx).wrapping_add(a).wrapping_sub(got) as u32, + ); + } else if sym.has_gottp(&ctx.symbols) { + let insn = relax_tlsdesc_to_ie(&buf[..off]); + if insn == 0 { + fatal!( + ctx, + "{}: illegal instruction sequence for TLSDESC", + isec.display(file) + ); + } + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write_u32( + &mut buf[off..], + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got) as u32, + ); + } else { + let insn = relax_tlsdesc_to_le(&buf[..off]); + if insn == 0 { + fatal!( + ctx, + "{}: illegal instruction sequence for TLSDESC", + isec.display(file) + ); + } + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write_u32( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.tp_addr) as u32, + ); + } + } + R_386_TLS_DESC_CALL => { + if !sym.has_tlsdesc(&ctx.symbols) { + // call *(%eax) -> nop + buf[off] = 0x66; + buf[off + 1] = 0x90; + } + } + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let off = rel.r_offset as usize; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), isec.rel_addend::(&rel) as u64), + }; + let frag_ref = frag.map(|(f, _)| f); + let got = ctx.got.hdr.shdr.sh_addr; + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + + match rel.r_type { + R_386_8 => { + check(s.wrapping_add(a) as i64, 0, 1 << 8); + buf[off] = s.wrapping_add(a) as u8; + } + R_386_16 => { + check(s.wrapping_add(a) as i64, 0, 1 << 16); + write_u16(&mut buf[off..], s.wrapping_add(a) as u16); + } + R_386_32 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => write_u32(&mut buf[off..], v as u32), + None => write_u32(&mut buf[off..], s.wrapping_add(a) as u32), + }, + R_386_PC8 => { + check(s.wrapping_add(a) as i64, -(1 << 7), 1 << 7); + buf[off] = s.wrapping_add(a) as u8; + } + R_386_PC16 => { + check(s.wrapping_add(a) as i64, -(1 << 15), 1 << 15); + write_u16(&mut buf[off..], s.wrapping_add(a) as u16); + } + R_386_PC32 => write_u32(&mut buf[off..], s.wrapping_add(a) as u32), + R_386_GOTPC => write_u32(&mut buf[off..], got.wrapping_add(a) as u32), + R_386_GOTOFF => { + write_u32(&mut buf[off..], s.wrapping_add(a).wrapping_sub(got) as u32) + } + R_386_TLS_LDO_32 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => write_u32(&mut buf[off..], v as u32), + None => write_u32( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.dtp_addr) as u32, + ), + }, + R_386_SIZE32 => write_u32( + &mut buf[off..], + sym.esym(ctx).st_size.wrapping_add(a) as u32, + ), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, _i: usize) -> u32 { + if rel.r_type == R_386_GOT32X && isec.is_alloc() { + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if !sym.has_got(&ctx.symbols) { + return R_386_GOTOFF; + } + } + rel.r_type + } + + fn write_addend(loc: &mut [u8], val: i64, rel: &ElfRel) { + match rel.r_type { + R_386_NONE => {} + R_386_8 | R_386_PC8 => loc[0] = val as u8, + R_386_16 | R_386_PC16 => write_u16(loc, val as u16), + R_386_32 | R_386_PC32 | R_386_GOT32 | R_386_GOT32X | R_386_PLT32 | R_386_GOTOFF + | R_386_GOTPC | R_386_TLS_LDM | R_386_TLS_GOTIE | R_386_TLS_LE | R_386_TLS_IE + | R_386_TLS_GD | R_386_TLS_LDO_32 | R_386_SIZE32 | R_386_TLS_GOTDESC => { + write_u32(loc, val as u32) + } + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + + fn get_addend(loc: &[u8], rel: &ElfRel) -> i64 { + match rel.r_type { + R_386_8 | R_386_PC8 => loc[0] as i8 as i64, + R_386_16 | R_386_PC16 => i16::from_le_bytes([loc[0], loc[1]]) as i64, + R_386_32 | R_386_PC32 | R_386_GOT32 | R_386_GOT32X | R_386_PLT32 | R_386_GOTOFF + | R_386_GOTPC | R_386_TLS_LDM | R_386_TLS_GOTIE | R_386_TLS_LE | R_386_TLS_IE + | R_386_TLS_GD | R_386_TLS_LDO_32 | R_386_SIZE32 | R_386_TLS_GOTDESC => { + i32::from_le_bytes([loc[0], loc[1], loc[2], loc[3]]) as i64 + } + _ => 0, + } + } +} + +fn write_u16(buf: &mut [u8], v: u16) { + buf[..2].copy_from_slice(&v.to_le_bytes()); +} + +fn write_u32(buf: &mut [u8], v: u32) { + buf[..4].copy_from_slice(&v.to_le_bytes()); +} + +/// The bytes of a section preceding a relocated location. +fn loc_before<'a>(isec: &'a InputSection, rel: &ElfRel) -> &'a [u8] { + &isec.contents()[..rel.r_offset as usize] +} + +/// The last two bytes before a relocated location, as an opcode. +fn last2(loc: &[u8]) -> u32 { + match loc { + [.., a, b] => ((*a as u32) << 8) | *b as u32, + _ => 0, + } +} + +/// `mov imm(%reg1), %reg2` -> `lea imm(%reg1), %reg2`, or 0 if the +/// instruction isn't a GOT load that can be relaxed. +fn relax_got32x(loc: &[u8]) -> u32 { + match loc { + [.., 0x8b, modrm] => 0x8d00 | *modrm as u32, + _ => 0, + } +} + +/// Rewrites a `__tls_get_addr` call sequence to compute a link-time +/// constant TP-relative address instead. +fn relax_gd_to_le(buf: &mut [u8], off: usize, rel: &ElfRel, val: u64) { + const INSN: [u8; 12] = [ + 0x65, 0xa1, 0, 0, 0, 0, // mov %gs:0, %eax + 0x81, 0xc0, 0, 0, 0, 0, // add $tp_offset, %eax + ]; + match rel.r_type { + R_386_PLT32 | R_386_PC32 => { + buf[off - 3..off + 9].copy_from_slice(&INSN); + write_u32(&mut buf[off + 5..], val as u32); + } + R_386_GOT32 | R_386_GOT32X => { + buf[off - 2..off + 10].copy_from_slice(&INSN); + write_u32(&mut buf[off + 6..], val as u32); + } + _ => unreachable!(), + } +} + +/// Like `relax_gd_to_le`, but materializes the address of the TLS block +/// rather than of a particular variable. +fn relax_ld_to_le(buf: &mut [u8], off: usize, rel: &ElfRel, tls_size: u64) { + match rel.r_type { + R_386_PLT32 | R_386_PC32 => { + const INSN: [u8; 11] = [ + 0x65, 0xa1, 0, 0, 0, 0, // mov %gs:0, %eax + 0x2d, 0, 0, 0, 0, // sub $tls_size, %eax + ]; + buf[off - 2..off + 9].copy_from_slice(&INSN); + write_u32(&mut buf[off + 5..], tls_size as u32); + } + R_386_GOT32 | R_386_GOT32X => { + const INSN: [u8; 12] = [ + 0x65, 0xa1, 0, 0, 0, 0, // mov %gs:0, %eax + 0x81, 0xe8, 0, 0, 0, 0, // sub $tls_size, %eax + ]; + buf[off - 2..off + 10].copy_from_slice(&INSN); + write_u32(&mut buf[off + 6..], tls_size as u32); + } + _ => unreachable!(), + } +} + +fn relax_tlsdesc_to_ie(loc: &[u8]) -> u32 { + match last2(loc) { + 0x8d83 => 0x8b83, // lea 0(%ebx), %eax -> mov 0(%ebx), %eax + 0x8d9b => 0x8b9b, // lea 0(%ebx), %ebx -> mov 0(%ebx), %ebx + 0x8d8b => 0x8b8b, // lea 0(%ebx), %ecx -> mov 0(%ebx), %ecx + 0x8d93 => 0x8b93, // lea 0(%ebx), %edx -> mov 0(%ebx), %edx + 0x8db3 => 0x8bb3, // lea 0(%ebx), %esi -> mov 0(%ebx), %esi + 0x8dbb => 0x8bbb, // lea 0(%ebx), %edi -> mov 0(%ebx), %edi + 0x8da3 => 0x8ba3, // lea 0(%ebx), %esp -> mov 0(%ebx), %esp + 0x8dab => 0x8bab, // lea 0(%ebx), %ebp -> mov 0(%ebx), %ebp + _ => 0, + } +} + +fn relax_tlsdesc_to_le(loc: &[u8]) -> u32 { + match last2(loc) { + 0x8d83 => 0x90b8, // lea 0(%ebx), %eax -> mov $0, %eax + 0x8d9b => 0x90bb, // lea 0(%ebx), %ebx -> mov $0, %ebx + 0x8d8b => 0x90b9, // lea 0(%ebx), %ecx -> mov $0, %ecx + 0x8d93 => 0x90ba, // lea 0(%ebx), %edx -> mov $0, %edx + 0x8db3 => 0x90be, // lea 0(%ebx), %esi -> mov $0, %esi + 0x8dbb => 0x90bf, // lea 0(%ebx), %edi -> mov $0, %edi + 0x8da3 => 0x90bc, // lea 0(%ebx), %esp -> mov $0, %esp + 0x8dab => 0x90bd, // lea 0(%ebx), %ebp -> mov $0, %ebp + _ => 0, + } +} diff --git a/src/arch/loongarch.rs b/src/arch/loongarch.rs new file mode 100644 index 000000000..43b8430db --- /dev/null +++ b/src/arch/loongarch.rs @@ -0,0 +1,1016 @@ +//! LoongArch, Loongson's RISC ISA from 2021. +//! +//! LoongArch feels like a modernized MIPS with a hint of RISC-V, without +//! being compatible with either. All instructions are 4 bytes long and +//! 4-byte aligned, and there are 32 general-purpose registers, of which +//! `$t0`–`$t8` (`$r12`–`$r20`) are temporaries the PLT may use. +//! +//! Like RISC-V, LoongArch relies on linker relaxation: certain +//! instruction sequences may be rewritten to shorter ones, so sections +//! are not copied as atomic units. See [`crate::relax`]. +//! +//! https://github.com/loongson/la-abi-specs/blob/release/laelf.adoc + +// Binary literals are grouped by instruction field. +#![allow(clippy::unusual_byte_groupings)] + +use std::marker::PhantomData; + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::input_sections::{ + check_tlsle, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection, RelocDelta, +}; +use crate::relax::compute_distance; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::util::{align_to, bits, is_int, overwrite_uleb, read_uleb, sign_extend}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct LoongArchTarget(PhantomData<()>); + +pub type LoongArch64 = LoongArchTarget; +pub type LoongArch32 = LoongArchTarget; + +impl Layout for LoongArchTarget { + type Endian = LittleEndian; + const IS_64: bool = IS_64; + const IS_RELA: bool = true; +} + +fn page(val: u64) -> u64 { + val & !0xfff +} + +/// The immediate of a `pcalau12i` that, followed by an `addi.d` with the +/// low 12 bits, materializes `val`. +/// +/// `pcalau12i` computes `pc + (imm << 12)` with the low 12 bits cleared, +/// and `addi.d` sign-extends its immediate. So when bit 11 of the +/// target is set, `pcalau12i` has to aim 0x1000 higher to compensate. +/// RISC-V's `auipc` differs in not clearing the low bits. +fn hi20(val: u64, pc: u64) -> u64 { + bits(page(val.wrapping_add(0x800)).wrapping_sub(page(pc)), 31, 12) +} + +/// The upper part of a 64-bit PC-relative address for the large code +/// model, which the psABI's formula compensates for the sign extensions +/// of the `addi.d`/`lu32i.d`/`lu52i.d` sequence following `pcalau12i`. +fn higher(val: u64, pc: u64) -> u64 { + let compensation = if val & 0x800 != 0 { + 0x1000u64.wrapping_sub(0x1_0000_0000) + } else { + 0 + }; + let val = val.wrapping_add(0x8000_0000).wrapping_add(compensation); + page(val).wrapping_sub(page(pc.wrapping_sub(8))) +} + +fn higher20(val: u64, pc: u64) -> u64 { + bits(higher(val, pc), 51, 32) +} + +fn highest12(val: u64, pc: u64) -> u64 { + bits(higher(val, pc), 63, 52) +} + +fn insn(loc: &[u8]) -> u32 { + LittleEndian::read_u32(loc) +} + +fn set_insn(loc: &mut [u8], v: u32) { + LittleEndian::write_u32(loc, v); +} + +/// Instruction formats, named after their immediate fields. +fn write_k12(loc: &mut [u8], val: u64) { + // opcode, [11:0], rj, rd + set_insn( + loc, + (insn(loc) & 0b1111111111_000000000000_11111_11111) | (bits(val, 11, 0) << 10) as u32, + ); +} + +fn write_k16(loc: &mut [u8], val: u64) { + // opcode, [15:0], rj, rd + set_insn( + loc, + (insn(loc) & 0b111111_0000000000000000_11111_11111) | (bits(val, 15, 0) << 10) as u32, + ); +} + +fn write_j20(loc: &mut [u8], val: u64) { + // opcode, [19:0], rd + set_insn( + loc, + (insn(loc) & 0b1111111_00000000000000000000_11111) | (bits(val, 19, 0) << 5) as u32, + ); +} + +fn write_d5k16(loc: &mut [u8], val: u64) { + // opcode, [15:0], rj, [20:16] + let v = (insn(loc) & 0b111111_0000000000000000_11111_00000) + | (bits(val, 15, 0) << 10) as u32 + | bits(val, 20, 16) as u32; + set_insn(loc, v); +} + +fn write_d10k16(loc: &mut [u8], val: u64) { + // opcode, [15:0], [25:16] + let v = (insn(loc) & 0b111111_0000000000000000_0000000000) + | (bits(val, 15, 0) << 10) as u32 + | bits(val, 25, 16) as u32; + set_insn(loc, v); +} + +fn rd(insn: u32) -> u32 { + bits(insn as u64, 4, 0) as u32 +} + +fn rj(insn: u32) -> u32 { + bits(insn as u64, 9, 5) as u32 +} + +fn set_rj(loc: &mut [u8], rj: u32) { + debug_assert!(rj < 32); + set_insn( + loc, + (insn(loc) & 0b111111_1111111111111111_00000_11111) | (rj << 5), + ); +} + +/// Rewrites the instruction at `loc` into `pcaddi $rd, imm`, keeping +/// its destination register. +fn write_pcaddi(loc: &mut [u8], val: u64) { + set_insn(loc, 0x1800_0000 | rd(insn(loc))); + write_j20(loc, val); +} + +/// Adds `val` to the value at `loc`, or subtracts it, for the ADD/SUB +/// relocation families that debug info and tables are built from. +fn add_bits(loc: &mut [u8], size: u32, val: u64, subtract: bool) { + let apply = |cur: u64| { + if subtract { + cur.wrapping_sub(val) + } else { + cur.wrapping_add(val) + } + }; + match size { + 6 => loc[0] = (loc[0] & 0b1100_0000) | (apply(loc[0] as u64) as u8 & 0b0011_1111), + 8 => loc[0] = apply(loc[0] as u64) as u8, + 16 => LittleEndian::write_u16(loc, apply(LittleEndian::read_u16(loc) as u64) as u16), + 32 => LittleEndian::write_u32(loc, apply(LittleEndian::read_u32(loc) as u64) as u32), + 64 => LittleEndian::write_u64(loc, apply(LittleEndian::read_u64(loc))), + _ => unreachable!(), + } +} + +fn add_uleb(loc: &mut [u8], val: u64, subtract: bool) { + let cur = read_uleb(&mut &*loc); + overwrite_uleb( + loc, + if subtract { + cur.wrapping_sub(val) + } else { + cur.wrapping_add(val) + }, + ); +} + +/// Whether the `i`th relocation refers to a GOT load of a symbol whose +/// PC-relative address is a link-time constant, which can become an +/// address materialization: +/// +/// ```text +/// pcalau12i $t0, 0 # R_LARCH_GOT_PC_HI20, R_LARCH_RELAX +/// ld.d $t0, $t0, 0 # R_LARCH_GOT_PC_LO12, R_LARCH_RELAX +/// ``` +fn is_relaxable_got_load(ctx: &Context, isec: &InputSection, i: usize) -> bool { + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rels.at(i).r_sym as usize]]; + let contents = isec.original_contents(file); + + if !ctx.args.relax + || !sym.is_pcrel_linktime_const(ctx) + || i + 3 >= rels.len() + || rels.at(i + 1).r_type != R_LARCH_RELAX + || rels.at(i + 2).r_type != R_LARCH_GOT_PC_LO12 + || rels.at(i + 2).r_offset != rels.at(i).r_offset + 4 + || rels.at(i + 3).r_type != R_LARCH_RELAX + { + return false; + } + let insn1 = insn(&contents[rels.at(i).r_offset as usize..]); + let insn2 = insn(&contents[rels.at(i).r_offset as usize + 4..]); + let is_ld_d = insn2 & 0xffc0_0000 == 0x28c0_0000; + rd(insn1) == rd(insn2) && rd(insn2) == rj(insn2) && is_ld_d +} + +/// Relocations that only guide relaxation and never relocate anything. +fn is_marker(r_type: u32) -> bool { + matches!( + r_type, + R_NONE | R_LARCH_RELAX | R_LARCH_MARK_LA | R_LARCH_MARK_PCREL | R_LARCH_ALIGN + ) +} + +const PLT_ENTRY_64: [u32; 4] = [ + 0x1a00_000f, // pcalau12i $t3, %pc_hi20(func@.got.plt) + 0x28c0_01ef, // ld.d $t3, $t3, %lo12(func@.got.plt) + 0x4c00_01ed, // jirl $t1, $t3, 0 + 0x002a_0000, // break +]; + +const PLT_ENTRY_32: [u32; 4] = [ + 0x1a00_000f, // pcalau12i $t3, %pc_hi20(func@.got.plt) + 0x2880_01ef, // ld.w $t3, $t3, %lo12(func@.got.plt) + 0x4c00_01ed, // jirl $t1, $t3, 0 + 0x002a_0000, // break +]; + +fn write_insns(buf: &mut [u8], insns: &[u32]) { + for (i, &insn) in insns.iter().enumerate() { + set_insn(&mut buf[i * 4..], insn); + } +} + +impl LoongArchTarget { + /// Writes a PLT-style entry that jumps to the address stored at `got`. + fn write_plt_stub(buf: &mut [u8], got: u64, plt: u64) { + write_insns(buf, if IS_64 { &PLT_ENTRY_64 } else { &PLT_ENTRY_32 }); + write_j20(buf, hi20(got, plt)); + write_k12(&mut buf[4..], got); + } +} + +impl Arch for LoongArchTarget { + const NAME: &'static str = if IS_64 { "loongarch64" } else { "loongarch32" }; + const FAMILY: Family = Family::LoongArch; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_LOONGARCH; + const PLT_HDR_SIZE: u64 = 32; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 16; + const TRAP: &'static [u8] = &[0x00, 0x00, 0x2a, 0x00]; // break 0 + + const R_COPY: u32 = R_LARCH_COPY; + const R_GLOB_DAT: u32 = if IS_64 { R_LARCH_64 } else { R_LARCH_32 }; + const R_JUMP_SLOT: u32 = R_LARCH_JUMP_SLOT; + const R_ABS: u32 = if IS_64 { R_LARCH_64 } else { R_LARCH_32 }; + const R_RELATIVE: u32 = R_LARCH_RELATIVE; + const R_IRELATIVE: Option = Some(R_LARCH_IRELATIVE); + const R_DTPOFF: u32 = if IS_64 { + R_LARCH_TLS_DTPREL64 + } else { + R_LARCH_TLS_DTPREL32 + }; + const R_TPOFF: u32 = if IS_64 { + R_LARCH_TLS_TPREL64 + } else { + R_LARCH_TLS_TPREL32 + }; + const R_DTPMOD: u32 = if IS_64 { + R_LARCH_TLS_DTPMOD64 + } else { + R_LARCH_TLS_DTPMOD32 + }; + const R_TLSDESC: Option = Some(if IS_64 { + R_LARCH_TLS_DESC64 + } else { + R_LARCH_TLS_DESC32 + }); + const R_FUNCALL: &'static [u32] = &[R_LARCH_B26, R_LARCH_CALL36]; + + fn rel_to_string(r_type: u32) -> String { + loongarch_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN_64: [u32; 8] = [ + 0x1a00_000e, // pcalau12i $t2, %pc_hi20(.got.plt) + 0x0011_bdad, // sub.d $t1, $t1, $t3 + 0x28c0_01cf, // ld.d $t3, $t2, %lo12(.got.plt) # _dl_runtime_resolve + 0x02ff_51ad, // addi.d $t1, $t1, -44 # .plt entry + 0x02c0_01cc, // addi.d $t0, $t2, %lo12(.got.plt) # &.got.plt + 0x0045_05ad, // srli.d $t1, $t1, 1 # .plt entry offset + 0x28c0_218c, // ld.d $t0, $t0, 8 # link map + 0x4c00_01e0, // jr $t3 + ]; + const INSN_32: [u32; 8] = [ + 0x1a00_000e, // pcalau12i $t2, %pc_hi20(.got.plt) + 0x0011_3dad, // sub.w $t1, $t1, $t3 + 0x2880_01cf, // ld.w $t3, $t2, %lo12(.got.plt) # _dl_runtime_resolve + 0x02bf_51ad, // addi.w $t1, $t1, -44 # .plt entry + 0x0280_01cc, // addi.w $t0, $t2, %lo12(.got.plt) # &.got.plt + 0x0044_89ad, // srli.w $t1, $t1, 2 # .plt entry offset + 0x2880_118c, // ld.w $t0, $t0, 4 # link map + 0x4c00_01e0, // jr $t3 + ]; + let gotplt = ctx.gotplt.hdr.shdr.sh_addr; + let plt = ctx.plt.hdr.shdr.sh_addr; + write_insns(buf, if IS_64 { &INSN_64 } else { &INSN_32 }); + write_j20(buf, hi20(gotplt, plt)); + write_k12(&mut buf[8..], gotplt); + write_k12(&mut buf[16..], gotplt); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + Self::write_plt_stub(buf, sym.gotplt_addr(ctx), sym.plt_addr(ctx)); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + Self::write_plt_stub(buf, sym.got_pltgot_addr(ctx), sym.plt_addr(ctx)); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_LARCH_ADD6 => add_bits(loc, 6, val, false), + R_LARCH_ADD8 => add_bits(loc, 8, val, false), + R_LARCH_ADD16 => add_bits(loc, 16, val, false), + R_LARCH_ADD32 => add_bits(loc, 32, val, false), + R_LARCH_ADD64 => add_bits(loc, 64, val, false), + R_LARCH_SUB6 => add_bits(loc, 6, val, true), + R_LARCH_SUB8 => add_bits(loc, 8, val, true), + R_LARCH_SUB16 => add_bits(loc, 16, val, true), + R_LARCH_SUB32 => add_bits(loc, 32, val, true), + R_LARCH_SUB64 => add_bits(loc, 64, val, true), + R_LARCH_32_PCREL => { + eh_frame::check_range( + ctx, + isec, + rel, + val.wrapping_sub(p) as i64, + -(1 << 31), + 1 << 31, + ); + LittleEndian::write_u32(loc, val.wrapping_sub(p) as u32); + } + R_LARCH_64_PCREL => LittleEndian::write_u64(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if is_marker(rel.r_type) || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_LARCH_32 => { + if IS_64 { + scan_absrel(ctx, isec, sym, &rel); + } + } + R_LARCH_B26 | R_LARCH_PCALA_HI20 | R_LARCH_CALL36 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_LARCH_GOT_HI20 | R_LARCH_GOT_PC_HI20 => sym.add_flags(NEEDS_GOT), + R_LARCH_TLS_IE_HI20 | R_LARCH_TLS_IE_PC_HI20 => sym.add_flags(NEEDS_GOTTP), + R_LARCH_TLS_GD_PC_HI20 + | R_LARCH_TLS_LD_PC_HI20 + | R_LARCH_TLS_GD_HI20 + | R_LARCH_TLS_LD_HI20 => sym.add_flags(NEEDS_TLSGD), + R_LARCH_32_PCREL | R_LARCH_64_PCREL => scan_pcrel(ctx, isec, sym, &rel), + R_LARCH_TLS_LE_HI20 + | R_LARCH_TLS_LE_LO12 + | R_LARCH_TLS_LE64_LO20 + | R_LARCH_TLS_LE64_HI12 + | R_LARCH_TLS_LE_HI20_R + | R_LARCH_TLS_LE_LO12_R => check_tlsle(ctx, isec, sym, &rel), + R_LARCH_TLS_DESC_CALL => scan_tlsdesc(ctx, sym), + R_LARCH_64 + | R_LARCH_B16 + | R_LARCH_B21 + | R_LARCH_ABS_HI20 + | R_LARCH_ABS_LO12 + | R_LARCH_ABS64_LO20 + | R_LARCH_ABS64_HI12 + | R_LARCH_PCALA_LO12 + | R_LARCH_PCALA64_LO20 + | R_LARCH_PCALA64_HI12 + | R_LARCH_GOT_PC_LO12 + | R_LARCH_GOT64_PC_LO20 + | R_LARCH_GOT64_PC_HI12 + | R_LARCH_GOT_LO12 + | R_LARCH_GOT64_LO20 + | R_LARCH_GOT64_HI12 + | R_LARCH_TLS_IE_PC_LO12 + | R_LARCH_TLS_IE64_PC_LO20 + | R_LARCH_TLS_IE64_PC_HI12 + | R_LARCH_TLS_IE_LO12 + | R_LARCH_TLS_IE64_LO20 + | R_LARCH_TLS_IE64_HI12 + | R_LARCH_ADD6 + | R_LARCH_SUB6 + | R_LARCH_ADD8 + | R_LARCH_SUB8 + | R_LARCH_ADD16 + | R_LARCH_SUB16 + | R_LARCH_ADD32 + | R_LARCH_SUB32 + | R_LARCH_ADD64 + | R_LARCH_SUB64 + | R_LARCH_ADD_ULEB128 + | R_LARCH_SUB_ULEB128 + | R_LARCH_TLS_DESC_PC_HI20 + | R_LARCH_TLS_DESC_PC_LO12 + | R_LARCH_TLS_DESC_LD + | R_LARCH_TLS_LE_ADD_R => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let contents = isec.original_contents(file); + let got = ctx.got.hdr.shdr.sh_addr; + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if is_marker(rel.r_type) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let (removed, delta) = isec.removed_at(rel); + let r_offset = rel.r_offset - delta as u64; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + r_offset; + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + + // Unlike other psABIs, LoongArch uses the same relocation types + // for GOT entries of thread-local and regular symbols, so G + // refers to whichever the symbol has. + // + // LoongArch defines TLSLD relocations but doesn't really support + // TLSLD: GCC and LLVM emit identical code for the global- and + // local-dynamic models, so TLSLD relocations are treated as + // TLSGD ones. Fixing the compilers is out of the question by + // now; the way forward is TLSDESC. + let g = || { + let entry = if sym.has_tlsgd(&ctx.symbols) { + sym.tlsgd_addr(ctx) + } else { + sym.got_addr(ctx) + }; + entry.wrapping_sub(got) + }; + let got_entry = || got.wrapping_add(g()).wrapping_add(a); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i - 1, val, lo, hi); + let check_branch = |val: i64, lo: i64, hi: i64| { + check(val, lo, hi); + if val & 0b11 != 0 { + error!( + ctx, + "{}: misaligned symbol {sym} for relocation {}", + isec.display(file), + rel.type_name::() + ); + } + }; + let loc = &mut buf[r_offset as usize..]; + + match rel.r_type { + R_LARCH_32 => { + debug_assert!(IS_64); + LittleEndian::write_u32(loc, sa as u32); + } + R_LARCH_B16 => { + check_branch(pcrel as i64, -(1 << 17), 1 << 17); + write_k16(loc, pcrel >> 2); + } + R_LARCH_B21 => { + check_branch(pcrel as i64, -(1 << 22), 1 << 22); + write_d5k16(loc, pcrel >> 2); + } + R_LARCH_B26 => { + if !sym.is_remaining_undef_weak() { + check_branch(pcrel as i64, -(1 << 27), 1 << 27); + } + write_d10k16(loc, pcrel >> 2); + } + R_LARCH_ABS_LO12 => write_k12(loc, sa), + R_LARCH_ABS_HI20 => write_j20(loc, sa >> 12), + R_LARCH_ABS64_LO20 => write_j20(loc, sa >> 32), + R_LARCH_ABS64_HI12 => write_k12(loc, sa >> 52), + R_LARCH_PCALA_LO12 => { + // Contrary to the psABI, R_LARCH_PCALA_LO12 is sometimes + // applied to a JIRL, whose immediate is 16 bits rather + // than 12. GNU ld accepts that, so this does too. + if insn(loc) & 0xfc00_0000 == 0x4c00_0000 { + write_k16(loc, (sign_extend(sa, 12) >> 2) as u64); + } else { + write_k12(loc, sa); + } + } + R_LARCH_PCALA_HI20 => { + if removed == 0 { + write_j20(loc, hi20(sa, p)); + } else { + // pcalau12i + addi.d became a pcaddi: the high part + // vanishes and the low part turns into the pcaddi's + // PC-relative relocation. + debug_assert_eq!(removed, 4); + write_pcaddi(loc, pcrel >> 2); + i += 3; + } + } + R_LARCH_PCALA64_LO20 => write_j20(loc, higher20(sa, p)), + R_LARCH_PCALA64_HI12 => write_k12(loc, highest12(sa, p)), + R_LARCH_GOT_PC_LO12 => write_k12(loc, got_entry()), + R_LARCH_GOT_PC_HI20 => { + if removed == 0 { + // A GOT load of a symbol whose PC-relative address + // is a link-time constant becomes an address + // materialization: the ld.d turns into an addi.d. + if is_relaxable_got_load(ctx, isec, i - 1) + && is_int(compute_distance(ctx, sym, isec, rel), 32) + { + let reg = rd(insn(loc)); + set_insn(&mut loc[4..], 0x02c0_0000 | (reg << 5) | reg); // addi.d + write_j20(loc, hi20(sa, p)); + write_k12(&mut loc[4..], sa); + i += 3; + } else { + write_j20(loc, hi20(got_entry(), p)); + } + } else { + // pcalau12i + ld.d became a pcaddi. + debug_assert_eq!(removed, 4); + write_pcaddi(loc, pcrel >> 2); + i += 3; + } + } + R_LARCH_GOT64_PC_LO20 => write_j20(loc, higher20(got_entry(), p)), + R_LARCH_GOT64_PC_HI12 => write_k12(loc, highest12(got_entry(), p)), + R_LARCH_GOT_LO12 => write_k12(loc, got_entry()), + R_LARCH_GOT_HI20 => write_j20(loc, got_entry() >> 12), + R_LARCH_GOT64_LO20 => write_j20(loc, got_entry() >> 32), + R_LARCH_GOT64_HI12 => write_k12(loc, got_entry() >> 52), + R_LARCH_TLS_LE_LO12 => write_k12(loc, sa.wrapping_sub(ctx.tp_addr)), + R_LARCH_TLS_LE_HI20 => write_j20(loc, sa.wrapping_sub(ctx.tp_addr) >> 12), + R_LARCH_TLS_LE64_LO20 => write_j20(loc, sa.wrapping_sub(ctx.tp_addr) >> 32), + R_LARCH_TLS_LE64_HI12 => write_k12(loc, sa.wrapping_sub(ctx.tp_addr) >> 52), + R_LARCH_TLS_IE_PC_LO12 => write_k12(loc, sym.gottp_addr(ctx).wrapping_add(a)), + R_LARCH_TLS_IE_PC_HI20 => { + write_j20(loc, hi20(sym.gottp_addr(ctx).wrapping_add(a), p)) + } + R_LARCH_TLS_IE64_PC_LO20 => { + write_j20(loc, higher20(sym.gottp_addr(ctx).wrapping_add(a), p)) + } + R_LARCH_TLS_IE64_PC_HI12 => { + write_k12(loc, highest12(sym.gottp_addr(ctx).wrapping_add(a), p)) + } + R_LARCH_TLS_IE_LO12 => write_k12(loc, sym.gottp_addr(ctx).wrapping_add(a)), + R_LARCH_TLS_IE_HI20 => write_j20(loc, sym.gottp_addr(ctx).wrapping_add(a) >> 12), + R_LARCH_TLS_IE64_LO20 => write_j20(loc, sym.gottp_addr(ctx).wrapping_add(a) >> 32), + R_LARCH_TLS_IE64_HI12 => write_k12(loc, sym.gottp_addr(ctx).wrapping_add(a) >> 52), + R_LARCH_TLS_GD_PC_HI20 | R_LARCH_TLS_LD_PC_HI20 => { + let val = sym.tlsgd_addr(ctx).wrapping_add(a); + check(val.wrapping_sub(p) as i64, -(1 << 31), 1 << 31); + write_j20(loc, hi20(val, p)); + } + R_LARCH_TLS_GD_HI20 | R_LARCH_TLS_LD_HI20 => { + write_j20(loc, sym.tlsgd_addr(ctx).wrapping_add(a) >> 12) + } + R_LARCH_ADD6 => add_bits(loc, 6, sa, false), + R_LARCH_ADD8 => add_bits(loc, 8, sa, false), + R_LARCH_ADD16 => add_bits(loc, 16, sa, false), + R_LARCH_ADD32 => add_bits(loc, 32, sa, false), + R_LARCH_ADD64 => add_bits(loc, 64, sa, false), + R_LARCH_SUB6 => add_bits(loc, 6, sa, true), + R_LARCH_SUB8 => add_bits(loc, 8, sa, true), + R_LARCH_SUB16 => add_bits(loc, 16, sa, true), + R_LARCH_SUB32 => add_bits(loc, 32, sa, true), + R_LARCH_SUB64 => add_bits(loc, 64, sa, true), + R_LARCH_32_PCREL => { + check(pcrel as i64, -(1 << 31), 1 << 31); + LittleEndian::write_u32(loc, pcrel as u32); + } + R_LARCH_64_PCREL => LittleEndian::write_u64(loc, pcrel), + R_LARCH_CALL36 => { + if removed == 0 { + if !sym.is_remaining_undef_weak() { + check_branch(pcrel as i64, -(1 << 37) - 0x20000, (1 << 37) - 0x20000); + } + write_j20(loc, pcrel.wrapping_add(0x20000) >> 18); + write_k16(&mut loc[4..], pcrel >> 2); + } else { + // pcaddu18i + jirl became a B or a BL, depending on + // whether the jirl linked. + debug_assert_eq!(removed, 4); + let jirl = insn(&contents[rel.r_offset as usize + 4..]); + set_insn( + loc, + if rd(jirl) == 0 { + 0x5000_0000 + } else { + 0x5400_0000 + }, + ); + write_d10k16(loc, pcrel >> 2); + } + } + R_LARCH_ADD_ULEB128 => add_uleb(loc, sa, false), + R_LARCH_SUB_ULEB128 => add_uleb(loc, sa, true), + // TLSDESC materializes a TP-relative address in $a0 with + // + // pcalau12i $a0, 0 R_LARCH_TLS_DESC_PC_HI20 + // addi.[dw] $a0, $a0, 0 R_LARCH_TLS_DESC_PC_LO12 + // ld.d $ra, $a0, 0 R_LARCH_TLS_DESC_LD + // jirl $ra, $ra, 0 R_LARCH_TLS_DESC_CALL + // + // If the TP-relative address is a link-time constant, the + // first two instructions go away and the last two become + // + // lu12i.w $a0, foo@TPOFF + // addi.w $a0, $a0, foo@TPOFF + // + // or, for a small TP offset, the first three go away and + // the last becomes `ori $a0, $zero, foo@TPOFF`. If the + // address is known at process start, the last two become + // + // pcalau12i $a0, foo@GOTTP + // ld.[dw] $a0, $a0, foo@GOTTP + // + // Failing all that, the first two can still fold into one + // `pcaddi $a0, foo@GOTDESC` if the descriptor is near. + R_LARCH_TLS_DESC_PC_HI20 => { + if sym.has_tlsdesc(&ctx.symbols) && removed == 0 { + write_j20(loc, hi20(sym.tlsdesc_addr(ctx).wrapping_add(a), p)); + } + } + R_LARCH_TLS_DESC_PC_LO12 => { + if sym.has_tlsdesc(&ctx.symbols) && removed == 0 { + let dist = sym.tlsdesc_addr(ctx).wrapping_add(a).wrapping_sub(p) as i64; + if is_int(dist, 22) { + write_pcaddi(loc, (dist >> 2) as u64); + } else { + write_k12(loc, sym.tlsdesc_addr(ctx).wrapping_add(a)); + } + } + } + R_LARCH_TLS_DESC_LD => { + // The ld.d slot holds the first instruction of the IE or + // LE sequence, unless that sequence is a single one. + if sym.has_tlsdesc(&ctx.symbols) || removed == 4 { + // Nothing to do. + } else if sym.has_gottp(&ctx.symbols) { + set_insn(loc, 0x1a00_0004); // pcalau12i $a0, 0 + write_j20(loc, hi20(sym.gottp_addr(ctx).wrapping_add(a), p)); + } else { + set_insn(loc, 0x1400_0004); // lu12i.w $a0, 0 + write_j20(loc, sa.wrapping_add(0x800).wrapping_sub(ctx.tp_addr) >> 12); + } + } + R_LARCH_TLS_DESC_CALL => { + // The jirl slot holds the second instruction of the IE + // or LE sequence. + if sym.has_tlsdesc(&ctx.symbols) { + // Nothing to do. + } else if sym.has_gottp(&ctx.symbols) { + set_insn(loc, if IS_64 { 0x28c0_0084 } else { 0x2880_0084 }); // ld.[dw] $a0, $a0, 0 + write_k12(loc, sym.gottp_addr(ctx).wrapping_add(a)); + } else { + let val = sa.wrapping_sub(ctx.tp_addr) as i64; + set_insn( + loc, + if (0..0x1000).contains(&val) { + 0x0380_0004 + } else { + 0x0280_0084 + }, + ); // ori $a0, $zero, 0 / addi.w $a0, $a0, 0 + write_k12(loc, val as u64); + } + } + // lu12i.w + add.d + addi.d reach TP ± 2 GiB; a variable + // within 2 KiB of TP needs only the addi.d, then relative + // to $tp itself. + R_LARCH_TLS_LE_HI20_R => { + if removed == 0 { + write_j20(loc, sa.wrapping_add(0x800).wrapping_sub(ctx.tp_addr) >> 12); + } + } + R_LARCH_TLS_LE_LO12_R => { + let val = sa.wrapping_sub(ctx.tp_addr) as i64; + write_k12(loc, val as u64); + if is_int(val, 12) { + set_rj(loc, 2); // $tp + } + } + R_LARCH_TLS_LE_ADD_R | R_LARCH_64 => {} + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let tombstone = || isec.tombstone(ctx, sym, frag.map(|(f, _)| f)); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_LARCH_32 => LittleEndian::write_u32(loc, sa as u32), + R_LARCH_64 => LittleEndian::write_u64(loc, tombstone().unwrap_or(sa)), + R_LARCH_ADD6 => add_bits(loc, 6, sa, false), + R_LARCH_ADD8 => add_bits(loc, 8, sa, false), + R_LARCH_ADD16 => add_bits(loc, 16, sa, false), + R_LARCH_ADD32 => add_bits(loc, 32, sa, false), + R_LARCH_ADD64 => add_bits(loc, 64, sa, false), + R_LARCH_SUB6 => add_bits(loc, 6, sa, true), + R_LARCH_SUB8 => add_bits(loc, 8, sa, true), + R_LARCH_SUB16 => add_bits(loc, 16, sa, true), + R_LARCH_SUB32 => add_bits(loc, 32, sa, true), + R_LARCH_SUB64 => add_bits(loc, 64, sa, true), + R_LARCH_TLS_DTPREL32 => LittleEndian::write_u32( + loc, + tombstone().unwrap_or(sa.wrapping_sub(ctx.dtp_addr)) as u32, + ), + R_LARCH_TLS_DTPREL64 => LittleEndian::write_u64( + loc, + tombstone().unwrap_or(sa.wrapping_sub(ctx.dtp_addr)), + ), + R_LARCH_ADD_ULEB128 => add_uleb(loc, sa, false), + R_LARCH_SUB_ULEB128 => add_uleb(loc, sa, true), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, i: usize) -> u32 { + if !isec.is_alloc() { + return rel.r_type; + } + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let (removed, delta) = isec.removed_at(rel); + + // The low half of a pair whose high half folded into a pcaddi. + let folded_into_pcaddi = || { + i >= 2 + && matches!( + rels.at(i - 2).r_type, + R_LARCH_PCALA_HI20 | R_LARCH_GOT_PC_HI20 + ) + && rels.at(i - 2).r_offset + 4 == rel.r_offset + && isec.removed_at(&rels.at(i - 2)).0 != 0 + }; + + match rel.r_type { + R_LARCH_PCALA_HI20 | R_LARCH_GOT_PC_HI20 if removed != 0 => R_NONE, + R_LARCH_PCALA_LO12 | R_LARCH_GOT_PC_LO12 if folded_into_pcaddi() => R_LARCH_PCREL20_S2, + R_LARCH_CALL36 if removed != 0 => R_LARCH_B26, + R_LARCH_TLS_DESC_PC_HI20 if !sym.has_tlsdesc(&ctx.symbols) || removed != 0 => R_NONE, + R_LARCH_TLS_DESC_PC_LO12 => { + if !sym.has_tlsdesc(&ctx.symbols) { + R_NONE + } else { + let p = isec.addr(ctx) + rel.r_offset - delta as u64; + let dist = sym + .tlsdesc_addr(ctx) + .wrapping_add(rel.r_addend as u64) + .wrapping_sub(p) as i64; + if removed == 0 && is_int(dist, 22) { + R_LARCH_TLS_DESC_PCREL20_S2 + } else { + rel.r_type + } + } + } + R_LARCH_TLS_DESC_LD if !sym.has_tlsdesc(&ctx.symbols) => { + if removed == 4 { + R_NONE + } else if sym.has_gottp(&ctx.symbols) { + R_LARCH_TLS_IE_PC_HI20 + } else { + R_LARCH_TLS_LE_HI20 + } + } + R_LARCH_TLS_DESC_CALL if !sym.has_tlsdesc(&ctx.symbols) => { + if sym.has_gottp(&ctx.symbols) { + R_LARCH_TLS_IE_PC_LO12 + } else { + R_LARCH_TLS_LE_LO12 + } + } + R_LARCH_TLS_LE_HI20_R | R_LARCH_TLS_LE_ADD_R if removed != 0 => R_NONE, + _ => rel.r_type, + } + } + + fn shrink_section(ctx: &Context, isec: &InputSection) -> Vec { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let contents = isec.original_contents(file); + let mut deltas: Vec = Vec::new(); + let mut delta: i64 = 0; + + // Records that `d` bytes go away at relocation `r`. + fn record(deltas: &mut Vec, delta: &mut i64, r: &ElfRel, d: i64) { + *delta += d; + deltas.push(RelocDelta { + offset: r.r_offset, + delta: *delta, + }); + } + + for i in 0..rels.len() { + let r = &rels.at(i); + let sym = &ctx.symbols[file.base.symbols[r.r_sym as usize]]; + + // An R_LARCH_ALIGN refers to a run of NOPs, some or all of which + // are removed so that the instruction after them is aligned. A + // request for 2^n alignment refers to 2^n - 4 bytes of NOPs. + if r.r_type == R_LARCH_ALIGN { + // The upper bits of r_addend can also carry an upper limit + // of the alignment, allowing the following instruction not + // to be aligned at all. That looks like a spec bug and + // isn't supported. + let alignment = if r.r_sym != 0 { + if r.r_addend >> 8 != 0 { + fatal!( + ctx, + "{}: ternary R_LARCH_ALIGN is not supported: {i}", + isec.display(file) + ); + } + 1u64 << r.r_addend + } else { + let alignment = r.r_addend as u64 + 4; + if !alignment.is_power_of_two() { + fatal!( + ctx, + "{}: R_LARCH_ALIGN: invalid alignment requirement: {i}", + isec.display(file) + ); + } + alignment + }; + let p = isec.addr(ctx) + r.r_offset - delta as u64; + let desired = align_to(p, alignment); + let actual = p + alignment - 4; + if desired != actual { + record(&mut deltas, &mut delta, r, (actual - desired) as i64); + } + continue; + } + + // Other relaxations are optional. + if !ctx.args.relax || i + 1 == rels.len() || rels.at(i + 1).r_type != R_LARCH_RELAX { + continue; + } + + // Linker-synthesized symbols get their values only once the + // layout is fixed, so they're never relaxed against. + if sym.file() == ctx.internal_obj.map(FileId::Obj) { + continue; + } + + let mut remove = |d: i64| record(&mut deltas, &mut delta, r, d); + + match r.r_type { + // lu12i.w + add.d + addi.d reach TP ± 2 GiB. A variable + // within TP ± 2 KiB needs only `addi.d $t0, $tp, offset`. + R_LARCH_TLS_LE_HI20_R | R_LARCH_TLS_LE_ADD_R => { + let val = sym + .addr(ctx) + .wrapping_add(r.r_addend as u64) + .wrapping_sub(ctx.tp_addr) as i64; + if is_int(val, 12) { + remove(4); + } + } + // pcalau12i + addi.d materialize a PC-relative address with + // a 32-bit displacement; within ±2 MiB a pcaddi does. + R_LARCH_PCALA_HI20 => { + if i + 3 < rels.len() + && rels.at(i + 2).r_type == R_LARCH_PCALA_LO12 + && rels.at(i + 2).r_offset == r.r_offset + 4 + && rels.at(i + 3).r_type == R_LARCH_RELAX + { + let dist = compute_distance(ctx, sym, isec, r); + let insn1 = insn(&contents[r.r_offset as usize..]); + let insn2 = insn(&contents[r.r_offset as usize + 4..]); + let is_addi_d = insn2 & 0xffc0_0000 == 0x02c0_0000; + if dist & 0b11 == 0 + && is_int(dist, 22) + && is_addi_d + && rd(insn1) == rd(insn2) + && rd(insn2) == rj(insn2) + { + remove(4); + } + } + } + // pcaddu18i + jirl reach PC ± 128 GiB; within ±128 MiB a B + // or BL does, provided the jirl's link register is $zero + // or $ra. + R_LARCH_CALL36 => { + let dist = compute_distance(ctx, sym, isec, r); + let jirl = insn(&contents[r.r_offset as usize + 4..]); + if is_int(dist, 28) && (rd(jirl) == 0 || rd(jirl) == 1) { + remove(4); + } + } + // pcalau12i + ld.d load a symbol's address from the GOT; + // if that address is a link-time constant within ±2 MiB, a + // pcaddi materializes it directly. + R_LARCH_GOT_PC_HI20 => { + if is_relaxable_got_load(ctx, isec, i) { + let dist = compute_distance(ctx, sym, isec, r); + if is_int(dist, 22) && dist & 3 == 0 { + remove(4); + } + } + } + R_LARCH_TLS_DESC_PC_HI20 => { + if sym.has_tlsdesc(&ctx.symbols) { + let p = isec.addr(ctx) + r.r_offset; + let dist = sym + .tlsdesc_addr(ctx) + .wrapping_add(r.r_addend as u64) + .wrapping_sub(p) as i64; + if is_int(dist, 22) { + remove(4); + } + } else { + remove(4); + } + } + R_LARCH_TLS_DESC_PC_LO12 => { + if !sym.has_tlsdesc(&ctx.symbols) { + remove(4); + } + } + R_LARCH_TLS_DESC_LD + if !sym.has_tlsdesc(&ctx.symbols) && !sym.has_gottp(&ctx.symbols) => + { + let val = sym + .addr(ctx) + .wrapping_add(r.r_addend as u64) + .wrapping_sub(ctx.tp_addr) as i64; + if (0..0x1000).contains(&val) { + remove(4); + } + } + _ => {} + } + } + deltas + } +} diff --git a/src/arch/m68k.rs b/src/arch/m68k.rs new file mode 100644 index 000000000..6bb0dbc71 --- /dev/null +++ b/src/arch/m68k.rs @@ -0,0 +1,293 @@ +//! m68k, the Motorola 68000 series. +//! +//! Running a Unix-like system on m68k is a retro-computing hobby today, +//! but the processor was a popular choice for Unix workstations in the +//! '80s, including the early Suns and the Macintosh until 1994. For a +//! linker it is straightforward: a 32-bit big-endian CISC ISA whose +//! PC-relative addressing modes make position-independent code simpler +//! than on the contemporary i386. +//! +//! https://github.com/rui314/psabi/blob/main/m68k.pdf + +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct M68k; + +impl Layout for M68k { + type Endian = BigEndian; + const IS_64: bool = false; + const IS_RELA: bool = true; +} + +fn w32(loc: &mut [u8], v: u32) { + BigEndian::write_u32(loc, v); +} + +impl Arch for M68k { + const NAME: &'static str = "m68k"; + const FAMILY: Family = Family::M68k; + const PAGE_SIZE: u64 = 8192; + const E_MACHINE: u32 = EM_68K; + const PLT_HDR_SIZE: u64 = 18; + const PLT_SIZE: u64 = 14; + const PLTGOT_SIZE: u64 = 8; + const TRAP: &'static [u8] = &[0x4a, 0xfc]; // illegal + + const R_COPY: u32 = R_68K_COPY; + const R_GLOB_DAT: u32 = R_68K_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_68K_JMP_SLOT; + const R_ABS: u32 = R_68K_32; + const R_RELATIVE: u32 = R_68K_RELATIVE; + const R_DTPOFF: u32 = R_68K_TLS_DTPREL32; + const R_TPOFF: u32 = R_68K_TLS_TPREL32; + const R_DTPMOD: u32 = R_68K_TLS_DTPMOD32; + const R_FUNCALL: &'static [u32] = &[R_68K_PLT32]; + + fn rel_to_string(r_type: u32) -> String { + m68k_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u8; 18] = [ + 0x2f, 0x00, // move.l %d0, -(%sp) + 0x2f, 0x3b, 0x01, 0x70, 0, 0, 0, 0, // move.l (GOTPLT+4, %pc), -(%sp) + 0x4e, 0xfb, 0x01, 0x71, 0, 0, 0, 0, // jmp ([GOTPLT+8, %pc]) + ]; + buf[..18].copy_from_slice(&INSN); + let gotplt = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr); + w32(&mut buf[6..], gotplt as u32); + w32(&mut buf[14..], gotplt.wrapping_sub(4) as u32); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u8; 14] = [ + 0x20, 0x3c, 0, 0, 0, 0, // move.l PLT_OFFSET, %d0 + 0x4e, 0xfb, 0x01, 0x71, 0, 0, 0, 0, // jmp ([GOTPLT_ENTRY, %pc]) + ]; + buf[..14].copy_from_slice(&INSN); + w32( + &mut buf[2..], + sym.plt_idx(&ctx.symbols).unwrap() * ElfRel::size::() as u32, + ); + w32( + &mut buf[10..], + sym.gotplt_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(8) as u32, + ); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u8; 8] = [0x4e, 0xfb, 0x01, 0x71, 0, 0, 0, 0]; // jmp ([GOT_ENTRY, %pc]) + buf[..8].copy_from_slice(&INSN); + w32( + &mut buf[4..], + sym.got_pltgot_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(2) as u32, + ); + } + + fn apply_eh_reloc( + ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_68K_32 => w32(loc, val as u32), + R_68K_PC32 => w32(loc, val.wrapping_sub(p) as u32), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + error!(ctx, "{sym}: GNU ifunc symbol is not supported on m68k"); + } + + match rel.r_type { + R_68K_16 | R_68K_8 => scan_absrel(ctx, isec, sym, &rel), + R_68K_PC32 | R_68K_PC16 | R_68K_PC8 => scan_pcrel(ctx, isec, sym, &rel), + R_68K_GOTPCREL32 | R_68K_GOTPCREL16 | R_68K_GOTPCREL8 | R_68K_GOTOFF32 + | R_68K_GOTOFF16 | R_68K_GOTOFF8 => sym.add_flags(NEEDS_GOT), + R_68K_PLT32 | R_68K_PLT16 | R_68K_PLT8 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_68K_TLS_GD32 | R_68K_TLS_GD16 | R_68K_TLS_GD8 => sym.add_flags(NEEDS_TLSGD), + R_68K_TLS_LDM32 | R_68K_TLS_LDM16 | R_68K_TLS_LDM8 => { + ctx.needs_tlsld.store(true, Ordering::Relaxed) + } + R_68K_TLS_IE32 | R_68K_TLS_IE16 | R_68K_TLS_IE8 => sym.add_flags(NEEDS_GOTTP), + R_68K_TLS_LE32 | R_68K_TLS_LE16 | R_68K_TLS_LE8 => { + check_tlsle(ctx, isec, sym, &rel) + } + R_68K_32 | R_68K_TLS_LDO32 | R_68K_TLS_LDO16 | R_68K_TLS_LDO8 => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let got = ctx.got.hdr.shdr.sh_addr; + + for (i, rel) in isec.rels::(file).iter().enumerate() { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + + // The narrower fields come in unsigned and signed flavors. + let write32 = |buf: &mut [u8], val: u64| w32(&mut buf[off..], val as u32); + let write16 = |buf: &mut [u8], val: u64| { + check(val as i64, 0, 1 << 16); + BigEndian::write_u16(&mut buf[off..], val as u16); + }; + let write16s = |buf: &mut [u8], val: u64| { + check(val as i64, -(1 << 15), 1 << 15); + BigEndian::write_u16(&mut buf[off..], val as u16); + }; + let write8 = |buf: &mut [u8], val: u64| { + check(val as i64, 0, 1 << 8); + buf[off] = val as u8; + }; + let write8s = |buf: &mut [u8], val: u64| { + check(val as i64, -(1 << 7), 1 << 7); + buf[off] = val as u8; + }; + + match rel.r_type { + // Handled as an absolute relocation by the output section. + R_68K_32 => {} + R_68K_16 => write16(buf, sa), + R_68K_8 => write8(buf, sa), + R_68K_PC32 | R_68K_PLT32 => write32(buf, sa.wrapping_sub(p)), + R_68K_PC16 | R_68K_PLT16 => write16s(buf, sa.wrapping_sub(p)), + R_68K_PC8 | R_68K_PLT8 => write8s(buf, sa.wrapping_sub(p)), + R_68K_GOTPCREL32 => write32(buf, got.wrapping_add(a).wrapping_sub(p)), + R_68K_GOTPCREL16 => write16s(buf, got.wrapping_add(a).wrapping_sub(p)), + R_68K_GOTPCREL8 => write8s(buf, got.wrapping_add(a).wrapping_sub(p)), + R_68K_GOTOFF32 => write32(buf, g().wrapping_add(a)), + R_68K_GOTOFF16 => write16(buf, g().wrapping_add(a)), + R_68K_GOTOFF8 => write8(buf, g().wrapping_add(a)), + R_68K_TLS_GD32 => { + write32(buf, sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got)) + } + R_68K_TLS_GD16 => { + write16(buf, sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got)) + } + R_68K_TLS_GD8 => write8(buf, sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got)), + R_68K_TLS_LDM32 => write32( + buf, + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + ), + R_68K_TLS_LDM16 => write16( + buf, + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + ), + R_68K_TLS_LDM8 => write8( + buf, + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + ), + R_68K_TLS_LDO32 => write32(buf, sa.wrapping_sub(ctx.dtp_addr)), + R_68K_TLS_LDO16 => write16s(buf, sa.wrapping_sub(ctx.dtp_addr)), + R_68K_TLS_LDO8 => write8s(buf, sa.wrapping_sub(ctx.dtp_addr)), + R_68K_TLS_IE32 => { + write32(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got)) + } + R_68K_TLS_IE16 => { + write16(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got)) + } + R_68K_TLS_IE8 => write8(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got)), + R_68K_TLS_LE32 => write32(buf, sa.wrapping_sub(ctx.tp_addr)), + R_68K_TLS_LE16 => write16(buf, sa.wrapping_sub(ctx.tp_addr)), + R_68K_TLS_LE8 => write8(buf, sa.wrapping_sub(ctx.tp_addr)), + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let tombstone = isec.tombstone(ctx, sym, frag.map(|(f, _)| f)); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_68K_32 => w32(loc, tombstone.unwrap_or(sa) as u32), + R_68K_TLS_LDO32 => w32( + loc, + tombstone.unwrap_or(sa.wrapping_sub(ctx.dtp_addr)) as u32, + ), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs new file mode 100644 index 000000000..23b654ef6 --- /dev/null +++ b/src/arch/mod.rs @@ -0,0 +1,399 @@ +//! Target architecture descriptions. +//! +//! Every target is a zero-sized marker type implementing [`Arch`]. The +//! trait carries the constants that vary between targets, such as the +//! relocation types used for dynamic linking, and the code generation +//! hooks that are inherently target-specific: relocation scanning and +//! application, and the PLT stubs. + +pub mod arm32; +pub mod arm64; +pub mod i386; +pub mod loongarch; +pub mod m68k; +pub mod ppc32; +pub mod ppc64v1; +pub mod ppc64v2; +pub mod riscv; +pub mod s390x; +pub mod sh4; +pub mod sparc64; +pub mod x86_64; + +pub use arm32::{Arm32, Arm32Be}; +pub use arm64::{Arm64, Arm64Be}; +pub use i386::I386; +pub use loongarch::{LoongArch32, LoongArch64}; +pub use m68k::M68k; +pub use ppc32::Ppc32; +pub use ppc64v1::Ppc64V1; +pub use ppc64v2::Ppc64V2; +pub use riscv::{Riscv32, Riscv32Be, Riscv64, Riscv64Be}; +pub use s390x::S390x; +pub use sh4::{Sh4, Sh4Be}; +pub use sparc64::Sparc64; +pub use x86_64::X86_64; + +use crate::context::Context; +use crate::elf::{ElfRel, Endian, Layout}; +use crate::input_sections::{InputSection, RelocDelta}; +use crate::symbol::Symbol; +use crate::thunks::Thunk; + +/// Coarse target families, for the few places where generic code needs +/// target-specific behavior that doesn't warrant a trait hook. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Family { + X86_64, + I386, + Arm64, + Arm32, + RiscV, + Ppc32, + Ppc64V1, + Ppc64V2, + S390x, + Sparc64, + M68k, + Sh4, + LoongArch, +} + +/// Sizes of range extension thunks, for targets whose branch instructions +/// can't reach the whole address space. +#[derive(Clone, Copy, Debug)] +pub struct ThunkLayout { + pub header_size: u64, + pub entry_size: u64, +} + +pub trait Arch: Layout { + const NAME: &'static str; + const FAMILY: Family; + const PAGE_SIZE: u64; + const E_MACHINE: u32; + const PLT_HDR_SIZE: u64; + const PLT_SIZE: u64; + const PLTGOT_SIZE: u64; + const THUNK: Option = None; + const SFRAME_ABI: Option = None; + + /// An instruction that traps, used to fill padding in executable sections. + const TRAP: &'static [u8]; + + const R_COPY: u32; + const R_GLOB_DAT: u32; + const R_JUMP_SLOT: u32; + const R_ABS: u32; + const R_RELATIVE: u32; + const R_DTPOFF: u32; + const R_TPOFF: u32; + const R_DTPMOD: u32; + const R_IRELATIVE: Option = None; + const R_TLSDESC: Option = None; + const R_SFRAME: Option = None; + + /// Relocation types that denote a function call. + const R_FUNCALL: &'static [u32]; + + const IS_LITTLE_ENDIAN: bool = Self::Endian::IS_LITTLE; + const SUPPORTS_IFUNC: bool = Self::R_IRELATIVE.is_some(); + const SUPPORTS_TLSDESC: bool = Self::R_TLSDESC.is_some(); + const SUPPORTS_SFRAME: bool = Self::SFRAME_ABI.is_some(); + const NEEDS_THUNK: bool = Self::THUNK.is_some(); + + const IS_X86: bool = matches!(Self::FAMILY, Family::X86_64 | Family::I386); + const IS_ARM: bool = matches!(Self::FAMILY, Family::Arm64 | Family::Arm32); + const IS_PPC: bool = matches!( + Self::FAMILY, + Family::Ppc32 | Family::Ppc64V1 | Family::Ppc64V2 + ); + const IS_PPC64: bool = matches!(Self::FAMILY, Family::Ppc64V1 | Family::Ppc64V2); + const IS_SPARC: bool = matches!(Self::FAMILY, Family::Sparc64); + const IS_RISCV: bool = matches!(Self::FAMILY, Family::RiscV); + const IS_LOONGARCH: bool = matches!(Self::FAMILY, Family::LoongArch); + + /// The maximum distance a direct branch can cover, for targets that need + /// range extension thunks. A small safety margin is subtracted so that + /// the exact branch origin, which differs slightly between targets, + /// doesn't matter. + fn branch_distance() -> i64 { + let bits = match Self::FAMILY { + Family::Arm64 => 27, + Family::Arm32 => 24, + _ => 25, + }; + (1 << bits) - 32 + } + + /// Returns the name of a relocation type, for diagnostics. + fn rel_to_string(r_type: u32) -> String; + + /// Rewrites input sections the compiler left half-baked, before + /// garbage collection. PPC64 ELFv1 dissolves the input `.opd` + /// sections here. + fn rewrite_input_sections(_ctx: &mut Context) {} + + /// Marks symbols that need target-specific synthetic entries, before + /// relocations are scanned. + fn scan_symbols(_ctx: &mut Context) {} + + /// Machine-specific `e_flags` of the output file. + fn eflags(_ctx: &Context) -> u32 { + 0 + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]); + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol); + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol); + + /// Applies a relocation in a reconstructed `.eh_frame` section. `loc` is + /// the relocated location, `p` its address and `val` the resolved value. + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ); + + /// Scans the relocations of an allocated section to find symbols that + /// need GOT, PLT or dynamic relocation entries. + fn scan_relocations(ctx: &Context, isec: &InputSection); + + /// Applies relocations to a copy of an allocated section's contents. + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]); + + /// Applies relocations to a copy of a non-allocated section's contents. + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]); + + /// The type the `i`th relocation of a section has in the output for + /// `--emit-relocs`. A relaxation that rewrote an instruction may + /// change it. + fn emitted_rel_type( + _ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + _i: usize, + ) -> u32 { + rel.r_type + } + + /// Whether a call must go through a thunk however close its target + /// is: a processor mode switch on ARM32, or TOC setup on PowerPC. + fn always_needs_thunk(_ctx: &Context, _sym: &Symbol, _rel: &ElfRel) -> bool { + false + } + + /// Lays out the entries of a thunk placed at `addr` once addresses are + /// known: the offset of each entry, followed by the thunk's size. + /// Entries have a fixed size unless the target says otherwise. + fn thunk_offsets(_ctx: &Context, thunk: &Thunk, _addr: u64) -> Vec { + thunk.fixed_offsets::() + } + + /// Writes the code of a thunk placed at `addr`. + fn write_thunk(_ctx: &Context, _thunk: &Thunk, _addr: u64, _buf: &mut [u8]) { + unreachable!("{} has no range extension thunks", Self::NAME) + } + + /// Finds the bytes linker relaxation can remove from an executable + /// section, for targets whose branches are shortened by the linker + /// (RISC-V, LoongArch). The result lists the cumulative number of + /// bytes removed up to each relocation that shrinks. + fn shrink_section(_ctx: &Context, _isec: &InputSection) -> Vec { + Vec::new() + } + + /// Rewrites the output image once every section is copied: big-endian + /// ARM32 converts code to BE8 form here, and x86-64 removes unneeded + /// ENDBR instructions. + fn finish_output(_ctx: &Context, _buf: &mut [u8]) {} + + /// Writes an addend into a relocated location, for REL-type targets + /// producing relocatable output. + fn write_addend(_loc: &mut [u8], _val: i64, _rel: &ElfRel) {} + + /// The addend of a relocation. REL-type targets store it in the + /// relocated location, given as `loc`. + fn get_addend(_loc: &[u8], rel: &ElfRel) -> i64 { + rel.r_addend + } +} + +/// A target that mold knows about, whether or not this build can link +/// for it. +#[derive(Clone, Copy, Debug)] +pub struct TargetInfo { + pub name: &'static str, + pub e_machine: u32, + pub is_64: bool, + pub is_little_endian: bool, +} + +pub const TARGETS: &[TargetInfo] = &[ + TargetInfo { + name: "x86_64", + e_machine: crate::elf::EM_X86_64, + is_64: true, + is_little_endian: true, + }, + TargetInfo { + name: "i386", + e_machine: crate::elf::EM_386, + is_64: false, + is_little_endian: true, + }, + TargetInfo { + name: "arm64", + e_machine: crate::elf::EM_AARCH64, + is_64: true, + is_little_endian: true, + }, + TargetInfo { + name: "arm64be", + e_machine: crate::elf::EM_AARCH64, + is_64: true, + is_little_endian: false, + }, + TargetInfo { + name: "arm32", + e_machine: crate::elf::EM_ARM, + is_64: false, + is_little_endian: true, + }, + TargetInfo { + name: "arm32be", + e_machine: crate::elf::EM_ARM, + is_64: false, + is_little_endian: false, + }, + TargetInfo { + name: "riscv64", + e_machine: crate::elf::EM_RISCV, + is_64: true, + is_little_endian: true, + }, + TargetInfo { + name: "riscv64be", + e_machine: crate::elf::EM_RISCV, + is_64: true, + is_little_endian: false, + }, + TargetInfo { + name: "riscv32", + e_machine: crate::elf::EM_RISCV, + is_64: false, + is_little_endian: true, + }, + TargetInfo { + name: "riscv32be", + e_machine: crate::elf::EM_RISCV, + is_64: false, + is_little_endian: false, + }, + TargetInfo { + name: "ppc32", + e_machine: crate::elf::EM_PPC, + is_64: false, + is_little_endian: false, + }, + TargetInfo { + name: "ppc64v1", + e_machine: crate::elf::EM_PPC64, + is_64: true, + is_little_endian: false, + }, + TargetInfo { + name: "ppc64v2", + e_machine: crate::elf::EM_PPC64, + is_64: true, + is_little_endian: true, + }, + TargetInfo { + name: "s390x", + e_machine: crate::elf::EM_S390X, + is_64: true, + is_little_endian: false, + }, + TargetInfo { + name: "sparc64", + e_machine: crate::elf::EM_SPARC64, + is_64: true, + is_little_endian: false, + }, + TargetInfo { + name: "m68k", + e_machine: crate::elf::EM_68K, + is_64: false, + is_little_endian: false, + }, + TargetInfo { + name: "sh4", + e_machine: crate::elf::EM_SH, + is_64: false, + is_little_endian: true, + }, + TargetInfo { + name: "sh4be", + e_machine: crate::elf::EM_SH, + is_64: false, + is_little_endian: false, + }, + TargetInfo { + name: "loongarch64", + e_machine: crate::elf::EM_LOONGARCH, + is_64: true, + is_little_endian: true, + }, + TargetInfo { + name: "loongarch32", + e_machine: crate::elf::EM_LOONGARCH, + is_64: false, + is_little_endian: true, + }, +]; + +/// Maps an `-m` emulation name to a target name. +pub fn emulation_to_target(emulation: &str) -> Option<&'static str> { + Some(match emulation { + "elf_x86_64" => "x86_64", + "elf_i386" => "i386", + "aarch64elf" | "aarch64linux" => "arm64", + "aarch64elfb" | "aarch64linuxb" => "arm64be", + "armelf_linux_eabi" => "arm32", + "armelfb_linux_eabi" => "arm32be", + "elf64lriscv" => "riscv64", + "elf64briscv" => "riscv64be", + "elf32lriscv" => "riscv32", + "elf32briscv" => "riscv32be", + "elf32ppc" | "elf32ppclinux" => "ppc32", + "elf64ppc" => "ppc64v1", + "elf64lppc" => "ppc64v2", + "elf64_s390" => "s390x", + "elf64_sparc" => "sparc64", + "m68kelf" => "m68k", + "shlelf" | "shlelf_linux" => "sh4", + "shelf" | "shelf_linux" => "sh4be", + "elf64loongarch" => "loongarch64", + "elf32loongarch" => "loongarch32", + _ => return None, + }) +} + +/// The relocation type to emit for `--emit-relocs`, accounting for +/// relaxations that changed the relocation's meaning. Relocatable output +/// applies no relaxation, so its relocations pass through unchanged. +pub fn emitted_rel_type( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + i: usize, +) -> u32 { + if ctx.args.relocatable { + rel.r_type + } else { + E::emitted_rel_type(ctx, isec, rel, i) + } +} diff --git a/src/arch/ppc32.rs b/src/arch/ppc32.rs new file mode 100644 index 000000000..4785d1d28 --- /dev/null +++ b/src/arch/ppc32.rs @@ -0,0 +1,384 @@ +//! PowerPC, 32-bit. +//! +//! PPC32 is a big-endian RISC ISA with 32 general-purpose registers, of +//! which `r0`, `r11` and `r12` are reserved for the linker's use in PLT +//! entries and thunks. The link register LR holds return addresses and +//! CTR is a branch target register. +//! +//! What complicates the psABI is the lack of PC-relative loads and +//! stores. A position-independent function finds its own address with +//! +//! ```text +//! mflr r0 // save the return address +//! bcl 20, 31, 4 // "call" the next instruction +//! mflr r12 // the return address is our own address +//! mtlr r0 // restore the return address +//! ``` +//! +//! and an object compiled with `-fPIC` has a `.got2` section holding the +//! addresses of the objects it refers to. A PIC function sets `r30` to +//! its own file's `.got2 + 0x8000` so that any of those addresses is one +//! 16-bit-offset load away. Each input file has its own `.got2`, so `r30` +//! means different things in functions from different files; GNU ld +//! exploits it anyway with per-file PLTs, but the PLT here simply +//! doesn't depend on `r30`. +//! +//! https://github.com/rui314/psabi/blob/main/ppc32.pdf + +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family, ThunkLayout}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::thunks::Thunk; +use crate::util::{bits, is_int}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Ppc32; + +impl Layout for Ppc32 { + type Endian = BigEndian; + const IS_64: bool = false; + const IS_RELA: bool = true; +} + +fn lo(x: u64) -> u64 { + x & 0xffff +} + +fn hi(x: u64) -> u64 { + x >> 16 +} + +fn ha(x: u64) -> u64 { + x.wrapping_add(0x8000) >> 16 +} + +fn higha(x: u64) -> u64 { + ha(x) & 0xffff +} + +fn w16(loc: &mut [u8], v: u64) { + BigEndian::write_u16(loc, v as u16); +} + +fn w32(loc: &mut [u8], v: u64) { + BigEndian::write_u32(loc, v as u32); +} + +fn or32(loc: &mut [u8], v: u64) { + let cur = BigEndian::read_u32(loc); + BigEndian::write_u32(loc, cur | v as u32); +} + +fn write_insns(buf: &mut [u8], insns: &[u32]) { + for (i, &insn) in insns.iter().enumerate() { + BigEndian::write_u32(&mut buf[i * 4..], insn); + } +} + +/// A PLT entry, also used as a thunk to a PLT symbol: it materializes +/// its own address, loads the destination from the GOT entry at a +/// known offset and jumps there. +const PLT_ENTRY: [u32; 9] = [ + 0x7c08_02a6, // mflr r0 + 0x429f_0005, // bcl 20, 31, 4 + 0x7d88_02a6, // mflr r12 + 0x7c08_03a6, // mtlr r0 + 0x3d6c_0000, // addis r11, r12, OFFSET@higha + 0x396b_0000, // addi r11, r11, OFFSET@lo + 0x818b_0000, // lwz r12, 0(r11) + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr +]; + +/// Writes a PLT entry that loads its destination from `got`. +fn write_plt_like(buf: &mut [u8], got: u64, entry_addr: u64) { + write_insns(buf, &PLT_ENTRY); + let offset = got.wrapping_sub(entry_addr).wrapping_sub(8); + or32(&mut buf[16..], higha(offset)); + or32(&mut buf[20..], lo(offset)); +} + +impl Arch for Ppc32 { + const NAME: &'static str = "ppc32"; + const FAMILY: Family = Family::Ppc32; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_PPC; + const PLT_HDR_SIZE: u64 = 64; + const PLT_SIZE: u64 = 36; + const PLTGOT_SIZE: u64 = 36; + const THUNK: Option = Some(ThunkLayout { + header_size: 0, + entry_size: 36, + }); + const TRAP: &'static [u8] = &[0x7f, 0xe0, 0x00, 0x08]; // trap + + const R_COPY: u32 = R_PPC_COPY; + const R_GLOB_DAT: u32 = R_PPC_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_PPC_JMP_SLOT; + const R_ABS: u32 = R_PPC_ADDR32; + const R_RELATIVE: u32 = R_PPC_RELATIVE; + const R_IRELATIVE: Option = Some(R_PPC_IRELATIVE); + const R_DTPOFF: u32 = R_PPC_DTPREL32; + const R_TPOFF: u32 = R_PPC_TPREL32; + const R_DTPMOD: u32 = R_PPC_DTPMOD32; + const R_FUNCALL: &'static [u32] = &[R_PPC_REL24, R_PPC_PLTREL24, R_PPC_LOCAL24PC]; + + fn rel_to_string(r_type: u32) -> String { + ppc32_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u32; 16] = [ + // Get the address of this PLT section. + 0x7c08_02a6, // mflr r0 + 0x429f_0005, // bcl 20, 31, 4 + 0x7d88_02a6, // 1: mflr r12 + 0x7c08_03a6, // mtlr r0 + // Compute the runtime address of GOTPLT+12. + 0x3d8c_0000, // addis r12, r12, (GOTPLT - 1b)@higha + 0x398c_0000, // addi r12, r12, (GOTPLT - 1b)@lo + // Compute the PLT entry offset. + 0x7d6c_5850, // sub r11, r11, r12 + 0x1d6b_0003, // mulli r11, r11, 3 + // Load GOTPLT[2] and branch to GOTPLT[1]. + 0x800c_fff8, // lwz r0, -8(r12) + 0x7c09_03a6, // mtctr r0 + 0x818c_fffc, // lwz r12, -4(r12) + 0x4e80_0420, // bctr + 0x6000_0000, // nop + 0x6000_0000, // nop + 0x6000_0000, // nop + 0x6000_0000, // nop + ]; + write_insns(buf, &INSN); + let offset = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr) + .wrapping_add(4); + or32(&mut buf[16..], higha(offset)); + or32(&mut buf[20..], lo(offset)); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_plt_like(buf, sym.gotplt_addr(ctx), sym.plt_addr(ctx)); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_plt_like(buf, sym.got_pltgot_addr(ctx), sym.plt_addr(ctx)); + } + + fn apply_eh_reloc( + ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_PPC_ADDR32 => w32(loc, val), + R_PPC_REL32 => w32(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_PPC_ADDR14 | R_PPC_ADDR16 | R_PPC_UADDR16 | R_PPC_ADDR16_LO | R_PPC_ADDR16_HI + | R_PPC_ADDR16_HA | R_PPC_ADDR24 | R_PPC_ADDR30 => { + scan_absrel(ctx, isec, sym, &rel) + } + R_PPC_REL14 | R_PPC_REL16 | R_PPC_REL16_LO | R_PPC_REL16_HI | R_PPC_REL16_HA + | R_PPC_REL32 => scan_pcrel(ctx, isec, sym, &rel), + R_PPC_GOT16 | R_PPC_GOT16_LO | R_PPC_GOT16_HI | R_PPC_GOT16_HA | R_PPC_PLT16_LO + | R_PPC_PLT16_HI | R_PPC_PLT16_HA | R_PPC_PLT32 => sym.add_flags(NEEDS_GOT), + R_PPC_REL24 | R_PPC_PLTREL24 | R_PPC_PLTREL32 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_PPC_GOT_TLSGD16 => sym.add_flags(NEEDS_TLSGD), + R_PPC_GOT_TLSLD16 => ctx.needs_tlsld.store(true, Ordering::Relaxed), + R_PPC_GOT_TPREL16 => sym.add_flags(NEEDS_GOTTP), + R_PPC_TPREL16_LO | R_PPC_TPREL16_HI | R_PPC_TPREL16_HA => { + check_tlsle(ctx, isec, sym, &rel) + } + R_PPC_ADDR32 | R_PPC_UADDR32 | R_PPC_LOCAL24PC | R_PPC_TLS | R_PPC_TLSGD + | R_PPC_TLSLD | R_PPC_DTPREL16_LO | R_PPC_DTPREL16_HI | R_PPC_DTPREL16_HA + | R_PPC_PLTSEQ | R_PPC_PLTCALL => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let got = ctx.got.hdr.shdr.sh_addr; + let got2 = file + .got2 + .map_or(0, |shndx| file.section_at(shndx).addr(ctx)); + + for rel in isec.rels::(file) { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + // PLT16/PLT32 relocations are relative to the file's .got2. + let plt = || sym.got_addr(ctx).wrapping_sub(a).wrapping_sub(got2); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_PPC_ADDR14 => or32(loc, bits(sa, 15, 2) << 2), + R_PPC_ADDR16 | R_PPC_UADDR16 | R_PPC_ADDR16_LO => w16(loc, lo(sa)), + R_PPC_ADDR16_HI => w16(loc, hi(sa)), + R_PPC_ADDR16_HA => w16(loc, ha(sa)), + R_PPC_ADDR24 => or32(loc, bits(sa, 25, 2) << 2), + R_PPC_ADDR30 => or32(loc, bits(sa, 31, 2) << 2), + R_PPC_PLT16_LO => w16(loc, lo(plt())), + R_PPC_PLT16_HI => w16(loc, hi(plt())), + R_PPC_PLT16_HA => w16(loc, ha(plt())), + R_PPC_PLT32 => w32(loc, plt()), + R_PPC_REL14 => or32(loc, bits(pcrel, 15, 2) << 2), + R_PPC_REL16 | R_PPC_REL16_LO => w16(loc, lo(pcrel)), + R_PPC_REL16_HI => w16(loc, hi(pcrel)), + R_PPC_REL16_HA => w16(loc, ha(pcrel)), + R_PPC_REL24 | R_PPC_LOCAL24PC => { + let mut val = pcrel as i64; + if !is_int(val, 26) { + val = sym.thunk_addr(ctx, p).wrapping_sub(p) as i64; + } + or32(loc, bits(val as u64, 25, 2) << 2); + } + R_PPC_PLTREL24 => { + let mut val = s.wrapping_sub(p) as i64; + if sym.has_plt(&ctx.symbols) || !is_int(val, 26) { + val = sym.thunk_addr(ctx, p).wrapping_sub(p) as i64; + } + or32(loc, bits(val as u64, 25, 2) << 2); + } + R_PPC_REL32 | R_PPC_PLTREL32 => w32(loc, pcrel), + R_PPC_GOT16 | R_PPC_GOT16_LO => w16(loc, lo(g().wrapping_add(a))), + R_PPC_GOT16_HI => w16(loc, hi(g().wrapping_add(a))), + R_PPC_GOT16_HA => w16(loc, ha(g().wrapping_add(a))), + R_PPC_TPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.tp_addr))), + R_PPC_TPREL16_HI => w16(loc, hi(sa.wrapping_sub(ctx.tp_addr))), + R_PPC_TPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.tp_addr))), + R_PPC_DTPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC_DTPREL16_HI => w16(loc, hi(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC_DTPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC_GOT_TLSGD16 => w16(loc, sym.tlsgd_addr(ctx).wrapping_sub(got)), + R_PPC_GOT_TLSLD16 => w16(loc, ctx.got.tlsld_addr::().wrapping_sub(got)), + R_PPC_GOT_TPREL16 => w16(loc, sym.gottp_addr(ctx).wrapping_sub(got)), + R_PPC_ADDR32 | R_PPC_UADDR32 | R_PPC_TLS | R_PPC_TLSGD | R_PPC_TLSLD + | R_PPC_PLTSEQ | R_PPC_PLTCALL => {} + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let tombstone = isec.tombstone(ctx, sym, frag.map(|(f, _)| f)); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_PPC_ADDR32 => w32(loc, tombstone.unwrap_or(sa)), + R_PPC_DTPREL32 => w32(loc, tombstone.unwrap_or(sa.wrapping_sub(ctx.dtp_addr))), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + /// All PLT calls go through thunks. + fn always_needs_thunk(ctx: &Context, sym: &Symbol, _rel: &ElfRel) -> bool { + sym.has_plt(&ctx.symbols) + } + + fn write_thunk(ctx: &Context, thunk: &Thunk, addr: u64, buf: &mut [u8]) { + const LOCAL_THUNK: [u32; 9] = [ + // Get this thunk's address. + 0x7c08_02a6, // mflr r0 + 0x429f_0005, // bcl 20, 31, 4 + 0x7d88_02a6, // mflr r12 + 0x7c08_03a6, // mtlr r0 + // Materialize the destination's address in r11 and jump there. + 0x3d6c_0000, // addis r11, r12, OFFSET@higha + 0x396b_0000, // addi r11, r11, OFFSET@lo + 0x7d69_03a6, // mtctr r11 + 0x4e80_0420, // bctr + 0x6000_0000, // nop + ]; + + for (i, &id) in thunk.symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + let p = addr + thunk.offsets[i]; + let entry = &mut buf[thunk.offsets[i] as usize..][..36]; + + if sym.has_plt(&ctx.symbols) { + let got = if sym.has_got(&ctx.symbols) { + sym.got_addr(ctx) + } else { + sym.gotplt_addr(ctx) + }; + write_plt_like(entry, got, p); + } else { + write_insns(entry, &LOCAL_THUNK); + let val = sym.addr(ctx).wrapping_sub(p).wrapping_sub(8); + or32(&mut entry[16..], higha(val)); + or32(&mut entry[20..], lo(val)); + } + } + } +} diff --git a/src/arch/ppc64v1.rs b/src/arch/ppc64v1.rs new file mode 100644 index 000000000..ac7e65655 --- /dev/null +++ b/src/arch/ppc64v1.rs @@ -0,0 +1,609 @@ +//! PowerPC64 ELFv1, the ABI of big-endian 64-bit PowerPC systems. The +//! little-endian systems use ELFv2; see [`crate::arch::ppc64v2`]. +//! +//! Beyond endianness, ELFv1's distinctive feature is that a function +//! pointer refers not to the function's code but to a *function +//! descriptor*: a 24-byte structure holding the entry point, the value +//! `r2` must have while the function runs, and an unused environment +//! pointer. The reason is the same TOC-pointer scheme as in ELFv2: +//! position-independent code addresses data relative to `r2 = GOT + +//! 0x8000`, each module has its own GOT, so calling into another module +//! requires switching `r2`, and a caller through a pointer can only do +//! that if the pointer tells it the callee's `r2`. Calls through the PLT +//! restore `r2` themselves, so only indirect calls pay for it. +//! +//! The descriptors live in `.opd` ("official procedure descriptors"). +//! Just as a function can have a PLT or GOT address on other targets, a +//! function here can also have an OPD address, which is what +//! address-taking relocations resolve to. +//! +//! https://github.com/rui314/psabi/blob/main/ppc64v1.pdf + +use std::collections::HashMap; +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family, ThunkLayout}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::input_sections::{check_tlsle, InputSection, SectionRef}; +use crate::symbol::{ + AddrFlags, Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_PPC_OPD, NEEDS_TLSGD, +}; +use crate::thunks::Thunk; +use crate::util::{bits, is_int}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Ppc64V1; + +impl Layout for Ppc64V1 { + type Endian = BigEndian; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +fn lo(x: u64) -> u64 { + x & 0xffff +} + +fn hi(x: u64) -> u64 { + x >> 16 +} + +fn ha(x: u64) -> u64 { + x.wrapping_add(0x8000) >> 16 +} + +fn high(x: u64) -> u64 { + hi(x) & 0xffff +} + +fn higha(x: u64) -> u64 { + ha(x) & 0xffff +} + +fn r32(loc: &[u8]) -> u32 { + BigEndian::read_u32(loc) +} + +fn w16(loc: &mut [u8], v: u64) { + BigEndian::write_u16(loc, v as u16); +} + +fn w32(loc: &mut [u8], v: u64) { + BigEndian::write_u32(loc, v as u32); +} + +fn w64(loc: &mut [u8], v: u64) { + BigEndian::write_u64(loc, v); +} + +fn or16(loc: &mut [u8], v: u64) { + let cur = BigEndian::read_u16(loc); + BigEndian::write_u16(loc, cur | v as u16); +} + +fn or32(loc: &mut [u8], v: u64) { + let cur = r32(loc); + BigEndian::write_u32(loc, cur | v as u32); +} + +fn write_insns(buf: &mut [u8], insns: &[u32]) { + for (i, &insn) in insns.iter().enumerate() { + BigEndian::write_u32(&mut buf[i * 4..], insn); + } +} + +/// The address of the TOC pointer, `.got + 0x8000`. +fn toc(ctx: &Context) -> u64 { + ctx.symbols[ctx.syms.toc.expect("PPC64 has a .TOC. symbol")].addr(ctx) +} + +/// Compilers emit a descriptor for every function into `.opd`, as if +/// the output `.opd` could be linked like any other section. It can't: +/// +/// 1. Function symbols refer to `.opd`, which suits address-taking +/// relocations but not branches, which need the code address that +/// only the descriptor's contents reveal. +/// 2. Only functions whose addresses are taken need output descriptors; +/// copying the input sections would keep plenty of dead ones. +/// 3. Every function being reachable from `.opd` defeats graph-based +/// passes such as garbage collection and identical code folding. +/// +/// So this undoes the compiler's work: function symbols move from +/// `.opd` to their code, relocations against `.opd` are redirected to +/// those symbols, and the input `.opd` sections die. Descriptors are then +/// synthesized for the functions marked `NEEDS_PPC_OPD`, like PLT +/// entries. +pub fn rewrite_opd(ctx: &mut Context) { + let _t = ctx.timer("rewrite_opd"); + + for i in 0..ctx.objs.len() { + let Some(opd) = ctx.objs[i] + .input_sections() + .find(|s| s.name(&ctx.objs[i]) == ".opd") + .map(|s| SectionRef { + file: s.file, + shndx: s.shndx, + }) + else { + continue; + }; + ctx.objs[i].kill_section(opd.shndx as usize); + + let local_symbols = ctx.objs[i].base.symbols.clone(); + let rels_at: HashMap = ctx.objs[i] + .section_at(opd.shndx) + .rels::(&ctx.objs[i]) + .iter() + .map(|r| (r.r_offset, r)) + .collect(); + + // Move function symbols from .opd to their code. + let mut descriptors: Vec<(u64, u32)> = Vec::new(); // (offset in .opd, local symbol index) + for (idx, &id) in local_symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + if sym.file() != Some(FileId::Obj(opd.file)) + || sym.input_section() != Some(opd) + || !matches!(sym.ty(), STT_FUNC | STT_GNU_IFUNC) + { + continue; + } + let Some(rel) = rels_at.get(&sym.value) else { + fatal!( + ctx, + "{}: cannot find a relocation in .opd for {sym} at offset {:#x}", + ctx.objs[i], + sym.value + ); + }; + let target = &ctx.symbols[local_symbols[rel.r_sym as usize]]; + if target.ty() != STT_SECTION { + fatal!( + ctx, + "{}: bad relocation in .opd referring to {target}", + ctx.objs[i] + ); + } + let origin = target.origin_state(); + descriptors.push((sym.value, idx as u32)); + let sym = &mut ctx.symbols[id]; + sym.set_origin_state(origin); + sym.value = rel.r_addend as u64; + } + descriptors.sort_by_key(|&(offset, _)| offset); + + // Redirect relocations against .opd to the function symbols. + let refers_to_opd: Vec = local_symbols + .iter() + .map(|&id| ctx.symbols[id].input_section() == Some(opd)) + .collect(); + let mut unresolved = None; + let mut rewrites = Vec::new(); + for isec in ctx.objs[i] + .input_sections() + .filter(|s| s.is_alive() && s.shndx != opd.shndx) + { + let mut rels: Vec = isec.rels::(&ctx.objs[i]).iter().collect(); + let mut redirected = false; + for rel in &mut rels { + if !refers_to_opd[rel.r_sym as usize] { + continue; + } + match descriptors + .binary_search_by_key(&(rel.r_addend as u64), |&(offset, _)| offset) + { + Ok(n) => { + rel.r_sym = descriptors[n].1; + rel.r_addend = 0; + redirected = true; + } + Err(_) => unresolved = unresolved.or(Some((isec.name(&ctx.objs[i]), *rel))), + } + } + if redirected { + rewrites.push((isec.shndx, rels)); + } + } + for (shndx, rels) in rewrites { + ctx.objs[i].rels_mut::(shndx).set_all(&rels); + } + if let Some((name, rel)) = unresolved { + fatal!( + ctx, + "{}:({name}): cannot find a symbol in .opd for {} at offset {:#x}", + ctx.objs[i], + rel.type_name::(), + rel.r_addend + ); + } + } +} + +/// An exported function's dynamic symbol refers to its descriptor, and so +/// do the ELF header's entry point and the `DT_INIT`/`DT_FINI` routines. +pub fn scan_symbols(ctx: &mut Context) { + let _t = ctx.timer("scan_symbols"); + let needs_descriptor = |sym: &Symbol| sym.add_flags(NEEDS_PPC_OPD); + + for id in ctx.symbols.global_ids() { + let sym = &ctx.symbols[id]; + if matches!(sym.file(), Some(FileId::Obj(_))) + && sym.is_exported() + && matches!(sym.ty(), STT_FUNC | STT_GNU_IFUNC) + { + needs_descriptor(sym); + } + } + for id in [ctx.syms.entry, ctx.syms.init, ctx.syms.fini] { + let sym = &ctx.symbols[id]; + if !sym.is_imported() { + needs_descriptor(sym); + } + } +} + +impl Arch for Ppc64V1 { + const NAME: &'static str = "ppc64v1"; + const FAMILY: Family = Family::Ppc64V1; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_PPC64; + const PLT_HDR_SIZE: u64 = 44; + const PLT_SIZE: u64 = 8; + const PLTGOT_SIZE: u64 = 0; + const THUNK: Option = Some(ThunkLayout { + header_size: 0, + entry_size: 28, + }); + const TRAP: &'static [u8] = &[0x7f, 0xe0, 0x00, 0x08]; // trap + + const R_COPY: u32 = R_PPC64_COPY; + const R_GLOB_DAT: u32 = R_PPC64_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_PPC64_JMP_SLOT; + const R_ABS: u32 = R_PPC64_ADDR64; + const R_RELATIVE: u32 = R_PPC64_RELATIVE; + const R_IRELATIVE: Option = Some(R_PPC64_IRELATIVE); + const R_DTPOFF: u32 = R_PPC64_DTPREL64; + const R_TPOFF: u32 = R_PPC64_TPREL64; + const R_DTPMOD: u32 = R_PPC64_DTPMOD64; + const R_FUNCALL: &'static [u32] = &[R_PPC64_REL24, R_PPC64_REL24_NOTOC]; + + fn rel_to_string(r_type: u32) -> String { + ppc64_rel_to_string(r_type) + } + + fn rewrite_input_sections(ctx: &mut Context) { + rewrite_opd(ctx); + } + + fn scan_symbols(ctx: &mut Context) { + scan_symbols(ctx); + } + + /// `.plt` serves lazy symbol resolution only. All PLT calls go through + /// thunks, which read the target's descriptor from `.got.plt` and jump + /// there, so once a symbol is resolved the PLT is skipped. + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u32; 11] = [ + 0x7d88_02a6, // mflr r12 + 0x429f_0005, // bcl 20, 31, 4 + 0x7d68_02a6, // mflr r11 + 0x7d88_03a6, // mtlr r12 + 0x3d6b_0000, // addis r11, r11, GOTPLT_OFFSET@ha + 0x396b_0000, // addi r11, r11, GOTPLT_OFFSET@lo + 0xe98b_0000, // ld r12,0(r11) + 0xe84b_0008, // ld r2,8(r11) + 0x7d89_03a6, // mtctr r12 + 0xe96b_0010, // ld r11,16(r11) + 0x4e80_0420, // bctr + ]; + write_insns(buf, &INSN); + let val = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr) + .wrapping_sub(8); + or32(&mut buf[16..], higha(val)); + or32(&mut buf[20..], lo(val)); + } + + /// The loader fills `.got.plt` itself and expects the layout the ABI + /// prescribes: entries of 8 bytes up to index 0x8000 and 12 bytes + /// beyond, since a larger index no longer fits an `li`. + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let idx = sym.plt_idx(&ctx.symbols).unwrap() as u64; + let plt0 = ctx.plt.hdr.shdr.sh_addr; + if idx < 0x8000 { + write_insns(buf, &[0x3800_0000, 0x4b00_0000]); // li r0, PLT_INDEX; b plt0 + or32(buf, idx); + or32( + &mut buf[4..], + plt0.wrapping_sub(sym.plt_addr(ctx)).wrapping_sub(4) & 0x00ff_ffff, + ); + } else { + // lis r0, PLT_INDEX@high; ori r0, r0, PLT_INDEX@lo; b plt0 + write_insns(buf, &[0x3c00_0000, 0x6000_0000, 0x4b00_0000]); + or32(buf, high(idx)); + or32(&mut buf[4..], lo(idx)); + or32( + &mut buf[8..], + plt0.wrapping_sub(sym.plt_addr(ctx)).wrapping_sub(8) & 0x00ff_ffff, + ); + } + } + + /// Thunks read GOT entries directly, so `.plt.got` has no entries. + fn write_pltgot_entry(_ctx: &Context, _buf: &mut [u8], _sym: &Symbol) {} + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_PPC64_ADDR64 => w64(loc, val), + R_PPC64_REL32 => { + eh_frame::check_range( + ctx, + isec, + rel, + val.wrapping_sub(p) as i64, + -(1 << 31), + 1 << 31, + ); + w32(loc, val.wrapping_sub(p)); + } + R_PPC64_REL64 => w64(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT | NEEDS_PPC_OPD); + } + + // Anything but a branch takes the function's address. + if rel.r_type != R_PPC64_REL24 && sym.ty() == STT_FUNC { + sym.add_flags(NEEDS_PPC_OPD); + } + + match rel.r_type { + R_PPC64_GOT_TPREL16_HA => sym.add_flags(NEEDS_GOTTP), + R_PPC64_REL24 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_PPC64_GOT16 | R_PPC64_PLT16_HA => sym.add_flags(NEEDS_GOT), + R_PPC64_GOT_TLSGD16_HA => sym.add_flags(NEEDS_TLSGD), + R_PPC64_GOT_TLSLD16_HA => ctx.needs_tlsld.store(true, Ordering::Relaxed), + R_PPC64_TPREL16_HA | R_PPC64_TPREL16_LO | R_PPC64_TPREL16_LO_DS => { + check_tlsle(ctx, isec, sym, &rel) + } + R_PPC64_ADDR64 + | R_PPC64_TOC + | R_PPC64_REL32 + | R_PPC64_REL64 + | R_PPC64_TOC16_HA + | R_PPC64_TOC16_LO + | R_PPC64_TOC16_LO_DS + | R_PPC64_TOC16_DS + | R_PPC64_REL16_HA + | R_PPC64_REL16_LO + | R_PPC64_PLT16_HI + | R_PPC64_PLT16_LO + | R_PPC64_PLT16_LO_DS + | R_PPC64_PLTSEQ + | R_PPC64_PLTCALL + | R_PPC64_GOT_TPREL16_LO_DS + | R_PPC64_GOT_TLSGD16_LO + | R_PPC64_GOT_TLSLD16_LO + | R_PPC64_TLS + | R_PPC64_TLSGD + | R_PPC64_TLSLD + | R_PPC64_DTPREL16_HA + | R_PPC64_DTPREL16_LO + | R_PPC64_DTPREL16_LO_DS => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let toc = toc(ctx); + let got = ctx.got.hdr.shdr.sh_addr; + + for (i, rel) in isec.rels::(file).iter().enumerate() { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_PPC64_TOC => {} + R_PPC64_TOC16_HA => w16(loc, ha(sa.wrapping_sub(toc))), + R_PPC64_TOC16_LO => w16(loc, lo(sa.wrapping_sub(toc))), + R_PPC64_TOC16_DS => { + isec.check_range(ctx, i, sa.wrapping_sub(toc) as i64, -(1 << 15), 1 << 15); + or16(loc, sa.wrapping_sub(toc) & 0xfffc); + } + R_PPC64_TOC16_LO_DS => or16(loc, sa.wrapping_sub(toc) & 0xfffc), + R_PPC64_REL24 => { + // A branch goes to the code, not the descriptor. + let code = sym.addr_with(ctx, AddrFlags::NO_OPD); + let mut val = code.wrapping_add(a).wrapping_sub(p) as i64; + if sym.has_plt(&ctx.symbols) || !is_int(val, 26) { + val = sym.thunk_addr(ctx, p).wrapping_add(a).wrapping_sub(p) as i64; + } + isec.check_range(ctx, i, val, -(1 << 25), 1 << 25); + or32(loc, bits(val as u64, 25, 2) << 2); + + // The PLT saves r2 to the caller's r2 save slot, which + // must be restored after the call returns. The + // placeholder for that is usually a NOP after the BL. + if sym.has_plt(&ctx.symbols) && loc.len() >= 8 && r32(&loc[4..]) == 0x6000_0000 + { + w32(&mut loc[4..], 0xe841_0028); // ld r2, 40(r1) + } + } + R_PPC64_REL32 => w32(loc, pcrel), + R_PPC64_REL64 => w64(loc, pcrel), + R_PPC64_REL16_HA => w16(loc, ha(pcrel)), + R_PPC64_REL16_LO => w16(loc, lo(pcrel)), + R_PPC64_GOT16 => w16(loc, g().wrapping_sub(toc)), + R_PPC64_PLT16_HA => w16(loc, ha(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_HI => w16(loc, hi(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_LO => w16(loc, lo(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_LO_DS => or16(loc, sym.got_addr(ctx).wrapping_sub(toc) & 0xfffc), + R_PPC64_GOT_TPREL16_HA => w16(loc, ha(sym.gottp_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TPREL16_LO_DS => { + or16(loc, sym.gottp_addr(ctx).wrapping_sub(toc) & 0xfffc) + } + R_PPC64_GOT_TLSGD16_HA => w16(loc, ha(sym.tlsgd_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TLSGD16_LO => w16(loc, lo(sym.tlsgd_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TLSLD16_HA => { + w16(loc, ha(ctx.got.tlsld_addr::().wrapping_sub(toc))) + } + R_PPC64_GOT_TLSLD16_LO => { + w16(loc, lo(ctx.got.tlsld_addr::().wrapping_sub(toc))) + } + R_PPC64_DTPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC64_DTPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC64_DTPREL16_LO_DS => or16(loc, sa.wrapping_sub(ctx.dtp_addr) & 0xfffc), + R_PPC64_TPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.tp_addr))), + R_PPC64_TPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.tp_addr))), + R_PPC64_TPREL16_LO_DS => or16(loc, sa.wrapping_sub(ctx.tp_addr) & 0xfffc), + R_PPC64_ADDR64 | R_PPC64_PLTSEQ | R_PPC64_PLTCALL | R_PPC64_TLS | R_PPC64_TLSGD + | R_PPC64_TLSLD => {} + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_PPC64_ADDR64 => w64( + loc, + isec.tombstone(ctx, sym, frag.map(|(f, _)| f)).unwrap_or(sa), + ), + R_PPC64_ADDR32 => { + isec.check_range(ctx, i, sa as i64, 0, 1 << 32); + w32(loc, sa); + } + R_PPC64_DTPREL64 => w64(loc, sa.wrapping_sub(ctx.dtp_addr)), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + /// All PLT calls go through thunks. + fn always_needs_thunk(ctx: &Context, sym: &Symbol, _rel: &ElfRel) -> bool { + sym.has_plt(&ctx.symbols) + } + + fn write_thunk(ctx: &Context, thunk: &Thunk, _addr: u64, buf: &mut [u8]) { + // A thunk to a function with a GOT entry saves the caller's r2, + // reads the descriptor's address from the GOT, sets the callee's + // r2 and jumps. + const PLTGOT_THUNK: [u32; 7] = [ + 0xf841_0028, // std r2, 40(r1) + 0x3d82_0000, // addis r12, r2, foo@got@toc@ha + 0xe98c_0000, // ld r12, foo@got@toc@lo(r12) + 0xe84c_0008, // ld r2, 8(r12) + 0xe98c_0000, // ld r12, 0(r12) + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + // A thunk to a PLT symbol finds the descriptor in .got.plt. + const PLT_THUNK: [u32; 7] = [ + 0xf841_0028, // std r2, 40(r1) + 0x3d82_0000, // addis r12, r2, foo@gotplt@toc@ha + 0x398c_0000, // addi r12, r12, foo@gotplt@toc@lo + 0xe84c_0008, // ld r2, 8(r12) + 0xe98c_0000, // ld r12, 0(r12) + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + // A thunk to a function in this module jumps straight to its code. + const LOCAL_THUNK: [u32; 7] = [ + 0x3d82_0000, // addis r12, r2, foo@toc@ha + 0x398c_0000, // addi r12, r12, foo@toc@lo + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + 0x6000_0000, // nop + 0x6000_0000, // nop + 0x6000_0000, // nop + ]; + + let toc = toc(ctx); + + for (i, &id) in thunk.symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + let entry = &mut buf[thunk.offsets[i] as usize..][..28]; + + let (insns, target, at) = if sym.has_got(&ctx.symbols) { + (&PLTGOT_THUNK, sym.got_addr(ctx), 4) + } else if sym.has_plt(&ctx.symbols) { + (&PLT_THUNK, sym.gotplt_addr(ctx), 4) + } else { + (&LOCAL_THUNK, sym.addr_with(ctx, AddrFlags::NO_OPD), 0) + }; + write_insns(entry, insns); + let val = target.wrapping_sub(toc); + or32(&mut entry[at..], higha(val)); + or32(&mut entry[at + 4..], lo(val)); + } + } +} diff --git a/src/arch/ppc64v2.rs b/src/arch/ppc64v2.rs new file mode 100644 index 000000000..fc1f2a282 --- /dev/null +++ b/src/arch/ppc64v2.rs @@ -0,0 +1,756 @@ +//! PowerPC64 ELFv2, the ABI of little-endian PowerPC systems ("ppc64le"). +//! +//! PC-relative loads and stores only arrived with Power10 in 2021, so +//! position-independent code for earlier processors can't address the +//! GOT relative to the program counter. Instead, most functions keep +//! `r2`, the TOC pointer, equal to `.got + 0x8000`, so that a GOT entry +//! is one `r2`-relative load away. Since a module has a single GOT, calls +//! within a module leave `r2` alone. +//! +//! A function compiled for such processors has two entry points. The +//! global entry point, which assumes its own address is in `r12`, +//! computes the callee's TOC pointer from `r12` with `addis`/`addi` and +//! falls through to the local entry point, which assumes `r2` is already +//! right. A call into another module therefore loads the callee's address +//! into `r12` (typically from `.got.plt`) and branches there; the callee +//! sets up its own `r2`. The distance between the two entry points is +//! encoded in the symbol's `st_other`. +//! +//! Power10 code uses 8-byte prefixed instructions with 34-bit PC-relative +//! offsets and treats `r2` as a scratch register. `_NOTOC` relocations +//! mark such call sites. When Power10 code calls a function that expects +//! the TOC pointer, or vice versa, the call goes through a range +//! extension thunk that recomputes `r2`. +//! +//! The psABI calls `.got.plt` `.plt`; the runtime only cares about +//! segments, so the sections keep their usual names here. +//! +//! https://github.com/rui314/psabi/blob/main/ppc64v2.pdf + +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family, ThunkLayout}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::thunks::Thunk; +use crate::util::{bits, is_int}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Ppc64V2; + +impl Layout for Ppc64V2 { + type Endian = LittleEndian; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +fn lo(x: u64) -> u64 { + x & 0xffff +} + +fn hi(x: u64) -> u64 { + x >> 16 +} + +fn ha(x: u64) -> u64 { + x.wrapping_add(0x8000) >> 16 +} + +fn higha(x: u64) -> u64 { + ha(x) & 0xffff +} + +fn r32(loc: &[u8]) -> u32 { + LittleEndian::read_u32(loc) +} + +fn w16(loc: &mut [u8], v: u64) { + LittleEndian::write_u16(loc, v as u16); +} + +fn w32(loc: &mut [u8], v: u32) { + LittleEndian::write_u32(loc, v); +} + +fn w64(loc: &mut [u8], v: u64) { + LittleEndian::write_u64(loc, v); +} + +fn or16(loc: &mut [u8], v: u64) { + let cur = LittleEndian::read_u16(loc); + LittleEndian::write_u16(loc, cur | v as u16); +} + +fn or32(loc: &mut [u8], v: u32) { + let cur = r32(loc); + w32(loc, cur | v); +} + +/// Writes the 34-bit immediate of a prefixed instruction, which is split +/// between the prefix word and the instruction word. +fn write34(loc: &mut [u8], x: u64) { + let prefix = (r32(loc) & 0xfffc_0000) | bits(x, 33, 16) as u32; + let insn = (r32(&loc[4..]) & 0xffff_0000) | bits(x, 15, 0) as u32; + w32(loc, prefix); + w32(&mut loc[4..], insn); +} + +/// The 26-bit displacement field of a branch. +fn branch_field(val: i64) -> u32 { + (bits(val as u64, 25, 2) << 2) as u32 +} + +/// The address of the TOC pointer, `.got + 0x8000`. +fn toc(ctx: &Context) -> u64 { + ctx.symbols[ctx.syms.toc.expect("PPC64 has a .TOC. symbol")].addr(ctx) +} + +fn is_power10(ctx: &Context) -> bool { + ctx.is_power10.load(Ordering::Relaxed) +} + +/// The distance from a function's global entry point to its local one. +fn local_entry_offset(ctx: &Context, sym: &Symbol) -> u64 { + match sym.esym(ctx).ppc64_local_entry() { + 0 | 1 => 0, + 7 => fatal!(ctx, "{sym}: local entry offset 7 is reserved"), + n => 1 << n, + } +} + +/// GCC may call these functions from prologues and epilogues under `-Os`. +/// They aren't in libgcc; the linker synthesizes them. Each label names +/// the instruction it precedes, and execution falls through the entries. +/// There are variants for general-purpose, floating-point and vector +/// registers. +pub const SAVE_RESTORE_INSNS: &[(&str, u32)] = &[ + ("_savegpr0_14", 0xf9c1ff70), // std r14,-144(r1) + ("_savegpr0_15", 0xf9e1ff78), // std r15,-136(r1) + ("_savegpr0_16", 0xfa01ff80), // std r16,-128(r1) + ("_savegpr0_17", 0xfa21ff88), // std r17,-120(r1) + ("_savegpr0_18", 0xfa41ff90), // std r18,-112(r1) + ("_savegpr0_19", 0xfa61ff98), // std r19,-104(r1) + ("_savegpr0_20", 0xfa81ffa0), // std r20,-96(r1) + ("_savegpr0_21", 0xfaa1ffa8), // std r21,-88(r1) + ("_savegpr0_22", 0xfac1ffb0), // std r22,-80(r1) + ("_savegpr0_23", 0xfae1ffb8), // std r23,-72(r1) + ("_savegpr0_24", 0xfb01ffc0), // std r24,-64(r1) + ("_savegpr0_25", 0xfb21ffc8), // std r25,-56(r1) + ("_savegpr0_26", 0xfb41ffd0), // std r26,-48(r1) + ("_savegpr0_27", 0xfb61ffd8), // std r27,-40(r1) + ("_savegpr0_28", 0xfb81ffe0), // std r28,-32(r1) + ("_savegpr0_29", 0xfba1ffe8), // std r29,-24(r1) + ("_savegpr0_30", 0xfbc1fff0), // std r30,-16(r1) + ("_savegpr0_31", 0xfbe1fff8), // std r31,-8(r1) + ("", 0xf8010010), // std r0,16(r1) + ("", 0x4e800020), // blr + ("_restgpr0_14", 0xe9c1ff70), // ld r14,-144(r1) + ("_restgpr0_15", 0xe9e1ff78), // ld r15,-136(r1) + ("_restgpr0_16", 0xea01ff80), // ld r16,-128(r1) + ("_restgpr0_17", 0xea21ff88), // ld r17,-120(r1) + ("_restgpr0_18", 0xea41ff90), // ld r18,-112(r1) + ("_restgpr0_19", 0xea61ff98), // ld r19,-104(r1) + ("_restgpr0_20", 0xea81ffa0), // ld r20,-96(r1) + ("_restgpr0_21", 0xeaa1ffa8), // ld r21,-88(r1) + ("_restgpr0_22", 0xeac1ffb0), // ld r22,-80(r1) + ("_restgpr0_23", 0xeae1ffb8), // ld r23,-72(r1) + ("_restgpr0_24", 0xeb01ffc0), // ld r24,-64(r1) + ("_restgpr0_25", 0xeb21ffc8), // ld r25,-56(r1) + ("_restgpr0_26", 0xeb41ffd0), // ld r26,-48(r1) + ("_restgpr0_27", 0xeb61ffd8), // ld r27,-40(r1) + ("_restgpr0_28", 0xeb81ffe0), // ld r28,-32(r1) + ("_restgpr0_29", 0xe8010010), // ld r0,16(r1) + ("", 0xeba1ffe8), // ld r29,-24(r1) + ("", 0x7c0803a6), // mtlr r0 + ("", 0xebc1fff0), // ld r30,-16(r1) + ("", 0xebe1fff8), // ld r31,-8(r1) + ("", 0x4e800020), // blr + ("_restgpr0_30", 0xebc1fff0), // ld r30,-16(r1) + ("_restgpr0_31", 0xe8010010), // ld r0,16(r1) + ("", 0xebe1fff8), // ld r31,-8(r1) + ("", 0x7c0803a6), // mtlr r0 + ("", 0x4e800020), // blr + ("_savegpr1_14", 0xf9ccff70), // std r14,-144(r12) + ("_savegpr1_15", 0xf9ecff78), // std r15,-136(r12) + ("_savegpr1_16", 0xfa0cff80), // std r16,-128(r12) + ("_savegpr1_17", 0xfa2cff88), // std r17,-120(r12) + ("_savegpr1_18", 0xfa4cff90), // std r18,-112(r12) + ("_savegpr1_19", 0xfa6cff98), // std r19,-104(r12) + ("_savegpr1_20", 0xfa8cffa0), // std r20,-96(r12) + ("_savegpr1_21", 0xfaacffa8), // std r21,-88(r12) + ("_savegpr1_22", 0xfaccffb0), // std r22,-80(r12) + ("_savegpr1_23", 0xfaecffb8), // std r23,-72(r12) + ("_savegpr1_24", 0xfb0cffc0), // std r24,-64(r12) + ("_savegpr1_25", 0xfb2cffc8), // std r25,-56(r12) + ("_savegpr1_26", 0xfb4cffd0), // std r26,-48(r12) + ("_savegpr1_27", 0xfb6cffd8), // std r27,-40(r12) + ("_savegpr1_28", 0xfb8cffe0), // std r28,-32(r12) + ("_savegpr1_29", 0xfbacffe8), // std r29,-24(r12) + ("_savegpr1_30", 0xfbccfff0), // std r30,-16(r12) + ("_savegpr1_31", 0xfbecfff8), // std r31,-8(r12) + ("", 0x4e800020), // blr + ("_restgpr1_14", 0xe9ccff70), // ld r14,-144(r12) + ("_restgpr1_15", 0xe9ecff78), // ld r15,-136(r12) + ("_restgpr1_16", 0xea0cff80), // ld r16,-128(r12) + ("_restgpr1_17", 0xea2cff88), // ld r17,-120(r12) + ("_restgpr1_18", 0xea4cff90), // ld r18,-112(r12) + ("_restgpr1_19", 0xea6cff98), // ld r19,-104(r12) + ("_restgpr1_20", 0xea8cffa0), // ld r20,-96(r12) + ("_restgpr1_21", 0xeaacffa8), // ld r21,-88(r12) + ("_restgpr1_22", 0xeaccffb0), // ld r22,-80(r12) + ("_restgpr1_23", 0xeaecffb8), // ld r23,-72(r12) + ("_restgpr1_24", 0xeb0cffc0), // ld r24,-64(r12) + ("_restgpr1_25", 0xeb2cffc8), // ld r25,-56(r12) + ("_restgpr1_26", 0xeb4cffd0), // ld r26,-48(r12) + ("_restgpr1_27", 0xeb6cffd8), // ld r27,-40(r12) + ("_restgpr1_28", 0xeb8cffe0), // ld r28,-32(r12) + ("_restgpr1_29", 0xebacffe8), // ld r29,-24(r12) + ("_restgpr1_30", 0xebccfff0), // ld r30,-16(r12) + ("_restgpr1_31", 0xebecfff8), // ld r31,-8(r12) + ("", 0x4e800020), // blr + ("_savefpr_14", 0xd9c1ff70), // stfd f14,-144(r1) + ("_savefpr_15", 0xd9e1ff78), // stfd f15,-136(r1) + ("_savefpr_16", 0xda01ff80), // stfd f16,-128(r1) + ("_savefpr_17", 0xda21ff88), // stfd f17,-120(r1) + ("_savefpr_18", 0xda41ff90), // stfd f18,-112(r1) + ("_savefpr_19", 0xda61ff98), // stfd f19,-104(r1) + ("_savefpr_20", 0xda81ffa0), // stfd f20,-96(r1) + ("_savefpr_21", 0xdaa1ffa8), // stfd f21,-88(r1) + ("_savefpr_22", 0xdac1ffb0), // stfd f22,-80(r1) + ("_savefpr_23", 0xdae1ffb8), // stfd f23,-72(r1) + ("_savefpr_24", 0xdb01ffc0), // stfd f24,-64(r1) + ("_savefpr_25", 0xdb21ffc8), // stfd f25,-56(r1) + ("_savefpr_26", 0xdb41ffd0), // stfd f26,-48(r1) + ("_savefpr_27", 0xdb61ffd8), // stfd f27,-40(r1) + ("_savefpr_28", 0xdb81ffe0), // stfd f28,-32(r1) + ("_savefpr_29", 0xdba1ffe8), // stfd f29,-24(r1) + ("_savefpr_30", 0xdbc1fff0), // stfd f30,-16(r1) + ("_savefpr_31", 0xdbe1fff8), // stfd f31,-8(r1) + ("", 0xf8010010), // std r0,16(r1) + ("", 0x4e800020), // blr + ("_restfpr_14", 0xc9c1ff70), // lfd f14,-144(r1) + ("_restfpr_15", 0xc9e1ff78), // lfd f15,-136(r1) + ("_restfpr_16", 0xca01ff80), // lfd f16,-128(r1) + ("_restfpr_17", 0xca21ff88), // lfd f17,-120(r1) + ("_restfpr_18", 0xca41ff90), // lfd f18,-112(r1) + ("_restfpr_19", 0xca61ff98), // lfd f19,-104(r1) + ("_restfpr_20", 0xca81ffa0), // lfd f20,-96(r1) + ("_restfpr_21", 0xcaa1ffa8), // lfd f21,-88(r1) + ("_restfpr_22", 0xcac1ffb0), // lfd f22,-80(r1) + ("_restfpr_23", 0xcae1ffb8), // lfd f23,-72(r1) + ("_restfpr_24", 0xcb01ffc0), // lfd f24,-64(r1) + ("_restfpr_25", 0xcb21ffc8), // lfd f25,-56(r1) + ("_restfpr_26", 0xcb41ffd0), // lfd f26,-48(r1) + ("_restfpr_27", 0xcb61ffd8), // lfd f27,-40(r1) + ("_restfpr_28", 0xcb81ffe0), // lfd f28,-32(r1) + ("_restfpr_29", 0xe8010010), // ld r0,16(r1) + ("", 0xcba1ffe8), // lfd f29,-24(r1) + ("", 0x7c0803a6), // mtlr r0 + ("", 0xcbc1fff0), // lfd f30,-16(r1) + ("", 0xcbe1fff8), // lfd f31,-8(r1) + ("", 0x4e800020), // blr + ("_restfpr_30", 0xcbc1fff0), // lfd f30,-16(r1) + ("_restfpr_31", 0xe8010010), // ld r0,16(r1) + ("", 0xcbe1fff8), // lfd f31,-8(r1) + ("", 0x7c0803a6), // mtlr r0 + ("", 0x4e800020), // blr + ("_savevr_20", 0x3980ff40), // li r12,-192 + ("", 0x7e8c01ce), // stvx v20,r12,r0 + ("_savevr_21", 0x3980ff50), // li r12,-176 + ("", 0x7eac01ce), // stvx v21,r12,r0 + ("_savevr_22", 0x3980ff60), // li r12,-160 + ("", 0x7ecc01ce), // stvx v22,r12,r0 + ("_savevr_23", 0x3980ff70), // li r12,-144 + ("", 0x7eec01ce), // stvx v23,r12,r0 + ("_savevr_24", 0x3980ff80), // li r12,-128 + ("", 0x7f0c01ce), // stvx v24,r12,r0 + ("_savevr_25", 0x3980ff90), // li r12,-112 + ("", 0x7f2c01ce), // stvx v25,r12,r0 + ("_savevr_26", 0x3980ffa0), // li r12,-96 + ("", 0x7f4c01ce), // stvx v26,r12,r0 + ("_savevr_27", 0x3980ffb0), // li r12,-80 + ("", 0x7f6c01ce), // stvx v27,r12,r0 + ("_savevr_28", 0x3980ffc0), // li r12,-64 + ("", 0x7f8c01ce), // stvx v28,r12,r0 + ("_savevr_29", 0x3980ffd0), // li r12,-48 + ("", 0x7fac01ce), // stvx v29,r12,r0 + ("_savevr_30", 0x3980ffe0), // li r12,-32 + ("", 0x7fcc01ce), // stvx v30,r12,r0 + ("_savevr_31", 0x3980fff0), // li r12,-16 + ("", 0x7fec01ce), // stvx v31,r12,r0 + ("", 0x4e800020), // blr + ("_restvr_20", 0x3980ff40), // li r12,-192 + ("", 0x7e8c00ce), // lvx v20,r12,r0 + ("_restvr_21", 0x3980ff50), // li r12,-176 + ("", 0x7eac00ce), // lvx v21,r12,r0 + ("_restvr_22", 0x3980ff60), // li r12,-160 + ("", 0x7ecc00ce), // lvx v22,r12,r0 + ("_restvr_23", 0x3980ff70), // li r12,-144 + ("", 0x7eec00ce), // lvx v23,r12,r0 + ("_restvr_24", 0x3980ff80), // li r12,-128 + ("", 0x7f0c00ce), // lvx v24,r12,r0 + ("_restvr_25", 0x3980ff90), // li r12,-112 + ("", 0x7f2c00ce), // lvx v25,r12,r0 + ("_restvr_26", 0x3980ffa0), // li r12,-96 + ("", 0x7f4c00ce), // lvx v26,r12,r0 + ("_restvr_27", 0x3980ffb0), // li r12,-80 + ("", 0x7f6c00ce), // lvx v27,r12,r0 + ("_restvr_28", 0x3980ffc0), // li r12,-64 + ("", 0x7f8c00ce), // lvx v28,r12,r0 + ("_restvr_29", 0x3980ffd0), // li r12,-48 + ("", 0x7fac00ce), // lvx v29,r12,r0 + ("_restvr_30", 0x3980ffe0), // li r12,-32 + ("", 0x7fcc00ce), // lvx v30,r12,r0 + ("_restvr_31", 0x3980fff0), // li r12,-16 + ("", 0x7fec00ce), // lvx v31,r12,r0 + ("", 0x4e800020), // blr +]; + +/// The contents of the `.save_restore_regs` section. +pub fn save_restore_contents() -> Vec { + SAVE_RESTORE_INSNS + .iter() + .flat_map(|&(_, insn)| insn.to_le_bytes()) + .collect() +} + +impl Arch for Ppc64V2 { + const NAME: &'static str = "ppc64v2"; + const FAMILY: Family = Family::Ppc64V2; + const PAGE_SIZE: u64 = 65536; + const E_MACHINE: u32 = EM_PPC64; + const PLT_HDR_SIZE: u64 = 52; + const PLT_SIZE: u64 = 4; + const PLTGOT_SIZE: u64 = 0; + const THUNK: Option = Some(ThunkLayout { + header_size: 0, + entry_size: 24, + }); + const TRAP: &'static [u8] = &[0x08, 0x00, 0xe0, 0x7f]; // trap + + const R_COPY: u32 = R_PPC64_COPY; + const R_GLOB_DAT: u32 = R_PPC64_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_PPC64_JMP_SLOT; + const R_ABS: u32 = R_PPC64_ADDR64; + const R_RELATIVE: u32 = R_PPC64_RELATIVE; + const R_IRELATIVE: Option = Some(R_PPC64_IRELATIVE); + const R_DTPOFF: u32 = R_PPC64_DTPREL64; + const R_TPOFF: u32 = R_PPC64_TPREL64; + const R_DTPMOD: u32 = R_PPC64_DTPMOD64; + const R_FUNCALL: &'static [u32] = &[R_PPC64_REL24, R_PPC64_REL24_NOTOC]; + + fn rel_to_string(r_type: u32) -> String { + ppc64_rel_to_string(r_type) + } + + fn eflags(_ctx: &Context) -> u32 { + 2 + } + + /// `.plt` serves lazy symbol resolution only. All PLT calls go through + /// thunks, which read the target from `.got.plt` and jump there, so + /// once a symbol is resolved the PLT is skipped. + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u32; 13] = [ + // Get PC. + 0x7c08_02a6, // mflr r0 + 0x429f_0005, // bcl 20, 31, 4 + 0x7d68_02a6, // mflr r11 + 0x7c08_03a6, // mtlr r0 + // Compute the PLT entry index. + 0x398c_ffd4, // addi r12, r12, -44 + 0x7c0b_6050, // subf r0, r11, r12 + 0x7800_f082, // rldicl r0, r0, 62, 2 + // Compute the address of .got.plt. + 0x3d6b_0000, // addis r11, r11, GOTPLT_OFFSET@ha + 0x396b_0000, // addi r11, r11, GOTPLT_OFFSET@lo + // Load .got.plt[0] and .got.plt[1] and branch to .got.plt[0]. + 0xe98b_0000, // ld r12, 0(r11) + 0x7d89_03a6, // mtctr r12 + 0xe96b_0008, // ld r11, 8(r11) + 0x4e80_0420, // bctr + ]; + for (i, &insn) in INSN.iter().enumerate() { + w32(&mut buf[i * 4..], insn); + } + let val = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr) + .wrapping_sub(8); + or32(&mut buf[28..], higha(val) as u32); + or32(&mut buf[32..], lo(val) as u32); + } + + /// The caller has already put the entry's address in `r12`. + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let offset = ctx.plt.hdr.shdr.sh_addr.wrapping_sub(sym.plt_addr(ctx)); + w32(buf, 0x4b00_0000 | (offset as u32 & 0x00ff_ffff)); // b plt0 + } + + /// Thunks read GOT entries directly, so `.plt.got` has no entries. + fn write_pltgot_entry(_ctx: &Context, _buf: &mut [u8], _sym: &Symbol) {} + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_PPC64_ADDR64 => w64(loc, val), + R_PPC64_REL32 => { + eh_frame::check_range( + ctx, + isec, + rel, + val.wrapping_sub(p) as i64, + -(1 << 31), + 1 << 31, + ); + w32(loc, val.wrapping_sub(p) as u32); + } + R_PPC64_REL64 => w64(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_PPC64_GOT_TPREL16_HA | R_PPC64_GOT_TPREL_PCREL34 => sym.add_flags(NEEDS_GOTTP), + R_PPC64_REL24 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_PPC64_REL24_NOTOC => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + ctx.is_power10.store(true, Ordering::Relaxed); + } + R_PPC64_GOT16 + | R_PPC64_GOT16_LO + | R_PPC64_GOT16_HI + | R_PPC64_GOT16_HA + | R_PPC64_PLT16_HA + | R_PPC64_PLT_PCREL34 + | R_PPC64_PLT_PCREL34_NOTOC + | R_PPC64_GOT_PCREL34 => sym.add_flags(NEEDS_GOT), + R_PPC64_GOT_TLSGD16_HA | R_PPC64_GOT_TLSGD_PCREL34 => sym.add_flags(NEEDS_TLSGD), + R_PPC64_GOT_TLSLD16_HA | R_PPC64_GOT_TLSLD_PCREL34 => { + ctx.needs_tlsld.store(true, Ordering::Relaxed) + } + R_PPC64_TPREL16_HA | R_PPC64_TPREL34 => check_tlsle(ctx, isec, sym, &rel), + R_PPC64_ADDR64 + | R_PPC64_REL14 + | R_PPC64_REL32 + | R_PPC64_REL64 + | R_PPC64_TOC16_HA + | R_PPC64_TOC16_LO + | R_PPC64_TOC16_LO_DS + | R_PPC64_TOC16_DS + | R_PPC64_REL16_HA + | R_PPC64_REL16_LO + | R_PPC64_PLT16_HI + | R_PPC64_PLT16_LO + | R_PPC64_PLT16_LO_DS + | R_PPC64_PCREL34 + | R_PPC64_PLTSEQ + | R_PPC64_PLTSEQ_NOTOC + | R_PPC64_PLTCALL + | R_PPC64_PLTCALL_NOTOC + | R_PPC64_GOT_TPREL16_LO_DS + | R_PPC64_GOT_TLSGD16_LO + | R_PPC64_GOT_TLSLD16_LO + | R_PPC64_TPREL16_LO + | R_PPC64_TPREL16_LO_DS + | R_PPC64_TLS + | R_PPC64_TLSGD + | R_PPC64_TLSLD + | R_PPC64_DTPREL16_HA + | R_PPC64_DTPREL16_LO + | R_PPC64_DTPREL16_LO_DS + | R_PPC64_DTPREL34 + | R_PPC64_ENTRY => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let toc = toc(ctx); + let got = ctx.got.hdr.shdr.sh_addr; + + for (i, rel) in isec.rels::(file).iter().enumerate() { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + let loc = &mut buf[off..]; + + // Thunk entries have two entry points: one that saves r2 to + // the caller's stack frame first, and one 8 bytes in that + // doesn't. + let r2save_thunk = || sym.thunk_addr(ctx, p); + let no_r2save_thunk = || sym.thunk_addr(ctx, p) + 8; + + match rel.r_type { + R_PPC64_TOC16_HA => w16(loc, ha(sa.wrapping_sub(toc))), + R_PPC64_TOC16_LO => w16(loc, lo(sa.wrapping_sub(toc))), + R_PPC64_TOC16_DS => { + isec.check_range(ctx, i, sa.wrapping_sub(toc) as i64, -(1 << 15), 1 << 15); + or16(loc, sa.wrapping_sub(toc) & 0xfffc); + } + R_PPC64_TOC16_LO_DS => or16(loc, sa.wrapping_sub(toc) & 0xfffc), + R_PPC64_REL24 => { + if sym.has_plt(&ctx.symbols) || !sym.esym(ctx).ppc64_preserves_r2() { + let val = r2save_thunk().wrapping_add(a).wrapping_sub(p) as i64; + or32(loc, branch_field(val)); + + // The thunk saves r2 to the caller's r2 save slot, + // which must be restored after the call returns. + // The placeholder for that is usually a NOP after + // the BL. + if loc.len() >= 8 && r32(&loc[4..]) == 0x6000_0000 { + w32(&mut loc[4..], 0xe841_0018); // ld r2, 24(r1) + } + } else { + let mut val = s + .wrapping_add(local_entry_offset(ctx, sym)) + .wrapping_add(a) + .wrapping_sub(p) as i64; + if !is_int(val, 26) { + val = no_r2save_thunk().wrapping_add(a).wrapping_sub(p) as i64; + } + or32(loc, branch_field(val)); + } + } + R_PPC64_REL24_NOTOC => { + let mut val = pcrel as i64; + if sym.has_plt(&ctx.symbols) + || sym.esym(ctx).ppc64_uses_toc() + || !is_int(val, 26) + { + val = no_r2save_thunk().wrapping_add(a).wrapping_sub(p) as i64; + } + or32(loc, branch_field(val)); + } + R_PPC64_REL14 => or32(loc, (bits(pcrel, 15, 2) << 2) as u32), + R_PPC64_REL32 => w32(loc, pcrel as u32), + R_PPC64_REL64 => w64(loc, pcrel), + R_PPC64_REL16_HA => w16(loc, ha(pcrel)), + R_PPC64_REL16_LO => w16(loc, lo(pcrel)), + R_PPC64_GOT16 => w16(loc, g().wrapping_sub(toc)), + R_PPC64_GOT16_LO => w16(loc, lo(g().wrapping_sub(toc))), + R_PPC64_GOT16_HI => w16(loc, hi(g().wrapping_sub(toc))), + R_PPC64_GOT16_HA => w16(loc, ha(g().wrapping_sub(toc))), + R_PPC64_PLT16_HA => w16(loc, ha(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_HI => w16(loc, hi(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_LO => w16(loc, lo(sym.got_addr(ctx).wrapping_sub(toc))), + R_PPC64_PLT16_LO_DS => or16(loc, sym.got_addr(ctx).wrapping_sub(toc) & 0xfffc), + R_PPC64_PLT_PCREL34 | R_PPC64_PLT_PCREL34_NOTOC | R_PPC64_GOT_PCREL34 => { + write34(loc, sym.got_addr(ctx).wrapping_sub(p)) + } + R_PPC64_PCREL34 => write34(loc, pcrel), + R_PPC64_GOT_TPREL16_HA => w16(loc, ha(sym.gottp_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TPREL16_LO_DS => { + or16(loc, sym.gottp_addr(ctx).wrapping_sub(toc) & 0xfffc) + } + R_PPC64_GOT_TPREL_PCREL34 => write34(loc, sym.gottp_addr(ctx).wrapping_sub(p)), + R_PPC64_GOT_TLSGD16_HA => w16(loc, ha(sym.tlsgd_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TLSGD16_LO => w16(loc, lo(sym.tlsgd_addr(ctx).wrapping_sub(toc))), + R_PPC64_GOT_TLSGD_PCREL34 => write34(loc, sym.tlsgd_addr(ctx).wrapping_sub(p)), + R_PPC64_GOT_TLSLD16_HA => { + w16(loc, ha(ctx.got.tlsld_addr::().wrapping_sub(toc))) + } + R_PPC64_GOT_TLSLD16_LO => { + w16(loc, lo(ctx.got.tlsld_addr::().wrapping_sub(toc))) + } + R_PPC64_GOT_TLSLD_PCREL34 => { + write34(loc, ctx.got.tlsld_addr::().wrapping_sub(p)) + } + R_PPC64_DTPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC64_DTPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.dtp_addr))), + R_PPC64_DTPREL16_LO_DS => or16(loc, sa.wrapping_sub(ctx.dtp_addr) & 0xfffc), + R_PPC64_DTPREL34 => write34(loc, sa.wrapping_sub(ctx.dtp_addr)), + R_PPC64_TPREL16_HA => w16(loc, ha(sa.wrapping_sub(ctx.tp_addr))), + R_PPC64_TPREL16_LO => w16(loc, lo(sa.wrapping_sub(ctx.tp_addr))), + R_PPC64_TPREL16_LO_DS => or16(loc, sa.wrapping_sub(ctx.tp_addr) & 0xfffc), + R_PPC64_TPREL34 => write34(loc, sa.wrapping_sub(ctx.tp_addr)), + R_PPC64_ADDR64 + | R_PPC64_PLTSEQ + | R_PPC64_PLTSEQ_NOTOC + | R_PPC64_PLTCALL + | R_PPC64_PLTCALL_NOTOC + | R_PPC64_TLS + | R_PPC64_TLSGD + | R_PPC64_TLSLD + | R_PPC64_ENTRY => {} + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_PPC64_ADDR64 => match isec.tombstone(ctx, sym, frag.map(|(f, _)| f)) { + Some(v) => w64(loc, v), + None => w64(loc, sa), + }, + R_PPC64_ADDR32 => { + isec.check_range(ctx, i, sa as i64, 0, 1 << 32); + w32(loc, sa as u32); + } + R_PPC64_DTPREL64 => w64(loc, sa.wrapping_sub(ctx.dtp_addr)), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + /// All PLT calls go through thunks. So do calls between code that + /// expects `r2` to hold the TOC pointer and code that doesn't. + fn always_needs_thunk(ctx: &Context, sym: &Symbol, rel: &ElfRel) -> bool { + sym.has_plt(&ctx.symbols) + || (rel.r_type == R_PPC64_REL24 && !sym.esym(ctx).ppc64_preserves_r2()) + || (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym(ctx).ppc64_uses_toc()) + } + + fn write_thunk(ctx: &Context, thunk: &Thunk, addr: u64, buf: &mut [u8]) { + // A PLT thunk reads the destination from .got.plt or .got and + // jumps there. + const PLT_THUNK: [u32; 6] = [ + 0xf841_0018, // std r2, 24(r1) + 0x6000_0000, // nop + 0x3d82_0000, // addis r12, r2, foo@gotplt@toc@ha + 0xe98c_0000, // ld r12, foo@gotplt@toc@lo(r12) + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + const PLT_THUNK_POWER10: [u32; 6] = [ + 0xf841_0018, // std r2, 24(r1) + 0x6000_0000, // nop + 0x0410_0000, // pld r12, foo@gotplt@pcrel + 0xe580_0000, + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + // A thunk to a function in this module jumps to its local entry + // point. + const LOCAL_THUNK: [u32; 6] = [ + 0xf841_0018, // std r2, 24(r1) + 0x6000_0000, // nop + 0x3d82_0000, // addis r12, r2, foo@toc@ha + 0x398c_0000, // addi r12, r12, foo@toc@lo + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + const LOCAL_THUNK_POWER10: [u32; 6] = [ + 0xf841_0018, // std r2, 24(r1) + 0x6000_0000, // nop + 0x0610_0000, // pla r12, foo@pcrel + 0x3980_0000, + 0x7d89_03a6, // mtctr r12 + 0x4e80_0420, // bctr + ]; + + let toc = toc(ctx); + let power10 = is_power10(ctx); + + for (i, &id) in thunk.symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + let p = addr + thunk.offsets[i]; + let entry = &mut buf[thunk.offsets[i] as usize..][..24]; + + let (insns, target) = if sym.has_plt(&ctx.symbols) { + let got = if sym.has_got(&ctx.symbols) { + sym.got_addr(ctx) + } else { + sym.gotplt_addr(ctx) + }; + ( + if power10 { + &PLT_THUNK_POWER10 + } else { + &PLT_THUNK + }, + got, + ) + } else { + ( + if power10 { + &LOCAL_THUNK_POWER10 + } else { + &LOCAL_THUNK + }, + sym.addr(ctx), + ) + }; + for (j, &insn) in insns.iter().enumerate() { + w32(&mut entry[j * 4..], insn); + } + if power10 { + write34(&mut entry[8..], target.wrapping_sub(p).wrapping_sub(8)); + } else { + or32(&mut entry[8..], higha(target.wrapping_sub(toc)) as u32); + or32(&mut entry[12..], lo(target.wrapping_sub(toc)) as u32); + } + } + } +} diff --git a/src/arch/riscv.rs b/src/arch/riscv.rs new file mode 100644 index 000000000..1a1c89ee3 --- /dev/null +++ b/src/arch/riscv.rs @@ -0,0 +1,1223 @@ +//! RISC-V, 64- and 32-bit, in either byte order. +//! +//! RISC-V is a clean RISC ISA with PC-relative loads and stores for +//! position-independent code, and RV32 is essentially RV64 without the +//! 64-bit operations. Big-endian RISC-V exists as an extension; even then +//! instructions are little-endian, only data is byte-swapped. +//! +//! What makes RISC-V unusual from the linker's point of view is that +//! sections can shrink while being copied: branches are emitted as +//! instruction pairs reaching ±2 GiB, and the linker replaces them with a +//! single instruction when the target is close enough. See `relax.rs`. +//! +//! https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc + +// Binary literals are grouped by instruction field. +#![allow(clippy::unusual_byte_groupings)] + +use std::marker::PhantomData; + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::input_sections::{ + check_tlsle, r_delta, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection, RelocDelta, +}; +use crate::relax::compute_distance; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::util::{align_to, bit, bits, encode_uleb, is_int, overwrite_uleb, read_uleb}; +use crate::{error, fatal}; + +/// RISC-V of a given word size and byte order. +#[derive(Clone, Copy, Debug, Default)] +pub struct RiscvTarget(PhantomData); + +pub type Riscv64 = RiscvTarget; +pub type Riscv64Be = RiscvTarget; +pub type Riscv32 = RiscvTarget; +pub type Riscv32Be = RiscvTarget; + +impl Layout for RiscvTarget { + type Endian = End; + const IS_64: bool = IS_64; + const IS_RELA: bool = true; +} + +// Instructions are always little-endian. + +fn insn32(loc: &[u8]) -> u32 { + u32::from_le_bytes([loc[0], loc[1], loc[2], loc[3]]) +} + +fn insn16(loc: &[u8]) -> u16 { + u16::from_le_bytes([loc[0], loc[1]]) +} + +fn write32(loc: &mut [u8], v: u32) { + loc[..4].copy_from_slice(&v.to_le_bytes()); +} + +fn write16(loc: &mut [u8], v: u16) { + loc[..2].copy_from_slice(&v.to_le_bytes()); +} + +fn b(val: u64, hi: u32, lo: u32) -> u32 { + bits(val, hi, lo) as u32 +} + +fn write_itype(loc: &mut [u8], val: u64) { + write32( + loc, + (insn32(loc) & 0b000000_00000_11111_111_11111_1111111) | (b(val, 11, 0) << 20), + ); +} + +fn write_stype(loc: &mut [u8], val: u64) { + write32( + loc, + (insn32(loc) & 0b000000_11111_11111_111_00000_1111111) + | (b(val, 11, 5) << 25) + | (b(val, 4, 0) << 7), + ); +} + +fn write_btype(loc: &mut [u8], val: u64) { + let imm = (bit(val, 12) as u32) << 31 + | b(val, 10, 5) << 25 + | b(val, 4, 1) << 8 + | (bit(val, 11) as u32) << 7; + write32( + loc, + (insn32(loc) & 0b000000_11111_11111_111_00000_1111111) | imm, + ); +} + +/// U-type instructions set the upper 20 bits of a register, and a +/// following I-type instruction adds a sign-extended 12-bit immediate; +/// 0x800 compensates for the sign extension. +fn write_utype(loc: &mut [u8], val: u64) { + write32( + loc, + (insn32(loc) & 0b000000_00000_00000_000_11111_1111111) + | ((val as u32).wrapping_add(0x800) & 0xffff_f000), + ); +} + +fn write_jtype(loc: &mut [u8], val: u64) { + let imm = (bit(val, 20) as u32) << 31 + | b(val, 10, 1) << 21 + | (bit(val, 11) as u32) << 20 + | b(val, 19, 12) << 12; + write32( + loc, + (insn32(loc) & 0b000000_00000_00000_000_11111_1111111) | imm, + ); +} + +fn write_citype(loc: &mut [u8], val: u64) { + let imm = (bit(val, 5) as u16) << 12 | (b(val, 4, 0) as u16) << 2; + write16(loc, (insn16(loc) & 0b111_0_11111_00000_11) | imm); +} + +fn write_cbtype(loc: &mut [u8], val: u64) { + let bt = |n: u32| bit(val, n) as u16; + let imm = bt(8) << 12 + | bt(4) << 11 + | bt(3) << 10 + | bt(7) << 6 + | bt(6) << 5 + | bt(2) << 4 + | bt(1) << 3 + | bt(5) << 2; + write16(loc, (insn16(loc) & 0b111_000_111_00000_11) | imm); +} + +fn write_cjtype(loc: &mut [u8], val: u64) { + let bt = |n: u32| bit(val, n) as u16; + let imm = bt(11) << 12 + | bt(4) << 11 + | bt(9) << 10 + | bt(8) << 9 + | bt(10) << 8 + | bt(6) << 7 + | bt(7) << 6 + | bt(3) << 5 + | bt(2) << 4 + | bt(1) << 3 + | bt(5) << 2; + write16(loc, (insn16(loc) & 0b111_00000000000_11) | imm); +} + +fn set_rs1(loc: &mut [u8], rs1: u32) { + debug_assert!(rs1 < 32); + write32( + loc, + (insn32(loc) & 0b111111_11111_00000_111_11111_1111111) | (rs1 << 15), + ); +} + +fn rd(loc: &[u8]) -> u32 { + b(insn32(loc) as u64, 11, 7) +} + +const NOP: u32 = 0x13; + +fn is_hi20(r_type: u32) -> bool { + matches!( + r_type, + R_RISCV_GOT_HI20 + | R_RISCV_TLS_GOT_HI20 + | R_RISCV_TLS_GD_HI20 + | R_RISCV_PCREL_HI20 + | R_RISCV_TLSDESC_HI20 + ) +} + +/// Finds the HI20 relocation a LO12 relocation is paired with. +/// +/// AUIPC materializes the upper 52 bits of a PC-relative address and a +/// following instruction the low 12 bits, but the pair need not be +/// adjacent. So the compiler creates a local symbol at the AUIPC and the +/// LO12 relocation refers to that symbol. The pair usually is adjacent, +/// which a linear search from `i` exploits. +fn find_paired_reloc( + ctx: &Context, + isec: &InputSection, + sym: &Symbol, + i: usize, +) -> usize { + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let value = sym.esym(ctx).st_value; + let candidates: Box> = if value <= rels.at(i).r_offset { + Box::new((0..i).rev()) + } else { + Box::new(i + 1..rels.len()) + }; + for j in candidates { + if is_hi20(rels.at(j).r_type) && value == rels.at(j).r_offset { + return j; + } + } + let file = &ctx.objs[isec.file.index()]; + fatal!( + ctx, + "{}: paired relocation is missing: {i}", + isec.display(file) + ); +} + +/// Whether the relocation at `i` heads the GOT-loading instruction pair +/// `la rd, foo` expands to: +/// +/// ```text +/// .L0: +/// auipc rd, 0 # R_RISCV_GOT_HI20(foo), R_RISCV_RELAX +/// ld rd, 0(rd) # R_RISCV_PCREL_LO12_I(.L0), R_RISCV_RELAX +/// ``` +fn is_got_load_pair(ctx: &Context, isec: &InputSection, i: usize) -> bool { + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let file = &ctx.objs[isec.file.index()]; + let contents = isec.original_contents(file); + i + 3 < rels.len() + && rels.at(i).r_type == R_RISCV_GOT_HI20 + && rels.at(i + 1).r_type == R_RISCV_RELAX + && rels.at(i + 2).r_type == R_RISCV_PCREL_LO12_I + && rels.at(i + 3).r_type == R_RISCV_RELAX + && rels.at(i).r_offset == rels.at(i + 2).r_offset - 4 + && rels.at(i).r_offset + == ctx.symbols[file.base.symbols[rels.at(i + 2).r_sym as usize]].value + && rd(&contents[rels.at(i).r_offset as usize..]) + == rd(&contents[rels.at(i + 2).r_offset as usize..]) +} + +impl Arch for RiscvTarget { + const NAME: &'static str = match (IS_64, End::IS_LITTLE) { + (true, true) => "riscv64", + (true, false) => "riscv64be", + (false, true) => "riscv32", + (false, false) => "riscv32be", + }; + const FAMILY: Family = Family::RiscV; + const PAGE_SIZE: u64 = 4096; + const E_MACHINE: u32 = EM_RISCV; + const PLT_HDR_SIZE: u64 = 32; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 16; + const TRAP: &'static [u8] = &[0x02, 0x90]; // c.ebreak + + const R_COPY: u32 = R_RISCV_COPY; + const R_GLOB_DAT: u32 = if IS_64 { R_RISCV_64 } else { R_RISCV_32 }; + const R_JUMP_SLOT: u32 = R_RISCV_JUMP_SLOT; + const R_ABS: u32 = if IS_64 { R_RISCV_64 } else { R_RISCV_32 }; + const R_RELATIVE: u32 = R_RISCV_RELATIVE; + const R_IRELATIVE: Option = Some(R_RISCV_IRELATIVE); + const R_DTPOFF: u32 = if IS_64 { + R_RISCV_TLS_DTPREL64 + } else { + R_RISCV_TLS_DTPREL32 + }; + const R_TPOFF: u32 = if IS_64 { + R_RISCV_TLS_TPREL64 + } else { + R_RISCV_TLS_TPREL32 + }; + const R_DTPMOD: u32 = if IS_64 { + R_RISCV_TLS_DTPMOD64 + } else { + R_RISCV_TLS_DTPMOD32 + }; + const R_TLSDESC: Option = Some(R_RISCV_TLSDESC); + const R_FUNCALL: &'static [u32] = &[R_RISCV_CALL, R_RISCV_CALL_PLT]; + + fn rel_to_string(r_type: u32) -> String { + riscv_rel_to_string(r_type) + } + + fn eflags(ctx: &Context) -> u32 { + let internal = ctx.internal_obj; + let mut objs = ctx + .objs + .iter() + .enumerate() + .filter(|&(i, _)| Some(crate::input_files::ObjId(i as u32)) != internal) + .map(|(_, f)| f); + let Some(first) = objs.next() else { return 0 }; + let mut ret = first.base.e_flags; + for file in objs { + let flags = file.base.e_flags; + if flags & EF_RISCV_RVC != 0 { + ret |= EF_RISCV_RVC; + } + if flags & EF_RISCV_FLOAT_ABI != ret & EF_RISCV_FLOAT_ABI { + error!(ctx, "{file}: cannot link object files with different floating-point ABI from {first}"); + } + if flags & EF_RISCV_RVE != ret & EF_RISCV_RVE { + error!( + ctx, + "{file}: cannot link object files with different EF_RISCV_RVE from {first}" + ); + } + } + ret + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN_64: [u32; 8] = [ + 0x0000_0397, // auipc t2, %pcrel_hi(.got.plt) + 0x41c3_0333, // sub t1, t1, t3 # .plt entry + hdr + 12 + 0x0003_be03, // ld t3, %pcrel_lo(1b)(t2) # _dl_runtime_resolve + 0xfd43_0313, // addi t1, t1, -44 # .plt entry + 0x0003_8293, // addi t0, t2, %pcrel_lo(1b) # &.got.plt + 0x0013_5313, // srli t1, t1, 1 # .plt entry offset + 0x0082_b283, // ld t0, 8(t0) # link map + 0x000e_0067, // jr t3 + ]; + const INSN_32: [u32; 8] = [ + 0x0000_0397, // auipc t2, %pcrel_hi(.got.plt) + 0x41c3_0333, // sub t1, t1, t3 # .plt entry + hdr + 12 + 0x0003_ae03, // lw t3, %pcrel_lo(1b)(t2) # _dl_runtime_resolve + 0xfd43_0313, // addi t1, t1, -44 # .plt entry + 0x0003_8293, // addi t0, t2, %pcrel_lo(1b) # &.got.plt + 0x0023_5313, // srli t1, t1, 2 # .plt entry offset + 0x0042_a283, // lw t0, 4(t0) # link map + 0x000e_0067, // jr t3 + ]; + let insn = if IS_64 { &INSN_64 } else { &INSN_32 }; + for (i, &v) in insn.iter().enumerate() { + write32(&mut buf[i * 4..], v); + } + let disp = ctx + .gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr); + write_utype(buf, disp); + write_itype(&mut buf[8..], disp); + write_itype(&mut buf[16..], disp); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_plt_stub::(buf, sym.gotplt_addr(ctx).wrapping_sub(sym.plt_addr(ctx))); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + write_plt_stub::( + buf, + sym.got_pltgot_addr(ctx).wrapping_sub(sym.plt_addr(ctx)), + ); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + let check = |val: i64, lo: i64, hi: i64| eh_frame::check_range(ctx, isec, rel, val, lo, hi); + match rel.r_type { + R_NONE => {} + R_RISCV_ADD32 => End::write_u32(loc, End::read_u32(loc).wrapping_add(val as u32)), + R_RISCV_SUB8 => loc[0] = loc[0].wrapping_sub(val as u8), + R_RISCV_SUB16 => End::write_u16(loc, End::read_u16(loc).wrapping_sub(val as u16)), + R_RISCV_SUB32 => End::write_u32(loc, End::read_u32(loc).wrapping_sub(val as u32)), + R_RISCV_SUB6 => { + loc[0] = (loc[0] & 0b1100_0000) | (loc[0].wrapping_sub(val as u8) & 0b0011_1111) + } + R_RISCV_SET6 => loc[0] = (loc[0] & 0b1100_0000) | (val as u8 & 0b0011_1111), + R_RISCV_SET8 => loc[0] = val as u8, + R_RISCV_SET16 => End::write_u16(loc, val as u16), + R_RISCV_SET32 => End::write_u32(loc, val as u32), + R_RISCV_32_PCREL => { + check(val.wrapping_sub(p) as i64, -(1 << 31), 1 << 31); + End::write_u32(loc, val.wrapping_sub(p) as u32); + } + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_RISCV_32 => { + if IS_64 { + scan_absrel(ctx, isec, sym, &rel); + } + } + R_RISCV_HI20 => scan_absrel(ctx, isec, sym, &rel), + R_RISCV_CALL | R_RISCV_CALL_PLT | R_RISCV_PLT32 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_RISCV_GOT_HI20 | R_RISCV_GOT32_PCREL => sym.add_flags(NEEDS_GOT), + R_RISCV_TLS_GOT_HI20 => sym.add_flags(NEEDS_GOTTP), + R_RISCV_TLS_GD_HI20 => sym.add_flags(NEEDS_TLSGD), + R_RISCV_TLSDESC_HI20 => scan_tlsdesc(ctx, sym), + R_RISCV_32_PCREL | R_RISCV_PCREL_HI20 => scan_pcrel(ctx, isec, sym, &rel), + R_RISCV_TPREL_HI20 => check_tlsle(ctx, isec, sym, &rel), + R_RISCV_64 + | R_RISCV_BRANCH + | R_RISCV_JAL + | R_RISCV_PCREL_LO12_I + | R_RISCV_PCREL_LO12_S + | R_RISCV_LO12_I + | R_RISCV_LO12_S + | R_RISCV_TPREL_LO12_I + | R_RISCV_TPREL_LO12_S + | R_RISCV_TPREL_ADD + | R_RISCV_TLSDESC_LOAD_LO12 + | R_RISCV_TLSDESC_ADD_LO12 + | R_RISCV_TLSDESC_CALL + | R_RISCV_ADD8 + | R_RISCV_ADD16 + | R_RISCV_ADD32 + | R_RISCV_ADD64 + | R_RISCV_SUB8 + | R_RISCV_SUB16 + | R_RISCV_SUB32 + | R_RISCV_SUB64 + | R_RISCV_ALIGN + | R_RISCV_RVC_BRANCH + | R_RISCV_RVC_JUMP + | R_RISCV_RELAX + | R_RISCV_SUB6 + | R_RISCV_SET6 + | R_RISCV_SET8 + | R_RISCV_SET16 + | R_RISCV_SET32 + | R_RISCV_SET_ULEB128 + | R_RISCV_SUB_ULEB128 => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let contents = isec.original_contents(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE || rel.r_type == R_RISCV_RELAX { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let (removed, delta) = isec.removed_at(rel); + let r_offset = rel.r_offset - delta as u64; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + r_offset; + let got = ctx.got.hdr.shdr.sh_addr; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i - 1, val, lo, hi); + let utype = |loc: &mut [u8], val: u64| { + check(val as i64, -(1i64 << 31) - 0x800, (1i64 << 31) - 0x800); + write_utype(loc, val); + }; + let loc = &mut buf[r_offset as usize..]; + let orig = &contents[rel.r_offset as usize..]; + + match rel.r_type { + R_RISCV_32 => { + if IS_64 { + End::write_u32(loc, sa as u32); + } + } + // Handled as absolute relocations by the output section. + R_RISCV_64 => {} + R_RISCV_BRANCH => { + check(pcrel as i64, -(1 << 12), 1 << 12); + write_btype(loc, pcrel); + } + R_RISCV_JAL => { + check(pcrel as i64, -(1 << 20), 1 << 20); + write_jtype(loc, pcrel); + } + R_RISCV_CALL | R_RISCV_CALL_PLT => { + let rd = rd(&orig[4..]); + if removed == 4 { + // auipc + jalr -> jal + write32(loc, (rd << 7) | 0b1101111); + write_jtype(loc, pcrel); + } else if removed == 6 && rd == 0 { + // auipc + jalr -> c.j + write16(loc, 0b101_00000000000_01); + write_cjtype(loc, pcrel); + } else if removed == 6 && rd == 1 { + // auipc + jalr -> c.jal (RV32 only) + debug_assert!(!IS_64); + write16(loc, 0b001_00000000000_01); + write_cjtype(loc, pcrel); + } else { + debug_assert_eq!(removed, 0); + if !sym.is_remaining_undef_weak() { + check(pcrel as i64, -(1i64 << 31) - 0x800, (1i64 << 31) - 0x800); + } + write_utype(loc, pcrel); + write_itype(&mut loc[4..], pcrel); + } + } + R_RISCV_GOT_HI20 => { + // This relocation usually heads an AUIPC+LD pair loading + // a symbol value from the GOT. If the value is a + // link-time constant, it can be materialized directly. + let rd = rd(orig); + if removed == 6 { + // c.li rd, val + write16(loc, 0b010_0_00000_00000_01 | (rd as u16) << 7); + write_citype(loc, s); + i += 3; + } else if removed == 4 { + // addi rd, zero, val + write32(loc, 0b0010011 | (rd << 7)); + write_itype(loc, s); + i += 3; + } else { + debug_assert_eq!(removed, 0); + if ctx.args.relax + && sym.is_pcrel_linktime_const(ctx) + && is_got_load_pair(ctx, isec, i - 1) + && is_int(pcrel as i64, 32) + { + // auipc rd, %hi20(val); addi rd, rd, %lo12(val) + utype(loc, pcrel); + write32(&mut loc[4..], 0b0010011 | (rd << 15) | (rd << 7)); + write_itype(&mut loc[4..], pcrel); + i += 3; + } else { + utype(loc, g().wrapping_add(got).wrapping_add(a).wrapping_sub(p)); + } + } + } + R_RISCV_TLS_GOT_HI20 => { + utype(loc, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p)) + } + R_RISCV_TLS_GD_HI20 => { + utype(loc, sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(p)) + } + R_RISCV_PCREL_HI20 => utype(loc, pcrel), + R_RISCV_PCREL_LO12_I | R_RISCV_PCREL_LO12_S => { + let j = find_paired_reloc(ctx, isec, sym, i - 1); + let rel2 = &rels.at(j); + let sym2 = &ctx.symbols[file.base.symbols[rel2.r_sym as usize]]; + let write = if rel.r_type == R_RISCV_PCREL_LO12_I { + write_itype + } else { + write_stype + }; + let a2 = rel2.r_addend as u64; + let p2 = isec.addr(ctx) + rel2.r_offset - r_delta(isec, rel2.r_offset) as u64; + match rel2.r_type { + R_RISCV_GOT_HI20 => { + write(loc, sym2.got_addr(ctx).wrapping_add(a2).wrapping_sub(p2)) + } + R_RISCV_TLS_GOT_HI20 => { + write(loc, sym2.gottp_addr(ctx).wrapping_add(a2).wrapping_sub(p2)) + } + R_RISCV_TLS_GD_HI20 => { + write(loc, sym2.tlsgd_addr(ctx).wrapping_add(a2).wrapping_sub(p2)) + } + R_RISCV_PCREL_HI20 => { + write(loc, sym2.addr(ctx).wrapping_add(a2).wrapping_sub(p2)) + } + _ => {} + } + } + R_RISCV_HI20 => { + // `lui` materializes the upper bits of a link-time + // constant; it may have been compressed to `c.lui` or + // removed by relaxation. + if removed == 2 { + let rd = rd(orig); + write16(loc, 0b011_0_00000_00000_01 | (rd as u16) << 7); + write_citype(loc, sa.wrapping_add(0x800) >> 12); + } else if removed == 0 { + utype(loc, sa); + } + } + R_RISCV_LO12_I | R_RISCV_LO12_S => { + if rel.r_type == R_RISCV_LO12_I { + write_itype(loc, sa); + } else { + write_stype(loc, sa); + } + // If the address fits in 12 bits, the `lui` may have + // been removed, so address relative to the zero + // register. + if is_int(sa as i64, 12) { + set_rs1(loc, 0); + } + } + R_RISCV_TPREL_HI20 => { + debug_assert!(removed == 0 || removed == 4); + if removed == 0 { + utype(loc, sa.wrapping_sub(ctx.tp_addr)); + } + } + // This only annotates an ADD that relaxation may remove. + R_RISCV_TPREL_ADD => {} + R_RISCV_TPREL_LO12_I | R_RISCV_TPREL_LO12_S => { + let val = sa.wrapping_sub(ctx.tp_addr); + if rel.r_type == R_RISCV_TPREL_LO12_I { + write_itype(loc, val); + } else { + write_stype(loc, val); + } + // If the offset fits in 12 bits, address relative to + // tp (x4) directly. + if is_int(val as i64, 12) { + set_rs1(loc, 4); + } + } + // TLSDESC materializes a TP-relative address in a0: + // + // .L0: + // auipc tX, 0 # R_RISCV_TLSDESC_HI20 foo + // l[d|w] tY, tX, 0 # R_RISCV_TLSDESC_LOAD_LO12_I .L0 + // addi a0, tX, 0 # R_RISCV_TLSDESC_ADD_LO12_I .L0 + // jalr t0, tY # R_RISCV_TLSDESC_CALL .L0 + // + // Without a descriptor, the first two instructions are + // deleted by relaxation and the rest becomes either + // `auipc a0, %gottp_hi; l[d|w] a0, %gottp_lo(a0)`, or for an + // executable `addi a0, zero, %tpoff_lo` (the addi also + // deleted) or `lui a0, %tpoff_hi; addi a0, a0, %tpoff_lo`. + // Without relaxation the useless instructions remain. + R_RISCV_TLSDESC_HI20 => { + if sym.has_tlsdesc(&ctx.symbols) && removed == 0 { + utype(loc, sym.tlsdesc_addr(ctx).wrapping_add(a).wrapping_sub(p)); + } + } + R_RISCV_TLSDESC_LOAD_LO12 | R_RISCV_TLSDESC_ADD_LO12 | R_RISCV_TLSDESC_CALL => { + if removed == 4 { + continue; + } + let j = find_paired_reloc(ctx, isec, sym, i - 1); + let rel2 = &rels.at(j); + let sym2 = &ctx.symbols[file.base.symbols[rel2.r_sym as usize]]; + let a2 = rel2.r_addend as u64; + let p2 = isec.addr(ctx) + rel2.r_offset - r_delta(isec, rel2.r_offset) as u64; + let tprel = sym2.addr(ctx).wrapping_add(a2).wrapping_sub(ctx.tp_addr); + match rel.r_type { + R_RISCV_TLSDESC_LOAD_LO12 => { + if sym2.has_tlsdesc(&ctx.symbols) { + write_itype( + loc, + sym2.tlsdesc_addr(ctx).wrapping_add(a2).wrapping_sub(p2), + ); + } else { + write32(loc, NOP); + } + } + R_RISCV_TLSDESC_ADD_LO12 => { + if sym2.has_tlsdesc(&ctx.symbols) { + write_itype( + loc, + sym2.tlsdesc_addr(ctx).wrapping_add(a2).wrapping_sub(p2), + ); + } else if sym2.has_gottp(&ctx.symbols) { + write32(loc, 0x517); // auipc a0, + utype(loc, sym2.gottp_addr(ctx).wrapping_add(a2).wrapping_sub(p2)); + } else { + write32(loc, 0x537); // lui a0, + utype(loc, tprel); + } + } + _ => { + if sym2.has_tlsdesc(&ctx.symbols) { + // Nothing to do. + } else if sym2.has_gottp(&ctx.symbols) { + write32(loc, if IS_64 { 0x53503 } else { 0x52503 }); // l[d|w] a0, + write_itype( + loc, + sym2.gottp_addr(ctx).wrapping_add(a2).wrapping_sub(p2), + ); + } else { + write32( + loc, + if is_int(tprel as i64, 12) { + 0x513 + } else { + 0x50513 + }, + ); // addi a0, zero|a0, + write_itype(loc, tprel); + } + } + } + } + R_RISCV_ADD8 => loc[0] = loc[0].wrapping_add(sa as u8), + R_RISCV_ADD16 => End::write_u16(loc, End::read_u16(loc).wrapping_add(sa as u16)), + R_RISCV_ADD32 => End::write_u32(loc, End::read_u32(loc).wrapping_add(sa as u32)), + R_RISCV_ADD64 => End::write_u64(loc, End::read_u64(loc).wrapping_add(sa)), + R_RISCV_SUB8 => loc[0] = loc[0].wrapping_sub(sa as u8), + R_RISCV_SUB16 => End::write_u16(loc, End::read_u16(loc).wrapping_sub(sa as u16)), + R_RISCV_SUB32 => End::write_u32(loc, End::read_u32(loc).wrapping_sub(sa as u32)), + R_RISCV_SUB64 => End::write_u64(loc, End::read_u64(loc).wrapping_sub(sa)), + R_RISCV_ALIGN => { + // R_RISCV_ALIGN is followed by NOPs, some of which may + // have been removed to align the next instruction. The + // whole NOP sequence is rewritten so that it stays valid + // (the first two bytes of a 4-byte NOP can't go alone). + let padding = (rel.r_addend - removed) as usize; + debug_assert_eq!(padding & 1, 0); + let mut k = 0; + while k + 4 <= padding { + write32(&mut loc[k..], NOP); + k += 4; + } + if k < padding { + write16(&mut loc[k..], 0x0001); // c.nop + } + } + R_RISCV_RVC_BRANCH => { + check(pcrel as i64, -(1 << 8), 1 << 8); + write_cbtype(loc, pcrel); + } + R_RISCV_RVC_JUMP => { + check(pcrel as i64, -(1 << 11), 1 << 11); + write_cjtype(loc, pcrel); + } + R_RISCV_SUB6 => { + loc[0] = (loc[0] & 0b1100_0000) | (loc[0].wrapping_sub(sa as u8) & 0b0011_1111) + } + R_RISCV_SET6 => loc[0] = (loc[0] & 0b1100_0000) | (sa as u8 & 0b0011_1111), + R_RISCV_SET8 => loc[0] = sa as u8, + R_RISCV_SET16 => End::write_u16(loc, sa as u16), + R_RISCV_SET32 => End::write_u32(loc, sa as u32), + R_RISCV_PLT32 | R_RISCV_32_PCREL => End::write_u32(loc, pcrel as u32), + R_RISCV_GOT32_PCREL => End::write_u32( + loc, + g().wrapping_add(got).wrapping_add(a).wrapping_sub(p) as u32, + ), + R_RISCV_SET_ULEB128 => overwrite_uleb(loc, sa), + R_RISCV_SUB_ULEB128 => { + let cur = read_uleb(&mut &loc[..]); + overwrite_uleb(loc, cur.wrapping_sub(sa)); + } + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let off = rel.r_offset as usize; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let frag_ref = frag.map(|(f, _)| f); + let sa = s.wrapping_add(a); + let loc = &mut buf[off..]; + + match rel.r_type { + R_RISCV_32 => End::write_u32(loc, sa as u32), + R_RISCV_64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => End::write_u64(loc, v), + None => End::write_u64(loc, sa), + }, + R_RISCV_TLS_DTPREL32 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => End::write_u32(loc, v as u32), + None => End::write_u32(loc, sa.wrapping_sub(ctx.dtp_addr) as u32), + }, + R_RISCV_TLS_DTPREL64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => End::write_u64(loc, v), + None => End::write_u64(loc, sa.wrapping_sub(ctx.dtp_addr)), + }, + R_RISCV_ADD8 => loc[0] = loc[0].wrapping_add(sa as u8), + R_RISCV_ADD16 => End::write_u16(loc, End::read_u16(loc).wrapping_add(sa as u16)), + R_RISCV_ADD32 => End::write_u32(loc, End::read_u32(loc).wrapping_add(sa as u32)), + R_RISCV_ADD64 => End::write_u64(loc, End::read_u64(loc).wrapping_add(sa)), + R_RISCV_SUB8 => loc[0] = loc[0].wrapping_sub(sa as u8), + R_RISCV_SUB16 => End::write_u16(loc, End::read_u16(loc).wrapping_sub(sa as u16)), + R_RISCV_SUB32 => End::write_u32(loc, End::read_u32(loc).wrapping_sub(sa as u32)), + R_RISCV_SUB64 => End::write_u64(loc, End::read_u64(loc).wrapping_sub(sa)), + R_RISCV_SUB6 => { + loc[0] = (loc[0] & 0b1100_0000) | (loc[0].wrapping_sub(sa as u8) & 0b0011_1111) + } + R_RISCV_SET6 => loc[0] = (loc[0] & 0b1100_0000) | (sa as u8 & 0b0011_1111), + R_RISCV_SET8 => loc[0] = sa as u8, + R_RISCV_SET16 => End::write_u16(loc, sa as u16), + R_RISCV_SET32 => End::write_u32(loc, sa as u32), + R_RISCV_SET_ULEB128 => overwrite_uleb(loc, sa), + R_RISCV_SUB_ULEB128 => { + let cur = read_uleb(&mut &loc[..]); + overwrite_uleb(loc, cur.wrapping_sub(sa)); + } + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, i: usize) -> u32 { + if !isec.is_alloc() { + return rel.r_type; + } + let rels = isec.rels::(&ctx.objs[isec.file.index()]); + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let (removed, _) = isec.removed_at(rel); + + match rel.r_type { + R_RISCV_CALL | R_RISCV_CALL_PLT if removed == 4 => R_RISCV_JAL, + R_RISCV_CALL | R_RISCV_CALL_PLT if removed == 6 => R_RISCV_RVC_JUMP, + R_RISCV_GOT_HI20 | R_RISCV_HI20 | R_RISCV_TPREL_HI20 | R_RISCV_TPREL_ADD + if removed != 0 => + { + R_NONE + } + R_RISCV_PCREL_LO12_I | R_RISCV_PCREL_LO12_S => { + // The load of a materialized GOT value is gone with it. + let j = find_paired_reloc(ctx, isec, sym, i); + if rels.at(j).r_type == R_RISCV_GOT_HI20 && isec.removed_at(&rels.at(j)).0 != 0 { + R_NONE + } else { + rel.r_type + } + } + R_RISCV_TLSDESC_HI20 if !sym.has_tlsdesc(&ctx.symbols) => R_NONE, + R_RISCV_TLSDESC_LOAD_LO12 | R_RISCV_TLSDESC_ADD_LO12 | R_RISCV_TLSDESC_CALL => { + let j = find_paired_reloc(ctx, isec, sym, i); + let sym2 = &ctx.symbols[file.base.symbols[rels.at(j).r_sym as usize]]; + if sym2.has_tlsdesc(&ctx.symbols) { + rel.r_type + } else { + R_NONE + } + } + _ => rel.r_type, + } + } + + fn shrink_section(ctx: &Context, isec: &InputSection) -> Vec { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let contents = isec.original_contents(file); + let mut deltas: Vec = Vec::new(); + let mut delta = 0i64; + + // Whether 2-byte instructions may be used. They usually may on + // Unix, since RV64GC is the common baseline. + let use_rvc = file.base.e_flags & EF_RISCV_RVC != 0; + + // Records that `d` bytes go away at relocation `r`. + fn record(deltas: &mut Vec, delta: &mut i64, r: &ElfRel, d: i64) { + *delta += d; + deltas.push(RelocDelta { + offset: r.r_offset, + delta: *delta, + }); + } + + for i in 0..rels.len() { + let r = &rels.at(i); + let sym = &ctx.symbols[file.base.symbols[r.r_sym as usize]]; + + // R_RISCV_ALIGN must be handled: it refers to NOPs, some or all + // of which are removed so that the following instruction is + // aligned. r_addend holds the number of NOP bytes, and the + // alignment is the smallest power of two greater than that, + // since the assembler emits enough NOPs for the worst case. + if r.r_type == R_RISCV_ALIGN { + let p = isec.addr(ctx) + r.r_offset - delta as u64; + let desired = align_to(p, (r.r_addend as u64 + 1).next_power_of_two()); + let actual = p + r.r_addend as u64; + if desired != actual { + record(&mut deltas, &mut delta, r, (actual - desired) as i64); + } + continue; + } + + // Other relaxations are optional. + if !ctx.args.relax || i + 1 == rels.len() || rels.at(i + 1).r_type != R_RISCV_RELAX { + continue; + } + + // Linker-synthesized symbols get their values only once the + // layout is fixed, so they're never relaxed against. + if sym.file() == ctx.internal_obj.map(FileId::Obj) { + continue; + } + + let mut remove = |d: i64| record(&mut deltas, &mut delta, r, d); + + match r.r_type { + R_RISCV_CALL | R_RISCV_CALL_PLT => { + // AUIPC+JALR reaches ±2 GiB; C.J, C.JAL or JAL do for + // nearer targets. + let dist = compute_distance(ctx, sym, isec, r); + if dist & 1 != 0 { + continue; + } + let rd = rd(&contents[r.r_offset as usize + 4..]); + if use_rvc && rd == 0 && is_int(dist, 12) { + // x0 and within ±2 KiB: C.J saves 6 bytes. + remove(6); + } else if use_rvc && !IS_64 && rd == 1 && is_int(dist, 12) { + // x1 and within ±2 KiB: C.JAL, which is RV32-only. + remove(6); + } else if is_int(dist, 21) { + // Within ±1 MiB: JAL. + remove(4); + } + } + R_RISCV_GOT_HI20 => { + // A GOT load of a link-time constant becomes a direct + // materialization of the value. + if sym.is_absolute() && is_got_load_pair(ctx, isec, i) { + let val = sym.addr(ctx).wrapping_add(r.r_addend as u64) as i64; + if use_rvc && is_int(val, 6) && rd(&contents[r.r_offset as usize..]) != 0 { + remove(6); // AUIPC+LD -> C.LI + } else if is_int(val, 12) { + remove(4); // AUIPC+LD -> ADDI + } + } + } + R_RISCV_HI20 => { + let val = sym.addr(ctx).wrapping_add(r.r_addend as u64) as i64; + let rd = rd(&contents[r.r_offset as usize..]); + if is_int(val, 12) { + // `lui t0, %hi(foo); add t0, t0, %lo(foo)` becomes + // `add t0, x0, %lo(foo)` if bits 32..11 of foo are all + // ones or all zeros. + remove(4); + } else if use_rvc && rd != 0 && rd != 2 && is_int(val + 0x800, 18) { + // The upper 20 bits fit in 6 bits: C.LUI. + remove(2); + } + } + R_RISCV_TPREL_HI20 | R_RISCV_TPREL_ADD => { + // `lui t0, %tprel_hi(foo); add t0, t0, tp` compute + // TP + %tprel_hi20(foo), which the low 12-bit access is + // relative to. Within TP ± 2 KiB that is TP itself, so + // both instructions go and the access uses tp directly. + let val = sym + .addr(ctx) + .wrapping_add(r.r_addend as u64) + .wrapping_sub(ctx.tp_addr) as i64; + if is_int(val, 12) { + remove(4); + } + } + R_RISCV_TLSDESC_HI20 => { + if !sym.has_tlsdesc(&ctx.symbols) { + remove(4); + } + } + R_RISCV_TLSDESC_LOAD_LO12 | R_RISCV_TLSDESC_ADD_LO12 => { + let j = find_paired_reloc(ctx, isec, sym, i); + let rel2 = &rels.at(j); + let sym2 = &ctx.symbols[file.base.symbols[rel2.r_sym as usize]]; + if r.r_type == R_RISCV_TLSDESC_LOAD_LO12 { + if !sym2.has_tlsdesc(&ctx.symbols) { + remove(4); + } + } else if !sym2.has_tlsdesc(&ctx.symbols) && !sym2.has_gottp(&ctx.symbols) { + let val = sym2 + .addr(ctx) + .wrapping_add(rel2.r_addend as u64) + .wrapping_sub(ctx.tp_addr) as i64; + if is_int(val, 12) { + remove(4); + } + } + } + _ => {} + } + } + deltas + } +} + +/// Writes a PLT stub reaching the GOT entry at `disp` from the stub. +fn write_plt_stub(buf: &mut [u8], disp: u64) { + const ENTRY_64: [u32; 4] = [ + 0x0000_0e17, // auipc t3, %pcrel_hi(function@.got.plt) + 0x000e_3e03, // ld t3, %pcrel_lo(1b)(t3) + 0x000e_0367, // jalr t1, t3 + 0x0010_0073, // ebreak + ]; + const ENTRY_32: [u32; 4] = [ + 0x0000_0e17, // auipc t3, %pcrel_hi(function@.got.plt) + 0x000e_2e03, // lw t3, %pcrel_lo(1b)(t3) + 0x000e_0367, // jalr t1, t3 + 0x0010_0073, // ebreak + ]; + let entry = if IS_64 { &ENTRY_64 } else { &ENTRY_32 }; + for (i, &v) in entry.iter().enumerate() { + write32(&mut buf[i * 4..], v); + } + write_utype(buf, disp); + write_itype(&mut buf[4..], disp); +} + +// ISA strings +// +// An ISA string such as "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0" names +// the base ISA followed by extensions, each with a mandatory major and +// minor version ("m2p0" is the "m" extension, version 2.0). Single-letter +// extensions come first; "z" extensions are named by several letters, +// and "s" and "x" prefixes are reserved for supervisor-level and private +// extensions. Every input object records the string of the extensions it +// uses, and the output gets the merged string. Extensions must appear in +// a specific, not quite alphabetical order for the string to be unique. + +#[derive(Clone, Debug)] +struct Extension { + name: String, + major: u64, + minor: u64, +} + +/// Whether extension `x` must precede extension `y`. For example, +/// rv64imafd is legal but rv64iafdm isn't. +fn extension_precedes(x: &str, y: &str) -> bool { + fn single_letter_rank(c: u8) -> i64 { + const ORDER: &[u8] = b"iemafdqlcbkjtpvnh"; + match ORDER.iter().position(|&e| e == c) { + Some(pos) => pos as i64, + None => (c - b'a') as i64 + ORDER.len() as i64, + } + } + fn rank(s: &str) -> i64 { + let bytes = s.as_bytes(); + match bytes[0] { + b'x' => 1 << 20, + b's' => 1 << 19, + b'z' => (1 << 18) + single_letter_rank(bytes[1]), + c => single_letter_rank(c), + } + } + (rank(x), x) < (rank(y), y) +} + +/// Parses an ISA string into its extensions. Each element is a name +/// (letters and digits, starting and ending with a letter) followed by +/// `p`; elements are separated by underscores. +fn parse_arch_string(s: &[u8]) -> Option> { + let mut result = Vec::new(); + for element in s.split(|&b| b == b'_') { + let element = std::str::from_utf8(element).ok()?; + let end_of_minor = element.len(); + let minor_start = element.rfind(|c: char| !c.is_ascii_digit())? + 1; + if minor_start == end_of_minor || !element[..minor_start].ends_with('p') { + return None; + } + let major_end = minor_start - 1; + let major_start = element[..major_end].rfind(|c: char| !c.is_ascii_digit())? + 1; + if major_start == major_end { + return None; + } + let name = &element[..major_start]; + let ok_name = name.bytes().next().is_some_and(|c| c.is_ascii_lowercase()) + && name.bytes().last().is_some_and(|c| c.is_ascii_lowercase()) + && name + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()); + if !ok_name { + return None; + } + result.push(Extension { + name: name.to_string(), + major: element[major_start..major_end].parse().ok()?, + minor: element[minor_start..].parse().ok()?, + }); + } + (!result.is_empty()).then_some(result) +} + +/// Merges two extension lists, keeping the newer version of an extension +/// present in both. The base ISAs must match. +fn merge_extensions(x: &[Extension], y: &[Extension]) -> Option> { + if x[0].name != y[0].name { + return None; + } + let mut result = Vec::new(); + let (mut x, mut y) = (x, y); + while let (Some(a), Some(b)) = (x.first(), y.first()) { + if a.name == b.name { + result.push(if (a.major, a.minor) < (b.major, b.minor) { + b.clone() + } else { + a.clone() + }); + x = &x[1..]; + y = &y[1..]; + } else if extension_precedes(&a.name, &b.name) { + result.push(a.clone()); + x = &x[1..]; + } else { + result.push(b.clone()); + y = &y[1..]; + } + } + result.extend(x.iter().cloned()); + result.extend(y.iter().cloned()); + Some(result) +} + +fn arch_string(extensions: &[Extension]) -> String { + extensions + .iter() + .map(|e| format!("{}{}p{}", e.name, e.major, e.minor)) + .collect::>() + .join("_") +} + +/// The contents of the output `.riscv.attributes`: the merged ISA string +/// and stack alignment of the inputs. +pub fn attributes_contents(ctx: &Context) -> Vec { + let mut stack: Option = None; + let mut arch: Vec = Vec::new(); + let mut unaligned = false; + + for file in &ctx.objs { + let attrs = &file.riscv_attributes; + if let Some(val) = attrs.stack_align { + if stack.is_some_and(|s| s != val) { + error!(ctx, "{file}: stack alignment requirement mistmatch"); + } + stack = Some(val); + } + if let Some(s) = attrs.arch { + let Some(arch2) = parse_arch_string(s) else { + error!( + ctx, + "{file}: corrupted .riscv.attributes ISA string: {}", + String::from_utf8_lossy(s) + ); + continue; + }; + if arch.is_empty() { + arch = arch2; + } else { + match merge_extensions(&arch, &arch2) { + Some(merged) => arch = merged, + None => error!( + ctx, + "{file}: incompatible .riscv.attributes ISA string: {}", + String::from_utf8_lossy(s) + ), + } + } + } + if attrs.unaligned_access { + unaligned = true; + } + } + if arch.is_empty() { + return Vec::new(); + } + + // Format version, then one "riscv" sub-section holding a file-scoped + // sub-sub-section of tagged attributes. Both carry their length. + let mut attributes = Vec::new(); + if let Some(stack) = stack { + encode_uleb(&mut attributes, ELF_TAG_RISCV_STACK_ALIGN as u64); + encode_uleb(&mut attributes, stack); + } + encode_uleb(&mut attributes, ELF_TAG_RISCV_ARCH as u64); + attributes.extend_from_slice(arch_string(&arch).as_bytes()); + attributes.push(0); + if unaligned { + encode_uleb(&mut attributes, ELF_TAG_RISCV_UNALIGNED_ACCESS as u64); + encode_uleb(&mut attributes, 1); + } + + let sub_sub_size = 1 + 4 + attributes.len(); + let sub_size = 4 + b"riscv\0".len() + sub_sub_size; + let u32_bytes = |v: u32| { + let mut bytes = [0u8; 4]; + E::Endian::write_u32(&mut bytes, v); + bytes + }; + let mut out = vec![b'A']; + out.extend_from_slice(&u32_bytes(sub_size as u32)); + out.extend_from_slice(b"riscv\0"); + out.push(ELF_TAG_FILE as u8); + out.extend_from_slice(&u32_bytes(sub_sub_size as u32)); + out.extend_from_slice(&attributes); + out +} diff --git a/src/arch/s390x.rs b/src/arch/s390x.rs new file mode 100644 index 000000000..637f832be --- /dev/null +++ b/src/arch/s390x.rs @@ -0,0 +1,503 @@ +//! s390x, IBM's 64-bit z/Architecture. +//! +//! z/Architecture is a big-endian CISC ISA with 16 general-purpose +//! registers and 2-, 4- or 6-byte instructions aligned to 2 bytes. From a +//! linker's point of view it feels like an x86-64 from a parallel +//! universe. `%r0` and `%r1` are scratch registers the PLT may use, `%r12` +//! holds the GOT address in position-independent code, `%r14` the return +//! address and `%r15` the stack pointer. +//! +//! Thread-local storage works as elsewhere except that `__tls_get_offset` +//! takes the place of `__tls_get_addr` and returns an offset from the +//! thread pointer rather than an address. +//! +//! https://github.com/IBM/s390x-abi/releases/download/v1.6.1/lzsabi_s390x.pdf + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::util::{bits, is_int}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct S390x; + +impl Layout for S390x { + type Endian = BigEndian; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +fn w16(loc: &mut [u8], v: u16) { + BigEndian::write_u16(loc, v); +} + +fn w32(loc: &mut [u8], v: u32) { + BigEndian::write_u32(loc, v); +} + +fn w64(loc: &mut [u8], v: u64) { + BigEndian::write_u64(loc, v); +} + +fn r16(loc: &[u8]) -> u16 { + BigEndian::read_u16(loc) +} + +fn r32(loc: &[u8]) -> u32 { + BigEndian::read_u32(loc) +} + +/// Sets the 12-bit displacement field of a halfword. +fn or12(loc: &mut [u8], val: u64) { + w16(loc, r16(loc) | bits(val, 11, 0) as u16); +} + +/// Writes a 20-bit displacement, which is split into a 12-bit low part +/// and an 8-bit high part. +fn write_mid20(loc: &mut [u8], val: u64) { + w32( + loc, + r32(loc) | ((bits(val, 11, 0) << 16) | (bits(val, 19, 12) << 8)) as u32, + ); +} + +/// Whether the GOT-loading LGRL at `loc` (opcode 0xc4?8, preceded by +/// the relocated operand) can become an address-materializing LARL. +fn relaxes_gotent(ctx: &Context, isec: &InputSection, rel: &ElfRel, sym: &Symbol) -> bool { + if !ctx.args.relax || !sym.is_pcrel_linktime_const(ctx) || rel.r_offset < 2 { + return false; + } + let op = r16(&isec.contents()[rel.r_offset as usize - 2..]); + let val = sym + .addr(ctx) + .wrapping_add(rel.r_addend as u64) + .wrapping_sub(isec.addr(ctx) + rel.r_offset) as i64; + op & 0xff0f == 0xc408 && rel.r_addend == 2 && val & 1 == 0 && is_int(val, 33) +} + +impl Arch for S390x { + const NAME: &'static str = "s390x"; + const FAMILY: Family = Family::S390x; + const PAGE_SIZE: u64 = 4096; + const E_MACHINE: u32 = EM_S390X; + const PLT_HDR_SIZE: u64 = 48; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 16; + const SFRAME_ABI: Option = Some(SFRAME_ABI_S390X_ENDIAN_BIG); + const TRAP: &'static [u8] = &[0x00, 0x00]; // invalid + + const R_COPY: u32 = R_390_COPY; + const R_GLOB_DAT: u32 = R_390_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_390_JMP_SLOT; + const R_ABS: u32 = R_390_64; + const R_RELATIVE: u32 = R_390_RELATIVE; + const R_IRELATIVE: Option = Some(R_390_IRELATIVE); + const R_DTPOFF: u32 = R_390_TLS_DTPOFF; + const R_TPOFF: u32 = R_390_TLS_TPOFF; + const R_DTPMOD: u32 = R_390_TLS_DTPMOD; + const R_SFRAME: Option = Some(R_390_PC64); + const R_FUNCALL: &'static [u32] = &[R_390_PLT32DBL]; + + fn rel_to_string(r_type: u32) -> String { + s390x_rel_to_string(r_type) + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u8; 48] = [ + // The offset into .rela.plt is (%r0 - %r1 - 48 - 14) * 3/2, + // where %r0 is the PLT entry address plus 14, %r1 the start of + // .plt, and 48 the size of this header; PLT entries are 16 + // bytes and .rela.plt entries 24. + 0xb9, 0x09, 0x00, 0x01, // sgr %r0, %r1 + 0xa7, 0x0b, 0xff, 0xc2, // aghi %r0, -62 + 0xeb, 0x10, 0x00, 0x01, 0x00, 0x0c, // srlg %r1, %r0, 1 + 0xb9, 0x08, 0x00, 0x01, // agr %r0, %r1 + // Store the result at 56(%r15) and .got.plt[1] at 48(%r15), + // %r15 being the stack pointer. + 0xe3, 0x00, 0xf0, 0x38, 0x00, 0x24, // stg %r0, 56(%r15) + 0xc0, 0x10, 0, 0, 0, 0, // larl %r1, GOTPLT_OFFSET + 0xd2, 0x07, 0xf0, 0x30, 0x10, 0x08, // mvc 48(8, %r15), 8(%r1) + // Branch to _dl_runtime_resolve. + 0xe3, 0x10, 0x10, 0x10, 0x00, 0x04, // lg %r1, 16(%r1) + 0x07, 0xf1, // br %r1 + 0x00, 0x00, 0x00, 0x00, // (filler) + ]; + buf[..48].copy_from_slice(&INSN); + w32( + &mut buf[26..], + (ctx.gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.plt.hdr.shdr.sh_addr) + .wrapping_sub(24) + >> 1) as u32, + ); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u8; 16] = [ + 0xc0, 0x10, 0, 0, 0, 0, // larl %r1, GOTPLT_ENTRY_OFFSET + 0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg %r1, (%r1) + 0x0d, 0x01, // basr %r0, %r1 + 0x00, 0x00, // (filler) + ]; + buf[..16].copy_from_slice(&INSN); + w32( + &mut buf[2..], + (sym.gotplt_addr(ctx).wrapping_sub(sym.plt_addr(ctx)) >> 1) as u32, + ); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u8; 16] = [ + 0xc0, 0x10, 0, 0, 0, 0, // larl %r1, GOT_ENTRY_OFFSET + 0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg %r1, (%r1) + 0x07, 0xf1, // br %r1 + 0x00, 0x00, // (filler) + ]; + buf[..16].copy_from_slice(&INSN); + w32( + &mut buf[2..], + (sym.got_pltgot_addr(ctx).wrapping_sub(sym.plt_addr(ctx)) >> 1) as u32, + ); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + let check = |val: i64, lo: i64, hi: i64| eh_frame::check_range(ctx, isec, rel, val, lo, hi); + match rel.r_type { + R_NONE => {} + R_390_PC32 => { + check(val.wrapping_sub(p) as i64, -(1 << 31), 1 << 31); + w32(loc, val.wrapping_sub(p) as u32); + } + R_390_64 => w64(loc, val), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match rel.r_type { + R_390_8 | R_390_12 | R_390_16 | R_390_20 | R_390_32 => { + scan_absrel(ctx, isec, sym, &rel) + } + R_390_PC12DBL | R_390_PC16 | R_390_PC16DBL | R_390_PC24DBL | R_390_PC32 + | R_390_PC32DBL | R_390_PC64 => scan_pcrel(ctx, isec, sym, &rel), + R_390_GOT12 | R_390_GOT16 | R_390_GOT20 | R_390_GOT32 | R_390_GOT64 + | R_390_GOTOFF16 | R_390_GOTOFF32 | R_390_GOTOFF64 | R_390_GOTPLT12 + | R_390_GOTPLT16 | R_390_GOTPLT20 | R_390_GOTPLT32 | R_390_GOTPLT64 + | R_390_GOTPC | R_390_GOTPCDBL => sym.add_flags(NEEDS_GOT), + R_390_GOTENT => { + if !relaxes_gotent(ctx, isec, &rel, sym) { + sym.add_flags(NEEDS_GOT); + } + } + R_390_PLT12DBL | R_390_PLT16DBL | R_390_PLT24DBL | R_390_PLT32 | R_390_PLT32DBL + | R_390_PLT64 | R_390_PLTOFF16 | R_390_PLTOFF32 | R_390_PLTOFF64 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_390_TLS_GOTIE20 | R_390_TLS_IEENT => sym.add_flags(NEEDS_GOTTP), + R_390_TLS_GD32 | R_390_TLS_GD64 => { + // Calls to __tls_get_offset are always relaxed in a + // static executable, since libc.a's just aborts. + if ctx.args.is_static || (ctx.args.relax && sym.is_tprel_linktime_const(ctx)) { + // Nothing to do. + } else if ctx.args.relax && sym.is_tprel_runtime_const(ctx) { + sym.add_flags(NEEDS_GOTTP); + } else { + sym.add_flags(NEEDS_TLSGD); + } + } + R_390_TLS_LDM32 | R_390_TLS_LDM64 => { + if !(ctx.args.is_static || (ctx.args.relax && !ctx.args.shared)) { + ctx.needs_tlsld + .store(true, std::sync::atomic::Ordering::Relaxed); + } + } + R_390_TLS_LE32 | R_390_TLS_LE64 => check_tlsle(ctx, isec, sym, &rel), + R_390_64 | R_390_TLS_LDO32 | R_390_TLS_LDO64 | R_390_TLS_GDCALL + | R_390_TLS_LDCALL => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for (i, rel) in isec.rels::(file).iter().enumerate() { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let got = ctx.got.hdr.shdr.sh_addr; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + // *DBL relocations must not refer to odd addresses. + let check_dbl = |val: i64, lo: i64, hi: i64| { + check(val, lo, hi); + if val & 1 != 0 { + error!( + ctx, + "{}: misaligned symbol {sym} for relocation {}", + isec.display(file), + rel.type_name::() + ); + } + }; + + match rel.r_type { + // Handled as an absolute relocation by the output section. + R_390_64 => {} + R_390_8 => { + check(sa as i64, 0, 1 << 8); + buf[off] = sa as u8; + } + R_390_12 => { + check(sa as i64, 0, 1 << 12); + or12(&mut buf[off..], sa); + } + R_390_16 => { + check(sa as i64, 0, 1 << 16); + w16(&mut buf[off..], sa as u16); + } + R_390_20 => { + check(sa as i64, 0, 1 << 20); + write_mid20(&mut buf[off..], sa); + } + R_390_32 | R_390_PLT32 => { + check(sa as i64, 0, 1 << 32); + w32(&mut buf[off..], sa as u32); + } + R_390_PC12DBL | R_390_PLT12DBL => { + check_dbl(pcrel as i64, -(1 << 12), 1 << 12); + let v = r16(&buf[off..]) | bits(pcrel, 12, 1) as u16; + w16(&mut buf[off..], v); + } + R_390_PC16 => { + check(pcrel as i64, -(1 << 15), 1 << 15); + w16(&mut buf[off..], pcrel as u16); + } + R_390_PC32 => { + check(pcrel as i64, -(1 << 31), 1 << 31); + w32(&mut buf[off..], pcrel as u32); + } + R_390_PC64 | R_390_PLT64 => w64(&mut buf[off..], pcrel), + R_390_PC16DBL | R_390_PLT16DBL => { + check_dbl(pcrel as i64, -(1 << 16), 1 << 16); + w16(&mut buf[off..], (pcrel >> 1) as u16); + } + R_390_PC24DBL | R_390_PLT24DBL => { + check_dbl(pcrel as i64, -(1 << 24), 1 << 24); + let v = r32(&buf[off..]) | bits(pcrel, 24, 1) as u32; + w32(&mut buf[off..], v); + } + R_390_PC32DBL => { + check_dbl(pcrel as i64, -(1 << 32), 1 << 32); + w32(&mut buf[off..], (pcrel >> 1) as u32); + } + R_390_PLT32DBL => { + if !sym.is_remaining_undef_weak() { + check_dbl(pcrel as i64, -(1 << 32), 1 << 32); + } + w32(&mut buf[off..], (pcrel >> 1) as u32); + } + R_390_GOT12 | R_390_GOTPLT12 => { + check(g().wrapping_add(a) as i64, 0, 1 << 12); + or12(&mut buf[off..], g().wrapping_add(a)); + } + R_390_GOT16 | R_390_GOTPLT16 => { + check(g().wrapping_add(a) as i64, 0, 1 << 16); + w16(&mut buf[off..], g().wrapping_add(a) as u16); + } + R_390_GOT20 | R_390_GOTPLT20 => { + check(g().wrapping_add(a) as i64, 0, 1 << 20); + write_mid20(&mut buf[off..], g().wrapping_add(a)); + } + R_390_GOT32 | R_390_GOTPLT32 => { + check(g().wrapping_add(a) as i64, 0, 1 << 32); + w32(&mut buf[off..], g().wrapping_add(a) as u32); + } + R_390_GOT64 | R_390_GOTPLT64 => w64(&mut buf[off..], g().wrapping_add(a)), + R_390_GOTOFF16 | R_390_PLTOFF16 => { + check(sa.wrapping_sub(got) as i64, -(1 << 15), 1 << 15); + w16(&mut buf[off..], sa.wrapping_sub(got) as u16); + } + R_390_GOTOFF32 | R_390_PLTOFF32 => { + check(sa.wrapping_sub(got) as i64, -(1 << 31), 1 << 31); + w32(&mut buf[off..], sa.wrapping_sub(got) as u32); + } + R_390_GOTOFF64 | R_390_PLTOFF64 => w64(&mut buf[off..], sa.wrapping_sub(got)), + R_390_GOTPC => w64(&mut buf[off..], got.wrapping_add(a).wrapping_sub(p)), + R_390_GOTPCDBL => { + let val = got.wrapping_add(a).wrapping_sub(p); + check_dbl(val as i64, -(1 << 32), 1 << 32); + w32(&mut buf[off..], (val >> 1) as u32); + } + R_390_GOTENT => { + // A GOT-loading LGRL (0xc4?8 followed by a 32-bit offset) + // becomes an address-materializing LARL (0xc0?0) if the + // address is a link-time constant. + if relaxes_gotent(ctx, isec, &rel, sym) { + let op = r16(&buf[off - 2..]); + w16(&mut buf[off - 2..], 0xc000 | (op & 0x00f0)); + w32(&mut buf[off..], (pcrel >> 1) as u32); + } else { + let val = got.wrapping_add(g()).wrapping_add(a).wrapping_sub(p); + check_dbl(val as i64, -(1 << 32), 1 << 32); + w32(&mut buf[off..], (val >> 1) as u32); + } + } + R_390_TLS_LE32 => w32(&mut buf[off..], sa.wrapping_sub(ctx.tp_addr) as u32), + R_390_TLS_LE64 => w64(&mut buf[off..], sa.wrapping_sub(ctx.tp_addr)), + R_390_TLS_GOTIE20 => write_mid20( + &mut buf[off..], + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), + ), + R_390_TLS_IEENT => w32( + &mut buf[off..], + (sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p) >> 1) as u32, + ), + R_390_TLS_GD32 | R_390_TLS_GD64 => { + let val = if sym.has_tlsgd(&ctx.symbols) { + sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got) + } else if sym.has_gottp(&ctx.symbols) { + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got) + } else { + sa.wrapping_sub(ctx.tp_addr) + }; + if rel.r_type == R_390_TLS_GD32 { + w32(&mut buf[off..], val as u32); + } else { + w64(&mut buf[off..], val); + } + } + R_390_TLS_GDCALL => { + if sym.has_tlsgd(&ctx.symbols) { + // Nothing to do. + } else if sym.has_gottp(&ctx.symbols) { + buf[off..off + 6].copy_from_slice(&[0xe3, 0x22, 0xc0, 0x00, 0x00, 0x04]); + // lg %r2, 0(%r2, %r12) + } else { + buf[off..off + 6].copy_from_slice(&[0xc0, 0x04, 0x00, 0x00, 0x00, 0x00]); + // nop + } + } + R_390_TLS_LDM32 | R_390_TLS_LDM64 => { + let val = if ctx.got.has_tlsld() { + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got) + } else { + ctx.dtp_addr.wrapping_sub(ctx.tp_addr) + }; + if rel.r_type == R_390_TLS_LDM32 { + w32(&mut buf[off..], val as u32); + } else { + w64(&mut buf[off..], val); + } + } + R_390_TLS_LDCALL => { + if !ctx.got.has_tlsld() { + buf[off..off + 6].copy_from_slice(&[0xc0, 0x04, 0x00, 0x00, 0x00, 0x00]); + // nop + } + } + R_390_TLS_LDO32 => w32(&mut buf[off..], sa.wrapping_sub(ctx.dtp_addr) as u32), + R_390_TLS_LDO64 => w64(&mut buf[off..], sa.wrapping_sub(ctx.dtp_addr)), + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let off = rel.r_offset as usize; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let frag_ref = frag.map(|(f, _)| f); + let sa = s.wrapping_add(a); + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + + match rel.r_type { + R_390_32 => { + check(sa as i64, 0, 1 << 32); + w32(&mut buf[off..], sa as u32); + } + R_390_64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => w64(&mut buf[off..], v), + None => w64(&mut buf[off..], sa), + }, + R_390_TLS_LDO64 => match isec.tombstone(ctx, sym, frag_ref) { + Some(v) => w64(&mut buf[off..], v), + None => w64(&mut buf[off..], sa.wrapping_sub(ctx.dtp_addr)), + }, + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, _i: usize) -> u32 { + if rel.r_type == R_390_GOTENT && isec.is_alloc() { + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if relaxes_gotent(ctx, isec, rel, sym) { + return R_390_PC32DBL; + } + } + rel.r_type + } +} diff --git a/src/arch/sh4.rs b/src/arch/sh4.rs new file mode 100644 index 000000000..1935162e1 --- /dev/null +++ b/src/arch/sh4.rs @@ -0,0 +1,336 @@ +//! SH-4 (SuperH 4), a 32-bit RISC ISA from Hitachi, best known from +//! Sega's Dreamcast and still sold by Renesas for embedded systems. +//! +//! Its instructions are 16 bits long for code density, which shapes the +//! ISA: 16 general-purpose registers, two-operand arithmetic, large +//! immediates loaded from memory with PC-relative `mov.l` rather than +//! built with load-high/load-low pairs, and few enough opcodes that the +//! set of relocations a linker has to support is small. Like MIPS and +//! SPARC of the same era, it has a branch delay slot. +//! +//! Notes on the psABI: +//! +//! - Position-independent functions start by loading the address of the +//! GOT into `r12`, which the PLT relies on for position-independent +//! output. +//! - The relocations are of the RELA type, yet object files store addends +//! in the relocated section contents. Dynamic relocations follow the +//! usual RELA convention. +//! - GCC tends to put dynamically relocated data in `.text`, so outputs +//! contain plenty of text relocations. +//! - The ecosystem has bit-rotted; some programs using C++ exceptions +//! don't work even when linked with GNU ld. + +use std::marker::PhantomData; +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_pcrel, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Sh4Target(PhantomData); + +pub type Sh4 = Sh4Target; +pub type Sh4Be = Sh4Target; + +impl Layout for Sh4Target { + type Endian = End; + const IS_64: bool = false; + const IS_RELA: bool = true; +} + +/// Whether the relocation's addend lives in the relocated word. +fn addend_in_place(r_type: u32) -> bool { + matches!( + r_type, + R_SH_DIR32 + | R_SH_REL32 + | R_SH_TLS_GD_32 + | R_SH_TLS_LD_32 + | R_SH_TLS_LDO_32 + | R_SH_TLS_IE_32 + | R_SH_TLS_LE_32 + | R_SH_TLS_DTPMOD32 + | R_SH_TLS_DTPOFF32 + | R_SH_TLS_TPOFF32 + | R_SH_GOT32 + | R_SH_PLT32 + | R_SH_GOTOFF + | R_SH_GOTPC + | R_SH_GOTPLT32 + ) +} + +impl Sh4Target { + fn write_insns(buf: &mut [u8], insns: &[u16]) { + for (i, &insn) in insns.iter().enumerate() { + End::write_u16(&mut buf[i * 2..], insn); + } + } +} + +impl Arch for Sh4Target { + const NAME: &'static str = if End::IS_LITTLE { "sh4" } else { "sh4be" }; + const FAMILY: Family = Family::Sh4; + const PAGE_SIZE: u64 = 4096; + const E_MACHINE: u32 = EM_SH; + const PLT_HDR_SIZE: u64 = 16; + const PLT_SIZE: u64 = 20; + const PLTGOT_SIZE: u64 = 12; + const TRAP: &'static [u8] = if End::IS_LITTLE { + &[0xfd, 0xff] + } else { + &[0xff, 0xfd] + }; // illegal + + const R_COPY: u32 = R_SH_COPY; + const R_GLOB_DAT: u32 = R_SH_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_SH_JMP_SLOT; + const R_ABS: u32 = R_SH_DIR32; + const R_RELATIVE: u32 = R_SH_RELATIVE; + const R_DTPOFF: u32 = R_SH_TLS_DTPOFF32; + const R_TPOFF: u32 = R_SH_TLS_TPOFF32; + const R_DTPMOD: u32 = R_SH_TLS_DTPMOD32; + const R_FUNCALL: &'static [u32] = &[R_SH_PLT32]; + + fn rel_to_string(r_type: u32) -> String { + sh4_rel_to_string(r_type) + } + + fn get_addend(loc: &[u8], rel: &ElfRel) -> i64 { + if addend_in_place(rel.r_type) { + End::read_u32(loc) as i32 as i64 + } else { + 0 + } + } + + fn write_addend(loc: &mut [u8], val: i64, rel: &ElfRel) { + if addend_in_place(rel.r_type) { + End::write_u32(loc, val as u32); + } + } + + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + if ctx.args.pic { + const INSN: [u16; 6] = [ + 0xd202, // mov.l 1f, r2 + 0x32cc, // add r12, r2 + 0x5022, // mov.l @(8, r2), r0 + 0x5221, // mov.l @(4, r2), r2 + 0x402b, // jmp @r0 + 0xe000, // mov #0, r0 + ]; // 1: .long GOTPLT + Self::write_insns(buf, &INSN); + End::write_u32( + &mut buf[12..], + ctx.gotplt + .hdr + .shdr + .sh_addr + .wrapping_sub(ctx.got.hdr.shdr.sh_addr) as u32, + ); + } else { + const INSN: [u16; 6] = [ + 0xd202, // mov.l 1f, r2 + 0x5022, // mov.l @(8, r2), r0 + 0x5221, // mov.l @(4, r2), r2 + 0x402b, // jmp @r0 + 0xe000, // mov #0, r0 + 0xfffd, // (illegal) + ]; // 1: .long GOTPLT + Self::write_insns(buf, &INSN); + End::write_u32(&mut buf[12..], ctx.gotplt.hdr.shdr.sh_addr as u32); + } + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let gotplt = sym.gotplt_addr(ctx); + if ctx.args.pic { + const INSN: [u16; 6] = [ + 0xd002, // mov.l 1f, r0 + 0x00ce, // mov.l @(r0, r12), r0 + 0xd102, // mov.l 2f, r1 + 0x402b, // jmp @r0 + 0x0009, // nop + 0x0009, // nop + ]; // 1: .long GOTPLT_ENTRY; 2: .long INDEX_IN_RELPLT + Self::write_insns(buf, &INSN); + End::write_u32( + &mut buf[12..], + gotplt.wrapping_sub(ctx.got.hdr.shdr.sh_addr) as u32, + ); + } else { + const INSN: [u16; 6] = [ + 0xd002, // mov.l 1f, r0 + 0x6002, // mov.l @r0, r0 + 0xd102, // mov.l 2f, r1 + 0x402b, // jmp @r0 + 0x0009, // nop + 0x0009, // nop + ]; // 1: .long GOTPLT_ENTRY; 2: .long INDEX_IN_RELPLT + Self::write_insns(buf, &INSN); + End::write_u32(&mut buf[12..], gotplt as u32); + } + End::write_u32( + &mut buf[16..], + sym.plt_idx(&ctx.symbols).unwrap() * ElfRel::size::() as u32, + ); + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let got = sym.got_pltgot_addr(ctx); + if ctx.args.pic { + const INSN: [u16; 4] = [ + 0xd001, // mov.l 1f, r0 + 0x00ce, // mov.l @(r0, r12), r0 + 0x402b, // jmp @r0 + 0x0009, // nop + ]; // 1: .long GOT_ENTRY + Self::write_insns(buf, &INSN); + End::write_u32( + &mut buf[8..], + got.wrapping_sub(ctx.got.hdr.shdr.sh_addr) as u32, + ); + } else { + const INSN: [u16; 4] = [ + 0xd001, // mov.l 1f, r0 + 0x6002, // mov.l @r0, r0 + 0x402b, // jmp @r0 + 0x0009, // nop + ]; // 1: .long GOT_ENTRY + Self::write_insns(buf, &INSN); + End::write_u32(&mut buf[8..], got as u32); + } + } + + fn apply_eh_reloc( + ctx: &Context, + _isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match rel.r_type { + R_NONE => {} + R_SH_DIR32 => End::write_u32(loc, val as u32), + R_SH_REL32 => End::write_u32(loc, val.wrapping_sub(p) as u32), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, _| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + error!(ctx, "{sym}: GNU ifunc symbol is not supported on sh4"); + } + + match rel.r_type { + R_SH_REL32 => scan_pcrel(ctx, isec, sym, &rel), + R_SH_GOT32 => sym.add_flags(NEEDS_GOT), + R_SH_PLT32 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_SH_TLS_GD_32 => sym.add_flags(NEEDS_TLSGD), + R_SH_TLS_LD_32 => ctx.needs_tlsld.store(true, Ordering::Relaxed), + R_SH_TLS_IE_32 => sym.add_flags(NEEDS_GOTTP), + R_SH_TLS_LE_32 => check_tlsle(ctx, isec, sym, &rel), + R_SH_DIR32 | R_SH_GOTPC | R_SH_GOTOFF | R_SH_TLS_LDO_32 => {} + _ => fatal!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let got = ctx.got.hdr.shdr.sh_addr; + + for rel in isec.rels::(file) { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let s = sym.addr(ctx); + let a = isec.rel_addend::(&rel) as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let loc = &mut buf[rel.r_offset as usize..]; + + let val = match rel.r_type { + // Handled as an absolute relocation by the output section. + R_SH_DIR32 => continue, + R_SH_REL32 | R_SH_PLT32 => sa.wrapping_sub(p), + R_SH_GOT32 => g(), + R_SH_GOTPC => got.wrapping_add(a).wrapping_sub(p), + R_SH_GOTOFF => sa.wrapping_sub(got), + R_SH_TLS_GD_32 => sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got), + R_SH_TLS_LD_32 => ctx + .got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + R_SH_TLS_LDO_32 => sa.wrapping_sub(ctx.dtp_addr), + R_SH_TLS_IE_32 => sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), + R_SH_TLS_LE_32 => sa.wrapping_sub(ctx.tp_addr), + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + }; + End::write_u32(loc, val as u32); + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), isec.rel_addend::(&rel) as u64), + }; + let sa = s.wrapping_add(a); + let tombstone = isec.tombstone(ctx, sym, frag.map(|(f, _)| f)); + let loc = &mut buf[rel.r_offset as usize..]; + + match rel.r_type { + R_SH_DIR32 => End::write_u32(loc, tombstone.unwrap_or(sa) as u32), + R_SH_TLS_LDO_32 => End::write_u32( + loc, + tombstone.unwrap_or(sa.wrapping_sub(ctx.dtp_addr)) as u32, + ), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + } + } +} diff --git a/src/arch/sparc64.rs b/src/arch/sparc64.rs new file mode 100644 index 000000000..261a8495e --- /dev/null +++ b/src/arch/sparc64.rs @@ -0,0 +1,681 @@ +//! SPARC, Sun's big-endian RISC ISA, in its 64-bit form. +//! +//! All instructions are 4 bytes and 4-byte aligned. Unusually for a +//! RISC, SPARC needs no range extension thunks: CALL has a 30-bit +//! displacement scaled by 4, reaching PC ± 2 GiB at the cost of a +//! quarter of the opcode space. CALL saves the return address in `%o7` +//! and the thread pointer lives in `%g7`. +//! +//! There are no PC-relative loads or stores. Position-independent code +//! materializes the address of the GOT with a `sethi`/`add` pair and a +//! call to a `__sparc_get_pc_thunk` routine that adds `%o7` to it, +//! with NOPs and ADDs in the branch delay slots: SPARC always executes +//! the instruction after a branch. The GOT address isn't shared between +//! functions, so each function computes it into whatever register it +//! likes. +//! +//! Relocations carry a second addend in the upper 24 bits of the type +//! field; only `R_SPARC_OLO10` uses it. +//! +//! https://github.com/rui314/psabi/blob/main/sparc.pdf + +use std::sync::atomic::Ordering; + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::chunks::got::plt::SPARC_NUM_SMALL_PLT; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::util::{bit, bits}; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Sparc64; + +impl Layout for Sparc64 { + type Endian = BigEndian; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +/// The relocation type proper, without the second addend. +fn r_type(rel: &ElfRel) -> u32 { + rel.r_type & 0xff +} + +/// The second addend of `R_SPARC_OLO10`. +fn r_type_data(rel: &ElfRel) -> u64 { + (rel.r_type >> 8) as u64 +} + +fn r32(loc: &[u8]) -> u32 { + BigEndian::read_u32(loc) +} + +fn w16(loc: &mut [u8], v: u64) { + BigEndian::write_u16(loc, v as u16); +} + +fn w32(loc: &mut [u8], v: u64) { + BigEndian::write_u32(loc, v as u32); +} + +fn w64(loc: &mut [u8], v: u64) { + BigEndian::write_u64(loc, v); +} + +fn or32(loc: &mut [u8], v: u64) { + let cur = r32(loc); + BigEndian::write_u32(loc, cur | v as u32); +} + +/// The high 22 bits of a value, for `sethi`, with the one's complement +/// trick the HIX22 family uses for negative values. +fn hix22(val: i64) -> u64 { + bits(if val < 0 { !val } else { val } as u64, 31, 10) +} + +/// The low 10 bits with the sign-extension bits of a `LOX10`-style +/// immediate. +fn lox10(val: i64) -> u64 { + bits(val as u64, 9, 0) | if val < 0 { 0b1_1100_0000_0000 } else { 0 } +} + +/// The byte offset within `.plt` of the data pointer of a large PLT +/// entry. See `write_plt_entry` for the layout this assumes. +pub fn plt_ptr_offset(num_plt_symbols: usize, plt_idx: u64) -> u64 { + let i = plt_idx - SPARC_NUM_SMALL_PLT; + let block = i / 160; + let num_large = num_plt_symbols as u64 - SPARC_NUM_SMALL_PLT; + let num_stubs = (num_large - block * 160).min(160); + 0x100000 + block * 5120 + num_stubs * 24 + (i % 160) * 8 +} + +impl Arch for Sparc64 { + const NAME: &'static str = "sparc64"; + const FAMILY: Family = Family::Sparc64; + const PAGE_SIZE: u64 = 8192; + const E_MACHINE: u32 = EM_SPARC64; + const PLT_HDR_SIZE: u64 = 128; + const PLT_SIZE: u64 = 32; + const PLTGOT_SIZE: u64 = 32; + const TRAP: &'static [u8] = &[0x91, 0xd0, 0x20, 0x05]; // ta 5 + + const R_COPY: u32 = R_SPARC_COPY; + const R_GLOB_DAT: u32 = R_SPARC_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_SPARC_JMP_SLOT; + const R_ABS: u32 = R_SPARC_64; + const R_RELATIVE: u32 = R_SPARC_RELATIVE; + const R_IRELATIVE: Option = Some(R_SPARC_IRELATIVE); + const R_DTPOFF: u32 = R_SPARC_TLS_DTPOFF64; + const R_TPOFF: u32 = R_SPARC_TLS_TPOFF64; + const R_DTPMOD: u32 = R_SPARC_TLS_DTPMOD64; + const R_FUNCALL: &'static [u32] = &[R_SPARC_WPLT30, R_SPARC_WDISP30]; + + fn rel_to_string(r_type: u32) -> String { + sparc64_rel_to_string(r_type & 0xff) + } + + /// The PLT is writable despite holding code: the loader patches + /// entries in place instead of writing to a `.got.plt`, and fills in + /// the header itself. The NOPs in entries leave it room. + fn write_plt_header(_ctx: &Context, buf: &mut [u8]) { + buf[..Self::PLT_HDR_SIZE as usize].fill(0); + } + + /// A "small" entry branches to the resolver stub at `.PLT1` with a + /// BPcc, whose reach is ±1 MiB. Past 0x100000 bytes of PLT, entries + /// switch to a "large" format that loads a 64-bit value from a + /// nearby data pointer and jumps to that value plus its own address. + /// + /// Large entries come in blocks of 160: 160 stubs, 24 bytes apart on + /// a fixed grid because the loader derives the symbol to resolve from + /// a stub's address, followed by 160 8-byte data pointers, which the + /// stubs reach with a signed 13-bit `ldx` offset. `buf` extends to + /// the end of the PLT, so the pointer region is in reach. + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let idx = sym.plt_idx(&ctx.symbols).unwrap() as u64; + let plt = ctx.plt.hdr.shdr.sh_addr; + let entry = sym.plt_addr(ctx); + + if idx < SPARC_NUM_SMALL_PLT { + const INSN: [u32; 8] = [ + 0x0300_0000, // sethi (. - .PLT0), %g1 + 0x3068_0000, // ba,a %xcc, .PLT1 + 0x0100_0000, // nop + 0x0100_0000, // nop + 0x0100_0000, // nop + 0x0100_0000, // nop + 0x0100_0000, // nop + 0x0100_0000, // nop + ]; + for (i, &insn) in INSN.iter().enumerate() { + w32(&mut buf[i * 4..], insn as u64); + } + let plt1 = plt + Self::PLT_SIZE; + or32(buf, bits(entry - plt, 21, 0)); + or32( + &mut buf[4..], + bits(plt1.wrapping_sub(entry).wrapping_sub(4), 20, 2), + ); + } else { + const INSN: [u32; 6] = [ + 0x8a10_000f, // mov %o7, %g5 + 0x4000_0002, // call . + 8 + 0x0100_0000, // nop + 0xc25b_e000, // ldx [ %o7 + .ptr ], %g1 + 0x83c3_c001, // jmpl %o7 + %g1, %g1 + 0x9e10_0005, // mov %g5, %o7 + ]; + for (i, &insn) in INSN.iter().enumerate() { + w32(&mut buf[i * 4..], insn as u64); + } + let call = entry + 4; + let ptroff = plt_ptr_offset(ctx.plt.symbols.len(), idx); + or32( + &mut buf[12..], + bits((plt + ptroff).wrapping_sub(call), 12, 0), + ); + + // The data pointer initially holds (.PLT0 - call) so that the + // first call reaches the loader's lazy resolver at .PLT0, + // which then replaces it with (target - call). + let ptr = (ptroff - (entry - plt)) as usize; + w64(&mut buf[ptr..], plt.wrapping_sub(call)); + } + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u32; 6] = [ + 0x8a10_000f, // mov %o7, %g5 + 0x4000_0002, // call . + 8 + 0xc25b_e014, // ldx [ %o7 + 20 ], %g1 + 0xc25b_c001, // ldx [ %o7 + %g1 ], %g1 + 0x81c0_4000, // jmp %g1 + 0x9e10_0005, // mov %g5, %o7 + ]; // .quad $plt_entry - $got_entry + for (i, &insn) in INSN.iter().enumerate() { + w32(&mut buf[i * 4..], insn as u64); + } + w64( + &mut buf[24..], + sym.got_pltgot_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(4), + ); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + match r_type(rel) { + R_NONE => {} + R_SPARC_64 | R_SPARC_UA64 => w64(loc, val), + R_SPARC_DISP32 => { + eh_frame::check_range( + ctx, + isec, + rel, + val.wrapping_sub(p) as i64, + -(1 << 31), + 1 << 31, + ); + w32(loc, val.wrapping_sub(p)); + } + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + let mut needs_tlsgd = false; + + for rel in isec.rels::(file) { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + match r_type(&rel) { + R_SPARC_8 | R_SPARC_5 | R_SPARC_6 | R_SPARC_7 | R_SPARC_10 | R_SPARC_11 + | R_SPARC_13 | R_SPARC_16 | R_SPARC_22 | R_SPARC_32 | R_SPARC_REGISTER + | R_SPARC_UA16 | R_SPARC_UA32 | R_SPARC_PC_HM10 | R_SPARC_OLO10 | R_SPARC_LOX10 + | R_SPARC_HM10 | R_SPARC_M44 | R_SPARC_HIX22 | R_SPARC_LO10 | R_SPARC_L44 + | R_SPARC_LM22 | R_SPARC_HI22 | R_SPARC_H44 | R_SPARC_HH22 => { + scan_absrel(ctx, isec, sym, &rel) + } + R_SPARC_PLT32 | R_SPARC_WPLT30 | R_SPARC_WDISP30 | R_SPARC_HIPLT22 + | R_SPARC_LOPLT10 | R_SPARC_PCPLT32 | R_SPARC_PCPLT22 | R_SPARC_PCPLT10 + | R_SPARC_PLT64 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_SPARC_GOT13 | R_SPARC_GOT10 | R_SPARC_GOT22 | R_SPARC_GOTDATA_HIX22 => { + sym.add_flags(NEEDS_GOT) + } + R_SPARC_GOTDATA_OP_HIX22 => { + if sym.is_imported() { + sym.add_flags(NEEDS_GOT); + } + } + R_SPARC_DISP16 | R_SPARC_DISP32 | R_SPARC_DISP64 | R_SPARC_DISP8 | R_SPARC_PC10 + | R_SPARC_PC22 | R_SPARC_PC_LM22 | R_SPARC_WDISP16 | R_SPARC_WDISP19 + | R_SPARC_WDISP22 | R_SPARC_PC_HH22 => scan_pcrel(ctx, isec, sym, &rel), + R_SPARC_TLS_GD_HI22 => { + // A static executable always relaxes, since libc.a + // has no __tls_get_addr. + if ctx.args.is_static || (ctx.args.relax && sym.is_tprel_linktime_const(ctx)) { + // Nothing to do. + } else if ctx.args.relax && sym.is_tprel_runtime_const(ctx) { + sym.add_flags(NEEDS_GOTTP); + } else { + sym.add_flags(NEEDS_TLSGD); + needs_tlsgd = true; + } + } + R_SPARC_TLS_LDM_HI22 => { + if !(ctx.args.is_static || (ctx.args.relax && !ctx.args.shared)) { + ctx.needs_tlsld.store(true, Ordering::Relaxed); + } + } + R_SPARC_TLS_IE_HI22 => sym.add_flags(NEEDS_GOTTP), + R_SPARC_TLS_LE_HIX22 | R_SPARC_TLS_LE_LOX10 => check_tlsle(ctx, isec, sym, &rel), + R_SPARC_64 + | R_SPARC_UA64 + | R_SPARC_GOTDATA_OP_LOX10 + | R_SPARC_GOTDATA_OP + | R_SPARC_GOTDATA_LOX10 + | R_SPARC_TLS_GD_LO10 + | R_SPARC_TLS_GD_ADD + | R_SPARC_TLS_GD_CALL + | R_SPARC_TLS_LDM_LO10 + | R_SPARC_TLS_LDM_ADD + | R_SPARC_TLS_LDM_CALL + | R_SPARC_TLS_LDO_HIX22 + | R_SPARC_TLS_LDO_LOX10 + | R_SPARC_TLS_LDO_ADD + | R_SPARC_TLS_IE_ADD + | R_SPARC_TLS_IE_LD + | R_SPARC_TLS_IE_LDX + | R_SPARC_TLS_IE_LO10 + | R_SPARC_SIZE32 => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + } + + // TLS_GD_CALL and TLS_LDM_CALL implicitly call __tls_get_addr, + // which may come from libc.so. + if let Some(id) = ctx.syms.tls_get_addr { + let sym = &ctx.symbols[id]; + if sym.is_imported() && (needs_tlsgd || ctx.needs_tlsld.load(Ordering::Relaxed)) { + sym.add_flags(NEEDS_PLT); + } + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let got = ctx.got.hdr.shdr.sh_addr; + let tls_get_addr = + || ctx.symbols[ctx.syms.tls_get_addr.expect("SPARC has __tls_get_addr")].addr(ctx); + + // Relocations are applied back to front so that rewriting a + // TLS_GD_CALL, which swaps the call with the instruction in its + // delay slot, sees that instruction already relocated. + for (i, rel) in isec.rels::(file).iter().enumerate().rev() { + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let g = || sym.got_addr(ctx).wrapping_sub(got); + let sa = s.wrapping_add(a); + let pcrel = sa.wrapping_sub(p); + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + + // Register fields of the instruction being rewritten. + let insn = r32(&buf[off..]); + let rs1 = insn & (0b11111 << 14); + let rs2 = insn & 0b11111; + let rd = insn & (0b11111 << 25); + let loc = &mut buf[off..]; + + match r_type(&rel) { + R_SPARC_5 => { + check(sa as i64, 0, 1 << 5); + or32(loc, bits(sa, 4, 0)); + } + R_SPARC_6 => { + check(sa as i64, 0, 1 << 6); + or32(loc, bits(sa, 5, 0)); + } + R_SPARC_7 => { + check(sa as i64, 0, 1 << 7); + or32(loc, bits(sa, 6, 0)); + } + R_SPARC_8 => { + check(sa as i64, 0, 1 << 8); + loc[0] = sa as u8; + } + R_SPARC_10 => { + check(sa as i64, 0, 1 << 10); + or32(loc, bits(sa, 9, 0)); + } + R_SPARC_LO10 | R_SPARC_LOPLT10 => or32(loc, bits(sa, 9, 0)), + R_SPARC_11 => { + check(sa as i64, 0, 1 << 11); + or32(loc, bits(sa, 10, 0)); + } + R_SPARC_13 => { + check(sa as i64, 0, 1 << 13); + or32(loc, bits(sa, 12, 0)); + } + R_SPARC_16 | R_SPARC_UA16 => { + check(sa as i64, 0, 1 << 16); + w16(loc, sa); + } + R_SPARC_22 => { + check(sa as i64, 0, 1 << 22); + or32(loc, bits(sa, 21, 0)); + } + R_SPARC_32 | R_SPARC_UA32 | R_SPARC_PLT32 => { + check(sa as i64, 0, 1 << 32); + w32(loc, sa); + } + R_SPARC_PLT64 | R_SPARC_REGISTER => w64(loc, sa), + R_SPARC_DISP8 => { + check(pcrel as i64, -(1 << 7), 1 << 7); + loc[0] = pcrel as u8; + } + R_SPARC_DISP16 => { + check(pcrel as i64, -(1 << 15), 1 << 15); + w16(loc, pcrel); + } + R_SPARC_DISP32 | R_SPARC_PCPLT32 => { + check(pcrel as i64, -(1 << 31), 1 << 31); + w32(loc, pcrel); + } + R_SPARC_DISP64 => w64(loc, pcrel), + R_SPARC_WDISP16 => { + check(pcrel as i64, -(1 << 16), 1 << 16); + let field = (bit(pcrel, 16) << 21) | bits(pcrel, 15, 2); + let cur = BigEndian::read_u16(loc); + w16(loc, cur as u64 | field); + } + R_SPARC_WDISP19 => { + check(pcrel as i64, -(1 << 20), 1 << 20); + or32(loc, bits(pcrel, 20, 2)); + } + R_SPARC_WDISP22 => { + check(pcrel as i64, -(1 << 23), 1 << 23); + or32(loc, bits(pcrel, 23, 2)); + } + R_SPARC_WDISP30 | R_SPARC_WPLT30 => { + if !sym.is_remaining_undef_weak() { + check(pcrel as i64, -(1 << 31), 1 << 31); + } + or32(loc, bits(pcrel, 31, 2)); + } + R_SPARC_HI22 | R_SPARC_HIPLT22 | R_SPARC_LM22 => or32(loc, bits(sa, 31, 10)), + R_SPARC_GOT10 => or32(loc, bits(g(), 9, 0)), + R_SPARC_GOT13 => { + check(g() as i64, 0, 1 << 12); + or32(loc, bits(g(), 12, 0)); + } + R_SPARC_GOT22 => or32(loc, bits(g(), 31, 10)), + R_SPARC_GOTDATA_HIX22 => or32(loc, hix22(sa.wrapping_sub(got) as i64)), + R_SPARC_GOTDATA_LOX10 => or32(loc, lox10(sa.wrapping_sub(got) as i64)), + // A GOT load of a local symbol is always relaxed to an + // immediate, since R_SPARC_GOTDATA_OP can't carry an + // addend for a local symbol. + R_SPARC_GOTDATA_OP_HIX22 => { + if sym.is_absolute() { + or32(loc, hix22(sa as i64)); + } else if sym.is_pcrel_linktime_const(ctx) { + or32(loc, hix22(sa.wrapping_sub(got) as i64)); + } else { + or32(loc, bits(g(), 31, 10)); + } + } + R_SPARC_GOTDATA_OP_LOX10 => { + if sym.is_absolute() { + or32(loc, lox10(sa as i64)); + } else if sym.is_pcrel_linktime_const(ctx) { + or32(loc, lox10(sa.wrapping_sub(got) as i64)); + } else { + or32(loc, bits(g(), 9, 0)); + } + } + R_SPARC_GOTDATA_OP => { + if sym.is_absolute() { + // ldx [ %rs1 + %rs2 ], %rd → mov %rs2, %rd + w32(loc, (0x8010_0000 | rs2 | rd) as u64); + } else if sym.is_pcrel_linktime_const(ctx) { + // ldx [ %rs1 + %rs2 ], %rd → add %rs1, %rs2, %rd + w32(loc, (0x8000_0000 | rs1 | rs2 | rd) as u64); + } + } + R_SPARC_PC10 | R_SPARC_PCPLT10 => or32(loc, bits(pcrel, 9, 0)), + R_SPARC_PC22 | R_SPARC_PCPLT22 | R_SPARC_PC_LM22 => or32(loc, bits(pcrel, 31, 10)), + R_SPARC_OLO10 => or32( + loc, + bits(bits(sa, 9, 0).wrapping_add(r_type_data(&rel)), 12, 0), + ), + R_SPARC_HH22 => or32(loc, bits(sa, 63, 42)), + R_SPARC_HM10 => or32(loc, bits(sa, 41, 32)), + R_SPARC_PC_HH22 => or32(loc, bits(pcrel, 63, 42)), + R_SPARC_PC_HM10 => or32(loc, bits(pcrel, 41, 32)), + R_SPARC_HIX22 => or32(loc, bits(!sa, 31, 10)), + R_SPARC_LOX10 => or32(loc, bits(sa, 9, 0) | 0b1_1100_0000_0000), + R_SPARC_H44 => or32(loc, bits(sa, 43, 22)), + R_SPARC_M44 => or32(loc, bits(sa, 21, 12)), + R_SPARC_L44 => or32(loc, bits(sa, 11, 0)), + R_SPARC_TLS_GD_HI22 => { + if sym.has_tlsgd(&ctx.symbols) { + or32( + loc, + bits( + sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got), + 31, + 10, + ), + ); + } else if sym.has_gottp(&ctx.symbols) { + or32( + loc, + bits( + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), + 31, + 10, + ), + ); + } else { + or32(loc, bits(!sa.wrapping_sub(ctx.tp_addr), 31, 10)); + } + } + R_SPARC_TLS_GD_LO10 => { + if sym.has_tlsgd(&ctx.symbols) { + or32( + loc, + bits(sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(got), 9, 0), + ); + } else if sym.has_gottp(&ctx.symbols) { + // add %rs1, %rs2, %rd → or %rs1, $imm, %rd + w32(loc, (0x8010_2000 | rs1 | rd) as u64); + or32( + loc, + bits(sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), 9, 0), + ); + } else { + // add %rs1, %rs2, %rd → xor %rs1, $imm, %rd + w32(loc, (0x8018_2000 | rs1 | rd) as u64); + or32( + loc, + bits(sa.wrapping_sub(ctx.tp_addr), 9, 0) | 0b1_1100_0000_0000, + ); + } + } + R_SPARC_TLS_GD_ADD => { + if sym.has_tlsgd(&ctx.symbols) { + // Nothing to do. + } else if sym.has_gottp(&ctx.symbols) { + // add %rs1, %rs2, %rd → ldx [ %rs1 + %rs2 ], %rd + w32(loc, (0xc058_0000 | rs1 | rs2 | rd) as u64); + } else { + // add %rs1, %rs2, %rd → add %g7, %rs2, %rd + w32(loc, (0x8001_c000 | rs2 | rd) as u64); + } + } + R_SPARC_TLS_GD_CALL => { + if sym.has_tlsgd(&ctx.symbols) { + or32( + loc, + bits(tls_get_addr().wrapping_add(a).wrapping_sub(p), 31, 2), + ); + } else if sym.has_gottp(&ctx.symbols) { + // The call becomes a non-branch, so it swaps + // places with the instruction in its delay slot to + // keep the original execution order. + let next = r32(&loc[4..]); + w32(loc, next as u64); + w32(&mut loc[4..], 0x9001_c008); // add %g7, %o0, %o0 + } else { + w32(loc, 0x0100_0000); // call → nop + } + } + R_SPARC_TLS_LDM_HI22 => { + if ctx.got.has_tlsld() { + or32( + loc, + bits( + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + 31, + 10, + ), + ); + } else { + or32(loc, bits(ctx.tp_addr.wrapping_sub(ctx.tls_begin), 31, 10)); + } + } + R_SPARC_TLS_LDM_LO10 => { + if ctx.got.has_tlsld() { + or32( + loc, + bits( + ctx.got + .tlsld_addr::() + .wrapping_add(a) + .wrapping_sub(got), + 9, + 0, + ), + ); + } else { + or32(loc, bits(ctx.tp_addr.wrapping_sub(ctx.tls_begin), 9, 0)); + } + } + R_SPARC_TLS_LDM_ADD => { + if !ctx.got.has_tlsld() { + w32(loc, (0x8021_c000 | rs2 | rd) as u64); // sub %g7, %rs2, %rd + } + } + R_SPARC_TLS_LDM_CALL => { + if ctx.got.has_tlsld() { + or32( + loc, + bits(tls_get_addr().wrapping_add(a).wrapping_sub(p), 31, 2), + ); + } else { + w32(loc, 0x0100_0000); // nop + } + } + R_SPARC_TLS_LDO_HIX22 => or32(loc, bits(sa.wrapping_sub(ctx.dtp_addr), 31, 10)), + R_SPARC_TLS_LDO_LOX10 => or32(loc, bits(sa.wrapping_sub(ctx.dtp_addr), 9, 0)), + R_SPARC_TLS_IE_HI22 => or32( + loc, + bits( + sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), + 31, + 10, + ), + ), + R_SPARC_TLS_IE_LO10 => or32( + loc, + bits(sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(got), 9, 0), + ), + R_SPARC_TLS_LE_HIX22 => or32(loc, bits(!sa.wrapping_sub(ctx.tp_addr), 31, 10)), + R_SPARC_TLS_LE_LOX10 => or32( + loc, + bits(sa.wrapping_sub(ctx.tp_addr), 9, 0) | 0b1_1100_0000_0000, + ), + R_SPARC_SIZE32 => w32(loc, sym.esym(ctx).st_size.wrapping_add(a)), + R_SPARC_64 | R_SPARC_UA64 | R_SPARC_TLS_LDO_ADD | R_SPARC_TLS_IE_LD + | R_SPARC_TLS_IE_LDX | R_SPARC_TLS_IE_ADD => {} + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error(ctx, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let frag = isec.fragment(ctx, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let sa = s.wrapping_add(a); + let loc = &mut buf[rel.r_offset as usize..]; + + match r_type(&rel) { + R_SPARC_64 | R_SPARC_UA64 => w64( + loc, + isec.tombstone(ctx, sym, frag.map(|(f, _)| f)).unwrap_or(sa), + ), + R_SPARC_32 | R_SPARC_UA32 => { + isec.check_range(ctx, i, sa as i64, 0, 1 << 32); + w32(loc, sa); + } + R_SPARC_TLS_DTPOFF32 => w32(loc, sa.wrapping_sub(ctx.dtp_addr)), + R_SPARC_TLS_DTPOFF64 => w64(loc, sa.wrapping_sub(ctx.dtp_addr)), + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } +} diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs new file mode 100644 index 000000000..9ac812f07 --- /dev/null +++ b/src/arch/x86_64.rs @@ -0,0 +1,943 @@ +//! x86-64. +//! +//! x86-64 is straightforward: it has PC-relative addressing for +//! position-independent code, and 32-bit branch displacements, so no +//! range extension thunks are needed. `%r11` is neither caller- nor +//! callee-saved, so the PLT uses it as a scratch register. +//! +//! The thread pointer lives in the `%fs` segment register, and for +//! historical reasons points past the end of the TLS block, so offsets to +//! thread-local variables in the main executable are negative. + +use crate::arch::{Arch, Family}; +use crate::chunks::eh_frame; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::input_sections::{check_tlsle, scan_absrel, scan_pcrel, scan_tlsdesc, InputSection}; +use crate::symbol::{Symbol, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSGD}; +use crate::util::is_int; +use crate::{error, fatal}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct X86_64; + +impl Layout for X86_64 { + type Endian = LittleEndian; + const IS_64: bool = true; + const IS_RELA: bool = true; +} + +impl Arch for X86_64 { + const NAME: &'static str = "x86_64"; + const FAMILY: Family = Family::X86_64; + const PAGE_SIZE: u64 = 4096; + const E_MACHINE: u32 = EM_X86_64; + const PLT_HDR_SIZE: u64 = 32; + const PLT_SIZE: u64 = 16; + const PLTGOT_SIZE: u64 = 8; + const SFRAME_ABI: Option = Some(SFRAME_ABI_AMD64_ENDIAN_LITTLE); + const TRAP: &'static [u8] = &[0xcc]; // int3 + + const R_COPY: u32 = R_X86_64_COPY; + const R_GLOB_DAT: u32 = R_X86_64_GLOB_DAT; + const R_JUMP_SLOT: u32 = R_X86_64_JUMP_SLOT; + const R_ABS: u32 = R_X86_64_64; + const R_RELATIVE: u32 = R_X86_64_RELATIVE; + const R_IRELATIVE: Option = Some(R_X86_64_IRELATIVE); + const R_DTPOFF: u32 = R_X86_64_DTPOFF64; + const R_TPOFF: u32 = R_X86_64_TPOFF64; + const R_DTPMOD: u32 = R_X86_64_DTPMOD64; + const R_TLSDESC: Option = Some(R_X86_64_TLSDESC); + const R_SFRAME: Option = Some(R_X86_64_PC64); + const R_FUNCALL: &'static [u32] = &[R_X86_64_PLT32, R_X86_64_PLTOFF64]; + + fn finish_output(ctx: &Context, buf: &mut [u8]) { + if ctx.args.z_rewrite_endbr { + rewrite_endbr(ctx, buf); + } + } + + fn rel_to_string(r_type: u32) -> String { + x86_64_rel_to_string(r_type) + } + + /// The PLT header and entries start with `endbr64` for Intel CET. + /// Unlike GNU ld's IBT PLT, which splits `.plt` and `.plt.sec` with + /// 32 bytes per entry, ours keeps 16-byte entries in one section. + /// Clobbering `%r11` is fine because the resolver does so anyway. + fn write_plt_header(ctx: &Context, buf: &mut [u8]) { + const INSN: [u8; 32] = [ + 0xf3, 0x0f, 0x1e, 0xfa, // endbr64 + 0x41, 0x53, // push %r11 + 0xff, 0x35, 0, 0, 0, 0, // push GOTPLT+8(%rip) + 0xff, 0x25, 0, 0, 0, 0, // jmp *GOTPLT+16(%rip) + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, // padding + ]; + buf[..32].copy_from_slice(&INSN); + let gotplt = ctx.gotplt.hdr.shdr.sh_addr; + let plt = ctx.plt.hdr.shdr.sh_addr; + write_u32( + &mut buf[8..], + gotplt.wrapping_sub(plt).wrapping_sub(4) as u32, + ); + write_u32( + &mut buf[14..], + gotplt.wrapping_sub(plt).wrapping_sub(2) as u32, + ); + } + + fn write_plt_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + let plt_idx = sym.plt_idx(&ctx.symbols).unwrap(); + let disp = sym.gotplt_addr(ctx).wrapping_sub(sym.plt_addr(ctx)); + // Only a canonical PLT entry can be address-taken, so only it + // needs a landing pad. + if sym.is_canonical() { + const INSN: [u8; 16] = [ + 0xf3, 0x0f, 0x1e, 0xfa, // endbr64 + 0x41, 0xbb, 0, 0, 0, 0, // mov $index_in_relplt, %r11d + 0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOTPLT + ]; + buf[..16].copy_from_slice(&INSN); + write_u32(&mut buf[6..], plt_idx); + write_u32(&mut buf[12..], disp.wrapping_sub(16) as u32); + } else { + const INSN: [u8; 16] = [ + 0x41, 0xbb, 0, 0, 0, 0, // mov $index_in_relplt, %r11d + 0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOTPLT + 0xcc, 0xcc, 0xcc, 0xcc, // padding + ]; + buf[..16].copy_from_slice(&INSN); + write_u32(&mut buf[2..], plt_idx); + write_u32(&mut buf[8..], disp.wrapping_sub(12) as u32); + } + } + + fn write_pltgot_entry(ctx: &Context, buf: &mut [u8], sym: &Symbol) { + const INSN: [u8; 8] = [ + 0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT + 0xcc, 0xcc, // padding + ]; + buf[..8].copy_from_slice(&INSN); + let disp = sym + .got_pltgot_addr(ctx) + .wrapping_sub(sym.plt_addr(ctx)) + .wrapping_sub(6); + write_u32(&mut buf[2..], disp as u32); + } + + fn apply_eh_reloc( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + loc: &mut [u8], + p: u64, + val: u64, + ) { + let check = |val: i64, lo: i64, hi: i64| eh_frame::check_range(ctx, isec, rel, val, lo, hi); + match rel.r_type { + R_NONE => {} + R_X86_64_32 => { + check(val as i64, 0, 1 << 32); + write_u32(loc, val as u32); + } + R_X86_64_64 => write_u64(loc, val), + R_X86_64_PC32 => { + check(val.wrapping_sub(p) as i64, -(1 << 31), 1 << 31); + write_u32(loc, val.wrapping_sub(p) as u32); + } + R_X86_64_PC64 => write_u64(loc, val.wrapping_sub(p)), + _ => eh_frame::unsupported(ctx, rel), + } + } + + fn scan_relocations(ctx: &Context, isec: &InputSection) { + debug_assert!(isec.is_alloc()); + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE || isec.record_undef_error_with_file(ctx, file, rel) { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let loc = &isec.contents()[rel.r_offset as usize..]; + + if sym.is_ifunc() { + sym.add_flags(NEEDS_GOT | NEEDS_PLT); + } + + if rel.r_type == R_X86_64_TLSGD || rel.r_type == R_X86_64_TLSLD { + let next = rels.get(i).map(|r| r.r_type); + let ok = matches!( + next, + Some( + R_X86_64_PLT32 + | R_X86_64_PC32 + | R_X86_64_PLTOFF64 + | R_X86_64_GOTPCREL + | R_X86_64_GOTPCRELX + ) + ); + if !ok { + fatal!( + ctx, + "{}: {} must be followed by PLT or GOTPCREL", + isec.display(file), + rel.type_name::() + ); + } + } + + match rel.r_type { + R_X86_64_8 | R_X86_64_16 | R_X86_64_32 | R_X86_64_32S => { + scan_absrel(ctx, isec, sym, rel) + } + R_X86_64_PC8 | R_X86_64_PC16 | R_X86_64_PC32 | R_X86_64_PC64 => { + scan_pcrel(ctx, isec, sym, rel) + } + R_X86_64_GOT32 + | R_X86_64_GOT64 + | R_X86_64_GOTPC32 + | R_X86_64_GOTPC64 + | R_X86_64_GOTPCREL + | R_X86_64_GOTPCREL64 + | R_X86_64_GOTPCRELX + | R_X86_64_REX_GOTPCRELX + | R_X86_64_CODE_4_GOTPCRELX => sym.add_flags(NEEDS_GOT), + R_X86_64_PLT32 | R_X86_64_PLTOFF64 => { + if sym.is_imported() { + sym.add_flags(NEEDS_PLT); + } + } + R_X86_64_TLSGD => { + // Always relax with -static because libc.a doesn't + // contain __tls_get_addr. + if ctx.args.is_static || (ctx.args.relax && sym.is_tprel_linktime_const(ctx)) { + i += 1; + } else if ctx.args.relax && sym.is_tprel_runtime_const(ctx) { + sym.add_flags(NEEDS_GOTTP); + i += 1; + } else { + sym.add_flags(NEEDS_TLSGD); + } + } + R_X86_64_TLSLD => { + if ctx.args.is_static || (ctx.args.relax && !ctx.args.shared) { + i += 1; + } else { + ctx.needs_tlsld + .store(true, std::sync::atomic::Ordering::Relaxed); + } + } + R_X86_64_GOTTPOFF | R_X86_64_CODE_4_GOTTPOFF => { + if !ctx.args.relax + || !sym.is_tprel_linktime_const(ctx) + || relax_gottpoff(loc_before(isec, rel), rel) == 0 + { + sym.add_flags(NEEDS_GOTTP); + } + } + R_X86_64_CODE_6_GOTTPOFF => sym.add_flags(NEEDS_GOTTP), + R_X86_64_TLSDESC_CALL => scan_tlsdesc(ctx, sym), + R_X86_64_TPOFF32 | R_X86_64_TPOFF64 => check_tlsle(ctx, isec, sym, rel), + R_X86_64_64 + | R_X86_64_GOTOFF64 + | R_X86_64_DTPOFF32 + | R_X86_64_DTPOFF64 + | R_X86_64_SIZE32 + | R_X86_64_SIZE64 + | R_X86_64_GOTPC32_TLSDESC + | R_X86_64_CODE_4_GOTPC32_TLSDESC => {} + _ => error!( + ctx, + "{}: unknown relocation: {}", + isec.display(file), + rel.type_name::() + ), + } + let _ = loc; + } + } + + fn apply_reloc_alloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + let rels = isec.rels::(file); + let mut i = 0; + + while i < rels.len() { + let rel = &rels.at(i); + i += 1; + if rel.r_type == R_NONE { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if sym.ty() == STT_TLS && sym.is_remaining_undef_weak() { + continue; + } + + let off = rel.r_offset as usize; + let s = sym.addr(ctx); + let a = rel.r_addend as u64; + let p = isec.addr(ctx) + rel.r_offset; + let got_base = ctx.gotplt.hdr.shdr.sh_addr; + let g = if sym.has_got(&ctx.symbols) { + sym.got_addr(ctx).wrapping_sub(got_base) + } else { + 0 + }; + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i - 1, val, lo, hi); + let write32 = |buf: &mut [u8], val: u64| { + check(val as i64, 0, 1 << 32); + write_u32(&mut buf[off..], val as u32); + }; + let write32s = |buf: &mut [u8], val: u64| { + check(val as i64, -(1 << 31), 1 << 31); + write_u32(&mut buf[off..], val as u32); + }; + + match rel.r_type { + R_X86_64_8 => { + check(s.wrapping_add(a) as i64, 0, 1 << 8); + buf[off] = s.wrapping_add(a) as u8; + } + R_X86_64_16 => { + check(s.wrapping_add(a) as i64, 0, 1 << 16); + write_u16(&mut buf[off..], s.wrapping_add(a) as u16); + } + R_X86_64_32 => write32(buf, s.wrapping_add(a)), + R_X86_64_32S => write32s(buf, s.wrapping_add(a)), + // Handled as an absolute relocation by the output section. + R_X86_64_64 => {} + R_X86_64_PC8 => { + let v = s.wrapping_add(a).wrapping_sub(p); + check(v as i64, -(1 << 7), 1 << 7); + buf[off] = v as u8; + } + R_X86_64_PC16 => { + let v = s.wrapping_add(a).wrapping_sub(p); + check(v as i64, -(1 << 15), 1 << 15); + write_u16(&mut buf[off..], v as u16); + } + R_X86_64_PC32 => write32s(buf, s.wrapping_add(a).wrapping_sub(p)), + R_X86_64_PLT32 => { + let v = s.wrapping_add(a).wrapping_sub(p); + if !sym.is_remaining_undef_weak() { + check(v as i64, -(1 << 31), 1 << 31); + } + write_u32(&mut buf[off..], v as u32); + } + R_X86_64_PC64 => write_u64(&mut buf[off..], s.wrapping_add(a).wrapping_sub(p)), + R_X86_64_GOT32 => write32(buf, g.wrapping_add(a)), + R_X86_64_GOT64 => write_u64(&mut buf[off..], g.wrapping_add(a)), + R_X86_64_GOTOFF64 | R_X86_64_PLTOFF64 => { + write_u64(&mut buf[off..], s.wrapping_add(a).wrapping_sub(got_base)) + } + R_X86_64_GOTPC32 => write32s(buf, got_base.wrapping_add(a).wrapping_sub(p)), + R_X86_64_GOTPC64 => { + write_u64(&mut buf[off..], got_base.wrapping_add(a).wrapping_sub(p)) + } + R_X86_64_GOTPCREL => write32s( + buf, + g.wrapping_add(got_base).wrapping_add(a).wrapping_sub(p), + ), + R_X86_64_GOTPCREL64 => write_u64( + &mut buf[off..], + g.wrapping_add(got_base).wrapping_add(a).wrapping_sub(p), + ), + R_X86_64_GOTPCRELX | R_X86_64_REX_GOTPCRELX | R_X86_64_CODE_4_GOTPCRELX => { + // GOTPCRELX is relaxed even with --no-relax because some + // static PIE runtime code depends on it. + let v = s.wrapping_add(a).wrapping_sub(p); + if sym.is_pcrel_linktime_const(ctx) && is_int(v as i64, 32) { + let insn = relax_gotpcrelx(&buf[..off], rel); + if insn != 0 { + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write_u32(&mut buf[off..], v as u32); + continue; + } + } + write32s( + buf, + g.wrapping_add(got_base).wrapping_add(a).wrapping_sub(p), + ); + } + R_X86_64_TLSGD => { + if sym.has_tlsgd(&ctx.symbols) { + write32s(buf, sym.tlsgd_addr(ctx).wrapping_add(a).wrapping_sub(p)); + } else if sym.has_gottp(&ctx.symbols) { + let next = &rels.at(i); + i += 1; + relax_gd_to_ie(buf, off, next, sym.gottp_addr(ctx).wrapping_sub(p)); + } else { + let next = &rels.at(i); + i += 1; + relax_gd_to_le(buf, off, next, s.wrapping_sub(ctx.tp_addr)); + } + } + R_X86_64_TLSLD => { + if ctx.got.has_tlsld() { + write32s( + buf, + ctx.got.tlsld_addr::().wrapping_add(a).wrapping_sub(p), + ); + } else { + let next = &rels.at(i); + i += 1; + relax_ld_to_le(buf, off, next, ctx.tp_addr.wrapping_sub(ctx.tls_begin)); + } + } + R_X86_64_DTPOFF32 => write32s(buf, s.wrapping_add(a).wrapping_sub(ctx.dtp_addr)), + R_X86_64_DTPOFF64 => write_u64( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.dtp_addr), + ), + R_X86_64_TPOFF32 => write32s(buf, s.wrapping_add(a).wrapping_sub(ctx.tp_addr)), + R_X86_64_TPOFF64 => { + write_u64(&mut buf[off..], s.wrapping_add(a).wrapping_sub(ctx.tp_addr)) + } + R_X86_64_GOTTPOFF | R_X86_64_CODE_4_GOTTPOFF => { + if sym.has_gottp(&ctx.symbols) { + write32s(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p)); + } else { + let insn = relax_gottpoff(&buf[..off], rel); + buf[off - 3] = (insn >> 16) as u8; + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write32s(buf, s.wrapping_sub(ctx.tp_addr)); + } + } + R_X86_64_CODE_6_GOTTPOFF => { + write32s(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p)) + } + R_X86_64_GOTPC32_TLSDESC | R_X86_64_CODE_4_GOTPC32_TLSDESC => { + // TLSDESC materializes a TP-relative address in %rax: + // + // lea 0(%rip), %rax # R_X86_64_GOTPC32_TLSDESC + // call *(%rax) # R_X86_64_TLSDESC_CALL + // + // If the address is known at link time it becomes + // `mov $foo@TPOFF, %rax; nop`, and if at load time + // `mov foo@GOTTPOFF(%rip), %rax; nop`. + if sym.has_tlsdesc(&ctx.symbols) { + write32s(buf, sym.tlsdesc_addr(ctx).wrapping_add(a).wrapping_sub(p)); + } else if sym.has_gottp(&ctx.symbols) { + let insn = relax_tlsdesc_to_ie(&buf[..off], rel); + if insn == 0 { + fatal!( + ctx, + "{}: illegal instruction sequence for {}", + isec.display(file), + rel.type_name::() + ); + } + buf[off - 3] = (insn >> 16) as u8; + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write32s(buf, sym.gottp_addr(ctx).wrapping_add(a).wrapping_sub(p)); + } else { + let insn = relax_tlsdesc_to_le(&buf[..off], rel); + if insn == 0 { + fatal!( + ctx, + "{}: illegal instruction sequence for {}", + isec.display(file), + rel.type_name::() + ); + } + buf[off - 3] = (insn >> 16) as u8; + buf[off - 2] = (insn >> 8) as u8; + buf[off - 1] = insn as u8; + write32s(buf, s.wrapping_sub(ctx.tp_addr)); + } + } + R_X86_64_TLSDESC_CALL => { + if !sym.has_tlsdesc(&ctx.symbols) { + // call *(%rax) -> nop + buf[off] = 0x66; + buf[off + 1] = 0x90; + } + } + R_X86_64_SIZE32 => write32(buf, sym.esym(ctx).st_size.wrapping_add(a)), + R_X86_64_SIZE64 => { + write_u64(&mut buf[off..], sym.esym(ctx).st_size.wrapping_add(a)) + } + _ => unreachable!("unexpected relocation {}", rel.type_name::()), + } + } + } + + /// Relocations against non-allocated sections (mostly debug info) + /// never need GOT or PLT entries, and aren't scanned beforehand. + fn apply_reloc_nonalloc(ctx: &Context, isec: &InputSection, buf: &mut [u8]) { + let file = &ctx.objs[isec.file.index()]; + isec.for_each_reloc::(ctx, |rel, i| { + if rel.r_type == R_NONE || isec.record_undef_error_with_file(ctx, file, &rel) { + return; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let off = rel.r_offset as usize; + let frag = isec.fragment_with_file::(file, &rel); + let (s, a) = match frag { + Some((frag, addend)) => (ctx.fragment_addr(frag), addend as u64), + None => (sym.addr(ctx), rel.r_addend as u64), + }; + let frag_ref = frag.map(|(f, _)| f); + + let check = |val: i64, lo: i64, hi: i64| isec.check_range(ctx, i, val, lo, hi); + let write32 = |buf: &mut [u8], val: u64| { + check(val as i64, 0, 1 << 32); + write_u32(&mut buf[off..], val as u32); + }; + let write32s = |buf: &mut [u8], val: u64| { + check(val as i64, -(1 << 31), 1 << 31); + write_u32(&mut buf[off..], val as u32); + }; + + match rel.r_type { + R_X86_64_8 => { + check(s.wrapping_add(a) as i64, 0, 1 << 8); + buf[off] = s.wrapping_add(a) as u8; + } + R_X86_64_16 => { + check(s.wrapping_add(a) as i64, 0, 1 << 16); + write_u16(&mut buf[off..], s.wrapping_add(a) as u16); + } + R_X86_64_32 => write32(buf, s.wrapping_add(a)), + R_X86_64_32S => write32s(buf, s.wrapping_add(a)), + R_X86_64_64 => match isec.tombstone_with_file(ctx, file, sym, frag_ref) { + Some(v) => write_u64(&mut buf[off..], v), + None => write_u64(&mut buf[off..], s.wrapping_add(a)), + }, + R_X86_64_DTPOFF32 => match isec.tombstone_with_file(ctx, file, sym, frag_ref) { + Some(v) => write_u32(&mut buf[off..], v as u32), + None => write32s(buf, s.wrapping_add(a).wrapping_sub(ctx.dtp_addr)), + }, + R_X86_64_DTPOFF64 => match isec.tombstone_with_file(ctx, file, sym, frag_ref) { + Some(v) => write_u64(&mut buf[off..], v), + None => write_u64( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.dtp_addr), + ), + }, + R_X86_64_GOTOFF64 => write_u64( + &mut buf[off..], + s.wrapping_add(a).wrapping_sub(ctx.gotplt.hdr.shdr.sh_addr), + ), + // GCC 6.3 emits this for _GLOBAL_OFFSET_TABLE_ even though a + // PC-relative relocation makes no sense here. + R_X86_64_GOTPC64 => { + write_u64(&mut buf[off..], ctx.gotplt.hdr.shdr.sh_addr.wrapping_add(a)) + } + R_X86_64_SIZE32 => write32(buf, sym.esym(ctx).st_size.wrapping_add(a)), + R_X86_64_SIZE64 => { + write_u64(&mut buf[off..], sym.esym(ctx).st_size.wrapping_add(a)) + } + _ => fatal!( + ctx, + "{}: invalid relocation for non-allocated sections: {}", + isec.display(file), + rel.type_name::() + ), + } + }); + } + + fn emitted_rel_type(ctx: &Context, isec: &InputSection, rel: &ElfRel, _i: usize) -> u32 { + if matches!( + rel.r_type, + R_X86_64_GOTPCRELX | R_X86_64_REX_GOTPCRELX | R_X86_64_CODE_4_GOTPCRELX + ) && isec.is_alloc() + { + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let s = sym.addr(ctx); + let p = isec.addr(ctx) + rel.r_offset; + let v = s.wrapping_add(rel.r_addend as u64).wrapping_sub(p); + if sym.is_pcrel_linktime_const(ctx) + && is_int(v as i64, 32) + && relax_gotpcrelx(loc_before(isec, rel), rel) != 0 + { + return R_X86_64_PC32; + } + } + rel.r_type + } +} + +fn write_u16(buf: &mut [u8], v: u16) { + buf[..2].copy_from_slice(&v.to_le_bytes()); +} + +fn write_u32(buf: &mut [u8], v: u32) { + buf[..4].copy_from_slice(&v.to_le_bytes()); +} + +fn write_u64(buf: &mut [u8], v: u64) { + buf[..8].copy_from_slice(&v.to_le_bytes()); +} + +/// The bytes of a section preceding a relocated location. +fn loc_before<'a>(isec: &'a InputSection, rel: &ElfRel) -> &'a [u8] { + &isec.contents()[..rel.r_offset as usize] +} + +fn last2(loc: &[u8]) -> u32 { + let n = loc.len(); + ((loc[n - 2] as u32) << 8) | loc[n - 1] as u32 +} + +fn last3(loc: &[u8]) -> u32 { + let n = loc.len(); + ((loc[n - 3] as u32) << 16) | ((loc[n - 2] as u32) << 8) | loc[n - 1] as u32 +} + +/// The instruction to substitute when a GOT load can become a direct +/// reference, or 0 if the instruction isn't relaxable. `loc` is the code +/// up to the relocated operand. +fn relax_gotpcrelx(loc: &[u8], rel: &ElfRel) -> u32 { + if rel.r_type == R_X86_64_GOTPCRELX { + if loc.len() < 2 { + return 0; + } + return match last2(loc) { + 0xff15 => 0x40e8, // call *0(%rip) -> call 0 + 0xff25 => 0x40e9, // jmp *0(%rip) -> jmp 0 + _ => 0, + }; + } + if loc.len() < 3 { + return 0; + } + match last3(loc) { + // mov 0(%rip), %reg -> lea 0(%rip), %reg + 0x488b05 | 0x4c8b05 => 0x8d05, + 0x488b0d | 0x4c8b0d => 0x8d0d, + 0x488b15 | 0x4c8b15 => 0x8d15, + 0x488b1d | 0x4c8b1d => 0x8d1d, + 0x488b25 | 0x4c8b25 => 0x8d25, + 0x488b2d | 0x4c8b2d => 0x8d2d, + 0x488b35 | 0x4c8b35 => 0x8d35, + 0x488b3d | 0x4c8b3d => 0x8d3d, + _ => 0, + } +} + +fn relax_gottpoff(loc: &[u8], rel: &ElfRel) -> u32 { + if loc.len() < 3 { + return 0; + } + let insn = last3(loc); + if rel.r_type == R_X86_64_GOTTPOFF { + match insn { + // mov 0(%rip), %reg -> mov $0, %reg + 0x488b05 => 0x48c7c0, + 0x488b0d => 0x48c7c1, + 0x488b15 => 0x48c7c2, + 0x488b1d => 0x48c7c3, + 0x488b25 => 0x48c7c4, + 0x488b2d => 0x48c7c5, + 0x488b35 => 0x48c7c6, + 0x488b3d => 0x48c7c7, + 0x4c8b05 => 0x49c7c0, + 0x4c8b0d => 0x49c7c1, + 0x4c8b15 => 0x49c7c2, + 0x4c8b1d => 0x49c7c3, + 0x4c8b25 => 0x49c7c4, + 0x4c8b2d => 0x49c7c5, + 0x4c8b35 => 0x49c7c6, + 0x4c8b3d => 0x49c7c7, + _ => 0, + } + } else { + debug_assert_eq!(rel.r_type, R_X86_64_CODE_4_GOTTPOFF); + match insn { + // mov 0(%rip), %r16..%r31 -> mov $0, %r16..%r31 + 0x488b05 => 0x18c7c0, + 0x488b0d => 0x18c7c1, + 0x488b15 => 0x18c7c2, + 0x488b1d => 0x18c7c3, + 0x488b25 => 0x18c7c4, + 0x488b2d => 0x18c7c5, + 0x488b35 => 0x18c7c6, + 0x488b3d => 0x18c7c7, + 0x4c8b05 => 0x19c7c0, + 0x4c8b0d => 0x19c7c1, + 0x4c8b15 => 0x19c7c2, + 0x4c8b1d => 0x19c7c3, + 0x4c8b25 => 0x19c7c4, + 0x4c8b2d => 0x19c7c5, + 0x4c8b35 => 0x19c7c6, + 0x4c8b3d => 0x19c7c7, + _ => 0, + } + } +} + +fn relax_tlsdesc_to_ie(loc: &[u8], rel: &ElfRel) -> u32 { + if loc.len() < 3 { + return 0; + } + let _ = rel; + match last3(loc) { + // lea 0(%rip), %reg -> mov 0(%rip), %reg + 0x488d05 => 0x488b05, + 0x488d0d => 0x488b0d, + 0x488d15 => 0x488b15, + 0x488d1d => 0x488b1d, + 0x488d25 => 0x488b25, + 0x488d2d => 0x488b2d, + 0x488d35 => 0x488b35, + 0x488d3d => 0x488b3d, + 0x4c8d05 => 0x4c8b05, + 0x4c8d0d => 0x4c8b0d, + 0x4c8d15 => 0x4c8b15, + 0x4c8d1d => 0x4c8b1d, + 0x4c8d25 => 0x4c8b25, + 0x4c8d2d => 0x4c8b2d, + 0x4c8d35 => 0x4c8b35, + 0x4c8d3d => 0x4c8b3d, + _ => 0, + } +} + +fn relax_tlsdesc_to_le(loc: &[u8], rel: &ElfRel) -> u32 { + if loc.len() < 3 { + return 0; + } + let insn = last3(loc); + if rel.r_type == R_X86_64_GOTPC32_TLSDESC { + match insn { + // lea 0(%rip), %reg -> mov $0, %reg + 0x488d05 => 0x48c7c0, + 0x488d0d => 0x48c7c1, + 0x488d15 => 0x48c7c2, + 0x488d1d => 0x48c7c3, + 0x488d25 => 0x48c7c4, + 0x488d2d => 0x48c7c5, + 0x488d35 => 0x48c7c6, + 0x488d3d => 0x48c7c7, + 0x4c8d05 => 0x49c7c0, + 0x4c8d0d => 0x49c7c1, + 0x4c8d15 => 0x49c7c2, + 0x4c8d1d => 0x49c7c3, + 0x4c8d25 => 0x49c7c4, + 0x4c8d2d => 0x49c7c5, + 0x4c8d35 => 0x49c7c6, + 0x4c8d3d => 0x49c7c7, + _ => 0, + } + } else { + match insn { + 0x488d05 => 0x18c7c0, + 0x488d0d => 0x18c7c1, + 0x488d15 => 0x18c7c2, + 0x488d1d => 0x18c7c3, + 0x488d25 => 0x18c7c4, + 0x488d2d => 0x18c7c5, + 0x488d35 => 0x18c7c6, + 0x488d3d => 0x18c7c7, + 0x4c8d05 => 0x19c7c0, + 0x4c8d0d => 0x19c7c1, + 0x4c8d15 => 0x19c7c2, + 0x4c8d1d => 0x19c7c3, + 0x4c8d25 => 0x19c7c4, + 0x4c8d2d => 0x19c7c5, + 0x4c8d35 => 0x19c7c6, + 0x4c8d3d => 0x19c7c7, + _ => 0, + } + } +} + +/// Rewrites a `__tls_get_addr` call sequence to compute a link-time +/// constant TP-relative address instead. +fn relax_gd_to_le(buf: &mut [u8], off: usize, rel: &ElfRel, val: u64) { + match rel.r_type { + R_X86_64_PLT32 | R_X86_64_PC32 | R_X86_64_GOTPCREL | R_X86_64_GOTPCRELX => { + // lea foo@tlsgd(%rip), %rdi; call __tls_get_addr + const INSN: [u8; 16] = [ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // mov %fs:0, %rax + 0x48, 0x81, 0xc0, 0, 0, 0, 0, // add $tp_offset, %rax + ]; + buf[off - 4..off + 12].copy_from_slice(&INSN); + write_u32(&mut buf[off + 8..], val as u32); + } + R_X86_64_PLTOFF64 => { + // lea foo@tlsgd(%rip), %rdi; movabs __tls_get_addr, %rax; + // add %rbx, %rax; call *%rax + const INSN: [u8; 22] = [ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // mov %fs:0, %rax + 0x48, 0x81, 0xc0, 0, 0, 0, 0, // add $tp_offset, %rax + 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, // nop + ]; + buf[off - 3..off + 19].copy_from_slice(&INSN); + write_u32(&mut buf[off + 9..], val as u32); + } + _ => unreachable!(), + } +} + +fn relax_gd_to_ie(buf: &mut [u8], off: usize, rel: &ElfRel, val: u64) { + match rel.r_type { + R_X86_64_PLT32 | R_X86_64_PC32 | R_X86_64_GOTPCREL | R_X86_64_GOTPCRELX => { + const INSN: [u8; 16] = [ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // mov %fs:0, %rax + 0x48, 0x03, 0x05, 0, 0, 0, 0, // add foo@gottpoff(%rip), %rax + ]; + buf[off - 4..off + 12].copy_from_slice(&INSN); + write_u32(&mut buf[off + 8..], val.wrapping_sub(12) as u32); + } + R_X86_64_PLTOFF64 => { + const INSN: [u8; 22] = [ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // mov %fs:0, %rax + 0x48, 0x03, 0x05, 0, 0, 0, 0, // add foo@gottpoff(%rip), %rax + 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, // nop + ]; + buf[off - 3..off + 19].copy_from_slice(&INSN); + write_u32(&mut buf[off + 9..], val.wrapping_sub(13) as u32); + } + _ => unreachable!(), + } +} + +/// Like `relax_gd_to_le`, but materializes the address of the TLS block +/// rather than of a particular variable. +fn relax_ld_to_le(buf: &mut [u8], off: usize, rel: &ElfRel, tls_size: u64) { + match rel.r_type { + R_X86_64_PLT32 | R_X86_64_PC32 => { + // lea foo@tlsld(%rip), %rdi; call __tls_get_addr + // + // The sequence is so short that `mov %fs:0, %rax` (9 bytes) + // doesn't fit; `xor %eax, %eax` plus `mov %fs:(%rax), %rax` does. + const INSN: [u8; 12] = [ + 0x31, 0xc0, // xor %eax, %eax + 0x64, 0x48, 0x8b, 0x00, // mov %fs:(%rax), %rax + 0x48, 0x2d, 0, 0, 0, 0, // sub $tls_size, %rax + ]; + buf[off - 3..off + 9].copy_from_slice(&INSN); + write_u32(&mut buf[off + 5..], tls_size as u32); + } + R_X86_64_GOTPCREL | R_X86_64_GOTPCRELX => { + // lea foo@tlsld(%rip), %rdi; call *__tls_get_addr@GOT(%rip) + const INSN: [u8; 13] = [ + 0x48, 0x31, 0xc0, // xor %rax, %rax + 0x64, 0x48, 0x8b, 0x00, // mov %fs:(%rax), %rax + 0x48, 0x2d, 0, 0, 0, 0, // sub $tls_size, %rax + ]; + buf[off - 3..off + 10].copy_from_slice(&INSN); + write_u32(&mut buf[off + 6..], tls_size as u32); + } + R_X86_64_PLTOFF64 => { + // lea foo@tlsld(%rip), %rdi; movabs __tls_get_addr@GOTOFF, %rax; + // add %rbx, %rax; call *%rax + const INSN: [u8; 22] = [ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // mov %fs:0, %rax + 0x48, 0x2d, 0, 0, 0, 0, // sub $tls_size, %rax + 0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00, // nop + ]; + buf[off - 3..off + 19].copy_from_slice(&INSN); + write_u32(&mut buf[off + 8..], tls_size as u32); + } + _ => unreachable!(), + } +} + +/// Rewrites `endbr64` landing pads of functions whose address is never +/// taken with NOPs. The compiler emits a landing pad for every global +/// function since it can't know whether the address is taken elsewhere; +/// the linker sees all translation units and can do better. +pub fn rewrite_endbr(ctx: &Context, buf: &mut [u8]) { + const ENDBR64: [u8; 4] = [0xf3, 0x0f, 0x1e, 0xfa]; + const NOP: [u8; 4] = [0x0f, 0x1f, 0x40, 0x00]; + + let output_offset = |isec: &InputSection| -> Option { + let osec = &ctx.output_sections[isec.output_section?.index()]; + Some(osec.hdr.shdr.sh_offset + isec.offset()) + }; + + // Rewrite the landing pads of all global functions. File-scoped + // functions don't get one unless their address is taken anyway. + for (fi, file) in ctx.objs.iter().enumerate() { + for &id in file.base.global_symbols() { + let sym = &ctx.symbols[id]; + if sym.file() != Some(FileId::Obj(crate::input_files::ObjId(fi as u32))) + || sym.st_type() != STT_FUNC + { + continue; + } + let Some(isec) = sym.input_section_ref() else { + continue; + }; + if isec.sh_flags & SHF_EXECINSTR as u64 == 0 { + continue; + } + let Some(base) = output_offset(isec) else { + continue; + }; + let pos = (base + sym.value) as usize; + if buf.get(pos..pos + 4) == Some(&ENDBR64) { + buf[pos..pos + 4].copy_from_slice(&NOP); + } + } + } + + // Restore the landing pads that address-taking relocations refer to. + let mut write_back = |isec: Option<&InputSection>, offset: i64| { + let Some(isec) = isec else { return }; + let size = isec.contents().len() as i64; + if isec.sh_flags & SHF_EXECINSTR as u64 == 0 || offset < 0 || offset > size - 4 { + return; + } + let Some(base) = output_offset(isec) else { + return; + }; + if isec.contents()[offset as usize..offset as usize + 4] == ENDBR64 { + let pos = (base as i64 + offset) as usize; + buf[pos..pos + 4].copy_from_slice(&ENDBR64); + } + }; + + for file in &ctx.objs { + for isec in file.input_sections() { + if !isec.is_alive() || !isec.is_alloc() { + continue; + } + for rel in isec.rels::(file) { + if rel.is_func_call::() { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let target = sym.input_section_ref(); + if sym.st_type() == STT_SECTION { + write_back(target, rel.r_addend); + } else { + write_back(target, sym.value as i64); + } + } + } + } + + // Symbols whose addresses are recorded in the ELF header, .dynamic or + // .dynsym keep their landing pads. + let mut keep = |id: SymbolId| { + let sym = &ctx.symbols[id]; + write_back(sym.input_section_ref(), sym.value as i64); + }; + keep(ctx.syms.entry); + keep(ctx.syms.init); + keep(ctx.syms.fini); + for &id in ctx.dynsym.symbols.iter().flatten() { + if ctx.symbols[id].is_exported() { + keep(id); + } + } +} + +use crate::symbol::SymbolId; diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 000000000..8033ac8b4 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,203 @@ +//! Archive (`.a`) file reading. +//! +//! An archive is a bundle of object files, similar to tar but without +//! compression. A regular ("fat") archive contains the object files +//! themselves; a thin archive contains only their paths. +//! +//! Giving an archive to the linker means something different from giving +//! its members directly: only the members needed to resolve undefined +//! symbols are linked. That is why linking `libc.a` doesn't pull the whole +//! C library into an executable. + +use crate::diagnostics::Diagnostics; +use crate::fatal; +use crate::mapped_file::MappedFile; +use crate::util; + +const HEADER_SIZE: usize = 60; + +/// A parsed archive member header. +struct ArHeader<'a> { + name: &'a [u8], + size: usize, +} + +impl<'a> ArHeader<'a> { + fn parse(bytes: &'a [u8]) -> Option { + let bytes = bytes.get(..HEADER_SIZE)?; + let size = parse_decimal(&bytes[48..58]); + Some(ArHeader { + name: &bytes[..16], + size, + }) + } + + fn is_strtab(&self) -> bool { + self.name.starts_with(b"// ") + } + + fn is_symtab(&self) -> bool { + self.name.starts_with(b"/ ") || self.name.starts_with(b"/SYM64/ ") + } + + /// Returns the member's file name. A BSD-style long name is stored + /// right after the header, so `body` is advanced past it. + fn read_name(&self, strtab: &[u8], body: &mut &'a [u8]) -> String { + // BSD-style long filename + if let Some(rest) = self.name.strip_prefix(b"#1/") { + let len = parse_decimal(rest); + let (name, remaining) = body.split_at(len.min(body.len())); + *body = remaining; + let name = name.split(|&b| b == 0).next().unwrap_or(&[]); + return String::from_utf8_lossy(name).into_owned(); + } + + // SysV-style long filename + if let Some(rest) = self.name.strip_prefix(b"/") { + let offset = parse_decimal(rest); + let start = strtab.get(offset..).unwrap_or(&[]); + let end = start + .windows(2) + .position(|w| w == b"/\n") + .unwrap_or(start.len()); + return String::from_utf8_lossy(&start[..end]).into_owned(); + } + + // Short filename + let end = self + .name + .iter() + .position(|&b| b == b'/') + .unwrap_or(self.name.len()); + String::from_utf8_lossy(&self.name[..end]).into_owned() + } +} + +/// Parses the leading decimal digits of a field, like `atoi`. +fn parse_decimal(bytes: &[u8]) -> usize { + bytes + .iter() + .skip_while(|b| b.is_ascii_whitespace()) + .take_while(|b| b.is_ascii_digit()) + .fold(0, |acc, &b| acc * 10 + (b - b'0') as usize) +} + +/// Iterates over the members of an archive as (header, name, body) +/// triples, skipping the symbol table and string table. +fn for_each_member( + diag: &Diagnostics, + mf: &'static MappedFile, + thin: bool, + mut f: impl FnMut(String, &'static [u8]), +) { + let data = mf.data(); + let mut pos = 8; + let mut strtab: &[u8] = &[]; + + while data.len() - pos >= 2 { + // Each header is aligned to a 2 byte boundary. + if pos % 2 != 0 { + pos += 1; + } + + let Some(hdr) = ArHeader::parse(&data[pos..]) else { + break; + }; + let body_start = pos + HEADER_SIZE; + let body_end = (body_start + hdr.size).min(data.len()); + let mut body = &data[body_start..body_end]; + + if hdr.is_strtab() { + strtab = body; + pos = body_end; + continue; + } + if hdr.is_symtab() { + pos = body_end; + continue; + } + + if thin && !hdr.name.starts_with(b"#1/") && !hdr.name.starts_with(b"/") { + fatal!( + diag, + "{}: filename is not stored as a long filename", + mf.name + ); + } + + let name = hdr.read_name(strtab, &mut body); + + if thin { + // A thin archive member's contents live elsewhere; only a + // BSD-style long name occupies space after the header. + pos = body_start + (body_end - body_start - body.len()); + } else { + pos = body_end; + } + + // Skip the symbol table + if name == "__.SYMDEF" || name == "__.SYMDEF SORTED" { + continue; + } + + f(name, body); + } +} + +/// Returns the paths of the members of a thin archive without opening them. +pub fn get_thin_archive_member_paths(diag: &Diagnostics, mf: &'static MappedFile) -> Vec { + let mut paths = Vec::new(); + for_each_member(diag, mf, true, |name, _| { + if name.starts_with('/') { + paths.push(name); + } else { + paths.push(format!("{}/{}", util::path_dirname(&mf.name), name)); + } + }); + paths +} + +pub fn read_thin_archive_members( + diag: &Diagnostics, + mf: &'static MappedFile, +) -> Vec<&'static MappedFile> { + get_thin_archive_member_paths(diag, mf) + .into_iter() + .map(|path| { + let member = MappedFile::must_open(diag, &path); + util::leak(MappedFile { + name: member.name.clone(), + data: member.data, + given_fullpath: true, + parent: None, + thin_parent: Some(mf), + is_dependency: std::sync::atomic::AtomicBool::new(true), + }) + }) + .collect::>() +} + +pub fn read_fat_archive_members( + diag: &Diagnostics, + mf: &'static MappedFile, +) -> Vec<&'static MappedFile> { + let mut members = Vec::new(); + let base = mf.data().as_ptr() as usize; + for_each_member(diag, mf, false, |name, body| { + let start = body.as_ptr() as usize - base; + members.push(mf.slice(name, start, body.len())); + }); + members +} + +pub fn read_archive_members( + diag: &Diagnostics, + mf: &'static MappedFile, +) -> Vec<&'static MappedFile> { + if mf.data().starts_with(b"!\n") { + read_fat_archive_members(diag, mf) + } else { + debug_assert!(mf.data().starts_with(b"!\n")); + read_thin_archive_members(diag, mf) + } +} diff --git a/src/args.rs b/src/args.rs new file mode 100644 index 000000000..1cf92911d --- /dev/null +++ b/src/args.rs @@ -0,0 +1,2060 @@ +//! Command-line argument parsing. + +use std::collections::{HashMap, HashSet}; + +use crate::arch; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::mapped_file::MappedFile; +use crate::util::glob::Glob; +use crate::util::{self, align_down, path_clean, path_filename}; +use crate::{fatal, out, warn}; + +const HELP: &str = " +Options: + --help Report usage information + -v, --version Report version information + -V Report version and target information + -(, --start-group Ignored + -), --end-group Ignored + -C DIR, --directory DIR Change to DIR before doing anything + -E, --export-dynamic Put symbols in the dynamic symbol table + --no-export-dynamic + -F LIBNAME, --filter LIBNAME + Set DT_FILTER to the specified value + -I FILE, --dynamic-linker FILE + Set dynamic linker path + --no-dynamic-linker + -L DIR, --library-path DIR Add DIR to library search path + -M, --print-map Write map file to stdout + -N, --omagic Do not page align data; do not make text readonly + --no-omagic + -O NUMBER Ignored + -P AUDITLIB, --depaudit AUDITLIB + Set DT_DEPAUDIT to the specified value + -S, --strip-debug Strip .debug_* sections + -T FILE, --script FILE Read linker script + -X, --discard-locals Discard temporary local symbols (default) + -e SYMBOL, --entry SYMBOL Set program entry point + -f SHLIB, --auxiliary SHLIB Set DT_AUXILIARY to the specified value + -h LIBNAME, --soname LIBNAME + Set shared library name + -l LIBNAME, --library LIBNAME + Search for a given library + -m TARGET Set target + -o FILE, --output FILE Set output filename + -q, --emit-relocs Leaves relocation sections in the output + -r, --relocatable Generate relocatable output + -s, --strip-all Strip .symtab section + -u SYMBOL, --undefined SYMBOL + Force to resolve SYMBOL + -w, --no-warnings Suppress warnings + -y SYMBOL, --trace-symbol SYMBOL + Trace references to SYMBOL + --Bdynamic, --dy Link against shared libraries (default) + --Bstatic, --dn, --static Do not link against shared libraries + --Bsymbolic Bind all symbols locally + --Bsymbolic-functions Bind function symbols locally + --Bsymbolic-non-weak Bind all but weak symbols locally + --Bsymbolic-non-weak-functions + Bind all but weak function symbols locally + --Bno-symbolic Cancel --Bsymbolic options + --Map FILE Write map file to a given file + --Tbss=ADDR Set address to .bss + --Tdata=ADDR Set address to .data + --Ttext=ADDR Set address to .text + --Ttext-segment=ADDR Set address of text segment + --allow-multiple-definition Allow multiple definitions + --apply-dynamic-relocs Apply link-time values for dynamic relocations (default) + --no-apply-dynamic-relocs + --as-needed Only set DT_NEEDED if used + --no-as-needed + --audit LIBNAME Set DT_AUDIT to the specified value + --build-id [none,md5,sha1,sha256,fast,uuid,HEXSTRING] + Generate build ID + --no-build-id + --chroot DIR Set a given path to the root directory + --color-diagnostics=[auto,always,never] + Use colors in diagnostics + --color-diagnostics Alias for --color-diagnostics=always + --compress-debug-sections=[none,zlib,zlib:0,...,zlib:9,zstd,zstd:1,...,zstd:22] + Compress .debug_* sections + --dc Ignored + --dependency-file=FILE Write Makefile-style dependency rules to FILE + --defsym=SYMBOL=VALUE Define a symbol alias + --demangle Demangle C++ symbols in log messages (default) + --no-demangle + --detach Create separate debug info file in the background (default) + --no-detach + --discard-none Keep all local symbols in the symbol table + --enable-new-dtags Emit DT_RUNPATH for --rpath (default) + --disable-new-dtags Emit DT_RPATH for --rpath + --execute-only Make executable segments unreadable + --dp Ignored + --dynamic-list=FILE Read a list of dynamic symbols (implies -Bsymbolic) + --dynamic-list-data Add data symbols to dynamic symbols + --eh-frame-hdr Create .eh_frame_hdr section + --no-eh-frame-hdr + --exclude-libs LIB,LIB,.. Mark all symbols in given libraries as hidden + --export-dynamic-symbol Put symbols matching glob in the dynamic symbol table + --export-dynamic-symbol-list=FILE + Read a list of dynamic symbols + --fatal-warnings Treat warnings as errors + --no-fatal-warnings Do not treat warnings as errors (default) + --fini SYMBOL Call SYMBOL at unload-time + --fork Spawn a child process (default) + --no-fork + --gc-sections Remove unreferenced sections + --no-gc-sections + --gdb-index Create .gdb_index for faster gdb startup + --hash-style [sysv,gnu,both,none] + Set hash style + --icf=[all,safe,none] Fold identical code + --no-icf + --ignore-data-address-equality + Allow merging non-executable sections with --icf + --image-base ADDR Set the base address to a given value + --init SYMBOL Call SYMBOL at load-time + --nmagic Do not page align sections + --no-nmagic + --no-undefined Report undefined symbols (even with --shared) + --noinhibit-exec Create an output file even if errors occur + --oformat=binary Omit ELF, section, and program headers + --pack-dyn-relocs=[relr,android,android+relr,none] + Pack dynamic relocations + --package-metadata=PERCENT_ENCODED_STRING + Set a given string to .note.package + --perf Print performance statistics + --pie, --pic-executable Create a position-independent executable + --no-pie, --no-pic-executable + --pop-state Restore the state of flags governing input file handling + --print-gc-sections[=FILE] Print, or save in FILE, removed unreferenced sections + --no-print-gc-sections + --print-icf-sections[=FILE] Print, or save in FILE, folded identical sections + --no-print-icf-sections + --push-state Save the state of flags governing input file handling + --quick-exit Use quick_exit to exit (default) + --no-quick-exit + --relax Optimize instructions (default) + --no-relax + --repro Embed input files in .repro section + --require-defined SYMBOL Require SYMBOL be defined in the final output + --retain-symbols-file FILE Keep only symbols listed in FILE + --reverse-sections Reverse input sections in the output file + --rosegment Put read-only non-executable sections in their own segment (default) + --no-rosegment Put read-only non-executable sections in an executable segment + --rpath DIR Add DIR to the runtime search path + --rpath-link DIR Ignored + --run COMMAND ARG... Run COMMAND with mold as /usr/bin/ld + --section-start=SECTION=ADDR Set address for section + --separate-debug-file[=FILE] Separate debug info to the specified file + --no-separate-debug-file + --shared, --Bshareable Create a shared library + --shuffle-sections[=SEED] Randomize the output by shuffling input sections + --sort-common Ignored + --sort-section Ignored + --spare-dynamic-tags NUMBER Reserve the given number of tags in the .dynamic section + --spare-program-headers NUMBER + Reserve the given number of slots in the program header + --start-lib Give following object files in-archive-file semantics + --end-lib End the effect of --start-lib + --stats Print input statistics + --sysroot DIR Set the target system root directory + --thread-count COUNT, --threads=COUNT + Use COUNT number of threads + --threads Use multiple threads (default) + --no-threads + --trace Print the name of each input file + --undefined-glob PATTERN Force to resolve all symbols that match a given pattern + --undefined-version Do not report version scripts that refer to undefined symbols + --no-undefined-version Report version scripts that refer to undefined symbols (default) + --unique PATTERN Don't merge input sections that match a given pattern + --unresolved-symbols [report-all,ignore-all,ignore-in-object-files,ignore-in-shared-libs] + Handle unresolved symbols + --version-script FILE Read version script + --warn-common Warn about common symbols + --no-warn-common + --warn-once Only warn once for each undefined symbol + --warn-shared-textrel Warn if the output .so needs text relocations + --warn-textrel Warn if the output file needs text relocations + --warn-unresolved-symbols Report unresolved symbols as warnings + --error-unresolved-symbols + Report unresolved symbols as errors (default) + --whole-archive Include all objects from static archives + --no-whole-archive + --wrap SYMBOL Use a wrapper function for a given symbol + --zero-to-bss Convert all-zero data sections into BSS + -z defs Report undefined symbols (even with --shared) + -z nodefs + -z common-page-size=VALUE Ignored + -z execstack Require an executable stack + -z noexecstack + -z execstack-if-needed Make the stack area executable if an input file explicitly requests it + -z initfirst Mark DSO to be initialized first at runtime + -z interpose Mark object to interpose all DSOs but the executable + -z keep-text-section-prefix Keep .text.{hot,unknown,unlikely,startup,exit} as separate sections in the final binary + -z nokeep-text-section-prefix + -z lazy Enable lazy function resolution (default) + -z max-page-size=VALUE Use VALUE as the memory page size + -z nocopyreloc Do not create copy relocations + -z nodefaultlib Make the dynamic loader ignore default search paths + -z nodelete Mark DSO non-deletable at runtime + -z nodlopen Mark DSO not available to dlopen + -z nodump Mark DSO not available to dldump + -z now Disable lazy function resolution + -z origin Mark object requiring immediate $ORIGIN processing at runtime + -z pack-relative-relocs Alias for --pack-dyn-relocs=relr + -z nopack-relative-relocs + -z sectionheader Do not omit section header (default) + -z nosectionheader Omit section header + -z start_stop_visibility=[hidden,protected] + Specify symbol visibility for \"__start_SECNAME\" and \"__stop_SECNAME\" symbols + -z separate-loadable-segments + Separate all loadable segments onto different pages + -z separate-code Separate code and data onto different pages + -z noseparate-code Allow overlap in pages + -z stack-size=VALUE Set the size of the stack segment + -z relro Make some sections read-only after relocation (default) + -z norelro + -z rewrite-endbr Rewrite indirect branch target instructions with NOPs + -z norewrite-endbr + -z rodynamic Make the .dynamic section read-only + -z text Report error if DT_TEXTREL is set + -z notext + -z textoff + +mold: supported targets: elf32-i386 elf64-x86-64 elf32-littlearm elf64-littleaarch64 elf64-bigaarch64 elf32-littleriscv elf32-bigriscv elf64-littleriscv elf64-bigriscv elf32-powerpc elf64-powerpc elf64-powerpc elf64-powerpcle elf64-s390 elf64-sparc elf32-m68k elf32-sh-linux elf64-loongarch elf32-loongarch +mold: supported emulations: elf_i386 elf_x86_64 armelf_linux_eabi aarch64elf aarch64linux aarch64elfb aarch64linuxb elf32lriscv elf32briscv elf64lriscv elf64briscv elf32ppc elf32ppclinux elf64ppc elf64lppc elf64_s390 elf64_sparc m68kelf shlelf_linux shelf_linux elf64loongarch elf32loongarch"; + +pub const VERSION: &str = "mold 2.42.0 (compatible with GNU ld)"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum BuildIdKind { + #[default] + None, + Hex, + Hash, + Uuid, +} + +#[derive(Clone, Debug, Default)] +pub struct BuildId { + pub kind: BuildIdKind, + pub value: Vec, + pub hash_size: usize, +} + +impl BuildId { + pub fn size(&self) -> usize { + match self.kind { + BuildIdKind::None => 0, + BuildIdKind::Hex => self.value.len(), + BuildIdKind::Hash => self.hash_size, + BuildIdKind::Uuid => 16, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum UnresolvedKind { + Error, + Warn, + #[default] + Ignore, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum BsymbolicKind { + #[default] + None, + All, + Functions, + NonWeak, + NonWeakFunctions, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum SeparateCodeKind { + SeparateLoadableSegments, + SeparateCode, + #[default] + NoSeparateCode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum CetReportKind { + #[default] + None, + Warning, + Error, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum ShuffleSectionsKind { + #[default] + None, + Shuffle, + Reverse, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SectionOrderKind { + Section, + Group, + Addr, + Align, + Symbol, +} + +#[derive(Clone, Debug)] +pub struct SectionOrder { + pub kind: SectionOrderKind, + pub name: String, + pub value: u64, + /// The original token, for error messages. + pub token: String, +} + +/// The right-hand side of `--defsym=SYMBOL=VALUE`. +#[derive(Clone, Debug)] +pub enum DefsymValue { + Addr(u64), + Symbol(String), +} + +/// A source of dynamic-list patterns, kept in command line order. +#[derive(Clone, Debug)] +pub enum DynamicListSource { + File(String), + Pattern(String), +} + +/// Options that apply to the input files following them on the command +/// line. +#[derive(Clone, Debug, Default)] +pub struct ReaderContext { + pub as_needed: bool, + pub in_lib: bool, + pub is_static: bool, + pub whole_archive: bool, + + /// The position of the file in the command line. Files inside other + /// files (archive members, GROUP entries) extend their parent's + /// position, so comparing positions lexicographically yields the + /// command line order. + pub pos: Vec, + + /// The number of files found so far inside the current file. + pub num_children: u32, +} + +impl ReaderContext { + /// Returns a context for the next file found inside the current one. + pub fn next_child(&mut self) -> ReaderContext { + let mut child = self.clone(); + child.pos.push(self.num_children); + child.num_children = 0; + self.num_children += 1; + child + } +} + +/// A file to read, with the reader state at its command line position. +#[derive(Clone, Debug, Default)] +pub struct ReaderJob { + pub rctx: ReaderContext, + pub name: String, + pub is_lib: bool, + pub mf: Option<&'static MappedFile>, + + /// For an archive member: the archive's name and, for a thin archive, + /// the archive itself. + pub archive_name: String, + pub thin_parent: Option<&'static MappedFile>, +} + +/// All command line options. +#[derive(Debug)] +pub struct Args { + pub bsymbolic: BsymbolicKind, + pub build_id: BuildId, + pub z_cet_report: CetReportKind, + pub undefined_glob: Glob, + pub unique: Glob, + pub z_separate_code: SeparateCodeKind, + pub shuffle_sections: ShuffleSectionsKind, + pub entry: String, + pub fini: String, + pub init: String, + pub unresolved_symbols: UnresolvedKind, + pub allow_multiple_definition: bool, + pub allow_shlib_undefined: bool, + pub apply_dynamic_relocs: bool, + pub be8: bool, + pub color_diagnostics: bool, + pub default_symver: bool, + pub demangle: bool, + pub detach: bool, + pub discard_all: bool, + pub discard_locals: bool, + pub dynamic_list_data: bool, + pub eh_frame_hdr: bool, + pub emit_relocs: bool, + pub enable_new_dtags: bool, + pub execute_only: bool, + pub export_dynamic: bool, + pub fatal_warnings: bool, + pub fork: bool, + pub gc_sections: bool, + pub gdb_index: bool, + pub hash_style_gnu: bool, + pub hash_style_sysv: bool, + pub icf: bool, + pub icf_all: bool, + pub ignore_data_address_equality: bool, + pub lto_pass2: bool, + pub nmagic: bool, + pub noinhibit_exec: bool, + pub oformat_binary: bool, + pub omagic: bool, + pub pack_dyn_relocs_android: bool, + pub pack_dyn_relocs_relr: bool, + pub perf: bool, + pub pic: bool, + pub pie: bool, + pub print_dependencies: bool, + pub print_map: bool, + pub quick_exit: bool, + pub relax: bool, + pub relocatable: bool, + pub relocatable_merge_sections: bool, + pub repro: bool, + pub rosegment: bool, + pub shared: bool, + pub start_stop: bool, + pub is_static: bool, + pub stats: bool, + pub strip_all: bool, + pub strip_debug: bool, + pub suppress_warnings: bool, + pub trace: bool, + pub undefined_version: bool, + pub use_android_relr_tags: bool, + pub warn_common: bool, + pub warn_once: bool, + pub warn_textrel: bool, + pub z_copyreloc: bool, + pub z_delete: bool, + pub z_dlopen: bool, + pub z_dump: bool, + pub z_dynamic_undefined_weak: bool, + pub z_execstack: bool, + pub z_execstack_if_needed: bool, + pub z_ibt: bool, + pub z_initfirst: bool, + pub z_interpose: bool, + pub z_keep_text_section_prefix: bool, + pub z_nodefaultlib: bool, + pub z_now: bool, + pub z_origin: bool, + pub z_relro: bool, + pub z_rewrite_endbr: bool, + pub z_rodynamic: bool, + pub z_sectionheader: bool, + pub z_shstk: bool, + pub z_start_stop_visibility_protected: bool, + pub z_text: bool, + pub zero_to_bss: bool, + pub compress_debug_sections: u32, + pub compress_debug_sections_level: i64, + pub filler: Option, + pub spare_dynamic_tags: i64, + pub spare_program_headers: i64, + pub z_stack_size: u64, + pub thread_count: Option, + pub retain_symbols_file: Option>, + pub physical_image_base: Option, + pub ttext_segment: Option, + pub map: String, + pub audit: String, + pub chroot: String, + pub depaudit: String, + pub dependency_file: String, + pub directory: String, + pub dynamic_linker: String, + pub output: String, + pub package_metadata: String, + pub plugin: String, + pub print_gc_sections: String, + pub print_icf_sections: String, + pub rpaths: String, + pub separate_debug_file: String, + pub soname: String, + pub sysroot: String, + pub emulation: String, + pub section_align: HashMap, + pub section_start: HashMap, + pub discard_section: HashSet, + pub exclude_libs: HashSet, + pub ignore_ir_file: HashSet, + pub wrap: HashSet, + pub section_order: Vec, + pub require_defined: Vec, + pub undefined: Vec, + pub defsyms: Vec<(String, DefsymValue)>, + pub library_paths: Vec, + pub plugin_opt: Vec, + pub version_definitions: Vec, + pub version_scripts: Vec, + pub dynamic_list: Vec, + pub auxiliary: Vec, + pub filter: Vec, + pub trace_symbol: Vec, + pub z_x86_64_isa_level: u32, + pub image_base: u64, + pub shuffle_sections_seed: u64, + pub page_size: u64, + + /// Whether an existing output file may be overwritten in place. + pub overwrite_output_file: bool, +} + +impl Default for Args { + fn default() -> Self { + Args { + bsymbolic: BsymbolicKind::None, + build_id: BuildId::default(), + z_cet_report: CetReportKind::None, + undefined_glob: Glob::new(), + unique: Glob::new(), + z_separate_code: SeparateCodeKind::NoSeparateCode, + shuffle_sections: ShuffleSectionsKind::None, + entry: "_start".to_string(), + fini: "_fini".to_string(), + init: "_init".to_string(), + unresolved_symbols: UnresolvedKind::Ignore, + allow_multiple_definition: false, + allow_shlib_undefined: true, + apply_dynamic_relocs: true, + be8: false, + color_diagnostics: false, + default_symver: false, + demangle: true, + detach: true, + discard_all: false, + discard_locals: true, + dynamic_list_data: false, + eh_frame_hdr: true, + emit_relocs: false, + enable_new_dtags: true, + execute_only: false, + export_dynamic: false, + fatal_warnings: false, + fork: true, + gc_sections: false, + gdb_index: false, + hash_style_gnu: true, + hash_style_sysv: true, + icf: false, + icf_all: false, + ignore_data_address_equality: false, + lto_pass2: false, + nmagic: false, + noinhibit_exec: false, + oformat_binary: false, + omagic: false, + pack_dyn_relocs_android: false, + pack_dyn_relocs_relr: false, + perf: false, + pic: false, + pie: false, + print_dependencies: false, + print_map: false, + quick_exit: true, + relax: true, + relocatable: false, + relocatable_merge_sections: false, + repro: false, + rosegment: true, + shared: false, + start_stop: false, + is_static: false, + stats: false, + strip_all: false, + strip_debug: false, + suppress_warnings: false, + trace: false, + undefined_version: false, + use_android_relr_tags: false, + warn_common: false, + warn_once: false, + warn_textrel: false, + z_copyreloc: true, + z_delete: true, + z_dlopen: true, + z_dump: true, + z_dynamic_undefined_weak: true, + z_execstack: false, + z_execstack_if_needed: false, + z_ibt: false, + z_initfirst: false, + z_interpose: false, + z_keep_text_section_prefix: false, + z_nodefaultlib: false, + z_now: false, + z_origin: false, + z_relro: true, + z_rewrite_endbr: false, + z_rodynamic: false, + z_sectionheader: true, + z_shstk: false, + z_start_stop_visibility_protected: false, + z_text: false, + zero_to_bss: false, + compress_debug_sections: ELFCOMPRESS_NONE, + compress_debug_sections_level: 0, + filler: None, + spare_dynamic_tags: 5, + spare_program_headers: 0, + z_stack_size: 0, + thread_count: None, + retain_symbols_file: None, + physical_image_base: None, + ttext_segment: None, + map: String::new(), + audit: String::new(), + chroot: String::new(), + depaudit: String::new(), + dependency_file: String::new(), + directory: String::new(), + dynamic_linker: String::new(), + output: "a.out".to_string(), + package_metadata: String::new(), + plugin: String::new(), + print_gc_sections: String::new(), + print_icf_sections: String::new(), + rpaths: String::new(), + separate_debug_file: String::new(), + soname: String::new(), + sysroot: String::new(), + emulation: String::new(), + section_align: HashMap::new(), + section_start: HashMap::new(), + discard_section: HashSet::new(), + exclude_libs: HashSet::new(), + ignore_ir_file: HashSet::new(), + wrap: HashSet::new(), + section_order: Vec::new(), + require_defined: Vec::new(), + undefined: Vec::new(), + defsyms: Vec::new(), + library_paths: Vec::new(), + plugin_opt: Vec::new(), + version_definitions: Vec::new(), + version_scripts: Vec::new(), + dynamic_list: Vec::new(), + auxiliary: Vec::new(), + filter: Vec::new(), + trace_symbol: Vec::new(), + z_x86_64_isa_level: 0, + image_base: 0x200000, + shuffle_sections_seed: 0, + page_size: 0, + overwrite_output_file: false, + } + } +} + +/// Properties of the target that influence option defaults and validation. +#[derive(Clone, Copy, Debug)] +pub struct TargetTraits { + pub name: &'static str, + pub is_rela: bool, + pub is_sparc: bool, + pub is_riscv: bool, + pub is_sh4: bool, + pub is_x86_64: bool, + pub page_size: u64, +} + +fn is_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// Reads a response file (`@file`) and returns its tokens, expanding +/// nested response files. +fn read_response_file(diag: &Diagnostics, path: &str, depth: usize) -> Vec { + if depth > 10 { + fatal!(diag, "{path}: response file nesting too deep"); + } + + let mf = MappedFile::must_open(diag, path); + mf.set_dependency(false); + let data = mf.data(); + let mut tokens = Vec::new(); + let mut i = 0; + + while i < data.len() { + if is_space(data[i]) { + i += 1; + continue; + } + + // A backslash escapes the next character; a quoted part may be + // followed by more characters of the same token. + let mut tok = Vec::new(); + let mut quote = None; + while i < data.len() { + let c = data[i]; + if c == b'\\' { + if i + 1 == data.len() { + fatal!(diag, "{path}: premature end of input"); + } + tok.push(data[i + 1]); + i += 2; + } else if let Some(q) = quote { + if c == q { + quote = None; + } else { + tok.push(c); + } + i += 1; + } else if c == b'\'' || c == b'"' { + quote = Some(c); + i += 1; + } else if is_space(c) { + break; + } else { + tok.push(c); + i += 1; + } + } + if quote.is_some() { + fatal!(diag, "{path}: premature end of input"); + } + tokens.push(String::from_utf8_lossy(&tok).into_owned()); + } + + let mut expanded = Vec::new(); + for tok in tokens { + if let Some(nested) = tok.strip_prefix('@') { + expanded.extend(read_response_file(diag, nested, depth + 1)); + } else { + expanded.push(tok); + } + } + expanded +} + +/// Replaces `@file` arguments with the contents of the named files. +pub fn expand_response_files(diag: &Diagnostics, argv: &[String]) -> Vec { + let mut args = Vec::new(); + for arg in argv { + if let Some(path) = arg.strip_prefix('@') { + args.extend(read_response_file(diag, path, 1)); + } else { + args.push(arg.clone()); + } + } + args +} + +/// Matches an argument against an option name. On success, returns the +/// remainder of the argument: matching "--foo=bar" against "foo" yields +/// "=bar", and matching "--foo" against "foo" yields "". +/// +/// Multi-letter option names accept a single or double dash, except names +/// starting with "o", which need a double dash so that "-omagic" means +/// "-o magic". Single-letter options take a single dash. +fn match_option<'a>(arg: &'a str, name: &str) -> Option<&'a str> { + let arg = arg.strip_prefix('-')?; + + if name.len() == 1 { + return arg.strip_prefix(name); + } + if name.starts_with('o') && !arg.starts_with('-') { + return None; + } + let arg = arg.strip_prefix('-').unwrap_or(arg); + arg.strip_prefix(name) +} + +fn parse_hex(diag: &Diagnostics, opt: &str, value: &str) -> u64 { + let digits = value + .strip_prefix("0x") + .or_else(|| value.strip_prefix("0X")) + .unwrap_or(value); + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_hexdigit()) { + fatal!(diag, "option -{opt}: not a hexadecimal number"); + } + u64::from_str_radix(digits, 16) + .unwrap_or_else(|_| fatal!(diag, "option -{opt}: not a hexadecimal number")) +} + +/// Parses an integer in C syntax (decimal, `0x` hex or leading-zero octal). +fn parse_c_number(s: &str) -> Option { + if let Some(hex) = s.strip_prefix("0x").or_else(|| s.strip_prefix("0X")) { + u64::from_str_radix(hex, 16).ok() + } else if s.len() > 1 && s.starts_with('0') { + u64::from_str_radix(&s[1..], 8).ok() + } else { + s.parse().ok() + } +} + +fn parse_number(diag: &Diagnostics, opt: &str, value: &str) -> i64 { + let (negative, digits) = match value.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, value), + }; + let n = parse_c_number(digits) + .unwrap_or_else(|| fatal!(diag, "option -{opt}: not a number: {value}")) as i64; + if negative { + -n + } else { + n + } +} + +fn from_hex(c: u8) -> u8 { + match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + _ => c - b'A' + 10, + } +} + +fn parse_hex_build_id(diag: &Diagnostics, arg: &str) -> Vec { + let digits = arg + .strip_prefix("0x") + .or_else(|| arg.strip_prefix("0X")) + .filter(|d| !d.is_empty() && d.len() % 2 == 0 && d.bytes().all(|b| b.is_ascii_hexdigit())) + .unwrap_or_else(|| fatal!(diag, "invalid build-id: {arg}")); + digits + .as_bytes() + .chunks(2) + .map(|pair| (from_hex(pair[0]) << 4) | from_hex(pair[1])) + .collect() +} + +fn parse_package_metadata(diag: &Diagnostics, arg: &str) -> String { + let bytes = arg.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + if i + 2 >= bytes.len() + || !bytes[i + 1].is_ascii_hexdigit() + || !bytes[i + 2].is_ascii_hexdigit() + { + fatal!(diag, "--package-metadata: invalid string: {arg}"); + } + out.push((from_hex(bytes[i + 1]) << 4) | from_hex(bytes[i + 2])); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8_lossy(&out).into_owned() +} + +fn read_retain_symbols_file(diag: &Diagnostics, path: &str) -> Vec { + let mf = MappedFile::must_open(diag, path); + String::from_utf8_lossy(mf.data()) + .lines() + .map(|line| line.trim_matches(|c| c == ' ' || c == '\t')) + .filter(|line| !line.is_empty()) + .map(str::to_string) + .collect() +} + +fn parse_section_order(diag: &Diagnostics, arg: &str) -> Vec { + let parse_value = |s: &str| -> Option { + if let Some(hex) = s.strip_prefix("0x").or_else(|| s.strip_prefix("0X")) { + u64::from_str_radix(hex, 16).ok() + } else { + s.parse().ok() + } + }; + let is_section_name = |s: &str| { + s.bytes() + .next() + .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'.') + && !s.bytes().any(|c| c.is_ascii_whitespace()) + }; + + let mut orders = Vec::new(); + for tok in arg.split([' ', '\t']).filter(|t| !t.is_empty()) { + let mut order = SectionOrder { + kind: SectionOrderKind::Section, + name: String::new(), + value: 0, + token: tok.to_string(), + }; + + if matches!( + tok.to_ascii_uppercase().as_str(), + "TEXT" | "DATA" | "RODATA" | "BSS" + ) { + order.kind = SectionOrderKind::Group; + order.name = tok.to_string(); + } else if let Some(v) = tok.strip_prefix('=').and_then(parse_value) { + order.kind = SectionOrderKind::Addr; + order.value = v; + } else if let Some(v) = tok.strip_prefix('%').and_then(parse_value) { + order.kind = SectionOrderKind::Align; + order.value = v; + } else if let Some(name) = tok.strip_prefix('!').filter(|n| !n.is_empty()) { + order.kind = SectionOrderKind::Symbol; + order.name = name.to_string(); + } else if is_section_name(tok) || tok == "EHDR" || tok == "PHDR" { + order.kind = SectionOrderKind::Section; + order.name = tok.to_string(); + } else { + fatal!(diag, "--section-order: parse error: {arg}"); + } + orders.push(order); + } + + let mut is_first = true; + for order in &orders { + if order.kind == SectionOrderKind::Section { + if is_first { + is_first = false; + } else if order.name == "EHDR" { + fatal!( + diag, + "--section-order: EHDR must be the first section specifier: {arg}" + ); + } + } + } + orders +} + +fn parse_defsym_value(s: &str) -> DefsymValue { + if let Some(hex) = s.strip_prefix("0x").or_else(|| s.strip_prefix("0X")) { + if let Ok(v) = u64::from_str_radix(hex, 16) { + return DefsymValue::Addr(v); + } + } + if !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) { + return DefsymValue::Addr(s.parse().unwrap_or(0)); + } + DefsymValue::Symbol(s.to_string()) +} + +/// Linux 6.11 and 6.12 don't return ETXTBSY for open(2) on a running +/// executable, which makes overwriting an output file in place unsafe. +fn returns_etxtbsy() -> bool { + let Ok(release) = std::fs::read_to_string("/proc/sys/kernel/osrelease") else { + return false; + }; + let mut parts = release + .trim() + .split(['.', '-']) + .map(|p| p.parse::().unwrap_or(0)); + let ver = ( + parts.next().unwrap_or(0), + parts.next().unwrap_or(0), + parts.next().unwrap_or(0), + ); + !((6, 11, 0)..(6, 13, 0)).contains(&ver) +} + +/// The result of parsing the command line. +pub struct ParsedArgs { + pub args: Args, + pub jobs: Vec, + /// The reader state at the end of the command line; `-static` there + /// means a static link. + pub final_rctx: ReaderContext, +} + +/// Parses all options. `cmdline` includes the program name. +pub fn parse_args(diag: &Diagnostics, target: &TargetTraits, cmdline: &[String]) -> ParsedArgs { + let mut a = Args::default(); + let mut jobs: Vec = Vec::new(); + let mut rctx = ReaderContext::default(); + let mut rctx_stack: Vec = Vec::new(); + let mut visited_libs: HashSet = HashSet::new(); + + a.color_diagnostics = stderr_is_tty(); + a.page_size = target.page_size; + + let mut version_shown = false; + let mut warn_shared_textrel = false; + let mut error_unresolved_symbols = true; + let mut z_separate_code: Option = None; + let mut allow_shlib_undefined: Option = None; + let mut report_undefined: Option = None; + let mut z_relro: Option = None; + let mut z_dynamic_undefined_weak: Option = None; + let mut separate_debug_file: Option = None; + let mut shuffle_sections_seed: Option = None; + let mut rpaths: HashSet = HashSet::new(); + + // We generally don't need to write addends to relocated places if the + // relocation type is RELA because RELA records contain addends. + // However, too much code wrongly assumes that addends are written to + // both, so we write them by default. SPARC's dynamic linker adds both + // r_addend and the value at the relocated place, and static PIEs + // crash on some RISC-V environments if addends are written, so those + // targets are exceptions. + a.apply_dynamic_relocs = !target.is_sparc && !target.is_riscv; + + let mut i = 1; + let mut arg = String::new(); + + macro_rules! read_arg { + ($name:expr) => {{ + let name: &str = $name; + match match_option(&cmdline[i], name) { + None => false, + Some(rest) if rest.is_empty() => { + if i + 1 == cmdline.len() { + fatal!(diag, "option -{name}: argument missing"); + } + arg = cmdline[i + 1].clone(); + i += 2; + true + } + Some(rest) if name.len() == 1 => { + arg = rest.to_string(); + i += 1; + true + } + Some(rest) => match rest.strip_prefix('=') { + Some(value) => { + arg = value.to_string(); + i += 1; + true + } + None => false, + }, + } + }}; + } + + macro_rules! read_eq { + ($name:expr) => {{ + match match_option(&cmdline[i], $name).and_then(|rest| rest.strip_prefix('=')) { + Some(value) => { + arg = value.to_string(); + i += 1; + true + } + None => false, + } + }}; + } + + macro_rules! read_flag { + ($name:expr) => {{ + if match_option(&cmdline[i], $name) == Some("") { + i += 1; + true + } else { + false + } + }}; + } + + macro_rules! read_z_flag { + ($name:expr) => {{ + let name: &str = $name; + if i + 1 < cmdline.len() && cmdline[i] == "-z" && cmdline[i + 1] == name { + i += 2; + true + } else if cmdline[i].strip_prefix("-z") == Some(name) { + i += 1; + true + } else { + false + } + }}; + } + + macro_rules! read_z_arg { + ($name:expr) => {{ + let name: &str = $name; + let prefix = format!("{name}="); + if i + 1 < cmdline.len() && cmdline[i] == "-z" && cmdline[i + 1].starts_with(&prefix) { + arg = cmdline[i + 1][prefix.len()..].to_string(); + i += 2; + true + } else if let Some(value) = cmdline[i] + .strip_prefix("-z") + .and_then(|s| s.strip_prefix(prefix.as_str())) + { + arg = value.to_string(); + i += 1; + true + } else { + false + } + }}; + } + + while i < cmdline.len() { + if !cmdline[i].starts_with('-') { + let mut job = ReaderJob { + rctx: rctx.clone(), + name: cmdline[i].clone(), + ..Default::default() + }; + job.rctx.pos = vec![jobs.len() as u32]; + jobs.push(job); + i += 1; + continue; + } + + if read_flag!("help") { + out!(diag, "Usage: {} [options] file...\n{}", cmdline[0], HELP); + std::process::exit(0); + } + + if read_arg!("o") || read_arg!("output") { + a.output = arg.clone(); + } else if read_arg!("dynamic-linker") || read_arg!("I") { + a.dynamic_linker = arg.clone(); + } else if read_flag!("no-dynamic-linker") { + a.dynamic_linker.clear(); + } else if read_flag!("v") { + out!(diag, "{VERSION}"); + version_shown = true; + } else if read_flag!("version") { + out!(diag, "{VERSION}"); + std::process::exit(0); + } else if read_flag!("V") { + out!( + diag, + "{VERSION}\n Supported emulations:\n elf_x86_64\n elf_i386\n aarch64elf\n \ + aarch64linux\n aarch64elfb\n aarch64linuxb\n armelf_linux_eabi\n elf64lriscv\n \ + elf64briscv\n elf32lriscv\n elf32briscv\n elf32ppc\n elf64ppc\n elf64lppc\n \ + elf64_s390\n elf64_sparc\n m68kelf\n shlelf_linux\n shelf_linux\n \ + elf64loongarch\n elf32loongarch" + ); + version_shown = true; + } else if read_arg!("mllvm") { + a.plugin_opt.push(arg.clone()); + } else if read_arg!("m") { + match arch::emulation_to_target(&arg) { + Some(name) => a.emulation = name.to_string(), + None => fatal!(diag, "unknown -m argument: {arg}"), + } + } else if read_flag!("end-lib") { + rctx.in_lib = false; + } else if read_flag!("export-dynamic") || read_flag!("E") { + a.export_dynamic = true; + } else if read_flag!("no-export-dynamic") { + a.export_dynamic = false; + } else if read_flag!("Bsymbolic") { + a.bsymbolic = BsymbolicKind::All; + } else if read_flag!("Bsymbolic-functions") { + a.bsymbolic = BsymbolicKind::Functions; + } else if read_flag!("Bsymbolic-non-weak") { + a.bsymbolic = BsymbolicKind::NonWeak; + } else if read_flag!("Bsymbolic-non-weak-functions") { + a.bsymbolic = BsymbolicKind::NonWeakFunctions; + } else if read_flag!("Bno-symbolic") { + a.bsymbolic = BsymbolicKind::None; + } else if read_arg!("exclude-libs") { + for lib in arg.split([',', ':']) { + a.exclude_libs.insert(lib.to_string()); + } + } else if read_flag!("q") || read_flag!("emit-relocs") { + a.emit_relocs = true; + a.discard_locals = false; + } else if read_arg!("e") || read_arg!("entry") { + a.entry = arg.clone(); + } else if read_arg!("Map") { + a.map = arg.clone(); + a.print_map = true; + } else if read_flag!("print-dependencies") { + a.print_dependencies = true; + } else if read_flag!("print-map") || read_flag!("M") { + a.print_map = true; + } else if read_flag!("Bstatic") || read_flag!("dn") || read_flag!("static") { + rctx.is_static = true; + } else if read_flag!("Bdynamic") || read_flag!("dy") { + rctx.is_static = false; + } else if read_flag!("shared") || read_flag!("Bshareable") { + a.shared = true; + } else if read_arg!("spare-dynamic-tags") { + a.spare_dynamic_tags = parse_number(diag, "spare-dynamic-tags", &arg); + } else if read_arg!("spare-program-headers") { + a.spare_program_headers = parse_number(diag, "spare-program-headers", &arg); + } else if read_flag!("start-lib") { + rctx.in_lib = true; + } else if read_flag!("start-stop") { + a.start_stop = true; + } else if read_arg!("dependency-file") { + a.dependency_file = arg.clone(); + } else if read_arg!("defsym") { + let Some((name, value)) = arg.split_once('=').filter(|(_, v)| !v.is_empty()) else { + fatal!(diag, "-defsym: syntax error: {arg}"); + }; + a.defsyms + .push((name.to_string(), parse_defsym_value(value))); + } else if read_flag!(":lto-pass2") { + a.lto_pass2 = true; + } else if read_arg!(":ignore-ir-file") { + a.ignore_ir_file.insert(arg.clone()); + } else if read_flag!("demangle") { + a.demangle = true; + } else if read_flag!("no-demangle") { + a.demangle = false; + } else if read_flag!("detach") { + a.detach = true; + } else if read_flag!("no-detach") { + a.detach = false; + } else if read_flag!("default-symver") { + a.default_symver = true; + } else if read_flag!("noinhibit-exec") { + a.noinhibit_exec = true; + } else if read_flag!("shuffle-sections") { + a.shuffle_sections = ShuffleSectionsKind::Shuffle; + } else if read_eq!("shuffle-sections") { + a.shuffle_sections = ShuffleSectionsKind::Shuffle; + shuffle_sections_seed = Some(parse_number(diag, "shuffle-sections", &arg) as u64); + } else if read_flag!("reverse-sections") { + a.shuffle_sections = ShuffleSectionsKind::Reverse; + } else if read_flag!("rosegment") { + a.rosegment = true; + } else if read_flag!("no-rosegment") { + a.rosegment = false; + } else if read_arg!("y") || read_arg!("trace-symbol") { + a.trace_symbol.push(arg.clone()); + } else if read_arg!("filler") { + a.filler = Some(parse_hex(diag, "filler", &arg) as u8); + } else if read_arg!("L") || read_arg!("library-path") { + a.library_paths.push(arg.clone()); + } else if read_arg!("sysroot") { + a.sysroot = arg.clone(); + } else if read_arg!("unique") { + if !a.unique.add(arg.as_bytes(), 1) { + fatal!(diag, "-unique: invalid glob pattern: {arg}"); + } + } else if read_arg!("unresolved-symbols") { + match arg.as_str() { + "report-all" | "ignore-in-shared-libs" => report_undefined = Some(true), + "ignore-all" | "ignore-in-object-files" => report_undefined = Some(false), + _ => fatal!(diag, "unknown --unresolved-symbols argument: {arg}"), + } + } else if read_arg!("undefined") || read_arg!("u") { + a.undefined.push(arg.clone()); + } else if read_arg!("undefined-glob") { + if !a.undefined_glob.add(arg.as_bytes(), 0) { + fatal!(diag, "--undefined-glob: invalid pattern: {arg}"); + } + } else if read_arg!("require-defined") { + a.require_defined.push(arg.clone()); + } else if read_arg!("init") { + a.init = arg.clone(); + } else if read_arg!("fini") { + a.fini = arg.clone(); + } else if read_arg!("hash-style") { + match arg.as_str() { + "sysv" => { + a.hash_style_sysv = true; + a.hash_style_gnu = false; + } + "gnu" => { + a.hash_style_sysv = false; + a.hash_style_gnu = true; + } + "both" => { + a.hash_style_sysv = true; + a.hash_style_gnu = true; + } + "none" => { + a.hash_style_sysv = false; + a.hash_style_gnu = false; + } + _ => fatal!(diag, "invalid --hash-style argument: {arg}"), + } + } else if read_arg!("soname") || read_arg!("h") { + a.soname = arg.clone(); + } else if read_arg!("audit") { + if !a.audit.is_empty() { + a.audit.push(':'); + } + a.audit.push_str(&arg); + } else if read_arg!("depaudit") || read_arg!("P") { + if !a.depaudit.is_empty() { + a.depaudit.push(':'); + } + a.depaudit.push_str(&arg); + } else if read_flag!("allow-multiple-definition") { + a.allow_multiple_definition = true; + } else if read_flag!("apply-dynamic-relocs") { + a.apply_dynamic_relocs = true; + } else if read_flag!("no-apply-dynamic-relocs") { + a.apply_dynamic_relocs = false; + } else if read_flag!("trace") { + a.trace = true; + } else if read_flag!("eh-frame-hdr") { + a.eh_frame_hdr = true; + } else if read_flag!("no-eh-frame-hdr") { + a.eh_frame_hdr = false; + } else if read_flag!("pie") || read_flag!("pic-executable") { + a.pic = true; + a.pie = true; + } else if read_flag!("no-pie") || read_flag!("no-pic-executable") || read_flag!("nopie") { + a.pic = false; + a.pie = false; + } else if read_flag!("relax") { + a.relax = true; + } else if read_flag!("no-relax") { + a.relax = false; + } else if read_flag!("gdb-index") { + a.gdb_index = true; + } else if read_flag!("no-gdb-index") { + a.gdb_index = false; + } else if read_flag!("r") || read_flag!("relocatable") { + a.relocatable = true; + a.emit_relocs = true; + a.discard_locals = false; + } else if read_flag!("relocatable-merge-sections") { + a.relocatable_merge_sections = true; + } else if read_flag!("perf") { + a.perf = true; + } else if read_flag!("pack-dyn-relocs=relr") || read_z_flag!("pack-relative-relocs") { + a.pack_dyn_relocs_relr = true; + a.pack_dyn_relocs_android = false; + } else if read_flag!("pack-dyn-relocs=android") { + a.pack_dyn_relocs_android = true; + a.pack_dyn_relocs_relr = false; + } else if read_flag!("pack-dyn-relocs=android+relr") { + a.pack_dyn_relocs_android = true; + a.pack_dyn_relocs_relr = true; + } else if read_flag!("pack-dyn-relocs=none") || read_z_flag!("nopack-relative-relocs") { + a.pack_dyn_relocs_relr = false; + a.pack_dyn_relocs_android = false; + } else if read_flag!("use-android-relr-tags") { + a.use_android_relr_tags = true; + } else if read_flag!("no-use-android-relr-tags") { + a.use_android_relr_tags = false; + } else if read_arg!("package-metadata") { + a.package_metadata = parse_package_metadata(diag, &arg); + } else if read_flag!("stats") { + a.stats = true; + } else if read_arg!("C") || read_arg!("directory") { + a.directory = arg.clone(); + } else if read_arg!("chroot") { + a.chroot = arg.clone(); + } else if read_flag!("color-diagnostics") || read_flag!("color-diagnostics=auto") { + a.color_diagnostics = stderr_is_tty(); + } else if read_flag!("color-diagnostics=always") { + a.color_diagnostics = true; + } else if read_flag!("color-diagnostics=never") { + a.color_diagnostics = false; + } else if read_flag!("warn-common") { + a.warn_common = true; + } else if read_flag!("no-warn-common") { + a.warn_common = false; + } else if read_flag!("warn-once") { + a.warn_once = true; + } else if read_flag!("warn-shared-textrel") { + warn_shared_textrel = true; + } else if read_flag!("warn-textrel") { + a.warn_textrel = true; + } else if read_flag!("enable-new-dtags") { + a.enable_new_dtags = true; + } else if read_flag!("disable-new-dtags") { + a.enable_new_dtags = false; + } else if read_flag!("execute-only") { + a.execute_only = true; + } else if read_flag!("zero-to-bss") { + a.zero_to_bss = true; + } else if read_arg!("compress-debug-sections") { + match arg.as_str() { + "zlib" | "zlib-gabi" => { + a.compress_debug_sections = ELFCOMPRESS_ZLIB; + a.compress_debug_sections_level = 1; + } + "zstd" => { + a.compress_debug_sections = ELFCOMPRESS_ZSTD; + a.compress_debug_sections_level = 3; + } + "none" => a.compress_debug_sections = ELFCOMPRESS_NONE, + s if s.starts_with("zlib:") => { + a.compress_debug_sections = ELFCOMPRESS_ZLIB; + let level = parse_number(diag, "compress-debug-sections", &s[5..]); + if !(0..=9).contains(&level) { + fatal!( + diag, + "invalid --compress-debug-sections argument: {arg} (zlib level must be between 0 and 9)" + ); + } + a.compress_debug_sections_level = level; + } + s if s.starts_with("zstd:") => { + a.compress_debug_sections = ELFCOMPRESS_ZSTD; + let level = parse_number(diag, "compress-debug-sections", &s[5..]); + if !(1..=22).contains(&level) { + fatal!( + diag, + "invalid --compress-debug-sections argument: {arg} (zstd level must be between 1 and 22)" + ); + } + a.compress_debug_sections_level = level; + } + _ => fatal!(diag, "invalid --compress-debug-sections argument: {arg}"), + } + } else if read_arg!("wrap") { + a.wrap.insert(arg.clone()); + } else if read_flag!("omagic") || read_flag!("N") { + a.omagic = true; + rctx.is_static = true; + } else if read_flag!("no-omagic") { + a.omagic = false; + } else if read_arg!("oformat") { + if arg != "binary" { + fatal!(diag, "-oformat: {arg} is not supported"); + } + a.oformat_binary = true; + } else if read_arg!("retain-symbols-file") { + a.retain_symbols_file = Some(read_retain_symbols_file(diag, &arg)); + } else if read_arg!("section-align") { + let Some((name, value)) = arg.split_once('=').filter(|(_, v)| !v.is_empty()) else { + fatal!(diag, "--section-align: syntax error: {arg}"); + }; + let value = parse_number(diag, "section-align", value); + if value <= 0 || !(value as u64).is_power_of_two() { + fatal!(diag, "--section-align={arg}: value must be a power of 2"); + } + a.section_align.insert(name.to_string(), value as u64); + } else if read_arg!("section-start") { + let Some((name, value)) = arg.split_once('=').filter(|(_, v)| !v.is_empty()) else { + fatal!(diag, "--section-start: syntax error: {arg}"); + }; + a.section_start + .insert(name.to_string(), parse_hex(diag, "section-start", value)); + } else if read_arg!("section-order") { + a.section_order = parse_section_order(diag, &arg); + } else if read_arg!("Tbss") { + a.section_start + .insert(".bss".to_string(), parse_hex(diag, "Tbss", &arg)); + } else if read_arg!("Tdata") { + a.section_start + .insert(".data".to_string(), parse_hex(diag, "Tdata", &arg)); + } else if read_arg!("Ttext") { + a.section_start + .insert(".text".to_string(), parse_hex(diag, "Ttext", &arg)); + } else if read_arg!("Ttext-segment") { + a.ttext_segment = Some(parse_number(diag, "Ttext-segment", &arg) as u64); + } else if read_flag!("repro") { + a.repro = true; + } else if read_z_flag!("now") { + a.z_now = true; + } else if read_z_flag!("lazy") { + a.z_now = false; + } else if read_z_flag!("cet-report=none") { + a.z_cet_report = CetReportKind::None; + } else if read_z_flag!("cet-report=warning") { + a.z_cet_report = CetReportKind::Warning; + } else if read_z_flag!("cet-report=error") { + a.z_cet_report = CetReportKind::Error; + } else if read_z_flag!("execstack") { + a.z_execstack = true; + } else if read_z_flag!("execstack-if-needed") { + a.z_execstack_if_needed = true; + } else if read_z_arg!("max-page-size") { + a.page_size = parse_number(diag, "-z max-page-size", &arg) as u64; + if !a.page_size.is_power_of_two() { + fatal!(diag, "-z max-page-size {arg}: value must be a power of 2"); + } + } else if read_z_flag!("start-stop-visibility=protected") { + a.z_start_stop_visibility_protected = true; + } else if read_z_flag!("start-stop-visibility=hidden") { + a.z_start_stop_visibility_protected = false; + } else if read_z_flag!("noexecstack") { + a.z_execstack = false; + } else if read_z_flag!("relro") { + z_relro = Some(true); + } else if read_z_flag!("norelro") { + z_relro = Some(false); + } else if read_z_flag!("defs") || read_flag!("no-undefined") { + report_undefined = Some(true); + } else if read_z_flag!("undefs") { + report_undefined = Some(false); + } else if read_z_flag!("nodlopen") { + a.z_dlopen = false; + } else if read_z_flag!("nodelete") { + a.z_delete = false; + } else if read_z_flag!("nocopyreloc") { + a.z_copyreloc = false; + } else if read_z_flag!("nodump") { + a.z_dump = false; + } else if read_z_flag!("initfirst") { + a.z_initfirst = true; + } else if read_z_flag!("interpose") { + a.z_interpose = true; + } else if read_z_flag!("ibt") { + a.z_ibt = true; + } else if read_z_flag!("ibtplt") { + } else if read_z_flag!("muldefs") { + a.allow_multiple_definition = true; + } else if read_z_flag!("keep-text-section-prefix") { + a.z_keep_text_section_prefix = true; + } else if read_z_flag!("nokeep-text-section-prefix") { + a.z_keep_text_section_prefix = false; + } else if read_z_flag!("shstk") { + a.z_shstk = true; + } else if read_z_flag!("text") { + a.z_text = true; + } else if read_z_flag!("notext") || read_z_flag!("textoff") { + a.z_text = false; + } else if read_z_flag!("origin") { + a.z_origin = true; + } else if read_z_flag!("nodefaultlib") { + a.z_nodefaultlib = true; + } else if read_eq!("separate-debug-file") { + separate_debug_file = Some(arg.clone()); + } else if read_flag!("separate-debug-file") { + separate_debug_file = Some(String::new()); + } else if read_flag!("no-separate-debug-file") { + separate_debug_file = None; + } else if read_z_flag!("separate-loadable-segments") { + z_separate_code = Some(SeparateCodeKind::SeparateLoadableSegments); + } else if read_z_flag!("separate-code") { + z_separate_code = Some(SeparateCodeKind::SeparateCode); + } else if read_z_flag!("noseparate-code") { + z_separate_code = Some(SeparateCodeKind::NoSeparateCode); + } else if read_z_arg!("stack-size") { + a.z_stack_size = parse_number(diag, "-z stack-size", &arg) as u64; + } else if read_z_flag!("dynamic-undefined-weak") { + z_dynamic_undefined_weak = Some(true); + } else if read_z_flag!("nodynamic-undefined-weak") { + z_dynamic_undefined_weak = Some(false); + } else if read_z_flag!("sectionheader") { + a.z_sectionheader = true; + } else if read_z_flag!("nosectionheader") { + a.z_sectionheader = false; + } else if read_z_flag!("rodynamic") { + a.z_rodynamic = true; + } else if read_z_flag!("x86-64-v2") { + a.z_x86_64_isa_level |= GNU_PROPERTY_X86_ISA_1_V2; + } else if read_z_flag!("x86-64-v3") { + a.z_x86_64_isa_level |= GNU_PROPERTY_X86_ISA_1_V3; + } else if read_z_flag!("x86-64-v4") { + a.z_x86_64_isa_level |= GNU_PROPERTY_X86_ISA_1_V4; + } else if read_z_flag!("rewrite-endbr") { + if !target.is_x86_64 { + fatal!(diag, "-z rewrite-endbr is supported only on x86-64"); + } + a.z_rewrite_endbr = true; + } else if read_z_flag!("norewrite-endbr") { + a.z_rewrite_endbr = false; + } else if read_flag!("nmagic") { + a.nmagic = true; + } else if read_flag!("no-nmagic") { + a.nmagic = false; + } else if read_flag!("fatal-warnings") { + a.fatal_warnings = true; + } else if read_flag!("no-fatal-warnings") { + a.fatal_warnings = false; + } else if read_flag!("w") || read_flag!("no-warnings") { + a.suppress_warnings = true; + } else if read_flag!("fork") { + a.fork = true; + } else if read_flag!("no-fork") { + a.fork = false; + } else if read_flag!("gc-sections") { + a.gc_sections = true; + } else if read_flag!("no-gc-sections") { + a.gc_sections = false; + } else if read_flag!("print-gc-sections") { + a.print_gc_sections = "-".to_string(); + } else if read_eq!("print-gc-sections") { + a.print_gc_sections = arg.clone(); + } else if read_flag!("no-print-gc-sections") { + a.print_gc_sections.clear(); + } else if read_arg!("discard-section") { + a.discard_section.insert(arg.clone()); + } else if read_arg!("no-discard-section") { + a.discard_section.remove(&arg); + } else if read_arg!("icf") { + match arg.as_str() { + "all" => { + a.icf = true; + a.icf_all = true; + } + "safe" => a.icf = true, + "none" => a.icf = false, + _ => fatal!(diag, "unknown --icf argument: {arg}"), + } + } else if read_flag!("no-icf") { + a.icf = false; + } else if read_flag!("ignore-data-address-equality") { + a.ignore_data_address_equality = true; + } else if read_arg!("image-base") { + a.image_base = parse_number(diag, "image-base", &arg) as u64; + } else if read_arg!("physical-image-base") { + a.physical_image_base = Some(parse_number(diag, "physical-image-base", &arg) as u64); + } else if read_flag!("print-icf-sections") { + a.print_icf_sections = "-".to_string(); + } else if read_eq!("print-icf-sections") { + a.print_icf_sections = arg.clone(); + } else if read_flag!("no-print-icf-sections") { + a.print_icf_sections.clear(); + } else if read_flag!("quick-exit") { + a.quick_exit = true; + } else if read_flag!("no-quick-exit") { + a.quick_exit = false; + } else if read_arg!("plugin") { + a.plugin = arg.clone(); + } else if read_arg!("plugin-opt") { + a.plugin_opt.push(arg.clone()); + } else if read_flag!("lto-cs-profile-generate") { + a.plugin_opt.push("cs-profile-generate".to_string()); + } else if read_arg!("lto-cs-profile-file") { + a.plugin_opt.push(format!("cs-profile-path={arg}")); + } else if read_flag!("lto-debug-pass-manager") { + a.plugin_opt.push("debug-pass-manager".to_string()); + } else if read_flag!("disable-verify") { + a.plugin_opt.push("disable-verify".to_string()); + } else if read_flag!("lto-emit-asm") { + a.plugin_opt.push("emit-asm".to_string()); + } else if read_flag!("no-legacy-pass-manager") { + a.plugin_opt.push("legacy-pass-manager".to_string()); + } else if read_arg!("lto-partitions") { + a.plugin_opt.push(format!("lto-partitions={arg}")); + } else if read_flag!("no-lto-legacy-pass-manager") { + a.plugin_opt.push("new-pass-manager".to_string()); + } else if read_arg!("lto-obj-path") { + a.plugin_opt.push(format!("obj-path={arg}")); + } else if read_arg!("opt-remarks-filename") { + a.plugin_opt.push(format!("opt-remarks-filename={arg}")); + } else if read_arg!("opt-remarks-format") { + a.plugin_opt.push(format!("opt-remarks-format={arg}")); + } else if read_arg!("opt-remarks-hotness-threshold") { + a.plugin_opt + .push(format!("opt-remarks-hotness-threshold={arg}")); + } else if read_arg!("opt-remarks-passes") { + a.plugin_opt.push(format!("opt-remarks-passes={arg}")); + } else if read_flag!("opt-remarks-with-hotness") { + a.plugin_opt.push("opt-remarks-with-hotness".to_string()); + } else if let Some(level) = cmdline[i].strip_prefix("-lto-O") { + a.plugin_opt.push(format!("O{level}")); + i += 1; + } else if let Some(level) = cmdline[i].strip_prefix("--lto-O") { + a.plugin_opt.push(format!("O{level}")); + i += 1; + } else if read_arg!("lto-pseudo-probe-for-profiling") { + a.plugin_opt + .push(format!("pseudo-probe-for-profiling={arg}")); + } else if read_arg!("lto-sample-profile") { + a.plugin_opt.push(format!("sample-profile={arg}")); + } else if read_flag!("save-temps") { + a.plugin_opt.push("save-temps".to_string()); + } else if read_flag!("thinlto-emit-imports-files") { + a.plugin_opt.push("thinlto-emit-imports-files".to_string()); + } else if read_arg!("thinlto-index-only") { + a.plugin_opt.push(format!("thinlto-index-only={arg}")); + } else if read_flag!("thinlto-index-only") { + a.plugin_opt.push("thinlto-index-only".to_string()); + } else if read_arg!("thinlto-object-suffix-replace") { + a.plugin_opt + .push(format!("thinlto-object-suffix-replace={arg}")); + } else if read_arg!("thinlto-prefix-replace") { + a.plugin_opt.push(format!("thinlto-prefix-replace={arg}")); + } else if read_arg!("thinlto-cache-dir") { + a.plugin_opt.push(format!("cache-dir={arg}")); + } else if read_arg!("thinlto-cache-policy") { + a.plugin_opt.push(format!("cache-policy={arg}")); + } else if read_arg!("thinlto-jobs") { + a.plugin_opt.push(format!("jobs={arg}")); + } else if read_arg!("thread-count") { + a.thread_count = Some(parse_number(diag, "thread-count", &arg).max(1) as usize); + } else if read_flag!("threads") { + a.thread_count = None; + } else if read_flag!("no-threads") { + a.thread_count = Some(1); + } else if read_eq!("threads") { + a.thread_count = Some(parse_number(diag, "threads", &arg).max(1) as usize); + } else if read_flag!("discard-all") || read_flag!("x") { + a.discard_all = true; + } else if read_flag!("discard-locals") || read_flag!("X") { + a.discard_locals = true; + } else if read_flag!("discard-none") { + a.discard_all = false; + a.discard_locals = false; + } else if read_flag!("strip-all") || read_flag!("s") { + a.strip_all = true; + } else if read_flag!("strip-debug") || read_flag!("S") { + a.strip_debug = true; + } else if read_flag!("warn-unresolved-symbols") { + error_unresolved_symbols = false; + } else if read_flag!("error-unresolved-symbols") { + error_unresolved_symbols = true; + } else if read_arg!("rpath") { + add_rpath(&mut a, &mut rpaths, &arg); + } else if read_arg!("R") { + if crate::mapped_file::is_file(&arg) { + fatal!( + diag, + "-R{arg}: -R as an alias for --just-symbols is not supported" + ); + } + add_rpath(&mut a, &mut rpaths, &arg); + } else if read_flag!("undefined-version") { + a.undefined_version = true; + } else if read_flag!("no-undefined-version") { + a.undefined_version = false; + } else if read_flag!("build-id") { + a.build_id.kind = BuildIdKind::Hash; + a.build_id.hash_size = 20; + } else if read_arg!("build-id") { + match arg.as_str() { + "none" => a.build_id.kind = BuildIdKind::None, + "uuid" => a.build_id.kind = BuildIdKind::Uuid, + "md5" => { + a.build_id.kind = BuildIdKind::Hash; + a.build_id.hash_size = 16; + } + "sha1" => { + a.build_id.kind = BuildIdKind::Hash; + a.build_id.hash_size = 20; + } + "sha256" | "fast" => { + a.build_id.kind = BuildIdKind::Hash; + a.build_id.hash_size = 32; + } + s if s.starts_with("0x") || s.starts_with("0X") => { + a.build_id.kind = BuildIdKind::Hex; + a.build_id.value = parse_hex_build_id(diag, s); + } + _ => fatal!(diag, "invalid --build-id argument: {arg}"), + } + } else if read_flag!("no-build-id") { + a.build_id.kind = BuildIdKind::None; + } else if read_flag!("be8") { + a.be8 = true; + } else if read_flag!("be32") { + a.be8 = false; + } else if read_arg!("format") || read_arg!("b") { + if arg == "binary" { + fatal!( + diag, + "mold does not support `-b binary`. If you want to convert a binary file into an \ + object file, use `objcopy -I binary -O default ` instead." + ); + } + fatal!(diag, "unknown command line option: -b {arg}"); + } else if read_arg!("fuse-ld") { + } else if read_arg!("auxiliary") || read_arg!("f") { + a.auxiliary.push(arg.clone()); + } else if read_arg!("filter") || read_arg!("F") { + a.filter.push(arg.clone()); + } else if read_flag!("allow-shlib-undefined") { + allow_shlib_undefined = Some(true); + } else if read_flag!("no-allow-shlib-undefined") { + allow_shlib_undefined = Some(false); + } else if read_arg!("O") + || read_flag!("EB") + || read_flag!("EL") + || read_flag!("O0") + || read_flag!("O1") + || read_flag!("O2") + || read_flag!("verbose") + || read_flag!("color-diagnostics") + || read_flag!("eh-frame-hdr") + || read_flag!("start-group") + || read_flag!("end-group") + || read_flag!("(") + || read_flag!(")") + || read_flag!("fatal-warnings") + || read_flag!("enable-new-dtags") + || read_flag!("disable-new-dtags") + || read_flag!("nostdlib") + || read_flag!("no-add-needed") + || read_flag!("no-call-graph-profile-sort") + || read_flag!("no-copy-dt-needed-entries") + || read_arg!("sort-section") + || read_flag!("sort-common") + || read_flag!("dc") + || read_flag!("dp") + || read_flag!("fix-cortex-a53-835769") + || read_flag!("fix-cortex-a53-843419") + || read_flag!("warn-once") + || read_flag!("nodefaultlibs") + || read_flag!("warn-constructors") + || read_flag!("warn-execstack") + || read_flag!("no-warn-execstack") + || read_flag!("long-plt") + || read_flag!("secure-plt") + || read_arg!("rpath-link") + || read_z_flag!("combreloc") + || read_z_flag!("nocombreloc") + || read_z_arg!("common-page-size") + || read_flag!("no-keep-memory") + || read_arg!("max-cache-size") + || read_flag!("mmap-output-file") + || read_flag!("no-mmap-output-file") + { + // Ignored for compatibility. + } else if read_arg!("version-script") { + a.version_scripts.push(arg.clone()); + } else if read_arg!("dynamic-list") { + a.bsymbolic = BsymbolicKind::All; + a.dynamic_list.push(DynamicListSource::File(arg.clone())); + } else if read_arg!("dynamic-list-data") { + a.dynamic_list_data = true; + } else if read_arg!("export-dynamic-symbol") { + a.dynamic_list.push(DynamicListSource::Pattern(arg.clone())); + } else if read_arg!("export-dynamic-symbol-list") { + a.dynamic_list.push(DynamicListSource::File(arg.clone())); + } else if read_flag!("as-needed") { + rctx.as_needed = true; + } else if read_flag!("no-as-needed") { + rctx.as_needed = false; + } else if read_flag!("whole-archive") { + rctx.whole_archive = true; + } else if read_flag!("no-whole-archive") { + rctx.whole_archive = false; + } else if read_arg!("l") || read_arg!("library") { + if visited_libs.insert(arg.clone()) { + let mut job = ReaderJob { + rctx: rctx.clone(), + name: arg.clone(), + is_lib: true, + ..Default::default() + }; + job.rctx.pos = vec![jobs.len() as u32]; + jobs.push(job); + } + } else if read_arg!("script") || read_arg!("T") { + let mut job = ReaderJob { + rctx: rctx.clone(), + name: arg.clone(), + ..Default::default() + }; + job.rctx.pos = vec![jobs.len() as u32]; + jobs.push(job); + } else if read_flag!("push-state") { + rctx_stack.push(rctx.clone()); + } else if read_flag!("pop-state") { + rctx = rctx_stack + .pop() + .unwrap_or_else(|| fatal!(diag, "no state pushed before popping")); + } else if cmdline[i].starts_with("-z") && cmdline[i].len() > 2 { + warn!(diag, "unknown command line option: {}", cmdline[i]); + i += 1; + } else if cmdline[i] == "-z" && i + 1 < cmdline.len() { + warn!(diag, "unknown command line option: -z {}", cmdline[i + 1]); + i += 2; + } else if cmdline[i] == "-dynamic" { + fatal!( + diag, + "unknown command line option: -dynamic; -dynamic is a macOS linker's option. mold does not support macOS." + ); + } else { + fatal!(diag, "unknown command line option: {}", cmdline[i]); + } + } + + if !a.chroot.is_empty() { + if !a.map.is_empty() { + a.map = format!("{}/{}", a.chroot, a.map); + } + if !a.dependency_file.is_empty() { + a.dependency_file = format!("{}/{}", a.chroot, a.dependency_file); + } + } + + if !a.directory.is_empty() { + if let Err(e) = std::env::set_current_dir(&a.directory) { + fatal!(diag, "chdir failed: {}: {e}", a.directory); + } + } + + if !a.sysroot.is_empty() { + for path in &mut a.library_paths { + if let Some(rest) = path.strip_prefix('=') { + *path = format!("{}{rest}", a.sysroot); + } else if let Some(rest) = path.strip_prefix("$SYSROOT") { + *path = format!("{}{rest}", a.sysroot); + } + } + } + + // Clean library paths by removing redundant `/..` and `/.` so that + // they are easier to read in log messages. + for path in &mut a.library_paths { + *path = path_clean(path); + } + + if a.shared { + a.pic = true; + } + + if let Some(val) = a.ttext_segment { + if val % a.page_size != 0 { + warn!(diag, "-Ttext-segment is not a multiple of page size: {val}"); + } + a.image_base = align_down(val, a.page_size); + } else if a.pic { + a.image_base = 0; + } + + // A shared library is loaded by the dynamic linker, so it doesn't need + // PT_INTERP. (-pie makes the output an executable even with -shared.) + if a.shared && !a.pie { + a.dynamic_linker.clear(); + } + + a.allow_shlib_undefined = allow_shlib_undefined.unwrap_or(a.shared); + + let report_undefined = report_undefined.unwrap_or(!a.shared); + a.unresolved_symbols = if report_undefined { + if error_unresolved_symbols { + UnresolvedKind::Error + } else { + UnresolvedKind::Warn + } + } else { + UnresolvedKind::Ignore + }; + + if a.retain_symbols_file.is_some() { + a.strip_all = false; + a.discard_all = false; + } + + if a.shuffle_sections == ShuffleSectionsKind::Shuffle { + a.shuffle_sections_seed = shuffle_sections_seed.unwrap_or_else(|| { + let mut buf = [0u8; 8]; + util::random_bytes(&mut buf); + u64::from_ne_bytes(buf) + }); + } + + // --section-order implies `-z separate-loadable-segments` + a.z_separate_code = z_separate_code.unwrap_or(if a.section_order.is_empty() { + SeparateCodeKind::NoSeparateCode + } else { + SeparateCodeKind::SeparateLoadableSegments + }); + + // `-z dynamic-undefined-weak` is enabled by default for DSOs. + a.z_dynamic_undefined_weak = z_dynamic_undefined_weak.unwrap_or(a.shared); + + // --section-order implies `-z norelro` + a.z_relro = z_relro.unwrap_or(a.section_order.is_empty()); + if a.nmagic || a.omagic { + a.z_relro = false; + } + + if !a.shared { + if !a.filter.is_empty() { + fatal!(diag, "-filter may not be used without -shared"); + } + if !a.auxiliary.is_empty() { + fatal!(diag, "-auxiliary may not be used without -shared"); + } + } + + // Even though SH4 is RELA, addends in its relocation records are + // always zero and actual addends are written to relocated places. + if (!target.is_rela || target.is_sh4) && !a.apply_dynamic_relocs { + fatal!( + diag, + "--no-apply-dynamic-relocs may not be used on {}", + target.name + ); + } + if target.is_sparc && a.apply_dynamic_relocs { + fatal!(diag, "--apply-dynamic-relocs may not be used on SPARC64"); + } + + if !a.section_start.is_empty() && !a.section_order.is_empty() { + fatal!(diag, "--section-start may not be used with --section-order"); + } + if a.image_base % a.page_size != 0 { + fatal!(diag, "-image-base must be a multiple of -max-page-size"); + } + if a.emulation == "arm32be" && !a.be8 { + fatal!(diag, "--be32 is not supported"); + } + + if std::env::var("MOLD_REPRO").is_ok_and(|v| !v.is_empty()) { + a.repro = true; + } + + if a.default_symver { + let ver = if a.soname.is_empty() { + path_filename(&a.output) + } else { + a.soname.clone() + }; + a.version_definitions.push(ver); + } + + if let Some(file) = separate_debug_file { + a.separate_debug_file = if file.is_empty() { + format!("{}.dbg", a.output) + } else { + file + }; + } + + if a.shared && warn_shared_textrel { + a.warn_textrel = true; + } + + // We don't want the background process to write to stdout. + if a.stats || a.perf { + a.detach = false; + } + + a.undefined.push(a.entry.clone()); + for (_, value) in &a.defsyms { + if let DefsymValue::Symbol(sym) = value { + a.undefined.push(sym.clone()); + } + } + + // --oformat=binary implies --strip-all because without a section + // header, there's no way to identify the location of a symbol table. + if a.oformat_binary { + a.strip_all = true; + } + + // Overwriting an existing output file in place is faster than + // creating a fresh one, but a running executable is protected by + // ETXTBSY only on some kernels, and shared objects never are. + a.overwrite_output_file = !a.shared && returns_etxtbsy(); + + if version_shown && jobs.is_empty() { + std::process::exit(0); + } + + if rctx.is_static || a.relocatable { + a.is_static = true; + a.dynamic_linker.clear(); + } + + ParsedArgs { + args: a, + jobs, + final_rctx: rctx, + } +} + +fn add_rpath(a: &mut Args, seen: &mut HashSet, path: &str) { + if seen.insert(path.to_string()) { + if !a.rpaths.is_empty() { + a.rpaths.push(':'); + } + a.rpaths.push_str(path); + } +} + +fn stderr_is_tty() -> bool { + // SAFETY: isatty only inspects a file descriptor. + unsafe { libc::isatty(libc::STDERR_FILENO) != 0 } +} diff --git a/src/chunks/arm_exidx.rs b/src/chunks/arm_exidx.rs new file mode 100644 index 000000000..a82c1f464 --- /dev/null +++ b/src/chunks/arm_exidx.rs @@ -0,0 +1,160 @@ +//! `.ARM.exidx`, the exception handling index of ARM32 executables. +//! +//! Where other targets look up the unwind record for a PC in +//! `.eh_frame_hdr`, ARM32 uses `.ARM.exidx`: a table of records, each a +//! signed 31-bit self-relative address of a function's start and a value +//! that is either CANTUNWIND, a compact unwind record encoded into the +//! word (its most significant bit set), or a self-relative address of a +//! larger record in `.ARM.extab`. The table must be sorted by address. +//! +//! The input `.ARM.exidx` sections are gathered into an ordinary output +//! section for layout, but the output is written by this chunk, which +//! sorts the records, appends a sentinel and merges adjacent functions +//! with identical unwind information. + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::{output_section, ChunkHeader, ChunkId, OutputSectionId}; +use crate::context::Context; +use crate::elf::*; +use crate::util::sign_extend; + +const CANTUNWIND: u32 = 1; +const ENTRY_SIZE: usize = 8; + +#[derive(Debug)] +pub struct ArmExidxSection { + pub hdr: ChunkHeader, + /// The output section holding the input `.ARM.exidx` sections. + pub output_section: OutputSectionId, +} + +/// Replaces the `.ARM.exidx` output section by the synthetic one. +pub fn create(ctx: &mut Context) { + let Some(i) = ctx.chunks.iter().position(|&id| match id { + ChunkId::Output(osec) => { + ctx.output_sections[osec.index()].hdr.shdr.sh_type == SHT_ARM_EXIDX + } + _ => false, + }) else { + return; + }; + let ChunkId::Output(osec) = ctx.chunks[i] else { + unreachable!() + }; + + let mut hdr = ChunkHeader::new(".ARM.exidx", SHT_ARM_EXIDX, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + ctx.arm_exidx = Some(ArmExidxSection { + hdr, + output_section: osec, + }); + ctx.chunks[i] = ChunkId::ArmExidx; + + // The input sections are consumed here rather than copied. + for m in ctx.output_sections[osec.index()].members.clone() { + let section = ctx.input_section(m).section_ref(); + ctx.objs[section.file.index()].kill_section(section.shndx as usize); + } +} + +pub fn compute_section_size(ctx: &mut Context) { + let osec = ctx.arm_exidx.as_ref().unwrap().output_section; + output_section::compute_section_size(ctx, osec); + let size = ctx.output_sections[osec.index()].hdr.shdr.sh_size; + ctx.arm_exidx.as_mut().unwrap().hdr.shdr.sh_size = size + ENTRY_SIZE as u64; + // plus the sentinel +} + +/// `sh_link` refers to `.text`. The runtime doesn't care, but `strip` +/// does. +pub fn update_shdr(ctx: &mut Context) { + if let Some(text) = ctx.find_chunk_by_name(b".text") { + let shndx = ctx.chunk_header(text).shndx; + ctx.arm_exidx.as_mut().unwrap().hdr.shdr.sh_link = shndx; + } +} + +/// Once addresses are known, merges the records of adjacent functions +/// with the same unwind information, which shrinks the section. +pub fn remove_duplicate_entries(ctx: &mut Context) { + // The input sections are laid out at the synthetic section's address, + // which their PC-relative records depend on. + let sec = ctx.arm_exidx.as_ref().unwrap(); + let (osec, addr) = (sec.output_section, sec.hdr.shdr.sh_addr); + ctx.output_sections[osec.index()].hdr.shdr.sh_addr = addr; + let size = contents(ctx).len() as u64; + ctx.arm_exidx.as_mut().unwrap().hdr.shdr.sh_size = size; +} + +pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let contents = contents(ctx); + debug_assert_eq!(contents.len(), buf.len()); + buf.copy_from_slice(&contents); +} + +/// The end of the text segment, which the sentinel record covers up to. +fn text_end(ctx: &Context) -> u64 { + ctx.chunks + .iter() + .map(|&id| ctx.chunk_header(id).shdr) + .filter(|shdr| shdr.sh_flags & SHF_EXECINSTR as u64 != 0) + .map(|shdr| shdr.sh_addr + shdr.sh_size) + .max() + .unwrap_or(0) +} + +fn contents(ctx: &Context) -> Vec { + let sec = ctx.arm_exidx.as_ref().unwrap(); + let osec = &ctx.output_sections[sec.output_section.index()]; + let base = sec.hdr.shdr.sh_addr; + + // The records as the input sections define them, plus a sentinel. + let num_entries = osec.hdr.shdr.sh_size as usize / ENTRY_SIZE + 1; + let mut buf = vec![0u8; num_entries * ENTRY_SIZE]; + output_section::write_to(ctx, sec.output_section, &mut buf); + let sentinel_addr = base + ((num_entries - 1) * ENTRY_SIZE) as u64; + let mut entries: Vec<(u32, u32)> = buf + .chunks_exact(ENTRY_SIZE) + .map(|e| (E::Endian::read_u32(e), E::Endian::read_u32(&e[4..]))) + .collect(); + entries[num_entries - 1] = (text_end(ctx).wrapping_sub(sentinel_addr) as u32, CANTUNWIND); + + // Addresses are relative to the records themselves. Make them + // relative to the section to sort by them. + let is_relative = |val: u32| val != CANTUNWIND && val & 0x8000_0000 == 0; + entries + .par_iter_mut() + .enumerate() + .for_each(|(i, (addr, val))| { + let offset = (i * ENTRY_SIZE) as u32; + *addr = (sign_extend(*addr as u64, 31) as u32).wrapping_add(offset); + if is_relative(*val) { + *val = 0x7fff_ffff & val.wrapping_add(offset); + } + }); + entries.sort_by_key(|&(addr, _)| addr); + + // Adjacent functions with the same unwind information, or which both + // can't be unwound, share one record. + entries.dedup_by_key(|&mut (_, val)| val); + + entries + .par_iter_mut() + .enumerate() + .for_each(|(i, (addr, val))| { + let offset = (i * ENTRY_SIZE) as u32; + *addr = 0x7fff_ffff & addr.wrapping_sub(offset); + if is_relative(*val) { + *val = 0x7fff_ffff & val.wrapping_sub(offset); + } + }); + + let mut out = vec![0u8; entries.len() * ENTRY_SIZE]; + for (i, (addr, val)) in entries.iter().enumerate() { + E::Endian::write_u32(&mut out[i * ENTRY_SIZE..], *addr); + E::Endian::write_u32(&mut out[i * ENTRY_SIZE + 4..], *val); + } + out +} diff --git a/src/chunks/dynamic.rs b/src/chunks/dynamic.rs new file mode 100644 index 000000000..51ee31238 --- /dev/null +++ b/src/chunks/dynamic.rs @@ -0,0 +1,667 @@ +//! `.rela.dyn`, `.relr.dyn` and `.dynamic`. + +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::chunks::{self, ChunkHeader, ChunkId}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::symbol::SymbolId; +use crate::util::encode_sleb; + +/// `.rela.dyn` holds the dynamic relocations of all other sections. +#[derive(Debug)] +pub struct RelDynSection { + pub hdr: ChunkHeader, + pub android_encoded: Vec, + pub keep_android_size: bool, +} + +impl RelDynSection { + pub fn new(args: &crate::args::Args) -> RelDynSection { + let name = if E::IS_RELA { ".rela.dyn" } else { ".rel.dyn" }; + let mut hdr = ChunkHeader::new(name, 0, SHF_ALLOC as u64); + if args.pack_dyn_relocs_android { + hdr.shdr.sh_type = if E::IS_RELA { + SHT_ANDROID_RELA + } else { + SHT_ANDROID_REL + }; + hdr.shdr.sh_entsize = 0; + hdr.shdr.sh_addralign = 1; + } else { + hdr.shdr.sh_type = if E::IS_RELA { SHT_RELA } else { SHT_REL }; + hdr.shdr.sh_entsize = ElfRel::size::() as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + } + RelDynSection { + hdr, + android_encoded: Vec::new(), + keep_android_size: false, + } + } +} + +pub mod reldyn { + use super::*; + + /// Gathers the dynamic relocations of all chunks. + pub fn collect_relocs(ctx: &Context) -> Vec { + let count: usize = ctx + .chunks + .iter() + .map(|&id| { + let hdr = ctx.chunk_header(id); + (hdr.num_dynrels - hdr.num_relrs) as usize + }) + .sum(); + let mut out = vec![ElfRel::default(); count]; + let mut rest = out.as_mut_slice(); + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + let count = (hdr.num_dynrels - hdr.num_relrs) as usize; + if count != 0 { + let (slots, tail) = std::mem::take(&mut rest).split_at_mut(count); + chunks::write_dynrels(ctx, id, chunks::DynRelBuffer::native(slots)); + rest = tail; + } + } + debug_assert!(rest.is_empty()); + out + } + + /// Encodes base relocations of each chunk in RELR form, using offsets + /// relative to the chunk. + pub fn construct_relr(ctx: &mut Context) { + debug_assert!(ctx.args.pack_dyn_relocs_relr); + let word = E::WORD_SIZE as u64; + let ids = ctx.chunks.clone(); + + for &id in &ids { + let n = chunks::num_dynrels(ctx, id); + let hdr = ctx.chunk_header_mut(id); + hdr.num_dynrels = n; + hdr.num_relrs = 0; + hdr.relr.clear(); + + // Executable chunks don't usually contain base relocations. + if hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0 { + continue; + } + // --section-start can override a chunk's alignment; use + // .rel[a].dyn if the assigned address isn't word-aligned. + let name = String::from_utf8_lossy(hdr.name).into_owned(); + if ctx + .args + .section_start + .get(&name) + .is_some_and(|&addr| addr % word != 0) + { + continue; + } + if n != 0 { + let offsets = chunks::relr_offsets(ctx, id); + let hdr = ctx.chunk_header_mut(id); + hdr.num_relrs = offsets.len() as u64; + debug_assert!(hdr.num_relrs <= hdr.num_dynrels); + hdr.relr = encode_relr::(&offsets); + } + } + + let size: u64 = ids + .iter() + .map(|&id| ctx.chunk_header(id).relr.len() as u64 * word) + .sum(); + if let Some(relrdyn) = &mut ctx.relrdyn { + relrdyn.hdr.shdr.sh_size = size; + } + } + + pub fn update_shdr(ctx: &mut Context) { + let ids = ctx.chunks.clone(); + if !ctx.args.pack_dyn_relocs_relr { + for &id in &ids { + let n = chunks::num_dynrels(ctx, id); + let hdr = ctx.chunk_header_mut(id); + hdr.num_dynrels = n; + hdr.num_relrs = 0; + } + } + + let mut num_relocs = 0; + let mut num_relrs = 0; + for &id in &ids { + let hdr = ctx.chunk_header(id); + num_relocs += hdr.num_dynrels; + num_relrs += hdr.num_relrs; + } + + if ctx.args.pack_dyn_relocs_android { + let relocs = collect_relocs(ctx); + // APS2 uses SLEB128-encoded deltas, so the size may oscillate + // as addresses move. If a shrink is followed by a growth, stop + // shrinking and pad the stream to converge. + let encoded = encode_android::(relocs); + let old_size = ctx.reldyn.hdr.shdr.sh_size as usize; + let reldyn = &mut ctx.reldyn; + if old_size != 0 && old_size < encoded.len() { + reldyn.keep_android_size = true; + } + reldyn.android_encoded = encoded; + if reldyn.keep_android_size && reldyn.android_encoded.len() < old_size { + reldyn.android_encoded.resize(old_size, 0); + } + reldyn.hdr.shdr.sh_size = reldyn.android_encoded.len() as u64; + } else { + ctx.reldyn.hdr.shdr.sh_size = (num_relocs - num_relrs) * ElfRel::size::() as u64; + } + ctx.reldyn.hdr.shdr.sh_link = ctx.dynsym.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + if ctx.args.pack_dyn_relocs_android { + buf[..ctx.reldyn.android_encoded.len()].copy_from_slice(&ctx.reldyn.android_encoded); + } else { + let size = ElfRel::size::(); + let mut rest = buf; + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + let count = (hdr.num_dynrels - hdr.num_relrs) as usize; + if count != 0 { + let (slots, tail) = std::mem::take(&mut rest).split_at_mut(count * size); + chunks::write_dynrels(ctx, id, chunks::DynRelBuffer::output(slots)); + rest = tail; + } + } + debug_assert!(rest.is_empty()); + } + } + + /// Sorts the dynamic relocations in the output so that the loader's + /// one-entry symbol cache is effective and IFUNC resolvers run last. + pub fn sort(ctx: &Context, buf: &mut [u8]) { + if ctx.args.pack_dyn_relocs_android { + return; + } + let rank = |r_type: u32| -> u32 { + if r_type == E::R_RELATIVE { + 0 + } else if Some(r_type) == E::R_IRELATIVE { + 2 + } else { + 1 + } + }; + match chunks::DynRelBuffer::::output(buf) { + chunks::DynRelBuffer::Native(relocs) => { + relocs.par_sort_by_key(|r| (rank(r.r_type), r.r_sym, r.r_offset)); + } + chunks::DynRelBuffer::Encoded(buf, _) => { + let mut relocs = ElfRel::parse_all::(buf); + relocs.par_sort_by_key(|r| (rank(r.r_type), r.r_sym, r.r_offset)); + ElfRel::write_all::(&relocs, buf); + } + } + } +} + +/// `.relr.dyn` stores base relocations compactly: a start address followed +/// by bitmaps, each bit covering one word after the address. +#[derive(Debug)] +pub struct RelrDynSection { + pub hdr: ChunkHeader, +} + +impl RelrDynSection { + pub fn new(args: &crate::args::Args) -> RelrDynSection { + let ty = if args.use_android_relr_tags { + SHT_ANDROID_RELR + } else { + SHT_RELR + }; + let mut hdr = ChunkHeader::new(".relr.dyn", ty, SHF_ALLOC as u64); + hdr.shdr.sh_entsize = E::WORD_SIZE as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + RelrDynSection { hdr } + } +} + +pub mod relrdyn { + use super::*; + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let w = E::WORD_SIZE; + let mut i = 0; + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + for &val in &hdr.relr { + let v = if val & 1 != 0 { + val + } else { + hdr.shdr.sh_addr + val + }; + if E::IS_64 { + E::Endian::write_u64(&mut buf[i * w..], v); + } else { + E::Endian::write_u32(&mut buf[i * w..], v as u32); + } + i += 1; + } + } + } +} + +/// Encodes sorted offsets in RELR form. Each address group is a start +/// address followed by bitmaps whose bit N means "also fix address + N * +/// word size". Addresses have LSB 0 and bitmaps LSB 1. +pub fn encode_relr(offsets: &[u64]) -> Vec { + let word = E::WORD_SIZE as u64; + let num_bits = if E::IS_64 { 63 } else { 31 }; + let max_delta = word * num_bits; + let mut vec = Vec::new(); + let mut i = 0; + + while i < offsets.len() { + let first = offsets[i]; + vec.push(first); + let mut base = first + word; + i += 1; + loop { + let mut bits = 0u64; + while i < offsets.len() && offsets[i] - base < max_delta { + bits |= 1 << ((offsets[i] - base) / word); + i += 1; + } + if bits == 0 { + break; + } + vec.push((bits << 1) | 1); + base += max_delta; + } + } + vec +} + +/// Encodes relocations in the Android packed format (APS2). Relocations +/// are grouped so that shared offset deltas, info values and addend deltas +/// are factored out. See bionic's linker_relocs.cpp for the decoder. +pub fn encode_android(mut rels: Vec) -> Vec { + const GROUPED_BY_INFO: i64 = 1; + const GROUPED_BY_OFFSET_DELTA: i64 = 2; + const GROUP_HAS_ADDEND: i64 = 8; + + let r_info = |r: &ElfRel| -> i64 { + if E::IS_64 { + (((r.r_sym as u64) << 32) | r.r_type as u64) as i64 + } else { + (((r.r_sym as u64) << 8) | (r.r_type & 0xff) as u64) as i64 + } + }; + + let mut buf = b"APS2".to_vec(); + encode_sleb(&mut buf, rels.len() as i64); + encode_sleb(&mut buf, 0); // initial offset state + if rels.is_empty() { + return buf; + } + + // Offset deltas are signed, so sort by (type, symbol, offset) to + // gather the dominant type (usually R_RELATIVE) in one run. + rels.sort_by_key(|r| (r.r_type, r.r_sym, r.r_offset)); + + let mut prev_offset = 0i64; + let mut prev_addend = 0i64; + let mut i = 0; + while i < rels.len() { + let offset_delta = rels[i].r_offset as i64 - prev_offset; + let cur_info = r_info(&rels[i]); + + let mut j = i + 1; + while j < rels.len() + && r_info(&rels[j]) == cur_info + && rels[j].r_offset as i64 - rels[j - 1].r_offset as i64 == offset_delta + { + j += 1; + } + + let size = (j - i) as i64; + let mut flags = 0; + if size > 1 { + flags = GROUPED_BY_INFO | GROUPED_BY_OFFSET_DELTA; + } + if E::IS_RELA { + flags |= GROUP_HAS_ADDEND; + } + encode_sleb(&mut buf, size); + encode_sleb(&mut buf, flags); + encode_sleb(&mut buf, offset_delta); + encode_sleb(&mut buf, cur_info); + + if E::IS_RELA { + for r in &rels[i..j] { + encode_sleb(&mut buf, r.r_addend - prev_addend); + prev_addend = r.r_addend; + } + } + prev_offset = rels[j - 1].r_offset as i64; + i = j; + } + buf +} + +/// `.dynamic` tells the dynamic linker where everything is. +#[derive(Debug)] +pub struct DynamicSection { + pub hdr: ChunkHeader, +} + +impl DynamicSection { + pub fn new(args: &crate::args::Args) -> DynamicSection { + let mut hdr = ChunkHeader::new(".dynamic", SHT_DYNAMIC, 0); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + hdr.shdr.sh_entsize = ElfDyn::size::() as u64; + if args.z_rodynamic { + hdr.shdr.sh_flags = SHF_ALLOC as u64; + hdr.is_relro = false; + } else { + hdr.shdr.sh_flags = (SHF_ALLOC | SHF_WRITE) as u64; + hdr.is_relro = true; + } + DynamicSection { hdr } + } +} + +// Named after the chunk, like the other chunks' modules in this directory. +#[allow(clippy::module_inception)] +pub mod dynamic { + use super::*; + + /// An AArch64 function with a non-standard calling convention is marked + /// with STO_AARCH64_VARIANT_PCS. It isn't safe to call through a lazy + /// PLT stub, so the loader resolves such symbols eagerly when + /// DT_AARCH64_VARIANT_PCS is set. RISC-V has the same feature under a + /// different name. + fn contains_variant_pcs(ctx: &Context) -> bool { + ctx.plt + .symbols + .iter() + .any(|&id| ctx.symbols[id].esym(ctx).arm64_variant_pcs()) + } + + fn sym_addr_if_defined(ctx: &Context, id: SymbolId) -> Option { + let sym = &ctx.symbols[id]; + match sym.file() { + Some(FileId::Obj(_)) => Some(sym.addr(ctx)), + _ => None, + } + } + + fn create_contents(ctx: &Context) -> Vec<(u64, u64)> { + let mut vec: Vec<(u64, u64)> = Vec::new(); + let mut define = |tag: u32, val: u64| vec.push((tag as u64, val)); + let dynstr = &ctx.dynstr; + + for dso in &ctx.dsos { + define(DT_NEEDED, dynstr.find_string(dso.soname.as_bytes())); + } + if !ctx.args.rpaths.is_empty() { + let tag = if ctx.args.enable_new_dtags { + DT_RUNPATH + } else { + DT_RPATH + }; + define(tag, dynstr.find_string(ctx.args.rpaths.as_bytes())); + } + if !ctx.args.soname.is_empty() { + define(DT_SONAME, dynstr.find_string(ctx.args.soname.as_bytes())); + } + for s in &ctx.args.auxiliary { + define(DT_AUXILIARY, dynstr.find_string(s.as_bytes())); + } + if !ctx.args.audit.is_empty() { + define(DT_AUDIT, dynstr.find_string(ctx.args.audit.as_bytes())); + } + if !ctx.args.depaudit.is_empty() { + define( + DT_DEPAUDIT, + dynstr.find_string(ctx.args.depaudit.as_bytes()), + ); + } + for s in &ctx.args.filter { + define(DT_FILTER, dynstr.find_string(s.as_bytes())); + } + + if ctx.reldyn.hdr.shdr.sh_size != 0 { + if ctx.args.pack_dyn_relocs_android { + define( + if E::IS_RELA { + DT_ANDROID_RELA + } else { + DT_ANDROID_REL + }, + ctx.reldyn.hdr.shdr.sh_addr, + ); + define( + if E::IS_RELA { + DT_ANDROID_RELASZ + } else { + DT_ANDROID_RELSZ + }, + ctx.reldyn.hdr.shdr.sh_size, + ); + } else { + define( + if E::IS_RELA { DT_RELA } else { DT_REL }, + ctx.reldyn.hdr.shdr.sh_addr, + ); + define( + if E::IS_RELA { DT_RELASZ } else { DT_RELSZ }, + ctx.reldyn.hdr.shdr.sh_size, + ); + define( + if E::IS_RELA { DT_RELAENT } else { DT_RELENT }, + ElfRel::size::() as u64, + ); + } + } + + if let Some(relrdyn) = &ctx.relrdyn { + if ctx.args.use_android_relr_tags { + define(DT_ANDROID_RELR, relrdyn.hdr.shdr.sh_addr); + define(DT_ANDROID_RELRSZ, relrdyn.hdr.shdr.sh_size); + define(DT_ANDROID_RELRENT, relrdyn.hdr.shdr.sh_entsize); + } else { + define(DT_RELR, relrdyn.hdr.shdr.sh_addr); + define(DT_RELRSZ, relrdyn.hdr.shdr.sh_size); + define(DT_RELRENT, relrdyn.hdr.shdr.sh_entsize); + } + } + + if ctx.relplt.hdr.shdr.sh_size != 0 { + define(DT_JMPREL, ctx.relplt.hdr.shdr.sh_addr); + define(DT_PLTRELSZ, ctx.relplt.hdr.shdr.sh_size); + define(DT_PLTREL, if E::IS_RELA { DT_RELA } else { DT_REL } as u64); + } + + if E::IS_SPARC { + if ctx.plt.hdr.shdr.sh_size != 0 { + define(DT_PLTGOT, ctx.plt.hdr.shdr.sh_addr); + } + } else if E::FAMILY == Family::Ppc32 { + if ctx.gotplt.hdr.shdr.sh_size != 0 { + define( + DT_PLTGOT, + ctx.gotplt.hdr.shdr.sh_addr + crate::chunks::got::gotplt::header_size::(), + ); + } + } else if ctx.gotplt.hdr.shdr.sh_size != 0 { + define(DT_PLTGOT, ctx.gotplt.hdr.shdr.sh_addr); + } + + if ctx.dynsym.hdr.shdr.sh_size != 0 { + define(DT_SYMTAB, ctx.dynsym.hdr.shdr.sh_addr); + define(DT_SYMENT, ElfSym::size::() as u64); + } + if ctx.dynstr.hdr.shdr.sh_size != 0 { + define(DT_STRTAB, ctx.dynstr.hdr.shdr.sh_addr); + define(DT_STRSZ, ctx.dynstr.hdr.shdr.sh_size); + } + + let value = |id: Option| id.map_or(0, |id| ctx.symbols[id].value); + if ctx.find_chunk_by_type(SHT_INIT_ARRAY).is_some() { + let start = value(ctx.syms.init_array_start); + define(DT_INIT_ARRAY, start); + define(DT_INIT_ARRAYSZ, value(ctx.syms.init_array_end) - start); + } + if ctx.find_chunk_by_type(SHT_PREINIT_ARRAY).is_some() { + let start = value(ctx.syms.preinit_array_start); + define(DT_PREINIT_ARRAY, start); + define( + DT_PREINIT_ARRAYSZ, + value(ctx.syms.preinit_array_end) - start, + ); + } + if ctx.find_chunk_by_type(SHT_FINI_ARRAY).is_some() { + let start = value(ctx.syms.fini_array_start); + define(DT_FINI_ARRAY, start); + define(DT_FINI_ARRAYSZ, value(ctx.syms.fini_array_end) - start); + } + + if ctx.versym.hdr.shdr.sh_size != 0 { + define(DT_VERSYM, ctx.versym.hdr.shdr.sh_addr); + } + if ctx.verneed.hdr.shdr.sh_size != 0 { + define(DT_VERNEED, ctx.verneed.hdr.shdr.sh_addr); + define(DT_VERNEEDNUM, ctx.verneed.hdr.shdr.sh_info as u64); + } + if let Some(verdef) = &ctx.verdef { + define(DT_VERDEF, verdef.hdr.shdr.sh_addr); + define(DT_VERDEFNUM, verdef.hdr.shdr.sh_info as u64); + } + + if let Some(addr) = sym_addr_if_defined(ctx, ctx.syms.init) { + define(DT_INIT, addr); + } + if let Some(addr) = sym_addr_if_defined(ctx, ctx.syms.fini) { + define(DT_FINI, addr); + } + + if let Some(hash) = &ctx.hash { + define(DT_HASH, hash.hdr.shdr.sh_addr); + } + if let Some(gnu_hash) = &ctx.gnu_hash { + define(DT_GNU_HASH, gnu_hash.hdr.shdr.sh_addr); + } + let has_textrel = ctx.has_textrel.load(std::sync::atomic::Ordering::Relaxed); + if has_textrel { + define(DT_TEXTREL, 0); + } + + let mut flags = 0u64; + let mut flags1 = 0u64; + if ctx.args.pie { + flags1 |= DF_1_PIE as u64; + } + if ctx.args.z_now { + flags |= DF_BIND_NOW as u64; + flags1 |= DF_1_NOW as u64; + } + if ctx.args.z_origin { + flags |= DF_ORIGIN as u64; + flags1 |= DF_1_ORIGIN as u64; + } + if !ctx.args.z_dlopen { + flags1 |= DF_1_NOOPEN as u64; + } + if ctx.args.z_nodefaultlib { + flags1 |= DF_1_NODEFLIB as u64; + } + if !ctx.args.z_delete { + flags1 |= DF_1_NODELETE as u64; + } + if !ctx.args.z_dump { + flags1 |= DF_1_NODUMP as u64; + } + if ctx.args.z_initfirst { + flags1 |= DF_1_INITFIRST as u64; + } + if ctx.args.z_interpose { + flags1 |= DF_1_INTERPOSE as u64; + } + if !ctx.got.gottp_syms.is_empty() { + flags |= DF_STATIC_TLS as u64; + } + if has_textrel { + flags |= DF_TEXTREL as u64; + } + if flags != 0 { + define(DT_FLAGS, flags); + } + if flags1 != 0 { + define(DT_FLAGS_1, flags1); + } + + if E::FAMILY == Family::Arm64 && contains_variant_pcs(ctx) { + define(DT_AARCH64_VARIANT_PCS, 0); + } + if E::IS_RISCV + && ctx + .plt + .symbols + .iter() + .any(|&id| ctx.symbols[id].esym(ctx).riscv_variant_cc()) + { + define(DT_RISCV_VARIANT_CC, 0); + } + if E::FAMILY == Family::Ppc32 { + define(DT_PPC_GOT, ctx.gotplt.hdr.shdr.sh_addr); + } + if E::IS_PPC64 { + // The psABI defines PPC64_GLINK as 32 bytes before the first + // PLT entry. + define( + DT_PPC64_GLINK, + ctx.plt.hdr.shdr.sh_addr + crate::chunks::got::plt::entry_offset::(0) - 32, + ); + } + + // GDB needs a DT_DEBUG entry in an executable for its own use. + if !ctx.args.shared && !ctx.args.z_rodynamic { + define(DT_DEBUG, 0); + } + + define(DT_NULL, 0); + for _ in 0..ctx.args.spare_dynamic_tags.max(0) { + define(DT_NULL, 0); + } + vec + } + + pub fn update_shdr(ctx: &mut Context) { + if ctx.args.is_static && !ctx.args.pie { + return; + } + let n = create_contents(ctx).len(); + let dynamic = ctx.dynamic.as_mut().unwrap(); + dynamic.hdr.shdr.sh_size = (n * ElfDyn::size::()) as u64; + dynamic.hdr.shdr.sh_link = ctx.dynstr.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let entries: Vec = create_contents(ctx) + .into_iter() + .map(|(d_tag, d_val)| ElfDyn { d_tag, d_val }) + .collect(); + debug_assert_eq!( + ctx.dynamic.as_ref().unwrap().hdr.shdr.sh_size as usize, + entries.len() * ElfDyn::size::() + ); + ElfDyn::write_all::(&entries, buf); + } + + /// A chunk id for the dynamic section, if it exists. + pub fn id(ctx: &Context) -> Option { + ctx.dynamic.as_ref().map(|_| ChunkId::Dynamic) + } +} diff --git a/src/chunks/eh_frame.rs b/src/chunks/eh_frame.rs new file mode 100644 index 000000000..4d9b02b95 --- /dev/null +++ b/src/chunks/eh_frame.rs @@ -0,0 +1,567 @@ +//! `.eh_frame`, `.eh_frame_hdr` and `.rela.eh_frame`. +//! +//! `.eh_frame` is reconstructed rather than copied: FDEs of dead functions +//! are dropped, identical CIEs are merged, and a sorted lookup table is +//! emitted as `.eh_frame_hdr` so that the unwinder can find the FDE for a +//! PC by binary search. + +use rayon::prelude::*; +use std::ptr::NonNull; + +use crate::arch::Arch; +use crate::chunks::ChunkHeader; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::ObjectFile; +use crate::input_sections::CieRecord; +use crate::output_file::split_at_offsets; +use crate::symbol::Symbol; +use crate::util::is_int; +use crate::{error, fatal}; + +/// A stable handle used while deduplicating CIEs. C++ mold keeps direct +/// `CieRecord *` leaders; the owner pointer is needed here because Rust keeps +/// the file-dependent relocation and symbol tables outside the record. +#[derive(Clone, Copy)] +pub(crate) struct CieHandle { + file: NonNull, + cie: NonNull, +} + +impl CieHandle { + /// Creates a handle while object files and their CIE vectors are stable. + /// + /// # Safety + /// + /// Both pointers must remain valid until the handle is discarded, and the + /// CIE vector must not be reallocated. Mutations through a handle must not + /// overlap a call to [`Self::equals`] involving that handle. + #[inline] + pub(crate) unsafe fn new(file: *mut ObjectFile, cie: *mut CieRecord) -> CieHandle { + CieHandle { + file: unsafe { NonNull::new_unchecked(file) }, + cie: unsafe { NonNull::new_unchecked(cie) }, + } + } + + #[inline] + pub(crate) fn equals(self, other: CieHandle) -> bool { + // SAFETY: construction guarantees that both records and owners remain + // live, and callers compare records only while they are not mutating. + unsafe { + cie_equals::( + self.file.as_ref(), + self.cie.as_ref(), + other.file.as_ref(), + other.cie.as_ref(), + ) + } + } + + #[inline] + pub(crate) fn size(self) -> usize { + // SAFETY: the handle's owner and record remain live. + unsafe { self.cie.as_ref().size::() } + } + + #[inline] + pub(crate) fn icf_idx(self, value: u32) { + // SAFETY: callers mutate only the current record, which is not yet in + // the leader list and is not being compared concurrently. + unsafe { (*self.cie.as_ptr()).icf_idx = value }; + } + + #[inline] + fn output_offset(self) -> u32 { + // SAFETY: the handle's record remains live. + unsafe { self.cie.as_ref().output_offset } + } + + #[inline] + fn set_output_offset(self, value: u32) { + // SAFETY: layout construction is serial and owns every CIE record. + unsafe { (*self.cie.as_ptr()).output_offset = value }; + } + + #[inline] + fn set_leader(self) { + // SAFETY: layout construction is serial and owns every CIE record. + unsafe { (*self.cie.as_ptr()).is_leader = true }; + } +} + +#[derive(Debug)] +pub struct EhFrameSection { + pub hdr: ChunkHeader, +} + +impl EhFrameSection { + pub fn new() -> EhFrameSection { + let mut hdr = ChunkHeader::new(".eh_frame", SHT_PROGBITS, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + EhFrameSection { hdr } + } +} + +/// Whether two CIEs are identical, including their relocations. +pub fn cie_equals( + a_file: &ObjectFile, + a: &CieRecord, + b_file: &ObjectFile, + b: &CieRecord, +) -> bool { + if a.contents::() != b.contents::() { + return false; + } + let x = a.rels::(a_file); + let y = b.rels::(b_file); + x.len() == y.len() + && x.iter().zip(y).all(|(rx, ry)| { + rx.r_offset - a.input_offset as u64 == ry.r_offset - b.input_offset as u64 + && rx.r_type == ry.r_type + && a_file.base.symbols[rx.r_sym as usize] == b_file.base.symbols[ry.r_sym as usize] + && rx.r_addend == ry.r_addend + }) +} + +/// Lays out the output `.eh_frame`: dead FDEs are removed, CIEs are +/// deduplicated, and each file gets a range for its FDEs. +pub fn construct(ctx: &mut Context) { + let _t = ctx.timer("eh_frame"); + + ctx.objs.par_iter_mut().for_each(|file| { + file.fdes.retain(|fde| fde.is_alive()); + let mut offset = 0; + let cies = &file.cies; + for fde in &mut file.fdes { + fde.output_offset = offset as u32; + offset += fde.size_with::(cies); + } + file.fde_size = offset as u64; + }); + + // Uniquify CIEs and assign offsets to them. + let mut leaders: Vec = Vec::new(); + let mut offset = 0u64; + for file in &mut ctx.objs { + let file = file.as_mut(); + let file_ptr = file as *mut ObjectFile; + let cies = file.cies.as_mut_ptr(); + for ci in 0..file.cies.len() { + // SAFETY: files are boxed and no CIE vector is resized during + // layout construction, so both addresses remain stable. + let cie = unsafe { CieHandle::new(file_ptr, cies.add(ci)) }; + let leader = leaders + .iter() + .copied() + .find(|&leader| leader.equals::(cie)); + match leader { + Some(leader) => cie.set_output_offset(leader.output_offset()), + None => { + cie.set_output_offset(offset as u32); + cie.set_leader(); + offset += cie.size::() as u64; + leaders.push(cie); + } + } + } + } + + // Assign FDE offsets to files. + let mut idx = 0u64; + for file in &mut ctx.objs { + file.fde_idx = idx; + idx += file.fdes.len() as u64; + file.fde_offset = offset; + offset += file.fde_size; + } + + // .eh_frame must end with a null word. + ctx.eh_frame.hdr.shdr.sh_size = offset + 4; +} + +/// Writes `.eh_frame` and the `.eh_frame_hdr` lookup table. +pub fn copy_buf(ctx: &Context, buf: &mut [u8], hdr_buf: Option<&mut [u8]>) { + let sh_addr = ctx.eh_frame.hdr.shdr.sh_addr; + let sh_size = ctx.eh_frame.hdr.shdr.sh_size; + + // Each file owns the range of its FDEs; leader CIEs are written by + // their files too. Since CIE ranges precede all FDE ranges, both are + // covered by splitting at CIE and FDE boundaries. A file without FDEs + // owns no range: its FDE offset coincides with the next file's. + let mut bounds: Vec = Vec::new(); + for file in &ctx.objs { + for cie in &file.cies { + if cie.is_leader { + bounds.push(cie.output_offset as u64); + } + } + } + for file in &ctx.objs { + if !file.fdes.is_empty() { + bounds.push(file.fde_offset); + } + } + debug_assert!(bounds.windows(2).all(|pair| pair[0] < pair[1])); + let mut slices = split_at_offsets(buf, &bounds).into_iter(); + + let hdr_addr = ctx.eh_frame_hdr.as_ref().map(|h| h.hdr.shdr.sh_addr); + + let write_cie = |file: &ObjectFile, cie: &CieRecord, dst: &mut [u8]| { + let contents = cie.contents::(); + dst[..contents.len()].copy_from_slice(contents); + if ctx.args.relocatable { + return; + } + for rel in cie.rels::(file) { + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let loc = (rel.r_offset - cie.input_offset as u64) as usize; + let val = sym + .addr(ctx) + .wrapping_add(file.section_at(cie.section).rel_addend::(&rel) as u64); + let p = sh_addr + cie.output_offset as u64 + loc as u64; + E::apply_eh_reloc( + ctx, + file.section_at(cie.section), + &rel, + &mut dst[loc..], + p, + val, + ); + } + }; + + // Gather per-file work items with their slices. + struct Item<'a> { + file: &'a ObjectFile, + cies: Vec<(usize, &'a mut [u8])>, + fdes: Option<&'a mut [u8]>, + hdr_entries: Option<&'a mut [u8]>, + } + let mut items: Vec = ctx + .objs + .iter() + .map(|file| Item { + file, + cies: Vec::new(), + fdes: None, + hdr_entries: None, + }) + .collect(); + + for (fi, file) in ctx.objs.iter().enumerate() { + for (ci, cie) in file.cies.iter().enumerate() { + if cie.is_leader { + let slice = slices.next().unwrap(); + items[fi].cies.push((ci, slice)); + } + } + } + for (fi, file) in ctx.objs.iter().enumerate() { + if !file.fdes.is_empty() { + items[fi].fdes = Some(slices.next().unwrap()); + } + } + debug_assert!(slices.next().is_none()); + + let base = EhFrameHdrSection::HEADER_SIZE as usize; + let mut hdr_table = hdr_buf.map(|buf| &mut buf[base..]); + if let Some(table) = hdr_table.as_deref_mut() { + let mut rest = table; + for (item, file) in items.iter_mut().zip(&ctx.objs) { + if !file.fdes.is_empty() { + let len = file.fdes.len() * 8; + let (own, tail) = std::mem::take(&mut rest).split_at_mut(len); + item.hdr_entries = Some(own); + rest = tail; + } + } + debug_assert!(rest.is_empty()); + } + + items.into_par_iter().for_each(|mut item| { + let file = item.file; + for (ci, dst) in item.cies { + write_cie(file, &file.cies[ci], dst); + } + let Some(fde_buf) = item.fdes else { + return; + }; + + for (i, fde) in file.fdes.iter().enumerate() { + let rels = fde.rels::(file); + let offset = file.fde_offset + fde.output_offset as u64; + let dst = &mut fde_buf[fde.output_offset as usize..]; + let contents = fde.contents::(file); + dst[..contents.len()].copy_from_slice(contents); + + let cie = &file.cies[fde.cie_idx as usize]; + E::Endian::write_u32( + &mut dst[4..], + (offset + 4 - cie.output_offset as u64) as u32, + ); + if ctx.args.relocatable { + continue; + } + + let mut func_addr = 0u64; + for (j, rel) in rels.iter().enumerate() { + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + let loc = (rel.r_offset - fde.input_offset as u64) as usize; + let val = sym + .addr(ctx) + .wrapping_add(file.section_at(cie.section).rel_addend::(&rel) as u64); + let p = sh_addr + offset + loc as u64; + E::apply_eh_reloc( + ctx, + file.section_at(cie.section), + &rel, + &mut dst[loc..], + p, + val, + ); + if j == 0 { + func_addr = val; + } + } + + let Some(origin) = hdr_addr else { + continue; + }; + let entry = &mut item.hdr_entries.as_deref_mut().unwrap()[i * 8..][..8]; + + // Compilers may emit a meaningless FDE covering a zero-length + // address range. Such an FDE must not be written to + // .eh_frame_hdr because another function may start at the same + // address, and a binary search on .eh_frame_hdr could then find + // the empty FDE instead of the real one. We write a tombstone + // value instead. + let ptr = &dst[8 + cie.fde_ptr_size as usize..]; + let range = if cie.fde_ptr_size == 4 { + E::Endian::read_u32(ptr) as u64 + } else { + E::Endian::read_u64(ptr) + }; + if range == 0 { + E::Endian::write_i32(entry, i32::MAX); + E::Endian::write_i32(&mut entry[4..], 0); + } else { + // The table entries are 32-bit offsets from .eh_frame_hdr. + if !is_int(func_addr.wrapping_sub(origin) as i64, 32) { + let sym = &ctx.symbols[file.base.symbols[rels.at(0).r_sym as usize]]; + error!(ctx, "{file}: {sym}: address out of range of .eh_frame_hdr"); + } + E::Endian::write_i32(entry, func_addr.wrapping_sub(origin) as i32); + E::Endian::write_i32( + &mut entry[4..], + (sh_addr + offset).wrapping_sub(origin) as i32, + ); + } + } + }); + + // Terminator + E::Endian::write_u32(&mut buf[sh_size as usize - 4..], 0); + + if let Some(table) = hdr_table { + let (entries, remainder) = table.as_chunks_mut::<8>(); + debug_assert!(remainder.is_empty()); + entries.par_sort_unstable_by_key(|entry| E::Endian::read_i32(entry)); + } +} + +/// Reports an `.eh_frame` relocation whose value doesn't fit. +pub fn check_range( + ctx: &Context, + isec: &crate::input_sections::InputSection, + rel: &ElfRel, + val: i64, + lo: i64, + hi: i64, +) { + if val < lo || hi <= val { + let file = &ctx.objs[isec.file.index()]; + let sym: &Symbol = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + error!( + ctx, + "{}: relocation {} against {sym} out of range: {val} is not in [{lo}, {hi})", + isec.display(file), + rel.type_name::() + ); + } +} + +/// `.eh_frame_hdr` is a sorted table of (function address, FDE address) +/// pairs. +#[derive(Debug)] +pub struct EhFrameHdrSection { + pub hdr: ChunkHeader, + pub num_fdes: u64, +} + +impl EhFrameHdrSection { + pub const HEADER_SIZE: u64 = 12; + + pub fn new() -> EhFrameHdrSection { + let mut hdr = ChunkHeader::new(".eh_frame_hdr", SHT_PROGBITS, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + hdr.shdr.sh_size = Self::HEADER_SIZE; + EhFrameHdrSection { hdr, num_fdes: 0 } + } +} + +impl Default for EhFrameHdrSection { + fn default() -> Self { + Self::new() + } +} + +pub mod eh_frame_hdr { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let num_fdes: u64 = ctx.objs.iter().map(|f| f.fdes.len() as u64).sum(); + let sec = ctx.eh_frame_hdr.as_mut().unwrap(); + sec.num_fdes = num_fdes; + sec.hdr.shdr.sh_size = EhFrameHdrSection::HEADER_SIZE + num_fdes * 8; + } + + /// Writes the header; the table is written by `eh_frame::copy_buf`. + pub fn write_header(ctx: &Context, buf: &mut [u8]) { + let sec = ctx.eh_frame_hdr.as_ref().unwrap(); + buf[0] = 1; + buf[1] = (DW_EH_PE_pcrel | DW_EH_PE_sdata4) as u8; + buf[2] = DW_EH_PE_udata4 as u8; + buf[3] = (DW_EH_PE_datarel | DW_EH_PE_sdata4) as u8; + E::Endian::write_u32( + &mut buf[4..], + ctx.eh_frame + .hdr + .shdr + .sh_addr + .wrapping_sub(sec.hdr.shdr.sh_addr) + .wrapping_sub(4) as u32, + ); + E::Endian::write_u32(&mut buf[8..], sec.num_fdes as u32); + } +} + +/// `.rela.eh_frame`, for relocatable outputs only. +#[derive(Debug)] +pub struct EhFrameRelocSection { + pub hdr: ChunkHeader, +} + +impl EhFrameRelocSection { + pub fn new() -> EhFrameRelocSection { + let (name, ty) = if E::IS_RELA { + (".rela.eh_frame", SHT_RELA) + } else { + (".rel.eh_frame", SHT_REL) + }; + let mut hdr = ChunkHeader::new(name, ty, SHF_INFO_LINK as u64); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + hdr.shdr.sh_entsize = ElfRel::size::() as u64; + EhFrameRelocSection { hdr } + } +} + +pub mod eh_frame_reloc { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let count: usize = ctx + .objs + .par_iter() + .map(|file| { + let cies: usize = file + .cies + .iter() + .filter(|c| c.is_leader) + .map(|c| c.rels::(file).len()) + .sum(); + let fdes: usize = file.fdes.iter().map(|f| f.rels::(file).len()).sum(); + cies + fdes + }) + .sum(); + let sec = ctx.eh_frame_reloc.as_mut().unwrap(); + sec.hdr.shdr.sh_size = (count * ElfRel::size::()) as u64; + sec.hdr.shdr.sh_link = ctx.symtab.hdr.shndx; + sec.hdr.shdr.sh_info = ctx.eh_frame.hdr.shndx; + } + + /// Writes the relocations; with REL and `-r`, addends are written into + /// `.eh_frame` itself, which is passed as `eh_frame_buf`. + pub fn copy_buf(ctx: &Context, buf: &mut [u8], eh_frame_buf: Option<&mut [u8]>) { + let size = ElfRel::size::(); + let mut eh_frame_buf = eh_frame_buf; + let mut n = 0; + + let mut copy = |file: &ObjectFile, + shndx: u32, + r: &ElfRel, + offset: u64, + eh_frame_buf: &mut Option<&mut [u8]>| { + let isec = file.section_at(shndx); + let sym = &ctx.symbols[file.base.symbols[r.r_sym as usize]]; + let mut out = ElfRel { + r_offset: ctx.eh_frame.hdr.shdr.sh_addr + offset, + r_type: r.r_type, + r_sym: 0, + r_addend: 0, + }; + + if sym.st_type() == STT_SECTION { + // Section symbols are recreated per output section, so a + // relocation's addend must account for the input section's + // offset in the output. + let target = sym.input_section_ref().unwrap(); + out.r_sym = ctx.output_section(target.output_section.unwrap()).hdr.shndx; + let addend = isec.rel_addend::(r) + target.offset() as i64; + if E::IS_RELA { + out.r_addend = addend; + } else if ctx.args.relocatable { + if let Some(eh) = eh_frame_buf { + E::write_addend(&mut eh[offset as usize..], addend, r); + } + } + } else { + out.r_sym = sym.output_sym_idx(ctx); + if E::IS_RELA { + out.r_addend = isec.rel_addend::(r); + } + } + out.write::(&mut buf[n * size..]); + n += 1; + }; + + for file in &ctx.objs { + for cie in &file.cies { + if cie.is_leader { + for rel in cie.rels::(file) { + let offset = + cie.output_offset as u64 + rel.r_offset - cie.input_offset as u64; + copy(file, cie.section, &rel, offset, &mut eh_frame_buf); + } + } + } + for fde in &file.fdes { + let cie = &file.cies[fde.cie_idx as usize]; + let base = file.fde_offset + fde.output_offset as u64; + for rel in fde.rels::(file) { + let offset = base + rel.r_offset - fde.input_offset as u64; + copy(file, cie.section, &rel, offset, &mut eh_frame_buf); + } + } + } + } +} + +/// Fatal error for `.eh_frame` contents that can't be handled. +pub fn unsupported(ctx: &Context, rel: &ElfRel) -> ! { + fatal!( + ctx, + "unsupported relocation in .eh_frame: {}", + rel.type_name::() + ) +} diff --git a/src/chunks/got.rs b/src/chunks/got.rs new file mode 100644 index 000000000..9be863998 --- /dev/null +++ b/src/chunks/got.rs @@ -0,0 +1,764 @@ +//! `.got`, `.got.plt`, `.plt`, `.plt.got` and `.rela.plt`. + +use crate::arch::{Arch, Family}; +use crate::chunks::{ChunkHeader, DynRelBuffer}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::SymtabBlock; +use crate::symbol::{AddrFlags, SymbolId}; + +/// `.got` is a linker-synthesized table of pointer-sized entries holding +/// runtime addresses of global variables and TP-relative offsets of +/// thread-local variables. +#[derive(Debug)] +pub struct GotSection { + pub hdr: ChunkHeader, + pub got_syms: Vec, + pub tlsgd_syms: Vec, + pub tlsdesc_syms: Vec, + pub gottp_syms: Vec, + pub tlsld_idx: Option, +} + +impl GotSection { + pub fn new() -> GotSection { + let mut hdr = ChunkHeader::new(".got", SHT_PROGBITS, (SHF_ALLOC | SHF_WRITE) as u64); + hdr.is_relro = true; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + // .got always exists so that _GLOBAL_OFFSET_TABLE_ has something + // to point to. s390x reserves GOT[1] and GOT[2] as well. + let reserved = if E::FAMILY == Family::S390x { 3 } else { 1 }; + hdr.shdr.sh_size = reserved * E::WORD_SIZE as u64; + GotSection { + hdr, + got_syms: Vec::new(), + tlsgd_syms: Vec::new(), + tlsdesc_syms: Vec::new(), + gottp_syms: Vec::new(), + tlsld_idx: None, + } + } + + pub fn has_tlsld(&self) -> bool { + self.tlsld_idx.is_some() + } + + pub fn tlsld_addr(&self) -> u64 { + self.hdr.shdr.sh_addr + self.tlsld_idx.unwrap() as u64 * E::WORD_SIZE as u64 + } +} + +// Named after the chunk, like the other chunks' modules in this directory. +#[allow(clippy::module_inception)] +pub mod got { + use super::*; + + fn word() -> u64 { + E::WORD_SIZE as u64 + } + + pub fn add_got_symbol(ctx: &mut Context, sym: SymbolId) { + let idx = (ctx.got.hdr.shdr.sh_size / word::()) as u32; + let is_pde_ifunc = ctx.symbols[sym].is_pde_ifunc(ctx); + ctx.symbols.aux_mut(sym).got_idx = Some(idx); + // An IFUNC in a position-dependent executable uses two GOT slots. + ctx.got.hdr.shdr.sh_size += if is_pde_ifunc { + 2 * word::() + } else { + word::() + }; + ctx.got.got_syms.push(sym); + } + + pub fn add_gottp_symbol(ctx: &mut Context, sym: SymbolId) { + let idx = (ctx.got.hdr.shdr.sh_size / word::()) as u32; + ctx.symbols.aux_mut(sym).gottp_idx = Some(idx); + ctx.got.hdr.shdr.sh_size += word::(); + ctx.got.gottp_syms.push(sym); + } + + pub fn add_tlsgd_symbol(ctx: &mut Context, sym: SymbolId) { + let idx = (ctx.got.hdr.shdr.sh_size / word::()) as u32; + ctx.symbols.aux_mut(sym).tlsgd_idx = Some(idx); + ctx.got.hdr.shdr.sh_size += 2 * word::(); + ctx.got.tlsgd_syms.push(sym); + } + + /// TLSDESC GOT slots always get a dynamic relocation, since their + /// values depend on the libc. A static executable has no dynamic + /// relocations, so TLSDESC is always relaxed away there. + pub fn add_tlsdesc_symbol(ctx: &mut Context, sym: SymbolId) { + debug_assert!(E::SUPPORTS_TLSDESC); + debug_assert!(!ctx.args.is_static); + let idx = (ctx.got.hdr.shdr.sh_size / word::()) as u32; + ctx.symbols.aux_mut(sym).tlsdesc_idx = Some(idx); + ctx.got.hdr.shdr.sh_size += 2 * word::(); + ctx.got.tlsdesc_syms.push(sym); + } + + pub fn add_tlsld(ctx: &mut Context) { + debug_assert!(ctx.got.tlsld_idx.is_none()); + ctx.got.tlsld_idx = Some((ctx.got.hdr.shdr.sh_size / word::()) as u32); + ctx.got.hdr.shdr.sh_size += 2 * word::(); + } + + struct GotEntry { + idx: u32, + val: u64, + r_type: u32, + sym: Option, + } + + /// Computes the GOT contents. An entry whose value is known at link + /// time is filled in directly; otherwise a dynamic relocation lets the + /// loader fill it. + fn got_entries(ctx: &Context) -> Vec { + let mut entries = Vec::new(); + let mut add = |idx: u32, val: u64, r_type: u32, sym: Option| { + entries.push(GotEntry { + idx, + val, + r_type, + sym, + }); + }; + let got = &ctx.got; + + for &id in &got.got_syms { + let sym = &ctx.symbols[id]; + let idx = sym.got_idx(&ctx.symbols).unwrap(); + + // IFUNC always needs to be fixed up by the dynamic linker. + if let Some(r_irelative) = E::R_IRELATIVE { + if sym.is_ifunc() { + if sym.is_pde_ifunc(ctx) { + add(idx, sym.plt_addr(ctx), R_NONE, None); + add( + idx + 1, + sym.addr_with(ctx, AddrFlags::NO_PLT), + r_irelative, + None, + ); + } else { + add( + idx, + sym.addr_with(ctx, AddrFlags::NO_PLT), + r_irelative, + None, + ); + } + continue; + } + } + + if sym.is_imported() { + add(idx, 0, E::R_GLOB_DAT, Some(id)); + } else if ctx.args.pic && sym.is_relative() { + add( + idx, + sym.addr_with(ctx, AddrFlags::NO_PLT), + E::R_RELATIVE, + None, + ); + } else { + add(idx, sym.addr_with(ctx, AddrFlags::NO_PLT), R_NONE, None); + } + } + + for &id in &got.tlsgd_syms { + let sym = &ctx.symbols[id]; + let idx = sym.tlsgd_idx(&ctx.symbols).unwrap(); + if sym.is_imported() { + add(idx, 0, E::R_DTPMOD, Some(id)); + add(idx + 1, 0, E::R_DTPOFF, Some(id)); + } else if ctx.args.shared { + // The offset within the TLS block is known but the module + // ID isn't. + add(idx, 0, E::R_DTPMOD, None); + add( + idx + 1, + sym.addr(ctx).wrapping_sub(ctx.dtp_addr), + R_NONE, + None, + ); + } else { + // Module ID 1 is the main executable. + add(idx, 1, R_NONE, None); + add( + idx + 1, + sym.addr(ctx).wrapping_sub(ctx.dtp_addr), + R_NONE, + None, + ); + } + } + + if let Some(r_tlsdesc) = E::R_TLSDESC { + for &id in &got.tlsdesc_syms { + let sym = &ctx.symbols[id]; + let idx = sym.tlsdesc_idx(&ctx.symbols).unwrap(); + if sym.is_imported() { + add(idx, 0, r_tlsdesc, Some(id)); + } else { + add( + idx, + sym.addr(ctx).wrapping_sub(ctx.tls_begin), + r_tlsdesc, + None, + ); + } + } + } + + for &id in &got.gottp_syms { + let sym = &ctx.symbols[id]; + let idx = sym.gottp_idx(&ctx.symbols).unwrap(); + if sym.is_imported() { + add(idx, 0, E::R_TPOFF, Some(id)); + } else if ctx.args.shared { + add( + idx, + sym.addr(ctx).wrapping_sub(ctx.tls_begin), + E::R_TPOFF, + None, + ); + } else { + add(idx, sym.addr(ctx).wrapping_sub(ctx.tp_addr), R_NONE, None); + } + } + + if let Some(idx) = got.tlsld_idx { + if ctx.args.shared { + add(idx, 0, E::R_DTPMOD, None); + } else { + add(idx, 1, R_NONE, None); + } + } + entries + } + + /// Counts the dynamic relocations the GOT emits without computing + /// entry values, which is too expensive for every layout iteration. + /// Must mirror `got_entries`. + pub fn num_dynrels(ctx: &Context) -> u64 { + let got = &ctx.got; + let mut n = 0; + for &id in &got.got_syms { + let sym = &ctx.symbols[id]; + if (E::SUPPORTS_IFUNC && sym.is_ifunc()) + || sym.is_imported() + || (ctx.args.pic && sym.is_relative()) + { + n += 1; + } + } + for &id in &got.tlsgd_syms { + let sym = &ctx.symbols[id]; + if sym.is_imported() { + n += 2; + } else if ctx.args.shared { + n += 1; + } + } + if E::SUPPORTS_TLSDESC { + n += got.tlsdesc_syms.len() as u64; + } + for &id in &got.gottp_syms { + if ctx.symbols[id].is_imported() || ctx.args.shared { + n += 1; + } + } + if got.tlsld_idx.is_some() && ctx.args.shared { + n += 1; + } + n + } + + pub fn relr_offsets(ctx: &Context) -> Vec { + if !ctx.args.pic { + return Vec::new(); + } + ctx.got + .got_syms + .iter() + .map(|&id| &ctx.symbols[id]) + .filter(|sym| !(E::SUPPORTS_IFUNC && sym.is_ifunc())) + .filter(|sym| !sym.is_imported() && sym.is_relative()) + .map(|sym| sym.got_idx(&ctx.symbols).unwrap() as u64 * word::()) + .collect() + } + + pub fn write_dynrels(ctx: &Context, mut out: DynRelBuffer<'_, E>) { + let mut i = 0; + for ent in got_entries(ctx) { + if ent.r_type == R_NONE { + continue; + } + let rel = ElfRel::new( + ctx.got.hdr.shdr.sh_addr + ent.idx as u64 * word::(), + ent.r_type, + ent.sym + .and_then(|s| ctx.symbols[s].dynsym_idx(&ctx.symbols)) + .unwrap_or(0), + ent.val as i64, + ); + let is_relr = rel.r_type == E::R_RELATIVE && rel.r_offset.is_multiple_of(word::()); + if !ctx.args.pack_dyn_relocs_relr || ctx.got.hdr.num_relrs == 0 || !is_relr { + out.write(i, rel); + i += 1; + } + } + debug_assert_eq!(i, out.len()); + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf.fill(0); + let w = word::() as usize; + let write = |buf: &mut [u8], idx: usize, val: u64| { + if E::IS_64 { + E::Endian::write_u64(&mut buf[idx * w..], val); + } else { + E::Endian::write_u32(&mut buf[idx * w..], val as u32); + } + }; + + // s390x's psABI requires GOT[0] to hold the address of _DYNAMIC. + // glibc's arm64 -static-pie code path wrongly assumes the same. + if let Some(dynamic) = &ctx.dynamic { + if E::FAMILY == Family::S390x + || (E::FAMILY == Family::Arm64 && ctx.args.is_static && ctx.args.pie) + { + write(buf, 0, dynamic.hdr.shdr.sh_addr); + } + } + + for ent in got_entries(ctx) { + let is_relr = ent.r_type == E::R_RELATIVE && ctx.args.pack_dyn_relocs_relr; + if is_relr || ent.r_type == R_NONE { + write(buf, ent.idx as usize, ent.val); + continue; + } + if ctx.args.apply_dynamic_relocs { + // A TLSDESC relocation fixes two consecutive slots: a + // function pointer and its argument. The addend applies to + // the argument, which is in the second slot, except on ARM32 + // which uses the inverted layout. + let mut i = ent.idx as usize; + if E::SUPPORTS_TLSDESC + && E::FAMILY != Family::Arm32 + && Some(ent.r_type) == E::R_TLSDESC + { + i += 1; + } + write(buf, i, ent.val); + } + } + } + + pub fn compute_symtab_size(ctx: &mut Context) { + let symbols = &ctx.symbols; + let got = &mut ctx.got; + got.hdr.strtab_size = 0; + got.hdr.num_local_symtab = 0; + let groups: [(&[SymbolId], &str); 4] = [ + (&got.got_syms, "$got"), + (&got.gottp_syms, "$gottp"), + (&got.tlsgd_syms, "$tlsgd"), + (&got.tlsdesc_syms, "$tlsdesc"), + ]; + let mut strtab_size = 0; + let mut count = 0; + for (syms, suffix) in groups { + for &id in syms { + strtab_size += symbols[id].name().len() as u64 + suffix.len() as u64 + 1; + count += 1; + } + } + if got.tlsld_idx.is_some() { + strtab_size += "$tlsld".len() as u64 + 1; + count += 1; + } + got.hdr.strtab_size = strtab_size; + got.hdr.num_local_symtab = count; + } + + pub fn populate_symtab(ctx: &Context, block: &mut SymtabBlock<'_>) { + let got = &ctx.got; + if got.hdr.num_local_symtab == 0 { + return; + } + let object = |value: u64| ElfSym { + st_info: STT_OBJECT as u8, + st_shndx: got.hdr.shndx as u16, + st_value: value, + ..ElfSym::default() + }; + for &id in &got.got_syms { + let sym = &ctx.symbols[id]; + block.push_synthetic::(sym.name(), b"$got", object(sym.got_addr(ctx))); + } + for &id in &got.gottp_syms { + let sym = &ctx.symbols[id]; + block.push_synthetic::(sym.name(), b"$gottp", object(sym.gottp_addr(ctx))); + } + for &id in &got.tlsgd_syms { + let sym = &ctx.symbols[id]; + block.push_synthetic::(sym.name(), b"$tlsgd", object(sym.tlsgd_addr(ctx))); + } + for &id in &got.tlsdesc_syms { + let sym = &ctx.symbols[id]; + block.push_synthetic::(sym.name(), b"$tlsdesc", object(sym.tlsdesc_addr(ctx))); + } + if got.tlsld_idx.is_some() { + block.push_synthetic::(b"", b"$tlsld", object(got.tlsld_addr::())); + } + } +} + +/// `.got.plt` holds the function pointers used by `.plt`. +#[derive(Debug)] +pub struct GotPltSection { + pub hdr: ChunkHeader, +} + +impl GotPltSection { + pub fn new(args: &crate::args::Args) -> GotPltSection { + let sh_type = if E::IS_PPC64 { + SHT_NOBITS + } else { + SHT_PROGBITS + }; + let mut hdr = ChunkHeader::new(".got.plt", sh_type, (SHF_ALLOC | SHF_WRITE) as u64); + hdr.is_relro = args.z_now; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + hdr.shdr.sh_size = gotplt::header_size::(); + GotPltSection { hdr } + } +} + +pub mod gotplt { + use super::*; + + pub fn header_size() -> u64 { + let words = if E::FAMILY == Family::Ppc64V2 { 2 } else { 3 }; + words * E::WORD_SIZE as u64 + } + + pub fn entry_size() -> u64 { + let words = if E::FAMILY == Family::Ppc64V1 { 3 } else { 1 }; + words * E::WORD_SIZE as u64 + } + + pub fn update_shdr(ctx: &mut Context) { + ctx.gotplt.hdr.shdr.sh_size = + header_size::() + ctx.plt.symbols.len() as u64 * entry_size::(); + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + // On PPC64 the loader fills .got.plt itself, finding the PLT + // through DT_PPC64_GLINK. + if E::IS_PPC64 { + return; + } + let w = E::WORD_SIZE; + let write = |buf: &mut [u8], idx: usize, val: u64| { + if E::IS_64 { + E::Endian::write_u64(&mut buf[idx * w..], val); + } else { + E::Endian::write_u32(&mut buf[idx * w..], val as u32); + } + }; + // The first slot points to _DYNAMIC as the psABI requests; the + // next two are reserved for the loader. + write( + buf, + 0, + ctx.dynamic.as_ref().map_or(0, |d| d.hdr.shdr.sh_addr), + ); + write(buf, 1, 0); + write(buf, 2, 0); + for i in 0..ctx.plt.symbols.len() { + write(buf, i + 3, ctx.plt.hdr.shdr.sh_addr); + } + } +} + +/// `.plt` contains stubs that branch to the real function entry points, +/// used for lazy symbol resolution. +#[derive(Debug)] +pub struct PltSection { + pub hdr: ChunkHeader, + pub symbols: Vec, +} + +impl PltSection { + pub fn new() -> PltSection { + let mut hdr = ChunkHeader::new(".plt", SHT_PROGBITS, (SHF_ALLOC | SHF_EXECINSTR) as u64); + if E::IS_SPARC { + hdr.shdr.sh_flags |= SHF_WRITE as u64; + hdr.shdr.sh_addralign = 256; + } else { + hdr.shdr.sh_addralign = 16; + } + PltSection { + hdr, + symbols: Vec::new(), + } + } +} + +pub mod plt { + use super::*; + + /// SPARC uses 32-byte "small" PLT entries until the table grows past + /// 0x100000 bytes, then switches to a "large" format. + pub const SPARC_NUM_SMALL_PLT: u64 = (0x100000 - 128) / 32; + + /// The offset of a PLT entry within `.plt`. + pub fn entry_offset(idx: u32) -> u64 { + let idx = idx as u64; + match E::FAMILY { + // PPC64 ELFv1 entries are 8 bytes for indices below 32768 and + // 12 bytes beyond. + Family::Ppc64V1 => { + if idx < 0x8000 { + E::PLT_HDR_SIZE + idx * 8 + } else { + E::PLT_HDR_SIZE + 0x8000 * 8 + (idx - 0x8000) * 12 + } + } + // SPARC large entries come in blocks of 160 24-byte stubs + // followed by 160 8-byte pointers. + Family::Sparc64 => { + if idx < SPARC_NUM_SMALL_PLT { + E::PLT_HDR_SIZE + idx * E::PLT_SIZE + } else { + let i = idx - SPARC_NUM_SMALL_PLT; + 0x100000 + (i / 160) * 5120 + (i % 160) * 24 + } + } + _ => E::PLT_HDR_SIZE + idx * E::PLT_SIZE, + } + } + + pub fn add_symbol(ctx: &mut Context, sym: SymbolId) { + debug_assert!(!ctx.symbols[sym].has_plt(&ctx.symbols)); + let idx = ctx.plt.symbols.len() as u32; + ctx.symbols.aux_mut(sym).plt_idx = Some(idx); + ctx.plt.symbols.push(sym); + crate::chunks::symtab::dynsym::add_symbol(ctx, sym); + } + + pub fn update_shdr(ctx: &mut Context) { + let n = ctx.plt.symbols.len() as u64; + ctx.plt.hdr.shdr.sh_size = if n == 0 { + 0 + } else if E::IS_SPARC { + E::PLT_HDR_SIZE + n * E::PLT_SIZE + } else { + entry_offset::(n as u32) + }; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + E::write_plt_header(ctx, buf); + for (i, &id) in ctx.plt.symbols.iter().enumerate() { + let off = entry_offset::(i as u32) as usize; + E::write_plt_entry(ctx, &mut buf[off..], &ctx.symbols[id]); + } + } + + pub fn compute_symtab_size(ctx: &mut Context) { + let n = ctx.plt.symbols.len() as u32; + let strtab_size: u64 = ctx + .plt + .symbols + .iter() + .map(|&id| ctx.symbols[id].name().len() as u64 + "$plt".len() as u64 + 1) + .sum(); + ctx.plt.hdr.num_local_symtab = if E::FAMILY == Family::Arm32 { + n * 3 + 2 + } else { + n + }; + ctx.plt.hdr.strtab_size = strtab_size; + } + + pub fn populate_symtab(ctx: &Context, block: &mut SymtabBlock<'_>) { + let plt = &ctx.plt; + if plt.hdr.num_local_symtab == 0 { + return; + } + let func = |addr: u64| ElfSym { + st_info: STT_FUNC as u8, + st_shndx: plt.hdr.shndx as u16, + st_value: addr, + ..ElfSym::default() + }; + use crate::chunks::symtab::strtab::{ARM, DATA}; + if E::FAMILY == Family::Arm32 { + block.push_mapping_symbol::(ARM, func(plt.hdr.shdr.sh_addr)); + block.push_mapping_symbol::(DATA, func(plt.hdr.shdr.sh_addr + 16)); + } + for &id in &plt.symbols { + let sym = &ctx.symbols[id]; + let addr = sym.plt_addr(ctx); + block.push_synthetic::(sym.name(), b"$plt", func(addr)); + if E::FAMILY == Family::Arm32 { + block.push_mapping_symbol::(ARM, func(addr)); + block.push_mapping_symbol::(DATA, func(addr + 12)); + } + } + } +} + +/// `.plt.got` is like `.plt` but for symbols that already have a GOT +/// entry; resolving them lazily through `.plt` would be pointless. +#[derive(Debug)] +pub struct PltGotSection { + pub hdr: ChunkHeader, + pub symbols: Vec, +} + +impl PltGotSection { + pub fn new() -> PltGotSection { + let mut hdr = + ChunkHeader::new(".plt.got", SHT_PROGBITS, (SHF_ALLOC | SHF_EXECINSTR) as u64); + hdr.shdr.sh_addralign = 16; + PltGotSection { + hdr, + symbols: Vec::new(), + } + } +} + +impl Default for PltGotSection { + fn default() -> Self { + Self::new() + } +} + +pub mod pltgot { + use super::*; + + pub fn add_symbol(ctx: &mut Context, sym: SymbolId) { + debug_assert!(!ctx.symbols[sym].has_plt(&ctx.symbols)); + debug_assert!(ctx.symbols[sym].has_got(&ctx.symbols)); + let idx = ctx.pltgot.symbols.len() as u32; + ctx.symbols.aux_mut(sym).pltgot_idx = Some(idx); + ctx.pltgot.symbols.push(sym); + ctx.pltgot.hdr.shdr.sh_size = ctx.pltgot.symbols.len() as u64 * E::PLTGOT_SIZE; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + for (i, &id) in ctx.pltgot.symbols.iter().enumerate() { + let off = i * E::PLTGOT_SIZE as usize; + E::write_pltgot_entry(ctx, &mut buf[off..], &ctx.symbols[id]); + } + } + + pub fn compute_symtab_size(ctx: &mut Context) { + let n = ctx.pltgot.symbols.len() as u32; + let strtab_size: u64 = ctx + .pltgot + .symbols + .iter() + .map(|&id| ctx.symbols[id].name().len() as u64 + "$pltgot".len() as u64 + 1) + .sum(); + ctx.pltgot.hdr.num_local_symtab = if E::FAMILY == Family::Arm32 { n * 3 } else { n }; + ctx.pltgot.hdr.strtab_size = strtab_size; + } + + pub fn populate_symtab(ctx: &Context, block: &mut SymtabBlock<'_>) { + let pltgot = &ctx.pltgot; + if pltgot.hdr.num_local_symtab == 0 { + return; + } + let func = |addr: u64| ElfSym { + st_info: STT_FUNC as u8, + st_shndx: pltgot.hdr.shndx as u16, + st_value: addr, + ..ElfSym::default() + }; + use crate::chunks::symtab::strtab::{ARM, DATA}; + for &id in &pltgot.symbols { + let sym = &ctx.symbols[id]; + let addr = sym.plt_addr(ctx); + block.push_synthetic::(sym.name(), b"$pltgot", func(addr)); + if E::FAMILY == Family::Arm32 { + block.push_mapping_symbol::(ARM, func(addr)); + block.push_mapping_symbol::(DATA, func(addr + 12)); + } + } + } +} + +/// `.rela.plt` holds the relocations for `.plt`. +#[derive(Debug)] +pub struct RelPltSection { + pub hdr: ChunkHeader, +} + +impl RelPltSection { + pub fn new() -> RelPltSection { + let (name, ty) = if E::IS_RELA { + (".rela.plt", SHT_RELA) + } else { + (".rel.plt", SHT_REL) + }; + let mut hdr = ChunkHeader::new(name, ty, SHF_ALLOC as u64); + hdr.shdr.sh_entsize = ElfRel::size::() as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + RelPltSection { hdr } + } +} + +pub mod relplt { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + ctx.relplt.hdr.shdr.sh_size = ctx.plt.symbols.len() as u64 * ElfRel::size::() as u64; + ctx.relplt.hdr.shdr.sh_link = ctx.dynsym.hdr.shndx; + if !E::IS_SPARC { + ctx.relplt.hdr.shdr.sh_info = ctx.gotplt.hdr.shndx; + } + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let size = ElfRel::size::(); + for (i, &id) in ctx.plt.symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + let rel = if E::IS_SPARC { + // SPARC has no .got.plt: its .plt is writable and the + // loader patches the stubs in place. + let idx = sym.plt_idx(&ctx.symbols).unwrap() as u64; + if idx < plt::SPARC_NUM_SMALL_PLT { + ElfRel::new( + sym.plt_addr(ctx), + E::R_JUMP_SLOT, + sym.dynsym_idx(&ctx.symbols).unwrap_or(0), + 0, + ) + } else { + let call = sym.plt_addr(ctx) + 4; + let ptr = ctx.plt.hdr.shdr.sh_addr + + crate::arch::sparc64::plt_ptr_offset(ctx.plt.symbols.len(), idx); + ElfRel::new( + ptr, + E::R_JUMP_SLOT, + sym.dynsym_idx(&ctx.symbols).unwrap_or(0), + -(call as i64), + ) + } + } else { + ElfRel::new( + sym.gotplt_addr(ctx), + E::R_JUMP_SLOT, + sym.dynsym_idx(&ctx.symbols).unwrap_or(0), + 0, + ) + }; + rel.write::(&mut buf[i * size..]); + } + } +} diff --git a/src/chunks/merged.rs b/src/chunks/merged.rs new file mode 100644 index 000000000..cc6b9cf99 --- /dev/null +++ b/src/chunks/merged.rs @@ -0,0 +1,551 @@ +//! Merged sections: constant pools such as string literals, deduplicated +//! across all input files. +//! +//! Fragments are deduplicated through a concurrent hash table sized for +//! 2/3 occupancy from an estimate of their number, and the output layout +//! follows the table: shard by shard, in the deterministic bucket order +//! of [`FrozenMap::sorted_entries`]. + +use std::fmt; +use std::sync::atomic::Ordering; +use std::sync::RwLock; + +use bstr::BStr; +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::args::Args; +use crate::chunks::ChunkHeader; +use crate::context::Context; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::input_sections::{InputSection, MergeableSection, SectionFragment, SectionRef}; +use crate::output_file::split_at_offsets; +use crate::util::align_to; +use crate::util::concurrent_map::{ConcurrentMap, EntryId, FrozenMap, NUM_SHARDS}; +use crate::util::hyperloglog::HyperLogLog; +use crate::util::timer::Timers; + +/// Index of a merged section in `Context::merged_sections`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct MergedSectionId(pub u32); + +impl MergedSectionId { + pub fn index(self) -> usize { + self.0 as usize + } +} + +/// The deterministically ordered fragments and the two output sizes of one +/// hash-table shard. +#[derive(Debug)] +struct ShardLayout { + near_size: u64, + far_size: u64, + fragments: Vec, +} + +#[derive(Debug)] +pub struct MergedSection { + pub hdr: ChunkHeader, + + /// The input sections merged into this one. + pub members: Vec, + + /// The fragment map while fragments are being inserted. + pub map: ConcurrentMap, + + /// The fragments once insertion is done. + pub fragments: FrozenMap, + pub resolved: bool, + + /// The fragments in deterministic order within each hash-table shard. + shards: Vec, + + /// Starts of the 32-bit and unrestricted halves of every shard, followed + /// by the section size. + shard_offsets: Vec, +} + +/// A direct reference to one input member while allocated merged sections +/// are resolved in parallel. C++ mold keeps the same pointers in +/// `MergedSection::members`; Rust keeps the durable section references there +/// and borrows the stable objects directly for this phase. +pub struct ResolveMember<'a> { + pub mergeable: &'a mut MergeableSection, + pub section: &'a InputSection, + pub filename: &'a str, + pub archive_name: &'a str, + pub name: &'static BStr, +} + +pub struct ResolveOptions<'a> { + pub allocated_only: bool, + pub gc_sections: bool, + pub diag: &'a Diagnostics, + pub comment: Option, + pub cmdline_args: &'a [String], + pub timers: &'a Timers, +} + +struct FileName<'a> { + filename: &'a str, + archive_name: &'a str, +} + +impl fmt::Display for FileName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.archive_name.is_empty() { + write!(f, "{}", crate::util::path_clean(self.filename)) + } else { + write!( + f, + "{}({})", + crate::util::path_clean(self.archive_name), + self.filename + ) + } + } +} + +/// Normalizes the name of a merged output section. GCC creates sections +/// named `.rodata.strN..M` or `.rodata.cst..cstN`; +/// the symbol name part is dropped. +fn merged_output_name( + args: &Args, + name: &'static BStr, + flags: u64, + entsize: u64, + addralign: u64, +) -> &'static BStr { + if args.relocatable && !args.relocatable_merge_sections { + return name; + } + if !args.unique.is_empty() && args.unique.find(name) != -1 { + return name; + } + if name.starts_with(b".rodata.") { + let name2 = if flags & SHF_STRINGS as u64 != 0 { + format!(".rodata.str{entsize}.{addralign}") + } else { + format!(".rodata.cst{entsize}") + }; + if name == name2.as_bytes() { + return name; + } + return BStr::new(crate::util::leak_bytes(name2.into_bytes())); + } + name +} + +impl MergedSection { + fn new(name: &'static BStr, flags: u64, sh_type: u32, entsize: u64) -> MergedSection { + let mut hdr = ChunkHeader::with_name(name, sh_type, flags); + hdr.shdr.sh_entsize = entsize; + MergedSection { + hdr, + members: Vec::new(), + map: ConcurrentMap::default(), + fragments: FrozenMap::default(), + resolved: false, + shards: Vec::new(), + shard_offsets: Vec::new(), + } + } + + /// Finds or creates the merged section an input section belongs to. + /// Files are converted in parallel, and nearly every lookup finds an + /// existing section, so the list is only write-locked to add one. + pub fn get_instance( + args: &Args, + sections: &RwLock>, + name: &'static BStr, + shdr: &ElfShdr, + ) -> Option { + if shdr.sh_flags & SHF_MERGE as u64 == 0 { + return None; + } + let addralign = shdr.sh_addralign.max(1); + let flags = shdr.sh_flags & !(SHF_GROUP as u64) & !(SHF_COMPRESSED as u64); + let mut entsize = shdr.sh_entsize; + if entsize == 0 { + entsize = if shdr.sh_flags & SHF_STRINGS as u64 != 0 { + 1 + } else { + shdr.sh_addralign + }; + } + if entsize == 0 { + return None; + } + + let name = merged_output_name(args, name, flags, entsize, addralign); + let find = |sections: &[MergedSection]| { + sections + .iter() + .position(|s| { + s.hdr.name == name + && s.hdr.shdr.sh_flags == flags + && s.hdr.shdr.sh_type == shdr.sh_type + && s.hdr.shdr.sh_entsize == entsize + }) + .map(|i| MergedSectionId(i as u32)) + }; + + if let Some(id) = find(§ions.read().unwrap()) { + return Some(id); + } + let mut sections = sections.write().unwrap(); + if let Some(id) = find(§ions) { + return Some(id); + } + sections.push(MergedSection::new(name, flags, shdr.sh_type, entsize)); + Some(MergedSectionId(sections.len() as u32 - 1)) + } + + /// Inserts a piece of data. Only memory-mapped strings are subject to + /// garbage collection; strings in debug info are kept. + pub fn insert( + &self, + data: &'static [u8], + hash: u64, + p2align: u8, + gc_sections: bool, + ) -> EntryId { + let is_alive = !gc_sections || !self.is_alloc(); + let (id, frag, _) = self + .map + .insert_with(data, hash, || SectionFragment::new(is_alive)); + // Most insertions find the fragment there already, so the alignment + // is only written when it grows: an atomic update on every insertion + // would bounce the cache line of a popular fragment between cores. + if frag.p2align.load(Ordering::Relaxed) < p2align { + frag.p2align.fetch_max(p2align, Ordering::Relaxed); + } + id + } + + pub fn is_alloc(&self) -> bool { + self.hdr.shdr.sh_flags & SHF_ALLOC as u64 != 0 + } +} + +/// Splits the members into fragments and deduplicates them. +pub fn resolve(ctx: &mut Context, id: MergedSectionId) { + let timers = ctx.timers.clone(); + let Context { + objs, + merged_sections, + args, + diag, + .. + } = ctx; + let msec = &merged_sections[id.index()]; + let gc_sections = args.gc_sections; + + // The members, by file: a file's sections are worked on by one task. + let mut member_shndx: Vec> = vec![Vec::new(); objs.len()]; + for m in &msec.members { + member_shndx[m.file.index()].push(m.shndx); + } + + // Split the members into pieces, estimating the number of distinct + // ones to size the table. + let t = timers.start("split_contents"); + let estimate = objs + .par_iter_mut() + .zip(&member_shndx) + .filter(|(_, shndx)| !shndx.is_empty()) + .fold(HyperLogLog::default, |mut sketch, (file, shndx)| { + let mut slots = std::mem::take(&mut file.sections); + for &i in shndx { + let (m, isec) = slots + .mergeable_with_section_mut(i as usize) + .expect("a mergeable section"); + let name = isec.name(file); + m.split_contents::(diag, file, isec, name, msec, &mut sketch); + } + file.sections = slots; + sketch + }) + .reduce(HyperLogLog::default, |a, b| a.merged(&b)); + drop(t); + + // We aim 2/3 occupation ratio + let t = timers.start("resize"); + let msec = &mut merged_sections[id.index()]; + msec.map = ConcurrentMap::with_capacity(estimate.cardinality() as usize * 3 / 2); + let msec = &*msec; + drop(t); + + let t = timers.start("resolve_contents"); + objs.par_iter_mut() + .zip(&member_shndx) + .filter(|(_, shndx)| !shndx.is_empty()) + .for_each(|(file, shndx)| { + for &i in shndx { + let (m, isec) = file + .sections + .mergeable_with_section_mut(i as usize) + .expect("a mergeable section"); + m.resolve_contents(isec, msec, gc_sections); + } + }); + drop(t); + + if ctx.comment == Some(id) { + let msec = &ctx.merged_sections[id.index()]; + add_comment_strings(msec, ctx.args.gc_sections, &ctx.cmdline_args); + } + + // Compute section alignment + let msec = &ctx.merged_sections[id.index()]; + let p2align = msec + .members + .iter() + .filter_map(|m| ctx.objs[m.file.index()].mergeable_section(m.shndx as usize)) + .map(|m| m.p2align) + .max() + .unwrap_or(0); + + let msec = &mut ctx.merged_sections[id.index()]; + msec.hdr.shdr.sh_addralign = 1 << p2align; + msec.fragments = std::mem::take(&mut msec.map).freeze(); + msec.resolved = true; +} + +/// Resolves selected merged sections concurrently, as C++ mold does. +/// Direct member borrows let different parent sections mutate disjoint +/// `MergeableSection`s even when they belong to the same object file. +pub fn resolve_sections( + sections: &mut [MergedSection], + members: &mut [Vec>], + options: ResolveOptions<'_>, +) { + let ResolveOptions { + allocated_only, + gc_sections, + diag, + comment, + cmdline_args, + timers, + } = options; + let t = timers.start("split_contents"); + let estimates: Vec> = sections + .par_iter() + .zip(members.par_iter_mut()) + .map(|(section, members)| { + (!section.resolved && (!allocated_only || section.is_alloc())).then(|| { + members + .par_iter_mut() + .fold(HyperLogLog::default, |mut sketch, member| { + member.mergeable.split_contents::( + diag, + &FileName { + filename: member.filename, + archive_name: member.archive_name, + }, + member.section, + member.name, + section, + &mut sketch, + ); + sketch + }) + .reduce(HyperLogLog::default, |a, b| a.merged(&b)) + }) + }) + .collect(); + drop(t); + + // We aim 2/3 occupation ratio + let t = timers.start("resize"); + sections + .par_iter_mut() + .zip(&estimates) + .for_each(|(section, estimate)| { + if let Some(estimate) = estimate { + section.map = ConcurrentMap::with_capacity(estimate.cardinality() as usize * 3 / 2); + } + }); + drop(t); + + let t = timers.start("resolve_contents"); + sections + .par_iter() + .zip(members.par_iter_mut()) + .filter(|(section, _)| !section.resolved && (!allocated_only || section.is_alloc())) + .for_each(|(section, members)| { + members.par_iter_mut().for_each(|member| { + member + .mergeable + .resolve_contents(member.section, section, gc_sections); + }); + }); + drop(t); + + sections + .par_iter_mut() + .zip(members.par_iter()) + .enumerate() + .filter(|(_, (section, _))| !section.resolved && (!allocated_only || section.is_alloc())) + .for_each(|(i, (section, members))| { + if comment == Some(MergedSectionId(i as u32)) { + add_comment_strings(section, gc_sections, cmdline_args); + } + + // Compute section alignment + let p2align = members + .iter() + .map(|member| member.mergeable.p2align) + .max() + .unwrap_or(0); + section.hdr.shdr.sh_addralign = 1 << p2align; + section.fragments = std::mem::take(&mut section.map).freeze(); + section.resolved = true; + }); +} + +/// Adds an identification string, and the command line under +/// `MOLD_DEBUG`, to `.comment`. +fn add_comment_strings(msec: &MergedSection, gc_sections: bool, cmdline_args: &[String]) { + let add = |s: String| { + let mut bytes = s.into_bytes(); + bytes.push(0); + let data = crate::util::leak_bytes(bytes); + msec.insert(data, xxhash_rust::xxh3::xxh3_64(data), 0, gc_sections); + }; + add(crate::args::VERSION.to_string()); + if std::env::var("MOLD_DEBUG").is_ok_and(|v| !v.is_empty()) { + add(format!( + "mold command line: {}", + cmdline_args[1..].join(" ") + )); + } +} + +/// Lays out the live fragments and computes the section size. +pub fn compute_section_size(ctx: &mut Context, id: MergedSectionId) { + if !ctx.merged_sections[id.index()].resolved { + resolve(ctx, id); + } + layout(&mut ctx.merged_sections[id.index()]); +} + +/// Lays out one resolved merged section. Different merged sections have no +/// shared mutable state, so callers can run this for all of them in parallel. +pub fn layout(msec: &mut MergedSection) { + debug_assert!(msec.resolved); + let frags = &msec.fragments; + + // Within a shard, fragments that must be reachable with 32-bit + // relocations come first; the 32-bit halves of all shards precede the + // other halves. + let shards: Vec = (0..NUM_SHARDS) + .into_par_iter() + .map(|shard| { + let fragments = frags.sorted_entries(shard); + let mut near_size = 0; + let mut far_size = 0; + for &entry in &fragments { + let frag = frags.get(entry); + if !frag.is_alive() { + continue; + } + if frag.is_32bit() { + near_size = align_to(near_size, 1 << frag.p2align.load(Ordering::Relaxed)); + frag.set_offset(near_size); + near_size += frags.key(entry).len() as u64; + } else { + far_size = align_to(far_size, 1 << frag.p2align.load(Ordering::Relaxed)); + frag.set_offset(far_size); + far_size += frags.key(entry).len() as u64; + } + } + ShardLayout { + near_size, + far_size, + fragments, + } + }) + .collect(); + + let addralign = msec.hdr.shdr.sh_addralign; + let mut shard_offsets = Vec::with_capacity(NUM_SHARDS * 2 + 1); + shard_offsets.push(0); + for size in shards + .iter() + .map(|shard| shard.near_size) + .chain(shards.iter().map(|shard| shard.far_size)) + { + shard_offsets.push(align_to( + shard_offsets.last().copied().unwrap() + size, + addralign, + )); + } + + shards.par_iter().enumerate().for_each(|(i, shard)| { + for &entry in &shard.fragments { + let frag = frags.get(entry); + if frag.is_alive() { + let base = if frag.is_32bit() { + shard_offsets[i] + } else { + shard_offsets[i + NUM_SHARDS] + }; + frag.set_offset(frag.offset() + base); + } + } + }); + + msec.hdr.shdr.sh_size = shard_offsets.last().copied().unwrap(); + msec.shards = shards; + msec.shard_offsets = shard_offsets; +} + +pub fn copy_buf(ctx: &Context, id: MergedSectionId, buf: &mut [u8]) { + write_to(ctx, id, buf); +} + +pub fn write_to(ctx: &Context, id: MergedSectionId, buf: &mut [u8]) { + let msec = &ctx.merged_sections[id.index()]; + let frags = &msec.fragments; + + // Alignment may leave gaps between fragments, which must be zeroed. + let has_gaps = + msec.hdr.shdr.sh_addralign > 1 && msec.hdr.shdr.sh_addralign != msec.hdr.shdr.sh_entsize; + if has_gaps { + split_at_offsets(buf, &msec.shard_offsets[..NUM_SHARDS * 2]) + .into_par_iter() + .for_each(|region| region.fill(0)); + } + + // Copy strings + let output = buf.as_mut_ptr() as usize; + let output_len = buf.len(); + msec.shards.par_iter().for_each(|shard| { + for &entry in &shard.fragments { + let frag = frags.get(entry); + if frag.is_alive() { + let key = frags.key(entry); + let offset = frag.offset() as usize; + debug_assert!(offset + key.len() <= output_len); + // SAFETY: layout assigns every live fragment a distinct range + // within `buf`; each entry occurs in exactly one shard, so the + // parallel copies do not overlap. + unsafe { + std::ptr::copy_nonoverlapping( + key.as_ptr(), + (output as *mut u8).add(offset), + key.len(), + ); + } + } + } + }); +} + +/// The number of unique fragments, for `--stats`. +pub fn num_fragments(msec: &MergedSection) -> usize { + msec.fragments.len() +} diff --git a/src/chunks/misc.rs b/src/chunks/misc.rs new file mode 100644 index 000000000..db785a259 --- /dev/null +++ b/src/chunks/misc.rs @@ -0,0 +1,753 @@ +//! Small synthesized sections. + +use std::collections::{BTreeMap, BTreeSet}; + +use bstr::BStr; +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::{self, ChunkHeader, ChunkId, DynRelBuffer, OutputSectionId}; +use crate::context::Context; +use crate::elf::*; +use crate::error; +use crate::input_files::FileId; +use crate::input_sections::{r_delta, InputSection}; +use crate::symbol::SymbolId; +use crate::util::align_to; +use crate::util::compress::Compressor; +use crate::util::{path_filename, write_cstr}; + +/// `.interp` names the dynamic linker. +#[derive(Debug)] +pub struct InterpSection { + pub hdr: ChunkHeader, +} + +impl InterpSection { + pub fn new() -> InterpSection { + InterpSection { + hdr: ChunkHeader::new(".interp", SHT_PROGBITS, SHF_ALLOC as u64), + } + } +} + +impl Default for InterpSection { + fn default() -> Self { + Self::new() + } +} + +pub mod interp { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let size = ctx.args.dynamic_linker.len() as u64 + 1; + ctx.interp.as_mut().unwrap().hdr.shdr.sh_size = size; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + write_cstr(buf, ctx.args.dynamic_linker.as_bytes()); + } +} + +/// `.copyrel` and `.copyrel.rel.ro` hold the space that copy relocations +/// fill from shared libraries at load time. +#[derive(Debug)] +pub struct CopyrelSection { + pub hdr: ChunkHeader, + pub symbols: Vec, +} + +impl CopyrelSection { + pub fn new(is_relro: bool) -> CopyrelSection { + let name = if is_relro { + ".copyrel.rel.ro" + } else { + ".copyrel" + }; + let mut hdr = ChunkHeader::new(name, SHT_NOBITS, (SHF_ALLOC | SHF_WRITE) as u64); + hdr.is_relro = is_relro; + CopyrelSection { + hdr, + symbols: Vec::new(), + } + } +} + +pub mod copyrel { + use super::*; + + pub fn add_symbol(ctx: &mut Context, relro: bool, id: SymbolId) { + debug_assert!(!ctx.args.shared); + let sym = &ctx.symbols[id]; + if sym.has_copyrel() { + return; + } + let Some(FileId::Dso(dso_id)) = sym.file() else { + debug_assert!(sym.is_undef_weak()); + let file = ctx.file_display(sym.file().unwrap()); + error!( + ctx, + "{file}: cannot create a copy relocation for {sym}; recompile with -fPIE or -fPIC" + ); + return; + }; + let dso = &ctx.dsos[dso_id.index()]; + if sym.esym(ctx).st_visibility() == STV_PROTECTED { + error!( + ctx, + "{dso}: cannot create a copy relocation for protected symbol '{sym}'; recompile with -fPIC" + ); + return; + } + if !ctx.args.z_copyreloc { + error!( + ctx, + "-z nocopyreloc: {dso}: cannot create a copy relocation for symbol '{sym}'; recompile with -fPIC" + ); + return; + } + + let alignment = dso.alignment(sym); + let size = sym.esym(ctx).st_size; + // Aliases at the same address must refer to the copied place too, + // e.g. `environ`, `_environ` and `__environ` in libc. + let aliases: Vec = dso.symbols_at(ctx, sym, dso_id).to_vec(); + + let sec = if relro { + &mut ctx.copyrel_relro + } else { + &mut ctx.copyrel + }; + sec.symbols.push(id); + let offset = align_to(sec.hdr.shdr.sh_size, alignment); + sec.hdr.shdr.sh_size = offset + size; + sec.hdr.shdr.sh_addralign = sec.hdr.shdr.sh_addralign.max(alignment); + + for alias in aliases { + ctx.symbols.aux_mut(alias); + let s = &mut ctx.symbols[alias]; + s.set_imported(true); + s.set_exported(true); + s.set_copyrel(true); + s.set_copyrel_readonly(relro); + s.value = offset; + chunks::symtab::dynsym::add_symbol(ctx, alias); + } + } + + pub fn write_dynrels( + ctx: &Context, + sec: &CopyrelSection, + mut out: DynRelBuffer<'_, E>, + ) { + for (i, &id) in sec.symbols.iter().enumerate() { + let sym = &ctx.symbols[id]; + out.write( + i, + ElfRel::new( + sym.addr(ctx), + E::R_COPY, + sym.dynsym_idx(&ctx.symbols).unwrap_or(0), + 0, + ), + ); + } + debug_assert_eq!(sec.symbols.len(), out.len()); + } +} + +/// `.note.gnu.build-id` identifies the output, usually by a hash of it. +#[derive(Debug)] +pub struct BuildIdSection { + pub hdr: ChunkHeader, + pub contents: Vec, +} + +impl BuildIdSection { + pub fn new() -> BuildIdSection { + let mut hdr = ChunkHeader::new(".note.gnu.build-id", SHT_NOTE, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + hdr.shdr.sh_size = 1; + BuildIdSection { + hdr, + contents: Vec::new(), + } + } +} + +impl Default for BuildIdSection { + fn default() -> Self { + Self::new() + } +} + +pub mod build_id { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let size = ctx.args.build_id.size() as u64 + 16; + ctx.buildid.as_mut().unwrap().hdr.shdr.sh_size = size; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let sec = ctx.buildid.as_ref().unwrap(); + buf.fill(0); + E::Endian::write_u32(buf, 4); + E::Endian::write_u32(&mut buf[4..], ctx.args.build_id.size() as u32); + E::Endian::write_u32(&mut buf[8..], NT_GNU_BUILD_ID); + buf[12..16].copy_from_slice(b"GNU\0"); + buf[16..16 + sec.contents.len()].copy_from_slice(&sec.contents); + } +} + +/// `.note.package` carries package metadata for distributions. +#[derive(Debug)] +pub struct NotePackageSection { + pub hdr: ChunkHeader, +} + +impl NotePackageSection { + pub fn new() -> NotePackageSection { + let mut hdr = ChunkHeader::new(".note.package", SHT_NOTE, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + NotePackageSection { hdr } + } +} + +impl Default for NotePackageSection { + fn default() -> Self { + Self::new() + } +} + +pub mod note_package { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + if !ctx.args.package_metadata.is_empty() { + // +17 for the header and the NUL terminator + ctx.note_package.hdr.shdr.sh_size = + align_to(ctx.args.package_metadata.len() as u64 + 17, 4); + } + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf.fill(0); + E::Endian::write_u32(buf, 4); + E::Endian::write_u32(&mut buf[4..], ctx.note_package.hdr.shdr.sh_size as u32 - 16); + E::Endian::write_u32(&mut buf[8..], NT_FDO_PACKAGING_METADATA); + buf[12..16].copy_from_slice(b"FDO\0"); + write_cstr(&mut buf[16..], ctx.args.package_metadata.as_bytes()); + } +} + +/// `.note.gnu.property` merges the input files' properties, such as CET +/// features on x86. +#[derive(Debug)] +pub struct NotePropertySection { + pub hdr: ChunkHeader, + pub contents: Vec<(u32, u32)>, +} + +impl NotePropertySection { + pub fn new() -> NotePropertySection { + let mut hdr = ChunkHeader::new(".note.gnu.property", SHT_NOTE, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + NotePropertySection { + hdr, + contents: Vec::new(), + } + } +} + +pub mod note_property { + use super::*; + + fn entry_size() -> usize { + if E::IS_64 { + 16 + } else { + 12 + } + } + + pub fn update_shdr(ctx: &mut Context) { + let files: Vec<&crate::input_files::ObjectFile> = ctx + .objs + .iter() + .enumerate() + .filter(|&(i, _)| !ctx.is_internal(crate::input_files::ObjId(i as u32))) + .map(|(_, f)| f.as_ref()) + .collect(); + let keys: BTreeSet = files + .iter() + .flat_map(|f| f.gnu_properties.keys().copied()) + .collect(); + let value = |f: &crate::input_files::ObjectFile, key: u32| { + f.gnu_properties.get(&key).copied().unwrap_or(0) + }; + + let mut map: BTreeMap = BTreeMap::new(); + for key in keys { + if (GNU_PROPERTY_X86_UINT32_AND_LO..=GNU_PROPERTY_X86_UINT32_AND_HI).contains(&key) { + // An AND feature is set if all inputs have it. + map.insert( + key, + files.iter().fold(u32::MAX, |acc, f| acc & value(f, key)), + ); + } else if (GNU_PROPERTY_X86_UINT32_OR_LO..=GNU_PROPERTY_X86_UINT32_OR_HI).contains(&key) + { + // An OR feature is set if some input has it. + map.insert(key, files.iter().fold(0, |acc, f| acc | value(f, key))); + } else if (GNU_PROPERTY_X86_UINT32_OR_AND_LO..=GNU_PROPERTY_X86_UINT32_OR_AND_HI) + .contains(&key) + { + // An OR-AND feature is set if all inputs have the property + // and some have the feature. + if files.iter().all(|f| f.gnu_properties.contains_key(&key)) { + map.insert(key, files.iter().fold(0, |acc, f| acc | value(f, key))); + } + } + } + + if ctx.args.z_ibt { + *map.entry(GNU_PROPERTY_X86_FEATURE_1_AND).or_insert(0) |= + GNU_PROPERTY_X86_FEATURE_1_IBT; + } + if ctx.args.z_shstk { + *map.entry(GNU_PROPERTY_X86_FEATURE_1_AND).or_insert(0) |= + GNU_PROPERTY_X86_FEATURE_1_SHSTK; + } + *map.entry(GNU_PROPERTY_X86_ISA_1_NEEDED).or_insert(0) |= ctx.args.z_x86_64_isa_level; + + let contents: Vec<(u32, u32)> = map.into_iter().filter(|&(_, v)| v != 0).collect(); + let sec = ctx.note_property.as_mut().unwrap(); + sec.hdr.shdr.sh_size = if contents.is_empty() { + 0 + } else { + (16 + contents.len() * entry_size::()) as u64 + }; + sec.contents = contents; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let sec = ctx.note_property.as_ref().unwrap(); + buf.fill(0); + E::Endian::write_u32(buf, 4); + E::Endian::write_u32(&mut buf[4..], sec.hdr.shdr.sh_size as u32 - 16); + E::Endian::write_u32(&mut buf[8..], NT_GNU_PROPERTY_TYPE_0); + buf[12..16].copy_from_slice(b"GNU\0"); + for (i, &(ty, val)) in sec.contents.iter().enumerate() { + let off = 16 + i * entry_size::(); + E::Endian::write_u32(&mut buf[off..], ty); + E::Endian::write_u32(&mut buf[off + 4..], 4); + E::Endian::write_u32(&mut buf[off + 8..], val); + } + } +} + +/// `.gnu_debuglink` names the separate debug file and its CRC32. +#[derive(Debug)] +pub struct GnuDebuglinkSection { + pub hdr: ChunkHeader, + pub filename: String, + pub crc32: u32, +} + +impl GnuDebuglinkSection { + pub fn new() -> GnuDebuglinkSection { + let mut hdr = ChunkHeader::new(".gnu_debuglink", SHT_PROGBITS, 0); + hdr.shdr.sh_addralign = 4; + GnuDebuglinkSection { + hdr, + filename: String::new(), + crc32: 0, + } + } +} + +impl Default for GnuDebuglinkSection { + fn default() -> Self { + Self::new() + } +} + +pub mod gnu_debuglink { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let filename = path_filename(&ctx.args.separate_debug_file); + let sec = ctx.gnu_debuglink.as_mut().unwrap(); + sec.hdr.shdr.sh_size = align_to(filename.len() as u64 + 1, 4) + 4; + sec.filename = filename; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let sec = ctx.gnu_debuglink.as_ref().unwrap(); + buf.fill(0); + write_cstr(buf, sec.filename.as_bytes()); + let n = buf.len(); + E::Endian::write_u32(&mut buf[n - 4..], sec.crc32); + } +} + +/// Pads a PT_GNU_RELRO segment to a page boundary. Leaving the trailing +/// part of the segment unused would work, but `strip` would delete it. +#[derive(Debug)] +pub struct RelroPaddingSection { + pub hdr: ChunkHeader, +} + +impl RelroPaddingSection { + pub fn new() -> RelroPaddingSection { + let mut hdr = + ChunkHeader::new(".relro_padding", SHT_NOBITS, (SHF_ALLOC | SHF_WRITE) as u64); + hdr.is_relro = true; + hdr.shdr.sh_size = 1; + RelroPaddingSection { hdr } + } +} + +impl Default for RelroPaddingSection { + fn default() -> Self { + Self::new() + } +} + +/// A debug section compressed with zlib or zstd. +#[derive(Debug)] +pub struct CompressedSection { + pub hdr: ChunkHeader, + pub chdr: ElfChdr, + pub compressor: Compressor, + /// Kept for --gdb-index, which reads the uncompressed contents. + pub uncompressed_data: Option>, + pub original: ChunkId, +} + +impl std::fmt::Debug for Compressor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Compressor({} bytes)", self.compressed_size()) + } +} + +pub mod compressed { + use super::*; + + pub fn new(ctx: &Context, original: ChunkId) -> CompressedSection { + let hdr = ctx.chunk_header(original); + let mut buf = vec![0u8; hdr.shdr.sh_size as usize]; + chunks::write_to(ctx, original, &mut buf); + + let level = ctx.args.compress_debug_sections_level; + let compressor = if ctx.args.compress_debug_sections == ELFCOMPRESS_ZLIB { + Compressor::zlib(&buf, level as u32) + } else { + Compressor::zstd(&buf, level as i32) + }; + + let chdr = ElfChdr { + ch_type: ctx.args.compress_debug_sections, + ch_size: hdr.shdr.sh_size, + ch_addralign: hdr.shdr.sh_addralign, + }; + let mut new_hdr = ChunkHeader::with_name( + hdr.name, + hdr.shdr.sh_type, + hdr.shdr.sh_flags | SHF_COMPRESSED as u64, + ); + new_hdr.shndx = hdr.shndx; + new_hdr.is_compressed = true; + new_hdr.shdr = hdr.shdr; + new_hdr.shdr.sh_flags |= SHF_COMPRESSED as u64; + new_hdr.shdr.sh_addralign = 1; + new_hdr.shdr.sh_size = (ElfChdr::size::() + compressor.compressed_size()) as u64; + + CompressedSection { + hdr: new_hdr, + chdr, + compressor, + uncompressed_data: ctx.args.gdb_index.then_some(buf), + original, + } + } + + pub fn copy_buf(ctx: &Context, i: u32, buf: &mut [u8]) { + let sec = &ctx.compressed_sections[i as usize]; + sec.chdr.write::(buf); + sec.compressor.write_to(&mut buf[ElfChdr::size::()..]); + } +} + +/// A relocation table for an output section, for `-r` and `--emit-relocs`. +#[derive(Debug)] +pub struct RelocSection { + pub hdr: ChunkHeader, + pub output_section: OutputSectionId, + /// The index of the first relocation of each member. + offsets: Vec, +} + +pub mod reloc { + use super::*; + + pub fn new(ctx: &Context, osec_id: OutputSectionId) -> RelocSection { + let osec = &ctx.output_sections[osec_id.index()]; + let name = format!( + "{}{}", + if E::IS_RELA { ".rela" } else { ".rel" }, + osec.hdr.name + ); + let name = BStr::new(crate::util::leak_bytes(name.into_bytes())); + let mut hdr = ChunkHeader::with_name( + name, + if E::IS_RELA { SHT_RELA } else { SHT_REL }, + SHF_INFO_LINK as u64, + ); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + hdr.shdr.sh_entsize = ElfRel::size::() as u64; + + let mut offsets = Vec::with_capacity(osec.members.len()); + let mut sum = 0u64; + for &m in &osec.members { + offsets.push(sum); + let isec = ctx.input_section(m); + let file = &ctx.objs[isec.file.index()]; + sum += isec.rels::(file).len() as u64; + } + hdr.shdr.sh_size = sum * ElfRel::size::() as u64; + RelocSection { + hdr, + output_section: osec_id, + offsets, + } + } + + pub fn update_shdr(ctx: &mut Context, i: u32) { + let symtab_shndx = ctx.symtab.hdr.shndx; + let osec = ctx.reloc_sections[i as usize].output_section; + let osec_shndx = ctx.output_sections[osec.index()].hdr.shndx; + let sec = &mut ctx.reloc_sections[i as usize]; + sec.hdr.shdr.sh_link = symtab_shndx; + sec.hdr.shdr.sh_info = osec_shndx; + } + + /// Translates a relocation's symbol reference into the {r_sym, addend} + /// pair valid in the output: either an output section index or an + /// output symbol table index. + fn symidx_addend(ctx: &Context, isec: &InputSection, rel: &ElfRel) -> (u32, i64) { + let file = &ctx.objs[isec.file.index()]; + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + + if !isec.is_alloc() { + if let Some((frag, addend)) = isec.fragment(ctx, rel) { + let msec = &ctx.merged_sections[frag.section.index()]; + return ( + msec.hdr.shndx, + msec.fragments.get(frag.entry).offset() as i64 + addend, + ); + } + } + + if sym.st_type() == STT_SECTION { + if let Some(frag) = sym.fragment() { + let msec = &ctx.merged_sections[frag.section.index()]; + return ( + msec.hdr.shndx, + msec.fragments.get(frag.entry).offset() as i64 + + sym.value as i64 + + isec.rel_addend::(rel), + ); + } + if let Some(target) = sym.input_section_ref() { + if let Some(osec) = target.output_section { + return ( + ctx.output_section(osec).hdr.shndx, + isec.rel_addend::(rel) + target.offset() as i64, + ); + } + } + // A dead debug section referring to a COMDAT-eliminated section. + return (0, 0); + } + + if sym.write_to_symtab() { + return (sym.output_sym_idx(ctx), isec.rel_addend::(rel)); + } + (0, 0) + } + + /// Writes the relocations. With `-r` on a REL target, the addends are + /// written into the output section's bytes, passed as `osec_buf`. + pub fn copy_buf( + ctx: &Context, + i: u32, + buf: &mut [u8], + osec_buf: Option<&mut [u8]>, + ) { + let sec = &ctx.reloc_sections[i as usize]; + let osec = &ctx.output_sections[sec.output_section.index()]; + let size = ElfRel::size::(); + let mut osec_buf = osec_buf; + + for (mi, &m) in osec.members.iter().enumerate() { + let isec = ctx.input_section(m); + let file = &ctx.objs[isec.file.index()]; + let base = sec.offsets[mi] as usize; + for (j, rel) in isec.rels::(file).iter().enumerate() { + let (symidx, addend) = symidx_addend(ctx, isec, &rel); + let mut r_offset = osec.hdr.shdr.sh_addr + isec.offset() + rel.r_offset; + if E::IS_RISCV || E::IS_LOONGARCH { + r_offset -= r_delta(isec, rel.r_offset) as u64; + } + // SH4 stores addends in the relocated places, and the + // records we emit follow that convention. + let out_addend = if E::FAMILY == crate::arch::Family::Sh4 { + 0 + } else { + addend + }; + let r_type = crate::arch::emitted_rel_type::(ctx, isec, &rel, j); + ElfRel::new(r_offset, r_type, symidx, out_addend) + .write::(&mut buf[(base + j) * size..]); + + if ctx.args.relocatable { + if let Some(osec_buf) = osec_buf.as_deref_mut() { + let loc = (isec.offset() + rel.r_offset) as usize; + E::write_addend(&mut osec_buf[loc..], addend, &rel); + } + } + } + } + } +} + +/// A COMDAT group in a relocatable output. +#[derive(Debug)] +pub struct ComdatGroupSection { + pub hdr: ChunkHeader, + pub sym: SymbolId, + pub members: Vec, +} + +impl ComdatGroupSection { + pub fn new(sym: SymbolId, members: Vec) -> ComdatGroupSection { + let mut hdr = ChunkHeader::new(".group", SHT_GROUP, 0); + hdr.shdr.sh_entsize = 4; + hdr.shdr.sh_addralign = 4; + hdr.shdr.sh_size = (members.len() * 4 + 4) as u64; + ComdatGroupSection { hdr, sym, members } + } +} + +pub mod comdat_group { + use super::*; + + pub fn update_shdr(ctx: &mut Context, i: u32) { + debug_assert!(ctx.args.relocatable); + let sec = &ctx.comdat_group_sections[i as usize]; + let sym = &ctx.symbols[sec.sym]; + let sh_info = if sym.st_type() == STT_SECTION { + let isec = sym.input_section_ref().unwrap(); + ctx.output_section(isec.output_section.unwrap()).hdr.shndx + } else { + sym.output_sym_idx(ctx) + }; + let symtab_shndx = ctx.symtab.hdr.shndx; + let sec = &mut ctx.comdat_group_sections[i as usize]; + sec.hdr.shdr.sh_link = symtab_shndx; + sec.hdr.shdr.sh_info = sh_info; + } + + pub fn copy_buf(ctx: &Context, i: u32, buf: &mut [u8]) { + let sec = &ctx.comdat_group_sections[i as usize]; + E::Endian::write_u32(buf, GRP_COMDAT); + for (j, &member) in sec.members.iter().enumerate() { + E::Endian::write_u32(&mut buf[4 + j * 4..], ctx.chunk_header(member).shndx); + } + } +} + +/// Runs a function over the output sections in parallel. +pub fn for_each_output_section(ctx: &Context, f: impl Fn(OutputSectionId) + Sync) { + (0..ctx.output_sections.len()) + .into_par_iter() + .for_each(|i| f(OutputSectionId::new(i as u32))); +} + +/// `.riscv.attributes` describes the ISA the output requires, merged +/// from the input files' attributes. +#[derive(Debug)] +pub struct RiscvAttributesSection { + pub hdr: ChunkHeader, + pub contents: Vec, +} + +impl RiscvAttributesSection { + pub fn new() -> RiscvAttributesSection { + RiscvAttributesSection { + hdr: ChunkHeader::new(".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0), + contents: Vec::new(), + } + } +} + +impl Default for RiscvAttributesSection { + fn default() -> Self { + Self::new() + } +} + +pub mod riscv_attributes { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + if !ctx.riscv_attributes.as_ref().unwrap().contents.is_empty() { + return; + } + let contents = crate::arch::riscv::attributes_contents(ctx); + let sec = ctx.riscv_attributes.as_mut().unwrap(); + sec.hdr.shdr.sh_size = contents.len() as u64; + sec.contents = contents; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let sec = ctx.riscv_attributes.as_ref().unwrap(); + buf[..sec.contents.len()].copy_from_slice(&sec.contents); + } +} + +/// `.save_restore_regs`, the register save and restore routines that GCC +/// expects the linker to provide on PowerPC64 ELFv2. +#[derive(Debug)] +pub struct Ppc64SaveRestoreSection { + pub hdr: ChunkHeader, +} + +impl Ppc64SaveRestoreSection { + pub fn new() -> Ppc64SaveRestoreSection { + let mut hdr = ChunkHeader::new( + ".save_restore_regs", + SHT_PROGBITS, + (SHF_ALLOC | SHF_EXECINSTR) as u64, + ); + hdr.shdr.sh_addralign = 16; + hdr.shdr.sh_size = (crate::arch::ppc64v2::SAVE_RESTORE_INSNS.len() * 4) as u64; + Ppc64SaveRestoreSection { hdr } + } +} + +impl Default for Ppc64SaveRestoreSection { + fn default() -> Self { + Self::new() + } +} + +pub mod ppc64_save_restore { + use super::*; + + pub fn copy_buf(_ctx: &Context, buf: &mut [u8]) { + buf.copy_from_slice(&crate::arch::ppc64v2::save_restore_contents()); + } +} diff --git a/src/chunks/mod.rs b/src/chunks/mod.rs new file mode 100644 index 000000000..12d319350 --- /dev/null +++ b/src/chunks/mod.rs @@ -0,0 +1,905 @@ +//! Output chunks: the contiguous regions that make up the output file. +//! +//! Besides the output sections built from input sections, the linker +//! synthesizes many sections of its own (`.got`, `.plt`, `.dynamic`, +//! `.symtab` and so on). Every such section has a [`ChunkHeader`] holding +//! its section header and bookkeeping, and is addressed by a [`ChunkId`]. +//! Operations common to all chunks dispatch on the id. + +pub mod arm_exidx; +pub mod dynamic; +pub mod eh_frame; +pub mod got; +pub mod merged; +pub mod misc; +pub mod opd; +pub mod output_section; +pub mod sframe; +pub mod symtab; +pub mod version; + +use std::marker::PhantomData; +use std::num::NonZeroU32; + +use bstr::BStr; + +use crate::arch::Arch; +use crate::args::SectionOrderKind; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::{FileId, SymtabBlock}; +use crate::symbol::AddrFlags; +use crate::tls; +use crate::{error, warn}; + +pub use merged::MergedSectionId; + +/// Index of an output section in `Context::output_sections`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct OutputSectionId(NonZeroU32); + +impl OutputSectionId { + #[inline] + pub fn new(index: u32) -> OutputSectionId { + let encoded = index.checked_add(1).expect("too many output sections"); + OutputSectionId(NonZeroU32::new(encoded).unwrap()) + } + + #[inline] + pub fn index(self) -> usize { + (self.0.get() - 1) as usize + } +} + +/// Identifies a chunk of the output file. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ChunkId { + Ehdr, + Phdr, + Shdr, + Interp, + Got, + GotPlt, + RelPlt, + RelDyn, + RelrDyn, + Dynamic, + Strtab, + Dynstr, + Hash, + GnuHash, + GnuDebuglink, + Shstrtab, + Plt, + PltGot, + Symtab, + SymtabShndx, + Dynsym, + EhFrame, + EhFrameHdr, + EhFrameReloc, + SFrame, + SFrameReloc, + Copyrel, + CopyrelRelro, + Versym, + Verneed, + Verdef, + BuildId, + NotePackage, + NoteProperty, + RiscvAttributes, + ArmExidx, + Ppc64SaveRestore, + Ppc64Opd, + GdbIndex, + RelroPadding, + Output(OutputSectionId), + Merged(MergedSectionId), + Reloc(u32), + ComdatGroup(u32), + Compressed(u32), + /// A section of the main output that a separate debug file lists + /// without contents. + Placeholder(u32), +} + +impl ChunkId { + /// Whether the chunk is one of the ELF headers rather than a section. + pub fn is_header(self) -> bool { + matches!(self, ChunkId::Ehdr | ChunkId::Phdr | ChunkId::Shdr) + } + + pub fn as_output_section(self) -> Option { + match self { + ChunkId::Output(id) => Some(id), + _ => None, + } + } +} + +/// State shared by all chunks. +#[derive(Debug)] +pub struct ChunkHeader { + pub name: &'static BStr, + pub shdr: ElfShdr, + + /// Index in the output section header table; 0 for headers. + pub shndx: u32, + + pub num_dynrels: u64, + pub num_relrs: u64, + pub relr: Vec, + pub is_relro: bool, + + /// For --gdb-index + pub is_compressed: bool, + + // Synthesized local symbols, e.g. `foo$got` or thunk labels. + pub local_symtab_idx: u32, + pub num_local_symtab: u32, + pub strtab_size: u64, + pub strtab_offset: u64, + + /// For --section-order + pub sect_order: i64, +} + +impl ChunkHeader { + pub fn new(name: &'static str, sh_type: u32, sh_flags: u64) -> ChunkHeader { + ChunkHeader { + name: BStr::new(name.as_bytes()), + shdr: ElfShdr { + sh_type, + sh_flags, + sh_addralign: 1, + ..ElfShdr::default() + }, + shndx: 0, + num_dynrels: 0, + num_relrs: 0, + relr: Vec::new(), + is_relro: false, + is_compressed: false, + local_symtab_idx: 0, + num_local_symtab: 0, + strtab_size: 0, + strtab_offset: 0, + sect_order: 0, + } + } + + pub fn with_name(name: &'static BStr, sh_type: u32, sh_flags: u64) -> ChunkHeader { + ChunkHeader { + name, + ..ChunkHeader::new("", sh_type, sh_flags) + } + } + + pub fn is_alloc(&self) -> bool { + self.shdr.sh_flags & SHF_ALLOC as u64 != 0 + } + + pub fn is_nobits(&self) -> bool { + self.shdr.sh_type == SHT_NOBITS + } +} + +/// The ELF file header. +#[derive(Debug)] +pub struct OutputEhdr { + pub hdr: ChunkHeader, +} + +impl OutputEhdr { + pub fn new(sh_flags: u64) -> OutputEhdr { + let mut hdr = ChunkHeader::new("EHDR", 0, sh_flags); + hdr.shdr.sh_size = ElfEhdr::size::() as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + OutputEhdr { hdr } + } +} + +/// The section header table, usually at the end of the file. It is not +/// needed at runtime; only the program header is. +#[derive(Debug)] +pub struct OutputShdr { + pub hdr: ChunkHeader, +} + +impl OutputShdr { + pub fn new() -> OutputShdr { + let mut hdr = ChunkHeader::new("SHDR", 0, 0); + hdr.shdr.sh_size = 1; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + OutputShdr { hdr } + } +} + +/// The program header, describing the segments the kernel maps. +#[derive(Debug)] +pub struct OutputPhdr { + pub hdr: ChunkHeader, + pub phdrs: Vec, +} + +impl OutputPhdr { + pub fn new(sh_flags: u64) -> OutputPhdr { + let mut hdr = ChunkHeader::new("PHDR", 0, sh_flags); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + OutputPhdr { + hdr, + phdrs: Vec::new(), + } + } +} + +/// `.gdb_index`, built after everything else has been written. +#[derive(Debug)] +pub struct GdbIndexSection { + pub hdr: ChunkHeader, +} + +impl GdbIndexSection { + pub fn new() -> GdbIndexSection { + let mut hdr = ChunkHeader::new(".gdb_index", SHT_PROGBITS, 0); + hdr.shdr.sh_addralign = 4; + GdbIndexSection { hdr } + } +} + +impl Default for GdbIndexSection { + fn default() -> Self { + Self::new() + } +} + +fn entry_addr(ctx: &Context) -> u64 { + if ctx.args.relocatable { + return 0; + } + let sym = &ctx.symbols[ctx.syms.entry]; + if let Some(FileId::Obj(_)) = sym.file() { + return sym.addr(ctx); + } + if !ctx.args.shared { + warn!(ctx, "entry symbol is not defined: {sym}"); + } + 0 +} + +fn write_ehdr(ctx: &Context, buf: &mut [u8]) { + let mut ehdr = ElfEhdr::default(); + ehdr.e_ident[..4].copy_from_slice(b"\x7fELF"); + ehdr.e_ident[EI_CLASS as usize] = if E::IS_64 { ELFCLASS64 } else { ELFCLASS32 } as u8; + ehdr.e_ident[EI_DATA as usize] = if E::IS_LITTLE_ENDIAN { + ELFDATA2LSB + } else { + ELFDATA2MSB + } as u8; + ehdr.e_ident[EI_VERSION as usize] = EV_CURRENT as u8; + ehdr.e_machine = E::E_MACHINE as u16; + ehdr.e_version = EV_CURRENT; + ehdr.e_entry = entry_addr(ctx); + ehdr.e_flags = E::eflags(ctx); + ehdr.e_ehsize = ElfEhdr::size::() as u16; + + // If e_shstrndx is too large, the real value goes to the zeroth + // section's sh_link. + if let Some(shstrtab) = &ctx.shstrtab { + ehdr.e_shstrndx = if shstrtab.hdr.shndx < SHN_LORESERVE { + shstrtab.hdr.shndx as u16 + } else { + SHN_XINDEX as u16 + }; + } + + ehdr.e_type = if ctx.args.relocatable { + ET_REL + } else if ctx.args.pie && ctx.args.ttext_segment.is_some() { + ET_EXEC + } else if ctx.args.pic { + ET_DYN + } else { + ET_EXEC + } as u16; + + if let Some(phdr) = &ctx.phdr { + ehdr.e_phoff = phdr.hdr.shdr.sh_offset; + ehdr.e_phentsize = ElfPhdr::size::() as u16; + ehdr.e_phnum = (phdr.hdr.shdr.sh_size / ElfPhdr::size::() as u64) as u16; + } + + if let Some(shdr) = &ctx.shdr { + ehdr.e_shoff = shdr.hdr.shdr.sh_offset; + ehdr.e_shentsize = ElfShdr::size::() as u16; + // e_shnum is 16 bits; a larger count is stored in the zeroth + // section's sh_size. + let shnum = shdr.hdr.shdr.sh_size / ElfShdr::size::() as u64; + ehdr.e_shnum = if shnum <= u16::MAX as u64 { + shnum as u16 + } else { + 0 + }; + } + + ehdr.write::(buf); +} + +fn write_shdr(ctx: &Context, buf: &mut [u8]) { + let size = ElfShdr::size::(); + buf.fill(0); + + let mut first = ElfShdr::default(); + if let Some(shstrtab) = &ctx.shstrtab { + if shstrtab.hdr.shndx >= SHN_LORESERVE { + first.sh_link = shstrtab.hdr.shndx; + } + } + let shnum = buf.len() / size; + if shnum > u16::MAX as usize { + first.sh_size = shnum as u64; + } + first.write::(buf); + + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + if hdr.shndx != 0 { + hdr.shdr.write::(&mut buf[hdr.shndx as usize * size..]); + } + } +} + +/// The segment flags a chunk requires. +pub fn to_phdr_flags(ctx: &Context, id: ChunkId) -> u32 { + // All sections are put into a single RWX segment if --omagic + if ctx.args.omagic { + return PF_R | PF_W | PF_X; + } + + let hdr = ctx.chunk_header(id); + let write = hdr.shdr.sh_flags & SHF_WRITE as u64 != 0; + let mut exec = hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0; + + // .text is not readable if --execute-only + if exec && ctx.args.execute_only { + if write { + error!( + ctx, + "--execute-only is not compatible with writable section: {}", hdr.name + ); + } + return PF_X; + } + + // .rodata is merged with .text if --no-rosegment + if !write && !ctx.args.rosegment { + exec = true; + } + + PF_R | if write { PF_W } else { 0 } | if exec { PF_X } else { 0 } +} + +fn create_phdr(ctx: &Context) -> Vec { + let mut vec: Vec = Vec::new(); + + let define = |vec: &mut Vec, p_type: u32, flags: u32, id: ChunkId| { + let shdr = &ctx.chunk_header(id).shdr; + let mut phdr = ElfPhdr { + p_type, + p_flags: flags, + p_align: shdr.sh_addralign, + ..ElfPhdr::default() + }; + if shdr.sh_type == SHT_NOBITS { + // p_offset is not significant for a segment with no file + // contents, but some loaders want it congruent with the + // virtual address modulo the page size. + phdr.p_offset = shdr.sh_addr % ctx.page_size; + } else { + phdr.p_offset = shdr.sh_offset; + phdr.p_filesz = shdr.sh_size; + } + phdr.p_vaddr = shdr.sh_addr; + phdr.p_paddr = shdr.sh_addr; + if shdr.sh_flags & SHF_ALLOC as u64 != 0 { + phdr.p_memsz = shdr.sh_size; + } + vec.push(phdr); + }; + + let append = |vec: &mut Vec, id: ChunkId| { + let shdr = &ctx.chunk_header(id).shdr; + let phdr = vec.last_mut().unwrap(); + phdr.p_align = phdr.p_align.max(shdr.sh_addralign); + phdr.p_memsz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr; + if shdr.sh_type != SHT_NOBITS { + phdr.p_filesz = phdr.p_memsz; + } + }; + + let is_bss = |id: ChunkId| ctx.chunk_header(id).shdr.sh_type == SHT_NOBITS; + let is_tbss = |id: ChunkId| { + let shdr = &ctx.chunk_header(id).shdr; + shdr.sh_type == SHT_NOBITS && shdr.sh_flags & SHF_TLS as u64 != 0 + }; + let is_note = |id: ChunkId| ctx.chunk_header(id).shdr.sh_type == SHT_NOTE; + + // Only these chunks are considered when creating PT_LOAD segments. + let mut chunks: Vec = ctx + .chunks + .iter() + .copied() + .filter(|&id| ctx.chunk_header(id).is_alloc() && !is_tbss(id)) + .collect(); + + // The ELF spec requires PT_LOAD entries to be sorted by p_vaddr. + chunks.sort_by_key(|&id| ctx.chunk_header(id).shdr.sh_addr); + + // PT_PHDR for the program header itself. + if let Some(phdr) = &ctx.phdr { + if phdr.hdr.is_alloc() { + define(&mut vec, PT_PHDR, PF_R, ChunkId::Phdr); + } + } + + if ctx.interp.is_some() { + define(&mut vec, PT_INTERP, PF_R, ChunkId::Interp); + } + + // PT_NOTE for note sections. + let mut i = 0; + while i < chunks.len() { + let first = chunks[i]; + i += 1; + if is_note(first) { + let flags = to_phdr_flags(ctx, first); + define(&mut vec, PT_NOTE, flags, first); + while i < chunks.len() && is_note(chunks[i]) && to_phdr_flags(ctx, chunks[i]) == flags { + append(&mut vec, chunks[i]); + i += 1; + } + } + } + + // PT_LOAD segments. + let mut i = 0; + while i < chunks.len() { + let first = chunks[i]; + i += 1; + let flags = to_phdr_flags(ctx, first); + define(&mut vec, PT_LOAD, flags, first); + if !ctx.args.nmagic && !ctx.args.omagic { + let last = vec.last_mut().unwrap(); + last.p_align = last.p_align.max(ctx.page_size); + } + + // Add contiguous ALLOC sections as long as they have the same + // section flags and there's no on-disk gap in between. + if !is_bss(first) { + let first_shdr = ctx.chunk_header(first).shdr; + while i < chunks.len() + && !is_bss(chunks[i]) + && to_phdr_flags(ctx, chunks[i]) == flags + && { + let shdr = &ctx.chunk_header(chunks[i]).shdr; + shdr.sh_offset.wrapping_sub(first_shdr.sh_offset) + == shdr.sh_addr.wrapping_sub(first_shdr.sh_addr) + } + { + append(&mut vec, chunks[i]); + i += 1; + } + } + while i < chunks.len() && is_bss(chunks[i]) && to_phdr_flags(ctx, chunks[i]) == flags { + append(&mut vec, chunks[i]); + i += 1; + } + } + + // PT_TLS + let is_tls = |id: ChunkId| ctx.chunk_header(id).shdr.sh_flags & SHF_TLS as u64 != 0; + let mut i = 0; + while i < ctx.chunks.len() { + let first = ctx.chunks[i]; + i += 1; + if is_tls(first) { + define(&mut vec, PT_TLS, PF_R, first); + while i < ctx.chunks.len() && is_tls(ctx.chunks[i]) { + append(&mut vec, ctx.chunks[i]); + i += 1; + } + } + } + + if let Some(dynamic) = &ctx.dynamic { + if dynamic.hdr.shdr.sh_size != 0 { + let flags = to_phdr_flags(ctx, ChunkId::Dynamic); + define(&mut vec, PT_DYNAMIC, flags, ChunkId::Dynamic); + } + } + + if ctx.eh_frame_hdr.is_some() { + define(&mut vec, PT_GNU_EH_FRAME, PF_R, ChunkId::EhFrameHdr); + } + + if ctx.sframe.hdr.shdr.sh_size != 0 && ctx.chunks.contains(&ChunkId::SFrame) { + define(&mut vec, PT_GNU_SFRAME, PF_R, ChunkId::SFrame); + } + + if let Some(id) = ctx.find_chunk_by_name(b".note.gnu.property") { + define(&mut vec, PT_GNU_PROPERTY, PF_R, id); + } + + if ctx + .riscv_attributes + .as_ref() + .is_some_and(|sec| sec.hdr.shdr.sh_size != 0) + { + define( + &mut vec, + PT_RISCV_ATTRIBUTES, + PF_R, + ChunkId::RiscvAttributes, + ); + } + + if ctx.arm_exidx.is_some() { + define(&mut vec, PT_ARM_EXIDX, PF_R, ChunkId::ArmExidx); + } + + // PT_GNU_STACK is a marker segment controlling the executable bit of + // the stack area. + vec.push(ElfPhdr { + p_type: PT_GNU_STACK, + p_flags: if ctx.args.z_execstack { + PF_R | PF_W | PF_X + } else { + PF_R | PF_W + }, + p_memsz: ctx.args.z_stack_size, + p_align: 1, + ..ElfPhdr::default() + }); + + // PT_GNU_RELRO + if ctx.args.z_relro { + let mut i = 0; + while i < chunks.len() { + let first = chunks[i]; + i += 1; + if ctx.chunk_header(first).is_relro { + define(&mut vec, PT_GNU_RELRO, PF_R, first); + while i < chunks.len() && ctx.chunk_header(chunks[i]).is_relro { + append(&mut vec, chunks[i]); + i += 1; + } + vec.last_mut().unwrap().p_align = 1; + } + } + } + + for &id in &ctx.chunks { + if ctx.chunk_header(id).name == b".openbsd.randomdata" { + define(&mut vec, PT_OPENBSD_RANDOMIZE, PF_R | PF_W, id); + } + } + + // --physical-image-base sets p_paddr for embedded programs whose + // segments start out in ROM. We keep vaddr == paddr for as many + // segments as possible so that they can be used in place, but give + // up once a gap between segments is two pages or larger. + if let Some(base) = ctx.args.physical_image_base { + if let Some(first) = vec.iter().position(|p| p.p_type == PT_LOAD) { + let mut addr = base; + let mut in_sync = vec[first].p_vaddr == addr; + vec[first].p_paddr = addr; + addr += vec[first].p_memsz; + + for p in vec[first + 1..] + .iter_mut() + .take_while(|p| p.p_type == PT_LOAD) + { + if in_sync && addr <= p.p_vaddr && p.p_vaddr < addr + ctx.page_size * 2 { + p.p_paddr = p.p_vaddr; + addr = p.p_vaddr + p.p_memsz; + } else { + in_sync = false; + p.p_paddr = addr; + addr += p.p_memsz; + } + } + } + } + + vec.resize( + vec.len() + ctx.args.spare_program_headers.max(0) as usize, + ElfPhdr::default(), + ); + vec +} + +/// Recomputes the program header and the TLS layout constants. +pub fn update_phdr(ctx: &mut Context) { + if ctx.phdr.is_none() { + return; + } + let phdrs = create_phdr(ctx); + for phdr in &phdrs { + if phdr.p_type == PT_TLS { + ctx.tls_begin = phdr.p_vaddr; + ctx.tp_addr = tls::tp_addr::(phdr); + ctx.dtp_addr = tls::dtp_addr::(phdr); + break; + } + } + let phdr = ctx.phdr.as_mut().unwrap(); + phdr.hdr.shdr.sh_size = (phdrs.len() * ElfPhdr::size::()) as u64; + phdr.phdrs = phdrs; +} + +/// Updates a chunk's section header for the current layout. Called at +/// least twice: once to size the section, and again after section +/// indices are known. +pub fn update_shdr(ctx: &mut Context, id: ChunkId) { + match id { + ChunkId::Phdr => update_phdr(ctx), + ChunkId::Interp => misc::interp::update_shdr(ctx), + ChunkId::GotPlt => got::gotplt::update_shdr(ctx), + ChunkId::RelPlt => got::relplt::update_shdr(ctx), + ChunkId::RelDyn => dynamic::reldyn::update_shdr(ctx), + ChunkId::Dynamic => dynamic::dynamic::update_shdr(ctx), + ChunkId::Strtab => symtab::strtab::update_shdr(ctx), + ChunkId::Shstrtab => symtab::shstrtab::update_shdr(ctx), + ChunkId::Plt => got::plt::update_shdr(ctx), + ChunkId::Symtab => symtab::symtab::update_shdr(ctx), + ChunkId::Dynsym => symtab::dynsym::update_shdr(ctx), + ChunkId::Hash => symtab::hash::update_shdr(ctx), + ChunkId::GnuHash => symtab::gnu_hash::update_shdr(ctx), + ChunkId::EhFrameHdr => eh_frame::eh_frame_hdr::update_shdr(ctx), + ChunkId::EhFrameReloc => eh_frame::eh_frame_reloc::update_shdr(ctx), + ChunkId::SFrameReloc => sframe::sframe_reloc::update_shdr(ctx), + ChunkId::Versym => version::versym::update_shdr(ctx), + ChunkId::Verneed => version::verneed::update_shdr(ctx), + ChunkId::Verdef => version::verdef::update_shdr(ctx), + ChunkId::BuildId => misc::build_id::update_shdr(ctx), + ChunkId::NotePackage => misc::note_package::update_shdr(ctx), + ChunkId::NoteProperty => misc::note_property::update_shdr(ctx), + ChunkId::RiscvAttributes => misc::riscv_attributes::update_shdr(ctx), + ChunkId::ArmExidx => arm_exidx::update_shdr(ctx), + ChunkId::GnuDebuglink => misc::gnu_debuglink::update_shdr(ctx), + ChunkId::Reloc(i) => misc::reloc::update_shdr(ctx, i), + ChunkId::ComdatGroup(i) => misc::comdat_group::update_shdr(ctx, i), + _ => {} + } +} + +/// Computes a chunk's size from its contents. For output sections this +/// also assigns offsets to the members. +pub fn compute_section_size(ctx: &mut Context, id: ChunkId) { + match id { + ChunkId::Output(id) => output_section::compute_section_size(ctx, id), + ChunkId::Merged(id) => merged::compute_section_size(ctx, id), + ChunkId::ArmExidx => arm_exidx::compute_section_size(ctx), + _ => {} + } +} + +/// The number of dynamic relocations a chunk emits. +pub fn num_dynrels(ctx: &Context, id: ChunkId) -> u64 { + match id { + ChunkId::Output(id) => output_section::num_dynrels(ctx, id), + ChunkId::Got => got::got::num_dynrels(ctx), + ChunkId::Copyrel => ctx.copyrel.symbols.len() as u64, + ChunkId::CopyrelRelro => ctx.copyrel_relro.symbols.len() as u64, + ChunkId::Ppc64Opd => opd::num_dynrels(ctx), + _ => 0, + } +} + +/// The offsets (relative to the chunk) of base relocations that can be +/// encoded in RELR form, marking them as such. +pub fn relr_offsets(ctx: &mut Context, id: ChunkId) -> Vec { + match id { + ChunkId::Output(id) => output_section::relr_offsets(ctx, id), + ChunkId::Got => got::got::relr_offsets(ctx), + ChunkId::Ppc64Opd => opd::relr_offsets(ctx), + _ => Vec::new(), + } +} + +/// Storage for a chunk's dynamic relocations. Native records are used by +/// Android packing and when the target's records have the host layout; +/// other targets are encoded directly into their output bytes. +pub enum DynRelBuffer<'a, E: Arch> { + Native(&'a mut [ElfRel]), + Encoded(&'a mut [u8], PhantomData), +} + +impl<'a, E: Arch> DynRelBuffer<'a, E> { + pub fn native(rels: &'a mut [ElfRel]) -> Self { + DynRelBuffer::Native(rels) + } + + pub fn output(buf: &'a mut [u8]) -> Self { + debug_assert!(buf.len().is_multiple_of(ElfRel::size::())); + + if E::IS_64 + && E::IS_RELA + && E::Endian::IS_NATIVE + && buf.as_ptr().align_offset(std::mem::align_of::()) == 0 + { + let len = buf.len() / std::mem::size_of::(); + // SAFETY: ElfRel has the native ELF64 Rela field layout when the + // target and host have the same byte order, the output section is + // suitably aligned, every bit pattern is valid for its integer + // fields, and `buf` is borrowed exclusively for the returned + // slice's lifetime. + let rels = + unsafe { std::slice::from_raw_parts_mut(buf.as_mut_ptr().cast::(), len) }; + return DynRelBuffer::Native(rels); + } + + DynRelBuffer::Encoded(buf, PhantomData) + } + + #[inline] + pub fn len(&self) -> usize { + match self { + DynRelBuffer::Native(rels) => rels.len(), + DynRelBuffer::Encoded(buf, _) => buf.len() / ElfRel::size::(), + } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn write(&mut self, i: usize, rel: ElfRel) { + match self { + DynRelBuffer::Native(rels) => rels[i] = rel, + DynRelBuffer::Encoded(buf, _) => { + let size = ElfRel::size::(); + rel.write::(&mut buf[i * size..(i + 1) * size]); + } + } + } + + pub fn split_at_offsets(self, offsets: &[u64]) -> Vec { + match self { + DynRelBuffer::Native(rels) => crate::output_file::split_at_offsets(rels, offsets) + .into_iter() + .map(DynRelBuffer::Native) + .collect(), + DynRelBuffer::Encoded(buf, _) => { + let size = ElfRel::size::() as u64; + let offsets: Vec = offsets.iter().map(|&offset| offset * size).collect(); + crate::output_file::split_at_offsets(buf, &offsets) + .into_iter() + .map(|buf| DynRelBuffer::Encoded(buf, PhantomData)) + .collect() + } + } + } +} + +/// Writes a chunk's dynamic relocations to its assigned output slots. +pub fn write_dynrels(ctx: &Context, id: ChunkId, out: DynRelBuffer<'_, E>) { + match id { + ChunkId::Output(id) => output_section::write_dynrels(ctx, id, out), + ChunkId::Got => got::got::write_dynrels(ctx, out), + ChunkId::Copyrel => misc::copyrel::write_dynrels(ctx, &ctx.copyrel, out), + ChunkId::CopyrelRelro => misc::copyrel::write_dynrels(ctx, &ctx.copyrel_relro, out), + ChunkId::Ppc64Opd => opd::write_dynrels(ctx, out), + _ => {} + } +} + +/// Sizes the local symbols a chunk synthesizes. +pub fn compute_symtab_size(ctx: &mut Context, id: ChunkId) { + match id { + ChunkId::Output(id) => output_section::compute_symtab_size(ctx, id), + ChunkId::Got => got::got::compute_symtab_size(ctx), + ChunkId::Plt => got::plt::compute_symtab_size(ctx), + ChunkId::PltGot => got::pltgot::compute_symtab_size(ctx), + _ => {} + } +} + +/// Produces the local symbols a chunk synthesizes. +pub fn populate_symtab(ctx: &Context, id: ChunkId, block: &mut SymtabBlock<'_>) { + match id { + ChunkId::Output(id) => output_section::populate_symtab(ctx, id, block), + ChunkId::Got => got::got::populate_symtab(ctx, block), + ChunkId::Plt => got::plt::populate_symtab(ctx, block), + ChunkId::PltGot => got::pltgot::populate_symtab(ctx, block), + _ => {} + } +} + +/// Writes a chunk's contents into its region of the output file. +/// +/// Chunks whose contents spill into other chunks (`.eh_frame` writes the +/// `.eh_frame_hdr` table, `.symtab` writes `.strtab`) are handled by the +/// output file writer, which hands them the extra buffers. +pub fn copy_buf(ctx: &Context, id: ChunkId, buf: &mut [u8]) { + match id { + ChunkId::Ehdr => write_ehdr(ctx, buf), + ChunkId::Shdr => write_shdr(ctx, buf), + ChunkId::Phdr => { + let phdrs = &ctx.phdr.as_ref().unwrap().phdrs; + ElfPhdr::write_all::(phdrs, buf); + } + ChunkId::Interp => misc::interp::copy_buf(ctx, buf), + ChunkId::Got => got::got::copy_buf(ctx, buf), + ChunkId::GotPlt => got::gotplt::copy_buf(ctx, buf), + ChunkId::RelPlt => got::relplt::copy_buf(ctx, buf), + ChunkId::RelDyn => dynamic::reldyn::copy_buf(ctx, buf), + ChunkId::RelrDyn => dynamic::relrdyn::copy_buf(ctx, buf), + ChunkId::Dynamic => dynamic::dynamic::copy_buf(ctx, buf), + ChunkId::Strtab => symtab::strtab::copy_buf(ctx, buf), + ChunkId::Dynstr => symtab::dynstr::copy_buf(ctx, buf), + ChunkId::Hash => symtab::hash::copy_buf(ctx, buf), + ChunkId::GnuHash => symtab::gnu_hash::copy_buf(ctx, buf), + ChunkId::GnuDebuglink => misc::gnu_debuglink::copy_buf(ctx, buf), + ChunkId::Shstrtab => symtab::shstrtab::copy_buf(ctx, buf), + ChunkId::Plt => got::plt::copy_buf(ctx, buf), + ChunkId::PltGot => got::pltgot::copy_buf(ctx, buf), + ChunkId::Symtab | ChunkId::SymtabShndx => {} + ChunkId::Dynsym => symtab::dynsym::copy_buf(ctx, buf), + ChunkId::EhFrame | ChunkId::EhFrameHdr => {} + ChunkId::EhFrameReloc => {} + ChunkId::SFrame => sframe::copy_buf(ctx, buf), + ChunkId::SFrameReloc => sframe::sframe_reloc::copy_buf(ctx, buf), + ChunkId::Copyrel | ChunkId::CopyrelRelro => {} + ChunkId::Versym => version::versym::copy_buf(ctx, buf), + ChunkId::Verneed => version::verneed::copy_buf(ctx, buf), + ChunkId::Verdef => version::verdef::copy_buf(ctx, buf), + ChunkId::BuildId => misc::build_id::copy_buf(ctx, buf), + ChunkId::NotePackage => misc::note_package::copy_buf(ctx, buf), + ChunkId::NoteProperty => misc::note_property::copy_buf(ctx, buf), + ChunkId::RiscvAttributes => misc::riscv_attributes::copy_buf(ctx, buf), + ChunkId::ArmExidx => arm_exidx::copy_buf(ctx, buf), + ChunkId::Ppc64SaveRestore => misc::ppc64_save_restore::copy_buf(ctx, buf), + ChunkId::Ppc64Opd => opd::copy_buf(ctx, buf), + ChunkId::GdbIndex | ChunkId::RelroPadding | ChunkId::Placeholder(_) => {} + ChunkId::Output(id) => output_section::copy_buf(ctx, id, buf), + ChunkId::Merged(id) => merged::copy_buf(ctx, id, buf), + ChunkId::Reloc(_) => {} + ChunkId::ComdatGroup(i) => misc::comdat_group::copy_buf(ctx, i, buf), + ChunkId::Compressed(i) => misc::compressed::copy_buf(ctx, i, buf), + } +} + +/// Writes a chunk's contents to a scratch buffer, for compression. +pub fn write_to(ctx: &Context, id: ChunkId, buf: &mut [u8]) { + match id { + ChunkId::Output(id) => output_section::write_to(ctx, id, buf), + ChunkId::Merged(id) => merged::write_to(ctx, id, buf), + _ => unreachable!("write_to is only for output and merged sections"), + } +} + +/// The address of a synthesized symbol placed at the start of a chunk. +pub fn chunk_start(ctx: &Context, id: ChunkId) -> u64 { + ctx.chunk_header(id).shdr.sh_addr +} + +/// Whether a chunk should be kept after `--section-order` filtering. +pub fn section_order_index( + ctx: &Context, + name: &[u8], + kind: SectionOrderKind, +) -> Option { + ctx.args + .section_order + .iter() + .position(|o| o.kind == kind && o.name.as_bytes() == name) +} + +/// Resolves a symbol's address without going through its PLT entry. +pub fn addr_no_plt(ctx: &Context, sym: &crate::symbol::Symbol) -> u64 { + sym.addr_with(ctx, AddrFlags::NO_PLT) +} diff --git a/src/chunks/opd.rs b/src/chunks/opd.rs new file mode 100644 index 000000000..e948d5712 --- /dev/null +++ b/src/chunks/opd.rs @@ -0,0 +1,118 @@ +//! `.opd`, the function descriptors of PowerPC64 ELFv1. +//! +//! In ELFv1 a function pointer doesn't refer to code but to a +//! descriptor: the entry point, the TOC pointer the function expects in +//! `r2`, and an environment pointer nobody uses. The compiler emits a +//! descriptor for every function into an input `.opd`; the linker +//! dissolves those (see [`crate::arch::ppc64v1`]) and synthesizes +//! descriptors here only for functions whose addresses are taken, in the +//! same way it synthesizes GOT and PLT entries. + +use crate::arch::Arch; +use crate::chunks::{ChunkHeader, DynRelBuffer}; +use crate::context::Context; +use crate::elf::*; +use crate::symbol::{AddrFlags, SymbolId}; + +pub const ENTRY_SIZE: u64 = 24; + +#[derive(Debug)] +pub struct Ppc64OpdSection { + pub hdr: ChunkHeader, + pub symbols: Vec, +} + +impl Ppc64OpdSection { + pub fn new() -> Ppc64OpdSection { + let mut hdr = ChunkHeader::new(".opd", SHT_PROGBITS, (SHF_ALLOC | SHF_WRITE) as u64); + hdr.shdr.sh_addralign = 8; + Ppc64OpdSection { + hdr, + symbols: Vec::new(), + } + } +} + +impl Default for Ppc64OpdSection { + fn default() -> Self { + Self::new() + } +} + +/// A descriptor holds the function's own address, not that of its PLT +/// entry or of the descriptor itself. +const ENTRY_POINT: AddrFlags = AddrFlags { + no_plt: true, + no_opd: true, +}; + +fn section(ctx: &Context) -> &Ppc64OpdSection { + ctx.ppc64_opd + .as_ref() + .expect("PPC64 ELFv1 has an .opd section") +} + +fn toc(ctx: &Context) -> u64 { + ctx.symbols[ctx.syms.toc.expect("PPC64 has a .TOC. symbol")].addr(ctx) +} + +pub fn add_symbol(ctx: &mut Context, sym: SymbolId) { + let opd = ctx + .ppc64_opd + .as_mut() + .expect("PPC64 ELFv1 has an .opd section"); + ctx.symbols.aux_mut(sym).opd_idx = Some(opd.symbols.len() as u32); + opd.symbols.push(sym); + opd.hdr.shdr.sh_size += ENTRY_SIZE; +} + +/// Position-independent output relocates both the entry point and the +/// TOC pointer at load time. +pub fn num_dynrels(ctx: &Context) -> u64 { + if ctx.args.pic { + section(ctx).symbols.len() as u64 * 2 + } else { + 0 + } +} + +pub fn relr_offsets(ctx: &Context) -> Vec { + if !ctx.args.pic { + return Vec::new(); + } + (0..section(ctx).symbols.len() as u64) + .flat_map(|i| [i * ENTRY_SIZE, i * ENTRY_SIZE + 8]) + .collect() +} + +pub fn write_dynrels(ctx: &Context, mut out: DynRelBuffer<'_, E>) { + if !ctx.args.pic { + debug_assert!(out.is_empty()); + return; + } + let opd = section(ctx); + let relr = ctx.args.pack_dyn_relocs_relr && opd.hdr.num_relrs != 0; + let toc = toc(ctx); + let mut j = 0; + for (i, &id) in opd.symbols.iter().enumerate() { + let loc = opd.hdr.shdr.sh_addr + i as u64 * ENTRY_SIZE; + let entry = ctx.symbols[id].addr_with(ctx, ENTRY_POINT); + for (addr, val) in [(loc, entry), (loc + 8, toc)] { + if !relr || addr % 8 != 0 { + out.write(j, ElfRel::new(addr, E::R_RELATIVE, 0, val as i64)); + j += 1; + } + } + } + debug_assert_eq!(j, out.len()); +} + +pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let toc = toc(ctx); + for (i, &id) in section(ctx).symbols.iter().enumerate() { + let entry = &mut buf[i * ENTRY_SIZE as usize..]; + E::Endian::write_u64(entry, ctx.symbols[id].addr_with(ctx, ENTRY_POINT)); + E::Endian::write_u64(&mut entry[8..], toc); + E::Endian::write_u64(&mut entry[16..], 0); + } +} diff --git a/src/chunks/output_section.rs b/src/chunks/output_section.rs new file mode 100644 index 000000000..5f427d2b1 --- /dev/null +++ b/src/chunks/output_section.rs @@ -0,0 +1,592 @@ +//! Output sections built from input sections, such as `.text` and `.data`. + +use std::sync::atomic::Ordering; + +use bstr::BStr; +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::chunks::{ChunkHeader, DynRelBuffer, OutputSectionId}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::SymtabBlock; +use crate::input_sections::{r_delta, InputSectionId}; +use crate::symbol::{AddrFlags, SymbolId, NEEDS_CANONICAL}; +use crate::thunks::Thunk; +use crate::util::align_to; +use crate::{error, warn}; + +/// How a word-size absolute relocation is resolved. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum AbsRelKind { + /// The value is known at link time. + #[default] + None, + /// A base relocation, applied by the loader. + BaseRel, + /// A base relocation encoded in `.relr.dyn`. + Relr, + /// An IFUNC relocation. + IFunc, + /// A symbolic dynamic relocation. + DynRel, +} + +/// A word-size absolute relocation (e.g. R_X86_64_64), which may need to +/// be promoted to a dynamic relocation. +#[derive(Clone, Debug)] +pub struct AbsRel { + pub isec: InputSectionId, + pub offset: u64, + pub sym: SymbolId, + pub addend: i64, + pub kind: AbsRelKind, +} + +#[derive(Debug)] +pub struct OutputSection { + pub hdr: ChunkHeader, + pub members: Vec, + pub thunks: Vec, + pub reloc_sec: Option, + pub abs_rels: Vec, + + /// The number of dynamic relocations before each shard of `abs_rels`. + pub dynrel_offsets: Vec, + pub relr_offsets: Vec, +} + +/// A pointer to an output section whose disjoint ranges are written in parallel. +struct OutputBuffer { + ptr: *mut u8, + len: usize, +} + +// SAFETY: Parallel loops use the pointer only for ranges that the ELF layout +// proves disjoint, just as their C++ counterparts do. +unsafe impl Sync for OutputBuffer {} + +impl OutputBuffer { + #[inline] + fn new(buf: &mut [u8]) -> OutputBuffer { + OutputBuffer { + ptr: buf.as_mut_ptr(), + len: buf.len(), + } + } + + /// Runs `f` on one range owned by the current parallel-loop iteration. + /// + /// # Safety + /// No other live access may overlap `range`. + #[inline] + unsafe fn with_slice( + &self, + range: std::ops::Range, + f: impl FnOnce(&mut [u8]) -> R, + ) -> R { + debug_assert!(range.start <= range.end && range.end <= self.len); + // SAFETY: the caller guarantees that this in-bounds range is the + // iteration's exclusive output range. + let slice = unsafe { + std::slice::from_raw_parts_mut(self.ptr.add(range.start), range.end - range.start) + }; + f(slice) + } + + /// Writes one relocated word. + /// + /// # Safety + /// No other live access may overlap the word at `offset`. + unsafe fn write_word(&self, offset: u64, value: u64) { + // SAFETY: The caller guarantees that the word is within the output + // section and exclusively owned by this relocation. + debug_assert!(offset as usize + E::WORD_SIZE <= self.len); + let ptr = unsafe { self.ptr.add(offset as usize) }; + let slot = unsafe { std::slice::from_raw_parts_mut(ptr, E::WORD_SIZE) }; + if E::IS_64 { + E::Endian::write_u64(slot, value); + } else { + E::Endian::write_u32(slot, value as u32); + } + } +} + +/// Shard size for parallel processing of absolute relocations. +pub const DYNREL_SHARD_SIZE: usize = 65536; + +impl OutputSection { + pub fn new(name: &'static BStr, sh_type: u32) -> OutputSection { + OutputSection { + hdr: ChunkHeader::with_name(name, sh_type, 0), + members: Vec::new(), + thunks: Vec::new(), + reloc_sec: None, + abs_rels: Vec::new(), + dynrel_offsets: Vec::new(), + relr_offsets: Vec::new(), + } + } +} + +/// Assigns offsets to the members of an output section and computes its +/// size. An output section may contain millions of input sections, so the +/// members are split into groups laid out in parallel. +pub fn compute_section_size(ctx: &mut Context, id: OutputSectionId) { + let size = layout(ctx, id); + ctx.output_sections[id.index()].hdr.shdr.sh_size = size; +} + +/// Assigns the members their offsets and returns the section size. +/// +/// An output section such as `.text` has a million members, so the +/// members are split into groups: the sizes of the groups are computed +/// in parallel, then the offsets within each group. +pub fn layout(ctx: &Context, id: OutputSectionId) -> u64 { + const GROUP_SIZE: usize = 10000; + + let osec = &ctx.output_sections[id.index()]; + debug_assert!( + !E::NEEDS_THUNK + || osec.hdr.shdr.sh_flags & SHF_EXECINSTR as u64 == 0 + || ctx.args.relocatable + ); + + struct Group { + size: u64, + offset: u64, + align: u64, + } + + let mut groups: Vec = osec + .members + .par_chunks(GROUP_SIZE) + .map(|members| { + let mut off = 0; + let mut align = 1; + for &m in members { + let isec = ctx.input_section(m); + off = align_to(off, 1 << isec.p2align()) + isec.sh_size; + align = align.max(1 << isec.p2align()); + } + Group { + size: off, + offset: 0, + align, + } + }) + .collect(); + + let mut off = 0; + for g in &mut groups { + off = align_to(off, g.align); + g.offset = off; + off += g.size; + } + + osec.members + .par_chunks(GROUP_SIZE) + .zip(&groups) + .for_each(|(members, g)| { + let mut off = g.offset; + for &m in members { + let isec = ctx.input_section(m); + off = align_to(off, 1 << isec.p2align()); + isec.set_offset(off); + off += isec.sh_size; + } + }); + off +} + +/// Copies the members into `buf`, fills the padding between them, and +/// applies relocations. +pub fn write_to(ctx: &Context, id: OutputSectionId, buf: &mut [u8]) { + let osec = &ctx.output_sections[id.index()]; + let members = &osec.members; + + let output = OutputBuffer::new(buf); + + // Copy section contents to an output file. + members.par_iter().enumerate().for_each(|(i, &member)| { + let isec = ctx.input_section(member); + let start = isec.offset() as usize; + let next_start = members + .get(i + 1) + .map_or(osec.hdr.shdr.sh_size as usize, |&next| { + ctx.input_section(next).offset() as usize + }); + // SAFETY: output-section member offsets are ordered and each + // iteration owns the bytes up to the next member. + unsafe { + output.with_slice(start..next_start, |slice| { + let (own, padding) = slice.split_at_mut(isec.sh_size as usize); + isec.write_to(ctx, own); + + // Clear trailing padding. We write trap instructions for an + // executable segment so that a disassembler wouldn't try to + // disassemble garbage as instructions. + if osec.hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0 { + // s390x's old CRT files use NOP slides in .init and .fini. + // https://sourceware.org/bugzilla/show_bug.cgi?id=31042 + let filler: &[u8] = if E::FAMILY == Family::S390x + && (osec.hdr.name == b".init" || osec.hdr.name == b".fini") + { + &[0x07, 0x00] + } else { + E::TRAP + }; + let mut pos = 0; + while pos + filler.len() <= padding.len() { + padding[pos..pos + filler.len()].copy_from_slice(filler); + pos += filler.len(); + } + } else { + padding.fill(0); + } + }); + } + }); + + // Emit range extension thunks. + if E::NEEDS_THUNK { + osec.thunks.par_iter().for_each(|thunk| { + let start = thunk.offset as usize; + let end = start + thunk.size() as usize; + // SAFETY: thunks have distinct ranges and the member loop has + // joined before this loop begins. + unsafe { + output.with_slice(start..end, |buf| { + crate::thunks::copy_buf(ctx, osec, thunk, buf); + }); + } + }); + } +} + +/// Writes the section and applies word-size absolute relocations. +pub fn copy_buf(ctx: &Context, id: OutputSectionId, buf: &mut [u8]) { + let osec = &ctx.output_sections[id.index()]; + if osec.hdr.shdr.sh_type == SHT_NOBITS { + return; + } + write_to(ctx, id, buf); + + // Apply absolute relocations. An output section can have a million + // of them, so this loop is parallel. + let word = E::WORD_SIZE; + let base_addr = osec.hdr.shdr.sh_addr; + let output = OutputBuffer::new(buf); + + osec.abs_rels.par_iter().for_each(|r| { + let isec = ctx.input_section(r.isec); + let sym = &ctx.symbols[r.sym]; + let mut loc = isec.offset() + r.offset; + let mut p = base_addr + loc; + if E::IS_RISCV || E::IS_LOONGARCH { + let delta = r_delta(isec, r.offset) as u64; + loc -= delta; + p -= delta; + } + let s = sym.addr(ctx); + let a = r.addend as u64; + + let can_relr = || { + ctx.args.pack_dyn_relocs_relr + && r.kind == AbsRelKind::BaseRel + && p.is_multiple_of(word as u64) + }; + + let value = match r.kind { + AbsRelKind::None | AbsRelKind::Relr => Some(s.wrapping_add(a)), + AbsRelKind::BaseRel => { + (can_relr() || ctx.args.apply_dynamic_relocs).then(|| s.wrapping_add(a)) + } + AbsRelKind::IFunc => (E::SUPPORTS_IFUNC && ctx.args.apply_dynamic_relocs) + .then(|| sym.addr_with(ctx, AddrFlags::NO_PLT).wrapping_add(a)), + AbsRelKind::DynRel => ctx.args.apply_dynamic_relocs.then_some(a), + }; + if let Some(value) = value { + // SAFETY: Input files do not contain overlapping relocations; + // each `AbsRel` therefore owns this output word. + unsafe { output.write_word::(loc, value) }; + } + }); +} + +pub fn num_dynrels(ctx: &Context, id: OutputSectionId) -> u64 { + ctx.output_sections[id.index()] + .dynrel_offsets + .last() + .copied() + .unwrap_or(0) +} + +/// Marks the base relocations that RELR can encode and returns their +/// offsets within the section. +pub fn relr_offsets(ctx: &mut Context, id: OutputSectionId) -> Vec { + let word = E::WORD_SIZE as u64; + let section_arena = &ctx.section_arena; + let osec = &mut ctx.output_sections[id.index()]; + let nshards = osec.dynrel_offsets.len().saturating_sub(1); + let mut relr_offsets = vec![0u64; nshards + 1]; + + let shards: Vec> = osec + .abs_rels + .par_chunks_mut(DYNREL_SHARD_SIZE) + .map(|shard| { + let mut out = Vec::new(); + for r in shard { + if r.kind != AbsRelKind::BaseRel { + continue; + } + let isec = section_arena.section(r.isec); + if (1u64 << isec.p2align()).is_multiple_of(word) && r.offset % word == 0 { + r.kind = AbsRelKind::Relr; + out.push(isec.offset() + r.offset); + } + } + out + }) + .collect(); + + let mut offsets = Vec::new(); + for (i, shard) in shards.into_iter().enumerate() { + relr_offsets[i + 1] = relr_offsets[i] + shard.len() as u64; + offsets.extend(shard); + } + osec.relr_offsets = relr_offsets; + offsets +} + +pub fn write_dynrels(ctx: &Context, id: OutputSectionId, out: DynRelBuffer<'_, E>) { + let osec = &ctx.output_sections[id.index()]; + // A single output section such as .data.rel.ro can account for + // most of an output's dynamic relocations, so we process its + // absolute relocations in parallel shards. + let nshards = osec.dynrel_offsets.len().saturating_sub(1); + let mut offsets = osec.dynrel_offsets.clone(); + + if ctx.args.pack_dyn_relocs_relr && osec.hdr.num_relrs != 0 { + debug_assert_eq!(osec.relr_offsets.len(), offsets.len()); + for (offset, &relr_offset) in offsets.iter_mut().zip(&osec.relr_offsets) { + *offset -= relr_offset; + } + } + + let count = offsets.last().copied().unwrap_or(0) as usize; + debug_assert_eq!(count as u64, osec.hdr.num_dynrels - osec.hdr.num_relrs); + debug_assert_eq!(out.len(), count); + let slices = out.split_at_offsets(&offsets[..nshards]); + + osec.abs_rels + .par_chunks(DYNREL_SHARD_SIZE) + .zip(slices.into_par_iter()) + .for_each(|(rels, mut slots)| { + let mut i = 0; + for r in rels { + let sym = &ctx.symbols[r.sym]; + let isec = ctx.input_section(r.isec); + let s = sym.addr(ctx); + let a = r.addend; + let mut p = osec.hdr.shdr.sh_addr + isec.offset() + r.offset; + if E::IS_RISCV || E::IS_LOONGARCH { + p -= r_delta(isec, r.offset) as u64; + } + let rel = match r.kind { + AbsRelKind::None | AbsRelKind::Relr => None, + AbsRelKind::BaseRel => Some(ElfRel::new( + p, + E::R_RELATIVE, + 0, + s.wrapping_add(a as u64) as i64, + )), + AbsRelKind::IFunc => E::R_IRELATIVE.map(|r_type| { + let val = sym.addr_with(ctx, AddrFlags::NO_PLT).wrapping_add(a as u64); + ElfRel::new(p, r_type, 0, val as i64) + }), + AbsRelKind::DynRel => Some(ElfRel::new( + p, + E::R_ABS, + sym.dynsym_idx(&ctx.symbols).unwrap_or(0), + a, + )), + }; + if let Some(rel) = rel { + slots.write(i, rel); + i += 1; + } + } + debug_assert_eq!(i, slots.len()); + }); +} + +fn abs_rel_kind(ctx: &Context, sym: &crate::symbol::Symbol) -> AbsRelKind { + if sym.is_ifunc() { + return if sym.is_pde_ifunc(ctx) { + AbsRelKind::None + } else { + AbsRelKind::IFunc + }; + } + if sym.is_absolute() { + return AbsRelKind::None; + } + // The symbol's address is in the output file. + if !sym.is_imported() || sym.flags() & NEEDS_CANONICAL != 0 { + return if ctx.args.pic { + AbsRelKind::BaseRel + } else { + AbsRelKind::None + }; + } + AbsRelKind::DynRel +} + +fn is_absrel(r: &ElfRel) -> bool { + match E::FAMILY { + // R_ARM_TARGET1 is typically used for entries in .init_array and + // is interpreted as either ABS32 or REL32 depending on the target; + // all targets we support treat it as ABS32. + Family::Arm32 => r.r_type == R_ARM_ABS32 || r.r_type == R_ARM_TARGET1, + // SPARC64 has separate relocations for aligned and unaligned words. + Family::Sparc64 => r.r_type == R_SPARC_64 || r.r_type == R_SPARC_UA64, + _ => r.r_type == E::R_ABS, + } +} + +/// Collects word-size absolute relocations, which are the only ones that +/// can be promoted to dynamic relocations, and classifies them. +pub fn scan_abs_relocations( + ctx: &Context, + id: OutputSectionId, +) -> (Vec, Vec) { + let osec = &ctx.output_sections[id.index()]; + + let mut abs_rels: Vec = osec + .members + .par_iter() + .flat_map_iter(|&m| { + let isec = ctx.input_section(m); + let file = &ctx.objs[isec.file.index()]; + isec.rels::(file) + .iter() + .filter(|r| is_absrel::(r)) + .map(move |r| AbsRel { + isec: m, + offset: r.r_offset, + sym: file.base.symbols[r.r_sym as usize], + addend: isec.rel_addend::(&r), + kind: AbsRelKind::None, + }) + }) + .collect(); + + // In a position-dependent executable, dynamic relocations in read-only + // sections can sometimes be avoided by promoting symbols to canonical + // PLT entries or copy relocations. + let promote = !ctx.args.pic && osec.hdr.shdr.sh_flags & SHF_WRITE as u64 == 0; + + let counts: Vec = abs_rels + .par_chunks_mut(DYNREL_SHARD_SIZE) + .map(|shard| { + let mut count = 0; + for r in shard { + let sym = &ctx.symbols[r.sym]; + if promote && sym.is_imported() && !sym.is_absolute() { + sym.add_flags(NEEDS_CANONICAL); + } + r.kind = abs_rel_kind(ctx, sym); + + let emit = matches!(r.kind, AbsRelKind::BaseRel | AbsRelKind::DynRel) + || (E::SUPPORTS_IFUNC && r.kind == AbsRelKind::IFunc); + if emit { + count += 1; + } + + // A relocation against a read-only section needs DT_TEXTREL. + let isec = ctx.input_section(r.isec); + if r.kind != AbsRelKind::None && isec.sh_flags & SHF_WRITE as u64 == 0 { + if ctx.args.z_text { + error!( + ctx, + "{}: relocation at offset 0x{:x} against symbol `{}' can not be used; recompile with -fPIC", + ctx.input_section_display(r.isec), + r.offset, + sym + ); + } else if ctx.args.warn_textrel { + warn!( + ctx, + "{}: relocation against symbol `{}' in read-only section", + ctx.input_section_display(r.isec), + sym + ); + } + ctx.has_textrel.store(true, Ordering::Relaxed); + } + } + count + }) + .collect(); + + let mut dynrel_offsets = vec![0u64; counts.len() + 1]; + for (i, c) in counts.iter().enumerate() { + dynrel_offsets[i + 1] = dynrel_offsets[i] + c; + } + (abs_rels, dynrel_offsets) +} + +/// Sizes the symbols synthesized for range extension thunks. +pub fn compute_symtab_size(ctx: &mut Context, id: OutputSectionId) { + if !E::NEEDS_THUNK { + return; + } + let symbols = &ctx.symbols; + let osec = &mut ctx.output_sections[id.index()]; + osec.hdr.strtab_size = 0; + osec.hdr.num_local_symtab = 0; + for thunk in &osec.thunks { + // ARM32 gets "$t", "$a" and "$d" mapping symbols per thunk entry. + let per_entry = if E::FAMILY == Family::Arm32 { 4 } else { 1 }; + osec.hdr.num_local_symtab += (thunk.symbols.len() * per_entry) as u32; + for &sym in &thunk.symbols { + osec.hdr.strtab_size += symbols[sym].name().len() as u64 + thunk.name.len() as u64 + 2; + } + } +} + +/// Produces the symbols marking thunk locations, which help disassembly +/// and debugging. +pub fn populate_symtab( + ctx: &Context, + id: OutputSectionId, + block: &mut SymtabBlock<'_>, +) { + if !E::NEEDS_THUNK { + return; + } + let osec = &ctx.output_sections[id.index()]; + let shndx = osec.hdr.shndx; + let func = |addr: u64| ElfSym { + st_info: STT_FUNC as u8, + st_shndx: shndx as u16, + st_value: addr, + ..ElfSym::default() + }; + + for thunk in &osec.thunks { + for (i, &sym) in thunk.symbols.iter().enumerate() { + let addr = osec.hdr.shdr.sh_addr + thunk.offset + thunk.offsets[i]; + let name = ctx.symbols[sym].name(); + let suffix = format!("${}", thunk.name); + block.push_synthetic::(name, suffix.as_bytes(), func(addr)); + if E::FAMILY == Family::Arm32 { + block.push_mapping_symbol::(crate::chunks::symtab::strtab::THUMB, func(addr)); + block.push_mapping_symbol::(crate::chunks::symtab::strtab::ARM, func(addr + 4)); + block + .push_mapping_symbol::(crate::chunks::symtab::strtab::DATA, func(addr + 12)); + } + } + } +} diff --git a/src/chunks/sframe.rs b/src/chunks/sframe.rs new file mode 100644 index 000000000..c7f7bc87e --- /dev/null +++ b/src/chunks/sframe.rs @@ -0,0 +1,188 @@ +//! `.sframe`, a compact stack-unwinding format. Like `.eh_frame` it is +//! reconstructed: live FDEs are gathered, their FREs concatenated, and the +//! FDE index sorted by address. + +use crate::arch::{Arch, Family}; +use crate::chunks::ChunkHeader; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::ObjId; + +#[derive(Debug)] +pub struct SFrameSection { + pub hdr: ChunkHeader, + pub header: SFrameHeader, + /// The live FDEs as (file, index into the file's `sframe_fdes`). + pub fdes: Vec<(ObjId, u32)>, +} + +impl SFrameSection { + pub fn new() -> SFrameSection { + let mut hdr = ChunkHeader::new(".sframe", SHT_GNU_SFRAME, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 8; + SFrameSection { + hdr, + header: SFrameHeader::default(), + fdes: Vec::new(), + } + } +} + +impl Default for SFrameSection { + fn default() -> Self { + Self::new() + } +} + +/// Lays out the output section. If no live function has unwind info the +/// section is left empty and dropped from the output. +pub fn construct(ctx: &mut Context) { + let Some(abi) = E::SFRAME_ABI else { + return; + }; + let _t = ctx.timer("sframe"); + + let mut fdes = Vec::new(); + for (fi, file) in ctx.objs.iter().enumerate() { + for (i, fde) in file.sframe_fdes.iter().enumerate() { + if file.section_at(fde.section).is_alive() { + fdes.push((ObjId(fi as u32), i as u32)); + } + } + } + if fdes.is_empty() { + return; + } + + // Function pointers are always PC-relative; the index is marked + // sorted unless this is a relocatable output. + let mut hdr = SFrameHeader { + magic: SFRAME_MAGIC, + version: 3, + flags: SFRAME_F_FDE_FUNC_START_PCREL + | if ctx.args.relocatable { + 0 + } else { + SFRAME_F_FDE_SORTED + }, + abi_arch: abi, + cfa_fixed_ra_offset: if E::FAMILY == Family::X86_64 { -8 } else { 0 }, + num_fdes: fdes.len() as u32, + freoff: (fdes.len() * SFrameFdeIdx::size::()) as u32, + ..SFrameHeader::default() + }; + for &(fi, i) in &fdes { + let fde = &ctx.objs[fi.index()].sframe_fdes[i as usize]; + hdr.fre_len += fde.fre.len() as u32; + hdr.num_fres += fde.num_fres; + } + + let sframe = &mut ctx.sframe; + sframe.hdr.shdr.sh_size = (SFrameHeader::size::() + fdes.len() * SFrameFdeIdx::size::()) + as u64 + + hdr.fre_len as u64; + sframe.header = hdr; + sframe.fdes = fdes; +} + +/// Sorts the FDEs by function address, which fixes the order in which +/// both `copy_buf` and `sframe_reloc::copy_buf` emit them. +pub fn sort(ctx: &mut Context) { + if ctx.args.relocatable { + return; + } + let addr = |ctx: &Context, (fi, i): (ObjId, u32)| { + let fde = &ctx.objs[fi.index()].sframe_fdes[i as usize]; + ctx.symbols[fde.sym] + .addr(ctx) + .wrapping_add(fde.addend as u64) + }; + let mut fdes = std::mem::take(&mut ctx.sframe.fdes); + fdes.sort_by_key(|&f| addr(ctx, f)); + ctx.sframe.fdes = fdes; +} + +pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let sframe = &ctx.sframe; + let hdr_size = SFrameHeader::size::(); + let idx_size = SFrameFdeIdx::size::(); + sframe.header.write::(buf); + + let fre_base = hdr_size + sframe.header.freoff as usize; + let mut fre_off = 0usize; + for (i, &(fi, fi_idx)) in sframe.fdes.iter().enumerate() { + let fde = &ctx.objs[fi.index()].sframe_fdes[fi_idx as usize]; + buf[fre_base + fre_off..fre_base + fre_off + fde.fre.len()].copy_from_slice(fde.fre); + + let func_start_offset = if ctx.args.relocatable { + 0 + } else { + let func_addr = ctx.symbols[fde.sym] + .addr(ctx) + .wrapping_add(fde.addend as u64); + let field_addr = sframe.hdr.shdr.sh_addr + hdr_size as u64 + (i * idx_size) as u64; + func_addr.wrapping_sub(field_addr) as i64 + }; + let ent = SFrameFdeIdx { + func_start_offset, + func_size: fde.func_size, + func_start_fre_off: fre_off as u32, + }; + ent.write::(&mut buf[hdr_size + i * idx_size..]); + fre_off += fde.fre.len(); + } +} + +/// `.rela.sframe`, for relocatable outputs. +#[derive(Debug)] +pub struct SFrameRelocSection { + pub hdr: ChunkHeader, +} + +impl SFrameRelocSection { + pub fn new() -> SFrameRelocSection { + let mut hdr = ChunkHeader::new(".rela.sframe", SHT_RELA, SHF_INFO_LINK as u64); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + hdr.shdr.sh_entsize = ElfRel::size::() as u64; + SFrameRelocSection { hdr } + } +} + +pub mod sframe_reloc { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let n = ctx.sframe.fdes.len(); + let sec = ctx.sframe_reloc.as_mut().unwrap(); + sec.hdr.shdr.sh_size = (n * ElfRel::size::()) as u64; + sec.hdr.shdr.sh_link = ctx.symtab.hdr.shndx; + sec.hdr.shdr.sh_info = ctx.sframe.hdr.shndx; + } + + /// One relocation per FDE for its func_start field, in index order. + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let Some(r_type) = E::R_SFRAME else { + return; + }; + let size = ElfRel::size::(); + for (i, &(fi, fi_idx)) in ctx.sframe.fdes.iter().enumerate() { + let fde = &ctx.objs[fi.index()].sframe_fdes[fi_idx as usize]; + let sym = &ctx.symbols[fde.sym]; + let r_offset = ctx.sframe.hdr.shdr.sh_addr + + SFrameHeader::size::() as u64 + + (i * SFrameFdeIdx::size::()) as u64; + + let (r_sym, r_addend) = if sym.st_type() == STT_SECTION { + // Section symbols are recreated per output section. + let target = sym.input_section_ref().unwrap(); + ( + ctx.output_section(target.output_section.unwrap()).hdr.shndx, + fde.addend + target.offset() as i64, + ) + } else { + (sym.output_sym_idx(ctx), fde.addend) + }; + ElfRel::new(r_offset, r_type, r_sym, r_addend).write::(&mut buf[i * size..]); + } + } +} diff --git a/src/chunks/symtab.rs b/src/chunks/symtab.rs new file mode 100644 index 000000000..2695578a2 --- /dev/null +++ b/src/chunks/symtab.rs @@ -0,0 +1,865 @@ +//! Symbol and string tables: `.symtab`, `.strtab`, `.shstrtab`, `.dynsym`, +//! `.dynstr`, `.hash` and `.gnu.hash`. + +use std::collections::HashMap; + +use bstr::BStr; +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::chunks::{self, ChunkHeader, ChunkId}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::{FileId, SymtabBlock, SymtabEntries}; +use crate::input_sections::r_delta; +use crate::symbol::{AddrFlags, Symbol, SymbolId}; +use crate::util::write_cstr; +use crate::{error, fatal}; + +/// `.strtab` holds the names of `.symtab` symbols. It isn't needed at +/// runtime; names the loader needs are in `.dynstr`. +#[derive(Debug)] +pub struct StrtabSection { + pub hdr: ChunkHeader, +} + +impl StrtabSection { + pub fn new() -> StrtabSection { + StrtabSection { + hdr: ChunkHeader::new(".strtab", SHT_STRTAB, 0), + } + } +} + +impl Default for StrtabSection { + fn default() -> Self { + Self::new() + } +} + +pub mod strtab { + use super::*; + + /// Offsets in `.strtab` of the ARM32 mapping symbol names. + pub const ARM: u32 = 1; + pub const THUMB: u32 = 4; + pub const DATA: u32 = 7; + + pub fn update_shdr(ctx: &mut Context) { + let mut offset = 1u64; + + // ARM32 uses $a, $t and $d mapping symbols to mark ARM code, Thumb + // code and data; they only help disassemblers. + if E::FAMILY == Family::Arm32 && !ctx.args.strip_all { + offset += b"$a\0$t\0$d\0".len() as u64; + } + + for id in ctx.chunks.clone() { + let hdr = ctx.chunk_header_mut(id); + hdr.strtab_offset = offset; + offset += hdr.strtab_size; + } + for file in &mut ctx.objs { + file.base.strtab_offset = offset; + offset += file.base.strtab_size; + } + for file in &mut ctx.dsos { + file.base.strtab_offset = offset; + offset += file.base.strtab_size; + } + ctx.strtab.hdr.shdr.sh_size = if offset == 1 { 0 } else { offset }; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf[0] = 0; + if E::FAMILY == Family::Arm32 && !ctx.args.strip_all { + buf[1..10].copy_from_slice(b"$a\0$t\0$d\0"); + } + } +} + +/// `.shstrtab` holds section names. +#[derive(Debug)] +pub struct ShstrtabSection { + pub hdr: ChunkHeader, +} + +impl ShstrtabSection { + pub fn new() -> ShstrtabSection { + ShstrtabSection { + hdr: ChunkHeader::new(".shstrtab", SHT_STRTAB, 0), + } + } +} + +impl Default for ShstrtabSection { + fn default() -> Self { + Self::new() + } +} + +pub mod shstrtab { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let mut map: HashMap<&'static BStr, u64> = HashMap::new(); + let mut offset = 1u64; + for id in ctx.chunks.clone() { + if id.is_header() { + continue; + } + let name = ctx.chunk_header(id).name; + if name.is_empty() { + continue; + } + let off = *map.entry(name).or_insert_with(|| { + let off = offset; + offset += name.len() as u64 + 1; + off + }); + ctx.chunk_header_mut(id).shdr.sh_name = off as u32; + } + ctx.shstrtab.as_mut().unwrap().hdr.shdr.sh_size = offset; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf[0] = 0; + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + if hdr.shdr.sh_name != 0 { + write_cstr(&mut buf[hdr.shdr.sh_name as usize..], hdr.name); + } + } + } +} + +/// `.dynstr` holds the strings the dynamic linker uses. +#[derive(Debug)] +pub struct DynstrSection { + pub hdr: ChunkHeader, + strings: HashMap, u64>, +} + +impl DynstrSection { + pub fn new() -> DynstrSection { + DynstrSection { + hdr: ChunkHeader::new(".dynstr", SHT_STRTAB, SHF_ALLOC as u64), + strings: HashMap::new(), + } + } + + pub fn add_string(&mut self, s: &[u8]) -> u64 { + if self.hdr.shdr.sh_size == 0 { + self.strings.insert(Vec::new(), 0); + self.hdr.shdr.sh_size = 1; + } + if let Some(&off) = self.strings.get(s) { + return off; + } + let off = self.hdr.shdr.sh_size; + self.strings.insert(s.to_vec(), off); + self.hdr.shdr.sh_size += s.len() as u64 + 1; + off + } + + pub fn find_string(&self, s: &[u8]) -> u64 { + *self.strings.get(s).expect("string not in .dynstr") + } +} + +impl Default for DynstrSection { + fn default() -> Self { + Self::new() + } +} + +pub mod dynstr { + use super::*; + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + for (s, &off) in &ctx.dynstr.strings { + write_cstr(&mut buf[off as usize..], s); + } + let mut off = ctx.dynsym.dynstr_offset as usize; + for &id in ctx.dynsym.symbols.iter().flatten() { + off += write_cstr(&mut buf[off..], ctx.symbols[id].name()); + } + } +} + +/// `.symtab` holds non-dynamic symbols, mainly for debugging. +#[derive(Debug)] +pub struct SymtabSection { + pub hdr: ChunkHeader, +} + +impl SymtabSection { + pub fn new() -> SymtabSection { + let mut hdr = ChunkHeader::new(".symtab", SHT_SYMTAB, 0); + hdr.shdr.sh_entsize = ElfSym::size::() as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + SymtabSection { hdr } + } +} + +/// `.symtab_shndx` holds section indices too large for the 16-bit +/// `st_shndx` field. Most files don't need it. +#[derive(Debug)] +pub struct SymtabShndxSection { + pub hdr: ChunkHeader, +} + +impl SymtabShndxSection { + pub fn new() -> SymtabShndxSection { + let mut hdr = ChunkHeader::new(".symtab_shndx", SHT_SYMTAB_SHNDX, 0); + hdr.shdr.sh_entsize = 4; + hdr.shdr.sh_addralign = 4; + SymtabShndxSection { hdr } + } +} + +impl Default for SymtabShndxSection { + fn default() -> Self { + Self::new() + } +} + +// Named after the chunk, like the other chunks' modules in this directory. +#[allow(clippy::module_inception)] +pub mod symtab { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + let mut nsyms = 1u32; + + // Section symbols + nsyms += ctx + .chunks + .iter() + .filter(|&&id| ctx.chunk_header(id).shndx != 0) + .count() as u32; + + // Linker-synthesized symbols + for id in ctx.chunks.clone() { + let hdr = ctx.chunk_header_mut(id); + hdr.local_symtab_idx = nsyms; + nsyms += hdr.num_local_symtab; + } + + // File local symbols, then global symbols + for file in &mut ctx.objs { + file.base.local_symtab_idx = nsyms; + nsyms += file.base.num_local_symtab; + } + for file in &mut ctx.objs { + file.base.global_symtab_idx = nsyms; + nsyms += file.base.num_global_symtab; + } + for file in &mut ctx.dsos { + file.base.global_symtab_idx = nsyms; + nsyms += file.base.num_global_symtab; + } + + ctx.symtab.hdr.shdr.sh_info = ctx.objs.first().map_or(nsyms, |f| f.base.global_symtab_idx); + ctx.symtab.hdr.shdr.sh_link = ctx.strtab.hdr.shndx; + ctx.symtab.hdr.shdr.sh_size = if nsyms == 1 { + 0 + } else { + nsyms as u64 * ElfSym::size::() as u64 + }; + } + + /// Writes `.symtab`, `.strtab` and `.symtab_shndx`. + pub fn copy_buf( + ctx: &Context, + symtab: &mut [u8], + strtab: &mut [u8], + mut xindex: Option<&mut [u8]>, + ) { + let size = ElfSym::size::(); + symtab[..size].fill(0); + if let Some(xindex) = xindex.as_deref_mut() { + xindex.fill(0); + } + + // Section symbols + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + if hdr.shndx == 0 { + continue; + } + let mut esym = ElfSym { + st_info: STT_SECTION as u8, + st_value: hdr.shdr.sh_addr, + ..ElfSym::default() + }; + match xindex.as_deref_mut() { + Some(xindex) => { + E::Endian::write_u32(&mut xindex[hdr.shndx as usize * 4..], hdr.shndx); + esym.st_shndx = SHN_XINDEX as u16; + } + None => esym.st_shndx = hdr.shndx as u16, + } + esym.write::(&mut symtab[hdr.shndx as usize * size..]); + } + + strtab::copy_buf(ctx, strtab); + + // Symbols synthesized by chunks, then symbols from input files. + // Each writer fills its own part of the tables: the local symbols + // come first, the chunks' then the files', and the global symbols + // after them, with the names laid out in the same order. + enum Writer { + Chunk(ChunkId), + Obj(usize), + Dso(usize), + } + struct Part { + writer: Writer, + locals: (u32, u32), + globals: (u32, u32), + strtab: (u64, u64), + } + let mut parts: Vec = Vec::new(); + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + if hdr.num_local_symtab != 0 { + let locals = (hdr.local_symtab_idx, hdr.num_local_symtab); + parts.push(Part { + writer: Writer::Chunk(id), + locals, + globals: (0, 0), + strtab: (hdr.strtab_offset, hdr.strtab_size), + }); + } + } + for (i, file) in ctx.objs.iter().enumerate() { + let base = &file.base; + let locals = (base.local_symtab_idx, base.num_local_symtab); + let globals = (base.global_symtab_idx, base.num_global_symtab); + parts.push(Part { + writer: Writer::Obj(i), + locals, + globals, + strtab: (base.strtab_offset, base.strtab_size), + }); + } + for (i, file) in ctx.dsos.iter().enumerate() { + let base = &file.base; + let globals = (base.global_symtab_idx, base.num_global_symtab); + parts.push(Part { + writer: Writer::Dso(i), + locals: (0, 0), + globals, + strtab: (base.strtab_offset, base.strtab_size), + }); + } + + // Carve each part's slices off the buffers, which the parts cover + // in order. + let (mut rest, mut pos) = (symtab, 0); + let (mut xrest, mut xpos) = (xindex, 0); + let mut entries = |(start, count): (u32, u32)| { + let syms = carve( + &mut rest, + &mut pos, + start as usize * size, + count as usize * size, + ); + let xindex = xrest + .as_mut() + .map(|x| carve(x, &mut xpos, start as usize * 4, count as usize * 4)); + (syms, xindex) + }; + let locals: Vec<_> = parts.iter().map(|p| entries(p.locals)).collect(); + let globals: Vec<_> = parts.iter().map(|p| entries(p.globals)).collect(); + let (mut srest, mut spos) = (strtab, 0); + let strtabs: Vec<_> = parts + .iter() + .map(|p| { + carve( + &mut srest, + &mut spos, + p.strtab.0 as usize, + p.strtab.1 as usize, + ) + }) + .collect(); + + parts + .par_iter() + .zip(locals) + .zip(globals) + .zip(strtabs) + .for_each(|(((part, (lsyms, lx)), (gsyms, gx)), strtab)| { + let mut block = SymtabBlock::new( + SymtabEntries::new(lsyms, lx), + SymtabEntries::new(gsyms, gx), + strtab, + part.strtab.0, + ); + match part.writer { + Writer::Chunk(id) => chunks::populate_symtab(ctx, id, &mut block), + Writer::Obj(i) => ctx.objs[i].populate_symtab( + ctx, + crate::input_files::ObjId(i as u32), + &mut block, + ), + Writer::Dso(i) => ctx.dsos[i].populate_symtab( + ctx, + crate::input_files::DsoId(i as u32), + &mut block, + ), + } + }); + } + + /// Splits `len` bytes starting at `start` off `rest`, which begins + /// at `pos` in the buffer; the pieces are taken in order. + fn carve<'a>( + rest: &mut &'a mut [u8], + pos: &mut usize, + start: usize, + len: usize, + ) -> &'a mut [u8] { + if len == 0 { + return &mut []; + } + let buf = std::mem::take(rest); + let (_, buf) = buf.split_at_mut(start - *pos); + let (piece, tail) = buf.split_at_mut(len); + *rest = tail; + *pos = start + len; + piece + } +} + +/// RISC-V and LoongArch relaxation may have removed instructions from a +/// function, so its size must be recomputed. +fn symbol_size(ctx: &Context, sym: &Symbol) -> u64 { + let esym = &sym.esym(ctx); + if (E::IS_RISCV || E::IS_LOONGARCH) && esym.st_size != 0 { + if let Some(isec) = sym.input_section_ref() { + if isec.sh_flags & SHF_EXECINSTR as u64 != 0 { + let end = esym.st_value + esym.st_size; + return (esym.st_size as i64 + esym.st_value as i64 + - sym.value as i64 + - r_delta(isec, end)) as u64; + } + } + } + esym.st_size +} + +/// Builds the output symbol table entry for a symbol. The returned index +/// is nonzero if the section index doesn't fit in `st_shndx` and must go +/// to `.symtab_shndx`. +pub fn to_output_esym(ctx: &Context, sym: &Symbol, st_name: u32) -> (ElfSym, u32) { + let mut esym = ElfSym { + st_name, + st_size: symbol_size(ctx, sym), + ..ElfSym::default() + }; + esym.set_type(sym.ty()); + + let file = sym.file().expect("symbol without a file in symbol table"); + esym.set_bind(if sym.is_local(ctx) { + STB_LOCAL + } else if sym.is_weak() { + STB_WEAK + } else if file.is_dso() { + STB_GLOBAL + } else { + sym.st_bind() + }); + + match E::FAMILY { + Family::Arm64 | Family::RiscV => esym.st_other |= sym.esym(ctx).st_other & 0x80, + Family::Ppc64V2 => esym.st_other |= sym.esym(ctx).st_other & 0xe0, + _ => {} + } + + let st_shndx_of = |sym: &Symbol| -> u32 { + if let Some(frag) = sym.fragment() { + if ctx.fragment(frag).is_alive() { + return ctx.merged_sections[frag.section.index()].hdr.shndx; + } + } + if E::FAMILY == Family::Ppc64V1 && sym.has_opd(&ctx.symbols) { + return ctx.ppc64_opd.as_ref().unwrap().hdr.shndx; + } + if let Some(isec) = sym.input_section_ref() { + if isec.is_alive() { + return ctx.output_section(isec.output_section.unwrap()).hdr.shndx; + } + if isec.is_icf_removed() { + let leader = ctx.section(isec.icf_leader().unwrap()); + return ctx.output_section(leader.output_section.unwrap()).hdr.shndx; + } + } + SHN_UNDEF + }; + + let mut shndx: Option = None; + let isec = sym.input_section_ref(); + + if sym.has_copyrel() { + shndx = Some(if sym.is_copyrel_readonly() { + ctx.copyrel_relro.hdr.shndx + } else { + ctx.copyrel.hdr.shndx + }); + esym.st_value = sym.addr(ctx); + } else if file.is_dso() || sym.is_undef() { + esym.st_shndx = SHN_UNDEF as u16; + esym.st_size = 0; + if sym.is_canonical() { + esym.st_value = sym.plt_addr(ctx); + } + } else if let Some(chunk) = sym.output_chunk() { + // Linker-synthesized symbol + shndx = Some(ctx.chunk_header(chunk).shndx); + esym.st_value = sym.addr(ctx); + } else if let Some(frag) = sym.fragment() { + shndx = Some(ctx.merged_sections[frag.section.index()].hdr.shndx); + esym.st_value = sym.addr(ctx); + } else if isec.is_none() { + if sym.is_common() { + // Common symbols are converted to .bss unless we're creating + // a relocatable output. st_value of a common symbol is its + // alignment. + debug_assert!(ctx.args.relocatable); + esym.st_shndx = SHN_COMMON as u16; + esym.st_value = sym.esym(ctx).st_value; + } else { + esym.st_shndx = SHN_ABS as u16; + esym.st_value = sym.addr(ctx); + } + } else if sym.ty() == STT_TLS { + shndx = Some(st_shndx_of(sym)); + esym.st_value = sym.addr(ctx) - ctx.tls_begin; + } else if sym.is_pde_ifunc(ctx) && sym.has_plt(&ctx.symbols) { + // An IFUNC in a PDE uses two GOT slots and its PLT address. + shndx = Some(st_shndx_of(sym)); + esym.set_type(STT_FUNC); + esym.set_visibility(sym.visibility()); + esym.st_value = sym.plt_addr(ctx); + } else if let Some(isec) = isec.filter(|isec| { + isec.sh_flags & SHF_MERGE as u64 != 0 && isec.sh_flags & SHF_ALLOC as u64 == 0 + }) { + // A symbol in a mergeable non-alloc section, such as .debug_str + let file = &ctx.objs[isec.file.index()]; + let m = file + .mergeable_section(file.shndx_at_in::(sym.sym_idx as usize)) + .expect("mergeable section"); + let (frag, addend) = m.fragment(sym.esym(ctx).st_value).expect("fragment"); + let msec = &ctx.merged_sections[m.parent.index()]; + shndx = Some(msec.hdr.shndx); + esym.set_visibility(sym.visibility()); + esym.st_value = + (msec.hdr.shdr.sh_addr + msec.fragments.get(frag).offset()).wrapping_add(addend as u64); + } else { + shndx = Some(st_shndx_of(sym)); + esym.set_visibility(sym.visibility()); + esym.st_value = sym.addr_with(ctx, AddrFlags::NO_PLT); + } + + // st_shndx is 16 bits; a large index goes to .symtab_shndx. + let mut xindex = 0; + if let Some(shndx) = shndx { + if shndx < SHN_LORESERVE { + esym.st_shndx = shndx as u16; + } else { + esym.st_shndx = SHN_XINDEX as u16; + xindex = shndx; + } + } + (esym, xindex) +} + +/// `.dynsym` holds the symbols used for dynamic linking. +#[derive(Debug)] +pub struct DynsymSection { + pub hdr: ChunkHeader, + /// Index 0 is the null symbol. + pub symbols: Vec>, + pub dynstr_offset: u64, +} + +impl DynsymSection { + pub fn new() -> DynsymSection { + let mut hdr = ChunkHeader::new(".dynsym", SHT_DYNSYM, SHF_ALLOC as u64); + hdr.shdr.sh_entsize = ElfSym::size::() as u64; + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + DynsymSection { + hdr, + symbols: Vec::new(), + dynstr_offset: 0, + } + } +} + +pub mod dynsym { + use super::*; + + pub fn add_symbol(ctx: &mut Context, sym: SymbolId) { + if ctx.dynsym.symbols.is_empty() { + ctx.dynsym.symbols.push(None); + } + if ctx.symbols[sym].dynsym_idx(&ctx.symbols).is_none() { + // A placeholder; the real index is assigned by sort_dynsyms. + ctx.symbols.aux_mut(sym).dynsym_idx = Some(u32::MAX); + ctx.dynsym.symbols.push(Some(sym)); + } + } + + pub fn update_shdr(ctx: &mut Context) { + ctx.dynsym.hdr.shdr.sh_link = ctx.dynstr.hdr.shndx; + ctx.dynsym.hdr.shdr.sh_size = ElfSym::size::() as u64 * ctx.dynsym.symbols.len() as u64; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let size = ElfSym::size::(); + buf[..size].fill(0); + let mut offset = ctx.dynsym.dynstr_offset as u32; + for &id in ctx.dynsym.symbols.iter().skip(1).flatten() { + let sym = &ctx.symbols[id]; + let (esym, xindex) = to_output_esym(ctx, sym, offset); + if xindex != 0 { + let nshdrs = ctx + .shdr + .as_ref() + .map_or(0, |s| s.hdr.shdr.sh_size / ElfShdr::size::() as u64); + error!( + ctx, + "{}: .dynsym: too many output sections: {nshdrs} requested, but ELF allows at most 65279", + ctx.args.output + ); + return; + } + esym.write::(&mut buf[sym.dynsym_idx(&ctx.symbols).unwrap() as usize * size..]); + offset += sym.name().len() as u32 + 1; + } + } +} + +/// The hash function for `.hash`. +pub fn elf_hash(name: &[u8]) -> u32 { + let mut h: u32 = 0; + for &c in name { + h = (h << 4).wrapping_add(c as u32); + let g = h & 0xf000_0000; + if g != 0 { + h ^= g >> 24; + } + h &= !g; + } + h +} + +/// The hash function for `.gnu.hash`. +pub fn djb_hash(name: &[u8]) -> u32 { + let mut h: u32 = 5381; + for &c in name { + h = (h << 5).wrapping_add(h).wrapping_add(c as u32); + } + h +} + +/// `.hash` is the classic on-disk hash table for `.dynsym`. +#[derive(Debug)] +pub struct HashSection { + pub hdr: ChunkHeader, +} + +impl HashSection { + pub fn new() -> HashSection { + let mut hdr = ChunkHeader::new(".hash", SHT_HASH, SHF_ALLOC as u64); + // s390x uses 64-bit entries; it looks like a spec bug but we + // follow suit for compatibility. + let entry = hash::entry_size::() as u64; + hdr.shdr.sh_entsize = entry; + hdr.shdr.sh_addralign = entry; + HashSection { hdr } + } +} + +pub mod hash { + use super::*; + + pub fn entry_size() -> usize { + if E::FAMILY == Family::S390x { + 8 + } else { + 4 + } + } + + pub fn update_shdr(ctx: &mut Context) { + if ctx.dynsym.symbols.is_empty() { + return; + } + let entry = entry_size::() as u64; + let num_slots = ctx.dynsym.symbols.len() as u64; + let hash = ctx.hash.as_mut().unwrap(); + hash.hdr.shdr.sh_size = entry * 2 + num_slots * entry * 2; + hash.hdr.shdr.sh_link = ctx.dynsym.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf.fill(0); + let entry = entry_size::(); + let write = |buf: &mut [u8], i: usize, v: u32| { + if entry == 8 { + E::Endian::write_u64(&mut buf[i * 8..], v as u64); + } else { + E::Endian::write_u32(&mut buf[i * 4..], v); + } + }; + let read = |buf: &[u8], i: usize| -> u32 { + if entry == 8 { + E::Endian::read_u64(&buf[i * 8..]) as u32 + } else { + E::Endian::read_u32(&buf[i * 4..]) + } + }; + + let n = ctx.dynsym.symbols.len(); + write(buf, 0, n as u32); + write(buf, 1, n as u32); + let buckets = 2; + let chains = 2 + n; + + for &id in ctx.dynsym.symbols.iter().skip(1).flatten() { + let sym = &ctx.symbols[id]; + let i = sym.dynsym_idx(&ctx.symbols).unwrap() as usize; + let h = elf_hash(sym.name()) as usize % n; + let head = read(buf, buckets + h); + write(buf, chains + i, head); + write(buf, buckets + h, i as u32); + } + } +} + +/// `.gnu.hash` adds a bloom filter to speed up negative lookups. +#[derive(Debug)] +pub struct GnuHashSection { + pub hdr: ChunkHeader, + pub num_buckets: u32, + pub num_bloom: u32, + pub num_exported: u32, +} + +impl GnuHashSection { + pub const LOAD_FACTOR: u32 = 8; + pub const HEADER_SIZE: u64 = 16; + pub const BLOOM_SHIFT: u32 = 26; + + pub fn new() -> GnuHashSection { + let mut hdr = ChunkHeader::new(".gnu.hash", SHT_GNU_HASH, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = E::WORD_SIZE as u64; + GnuHashSection { + hdr, + num_buckets: 0, + num_bloom: 1, + num_exported: 0, + } + } +} + +pub mod gnu_hash { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + if ctx.dynsym.symbols.is_empty() { + return; + } + let word = E::WORD_SIZE as u64; + let gh = ctx.gnu_hash.as_mut().unwrap(); + // 12 bits per symbol in the bloom filter. + gh.num_bloom = ((gh.num_exported as u64 * 12) / (word * 8)) + .max(1) + .next_power_of_two() as u32; + gh.hdr.shdr.sh_size = GnuHashSection::HEADER_SIZE + + gh.num_bloom as u64 * word + + gh.num_buckets as u64 * 4 + + gh.num_exported as u64 * 4; + gh.hdr.shdr.sh_link = ctx.dynsym.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf.fill(0); + let gh = ctx.gnu_hash.as_ref().unwrap(); + let word = E::WORD_SIZE; + let first_exported = ctx.dynsym.symbols.len() - gh.num_exported as usize; + + E::Endian::write_u32(buf, gh.num_buckets); + E::Endian::write_u32(&mut buf[4..], first_exported as u32); + E::Endian::write_u32(&mut buf[8..], gh.num_bloom); + E::Endian::write_u32(&mut buf[12..], GnuHashSection::BLOOM_SHIFT); + + let syms: Vec = ctx.dynsym.symbols[first_exported..] + .iter() + .flatten() + .copied() + .collect(); + if syms.is_empty() { + return; + } + + // Bloom filter + let bloom_off = GnuHashSection::HEADER_SIZE as usize; + let word_bits = word * 8; + let mut indices = Vec::with_capacity(syms.len()); + for &id in &syms { + let h = ctx.symbols[id].aux(&ctx.symbols).unwrap().djb_hash; + indices.push(h % gh.num_buckets); + let idx = (h as usize / word_bits) % gh.num_bloom as usize; + let bits = (1u64 << (h as usize % word_bits)) + | (1u64 << ((h >> GnuHashSection::BLOOM_SHIFT) as usize % word_bits)); + let slot = &mut buf[bloom_off + idx * word..]; + if E::IS_64 { + E::Endian::write_u64(slot, E::Endian::read_u64(slot) | bits); + } else { + E::Endian::write_u32(slot, E::Endian::read_u32(slot) | bits as u32); + } + } + + // Hash buckets + let buckets_off = bloom_off + gh.num_bloom as usize * word; + for (i, &bucket) in indices.iter().enumerate().rev() { + E::Endian::write_u32( + &mut buf[buckets_off + bucket as usize * 4..], + (first_exported + i) as u32, + ); + } + + // Hash values; the last entry of a chain has its LSB set. + let table_off = buckets_off + gh.num_buckets as usize * 4; + for (i, &id) in syms.iter().enumerate() { + let h = ctx.symbols[id].aux(&ctx.symbols).unwrap().djb_hash; + let last = i + 1 == syms.len() || indices[i] != indices[i + 1]; + E::Endian::write_u32( + &mut buf[table_off + i * 4..], + if last { h | 1 } else { h & !1 }, + ); + } + } +} + +/// Chunk ids for which dynamic symbol information must exist. +pub fn require_dynsym(ctx: &Context) { + if ctx.dynsym.symbols.is_empty() { + fatal!(ctx, "internal error: dynamic symbol table required"); + } +} + +/// Whether the symbol belongs to the output rather than a DSO. +pub fn is_defined_in_output(sym: &Symbol) -> bool { + matches!(sym.file(), Some(FileId::Obj(_))) +} + +/// The chunk id of `.symtab_shndx` if it exists. +pub fn symtab_shndx_id(ctx: &Context) -> Option { + ctx.symtab_shndx.as_ref().map(|_| ChunkId::SymtabShndx) +} diff --git a/src/chunks/version.rs b/src/chunks/version.rs new file mode 100644 index 000000000..2baa47169 --- /dev/null +++ b/src/chunks/version.rs @@ -0,0 +1,371 @@ +//! Symbol versioning sections: `.gnu.version`, `.gnu.version_r` and +//! `.gnu.version_d`. + +use crate::arch::Arch; +use crate::chunks::symtab::elf_hash; +use crate::chunks::ChunkHeader; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::{DsoId, FileId}; +use crate::symbol::SymbolId; +use crate::util::path_filename; + +/// `.gnu.version` is a table parallel to `.dynsym` giving each symbol's +/// version index. +#[derive(Debug)] +pub struct VersymSection { + pub hdr: ChunkHeader, + pub contents: Vec, +} + +impl VersymSection { + pub fn new() -> VersymSection { + let mut hdr = ChunkHeader::new(".gnu.version", SHT_GNU_VERSYM, SHF_ALLOC as u64); + hdr.shdr.sh_entsize = 2; + hdr.shdr.sh_addralign = 2; + VersymSection { + hdr, + contents: Vec::new(), + } + } +} + +impl Default for VersymSection { + fn default() -> Self { + Self::new() + } +} + +pub mod versym { + use super::*; + + pub fn update_shdr(ctx: &mut Context) { + ctx.versym.hdr.shdr.sh_size = ctx.versym.contents.len() as u64 * 2; + ctx.versym.hdr.shdr.sh_link = ctx.dynsym.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + for (i, &v) in ctx.versym.contents.iter().enumerate() { + E::Endian::write_u16(&mut buf[i * 2..], v); + } + } +} + +/// `.gnu.version_r` lists the versions required from each shared library. +#[derive(Debug)] +pub struct VerneedSection { + pub hdr: ChunkHeader, + pub contents: Vec, +} + +impl VerneedSection { + pub fn new() -> VerneedSection { + let mut hdr = ChunkHeader::new(".gnu.version_r", SHT_GNU_VERNEED, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + VerneedSection { + hdr, + contents: Vec::new(), + } + } +} + +impl Default for VerneedSection { + fn default() -> Self { + Self::new() + } +} + +/// `-z pack-relative-relocs` outputs crash on glibc before 2.38, which +/// doesn't understand RELR. Adding a dependency on the dummy version +/// `GLIBC_ABI_DT_RELR` turns that into a friendlier "version not found" +/// error; newer glibc knows the name and ignores it. +fn is_glibc2(dso: &crate::input_files::SharedFile) -> bool { + dso.soname.starts_with("libc.so.") + && dso + .version_strings + .iter() + .any(|v| v.starts_with(b"GLIBC_2.")) +} + +pub mod verneed { + use super::*; + + pub fn construct(ctx: &mut Context) { + let _t = ctx.timer("fill_verneed"); + + // Versioned symbols, sorted by file and version. + let mut syms: Vec<(DsoId, SymbolId)> = ctx + .dynsym + .symbols + .iter() + .skip(1) + .flatten() + .filter_map(|&id| { + let sym = &ctx.symbols[id]; + match sym.file() { + Some(FileId::Dso(dso)) if (sym.ver_idx as u32) > VER_NDX_LAST_RESERVED => { + Some((dso, id)) + } + _ => None, + } + }) + .collect(); + if syms.is_empty() { + return; + } + syms.sort_by_key(|&(dso, id)| { + ( + ctx.dsos[dso.index()].soname.clone(), + ctx.symbols[id].ver_idx, + ) + }); + + let n = ctx.dynsym.symbols.len(); + ctx.versym.contents.resize(n, VER_NDX_GLOBAL as u16); + ctx.versym.contents[0] = VER_NDX_LOCAL as u16; + + let mut builder = VerneedBuilder:: { + contents: Vec::new(), + veridx: VER_NDX_LAST_RESERVED as u16 + ctx.args.version_definitions.len() as u16, + num_groups: 0, + group_pos: None, + aux_pos: None, + _arch: std::marker::PhantomData, + }; + + for i in 0..syms.len() { + let (dso, id) = syms[i]; + let start_group = i == 0 || syms[i - 1].0 != dso; + if start_group { + let soname = ctx.dsos[dso.index()].soname.clone(); + let vn_file = ctx.dynstr.find_string(soname.as_bytes()) as u32; + builder.start_group(vn_file); + if ctx.args.pack_dyn_relocs_relr && is_glibc2(&ctx.dsos[dso.index()]) { + builder.add_entry(&mut ctx.dynstr, b"GLIBC_ABI_DT_RELR"); + } + let version = ctx.symbols[id].version(ctx); + builder.add_entry(&mut ctx.dynstr, version); + } else if ctx.symbols[syms[i - 1].1].ver_idx != ctx.symbols[id].ver_idx { + let version = ctx.symbols[id].version(ctx); + builder.add_entry(&mut ctx.dynstr, version); + } + let dynsym_idx = ctx.symbols[id].dynsym_idx(&ctx.symbols).unwrap() as usize; + ctx.versym.contents[dynsym_idx] = builder.veridx; + } + + ctx.verneed.hdr.shdr.sh_info = builder.num_groups; + ctx.verneed.contents = builder.contents; + } + + /// Builds `.gnu.version_r` incrementally: a verneed entry per shared + /// library, each followed by its vernaux entries. + struct VerneedBuilder { + contents: Vec, + veridx: u16, + num_groups: u32, + /// Positions of the current group's verneed and last vernaux. + group_pos: Option, + aux_pos: Option, + _arch: std::marker::PhantomData, + } + + impl VerneedBuilder { + fn start_group(&mut self, vn_file: u32) { + self.num_groups += 1; + if let Some(gp) = self.group_pos { + let mut vn = ElfVerneed::parse::(&self.contents[gp..]); + vn.vn_next = (self.contents.len() - gp) as u32; + vn.write::(&mut self.contents[gp..]); + } + let pos = self.contents.len(); + self.group_pos = Some(pos); + self.aux_pos = None; + self.contents.resize(pos + ElfVerneed::size::(), 0); + ElfVerneed { + vn_version: 1, + vn_cnt: 0, + vn_file, + vn_aux: ElfVerneed::size::() as u32, + vn_next: 0, + } + .write::(&mut self.contents[pos..]); + } + + fn add_entry(&mut self, dynstr: &mut super::super::symtab::DynstrSection, verstr: &[u8]) { + let gp = self.group_pos.unwrap(); + let mut vn = ElfVerneed::parse::(&self.contents[gp..]); + vn.vn_cnt += 1; + vn.write::(&mut self.contents[gp..]); + if let Some(ap) = self.aux_pos { + let mut aux = ElfVernaux::parse::(&self.contents[ap..]); + aux.vna_next = ElfVernaux::size::() as u32; + aux.write::(&mut self.contents[ap..]); + } + self.veridx += 1; + let aux = ElfVernaux { + vna_hash: elf_hash(verstr), + vna_flags: 0, + vna_other: self.veridx, + vna_name: dynstr.add_string(verstr) as u32, + vna_next: 0, + }; + let pos = self.contents.len(); + self.aux_pos = Some(pos); + self.contents.resize(pos + ElfVernaux::size::(), 0); + aux.write::(&mut self.contents[pos..]); + } + } + + pub fn update_shdr(ctx: &mut Context) { + ctx.verneed.hdr.shdr.sh_size = ctx.verneed.contents.len() as u64; + ctx.verneed.hdr.shdr.sh_link = ctx.dynstr.hdr.shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + buf[..ctx.verneed.contents.len()].copy_from_slice(&ctx.verneed.contents); + } +} + +/// `.gnu.version_d` defines the versions of the symbols this file +/// exports. It exists only in shared libraries. +#[derive(Debug)] +pub struct VerdefSection { + pub hdr: ChunkHeader, + pub contents: Vec, +} + +impl VerdefSection { + pub fn new() -> VerdefSection { + let mut hdr = ChunkHeader::new(".gnu.version_d", SHT_GNU_VERDEF, SHF_ALLOC as u64); + hdr.shdr.sh_addralign = 4; + VerdefSection { + hdr, + contents: Vec::new(), + } + } +} + +impl Default for VerdefSection { + fn default() -> Self { + Self::new() + } +} + +pub mod verdef { + use super::*; + + pub fn construct(ctx: &mut Context) { + let _t = ctx.timer("fill_verdef"); + if ctx.args.version_definitions.is_empty() { + return; + } + + // --default-symver + if ctx.args.default_symver { + for &id in ctx.dynsym.symbols.iter().flatten() { + let sym = &mut ctx.symbols[id]; + if matches!(sym.file(), Some(FileId::Obj(_))) && !sym.is_undef() { + let ver = sym.ver_idx as u32; + if ver == VER_NDX_GLOBAL || ver == VER_NDX_UNSPECIFIED { + sym.ver_idx = VER_NDX_LAST_RESERVED as u16 + 1; + } + } + } + } + + let n = ctx.dynsym.symbols.len(); + ctx.versym.contents.resize(n, VER_NDX_GLOBAL as u16); + ctx.versym.contents[0] = VER_NDX_LOCAL as u16; + + for &id in ctx.dynsym.symbols.iter().flatten() { + let sym = &ctx.symbols[id]; + if !matches!(sym.file(), Some(FileId::Obj(_))) { + continue; + } + let idx = sym.dynsym_idx(&ctx.symbols).unwrap() as usize; + // An unversioned undefined symbol takes version index 0. + if sym.ver_idx as u32 != VER_NDX_UNSPECIFIED { + ctx.versym.contents[idx] = sym.ver_idx; + } else if sym.is_undef() { + ctx.versym.contents[idx] = VER_NDX_LOCAL as u16; + } + } + + let mut contents: Vec = Vec::new(); + let verdef_size = ElfVerdef::size::(); + let verdaux_size = ElfVerdaux::size::(); + let mut prev: Option = None; + let mut count = 0u32; + + let mut write = |contents: &mut Vec, + dynstr: &mut super::super::symtab::DynstrSection, + verstr: &[u8], + idx: u16, + flags: u16| { + count += 1; + if let Some(p) = prev { + let mut vd = ElfVerdef::parse::(&contents[p..]); + vd.vd_next = (contents.len() - p) as u32; + vd.write::(&mut contents[p..]); + } + let pos = contents.len(); + prev = Some(pos); + contents.resize(pos + verdef_size + verdaux_size, 0); + ElfVerdef { + vd_version: 1, + vd_flags: flags, + vd_ndx: idx, + vd_cnt: 1, + vd_hash: elf_hash(verstr), + vd_aux: verdef_size as u32, + vd_next: 0, + } + .write::(&mut contents[pos..]); + ElfVerdaux { + vda_name: dynstr.add_string(verstr) as u32, + vda_next: 0, + } + .write::(&mut contents[pos + verdef_size..]); + }; + + let soname = if ctx.args.soname.is_empty() { + path_filename(&ctx.args.output) + } else { + ctx.args.soname.clone() + }; + write( + &mut contents, + &mut ctx.dynstr, + soname.as_bytes(), + 1, + VER_FLG_BASE as u16, + ); + + let defs = ctx.args.version_definitions.clone(); + for (i, verstr) in defs.iter().enumerate() { + write( + &mut contents, + &mut ctx.dynstr, + verstr.as_bytes(), + VER_NDX_LAST_RESERVED as u16 + 1 + i as u16, + 0, + ); + } + + let verdef = ctx.verdef.as_mut().unwrap(); + verdef.hdr.shdr.sh_info = count; + verdef.contents = contents; + } + + pub fn update_shdr(ctx: &mut Context) { + let shndx = ctx.dynstr.hdr.shndx; + let verdef = ctx.verdef.as_mut().unwrap(); + verdef.hdr.shdr.sh_size = verdef.contents.len() as u64; + verdef.hdr.shdr.sh_link = shndx; + } + + pub fn copy_buf(ctx: &Context, buf: &mut [u8]) { + let verdef = ctx.verdef.as_ref().unwrap(); + buf[..verdef.contents.len()].copy_from_slice(&verdef.contents); + } +} diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 000000000..bd05752b2 --- /dev/null +++ b/src/context.rs @@ -0,0 +1,598 @@ +//! The linker context: everything about one link, from parsed arguments +//! to output chunks. + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::marker::PhantomData; +use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; + +use crate::arch::Arch; +use crate::args::Args; +use crate::chunks::dynamic::{DynamicSection, RelDynSection, RelrDynSection}; +use crate::chunks::eh_frame::{EhFrameHdrSection, EhFrameRelocSection, EhFrameSection}; +use crate::chunks::got::{GotPltSection, GotSection, PltGotSection, PltSection, RelPltSection}; +use crate::chunks::merged::{MergedSection, MergedSectionId}; +use crate::chunks::misc::{ + BuildIdSection, ComdatGroupSection, CompressedSection, CopyrelSection, GnuDebuglinkSection, + InterpSection, NotePackageSection, NotePropertySection, RelocSection, RelroPaddingSection, + RiscvAttributesSection, +}; +use crate::chunks::output_section::OutputSection; +use crate::chunks::sframe::{SFrameRelocSection, SFrameSection}; +use crate::chunks::symtab::{ + DynstrSection, DynsymSection, GnuHashSection, HashSection, ShstrtabSection, StrtabSection, + SymtabSection, SymtabShndxSection, +}; +use crate::chunks::version::{VerdefSection, VerneedSection, VersymSection}; +use crate::chunks::{ + ChunkHeader, ChunkId, GdbIndexSection, OutputEhdr, OutputPhdr, OutputSectionId, OutputShdr, +}; +use crate::diagnostics::{Diagnostics, HasDiagnostics}; +use crate::elf::ElfSym; +use crate::input_files::{DsoId, FileId, InputFile, ObjId, ObjectFile, SharedFile}; +use crate::input_sections::{ + FragmentRef, InputSection, InputSectionId, SectionArena, SectionFragment, SectionRef, +}; +use crate::linker_script::{DynamicPattern, VersionPattern}; +use crate::mapped_file::MappedFile; +use crate::symbol::{Bins, SymbolId, SymbolSlot, SymbolTable}; +use crate::util::timer::Timers; + +/// Linker-synthesized symbols with well-known names. +#[derive(Debug, Default)] +pub struct SyntheticSymbols { + pub dynamic: Option, + pub global_offset_table: Option, + pub procedure_linkage_table: Option, + pub tls_module_base: Option, + pub gnu_eh_frame_hdr: Option, + pub bss_start: Option, + pub dso_handle: Option, + pub ehdr_start: Option, + pub executable_start: Option, + pub exidx_end: Option, + pub exidx_start: Option, + pub fini_array_end: Option, + pub fini_array_start: Option, + pub global_pointer: Option, + pub init_array_end: Option, + pub init_array_start: Option, + pub preinit_array_end: Option, + pub preinit_array_start: Option, + pub rel_iplt_end: Option, + pub rel_iplt_start: Option, + pub edata_: Option, + pub end_: Option, + pub etext_: Option, + pub edata: Option, + pub end: Option, + pub etext: Option, + pub toc: Option, + pub sda_base: Option, + pub tls_get_addr: Option, + + /// The entry point, `-init` and `-fini` symbols. + pub entry: SymbolId, + pub init: SymbolId, + pub fini: SymbolId, +} + +pub struct Context { + pub args: Args, + pub diag: Arc, + pub cmdline_args: Vec, + pub timers: Timers, + + pub symbols: SymbolTable, + + /// Global symbol keys recorded while input files are parsed, one bin per + /// Rayon worker plus one for callers outside the pool. + symbol_bins: OnceLock>>>, + + pub objs: Vec>, + pub dsos: Vec>, + + /// Files removed from the live vectors remain allocated until teardown, + /// as C++ mold's arena-owned InputFiles do. + pub discarded_objs: Vec>, + pub discarded_dsos: Vec>, + + // Declared after all object files so their SectionLists are dropped + // before the arena releases its backing mapping. + pub(crate) section_arena: SectionArena, + + /// Files indexed by priority, for decoding symbol resolution results. + pub file_by_priority: Vec>, + + /// Files read so far with their command line positions, in the + /// arbitrary order the parallel reader found them. + pub pending_files: Vec<(Vec, FileId)>, + + /// Sonames of all shared libraries given to the linker, including + /// ones later dropped as unneeded; --no-allow-shlib-undefined can + /// only be checked if the set of libraries is complete. + pub dso_sonames: HashSet, + + /// Symbols defined by shared libraries that were dropped as unneeded. + /// Like mold, --no-allow-shlib-undefined still considers them provided. + pub dropped_dso_definitions: HashSet, + + /// IR objects the LTO plugin consumed. They leave the link once + /// compiled, but the output still depends on them. + pub lto_input_files: Vec<(u32, &'static MappedFile)>, + + pub internal_obj: Option, + pub internal_esyms: Vec, + + pub output_sections: Vec, + pub merged_sections: Vec, + pub reloc_sections: Vec, + pub comdat_group_sections: Vec, + pub compressed_sections: Vec, + pub placeholders: Vec, + + /// The chunks that make up the output, in file order. + pub chunks: Vec, + + /// Debug chunks set aside for `--separate-debug-file`. + pub debug_chunks: Vec, + + pub ehdr: Option, + pub phdr: Option, + pub shdr: Option, + pub interp: Option, + pub got: GotSection, + pub gotplt: GotPltSection, + pub relplt: RelPltSection, + pub reldyn: RelDynSection, + pub relrdyn: Option, + pub dynamic: Option, + pub strtab: StrtabSection, + pub dynstr: DynstrSection, + pub hash: Option, + pub gnu_hash: Option, + pub gnu_debuglink: Option, + pub shstrtab: Option, + pub plt: PltSection, + pub pltgot: PltGotSection, + pub symtab: SymtabSection, + pub symtab_shndx: Option, + pub dynsym: DynsymSection, + pub eh_frame: EhFrameSection, + pub eh_frame_hdr: Option, + pub eh_frame_reloc: Option, + pub sframe: SFrameSection, + pub sframe_reloc: Option, + pub copyrel: CopyrelSection, + pub copyrel_relro: CopyrelSection, + pub versym: VersymSection, + pub verneed: VerneedSection, + pub verdef: Option, + pub buildid: Option, + pub note_package: NotePackageSection, + pub note_property: Option, + pub riscv_attributes: Option, + pub arm_exidx: Option, + pub ppc64_save_restore: Option, + pub ppc64_opd: Option, + /// Whether any input uses Power10 PC-relative calls, which decides + /// how thunks address their targets. + pub is_power10: AtomicBool, + pub gdb_index: Option, + pub gdb_index_data: Option, + pub relro_padding: Option, + pub comment: Option, + + pub needs_tlsld: AtomicBool, + pub has_textrel: AtomicBool, + pub num_ifunc_dynrels: AtomicU32, + + pub undef_errors: Mutex>>, + + pub version_patterns: Vec, + pub dynamic_list_patterns: Vec, + pub default_version: u16, + pub page_size: u64, + + pub tls_begin: u64, + pub tp_addr: u64, + pub dtp_addr: u64, + + pub syms: SyntheticSymbols, + + /// The size of the output file once the layout is fixed. + pub filesize: u64, + + _arch: PhantomData, +} + +impl HasDiagnostics for Context { + fn diagnostics(&self) -> &Diagnostics { + &self.diag + } +} + +impl Context { + pub fn new(args: Args, diag: Diagnostics, cmdline_args: Vec) -> Context { + let mut symbols = SymbolTable::new(); + let syms = SyntheticSymbols { + entry: symbols.intern(crate::util::leak_bytes(args.entry.clone().into_bytes())), + init: symbols.intern(crate::util::leak_bytes(args.init.clone().into_bytes())), + fini: symbols.intern(crate::util::leak_bytes(args.fini.clone().into_bytes())), + ..SyntheticSymbols::default() + }; + let page_size = args.page_size; + + Context { + reldyn: RelDynSection::new::(&args), + got: GotSection::new::(), + gotplt: GotPltSection::new::(&args), + relplt: RelPltSection::new::(), + strtab: StrtabSection::new(), + dynstr: DynstrSection::new(), + plt: PltSection::new::(), + pltgot: PltGotSection::new(), + symtab: SymtabSection::new::(), + dynsym: DynsymSection::new::(), + eh_frame: EhFrameSection::new::(), + sframe: SFrameSection::new(), + copyrel: CopyrelSection::new(false), + copyrel_relro: CopyrelSection::new(true), + versym: VersymSection::new(), + verneed: VerneedSection::new(), + note_package: NotePackageSection::new(), + args, + diag: Arc::new(diag), + cmdline_args, + timers: Timers::new(), + symbols, + symbol_bins: OnceLock::new(), + objs: Vec::new(), + dsos: Vec::new(), + discarded_objs: Vec::new(), + discarded_dsos: Vec::new(), + section_arena: SectionArena::new(), + file_by_priority: Vec::new(), + pending_files: Vec::new(), + dso_sonames: HashSet::new(), + dropped_dso_definitions: HashSet::new(), + lto_input_files: Vec::new(), + internal_obj: None, + internal_esyms: Vec::new(), + output_sections: Vec::new(), + merged_sections: Vec::new(), + reloc_sections: Vec::new(), + comdat_group_sections: Vec::new(), + compressed_sections: Vec::new(), + placeholders: Vec::new(), + chunks: Vec::new(), + debug_chunks: Vec::new(), + ehdr: None, + phdr: None, + shdr: None, + interp: None, + relrdyn: None, + dynamic: None, + hash: None, + gnu_hash: None, + gnu_debuglink: None, + shstrtab: None, + symtab_shndx: None, + eh_frame_hdr: None, + eh_frame_reloc: None, + sframe_reloc: None, + verdef: None, + buildid: None, + note_property: None, + riscv_attributes: None, + arm_exidx: None, + ppc64_save_restore: None, + ppc64_opd: None, + is_power10: AtomicBool::new(false), + gdb_index: None, + gdb_index_data: None, + relro_padding: None, + comment: None, + needs_tlsld: AtomicBool::new(false), + has_textrel: AtomicBool::new(false), + num_ifunc_dynrels: AtomicU32::new(0), + undef_errors: Mutex::new(HashMap::new()), + version_patterns: Vec::new(), + dynamic_list_patterns: Vec::new(), + default_version: crate::elf::VER_NDX_UNSPECIFIED as u16, + page_size, + tls_begin: 0, + tp_addr: 0, + dtp_addr: 0, + syms, + filesize: 0, + _arch: PhantomData, + } + } + + pub fn checkpoint(&self) { + self.diag.checkpoint(); + } + + /// Returns this worker's symbol bin. Looking it up once per file keeps the + /// synchronization cost outside the per-symbol loop. + pub(crate) fn symbol_bin(&self) -> MutexGuard<'_, Bins> { + let bins = self.symbol_bins.get_or_init(|| { + let workers = rayon::current_num_threads(); + (0..=workers).map(|_| Mutex::new(Bins::new())).collect() + }); + let fallback = bins.len() - 1; + let index = rayon::current_thread_index() + .unwrap_or(fallback) + .min(fallback); + bins[index].lock().unwrap() + } + + /// Takes all keys recorded since the previous gather, leaving empty bins + /// in place for an LTO output file to use before the second gather. + pub(crate) fn take_symbol_bins(&mut self) -> Vec> { + self.symbol_bins + .get_mut() + .map(|bins| { + bins.iter_mut() + .map(|bin| std::mem::take(bin.get_mut().unwrap())) + .collect() + }) + .unwrap_or_default() + } + + /// Interns a global symbol by name. + pub fn get_symbol(&mut self, name: &[u8]) -> SymbolId { + if let Some(id) = self.symbols.lookup(name) { + return id; + } + self.symbols.intern(crate::util::leak_bytes(name.to_vec())) + } + + pub fn obj(&self, id: ObjId) -> &ObjectFile { + &self.objs[id.index()] + } + + pub fn dso(&self, id: DsoId) -> &SharedFile { + &self.dsos[id.index()] + } + + /// The common part of a file. + pub fn file(&self, id: FileId) -> &InputFile { + match id { + FileId::Obj(id) => &self.objs[id.index()].base, + FileId::Dso(id) => &self.dsos[id.index()].base, + } + } + + pub fn file_mut(&mut self, id: FileId) -> &mut InputFile { + match id { + FileId::Obj(id) => &mut self.objs[id.index()].base, + FileId::Dso(id) => &mut self.dsos[id.index()].base, + } + } + + /// Formats a file for diagnostics. + pub fn file_display(&self, id: FileId) -> &dyn fmt::Display { + match id { + FileId::Obj(id) => &self.objs[id.index()], + FileId::Dso(id) => &self.dsos[id.index()], + } + } + + pub fn section(&self, r: SectionRef) -> &InputSection { + self.objs[r.file.index()].section_at(r.shndx) + } + + pub fn try_section(&self, r: SectionRef) -> Option<&InputSection> { + self.objs[r.file.index()].section(r.shndx as usize) + } + + pub fn section_mut(&mut self, r: SectionRef) -> &mut InputSection { + self.objs[r.file.index()] + .section_mut(r.shndx as usize) + .expect("no such input section") + } + + #[inline] + pub fn input_section(&self, id: InputSectionId) -> &InputSection { + self.section_arena.section(id) + } + + #[inline] + pub fn input_section_mut(&mut self, id: InputSectionId) -> &mut InputSection { + self.section_arena.section_mut(id) + } + + pub fn set_r_deltas( + &mut self, + r: SectionRef, + deltas: Box<[crate::input_sections::RelocDelta]>, + ) { + let Context { + objs, + section_arena, + .. + } = self; + objs[r.file.index()] + .section_mut(r.shndx as usize) + .expect("no such input section") + .set_r_deltas(deltas, section_arena); + } + + /// Formats an input section for diagnostics. + pub fn section_display(&self, r: SectionRef) -> impl fmt::Display + '_ { + let file = &self.objs[r.file.index()]; + file.section_at(r.shndx).display(file) + } + + /// Formats an arena-addressed input section for diagnostics. + pub fn input_section_display(&self, id: InputSectionId) -> impl fmt::Display + '_ { + let isec = self.input_section(id); + isec.display(&self.objs[isec.file.index()]) + } + + pub fn fragment(&self, r: FragmentRef) -> &SectionFragment { + self.merged_sections[r.section.index()] + .fragments + .get(r.entry) + } + + pub fn fragment_addr(&self, r: FragmentRef) -> u64 { + let msec = &self.merged_sections[r.section.index()]; + msec.hdr.shdr.sh_addr + msec.fragments.get(r.entry).offset() + } + + pub fn output_section(&self, id: OutputSectionId) -> &OutputSection { + &self.output_sections[id.index()] + } + + /// The address of a PPC64 ELFv1 function descriptor. + pub fn opd_addr(&self, idx: u32) -> u64 { + let opd = self + .ppc64_opd + .as_ref() + .expect("PPC64 ELFv1 has an .opd section"); + opd.hdr.shdr.sh_addr + idx as u64 * crate::chunks::opd::ENTRY_SIZE + } + + /// Whether the file is the internal object holding synthesized symbols. + pub fn is_internal(&self, id: ObjId) -> bool { + self.internal_obj == Some(id) + } + + /// The header of any chunk. Panics if the chunk does not exist. + pub fn chunk_header(&self, id: ChunkId) -> &ChunkHeader { + macro_rules! opt { + ($e:expr) => { + &$e.as_ref().expect("chunk does not exist").hdr + }; + } + match id { + ChunkId::Ehdr => opt!(self.ehdr), + ChunkId::Phdr => opt!(self.phdr), + ChunkId::Shdr => opt!(self.shdr), + ChunkId::Interp => opt!(self.interp), + ChunkId::Got => &self.got.hdr, + ChunkId::GotPlt => &self.gotplt.hdr, + ChunkId::RelPlt => &self.relplt.hdr, + ChunkId::RelDyn => &self.reldyn.hdr, + ChunkId::RelrDyn => opt!(self.relrdyn), + ChunkId::Dynamic => opt!(self.dynamic), + ChunkId::Strtab => &self.strtab.hdr, + ChunkId::Dynstr => &self.dynstr.hdr, + ChunkId::Hash => opt!(self.hash), + ChunkId::GnuHash => opt!(self.gnu_hash), + ChunkId::GnuDebuglink => opt!(self.gnu_debuglink), + ChunkId::Shstrtab => opt!(self.shstrtab), + ChunkId::Plt => &self.plt.hdr, + ChunkId::PltGot => &self.pltgot.hdr, + ChunkId::Symtab => &self.symtab.hdr, + ChunkId::SymtabShndx => opt!(self.symtab_shndx), + ChunkId::Dynsym => &self.dynsym.hdr, + ChunkId::EhFrame => &self.eh_frame.hdr, + ChunkId::EhFrameHdr => opt!(self.eh_frame_hdr), + ChunkId::EhFrameReloc => opt!(self.eh_frame_reloc), + ChunkId::SFrame => &self.sframe.hdr, + ChunkId::SFrameReloc => opt!(self.sframe_reloc), + ChunkId::Copyrel => &self.copyrel.hdr, + ChunkId::CopyrelRelro => &self.copyrel_relro.hdr, + ChunkId::Versym => &self.versym.hdr, + ChunkId::Verneed => &self.verneed.hdr, + ChunkId::Verdef => opt!(self.verdef), + ChunkId::BuildId => opt!(self.buildid), + ChunkId::NotePackage => &self.note_package.hdr, + ChunkId::NoteProperty => opt!(self.note_property), + ChunkId::RiscvAttributes => opt!(self.riscv_attributes), + ChunkId::ArmExidx => opt!(self.arm_exidx), + ChunkId::Ppc64SaveRestore => opt!(self.ppc64_save_restore), + ChunkId::Ppc64Opd => opt!(self.ppc64_opd), + ChunkId::GdbIndex => opt!(self.gdb_index), + ChunkId::RelroPadding => opt!(self.relro_padding), + ChunkId::Output(id) => &self.output_sections[id.index()].hdr, + ChunkId::Merged(id) => &self.merged_sections[id.index()].hdr, + ChunkId::Reloc(i) => &self.reloc_sections[i as usize].hdr, + ChunkId::ComdatGroup(i) => &self.comdat_group_sections[i as usize].hdr, + ChunkId::Compressed(i) => &self.compressed_sections[i as usize].hdr, + ChunkId::Placeholder(i) => &self.placeholders[i as usize], + } + } + + pub fn chunk_header_mut(&mut self, id: ChunkId) -> &mut ChunkHeader { + macro_rules! opt { + ($e:expr) => { + &mut $e.as_mut().expect("chunk does not exist").hdr + }; + } + match id { + ChunkId::Ehdr => opt!(self.ehdr), + ChunkId::Phdr => opt!(self.phdr), + ChunkId::Shdr => opt!(self.shdr), + ChunkId::Interp => opt!(self.interp), + ChunkId::Got => &mut self.got.hdr, + ChunkId::GotPlt => &mut self.gotplt.hdr, + ChunkId::RelPlt => &mut self.relplt.hdr, + ChunkId::RelDyn => &mut self.reldyn.hdr, + ChunkId::RelrDyn => opt!(self.relrdyn), + ChunkId::Dynamic => opt!(self.dynamic), + ChunkId::Strtab => &mut self.strtab.hdr, + ChunkId::Dynstr => &mut self.dynstr.hdr, + ChunkId::Hash => opt!(self.hash), + ChunkId::GnuHash => opt!(self.gnu_hash), + ChunkId::GnuDebuglink => opt!(self.gnu_debuglink), + ChunkId::Shstrtab => opt!(self.shstrtab), + ChunkId::Plt => &mut self.plt.hdr, + ChunkId::PltGot => &mut self.pltgot.hdr, + ChunkId::Symtab => &mut self.symtab.hdr, + ChunkId::SymtabShndx => opt!(self.symtab_shndx), + ChunkId::Dynsym => &mut self.dynsym.hdr, + ChunkId::EhFrame => &mut self.eh_frame.hdr, + ChunkId::EhFrameHdr => opt!(self.eh_frame_hdr), + ChunkId::EhFrameReloc => opt!(self.eh_frame_reloc), + ChunkId::SFrame => &mut self.sframe.hdr, + ChunkId::SFrameReloc => opt!(self.sframe_reloc), + ChunkId::Copyrel => &mut self.copyrel.hdr, + ChunkId::CopyrelRelro => &mut self.copyrel_relro.hdr, + ChunkId::Versym => &mut self.versym.hdr, + ChunkId::Verneed => &mut self.verneed.hdr, + ChunkId::Verdef => opt!(self.verdef), + ChunkId::BuildId => opt!(self.buildid), + ChunkId::NotePackage => &mut self.note_package.hdr, + ChunkId::NoteProperty => opt!(self.note_property), + ChunkId::RiscvAttributes => opt!(self.riscv_attributes), + ChunkId::ArmExidx => opt!(self.arm_exidx), + ChunkId::Ppc64SaveRestore => opt!(self.ppc64_save_restore), + ChunkId::Ppc64Opd => opt!(self.ppc64_opd), + ChunkId::GdbIndex => opt!(self.gdb_index), + ChunkId::RelroPadding => opt!(self.relro_padding), + ChunkId::Output(id) => &mut self.output_sections[id.index()].hdr, + ChunkId::Merged(id) => &mut self.merged_sections[id.index()].hdr, + ChunkId::Reloc(i) => &mut self.reloc_sections[i as usize].hdr, + ChunkId::ComdatGroup(i) => &mut self.comdat_group_sections[i as usize].hdr, + ChunkId::Compressed(i) => &mut self.compressed_sections[i as usize].hdr, + ChunkId::Placeholder(i) => &mut self.placeholders[i as usize], + } + } + + /// Finds the first chunk of a section type. + pub fn find_chunk_by_type(&self, sh_type: u32) -> Option { + self.chunks + .iter() + .copied() + .find(|&c| self.chunk_header(c).shdr.sh_type == sh_type) + } + + /// Finds the first chunk with a name. + pub fn find_chunk_by_name(&self, name: &[u8]) -> Option { + self.chunks + .iter() + .copied() + .find(|&c| self.chunk_header(c).name == name) + } + + /// Starts a `--perf` timer for a pass. + pub fn timer(&self, name: &str) -> crate::util::timer::Timer { + self.timers.start(name) + } +} diff --git a/src/diagnostics.rs b/src/diagnostics.rs new file mode 100644 index 000000000..6a99944a5 --- /dev/null +++ b/src/diagnostics.rs @@ -0,0 +1,195 @@ +//! Error, warning and fatal-error reporting. +//! +//! Errors don't abort the link immediately: the linker keeps going so that +//! all problems are reported in one run, then exits before writing the +//! output. Fatal errors are for conditions the linker can't continue past. + +use std::fmt; +use std::io::{self, Write}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Whether to demangle symbol names in diagnostics. This is process-wide +/// state so that `Display` implementations, which have no access to the +/// linker context, can consult it. +static DEMANGLE: AtomicBool = AtomicBool::new(true); + +pub fn set_demangle(enabled: bool) { + DEMANGLE.store(enabled, Ordering::Relaxed); +} + +pub fn demangle_enabled() -> bool { + DEMANGLE.load(Ordering::Relaxed) +} + +/// Diagnostic settings and state, shared by all threads. +#[derive(Debug, Default)] +pub struct Diagnostics { + color: AtomicBool, + fatal_warnings: AtomicBool, + suppress_warnings: AtomicBool, + noinhibit_exec: AtomicBool, + has_error: AtomicBool, + lock: Mutex<()>, +} + +impl Diagnostics { + pub fn new(color: bool) -> Self { + let diag = Diagnostics::default(); + diag.color.store(color, Ordering::Relaxed); + diag + } + + pub fn set_color(&self, on: bool) { + self.color.store(on, Ordering::Relaxed); + } + + pub fn set_fatal_warnings(&self, on: bool) { + self.fatal_warnings.store(on, Ordering::Relaxed); + } + + pub fn set_suppress_warnings(&self, on: bool) { + self.suppress_warnings.store(on, Ordering::Relaxed); + } + + pub fn set_noinhibit_exec(&self, on: bool) { + self.noinhibit_exec.store(on, Ordering::Relaxed); + } + + pub fn has_error(&self) -> bool { + self.has_error.load(Ordering::Relaxed) + } + + fn color(&self) -> bool { + self.color.load(Ordering::Relaxed) + } + + fn emit(&self, prefix_mono: &str, prefix_color: &str, msg: fmt::Arguments) { + let prefix = if self.color() { + prefix_color + } else { + prefix_mono + }; + let text = format!("{prefix}{msg}\n"); + let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner()); + let _ = io::stderr().write_all(text.as_bytes()); + } + + /// Reports an unrecoverable error and exits. + pub fn fatal(&self, msg: fmt::Arguments) -> ! { + self.emit("mold: fatal: ", "mold: \x1b[0;1;31mfatal:\x1b[0m ", msg); + exit_after_cleanup(1); + } + + /// Reports an error. With `--noinhibit-exec` it is downgraded to a warning. + pub fn error(&self, msg: fmt::Arguments) { + if self.noinhibit_exec.load(Ordering::Relaxed) { + self.emit("mold: warning: ", "mold: \x1b[0;1;35mwarning:\x1b[0m ", msg); + } else { + self.emit("mold: error: ", "mold: \x1b[0;1;31merror:\x1b[0m ", msg); + self.has_error.store(true, Ordering::Relaxed); + } + } + + /// Reports a warning. With `--fatal-warnings` it is promoted to an error. + pub fn warn(&self, msg: fmt::Arguments) { + if self.suppress_warnings.load(Ordering::Relaxed) { + return; + } + if self.fatal_warnings.load(Ordering::Relaxed) { + self.emit("mold: error: ", "mold: \x1b[0;1;31merror:\x1b[0m ", msg); + self.has_error.store(true, Ordering::Relaxed); + } else { + self.emit("mold: warning: ", "mold: \x1b[0;1;35mwarning:\x1b[0m ", msg); + } + } + + /// Prints an informational message to stdout. + pub fn out(&self, msg: fmt::Arguments) { + let text = format!("{msg}\n"); + let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner()); + let _ = io::stdout().write_all(text.as_bytes()); + } + + /// Exits with a failure status if any error has been reported. + pub fn checkpoint(&self) { + if self.has_error() { + exit_after_cleanup(1); + } + } +} + +/// Anything that can report diagnostics; implemented by the linker +/// context and by [`Diagnostics`] itself so that the macros below accept +/// either. +pub trait HasDiagnostics { + fn diagnostics(&self) -> &Diagnostics; +} + +impl HasDiagnostics for Diagnostics { + fn diagnostics(&self) -> &Diagnostics { + self + } +} + +/// Removes a partially-written output file, then terminates the process +/// without running destructors. Input files are mapped for the process's +/// lifetime, so there is nothing else to release. +pub fn exit_after_cleanup(status: i32) -> ! { + crate::output_file::cleanup(); + let _ = io::stdout().flush(); + let _ = io::stderr().flush(); + // SAFETY: `_exit` only terminates the process. + unsafe { libc::_exit(status) } +} + +#[macro_export] +macro_rules! fatal { + ($ctx:expr, $($arg:tt)*) => { + $crate::diagnostics::HasDiagnostics::diagnostics(&$ctx).fatal(format_args!($($arg)*)) + }; +} + +#[macro_export] +macro_rules! error { + ($ctx:expr, $($arg:tt)*) => { + $crate::diagnostics::HasDiagnostics::diagnostics(&$ctx).error(format_args!($($arg)*)) + }; +} + +#[macro_export] +macro_rules! warn { + ($ctx:expr, $($arg:tt)*) => { + $crate::diagnostics::HasDiagnostics::diagnostics(&$ctx).warn(format_args!($($arg)*)) + }; +} + +#[macro_export] +macro_rules! out { + ($ctx:expr, $($arg:tt)*) => { + $crate::diagnostics::HasDiagnostics::diagnostics(&$ctx).out(format_args!($($arg)*)) + }; +} + +/// Formats an OS error the way `strerror` would. +pub fn errno_string() -> String { + io::Error::last_os_error().to_string() +} + +impl HasDiagnostics for &T { + fn diagnostics(&self) -> &Diagnostics { + (**self).diagnostics() + } +} + +impl HasDiagnostics for &mut T { + fn diagnostics(&self) -> &Diagnostics { + (**self).diagnostics() + } +} + +impl HasDiagnostics for Arc { + fn diagnostics(&self) -> &Diagnostics { + (**self).diagnostics() + } +} diff --git a/src/driver.rs b/src/driver.rs new file mode 100644 index 000000000..0bec8d7a8 --- /dev/null +++ b/src/driver.rs @@ -0,0 +1,736 @@ +//! The linker driver: runs the passes in order. + +use std::fmt; +use std::sync::{mpsc, Arc}; + +use rayon::prelude::*; + +use crate::arch::{self, Arch}; +use crate::args::{self, Args, TargetTraits}; +use crate::chunks::{self, ChunkId}; +use crate::context::Context; +use crate::diagnostics::{self, Diagnostics}; +use crate::elf::*; +use crate::input_files::FileId; +use crate::output_file::{split_ranges, OutputFile, Range}; +use crate::{error, fatal, out, passes}; + +/// Runs the linker with the given command line. Returns the exit status. +/// +/// `link_for_target` links for a named target, or reports the target the +/// inputs are actually for; the executable provides it, as the targets +/// are instantiated in crates of their own. +pub fn main( + argv: Vec, + link_for_target: impl Fn(&str, &[String], &Diagnostics) -> Result, +) -> i32 { + let diag = Diagnostics::new(false); + + if argv.get(1).is_some_and(|a| a == "-run" || a == "--run") { + crate::subprocess::process_run_subcommand(&diag, &argv); + } + + // -C may change the directory; remember the original so that a + // restart for another target resolves relative paths the same way. + let orig_cwd = std::env::current_dir().ok(); + let cmdline = args::expand_response_files(&diag, &argv); + + // Parse with x86-64 defaults; if the target turns out to be different, + // start over with the right one. + let mut target = "x86_64".to_string(); + loop { + if let Some(cwd) = &orig_cwd { + let _ = std::env::set_current_dir(cwd); + } + match link_for_target(&target, &cmdline, &diag) { + Ok(status) => return status, + Err(actual) => target = actual, + } + } +} + +fn target_traits() -> TargetTraits { + TargetTraits { + name: E::NAME, + is_rela: E::IS_RELA, + is_sparc: E::IS_SPARC, + is_riscv: E::IS_RISCV, + is_sh4: E::FAMILY == arch::Family::Sh4, + is_x86_64: E::FAMILY == arch::Family::X86_64, + page_size: E::PAGE_SIZE, + } +} + +fn configure_diagnostics(diag: &Diagnostics, args: &Args) { + diag.set_color(args.color_diagnostics); + diag.set_fatal_warnings(args.fatal_warnings); + diag.set_suppress_warnings(args.suppress_warnings); + diag.set_noinhibit_exec(args.noinhibit_exec); + diagnostics::set_demangle(args.demangle); +} + +fn thread_count(args: &Args) -> usize { + // mold doesn't scale well beyond 32 threads. + args.thread_count.unwrap_or_else(|| { + std::thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(32) + }) +} + +fn wait_for_background(receiver: mpsc::Receiver, name: &str) -> T { + loop { + match receiver.try_recv() { + Ok(value) => return value, + Err(mpsc::TryRecvError::Disconnected) => panic!("{name} task failed"), + Err(mpsc::TryRecvError::Empty) => { + if !matches!(rayon::yield_now(), Some(rayon::Yield::Executed)) { + std::thread::yield_now(); + } + } + } + } +} + +/// Links for the target `E`, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + let parsed = args::parse_args(diag, &target_traits::(), cmdline); + let args::ParsedArgs { args, jobs, .. } = parsed; + configure_diagnostics(diag, &args); + + let mut ctx = Context::::new(args, Diagnostics::new(false), cmdline.to_vec()); + configure_diagnostics(&ctx.diag, &ctx.args); + + // If no -m option is given, deduce it from the input files. + if ctx.args.emulation.is_empty() { + ctx.args.emulation = crate::reader::detect_machine_type(&mut ctx, &jobs); + } + if ctx.args.emulation != E::NAME { + return Err(ctx.args.emulation.clone()); + } + + let t_all = ctx.timer("all"); + crate::subprocess::install_signal_handler(); + + if ctx.args.fork { + crate::subprocess::fork_child(); + } + + let threads = thread_count(&ctx.args); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("failed to build linker thread pool"); + + pool.install(|| { + // --wrap, --retain-symbols-file and --trace-symbol + for name in ctx.args.wrap.clone() { + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].set_wrapped(true); + } + if let Some(names) = ctx.args.retain_symbols_file.clone() { + for name in names { + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].set_write_to_symtab(); + } + } + for name in ctx.args.trace_symbol.clone() { + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].set_traced(true); + } + + // Version scripts and dynamic lists given on the command line. + for path in ctx.args.version_scripts.clone() { + let chroot = ctx.args.chroot.clone(); + let mf = crate::mapped_file::open_file(&ctx.diag, &chroot, &path).or_else(|| { + ctx.args.library_paths.iter().find_map(|dir| { + crate::mapped_file::open_file(&ctx.diag, &chroot, &format!("{dir}/{path}")) + }) + }); + let Some(mf) = mf else { + fatal!(ctx, "--version-script: file not found: {path}"); + }; + let mut rctx = args::ReaderContext::default(); + crate::linker_script::Script::new(&mut ctx, &mut rctx, mf).parse_version_script(); + } + for source in ctx.args.dynamic_list.clone() { + match source { + args::DynamicListSource::File(path) => { + let patterns = crate::linker_script::parse_dynamic_list(&mut ctx, &path); + ctx.dynamic_list_patterns.extend(patterns); + } + args::DynamicListSource::Pattern(pattern) => { + ctx.dynamic_list_patterns + .push(crate::linker_script::DynamicPattern { + pattern: crate::util::leak_bytes(pattern.into_bytes()), + source: "".to_string(), + is_cpp: false, + }); + } + } + } + + crate::reader::read_input_files(&mut ctx, jobs); + + // Uniquify shared object files by soname, keeping the first of each. + { + let dsos = std::mem::take(&mut ctx.dsos); + let mut map = Vec::with_capacity(dsos.len()); + let mut retained = Vec::with_capacity(dsos.len()); + for file in dsos { + if ctx.dso_sonames.insert(file.soname.clone()) { + map.push(Some(crate::input_files::DsoId(retained.len() as u32))); + retained.push(file); + } else { + map.push(None); + } + } + for slot in &mut ctx.file_by_priority { + if let Some(FileId::Dso(d)) = *slot { + *slot = map[d.index()].map(FileId::Dso); + } + } + + // Bins keep pointers to the files' SymbolId slots until gather. + // Register only the DSOs retained above, so every slot outlives that + // gather even when duplicate sonames were present on the command line. + { + let mut bin = ctx.symbol_bin(); + for file in &mut retained { + file.record_global_symbols(&mut bin); + } + } + ctx.dsos = retained; + } + + if ctx.args.repro { + passes::write_repro_file(&ctx); + } + + let mut t_before_copy = ctx.timer("before_copy"); + + passes::apply_exclude_libs(&mut ctx); + + if !ctx.args.relocatable { + let t = ctx.timer("create_internal_file"); + passes::create_internal_file(&mut ctx); + drop(t); + } + + // Resolve symbols by choosing the most appropriate file for each + // symbol. This pass also removes redundant comdat sections (e.g. + // duplicate inline functions). + passes::resolve_symbols(&mut ctx); + + // If there's an object file compiled with -flto, do link-time + // optimization. + if passes::has_lto_obj(&ctx) { + passes::do_lto(&mut ctx); + } + + // Now that we know which object files are to be included to the + // final output, we can remove unnecessary files. + let t = ctx.timer("remove_unreachable_files"); + passes::remove_unreachable_files(&mut ctx); + drop(t); + + // Building .gdb_index is split into three stages because the required data + // becomes available at different points in the link. Compilation units and + // public names depend only on input sections, so read them now in a + // low-priority arena while foreground passes continue. + let gdb_input_job = if ctx.args.gdb_index && !ctx.args.relocatable { + let timer = t_before_copy.handle(); + let inputs = crate::gdb_index::prepare_inputs(&mut ctx); + let diag = Arc::clone(&ctx.diag); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let job = move || { + let timer = timer.child("read_gdb_index_inputs"); + let data = crate::gdb_index::read_inputs::(timer, &diag, inputs); + let _ = sender.send(data); + }; + // Rayon has no task priorities, so queue this as an ordinary + // background task. + rayon::spawn(job); + Some(receiver) + } else { + None + }; + + passes::parse_eh_frame_sections(&mut ctx); + passes::parse_sframe_sections(&mut ctx); + passes::create_merged_sections(&mut ctx); + + if ctx.args.relocatable { + crate::relocatable::combine_objects(&mut ctx); + return Ok(0); + } + + passes::convert_common_symbols(&mut ctx); + passes::apply_version_script(&mut ctx); + passes::parse_symbol_version(&mut ctx); + passes::compute_import_export(&mut ctx); + + if !ctx.args.allow_multiple_definition { + passes::check_duplicate_symbols(&ctx); + } + if ctx.args.zero_to_bss { + let t = ctx.timer("convert_zero_to_bss"); + passes::convert_zero_to_bss(&mut ctx); + drop(t); + } + if ctx.args.icf { + let t = ctx.timer("compute_address_significance"); + passes::compute_address_significance(&mut ctx); + drop(t); + } + E::rewrite_input_sections(&mut ctx); + if ctx.args.gc_sections { + crate::gc_sections::gc_sections(&mut ctx); + } + if ctx.args.icf { + crate::icf::icf_sections(&mut ctx); + } + + let t = ctx.timer("create_synthetic_sections"); + passes::create_synthetic_sections(&mut ctx); + drop(t); + + if !ctx.args.allow_shlib_undefined { + passes::check_shlib_undefined(&mut ctx); + } + passes::check_symbol_types(&ctx); + passes::create_output_sections(&mut ctx); + + if E::FAMILY == arch::Family::Arm32 { + chunks::arm_exidx::create(&mut ctx); + } + + if !ctx.args.section_align.is_empty() { + passes::apply_section_align(&mut ctx); + } + let t = ctx.timer("add_synthetic_symbols"); + passes::add_synthetic_symbols(&mut ctx); + drop(t); + + // Beyond this point, no new files are added. + + if ctx.args.z_cet_report != args::CetReportKind::None { + passes::check_cet_errors(&ctx); + } + if ctx.args.z_execstack_if_needed && ctx.objs.iter().any(|f| f.needs_executable_stack) { + ctx.args.z_execstack = true; + } + + // Remaining undefined symbols become imports in a DSO, and weak + // undefined symbols get another chance to be resolved at runtime. + passes::claim_unresolved_symbols(&mut ctx); + + // Beyond this point, no new symbols are added. + + if ctx.args.print_dependencies { + passes::print_dependencies(&ctx); + } + for name in ctx.args.require_defined.clone() { + let id = ctx.get_symbol(name.as_bytes()); + if ctx.symbols[id].file().is_none() { + error!( + ctx, + "--require-defined: undefined symbol: {}", ctx.symbols[id] + ); + } + } + + passes::sort_init_fini(&mut ctx); + passes::sort_ctor_dtor(&mut ctx); + passes::fixup_ctors_in_init_array(&mut ctx); + + if ctx.args.shuffle_sections != args::ShuffleSectionsKind::None { + passes::shuffle_sections(&mut ctx); + } + let t = ctx.timer("add_dynamic_strings"); + passes::add_dynamic_strings(&mut ctx); + drop(t); + E::scan_symbols(&mut ctx); + passes::scan_relocations(&mut ctx); + passes::check_symbol_version_conflicts(&ctx); + passes::compute_imported_symbol_weakness(&mut ctx); + let t = ctx.timer("sort_output_sections"); + passes::sort_output_sections(&mut ctx); + drop(t); + + if ctx.gnu_debuglink.is_some() { + passes::separate_debug_sections(&mut ctx); + } + passes::compute_section_sizes(&mut ctx); + + if ctx.args.pack_dyn_relocs_relr { + chunks::dynamic::reldyn::construct_relr(&mut ctx); + } + passes::sort_dynsyms(&mut ctx); + // sort_debug_info_sections may uncompress the same .debug_info sections. + if let Some(job) = gdb_input_job { + ctx.gdb_index_data = Some(wait_for_background(job, ".gdb_index input")); + } + passes::sort_debug_info_sections(&mut ctx); + + // Type vectors identify compilation units by their order in the output + // .debug_info section. That order is now fixed, so build the table while the + // remaining layout passes continue. This stage stops scaling after about 12 + // workers, so limit its arena to avoid competing with foreground work. + let gdb_table_workers = (threads * 3 / 8).clamp(1, 12); + let mut gdb_table_job = if ctx.gdb_index.is_some() && ctx.gnu_debuglink.is_none() { + let timer = t_all.handle(); + let mut data = ctx + .gdb_index_data + .take() + .expect("missing .gdb_index input data"); + crate::gdb_index::prepare_tables(&ctx, &mut data); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let job = move || { + let timer = timer.child("build_gdb_index_tables"); + let data = crate::gdb_index::build_tables(timer, data, gdb_table_workers); + let _ = sender.send(data); + }; + // Rayon has no task priorities, so queue this as an ordinary + // background task. + rayon::spawn(job); + Some(receiver) + } else { + None + }; + + if ctx.args.unresolved_symbols == args::UnresolvedKind::Error { + passes::report_undef_errors(&ctx); + } + + if ctx.verdef.is_some() { + chunks::version::verdef::construct(&mut ctx); + } + chunks::version::verneed::construct(&mut ctx); + chunks::eh_frame::construct(&mut ctx); + chunks::sframe::construct(&mut ctx); + + if ctx.args.emit_relocs { + passes::create_reloc_sections(&mut ctx); + } + if !ctx.args.strip_all { + passes::create_output_symtab(&mut ctx); + } + let t = ctx.timer("compute_section_headers"); + passes::compute_section_headers(&mut ctx); + drop(t); + + let mut filesize = passes::set_osec_offsets(&mut ctx); + + // RISC-V and LoongArch branches are encoded with instruction pairs + // that reach ±2 GiB; those whose targets are close enough are now + // shortened. + if E::IS_RISCV || E::IS_LOONGARCH { + crate::relax::shrink_sections(&mut ctx); + filesize = passes::set_osec_offsets(&mut ctx); + } + + if E::NEEDS_THUNK { + crate::thunks::remove_redundant_thunks(&mut ctx); + filesize = passes::set_osec_offsets(&mut ctx); + } + if ctx.arm_exidx.is_some() { + chunks::arm_exidx::remove_duplicate_entries(&mut ctx); + filesize = passes::set_osec_offsets(&mut ctx); + } + + // The memory layout is fixed; set the addresses of synthesized symbols. + let t = ctx.timer("fix_synthetic_symbols"); + passes::fix_synthetic_symbols(&mut ctx); + drop(t); + chunks::sframe::sort(&mut ctx); + + if ctx.args.compress_debug_sections != ELFCOMPRESS_NONE { + passes::compress_debug_sections(&mut ctx); + filesize = passes::set_osec_offsets(&mut ctx); + } + if E::NEEDS_THUNK { + crate::thunks::gather_thunk_addresses(&mut ctx); + } + // Addends of dynamic relocations against synthetic symbols may have + // changed the packed relocation size. + if ctx.args.pack_dyn_relocs_android { + filesize = passes::set_osec_offsets(&mut ctx); + } + + let t = ctx.timer("update_reldyn"); + chunks::dynamic::reldyn::update_shdr(&mut ctx); + drop(t); + ctx.filesize = filesize; + t_before_copy.stop(); + + let t_open = ctx.timer("open_file"); + let mut output = OutputFile::open( + &ctx.diag, + &ctx.args.output, + filesize, + 0o777, + ctx.args.overwrite_output_file, + ); + drop(t_open); + { + let mut t_copy = ctx.timer("copy"); + let buf = output.buf(); + copy_chunks(&ctx, buf); + + E::finish_output(&ctx, buf); + + // The dynamic linker works better with sorted .rela.dyn. + let reldyn = ctx.reldyn.hdr.shdr; + if ctx.chunks.contains(&ChunkId::RelDyn) && reldyn.sh_size != 0 { + chunks::dynamic::reldyn::sort( + &ctx, + &mut buf + [reldyn.sh_offset as usize..(reldyn.sh_offset + reldyn.sh_size) as usize], + ); + } + + // The address table of .gdb_index needs relocated debug info. + if ctx.gdb_index.is_some() && ctx.gnu_debuglink.is_none() { + if let Some(job) = gdb_table_job.take() { + ctx.gdb_index_data = Some(wait_for_background(job, ".gdb_index table")); + } + crate::gdb_index::write(&mut ctx, &mut output); + } + + if ctx.buildid.is_some() { + let is_mmapped = output.is_mmapped(); + passes::write_build_id(&mut ctx, output.buf(), is_mmapped); + } + if ctx.gnu_debuglink.is_some() { + passes::write_gnu_debuglink(&mut ctx, output.buf()); + } + t_copy.stop(); + } + ctx.checkpoint(); + + let t_close = ctx.timer("close_file"); + output.close(&ctx.diag); + drop(t_close); + + if !ctx.args.dependency_file.is_empty() { + passes::write_dependency_file(&ctx); + } + if !ctx.args.plugin.is_empty() { + crate::lto::cleanup(); + } + drop(t_all); + + if ctx.args.print_map { + crate::mapfile::print_map(&ctx); + } + if ctx.gnu_debuglink.is_some() { + passes::write_separate_debug_file(&mut ctx); + } + if ctx.args.stats { + passes::show_stats(&ctx); + } + if ctx.args.perf { + ctx.timers.print(); + } + + let _ = std::io::Write::flush(&mut std::io::stdout()); + let _ = std::io::Write::flush(&mut std::io::stderr()); + crate::subprocess::notify_parent(); + + crate::mapped_file::drop_mappings(); + + if ctx.args.quick_exit { + diagnostics::exit_after_cleanup(0); + } + ctx.checkpoint(); + Ok(0) + }) +} + +fn file_range(ctx: &Context, id: ChunkId) -> Range { + let hdr = ctx.chunk_header(id); + let size = if hdr.shdr.sh_type == SHT_NOBITS { + 0 + } else { + hdr.shdr.sh_size + }; + Range { + offset: hdr.shdr.sh_offset, + size, + } +} + +/// A chunk together with the other chunks whose bytes it writes. +struct Task { + chunk: ChunkId, + extra: Vec, +} + +/// Copies every chunk into the output buffer. +/// +/// Each chunk owns its own byte range, and a few also write into another +/// chunk's range (`.eh_frame` fills the `.eh_frame_hdr` table, `.symtab` +/// writes `.strtab`, and relocation sections may patch addends into their +/// target sections). The buffer is split into the disjoint ranges each +/// task needs so that all tasks can run in parallel safely. +pub fn copy_chunks(ctx: &Context, buf: &mut [u8]) { + let t = ctx.timer("copy_chunks"); + + let mut first: Vec = Vec::new(); + let mut last: Vec = Vec::new(); + let is_reloc_sec = |id: ChunkId| { + let ty = ctx.chunk_header(id).shdr.sh_type; + matches!( + id, + ChunkId::Reloc(_) | ChunkId::EhFrameReloc | ChunkId::SFrameReloc + ) || ty == SHT_REL + || (E::FAMILY == arch::Family::Sh4 && ty == SHT_RELA) + }; + + for &id in &ctx.chunks { + match id { + ChunkId::EhFrameHdr | ChunkId::Strtab | ChunkId::SymtabShndx => {} + ChunkId::EhFrame => { + let extra = ctx + .eh_frame_hdr + .as_ref() + .map(|_| ChunkId::EhFrameHdr) + .into_iter() + .collect(); + first.push(Task { chunk: id, extra }); + } + ChunkId::Symtab => { + let mut extra = vec![ChunkId::Strtab]; + if ctx.symtab_shndx.is_some() { + extra.push(ChunkId::SymtabShndx); + } + first.push(Task { chunk: id, extra }); + } + ChunkId::Reloc(i) => { + let osec = ctx.reloc_sections[i as usize].output_section; + last.push(Task { + chunk: id, + extra: vec![ChunkId::Output(osec)], + }); + } + ChunkId::EhFrameReloc => last.push(Task { + chunk: id, + extra: vec![ChunkId::EhFrame], + }), + _ if is_reloc_sec(id) => last.push(Task { + chunk: id, + extra: Vec::new(), + }), + _ => first.push(Task { + chunk: id, + extra: Vec::new(), + }), + } + } + + // For --relocatable and --emit-relocs, we want to copy non-relocation + // sections first, for two reasons. First, REL-type relocation sections (as + // opposed to RELA-type) store relocation addends to target sections, so the + // targets must be written first. Second, relaxation may retype an emitted + // relocation in place while applying relocations (e.g. AArch64 GOT/TLS + // relaxations), and RelocSection has to observe the updated type, so it + // must run after the target sections. + // + // We also do that for SH4 because despite being RELA, we always need + // to write addends to relocated places for SH4. + run_tasks(ctx, buf, &first, &t); + run_tasks(ctx, buf, &last, &t); + + // Undefined symbols in non-alloc sections are found only now. + passes::report_undef_errors(ctx); + + // Zero the padding between chunks. + let mut ranges: Vec = ctx + .chunks + .iter() + .map(|&id| file_range(ctx, id)) + .filter(|r| r.size != 0) + .collect(); + ranges.sort_by_key(|r| r.offset); + let mut pos = 0usize; + for r in ranges { + let start = r.offset as usize; + if start > pos { + buf[pos..start].fill(0); + } + pos = pos.max(start + r.size as usize); + } + buf[pos..].fill(0); +} + +fn run_tasks( + ctx: &Context, + buf: &mut [u8], + tasks: &[Task], + timer: &crate::util::timer::Timer, +) { + let mut ranges: Vec = Vec::new(); + let mut task_ranges: Vec> = Vec::new(); + for task in tasks { + let mut idx = Vec::new(); + for id in std::iter::once(task.chunk).chain(task.extra.iter().copied()) { + let r = file_range(ctx, id); + idx.push(ranges.len()); + ranges.push(r); + } + task_ranges.push(idx); + } + + let mut slices: Vec> = + split_ranges(buf, &ranges).into_iter().map(Some).collect(); + let mut work: Vec<(&Task, Vec<&mut [u8]>)> = Vec::new(); + for (task, idx) in tasks.iter().zip(task_ranges) { + let bufs = idx.into_iter().map(|i| slices[i].take().unwrap()).collect(); + work.push((task, bufs)); + } + + work.into_par_iter().for_each(|(task, mut bufs)| { + let name = ctx.chunk_header(task.chunk).name; + let _t = timer.child(&if name.is_empty() { + "(header)".to_string() + } else { + name.to_string() + }); + let mut bufs = bufs.drain(..); + let own = bufs.next().unwrap(); + match task.chunk { + ChunkId::EhFrame => chunks::eh_frame::copy_buf(ctx, own, bufs.next()), + ChunkId::Symtab => { + let strtab = bufs.next().unwrap(); + chunks::symtab::symtab::copy_buf(ctx, own, strtab, bufs.next()); + } + ChunkId::Reloc(i) => chunks::misc::reloc::copy_buf(ctx, i, own, bufs.next()), + ChunkId::EhFrameReloc => { + chunks::eh_frame::eh_frame_reloc::copy_buf(ctx, own, bufs.next()) + } + id => chunks::copy_buf(ctx, id, own), + } + }); + + // .eh_frame_hdr's header, whose table .eh_frame wrote. + if tasks.iter().any(|t| t.chunk == ChunkId::EhFrame) && ctx.eh_frame_hdr.is_some() { + let r = file_range(ctx, ChunkId::EhFrameHdr); + chunks::eh_frame::eh_frame_hdr::write_header( + ctx, + &mut buf[r.offset as usize..(r.offset + r.size) as usize], + ); + } +} + +impl fmt::Debug for Context { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Context<{}>", E::NAME) + } +} + +/// Prints the version banner, for `-v`. +pub fn print_version(diag: &Diagnostics) { + out!(diag, "{}", args::VERSION); +} diff --git a/src/elf/consts.rs b/src/elf/consts.rs new file mode 100644 index 000000000..d024a9d69 --- /dev/null +++ b/src/elf/consts.rs @@ -0,0 +1,1308 @@ +// ELF constants, transcribed from mold/src/elf.h. + +#![allow(dead_code, non_upper_case_globals)] + +pub const SHN_UNDEF: u32 = 0; +pub const SHN_LORESERVE: u32 = 0xff00; +pub const SHN_ABS: u32 = 0xfff1; +pub const SHN_COMMON: u32 = 0xfff2; +pub const SHN_XINDEX: u32 = 0xffff; + +pub const SHT_NULL: u32 = 0; +pub const SHT_PROGBITS: u32 = 1; +pub const SHT_SYMTAB: u32 = 2; +pub const SHT_STRTAB: u32 = 3; +pub const SHT_RELA: u32 = 4; +pub const SHT_HASH: u32 = 5; +pub const SHT_DYNAMIC: u32 = 6; +pub const SHT_NOTE: u32 = 7; +pub const SHT_NOBITS: u32 = 8; +pub const SHT_REL: u32 = 9; +pub const SHT_SHLIB: u32 = 10; +pub const SHT_DYNSYM: u32 = 11; +pub const SHT_INIT_ARRAY: u32 = 14; +pub const SHT_FINI_ARRAY: u32 = 15; +pub const SHT_PREINIT_ARRAY: u32 = 16; +pub const SHT_GROUP: u32 = 17; +pub const SHT_SYMTAB_SHNDX: u32 = 18; +pub const SHT_RELR: u32 = 19; +pub const SHT_CREL: u32 = 0x40000014; +pub const SHT_LOOS: u32 = 0x60000000; +pub const SHT_ANDROID_REL: u32 = 0x60000001; +pub const SHT_ANDROID_RELA: u32 = 0x60000002; +pub const SHT_LLVM_ADDRSIG: u32 = 0x6fff4c03; +pub const SHT_ANDROID_RELR: u32 = 0x6fffff00; +pub const SHT_GNU_SFRAME: u32 = 0x6ffffff4; +pub const SHT_GNU_HASH: u32 = 0x6ffffff6; +pub const SHT_GNU_VERDEF: u32 = 0x6ffffffd; +pub const SHT_GNU_VERNEED: u32 = 0x6ffffffe; +pub const SHT_GNU_VERSYM: u32 = 0x6fffffff; +pub const SHT_HIOS: u32 = 0x6fffffff; +pub const SHT_X86_64_UNWIND: u32 = 0x70000001; +pub const SHT_ARM_EXIDX: u32 = 0x70000001; +pub const SHT_ARM_ATTRIBUTES: u32 = 0x70000003; +pub const SHT_RISCV_ATTRIBUTES: u32 = 0x70000003; +pub const SHT_LOUSER: u32 = 0x80000000; +pub const SHT_HIUSER: u32 = 0xffffffff; + +pub const SHF_WRITE: u32 = 0x1; +pub const SHF_ALLOC: u32 = 0x2; +pub const SHF_EXECINSTR: u32 = 0x4; +pub const SHF_MERGE: u32 = 0x10; +pub const SHF_STRINGS: u32 = 0x20; +pub const SHF_INFO_LINK: u32 = 0x40; +pub const SHF_LINK_ORDER: u32 = 0x80; +pub const SHF_OS_NONCONFORMING: u32 = 0x100; +pub const SHF_GROUP: u32 = 0x200; +pub const SHF_TLS: u32 = 0x400; +pub const SHF_COMPRESSED: u32 = 0x800; +pub const SHF_GNU_RETAIN: u32 = 0x200000; +pub const SHF_EXCLUDE: u32 = 0x80000000; + +pub const GRP_COMDAT: u32 = 1; + +pub const STT_NOTYPE: u32 = 0; +pub const STT_OBJECT: u32 = 1; +pub const STT_FUNC: u32 = 2; +pub const STT_SECTION: u32 = 3; +pub const STT_FILE: u32 = 4; +pub const STT_COMMON: u32 = 5; +pub const STT_TLS: u32 = 6; +pub const STT_GNU_IFUNC: u32 = 10; +pub const STT_SPARC_REGISTER: u32 = 13; + +pub const STB_LOCAL: u32 = 0; +pub const STB_GLOBAL: u32 = 1; +pub const STB_WEAK: u32 = 2; +pub const STB_GNU_UNIQUE: u32 = 10; + +pub const STV_DEFAULT: u32 = 0; +pub const STV_INTERNAL: u32 = 1; +pub const STV_HIDDEN: u32 = 2; +pub const STV_PROTECTED: u32 = 3; + +pub const VER_NDX_LOCAL: u32 = 0; +pub const VER_NDX_GLOBAL: u32 = 1; +pub const VER_NDX_LAST_RESERVED: u32 = 1; +pub const VER_NDX_UNSPECIFIED: u32 = 0xffff; +pub const VERSYM_HIDDEN: u32 = 0x8000; + +pub const VER_FLG_BASE: u32 = 1; +pub const VER_FLG_WEAK: u32 = 2; +pub const VER_FLG_INFO: u32 = 4; + +pub const PT_NULL: u32 = 0; +pub const PT_LOAD: u32 = 1; +pub const PT_DYNAMIC: u32 = 2; +pub const PT_INTERP: u32 = 3; +pub const PT_NOTE: u32 = 4; +pub const PT_SHLIB: u32 = 5; +pub const PT_PHDR: u32 = 6; +pub const PT_TLS: u32 = 7; +pub const PT_GNU_EH_FRAME: u32 = 0x6474e550; +pub const PT_GNU_STACK: u32 = 0x6474e551; +pub const PT_GNU_RELRO: u32 = 0x6474e552; +pub const PT_GNU_PROPERTY: u32 = 0x6474e553; +pub const PT_GNU_SFRAME: u32 = 0x6474e554; +pub const PT_OPENBSD_RANDOMIZE: u32 = 0x65a3dbe6; +pub const PT_ARM_EXIDX: u32 = 0x70000001; +pub const PT_RISCV_ATTRIBUTES: u32 = 0x70000003; + +pub const PF_NONE: u32 = 0; +pub const PF_X: u32 = 1; +pub const PF_W: u32 = 2; +pub const PF_R: u32 = 4; + +pub const ET_NONE: u32 = 0; +pub const ET_REL: u32 = 1; +pub const ET_EXEC: u32 = 2; +pub const ET_DYN: u32 = 3; + +pub const ELFDATA2LSB: u32 = 1; +pub const ELFDATA2MSB: u32 = 2; + +pub const ELFCLASS32: u32 = 1; +pub const ELFCLASS64: u32 = 2; + +pub const EV_CURRENT: u32 = 1; + +pub const EM_NONE: u32 = 0; +pub const EM_386: u32 = 3; +pub const EM_68K: u32 = 4; +pub const EM_PPC: u32 = 20; +pub const EM_PPC64: u32 = 21; +pub const EM_S390X: u32 = 22; +pub const EM_ARM: u32 = 40; +pub const EM_SH: u32 = 42; +pub const EM_SPARC64: u32 = 43; +pub const EM_X86_64: u32 = 62; +pub const EM_AARCH64: u32 = 183; +pub const EM_RISCV: u32 = 243; +pub const EM_LOONGARCH: u32 = 258; + +pub const EI_CLASS: u32 = 4; +pub const EI_DATA: u32 = 5; +pub const EI_VERSION: u32 = 6; +pub const EI_OSABI: u32 = 7; +pub const EI_ABIVERSION: u32 = 8; + +pub const DT_NULL: u32 = 0; +pub const DT_NEEDED: u32 = 1; +pub const DT_PLTRELSZ: u32 = 2; +pub const DT_PLTGOT: u32 = 3; +pub const DT_HASH: u32 = 4; +pub const DT_STRTAB: u32 = 5; +pub const DT_SYMTAB: u32 = 6; +pub const DT_RELA: u32 = 7; +pub const DT_RELASZ: u32 = 8; +pub const DT_RELAENT: u32 = 9; +pub const DT_STRSZ: u32 = 10; +pub const DT_SYMENT: u32 = 11; +pub const DT_INIT: u32 = 12; +pub const DT_FINI: u32 = 13; +pub const DT_SONAME: u32 = 14; +pub const DT_RPATH: u32 = 15; +pub const DT_SYMBOLIC: u32 = 16; +pub const DT_REL: u32 = 17; +pub const DT_RELSZ: u32 = 18; +pub const DT_RELENT: u32 = 19; +pub const DT_PLTREL: u32 = 20; +pub const DT_DEBUG: u32 = 21; +pub const DT_TEXTREL: u32 = 22; +pub const DT_JMPREL: u32 = 23; +pub const DT_BIND_NOW: u32 = 24; +pub const DT_INIT_ARRAY: u32 = 25; +pub const DT_FINI_ARRAY: u32 = 26; +pub const DT_INIT_ARRAYSZ: u32 = 27; +pub const DT_FINI_ARRAYSZ: u32 = 28; +pub const DT_RUNPATH: u32 = 29; +pub const DT_FLAGS: u32 = 30; +pub const DT_PREINIT_ARRAY: u32 = 32; +pub const DT_PREINIT_ARRAYSZ: u32 = 33; +pub const DT_RELRSZ: u32 = 35; +pub const DT_RELR: u32 = 36; +pub const DT_RELRENT: u32 = 37; +pub const DT_ANDROID_REL: u32 = 0x6000000f; +pub const DT_ANDROID_RELSZ: u32 = 0x60000010; +pub const DT_ANDROID_RELA: u32 = 0x60000011; +pub const DT_ANDROID_RELASZ: u32 = 0x60000012; +pub const DT_ANDROID_RELR: u32 = 0x6fffe000; +pub const DT_ANDROID_RELRSZ: u32 = 0x6fffe001; +pub const DT_ANDROID_RELRENT: u32 = 0x6fffe003; +pub const DT_GNU_HASH: u32 = 0x6ffffef5; +pub const DT_DEPAUDIT: u32 = 0x6ffffefb; +pub const DT_AUDIT: u32 = 0x6ffffefc; +pub const DT_VERSYM: u32 = 0x6ffffff0; +pub const DT_RELACOUNT: u32 = 0x6ffffff9; +pub const DT_RELCOUNT: u32 = 0x6ffffffa; +pub const DT_FLAGS_1: u32 = 0x6ffffffb; +pub const DT_VERDEF: u32 = 0x6ffffffc; +pub const DT_VERDEFNUM: u32 = 0x6ffffffd; +pub const DT_VERNEED: u32 = 0x6ffffffe; +pub const DT_VERNEEDNUM: u32 = 0x6fffffff; +pub const DT_PPC_GOT: u32 = 0x70000000; +pub const DT_PPC64_GLINK: u32 = 0x70000000; +pub const DT_RISCV_VARIANT_CC: u32 = 0x70000001; +pub const DT_AARCH64_VARIANT_PCS: u32 = 0x70000005; +pub const DT_AUXILIARY: u32 = 0x7ffffffd; +pub const DT_FILTER: u32 = 0x7fffffff; + +pub const DF_ORIGIN: u32 = 0x01; +pub const DF_SYMBOLIC: u32 = 0x02; +pub const DF_TEXTREL: u32 = 0x04; +pub const DF_BIND_NOW: u32 = 0x08; +pub const DF_STATIC_TLS: u32 = 0x10; + +pub const DF_1_NOW: u32 = 0x00000001; +pub const DF_1_NODELETE: u32 = 0x00000008; +pub const DF_1_INITFIRST: u32 = 0x00000020; +pub const DF_1_NOOPEN: u32 = 0x00000040; +pub const DF_1_ORIGIN: u32 = 0x00000080; +pub const DF_1_INTERPOSE: u32 = 0x00000400; +pub const DF_1_NODEFLIB: u32 = 0x00000800; +pub const DF_1_NODUMP: u32 = 0x00001000; +pub const DF_1_PIE: u32 = 0x08000000; + +pub const NT_GNU_ABI_TAG: u32 = 1; +pub const NT_GNU_HWCAP: u32 = 2; +pub const NT_GNU_BUILD_ID: u32 = 3; +pub const NT_GNU_GOLD_VERSION: u32 = 4; +pub const NT_GNU_PROPERTY_TYPE_0: u32 = 5; +pub const NT_FDO_PACKAGING_METADATA: u32 = 0xcafe1a7e; + +pub const GNU_PROPERTY_X86_UINT32_AND_LO: u32 = 0xc0000002; +pub const GNU_PROPERTY_X86_UINT32_AND_HI: u32 = 0xc0007fff; +pub const GNU_PROPERTY_X86_UINT32_OR_LO: u32 = 0xc0008000; +pub const GNU_PROPERTY_X86_UINT32_OR_HI: u32 = 0xc000ffff; +pub const GNU_PROPERTY_X86_UINT32_OR_AND_LO: u32 = 0xc0010000; +pub const GNU_PROPERTY_X86_UINT32_OR_AND_HI: u32 = 0xc0017fff; + +pub const GNU_PROPERTY_X86_FEATURE_1_IBT: u32 = 1; +pub const GNU_PROPERTY_X86_FEATURE_1_SHSTK: u32 = 2; +pub const GNU_PROPERTY_X86_FEATURE_1_AND: u32 = 0xc0000002; + +pub const GNU_PROPERTY_X86_ISA_1_NEEDED: u32 = 0xc0008002; +pub const GNU_PROPERTY_X86_ISA_1_BASELINE: u32 = 1; +pub const GNU_PROPERTY_X86_ISA_1_V2: u32 = 2; +pub const GNU_PROPERTY_X86_ISA_1_V3: u32 = 4; +pub const GNU_PROPERTY_X86_ISA_1_V4: u32 = 8; + +pub const ELFCOMPRESS_NONE: u32 = 0; +pub const ELFCOMPRESS_ZLIB: u32 = 1; +pub const ELFCOMPRESS_ZSTD: u32 = 2; + +pub const EF_ARM_ABI_FLOAT_SOFT: u32 = 0x200; +pub const EF_ARM_ABI_FLOAT_HARD: u32 = 0x400; +pub const EF_ARM_BE8: u32 = 0x800000; +pub const EF_ARM_EABI_VER5: u32 = 0x5000000; + +pub const EF_PPC64_ABI: u32 = 3; + +pub const EF_RISCV_RVC: u32 = 1; +pub const EF_RISCV_FLOAT_ABI: u32 = 6; +pub const EF_RISCV_FLOAT_ABI_SOFT: u32 = 0; +pub const EF_RISCV_FLOAT_ABI_SINGLE: u32 = 2; +pub const EF_RISCV_FLOAT_ABI_DOUBLE: u32 = 4; +pub const EF_RISCV_FLOAT_ABI_QUAD: u32 = 6; +pub const EF_RISCV_RVE: u32 = 8; +pub const EF_RISCV_TSO: u32 = 16; + +pub const EF_SPARC64_MM: u32 = 0x3; +pub const EF_SPARC64_TSO: u32 = 0x0; +pub const EF_SPARC64_PSO: u32 = 0x1; +pub const EF_SPARC64_RMO: u32 = 0x2; +pub const EF_SPARC_EXT_MASK: u32 = 0xffff00; +pub const EF_SPARC_SUN_US1: u32 = 0x000200; +pub const EF_SPARC_HAL_R1: u32 = 0x000400; +pub const EF_SPARC_SUN_US3: u32 = 0x000800; + +pub const STO_RISCV_VARIANT_CC: u32 = 0x80; + +pub const ELF_TAG_FILE: u32 = 1; +pub const ELF_TAG_SECTION: u32 = 2; +pub const ELF_TAG_SYMBOL: u32 = 3; +pub const ELF_TAG_RISCV_STACK_ALIGN: u32 = 4; +pub const ELF_TAG_RISCV_ARCH: u32 = 5; +pub const ELF_TAG_RISCV_UNALIGNED_ACCESS: u32 = 6; + +pub const EF_LOONGARCH_ABI_SOFT_FLOAT: u32 = 0x1; +pub const EF_LOONGARCH_ABI_SINGLE_FLOAT: u32 = 0x2; +pub const EF_LOONGARCH_ABI_DOUBLE_FLOAT: u32 = 0x3; +pub const EF_LOONGARCH_ABI_MODIFIER_MASK: u32 = 0x7; +pub const EF_LOONGARCH_OBJABI_V1: u32 = 0x40; +pub const EF_LOONGARCH_OBJABI_MASK: u32 = 0xC0; + +pub const R_NONE: u32 = 0; + +pub const R_X86_64_NONE: u32 = 0; +pub const R_X86_64_64: u32 = 1; +pub const R_X86_64_PC32: u32 = 2; +pub const R_X86_64_GOT32: u32 = 3; +pub const R_X86_64_PLT32: u32 = 4; +pub const R_X86_64_COPY: u32 = 5; +pub const R_X86_64_GLOB_DAT: u32 = 6; +pub const R_X86_64_JUMP_SLOT: u32 = 7; +pub const R_X86_64_RELATIVE: u32 = 8; +pub const R_X86_64_GOTPCREL: u32 = 9; +pub const R_X86_64_32: u32 = 10; +pub const R_X86_64_32S: u32 = 11; +pub const R_X86_64_16: u32 = 12; +pub const R_X86_64_PC16: u32 = 13; +pub const R_X86_64_8: u32 = 14; +pub const R_X86_64_PC8: u32 = 15; +pub const R_X86_64_DTPMOD64: u32 = 16; +pub const R_X86_64_DTPOFF64: u32 = 17; +pub const R_X86_64_TPOFF64: u32 = 18; +pub const R_X86_64_TLSGD: u32 = 19; +pub const R_X86_64_TLSLD: u32 = 20; +pub const R_X86_64_DTPOFF32: u32 = 21; +pub const R_X86_64_GOTTPOFF: u32 = 22; +pub const R_X86_64_TPOFF32: u32 = 23; +pub const R_X86_64_PC64: u32 = 24; +pub const R_X86_64_GOTOFF64: u32 = 25; +pub const R_X86_64_GOTPC32: u32 = 26; +pub const R_X86_64_GOT64: u32 = 27; +pub const R_X86_64_GOTPCREL64: u32 = 28; +pub const R_X86_64_GOTPC64: u32 = 29; +pub const R_X86_64_GOTPLT64: u32 = 30; +pub const R_X86_64_PLTOFF64: u32 = 31; +pub const R_X86_64_SIZE32: u32 = 32; +pub const R_X86_64_SIZE64: u32 = 33; +pub const R_X86_64_GOTPC32_TLSDESC: u32 = 34; +pub const R_X86_64_TLSDESC_CALL: u32 = 35; +pub const R_X86_64_TLSDESC: u32 = 36; +pub const R_X86_64_IRELATIVE: u32 = 37; +pub const R_X86_64_GOTPCRELX: u32 = 41; +pub const R_X86_64_REX_GOTPCRELX: u32 = 42; +pub const R_X86_64_CODE_4_GOTPCRELX: u32 = 43; +pub const R_X86_64_CODE_4_GOTTPOFF: u32 = 44; +pub const R_X86_64_CODE_4_GOTPC32_TLSDESC: u32 = 45; +pub const R_X86_64_CODE_5_GOTPCRELX: u32 = 46; +pub const R_X86_64_CODE_5_GOTTPOFF: u32 = 47; +pub const R_X86_64_CODE_5_GOTPC32_TLSDESC: u32 = 48; +pub const R_X86_64_CODE_6_GOTPCRELX: u32 = 49; +pub const R_X86_64_CODE_6_GOTTPOFF: u32 = 50; +pub const R_X86_64_CODE_6_GOTPC32_TLSDESC: u32 = 51; + +pub const R_386_NONE: u32 = 0; +pub const R_386_32: u32 = 1; +pub const R_386_PC32: u32 = 2; +pub const R_386_GOT32: u32 = 3; +pub const R_386_PLT32: u32 = 4; +pub const R_386_COPY: u32 = 5; +pub const R_386_GLOB_DAT: u32 = 6; +pub const R_386_JUMP_SLOT: u32 = 7; +pub const R_386_RELATIVE: u32 = 8; +pub const R_386_GOTOFF: u32 = 9; +pub const R_386_GOTPC: u32 = 10; +pub const R_386_32PLT: u32 = 11; +pub const R_386_TLS_TPOFF: u32 = 14; +pub const R_386_TLS_IE: u32 = 15; +pub const R_386_TLS_GOTIE: u32 = 16; +pub const R_386_TLS_LE: u32 = 17; +pub const R_386_TLS_GD: u32 = 18; +pub const R_386_TLS_LDM: u32 = 19; +pub const R_386_16: u32 = 20; +pub const R_386_PC16: u32 = 21; +pub const R_386_8: u32 = 22; +pub const R_386_PC8: u32 = 23; +pub const R_386_TLS_GD_32: u32 = 24; +pub const R_386_TLS_GD_PUSH: u32 = 25; +pub const R_386_TLS_GD_CALL: u32 = 26; +pub const R_386_TLS_GD_POP: u32 = 27; +pub const R_386_TLS_LDM_32: u32 = 28; +pub const R_386_TLS_LDM_PUSH: u32 = 29; +pub const R_386_TLS_LDM_CALL: u32 = 30; +pub const R_386_TLS_LDM_POP: u32 = 31; +pub const R_386_TLS_LDO_32: u32 = 32; +pub const R_386_TLS_IE_32: u32 = 33; +pub const R_386_TLS_LE_32: u32 = 34; +pub const R_386_TLS_DTPMOD32: u32 = 35; +pub const R_386_TLS_DTPOFF32: u32 = 36; +pub const R_386_TLS_TPOFF32: u32 = 37; +pub const R_386_SIZE32: u32 = 38; +pub const R_386_TLS_GOTDESC: u32 = 39; +pub const R_386_TLS_DESC_CALL: u32 = 40; +pub const R_386_TLS_DESC: u32 = 41; +pub const R_386_IRELATIVE: u32 = 42; +pub const R_386_GOT32X: u32 = 43; + +pub const R_AARCH64_NONE: u32 = 0; +pub const R_AARCH64_ABS64: u32 = 0x101; +pub const R_AARCH64_ABS32: u32 = 0x102; +pub const R_AARCH64_ABS16: u32 = 0x103; +pub const R_AARCH64_PREL64: u32 = 0x104; +pub const R_AARCH64_PREL32: u32 = 0x105; +pub const R_AARCH64_PREL16: u32 = 0x106; +pub const R_AARCH64_MOVW_UABS_G0: u32 = 0x107; +pub const R_AARCH64_MOVW_UABS_G0_NC: u32 = 0x108; +pub const R_AARCH64_MOVW_UABS_G1: u32 = 0x109; +pub const R_AARCH64_MOVW_UABS_G1_NC: u32 = 0x10a; +pub const R_AARCH64_MOVW_UABS_G2: u32 = 0x10b; +pub const R_AARCH64_MOVW_UABS_G2_NC: u32 = 0x10c; +pub const R_AARCH64_MOVW_UABS_G3: u32 = 0x10d; +pub const R_AARCH64_MOVW_SABS_G0: u32 = 0x10e; +pub const R_AARCH64_MOVW_SABS_G1: u32 = 0x10f; +pub const R_AARCH64_MOVW_SABS_G2: u32 = 0x110; +pub const R_AARCH64_LD_PREL_LO19: u32 = 0x111; +pub const R_AARCH64_ADR_PREL_LO21: u32 = 0x112; +pub const R_AARCH64_ADR_PREL_PG_HI21: u32 = 0x113; +pub const R_AARCH64_ADR_PREL_PG_HI21_NC: u32 = 0x114; +pub const R_AARCH64_ADD_ABS_LO12_NC: u32 = 0x115; +pub const R_AARCH64_LDST8_ABS_LO12_NC: u32 = 0x116; +pub const R_AARCH64_TSTBR14: u32 = 0x117; +pub const R_AARCH64_CONDBR19: u32 = 0x118; +pub const R_AARCH64_JUMP26: u32 = 0x11a; +pub const R_AARCH64_CALL26: u32 = 0x11b; +pub const R_AARCH64_LDST16_ABS_LO12_NC: u32 = 0x11c; +pub const R_AARCH64_LDST32_ABS_LO12_NC: u32 = 0x11d; +pub const R_AARCH64_LDST64_ABS_LO12_NC: u32 = 0x11e; +pub const R_AARCH64_MOVW_PREL_G0: u32 = 0x11f; +pub const R_AARCH64_MOVW_PREL_G0_NC: u32 = 0x120; +pub const R_AARCH64_MOVW_PREL_G1: u32 = 0x121; +pub const R_AARCH64_MOVW_PREL_G1_NC: u32 = 0x122; +pub const R_AARCH64_MOVW_PREL_G2: u32 = 0x123; +pub const R_AARCH64_MOVW_PREL_G2_NC: u32 = 0x124; +pub const R_AARCH64_MOVW_PREL_G3: u32 = 0x125; +pub const R_AARCH64_LDST128_ABS_LO12_NC: u32 = 0x12b; +pub const R_AARCH64_ADR_GOT_PAGE: u32 = 0x137; +pub const R_AARCH64_LD64_GOT_LO12_NC: u32 = 0x138; +pub const R_AARCH64_LD64_GOTPAGE_LO15: u32 = 0x139; +pub const R_AARCH64_PLT32: u32 = 0x13a; +pub const R_AARCH64_GOTPCREL32: u32 = 0x13b; +pub const R_AARCH64_TLSGD_ADR_PREL21: u32 = 0x200; +pub const R_AARCH64_TLSGD_ADR_PAGE21: u32 = 0x201; +pub const R_AARCH64_TLSGD_ADD_LO12_NC: u32 = 0x202; +pub const R_AARCH64_TLSGD_MOVW_G1: u32 = 0x203; +pub const R_AARCH64_TLSGD_MOVW_G0_NC: u32 = 0x204; +pub const R_AARCH64_TLSLD_ADR_PREL21: u32 = 0x205; +pub const R_AARCH64_TLSLD_ADR_PAGE21: u32 = 0x206; +pub const R_AARCH64_TLSLD_ADD_LO12_NC: u32 = 0x207; +pub const R_AARCH64_TLSLD_MOVW_G1: u32 = 0x208; +pub const R_AARCH64_TLSLD_MOVW_G0_NC: u32 = 0x209; +pub const R_AARCH64_TLSLD_LD_PREL19: u32 = 0x20a; +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G2: u32 = 0x20b; +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G1: u32 = 0x20c; +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC: u32 = 0x20d; +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G0: u32 = 0x20e; +pub const R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC: u32 = 0x20f; +pub const R_AARCH64_TLSLD_ADD_DTPREL_HI12: u32 = 0x210; +pub const R_AARCH64_TLSLD_ADD_DTPREL_LO12: u32 = 0x211; +pub const R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC: u32 = 0x212; +pub const R_AARCH64_TLSLD_LDST8_DTPREL_LO12: u32 = 0x213; +pub const R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC: u32 = 0x214; +pub const R_AARCH64_TLSLD_LDST16_DTPREL_LO12: u32 = 0x215; +pub const R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC: u32 = 0x216; +pub const R_AARCH64_TLSLD_LDST32_DTPREL_LO12: u32 = 0x217; +pub const R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC: u32 = 0x218; +pub const R_AARCH64_TLSLD_LDST64_DTPREL_LO12: u32 = 0x219; +pub const R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC: u32 = 0x21a; +pub const R_AARCH64_TLSIE_MOVW_GOTTPREL_G1: u32 = 0x21b; +pub const R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC: u32 = 0x21c; +pub const R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: u32 = 0x21d; +pub const R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: u32 = 0x21e; +pub const R_AARCH64_TLSIE_LD_GOTTPREL_PREL19: u32 = 0x21f; +pub const R_AARCH64_TLSLE_MOVW_TPREL_G2: u32 = 0x220; +pub const R_AARCH64_TLSLE_MOVW_TPREL_G1: u32 = 0x221; +pub const R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: u32 = 0x222; +pub const R_AARCH64_TLSLE_MOVW_TPREL_G0: u32 = 0x223; +pub const R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: u32 = 0x224; +pub const R_AARCH64_TLSLE_ADD_TPREL_HI12: u32 = 0x225; +pub const R_AARCH64_TLSLE_ADD_TPREL_LO12: u32 = 0x226; +pub const R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: u32 = 0x227; +pub const R_AARCH64_TLSLE_LDST8_TPREL_LO12: u32 = 0x228; +pub const R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: u32 = 0x229; +pub const R_AARCH64_TLSLE_LDST16_TPREL_LO12: u32 = 0x22a; +pub const R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: u32 = 0x22b; +pub const R_AARCH64_TLSLE_LDST32_TPREL_LO12: u32 = 0x22c; +pub const R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: u32 = 0x22d; +pub const R_AARCH64_TLSLE_LDST64_TPREL_LO12: u32 = 0x22e; +pub const R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: u32 = 0x22f; +pub const R_AARCH64_TLSDESC_ADR_PAGE21: u32 = 0x232; +pub const R_AARCH64_TLSDESC_LD64_LO12: u32 = 0x233; +pub const R_AARCH64_TLSDESC_ADD_LO12: u32 = 0x234; +pub const R_AARCH64_TLSDESC_CALL: u32 = 0x239; +pub const R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: u32 = 0x23b; +pub const R_AARCH64_COPY: u32 = 0x400; +pub const R_AARCH64_GLOB_DAT: u32 = 0x401; +pub const R_AARCH64_JUMP_SLOT: u32 = 0x402; +pub const R_AARCH64_RELATIVE: u32 = 0x403; +pub const R_AARCH64_TLS_DTPMOD64: u32 = 0x404; +pub const R_AARCH64_TLS_DTPREL64: u32 = 0x405; +pub const R_AARCH64_TLS_TPREL64: u32 = 0x406; +pub const R_AARCH64_TLSDESC: u32 = 0x407; +pub const R_AARCH64_IRELATIVE: u32 = 0x408; + +pub const R_ARM_NONE: u32 = 0x0; +pub const R_ARM_PC24: u32 = 0x1; +pub const R_ARM_ABS32: u32 = 0x2; +pub const R_ARM_REL32: u32 = 0x3; +pub const R_ARM_LDR_PC_G0: u32 = 0x4; +pub const R_ARM_ABS16: u32 = 0x5; +pub const R_ARM_ABS12: u32 = 0x6; +pub const R_ARM_THM_ABS5: u32 = 0x7; +pub const R_ARM_ABS8: u32 = 0x8; +pub const R_ARM_SBREL32: u32 = 0x9; +pub const R_ARM_THM_CALL: u32 = 0xa; +pub const R_ARM_THM_PC8: u32 = 0xb; +pub const R_ARM_BREL_ADJ: u32 = 0xc; +pub const R_ARM_TLS_DESC: u32 = 0xd; +pub const R_ARM_THM_SWI8: u32 = 0xe; +pub const R_ARM_XPC25: u32 = 0xf; +pub const R_ARM_THM_XPC22: u32 = 0x10; +pub const R_ARM_TLS_DTPMOD32: u32 = 0x11; +pub const R_ARM_TLS_DTPOFF32: u32 = 0x12; +pub const R_ARM_TLS_TPOFF32: u32 = 0x13; +pub const R_ARM_COPY: u32 = 0x14; +pub const R_ARM_GLOB_DAT: u32 = 0x15; +pub const R_ARM_JUMP_SLOT: u32 = 0x16; +pub const R_ARM_RELATIVE: u32 = 0x17; +pub const R_ARM_GOTOFF32: u32 = 0x18; +pub const R_ARM_BASE_PREL: u32 = 0x19; +pub const R_ARM_GOT_BREL: u32 = 0x1a; +pub const R_ARM_PLT32: u32 = 0x1b; +pub const R_ARM_CALL: u32 = 0x1c; +pub const R_ARM_JUMP24: u32 = 0x1d; +pub const R_ARM_THM_JUMP24: u32 = 0x1e; +pub const R_ARM_BASE_ABS: u32 = 0x1f; +pub const R_ARM_ALU_PCREL_7_0: u32 = 0x20; +pub const R_ARM_ALU_PCREL_15_8: u32 = 0x21; +pub const R_ARM_ALU_PCREL_23_15: u32 = 0x22; +pub const R_ARM_LDR_SBREL_11_0_NC: u32 = 0x23; +pub const R_ARM_ALU_SBREL_19_12_NC: u32 = 0x24; +pub const R_ARM_ALU_SBREL_27_20_CK: u32 = 0x25; +pub const R_ARM_TARGET1: u32 = 0x26; +pub const R_ARM_SBREL31: u32 = 0x27; +pub const R_ARM_V4BX: u32 = 0x28; +pub const R_ARM_TARGET2: u32 = 0x29; +pub const R_ARM_PREL31: u32 = 0x2a; +pub const R_ARM_MOVW_ABS_NC: u32 = 0x2b; +pub const R_ARM_MOVT_ABS: u32 = 0x2c; +pub const R_ARM_MOVW_PREL_NC: u32 = 0x2d; +pub const R_ARM_MOVT_PREL: u32 = 0x2e; +pub const R_ARM_THM_MOVW_ABS_NC: u32 = 0x2f; +pub const R_ARM_THM_MOVT_ABS: u32 = 0x30; +pub const R_ARM_THM_MOVW_PREL_NC: u32 = 0x31; +pub const R_ARM_THM_MOVT_PREL: u32 = 0x32; +pub const R_ARM_THM_JUMP19: u32 = 0x33; +pub const R_ARM_THM_JUMP6: u32 = 0x34; +pub const R_ARM_THM_ALU_PREL_11_0: u32 = 0x35; +pub const R_ARM_THM_PC12: u32 = 0x36; +pub const R_ARM_ABS32_NOI: u32 = 0x37; +pub const R_ARM_REL32_NOI: u32 = 0x38; +pub const R_ARM_ALU_PC_G0_NC: u32 = 0x39; +pub const R_ARM_ALU_PC_G0: u32 = 0x3a; +pub const R_ARM_ALU_PC_G1_NC: u32 = 0x3b; +pub const R_ARM_ALU_PC_G1: u32 = 0x3c; +pub const R_ARM_ALU_PC_G2: u32 = 0x3d; +pub const R_ARM_LDR_PC_G1: u32 = 0x3e; +pub const R_ARM_LDR_PC_G2: u32 = 0x3f; +pub const R_ARM_LDRS_PC_G0: u32 = 0x40; +pub const R_ARM_LDRS_PC_G1: u32 = 0x41; +pub const R_ARM_LDRS_PC_G2: u32 = 0x42; +pub const R_ARM_LDC_PC_G0: u32 = 0x43; +pub const R_ARM_LDC_PC_G1: u32 = 0x44; +pub const R_ARM_LDC_PC_G2: u32 = 0x45; +pub const R_ARM_ALU_SB_G0_NC: u32 = 0x46; +pub const R_ARM_ALU_SB_G0: u32 = 0x47; +pub const R_ARM_ALU_SB_G1_NC: u32 = 0x48; +pub const R_ARM_ALU_SB_G1: u32 = 0x49; +pub const R_ARM_ALU_SB_G2: u32 = 0x4a; +pub const R_ARM_LDR_SB_G0: u32 = 0x4b; +pub const R_ARM_LDR_SB_G1: u32 = 0x4c; +pub const R_ARM_LDR_SB_G2: u32 = 0x4d; +pub const R_ARM_LDRS_SB_G0: u32 = 0x4e; +pub const R_ARM_LDRS_SB_G1: u32 = 0x4f; +pub const R_ARM_LDRS_SB_G2: u32 = 0x50; +pub const R_ARM_LDC_SB_G0: u32 = 0x51; +pub const R_ARM_LDC_SB_G1: u32 = 0x52; +pub const R_ARM_LDC_SB_G2: u32 = 0x53; +pub const R_ARM_MOVW_BREL_NC: u32 = 0x54; +pub const R_ARM_MOVT_BREL: u32 = 0x55; +pub const R_ARM_MOVW_BREL: u32 = 0x56; +pub const R_ARM_THM_MOVW_BREL_NC: u32 = 0x57; +pub const R_ARM_THM_MOVT_BREL: u32 = 0x58; +pub const R_ARM_THM_MOVW_BREL: u32 = 0x59; +pub const R_ARM_TLS_GOTDESC: u32 = 0x5a; +pub const R_ARM_TLS_CALL: u32 = 0x5b; +pub const R_ARM_TLS_DESCSEQ: u32 = 0x5c; +pub const R_ARM_THM_TLS_CALL: u32 = 0x5d; +pub const R_ARM_PLT32_ABS: u32 = 0x5e; +pub const R_ARM_GOT_ABS: u32 = 0x5f; +pub const R_ARM_GOT_PREL: u32 = 0x60; +pub const R_ARM_GOT_BREL12: u32 = 0x61; +pub const R_ARM_GOTOFF12: u32 = 0x62; +pub const R_ARM_GOTRELAX: u32 = 0x63; +pub const R_ARM_GNU_VTENTRY: u32 = 0x64; +pub const R_ARM_GNU_VTINHERIT: u32 = 0x65; +pub const R_ARM_THM_JUMP11: u32 = 0x66; +pub const R_ARM_THM_JUMP8: u32 = 0x67; +pub const R_ARM_TLS_GD32: u32 = 0x68; +pub const R_ARM_TLS_LDM32: u32 = 0x69; +pub const R_ARM_TLS_LDO32: u32 = 0x6a; +pub const R_ARM_TLS_IE32: u32 = 0x6b; +pub const R_ARM_TLS_LE32: u32 = 0x6c; +pub const R_ARM_TLS_LDO12: u32 = 0x6d; +pub const R_ARM_TLS_LE12: u32 = 0x6e; +pub const R_ARM_TLS_IE12GP: u32 = 0x6f; +pub const R_ARM_PRIVATE_0: u32 = 0x70; +pub const R_ARM_PRIVATE_1: u32 = 0x71; +pub const R_ARM_PRIVATE_2: u32 = 0x72; +pub const R_ARM_PRIVATE_3: u32 = 0x73; +pub const R_ARM_PRIVATE_4: u32 = 0x74; +pub const R_ARM_PRIVATE_5: u32 = 0x75; +pub const R_ARM_PRIVATE_6: u32 = 0x76; +pub const R_ARM_PRIVATE_7: u32 = 0x77; +pub const R_ARM_PRIVATE_8: u32 = 0x78; +pub const R_ARM_PRIVATE_9: u32 = 0x79; +pub const R_ARM_PRIVATE_10: u32 = 0x7a; +pub const R_ARM_PRIVATE_11: u32 = 0x7b; +pub const R_ARM_PRIVATE_12: u32 = 0x7c; +pub const R_ARM_PRIVATE_13: u32 = 0x7d; +pub const R_ARM_PRIVATE_14: u32 = 0x7e; +pub const R_ARM_PRIVATE_15: u32 = 0x7f; +pub const R_ARM_ME_TOO: u32 = 0x80; +pub const R_ARM_THM_TLS_DESCSEQ16: u32 = 0x81; +pub const R_ARM_THM_TLS_DESCSEQ32: u32 = 0x82; +pub const R_ARM_THM_BF16: u32 = 0x88; +pub const R_ARM_THM_BF12: u32 = 0x89; +pub const R_ARM_THM_BF18: u32 = 0x8a; +pub const R_ARM_IRELATIVE: u32 = 0xa0; + +pub const R_RISCV_NONE: u32 = 0; +pub const R_RISCV_32: u32 = 1; +pub const R_RISCV_64: u32 = 2; +pub const R_RISCV_RELATIVE: u32 = 3; +pub const R_RISCV_COPY: u32 = 4; +pub const R_RISCV_JUMP_SLOT: u32 = 5; +pub const R_RISCV_TLS_DTPMOD32: u32 = 6; +pub const R_RISCV_TLS_DTPMOD64: u32 = 7; +pub const R_RISCV_TLS_DTPREL32: u32 = 8; +pub const R_RISCV_TLS_DTPREL64: u32 = 9; +pub const R_RISCV_TLS_TPREL32: u32 = 10; +pub const R_RISCV_TLS_TPREL64: u32 = 11; +pub const R_RISCV_TLSDESC: u32 = 12; +pub const R_RISCV_BRANCH: u32 = 16; +pub const R_RISCV_JAL: u32 = 17; +pub const R_RISCV_CALL: u32 = 18; +pub const R_RISCV_CALL_PLT: u32 = 19; +pub const R_RISCV_GOT_HI20: u32 = 20; +pub const R_RISCV_TLS_GOT_HI20: u32 = 21; +pub const R_RISCV_TLS_GD_HI20: u32 = 22; +pub const R_RISCV_PCREL_HI20: u32 = 23; +pub const R_RISCV_PCREL_LO12_I: u32 = 24; +pub const R_RISCV_PCREL_LO12_S: u32 = 25; +pub const R_RISCV_HI20: u32 = 26; +pub const R_RISCV_LO12_I: u32 = 27; +pub const R_RISCV_LO12_S: u32 = 28; +pub const R_RISCV_TPREL_HI20: u32 = 29; +pub const R_RISCV_TPREL_LO12_I: u32 = 30; +pub const R_RISCV_TPREL_LO12_S: u32 = 31; +pub const R_RISCV_TPREL_ADD: u32 = 32; +pub const R_RISCV_ADD8: u32 = 33; +pub const R_RISCV_ADD16: u32 = 34; +pub const R_RISCV_ADD32: u32 = 35; +pub const R_RISCV_ADD64: u32 = 36; +pub const R_RISCV_SUB8: u32 = 37; +pub const R_RISCV_SUB16: u32 = 38; +pub const R_RISCV_SUB32: u32 = 39; +pub const R_RISCV_SUB64: u32 = 40; +pub const R_RISCV_ALIGN: u32 = 43; +pub const R_RISCV_RVC_BRANCH: u32 = 44; +pub const R_RISCV_RVC_JUMP: u32 = 45; +pub const R_RISCV_RELAX: u32 = 51; +pub const R_RISCV_SUB6: u32 = 52; +pub const R_RISCV_SET6: u32 = 53; +pub const R_RISCV_SET8: u32 = 54; +pub const R_RISCV_SET16: u32 = 55; +pub const R_RISCV_SET32: u32 = 56; +pub const R_RISCV_GOT32_PCREL: u32 = 41; +pub const R_RISCV_32_PCREL: u32 = 57; +pub const R_RISCV_IRELATIVE: u32 = 58; +pub const R_RISCV_PLT32: u32 = 59; +pub const R_RISCV_SET_ULEB128: u32 = 60; +pub const R_RISCV_SUB_ULEB128: u32 = 61; +pub const R_RISCV_TLSDESC_HI20: u32 = 62; +pub const R_RISCV_TLSDESC_LOAD_LO12: u32 = 63; +pub const R_RISCV_TLSDESC_ADD_LO12: u32 = 64; +pub const R_RISCV_TLSDESC_CALL: u32 = 65; + +pub const R_PPC_NONE: u32 = 0; +pub const R_PPC_ADDR32: u32 = 1; +pub const R_PPC_ADDR24: u32 = 2; +pub const R_PPC_ADDR16: u32 = 3; +pub const R_PPC_ADDR16_LO: u32 = 4; +pub const R_PPC_ADDR16_HI: u32 = 5; +pub const R_PPC_ADDR16_HA: u32 = 6; +pub const R_PPC_ADDR14: u32 = 7; +pub const R_PPC_ADDR14_BRTAKEN: u32 = 8; +pub const R_PPC_ADDR14_BRNTAKEN: u32 = 9; +pub const R_PPC_REL24: u32 = 10; +pub const R_PPC_REL14: u32 = 11; +pub const R_PPC_REL14_BRTAKEN: u32 = 12; +pub const R_PPC_REL14_BRNTAKEN: u32 = 13; +pub const R_PPC_GOT16: u32 = 14; +pub const R_PPC_GOT16_LO: u32 = 15; +pub const R_PPC_GOT16_HI: u32 = 16; +pub const R_PPC_GOT16_HA: u32 = 17; +pub const R_PPC_PLTREL24: u32 = 18; +pub const R_PPC_COPY: u32 = 19; +pub const R_PPC_GLOB_DAT: u32 = 20; +pub const R_PPC_JMP_SLOT: u32 = 21; +pub const R_PPC_RELATIVE: u32 = 22; +pub const R_PPC_LOCAL24PC: u32 = 23; +pub const R_PPC_UADDR32: u32 = 24; +pub const R_PPC_UADDR16: u32 = 25; +pub const R_PPC_REL32: u32 = 26; +pub const R_PPC_PLT32: u32 = 27; +pub const R_PPC_PLTREL32: u32 = 28; +pub const R_PPC_PLT16_LO: u32 = 29; +pub const R_PPC_PLT16_HI: u32 = 30; +pub const R_PPC_PLT16_HA: u32 = 31; +pub const R_PPC_SDAREL16: u32 = 32; +pub const R_PPC_SECTOFF: u32 = 33; +pub const R_PPC_SECTOFF_LO: u32 = 34; +pub const R_PPC_SECTOFF_HI: u32 = 35; +pub const R_PPC_SECTOFF_HA: u32 = 36; +pub const R_PPC_ADDR30: u32 = 37; +pub const R_PPC_TLS: u32 = 67; +pub const R_PPC_DTPMOD32: u32 = 68; +pub const R_PPC_TPREL16: u32 = 69; +pub const R_PPC_TPREL16_LO: u32 = 70; +pub const R_PPC_TPREL16_HI: u32 = 71; +pub const R_PPC_TPREL16_HA: u32 = 72; +pub const R_PPC_TPREL32: u32 = 73; +pub const R_PPC_DTPREL16: u32 = 74; +pub const R_PPC_DTPREL16_LO: u32 = 75; +pub const R_PPC_DTPREL16_HI: u32 = 76; +pub const R_PPC_DTPREL16_HA: u32 = 77; +pub const R_PPC_DTPREL32: u32 = 78; +pub const R_PPC_GOT_TLSGD16: u32 = 79; +pub const R_PPC_GOT_TLSGD16_LO: u32 = 80; +pub const R_PPC_GOT_TLSGD16_HI: u32 = 81; +pub const R_PPC_GOT_TLSGD16_HA: u32 = 82; +pub const R_PPC_GOT_TLSLD16: u32 = 83; +pub const R_PPC_GOT_TLSLD16_LO: u32 = 84; +pub const R_PPC_GOT_TLSLD16_HI: u32 = 85; +pub const R_PPC_GOT_TLSLD16_HA: u32 = 86; +pub const R_PPC_GOT_TPREL16: u32 = 87; +pub const R_PPC_GOT_TPREL16_LO: u32 = 88; +pub const R_PPC_GOT_TPREL16_HI: u32 = 89; +pub const R_PPC_GOT_TPREL16_HA: u32 = 90; +pub const R_PPC_GOT_DTPREL16: u32 = 91; +pub const R_PPC_GOT_DTPREL16_LO: u32 = 92; +pub const R_PPC_GOT_DTPREL16_HI: u32 = 93; +pub const R_PPC_GOT_DTPREL16_HA: u32 = 94; +pub const R_PPC_TLSGD: u32 = 95; +pub const R_PPC_TLSLD: u32 = 96; +pub const R_PPC_PLTSEQ: u32 = 119; +pub const R_PPC_PLTCALL: u32 = 120; +pub const R_PPC_IRELATIVE: u32 = 248; +pub const R_PPC_REL16: u32 = 249; +pub const R_PPC_REL16_LO: u32 = 250; +pub const R_PPC_REL16_HI: u32 = 251; +pub const R_PPC_REL16_HA: u32 = 252; + +pub const R_PPC64_NONE: u32 = 0; +pub const R_PPC64_ADDR32: u32 = 1; +pub const R_PPC64_ADDR24: u32 = 2; +pub const R_PPC64_ADDR16: u32 = 3; +pub const R_PPC64_ADDR16_LO: u32 = 4; +pub const R_PPC64_ADDR16_HI: u32 = 5; +pub const R_PPC64_ADDR16_HA: u32 = 6; +pub const R_PPC64_ADDR14: u32 = 7; +pub const R_PPC64_ADDR14_BRTAKEN: u32 = 8; +pub const R_PPC64_ADDR14_BRNTAKEN: u32 = 9; +pub const R_PPC64_REL24: u32 = 10; +pub const R_PPC64_REL14: u32 = 11; +pub const R_PPC64_REL14_BRTAKEN: u32 = 12; +pub const R_PPC64_REL14_BRNTAKEN: u32 = 13; +pub const R_PPC64_GOT16: u32 = 14; +pub const R_PPC64_GOT16_LO: u32 = 15; +pub const R_PPC64_GOT16_HI: u32 = 16; +pub const R_PPC64_GOT16_HA: u32 = 17; +pub const R_PPC64_COPY: u32 = 19; +pub const R_PPC64_GLOB_DAT: u32 = 20; +pub const R_PPC64_JMP_SLOT: u32 = 21; +pub const R_PPC64_RELATIVE: u32 = 22; +pub const R_PPC64_REL32: u32 = 26; +pub const R_PPC64_PLT16_LO: u32 = 29; +pub const R_PPC64_PLT16_HI: u32 = 30; +pub const R_PPC64_PLT16_HA: u32 = 31; +pub const R_PPC64_ADDR64: u32 = 38; +pub const R_PPC64_ADDR16_HIGHER: u32 = 39; +pub const R_PPC64_ADDR16_HIGHERA: u32 = 40; +pub const R_PPC64_ADDR16_HIGHEST: u32 = 41; +pub const R_PPC64_ADDR16_HIGHESTA: u32 = 42; +pub const R_PPC64_REL64: u32 = 44; +pub const R_PPC64_TOC16: u32 = 47; +pub const R_PPC64_TOC16_LO: u32 = 48; +pub const R_PPC64_TOC16_HI: u32 = 49; +pub const R_PPC64_TOC16_HA: u32 = 50; +pub const R_PPC64_TOC: u32 = 51; +pub const R_PPC64_ADDR16_DS: u32 = 56; +pub const R_PPC64_ADDR16_LO_DS: u32 = 57; +pub const R_PPC64_GOT16_DS: u32 = 58; +pub const R_PPC64_GOT16_LO_DS: u32 = 59; +pub const R_PPC64_PLT16_LO_DS: u32 = 60; +pub const R_PPC64_TOC16_DS: u32 = 63; +pub const R_PPC64_TOC16_LO_DS: u32 = 64; +pub const R_PPC64_TLS: u32 = 67; +pub const R_PPC64_DTPMOD64: u32 = 68; +pub const R_PPC64_TPREL16: u32 = 69; +pub const R_PPC64_TPREL16_LO: u32 = 70; +pub const R_PPC64_TPREL16_HI: u32 = 71; +pub const R_PPC64_TPREL16_HA: u32 = 72; +pub const R_PPC64_TPREL64: u32 = 73; +pub const R_PPC64_DTPREL16: u32 = 74; +pub const R_PPC64_DTPREL16_LO: u32 = 75; +pub const R_PPC64_DTPREL16_HI: u32 = 76; +pub const R_PPC64_DTPREL16_HA: u32 = 77; +pub const R_PPC64_DTPREL64: u32 = 78; +pub const R_PPC64_GOT_TLSGD16: u32 = 79; +pub const R_PPC64_GOT_TLSGD16_LO: u32 = 80; +pub const R_PPC64_GOT_TLSGD16_HI: u32 = 81; +pub const R_PPC64_GOT_TLSGD16_HA: u32 = 82; +pub const R_PPC64_GOT_TLSLD16: u32 = 83; +pub const R_PPC64_GOT_TLSLD16_LO: u32 = 84; +pub const R_PPC64_GOT_TLSLD16_HI: u32 = 85; +pub const R_PPC64_GOT_TLSLD16_HA: u32 = 86; +pub const R_PPC64_GOT_TPREL16_DS: u32 = 87; +pub const R_PPC64_GOT_TPREL16_LO_DS: u32 = 88; +pub const R_PPC64_GOT_TPREL16_HI: u32 = 89; +pub const R_PPC64_GOT_TPREL16_HA: u32 = 90; +pub const R_PPC64_GOT_DTPREL16_DS: u32 = 91; +pub const R_PPC64_GOT_DTPREL16_LO_DS: u32 = 92; +pub const R_PPC64_GOT_DTPREL16_HI: u32 = 93; +pub const R_PPC64_GOT_DTPREL16_HA: u32 = 94; +pub const R_PPC64_TPREL16_DS: u32 = 95; +pub const R_PPC64_TPREL16_LO_DS: u32 = 96; +pub const R_PPC64_TPREL16_HIGHER: u32 = 97; +pub const R_PPC64_TPREL16_HIGHERA: u32 = 98; +pub const R_PPC64_TPREL16_HIGHEST: u32 = 99; +pub const R_PPC64_TPREL16_HIGHESTA: u32 = 100; +pub const R_PPC64_DTPREL16_DS: u32 = 101; +pub const R_PPC64_DTPREL16_LO_DS: u32 = 102; +pub const R_PPC64_DTPREL16_HIGHER: u32 = 103; +pub const R_PPC64_DTPREL16_HIGHERA: u32 = 104; +pub const R_PPC64_DTPREL16_HIGHEST: u32 = 105; +pub const R_PPC64_DTPREL16_HIGHESTA: u32 = 106; +pub const R_PPC64_TLSGD: u32 = 107; +pub const R_PPC64_TLSLD: u32 = 108; +pub const R_PPC64_ADDR16_HIGH: u32 = 110; +pub const R_PPC64_ADDR16_HIGHA: u32 = 111; +pub const R_PPC64_TPREL16_HIGH: u32 = 112; +pub const R_PPC64_TPREL16_HIGHA: u32 = 113; +pub const R_PPC64_DTPREL16_HIGH: u32 = 114; +pub const R_PPC64_DTPREL16_HIGHA: u32 = 115; +pub const R_PPC64_REL24_NOTOC: u32 = 116; +pub const R_PPC64_ENTRY: u32 = 118; +pub const R_PPC64_PLTSEQ: u32 = 119; +pub const R_PPC64_PLTCALL: u32 = 120; +pub const R_PPC64_PLTSEQ_NOTOC: u32 = 121; +pub const R_PPC64_PLTCALL_NOTOC: u32 = 122; +pub const R_PPC64_PCREL_OPT: u32 = 123; +pub const R_PPC64_PCREL34: u32 = 132; +pub const R_PPC64_GOT_PCREL34: u32 = 133; +pub const R_PPC64_PLT_PCREL34: u32 = 134; +pub const R_PPC64_PLT_PCREL34_NOTOC: u32 = 135; +pub const R_PPC64_TPREL34: u32 = 146; +pub const R_PPC64_DTPREL34: u32 = 147; +pub const R_PPC64_GOT_TLSGD_PCREL34: u32 = 148; +pub const R_PPC64_GOT_TLSLD_PCREL34: u32 = 149; +pub const R_PPC64_GOT_TPREL_PCREL34: u32 = 150; +pub const R_PPC64_IRELATIVE: u32 = 248; +pub const R_PPC64_REL16: u32 = 249; +pub const R_PPC64_REL16_LO: u32 = 250; +pub const R_PPC64_REL16_HI: u32 = 251; +pub const R_PPC64_REL16_HA: u32 = 252; + +pub const R_SPARC_NONE: u32 = 0; +pub const R_SPARC_8: u32 = 1; +pub const R_SPARC_16: u32 = 2; +pub const R_SPARC_32: u32 = 3; +pub const R_SPARC_DISP8: u32 = 4; +pub const R_SPARC_DISP16: u32 = 5; +pub const R_SPARC_DISP32: u32 = 6; +pub const R_SPARC_WDISP30: u32 = 7; +pub const R_SPARC_WDISP22: u32 = 8; +pub const R_SPARC_HI22: u32 = 9; +pub const R_SPARC_22: u32 = 10; +pub const R_SPARC_13: u32 = 11; +pub const R_SPARC_LO10: u32 = 12; +pub const R_SPARC_GOT10: u32 = 13; +pub const R_SPARC_GOT13: u32 = 14; +pub const R_SPARC_GOT22: u32 = 15; +pub const R_SPARC_PC10: u32 = 16; +pub const R_SPARC_PC22: u32 = 17; +pub const R_SPARC_WPLT30: u32 = 18; +pub const R_SPARC_COPY: u32 = 19; +pub const R_SPARC_GLOB_DAT: u32 = 20; +pub const R_SPARC_JMP_SLOT: u32 = 21; +pub const R_SPARC_RELATIVE: u32 = 22; +pub const R_SPARC_UA32: u32 = 23; +pub const R_SPARC_PLT32: u32 = 24; +pub const R_SPARC_HIPLT22: u32 = 25; +pub const R_SPARC_LOPLT10: u32 = 26; +pub const R_SPARC_PCPLT32: u32 = 27; +pub const R_SPARC_PCPLT22: u32 = 28; +pub const R_SPARC_PCPLT10: u32 = 29; +pub const R_SPARC_10: u32 = 30; +pub const R_SPARC_11: u32 = 31; +pub const R_SPARC_64: u32 = 32; +pub const R_SPARC_OLO10: u32 = 33; +pub const R_SPARC_HH22: u32 = 34; +pub const R_SPARC_HM10: u32 = 35; +pub const R_SPARC_LM22: u32 = 36; +pub const R_SPARC_PC_HH22: u32 = 37; +pub const R_SPARC_PC_HM10: u32 = 38; +pub const R_SPARC_PC_LM22: u32 = 39; +pub const R_SPARC_WDISP16: u32 = 40; +pub const R_SPARC_WDISP19: u32 = 41; +pub const R_SPARC_7: u32 = 43; +pub const R_SPARC_5: u32 = 44; +pub const R_SPARC_6: u32 = 45; +pub const R_SPARC_DISP64: u32 = 46; +pub const R_SPARC_PLT64: u32 = 47; +pub const R_SPARC_HIX22: u32 = 48; +pub const R_SPARC_LOX10: u32 = 49; +pub const R_SPARC_H44: u32 = 50; +pub const R_SPARC_M44: u32 = 51; +pub const R_SPARC_L44: u32 = 52; +pub const R_SPARC_REGISTER: u32 = 53; +pub const R_SPARC_UA64: u32 = 54; +pub const R_SPARC_UA16: u32 = 55; +pub const R_SPARC_TLS_GD_HI22: u32 = 56; +pub const R_SPARC_TLS_GD_LO10: u32 = 57; +pub const R_SPARC_TLS_GD_ADD: u32 = 58; +pub const R_SPARC_TLS_GD_CALL: u32 = 59; +pub const R_SPARC_TLS_LDM_HI22: u32 = 60; +pub const R_SPARC_TLS_LDM_LO10: u32 = 61; +pub const R_SPARC_TLS_LDM_ADD: u32 = 62; +pub const R_SPARC_TLS_LDM_CALL: u32 = 63; +pub const R_SPARC_TLS_LDO_HIX22: u32 = 64; +pub const R_SPARC_TLS_LDO_LOX10: u32 = 65; +pub const R_SPARC_TLS_LDO_ADD: u32 = 66; +pub const R_SPARC_TLS_IE_HI22: u32 = 67; +pub const R_SPARC_TLS_IE_LO10: u32 = 68; +pub const R_SPARC_TLS_IE_LD: u32 = 69; +pub const R_SPARC_TLS_IE_LDX: u32 = 70; +pub const R_SPARC_TLS_IE_ADD: u32 = 71; +pub const R_SPARC_TLS_LE_HIX22: u32 = 72; +pub const R_SPARC_TLS_LE_LOX10: u32 = 73; +pub const R_SPARC_TLS_DTPMOD32: u32 = 74; +pub const R_SPARC_TLS_DTPMOD64: u32 = 75; +pub const R_SPARC_TLS_DTPOFF32: u32 = 76; +pub const R_SPARC_TLS_DTPOFF64: u32 = 77; +pub const R_SPARC_TLS_TPOFF32: u32 = 78; +pub const R_SPARC_TLS_TPOFF64: u32 = 79; +pub const R_SPARC_GOTDATA_HIX22: u32 = 80; +pub const R_SPARC_GOTDATA_LOX10: u32 = 81; +pub const R_SPARC_GOTDATA_OP_HIX22: u32 = 82; +pub const R_SPARC_GOTDATA_OP_LOX10: u32 = 83; +pub const R_SPARC_GOTDATA_OP: u32 = 84; +pub const R_SPARC_SIZE32: u32 = 86; +pub const R_SPARC_JMP_IREL: u32 = 248; +pub const R_SPARC_IRELATIVE: u32 = 249; + +pub const R_390_NONE: u32 = 0; +pub const R_390_8: u32 = 1; +pub const R_390_12: u32 = 2; +pub const R_390_16: u32 = 3; +pub const R_390_32: u32 = 4; +pub const R_390_PC32: u32 = 5; +pub const R_390_GOT12: u32 = 6; +pub const R_390_GOT32: u32 = 7; +pub const R_390_PLT32: u32 = 8; +pub const R_390_COPY: u32 = 9; +pub const R_390_GLOB_DAT: u32 = 10; +pub const R_390_JMP_SLOT: u32 = 11; +pub const R_390_RELATIVE: u32 = 12; +pub const R_390_GOTOFF32: u32 = 13; +pub const R_390_GOTPC: u32 = 14; +pub const R_390_GOT16: u32 = 15; +pub const R_390_PC16: u32 = 16; +pub const R_390_PC16DBL: u32 = 17; +pub const R_390_PLT16DBL: u32 = 18; +pub const R_390_PC32DBL: u32 = 19; +pub const R_390_PLT32DBL: u32 = 20; +pub const R_390_GOTPCDBL: u32 = 21; +pub const R_390_64: u32 = 22; +pub const R_390_PC64: u32 = 23; +pub const R_390_GOT64: u32 = 24; +pub const R_390_PLT64: u32 = 25; +pub const R_390_GOTENT: u32 = 26; +pub const R_390_GOTOFF16: u32 = 27; +pub const R_390_GOTOFF64: u32 = 28; +pub const R_390_GOTPLT12: u32 = 29; +pub const R_390_GOTPLT16: u32 = 30; +pub const R_390_GOTPLT32: u32 = 31; +pub const R_390_GOTPLT64: u32 = 32; +pub const R_390_GOTPLTENT: u32 = 33; +pub const R_390_PLTOFF16: u32 = 34; +pub const R_390_PLTOFF32: u32 = 35; +pub const R_390_PLTOFF64: u32 = 36; +pub const R_390_TLS_LOAD: u32 = 37; +pub const R_390_TLS_GDCALL: u32 = 38; +pub const R_390_TLS_LDCALL: u32 = 39; +pub const R_390_TLS_GD32: u32 = 40; +pub const R_390_TLS_GD64: u32 = 41; +pub const R_390_TLS_GOTIE12: u32 = 42; +pub const R_390_TLS_GOTIE32: u32 = 43; +pub const R_390_TLS_GOTIE64: u32 = 44; +pub const R_390_TLS_LDM32: u32 = 45; +pub const R_390_TLS_LDM64: u32 = 46; +pub const R_390_TLS_IE32: u32 = 47; +pub const R_390_TLS_IE64: u32 = 48; +pub const R_390_TLS_IEENT: u32 = 49; +pub const R_390_TLS_LE32: u32 = 50; +pub const R_390_TLS_LE64: u32 = 51; +pub const R_390_TLS_LDO32: u32 = 52; +pub const R_390_TLS_LDO64: u32 = 53; +pub const R_390_TLS_DTPMOD: u32 = 54; +pub const R_390_TLS_DTPOFF: u32 = 55; +pub const R_390_TLS_TPOFF: u32 = 56; +pub const R_390_20: u32 = 57; +pub const R_390_GOT20: u32 = 58; +pub const R_390_GOTPLT20: u32 = 59; +pub const R_390_TLS_GOTIE20: u32 = 60; +pub const R_390_IRELATIVE: u32 = 61; +pub const R_390_PC12DBL: u32 = 62; +pub const R_390_PLT12DBL: u32 = 63; +pub const R_390_PC24DBL: u32 = 64; +pub const R_390_PLT24DBL: u32 = 65; + +pub const R_68K_NONE: u32 = 0; +pub const R_68K_32: u32 = 1; +pub const R_68K_16: u32 = 2; +pub const R_68K_8: u32 = 3; +pub const R_68K_PC32: u32 = 4; +pub const R_68K_PC16: u32 = 5; +pub const R_68K_PC8: u32 = 6; +pub const R_68K_GOTPCREL32: u32 = 7; +pub const R_68K_GOTPCREL16: u32 = 8; +pub const R_68K_GOTPCREL8: u32 = 9; +pub const R_68K_GOTOFF32: u32 = 10; +pub const R_68K_GOTOFF16: u32 = 11; +pub const R_68K_GOTOFF8: u32 = 12; +pub const R_68K_PLT32: u32 = 13; +pub const R_68K_PLT16: u32 = 14; +pub const R_68K_PLT8: u32 = 15; +pub const R_68K_PLTOFF32: u32 = 16; +pub const R_68K_PLTOFF16: u32 = 17; +pub const R_68K_PLTOFF8: u32 = 18; +pub const R_68K_COPY: u32 = 19; +pub const R_68K_GLOB_DAT: u32 = 20; +pub const R_68K_JMP_SLOT: u32 = 21; +pub const R_68K_RELATIVE: u32 = 22; +pub const R_68K_TLS_GD32: u32 = 25; +pub const R_68K_TLS_GD16: u32 = 26; +pub const R_68K_TLS_GD8: u32 = 27; +pub const R_68K_TLS_LDM32: u32 = 28; +pub const R_68K_TLS_LDM16: u32 = 29; +pub const R_68K_TLS_LDM8: u32 = 30; +pub const R_68K_TLS_LDO32: u32 = 31; +pub const R_68K_TLS_LDO16: u32 = 32; +pub const R_68K_TLS_LDO8: u32 = 33; +pub const R_68K_TLS_IE32: u32 = 34; +pub const R_68K_TLS_IE16: u32 = 35; +pub const R_68K_TLS_IE8: u32 = 36; +pub const R_68K_TLS_LE32: u32 = 37; +pub const R_68K_TLS_LE16: u32 = 38; +pub const R_68K_TLS_LE8: u32 = 39; +pub const R_68K_TLS_DTPMOD32: u32 = 40; +pub const R_68K_TLS_DTPREL32: u32 = 41; +pub const R_68K_TLS_TPREL32: u32 = 42; + +pub const R_SH_NONE: u32 = 0; +pub const R_SH_DIR32: u32 = 1; +pub const R_SH_REL32: u32 = 2; +pub const R_SH_DIR8WPN: u32 = 3; +pub const R_SH_IND12W: u32 = 4; +pub const R_SH_DIR8WPL: u32 = 5; +pub const R_SH_DIR8WPZ: u32 = 6; +pub const R_SH_DIR8BP: u32 = 7; +pub const R_SH_DIR8W: u32 = 8; +pub const R_SH_DIR8L: u32 = 9; +pub const R_SH_TLS_GD_32: u32 = 144; +pub const R_SH_TLS_LD_32: u32 = 145; +pub const R_SH_TLS_LDO_32: u32 = 146; +pub const R_SH_TLS_IE_32: u32 = 147; +pub const R_SH_TLS_LE_32: u32 = 148; +pub const R_SH_TLS_DTPMOD32: u32 = 149; +pub const R_SH_TLS_DTPOFF32: u32 = 150; +pub const R_SH_TLS_TPOFF32: u32 = 151; +pub const R_SH_GOT32: u32 = 160; +pub const R_SH_PLT32: u32 = 161; +pub const R_SH_COPY: u32 = 162; +pub const R_SH_GLOB_DAT: u32 = 163; +pub const R_SH_JMP_SLOT: u32 = 164; +pub const R_SH_RELATIVE: u32 = 165; +pub const R_SH_GOTOFF: u32 = 166; +pub const R_SH_GOTPC: u32 = 167; +pub const R_SH_GOTPLT32: u32 = 168; + +pub const R_LARCH_NONE: u32 = 0; +pub const R_LARCH_32: u32 = 1; +pub const R_LARCH_64: u32 = 2; +pub const R_LARCH_RELATIVE: u32 = 3; +pub const R_LARCH_COPY: u32 = 4; +pub const R_LARCH_JUMP_SLOT: u32 = 5; +pub const R_LARCH_TLS_DTPMOD32: u32 = 6; +pub const R_LARCH_TLS_DTPMOD64: u32 = 7; +pub const R_LARCH_TLS_DTPREL32: u32 = 8; +pub const R_LARCH_TLS_DTPREL64: u32 = 9; +pub const R_LARCH_TLS_TPREL32: u32 = 10; +pub const R_LARCH_TLS_TPREL64: u32 = 11; +pub const R_LARCH_IRELATIVE: u32 = 12; +pub const R_LARCH_TLS_DESC32: u32 = 13; +pub const R_LARCH_TLS_DESC64: u32 = 14; +pub const R_LARCH_MARK_LA: u32 = 20; +pub const R_LARCH_MARK_PCREL: u32 = 21; +pub const R_LARCH_SOP_PUSH_PCREL: u32 = 22; +pub const R_LARCH_SOP_PUSH_ABSOLUTE: u32 = 23; +pub const R_LARCH_SOP_PUSH_DUP: u32 = 24; +pub const R_LARCH_SOP_PUSH_GPREL: u32 = 25; +pub const R_LARCH_SOP_PUSH_TLS_TPREL: u32 = 26; +pub const R_LARCH_SOP_PUSH_TLS_GOT: u32 = 27; +pub const R_LARCH_SOP_PUSH_TLS_GD: u32 = 28; +pub const R_LARCH_SOP_PUSH_PLT_PCREL: u32 = 29; +pub const R_LARCH_SOP_ASSERT: u32 = 30; +pub const R_LARCH_SOP_NOT: u32 = 31; +pub const R_LARCH_SOP_SUB: u32 = 32; +pub const R_LARCH_SOP_SL: u32 = 33; +pub const R_LARCH_SOP_SR: u32 = 34; +pub const R_LARCH_SOP_ADD: u32 = 35; +pub const R_LARCH_SOP_AND: u32 = 36; +pub const R_LARCH_SOP_IF_ELSE: u32 = 37; +pub const R_LARCH_SOP_POP_32_S_10_5: u32 = 38; +pub const R_LARCH_SOP_POP_32_U_10_12: u32 = 39; +pub const R_LARCH_SOP_POP_32_S_10_12: u32 = 40; +pub const R_LARCH_SOP_POP_32_S_10_16: u32 = 41; +pub const R_LARCH_SOP_POP_32_S_10_16_S2: u32 = 42; +pub const R_LARCH_SOP_POP_32_S_5_20: u32 = 43; +pub const R_LARCH_SOP_POP_32_S_0_5_10_16_S2: u32 = 44; +pub const R_LARCH_SOP_POP_32_S_0_10_10_16_S2: u32 = 45; +pub const R_LARCH_SOP_POP_32_U: u32 = 46; +pub const R_LARCH_ADD8: u32 = 47; +pub const R_LARCH_ADD16: u32 = 48; +pub const R_LARCH_ADD24: u32 = 49; +pub const R_LARCH_ADD32: u32 = 50; +pub const R_LARCH_ADD64: u32 = 51; +pub const R_LARCH_SUB8: u32 = 52; +pub const R_LARCH_SUB16: u32 = 53; +pub const R_LARCH_SUB24: u32 = 54; +pub const R_LARCH_SUB32: u32 = 55; +pub const R_LARCH_SUB64: u32 = 56; +pub const R_LARCH_GNU_VTINHERIT: u32 = 57; +pub const R_LARCH_GNU_VTENTRY: u32 = 58; +pub const R_LARCH_B16: u32 = 64; +pub const R_LARCH_B21: u32 = 65; +pub const R_LARCH_B26: u32 = 66; +pub const R_LARCH_ABS_HI20: u32 = 67; +pub const R_LARCH_ABS_LO12: u32 = 68; +pub const R_LARCH_ABS64_LO20: u32 = 69; +pub const R_LARCH_ABS64_HI12: u32 = 70; +pub const R_LARCH_PCALA_HI20: u32 = 71; +pub const R_LARCH_PCALA_LO12: u32 = 72; +pub const R_LARCH_PCALA64_LO20: u32 = 73; +pub const R_LARCH_PCALA64_HI12: u32 = 74; +pub const R_LARCH_GOT_PC_HI20: u32 = 75; +pub const R_LARCH_GOT_PC_LO12: u32 = 76; +pub const R_LARCH_GOT64_PC_LO20: u32 = 77; +pub const R_LARCH_GOT64_PC_HI12: u32 = 78; +pub const R_LARCH_GOT_HI20: u32 = 79; +pub const R_LARCH_GOT_LO12: u32 = 80; +pub const R_LARCH_GOT64_LO20: u32 = 81; +pub const R_LARCH_GOT64_HI12: u32 = 82; +pub const R_LARCH_TLS_LE_HI20: u32 = 83; +pub const R_LARCH_TLS_LE_LO12: u32 = 84; +pub const R_LARCH_TLS_LE64_LO20: u32 = 85; +pub const R_LARCH_TLS_LE64_HI12: u32 = 86; +pub const R_LARCH_TLS_IE_PC_HI20: u32 = 87; +pub const R_LARCH_TLS_IE_PC_LO12: u32 = 88; +pub const R_LARCH_TLS_IE64_PC_LO20: u32 = 89; +pub const R_LARCH_TLS_IE64_PC_HI12: u32 = 90; +pub const R_LARCH_TLS_IE_HI20: u32 = 91; +pub const R_LARCH_TLS_IE_LO12: u32 = 92; +pub const R_LARCH_TLS_IE64_LO20: u32 = 93; +pub const R_LARCH_TLS_IE64_HI12: u32 = 94; +pub const R_LARCH_TLS_LD_PC_HI20: u32 = 95; +pub const R_LARCH_TLS_LD_HI20: u32 = 96; +pub const R_LARCH_TLS_GD_PC_HI20: u32 = 97; +pub const R_LARCH_TLS_GD_HI20: u32 = 98; +pub const R_LARCH_32_PCREL: u32 = 99; +pub const R_LARCH_RELAX: u32 = 100; +pub const R_LARCH_DELETE: u32 = 101; +pub const R_LARCH_ALIGN: u32 = 102; +pub const R_LARCH_PCREL20_S2: u32 = 103; +pub const R_LARCH_CFA: u32 = 104; +pub const R_LARCH_ADD6: u32 = 105; +pub const R_LARCH_SUB6: u32 = 106; +pub const R_LARCH_ADD_ULEB128: u32 = 107; +pub const R_LARCH_SUB_ULEB128: u32 = 108; +pub const R_LARCH_64_PCREL: u32 = 109; +pub const R_LARCH_CALL36: u32 = 110; +pub const R_LARCH_TLS_DESC_PC_HI20: u32 = 111; +pub const R_LARCH_TLS_DESC_PC_LO12: u32 = 112; +pub const R_LARCH_TLS_DESC64_PC_LO20: u32 = 113; +pub const R_LARCH_TLS_DESC64_PC_HI12: u32 = 114; +pub const R_LARCH_TLS_DESC_HI20: u32 = 115; +pub const R_LARCH_TLS_DESC_LO12: u32 = 116; +pub const R_LARCH_TLS_DESC64_LO20: u32 = 117; +pub const R_LARCH_TLS_DESC64_HI12: u32 = 118; +pub const R_LARCH_TLS_DESC_LD: u32 = 119; +pub const R_LARCH_TLS_DESC_CALL: u32 = 120; +pub const R_LARCH_TLS_LE_HI20_R: u32 = 121; +pub const R_LARCH_TLS_LE_ADD_R: u32 = 122; +pub const R_LARCH_TLS_LE_LO12_R: u32 = 123; +pub const R_LARCH_TLS_LD_PCREL20_S2: u32 = 124; +pub const R_LARCH_TLS_GD_PCREL20_S2: u32 = 125; +pub const R_LARCH_TLS_DESC_PCREL20_S2: u32 = 126; + +pub const DW_EH_PE_absptr: u32 = 0; +pub const DW_EH_PE_omit: u32 = 0xff; +pub const DW_EH_PE_uleb128: u32 = 0x01; +pub const DW_EH_PE_udata2: u32 = 0x02; +pub const DW_EH_PE_udata4: u32 = 0x03; +pub const DW_EH_PE_udata8: u32 = 0x04; +pub const DW_EH_PE_signed: u32 = 0x08; +pub const DW_EH_PE_sleb128: u32 = 0x09; +pub const DW_EH_PE_sdata2: u32 = 0x0a; +pub const DW_EH_PE_sdata4: u32 = 0x0b; +pub const DW_EH_PE_sdata8: u32 = 0x0c; +pub const DW_EH_PE_pcrel: u32 = 0x10; +pub const DW_EH_PE_textrel: u32 = 0x20; +pub const DW_EH_PE_datarel: u32 = 0x30; +pub const DW_EH_PE_funcrel: u32 = 0x40; +pub const DW_EH_PE_aligned: u32 = 0x50; + +pub const DW_AT_low_pc: u32 = 0x11; +pub const DW_AT_high_pc: u32 = 0x12; +pub const DW_AT_producer: u32 = 0x25; +pub const DW_AT_ranges: u32 = 0x55; +pub const DW_AT_addr_base: u32 = 0x73; +pub const DW_AT_rnglists_base: u32 = 0x74; + +pub const DW_TAG_compile_unit: u32 = 0x11; +pub const DW_TAG_skeleton_unit: u32 = 0x4a; + +pub const DW_UT_compile: u32 = 0x01; +pub const DW_UT_type: u32 = 0x02; +pub const DW_UT_partial: u32 = 0x03; +pub const DW_UT_skeleton: u32 = 0x04; +pub const DW_UT_split_compile: u32 = 0x05; +pub const DW_UT_split_type: u32 = 0x06; + +pub const DW_FORM_addr: u32 = 0x01; +pub const DW_FORM_block2: u32 = 0x03; +pub const DW_FORM_block4: u32 = 0x04; +pub const DW_FORM_data2: u32 = 0x05; +pub const DW_FORM_data4: u32 = 0x06; +pub const DW_FORM_data8: u32 = 0x07; +pub const DW_FORM_string: u32 = 0x08; +pub const DW_FORM_block: u32 = 0x09; +pub const DW_FORM_block1: u32 = 0x0a; +pub const DW_FORM_data1: u32 = 0x0b; +pub const DW_FORM_flag: u32 = 0x0c; +pub const DW_FORM_sdata: u32 = 0x0d; +pub const DW_FORM_strp: u32 = 0x0e; +pub const DW_FORM_udata: u32 = 0x0f; +pub const DW_FORM_ref_addr: u32 = 0x10; +pub const DW_FORM_ref1: u32 = 0x11; +pub const DW_FORM_ref2: u32 = 0x12; +pub const DW_FORM_ref4: u32 = 0x13; +pub const DW_FORM_ref8: u32 = 0x14; +pub const DW_FORM_ref_udata: u32 = 0x15; +pub const DW_FORM_indirect: u32 = 0x16; +pub const DW_FORM_sec_offset: u32 = 0x17; +pub const DW_FORM_exprloc: u32 = 0x18; +pub const DW_FORM_flag_present: u32 = 0x19; +pub const DW_FORM_strx: u32 = 0x1a; +pub const DW_FORM_addrx: u32 = 0x1b; +pub const DW_FORM_ref_sup4: u32 = 0x1c; +pub const DW_FORM_strp_sup: u32 = 0x1d; +pub const DW_FORM_data16: u32 = 0x1e; +pub const DW_FORM_line_strp: u32 = 0x1f; +pub const DW_FORM_ref_sig8: u32 = 0x20; +pub const DW_FORM_implicit_const: u32 = 0x21; +pub const DW_FORM_loclistx: u32 = 0x22; +pub const DW_FORM_rnglistx: u32 = 0x23; +pub const DW_FORM_ref_sup8: u32 = 0x24; +pub const DW_FORM_strx1: u32 = 0x25; +pub const DW_FORM_strx2: u32 = 0x26; +pub const DW_FORM_strx3: u32 = 0x27; +pub const DW_FORM_strx4: u32 = 0x28; +pub const DW_FORM_addrx1: u32 = 0x29; +pub const DW_FORM_addrx2: u32 = 0x2a; +pub const DW_FORM_addrx3: u32 = 0x2b; +pub const DW_FORM_addrx4: u32 = 0x2c; + +pub const DW_RLE_end_of_list: u32 = 0x00; +pub const DW_RLE_base_addressx: u32 = 0x01; +pub const DW_RLE_startx_endx: u32 = 0x02; +pub const DW_RLE_startx_length: u32 = 0x03; +pub const DW_RLE_offset_pair: u32 = 0x04; +pub const DW_RLE_base_address: u32 = 0x05; +pub const DW_RLE_start_end: u32 = 0x06; +pub const DW_RLE_start_length: u32 = 0x07; diff --git a/src/elf/mod.rs b/src/elf/mod.rs new file mode 100644 index 000000000..f7b5beece --- /dev/null +++ b/src/elf/mod.rs @@ -0,0 +1,2112 @@ +//! ELF file format definitions. +//! +//! Records are decoded into the host-native structs defined here as they +//! are used: the on-disk encoding differs in byte order and word size +//! between targets, and decoding on access keeps the rest of the linker +//! free of both concerns while the big tables — section headers, symbols +//! and relocations — stay in the input files rather than being copied. +//! Encoding and decoding are driven by an [`Arch`](crate::arch::Arch), +//! whose associated [`Endian`] and `IS_64` constant select the layout at +//! compile time, or by a [`RecordLayout`] value where the target type +//! isn't at hand. + +mod consts; +mod relnames; + +use std::fmt; +use std::marker::PhantomData; + +pub use consts::*; +pub use relnames::*; + +use crate::arch::Arch; + +/// The on-disk layout of an ELF file: word size, byte order and +/// relocation record format. Targets implement this through [`Arch`], and +/// a few plain layouts exist for peeking into files before the target is +/// known. +pub trait Layout: Copy + Send + Sync + 'static { + type Endian: Endian; + const IS_64: bool; + const IS_RELA: bool; + + const WORD_SIZE: usize = if Self::IS_64 { 8 } else { 4 }; +} + +macro_rules! plain_layout { + ($name:ident, $endian:ty, $is_64:expr) => { + #[derive(Clone, Copy, Debug, Default)] + pub struct $name; + + impl Layout for $name { + type Endian = $endian; + const IS_64: bool = $is_64; + const IS_RELA: bool = true; + } + }; +} + +plain_layout!(Elf32Le, LittleEndian, false); +plain_layout!(Elf64Le, LittleEndian, true); +plain_layout!(Elf32Be, BigEndian, false); +plain_layout!(Elf64Be, BigEndian, true); + +/// Byte order of an ELF file, as a type-level marker. +pub trait Endian: Copy + Default + Send + Sync + fmt::Debug + 'static { + const IS_LITTLE: bool; + const IS_NATIVE: bool = Self::IS_LITTLE == cfg!(target_endian = "little"); + + fn read_u16(bytes: &[u8]) -> u16 { + let bytes = bytes[..2].try_into().unwrap(); + if Self::IS_LITTLE { + u16::from_le_bytes(bytes) + } else { + u16::from_be_bytes(bytes) + } + } + + fn read_u32(bytes: &[u8]) -> u32 { + let bytes = bytes[..4].try_into().unwrap(); + if Self::IS_LITTLE { + u32::from_le_bytes(bytes) + } else { + u32::from_be_bytes(bytes) + } + } + + fn read_u64(bytes: &[u8]) -> u64 { + let bytes = bytes[..8].try_into().unwrap(); + if Self::IS_LITTLE { + u64::from_le_bytes(bytes) + } else { + u64::from_be_bytes(bytes) + } + } + + fn read_i32(bytes: &[u8]) -> i32 { + Self::read_u32(bytes) as i32 + } + + fn read_i64(bytes: &[u8]) -> i64 { + Self::read_u64(bytes) as i64 + } + + fn write_u16(bytes: &mut [u8], value: u16) { + let encoded = if Self::IS_LITTLE { + value.to_le_bytes() + } else { + value.to_be_bytes() + }; + bytes[..2].copy_from_slice(&encoded); + } + + fn write_u32(bytes: &mut [u8], value: u32) { + let encoded = if Self::IS_LITTLE { + value.to_le_bytes() + } else { + value.to_be_bytes() + }; + bytes[..4].copy_from_slice(&encoded); + } + + fn write_u64(bytes: &mut [u8], value: u64) { + let encoded = if Self::IS_LITTLE { + value.to_le_bytes() + } else { + value.to_be_bytes() + }; + bytes[..8].copy_from_slice(&encoded); + } + + fn write_i32(bytes: &mut [u8], value: i32) { + Self::write_u32(bytes, value as u32); + } + + fn write_i64(bytes: &mut [u8], value: i64) { + Self::write_u64(bytes, value as u64); + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct LittleEndian; + +#[derive(Clone, Copy, Debug, Default)] +pub struct BigEndian; + +impl Endian for LittleEndian { + const IS_LITTLE: bool = true; +} + +impl Endian for BigEndian { + const IS_LITTLE: bool = false; +} + +/// A cursor for decoding fixed-layout records. +struct Reader<'a, E: Layout> { + bytes: &'a [u8], + _arch: std::marker::PhantomData, +} + +impl<'a, E: Layout> Reader<'a, E> { + fn new(bytes: &'a [u8]) -> Self { + Reader { + bytes, + _arch: std::marker::PhantomData, + } + } + + fn u8(&self, offset: usize) -> u8 { + self.bytes[offset] + } + + #[inline] + fn u16(&self, offset: usize) -> u16 { + E::Endian::read_u16(&self.bytes[offset..]) + } + + #[inline] + fn u32(&self, offset: usize) -> u32 { + E::Endian::read_u32(&self.bytes[offset..]) + } + + #[inline] + fn u64(&self, offset: usize) -> u64 { + E::Endian::read_u64(&self.bytes[offset..]) + } + + /// Reads a word-sized (4 or 8 bytes) unsigned integer. + #[inline] + fn word(&self, offset: usize) -> u64 { + if E::IS_64 { + self.u64(offset) + } else { + self.u32(offset) as u64 + } + } +} + +/// A cursor for encoding fixed-layout records. +struct Writer<'a, E: Layout> { + bytes: &'a mut [u8], + _arch: std::marker::PhantomData, +} + +impl<'a, E: Layout> Writer<'a, E> { + fn new(bytes: &'a mut [u8]) -> Self { + Writer { + bytes, + _arch: std::marker::PhantomData, + } + } + + fn u8(&mut self, offset: usize, value: u8) { + self.bytes[offset] = value; + } + + #[inline] + fn u16(&mut self, offset: usize, value: u16) { + E::Endian::write_u16(&mut self.bytes[offset..], value); + } + + #[inline] + fn u32(&mut self, offset: usize, value: u32) { + E::Endian::write_u32(&mut self.bytes[offset..], value); + } + + #[inline] + fn u64(&mut self, offset: usize, value: u64) { + E::Endian::write_u64(&mut self.bytes[offset..], value); + } + + #[inline] + fn word(&mut self, offset: usize, value: u64) { + if E::IS_64 { + self.u64(offset, value); + } else { + self.u32(offset, value as u32); + } + } +} + +/// A record with a target-dependent on-disk encoding. +pub trait Record: Sized { + /// The encoded size in bytes. + fn size() -> usize; + + /// Decodes a record from the beginning of `bytes`. + fn parse(bytes: &[u8]) -> Self; + + /// Encodes the record to the beginning of `buf`. + fn write(&self, buf: &mut [u8]); + + /// Decodes a contiguous table of records. + fn parse_all(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(Self::size::()) + .map(Self::parse::) + .collect() + } + + /// Encodes a slice of records back to back. + fn write_all(records: &[Self], buf: &mut [u8]) { + for (record, slot) in records.iter().zip(buf.chunks_exact_mut(Self::size::())) { + record.write::(slot); + } + } +} + +/// The ELF file header. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfEhdr { + pub e_ident: [u8; 16], + pub e_type: u16, + pub e_machine: u16, + pub e_version: u32, + pub e_entry: u64, + pub e_phoff: u64, + pub e_shoff: u64, + pub e_flags: u32, + pub e_ehsize: u16, + pub e_phentsize: u16, + pub e_phnum: u16, + pub e_shentsize: u16, + pub e_shnum: u16, + pub e_shstrndx: u16, +} + +impl Record for ElfEhdr { + fn size() -> usize { + if E::IS_64 { + 64 + } else { + 52 + } + } + + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfEhdr has the native ELF64 field layout, `bytes` + // contains a complete record, and an unaligned read accepts the + // file's alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + let w = E::WORD_SIZE; + ElfEhdr { + e_ident: bytes[..16].try_into().unwrap(), + e_type: r.u16(16), + e_machine: r.u16(18), + e_version: r.u32(20), + e_entry: r.word(24), + e_phoff: r.word(24 + w), + e_shoff: r.word(24 + 2 * w), + e_flags: r.u32(24 + 3 * w), + e_ehsize: r.u16(28 + 3 * w), + e_phentsize: r.u16(30 + 3 * w), + e_phnum: r.u16(32 + 3 * w), + e_shentsize: r.u16(34 + 3 * w), + e_shnum: r.u16(36 + 3 * w), + e_shstrndx: r.u16(38 + 3 * w), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + let ws = E::WORD_SIZE; + w.bytes[..16].copy_from_slice(&self.e_ident); + w.u16(16, self.e_type); + w.u16(18, self.e_machine); + w.u32(20, self.e_version); + w.word(24, self.e_entry); + w.word(24 + ws, self.e_phoff); + w.word(24 + 2 * ws, self.e_shoff); + w.u32(24 + 3 * ws, self.e_flags); + w.u16(28 + 3 * ws, self.e_ehsize); + w.u16(30 + 3 * ws, self.e_phentsize); + w.u16(32 + 3 * ws, self.e_phnum); + w.u16(34 + 3 * ws, self.e_shentsize); + w.u16(36 + 3 * ws, self.e_shnum); + w.u16(38 + 3 * ws, self.e_shstrndx); + } +} + +const _: () = assert!(std::mem::size_of::() == 64); + +/// A section header. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfShdr { + pub sh_name: u32, + pub sh_type: u32, + pub sh_flags: u64, + pub sh_addr: u64, + pub sh_offset: u64, + pub sh_size: u64, + pub sh_link: u32, + pub sh_info: u32, + pub sh_addralign: u64, + pub sh_entsize: u64, +} + +impl Record for ElfShdr { + fn size() -> usize { + if E::IS_64 { + 64 + } else { + 40 + } + } + + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfShdr has the native ELF64 field layout, `bytes` + // contains a complete record, and an unaligned read accepts the + // file's alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + let w = E::WORD_SIZE; + ElfShdr { + sh_name: r.u32(0), + sh_type: r.u32(4), + sh_flags: r.word(8), + sh_addr: r.word(8 + w), + sh_offset: r.word(8 + 2 * w), + sh_size: r.word(8 + 3 * w), + sh_link: r.u32(8 + 4 * w), + sh_info: r.u32(12 + 4 * w), + sh_addralign: r.word(16 + 4 * w), + sh_entsize: r.word(16 + 5 * w), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + let ws = E::WORD_SIZE; + w.u32(0, self.sh_name); + w.u32(4, self.sh_type); + w.word(8, self.sh_flags); + w.word(8 + ws, self.sh_addr); + w.word(8 + 2 * ws, self.sh_offset); + w.word(8 + 3 * ws, self.sh_size); + w.u32(8 + 4 * ws, self.sh_link); + w.u32(12 + 4 * ws, self.sh_info); + w.word(16 + 4 * ws, self.sh_addralign); + w.word(16 + 5 * ws, self.sh_entsize); + } +} + +const _: () = assert!(std::mem::size_of::() == 64); + +/// A program header. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfPhdr { + pub p_type: u32, + pub p_flags: u32, + pub p_offset: u64, + pub p_vaddr: u64, + pub p_paddr: u64, + pub p_filesz: u64, + pub p_memsz: u64, + pub p_align: u64, +} + +impl Record for ElfPhdr { + fn size() -> usize { + if E::IS_64 { + 56 + } else { + 32 + } + } + + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfPhdr has the native ELF64 field layout, `bytes` + // contains a complete record, and an unaligned read accepts the + // file's alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + if E::IS_64 { + ElfPhdr { + p_type: r.u32(0), + p_flags: r.u32(4), + p_offset: r.u64(8), + p_vaddr: r.u64(16), + p_paddr: r.u64(24), + p_filesz: r.u64(32), + p_memsz: r.u64(40), + p_align: r.u64(48), + } + } else { + ElfPhdr { + p_type: r.u32(0), + p_offset: r.u32(4) as u64, + p_vaddr: r.u32(8) as u64, + p_paddr: r.u32(12) as u64, + p_filesz: r.u32(16) as u64, + p_memsz: r.u32(20) as u64, + p_flags: r.u32(24), + p_align: r.u32(28) as u64, + } + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + if E::IS_64 { + w.u32(0, self.p_type); + w.u32(4, self.p_flags); + w.u64(8, self.p_offset); + w.u64(16, self.p_vaddr); + w.u64(24, self.p_paddr); + w.u64(32, self.p_filesz); + w.u64(40, self.p_memsz); + w.u64(48, self.p_align); + } else { + w.u32(0, self.p_type); + w.u32(4, self.p_offset as u32); + w.u32(8, self.p_vaddr as u32); + w.u32(12, self.p_paddr as u32); + w.u32(16, self.p_filesz as u32); + w.u32(20, self.p_memsz as u32); + w.u32(24, self.p_flags); + w.u32(28, self.p_align as u32); + } + } +} + +const _: () = assert!(std::mem::size_of::() == 56); + +/// A symbol table entry. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfSym { + pub st_name: u32, + pub st_info: u8, + pub st_other: u8, + pub st_shndx: u16, + pub st_value: u64, + pub st_size: u64, +} + +impl ElfSym { + pub fn st_type(&self) -> u32 { + (self.st_info & 0xf) as u32 + } + + pub fn st_bind(&self) -> u32 { + (self.st_info >> 4) as u32 + } + + pub fn st_visibility(&self) -> u32 { + (self.st_other & 3) as u32 + } + + pub fn set_type(&mut self, ty: u32) { + self.st_info = (self.st_info & 0xf0) | (ty as u8 & 0xf); + } + + pub fn set_bind(&mut self, bind: u32) { + self.st_info = (self.st_info & 0x0f) | ((bind as u8) << 4); + } + + pub fn set_visibility(&mut self, visibility: u32) { + self.st_other = (self.st_other & !3) | (visibility as u8 & 3); + } + + pub fn is_undef(&self) -> bool { + self.st_shndx == SHN_UNDEF as u16 + } + + pub fn is_abs(&self) -> bool { + self.st_shndx == SHN_ABS as u16 + } + + pub fn is_common(&self) -> bool { + self.st_shndx == SHN_COMMON as u16 + } + + pub fn is_weak(&self) -> bool { + self.st_bind() == STB_WEAK + } + + pub fn is_undef_weak(&self) -> bool { + self.is_undef() && self.is_weak() + } + + /// The `st_other` bit that AArch64 uses to mark functions with a + /// non-standard calling convention. + pub fn arm64_variant_pcs(&self) -> bool { + self.st_other & 0x80 != 0 + } + + /// The RISC-V counterpart of [`Self::arm64_variant_pcs`]. + pub fn riscv_variant_cc(&self) -> bool { + self.st_other & 0x80 != 0 + } + + /// The distance between a PPC64 ELFv2 function's global and local entry + /// points, encoded in the top three bits of `st_other`. + pub fn ppc64_local_entry(&self) -> u8 { + self.st_other >> 5 + } + + pub fn ppc64_preserves_r2(&self) -> bool { + self.ppc64_local_entry() != 1 + } + + pub fn ppc64_uses_toc(&self) -> bool { + self.ppc64_local_entry() > 1 + } +} + +impl Record for ElfSym { + fn size() -> usize { + if E::IS_64 { + 24 + } else { + 16 + } + } + + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfSym has the native ELF64 field layout, `bytes` + // contains a complete record, and an unaligned read accepts the + // file's alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + if E::IS_64 { + ElfSym { + st_name: r.u32(0), + st_info: r.u8(4), + st_other: r.u8(5), + st_shndx: r.u16(6), + st_value: r.u64(8), + st_size: r.u64(16), + } + } else { + ElfSym { + st_name: r.u32(0), + st_value: r.u32(4) as u64, + st_size: r.u32(8) as u64, + st_info: r.u8(12), + st_other: r.u8(13), + st_shndx: r.u16(14), + } + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + if E::IS_64 { + w.u32(0, self.st_name); + w.u8(4, self.st_info); + w.u8(5, self.st_other); + w.u16(6, self.st_shndx); + w.u64(8, self.st_value); + w.u64(16, self.st_size); + } else { + w.u32(0, self.st_name); + w.u32(4, self.st_value as u32); + w.u32(8, self.st_size as u32); + w.u8(12, self.st_info); + w.u8(13, self.st_other); + w.u16(14, self.st_shndx); + } + } +} + +const _: () = assert!(std::mem::size_of::() == 24); + +/// A relocation record. +/// +/// Targets use either REL or RELA relocation tables; the former store the +/// addend in the relocated location rather than in the record. To keep the +/// distinction out of most of the linker, `r_addend` is always present and +/// simply zero when read from a REL table. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfRel { + pub r_offset: u64, + // Match the two halves of ELF64 r_info in the host's byte order, so a + // same-endian target can use the native record layout on any host. + #[cfg(target_endian = "big")] + pub r_sym: u32, + pub r_type: u32, + #[cfg(target_endian = "little")] + pub r_sym: u32, + pub r_addend: i64, +} + +impl ElfRel { + pub fn new(r_offset: u64, r_type: u32, r_sym: u32, r_addend: i64) -> Self { + ElfRel { + r_offset, + r_type, + r_sym, + r_addend, + } + } + + /// Whether the relocation is one of the target's function-call relocations. + #[inline(always)] + pub fn is_func_call(&self) -> bool { + E::R_FUNCALL.contains(&self.r_type) + } + + /// Formats the relocation type name for the given target. + pub fn type_name(&self) -> String { + E::rel_to_string(self.r_type) + } +} + +const _: () = assert!(std::mem::size_of::() == 24); + +impl Record for ElfRel { + fn size() -> usize { + match (E::IS_64, E::IS_RELA) { + (true, true) => 24, + (true, false) => 16, + (false, true) => 12, + (false, false) => 8, + } + } + + #[inline] + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::IS_RELA && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfRel has the native ELF64 Rela field layout when the + // target and host have the same byte order, `bytes` contains a + // complete record, and an unaligned read accepts its alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + let (r_offset, r_type, r_sym) = if E::IS_64 { + // SPARC keeps a second addend in the upper bits of the type + // field; its backend separates the two. + let info = r.u64(8); + (r.u64(0), info as u32, (info >> 32) as u32) + } else { + let info = r.u32(4); + (r.u32(0) as u64, info & 0xff, info >> 8) + }; + let r_addend = if E::IS_RELA { + if E::IS_64 { + r.u64(16) as i64 + } else { + r.u32(8) as i32 as i64 + } + } else { + 0 + }; + ElfRel { + r_offset, + r_type, + r_sym, + r_addend, + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + if E::IS_64 { + w.u64(0, self.r_offset); + w.u64(8, ((self.r_sym as u64) << 32) | self.r_type as u64); + if E::IS_RELA { + w.u64(16, self.r_addend as u64); + } + } else { + w.u32(0, self.r_offset as u32); + w.u32(4, (self.r_sym << 8) | (self.r_type & 0xff)); + if E::IS_RELA { + w.u32(8, self.r_addend as u32); + } + } + } +} + +/// Relocation records as they are laid out in a file, read as they are +/// used rather than copied out. Input files hold tens of millions of +/// relocations, which most passes go through once. +#[derive(Clone, Copy)] +enum RelocationData<'a> { + Encoded(&'a [u8]), + Decoded(&'a [ElfRel]), +} + +#[derive(Clone, Copy)] +pub struct Rels<'a, E> { + data: RelocationData<'a>, + layout: PhantomData, +} + +impl<'a, E: Layout> Rels<'a, E> { + pub fn new(data: &'a [u8]) -> Self { + debug_assert!(data.len().is_multiple_of(ElfRel::size::())); + Rels { + data: RelocationData::Encoded(data), + layout: PhantomData, + } + } + + pub fn decoded(data: &'a [ElfRel]) -> Self { + Rels { + data: RelocationData::Decoded(data), + layout: PhantomData, + } + } + + #[inline] + pub fn len(&self) -> usize { + match self.data { + RelocationData::Encoded(data) => data.len() / ElfRel::size::(), + RelocationData::Decoded(data) => data.len(), + } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn at(&self, i: usize) -> ElfRel { + match self.data { + RelocationData::Encoded(data) => { + let size = ElfRel::size::(); + ElfRel::parse::(&data[i * size..(i + 1) * size]) + } + RelocationData::Decoded(data) => data[i], + } + } + + /// The record at index `i`, if there is one. + #[inline] + pub fn get(&self, i: usize) -> Option { + (i < self.len()).then(|| self.at(i)) + } + + pub fn last(&self) -> Option { + self.len().checked_sub(1).map(|i| self.at(i)) + } + + /// The symbol index of record `i` alone, for loops that decode only + /// the records referring to symbols of interest. + #[inline] + pub fn r_sym(&self, i: usize) -> u32 { + match self.data { + RelocationData::Encoded(data) => { + let bytes = &data[i * ElfRel::size::()..]; + if E::IS_64 { + (E::Endian::read_u64(&bytes[8..]) >> 32) as u32 + } else { + E::Endian::read_u32(&bytes[4..]) >> 8 + } + } + RelocationData::Decoded(data) => data[i].r_sym, + } + } + + /// Iterates over symbol indices without decoding the other relocation + /// fields. + #[inline] + pub fn sym_indices(&self) -> impl DoubleEndedIterator + ExactSizeIterator + '_ { + (0..self.len()).map(|i| self.r_sym(i)) + } + + pub fn slice(&self, range: std::ops::Range) -> Rels<'a, E> { + match self.data { + RelocationData::Encoded(data) => { + let size = ElfRel::size::(); + Rels::new(&data[range.start * size..range.end * size]) + } + RelocationData::Decoded(data) => Rels::decoded(&data[range]), + } + } + + pub fn iter(&self) -> RelsIter<'a, E> { + let rels = *self; + RelsIter { + range: 0..rels.len(), + rels, + } + } + + /// The index of the first record for which `pred` is false, the + /// records being partitioned by it. + pub fn partition_point(&self, pred: impl Fn(&ElfRel) -> bool) -> usize { + let (mut lo, mut hi) = (0, self.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if pred(&self.at(mid)) { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + } +} + +pub struct RelsIter<'a, E> { + rels: Rels<'a, E>, + range: std::ops::Range, +} + +impl Iterator for RelsIter<'_, E> { + type Item = ElfRel; + + #[inline] + fn next(&mut self) -> Option { + self.range.next().map(|i| self.rels.at(i)) + } + + fn size_hint(&self) -> (usize, Option) { + self.range.size_hint() + } +} + +impl DoubleEndedIterator for RelsIter<'_, E> { + #[inline] + fn next_back(&mut self) -> Option { + self.range.next_back().map(|i| self.rels.at(i)) + } +} + +impl ExactSizeIterator for RelsIter<'_, E> {} + +impl<'a, E: Layout> IntoIterator for Rels<'a, E> { + type Item = ElfRel; + type IntoIter = RelsIter<'a, E>; + + fn into_iter(self) -> Self::IntoIter { + RelsIter { + range: 0..self.len(), + rels: self, + } + } +} + +/// Mutable access to relocation records stored in the file's layout. +enum MutableRelocationData<'a> { + Encoded(&'a mut [u8]), + Decoded(&'a mut [ElfRel]), +} + +pub struct RelsMut<'a, E> { + data: MutableRelocationData<'a>, + layout: PhantomData, +} + +impl<'a, E: Layout> RelsMut<'a, E> { + pub fn new(data: &'a mut [u8]) -> Self { + debug_assert!(data.len().is_multiple_of(ElfRel::size::())); + RelsMut { + data: MutableRelocationData::Encoded(data), + layout: PhantomData, + } + } + + pub fn decoded(data: &'a mut [ElfRel]) -> Self { + RelsMut { + data: MutableRelocationData::Decoded(data), + layout: PhantomData, + } + } + + #[inline] + pub fn len(&self) -> usize { + match &self.data { + MutableRelocationData::Encoded(data) => data.len() / ElfRel::size::(), + MutableRelocationData::Decoded(data) => data.len(), + } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn at(&self, i: usize) -> ElfRel { + match &self.data { + MutableRelocationData::Encoded(data) => { + let size = ElfRel::size::(); + ElfRel::parse::(&data[i * size..(i + 1) * size]) + } + MutableRelocationData::Decoded(data) => data[i], + } + } + + /// The symbol index of record `i` alone, for loops that decode only + /// the records referring to symbols of interest. + #[inline] + pub fn r_sym(&self, i: usize) -> u32 { + match &self.data { + MutableRelocationData::Encoded(data) => { + let bytes = &data[i * ElfRel::size::()..]; + if E::IS_64 { + (E::Endian::read_u64(&bytes[8..]) >> 32) as u32 + } else { + E::Endian::read_u32(&bytes[4..]) >> 8 + } + } + MutableRelocationData::Decoded(data) => data[i].r_sym, + } + } + + /// Redirects record `i` without decoding and rewriting its other fields. + #[inline] + pub fn set_r_sym(&mut self, i: usize, r_sym: u32) { + match &mut self.data { + MutableRelocationData::Encoded(data) => { + let bytes = &mut data[i * ElfRel::size::()..]; + if E::IS_64 { + let r_info = E::Endian::read_u64(&bytes[8..]); + E::Endian::write_u64( + &mut bytes[8..], + (u64::from(r_sym) << 32) | (r_info as u32 as u64), + ); + } else { + let r_info = E::Endian::read_u32(&bytes[4..]); + E::Endian::write_u32(&mut bytes[4..], (r_sym << 8) | (r_info & 0xff)); + } + } + MutableRelocationData::Decoded(data) => data[i].r_sym = r_sym, + } + } + + pub fn set(&mut self, i: usize, rel: ElfRel) { + match &mut self.data { + MutableRelocationData::Encoded(data) => { + let size = ElfRel::size::(); + rel.write::(&mut data[i * size..(i + 1) * size]); + } + MutableRelocationData::Decoded(data) => data[i] = rel, + } + } + + /// Replaces the records by `rels`, which has the same length. + pub fn set_all(&mut self, rels: &[ElfRel]) { + debug_assert_eq!(rels.len(), self.len()); + match &mut self.data { + MutableRelocationData::Encoded(data) => ElfRel::write_all::(rels, data), + MutableRelocationData::Decoded(data) => data.copy_from_slice(rels), + } + } +} + +/// The layout of a file's records — word size and byte order — as a +/// value, for the tables that are read in place by code that isn't +/// specialized for a target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RecordLayout { + pub is_64: bool, + pub is_little_endian: bool, +} + +impl RecordLayout { + #[inline] + pub fn of() -> RecordLayout { + RecordLayout { + is_64: E::IS_64, + is_little_endian: E::Endian::IS_LITTLE, + } + } + + #[inline] + fn u16(self, bytes: &[u8], offset: usize) -> u16 { + let bytes = bytes[offset..offset + 2].try_into().unwrap(); + if self.is_little_endian { + u16::from_le_bytes(bytes) + } else { + u16::from_be_bytes(bytes) + } + } + + #[inline] + fn u32(self, bytes: &[u8], offset: usize) -> u32 { + let bytes = bytes[offset..offset + 4].try_into().unwrap(); + if self.is_little_endian { + u32::from_le_bytes(bytes) + } else { + u32::from_be_bytes(bytes) + } + } + + #[inline] + fn u64(self, bytes: &[u8], offset: usize) -> u64 { + let bytes = bytes[offset..offset + 8].try_into().unwrap(); + if self.is_little_endian { + u64::from_le_bytes(bytes) + } else { + u64::from_be_bytes(bytes) + } + } + + #[inline] + fn word(self, bytes: &[u8], offset: usize) -> u64 { + if self.is_64 { + self.u64(bytes, offset) + } else { + self.u32(bytes, offset) as u64 + } + } + + #[inline] + fn put_u16(self, buf: &mut [u8], offset: usize, value: u16) { + buf[offset..offset + 2].copy_from_slice(&if self.is_little_endian { + value.to_le_bytes() + } else { + value.to_be_bytes() + }); + } + + #[inline] + fn put_u32(self, buf: &mut [u8], offset: usize, value: u32) { + buf[offset..offset + 4].copy_from_slice(&if self.is_little_endian { + value.to_le_bytes() + } else { + value.to_be_bytes() + }); + } + + #[inline] + fn put_u64(self, buf: &mut [u8], offset: usize, value: u64) { + buf[offset..offset + 8].copy_from_slice(&if self.is_little_endian { + value.to_le_bytes() + } else { + value.to_be_bytes() + }); + } + + #[inline] + fn put_word(self, buf: &mut [u8], offset: usize, value: u64) { + if self.is_64 { + self.put_u64(buf, offset, value); + } else { + self.put_u32(buf, offset, value as u32); + } + } + + #[inline] + pub fn sym_size(self) -> usize { + if self.is_64 { + 24 + } else { + 16 + } + } + + #[inline] + pub fn read_sym(self, bytes: &[u8]) -> ElfSym { + if self.is_64 { + ElfSym { + st_name: self.u32(bytes, 0), + st_info: bytes[4], + st_other: bytes[5], + st_shndx: self.u16(bytes, 6), + st_value: self.u64(bytes, 8), + st_size: self.u64(bytes, 16), + } + } else { + ElfSym { + st_name: self.u32(bytes, 0), + st_value: self.u32(bytes, 4) as u64, + st_size: self.u32(bytes, 8) as u64, + st_info: bytes[12], + st_other: bytes[13], + st_shndx: self.u16(bytes, 14), + } + } + } + + #[inline] + pub fn write_sym(self, sym: &ElfSym, buf: &mut [u8]) { + if self.is_64 { + self.put_u32(buf, 0, sym.st_name); + buf[4] = sym.st_info; + buf[5] = sym.st_other; + self.put_u16(buf, 6, sym.st_shndx); + self.put_u64(buf, 8, sym.st_value); + self.put_u64(buf, 16, sym.st_size); + } else { + self.put_u32(buf, 0, sym.st_name); + self.put_u32(buf, 4, sym.st_value as u32); + self.put_u32(buf, 8, sym.st_size as u32); + buf[12] = sym.st_info; + buf[13] = sym.st_other; + self.put_u16(buf, 14, sym.st_shndx); + } + } + + #[inline] + pub fn shdr_size(self) -> usize { + if self.is_64 { + 64 + } else { + 40 + } + } + + #[inline] + pub fn read_shdr(self, bytes: &[u8]) -> ElfShdr { + let w = if self.is_64 { 8 } else { 4 }; + ElfShdr { + sh_name: self.u32(bytes, 0), + sh_type: self.u32(bytes, 4), + sh_flags: self.word(bytes, 8), + sh_addr: self.word(bytes, 8 + w), + sh_offset: self.word(bytes, 8 + 2 * w), + sh_size: self.word(bytes, 8 + 3 * w), + sh_link: self.u32(bytes, 8 + 4 * w), + sh_info: self.u32(bytes, 12 + 4 * w), + sh_addralign: self.word(bytes, 16 + 4 * w), + sh_entsize: self.word(bytes, 16 + 5 * w), + } + } + + #[inline] + pub fn write_shdr(self, shdr: &ElfShdr, buf: &mut [u8]) { + let w = if self.is_64 { 8 } else { 4 }; + self.put_u32(buf, 0, shdr.sh_name); + self.put_u32(buf, 4, shdr.sh_type); + self.put_word(buf, 8, shdr.sh_flags); + self.put_word(buf, 8 + w, shdr.sh_addr); + self.put_word(buf, 8 + 2 * w, shdr.sh_offset); + self.put_word(buf, 8 + 3 * w, shdr.sh_size); + self.put_u32(buf, 8 + 4 * w, shdr.sh_link); + self.put_u32(buf, 12 + 4 * w, shdr.sh_info); + self.put_word(buf, 16 + 4 * w, shdr.sh_addralign); + self.put_word(buf, 16 + 5 * w, shdr.sh_entsize); + } +} + +/// A file's symbol table, read in place from the file; the linker's own +/// files build theirs. Symbols are decoded as they are used. +#[derive(Clone, Debug, Default)] +pub struct SymTable { + data: std::borrow::Cow<'static, [u8]>, + layout: RecordLayout, + len: usize, +} + +impl SymTable { + pub fn in_file(data: &'static [u8], layout: RecordLayout) -> SymTable { + debug_assert!(data.len().is_multiple_of(layout.sym_size())); + SymTable { + data: std::borrow::Cow::Borrowed(data), + layout, + len: data.len() / layout.sym_size(), + } + } + + /// An empty table the linker fills, in the target's layout. + pub fn new(layout: RecordLayout) -> SymTable { + SymTable { + data: std::borrow::Cow::Owned(Vec::new()), + layout, + len: 0, + } + } + + pub fn from_records(layout: RecordLayout, syms: &[ElfSym]) -> SymTable { + let mut table = SymTable::new(layout); + for sym in syms { + table.push(*sym); + } + table + } + + #[inline] + pub fn len(&self) -> usize { + self.len + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + #[inline] + pub fn at(&self, i: usize) -> ElfSym { + let size = self.layout.sym_size(); + self.layout.read_sym(&self.data[i * size..(i + 1) * size]) + } + + #[inline] + pub fn get(&self, i: usize) -> Option { + (i < self.len()).then(|| self.at(i)) + } + + /// Like [`Self::at`] for code specialized for the target, which + /// knows the layout at compile time; relocation loops read a symbol + /// per relocation. + #[inline] + pub fn at_in(&self, i: usize) -> ElfSym { + debug_assert_eq!(self.layout, RecordLayout::of::()); + let size = ElfSym::size::(); + ElfSym::parse::(&self.data[i * size..(i + 1) * size]) + } + + #[inline] + pub fn get_in(&self, i: usize) -> Option { + (i < self.len()).then(|| self.at_in::(i)) + } + + /// Replaces a symbol in a table the linker builds. + pub fn set_in(&mut self, i: usize, sym: ElfSym) { + debug_assert_eq!(self.layout, RecordLayout::of::()); + let size = ElfSym::size::(); + sym.write::(&mut self.data.to_mut()[i * size..(i + 1) * size]); + } + + /// The type and binding byte of symbol `i` alone, for relocation loops + /// that look for section symbols. + #[inline] + pub fn st_info(&self, i: usize) -> u8 { + let offset = if self.layout.is_64 { 4 } else { 12 }; + self.data[i * self.layout.sym_size() + offset] + } + + /// Like [`Self::st_info`] for code specialized for the target. + #[inline] + pub fn st_info_in(&self, i: usize) -> u8 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + let offset = if E::IS_64 { 4 } else { 12 }; + self.data[i * ElfSym::size::() + offset] + } + + /// Reads the type and binding byte after checking the symbol index once. + #[inline(always)] + pub fn st_info_in_checked(&self, i: usize) -> Option { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i >= self.len { + return None; + } + let offset = i * ElfSym::size::() + if E::IS_64 { 4 } else { 12 }; + // SAFETY: `i < self.len` and every symbol record has this field. + Some(unsafe { *self.data.as_ptr().add(offset) }) + } + + /// The section index of symbol `i` alone. + #[inline] + pub fn st_shndx(&self, i: usize) -> u16 { + let offset = if self.layout.is_64 { 6 } else { 14 }; + self.layout + .u16(&self.data, i * self.layout.sym_size() + offset) + } + + /// Like [`Self::st_shndx`] for code specialized for the target. + #[inline] + pub fn st_shndx_in(&self, i: usize) -> u16 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + let offset = if E::IS_64 { 6 } else { 14 }; + E::Endian::read_u16(&self.data[i * ElfSym::size::() + offset..]) + } + + /// Reads a section index after the caller has checked `i < self.len()`. + #[inline(always)] + pub(crate) unsafe fn st_shndx_in_unchecked(&self, i: usize) -> u16 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.len); + let offset = i * ElfSym::size::() + if E::IS_64 { 6 } else { 14 }; + // SAFETY: guaranteed by the caller and the fixed symbol layout. + let raw = unsafe { std::ptr::read_unaligned(self.data.as_ptr().add(offset).cast::()) }; + if E::Endian::IS_LITTLE { + u16::from_le(raw) + } else { + u16::from_be(raw) + } + } + + /// The name offset of symbol `i` alone. + #[inline] + pub fn st_name(&self, i: usize) -> u32 { + self.layout.u32(&self.data, i * self.layout.sym_size()) + } + + /// Like [`Self::st_name`] for code specialized for the target. + #[inline] + pub fn st_name_in(&self, i: usize) -> u32 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + E::Endian::read_u32(&self.data[i * ElfSym::size::()..]) + } + + /// Walks the name offsets without decoding the rest of each symbol. + #[inline] + pub fn name_offsets_in( + &self, + ) -> impl DoubleEndedIterator + ExactSizeIterator + '_ { + debug_assert_eq!(self.layout, RecordLayout::of::()); + self.data + .chunks_exact(ElfSym::size::()) + .map(E::Endian::read_u32) + } + + pub fn iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + '_ { + let layout = self.layout; + self.data + .chunks_exact(layout.sym_size()) + .map(move |bytes| layout.read_sym(bytes)) + } + + /// Appends a symbol to a table the linker builds. + pub fn push(&mut self, sym: ElfSym) { + let size = self.layout.sym_size(); + let data = self.data.to_mut(); + let start = data.len(); + data.resize(start + size, 0); + self.layout.write_sym(&sym, &mut data[start..]); + self.len += 1; + } +} + +/// A file's section headers, read in place, followed by those the linker +/// adds for the sections it makes up (for common symbols). +#[derive(Clone, Debug, Default)] +pub struct ShdrTable<'a> { + data: &'a [u8], + layout: RecordLayout, + /// The number of headers in `data`. + num_in_file: usize, + extra: Vec, +} + +impl<'a> ShdrTable<'a> { + pub fn in_file(data: &'a [u8], layout: RecordLayout) -> ShdrTable<'a> { + debug_assert!(data.len().is_multiple_of(layout.shdr_size())); + ShdrTable { + data, + layout, + num_in_file: data.len() / layout.shdr_size(), + extra: Vec::new(), + } + } + + /// The number of headers in the file. + #[inline] + pub fn num_in_file(&self) -> usize { + self.num_in_file + } + + /// The name offset of header `i` alone. + #[inline] + pub fn sh_name(&self, i: usize) -> u32 { + if i < self.num_in_file { + self.layout.u32(self.data, i * self.layout.shdr_size()) + } else { + self.extra[i - self.num_in_file].sh_name + } + } + + /// Like [`Self::sh_name`] for code specialized for the target. + #[inline] + pub fn sh_name_in(&self, i: usize) -> u32 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i < self.num_in_file { + E::Endian::read_u32(&self.data[i * ElfShdr::size::()..]) + } else { + self.extra[i - self.num_in_file].sh_name + } + } + + #[inline] + pub fn len(&self) -> usize { + self.num_in_file() + self.extra.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn at(&self, i: usize) -> ElfShdr { + let n = self.num_in_file(); + if i < n { + let size = self.layout.shdr_size(); + self.layout.read_shdr(&self.data[i * size..(i + 1) * size]) + } else { + self.extra[i - n] + } + } + + #[inline] + pub fn get(&self, i: usize) -> Option { + (i < self.len()).then(|| self.at(i)) + } + + /// Like [`Self::at`] for code specialized for the target, which knows + /// the layout at compile time. + #[inline] + pub fn at_in(&self, i: usize) -> ElfShdr { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i < self.num_in_file { + let size = ElfShdr::size::(); + ElfShdr::parse::(&self.data[i * size..(i + 1) * size]) + } else { + self.extra[i - self.num_in_file] + } + } + + /// Like [`Self::at_in`], for a caller already iterating within the + /// table's length. C++ mold walks the same validated section-header span + /// directly, without checking the index again for every field. + /// + /// # Safety + /// + /// `i` must be less than [`Self::len`]. + #[inline(always)] + pub unsafe fn at_in_unchecked(&self, i: usize) -> ElfShdr { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.len()); + if i < self.num_in_file { + let size = ElfShdr::size::(); + // SAFETY: the caller proved `i`, and `in_file` accepted only a + // whole number of section-header records. + let bytes = + unsafe { std::slice::from_raw_parts(self.data.as_ptr().add(i * size), size) }; + ElfShdr::parse::(bytes) + } else { + // SAFETY: the total-length precondition proves this extra index. + unsafe { *self.extra.get_unchecked(i - self.num_in_file) } + } + } + + /// Reads a header from the validated file-backed part of the table. + /// + /// # Safety + /// + /// `i` must be less than [`Self::num_in_file`]. + #[inline(always)] + pub unsafe fn file_at_in_unchecked(&self, i: usize) -> ElfShdr { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.num_in_file); + let size = ElfShdr::size::(); + // SAFETY: the caller proved `i`, and `in_file` accepted only a whole + // number of section-header records. + let bytes = unsafe { std::slice::from_raw_parts(self.data.as_ptr().add(i * size), size) }; + ElfShdr::parse::(bytes) + } + + /// Reads the two fields needed to classify a section without decoding + /// the rest of its header. + /// + /// # Safety + /// + /// `i` must be less than [`Self::len`]. + #[inline(always)] + pub unsafe fn sh_type_and_flags_in_unchecked(&self, i: usize) -> (u32, u64) { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.len()); + if i < self.num_in_file { + let ptr = unsafe { self.data.as_ptr().add(i * ElfShdr::size::()) }; + // SAFETY: a complete header has a u32 at offset 4 and a target + // word at offset 8. Unaligned reads match ELF's file layout. + let sh_type = unsafe { std::ptr::read_unaligned(ptr.add(4).cast::()) }; + let sh_type = if E::Endian::IS_LITTLE { + u32::from_le(sh_type) + } else { + u32::from_be(sh_type) + }; + let sh_flags = if E::IS_64 { + let value = unsafe { std::ptr::read_unaligned(ptr.add(8).cast::()) }; + if E::Endian::IS_LITTLE { + u64::from_le(value) + } else { + u64::from_be(value) + } + } else { + let value = unsafe { std::ptr::read_unaligned(ptr.add(8).cast::()) }; + if E::Endian::IS_LITTLE { + u32::from_le(value) as u64 + } else { + u32::from_be(value) as u64 + } + }; + (sh_type, sh_flags) + } else { + // SAFETY: the total-length precondition proves this extra index. + let shdr = unsafe { self.extra.get_unchecked(i - self.num_in_file) }; + (shdr.sh_type, shdr.sh_flags) + } + } + + /// Reads the type and flags from the validated file-backed span. + /// + /// # Safety + /// + /// `i` must be less than [`Self::num_in_file`]. + #[inline(always)] + pub unsafe fn file_type_and_flags_in_unchecked(&self, i: usize) -> (u32, u64) { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.num_in_file); + let ptr = unsafe { self.data.as_ptr().add(i * ElfShdr::size::()) }; + // SAFETY: a complete header has a u32 at offset 4 and a target word at + // offset 8. Unaligned reads match ELF's file layout. + let sh_type = unsafe { std::ptr::read_unaligned(ptr.add(4).cast::()) }; + let sh_type = if E::Endian::IS_LITTLE { + u32::from_le(sh_type) + } else { + u32::from_be(sh_type) + }; + let sh_flags = if E::IS_64 { + let value = unsafe { std::ptr::read_unaligned(ptr.add(8).cast::()) }; + if E::Endian::IS_LITTLE { + u64::from_le(value) + } else { + u64::from_be(value) + } + } else { + let value = unsafe { std::ptr::read_unaligned(ptr.add(8).cast::()) }; + if E::Endian::IS_LITTLE { + u32::from_le(value) as u64 + } else { + u32::from_be(value) as u64 + } + }; + (sh_type, sh_flags) + } + + /// Reads the type from the validated file-backed span. + /// + /// # Safety + /// + /// `i` must be less than [`Self::num_in_file`]. + #[inline(always)] + pub unsafe fn file_type_in_unchecked(&self, i: usize) -> u32 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + debug_assert!(i < self.num_in_file); + let ptr = unsafe { self.data.as_ptr().add(i * ElfShdr::size::() + 4) }; + // SAFETY: a complete header has a u32 at offset 4. An unaligned read + // matches ELF's file layout. + let value = unsafe { std::ptr::read_unaligned(ptr.cast::()) }; + if E::Endian::IS_LITTLE { + u32::from_le(value) + } else { + u32::from_be(value) + } + } + + /// The type of header `i` alone, for loops that decode only the + /// headers they are after. + #[inline] + pub fn sh_type(&self, i: usize) -> u32 { + let n = self.num_in_file(); + if i < n { + self.layout.u32(self.data, i * self.layout.shdr_size() + 4) + } else { + self.extra[i - n].sh_type + } + } + + /// Like [`Self::sh_type`] for code specialized for the target. + #[inline] + pub fn sh_type_in(&self, i: usize) -> u32 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i < self.num_in_file { + E::Endian::read_u32(&self.data[i * ElfShdr::size::() + 4..]) + } else { + self.extra[i - self.num_in_file].sh_type + } + } + + /// The flags of header `i` alone. + #[inline] + pub fn sh_flags(&self, i: usize) -> u64 { + let n = self.num_in_file(); + if i < n { + self.layout.word(self.data, i * self.layout.shdr_size() + 8) + } else { + self.extra[i - n].sh_flags + } + } + + /// Like [`Self::sh_flags`] for code specialized for the target. + #[inline] + pub fn sh_flags_in(&self, i: usize) -> u64 { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i < self.num_in_file { + let bytes = &self.data[i * ElfShdr::size::() + 8..]; + if E::IS_64 { + E::Endian::read_u64(bytes) + } else { + E::Endian::read_u32(bytes) as u64 + } + } else { + self.extra[i - self.num_in_file].sh_flags + } + } + + /// Reads section flags after checking the section index once. + #[inline(always)] + pub fn sh_flags_in_checked(&self, i: usize) -> Option { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i >= self.len() { + return None; + } + if i < self.num_in_file { + let offset = i * ElfShdr::size::() + 8; + if E::IS_64 { + // SAFETY: `i` names a complete 64-bit section header. + let raw = unsafe { + std::ptr::read_unaligned(self.data.as_ptr().add(offset).cast::()) + }; + Some(if E::Endian::IS_LITTLE { + u64::from_le(raw) + } else { + u64::from_be(raw) + }) + } else { + // SAFETY: `i` names a complete 32-bit section header. + let raw = unsafe { + std::ptr::read_unaligned(self.data.as_ptr().add(offset).cast::()) + }; + Some(if E::Endian::IS_LITTLE { + u32::from_le(raw) as u64 + } else { + u32::from_be(raw) as u64 + }) + } + } else { + // SAFETY: the total-length check above proves this extra index. + Some(unsafe { self.extra.get_unchecked(i - self.num_in_file).sh_flags }) + } + } + + /// The file offset and size of section `i` alone. + #[inline] + pub fn sh_offset_and_size(&self, i: usize) -> (u64, u64) { + let n = self.num_in_file(); + if i < n { + let w = if self.layout.is_64 { 8 } else { 4 }; + let base = i * self.layout.shdr_size(); + ( + self.layout.word(self.data, base + 8 + 2 * w), + self.layout.word(self.data, base + 8 + 3 * w), + ) + } else { + let shdr = &self.extra[i - n]; + (shdr.sh_offset, shdr.sh_size) + } + } + + /// Like [`Self::sh_offset_and_size`] for target-specialized code. + #[inline] + pub fn sh_offset_and_size_in(&self, i: usize) -> (u64, u64) { + debug_assert_eq!(self.layout, RecordLayout::of::()); + if i < self.num_in_file { + let base = i * ElfShdr::size::(); + if E::IS_64 { + ( + E::Endian::read_u64(&self.data[base + 24..]), + E::Endian::read_u64(&self.data[base + 32..]), + ) + } else { + ( + E::Endian::read_u32(&self.data[base + 16..]) as u64, + E::Endian::read_u32(&self.data[base + 20..]) as u64, + ) + } + } else { + let shdr = &self.extra[i - self.num_in_file]; + (shdr.sh_offset, shdr.sh_size) + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.len()).map(|i| self.at(i)) + } + + pub fn push(&mut self, shdr: ElfShdr) { + self.extra.push(shdr); + } +} + +/// An entry of the `.dynamic` section. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfDyn { + pub d_tag: u64, + pub d_val: u64, +} + +impl Record for ElfDyn { + fn size() -> usize { + 2 * E::WORD_SIZE + } + + fn parse(bytes: &[u8]) -> Self { + if E::IS_64 && E::Endian::IS_NATIVE { + let bytes = &bytes[..Self::size::()]; + // SAFETY: ElfDyn has the native ELF64 field layout, `bytes` + // contains a complete record, and an unaligned read accepts the + // file's alignment. + return unsafe { bytes.as_ptr().cast::().read_unaligned() }; + } + + let r = Reader::::new(bytes); + ElfDyn { + d_tag: r.word(0), + d_val: r.word(E::WORD_SIZE), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.word(0, self.d_tag); + w.word(E::WORD_SIZE, self.d_val); + } +} + +const _: () = assert!(std::mem::size_of::() == 16); + +/// The header of a compressed section. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfChdr { + pub ch_type: u32, + pub ch_size: u64, + pub ch_addralign: u64, +} + +impl Record for ElfChdr { + fn size() -> usize { + if E::IS_64 { + 24 + } else { + 12 + } + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + if E::IS_64 { + ElfChdr { + ch_type: r.u32(0), + ch_size: r.u64(8), + ch_addralign: r.u64(16), + } + } else { + ElfChdr { + ch_type: r.u32(0), + ch_size: r.u32(4) as u64, + ch_addralign: r.u32(8) as u64, + } + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + if E::IS_64 { + w.u32(0, self.ch_type); + w.u32(4, 0); + w.u64(8, self.ch_size); + w.u64(16, self.ch_addralign); + } else { + w.u32(0, self.ch_type); + w.u32(4, self.ch_size as u32); + w.u32(8, self.ch_addralign as u32); + } + } +} + +/// A note header. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfNhdr { + pub n_namesz: u32, + pub n_descsz: u32, + pub n_type: u32, +} + +impl Record for ElfNhdr { + fn size() -> usize { + 12 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + ElfNhdr { + n_namesz: r.u32(0), + n_descsz: r.u32(4), + n_type: r.u32(8), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u32(0, self.n_namesz); + w.u32(4, self.n_descsz); + w.u32(8, self.n_type); + } +} + +/// A `.gnu.version_r` file entry. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfVerneed { + pub vn_version: u16, + pub vn_cnt: u16, + pub vn_file: u32, + pub vn_aux: u32, + pub vn_next: u32, +} + +impl Record for ElfVerneed { + fn size() -> usize { + 16 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + ElfVerneed { + vn_version: r.u16(0), + vn_cnt: r.u16(2), + vn_file: r.u32(4), + vn_aux: r.u32(8), + vn_next: r.u32(12), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u16(0, self.vn_version); + w.u16(2, self.vn_cnt); + w.u32(4, self.vn_file); + w.u32(8, self.vn_aux); + w.u32(12, self.vn_next); + } +} + +/// A `.gnu.version_r` version entry. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfVernaux { + pub vna_hash: u32, + pub vna_flags: u16, + pub vna_other: u16, + pub vna_name: u32, + pub vna_next: u32, +} + +impl Record for ElfVernaux { + fn size() -> usize { + 16 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + ElfVernaux { + vna_hash: r.u32(0), + vna_flags: r.u16(4), + vna_other: r.u16(6), + vna_name: r.u32(8), + vna_next: r.u32(12), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u32(0, self.vna_hash); + w.u16(4, self.vna_flags); + w.u16(6, self.vna_other); + w.u32(8, self.vna_name); + w.u32(12, self.vna_next); + } +} + +/// A `.gnu.version_d` definition entry. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfVerdef { + pub vd_version: u16, + pub vd_flags: u16, + pub vd_ndx: u16, + pub vd_cnt: u16, + pub vd_hash: u32, + pub vd_aux: u32, + pub vd_next: u32, +} + +impl Record for ElfVerdef { + fn size() -> usize { + 20 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + ElfVerdef { + vd_version: r.u16(0), + vd_flags: r.u16(2), + vd_ndx: r.u16(4), + vd_cnt: r.u16(6), + vd_hash: r.u32(8), + vd_aux: r.u32(12), + vd_next: r.u32(16), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u16(0, self.vd_version); + w.u16(2, self.vd_flags); + w.u16(4, self.vd_ndx); + w.u16(6, self.vd_cnt); + w.u32(8, self.vd_hash); + w.u32(12, self.vd_aux); + w.u32(16, self.vd_next); + } +} + +/// A `.gnu.version_d` name entry. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ElfVerdaux { + pub vda_name: u32, + pub vda_next: u32, +} + +impl Record for ElfVerdaux { + fn size() -> usize { + 8 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + ElfVerdaux { + vda_name: r.u32(0), + vda_next: r.u32(4), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u32(0, self.vda_name); + w.u32(4, self.vda_next); + } +} + +/// The header of an SFrame section (version 3). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SFrameHeader { + pub magic: u16, + pub version: u8, + pub flags: u8, + pub abi_arch: u8, + pub cfa_fixed_fp_offset: i8, + pub cfa_fixed_ra_offset: i8, + pub auxhdr_len: u8, + pub num_fdes: u32, + pub num_fres: u32, + pub fre_len: u32, + pub fdeoff: u32, + pub freoff: u32, +} + +pub const SFRAME_MAGIC: u16 = 0xdee2; +pub const SFRAME_F_FDE_SORTED: u8 = 0x1; +pub const SFRAME_F_FRAME_POINTER: u8 = 0x2; +pub const SFRAME_F_FDE_FUNC_START_PCREL: u8 = 0x4; + +pub const SFRAME_ABI_AARCH64_ENDIAN_BIG: u8 = 1; +pub const SFRAME_ABI_AARCH64_ENDIAN_LITTLE: u8 = 2; +pub const SFRAME_ABI_AMD64_ENDIAN_LITTLE: u8 = 3; +pub const SFRAME_ABI_S390X_ENDIAN_BIG: u8 = 4; + +impl Record for SFrameHeader { + fn size() -> usize { + 28 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + SFrameHeader { + magic: r.u16(0), + version: r.u8(2), + flags: r.u8(3), + abi_arch: r.u8(4), + cfa_fixed_fp_offset: r.u8(5) as i8, + cfa_fixed_ra_offset: r.u8(6) as i8, + auxhdr_len: r.u8(7), + num_fdes: r.u32(8), + num_fres: r.u32(12), + fre_len: r.u32(16), + fdeoff: r.u32(20), + freoff: r.u32(24), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u16(0, self.magic); + w.u8(2, self.version); + w.u8(3, self.flags); + w.u8(4, self.abi_arch); + w.u8(5, self.cfa_fixed_fp_offset as u8); + w.u8(6, self.cfa_fixed_ra_offset as u8); + w.u8(7, self.auxhdr_len); + w.u32(8, self.num_fdes); + w.u32(12, self.num_fres); + w.u32(16, self.fre_len); + w.u32(20, self.fdeoff); + w.u32(24, self.freoff); + } +} + +/// The index part of an SFrame function descriptor entry. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SFrameFdeIdx { + pub func_start_offset: i64, + pub func_size: u32, + pub func_start_fre_off: u32, +} + +impl Record for SFrameFdeIdx { + fn size() -> usize { + 16 + } + + fn parse(bytes: &[u8]) -> Self { + let r = Reader::::new(bytes); + SFrameFdeIdx { + func_start_offset: r.u64(0) as i64, + func_size: r.u32(8), + func_start_fre_off: r.u32(12), + } + } + + fn write(&self, buf: &mut [u8]) { + let mut w = Writer::::new(buf); + w.u64(0, self.func_start_offset as u64); + w.u32(8, self.func_size); + w.u32(12, self.func_start_fre_off); + } +} + +/// Returns a symbol type's name for diagnostics. +pub fn stt_to_string(st_type: u32) -> String { + match st_type { + STT_NOTYPE => "STT_NOTYPE".into(), + STT_OBJECT => "STT_OBJECT".into(), + STT_FUNC => "STT_FUNC".into(), + STT_SECTION => "STT_SECTION".into(), + STT_FILE => "STT_FILE".into(), + STT_COMMON => "STT_COMMON".into(), + STT_TLS => "STT_TLS".into(), + STT_GNU_IFUNC => "STT_GNU_IFUNC".into(), + STT_SPARC_REGISTER => "STT_SPARC_REGISTER".into(), + _ => format!("unknown st_type ({st_type})"), + } +} diff --git a/src/elf/relnames.rs b/src/elf/relnames.rs new file mode 100644 index 000000000..7413f12d3 --- /dev/null +++ b/src/elf/relnames.rs @@ -0,0 +1,999 @@ +// Relocation type names for diagnostics. Generated from consts.rs. + +use super::consts::*; + +pub fn x86_64_rel_to_string(r_type: u32) -> String { + match r_type { + R_X86_64_NONE => "R_X86_64_NONE".to_string(), + R_X86_64_64 => "R_X86_64_64".to_string(), + R_X86_64_PC32 => "R_X86_64_PC32".to_string(), + R_X86_64_GOT32 => "R_X86_64_GOT32".to_string(), + R_X86_64_PLT32 => "R_X86_64_PLT32".to_string(), + R_X86_64_COPY => "R_X86_64_COPY".to_string(), + R_X86_64_GLOB_DAT => "R_X86_64_GLOB_DAT".to_string(), + R_X86_64_JUMP_SLOT => "R_X86_64_JUMP_SLOT".to_string(), + R_X86_64_RELATIVE => "R_X86_64_RELATIVE".to_string(), + R_X86_64_GOTPCREL => "R_X86_64_GOTPCREL".to_string(), + R_X86_64_32 => "R_X86_64_32".to_string(), + R_X86_64_32S => "R_X86_64_32S".to_string(), + R_X86_64_16 => "R_X86_64_16".to_string(), + R_X86_64_PC16 => "R_X86_64_PC16".to_string(), + R_X86_64_8 => "R_X86_64_8".to_string(), + R_X86_64_PC8 => "R_X86_64_PC8".to_string(), + R_X86_64_DTPMOD64 => "R_X86_64_DTPMOD64".to_string(), + R_X86_64_DTPOFF64 => "R_X86_64_DTPOFF64".to_string(), + R_X86_64_TPOFF64 => "R_X86_64_TPOFF64".to_string(), + R_X86_64_TLSGD => "R_X86_64_TLSGD".to_string(), + R_X86_64_TLSLD => "R_X86_64_TLSLD".to_string(), + R_X86_64_DTPOFF32 => "R_X86_64_DTPOFF32".to_string(), + R_X86_64_GOTTPOFF => "R_X86_64_GOTTPOFF".to_string(), + R_X86_64_TPOFF32 => "R_X86_64_TPOFF32".to_string(), + R_X86_64_PC64 => "R_X86_64_PC64".to_string(), + R_X86_64_GOTOFF64 => "R_X86_64_GOTOFF64".to_string(), + R_X86_64_GOTPC32 => "R_X86_64_GOTPC32".to_string(), + R_X86_64_GOT64 => "R_X86_64_GOT64".to_string(), + R_X86_64_GOTPCREL64 => "R_X86_64_GOTPCREL64".to_string(), + R_X86_64_GOTPC64 => "R_X86_64_GOTPC64".to_string(), + R_X86_64_GOTPLT64 => "R_X86_64_GOTPLT64".to_string(), + R_X86_64_PLTOFF64 => "R_X86_64_PLTOFF64".to_string(), + R_X86_64_SIZE32 => "R_X86_64_SIZE32".to_string(), + R_X86_64_SIZE64 => "R_X86_64_SIZE64".to_string(), + R_X86_64_GOTPC32_TLSDESC => "R_X86_64_GOTPC32_TLSDESC".to_string(), + R_X86_64_TLSDESC_CALL => "R_X86_64_TLSDESC_CALL".to_string(), + R_X86_64_TLSDESC => "R_X86_64_TLSDESC".to_string(), + R_X86_64_IRELATIVE => "R_X86_64_IRELATIVE".to_string(), + R_X86_64_GOTPCRELX => "R_X86_64_GOTPCRELX".to_string(), + R_X86_64_REX_GOTPCRELX => "R_X86_64_REX_GOTPCRELX".to_string(), + R_X86_64_CODE_4_GOTPCRELX => "R_X86_64_CODE_4_GOTPCRELX".to_string(), + R_X86_64_CODE_4_GOTTPOFF => "R_X86_64_CODE_4_GOTTPOFF".to_string(), + R_X86_64_CODE_4_GOTPC32_TLSDESC => "R_X86_64_CODE_4_GOTPC32_TLSDESC".to_string(), + R_X86_64_CODE_5_GOTPCRELX => "R_X86_64_CODE_5_GOTPCRELX".to_string(), + R_X86_64_CODE_5_GOTTPOFF => "R_X86_64_CODE_5_GOTTPOFF".to_string(), + R_X86_64_CODE_5_GOTPC32_TLSDESC => "R_X86_64_CODE_5_GOTPC32_TLSDESC".to_string(), + R_X86_64_CODE_6_GOTPCRELX => "R_X86_64_CODE_6_GOTPCRELX".to_string(), + R_X86_64_CODE_6_GOTTPOFF => "R_X86_64_CODE_6_GOTTPOFF".to_string(), + R_X86_64_CODE_6_GOTPC32_TLSDESC => "R_X86_64_CODE_6_GOTPC32_TLSDESC".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn i386_rel_to_string(r_type: u32) -> String { + match r_type { + R_386_NONE => "R_386_NONE".to_string(), + R_386_32 => "R_386_32".to_string(), + R_386_PC32 => "R_386_PC32".to_string(), + R_386_GOT32 => "R_386_GOT32".to_string(), + R_386_PLT32 => "R_386_PLT32".to_string(), + R_386_COPY => "R_386_COPY".to_string(), + R_386_GLOB_DAT => "R_386_GLOB_DAT".to_string(), + R_386_JUMP_SLOT => "R_386_JUMP_SLOT".to_string(), + R_386_RELATIVE => "R_386_RELATIVE".to_string(), + R_386_GOTOFF => "R_386_GOTOFF".to_string(), + R_386_GOTPC => "R_386_GOTPC".to_string(), + R_386_32PLT => "R_386_32PLT".to_string(), + R_386_TLS_TPOFF => "R_386_TLS_TPOFF".to_string(), + R_386_TLS_IE => "R_386_TLS_IE".to_string(), + R_386_TLS_GOTIE => "R_386_TLS_GOTIE".to_string(), + R_386_TLS_LE => "R_386_TLS_LE".to_string(), + R_386_TLS_GD => "R_386_TLS_GD".to_string(), + R_386_TLS_LDM => "R_386_TLS_LDM".to_string(), + R_386_16 => "R_386_16".to_string(), + R_386_PC16 => "R_386_PC16".to_string(), + R_386_8 => "R_386_8".to_string(), + R_386_PC8 => "R_386_PC8".to_string(), + R_386_TLS_GD_32 => "R_386_TLS_GD_32".to_string(), + R_386_TLS_GD_PUSH => "R_386_TLS_GD_PUSH".to_string(), + R_386_TLS_GD_CALL => "R_386_TLS_GD_CALL".to_string(), + R_386_TLS_GD_POP => "R_386_TLS_GD_POP".to_string(), + R_386_TLS_LDM_32 => "R_386_TLS_LDM_32".to_string(), + R_386_TLS_LDM_PUSH => "R_386_TLS_LDM_PUSH".to_string(), + R_386_TLS_LDM_CALL => "R_386_TLS_LDM_CALL".to_string(), + R_386_TLS_LDM_POP => "R_386_TLS_LDM_POP".to_string(), + R_386_TLS_LDO_32 => "R_386_TLS_LDO_32".to_string(), + R_386_TLS_IE_32 => "R_386_TLS_IE_32".to_string(), + R_386_TLS_LE_32 => "R_386_TLS_LE_32".to_string(), + R_386_TLS_DTPMOD32 => "R_386_TLS_DTPMOD32".to_string(), + R_386_TLS_DTPOFF32 => "R_386_TLS_DTPOFF32".to_string(), + R_386_TLS_TPOFF32 => "R_386_TLS_TPOFF32".to_string(), + R_386_SIZE32 => "R_386_SIZE32".to_string(), + R_386_TLS_GOTDESC => "R_386_TLS_GOTDESC".to_string(), + R_386_TLS_DESC_CALL => "R_386_TLS_DESC_CALL".to_string(), + R_386_TLS_DESC => "R_386_TLS_DESC".to_string(), + R_386_IRELATIVE => "R_386_IRELATIVE".to_string(), + R_386_GOT32X => "R_386_GOT32X".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn arm64_rel_to_string(r_type: u32) -> String { + match r_type { + R_AARCH64_NONE => "R_AARCH64_NONE".to_string(), + R_AARCH64_ABS64 => "R_AARCH64_ABS64".to_string(), + R_AARCH64_ABS32 => "R_AARCH64_ABS32".to_string(), + R_AARCH64_ABS16 => "R_AARCH64_ABS16".to_string(), + R_AARCH64_PREL64 => "R_AARCH64_PREL64".to_string(), + R_AARCH64_PREL32 => "R_AARCH64_PREL32".to_string(), + R_AARCH64_PREL16 => "R_AARCH64_PREL16".to_string(), + R_AARCH64_MOVW_UABS_G0 => "R_AARCH64_MOVW_UABS_G0".to_string(), + R_AARCH64_MOVW_UABS_G0_NC => "R_AARCH64_MOVW_UABS_G0_NC".to_string(), + R_AARCH64_MOVW_UABS_G1 => "R_AARCH64_MOVW_UABS_G1".to_string(), + R_AARCH64_MOVW_UABS_G1_NC => "R_AARCH64_MOVW_UABS_G1_NC".to_string(), + R_AARCH64_MOVW_UABS_G2 => "R_AARCH64_MOVW_UABS_G2".to_string(), + R_AARCH64_MOVW_UABS_G2_NC => "R_AARCH64_MOVW_UABS_G2_NC".to_string(), + R_AARCH64_MOVW_UABS_G3 => "R_AARCH64_MOVW_UABS_G3".to_string(), + R_AARCH64_MOVW_SABS_G0 => "R_AARCH64_MOVW_SABS_G0".to_string(), + R_AARCH64_MOVW_SABS_G1 => "R_AARCH64_MOVW_SABS_G1".to_string(), + R_AARCH64_MOVW_SABS_G2 => "R_AARCH64_MOVW_SABS_G2".to_string(), + R_AARCH64_LD_PREL_LO19 => "R_AARCH64_LD_PREL_LO19".to_string(), + R_AARCH64_ADR_PREL_LO21 => "R_AARCH64_ADR_PREL_LO21".to_string(), + R_AARCH64_ADR_PREL_PG_HI21 => "R_AARCH64_ADR_PREL_PG_HI21".to_string(), + R_AARCH64_ADR_PREL_PG_HI21_NC => "R_AARCH64_ADR_PREL_PG_HI21_NC".to_string(), + R_AARCH64_ADD_ABS_LO12_NC => "R_AARCH64_ADD_ABS_LO12_NC".to_string(), + R_AARCH64_LDST8_ABS_LO12_NC => "R_AARCH64_LDST8_ABS_LO12_NC".to_string(), + R_AARCH64_TSTBR14 => "R_AARCH64_TSTBR14".to_string(), + R_AARCH64_CONDBR19 => "R_AARCH64_CONDBR19".to_string(), + R_AARCH64_JUMP26 => "R_AARCH64_JUMP26".to_string(), + R_AARCH64_CALL26 => "R_AARCH64_CALL26".to_string(), + R_AARCH64_LDST16_ABS_LO12_NC => "R_AARCH64_LDST16_ABS_LO12_NC".to_string(), + R_AARCH64_LDST32_ABS_LO12_NC => "R_AARCH64_LDST32_ABS_LO12_NC".to_string(), + R_AARCH64_LDST64_ABS_LO12_NC => "R_AARCH64_LDST64_ABS_LO12_NC".to_string(), + R_AARCH64_MOVW_PREL_G0 => "R_AARCH64_MOVW_PREL_G0".to_string(), + R_AARCH64_MOVW_PREL_G0_NC => "R_AARCH64_MOVW_PREL_G0_NC".to_string(), + R_AARCH64_MOVW_PREL_G1 => "R_AARCH64_MOVW_PREL_G1".to_string(), + R_AARCH64_MOVW_PREL_G1_NC => "R_AARCH64_MOVW_PREL_G1_NC".to_string(), + R_AARCH64_MOVW_PREL_G2 => "R_AARCH64_MOVW_PREL_G2".to_string(), + R_AARCH64_MOVW_PREL_G2_NC => "R_AARCH64_MOVW_PREL_G2_NC".to_string(), + R_AARCH64_MOVW_PREL_G3 => "R_AARCH64_MOVW_PREL_G3".to_string(), + R_AARCH64_LDST128_ABS_LO12_NC => "R_AARCH64_LDST128_ABS_LO12_NC".to_string(), + R_AARCH64_ADR_GOT_PAGE => "R_AARCH64_ADR_GOT_PAGE".to_string(), + R_AARCH64_LD64_GOT_LO12_NC => "R_AARCH64_LD64_GOT_LO12_NC".to_string(), + R_AARCH64_LD64_GOTPAGE_LO15 => "R_AARCH64_LD64_GOTPAGE_LO15".to_string(), + R_AARCH64_PLT32 => "R_AARCH64_PLT32".to_string(), + R_AARCH64_GOTPCREL32 => "R_AARCH64_GOTPCREL32".to_string(), + R_AARCH64_TLSGD_ADR_PREL21 => "R_AARCH64_TLSGD_ADR_PREL21".to_string(), + R_AARCH64_TLSGD_ADR_PAGE21 => "R_AARCH64_TLSGD_ADR_PAGE21".to_string(), + R_AARCH64_TLSGD_ADD_LO12_NC => "R_AARCH64_TLSGD_ADD_LO12_NC".to_string(), + R_AARCH64_TLSGD_MOVW_G1 => "R_AARCH64_TLSGD_MOVW_G1".to_string(), + R_AARCH64_TLSGD_MOVW_G0_NC => "R_AARCH64_TLSGD_MOVW_G0_NC".to_string(), + R_AARCH64_TLSLD_ADR_PREL21 => "R_AARCH64_TLSLD_ADR_PREL21".to_string(), + R_AARCH64_TLSLD_ADR_PAGE21 => "R_AARCH64_TLSLD_ADR_PAGE21".to_string(), + R_AARCH64_TLSLD_ADD_LO12_NC => "R_AARCH64_TLSLD_ADD_LO12_NC".to_string(), + R_AARCH64_TLSLD_MOVW_G1 => "R_AARCH64_TLSLD_MOVW_G1".to_string(), + R_AARCH64_TLSLD_MOVW_G0_NC => "R_AARCH64_TLSLD_MOVW_G0_NC".to_string(), + R_AARCH64_TLSLD_LD_PREL19 => "R_AARCH64_TLSLD_LD_PREL19".to_string(), + R_AARCH64_TLSLD_MOVW_DTPREL_G2 => "R_AARCH64_TLSLD_MOVW_DTPREL_G2".to_string(), + R_AARCH64_TLSLD_MOVW_DTPREL_G1 => "R_AARCH64_TLSLD_MOVW_DTPREL_G1".to_string(), + R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC => "R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC".to_string(), + R_AARCH64_TLSLD_MOVW_DTPREL_G0 => "R_AARCH64_TLSLD_MOVW_DTPREL_G0".to_string(), + R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC => "R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC".to_string(), + R_AARCH64_TLSLD_ADD_DTPREL_HI12 => "R_AARCH64_TLSLD_ADD_DTPREL_HI12".to_string(), + R_AARCH64_TLSLD_ADD_DTPREL_LO12 => "R_AARCH64_TLSLD_ADD_DTPREL_LO12".to_string(), + R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC => "R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC".to_string(), + R_AARCH64_TLSLD_LDST8_DTPREL_LO12 => "R_AARCH64_TLSLD_LDST8_DTPREL_LO12".to_string(), + R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC => "R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC".to_string(), + R_AARCH64_TLSLD_LDST16_DTPREL_LO12 => "R_AARCH64_TLSLD_LDST16_DTPREL_LO12".to_string(), + R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC => { + "R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC".to_string() + } + R_AARCH64_TLSLD_LDST32_DTPREL_LO12 => "R_AARCH64_TLSLD_LDST32_DTPREL_LO12".to_string(), + R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC => { + "R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC".to_string() + } + R_AARCH64_TLSLD_LDST64_DTPREL_LO12 => "R_AARCH64_TLSLD_LDST64_DTPREL_LO12".to_string(), + R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC => { + "R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC".to_string() + } + R_AARCH64_TLSIE_MOVW_GOTTPREL_G1 => "R_AARCH64_TLSIE_MOVW_GOTTPREL_G1".to_string(), + R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC => "R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC".to_string(), + R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 => "R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21".to_string(), + R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC => { + "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC".to_string() + } + R_AARCH64_TLSIE_LD_GOTTPREL_PREL19 => "R_AARCH64_TLSIE_LD_GOTTPREL_PREL19".to_string(), + R_AARCH64_TLSLE_MOVW_TPREL_G2 => "R_AARCH64_TLSLE_MOVW_TPREL_G2".to_string(), + R_AARCH64_TLSLE_MOVW_TPREL_G1 => "R_AARCH64_TLSLE_MOVW_TPREL_G1".to_string(), + R_AARCH64_TLSLE_MOVW_TPREL_G1_NC => "R_AARCH64_TLSLE_MOVW_TPREL_G1_NC".to_string(), + R_AARCH64_TLSLE_MOVW_TPREL_G0 => "R_AARCH64_TLSLE_MOVW_TPREL_G0".to_string(), + R_AARCH64_TLSLE_MOVW_TPREL_G0_NC => "R_AARCH64_TLSLE_MOVW_TPREL_G0_NC".to_string(), + R_AARCH64_TLSLE_ADD_TPREL_HI12 => "R_AARCH64_TLSLE_ADD_TPREL_HI12".to_string(), + R_AARCH64_TLSLE_ADD_TPREL_LO12 => "R_AARCH64_TLSLE_ADD_TPREL_LO12".to_string(), + R_AARCH64_TLSLE_ADD_TPREL_LO12_NC => "R_AARCH64_TLSLE_ADD_TPREL_LO12_NC".to_string(), + R_AARCH64_TLSLE_LDST8_TPREL_LO12 => "R_AARCH64_TLSLE_LDST8_TPREL_LO12".to_string(), + R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC => "R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC".to_string(), + R_AARCH64_TLSLE_LDST16_TPREL_LO12 => "R_AARCH64_TLSLE_LDST16_TPREL_LO12".to_string(), + R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC => "R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC".to_string(), + R_AARCH64_TLSLE_LDST32_TPREL_LO12 => "R_AARCH64_TLSLE_LDST32_TPREL_LO12".to_string(), + R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC => "R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC".to_string(), + R_AARCH64_TLSLE_LDST64_TPREL_LO12 => "R_AARCH64_TLSLE_LDST64_TPREL_LO12".to_string(), + R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC => "R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC".to_string(), + R_AARCH64_TLSDESC_ADR_PAGE21 => "R_AARCH64_TLSDESC_ADR_PAGE21".to_string(), + R_AARCH64_TLSDESC_LD64_LO12 => "R_AARCH64_TLSDESC_LD64_LO12".to_string(), + R_AARCH64_TLSDESC_ADD_LO12 => "R_AARCH64_TLSDESC_ADD_LO12".to_string(), + R_AARCH64_TLSDESC_CALL => "R_AARCH64_TLSDESC_CALL".to_string(), + R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC => { + "R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC".to_string() + } + R_AARCH64_COPY => "R_AARCH64_COPY".to_string(), + R_AARCH64_GLOB_DAT => "R_AARCH64_GLOB_DAT".to_string(), + R_AARCH64_JUMP_SLOT => "R_AARCH64_JUMP_SLOT".to_string(), + R_AARCH64_RELATIVE => "R_AARCH64_RELATIVE".to_string(), + R_AARCH64_TLS_DTPMOD64 => "R_AARCH64_TLS_DTPMOD64".to_string(), + R_AARCH64_TLS_DTPREL64 => "R_AARCH64_TLS_DTPREL64".to_string(), + R_AARCH64_TLS_TPREL64 => "R_AARCH64_TLS_TPREL64".to_string(), + R_AARCH64_TLSDESC => "R_AARCH64_TLSDESC".to_string(), + R_AARCH64_IRELATIVE => "R_AARCH64_IRELATIVE".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn arm32_rel_to_string(r_type: u32) -> String { + match r_type { + R_ARM_NONE => "R_ARM_NONE".to_string(), + R_ARM_PC24 => "R_ARM_PC24".to_string(), + R_ARM_ABS32 => "R_ARM_ABS32".to_string(), + R_ARM_REL32 => "R_ARM_REL32".to_string(), + R_ARM_LDR_PC_G0 => "R_ARM_LDR_PC_G0".to_string(), + R_ARM_ABS16 => "R_ARM_ABS16".to_string(), + R_ARM_ABS12 => "R_ARM_ABS12".to_string(), + R_ARM_THM_ABS5 => "R_ARM_THM_ABS5".to_string(), + R_ARM_ABS8 => "R_ARM_ABS8".to_string(), + R_ARM_SBREL32 => "R_ARM_SBREL32".to_string(), + R_ARM_THM_CALL => "R_ARM_THM_CALL".to_string(), + R_ARM_THM_PC8 => "R_ARM_THM_PC8".to_string(), + R_ARM_BREL_ADJ => "R_ARM_BREL_ADJ".to_string(), + R_ARM_TLS_DESC => "R_ARM_TLS_DESC".to_string(), + R_ARM_THM_SWI8 => "R_ARM_THM_SWI8".to_string(), + R_ARM_XPC25 => "R_ARM_XPC25".to_string(), + R_ARM_THM_XPC22 => "R_ARM_THM_XPC22".to_string(), + R_ARM_TLS_DTPMOD32 => "R_ARM_TLS_DTPMOD32".to_string(), + R_ARM_TLS_DTPOFF32 => "R_ARM_TLS_DTPOFF32".to_string(), + R_ARM_TLS_TPOFF32 => "R_ARM_TLS_TPOFF32".to_string(), + R_ARM_COPY => "R_ARM_COPY".to_string(), + R_ARM_GLOB_DAT => "R_ARM_GLOB_DAT".to_string(), + R_ARM_JUMP_SLOT => "R_ARM_JUMP_SLOT".to_string(), + R_ARM_RELATIVE => "R_ARM_RELATIVE".to_string(), + R_ARM_GOTOFF32 => "R_ARM_GOTOFF32".to_string(), + R_ARM_BASE_PREL => "R_ARM_BASE_PREL".to_string(), + R_ARM_GOT_BREL => "R_ARM_GOT_BREL".to_string(), + R_ARM_PLT32 => "R_ARM_PLT32".to_string(), + R_ARM_CALL => "R_ARM_CALL".to_string(), + R_ARM_JUMP24 => "R_ARM_JUMP24".to_string(), + R_ARM_THM_JUMP24 => "R_ARM_THM_JUMP24".to_string(), + R_ARM_BASE_ABS => "R_ARM_BASE_ABS".to_string(), + R_ARM_ALU_PCREL_7_0 => "R_ARM_ALU_PCREL_7_0".to_string(), + R_ARM_ALU_PCREL_15_8 => "R_ARM_ALU_PCREL_15_8".to_string(), + R_ARM_ALU_PCREL_23_15 => "R_ARM_ALU_PCREL_23_15".to_string(), + R_ARM_LDR_SBREL_11_0_NC => "R_ARM_LDR_SBREL_11_0_NC".to_string(), + R_ARM_ALU_SBREL_19_12_NC => "R_ARM_ALU_SBREL_19_12_NC".to_string(), + R_ARM_ALU_SBREL_27_20_CK => "R_ARM_ALU_SBREL_27_20_CK".to_string(), + R_ARM_TARGET1 => "R_ARM_TARGET1".to_string(), + R_ARM_SBREL31 => "R_ARM_SBREL31".to_string(), + R_ARM_V4BX => "R_ARM_V4BX".to_string(), + R_ARM_TARGET2 => "R_ARM_TARGET2".to_string(), + R_ARM_PREL31 => "R_ARM_PREL31".to_string(), + R_ARM_MOVW_ABS_NC => "R_ARM_MOVW_ABS_NC".to_string(), + R_ARM_MOVT_ABS => "R_ARM_MOVT_ABS".to_string(), + R_ARM_MOVW_PREL_NC => "R_ARM_MOVW_PREL_NC".to_string(), + R_ARM_MOVT_PREL => "R_ARM_MOVT_PREL".to_string(), + R_ARM_THM_MOVW_ABS_NC => "R_ARM_THM_MOVW_ABS_NC".to_string(), + R_ARM_THM_MOVT_ABS => "R_ARM_THM_MOVT_ABS".to_string(), + R_ARM_THM_MOVW_PREL_NC => "R_ARM_THM_MOVW_PREL_NC".to_string(), + R_ARM_THM_MOVT_PREL => "R_ARM_THM_MOVT_PREL".to_string(), + R_ARM_THM_JUMP19 => "R_ARM_THM_JUMP19".to_string(), + R_ARM_THM_JUMP6 => "R_ARM_THM_JUMP6".to_string(), + R_ARM_THM_ALU_PREL_11_0 => "R_ARM_THM_ALU_PREL_11_0".to_string(), + R_ARM_THM_PC12 => "R_ARM_THM_PC12".to_string(), + R_ARM_ABS32_NOI => "R_ARM_ABS32_NOI".to_string(), + R_ARM_REL32_NOI => "R_ARM_REL32_NOI".to_string(), + R_ARM_ALU_PC_G0_NC => "R_ARM_ALU_PC_G0_NC".to_string(), + R_ARM_ALU_PC_G0 => "R_ARM_ALU_PC_G0".to_string(), + R_ARM_ALU_PC_G1_NC => "R_ARM_ALU_PC_G1_NC".to_string(), + R_ARM_ALU_PC_G1 => "R_ARM_ALU_PC_G1".to_string(), + R_ARM_ALU_PC_G2 => "R_ARM_ALU_PC_G2".to_string(), + R_ARM_LDR_PC_G1 => "R_ARM_LDR_PC_G1".to_string(), + R_ARM_LDR_PC_G2 => "R_ARM_LDR_PC_G2".to_string(), + R_ARM_LDRS_PC_G0 => "R_ARM_LDRS_PC_G0".to_string(), + R_ARM_LDRS_PC_G1 => "R_ARM_LDRS_PC_G1".to_string(), + R_ARM_LDRS_PC_G2 => "R_ARM_LDRS_PC_G2".to_string(), + R_ARM_LDC_PC_G0 => "R_ARM_LDC_PC_G0".to_string(), + R_ARM_LDC_PC_G1 => "R_ARM_LDC_PC_G1".to_string(), + R_ARM_LDC_PC_G2 => "R_ARM_LDC_PC_G2".to_string(), + R_ARM_ALU_SB_G0_NC => "R_ARM_ALU_SB_G0_NC".to_string(), + R_ARM_ALU_SB_G0 => "R_ARM_ALU_SB_G0".to_string(), + R_ARM_ALU_SB_G1_NC => "R_ARM_ALU_SB_G1_NC".to_string(), + R_ARM_ALU_SB_G1 => "R_ARM_ALU_SB_G1".to_string(), + R_ARM_ALU_SB_G2 => "R_ARM_ALU_SB_G2".to_string(), + R_ARM_LDR_SB_G0 => "R_ARM_LDR_SB_G0".to_string(), + R_ARM_LDR_SB_G1 => "R_ARM_LDR_SB_G1".to_string(), + R_ARM_LDR_SB_G2 => "R_ARM_LDR_SB_G2".to_string(), + R_ARM_LDRS_SB_G0 => "R_ARM_LDRS_SB_G0".to_string(), + R_ARM_LDRS_SB_G1 => "R_ARM_LDRS_SB_G1".to_string(), + R_ARM_LDRS_SB_G2 => "R_ARM_LDRS_SB_G2".to_string(), + R_ARM_LDC_SB_G0 => "R_ARM_LDC_SB_G0".to_string(), + R_ARM_LDC_SB_G1 => "R_ARM_LDC_SB_G1".to_string(), + R_ARM_LDC_SB_G2 => "R_ARM_LDC_SB_G2".to_string(), + R_ARM_MOVW_BREL_NC => "R_ARM_MOVW_BREL_NC".to_string(), + R_ARM_MOVT_BREL => "R_ARM_MOVT_BREL".to_string(), + R_ARM_MOVW_BREL => "R_ARM_MOVW_BREL".to_string(), + R_ARM_THM_MOVW_BREL_NC => "R_ARM_THM_MOVW_BREL_NC".to_string(), + R_ARM_THM_MOVT_BREL => "R_ARM_THM_MOVT_BREL".to_string(), + R_ARM_THM_MOVW_BREL => "R_ARM_THM_MOVW_BREL".to_string(), + R_ARM_TLS_GOTDESC => "R_ARM_TLS_GOTDESC".to_string(), + R_ARM_TLS_CALL => "R_ARM_TLS_CALL".to_string(), + R_ARM_TLS_DESCSEQ => "R_ARM_TLS_DESCSEQ".to_string(), + R_ARM_THM_TLS_CALL => "R_ARM_THM_TLS_CALL".to_string(), + R_ARM_PLT32_ABS => "R_ARM_PLT32_ABS".to_string(), + R_ARM_GOT_ABS => "R_ARM_GOT_ABS".to_string(), + R_ARM_GOT_PREL => "R_ARM_GOT_PREL".to_string(), + R_ARM_GOT_BREL12 => "R_ARM_GOT_BREL12".to_string(), + R_ARM_GOTOFF12 => "R_ARM_GOTOFF12".to_string(), + R_ARM_GOTRELAX => "R_ARM_GOTRELAX".to_string(), + R_ARM_GNU_VTENTRY => "R_ARM_GNU_VTENTRY".to_string(), + R_ARM_GNU_VTINHERIT => "R_ARM_GNU_VTINHERIT".to_string(), + R_ARM_THM_JUMP11 => "R_ARM_THM_JUMP11".to_string(), + R_ARM_THM_JUMP8 => "R_ARM_THM_JUMP8".to_string(), + R_ARM_TLS_GD32 => "R_ARM_TLS_GD32".to_string(), + R_ARM_TLS_LDM32 => "R_ARM_TLS_LDM32".to_string(), + R_ARM_TLS_LDO32 => "R_ARM_TLS_LDO32".to_string(), + R_ARM_TLS_IE32 => "R_ARM_TLS_IE32".to_string(), + R_ARM_TLS_LE32 => "R_ARM_TLS_LE32".to_string(), + R_ARM_TLS_LDO12 => "R_ARM_TLS_LDO12".to_string(), + R_ARM_TLS_LE12 => "R_ARM_TLS_LE12".to_string(), + R_ARM_TLS_IE12GP => "R_ARM_TLS_IE12GP".to_string(), + R_ARM_PRIVATE_0 => "R_ARM_PRIVATE_0".to_string(), + R_ARM_PRIVATE_1 => "R_ARM_PRIVATE_1".to_string(), + R_ARM_PRIVATE_2 => "R_ARM_PRIVATE_2".to_string(), + R_ARM_PRIVATE_3 => "R_ARM_PRIVATE_3".to_string(), + R_ARM_PRIVATE_4 => "R_ARM_PRIVATE_4".to_string(), + R_ARM_PRIVATE_5 => "R_ARM_PRIVATE_5".to_string(), + R_ARM_PRIVATE_6 => "R_ARM_PRIVATE_6".to_string(), + R_ARM_PRIVATE_7 => "R_ARM_PRIVATE_7".to_string(), + R_ARM_PRIVATE_8 => "R_ARM_PRIVATE_8".to_string(), + R_ARM_PRIVATE_9 => "R_ARM_PRIVATE_9".to_string(), + R_ARM_PRIVATE_10 => "R_ARM_PRIVATE_10".to_string(), + R_ARM_PRIVATE_11 => "R_ARM_PRIVATE_11".to_string(), + R_ARM_PRIVATE_12 => "R_ARM_PRIVATE_12".to_string(), + R_ARM_PRIVATE_13 => "R_ARM_PRIVATE_13".to_string(), + R_ARM_PRIVATE_14 => "R_ARM_PRIVATE_14".to_string(), + R_ARM_PRIVATE_15 => "R_ARM_PRIVATE_15".to_string(), + R_ARM_ME_TOO => "R_ARM_ME_TOO".to_string(), + R_ARM_THM_TLS_DESCSEQ16 => "R_ARM_THM_TLS_DESCSEQ16".to_string(), + R_ARM_THM_TLS_DESCSEQ32 => "R_ARM_THM_TLS_DESCSEQ32".to_string(), + R_ARM_THM_BF16 => "R_ARM_THM_BF16".to_string(), + R_ARM_THM_BF12 => "R_ARM_THM_BF12".to_string(), + R_ARM_THM_BF18 => "R_ARM_THM_BF18".to_string(), + R_ARM_IRELATIVE => "R_ARM_IRELATIVE".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn riscv_rel_to_string(r_type: u32) -> String { + match r_type { + R_RISCV_NONE => "R_RISCV_NONE".to_string(), + R_RISCV_32 => "R_RISCV_32".to_string(), + R_RISCV_64 => "R_RISCV_64".to_string(), + R_RISCV_RELATIVE => "R_RISCV_RELATIVE".to_string(), + R_RISCV_COPY => "R_RISCV_COPY".to_string(), + R_RISCV_JUMP_SLOT => "R_RISCV_JUMP_SLOT".to_string(), + R_RISCV_TLS_DTPMOD32 => "R_RISCV_TLS_DTPMOD32".to_string(), + R_RISCV_TLS_DTPMOD64 => "R_RISCV_TLS_DTPMOD64".to_string(), + R_RISCV_TLS_DTPREL32 => "R_RISCV_TLS_DTPREL32".to_string(), + R_RISCV_TLS_DTPREL64 => "R_RISCV_TLS_DTPREL64".to_string(), + R_RISCV_TLS_TPREL32 => "R_RISCV_TLS_TPREL32".to_string(), + R_RISCV_TLS_TPREL64 => "R_RISCV_TLS_TPREL64".to_string(), + R_RISCV_TLSDESC => "R_RISCV_TLSDESC".to_string(), + R_RISCV_BRANCH => "R_RISCV_BRANCH".to_string(), + R_RISCV_JAL => "R_RISCV_JAL".to_string(), + R_RISCV_CALL => "R_RISCV_CALL".to_string(), + R_RISCV_CALL_PLT => "R_RISCV_CALL_PLT".to_string(), + R_RISCV_GOT_HI20 => "R_RISCV_GOT_HI20".to_string(), + R_RISCV_TLS_GOT_HI20 => "R_RISCV_TLS_GOT_HI20".to_string(), + R_RISCV_TLS_GD_HI20 => "R_RISCV_TLS_GD_HI20".to_string(), + R_RISCV_PCREL_HI20 => "R_RISCV_PCREL_HI20".to_string(), + R_RISCV_PCREL_LO12_I => "R_RISCV_PCREL_LO12_I".to_string(), + R_RISCV_PCREL_LO12_S => "R_RISCV_PCREL_LO12_S".to_string(), + R_RISCV_HI20 => "R_RISCV_HI20".to_string(), + R_RISCV_LO12_I => "R_RISCV_LO12_I".to_string(), + R_RISCV_LO12_S => "R_RISCV_LO12_S".to_string(), + R_RISCV_TPREL_HI20 => "R_RISCV_TPREL_HI20".to_string(), + R_RISCV_TPREL_LO12_I => "R_RISCV_TPREL_LO12_I".to_string(), + R_RISCV_TPREL_LO12_S => "R_RISCV_TPREL_LO12_S".to_string(), + R_RISCV_TPREL_ADD => "R_RISCV_TPREL_ADD".to_string(), + R_RISCV_ADD8 => "R_RISCV_ADD8".to_string(), + R_RISCV_ADD16 => "R_RISCV_ADD16".to_string(), + R_RISCV_ADD32 => "R_RISCV_ADD32".to_string(), + R_RISCV_ADD64 => "R_RISCV_ADD64".to_string(), + R_RISCV_SUB8 => "R_RISCV_SUB8".to_string(), + R_RISCV_SUB16 => "R_RISCV_SUB16".to_string(), + R_RISCV_SUB32 => "R_RISCV_SUB32".to_string(), + R_RISCV_SUB64 => "R_RISCV_SUB64".to_string(), + R_RISCV_ALIGN => "R_RISCV_ALIGN".to_string(), + R_RISCV_RVC_BRANCH => "R_RISCV_RVC_BRANCH".to_string(), + R_RISCV_RVC_JUMP => "R_RISCV_RVC_JUMP".to_string(), + R_RISCV_RELAX => "R_RISCV_RELAX".to_string(), + R_RISCV_SUB6 => "R_RISCV_SUB6".to_string(), + R_RISCV_SET6 => "R_RISCV_SET6".to_string(), + R_RISCV_SET8 => "R_RISCV_SET8".to_string(), + R_RISCV_SET16 => "R_RISCV_SET16".to_string(), + R_RISCV_SET32 => "R_RISCV_SET32".to_string(), + R_RISCV_GOT32_PCREL => "R_RISCV_GOT32_PCREL".to_string(), + R_RISCV_32_PCREL => "R_RISCV_32_PCREL".to_string(), + R_RISCV_IRELATIVE => "R_RISCV_IRELATIVE".to_string(), + R_RISCV_PLT32 => "R_RISCV_PLT32".to_string(), + R_RISCV_SET_ULEB128 => "R_RISCV_SET_ULEB128".to_string(), + R_RISCV_SUB_ULEB128 => "R_RISCV_SUB_ULEB128".to_string(), + R_RISCV_TLSDESC_HI20 => "R_RISCV_TLSDESC_HI20".to_string(), + R_RISCV_TLSDESC_LOAD_LO12 => "R_RISCV_TLSDESC_LOAD_LO12".to_string(), + R_RISCV_TLSDESC_ADD_LO12 => "R_RISCV_TLSDESC_ADD_LO12".to_string(), + R_RISCV_TLSDESC_CALL => "R_RISCV_TLSDESC_CALL".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn ppc32_rel_to_string(r_type: u32) -> String { + match r_type { + R_PPC_NONE => "R_PPC_NONE".to_string(), + R_PPC_ADDR32 => "R_PPC_ADDR32".to_string(), + R_PPC_ADDR24 => "R_PPC_ADDR24".to_string(), + R_PPC_ADDR16 => "R_PPC_ADDR16".to_string(), + R_PPC_ADDR16_LO => "R_PPC_ADDR16_LO".to_string(), + R_PPC_ADDR16_HI => "R_PPC_ADDR16_HI".to_string(), + R_PPC_ADDR16_HA => "R_PPC_ADDR16_HA".to_string(), + R_PPC_ADDR14 => "R_PPC_ADDR14".to_string(), + R_PPC_ADDR14_BRTAKEN => "R_PPC_ADDR14_BRTAKEN".to_string(), + R_PPC_ADDR14_BRNTAKEN => "R_PPC_ADDR14_BRNTAKEN".to_string(), + R_PPC_REL24 => "R_PPC_REL24".to_string(), + R_PPC_REL14 => "R_PPC_REL14".to_string(), + R_PPC_REL14_BRTAKEN => "R_PPC_REL14_BRTAKEN".to_string(), + R_PPC_REL14_BRNTAKEN => "R_PPC_REL14_BRNTAKEN".to_string(), + R_PPC_GOT16 => "R_PPC_GOT16".to_string(), + R_PPC_GOT16_LO => "R_PPC_GOT16_LO".to_string(), + R_PPC_GOT16_HI => "R_PPC_GOT16_HI".to_string(), + R_PPC_GOT16_HA => "R_PPC_GOT16_HA".to_string(), + R_PPC_PLTREL24 => "R_PPC_PLTREL24".to_string(), + R_PPC_COPY => "R_PPC_COPY".to_string(), + R_PPC_GLOB_DAT => "R_PPC_GLOB_DAT".to_string(), + R_PPC_JMP_SLOT => "R_PPC_JMP_SLOT".to_string(), + R_PPC_RELATIVE => "R_PPC_RELATIVE".to_string(), + R_PPC_LOCAL24PC => "R_PPC_LOCAL24PC".to_string(), + R_PPC_UADDR32 => "R_PPC_UADDR32".to_string(), + R_PPC_UADDR16 => "R_PPC_UADDR16".to_string(), + R_PPC_REL32 => "R_PPC_REL32".to_string(), + R_PPC_PLT32 => "R_PPC_PLT32".to_string(), + R_PPC_PLTREL32 => "R_PPC_PLTREL32".to_string(), + R_PPC_PLT16_LO => "R_PPC_PLT16_LO".to_string(), + R_PPC_PLT16_HI => "R_PPC_PLT16_HI".to_string(), + R_PPC_PLT16_HA => "R_PPC_PLT16_HA".to_string(), + R_PPC_SDAREL16 => "R_PPC_SDAREL16".to_string(), + R_PPC_SECTOFF => "R_PPC_SECTOFF".to_string(), + R_PPC_SECTOFF_LO => "R_PPC_SECTOFF_LO".to_string(), + R_PPC_SECTOFF_HI => "R_PPC_SECTOFF_HI".to_string(), + R_PPC_SECTOFF_HA => "R_PPC_SECTOFF_HA".to_string(), + R_PPC_ADDR30 => "R_PPC_ADDR30".to_string(), + R_PPC_TLS => "R_PPC_TLS".to_string(), + R_PPC_DTPMOD32 => "R_PPC_DTPMOD32".to_string(), + R_PPC_TPREL16 => "R_PPC_TPREL16".to_string(), + R_PPC_TPREL16_LO => "R_PPC_TPREL16_LO".to_string(), + R_PPC_TPREL16_HI => "R_PPC_TPREL16_HI".to_string(), + R_PPC_TPREL16_HA => "R_PPC_TPREL16_HA".to_string(), + R_PPC_TPREL32 => "R_PPC_TPREL32".to_string(), + R_PPC_DTPREL16 => "R_PPC_DTPREL16".to_string(), + R_PPC_DTPREL16_LO => "R_PPC_DTPREL16_LO".to_string(), + R_PPC_DTPREL16_HI => "R_PPC_DTPREL16_HI".to_string(), + R_PPC_DTPREL16_HA => "R_PPC_DTPREL16_HA".to_string(), + R_PPC_DTPREL32 => "R_PPC_DTPREL32".to_string(), + R_PPC_GOT_TLSGD16 => "R_PPC_GOT_TLSGD16".to_string(), + R_PPC_GOT_TLSGD16_LO => "R_PPC_GOT_TLSGD16_LO".to_string(), + R_PPC_GOT_TLSGD16_HI => "R_PPC_GOT_TLSGD16_HI".to_string(), + R_PPC_GOT_TLSGD16_HA => "R_PPC_GOT_TLSGD16_HA".to_string(), + R_PPC_GOT_TLSLD16 => "R_PPC_GOT_TLSLD16".to_string(), + R_PPC_GOT_TLSLD16_LO => "R_PPC_GOT_TLSLD16_LO".to_string(), + R_PPC_GOT_TLSLD16_HI => "R_PPC_GOT_TLSLD16_HI".to_string(), + R_PPC_GOT_TLSLD16_HA => "R_PPC_GOT_TLSLD16_HA".to_string(), + R_PPC_GOT_TPREL16 => "R_PPC_GOT_TPREL16".to_string(), + R_PPC_GOT_TPREL16_LO => "R_PPC_GOT_TPREL16_LO".to_string(), + R_PPC_GOT_TPREL16_HI => "R_PPC_GOT_TPREL16_HI".to_string(), + R_PPC_GOT_TPREL16_HA => "R_PPC_GOT_TPREL16_HA".to_string(), + R_PPC_GOT_DTPREL16 => "R_PPC_GOT_DTPREL16".to_string(), + R_PPC_GOT_DTPREL16_LO => "R_PPC_GOT_DTPREL16_LO".to_string(), + R_PPC_GOT_DTPREL16_HI => "R_PPC_GOT_DTPREL16_HI".to_string(), + R_PPC_GOT_DTPREL16_HA => "R_PPC_GOT_DTPREL16_HA".to_string(), + R_PPC_TLSGD => "R_PPC_TLSGD".to_string(), + R_PPC_TLSLD => "R_PPC_TLSLD".to_string(), + R_PPC_PLTSEQ => "R_PPC_PLTSEQ".to_string(), + R_PPC_PLTCALL => "R_PPC_PLTCALL".to_string(), + R_PPC_IRELATIVE => "R_PPC_IRELATIVE".to_string(), + R_PPC_REL16 => "R_PPC_REL16".to_string(), + R_PPC_REL16_LO => "R_PPC_REL16_LO".to_string(), + R_PPC_REL16_HI => "R_PPC_REL16_HI".to_string(), + R_PPC_REL16_HA => "R_PPC_REL16_HA".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn ppc64_rel_to_string(r_type: u32) -> String { + match r_type { + R_PPC64_NONE => "R_PPC64_NONE".to_string(), + R_PPC64_ADDR32 => "R_PPC64_ADDR32".to_string(), + R_PPC64_ADDR24 => "R_PPC64_ADDR24".to_string(), + R_PPC64_ADDR16 => "R_PPC64_ADDR16".to_string(), + R_PPC64_ADDR16_LO => "R_PPC64_ADDR16_LO".to_string(), + R_PPC64_ADDR16_HI => "R_PPC64_ADDR16_HI".to_string(), + R_PPC64_ADDR16_HA => "R_PPC64_ADDR16_HA".to_string(), + R_PPC64_ADDR14 => "R_PPC64_ADDR14".to_string(), + R_PPC64_ADDR14_BRTAKEN => "R_PPC64_ADDR14_BRTAKEN".to_string(), + R_PPC64_ADDR14_BRNTAKEN => "R_PPC64_ADDR14_BRNTAKEN".to_string(), + R_PPC64_REL24 => "R_PPC64_REL24".to_string(), + R_PPC64_REL14 => "R_PPC64_REL14".to_string(), + R_PPC64_REL14_BRTAKEN => "R_PPC64_REL14_BRTAKEN".to_string(), + R_PPC64_REL14_BRNTAKEN => "R_PPC64_REL14_BRNTAKEN".to_string(), + R_PPC64_GOT16 => "R_PPC64_GOT16".to_string(), + R_PPC64_GOT16_LO => "R_PPC64_GOT16_LO".to_string(), + R_PPC64_GOT16_HI => "R_PPC64_GOT16_HI".to_string(), + R_PPC64_GOT16_HA => "R_PPC64_GOT16_HA".to_string(), + R_PPC64_COPY => "R_PPC64_COPY".to_string(), + R_PPC64_GLOB_DAT => "R_PPC64_GLOB_DAT".to_string(), + R_PPC64_JMP_SLOT => "R_PPC64_JMP_SLOT".to_string(), + R_PPC64_RELATIVE => "R_PPC64_RELATIVE".to_string(), + R_PPC64_REL32 => "R_PPC64_REL32".to_string(), + R_PPC64_PLT16_LO => "R_PPC64_PLT16_LO".to_string(), + R_PPC64_PLT16_HI => "R_PPC64_PLT16_HI".to_string(), + R_PPC64_PLT16_HA => "R_PPC64_PLT16_HA".to_string(), + R_PPC64_ADDR64 => "R_PPC64_ADDR64".to_string(), + R_PPC64_ADDR16_HIGHER => "R_PPC64_ADDR16_HIGHER".to_string(), + R_PPC64_ADDR16_HIGHERA => "R_PPC64_ADDR16_HIGHERA".to_string(), + R_PPC64_ADDR16_HIGHEST => "R_PPC64_ADDR16_HIGHEST".to_string(), + R_PPC64_ADDR16_HIGHESTA => "R_PPC64_ADDR16_HIGHESTA".to_string(), + R_PPC64_REL64 => "R_PPC64_REL64".to_string(), + R_PPC64_TOC16 => "R_PPC64_TOC16".to_string(), + R_PPC64_TOC16_LO => "R_PPC64_TOC16_LO".to_string(), + R_PPC64_TOC16_HI => "R_PPC64_TOC16_HI".to_string(), + R_PPC64_TOC16_HA => "R_PPC64_TOC16_HA".to_string(), + R_PPC64_TOC => "R_PPC64_TOC".to_string(), + R_PPC64_ADDR16_DS => "R_PPC64_ADDR16_DS".to_string(), + R_PPC64_ADDR16_LO_DS => "R_PPC64_ADDR16_LO_DS".to_string(), + R_PPC64_GOT16_DS => "R_PPC64_GOT16_DS".to_string(), + R_PPC64_GOT16_LO_DS => "R_PPC64_GOT16_LO_DS".to_string(), + R_PPC64_PLT16_LO_DS => "R_PPC64_PLT16_LO_DS".to_string(), + R_PPC64_TOC16_DS => "R_PPC64_TOC16_DS".to_string(), + R_PPC64_TOC16_LO_DS => "R_PPC64_TOC16_LO_DS".to_string(), + R_PPC64_TLS => "R_PPC64_TLS".to_string(), + R_PPC64_DTPMOD64 => "R_PPC64_DTPMOD64".to_string(), + R_PPC64_TPREL16 => "R_PPC64_TPREL16".to_string(), + R_PPC64_TPREL16_LO => "R_PPC64_TPREL16_LO".to_string(), + R_PPC64_TPREL16_HI => "R_PPC64_TPREL16_HI".to_string(), + R_PPC64_TPREL16_HA => "R_PPC64_TPREL16_HA".to_string(), + R_PPC64_TPREL64 => "R_PPC64_TPREL64".to_string(), + R_PPC64_DTPREL16 => "R_PPC64_DTPREL16".to_string(), + R_PPC64_DTPREL16_LO => "R_PPC64_DTPREL16_LO".to_string(), + R_PPC64_DTPREL16_HI => "R_PPC64_DTPREL16_HI".to_string(), + R_PPC64_DTPREL16_HA => "R_PPC64_DTPREL16_HA".to_string(), + R_PPC64_DTPREL64 => "R_PPC64_DTPREL64".to_string(), + R_PPC64_GOT_TLSGD16 => "R_PPC64_GOT_TLSGD16".to_string(), + R_PPC64_GOT_TLSGD16_LO => "R_PPC64_GOT_TLSGD16_LO".to_string(), + R_PPC64_GOT_TLSGD16_HI => "R_PPC64_GOT_TLSGD16_HI".to_string(), + R_PPC64_GOT_TLSGD16_HA => "R_PPC64_GOT_TLSGD16_HA".to_string(), + R_PPC64_GOT_TLSLD16 => "R_PPC64_GOT_TLSLD16".to_string(), + R_PPC64_GOT_TLSLD16_LO => "R_PPC64_GOT_TLSLD16_LO".to_string(), + R_PPC64_GOT_TLSLD16_HI => "R_PPC64_GOT_TLSLD16_HI".to_string(), + R_PPC64_GOT_TLSLD16_HA => "R_PPC64_GOT_TLSLD16_HA".to_string(), + R_PPC64_GOT_TPREL16_DS => "R_PPC64_GOT_TPREL16_DS".to_string(), + R_PPC64_GOT_TPREL16_LO_DS => "R_PPC64_GOT_TPREL16_LO_DS".to_string(), + R_PPC64_GOT_TPREL16_HI => "R_PPC64_GOT_TPREL16_HI".to_string(), + R_PPC64_GOT_TPREL16_HA => "R_PPC64_GOT_TPREL16_HA".to_string(), + R_PPC64_GOT_DTPREL16_DS => "R_PPC64_GOT_DTPREL16_DS".to_string(), + R_PPC64_GOT_DTPREL16_LO_DS => "R_PPC64_GOT_DTPREL16_LO_DS".to_string(), + R_PPC64_GOT_DTPREL16_HI => "R_PPC64_GOT_DTPREL16_HI".to_string(), + R_PPC64_GOT_DTPREL16_HA => "R_PPC64_GOT_DTPREL16_HA".to_string(), + R_PPC64_TPREL16_DS => "R_PPC64_TPREL16_DS".to_string(), + R_PPC64_TPREL16_LO_DS => "R_PPC64_TPREL16_LO_DS".to_string(), + R_PPC64_TPREL16_HIGHER => "R_PPC64_TPREL16_HIGHER".to_string(), + R_PPC64_TPREL16_HIGHERA => "R_PPC64_TPREL16_HIGHERA".to_string(), + R_PPC64_TPREL16_HIGHEST => "R_PPC64_TPREL16_HIGHEST".to_string(), + R_PPC64_TPREL16_HIGHESTA => "R_PPC64_TPREL16_HIGHESTA".to_string(), + R_PPC64_DTPREL16_DS => "R_PPC64_DTPREL16_DS".to_string(), + R_PPC64_DTPREL16_LO_DS => "R_PPC64_DTPREL16_LO_DS".to_string(), + R_PPC64_DTPREL16_HIGHER => "R_PPC64_DTPREL16_HIGHER".to_string(), + R_PPC64_DTPREL16_HIGHERA => "R_PPC64_DTPREL16_HIGHERA".to_string(), + R_PPC64_DTPREL16_HIGHEST => "R_PPC64_DTPREL16_HIGHEST".to_string(), + R_PPC64_DTPREL16_HIGHESTA => "R_PPC64_DTPREL16_HIGHESTA".to_string(), + R_PPC64_TLSGD => "R_PPC64_TLSGD".to_string(), + R_PPC64_TLSLD => "R_PPC64_TLSLD".to_string(), + R_PPC64_ADDR16_HIGH => "R_PPC64_ADDR16_HIGH".to_string(), + R_PPC64_ADDR16_HIGHA => "R_PPC64_ADDR16_HIGHA".to_string(), + R_PPC64_TPREL16_HIGH => "R_PPC64_TPREL16_HIGH".to_string(), + R_PPC64_TPREL16_HIGHA => "R_PPC64_TPREL16_HIGHA".to_string(), + R_PPC64_DTPREL16_HIGH => "R_PPC64_DTPREL16_HIGH".to_string(), + R_PPC64_DTPREL16_HIGHA => "R_PPC64_DTPREL16_HIGHA".to_string(), + R_PPC64_REL24_NOTOC => "R_PPC64_REL24_NOTOC".to_string(), + R_PPC64_ENTRY => "R_PPC64_ENTRY".to_string(), + R_PPC64_PLTSEQ => "R_PPC64_PLTSEQ".to_string(), + R_PPC64_PLTCALL => "R_PPC64_PLTCALL".to_string(), + R_PPC64_PLTSEQ_NOTOC => "R_PPC64_PLTSEQ_NOTOC".to_string(), + R_PPC64_PLTCALL_NOTOC => "R_PPC64_PLTCALL_NOTOC".to_string(), + R_PPC64_PCREL_OPT => "R_PPC64_PCREL_OPT".to_string(), + R_PPC64_PCREL34 => "R_PPC64_PCREL34".to_string(), + R_PPC64_GOT_PCREL34 => "R_PPC64_GOT_PCREL34".to_string(), + R_PPC64_PLT_PCREL34 => "R_PPC64_PLT_PCREL34".to_string(), + R_PPC64_PLT_PCREL34_NOTOC => "R_PPC64_PLT_PCREL34_NOTOC".to_string(), + R_PPC64_TPREL34 => "R_PPC64_TPREL34".to_string(), + R_PPC64_DTPREL34 => "R_PPC64_DTPREL34".to_string(), + R_PPC64_GOT_TLSGD_PCREL34 => "R_PPC64_GOT_TLSGD_PCREL34".to_string(), + R_PPC64_GOT_TLSLD_PCREL34 => "R_PPC64_GOT_TLSLD_PCREL34".to_string(), + R_PPC64_GOT_TPREL_PCREL34 => "R_PPC64_GOT_TPREL_PCREL34".to_string(), + R_PPC64_IRELATIVE => "R_PPC64_IRELATIVE".to_string(), + R_PPC64_REL16 => "R_PPC64_REL16".to_string(), + R_PPC64_REL16_LO => "R_PPC64_REL16_LO".to_string(), + R_PPC64_REL16_HI => "R_PPC64_REL16_HI".to_string(), + R_PPC64_REL16_HA => "R_PPC64_REL16_HA".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn sparc64_rel_to_string(r_type: u32) -> String { + match r_type { + R_SPARC_NONE => "R_SPARC_NONE".to_string(), + R_SPARC_8 => "R_SPARC_8".to_string(), + R_SPARC_16 => "R_SPARC_16".to_string(), + R_SPARC_32 => "R_SPARC_32".to_string(), + R_SPARC_DISP8 => "R_SPARC_DISP8".to_string(), + R_SPARC_DISP16 => "R_SPARC_DISP16".to_string(), + R_SPARC_DISP32 => "R_SPARC_DISP32".to_string(), + R_SPARC_WDISP30 => "R_SPARC_WDISP30".to_string(), + R_SPARC_WDISP22 => "R_SPARC_WDISP22".to_string(), + R_SPARC_HI22 => "R_SPARC_HI22".to_string(), + R_SPARC_22 => "R_SPARC_22".to_string(), + R_SPARC_13 => "R_SPARC_13".to_string(), + R_SPARC_LO10 => "R_SPARC_LO10".to_string(), + R_SPARC_GOT10 => "R_SPARC_GOT10".to_string(), + R_SPARC_GOT13 => "R_SPARC_GOT13".to_string(), + R_SPARC_GOT22 => "R_SPARC_GOT22".to_string(), + R_SPARC_PC10 => "R_SPARC_PC10".to_string(), + R_SPARC_PC22 => "R_SPARC_PC22".to_string(), + R_SPARC_WPLT30 => "R_SPARC_WPLT30".to_string(), + R_SPARC_COPY => "R_SPARC_COPY".to_string(), + R_SPARC_GLOB_DAT => "R_SPARC_GLOB_DAT".to_string(), + R_SPARC_JMP_SLOT => "R_SPARC_JMP_SLOT".to_string(), + R_SPARC_RELATIVE => "R_SPARC_RELATIVE".to_string(), + R_SPARC_UA32 => "R_SPARC_UA32".to_string(), + R_SPARC_PLT32 => "R_SPARC_PLT32".to_string(), + R_SPARC_HIPLT22 => "R_SPARC_HIPLT22".to_string(), + R_SPARC_LOPLT10 => "R_SPARC_LOPLT10".to_string(), + R_SPARC_PCPLT32 => "R_SPARC_PCPLT32".to_string(), + R_SPARC_PCPLT22 => "R_SPARC_PCPLT22".to_string(), + R_SPARC_PCPLT10 => "R_SPARC_PCPLT10".to_string(), + R_SPARC_10 => "R_SPARC_10".to_string(), + R_SPARC_11 => "R_SPARC_11".to_string(), + R_SPARC_64 => "R_SPARC_64".to_string(), + R_SPARC_OLO10 => "R_SPARC_OLO10".to_string(), + R_SPARC_HH22 => "R_SPARC_HH22".to_string(), + R_SPARC_HM10 => "R_SPARC_HM10".to_string(), + R_SPARC_LM22 => "R_SPARC_LM22".to_string(), + R_SPARC_PC_HH22 => "R_SPARC_PC_HH22".to_string(), + R_SPARC_PC_HM10 => "R_SPARC_PC_HM10".to_string(), + R_SPARC_PC_LM22 => "R_SPARC_PC_LM22".to_string(), + R_SPARC_WDISP16 => "R_SPARC_WDISP16".to_string(), + R_SPARC_WDISP19 => "R_SPARC_WDISP19".to_string(), + R_SPARC_7 => "R_SPARC_7".to_string(), + R_SPARC_5 => "R_SPARC_5".to_string(), + R_SPARC_6 => "R_SPARC_6".to_string(), + R_SPARC_DISP64 => "R_SPARC_DISP64".to_string(), + R_SPARC_PLT64 => "R_SPARC_PLT64".to_string(), + R_SPARC_HIX22 => "R_SPARC_HIX22".to_string(), + R_SPARC_LOX10 => "R_SPARC_LOX10".to_string(), + R_SPARC_H44 => "R_SPARC_H44".to_string(), + R_SPARC_M44 => "R_SPARC_M44".to_string(), + R_SPARC_L44 => "R_SPARC_L44".to_string(), + R_SPARC_REGISTER => "R_SPARC_REGISTER".to_string(), + R_SPARC_UA64 => "R_SPARC_UA64".to_string(), + R_SPARC_UA16 => "R_SPARC_UA16".to_string(), + R_SPARC_TLS_GD_HI22 => "R_SPARC_TLS_GD_HI22".to_string(), + R_SPARC_TLS_GD_LO10 => "R_SPARC_TLS_GD_LO10".to_string(), + R_SPARC_TLS_GD_ADD => "R_SPARC_TLS_GD_ADD".to_string(), + R_SPARC_TLS_GD_CALL => "R_SPARC_TLS_GD_CALL".to_string(), + R_SPARC_TLS_LDM_HI22 => "R_SPARC_TLS_LDM_HI22".to_string(), + R_SPARC_TLS_LDM_LO10 => "R_SPARC_TLS_LDM_LO10".to_string(), + R_SPARC_TLS_LDM_ADD => "R_SPARC_TLS_LDM_ADD".to_string(), + R_SPARC_TLS_LDM_CALL => "R_SPARC_TLS_LDM_CALL".to_string(), + R_SPARC_TLS_LDO_HIX22 => "R_SPARC_TLS_LDO_HIX22".to_string(), + R_SPARC_TLS_LDO_LOX10 => "R_SPARC_TLS_LDO_LOX10".to_string(), + R_SPARC_TLS_LDO_ADD => "R_SPARC_TLS_LDO_ADD".to_string(), + R_SPARC_TLS_IE_HI22 => "R_SPARC_TLS_IE_HI22".to_string(), + R_SPARC_TLS_IE_LO10 => "R_SPARC_TLS_IE_LO10".to_string(), + R_SPARC_TLS_IE_LD => "R_SPARC_TLS_IE_LD".to_string(), + R_SPARC_TLS_IE_LDX => "R_SPARC_TLS_IE_LDX".to_string(), + R_SPARC_TLS_IE_ADD => "R_SPARC_TLS_IE_ADD".to_string(), + R_SPARC_TLS_LE_HIX22 => "R_SPARC_TLS_LE_HIX22".to_string(), + R_SPARC_TLS_LE_LOX10 => "R_SPARC_TLS_LE_LOX10".to_string(), + R_SPARC_TLS_DTPMOD32 => "R_SPARC_TLS_DTPMOD32".to_string(), + R_SPARC_TLS_DTPMOD64 => "R_SPARC_TLS_DTPMOD64".to_string(), + R_SPARC_TLS_DTPOFF32 => "R_SPARC_TLS_DTPOFF32".to_string(), + R_SPARC_TLS_DTPOFF64 => "R_SPARC_TLS_DTPOFF64".to_string(), + R_SPARC_TLS_TPOFF32 => "R_SPARC_TLS_TPOFF32".to_string(), + R_SPARC_TLS_TPOFF64 => "R_SPARC_TLS_TPOFF64".to_string(), + R_SPARC_GOTDATA_HIX22 => "R_SPARC_GOTDATA_HIX22".to_string(), + R_SPARC_GOTDATA_LOX10 => "R_SPARC_GOTDATA_LOX10".to_string(), + R_SPARC_GOTDATA_OP_HIX22 => "R_SPARC_GOTDATA_OP_HIX22".to_string(), + R_SPARC_GOTDATA_OP_LOX10 => "R_SPARC_GOTDATA_OP_LOX10".to_string(), + R_SPARC_GOTDATA_OP => "R_SPARC_GOTDATA_OP".to_string(), + R_SPARC_SIZE32 => "R_SPARC_SIZE32".to_string(), + R_SPARC_JMP_IREL => "R_SPARC_JMP_IREL".to_string(), + R_SPARC_IRELATIVE => "R_SPARC_IRELATIVE".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn s390x_rel_to_string(r_type: u32) -> String { + match r_type { + R_390_NONE => "R_390_NONE".to_string(), + R_390_8 => "R_390_8".to_string(), + R_390_12 => "R_390_12".to_string(), + R_390_16 => "R_390_16".to_string(), + R_390_32 => "R_390_32".to_string(), + R_390_PC32 => "R_390_PC32".to_string(), + R_390_GOT12 => "R_390_GOT12".to_string(), + R_390_GOT32 => "R_390_GOT32".to_string(), + R_390_PLT32 => "R_390_PLT32".to_string(), + R_390_COPY => "R_390_COPY".to_string(), + R_390_GLOB_DAT => "R_390_GLOB_DAT".to_string(), + R_390_JMP_SLOT => "R_390_JMP_SLOT".to_string(), + R_390_RELATIVE => "R_390_RELATIVE".to_string(), + R_390_GOTOFF32 => "R_390_GOTOFF32".to_string(), + R_390_GOTPC => "R_390_GOTPC".to_string(), + R_390_GOT16 => "R_390_GOT16".to_string(), + R_390_PC16 => "R_390_PC16".to_string(), + R_390_PC16DBL => "R_390_PC16DBL".to_string(), + R_390_PLT16DBL => "R_390_PLT16DBL".to_string(), + R_390_PC32DBL => "R_390_PC32DBL".to_string(), + R_390_PLT32DBL => "R_390_PLT32DBL".to_string(), + R_390_GOTPCDBL => "R_390_GOTPCDBL".to_string(), + R_390_64 => "R_390_64".to_string(), + R_390_PC64 => "R_390_PC64".to_string(), + R_390_GOT64 => "R_390_GOT64".to_string(), + R_390_PLT64 => "R_390_PLT64".to_string(), + R_390_GOTENT => "R_390_GOTENT".to_string(), + R_390_GOTOFF16 => "R_390_GOTOFF16".to_string(), + R_390_GOTOFF64 => "R_390_GOTOFF64".to_string(), + R_390_GOTPLT12 => "R_390_GOTPLT12".to_string(), + R_390_GOTPLT16 => "R_390_GOTPLT16".to_string(), + R_390_GOTPLT32 => "R_390_GOTPLT32".to_string(), + R_390_GOTPLT64 => "R_390_GOTPLT64".to_string(), + R_390_GOTPLTENT => "R_390_GOTPLTENT".to_string(), + R_390_PLTOFF16 => "R_390_PLTOFF16".to_string(), + R_390_PLTOFF32 => "R_390_PLTOFF32".to_string(), + R_390_PLTOFF64 => "R_390_PLTOFF64".to_string(), + R_390_TLS_LOAD => "R_390_TLS_LOAD".to_string(), + R_390_TLS_GDCALL => "R_390_TLS_GDCALL".to_string(), + R_390_TLS_LDCALL => "R_390_TLS_LDCALL".to_string(), + R_390_TLS_GD32 => "R_390_TLS_GD32".to_string(), + R_390_TLS_GD64 => "R_390_TLS_GD64".to_string(), + R_390_TLS_GOTIE12 => "R_390_TLS_GOTIE12".to_string(), + R_390_TLS_GOTIE32 => "R_390_TLS_GOTIE32".to_string(), + R_390_TLS_GOTIE64 => "R_390_TLS_GOTIE64".to_string(), + R_390_TLS_LDM32 => "R_390_TLS_LDM32".to_string(), + R_390_TLS_LDM64 => "R_390_TLS_LDM64".to_string(), + R_390_TLS_IE32 => "R_390_TLS_IE32".to_string(), + R_390_TLS_IE64 => "R_390_TLS_IE64".to_string(), + R_390_TLS_IEENT => "R_390_TLS_IEENT".to_string(), + R_390_TLS_LE32 => "R_390_TLS_LE32".to_string(), + R_390_TLS_LE64 => "R_390_TLS_LE64".to_string(), + R_390_TLS_LDO32 => "R_390_TLS_LDO32".to_string(), + R_390_TLS_LDO64 => "R_390_TLS_LDO64".to_string(), + R_390_TLS_DTPMOD => "R_390_TLS_DTPMOD".to_string(), + R_390_TLS_DTPOFF => "R_390_TLS_DTPOFF".to_string(), + R_390_TLS_TPOFF => "R_390_TLS_TPOFF".to_string(), + R_390_20 => "R_390_20".to_string(), + R_390_GOT20 => "R_390_GOT20".to_string(), + R_390_GOTPLT20 => "R_390_GOTPLT20".to_string(), + R_390_TLS_GOTIE20 => "R_390_TLS_GOTIE20".to_string(), + R_390_IRELATIVE => "R_390_IRELATIVE".to_string(), + R_390_PC12DBL => "R_390_PC12DBL".to_string(), + R_390_PLT12DBL => "R_390_PLT12DBL".to_string(), + R_390_PC24DBL => "R_390_PC24DBL".to_string(), + R_390_PLT24DBL => "R_390_PLT24DBL".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn m68k_rel_to_string(r_type: u32) -> String { + match r_type { + R_68K_NONE => "R_68K_NONE".to_string(), + R_68K_32 => "R_68K_32".to_string(), + R_68K_16 => "R_68K_16".to_string(), + R_68K_8 => "R_68K_8".to_string(), + R_68K_PC32 => "R_68K_PC32".to_string(), + R_68K_PC16 => "R_68K_PC16".to_string(), + R_68K_PC8 => "R_68K_PC8".to_string(), + R_68K_GOTPCREL32 => "R_68K_GOTPCREL32".to_string(), + R_68K_GOTPCREL16 => "R_68K_GOTPCREL16".to_string(), + R_68K_GOTPCREL8 => "R_68K_GOTPCREL8".to_string(), + R_68K_GOTOFF32 => "R_68K_GOTOFF32".to_string(), + R_68K_GOTOFF16 => "R_68K_GOTOFF16".to_string(), + R_68K_GOTOFF8 => "R_68K_GOTOFF8".to_string(), + R_68K_PLT32 => "R_68K_PLT32".to_string(), + R_68K_PLT16 => "R_68K_PLT16".to_string(), + R_68K_PLT8 => "R_68K_PLT8".to_string(), + R_68K_PLTOFF32 => "R_68K_PLTOFF32".to_string(), + R_68K_PLTOFF16 => "R_68K_PLTOFF16".to_string(), + R_68K_PLTOFF8 => "R_68K_PLTOFF8".to_string(), + R_68K_COPY => "R_68K_COPY".to_string(), + R_68K_GLOB_DAT => "R_68K_GLOB_DAT".to_string(), + R_68K_JMP_SLOT => "R_68K_JMP_SLOT".to_string(), + R_68K_RELATIVE => "R_68K_RELATIVE".to_string(), + R_68K_TLS_GD32 => "R_68K_TLS_GD32".to_string(), + R_68K_TLS_GD16 => "R_68K_TLS_GD16".to_string(), + R_68K_TLS_GD8 => "R_68K_TLS_GD8".to_string(), + R_68K_TLS_LDM32 => "R_68K_TLS_LDM32".to_string(), + R_68K_TLS_LDM16 => "R_68K_TLS_LDM16".to_string(), + R_68K_TLS_LDM8 => "R_68K_TLS_LDM8".to_string(), + R_68K_TLS_LDO32 => "R_68K_TLS_LDO32".to_string(), + R_68K_TLS_LDO16 => "R_68K_TLS_LDO16".to_string(), + R_68K_TLS_LDO8 => "R_68K_TLS_LDO8".to_string(), + R_68K_TLS_IE32 => "R_68K_TLS_IE32".to_string(), + R_68K_TLS_IE16 => "R_68K_TLS_IE16".to_string(), + R_68K_TLS_IE8 => "R_68K_TLS_IE8".to_string(), + R_68K_TLS_LE32 => "R_68K_TLS_LE32".to_string(), + R_68K_TLS_LE16 => "R_68K_TLS_LE16".to_string(), + R_68K_TLS_LE8 => "R_68K_TLS_LE8".to_string(), + R_68K_TLS_DTPMOD32 => "R_68K_TLS_DTPMOD32".to_string(), + R_68K_TLS_DTPREL32 => "R_68K_TLS_DTPREL32".to_string(), + R_68K_TLS_TPREL32 => "R_68K_TLS_TPREL32".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn sh4_rel_to_string(r_type: u32) -> String { + match r_type { + R_SH_NONE => "R_SH_NONE".to_string(), + R_SH_DIR32 => "R_SH_DIR32".to_string(), + R_SH_REL32 => "R_SH_REL32".to_string(), + R_SH_DIR8WPN => "R_SH_DIR8WPN".to_string(), + R_SH_IND12W => "R_SH_IND12W".to_string(), + R_SH_DIR8WPL => "R_SH_DIR8WPL".to_string(), + R_SH_DIR8WPZ => "R_SH_DIR8WPZ".to_string(), + R_SH_DIR8BP => "R_SH_DIR8BP".to_string(), + R_SH_DIR8W => "R_SH_DIR8W".to_string(), + R_SH_DIR8L => "R_SH_DIR8L".to_string(), + R_SH_TLS_GD_32 => "R_SH_TLS_GD_32".to_string(), + R_SH_TLS_LD_32 => "R_SH_TLS_LD_32".to_string(), + R_SH_TLS_LDO_32 => "R_SH_TLS_LDO_32".to_string(), + R_SH_TLS_IE_32 => "R_SH_TLS_IE_32".to_string(), + R_SH_TLS_LE_32 => "R_SH_TLS_LE_32".to_string(), + R_SH_TLS_DTPMOD32 => "R_SH_TLS_DTPMOD32".to_string(), + R_SH_TLS_DTPOFF32 => "R_SH_TLS_DTPOFF32".to_string(), + R_SH_TLS_TPOFF32 => "R_SH_TLS_TPOFF32".to_string(), + R_SH_GOT32 => "R_SH_GOT32".to_string(), + R_SH_PLT32 => "R_SH_PLT32".to_string(), + R_SH_COPY => "R_SH_COPY".to_string(), + R_SH_GLOB_DAT => "R_SH_GLOB_DAT".to_string(), + R_SH_JMP_SLOT => "R_SH_JMP_SLOT".to_string(), + R_SH_RELATIVE => "R_SH_RELATIVE".to_string(), + R_SH_GOTOFF => "R_SH_GOTOFF".to_string(), + R_SH_GOTPC => "R_SH_GOTPC".to_string(), + R_SH_GOTPLT32 => "R_SH_GOTPLT32".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} + +pub fn loongarch_rel_to_string(r_type: u32) -> String { + match r_type { + R_LARCH_NONE => "R_LARCH_NONE".to_string(), + R_LARCH_32 => "R_LARCH_32".to_string(), + R_LARCH_64 => "R_LARCH_64".to_string(), + R_LARCH_RELATIVE => "R_LARCH_RELATIVE".to_string(), + R_LARCH_COPY => "R_LARCH_COPY".to_string(), + R_LARCH_JUMP_SLOT => "R_LARCH_JUMP_SLOT".to_string(), + R_LARCH_TLS_DTPMOD32 => "R_LARCH_TLS_DTPMOD32".to_string(), + R_LARCH_TLS_DTPMOD64 => "R_LARCH_TLS_DTPMOD64".to_string(), + R_LARCH_TLS_DTPREL32 => "R_LARCH_TLS_DTPREL32".to_string(), + R_LARCH_TLS_DTPREL64 => "R_LARCH_TLS_DTPREL64".to_string(), + R_LARCH_TLS_TPREL32 => "R_LARCH_TLS_TPREL32".to_string(), + R_LARCH_TLS_TPREL64 => "R_LARCH_TLS_TPREL64".to_string(), + R_LARCH_IRELATIVE => "R_LARCH_IRELATIVE".to_string(), + R_LARCH_TLS_DESC32 => "R_LARCH_TLS_DESC32".to_string(), + R_LARCH_TLS_DESC64 => "R_LARCH_TLS_DESC64".to_string(), + R_LARCH_MARK_LA => "R_LARCH_MARK_LA".to_string(), + R_LARCH_MARK_PCREL => "R_LARCH_MARK_PCREL".to_string(), + R_LARCH_SOP_PUSH_PCREL => "R_LARCH_SOP_PUSH_PCREL".to_string(), + R_LARCH_SOP_PUSH_ABSOLUTE => "R_LARCH_SOP_PUSH_ABSOLUTE".to_string(), + R_LARCH_SOP_PUSH_DUP => "R_LARCH_SOP_PUSH_DUP".to_string(), + R_LARCH_SOP_PUSH_GPREL => "R_LARCH_SOP_PUSH_GPREL".to_string(), + R_LARCH_SOP_PUSH_TLS_TPREL => "R_LARCH_SOP_PUSH_TLS_TPREL".to_string(), + R_LARCH_SOP_PUSH_TLS_GOT => "R_LARCH_SOP_PUSH_TLS_GOT".to_string(), + R_LARCH_SOP_PUSH_TLS_GD => "R_LARCH_SOP_PUSH_TLS_GD".to_string(), + R_LARCH_SOP_PUSH_PLT_PCREL => "R_LARCH_SOP_PUSH_PLT_PCREL".to_string(), + R_LARCH_SOP_ASSERT => "R_LARCH_SOP_ASSERT".to_string(), + R_LARCH_SOP_NOT => "R_LARCH_SOP_NOT".to_string(), + R_LARCH_SOP_SUB => "R_LARCH_SOP_SUB".to_string(), + R_LARCH_SOP_SL => "R_LARCH_SOP_SL".to_string(), + R_LARCH_SOP_SR => "R_LARCH_SOP_SR".to_string(), + R_LARCH_SOP_ADD => "R_LARCH_SOP_ADD".to_string(), + R_LARCH_SOP_AND => "R_LARCH_SOP_AND".to_string(), + R_LARCH_SOP_IF_ELSE => "R_LARCH_SOP_IF_ELSE".to_string(), + R_LARCH_SOP_POP_32_S_10_5 => "R_LARCH_SOP_POP_32_S_10_5".to_string(), + R_LARCH_SOP_POP_32_U_10_12 => "R_LARCH_SOP_POP_32_U_10_12".to_string(), + R_LARCH_SOP_POP_32_S_10_12 => "R_LARCH_SOP_POP_32_S_10_12".to_string(), + R_LARCH_SOP_POP_32_S_10_16 => "R_LARCH_SOP_POP_32_S_10_16".to_string(), + R_LARCH_SOP_POP_32_S_10_16_S2 => "R_LARCH_SOP_POP_32_S_10_16_S2".to_string(), + R_LARCH_SOP_POP_32_S_5_20 => "R_LARCH_SOP_POP_32_S_5_20".to_string(), + R_LARCH_SOP_POP_32_S_0_5_10_16_S2 => "R_LARCH_SOP_POP_32_S_0_5_10_16_S2".to_string(), + R_LARCH_SOP_POP_32_S_0_10_10_16_S2 => "R_LARCH_SOP_POP_32_S_0_10_10_16_S2".to_string(), + R_LARCH_SOP_POP_32_U => "R_LARCH_SOP_POP_32_U".to_string(), + R_LARCH_ADD8 => "R_LARCH_ADD8".to_string(), + R_LARCH_ADD16 => "R_LARCH_ADD16".to_string(), + R_LARCH_ADD24 => "R_LARCH_ADD24".to_string(), + R_LARCH_ADD32 => "R_LARCH_ADD32".to_string(), + R_LARCH_ADD64 => "R_LARCH_ADD64".to_string(), + R_LARCH_SUB8 => "R_LARCH_SUB8".to_string(), + R_LARCH_SUB16 => "R_LARCH_SUB16".to_string(), + R_LARCH_SUB24 => "R_LARCH_SUB24".to_string(), + R_LARCH_SUB32 => "R_LARCH_SUB32".to_string(), + R_LARCH_SUB64 => "R_LARCH_SUB64".to_string(), + R_LARCH_GNU_VTINHERIT => "R_LARCH_GNU_VTINHERIT".to_string(), + R_LARCH_GNU_VTENTRY => "R_LARCH_GNU_VTENTRY".to_string(), + R_LARCH_B16 => "R_LARCH_B16".to_string(), + R_LARCH_B21 => "R_LARCH_B21".to_string(), + R_LARCH_B26 => "R_LARCH_B26".to_string(), + R_LARCH_ABS_HI20 => "R_LARCH_ABS_HI20".to_string(), + R_LARCH_ABS_LO12 => "R_LARCH_ABS_LO12".to_string(), + R_LARCH_ABS64_LO20 => "R_LARCH_ABS64_LO20".to_string(), + R_LARCH_ABS64_HI12 => "R_LARCH_ABS64_HI12".to_string(), + R_LARCH_PCALA_HI20 => "R_LARCH_PCALA_HI20".to_string(), + R_LARCH_PCALA_LO12 => "R_LARCH_PCALA_LO12".to_string(), + R_LARCH_PCALA64_LO20 => "R_LARCH_PCALA64_LO20".to_string(), + R_LARCH_PCALA64_HI12 => "R_LARCH_PCALA64_HI12".to_string(), + R_LARCH_GOT_PC_HI20 => "R_LARCH_GOT_PC_HI20".to_string(), + R_LARCH_GOT_PC_LO12 => "R_LARCH_GOT_PC_LO12".to_string(), + R_LARCH_GOT64_PC_LO20 => "R_LARCH_GOT64_PC_LO20".to_string(), + R_LARCH_GOT64_PC_HI12 => "R_LARCH_GOT64_PC_HI12".to_string(), + R_LARCH_GOT_HI20 => "R_LARCH_GOT_HI20".to_string(), + R_LARCH_GOT_LO12 => "R_LARCH_GOT_LO12".to_string(), + R_LARCH_GOT64_LO20 => "R_LARCH_GOT64_LO20".to_string(), + R_LARCH_GOT64_HI12 => "R_LARCH_GOT64_HI12".to_string(), + R_LARCH_TLS_LE_HI20 => "R_LARCH_TLS_LE_HI20".to_string(), + R_LARCH_TLS_LE_LO12 => "R_LARCH_TLS_LE_LO12".to_string(), + R_LARCH_TLS_LE64_LO20 => "R_LARCH_TLS_LE64_LO20".to_string(), + R_LARCH_TLS_LE64_HI12 => "R_LARCH_TLS_LE64_HI12".to_string(), + R_LARCH_TLS_IE_PC_HI20 => "R_LARCH_TLS_IE_PC_HI20".to_string(), + R_LARCH_TLS_IE_PC_LO12 => "R_LARCH_TLS_IE_PC_LO12".to_string(), + R_LARCH_TLS_IE64_PC_LO20 => "R_LARCH_TLS_IE64_PC_LO20".to_string(), + R_LARCH_TLS_IE64_PC_HI12 => "R_LARCH_TLS_IE64_PC_HI12".to_string(), + R_LARCH_TLS_IE_HI20 => "R_LARCH_TLS_IE_HI20".to_string(), + R_LARCH_TLS_IE_LO12 => "R_LARCH_TLS_IE_LO12".to_string(), + R_LARCH_TLS_IE64_LO20 => "R_LARCH_TLS_IE64_LO20".to_string(), + R_LARCH_TLS_IE64_HI12 => "R_LARCH_TLS_IE64_HI12".to_string(), + R_LARCH_TLS_LD_PC_HI20 => "R_LARCH_TLS_LD_PC_HI20".to_string(), + R_LARCH_TLS_LD_HI20 => "R_LARCH_TLS_LD_HI20".to_string(), + R_LARCH_TLS_GD_PC_HI20 => "R_LARCH_TLS_GD_PC_HI20".to_string(), + R_LARCH_TLS_GD_HI20 => "R_LARCH_TLS_GD_HI20".to_string(), + R_LARCH_32_PCREL => "R_LARCH_32_PCREL".to_string(), + R_LARCH_RELAX => "R_LARCH_RELAX".to_string(), + R_LARCH_DELETE => "R_LARCH_DELETE".to_string(), + R_LARCH_ALIGN => "R_LARCH_ALIGN".to_string(), + R_LARCH_PCREL20_S2 => "R_LARCH_PCREL20_S2".to_string(), + R_LARCH_CFA => "R_LARCH_CFA".to_string(), + R_LARCH_ADD6 => "R_LARCH_ADD6".to_string(), + R_LARCH_SUB6 => "R_LARCH_SUB6".to_string(), + R_LARCH_ADD_ULEB128 => "R_LARCH_ADD_ULEB128".to_string(), + R_LARCH_SUB_ULEB128 => "R_LARCH_SUB_ULEB128".to_string(), + R_LARCH_64_PCREL => "R_LARCH_64_PCREL".to_string(), + R_LARCH_CALL36 => "R_LARCH_CALL36".to_string(), + R_LARCH_TLS_DESC_PC_HI20 => "R_LARCH_TLS_DESC_PC_HI20".to_string(), + R_LARCH_TLS_DESC_PC_LO12 => "R_LARCH_TLS_DESC_PC_LO12".to_string(), + R_LARCH_TLS_DESC64_PC_LO20 => "R_LARCH_TLS_DESC64_PC_LO20".to_string(), + R_LARCH_TLS_DESC64_PC_HI12 => "R_LARCH_TLS_DESC64_PC_HI12".to_string(), + R_LARCH_TLS_DESC_HI20 => "R_LARCH_TLS_DESC_HI20".to_string(), + R_LARCH_TLS_DESC_LO12 => "R_LARCH_TLS_DESC_LO12".to_string(), + R_LARCH_TLS_DESC64_LO20 => "R_LARCH_TLS_DESC64_LO20".to_string(), + R_LARCH_TLS_DESC64_HI12 => "R_LARCH_TLS_DESC64_HI12".to_string(), + R_LARCH_TLS_DESC_LD => "R_LARCH_TLS_DESC_LD".to_string(), + R_LARCH_TLS_DESC_CALL => "R_LARCH_TLS_DESC_CALL".to_string(), + R_LARCH_TLS_LE_HI20_R => "R_LARCH_TLS_LE_HI20_R".to_string(), + R_LARCH_TLS_LE_ADD_R => "R_LARCH_TLS_LE_ADD_R".to_string(), + R_LARCH_TLS_LE_LO12_R => "R_LARCH_TLS_LE_LO12_R".to_string(), + R_LARCH_TLS_LD_PCREL20_S2 => "R_LARCH_TLS_LD_PCREL20_S2".to_string(), + R_LARCH_TLS_GD_PCREL20_S2 => "R_LARCH_TLS_GD_PCREL20_S2".to_string(), + R_LARCH_TLS_DESC_PCREL20_S2 => "R_LARCH_TLS_DESC_PCREL20_S2".to_string(), + _ => format!("unknown (0x{r_type:x})"), + } +} diff --git a/src/filetype.rs b/src/filetype.rs new file mode 100644 index 000000000..dcb3d9a81 --- /dev/null +++ b/src/filetype.rs @@ -0,0 +1,281 @@ +//! Input file classification. + +use crate::arch::{self, TargetInfo}; +use crate::archive; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::mapped_file::MappedFile; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FileType { + Unknown, + Empty, + ElfObj, + ElfDso, + Ar, + ThinAr, + Text, + GccLtoObj, + LlvmBitcode, +} + +fn is_text_file(data: &[u8]) -> bool { + let is_text = |c: u8| c.is_ascii_graphic() || c == b' ' || c == b'\n' || c == b'\t'; + data.len() >= 4 && data[..4].iter().all(|&c| is_text(c)) +} + +/// Whether an ELF relocatable object is really a GCC LTO object. +fn is_gcc_lto_obj(data: &[u8], has_gcc_plugin: bool) -> bool { + let Some(ehdr) = data.get(..ElfEhdr::size::()).map(ElfEhdr::parse::) else { + return false; + }; + let shoff = ehdr.e_shoff as usize; + let shdr_size = ElfShdr::size::(); + let Some(shdr_bytes) = data.get(shoff..shoff + ehdr.e_shnum as usize * shdr_size) else { + return false; + }; + let shdrs = ShdrTable::in_file(shdr_bytes, RecordLayout::of::()); + let Some(first) = shdrs.get(0) else { + return false; + }; + + // e_shstrndx is a 16-bit field. If .shstrtab's section index is too + // large, the actual number is stored in sh_link of the first header. + let shstrtab_idx = if ehdr.e_shstrndx as u32 == SHN_XINDEX { + first.sh_link as usize + } else { + ehdr.e_shstrndx as usize + }; + let shstrtab_offset = if has_gcc_plugin { + shdrs.get(shstrtab_idx).map(|shdr| shdr.sh_offset as usize) + } else { + None + }; + + for i in 0..shdrs.len() { + // GCC FAT LTO objects contain both regular ELF sections and + // GCC-specific LTO sections, so that they can be linked as LTO + // objects if the plugin is available and as regular objects + // otherwise. They can be identified by a `.gnu.lto_.symtab.` section. + if let Some(offset) = shstrtab_offset { + let name = crate::util::cstr_at(data, offset + shdrs.sh_name_in::(i) as usize); + if name.starts_with(b".gnu.lto_.symtab.") { + return true; + } + } + + if shdrs.sh_type_in::(i) != SHT_SYMTAB { + continue; + } + let shdr = shdrs.at_in::(i); + + // A non-FAT GCC LTO object contains only section symbols followed + // by a common symbol named `__gnu_lto_slim` (or `__gnu_lto_v1` in + // older releases). + let off = shdr.sh_offset as usize; + let Some(bytes) = data.get(off..off + shdr.sh_size as usize) else { + return false; + }; + let syms = bytes + .chunks_exact(ElfSym::size::()) + .map(ElfSym::parse::); + let skip = |ty: u32| ty == STT_NOTYPE || ty == STT_FILE || ty == STT_SECTION; + + if let Some(sym) = syms.skip(1).find(|s| !skip(s.st_type())) { + if sym.st_shndx as u32 == SHN_COMMON { + let Some(strtab) = shdrs.get(shdr.sh_link as usize) else { + return false; + }; + let name = + crate::util::cstr_at(data, strtab.sh_offset as usize + sym.st_name as usize); + if name.starts_with(b"__gnu_lto_") { + return true; + } + } + } + break; + } + false +} + +/// Classifies a file by its contents. +pub fn get_file_type(plugin: &str, mf: &MappedFile) -> FileType { + let data = mf.data(); + if data.is_empty() { + return FileType::Empty; + } + + // GCC FAT LTO objects can be linked as regular ELF objects. If the + // active plugin is LLVM's, treat them as regular objects so that we + // fall back to native code instead of routing them through GCC LTO. + let has_gcc_plugin = !plugin.is_empty() && !plugin.contains("LLVMgold."); + + if data.starts_with(b"\x7fELF") && data.len() >= 20 { + let is_le = data[EI_DATA as usize] == ELFDATA2LSB as u8; + let is_32 = data[EI_CLASS as usize] == ELFCLASS32 as u8; + let e_type = if is_le { + u16::from_le_bytes([data[16], data[17]]) as u32 + } else { + u16::from_be_bytes([data[16], data[17]]) as u32 + }; + + if e_type == ET_REL { + let is_lto = match (is_le, is_32) { + (true, true) => is_gcc_lto_obj::(data, has_gcc_plugin), + (true, false) => is_gcc_lto_obj::(data, has_gcc_plugin), + (false, true) => is_gcc_lto_obj::(data, has_gcc_plugin), + (false, false) => is_gcc_lto_obj::(data, has_gcc_plugin), + }; + return if is_lto { + FileType::GccLtoObj + } else { + FileType::ElfObj + }; + } + if e_type == ET_DYN { + return FileType::ElfDso; + } + return FileType::Unknown; + } + + if data.starts_with(b"!\n") { + return FileType::Ar; + } + if data.starts_with(b"!\n") { + return FileType::ThinAr; + } + if is_text_file(data) { + return FileType::Text; + } + if data.starts_with(b"\xde\xc0\x17\x0b") || data.starts_with(b"BC\xc0\xde") { + return FileType::LlvmBitcode; + } + FileType::Unknown +} + +/// Returns the target name of an ELF file, or `None` if its machine type +/// is not one we recognize. +pub fn get_elf_target(data: &[u8]) -> Option<&'static str> { + if data.len() < 52 { + return None; + } + let is_le = data[EI_DATA as usize] == ELFDATA2LSB as u8; + let is_64 = data[EI_CLASS as usize] == ELFCLASS64 as u8; + let read_u16 = |off: usize| { + let b = [data[off], data[off + 1]]; + if is_le { + u16::from_le_bytes(b) + } else { + u16::from_be_bytes(b) + } + }; + let read_u32 = |off: usize| { + let b = [data[off], data[off + 1], data[off + 2], data[off + 3]]; + if is_le { + u32::from_le_bytes(b) + } else { + u32::from_be_bytes(b) + } + }; + let e_machine = read_u16(18) as u32; + // e_flags follows the three word-sized fields after e_version. + let e_flags = read_u32(if is_64 { 48 } else { 36 }); + + let name = match e_machine { + EM_386 => "i386", + EM_X86_64 => "x86_64", + EM_ARM => { + if is_le { + "arm32" + } else { + "arm32be" + } + } + EM_AARCH64 => { + if is_le { + "arm64" + } else { + "arm64be" + } + } + EM_RISCV => match (is_le, is_64) { + (true, true) => "riscv64", + (true, false) => "riscv32", + (false, true) => "riscv64be", + (false, false) => "riscv32be", + }, + EM_PPC => "ppc32", + EM_PPC64 => { + // ELFv1 is big-endian and ELFv2 is little-endian by convention, + // but that's not a rule; musl for example uses ELFv2 on + // big-endian too. We support only the usual combinations, so + // treat the others as unrecognizable rather than silently + // linking them against the wrong ABI. + let abi = e_flags & EF_PPC64_ABI; + if !is_le && (abi == 0 || abi == 1) { + "ppc64v1" + } else if is_le && (abi == 0 || abi == 2) { + "ppc64v2" + } else { + return None; + } + } + EM_S390X => "s390x", + EM_SPARC64 => "sparc64", + EM_68K => "m68k", + EM_SH => { + if is_le { + "sh4" + } else { + "sh4be" + } + } + EM_LOONGARCH => { + if is_64 { + "loongarch64" + } else { + "loongarch32" + } + } + _ => return None, + }; + Some(name) +} + +/// Reads the beginning of a file and returns the target it was compiled +/// for, looking inside archives if necessary. +pub fn get_machine_type( + diag: &Diagnostics, + plugin: &str, + mf: &'static MappedFile, + script_target: impl FnOnce() -> Option<&'static str>, +) -> Option<&'static str> { + match get_file_type(plugin, mf) { + FileType::ElfObj | FileType::ElfDso | FileType::GccLtoObj => get_elf_target(mf.data()), + FileType::Ar => archive::read_fat_archive_members(diag, mf) + .into_iter() + .find(|child| { + matches!( + get_file_type(plugin, child), + FileType::ElfObj | FileType::GccLtoObj + ) + }) + .and_then(|child| get_elf_target(child.data())), + FileType::ThinAr => archive::read_thin_archive_members(diag, mf) + .into_iter() + .find(|child| { + matches!( + get_file_type(plugin, child), + FileType::ElfObj | FileType::GccLtoObj + ) + }) + .and_then(|child| get_elf_target(child.data())), + FileType::Text => script_target(), + _ => None, + } +} + +/// Looks up the description of a target by name. +pub fn target_info(name: &str) -> Option<&'static TargetInfo> { + arch::TARGETS.iter().find(|t| t.name == name) +} diff --git a/src/gc_sections.rs b/src/gc_sections.rs new file mode 100644 index 000000000..b29f93963 --- /dev/null +++ b/src/gc_sections.rs @@ -0,0 +1,311 @@ +//! A mark-sweep garbage collector for `--gc-sections`: sections are +//! vertices, relocations are edges, and anything reachable from a root +//! section is kept. + +use std::collections::HashMap; +use std::io::Write; + +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::context::Context; +use crate::elf::*; +use crate::fatal; +use crate::input_files::{FileId, ObjectFile}; +use crate::input_sections::{InputSection, SectionRef}; +use crate::symbol::{is_c_identifier, SymbolId}; + +fn should_keep(file: &ObjectFile, isec: &InputSection) -> bool { + let ty = isec.sh_type(file); + let flags = isec.sh_flags as u32; + let name: &[u8] = isec.name(file); + if E::FAMILY == Family::Ppc32 && name == b".got2" { + return true; + } + flags & SHF_GNU_RETAIN != 0 + || matches!( + ty, + SHT_NOTE | SHT_INIT_ARRAY | SHT_FINI_ARRAY | SHT_PREINIT_ARRAY + ) + || name.starts_with(b".ctors") + || name.starts_with(b".dtors") + || name.starts_with(b".init") + || name.starts_with(b".fini") +} + +/// Sections with C identifier names can be referenced through the +/// synthesized `__start_`/`__stop_` symbols, so a reference +/// to such a symbol keeps every section of that name alive. +type StartStopMap<'a> = HashMap<&'static [u8], Vec<&'a InputSection>>; + +fn build_start_stop_map<'a, E: Arch>(ctx: &'a Context) -> StartStopMap<'a> { + let per_file: Vec> = ctx + .objs + .par_iter() + .map(|file| { + file.input_sections() + .filter(|isec| { + isec.is_alive() && isec.is_alloc() && is_c_identifier(isec.name(file)) + }) + .map(|isec| { + let name: &'static [u8] = isec.name(file); + (name, isec) + }) + .collect() + }) + .collect(); + let mut map: StartStopMap<'a> = HashMap::new(); + for (name, isec) in per_file.into_iter().flatten() { + map.entry(name).or_default().push(isec); + } + map +} + +#[inline] +fn mark_section(isec: &InputSection) -> bool { + isec.is_alive() && isec.visit() +} + +fn collect_root_set<'a, E: Arch>(ctx: &'a Context) -> Vec<&'a InputSection> { + let _t = ctx.timer("collect_root_set"); + + let enqueue_symbol = |id: SymbolId, out: &mut Vec<&'a InputSection>| { + let sym = &ctx.symbols[id]; + if let Some(frag) = sym.fragment() { + ctx.fragment(frag).set_alive(); + } else if let Some(isec) = sym.input_section_ref() { + if mark_section(isec) { + out.push(isec); + } + } + }; + + ctx.objs + .par_iter() + .enumerate() + .flat_map_iter(|(fi, file)| { + let file_id = FileId::Obj(crate::input_files::ObjId(fi as u32)); + let mut roots = Vec::new(); + + // Sections not subject to garbage collection. Only SHF_ALLOC + // sections are discarded; use `strip` for the rest. + for isec in file.input_sections() { + if !isec.is_alive() { + continue; + } + if !isec.is_alloc() { + isec.set_visited(); + continue; + } + if should_keep::(file, isec) && mark_section(isec) { + roots.push(isec); + } + } + + // Sections containing GC roots or exported symbols. + for &id in &file.base.symbols { + let sym = &ctx.symbols[id]; + if sym.file() == Some(file_id) && (sym.gc_root() || sym.is_exported()) { + enqueue_symbol(id, &mut roots); + } + } + + // CIEs are always kept along with everything they reference. + for cie in &file.cies { + for rel in cie.rels::(file) { + enqueue_symbol(file.base.symbols[rel.r_sym as usize], &mut roots); + } + } + roots + }) + .collect() +} + +#[inline] +fn start_stop_name(name: &[u8]) -> Option<&[u8]> { + name.strip_prefix(b"__start_") + .or_else(|| name.strip_prefix(b"__stop_")) +} + +fn visit_section<'scope, E: Arch>( + ctx: &'scope Context, + isec: &'scope InputSection, + depth: usize, + map: &'scope StartStopMap<'scope>, + scope: &rayon::Scope<'scope>, + next: &mut Vec<&'scope InputSection>, +) { + let file = &ctx.objs[isec.file.index()]; + debug_assert!(isec.is_visited()); + + // Mark a section alive. For better performance, we don't add work + // to the pool too often. + let mut mark = |target: &'scope InputSection| { + if mark_section(target) { + if depth < 3 { + visit_section(ctx, target, depth + 1, map, scope, next); + } else { + next.push(target); + } + } + }; + + // Keep the .eh_frame records describing this section's functions. + for fde in isec.fdes(file) { + for rel in fde.rels::(file).iter().skip(1) { + if let Some(target) = + ctx.symbols[file.base.symbols[rel.r_sym as usize]].input_section_ref() + { + mark(target); + } + } + } + + for rel in isec.rels::(file) { + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if let Some(FileId::Dso(dso)) = sym.file() { + ctx.dsos[dso.index()].base.set_reachable(true); + continue; + } + if let Some(frag) = sym.fragment() { + ctx.fragment(frag).set_alive(); + continue; + } + if let Some(target) = sym.input_section_ref() { + mark(target); + } + + // A reference to __start_ or __stop_ keeps every + // section named alive. + if let Some(name) = start_stop_name(sym.name()) { + if let Some(sections) = map.get(name) { + // Mark and visit the sections with a nested parallel loop. + // As in mark() below, a section added to the work pool has + // already been marked, so a task must visit it unconditionally. + sections.par_chunks(GC_BATCH).for_each(|sections| { + let mut found = Vec::with_capacity(sections.len()); + for &target in sections { + if mark_section(target) { + found.push(target); + } + } + if !found.is_empty() { + visit_batch(ctx, &found, map, scope); + } + }); + } + } + } + + if E::FAMILY == Family::Arm32 { + if let Some(exidx) = isec.exidx() { + mark(file.section_at(exidx)); + } + } +} + +const GC_BATCH: usize = 16; + +/// Visits marked sections and publishes newly found work in batches. Rayon +/// tasks are heavier than TBB feeder items, so a small batch preserves the +/// feeder's dynamic load balancing while amortizing task allocation. +fn visit_batch<'scope, E: Arch>( + ctx: &'scope Context, + batch: &[&'scope InputSection], + map: &'scope StartStopMap<'scope>, + scope: &rayon::Scope<'scope>, +) { + let mut next = Vec::with_capacity(GC_BATCH); + for &isec in batch { + visit_section(ctx, isec, 0, map, scope, &mut next); + if next.len() >= GC_BATCH { + let found = std::mem::replace(&mut next, Vec::with_capacity(GC_BATCH)); + scope.spawn(move |scope| visit_batch(ctx, &found, map, scope)); + } + } + if !next.is_empty() { + scope.spawn(move |scope| visit_batch(ctx, &next, map, scope)); + } +} + +// Mark all reachable sections. +fn mark<'a, E: Arch>(ctx: &'a Context, roots: Vec<&'a InputSection>, map: &'a StartStopMap<'a>) { + let _t = ctx.timer("mark"); + + rayon::scope(|scope| { + roots + .par_chunks(GC_BATCH) + .for_each(|batch| visit_batch(ctx, batch, map, scope)); + }); +} + +fn sweep(ctx: &Context) { + let _t = ctx.timer("sweep"); + let removed: Vec> = ctx + .objs + .par_iter() + .map(|file| { + let mut removed = Vec::new(); + for isec in file.input_sections() { + if isec.is_alive() && !isec.is_visited() { + file.kill_section(isec.shndx as usize); + removed.push(SectionRef { + file: isec.file, + shndx: isec.shndx, + }); + } + } + removed + }) + .collect(); + + let path = &ctx.args.print_gc_sections; + if path.is_empty() { + return; + } + let mut out = String::new(); + for r in removed.iter().flatten() { + out.push_str(&format!( + "removing unused section {}\n", + ctx.section_display(*r) + )); + } + out.push_str("GC saved 0 bytes\n"); + if path == "-" { + let _ = std::io::stdout().write_all(out.as_bytes()); + } else { + std::fs::write(path, out) + .unwrap_or_else(|e| fatal!(ctx, "--print-gc-sections: cannot open {path}: {e}")); + } +} + +pub fn gc_sections(ctx: &mut Context) { + let _t = ctx.timer("gc"); + + for file in &ctx.dsos { + if file.base.as_needed { + file.base.set_reachable(false); + } + } + for name in ctx + .args + .undefined + .clone() + .iter() + .chain(ctx.args.require_defined.clone().iter()) + { + let id = ctx.get_symbol(name.as_bytes()); + if let Some(FileId::Dso(dso)) = ctx.symbols[id].file() { + ctx.dsos[dso.index()].base.set_reachable(true); + } + } + + let roots = collect_root_set(ctx); + let t = ctx.timer("build_start_stop_map"); + let map = build_start_stop_map(ctx); + drop(t); + mark(ctx, roots, &map); + sweep(ctx); + + crate::passes::remove_unreachable_dsos(ctx); +} diff --git a/src/gdb_index.rs b/src/gdb_index.rs new file mode 100644 index 000000000..1bb2df3e2 --- /dev/null +++ b/src/gdb_index.rs @@ -0,0 +1,1388 @@ +//! `.gdb_index` generation for `--gdb-index`. +//! +//! `.gdb_index` speeds up gdb start-up. It maps the names of functions, +//! variables and types to the units defining them, and address ranges to +//! compilation units, so that gdb can find the unit for a name or a PC +//! without reading all of the debug info first. The format is described +//! at https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html. +//! +//! Names come from `.debug_gnu_pubnames` and `.debug_gnu_pubtypes`, which +//! compilers emit under `-ggnu-pubnames`; those sections are consumed +//! rather than copied to the output. Address ranges come from the +//! compilation units in `.debug_info`, possibly through `.debug_ranges`, +//! `.debug_rnglists` and `.debug_addr`, and are only known once +//! relocations have been applied. The index is therefore assembled in +//! three stages: units and names are read from the inputs, the name +//! table is built once the order of units in the output is fixed, and +//! the address table is written after everything else, at the very end +//! of the file. + +// DWARF constants keep the spelling of the specification. +#![allow(non_upper_case_globals)] + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::ChunkId; +use crate::context::Context; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::fatal; +use crate::output_file::{split_at_offsets, OutputFile}; +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use crate::util::concurrent_map::{ConcurrentMap, EntryId, FrozenMap}; +use crate::util::hyperloglog::HyperLogLog; +use crate::util::read_uleb; +use crate::util::timer::Timer; + +/// A public name and its GNU kind before the name is interned. +#[derive(Clone, Copy)] +#[repr(C)] +struct NameType { + // The low byte stores the type. Hash collisions, including collisions in + // the discarded high byte, are resolved by comparing the complete name. + hash_and_type: u64, + name: usize, +} + +impl NameType { + fn new(hash: u64, kind: u8, name: &'static [u8]) -> NameType { + NameType { + hash_and_type: (hash << 8) | kind as u64, + name: name.as_ptr() as usize, + } + } + + fn hash(self) -> u64 { + self.hash_and_type >> 8 + } + + fn kind(self) -> u8 { + self.hash_and_type as u8 + } + + fn same_name(self, other: NameType) -> bool { + self.hash_and_type == other.hash_and_type + // SAFETY: public names point into NUL-terminated debug sections + // that remain live for the complete link. + && unsafe { + libc::strcmp( + (self.name as *const u8).cast(), + (other.name as *const u8).cast(), + ) == 0 + } + } +} + +/// The interned name's map entry and reserved slot in its type vector. +#[derive(Clone, Copy)] +#[repr(C)] +struct IndexedName { + entry: EntryId, + type_vector_idx: u32, + kind: u8, +} + +/// A name record is read as a NameType and replaced with an IndexedName after +/// the name has been interned. The union avoids retaining both forms for the +/// millions of public names in a large debug link. +#[derive(Clone, Copy)] +union NameRecord { + nametype: NameType, + indexed: IndexedName, +} + +impl NameRecord { + fn new(hash: u64, kind: u8, name: &'static [u8]) -> NameRecord { + NameRecord { + nametype: NameType::new(hash, kind, name), + } + } + + fn nametype(self) -> NameType { + // SAFETY: called only before the interning phase replaces this record. + unsafe { self.nametype } + } + + fn indexed(self) -> IndexedName { + // SAFETY: called only after every record has been interned. + unsafe { self.indexed } + } + + fn set_indexed(&mut self, indexed: IndexedName) { + self.indexed = indexed; + } +} + +const _: () = assert!(std::mem::size_of::() == 16); + +/// A compilation unit. Its offset is relative to its input section until +/// `build_tables` rebases it to the output `.debug_info`. +struct Compunit { + offset: u64, + size: u64, + file: u32, + shndx: u32, + names: Vec, + ranges: Vec<(u64, u64)>, +} + +/// A DWARF 5 type unit. Type units have no address ranges, because code +/// belongs to compilation units. +struct Typeunit { + offset: u64, + type_die_offset: u64, + signature: u64, + file: u32, + shndx: u32, + names: Vec, +} + +/// A unique name in the index. +struct NameEntry { + /// gdb's own hash of the name, which decides its hash table slot. + gdb_hash: u32, + /// The number of units the name occurs in. + count: AtomicU32, + /// Offsets in the constant pool. + type_vector_offset: u32, + name_offset: u32, +} + +#[derive(Clone, Copy, Default)] +struct PoolSize { + type_bytes: u32, + name_bytes: u32, +} + +pub struct GdbIndexData { + cus: Vec, + tus: Vec, + names: Option>, + /// The names in output order. + entries: Vec, + type_pool_size: u32, + name_pool_size: u32, + ht_size: u32, + /// The hash table followed by the constant pool, once built. + tables: Vec, +} + +/// Views word-aligned table storage as its serialized bytes. +fn words_as_bytes(words: &[u32]) -> &[u8] { + // SAFETY: u8 has alignment one and the byte length covers the same + // allocation without outliving it. + unsafe { std::slice::from_raw_parts(words.as_ptr().cast::(), std::mem::size_of_val(words)) } +} + +/// Normalized view of a DWARF unit header. DWARF32 and DWARF64 describe +/// the width of section offsets, not the target address size. +struct UnitHeader { + size: u64, + header_size: u64, + abbrev_offset: u64, + type_die_offset: u64, + signature: u64, + version: u16, + unit_type: u8, + address_size: u8, + offset_size: u8, +} + +/// A cursor over DWARF data. +struct Reader<'d, 'a, E: Arch> { + diag: &'d Diagnostics, + data: &'a [u8], + pos: usize, + marker: std::marker::PhantomData, +} + +impl<'d, 'a, E: Arch> Reader<'d, 'a, E> { + fn new(diag: &'d Diagnostics, data: &'a [u8], pos: usize) -> Reader<'d, 'a, E> { + Reader { + diag, + data, + pos, + marker: std::marker::PhantomData, + } + } + + fn take(&mut self, n: usize) -> &'a [u8] { + let Some(bytes) = self.data.get(self.pos..self.pos + n) else { + fatal!(self.diag, "--gdb-index: truncated debug info"); + }; + self.pos += n; + bytes + } + + fn uint(&mut self, n: usize) -> u64 { + let bytes = self.take(n); + match n { + 1 => bytes[0] as u64, + 2 => E::Endian::read_u16(bytes) as u64, + 3 => { + if E::IS_LITTLE_ENDIAN { + bytes[0] as u64 | (bytes[1] as u64) << 8 | (bytes[2] as u64) << 16 + } else { + (bytes[0] as u64) << 16 | (bytes[1] as u64) << 8 | bytes[2] as u64 + } + } + 4 => E::Endian::read_u32(bytes) as u64, + 8 => E::Endian::read_u64(bytes), + _ => unreachable!("unsupported integer size"), + } + } + + fn u8(&mut self) -> u8 { + self.take(1)[0] + } + + fn u16(&mut self) -> u16 { + self.uint(2) as u16 + } + + fn u32(&mut self) -> u32 { + self.uint(4) as u32 + } + + fn u64(&mut self) -> u64 { + self.uint(8) + } + + /// A section offset, whose width depends on the DWARF32/64 format. + fn offset(&mut self, offset_size: u8) -> u64 { + self.uint(offset_size as usize) + } + + fn uleb(&mut self) -> u64 { + let mut rest = &self.data[self.pos..]; + let before = rest.len(); + let val = read_uleb(&mut rest); + self.pos += before - rest.len(); + val + } + + fn cstr(&mut self) -> &'a [u8] { + let rest = &self.data[self.pos..]; + // SAFETY: strnlen reads at most rest.len() bytes from the slice. + let len = unsafe { libc::strnlen(rest.as_ptr().cast(), rest.len()) }; + if len == rest.len() { + fatal!(self.diag, "--gdb-index: unterminated string in debug info"); + } + self.pos += len + 1; + &rest[..len] + } +} + +fn parse_unit_header(diag: &Diagnostics, data: &[u8], pos: usize) -> UnitHeader { + // The first word is either a DWARF32 unit length or DWARF64's reserved + // marker. The length excludes its own encoding: four bytes in DWARF32, + // or the marker plus an eight-byte length in DWARF64. + let mut r = Reader::::new(diag, data, pos); + let mut unit_length = r.u32() as u64; + let mut initial_length_size = 4; + let mut offset_size = 4; + if unit_length == u32::MAX as u64 { + unit_length = r.u64(); + initial_length_size = 12; + offset_size = 8; + } + + let version = r.u16(); + if version > 5 { + fatal!( + diag, + "--gdb-index: DWARF version {version} is not supported" + ); + } + + let mut hdr = UnitHeader { + size: unit_length + initial_length_size, + header_size: 0, + abbrev_offset: 0, + type_die_offset: 0, + signature: 0, + version, + unit_type: DW_UT_compile as u8, + address_size: 0, + offset_size, + }; + + if version < 5 { + hdr.abbrev_offset = r.offset(offset_size); + hdr.address_size = r.u8(); + } else { + hdr.unit_type = r.u8(); + hdr.address_size = r.u8(); + hdr.abbrev_offset = r.offset(offset_size); + match hdr.unit_type as u32 { + DW_UT_skeleton | DW_UT_split_compile => { + r.u64(); // dwo_id + } + DW_UT_type | DW_UT_split_type => { + hdr.signature = r.u64(); + hdr.type_die_offset = r.offset(offset_size); + } + _ => {} + } + } + hdr.header_size = (r.pos - pos) as u64; + hdr +} + +/// The address ranges a compilation unit covers. +struct RangeSections<'a> { + info: &'a [u8], + abbrev: &'a [u8], + ranges: &'a [u8], + addr: &'a [u8], + rnglists: &'a [u8], +} + +/// Positions an abbreviation reader at the attribute specifications of +/// the unit's first DIE, which must be the compilation unit DIE. +fn find_cu_abbrev<'d, 'a, E: Arch>( + diag: &'d Diagnostics, + die: &mut Reader<'d, 'a, E>, + abbrev_section: &'a [u8], + hdr: &UnitHeader, +) -> Reader<'d, 'a, E> { + if hdr.address_size as usize != E::WORD_SIZE { + fatal!( + diag, + "--gdb-index: unsupported address size {}", + hdr.address_size + ); + } + let abbrev_code = die.uleb(); + let mut abbrev = Reader::::new(diag, abbrev_section, hdr.abbrev_offset as usize); + loop { + let code = abbrev.uleb(); + if code == 0 { + fatal!(diag, "--gdb-index: .debug_abbrev does not contain a record for the first .debug_info record"); + } + let tag = abbrev.uleb(); + abbrev.u8(); // has_children + if code == abbrev_code { + if tag != DW_TAG_compile_unit as u64 && tag != DW_TAG_skeleton_unit as u64 { + fatal!(diag, "--gdb-index: the first entry's tag is not DW_TAG_compile_unit/DW_TAG_skeleton_unit but {tag:#x}"); + } + return abbrev; + } + // Skip the attribute specifications of an uninteresting record. + loop { + let name = abbrev.uleb(); + let form = abbrev.uleb(); + if name == 0 && form == 0 { + break; + } + if form == DW_FORM_implicit_const as u64 { + abbrev.uleb(); + } + } + } +} + +/// Reads one attribute value of the given form. Only the forms used by +/// the attributes of a compilation unit DIE are supported. +fn read_scalar(diag: &Diagnostics, r: &mut Reader, form: u64, offset_size: u8) -> u64 { + match form as u32 { + DW_FORM_flag_present => 0, + DW_FORM_data1 | DW_FORM_flag | DW_FORM_strx1 | DW_FORM_addrx1 | DW_FORM_ref1 => { + r.u8() as u64 + } + DW_FORM_data2 | DW_FORM_strx2 | DW_FORM_addrx2 | DW_FORM_ref2 => r.u16() as u64, + DW_FORM_strx3 | DW_FORM_addrx3 => r.uint(3), + DW_FORM_data4 | DW_FORM_strx4 | DW_FORM_addrx4 | DW_FORM_ref4 => r.u32() as u64, + DW_FORM_data8 | DW_FORM_ref8 => r.u64(), + DW_FORM_strp | DW_FORM_sec_offset | DW_FORM_line_strp => r.offset(offset_size), + DW_FORM_addr | DW_FORM_ref_addr => r.uint(E::WORD_SIZE), + DW_FORM_strx | DW_FORM_addrx | DW_FORM_udata | DW_FORM_ref_udata | DW_FORM_loclistx + | DW_FORM_rnglistx => r.uleb(), + DW_FORM_string => { + r.cstr(); + 0 + } + _ => fatal!(diag, "--gdb-index: unhandled debug info form: {form:#x}"), + } +} + +/// Reads a DWARF 2–4 range list from `.debug_ranges`. +fn read_debug_ranges(r: &mut Reader, mut base: u64) -> Vec<(u64, u64)> { + let mut vec = Vec::new(); + loop { + let start = r.uint(E::WORD_SIZE); + let end = r.uint(E::WORD_SIZE); + if start == 0 && end == 0 { + return vec; + } + if start == (1u64 << (8 * E::WORD_SIZE as u32)).wrapping_sub(1) { + base = end; + } else { + vec.push((start + base, end + base)); + } + } +} + +/// Reads a DWARF 5 range list from `.debug_rnglists`. `addrx` is the +/// address table the list may index into. +fn read_rnglist( + diag: &Diagnostics, + r: &mut Reader, + addrx: &[u8], + mut base: u64, +) -> Vec<(u64, u64)> { + let addr_at = |i: u64| -> u64 { + let mut a = Reader::::new(diag, addrx, i as usize * E::WORD_SIZE); + a.uint(E::WORD_SIZE) + }; + let mut vec = Vec::new(); + loop { + match r.u8() as u32 { + DW_RLE_end_of_list => return vec, + DW_RLE_base_addressx => base = addr_at(r.uleb()), + DW_RLE_startx_endx => { + let (a, b) = (r.uleb(), r.uleb()); + vec.push((addr_at(a), addr_at(b))); + } + DW_RLE_startx_length => { + let (a, len) = (r.uleb(), r.uleb()); + let start = addr_at(a); + vec.push((start, start + len)); + } + DW_RLE_offset_pair => { + let (a, b) = (r.uleb(), r.uleb()); + // A zero base means the range belongs to a discarded + // section. + if base != 0 { + vec.push((base + a, base + b)); + } + } + DW_RLE_base_address => base = r.uint(E::WORD_SIZE), + DW_RLE_start_end => { + let (a, b) = (r.uint(E::WORD_SIZE), r.uint(E::WORD_SIZE)); + vec.push((a, b)); + } + DW_RLE_start_length => { + let a = r.uint(E::WORD_SIZE); + let len = r.uleb(); + vec.push((a, a + len)); + } + kind => fatal!( + diag, + "--gdb-index: unknown .debug_rnglists entry kind: {kind:#x}" + ), + } + } +} + +/// Reads the address ranges of the compilation unit at `cu.offset` in the +/// output `.debug_info`. A unit with several ranges refers to +/// `.debug_ranges` (or `.debug_rnglists` in DWARF 5); a contiguous one +/// stores its bounds directly, possibly through `.debug_addr`. +fn read_address_ranges( + diag: &Diagnostics, + secs: &RangeSections, + cu: &Compunit, +) -> Vec<(u64, u64)> { + let hdr = parse_unit_header::(diag, secs.info, cu.offset as usize); + let mut die = Reader::::new(diag, secs.info, (cu.offset + hdr.header_size) as usize); + let mut abbrev = find_cu_abbrev::(diag, &mut die, secs.abbrev, &hdr); + + let mut low_pc: Option<(u64, u64)> = None; + let mut high_pc: Option<(u64, u64)> = None; + let mut ranges: Option<(u64, u64)> = None; + let mut rnglists_base: Option = None; + let mut addrx: &[u8] = &[]; + + loop { + let name = abbrev.uleb(); + let form = abbrev.uleb(); + if name == 0 && form == 0 { + break; + } + let val = read_scalar::(diag, &mut die, form, hdr.offset_size); + match name as u32 { + DW_AT_low_pc => low_pc = Some((form, val)), + DW_AT_high_pc => high_pc = Some((form, val)), + DW_AT_rnglists_base => rnglists_base = Some(val), + DW_AT_addr_base => addrx = &secs.addr[val as usize..], + DW_AT_ranges => ranges = Some((form, val)), + _ => {} + } + } + + let addr_at = |i: u64| -> u64 { + Reader::::new(diag, addrx, i as usize * E::WORD_SIZE).uint(E::WORD_SIZE) + }; + let base = low_pc.map_or(0, |(_, val)| val); + + // Before DWARF 5, DW_AT_ranges is an offset into .debug_ranges. In + // DWARF 5 it is either an offset into .debug_rnglists or an index into + // the offset table at DW_AT_rnglists_base. + if let Some((form, val)) = ranges { + if hdr.version <= 4 { + let mut r = Reader::::new(diag, secs.ranges, val as usize); + return read_debug_ranges::(&mut r, base); + } + if form == DW_FORM_sec_offset as u64 { + let mut r = Reader::::new(diag, secs.rnglists, val as usize); + return read_rnglist::(diag, &mut r, addrx, base); + } + let Some(list_base) = rnglists_base else { + fatal!(diag, "--gdb-index: missing DW_AT_rnglists_base"); + }; + let mut entry = Reader::::new( + diag, + secs.rnglists, + (list_base + val * hdr.offset_size as u64) as usize, + ); + let offset = entry.offset(hdr.offset_size); + let mut r = Reader::::new(diag, secs.rnglists, (list_base + offset) as usize); + return read_rnglist::(diag, &mut r, addrx, base); + } + + // For one contiguous range, high_pc is either an address or a length, + // as indicated by its form. DWARF 5 may store either endpoint as an + // index into the address table. + let (Some((lo_form, lo_val)), Some((hi_form, hi_val))) = (low_pc, high_pc) else { + return Vec::new(); + }; + let lo = match lo_form as u32 { + DW_FORM_addr => lo_val, + DW_FORM_addrx | DW_FORM_addrx1 | DW_FORM_addrx2 | DW_FORM_addrx4 => addr_at(lo_val), + _ => fatal!( + diag, + "--gdb-index: unhandled form for DW_AT_low_pc: {lo_form:#x}" + ), + }; + let hi = match hi_form as u32 { + DW_FORM_addr => hi_val, + DW_FORM_addrx | DW_FORM_addrx1 | DW_FORM_addrx2 | DW_FORM_addrx4 => addr_at(hi_val), + DW_FORM_udata | DW_FORM_data1 | DW_FORM_data2 | DW_FORM_data4 | DW_FORM_data8 => { + lo + hi_val + } + _ => fatal!( + diag, + "--gdb-index: unhandled form for DW_AT_high_pc: {hi_form:#x}" + ), + }; + vec![(lo, hi)] +} + +/// Units read from one object file. +#[derive(Default)] +struct FileUnits { + cus: Vec, + tus: Vec, +} + +/// One live `.debug_info` contribution needed by the background reader. +struct DebugInfoInput { + shndx: u32, + contents: &'static [u8], +} + +/// A relocated `.debug_gnu_pubnames` header field and the unit it names. +struct PubnamesRelocation { + offset: u64, + target_shndx: u32, + unit_offset: u64, +} + +/// One GNU pubnames or pubtypes contribution needed by the background reader. +struct PubnamesInput { + contents: &'static [u8], + relocations: Vec, +} + +/// Immutable input owned by the background `.gdb_index` reader. The section +/// contents are either input mappings or leaked decompression buffers, so the +/// foreground passes may continue mutating their ObjectFiles independently. +pub struct GdbInputFile { + name: String, + debug_info: Vec, + pubnames: Vec, +} + +/// Prepares immutable views of the debug sections before foreground and +/// `.gdb_index` work diverge. Relocations are reduced to the unit associations +/// the reader needs, so the background task never aliases an ObjectFile. +pub fn prepare_inputs(ctx: &mut Context) -> Vec { + let diag = &*ctx.diag; + ctx.objs + .par_iter_mut() + .map(|file| { + let name = file.to_string(); + let mut debug_info = Vec::new(); + for shndx in file.debug_info_sections.clone() { + let Some(section_name) = file.section(shndx as usize).map(|isec| isec.name(file)) + else { + continue; + }; + let input_size = file.base.shdrs.sh_offset_and_size(shndx as usize).1 as usize; + let Some(isec) = file.section_mut(shndx as usize) else { + continue; + }; + + // Skip type units that lost COMDAT group selection. + if !isec.is_alive() { + continue; + } + isec.uncompress::(diag, &name, section_name, input_size); + debug_info.push(DebugInfoInput { + shndx, + contents: isec.contents(), + }); + } + + let mut pubnames = Vec::new(); + for shndx in [file.debug_pubnames, file.debug_pubtypes] + .into_iter() + .flatten() + { + let Some(section_name) = file.section(shndx as usize).map(|isec| isec.name(file)) + else { + continue; + }; + let input_size = file.base.shdrs.sh_offset_and_size(shndx as usize).1 as usize; + let Some(isec) = file.section_mut(shndx as usize) else { + continue; + }; + isec.uncompress::(diag, &name, section_name, input_size); + + let isec = file.section_at(shndx); + let mut relocations = Vec::new(); + file.for_each_relocation::(diag, isec.relsec_idx(), |rel, _| { + let esym = file.base.elf_syms.at(rel.r_sym as usize); + if let Some(target) = file.symbol_section(rel.r_sym as usize) { + relocations.push(PubnamesRelocation { + offset: rel.r_offset, + target_shndx: target.shndx, + unit_offset: esym + .st_value + .wrapping_add(isec.rel_addend::(&rel) as u64), + }); + } + }); + pubnames.push(PubnamesInput { + contents: isec.contents(), + relocations, + }); + } + + GdbInputFile { + name, + debug_info, + pubnames, + } + }) + .collect() +} + +/// Reads the units of every live `.debug_info` section of a file. +fn read_debug_units(diag: &Diagnostics, file: &GdbInputFile, file_idx: u32) -> FileUnits { + let mut units = FileUnits::default(); + for input in &file.debug_info { + let contents = input.contents; + let mut pos = 0; + while pos < contents.len() { + let unit = parse_unit_header::(diag, contents, pos); + match unit.unit_type as u32 { + DW_UT_compile | DW_UT_partial | DW_UT_skeleton | DW_UT_split_compile => { + units.cus.push(Compunit { + offset: pos as u64, + size: unit.size, + file: file_idx, + shndx: input.shndx, + names: Vec::new(), + ranges: Vec::new(), + }) + } + DW_UT_type | DW_UT_split_type => units.tus.push(Typeunit { + offset: pos as u64, + type_die_offset: unit.type_die_offset, + signature: unit.signature, + file: file_idx, + shndx: input.shndx, + names: Vec::new(), + }), + kind => fatal!(diag, "--gdb-index: unknown unit type: {kind:#x}"), + } + pos += unit.size as usize; + } + } + units +} + +/// Finds the unit a pubnames set refers to. The set header's +/// `debug_info_offset` field is relocated against the input `.debug_info` +/// contribution containing the unit, which matters for DWARF 5 type units +/// in their own COMDAT sections. +fn pubnames_unit<'a>( + input: &PubnamesInput, + field_offset: u64, + units: &'a mut FileUnits, +) -> Option<&'a mut Vec> { + let rel = input + .relocations + .get( + input + .relocations + .partition_point(|r| r.offset < field_offset), + ) + .filter(|r| r.offset == field_offset)?; + let key = (rel.target_shndx, rel.unit_offset); + + // Units were appended in section and offset order, so both lists are + // sorted by this key. + if let Ok(i) = units + .cus + .binary_search_by_key(&key, |cu| (cu.shndx, cu.offset)) + { + return Some(&mut units.cus[i].names); + } + if let Ok(i) = units + .tus + .binary_search_by_key(&key, |tu| (tu.shndx, tu.offset)) + { + return Some(&mut units.tus[i].names); + } + None +} + +/// Parses `.debug_gnu_pubnames` and `.debug_gnu_pubtypes`. Each set starts +/// with a DWARF32 or DWARF64 header naming one unit, followed by +/// (DIE offset, kind byte, name) tuples until a zero offset. +fn read_pubnames(diag: &Diagnostics, file: &GdbInputFile, units: &mut FileUnits) { + for input in &file.pubnames { + let contents = input.contents; + let mut pos = 0; + while pos < contents.len() { + let mut r = Reader::::new(diag, contents, pos); + let (set_size, offset_size, field_offset) = if r.u32() == u32::MAX { + let size = r.u64(); + (size + 12, 8, pos as u64 + 14) + } else { + r.pos = pos; + let size = r.u32() as u64; + (size + 4, 4, pos as u64 + 6) + }; + r.u16(); // version + r.offset(offset_size); // debug_info_offset + r.offset(offset_size); // debug_info_size + + let Some(names) = pubnames_unit(input, field_offset, units) else { + fatal!(diag, "{}: corrupted debug_info_offset", file.name); + }; + let end = pos + set_size as usize; + while r.pos < end { + if r.offset(offset_size) == 0 { + break; + } + let kind = r.u8(); + let name = r.cstr(); + names.push(NameRecord::new( + xxhash_rust::xxh3::xxh3_64(name), + kind, + name, + )); + } + pos = end; + } + } +} + +/// GCC emits a name once per COMDAT group it appears in. +fn dedup_names(names: &mut Vec) { + if names.len() < 2 { + return; + } + + let capacity = (names.len() * 2).next_power_of_two(); + let mut buckets = vec![u32::MAX; capacity]; + let mask = capacity - 1; + let mut out = 0; + + for i in 0..names.len() { + let nametype = names[i].nametype(); + let mut idx = (nametype.hash() ^ (nametype.kind() as u64 * 0x9e37_79b9)) as usize & mask; + let mut duplicate = false; + + while buckets[idx] != u32::MAX { + if names[buckets[idx] as usize].nametype().same_name(nametype) { + duplicate = true; + break; + } + idx = (idx + 1) & mask; + } + + if duplicate { + continue; + } + if i != out { + names[out] = names[i]; + } + buckets[idx] = out as u32; + out += 1; + } + names.truncate(out); +} + +/// Computes the `.gdb_index` hash and length together when a name is first +/// inserted. +fn initialize_gdb_name(name: *const u8) -> (u32, NameEntry) { + let mut hash = 0u32; + let mut size = 0u32; + loop { + // SAFETY: NameType points to a NUL-terminated public name. + let mut c = unsafe { *name.add(size as usize) }; + if c == 0 { + break; + } + if c.is_ascii_uppercase() { + c = c.to_ascii_lowercase(); + } + hash = hash + .wrapping_mul(67) + .wrapping_add(c as u32) + .wrapping_sub(113); + size += 1; + } + ( + size, + NameEntry { + gdb_hash: hash, + count: AtomicU32::new(0), + type_vector_offset: 0, + name_offset: 0, + }, + ) +} + +// GDB index type lists contain fixed-width 32-bit records. Four byte-wise +// passes are faster than comparison sorting once a list is sufficiently large. +fn radix_sort(values: &mut [u32], scratch: &mut Vec) { + scratch.resize(values.len(), 0); + let mut in_values = true; + + for shift in (0..32).step_by(8) { + let mut counts = [0usize; 256]; + let src = if in_values { &*values } else { &*scratch }; + for &value in src { + counts[(value >> shift) as usize & 255] += 1; + } + + let mut offsets = [0usize; 256]; + for i in 1..256 { + offsets[i] = offsets[i - 1] + counts[i - 1]; + } + + if in_values { + for &value in &*values { + let bucket = (value >> shift) as usize & 255; + scratch[offsets[bucket]] = value; + offsets[bucket] += 1; + } + } else { + for &value in &*scratch { + let bucket = (value >> shift) as usize & 255; + values[offsets[bucket]] = value; + offsets[bucket] += 1; + } + } + in_values = !in_values; + } + + debug_assert!(in_values); +} + +fn estimate_names(units: &[T], names: impl Fn(&T) -> &[NameRecord] + Sync) -> HyperLogLog { + units + .par_iter() + .fold(HyperLogLog::default, |mut sketch, unit| { + // Name records keep 56 hash bits. Spread them across a 64-bit + // word because HyperLogLog uses the number of leading zero bits. + for record in names(unit) { + sketch.insert(record.nametype().hash().wrapping_mul(0x9e37_79b9_7f4a_7c15)); + } + sketch + }) + .reduce(HyperLogLog::default, |a, b| a.merged(&b)) +} + +/// Reads compilation units and their public names from the input files, +/// and interns the names. +pub fn read_inputs( + timer: Timer, + diag: &Diagnostics, + files: Vec, +) -> GdbIndexData { + let _timer = timer; + let per_file: Vec = files + .par_iter() + .enumerate() + .map(|(i, file)| { + let mut units = read_debug_units::(diag, file, i as u32); + read_pubnames::(diag, file, &mut units); + for cu in &mut units.cus { + dedup_names(&mut cu.names); + } + for tu in &mut units.tus { + dedup_names(&mut tu.names); + } + units + }) + .collect(); + + let num_cus = per_file.iter().map(|units| units.cus.len()).sum(); + let num_tus = per_file.iter().map(|units| units.tus.len()).sum(); + let mut cus = Vec::with_capacity(num_cus); + let mut tus = Vec::with_capacity(num_tus); + for mut units in per_file { + cus.append(&mut units.cus); + tus.append(&mut units.tus); + } + + let mut estimate = estimate_names(&cus, |cu| &cu.names); + estimate = estimate.merged(&estimate_names(&tus, |tu| &tu.names)); + let map: ConcurrentMap = + ConcurrentMap::with_capacity(estimate.cardinality() as usize * 3 / 2); + let intern = |names: &mut [NameRecord]| { + for record in names { + let nametype = record.nametype(); + // SAFETY: NameType stores a NUL-terminated string that remains + // live for the complete link. + let (entry, value, _) = unsafe { + map.insert_cstr_with( + nametype.name as *const u8, + nametype.hash(), + initialize_gdb_name, + ) + }; + record.set_indexed(IndexedName { + entry, + type_vector_idx: value.count.fetch_add(1, Ordering::Relaxed) + 1, + kind: nametype.kind(), + }); + } + }; + cus.par_iter_mut().for_each(|cu| intern(&mut cu.names)); + tus.par_iter_mut().for_each(|tu| intern(&mut tu.names)); + + // Lay out the constant pool: all type vectors, then all names, in a + // deterministic order. + let entries = map.sorted_entries_all(); + let names = map.freeze(); + + // The map may contain millions of names. Assign their type and string + // ranges with a parallel prefix sum. + let chunk_size = entries.len().div_ceil(rayon::current_num_threads()).max(1); + let chunk_sizes: Vec = entries + .par_chunks(chunk_size) + .map(|chunk| { + let mut size = PoolSize::default(); + for &id in chunk { + let entry = names.get(id); + size.type_bytes += entry.count.load(Ordering::Relaxed) * 4 + 4; + size.name_bytes += names.key(id).len() as u32 + 1; + } + size + }) + .collect(); + let mut pool_size = PoolSize::default(); + let chunk_offsets: Vec = chunk_sizes + .iter() + .map(|&size| { + let offset = pool_size; + pool_size.type_bytes += size.type_bytes; + pool_size.name_bytes += size.name_bytes; + offset + }) + .collect(); + entries + .par_chunks(chunk_size) + .zip(chunk_offsets) + .for_each(|(chunk, mut size)| { + for &id in chunk { + // SAFETY: entries contains each map entry exactly once, so + // parallel chunks update disjoint values. + let entry = unsafe { &mut *names.value_mut_ptr(id) }; + entry.type_vector_offset = size.type_bytes; + entry.name_offset = size.name_bytes; + size.type_bytes += entry.count.load(Ordering::Relaxed) * 4 + 4; + size.name_bytes += names.key(id).len() as u32 + 1; + } + }); + entries.par_iter().for_each(|&id| { + // SAFETY: as above, each entry appears once. + unsafe { &mut *names.value_mut_ptr(id) }.name_offset += pool_size.type_bytes; + }); + + let ht_size = (entries.len() as u32 * 5 / 4 + 1).next_power_of_two(); + GdbIndexData { + cus, + tus, + names: Some(names), + entries, + type_pool_size: pool_size.type_bytes, + name_pool_size: pool_size.name_bytes, + ht_size, + tables: Vec::new(), + } +} + +/// Rebases unit offsets while the ObjectFiles are still available to the +/// foreground driver. The returned data is then self-contained. +pub fn prepare_tables(ctx: &Context, data: &mut GdbIndexData) { + let output_offset = |file: u32, shndx: u32| ctx.objs[file as usize].section_at(shndx).offset(); + for cu in &mut data.cus { + cu.offset += output_offset(cu.file, cu.shndx); + } + for tu in &mut data.tus { + tu.offset += output_offset(tu.file, tu.shndx); + } +} + +fn limited_parallel_for_mut( + values: &mut [T], + workers: usize, + op: impl Fn(usize, &mut T) + Send + Sync, +) { + limited_parallel_for_mut_init(values, workers, || (), |_, i, value| op(i, value)); +} + +fn limited_parallel_for_mut_init( + values: &mut [T], + workers: usize, + init: impl Fn() -> S + Send + Sync, + op: impl Fn(&mut S, usize, &mut T) + Send + Sync, +) { + if values.is_empty() { + return; + } + + let len = values.len(); + let workers = workers.max(1).min(len); + let chunk_size = len.div_ceil(workers.saturating_mul(2)).max(1); + let next = AtomicUsize::new(0); + let addr = values.as_mut_ptr() as usize; + (0..workers).into_par_iter().for_each(|_| { + let mut state = init(); + loop { + let begin = next.fetch_add(chunk_size, Ordering::Relaxed); + if begin >= len { + break; + } + let end = (begin + chunk_size).min(len); + for i in begin..end { + // SAFETY: every range is returned by one fetch_add, so parallel + // workers receive disjoint elements of values. + op(&mut state, i, unsafe { &mut *(addr as *mut T).add(i) }); + } + } + }); +} + +/// Builds the name hash table and the constant pool. They depend on the +/// order of units in the output `.debug_info` but not on addresses, so +/// this can run before relocations are applied. +pub fn build_tables(timer: Timer, mut data: GdbIndexData, workers: usize) -> GdbIndexData { + let _timer = timer; + if data.cus.is_empty() && data.tus.is_empty() { + return data; + } + + // Units are numbered by their position in the output: all + // compilation units, then all type units. + data.cus.sort_by_key(|cu| cu.offset); + data.tus.sort_by_key(|tu| tu.offset); + + let names = data.names.as_ref().unwrap(); + let symtab_size = data.ht_size as usize * 8; + let pool_size = (data.type_pool_size + data.name_pool_size) as usize; + let table_size = symtab_size + pool_size; + let table_words = table_size.div_ceil(4); + let mut tables = Vec::>::with_capacity(table_words); + // SAFETY: MaybeUninit may be left uninitialized. Every serialized byte is + // written below before the storage is converted to u32 words. + unsafe { tables.set_len(table_words) }; + let table_addr = tables.as_mut_ptr() as usize; + + // `tables` contains the name hash table followed by the constant pool. The + // constant pool contains all type vectors followed by all name strings. + // Each occupied hash-table slot contains the constant-pool offsets of a + // name and its type vector. + // SAFETY: table storage has at least symtab_size bytes. Initializing them + // before making the u32 slice avoids forming references to uninitialized + // integers. + unsafe { std::ptr::write_bytes(table_addr as *mut u8, 0, symtab_size) }; + let ht = unsafe { std::slice::from_raw_parts_mut(table_addr as *mut u32, symtab_size / 4) }; + + // This probing sequence is part of the .gdb_index format. The table size + // is a power of two, so an odd step visits every slot. + let mask = data.ht_size - 1; + for &id in &data.entries { + let entry = names.get(id); + let step = ((entry.gdb_hash.wrapping_mul(17)) & mask) | 1; + let mut i = entry.gdb_hash & mask; + while ht[i as usize * 2] != 0 || ht[i as usize * 2 + 1] != 0 { + i = (i + step) & mask; + } + ht[i as usize * 2] = entry.name_offset.to_le(); + ht[i as usize * 2 + 1] = entry.type_vector_offset.to_le(); + } + + let pool_addr = table_addr + symtab_size; + + // Each occurrence of a name contributes one value to its type vector. Each + // occurrence was assigned a distinct slot while the names were interned, so + // the vectors can be filled in parallel. The high byte is the name's type + // and the low 24 bits are the unit number. TU numbers follow CU numbers. + let write_names = |records: &[NameRecord], unit: usize| { + for record in records { + let name = record.indexed(); + let entry = names.get(name.entry); + let offset = entry.type_vector_offset as usize + name.type_vector_idx as usize * 4; + debug_assert!(offset + 4 <= data.type_pool_size as usize); + let value = (name.kind as u32) << 24 | unit as u32; + // SAFETY: the pool starts on a u32 boundary, all offsets are + // multiples of four, and every occurrence reserved a distinct + // slot before this parallel phase. + unsafe { (pool_addr as *mut u32).add(offset / 4).write(value) }; + } + }; + let num_cus = data.cus.len(); + limited_parallel_for_mut(&mut data.cus, workers, |i, cu| { + write_names(&cu.names, i); + }); + limited_parallel_for_mut(&mut data.tus, workers, |i, tu| { + write_names(&tu.names, num_cus + i); + }); + + // Prefix each type vector with its length and sort it for deterministic + // output. Store the NUL-terminated name at its assigned string-pool offset. + limited_parallel_for_mut_init(&mut data.entries, workers, Vec::new, |scratch, _, id| { + let entry = names.get(*id); + let count = entry.count.load(Ordering::Relaxed); + let words = (pool_addr as *mut u32).wrapping_add(entry.type_vector_offset as usize / 4); + // SAFETY: every occurrence filled its distinct value slot above; + // writing the prefix completes this vector before a slice is made. + unsafe { words.write(count) }; + let values = unsafe { std::slice::from_raw_parts_mut(words.add(1), count as usize) }; + if values.len() < 256 { + values.sort_unstable(); + } else { + radix_sort(values, scratch); + } + // SAFETY: the prefix and all count values are now initialized. + let words = unsafe { std::slice::from_raw_parts_mut(words, count as usize + 1) }; + for word in &mut *words { + *word = word.to_le(); + } + let key = names.key(*id); + let name = (pool_addr as *mut u8).wrapping_add(entry.name_offset as usize); + // SAFETY: prefix-scan offsets assign this entry a distinct + // key.len()+1 byte range in the name pool. + unsafe { + name.copy_from_nonoverlapping(key.as_ptr(), key.len()); + name.add(key.len()).write(0); + } + }); + + // Vec rounds the byte allocation up to a whole word; initialize only + // those padding bytes, which are not part of the serialized tables. + let padded_size = table_words * 4; + for i in table_size..padded_size { + // SAFETY: these are the allocation's final padding bytes and no worker + // remains active. + unsafe { (table_addr as *mut u8).add(i).write(0) }; + } + let (table_ptr, table_len, table_capacity) = ( + tables.as_mut_ptr().cast::(), + tables.len(), + tables.capacity(), + ); + std::mem::forget(tables); + // SAFETY: the hash table, every type vector, every name including its NUL, + // and the final allocation padding have all been initialized. u32 and + // MaybeUninit have identical allocation layouts. + let tables = unsafe { Vec::from_raw_parts(table_ptr, table_len, table_capacity) }; + + // The serialized tables contain everything needed from names and the map. + // Release their storage here so reclamation remains part of this background + // phase rather than delaying the final output path. + limited_parallel_for_mut(&mut data.cus, workers, |_, cu| { + cu.names = Vec::new(); + }); + limited_parallel_for_mut(&mut data.tus, workers, |_, tu| { + tu.names = Vec::new(); + }); + data.entries = Vec::new(); + data.names = None; + data.tables = tables; + data +} + +/// Builds the tables synchronously for the separate-debug-file path, where +/// the output's section ordering differs from the main file. +pub fn build_tables_now(ctx: &mut Context) { + let timer = ctx.timer("build_gdb_index_tables"); + let Some(mut data) = ctx.gdb_index_data.take() else { + return; + }; + prepare_tables(ctx, &mut data); + ctx.gdb_index_data = Some(build_tables(timer, data, rayon::current_num_threads())); +} + +/// The contents of a debug section in the output, or its uncompressed +/// contents if it was compressed. +fn section_contents<'a, E: Arch>(ctx: &'a Context, buf: &'a [u8], name: &str) -> &'a [u8] { + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + if hdr.name != name { + continue; + } + if let ChunkId::Compressed(i) = id { + return ctx.compressed_sections[i as usize] + .uncompressed_data + .as_deref() + .unwrap_or(&[]); + } + return &buf[hdr.shdr.sh_offset as usize..(hdr.shdr.sh_offset + hdr.shdr.sh_size) as usize]; + } + &[] +} + +fn parallel_copy(dst: &mut [u8], src: &[u8]) { + const BLOCK_SIZE: usize = 2 * 1024 * 1024; + + debug_assert_eq!(dst.len(), src.len()); + dst.par_chunks_mut(BLOCK_SIZE) + .zip(src.par_chunks(BLOCK_SIZE)) + .for_each(|(dst, src)| dst.copy_from_slice(src)); +} + +/// Reads the relocated address ranges and appends the finished index to +/// the output file. +pub fn write(ctx: &mut Context, output: &mut OutputFile) { + let _t = ctx.timer("write_gdb_index"); + let Some(mut data) = ctx.gdb_index_data.take() else { + return; + }; + if data.cus.is_empty() && data.tus.is_empty() { + return; + } + + { + let buf = output.buf(); + let secs = RangeSections { + info: section_contents(ctx, buf, ".debug_info"), + abbrev: section_contents(ctx, buf, ".debug_abbrev"), + ranges: section_contents(ctx, buf, ".debug_ranges"), + addr: section_contents(ctx, buf, ".debug_addr"), + rnglists: section_contents(ctx, buf, ".debug_rnglists"), + }; + let diag = &ctx.diag; + data.cus.par_iter_mut().for_each(|cu| { + cu.ranges = read_address_ranges::(diag, &secs, cu); + cu.ranges.retain(|&(start, end)| start != 0 && start != end); + }); + } + + // Version 8 made symbol table entries refer to type units directly, as + // ours do. gdb 12 accepts version 8 but assumes its type unit list + // refers to .debug_types and crashes on DWARF 5 type units in + // .debug_info; version 9 makes it ignore the index instead. Version 7 + // is kept for indices without type units so that older gdbs can use + // them. + let has_tus = !data.tus.is_empty(); + let header_size = if has_tus { 28 } else { 24 }; + let cu_list_offset = header_size; + let cu_types_offset = cu_list_offset + data.cus.len() * 16; + let ranges_offset = cu_types_offset + data.tus.len() * 24; + let mut range_offsets = Vec::with_capacity(data.cus.len()); + let mut ranges_size = 0usize; + for cu in &data.cus { + range_offsets.push(ranges_size as u64); + ranges_size += cu.ranges.len() * 20; + } + let symtab_offset = ranges_offset + ranges_size; + let shortcut_offset = symtab_offset + data.ht_size as usize * 8; + let const_pool_offset = shortcut_offset + if has_tus { 8 } else { 0 }; + let size = const_pool_offset + (data.type_pool_size + data.name_pool_size) as usize; + + let file_size = output.len(); + output.extend(&ctx.diag, size); + let buf = &mut output.buf()[file_size..]; + + let mut header: Vec = vec![ + if has_tus { 9 } else { 7 }, + cu_list_offset as u32, + cu_types_offset as u32, + ranges_offset as u32, + symtab_offset as u32, + ]; + if has_tus { + header.push(shortcut_offset as u32); + } + header.push(const_pool_offset as u32); + for (i, word) in header.iter().enumerate() { + buf[i * 4..i * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + + let mut p = cu_list_offset; + for cu in &data.cus { + buf[p..p + 8].copy_from_slice(&cu.offset.to_le_bytes()); + buf[p + 8..p + 16].copy_from_slice(&cu.size.to_le_bytes()); + p += 16; + } + for tu in &data.tus { + buf[p..p + 8].copy_from_slice(&tu.offset.to_le_bytes()); + buf[p + 8..p + 16].copy_from_slice(&tu.type_die_offset.to_le_bytes()); + buf[p + 16..p + 24].copy_from_slice(&tu.signature.to_le_bytes()); + p += 24; + } + + // An address-area entry is {start address, end address, CU-list index}. + data.cus + .par_iter() + .enumerate() + .zip(split_at_offsets( + &mut buf[ranges_offset..symtab_offset], + &range_offsets, + )) + .for_each(|((i, cu), buf)| { + for (range, entry) in cu.ranges.iter().zip(buf.chunks_exact_mut(20)) { + entry[..8].copy_from_slice(&range.0.to_le_bytes()); + entry[8..16].copy_from_slice(&range.1.to_le_bytes()); + entry[16..20].copy_from_slice(&(i as u32).to_le_bytes()); + } + }); + + let symtab_size = data.ht_size as usize * 8; + let tables = words_as_bytes(&data.tables); + parallel_copy( + &mut buf[symtab_offset..symtab_offset + symtab_size], + &tables[..symtab_size], + ); + parallel_copy( + &mut buf[const_pool_offset..size], + &tables[symtab_size..symtab_size + size - const_pool_offset], + ); + + // The section header table was written before the index existed. + if let Some(gdb_index) = &mut ctx.gdb_index { + gdb_index.hdr.shdr.sh_size = size as u64; + } + if let Some(section_headers) = &ctx.shdr { + let shdr = section_headers.hdr.shdr; + let buf = output.buf(); + crate::chunks::copy_buf( + ctx, + ChunkId::Shdr, + &mut buf[shdr.sh_offset as usize..(shdr.sh_offset + shdr.sh_size) as usize], + ); + } +} diff --git a/src/icf.rs b/src/icf.rs new file mode 100644 index 000000000..39fa64db8 --- /dev/null +++ b/src/icf.rs @@ -0,0 +1,778 @@ +//! Identical Code Folding. +//! +//! ICF merges read-only sections with identical contents, metadata and +//! relocations. Two relocations count as identical if they refer to +//! sections that are themselves identical, which makes this a graph +//! problem: sections are vertices and relocations are edges, and two +//! vertices are equivalent if their (possibly infinite) unfoldings into +//! trees are equal. +//! +//! The approach is to hash each section together with the hashes of the +//! sections it refers to, repeatedly: after n rounds a section's digest +//! covers its references up to depth n. The number of distinct digests +//! can only grow, so once two rounds produce the same count the partition +//! into equivalence classes has converged. + +use std::io::Write; +use std::sync::atomic::{AtomicU64, Ordering}; + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::context::Context; +use crate::elf::*; +use crate::fatal; +use crate::input_files::{ObjId, ObjectFile}; +use crate::input_sections::{InputSection, SectionRef}; +use crate::symbol::{is_c_identifier, Symbol}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Digest { + hi: u64, + lo: u64, +} + +// This is an implementation of SipHash based on the reference +// implementation at https://github.com/rui314/siphash. +struct SipHash13_128 { + v0: u64, + v1: u64, + v2: u64, + v3: u64, + buf: [u8; 8], + buflen: u8, + sum: u8, +} + +impl SipHash13_128 { + #[inline] + fn new(key: &[u8; 16]) -> SipHash13_128 { + let k0 = u64::from_le_bytes(key[..8].try_into().unwrap()); + let k1 = u64::from_le_bytes(key[8..].try_into().unwrap()); + SipHash13_128 { + v0: 0x736f_6d65_7073_6575 ^ k0, + v1: 0x646f_7261_6e64_6f6d ^ k1 ^ 0xee, + v2: 0x6c79_6765_6e65_7261 ^ k0, + v3: 0x7465_6462_7974_6573 ^ k1, + buf: [0; 8], + buflen: 0, + sum: 0, + } + } + + #[inline] + fn update(&mut self, mut msg: &[u8]) { + self.sum = self.sum.wrapping_add(msg.len() as u8); + + if self.buflen != 0 { + let buflen = self.buflen as usize; + if buflen + msg.len() < 8 { + self.buf[buflen..buflen + msg.len()].copy_from_slice(msg); + self.buflen += msg.len() as u8; + return; + } + + let n = 8 - buflen; + self.buf[buflen..].copy_from_slice(&msg[..n]); + self.compress(u64::from_le_bytes(self.buf)); + msg = &msg[n..]; + self.buflen = 0; + } + + while msg.len() >= 8 { + self.compress(u64::from_le_bytes(msg[..8].try_into().unwrap())); + msg = &msg[8..]; + } + + self.buf[..msg.len()].copy_from_slice(msg); + self.buflen = msg.len() as u8; + } + + /// Updates the hash with an in-memory `Digest`. Propagation hashes only + /// complete digests, so this is the aligned 16-byte path through `update`. + #[inline(always)] + fn update_digest(&mut self, digest: Digest) { + debug_assert_eq!(self.buflen, 0); + self.sum = self.sum.wrapping_add(16); + self.compress(u64::from_le_bytes(digest.hi.to_ne_bytes())); + self.compress(u64::from_le_bytes(digest.lo.to_ne_bytes())); + } + + #[inline] + fn finish(mut self) -> Digest { + self.buf[self.buflen as usize..].fill(0); + self.compress((u64::from(self.sum) << 56) | u64::from_le_bytes(self.buf)); + + self.v2 ^= 0xee; + self.finalize(); + let hi = self.v0 ^ self.v1 ^ self.v2 ^ self.v3; + + self.v1 ^= 0xdd; + self.finalize(); + let lo = self.v0 ^ self.v1 ^ self.v2 ^ self.v3; + Digest { hi, lo } + } + + #[inline(always)] + fn round(&mut self) { + self.v0 = self.v0.wrapping_add(self.v1); + self.v1 = self.v1.rotate_left(13); + self.v1 ^= self.v0; + self.v0 = self.v0.rotate_left(32); + self.v2 = self.v2.wrapping_add(self.v3); + self.v3 = self.v3.rotate_left(16); + self.v3 ^= self.v2; + self.v0 = self.v0.wrapping_add(self.v3); + self.v3 = self.v3.rotate_left(21); + self.v3 ^= self.v0; + self.v2 = self.v2.wrapping_add(self.v1); + self.v1 = self.v1.rotate_left(17); + self.v1 ^= self.v2; + self.v2 = self.v2.rotate_left(32); + } + + #[inline(always)] + fn compress(&mut self, m: u64) { + self.v3 ^= m; + self.round(); + self.v0 ^= m; + } + + #[inline(always)] + fn finalize(&mut self) { + self.round(); + self.round(); + self.round(); + } +} + +// A concurrent hash map from digest to section. We use it to count the +// number of distinct digests and to elect the leader section of each +// digest's equivalence class. +// +// The number of distinct digests cannot exceed the number of digests, +// so we can allocate a large enough table upfront and never need to +// grow it. +// +// The map is reused across propagation rounds. Instead of clearing the +// table at the start of each round, we stamp each slot with the round +// number in which it was written; a slot stamped with an earlier round +// is treated as vacant. +// +// Each slot consists of two 64-bit words and a section pointer. The +// first word packs the round number, a busy bit, and 48 bits of the +// digest; the second word holds another 64 bits. An inserter claims a +// vacant slot by installing the first word with compare-and-swap with +// the busy bit set, writes the second word and the pointer, and then +// rewrites the first word with the busy bit cleared to publish the +// slot. Since the slot index is derived from digest bits that the +// first word doesn't contain, a successful match effectively compares +// an entire 128-bit digest, so the map is exact under the same +// hash-collision assumption the surrounding algorithm is built on. +// +// Of all sections inserted with the same digest in the same round, the +// slot ends up pointing to the one with the lowest priority, which ICF +// uses as the leader of the digest's equivalence class. +struct DigestMap { + // Time begins in round 1 so that all-zero slots, the initial state + // of the table, read as vacant. + round: u64, + mask: usize, + slots: Vec, +} + +#[derive(Default)] +struct DigestSlot { + hi: AtomicU64, + lo: AtomicU64, + leader: AtomicU64, +} + +impl DigestMap { + fn new(n: usize) -> DigestMap { + let len = n.saturating_mul(2).next_power_of_two(); + DigestMap { + round: 1, + mask: len - 1, + slots: (0..len).map(|_| DigestSlot::default()).collect(), + } + } + + fn next_round(&mut self) { + self.round += 1; + if self.round == 1 << 15 { + // The round number wrapped around, making stale slot stamps + // ambiguous, so reset the table. In practice, ICF converges long + // before this point. + self.slots + .par_iter() + .for_each(|slot| slot.hi.store(0, Ordering::Relaxed)); + self.round = 1; + } + } + + // Returns true if the digest was not in the table. + fn insert(&self, ctx: &Context, digest: Digest, isec: SectionRef) -> bool { + const BUSY_BIT: u64 = 1 << 48; + let tag = digest.hi >> 16; + let value = (self.round << 49) | tag; + + let mut i = digest.hi as usize & self.mask; + loop { + let slot = &self.slots[i]; + let mut x = slot.hi.load(Ordering::Acquire); + + // If the slot was last written in an earlier round, it's vacant; + // try to claim it. + while x >> 49 != self.round { + match slot.hi.compare_exchange_weak( + x, + value | BUSY_BIT, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => { + slot.lo.store(digest.lo, Ordering::Relaxed); + slot.leader.store(isec.encode(), Ordering::Relaxed); + slot.hi.store(value, Ordering::Release); + return true; + } + Err(actual) => x = actual, + } + } + + // The slot is occupied. If it holds a different digest, try the + // next slot. + if x & 0xffff_ffff_ffff != tag { + i = (i + 1) & self.mask; + continue; + } + + // The tags match; compare the digest bits in the second word, + // waiting for the writer to publish them if the slot is still + // being claimed. + while x & BUSY_BIT != 0 { + std::hint::spin_loop(); + x = slot.hi.load(Ordering::Acquire); + } + if slot.lo.load(Ordering::Relaxed) != digest.lo { + i = (i + 1) & self.mask; + continue; + } + + // The digest is already in the table; keep the slot pointing to + // the lowest-priority section. + let candidate_priority = ctx.section(isec).priority(&ctx.objs[isec.file.index()]); + let candidate = isec.encode(); + let mut cur = slot.leader.load(Ordering::Relaxed); + loop { + let current = SectionRef::decode(cur); + if candidate_priority + >= ctx + .section(current) + .priority(&ctx.objs[current.file.index()]) + { + break; + } + match slot.leader.compare_exchange_weak( + cur, + candidate, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => cur = actual, + } + } + return false; + } + } + + // Returns the section associated with a digest. The digest must have + // been inserted in the current round, and no insertion may be running + // concurrently. + fn find(&self, digest: Digest) -> SectionRef { + let value = (self.round << 49) | (digest.hi >> 16); + let mut i = digest.hi as usize & self.mask; + loop { + let slot = &self.slots[i]; + if slot.hi.load(Ordering::Relaxed) == value + && slot.lo.load(Ordering::Relaxed) == digest.lo + { + return SectionRef::decode(slot.leader.load(Ordering::Relaxed)); + } + i = (i + 1) & self.mask; + } + } +} + +fn uniquify_cies(ctx: &mut Context) { + let _t = ctx.timer("uniquify_cies"); + let mut leaders: Vec = Vec::new(); + for file in &mut ctx.objs { + let file = file.as_mut(); + let file_ptr = file as *mut ObjectFile; + let cies = file.cies.as_mut_ptr(); + for ci in 0..file.cies.len() { + // SAFETY: object files are boxed and ICF does not resize their CIE + // vectors, so leaders remain stable through this serial pass. + let cie = unsafe { crate::chunks::eh_frame::CieHandle::new(file_ptr, cies.add(ci)) }; + let found = leaders.iter().position(|&leader| leader.equals::(cie)); + match found { + Some(idx) => cie.icf_idx(idx as u32), + None => { + cie.icf_idx(leaders.len() as u32); + leaders.push(cie); + } + } + } + } +} + +fn is_eligible(ctx: &Context, isec: &InputSection) -> bool { + let file = &ctx.objs[isec.file.index()]; + let name: &[u8] = isec.name(file); + if isec.sh_size == 0 + || !isec.is_alloc() + || isec.sh_type(file) == SHT_NOBITS + || is_c_identifier(name) + { + return false; + } + if isec.sh_flags & SHF_EXECINSTR as u64 != 0 { + return (ctx.args.icf_all || !isec.is_address_taken()) + && name != b".init" + && name != b".fini"; + } + // .gcc_except_table holds compiler-generated tables whose pointer + // identity doesn't matter. + if name == b".gcc_except_table" || name.starts_with(b".gcc_except_table.") { + return true; + } + let is_readonly = isec.sh_flags & SHF_WRITE as u64 == 0; + let is_relro = name.starts_with(b".data.rel.ro"); + (ctx.args.ignore_data_address_equality || !isec.is_address_taken()) && (is_readonly || is_relro) +} + +fn compute_digest(ctx: &Context, key: &[u8; 16], r: SectionRef) -> Digest { + let file = &ctx.objs[r.file.index()]; + let isec = file.section_at(r.shndx); + let mut h = SipHash13_128::new(key); + + let hash_u32 = |h: &mut SipHash13_128, v: u32| h.update(&v.to_ne_bytes()); + let hash_u64 = |h: &mut SipHash13_128, v: u64| h.update(&v.to_ne_bytes()); + let hash_i64 = |h: &mut SipHash13_128, v: i64| h.update(&v.to_ne_bytes()); + let hash_bytes = |h: &mut SipHash13_128, b: &[u8]| { + h.update(&b.len().to_ne_bytes()); + h.update(b); + }; + let hash_symbol = |h: &mut SipHash13_128, id: crate::symbol::SymbolId, sym: &Symbol| { + if sym.file().is_none() || sym.is_imported() { + h.update(b"1"); + hash_u64(h, id.0 as u64); + } else if let Some(frag) = sym.fragment() { + h.update(b"2"); + hash_u64(h, ((frag.section.0 as u64) << 32) | frag.entry.raw() as u64); + } else if let Some(isec) = sym.input_section_ref() { + if isec.icf_index().is_some() { + h.update(b"4"); + } else { + h.update(b"5"); + let sec = SectionRef { + file: isec.file, + shndx: isec.shndx, + }; + hash_u64(h, ((sec.file.0 as u64) << 32) | sec.shndx as u64); + } + } else { + h.update(b"3"); + } + hash_u64(h, sym.value); + }; + + hash_bytes(&mut h, isec.contents()); + hash_u64(&mut h, isec.sh_flags); + let fdes = isec.fdes(file); + h.update(&fdes.len().to_ne_bytes()); + h.update(&isec.rels::(file).len().to_ne_bytes()); + + for fde in fdes { + let cie = &file.cies[fde.cie_idx as usize]; + hash_u32(&mut h, cie.icf_idx); + // Bytes 0..4 hold the length and 4..8 the CIE pointer. + hash_bytes(&mut h, &fde.contents::(file)[8..]); + let rels = fde.rels::(file); + h.update(&rels.len().to_ne_bytes()); + for rel in rels.iter().skip(1) { + let id = file.base.symbols[rel.r_sym as usize]; + hash_symbol(&mut h, id, &ctx.symbols[id]); + hash_u32(&mut h, rel.r_type); + hash_u64(&mut h, rel.r_offset - fde.input_offset as u64); + hash_i64(&mut h, file.section_at(cie.section).rel_addend::(&rel)); + } + } + + for rel in isec.rels::(file) { + hash_u64(&mut h, rel.r_offset); + hash_u32(&mut h, rel.r_type); + hash_i64(&mut h, isec.rel_addend::(&rel)); + let id = file.base.symbols[rel.r_sym as usize]; + hash_symbol(&mut h, id, &ctx.symbols[id]); + } + h.finish() +} + +fn gather_sections(ctx: &Context) -> Vec { + let _t = ctx.timer("gather_sections"); + + // Count the number of eligible input sections for each input file + // and turn the counts into starting indices with a prefix sum. + let counts: Vec = ctx + .objs + .par_iter() + .map(|file| { + let mut count = 0; + for isec in file.input_sections() { + if !isec.is_alive() { + continue; + } + if is_eligible(ctx, isec) { + isec.set_icf_index(0); + count += 1; + } + } + count + }) + .collect(); + + let mut file_indices = Vec::with_capacity(counts.len() + 1); + file_indices.push(0usize); + for count in counts { + file_indices.push(file_indices.last().unwrap() + count); + } + let total = *file_indices.last().unwrap(); + assert!( + u32::try_from(total).is_ok(), + "too many ICF-eligible sections" + ); + let mut sections = vec![ + SectionRef { + file: ObjId(0), + shndx: 0, + }; + total + ]; + + let mut rest = sections.as_mut_slice(); + let mut section_chunks = Vec::with_capacity(ctx.objs.len()); + for range in file_indices.windows(2) { + let (chunk, tail) = rest.split_at_mut(range[1] - range[0]); + section_chunks.push(chunk); + rest = tail; + } + + // Fill `sections` contents. + ctx.objs + .par_iter() + .zip(section_chunks.into_par_iter()) + .enumerate() + .for_each(|(fi, (file, out))| { + let base = file_indices[fi]; + let mut local = 0; + for isec in file.input_sections() { + if isec.icf_index().is_some() { + isec.set_icf_index((base + local) as u32); + out[local] = SectionRef { + file: ObjId(fi as u32), + shndx: isec.shndx, + }; + local += 1; + } + } + debug_assert_eq!(local, out.len()); + }); + + sections +} + +#[inline] +fn for_each_edge(ctx: &Context, r: SectionRef, mut f: impl FnMut(u32)) { + let file: &ObjectFile = &ctx.objs[r.file.index()]; + let isec = file.section_at(r.shndx); + let mut add = |sym_idx: u32| { + let sym = &ctx.symbols[file.base.symbols[sym_idx as usize]]; + if let Some(target) = sym.input_section_ref().and_then(InputSection::icf_index) { + f(target); + } + }; + for fde in isec.fdes(file) { + for rel in fde.rels::(file).iter().skip(1) { + add(rel.r_sym); + } + } + for rel in isec.rels::(file) { + add(rel.r_sym); + } +} + +struct Edges { + values: Vec, + indices: Vec, +} + +/// The pre-sized edge array filled in parallel at disjoint prefix-sum ranges. +struct EdgeBuffer(*mut u32); + +// SAFETY: `gather_edges` assigns each parallel vertex its own range from +// `indices[i]` to `indices[i + 1]`, exactly as C++ mold does. +unsafe impl Sync for EdgeBuffer {} + +impl EdgeBuffer { + /// # Safety + /// `index` must belong exclusively to the calling vertex's edge range. + #[inline] + unsafe fn write(&self, index: usize, edge: u32) { + // SAFETY: guaranteed by the caller and the prefix-sum ranges above. + unsafe { self.0.add(index).write(edge) }; + } +} + +// Build a graph, treating every function as a vertex and every function call +// as an edge. See the description at the top for a more detailed formulation. +// We use u32 indices here to improve cache locality. +// +// Relocations in a section's FDEs are edges too, because compute_digest +// hashes eligible relocation targets without identity, and every such +// target must be represented as an edge to remain distinguishable. In +// particular, an FDE's reference to an LSDA is an edge; without it, two +// identical functions whose exception tables catch different types would +// be folded into one. +fn gather_edges(ctx: &Context, sections: &[SectionRef]) -> Edges { + let _t = ctx.timer("gather_edges"); + + // Count the number of outgoing edges for each vertex and turn the + // counts into starting indices with a prefix sum. The extra entry at + // the end makes edge_indices[i + 1] valid for every vertex, so that + // vertex i's edges are edge_indices[i] to edge_indices[i + 1]. + let counts: Vec = sections + .par_iter() + .map(|&r| { + let mut count = 0u32; + for_each_edge::(ctx, r, |_| count += 1); + count + }) + .collect(); + + let mut indices = Vec::with_capacity(sections.len() + 1); + indices.push(0u32); + for count in counts { + indices.push( + indices + .last() + .unwrap() + .checked_add(count) + .expect("too many ICF edges"), + ); + } + + let mut values = vec![0; *indices.last().unwrap() as usize]; + let out = EdgeBuffer(values.as_mut_ptr()); + sections.par_iter().enumerate().for_each(|(vertex, &r)| { + let mut i = indices[vertex] as usize; + for_each_edge::(ctx, r, |edge| { + // SAFETY: this vertex alone owns its prefix-sum range. + unsafe { out.write(i, edge) }; + i += 1; + }); + debug_assert_eq!(i, indices[vertex + 1] as usize); + }); + + Edges { values, indices } +} + +// Compute the next-round digest of each vertex by hashing its current +// digest and the current digests of the vertices it refers to. A +// vertex's digest after the nth round is therefore a hash of its +// unfolding into a tree of depth n. +fn propagate(key: &[u8; 16], cur: &mut Vec, next: &mut Vec, edges: &Edges) { + next.par_iter_mut().enumerate().for_each(|(i, out)| { + let mut h = SipHash13_128::new(key); + h.update_digest(cur[i]); + let begin = edges.indices[i] as usize; + let end = edges.indices[i + 1] as usize; + for &j in &edges.values[begin..end] { + h.update_digest(cur[j as usize]); + } + *out = h.finish(); + }); + std::mem::swap(cur, next); +} + +fn count_num_classes( + ctx: &Context, + digests: &[Digest], + sections: &[SectionRef], + map: &mut DigestMap, +) -> usize { + map.next_round(); + digests + .par_iter() + .zip(sections) + .map(|(&digest, &isec)| usize::from(map.insert(ctx, digest, isec))) + .sum() +} + +fn print_icf_sections(ctx: &Context, sections: &[SectionRef]) { + let mut leaders: Vec<(SectionRef, Vec)> = Vec::new(); + let mut map: std::collections::HashMap = std::collections::HashMap::new(); + for &r in sections { + let leader = ctx.section(r).icf_leader_in_round(); + if leader == r { + map.entry(leader).or_insert_with(|| { + leaders.push((leader, Vec::new())); + leaders.len() - 1 + }); + } + } + for &r in sections { + let leader = ctx.section(r).icf_leader_in_round(); + if leader != r { + let idx = *map.entry(leader).or_insert_with(|| { + leaders.push((leader, Vec::new())); + leaders.len() - 1 + }); + leaders[idx].1.push(r); + } + } + leaders.sort_by_key(|(l, _)| ctx.section(*l).priority(&ctx.objs[l.file.index()])); + + let mut out = String::new(); + let mut saved = 0usize; + for (leader, members) in &leaders { + if members.is_empty() { + continue; + } + out.push_str(&format!( + "selected section {}\n", + ctx.section_display(*leader) + )); + for m in members { + out.push_str(&format!( + " removing identical section {}\n", + ctx.section_display(*m) + )); + saved += ctx.section(*leader).contents().len(); + } + } + out.push_str(&format!("ICF saved {saved} bytes\n")); + + let path = &ctx.args.print_icf_sections; + if path == "-" { + let _ = std::io::stdout().write_all(out.as_bytes()); + } else { + std::fs::write(path, out) + .unwrap_or_else(|e| fatal!(ctx, "--print-icf-sections: cannot open {path}: {e}")); + } +} + +pub fn icf_sections(ctx: &mut Context) { + let _t = ctx.timer("icf"); + if ctx.objs.is_empty() { + return; + } + let mut key = [0u8; 16]; + crate::util::random_bytes(&mut key); + + uniquify_cies(ctx); + let sections = gather_sections(ctx); + + // `digests` holds the current digest of each vertex. + let mut digests: Vec = { + let _t = ctx.timer("compute_digests"); + sections + .par_iter() + .map(|&r| compute_digest(ctx, &key, r)) + .collect() + }; + + let edges = gather_edges(ctx, §ions); + + // The digest map is used to count the number of distinct digests in + // the loop below. As a side effect, it records the lowest-priority + // section for each digest, which the grouping step after the loop + // uses as the leader of each equivalence class. + let mut map = DigestMap::new(digests.len()); + + // Execute the propagation rounds until convergence is obtained. + // + // The number of distinct digests can only monotonically increase as + // the rounds hash ever-deeper trees, so once two consecutive rounds + // yield the same count, the partition of sections into equivalence + // classes has stopped changing and will remain unchanged for further + // iterations (proof omitted for brevity). Note that individual + // digests may well still be changing at that point; sections that + // have a cycle in downstream (i.e. recursive functions and functions + // that call them) never settle on a digest. That doesn't matter + // because sections in the same class change their digests in + // lockstep, keeping the partition intact. + { + let _t = ctx.timer("propagate"); + let mut num_classes = usize::MAX; + let mut scratch = vec![Digest::default(); digests.len()]; + loop { + // count_num_classes is as expensive as propagate, so we propagate + // a few times before counting the number of distinct groups. + propagate(&key, &mut digests, &mut scratch, &edges); + propagate(&key, &mut digests, &mut scratch, &edges); + propagate(&key, &mut digests, &mut scratch, &edges); + + let n = count_num_classes(ctx, &digests, §ions, &mut map); + if n == num_classes { + break; + } + num_classes = n; + } + } + + // Group sections by digest. The final counting round has already + // elected a leader for each digest; look it up. + { + let _t = ctx.timer("group"); + sections + .par_iter() + .zip(&digests) + .for_each(|(&r, &digest)| ctx.section(r).set_icf_leader(map.find(digest))); + } + + if !ctx.args.print_icf_sections.is_empty() { + print_icf_sections(ctx, §ions); + } + + // Update alignment of leaders. + { + let _t = ctx.timer("update_alignment"); + sections.par_iter().for_each(|&r| { + let isec = ctx.section(r); + let leader = isec.icf_leader_in_round(); + if leader != r { + ctx.section(leader).update_p2align(isec.p2align()); + } + }); + } + + // Eliminate duplicate sections. + // Symbols pointing to eliminated sections will be redirected on the fly when + // exporting to the symtab. + { + let _t = ctx.timer("sweep"); + sections.par_iter().for_each(|&r| { + let isec = ctx.section(r); + if isec.icf_leader_in_round() != r { + isec.set_icf_removed(); + ctx.objs[r.file.index()].kill_section(r.shndx as usize); + } else { + isec.set_offset(u64::MAX); + } + }); + } +} diff --git a/src/input_files.rs b/src/input_files.rs new file mode 100644 index 000000000..02a3abb67 --- /dev/null +++ b/src/input_files.rs @@ -0,0 +1,3212 @@ +//! Input object files and shared libraries. + +// DWARF constants keep the spelling of the specification. +#![allow(non_upper_case_globals)] + +use std::collections::BTreeMap; +use std::fmt; +use std::marker::PhantomData; +use std::mem::MaybeUninit; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{OnceLock, RwLock}; + +use crate::arch::{Arch, Family}; +use crate::args::Args; +use crate::chunks::merged::MergedSection; +use crate::context::Context; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::input_sections::{ + CieRecord, FdeRecord, FragmentRef, InputSection, MergeableSection, RelocationSpan, SFrameFde, + SectionList, +}; +use crate::mapped_file::MappedFile; +use crate::symbol::{ + hash_key, Bins, ParallelSymbolAllocator, Symbol, SymbolId, SymbolSlot, SymbolTable, NEEDS_PLT, +}; +use crate::util::{ + self, align_to, bits, cstr_at, leak_bytes, path_clean, path_filename, read_uleb, +}; +use crate::{error, fatal, out, warn}; +use bstr::BStr; + +#[derive(Clone, Copy, Debug, Default)] +struct NameLen(u8); + +impl NameLen { + const LONG_NAME: usize = 240; + + #[inline] + fn new(len: usize) -> NameLen { + if len < Self::LONG_NAME { + return NameLen(len as u8); + } + let log2 = (len - Self::LONG_NAME + 1).ilog2() as usize; + NameLen((Self::LONG_NAME + log2.min(u8::MAX as usize - Self::LONG_NAME)) as u8) + } + + #[inline] + fn is_long(self) -> bool { + self.0 as usize >= Self::LONG_NAME + } + + #[inline] + fn lower_bound(self) -> usize { + if !self.is_long() { + return self.0 as usize; + } + Self::LONG_NAME - 1 + (1 << (self.0 as usize - Self::LONG_NAME)) + } + + #[inline] + fn get(self, strtab: &'static [u8], offset: usize) -> &'static [u8] { + let rest = strtab.get(offset..).unwrap_or(&[]); + let lower = self.lower_bound().min(rest.len()); + let len = if self.is_long() { + lower + cstr_at(rest, lower).len() + } else { + lower + }; + &rest[..len] + } +} + +/// Index of an object file in `Context::objs`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ObjId(pub u32); + +/// Index of a shared library in `Context::dsos`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DsoId(pub u32); + +impl ObjId { + pub fn index(self) -> usize { + self.0 as usize + } +} + +impl DsoId { + pub fn index(self) -> usize { + self.0 as usize + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum FileId { + Obj(ObjId), + Dso(DsoId), +} + +impl FileId { + pub fn is_dso(self) -> bool { + matches!(self, FileId::Dso(_)) + } + + pub fn as_obj(self) -> Option { + match self { + FileId::Obj(id) => Some(id), + FileId::Dso(_) => None, + } + } + + pub fn as_dso(self) -> Option { + match self { + FileId::Dso(id) => Some(id), + FileId::Obj(_) => None, + } + } +} + +impl From for FileId { + fn from(id: ObjId) -> FileId { + FileId::Obj(id) + } +} + +impl From for FileId { + fn from(id: DsoId) -> FileId { + FileId::Dso(id) + } +} + +/// State shared by object files and shared libraries. +#[derive(Debug)] +pub struct InputFile { + pub mf: Option<&'static MappedFile>, + pub filename: String, + + /// Position in the command line; lower is earlier. Symbol resolution + /// breaks ties in favor of earlier files. + pub priority: u32, + pub is_reachable: AtomicBool, + pub is_little_endian: bool, + pub e_flags: u32, + + pub shdrs: ShdrTable<'static>, + pub shstrtab: &'static [u8], + pub elf_syms: SymTable, + pub symbol_strtab: &'static [u8], + + // Parallel to elf_syms; avoids rescanning complete symbol names. + symbol_name_lengths: Vec, + + /// The symbol for each entry of `elf_syms`, plus any fragment dummies + /// appended after them. + pub symbols: Vec, + pub first_global: usize, + + pub as_needed: bool, + + // Output symbol table layout + pub local_symtab_idx: u32, + pub global_symtab_idx: u32, + pub num_local_symtab: u32, + pub num_global_symtab: u32, + pub strtab_offset: u64, + pub strtab_size: u64, + + /// Output symbol table index (relative to the file's block) of each + /// symbol, or -1 if not written. For `--emit-relocs`. + pub output_sym_indices: Vec, +} + +impl InputFile { + fn empty(filename: &str) -> InputFile { + InputFile { + mf: None, + filename: filename.to_string(), + priority: 0, + is_reachable: AtomicBool::new(false), + is_little_endian: true, + e_flags: 0, + shdrs: ShdrTable::default(), + shstrtab: &[], + elf_syms: SymTable::default(), + symbol_strtab: &[], + symbol_name_lengths: Vec::new(), + symbols: Vec::new(), + first_global: 0, + as_needed: false, + local_symtab_idx: 0, + global_symtab_idx: 0, + num_local_symtab: 0, + num_global_symtab: 0, + strtab_offset: 0, + strtab_size: 0, + output_sym_indices: Vec::new(), + } + } + + /// Reads the ELF and section headers. + fn parse( + diag: &Diagnostics, + mf: &'static MappedFile, + display: &dyn fmt::Display, + ) -> InputFile { + let data = mf.data(); + if data.len() < ElfEhdr::size::() { + fatal!(diag, "{display}: file too small"); + } + if !data.starts_with(b"\x7fELF") { + fatal!(diag, "{display}: not an ELF file"); + } + + let ehdr = ElfEhdr::parse::(data); + let shoff = ehdr.e_shoff as usize; + let shdr_size = ElfShdr::size::(); + + // e_shnum is a 16-bit field. A file with more than 65535 sections + // stores the real number in the first section header's sh_size. + let first = data.get(shoff..shoff + shdr_size).map(ElfShdr::parse::); + let num_sections = match (ehdr.e_shnum, &first) { + (0, Some(first)) => first.sh_size as usize, + (n, _) => n as usize, + }; + + let Some(shdr_bytes) = data.get(shoff..shoff + num_sections * shdr_size) else { + fatal!( + diag, + "{}: e_shoff or e_shnum corrupted: {} {num_sections}", + mf.name, + data.len() + ); + }; + let shdrs = ShdrTable::in_file(shdr_bytes, RecordLayout::of::()); + + let mut file = InputFile { + mf: Some(mf), + is_little_endian: E::IS_LITTLE_ENDIAN, + e_flags: ehdr.e_flags, + shdrs, + ..InputFile::empty(&mf.name) + }; + + // e_shstrndx is likewise a 16-bit field; a large index is stored + // in the first section header's sh_link. + let shstrtab_idx = if ehdr.e_shstrndx as u32 == SHN_XINDEX { + file.shdrs.get(0).map_or(0, |s| s.sh_link as usize) + } else { + ehdr.e_shstrndx as usize + }; + file.shstrtab = file.section_contents_checked(diag, shstrtab_idx, display); + file + } + + #[inline] + pub fn is_reachable(&self) -> bool { + self.is_reachable.load(Ordering::Relaxed) + } + + #[inline] + pub fn set_reachable(&self, value: bool) { + self.is_reachable.store(value, Ordering::Relaxed); + } + + /// Marks the file reachable, returning true if it wasn't already. + #[inline] + pub fn mark_reachable(&self) -> bool { + // A relaxed load + branch (assuming miss) takes only around 20 cycles, + // while an atomic RMW can easily take hundreds on x86. It is common + // that another thread beat us in marking, so test optimistically first. + if self.is_reachable.load(Ordering::Relaxed) { + return false; + } + !self.is_reachable.swap(true, Ordering::Relaxed) + } + + #[inline] + pub fn data(&self) -> &'static [u8] { + self.mf.map_or(&[], MappedFile::data) + } + + /// The contents of a section, checked against the file size. + #[inline] + pub fn section_contents(&self, diag: &Diagnostics, idx: usize) -> &'static [u8] { + self.section_contents_checked(diag, idx, &self.filename) + } + + fn section_contents_checked( + &self, + diag: &Diagnostics, + idx: usize, + display: &dyn fmt::Display, + ) -> &'static [u8] { + if idx >= self.shdrs.len() { + fatal!(diag, "{display}: invalid section index: {idx}"); + } + let (sh_offset, sh_size) = self.shdrs.sh_offset_and_size(idx); + self.section_contents_range_checked(diag, sh_offset, sh_size, display) + } + + /// The contents described by a section header already decoded by the + /// caller. This avoids decoding its offset and size a second time. + #[inline] + pub(crate) fn section_contents_from_shdr( + &self, + diag: &Diagnostics, + shdr: &ElfShdr, + ) -> &'static [u8] { + self.section_contents_range_checked(diag, shdr.sh_offset, shdr.sh_size, &self.filename) + } + + fn section_contents_range_checked( + &self, + diag: &Diagnostics, + sh_offset: u64, + sh_size: u64, + display: &dyn fmt::Display, + ) -> &'static [u8] { + let data = self.data(); + let start = sh_offset as usize; + let end = start.saturating_add(sh_size as usize); + if end > data.len() { + fatal!( + diag, + "{display}: section header is out of range: {sh_offset}" + ); + } + &data[start..end] + } + + #[inline] + pub fn find_section(&self, sh_type: u32) -> Option { + (0..self.shdrs.len()).find(|&i| self.shdrs.sh_type(i) == sh_type) + } + + #[inline] + pub fn section_name(&self, shndx: usize) -> &'static [u8] { + cstr_at(self.shstrtab, self.shdrs.sh_name(shndx) as usize) + } + + #[inline] + pub fn symbol_name(&self, i: usize) -> &'static [u8] { + let offset = self.elf_syms.st_name(i) as usize; + self.symbol_name_lengths.get(i).map_or_else( + || cstr_at(self.symbol_strtab, offset), + |len| len.get(self.symbol_strtab, offset), + ) + } + + /// Like [`Self::symbol_name`] for code specialized for the target. + #[inline(always)] + pub fn symbol_name_in(&self, i: usize) -> &'static [u8] { + let offset = self.elf_syms.st_name_in::(i) as usize; + if let Some(len) = self.symbol_name_lengths.get(i) { + len.get(self.symbol_strtab, offset) + } else { + cstr_at(self.symbol_strtab, offset) + } + } + + fn populate_symbol_name_lengths(&mut self) { + self.symbol_name_lengths = self + .elf_syms + .name_offsets_in::() + .map(|offset| { + let offset = offset as usize; + NameLen::new(cstr_at(self.symbol_strtab, offset).len()) + }) + .collect(); + } + + pub fn global_symbols(&self) -> &[SymbolId] { + &self.symbols[self.first_global.min(self.symbols.len())..] + } + + pub fn local_symbols(&self) -> &[SymbolId] { + &self.symbols[..self.first_global.min(self.symbols.len())] + } + + /// The source file name recorded as an STT_FILE symbol. + pub fn source_name(&self, symbols: &SymbolTable) -> Option<&'static [u8]> { + self.local_symbols() + .iter() + .map(|&id| &symbols[id]) + .find(|sym| sym.ty() == STT_FILE) + .map(|sym| { + let name: &'static BStr = sym.name(); + &**name + }) + } +} + +// A COMDAT group usually contains an inline function's code and related data, +// such as its string literals. Groups are identified by a signature. If two +// groups have the same signature, the linker keeps one and discards the +// sections in the other. +#[derive(Debug)] +pub struct ComdatGroupRef { + pub sect_idx: u32, + + // The vector backing comdat_groups is outside the arena. The symbol arena + // has fewer than 2^31 slots, leaving the high bit for is_owner. + signature_and_owner: u32, +} + +impl ComdatGroupRef { + const IS_OWNER: u32 = 1 << 31; + + fn new(sect_idx: u32, signature: SymbolId) -> ComdatGroupRef { + assert!(signature.0 < Self::IS_OWNER); + ComdatGroupRef { + sect_idx, + signature_and_owner: signature.0, + } + } + + #[inline] + pub fn signature(&self) -> SymbolId { + SymbolId(self.signature_and_owner & !Self::IS_OWNER) + } + + #[inline] + pub fn is_owner(&self) -> bool { + self.signature_and_owner & Self::IS_OWNER != 0 + } + + #[inline] + pub fn set_owner(&mut self, is_owner: bool) { + if is_owner { + self.signature_and_owner |= Self::IS_OWNER; + } else { + self.signature_and_owner &= !Self::IS_OWNER; + } + } + + /// Returns the word that receives an exceptional signature while COMDAT + /// metadata is gathered. Ownership has not been selected at that point. + #[inline] + pub(crate) fn signature_word_mut(&mut self) -> &mut u32 { + debug_assert!(!self.is_owner()); + &mut self.signature_and_owner + } +} + +const _: () = assert!(std::mem::size_of::() == 8); + +/// An exceptional COMDAT signature that must be interned by name. Most groups +/// use their file's existing global symbol and need only `ComdatGroupRef`. +#[derive(Debug)] +pub(crate) struct PendingComdatSignature { + pub key: &'static [u8], + pub group_idx: u32, + pub name_len: u32, +} + +const _: () = assert!(std::mem::size_of::() == 24); + +/// RISC-V attributes read from `.riscv.attributes`. +#[derive(Debug, Default)] +pub struct RiscvAttributes { + pub stack_align: Option, + pub arch: Option<&'static [u8]>, + pub unaligned_access: bool, +} + +/// An input relocatable object file. +#[derive(Debug)] +pub struct ObjectFile { + pub base: InputFile, + pub archive_name: String, + + /// The sections by section header index, plus sections synthesized + /// for common symbols. + pub sections: SectionList, + pub sections_parsed: bool, + + /// CREL relocation tables decoded into ordinary records, indexed by + /// relocation section. + decoded_crel: Vec>>, + + /// The number of section headers in the file; `base.shdrs` may have + /// synthesized headers appended after them. + pub num_elf_sections: usize, + + pub cies: Vec, + pub fdes: Vec, + pub has_symver: Vec, + pub comdat_groups: Vec, + pub(crate) pending_comdat_signatures: Vec, + pub comdat_discarded: Vec, + + pub eh_frame_sections: Vec, + pub sframe_sections: Vec, + pub sframe_fdes: Vec, + + pub exclude_libs: bool, + pub gnu_properties: BTreeMap, + pub needs_executable_stack: bool, + pub is_lto_input: bool, + pub is_lto_output: bool, + pub is_gcc_offload_obj: bool, + pub is_rust_obj: bool, + pub is_dwarf32: bool, + pub has_init_array: bool, + pub has_ctors: bool, + + // Output .eh_frame layout + pub fde_idx: u64, + pub fde_offset: u64, + pub fde_size: u64, + + /// `.llvm_addrsig`, for `--icf=safe`. + pub llvm_addrsig: Option, + + /// Non-allocated `.debug_info` sections. + pub debug_info_sections: Vec, + pub debug_pubnames: Option, + pub debug_pubtypes: Option, + + // For LTO + /// COMDAT group signatures of the symbols of an IR object, by symbol + /// index. IR objects have no sections, so their COMDAT groups are + /// tracked per symbol. + pub lto_comdat_keys: Vec>, + pub lto_comdat_signatures: Vec>, + pub lto_comdat_discarded: Vec, + + pub riscv_attributes: RiscvAttributes, + /// `.got2` for PPC32. + pub got2: Option, + + symtab_shndx: Vec, + num_frag_syms: usize, + num_common_symbols: u32, +} + +impl fmt::Display for ObjectFile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.archive_name.is_empty() { + write!(f, "{}", path_clean(&self.base.filename)) + } else { + write!( + f, + "{}({})", + path_clean(&self.archive_name), + self.base.filename + ) + } + } +} + +#[cold] +#[inline(never)] +fn invalid_relocation_symbol(file: &ObjectFile, r_sym: usize) -> ! { + panic!("{file}: invalid relocation symbol index {r_sym}") +} + +#[cold] +#[inline(never)] +fn invalid_relocation_section(file: &ObjectFile, r_sym: usize, shndx: usize) -> ! { + panic!("{file}: relocation symbol {r_sym} has invalid section index {shndx}") +} + +/// Formats a file the way it appears in diagnostics before the file +/// object exists. +struct FileName<'a>(&'a str, &'a str); + +impl fmt::Display for FileName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.1.is_empty() { + write!(f, "{}", path_clean(self.0)) + } else { + write!(f, "{}({})", path_clean(self.1), self.0) + } + } +} + +fn is_debug_section(shdr: &ElfShdr, name: &[u8]) -> bool { + shdr.sh_flags & SHF_ALLOC as u64 == 0 && name.starts_with(b".debug_") +} + +fn is_known_section_type(shdr: &ElfShdr) -> bool { + let ty = shdr.sh_type; + let flags = shdr.sh_flags as u32; + if matches!( + ty, + SHT_PROGBITS | SHT_NOTE | SHT_NOBITS | SHT_INIT_ARRAY | SHT_FINI_ARRAY | SHT_PREINIT_ARRAY + ) { + return true; + } + if (SHT_LOUSER..=SHT_HIUSER).contains(&ty) && flags & SHF_ALLOC == 0 { + return true; + } + if (SHT_LOOS..=SHT_HIOS).contains(&ty) && flags & SHF_OS_NONCONFORMING == 0 { + return true; + } + match E::FAMILY { + Family::X86_64 if ty == SHT_X86_64_UNWIND => true, + Family::Arm32 if ty == SHT_ARM_EXIDX || ty == SHT_ARM_ATTRIBUTES => true, + Family::RiscV if ty == SHT_RISCV_ATTRIBUTES => true, + _ => false, + } +} + +/// A streaming reader for CREL, an experimental compact relocation encoding +/// that only LLVM produces at the moment. +struct CrelReader<'a> { + data: &'a [u8], + nrels: usize, + scale: u32, + is_rela: bool, +} + +impl<'a> CrelReader<'a> { + fn new(diag: &Diagnostics, file: &dyn fmt::Display, mut data: &'a [u8]) -> Self { + let hdr = read_uleb(&mut data); + let is_rela = hdr & 0b100 != 0; + if is_rela && !E::IS_RELA { + fatal!( + diag, + "{file}: CREL with addends is not supported for {}", + E::NAME + ); + } + CrelReader { + data, + nrels: (hdr >> 3) as usize, + scale: (hdr & 0b11) as u32, + is_rela, + } + } + + fn len(&self) -> usize { + self.nrels + } + + fn for_each(&self, mut f: impl FnMut(ElfRel, usize)) { + let mut data = self.data; + let nflags = if self.is_rela { 3 } else { 2 }; + let (mut offset, mut r_type, mut r_sym, mut addend) = (0u64, 0i64, 0i64, 0i64); + + for i in 0..self.nrels { + let flags = data[0]; + data = &data[1..]; + + // The first byte combines flags with the low bits of an offset + // delta. A large delta continues as ULEB128 and can wrap the + // current offset. + let delta = if flags & 0x80 != 0 { + (read_uleb(&mut data) << (7 - nflags)) | ((flags & 0x7f) as u64 >> nflags) + } else { + (flags >> nflags) as u64 + }; + offset = offset.wrapping_add(delta << self.scale); + + if flags & 1 != 0 { + r_sym += util::read_sleb(&mut data); + } + if flags & 2 != 0 { + r_type += util::read_sleb(&mut data); + } + if self.is_rela && flags & 4 != 0 { + addend += util::read_sleb(&mut data); + } + + f(ElfRel::new(offset, r_type as u32, r_sym as u32, addend), i); + } + } +} + +// SHT_CREL is an experimental alternative relocation table format +// designed to reduce the size of the table. Only LLVM supports it +// at the moment. +// +// This function converts a CREL relocation table to a regular one. +fn decode_crel(diag: &Diagnostics, file: &dyn fmt::Display, data: &[u8]) -> Box<[ElfRel]> { + let reader = CrelReader::new::(diag, file, data); + // Own a fixed-size array without value-initializing elements that the + // caller is about to overwrite. + let mut rels = Box::<[ElfRel]>::new_uninit_slice(reader.len()); + reader.for_each(|rel, i| { + rels[i].write(rel); + }); + // SAFETY: CrelReader::for_each visits every index from zero to len once. + unsafe { rels.assume_init() } +} + +impl ObjectFile { + /// Creates the internal object file that holds linker-synthesized + /// symbols. + pub fn internal() -> ObjectFile { + let mut file = ObjectFile::with_base(InputFile::empty(""), String::new()); + file.sections_parsed = true; + file.base.set_reachable(true); + file + } + + fn with_base(base: InputFile, archive_name: String) -> ObjectFile { + ObjectFile { + num_elf_sections: base.shdrs.len(), + base, + archive_name, + sections: SectionList::default(), + sections_parsed: false, + decoded_crel: Vec::new(), + cies: Vec::new(), + fdes: Vec::new(), + has_symver: Vec::new(), + comdat_groups: Vec::new(), + pending_comdat_signatures: Vec::new(), + comdat_discarded: Vec::new(), + eh_frame_sections: Vec::new(), + sframe_sections: Vec::new(), + sframe_fdes: Vec::new(), + exclude_libs: false, + gnu_properties: BTreeMap::new(), + needs_executable_stack: false, + is_lto_input: false, + is_lto_output: false, + is_gcc_offload_obj: false, + is_rust_obj: false, + is_dwarf32: false, + has_init_array: false, + has_ctors: false, + fde_idx: 0, + fde_offset: 0, + fde_size: 0, + llvm_addrsig: None, + debug_info_sections: Vec::new(), + debug_pubnames: None, + debug_pubtypes: None, + lto_comdat_keys: Vec::new(), + lto_comdat_signatures: Vec::new(), + lto_comdat_discarded: Vec::new(), + riscv_attributes: RiscvAttributes::default(), + got2: None, + symtab_shndx: Vec::new(), + num_frag_syms: 0, + num_common_symbols: 0, + } + } + + /// Opens an object file and reads its symbol table. Sections are read + /// later, once COMDAT group selection is done. + pub fn new( + diag: &Diagnostics, + mf: &'static MappedFile, + archive_name: String, + ) -> ObjectFile { + let display = FileName(&mf.name, &archive_name); + let base = InputFile::parse::(diag, mf, &display); + let mut file = ObjectFile::with_base(base, archive_name); + file.parse_symbols::(diag); + file + } + + /// Creates the object for an IR file claimed by the LTO plugin. Its + /// symbols come from the plugin rather than from an ELF symbol table, + /// and it has no sections. + pub fn lto_input( + mf: &'static MappedFile, + archive_name: String, + elf_syms: SymTable, + strtab: &'static [u8], + comdat_keys: Vec>, + ) -> ObjectFile { + let mut base = InputFile::empty(&mf.name); + base.mf = Some(mf); + base.elf_syms = elf_syms; + base.symbol_strtab = strtab; + base.populate_symbol_name_lengths::(); + base.first_global = 1; + let mut file = ObjectFile::with_base(base, archive_name); + file.is_lto_input = true; + file.lto_comdat_signatures = vec![None; comdat_keys.len()]; + file.lto_comdat_discarded = vec![false; comdat_keys.len()]; + file.lto_comdat_keys = comdat_keys; + file + } + + pub fn id_display(&self) -> &dyn fmt::Display { + self + } + + /// The section index of the symbol at `idx`. Indices too large for + /// the 16-bit `st_shndx` field are stored in `.symtab_shndx`. + #[inline] + pub fn shndx_at(&self, idx: usize) -> usize { + let st_shndx = self.base.elf_syms.st_shndx(idx); + self.shndx_from(idx, st_shndx) + } + + /// Like [`Self::shndx_at`] for code specialized for the target. + #[inline] + pub fn shndx_at_in(&self, idx: usize) -> usize { + let st_shndx = self.base.elf_syms.st_shndx_in::(idx); + self.shndx_from(idx, st_shndx) + } + + /// Resolves an already-read symbol's section index. + #[inline] + pub(crate) fn shndx_from(&self, idx: usize, st_shndx: u16) -> usize { + if st_shndx as u32 == SHN_XINDEX { + self.symtab_shndx.get(idx).copied().unwrap_or(0) as usize + } else if st_shndx as u32 >= SHN_LORESERVE { + 0 + } else { + st_shndx as usize + } + } + + /// The section at `shndx`. A section that has been converted to a + /// mergeable one is returned as well, as a dead section: symbols + /// defined in it keep referring to it until they're attached to + /// fragments. + #[inline] + pub fn section(&self, shndx: usize) -> Option<&InputSection> { + self.sections.section(shndx) + } + + #[inline] + pub fn section_mut(&mut self, shndx: usize) -> Option<&mut InputSection> { + self.sections.section_mut(shndx) + } + + /// The regular section at `shndx`, which must exist. + #[inline] + pub fn section_at(&self, shndx: u32) -> &InputSection { + self.section(shndx as usize).expect("no such input section") + } + + /// Returns a relocation table from the file unless a compressed table + /// was decoded into the side table. + #[inline(always)] + pub(crate) fn relocations(&self, relsec_idx: Option) -> Rels<'_, E> { + let Some(relsec_idx) = relsec_idx else { + return Rels::new(&[]); + }; + if let Some(Some(rels)) = self.decoded_crel.get(relsec_idx as usize) { + return Rels::decoded(rels); + } + + Rels::new(self.input_relocation_data::(relsec_idx)) + } + + /// Visits relocations without materializing a deferred CREL table. + #[inline] + pub(crate) fn for_each_relocation( + &self, + diag: &Diagnostics, + relsec_idx: Option, + mut f: impl FnMut(ElfRel, usize), + ) { + let Some(relsec_idx) = relsec_idx else { + return; + }; + let index = relsec_idx as usize; + if self.base.shdrs.at_in::(index).sh_type == SHT_CREL + && !self.decoded_crel.get(index).is_some_and(Option::is_some) + { + let data = self.input_relocation_data::(relsec_idx); + CrelReader::new::(diag, self, data).for_each(f); + return; + } + + for (i, rel) in self.relocations::(Some(relsec_idx)).iter().enumerate() { + f(rel, i); + } + } + + #[inline(always)] + fn input_relocation_data(&self, relsec_idx: u32) -> &'static [u8] { + // Relocation sections are range-checked when sections are parsed. + let (offset, size) = self + .base + .shdrs + .sh_offset_and_size_in::(relsec_idx as usize); + &self.base.data()[offset as usize..(offset + size) as usize] + } + + #[inline] + fn relocation_span(&self, relsec_idx: Option) -> RelocationSpan { + let Some(relsec_idx) = relsec_idx else { + return RelocationSpan::Input(&[]); + }; + if self + .decoded_crel + .get(relsec_idx as usize) + .is_some_and(Option::is_some) + { + RelocationSpan::SideTable(relsec_idx) + } else { + RelocationSpan::Input(self.input_relocation_data::(relsec_idx)) + } + } + + /// Returns relocations for rewriting. Ordinary records live in the + /// input's private writable mapping; decoded CREL records use the side + /// table because they have no ordinary in-file representation. + pub fn rels_mut(&mut self, shndx: u32) -> RelsMut<'_, E> { + let Some(relsec_idx) = self.section_at(shndx).relsec_idx() else { + return RelsMut::new(&mut []); + }; + let index = relsec_idx as usize; + if self.decoded_crel.get(index).is_some_and(Option::is_some) { + return RelsMut::decoded(self.decoded_crel[index].as_deref_mut().unwrap()); + } + + let (offset, size) = self.base.shdrs.sh_offset_and_size_in::(index); + let mf = self + .base + .mf + .expect("input relocations without a mapped file"); + // SAFETY: a mutable ObjectFile owns this relocation section for the + // duration of the pass, and section parsing checked its range. + let data = unsafe { mf.data_mut_ptr(offset as usize..(offset + size) as usize) }; + // SAFETY: the same exclusive ownership applies while the returned + // relocation view is alive. + RelsMut::new(unsafe { &mut *data }) + } + + fn set_decoded_crel(&mut self, index: usize, rels: Box<[ElfRel]>) { + if self.decoded_crel.len() <= index { + self.decoded_crel.resize_with(index + 1, || None); + } + debug_assert!(self.decoded_crel[index].is_none()); + self.decoded_crel[index] = Some(rels); + } + + #[inline] + pub fn mergeable_section(&self, shndx: usize) -> Option<&MergeableSection> { + self.sections.mergeable(shndx) + } + + #[inline] + pub fn mergeable_section_mut(&mut self, shndx: usize) -> Option<&mut MergeableSection> { + self.sections.mergeable_mut(shndx) + } + + /// The section the symbol at `idx` is defined in. + #[inline] + pub fn symbol_section(&self, idx: usize) -> Option<&InputSection> { + self.section(self.shndx_at(idx)) + } + + /// Whether the symbol at `idx` is defined in a discarded COMDAT group. + #[inline] + pub fn is_discarded_comdat(&self, idx: usize) -> bool { + if self.comdat_discarded.is_empty() { + return false; + } + let st_shndx = self.base.elf_syms.st_shndx(idx) as u32; + if st_shndx == SHN_ABS || st_shndx == SHN_COMMON { + return false; + } + self.comdat_discarded[self.shndx_from(idx, st_shndx as u16)] + } + + #[inline] + fn is_discarded_comdat_sym(&self, idx: usize, esym: &ElfSym) -> bool { + if self.comdat_discarded.is_empty() || esym.is_abs() || esym.is_common() { + return false; + } + self.comdat_discarded[self.shndx_from(idx, esym.st_shndx)] + } + + /// Iterates over the live regular sections. + #[inline] + pub fn input_sections(&self) -> impl Iterator { + self.sections.regular() + } + + #[inline] + pub fn input_sections_mut(&mut self) -> impl Iterator { + self.sections.regular_mut() + } + + #[inline] + pub fn mergeable_sections(&self) -> impl Iterator { + self.sections.mergeable_sections() + } + + #[inline] + pub fn mergeable_sections_mut(&mut self) -> impl Iterator { + self.sections.mergeable_sections_mut() + } + + /// The section indices of a COMDAT group's members, read from the + /// group section as they are needed. + #[inline] + pub fn comdat_members(&self, group: &ComdatGroupRef) -> impl Iterator + '_ { + let data = self.base.data(); + let (sh_offset, sh_size) = self.base.shdrs.sh_offset_and_size(group.sect_idx as usize); + let start = sh_offset as usize; + let bytes = &data[start..start + sh_size as usize]; + let is_little_endian = self.base.is_little_endian; + bytes.chunks_exact(4).skip(1).map(move |b| { + let b = [b[0], b[1], b[2], b[3]]; + if is_little_endian { + u32::from_le_bytes(b) + } else { + u32::from_be_bytes(b) + } + }) + } + + /// Marks a section dead along with its FDEs. + pub fn kill_section(&self, shndx: usize) { + if let Some(isec) = self.section(shndx) { + if isec.kill() { + for fde in isec.fdes(self) { + fde.kill(); + } + } + } + } + + fn parse_symbols(&mut self, diag: &Diagnostics) { + if let Some(idx) = self.base.find_section(SHT_SYMTAB) { + let shdr = self.base.shdrs.at(idx); + // In ELF, all local symbols precede global symbols in the + // symbol table; sh_info is the index of the first global. + self.base.first_global = shdr.sh_info as usize; + let contents = self.base.section_contents(diag, idx); + if !contents.len().is_multiple_of(ElfSym::size::()) { + fatal!(diag, "{self}: corrupted section"); + } + self.base.elf_syms = SymTable::in_file(contents, RecordLayout::of::()); + self.base.symbol_strtab = self.base.section_contents(diag, shdr.sh_link as usize); + self.base.populate_symbol_name_lengths::(); + + if let Some(idx) = self.base.find_section(SHT_SYMTAB_SHNDX) { + let bytes = self.base.section_contents(diag, idx); + self.symtab_shndx = bytes.chunks_exact(4).map(E::Endian::read_u32).collect(); + } + } + } + + /// Registers the global symbols in this worker's symbol-table bin. The + /// complete slot array is allocated first and does not move before gather. + pub(crate) fn register_global_symbols( + &mut self, + args: &Args, + bins: &mut Bins, + ) { + let n = self.base.elf_syms.len(); + if n == 0 { + return; + } + self.base.symbols = vec![SymbolId::DISCARDED_COMDAT; n]; + let num_globals = n.saturating_sub(self.base.first_global); + self.has_symver = vec![false; num_globals]; + + for i in self.base.first_global..n { + let esym = self.base.elf_syms.at_in::(i); + if esym.is_common() { + self.num_common_symbols += 1; + } + + let mut key = self.base.symbol_name_in::(i); + let mut name = key; + + // Parse a symbol version after an atsign. A default version + // (`@@`) is dropped from the key; any other stays part of it, + // and its length is remembered so that the key's name needn't + // be scanned for the atsign again. + let mut ver_len = 0; + if let Some(pos) = util::find_byte(b'@', name) { + let ver = &name[pos..]; + name = &name[..pos]; + if ver.starts_with(b"@@") { + key = name; + } else { + ver_len = ver.len(); + } + if ver != b"@" { + self.has_symver[i - self.base.first_global] = true; + } + } + + // Handle --wrap + if esym.is_undef() && !args.wrap.is_empty() { + let as_str = |s: &[u8]| String::from_utf8_lossy(s).into_owned(); + if let Some(real) = name.strip_prefix(b"__real_") { + if args.wrap.contains(&as_str(real)) { + key = &key[7..]; + } + } else if args.wrap.contains(&as_str(key)) { + key = leak_bytes([b"__wrap_", key].concat()); + } + } + bins.record_hashed( + key, + hash_key(key), + key.len() - ver_len, + SymbolSlot::new(&mut self.base.symbols[i]), + ); + } + } + + /// Reads COMDAT groups and detects GCC offload objects. Both affect + /// which input sections can be discarded before LTO. + pub fn read_section_metadata(&mut self, diag: &Diagnostics) { + debug_assert!(!self.sections_parsed); + + for i in 0..self.num_elf_sections { + // SAFETY: `i` comes from the section-header table's range. + let (sh_type, sh_flags) = + unsafe { self.base.shdrs.file_type_and_flags_in_unchecked::(i) }; + + if sh_flags & SHF_EXCLUDE as u64 != 0 + && self + .base + .section_name(i) + .starts_with(b".gnu.offload_lto_.symtab.") + { + self.is_gcc_offload_obj = true; + } + + if sh_type != SHT_GROUP { + continue; + } + // SAFETY: `i` comes from the section-header table's range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + if shdr.sh_info as usize >= self.base.elf_syms.len() { + fatal!(diag, "{self}: invalid symbol index"); + } + + let esym = self.base.elf_syms.at(shdr.sh_info as usize); + let name = if esym.st_type() == STT_SECTION { + self.base + .section_name(self.shndx_from(shdr.sh_info as usize, esym.st_shndx)) + } else { + self.base.symbol_name_in::(shdr.sh_info as usize) + }; + + // Ignore a broken comdat group GCC emits for .debug_macros. + // https://github.com/rui314/mold/issues/438 + if name.starts_with(b"wm4.") { + continue; + } + + let contents = self.base.section_contents_from_shdr(diag, &shdr); + if contents.len() < 4 { + fatal!(diag, "{self}: empty SHT_GROUP"); + } + let kind = E::Endian::read_u32(contents); + if kind == 0 { + continue; + } + if kind != GRP_COMDAT { + fatal!(diag, "{self}: unsupported SHT_GROUP format"); + } + + // Ordinary global signatures already have a Symbol. Local, section + // and versioned signatures use the same symbol table through their + // full name. + let version = util::find_byte(b'@', name); + let is_own_global = esym.st_type() != STT_SECTION + && esym.st_bind() != STB_LOCAL + && !esym.is_undef() + && version.is_none(); + + let signature = if is_own_global { + self.base.symbols[shdr.sh_info as usize] + } else { + SymbolId::DISCARDED_COMDAT + }; + let group_idx = self.comdat_groups.len() as u32; + self.comdat_groups + .push(ComdatGroupRef::new(i as u32, signature)); + if !is_own_global { + self.pending_comdat_signatures.push(PendingComdatSignature { + key: name, + group_idx, + name_len: version.unwrap_or(name.len()) as u32, + }); + } + } + } + + /// Counts relocations referring to the section symbol of a mergeable + /// section; each gets a dummy symbol in `reattach_section_pieces`. + fn count_frag_syms(&self, rels: Rels<'_, E>) -> usize { + let mut count = 0; + for r_sym in rels.sym_indices().map(|i| i as usize) { + let Some(st_info) = self.base.elf_syms.st_info_in_checked::(r_sym) else { + invalid_relocation_symbol(self, r_sym); + }; + if (st_info & 0xf) as u32 != STT_SECTION { + continue; + } + + // SAFETY: `st_info_in_checked` just proved the symbol index. + let st_shndx = unsafe { self.base.elf_syms.st_shndx_in_unchecked::(r_sym) }; + let shndx = self.shndx_from(r_sym, st_shndx); + let Some(flags) = self.base.shdrs.sh_flags_in_checked::(shndx) else { + invalid_relocation_section(self, r_sym, shndx); + }; + count += usize::from(flags & SHF_MERGE as u64 != 0); + } + count + } + + fn parse_note_gnu_property(&mut self, mut data: &'static [u8]) { + while data.len() >= ElfNhdr::size::() { + let hdr = ElfNhdr::parse::(data); + data = &data[ElfNhdr::size::()..]; + + let name_len = hdr.n_namesz as usize; + let name = &data[..name_len.saturating_sub(1).min(data.len())]; + data = &data[(align_to(name_len as u64, 4) as usize).min(data.len())..]; + + let desc_len = hdr.n_descsz as usize; + let mut desc = &data[..desc_len.min(data.len())]; + data = + &data[(align_to(desc_len as u64, E::WORD_SIZE as u64) as usize).min(data.len())..]; + + if hdr.n_type != NT_GNU_PROPERTY_TYPE_0 || name != b"GNU" { + continue; + } + + while desc.len() >= 8 { + let ty = E::Endian::read_u32(desc); + let size = E::Endian::read_u32(&desc[4..]) as usize; + desc = &desc[8..]; + + // Most properties are 32-bit values; skip anything else + // (e.g. GNU_PROPERTY_STACK_SIZE). + if size == 4 && desc.len() >= 4 { + *self.gnu_properties.entry(ty).or_insert(0) |= E::Endian::read_u32(desc); + } + desc = + &desc[(align_to(size as u64, E::WORD_SIZE as u64) as usize).min(desc.len())..]; + } + } + } + + fn read_riscv_attributes(&mut self, diag: &Diagnostics, data: &'static [u8]) { + if data.is_empty() { + fatal!(diag, "{self}: corrupted .riscv.attributes section"); + } + if data[0] != b'A' { + return; + } + let mut data = &data[1..]; + + while !data.is_empty() { + let sz = E::Endian::read_u32(data) as usize; + if data.len() < sz || sz < 4 { + fatal!(diag, "{self}: corrupted .riscv.attributes section"); + } + let mut p = &data[4..sz]; + data = &data[sz..]; + + let Some(rest) = p.strip_prefix(b"riscv\0") else { + continue; + }; + p = rest; + if p.first() != Some(&(ELF_TAG_FILE as u8)) { + fatal!(diag, "{self}: corrupted .riscv.attributes section"); + } + p = &p[5..]; // skip the tag and the sub-sub-section size + + while !p.is_empty() { + let tag = read_uleb(&mut p) as u32; + match tag { + ELF_TAG_RISCV_STACK_ALIGN => { + self.riscv_attributes.stack_align = Some(read_uleb(&mut p)) + } + ELF_TAG_RISCV_ARCH => { + let end = p.iter().position(|&b| b == 0).unwrap_or(p.len()); + self.riscv_attributes.arch = Some(&p[..end]); + p = &p[(end + 1).min(p.len())..]; + } + ELF_TAG_RISCV_UNALIGNED_ACCESS => { + self.riscv_attributes.unaligned_access = read_uleb(&mut p) != 0 + } + _ => {} + } + } + } + } + + fn initialize_sections( + &mut self, + diag: &Diagnostics, + args: &Args, + id: ObjId, + section_arena: &crate::input_sections::SectionArena, + ) { + let nsections = self.num_elf_sections; + let expected_reloc_type = if E::IS_RELA { SHT_RELA } else { SHT_REL }; + debug_assert!(self.comdat_discarded.is_empty() || self.comdat_discarded.len() == nsections); + for i in 0..nsections { + if !self.comdat_discarded.is_empty() + // SAFETY: a nonempty COMDAT bitmap has one entry per input + // section, and `i` is in that range. + && unsafe { *self.comdat_discarded.get_unchecked(i) } + { + continue; + } + + // SAFETY: `i` comes from the file's section-header range. + let (sh_type, flags) = + unsafe { self.base.shdrs.file_type_and_flags_in_unchecked::(i) }; + if flags & SHF_EXCLUDE as u64 != 0 + && flags & SHF_ALLOC as u64 == 0 + && sh_type != SHT_LLVM_ADDRSIG + && !args.relocatable + { + continue; + } + + if E::IS_ARM && sh_type == SHT_ARM_ATTRIBUTES { + continue; + } + if E::IS_RISCV && sh_type == SHT_RISCV_ATTRIBUTES { + // SAFETY: `i` comes from the file's section-header range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + let contents = self.base.section_contents_from_shdr(diag, &shdr); + self.read_riscv_attributes::(diag, contents); + continue; + } + + match sh_type { + SHT_GROUP | SHT_SYMTAB | SHT_SYMTAB_SHNDX | SHT_STRTAB | SHT_NULL => {} + SHT_REL | SHT_RELA => { + if sh_type != expected_reloc_type { + continue; + } + // SAFETY: `i` comes from the file's section-header range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + let target = shdr.sh_info as usize; + let Some(target_flags) = self.base.shdrs.sh_flags_in_checked::(target) + else { + continue; + }; + if target_flags & SHF_ALLOC as u64 != 0 { + let contents = self.base.section_contents_from_shdr(diag, &shdr); + if !contents.len().is_multiple_of(ElfRel::size::()) { + fatal!(diag, "{self}: corrupted section"); + } + self.num_frag_syms += self.count_frag_syms(Rels::::new(contents)); + } + // Relocations are attached to their sections below. + } + SHT_CREL => { + // SAFETY: `i` comes from the file's section-header range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + let target = shdr.sh_info as usize; + let Some(target_flags) = self.base.shdrs.sh_flags_in_checked::(target) + else { + continue; + }; + let target_is_alloc = target_flags & SHF_ALLOC as u64 != 0; + if target_is_alloc || args.relocatable || args.emit_relocs { + let contents = self.base.section_contents_from_shdr(diag, &shdr); + let decoded = decode_crel::(diag, self, contents); + + // Count the relocations just decoded while they are in cache. + if target_is_alloc { + self.num_frag_syms += + self.count_frag_syms(Rels::::decoded(&decoded)); + } + self.set_decoded_crel(i, decoded); + } + // Relocations are attached to their sections below. + } + _ => { + // SAFETY: `i` comes from the file's section-header range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + let name = cstr_at(self.base.shstrtab, shdr.sh_name as usize); + if !is_known_section_type::(&shdr) { + fatal!( + diag, + "{self}: {}: unsupported section type: 0x{:x}", + util::display(name), + shdr.sh_type + ); + } + + // .note.GNU-stack controls executable-ness of the stack + // in GNU linkers. We ignore it because silently making + // the stack executable is too dangerous, but tell the + // user if that matters. + if name == b".note.GNU-stack" && !args.relocatable { + if flags & SHF_EXECINSTR as u64 != 0 { + if !args.z_execstack && !args.z_execstack_if_needed { + warn!( + diag, + "{self}: this file may cause a segmentation fault because it requires an executable stack. See https://github.com/rui314/mold/tree/main/docs/execstack.md for more info." + ); + } + self.needs_executable_stack = true; + } + continue; + } + + if name == b".note.gnu.property" { + let contents = self.base.section_contents_from_shdr(diag, &shdr); + self.parse_note_gnu_property::(contents); + continue; + } + + // A build-id section in an input file is unusual but + // possible (`ld.bfd -r --build-id`). + if name == b".note.gnu.build-id" { + continue; + } + + // Old glibc i386 CRT files and ICC emit these. + if name == b".gnu.linkonce.t.__x86.get_pc_thunk.bx" + || name == b".gnu.linkonce.t.__i686.get_pc_thunk.bx" + || name == b".gnu.linkonce.d.DW.ref.__gxx_personality_v0" + { + continue; + } + + if (args.strip_all || args.strip_debug) && is_debug_section(&shdr, name) { + continue; + } + + if !args.discard_section.is_empty() + && args + .discard_section + .contains(&*String::from_utf8_lossy(name)) + { + continue; + } + + if name == b".comment" + && self + .base + .section_contents_from_shdr(diag, &shdr) + .starts_with(b"rustc ") + { + self.is_rust_obj = true; + } + + // Without a section header (--oformat=binary), non-alloc + // sections have no place in the output. + if args.oformat_binary && flags & SHF_ALLOC as u64 == 0 { + continue; + } + + let isec = + InputSection::new::(diag, self, id, i as u32, &shdr, BStr::new(name)); + + // Save .llvm_addrsig for --icf=safe. Tools that mutate the + // symbol table tend not to preserve sh_link, so a section + // with sh_link == 0 is ignored. + if shdr.sh_type == SHT_LLVM_ADDRSIG && !args.relocatable { + if shdr.sh_link != 0 { + self.llvm_addrsig = Some(isec); + } + continue; + } + + if matches!( + shdr.sh_type, + SHT_INIT_ARRAY | SHT_FINI_ARRAY | SHT_PREINIT_ARRAY + ) { + self.has_init_array = true; + } + if name == b".ctors" + || name.starts_with(b".ctors.") + || name == b".dtors" + || name.starts_with(b".dtors.") + { + self.has_ctors = true; + } + if name == b".eh_frame" { + self.eh_frame_sections.push(i as u32); + } + if E::SUPPORTS_SFRAME && name == b".sframe" { + self.sframe_sections.push(i as u32); + } + if name == b".debug_info" && flags & SHF_ALLOC as u64 == 0 { + self.debug_info_sections.push(i as u32); + } + if E::FAMILY == Family::Ppc32 && name == b".got2" { + self.got2 = Some(i as u32); + } + + // With --gdb-index, .debug_gnu_pubnames and + // .debug_gnu_pubtypes are consumed by .gdb_index. + if args.gdb_index { + if name == b".debug_gnu_pubnames" { + self.debug_pubnames = Some(i as u32); + isec.kill(); + } + if name == b".debug_gnu_pubtypes" { + self.debug_pubtypes = Some(i as u32); + isec.kill(); + } + if name == b".debug_types" { + fatal!( + diag, + "{self}: mold's --gdb-index is not compatible with .debug_types; to fix this error, remove -fdebug-types-section and recompile" + ); + } + } + + self.sections.insert(i, isec, section_arena); + } + } + } + + // Attach relocation tables to their sections. + for i in 0..nsections { + // SAFETY: `i` comes from the file's section-header range. + let sh_type = unsafe { self.base.shdrs.file_type_in_unchecked::(i) }; + if sh_type != expected_reloc_type && sh_type != SHT_CREL { + continue; + } + // SAFETY: `i` comes from the file's section-header range. + let shdr = unsafe { self.base.shdrs.file_at_in_unchecked::(i) }; + let target = shdr.sh_info as usize; + if self.section(target).is_none() { + continue; + } + + let has_relocs = if sh_type == SHT_CREL { + if let Some(Some(decoded)) = self.decoded_crel.get(i) { + !decoded.is_empty() + } else { + let contents = self.base.section_contents_from_shdr(diag, &shdr); + CrelReader::new::(diag, self, contents).len() != 0 + } + } else { + shdr.sh_size != 0 + }; + + let isec = self.section_mut(target).unwrap(); + debug_assert!(!isec.has_relsec()); + isec.set_relsec(i as u32, has_relocs); + } + + // Attach .ARM.exidx sections to the sections they describe. + if E::FAMILY == Family::Arm32 { + let pairs: Vec<(usize, usize)> = self + .input_sections() + .filter(|isec| isec.sh_type(self) == SHT_ARM_EXIDX) + .map(|isec| { + ( + self.base.shdrs.at(isec.shndx as usize).sh_link as usize, + isec.shndx as usize, + ) + }) + .collect(); + for (target, exidx) in pairs { + if let Some(isec) = self.section_mut(target) { + isec.set_exidx(exidx as u32, section_arena); + } + } + } + } + + /// Relocations are usually sorted by offset, but RISC-V and LoongArch + /// object files don't always follow that convention. + fn sort_relocations(&mut self) { + if !E::IS_RISCV && !E::IS_LOONGARCH { + return; + } + let sections: Vec = self.input_sections().map(|isec| isec.shndx).collect(); + for shndx in sections { + let isec = self.section_at(shndx); + if !isec.is_alive() || !isec.is_alloc() { + continue; + } + let rels = isec.rels::(self); + if !rels.iter().map(|r| r.r_offset).is_sorted() { + let mut sorted: Vec = rels.iter().collect(); + sorted.sort_by_key(|r| r.r_offset); + self.rels_mut::(shndx).set_all(&sorted); + } + } + } + + /// Constructs sections after COMDAT ownership is known. Members of losing + /// groups are normally skipped. If another selection will run after LTO, + /// construct them too so a different copy can become live. + pub(crate) fn parse_sections( + &mut self, + diag: &Diagnostics, + args: &Args, + id: ObjId, + section_arena: &crate::input_sections::SectionArena, + allocator: &ParallelSymbolAllocator<'_>, + keep_discarded_comdat: bool, + ) { + debug_assert!(!self.sections_parsed); + let n = self.base.shdrs.len(); + self.sections = SectionList::new(n, self.num_common_symbols as usize, section_arena); + + if !keep_discarded_comdat && !self.comdat_groups.is_empty() { + let mut discarded = vec![false; n]; + for group in self.comdat_groups.iter().filter(|group| !group.is_owner()) { + for member in self.comdat_members(group) { + if let Some(slot) = discarded.get_mut(member as usize) { + *slot = true; + } + } + } + self.comdat_discarded = discarded; + } + + let count = self.num_local_symbols(); + // SAFETY: initialize_local_symbols fills all `count` slots, including + // any unused tail, before the parallel allocator publishes the final + // symbol-table length. + unsafe { + allocator.allocate(count, |base_id, slots| { + self.initialize_sections::(diag, args, id, section_arena); + self.initialize_local_symbols::(diag, id, base_id, slots); + self.sort_relocations::(); + self.sections_parsed = true; + }); + } + } + + /// The number of local symbols this file contributes to the symbol + /// table: the null symbol plus every local not in a discarded COMDAT. + #[inline] + pub fn num_local_symbols(&self) -> usize { + if self.base.elf_syms.is_empty() { + return 0; + } + 1 + (1..self.base.first_global) + .filter(|&i| !self.is_discarded_comdat(i)) + .count() + } + + /// Fills in the local symbols. `slots` has `num_local_symbols()` + /// entries whose ids start at `base_id`; every one of them is + /// initialized, those left over with blank symbols. + pub fn initialize_local_symbols( + &mut self, + diag: &Diagnostics, + id: ObjId, + base_id: SymbolId, + slots: &mut [MaybeUninit], + ) { + let mut next = 0; + if !slots.is_empty() && !self.base.elf_syms.is_empty() { + next = self.initialize_local_symbols_into::(diag, id, base_id, slots); + } + for slot in &mut slots[next..] { + slot.write(Symbol::new(BStr::new(b""))); + } + } + + /// Returns the number of slots written. + fn initialize_local_symbols_into( + &mut self, + diag: &Diagnostics, + id: ObjId, + base_id: SymbolId, + slots: &mut [MaybeUninit], + ) -> usize { + let file_id = FileId::Obj(id); + + let mut first = Symbol::new(BStr::new(b"")); + first.set_file(file_id); + first.sym_idx = 0; + slots[0].write(first); + self.base.symbols[0] = base_id; + + let mut next = 1; + for i in 1..self.base.first_global { + let esym = self.base.elf_syms.at_in::(i); + if esym.is_common() { + fatal!(diag, "{self}: common local symbol?"); + } + if self.is_discarded_comdat_sym(i, &esym) { + self.base.symbols[i] = SymbolId::DISCARDED_COMDAT; + continue; + } + + let shndx = (!esym.is_abs()).then(|| self.shndx_from(i, esym.st_shndx)); + + let name: &'static [u8] = if esym.st_type() == STT_SECTION { + let shndx = shndx.unwrap(); + match self.section(shndx) { + Some(isec) => isec.name(self), + None => self.base.section_name(shndx), + } + } else { + self.base.symbol_name_in::(i) + }; + + let mut sym = Symbol::new(BStr::new(name)); + sym.set_file(file_id); + sym.value = esym.st_value; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.set_rust(self.is_rust_obj); + if let Some(shndx) = shndx { + if let Some(section) = self.section(shndx) { + sym.set_input_section(section); + } + } + slots[next].write(sym); + self.base.symbols[i] = SymbolId(base_id.0 + next as u32); + next += 1; + } + debug_assert_eq!(next, slots.len()); + next + } + + /// Parses `.eh_frame` sections into CIE and FDE records. + /// + /// Unlike most sections, `.eh_frame` is not copied opaquely: FDEs of + /// dead functions are dropped, identical CIEs are merged, and the + /// records are indexed for `.eh_frame_hdr`. + pub fn parse_ehframe(&mut self, diag: &Diagnostics) { + let eh_frame_sections = std::mem::take(&mut self.eh_frame_sections); + for &shndx in &eh_frame_sections { + let isec = self.section_at(shndx); + let contents = isec.contents(); + let relocations = self.relocation_span::(isec.relsec_idx()); + let rels = isec.rels::(self); + let cies_begin = self.cies.len(); + let fdes_begin = self.fdes.len(); + let mut new_cies: Vec = Vec::new(); + let mut new_fdes: Vec = Vec::new(); + + let mut rel_idx = 0; + let mut pos = 0; + while pos + 4 <= contents.len() { + let size = E::Endian::read_u32(&contents[pos..]) as usize; + if size == 0 { + break; + } + let begin_offset = pos; + let end_offset = pos + size + 4; + let id = E::Endian::read_u32(&contents[pos + 4..]); + pos = end_offset; + + let rel_begin = rel_idx; + while rel_idx < rels.len() && (rels.at(rel_idx).r_offset as usize) < end_offset { + rel_idx += 1; + } + + if id == 0 { + let mut cie = CieRecord { + section: shndx, + contents, + input_offset: begin_offset as u32, + output_offset: 0, + rel_idx: rel_begin as u32, + relocations, + icf_idx: 0, + fde_ptr_size: 0, + is_leader: false, + }; + cie.fde_ptr_size = parse_fde_encoding::( + diag, + self, + isec, + &contents[begin_offset..end_offset], + ); + new_cies.push(cie); + } else { + // An FDE without a valid relocation is dead from the + // start; `ld -r` tends to produce such FDEs. + if rel_begin == rel_idx || rels.at(rel_begin).r_sym == 0 { + continue; + } + if rels.at(rel_begin).r_offset as usize - begin_offset != 8 { + fatal!( + diag, + "{}: FDE's first relocation should have offset 8", + isec.display(self) + ); + } + // The function may belong to a discarded COMDAT group. + if self + .symbol_section(rels.at(rel_begin).r_sym as usize) + .is_none() + { + continue; + } + new_fdes.push(FdeRecord::new(begin_offset as u32, rel_begin as u32)); + } + } + + // Associate CIEs with FDEs. + for fde in &mut new_fdes { + let off = fde.input_offset as usize + 4; + let cie_offset = E::Endian::read_i32(&contents[off..]) as i64; + let target = off as i64 - cie_offset; + let Some(ci) = new_cies + .iter() + .position(|c| c.input_offset as i64 == target) + else { + fatal!(diag, "{}: bad FDE pointer", isec.display(self)); + }; + fde.cie_idx = (cies_begin + ci) as u16; + } + + self.cies.extend(new_cies); + self.fdes.extend(new_fdes); + let _ = fdes_begin; + self.kill_section(shndx as usize); + } + self.eh_frame_sections = eh_frame_sections; + + // Group FDEs by the section they describe, keeping FDEs of the + // same section contiguous. + let section_of = |file: &ObjectFile, fde: &FdeRecord| -> usize { + let rel = fde.rels::(file).at(0); + file.shndx_at_in::(rel.r_sym as usize) + }; + let mut order: Vec<(u64, usize, usize)> = self + .fdes + .iter() + .enumerate() + .map(|(i, fde)| { + let shndx = section_of(self, fde); + (self.section_at(shndx as u32).priority(self), i, shndx) + }) + .collect(); + order.sort(); + let fdes = std::mem::take(&mut self.fdes); + let mut sorted: Vec> = fdes.into_iter().map(Some).collect(); + self.fdes = order + .iter() + .map(|&(_, i, _)| sorted[i].take().unwrap()) + .collect(); + let shndxs: Vec = order.iter().map(|&(_, _, s)| s).collect(); + + // Associate FDEs with input sections. + let mut i = 0; + while i < shndxs.len() { + let begin = i; + let shndx = shndxs[i]; + while i < shndxs.len() && shndxs[i] == shndx { + i += 1; + } + self.fdes[i - 1].is_last = true; + if self.section_at(shndx as u32).is_alive() { + self.section_mut(shndx).unwrap().fde_begin = begin as u32; + } else { + for fde in &self.fdes[begin..i] { + fde.kill(); + } + } + } + } + + /// Parses `.sframe` sections into function descriptor entries. + pub fn parse_sframe(&mut self, diag: &Diagnostics) { + let Some(abi) = E::SFRAME_ABI else { + return; + }; + let sections = std::mem::take(&mut self.sframe_sections); + for &shndx in §ions { + // The section is reconstructed into the output .sframe. + self.kill_section(shndx as usize); + let isec = self.section_at(shndx); + let data = isec.contents(); + + if data.len() < SFrameHeader::size::() { + fatal!(diag, "{}: corrupted .sframe section", isec.display(self)); + } + let hdr = SFrameHeader::parse::(data); + if hdr.magic != SFRAME_MAGIC { + fatal!(diag, "{}: corrupted .sframe section", isec.display(self)); + } + if hdr.abi_arch != abi { + fatal!( + diag, + "{}: .sframe is incompatible with {}", + isec.display(self), + E::NAME + ); + } + // Only SFrame version 3 is supported; older sections are + // ignored rather than rejected. + if hdr.version != 3 { + continue; + } + + let hdr_len = SFrameHeader::size::() + hdr.auxhdr_len as usize; + let fde_off = hdr_len + hdr.fdeoff as usize; + let fre_off = hdr_len + hdr.freoff as usize; + let rels = isec.rels::(self); + let mut rel_idx = 0; + let mut new_fdes = Vec::new(); + + for i in 0..hdr.num_fdes as usize { + let idx_off = fde_off + i * SFrameFdeIdx::size::(); + let ent = SFrameFdeIdx::parse::(&data[idx_off..]); + + // An FDE without a func_start relocation isn't tied to any + // function, so drop it. + while rel_idx < rels.len() && (rels.at(rel_idx).r_offset as usize) < idx_off { + rel_idx += 1; + } + if rel_idx == rels.len() || rels.at(rel_idx).r_offset as usize != idx_off { + continue; + } + let rel = &rels.at(rel_idx); + let off = fre_off + ent.func_start_fre_off as usize; + + let Some(func) = self.symbol_section(rel.r_sym as usize) else { + continue; + }; + let fre = &data[off..off + sframe_fre_block_size::(data, off)]; + new_fdes.push(SFrameFde { + section: func.shndx, + sym: self.base.symbols[rel.r_sym as usize], + addend: rel.r_addend, + fre, + func_size: ent.func_size, + num_fres: E::Endian::read_u16(&data[off..]) as u32, + }); + } + self.sframe_fdes.extend(new_fdes); + } + self.sframe_sections = sections; + } + + /// Converts sections with SHF_MERGE into mergeable sections. Files + /// are converted in parallel, so the merged sections are behind a + /// lock while they are being looked up and created. + pub fn convert_mergeable_sections( + &mut self, + ctx_args: &Args, + merged: &RwLock>, + diag: &Diagnostics, + ) { + let mut sections = std::mem::take(&mut self.sections); + for i in 0..sections.len() { + let Some(isec) = sections.section(i) else { + continue; + }; + if isec.sh_size == 0 || isec.has_relsec() || isec.sh_flags & SHF_MERGE as u64 == 0 { + continue; + } + let name = isec.name(self); + let Some(parent) = + MergedSection::get_instance(ctx_args, merged, name, &self.base.shdrs.at_in::(i)) + else { + continue; + }; + sections + .regular_section_mut(i) + .expect("a regular section") + .uncompress::( + diag, + self, + name, + self.base.shdrs.sh_offset_and_size(i).1 as usize, + ); + sections.set_mergeable(i, parent); + } + self.sections = sections; + } + + /// Attaches symbols in mergeable sections to the section fragments they + /// refer to. + /// + /// Section GC and ICF work on a graph of sections and fragments, so + /// every non-absolute symbol must refer to a non-mergeable section or + /// a fragment. This is done only for SHF_ALLOC sections, since GC and + /// ICF work only on them. + pub(crate) fn reattach_section_symbols( + &self, + diag: &Diagnostics, + id: ObjId, + symbols: &SymbolEditor<'_>, + merged: &[MergedSection], + ) { + for i in 1..self.base.elf_syms.len() { + let esym = self.base.elf_syms.at_in::(i); + if esym.is_abs() || esym.is_common() || esym.is_undef() { + continue; + } + let sym_id = self.base.symbols[i]; + let shndx = self.shndx_from(i, esym.st_shndx); + let Some(m) = self.mergeable_section(shndx) else { + continue; + }; + if !merged[m.parent.index()].resolved { + continue; + } + let Some((frag, offset)) = m.fragment(esym.st_value) else { + fatal!(diag, "{self}: bad symbol value: {}", esym.st_value); + }; + let frag = FragmentRef { + section: m.parent, + entry: frag, + }; + symbols.with_symbol(sym_id, |sym| { + if sym.file() != Some(FileId::Obj(id)) { + return; + } + sym.set_fragment(frag); + sym.value = offset as u64; + }); + } + } + + pub(crate) fn num_fragment_dummies(&self) -> usize { + self.num_frag_syms + } + + /// For each relocation referring to a mergeable section symbol, creates + /// a dummy non-section symbol and redirects the relocation to it. + pub(crate) fn reattach_fragment_relocations( + &mut self, + diag: &Diagnostics, + id: ObjId, + merged: &[MergedSection], + base_id: SymbolId, + slots: &mut [MaybeUninit], + ) { + debug_assert_eq!(slots.len(), self.num_frag_syms); + self.base.symbols.reserve(slots.len()); + let mut next = 0; + + for shndx in 0..self.sections.len() { + let (relsec_idx, contents) = { + let Some(isec) = self.section(shndx) else { + continue; + }; + if !isec.is_alloc() { + continue; + } + let Some(relsec_idx) = isec.relsec_idx() else { + continue; + }; + (relsec_idx, isec.contents()) + }; + + // Rewrite a relocation table in one visit, as the in-memory + // ElfRel array is updated in place in C++. Ordinary records live + // in the private input mapping; decoded CREL records remain in + // the side table. + let relsec_idx = relsec_idx as usize; + let mut decoded = self.decoded_crel.get_mut(relsec_idx).and_then(Option::take); + let mut rels = match decoded.as_deref_mut() { + Some(data) => RelsMut::::decoded(data), + None => { + let (offset, size) = self.base.shdrs.sh_offset_and_size_in::(relsec_idx); + let mf = self + .base + .mf + .expect("input relocations without a mapped file"); + // SAFETY: this file-parallel pass exclusively owns the + // relocation section, whose range was checked at parse + // time. No overlapping shared slice is used here. + let data = + unsafe { mf.data_mut_ptr(offset as usize..(offset + size) as usize) }; + // SAFETY: the exclusive access described above lasts + // until this relocation view is dropped. + RelsMut::::new(unsafe { &mut *data }) + } + }; + + for ri in 0..rels.len() { + let r_sym = rels.r_sym(ri) as usize; + if (self.base.elf_syms.st_info_in::(r_sym) & 0xf) as u32 != STT_SECTION { + continue; + } + let found = { + let esym = self.base.elf_syms.at_in::(r_sym); + let sym_shndx = self.shndx_from(r_sym, esym.st_shndx); + self.mergeable_section(sym_shndx).map(|m| { + debug_assert!(merged[m.parent.index()].resolved); + let rel = rels.at(ri); + let addend = if E::IS_RELA && E::FAMILY != Family::Sh4 { + rel.r_addend + } else { + E::get_addend(&contents[rel.r_offset as usize..], &rel) + }; + let Some((frag, in_frag_offset)) = + m.fragment(esym.st_value.wrapping_add(addend as u64)) + else { + fatal!(diag, "{self}: bad relocation at {}", rel.r_sym); + }; + ( + rel, + FragmentRef { + section: m.parent, + entry: frag, + }, + (in_frag_offset - addend) as u64, + ) + }) + }; + let Some((rel, frag, value)) = found else { + continue; + }; + + let dummy_idx = self.base.symbols.len() as u32; + self.base.symbols.push(SymbolId(base_id.0 + next as u32)); + + let mut sym = Symbol::new(BStr::new(b"")); + sym.set_file(FileId::Obj(id)); + sym.set_fragment_dummy(true); + sym.sym_idx = rel.r_sym; + sym.set_esym(&self.base.elf_syms.at(rel.r_sym as usize)); + sym.set_visibility(STV_HIDDEN); + sym.set_fragment(frag); + sym.value = value; + slots[next].write(sym); + next += 1; + + rels.set_r_sym(ri, dummy_idx); + } + + if let Some(data) = decoded { + debug_assert!(self.decoded_crel[relsec_idx].is_none()); + self.decoded_crel[relsec_idx] = Some(data); + } + } + + // num_frag_syms may include references to mergeable sections that + // were not converted; the extra symbols stay unused. + for slot in &mut slots[next..] { + self.base.symbols.push(SymbolId(base_id.0 + next as u32)); + slot.write(Symbol::new(BStr::new(b""))); + next += 1; + } + } + + /// Scans relocations of live sections and of CIEs. + pub fn scan_relocations(&self, ctx: &Context) { + for isec in self.input_sections() { + if isec.is_alive() && isec.is_alloc() { + E::scan_relocations(ctx, isec); + } + } + + for cie in &self.cies { + for rel in cie.rels::(self) { + let sym = &ctx.symbols[self.base.symbols[rel.r_sym as usize]]; + if ctx.args.pic && rel.r_type == E::R_ABS { + error!( + ctx, + "{self}: relocation {} in .eh_frame can not be used when making a position-independent output; recompile with -fPIE or -fPIC", + rel.type_name::() + ); + } + if sym.is_imported() { + if sym.ty() != STT_FUNC { + fatal!( + ctx, + "{self}: {sym}: .eh_frame CIE record with an external data reference is not supported" + ); + } + sym.add_flags(NEEDS_PLT); + } + } + } + } + + /// Allocates space in `.common`/`.tls_common` for common symbols + /// that weren't resolved to real definitions. + /// + /// Common symbols come from C tentative definitions: `int foo;` in a + /// header included from several translation units yields a common + /// symbol in each object, which the linker merges into one. + pub fn convert_common_symbols( + &mut self, + diag: &Diagnostics, + args: &Args, + id: ObjId, + symbols: &mut SymbolTable, + default_version: u16, + section_arena: &crate::input_sections::SectionArena, + ) { + if self.num_common_symbols == 0 { + return; + } + for i in self.base.first_global..self.base.elf_syms.len() { + let esym = self.base.elf_syms.at_in::(i); + if !esym.is_common() { + continue; + } + let sym_id = self.base.symbols[i]; + let sym = &symbols[sym_id]; + if sym.file() != Some(FileId::Obj(id)) { + if args.warn_common { + warn!(diag, "{self}: multiple common symbols: {sym}"); + } + continue; + } + + let mut shdr = ElfShdr { + sh_type: SHT_NOBITS, + sh_size: esym.st_size, + sh_addralign: esym.st_value, + ..ElfShdr::default() + }; + shdr.sh_flags = if sym.ty() == STT_TLS { + (SHF_ALLOC | SHF_WRITE | SHF_TLS) as u64 + } else { + (SHF_ALLOC | SHF_WRITE) as u64 + }; + let name: &'static [u8] = if sym.ty() == STT_TLS { + b".tls_common" + } else { + b".common" + }; + + self.base.shdrs.push(shdr); + let shndx = self.base.shdrs.len() - 1; + let isec = InputSection::new::(diag, self, id, shndx as u32, &shdr, BStr::new(name)); + self.sections.push(isec, section_arena); + + let sym = &mut symbols[sym_id]; + sym.set_input_section(self.section_at(shndx as u32)); + sym.value = 0; + sym.sym_idx = i as u32; + sym.ver_idx = default_version; + sym.set_weak(false); + } + } + + /// Decides which symbols go to the output symbol table and sizes the + /// file's block of `.symtab` and `.strtab`. + pub fn plan_symtab(&self, ctx: &Context, id: ObjId) -> SymtabPlan { + let mut plan = SymtabPlan { + output_sym_indices: vec![-1; self.base.elf_syms.len()], + ..SymtabPlan::default() + }; + let file_id = FileId::Obj(id); + + // Symbols in garbage-collected sections and fragments are dropped + // along with them. + let is_alive = |sym: &Symbol| -> bool { + if !ctx.args.gc_sections { + return true; + } + if let Some(frag) = sym.fragment() { + return ctx.fragment(frag).is_alive(); + } + if let Some(isec) = sym.input_section_ref() { + return isec.is_alive(); + } + true + }; + + if !ctx.args.discard_all && !ctx.args.strip_all && ctx.args.retain_symbols_file.is_none() { + for i in 1..self.base.first_global.min(self.base.elf_syms.len()) { + if self.is_discarded_comdat(i) { + continue; + } + let sym_id = self.base.symbols[i]; + let sym = &ctx.symbols[sym_id]; + if is_alive(sym) && should_write_to_local_symtab(ctx, sym) { + plan.strtab_size += sym.name().len() as u64 + 1; + plan.output_sym_indices[i] = plan.num_local_symtab as i32; + plan.num_local_symtab += 1; + sym.set_write_to_symtab(); + } + } + } + + for i in self.base.first_global..self.base.elf_syms.len() { + let sym_id = self.base.symbols[i]; + let sym = &ctx.symbols[sym_id]; + if sym.file() == Some(file_id) + && is_alive(sym) + && (ctx.args.retain_symbols_file.is_none() || sym.write_to_symtab()) + { + plan.strtab_size += sym.name().len() as u64 + 1; + // Global symbols can be demoted to local by visibility or + // version scripts. + if sym.is_local(ctx) { + plan.output_sym_indices[i] = plan.num_local_symtab as i32; + plan.num_local_symtab += 1; + } else { + plan.output_sym_indices[i] = plan.num_global_symtab as i32; + plan.num_global_symtab += 1; + } + sym.set_write_to_symtab(); + } + } + plan + } + + /// Whether the file's `.debug_info` contains only DWARF32 units. + pub fn is_dwarf32(&mut self, diag: &Diagnostics) -> bool { + let name = self.to_string(); + for shndx in self.debug_info_sections.clone() { + let isec = self.section_at(shndx); + let section_name = isec.name(self); + if isec.sh_size < 12 { + // Too short to be valid; garbage in, garbage out. + return true; + } + let mut buf = [0u8; 12]; + let input_size = self.base.shdrs.sh_offset_and_size(shndx as usize).1 as usize; + isec.copy_contents_to::(diag, &name, section_name, input_size, &mut buf); + // A 32-bit CU starts with a 32-bit size; a 64-bit CU starts + // with 0xffffffff followed by a 64-bit size. + if E::Endian::read_u32(&buf) != 0xffff_ffff { + return true; + } + let first_size = E::Endian::read_u64(&buf[4..]) as usize + 12; + if first_size as u64 == isec.sh_size { + continue; + } + // A section combined by `ld -r` may contain several CUs. + let isec = self.section_mut(shndx as usize).unwrap(); + isec.uncompress::(diag, &name, section_name, input_size); + let contents = isec.contents(); + let mut p = first_size; + while contents.len() - p >= 12 { + if E::Endian::read_u32(&contents[p..]) != 0xffff_ffff { + return true; + } + p += E::Endian::read_u64(&contents[p + 4..]) as usize + 12; + } + } + false + } + + /// Produces this file's block of the output symbol table. + pub fn populate_symtab( + &self, + ctx: &Context, + id: ObjId, + block: &mut SymtabBlock<'_>, + ) { + let id = FileId::Obj(id); + + for i in 1..self.base.first_global.min(self.base.symbols.len()) { + let sym = &ctx.symbols[self.base.symbols[i]]; + if sym.write_to_symtab() && self.base.output_sym_indices[i] >= 0 { + block.push_local::(ctx, sym); + } + } + for i in self.base.first_global..self.base.elf_syms.len() { + let sym = &ctx.symbols[self.base.symbols[i]]; + if sym.file() == Some(id) && sym.write_to_symtab() { + if sym.is_local(ctx) { + block.push_local::(ctx, sym); + } else { + block.push_global::(ctx, sym); + } + } + } + } +} + +/// The layout of a file's block of the output symbol table. +#[derive(Debug, Default)] +pub struct SymtabPlan { + pub output_sym_indices: Vec, + pub num_local_symtab: u32, + pub num_global_symtab: u32, + pub strtab_size: u64, +} + +impl InputFile { + pub fn apply_symtab_plan(&mut self, plan: SymtabPlan) { + self.output_sym_indices = plan.output_sym_indices; + self.num_local_symtab = plan.num_local_symtab; + self.num_global_symtab = plan.num_global_symtab; + self.strtab_size = plan.strtab_size; + } +} + +/// A file's or chunk's part of `.symtab`, `.strtab` and `.symtab_shndx`: +/// its local symbols, its global symbols and their names, written in +/// place. The parts of different files don't overlap, so they are +/// written in parallel. +pub struct SymtabBlock<'a> { + locals: SymtabEntries<'a>, + globals: SymtabEntries<'a>, + strtab: &'a mut [u8], + /// The offset of `strtab` within `.strtab`. + strtab_base: u64, + strtab_len: usize, +} + +/// A run of `.symtab` entries and, if the output has the section, the +/// matching `.symtab_shndx` entries. +pub struct SymtabEntries<'a> { + syms: &'a mut [u8], + xindex: Option<&'a mut [u8]>, + len: usize, +} + +impl<'a> SymtabEntries<'a> { + pub fn new(syms: &'a mut [u8], xindex: Option<&'a mut [u8]>) -> SymtabEntries<'a> { + SymtabEntries { + syms, + xindex, + len: 0, + } + } + + fn push(&mut self, esym: ElfSym, xindex: u32) { + let size = ElfSym::size::(); + esym.write::(&mut self.syms[self.len * size..(self.len + 1) * size]); + if let Some(entries) = &mut self.xindex { + E::Endian::write_u32(&mut entries[self.len * 4..], xindex); + } + self.len += 1; + } +} + +impl<'a> SymtabBlock<'a> { + pub fn new( + locals: SymtabEntries<'a>, + globals: SymtabEntries<'a>, + strtab: &'a mut [u8], + strtab_base: u64, + ) -> SymtabBlock<'a> { + SymtabBlock { + locals, + globals, + strtab, + strtab_base, + strtab_len: 0, + } + } + + /// Adds a name made of `parts`, returning its `.strtab` offset. + fn add_string(&mut self, parts: &[&[u8]]) -> u32 { + let offset = self.strtab_base + self.strtab_len as u64; + for part in parts { + self.strtab[self.strtab_len..self.strtab_len + part.len()].copy_from_slice(part); + self.strtab_len += part.len(); + } + self.strtab[self.strtab_len] = 0; + self.strtab_len += 1; + offset as u32 + } + + pub fn push_local(&mut self, ctx: &Context, sym: &Symbol) { + let st_name = self.add_string(&[sym.name().as_ref()]); + let (esym, xindex) = crate::chunks::symtab::to_output_esym(ctx, sym, st_name); + self.locals.push::(esym, xindex); + } + + pub fn push_global(&mut self, ctx: &Context, sym: &Symbol) { + let st_name = self.add_string(&[sym.name().as_ref()]); + let (esym, xindex) = crate::chunks::symtab::to_output_esym(ctx, sym, st_name); + self.globals.push::(esym, xindex); + } + + /// Adds a synthesized local symbol with a name built from `name` and + /// `suffix`. + pub fn push_synthetic(&mut self, name: &[u8], suffix: &[u8], esym: ElfSym) { + let st_name = self.add_string(&[name, suffix]); + self.locals.push::(ElfSym { st_name, ..esym }, 0); + } + + /// Adds a local symbol whose name is a fixed `.strtab` entry, such as + /// an ARM32 mapping symbol. + pub fn push_mapping_symbol(&mut self, st_name: u32, esym: ElfSym) { + self.locals.push::(ElfSym { st_name, ..esym }, 0); + } +} + +fn should_write_to_local_symtab(ctx: &Context, sym: &Symbol) -> bool { + if sym.ty() == STT_SECTION { + return false; + } + + // Temporary local symbols such as .L.str.42 are compiler-internal and + // numerous, and not referenced by DWARF, so they are discarded by + // default, as lld does. Even with --discard-none, temporary symbols + // in mergeable sections are discarded, as traditional linkers do. + let name: &[u8] = sym.name(); + if name.starts_with(b".L") || name == b"L0\x01" { + if ctx.args.discard_locals { + return false; + } + if let Some(isec) = sym.input_section_ref() { + if isec.sh_flags & SHF_MERGE as u64 != 0 { + return false; + } + } + } + true +} + +/// Determines how FDE pointers are encoded by parsing a CIE's +/// augmentation string, so that FDEs covering an empty range can be +/// dropped from `.eh_frame_hdr`. +fn parse_fde_encoding( + diag: &Diagnostics, + file: &ObjectFile, + isec: &InputSection, + data: &[u8], +) -> u8 { + let ptr_size = |enc: u8| -> u8 { + match enc as u32 & 0xf { + DW_EH_PE_absptr => E::WORD_SIZE as u8, + DW_EH_PE_udata4 | DW_EH_PE_sdata4 => 4, + DW_EH_PE_udata8 | DW_EH_PE_sdata8 => 8, + _ => fatal!( + diag, + "{}: unsupported FDE pointer encoding: {enc}", + isec.display(file) + ), + } + }; + + // Skip the length, CIE ID and version fields. + let version = data[8]; + if version != 1 && version != 3 { + fatal!( + diag, + "{}: unsupported CIE version: {version}", + isec.display(file) + ); + } + let mut rest = &data[9..]; + let aug_len = rest.iter().position(|&b| b == 0).unwrap_or(rest.len()); + let aug = &rest[..aug_len]; + rest = &rest[(aug_len + 1).min(rest.len())..]; + + let enc = 'enc: { + // An empty augmentation string means raw absolute pointers. A + // string not starting with 'z' is a legacy augmentation whose + // layout we don't know. + if aug.is_empty() { + break 'enc DW_EH_PE_absptr as u8; + } + if aug[0] != b'z' { + fatal!( + diag, + "{}: unsupported CIE augmentation string: {}", + isec.display(file), + util::display(aug) + ); + } + + read_uleb(&mut rest); // code alignment factor + read_uleb(&mut rest); // data alignment factor + if version == 1 { + rest = &rest[1..]; // return address register + } else { + read_uleb(&mut rest); + } + read_uleb(&mut rest); // augmentation data length + + for &c in &aug[1..] { + match c { + b'R' => break 'enc rest[0], + b'L' => rest = &rest[1..], + b'P' => rest = &rest[ptr_size(rest[0]) as usize + 1..], + b'S' | b'B' | b'G' => {} + _ => fatal!( + diag, + "{}: unsupported CIE augmentation string: {}", + isec.display(file), + util::display(aug) + ), + } + } + DW_EH_PE_absptr as u8 + }; + + if enc & 0xf0 != 0 && enc as u32 & 0xf0 != DW_EH_PE_pcrel { + fatal!( + diag, + "{}: unsupported FDE pointer encoding: {enc}", + isec.display(file) + ); + } + ptr_size(enc) +} + +/// The byte length of an SFrame FRE block: a 5-byte attribute header +/// followed by frame row entries of variable width. +fn sframe_fre_block_size(data: &[u8], offset: usize) -> usize { + let num_fres = E::Endian::read_u16(&data[offset..]) as usize; + let addr_size = 1usize << bits(data[offset + 2] as u64, 3, 0); + let mut p = offset + 5; + for _ in 0..num_fres { + let info = data[p + addr_size] as u64; + let num_words = bits(info, 4, 1) as usize; + let word_size = 1usize << bits(info, 6, 5); + p += addr_size + 1 + num_words * word_size; + } + p - offset +} + +/// A shared library given as an input. +#[derive(Debug)] +pub struct SharedFile { + pub base: InputFile, + pub soname: String, + pub version_strings: Vec<&'static [u8]>, + + /// For each symbol, the `foo@VERSION` alias of a default-versioned + /// definition `foo@@VERSION`. + pub symbols2: Vec, + pub versyms: Vec, + + /// Symbol table keys for `base.symbols` and `symbols2`, interned by + /// `gather_symbols`. + pub symbol_keys: Vec<(&'static [u8], &'static [u8])>, + pub symbol2_keys: Vec>, + + sorted_syms: OnceLock>, +} + +impl fmt::Display for SharedFile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", path_clean(&self.base.filename)) + } +} + +impl SharedFile { + pub fn new(diag: &Diagnostics, mf: &'static MappedFile) -> SharedFile { + let base = InputFile::parse::(diag, mf, &FileName(&mf.name, "")); + let mut file = SharedFile { + base, + soname: String::new(), + version_strings: Vec::new(), + symbols2: Vec::new(), + versyms: Vec::new(), + symbol_keys: Vec::new(), + symbol2_keys: Vec::new(), + sorted_syms: OnceLock::new(), + }; + file.parse::(diag); + file + } + + /// The strings of the dynamic entries with the given tag, such as the + /// DT_NEEDED libraries. + fn dynamic_strings(&self, diag: &Diagnostics, tag: u64) -> Vec<&'static [u8]> { + let Some(idx) = self.base.find_section(SHT_DYNAMIC) else { + return Vec::new(); + }; + let shdr = self.base.shdrs.at(idx); + let strtab = self.base.section_contents(diag, shdr.sh_link as usize); + ElfDyn::parse_all::(self.base.section_contents(diag, idx)) + .into_iter() + .filter(|entry| entry.d_tag == tag) + .map(|entry| cstr_at(strtab, entry.d_val as usize)) + .collect() + } + + fn get_soname(&self, diag: &Diagnostics) -> String { + if let Some(soname) = self.dynamic_strings::(diag, DT_SONAME as u64).first() { + return String::from_utf8_lossy(soname).into_owned(); + } + if self.base.mf.is_none_or(|mf| mf.given_fullpath) { + return self.base.filename.clone(); + } + path_filename(&self.base.filename) + } + + fn parse(&mut self, diag: &Diagnostics) { + let Some(symtab_idx) = self.base.find_section(SHT_DYNSYM) else { + return; + }; + let symtab_shdr = self.base.shdrs.at(symtab_idx); + self.base.symbol_strtab = self + .base + .section_contents(diag, symtab_shdr.sh_link as usize); + self.soname = self.get_soname::(diag); + self.version_strings = self.read_version_strings::(diag); + + let esyms = ElfSym::parse_all::(self.base.section_contents(diag, symtab_idx)); + let first = symtab_shdr.sh_info as usize; + if esyms.len() < first { + fatal!(diag, "{self}: invalid symbol table"); + } + // Only the symbols this file exports are kept, so the table is + // rebuilt rather than read in place. + self.base.elf_syms = SymTable::new(RecordLayout::of::()); + + let vers: Vec = match self.base.find_section(SHT_GNU_VERSYM) { + Some(idx) => self + .base + .section_contents(diag, idx) + .chunks_exact(2) + .map(E::Endian::read_u16) + .collect(), + None => Vec::new(), + }; + + for i in first..esyms.len() { + let esym = esyms[i]; + let mut ver = if vers.is_empty() { + VER_NDX_GLOBAL as u16 + } else { + vers[i] & !(VERSYM_HIDDEN as u16) + }; + + // Version index 0 is valid only for unversioned undefined + // symbols; a defined symbol with index 0 is ill-formed. + if ver as u32 == VER_NDX_LOCAL { + if !esym.is_undef() { + fatal!( + diag, + "{self}: invalid version index 0 for defined symbol {}", + util::display(cstr_at(self.base.symbol_strtab, esym.st_name as usize)) + ); + } + ver = VER_NDX_GLOBAL as u16; + } + + // A definition whose versym is exactly VERSYM_HIDDEN | + // VER_NDX_GLOBAL is a compatibility alias for binaries linked + // before the library adopted versioning. New references must + // not bind to it. + if !vers.is_empty() + && vers[i] == (VERSYM_HIDDEN | VER_NDX_GLOBAL) as u16 + && !esym.is_undef() + { + continue; + } + + self.base.elf_syms.push(esym); + self.versyms.push(if esym.is_undef() { + VER_NDX_GLOBAL as u16 + } else { + ver + }); + + let name = cstr_at(self.base.symbol_strtab, esym.st_name as usize); + let has_version = ver as u32 != VER_NDX_GLOBAL + && (ver as usize) < self.version_strings.len() + && !self.version_strings[ver as usize].is_empty(); + + let versioned_key = || -> (&'static [u8], &'static [u8]) { + let key = leak_bytes([name, b"@", self.version_strings[ver as usize]].concat()); + (key, name) + }; + + // A default-versioned symbol `foo@@VER` can be referred to as + // either `foo` or `foo@VER`. We resolve `foo` normally and make + // `foo@VER` a forwarding alias. + if !has_version { + self.symbol_keys.push((name, name)); + self.symbol2_keys.push(None); + } else if esym.is_undef() || vers[i] & VERSYM_HIDDEN as u16 != 0 { + self.symbol_keys.push(versioned_key()); + self.symbol2_keys.push(None); + } else { + self.symbol_keys.push((name, name)); + self.symbol2_keys.push(Some(versioned_key())); + } + } + + self.base.first_global = 0; + self.base.symbols = vec![SymbolId::DISCARDED_COMDAT; self.base.elf_syms.len()]; + self.symbols2 = vec![SymbolId::NONE; self.base.elf_syms.len()]; + } + + /// Records symbols and default-version aliases in this worker's bin. + /// Both slot arrays are complete and stable until gather. + pub(crate) fn record_global_symbols(&mut self, bins: &mut Bins) { + for (i, (key, name)) in std::mem::take(&mut self.symbol_keys) + .into_iter() + .enumerate() + { + bins.record(key, name.len(), SymbolSlot::new(&mut self.base.symbols[i])); + } + for (i, alias) in std::mem::take(&mut self.symbol2_keys) + .into_iter() + .enumerate() + { + if let Some((key, name)) = alias { + bins.record(key, name.len(), SymbolSlot::new(&mut self.symbols2[i])); + } + } + } + + pub fn dt_needed(&self, diag: &Diagnostics) -> Vec<&'static [u8]> { + self.dynamic_strings::(diag, DT_NEEDED as u64) + } + + pub fn dt_audit(&self, diag: &Diagnostics) -> &'static [u8] { + self.dynamic_strings::(diag, DT_AUDIT as u64) + .first() + .copied() + .unwrap_or(b"") + } + + /// Reads `.gnu.version_d` and `.gnu.version_r` into a table of version + /// names indexed by version index. Both sections share the index space. + fn read_version_strings(&self, diag: &Diagnostics) -> Vec<&'static [u8]> { + let mut vec: Vec<&'static [u8]> = Vec::new(); + let mut set = |idx: usize, name: &'static [u8]| { + if vec.len() <= idx { + vec.resize(idx + 1, b""); + } + vec[idx] = name; + }; + + if let Some(idx) = self.base.find_section(SHT_GNU_VERDEF) { + let verdef = self.base.section_contents(diag, idx); + let strtab = self + .base + .section_contents(diag, self.base.shdrs.at(idx).sh_link as usize); + let mut pos = 0; + loop { + let ver = ElfVerdef::parse::(&verdef[pos..]); + if ver.vd_ndx as u32 == VER_NDX_UNSPECIFIED { + fatal!(diag, "{self}: symbol version too large"); + } + let aux = ElfVerdaux::parse::(&verdef[pos + ver.vd_aux as usize..]); + set(ver.vd_ndx as usize, cstr_at(strtab, aux.vda_name as usize)); + if ver.vd_next == 0 { + break; + } + pos += ver.vd_next as usize; + } + } + + if let Some(idx) = self.base.find_section(SHT_GNU_VERNEED) { + let verneed = self.base.section_contents(diag, idx); + let strtab = self + .base + .section_contents(diag, self.base.shdrs.at(idx).sh_link as usize); + let mut pos = 0; + loop { + let vn = ElfVerneed::parse::(&verneed[pos..]); + let mut aux_pos = pos + vn.vn_aux as usize; + for _ in 0..vn.vn_cnt { + let aux = ElfVernaux::parse::(&verneed[aux_pos..]); + let idx = (aux.vna_other & !(VERSYM_HIDDEN as u16)) as usize; + set(idx, cstr_at(strtab, aux.vna_name as usize)); + if aux.vna_next == 0 { + break; + } + aux_pos += aux.vna_next as usize; + } + if vn.vn_next == 0 { + break; + } + pos += vn.vn_next as usize; + } + } + vec + } + + /// The symbols this file defines at the same address as `sym`. + pub fn symbols_at(&self, ctx: &Context, sym: &Symbol, id: DsoId) -> &[SymbolId] { + let sorted = self.sorted_syms.get_or_init(|| { + let mut syms: Vec = self + .base + .symbols + .iter() + .copied() + .filter(|&s| ctx.symbols[s].file() == Some(FileId::Dso(id))) + .collect(); + syms.sort_by_key(|&s| (ctx.symbols[s].esym(ctx).st_value, s)); + syms + }); + let value = sym.esym(ctx).st_value; + let begin = sorted.partition_point(|&s| ctx.symbols[s].esym(ctx).st_value < value); + let end = sorted.partition_point(|&s| ctx.symbols[s].esym(ctx).st_value <= value); + &sorted[begin..end] + } + + /// Infers the alignment of a symbol from its address and its section's + /// alignment, for copy relocations. + pub fn alignment(&self, sym: &Symbol) -> u64 { + let shdr = &self + .base + .shdrs + .at(self.base.elf_syms.at(sym.sym_idx as usize).st_shndx as usize); + let mut align = shdr.sh_addralign.max(1); + if sym.value != 0 { + align = align.min(1 << sym.value.trailing_zeros()); + } + align + } + + /// Whether a symbol lives in a read-only segment. + pub fn is_readonly(&self, sym: &Symbol) -> bool { + let data = self.base.data(); + let ehdr = ElfEhdr::parse::(data); + let val = self.base.elf_syms.at(sym.sym_idx as usize).st_value; + let phoff = ehdr.e_phoff as usize; + let size = ElfPhdr::size::(); + (0..ehdr.e_phnum as usize) + .map(|i| ElfPhdr::parse::(&data[phoff + i * size..])) + .any(|phdr| { + (phdr.p_type == PT_LOAD || phdr.p_type == PT_GNU_RELRO) + && phdr.p_flags & PF_W == 0 + && phdr.p_vaddr <= val + && val < phdr.p_vaddr + phdr.p_memsz + }) + } + + pub fn plan_symtab(&self, ctx: &Context, id: DsoId) -> SymtabPlan { + let mut plan = SymtabPlan { + output_sym_indices: vec![-1; self.base.elf_syms.len()], + ..SymtabPlan::default() + }; + for i in 0..self.base.symbols.len() { + let sym_id = self.base.symbols[i]; + let sym = &ctx.symbols[sym_id]; + if sym.file() == Some(FileId::Dso(id)) + && (sym.is_imported() || sym.is_exported()) + && (ctx.args.retain_symbols_file.is_none() || sym.write_to_symtab()) + { + plan.strtab_size += sym.name().len() as u64 + 1; + plan.output_sym_indices[i] = plan.num_global_symtab as i32; + plan.num_global_symtab += 1; + sym.set_write_to_symtab(); + } + } + plan + } + + pub fn populate_symtab( + &self, + ctx: &Context, + id: DsoId, + block: &mut SymtabBlock<'_>, + ) { + for &sym_id in &self.base.symbols { + let sym = &ctx.symbols[sym_id]; + if sym.file() == Some(FileId::Dso(id)) && sym.write_to_symtab() { + block.push_global::(ctx, sym); + } + } + } +} + +/// Symbol definition strength, from strongest to weakest: +/// +/// 1. Strong defined symbol +/// 2. Weak defined symbol +/// 3. Strong defined symbol in a DSO or archive +/// 4. Weak defined symbol in a DSO or archive +/// 5. Common symbol +/// 6. Common symbol in an archive +/// 7. Undefined +/// +/// These are heuristics rather than exact science; they avoid link errors +/// in all programs we've tested. Ties are broken by file priority. +#[inline] +fn symbol_rank_from_fields( + is_common: bool, + st_bind: u32, + is_dso: bool, + is_in_archive: bool, +) -> u64 { + if is_common { + debug_assert!(!is_dso); + return if is_in_archive { 6 } else { 5 }; + } + if is_dso || is_in_archive { + return if st_bind == STB_WEAK { 4 } else { 3 }; + } + if st_bind == STB_WEAK { + 2 + } else { + 1 + } +} + +#[inline] +pub fn symbol_rank(esym: &ElfSym, is_dso: bool, is_in_archive: bool) -> u64 { + symbol_rank_from_fields(esym.is_common(), esym.st_bind(), is_dso, is_in_archive) +} + +/// The rank used to choose a symbol definition. Ties in definition +/// strength are broken in favor of the file occurring first on the command +/// line. +#[inline] +pub fn symbol_resolution_rank( + esym: &ElfSym, + is_dso: bool, + is_in_archive: bool, + priority: u32, +) -> u64 { + (symbol_rank(esym, is_dso, is_in_archive) << 32) | priority as u64 +} + +/// Returns the resolution rank of a symbol that already owns a definition. +/// `Symbol` caches the defining ELF symbol's fields, so we don't need to +/// decode that record again for every competing definition. +#[inline] +pub fn resolved_symbol_rank(sym: &Symbol, is_dso: bool, is_in_archive: bool, priority: u32) -> u64 { + (symbol_rank_from_fields(sym.is_common(), sym.st_bind(), is_dso, is_in_archive) << 32) + | priority as u64 +} + +/// An exclusive view of the symbol table shared by file-parallel passes. +/// Individual symbols are serialized by their byte-sized spin locks. +pub(crate) struct SymbolEditor<'a> { + symbols: *mut Symbol, + len: usize, + _symbols: PhantomData<&'a mut [Symbol]>, +} + +// The exclusive slice represented by `symbols` cannot be accessed except +// through `with_symbol`, which serializes accesses to each element. +unsafe impl Sync for SymbolEditor<'_> {} + +impl<'a> SymbolEditor<'a> { + pub(crate) fn new(symbols: &'a mut [Symbol]) -> SymbolEditor<'a> { + SymbolEditor { + symbols: symbols.as_mut_ptr(), + len: symbols.len(), + _symbols: PhantomData, + } + } + + #[inline] + pub(crate) fn with_symbol(&self, id: SymbolId, f: impl FnOnce(&mut Symbol) -> R) -> R { + debug_assert!(id.index() < self.len); + // SAFETY: the editor owns an exclusive borrow of the whole table, + // and Symbol::with_resolution_lock serializes accesses to this slot. + unsafe { Symbol::with_resolution_lock(self.symbols.add(id.index()), f) } + } + + #[inline] + pub(crate) fn skip_dso(&self, id: SymbolId) -> bool { + debug_assert!(id.index() < self.len); + // SAFETY: the editor owns a live symbol array; skip_dso_at reads only + // the atomic state byte shared with the resolution lock. + unsafe { Symbol::skip_dso_at(self.symbols.add(id.index())) } + } +} + +/// The files and defaults needed while editing symbols during resolution. +pub struct SymbolResolver<'a> { + editor: SymbolEditor<'a>, + objs: &'a [Box], + dsos: &'a [Box], + default_version: u16, +} + +// SymbolResolver's mutable symbol-table access is serialized by its editor. +unsafe impl Sync for SymbolResolver<'_> {} + +impl<'a> SymbolResolver<'a> { + pub fn new( + symbols: &'a mut [Symbol], + objs: &'a [Box], + dsos: &'a [Box], + default_version: u16, + ) -> SymbolResolver<'a> { + SymbolResolver { + editor: SymbolEditor::new(symbols), + objs, + dsos, + default_version, + } + } + + #[inline] + fn with_symbol(&self, id: SymbolId, f: impl FnOnce(&mut Symbol) -> R) -> R { + self.editor.with_symbol(id, f) + } + + #[inline] + fn skip_dso(&self, id: SymbolId) -> bool { + self.editor.skip_dso(id) + } + + #[inline] + fn current_rank(&self, sym: &Symbol) -> u64 { + let Some(file) = sym.file() else { + return 7 << 32; + }; + let base = match file { + FileId::Obj(id) => &self.objs[id.index()].base, + FileId::Dso(id) => &self.dsos[id.index()].base, + }; + resolved_symbol_rank(sym, file.is_dso(), !base.is_reachable(), base.priority) + } +} + +impl ObjectFile { + /// Resolves this file's global definitions in place. Before sections + /// are parsed, all definitions are treated as live; the final round uses + /// the actual section state. + pub fn resolve_symbols(&self, resolver: &SymbolResolver<'_>, id: ObjId) { + let in_archive = !self.base.is_reachable(); + for i in self.base.first_global..self.base.elf_syms.len() { + let sym_id = self.base.symbols[i]; + self.resolve_symbol::(resolver, id, i, sym_id, in_archive); + } + } + + /// Resolves only symbols marked to ignore DSO definitions. This is the + /// rare hidden-symbol retry; filtering before `resolve_symbol` avoids + /// decoding and locking every other definition, as in C++. + pub fn resolve_skip_dso_symbols(&self, resolver: &SymbolResolver<'_>, id: ObjId) { + let in_archive = !self.base.is_reachable(); + for i in self.base.first_global..self.base.elf_syms.len() { + let sym_id = self.base.symbols[i]; + if resolver.skip_dso(sym_id) { + self.resolve_symbol::(resolver, id, i, sym_id, in_archive); + } + } + } + + // Makes this file's i'th symbol the definition of the global symbol it + // refers to if it is the best one seen so far. + fn resolve_symbol( + &self, + resolver: &SymbolResolver<'_>, + id: ObjId, + i: usize, + sym_id: SymbolId, + in_archive: bool, + ) { + let esym = self.base.elf_syms.at_in::(i); + if esym.is_undef() { + return; + } + + let mut origin = None; + if !esym.is_abs() && !esym.is_common() && self.sections_parsed { + let shndx = self.shndx_from(i, esym.st_shndx); + let Some(isec) = self.section(shndx) else { + return; + }; + if !isec.is_alive() { + return; + } + origin = Some(isec); + } + + let rank = symbol_resolution_rank(&esym, false, in_archive, self.base.priority); + resolver.with_symbol(sym_id, |sym| { + if rank < resolver.current_rank(sym) { + sym.set_file(FileId::Obj(id)); + match origin { + Some(section) => sym.set_input_section(section), + None => sym.clear_origin(), + } + sym.value = esym.st_value; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.ver_idx = resolver.default_version; + sym.set_weak(esym.is_weak()); + sym.set_versioned_default(false); + sym.set_rust(self.is_rust_obj); + } + }); + } +} + +impl SharedFile { + /// Resolves this shared library's definitions in place, including the + /// forwarding aliases for default symbol versions. + pub fn resolve_symbols(&self, resolver: &SymbolResolver<'_>, id: DsoId) { + for i in 0..self.base.elf_syms.len() { + let esym = self.base.elf_syms.at_in::(i); + let sym_id = self.base.symbols[i]; + if esym.is_undef() || resolver.skip_dso(sym_id) { + continue; + } + + let rank = symbol_resolution_rank(&esym, true, false, self.base.priority); + resolver.with_symbol(sym_id, |sym| { + if rank < resolver.current_rank(sym) { + sym.set_file(FileId::Dso(id)); + sym.clear_origin(); + sym.value = esym.st_value; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.ver_idx = self.versyms[i]; + sym.set_weak(true); + sym.set_versioned_default(false); + sym.set_rust(false); + } + }); + + // A symbol with the default version is a special case because, + // unlike other symbols, the symbol can be referred by two names, + // `foo` and `foo@VERSION`. Resolve the latter as a proxy of the + // former. + let alias_id = self.symbols2[i]; + if alias_id != SymbolId::NONE && alias_id != sym_id { + resolver.with_symbol(alias_id, |sym| { + if rank < resolver.current_rank(sym) { + sym.set_file(FileId::Dso(id)); + sym.set_symbol_origin(sym_id); + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.set_rust(false); + sym.set_versioned_default(true); + } + }); + } + } + } +} + +/// Prints a `--trace-symbol` line for a reference or definition. +pub fn print_trace_symbol( + diag: &Diagnostics, + file: &dyn fmt::Display, + esym: &ElfSym, + sym: &Symbol, +) { + if !esym.is_undef() { + out!(diag, "trace-symbol: {file}: definition of {sym}"); + } else if esym.is_weak() { + out!(diag, "trace-symbol: {file}: weak reference to {sym}"); + } else { + out!(diag, "trace-symbol: {file}: reference to {sym}"); + } +} diff --git a/src/input_sections.rs b/src/input_sections.rs new file mode 100644 index 000000000..61d4e4d50 --- /dev/null +++ b/src/input_sections.rs @@ -0,0 +1,1971 @@ +//! Input sections and the records the linker parses out of them. + +use std::cell::Cell; +use std::fmt; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; + +use bstr::BStr; + +use crate::arch::{Arch, Family}; +use crate::args::UnresolvedKind; +use crate::chunks::merged::{MergedSection, MergedSectionId}; +use crate::chunks::OutputSectionId; +use crate::context::Context; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::input_files::{ObjId, ObjectFile}; +use crate::symbol::{Symbol, SymbolId, NEEDS_CANONICAL, NEEDS_GOTTP, NEEDS_PLT, NEEDS_TLSDESC}; +use crate::util::compress::{zlib_decompress, zstd_decompress}; +use crate::util::concurrent_map::EntryId; +use crate::util::hyperloglog::HyperLogLog; +use crate::util::{self, cstr_at, leak_bytes}; +use crate::{error, fatal}; + +/// Identifies an input section by its file and section index. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SectionRef { + pub file: ObjId, + pub shndx: u32, +} + +impl SectionRef { + #[inline] + pub(crate) fn encode(self) -> u64 { + (u64::from(self.file.0) << 32) | u64::from(self.shndx) + } + + #[inline] + pub(crate) fn decode(value: u64) -> SectionRef { + SectionRef { + file: ObjId((value >> 32) as u32), + shndx: value as u32, + } + } +} + +/// A compact arena pointer to an input section. C++ mold stores output-section +/// members as `ArenaPtr` rather than looking them up by file and +/// ELF section index. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct InputSectionId(u32); + +const _: () = assert!(std::mem::size_of::() == 4); + +impl InputSectionId { + /// A placeholder used only while a member array is being filled. + pub(crate) const NONE: InputSectionId = InputSectionId(0); +} + +/// Identifies a section fragment in a merged output section. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FragmentRef { + pub section: MergedSectionId, + pub entry: EntryId, +} + +/// A byte range that RISC-V or LoongArch relaxation removed from a +/// section: the contents at and after `offset` shift towards the +/// beginning of the section by `delta` bytes in total. +#[derive(Clone, Copy, Debug)] +pub struct RelocDelta { + pub offset: u64, + pub delta: i64, +} + +const IS_ALIVE: u8 = 1 << 0; +const IS_VISITED: u8 = 1 << 1; +const IS_ADDRESS_TAKEN: u8 = 1 << 2; +const IS_UNCOMPRESSED: u8 = 1 << 3; +const IS_ICF_REMOVED: u8 = 1 << 4; +const HAS_RELOCS: u8 = 1 << 5; +const IS_NOBITS: u8 = 1 << 6; +const NO_RELSEC: u32 = u32::MAX; +const NO_FDE: u32 = u32::MAX; + +/// Rarely used input-section state, allocated only when needed. +#[derive(Debug, Default)] +struct InputSectionExtras { + /// The `.ARM.exidx` section describing this section (ARM32). + exidx: Option, + + /// Bytes removed by linker relaxation (RISC-V, LoongArch). + r_deltas: Box<[RelocDelta]>, +} + +/// A section of an input object file. +#[derive(Debug)] +pub struct InputSection { + pub file: ObjId, + pub shndx: u32, + + /// Offset in the owner file's section-name string table. + name_offset: u32, + + // UINT16_MAX means that name() must scan the remaining suffix. + namelen: u16, + + /// The section header's flags; the rest of the header is read from the + /// file when needed. + pub sh_flags: u64, + + /// The section contents as they appear in the file, or the + /// decompressed contents once [`Self::uncompress`] has run. + contents: usize, + + /// The size after decompression; may shrink during relaxation. + pub sh_size: u64, + p2align: AtomicU8, + + pub output_section: Option, + + /// Offset within the output section. During ICF it temporarily holds the + /// dense section index and then the encoded leader, as in C++ mold's + /// union. Sections are processed in parallel, so it is atomic. + offset: AtomicU64, + + /// Index of the relocation section applying to this section, or + /// `NO_RELSEC`. Rewritten relocations live at the same index in the + /// owner file's side table. + relsec_idx: u32, + + /// First FDE in the owner file's contiguous run for this section. + pub(crate) fde_begin: u32, + + /// A self-relative pointer to rarely used state, in four-byte units. + extra: i32, + + flags: AtomicU8, +} + +const _: () = assert!(std::mem::size_of::() == 64); + +#[inline] +fn to_p2align(alignment: u64) -> u8 { + if alignment == 0 { + 0 + } else { + alignment.trailing_zeros() as u8 + } +} + +impl InputSection { + #[inline] + pub fn new( + diag: &Diagnostics, + file: &ObjectFile, + file_id: ObjId, + shndx: u32, + shdr: &ElfShdr, + name: &'static BStr, + ) -> InputSection { + let contents: &'static [u8] = + if shdr.sh_type == SHT_NOBITS || (shndx as usize) >= file.num_elf_sections { + &[] + } else { + file.base.section_contents_from_shdr(diag, shdr) + }; + + let (sh_size, p2align) = if shdr.sh_flags & SHF_COMPRESSED as u64 != 0 { + let chdr = ElfChdr::parse::(contents); + (chdr.ch_size, to_p2align(chdr.ch_addralign)) + } else { + (shdr.sh_size, to_p2align(shdr.sh_addralign)) + }; + + let mut isec = InputSection { + file: file_id, + shndx, + name_offset: if (shndx as usize) < file.num_elf_sections { + shdr.sh_name + } else { + 0 + }, + namelen: name.len().min(u16::MAX as usize) as u16, + sh_flags: shdr.sh_flags, + contents: contents.as_ptr() as usize, + sh_size, + p2align: AtomicU8::new(p2align), + output_section: None, + offset: AtomicU64::new(u64::MAX), + relsec_idx: NO_RELSEC, + fde_begin: NO_FDE, + extra: 0, + flags: AtomicU8::new(IS_ALIVE), + }; + + // Compressed sections are usually decompressed straight into the + // output file, but REL-type targets read relocation addends from + // section contents, so those need the contents early. SH4 stores + // addends in sections despite being RELA. + if !E::IS_RELA || E::FAMILY == Family::Sh4 { + isec.uncompress::(diag, file, name, shdr.sh_size as usize); + } + isec + } + + #[inline] + pub fn is_alive(&self) -> bool { + self.flags.load(Ordering::Relaxed) & IS_ALIVE != 0 + } + + #[inline] + pub fn section_ref(&self) -> SectionRef { + SectionRef { + file: self.file, + shndx: self.shndx, + } + } + + #[inline] + pub fn is_visited(&self) -> bool { + self.flags.load(Ordering::Relaxed) & IS_VISITED != 0 + } + + #[inline] + pub fn p2align(&self) -> u8 { + self.p2align.load(Ordering::Relaxed) + } + + #[inline] + pub fn update_p2align(&self, p2align: u8) { + self.p2align.fetch_max(p2align, Ordering::Relaxed); + } + + #[inline] + pub fn is_address_taken(&self) -> bool { + self.flags.load(Ordering::Relaxed) & IS_ADDRESS_TAKEN != 0 + } + + #[inline] + pub fn is_uncompressed(&self) -> bool { + self.flags.load(Ordering::Relaxed) & IS_UNCOMPRESSED != 0 + } + + #[inline] + pub fn is_icf_removed(&self) -> bool { + self.flags.load(Ordering::Relaxed) & IS_ICF_REMOVED != 0 + } + + #[inline] + pub fn sh_type(&self, file: &ObjectFile) -> u32 { + self.sh_type_from(&file.base.shdrs) + } + + #[inline] + pub(crate) fn sh_type_from(&self, shdrs: &ShdrTable<'_>) -> u32 { + if self.flags.load(Ordering::Relaxed) & IS_NOBITS != 0 { + SHT_NOBITS + } else { + shdrs.sh_type(self.shndx as usize) + } + } + + #[inline] + pub fn set_nobits(&self) { + self.flags.fetch_or(IS_NOBITS, Ordering::Relaxed); + } + + #[inline] + pub fn is_compressed(&self) -> bool { + self.sh_flags & SHF_COMPRESSED as u64 != 0 && !self.is_uncompressed() + } + + /// Marks the section visited, returning true if it was not already. + #[inline] + pub fn visit(&self) -> bool { + // Most sections are referenced many times, so check with a plain + // load before the atomic read-modify-write. + if self.is_visited() { + return false; + } + self.flags.fetch_or(IS_VISITED, Ordering::Relaxed) & IS_VISITED == 0 + } + + #[inline] + pub fn set_visited(&self) { + self.flags.fetch_or(IS_VISITED, Ordering::Relaxed); + } + + #[inline] + pub fn set_address_taken(&self) { + self.flags.fetch_or(IS_ADDRESS_TAKEN, Ordering::Relaxed); + } + + #[inline] + pub fn set_icf_removed(&self) { + self.flags.fetch_or(IS_ICF_REMOVED, Ordering::Relaxed); + } + + /// Marks the section dead. Returns true if it was alive; the caller is + /// then responsible for killing the section's FDEs. + #[inline] + pub fn kill(&self) -> bool { + self.flags.fetch_and(!IS_ALIVE, Ordering::Relaxed) & IS_ALIVE != 0 + } + + #[inline] + pub fn revive(&self) { + self.flags.fetch_or(IS_ALIVE, Ordering::Relaxed); + } + + /// The section contents, decompressed if [`Self::uncompress`] has run. + #[inline] + pub fn contents(&self) -> &'static [u8] { + if self.contents == 0 { + return &[]; + } + debug_assert!(!self.is_compressed()); + // SAFETY: contents points into an input mapping or a leaked + // decompression buffer, both live for the complete link. sh_size is + // their current logical size except during relaxation, whose callers + // use original_contents below. + unsafe { std::slice::from_raw_parts(self.contents as *const u8, self.sh_size as usize) } + } + + /// The complete input contents, including bytes removed by relaxation. + #[inline] + pub fn original_contents(&self, file: &ObjectFile) -> &'static [u8] { + if self.contents == 0 { + return &[]; + } + debug_assert!(!self.is_compressed()); + let size = if self.is_uncompressed() { + self.sh_size + } else { + file.base.shdrs.sh_offset_and_size(self.shndx as usize).1 + }; + // SAFETY: as in contents; an uncompressed buffer has sh_size bytes, + // while an ordinary input view has its ELF section header's size. + unsafe { std::slice::from_raw_parts(self.contents as *const u8, size as usize) } + } + + /// The section name, read from the owner file's string table. + #[inline] + pub fn name(&self, file: &ObjectFile) -> &'static BStr { + self.name_in(file.base.shstrtab, file.num_elf_sections) + } + + #[inline] + pub(crate) fn name_in( + &self, + shstrtab: &'static [u8], + num_elf_sections: usize, + ) -> &'static BStr { + if self.shndx as usize >= num_elf_sections { + let name: &[u8] = if self.sh_flags & SHF_TLS as u64 != 0 { + b".tls_common" + } else { + b".common" + }; + return BStr::new(name); + } + + let start = self.name_offset as usize; + if self.namelen == u16::MAX { + BStr::new(cstr_at(shstrtab, start)) + } else { + BStr::new(&shstrtab[start..start + self.namelen as usize]) + } + } + + /// The offset within the output section. + #[inline] + pub fn offset(&self) -> u64 { + self.offset.load(Ordering::Relaxed) + } + + #[inline] + pub fn set_offset(&self, offset: u64) { + self.offset.store(offset, Ordering::Relaxed); + } + + /// The dense section index used while ICF constructs its graph. + #[inline] + pub(crate) fn icf_index(&self) -> Option { + let index = self.offset(); + (index != u64::MAX).then(|| u32::try_from(index).expect("invalid ICF section index")) + } + + #[inline] + pub(crate) fn set_icf_index(&self, index: u32) { + self.set_offset(index as u64); + } + + #[inline] + pub fn addr(&self, ctx: &Context) -> u64 { + let osec = self.output_section.expect("section has no output section"); + ctx.output_sections[osec.index()].hdr.shdr.sh_addr + self.offset() + } + + #[inline] + pub fn is_alloc(&self) -> bool { + self.sh_flags & SHF_ALLOC as u64 != 0 + } + + /// Sort key for deterministic ordering: files are processed in + /// command line order, sections in file order. + pub fn priority(&self, file: &ObjectFile) -> u64 { + ((file.base.priority as u64) << 32) | self.shndx as u64 + } + + /// Replaces compressed contents with a decompressed copy. `file` is + /// the owning file's name, for diagnostics. + pub fn uncompress( + &mut self, + diag: &Diagnostics, + file: &dyn fmt::Display, + name: &BStr, + input_size: usize, + ) { + if !self.is_compressed() { + return; + } + let mut buf = vec![0u8; self.sh_size as usize]; + self.copy_contents_to::(diag, file, name, input_size, &mut buf); + self.contents = leak_bytes(buf).as_ptr() as usize; + self.flags.fetch_or(IS_UNCOMPRESSED, Ordering::Relaxed); + } + + /// Copies the (decompressed) contents into `buf`, which must be + /// `sh_size` bytes long. + pub fn copy_contents_to( + &self, + diag: &Diagnostics, + file: &dyn fmt::Display, + name: &BStr, + input_size: usize, + buf: &mut [u8], + ) { + if !self.is_compressed() { + // SAFETY: an ordinary input section has at least sh_size bytes, + // and replacement buffers are created with that same size. + let contents = unsafe { + std::slice::from_raw_parts(self.contents as *const u8, self.sh_size as usize) + }; + buf.copy_from_slice(&contents[..buf.len()]); + return; + } + + let hdr_size = ElfChdr::size::(); + if input_size < hdr_size { + fatal!(diag, "{file}:({name}): corrupted compressed section"); + } + // SAFETY: input_size comes from this section's validated ELF header. + let contents = + unsafe { std::slice::from_raw_parts(self.contents as *const u8, input_size) }; + let chdr = ElfChdr::parse::(contents); + let data = &contents[hdr_size..]; + + let result = match chdr.ch_type { + ELFCOMPRESS_ZLIB => zlib_decompress(data, buf), + ELFCOMPRESS_ZSTD => zstd_decompress(data, buf), + ty => fatal!( + diag, + "{file}:({name}): unsupported compression type: 0x{ty:x}" + ), + }; + if let Err(msg) = result { + fatal!(diag, "{file}:({name}): uncompress failed: {msg}"); + } + } + + /// Replaces the contents, e.g. after reversing a `.ctors` section. + pub fn set_contents(&mut self, contents: &'static [u8]) { + debug_assert_eq!(contents.len(), self.sh_size as usize); + self.contents = contents.as_ptr() as usize; + } + + /// Drops the contents, for a section converted to BSS. + pub fn clear_contents(&mut self) { + self.contents = 0; + } + + /// FDE records describing this section. FDEs for one input section are + /// contiguous, and the last record carries the end marker, as in C++. + #[inline] + pub fn fdes<'a>(&self, file: &'a ObjectFile) -> &'a [FdeRecord] { + if self.fde_begin == NO_FDE { + return &[]; + } + let begin = self.fde_begin as usize; + let end = begin + + file.fdes[begin..] + .iter() + .position(|fde| fde.is_last) + .expect("unterminated FDE run") + + 1; + &file.fdes[begin..end] + } + + /// The addend of a relocation against this section. + #[inline] + pub fn rel_addend(&self, rel: &ElfRel) -> i64 { + if E::IS_RELA && E::FAMILY != Family::Sh4 { + rel.r_addend + } else { + E::get_addend(&self.contents()[rel.r_offset as usize..], rel) + } + } + + /// Formats the section as `file:(name)` for diagnostics. + pub fn display<'a>(&'a self, file: &'a ObjectFile) -> impl fmt::Display + 'a { + struct Display<'a>(&'a InputSection, &'a ObjectFile); + impl fmt::Display for Display<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:({})", self.1, self.0.name(self.1)) + } + } + Display(self, file) + } + + /// Whether a relocation can be encoded in the RELR format. + #[inline] + pub fn is_relr_reloc(&self, ctx: &Context, rel: &ElfRel) -> bool { + ctx.args.pack_dyn_relocs_relr + && (1u64 << self.p2align()).is_multiple_of(E::WORD_SIZE as u64) + && rel.r_offset.is_multiple_of(E::WORD_SIZE as u64) + } + + /// Returns the name of the function containing `offset`. + pub fn func_name(&self, ctx: &Context, offset: u64) -> Option { + let file = &ctx.objs[self.file.index()]; + for &id in &file.base.symbols { + let sym = &ctx.symbols[id]; + if sym.file() != Some(self.file.into()) { + continue; + } + let esym = &sym.esym(ctx); + if esym.st_shndx as u32 == self.shndx + && esym.st_type() == STT_FUNC + && esym.st_value <= offset + && offset < esym.st_value + esym.st_size + { + return Some(sym.to_string()); + } + } + None + } + + /// Whether the file has a relocation section for this section. + #[inline] + pub fn has_relsec(&self) -> bool { + self.relsec_idx != NO_RELSEC + } + + /// Attaches the section's relocation section. + pub fn set_relsec(&mut self, relsec_idx: u32, has_relocs: bool) { + debug_assert!(!self.has_relsec()); + debug_assert_ne!(relsec_idx, NO_RELSEC); + self.relsec_idx = relsec_idx; + if has_relocs { + self.flags.fetch_or(HAS_RELOCS, Ordering::Relaxed); + } + } + + #[inline] + pub fn relsec_idx(&self) -> Option { + (self.relsec_idx != NO_RELSEC).then_some(self.relsec_idx) + } + + #[inline] + pub fn has_relocations(&self) -> bool { + self.flags.load(Ordering::Relaxed) & HAS_RELOCS != 0 + } + + #[inline] + pub fn rels<'a, E: Layout>(&self, file: &'a ObjectFile) -> Rels<'a, E> { + file.relocations::(self.relsec_idx()) + } + + /// Visits relocations without materializing a deferred CREL table. + #[inline] + pub fn for_each_reloc(&self, ctx: &Context, f: impl FnMut(ElfRel, usize)) { + let file = &ctx.objs[self.file.index()]; + file.for_each_relocation::(&ctx.diag, self.relsec_idx(), f); + } + + #[inline] + fn extra_ptr(&self) -> Option<*mut InputSectionExtras> { + if self.extra == 0 { + return None; + } + let field = std::ptr::addr_of!(self.extra).cast::(); + // SAFETY: a nonzero offset was created from an extras allocation in + // the same arena and InputSection has not moved since then. + Some(unsafe { + field + .offset(self.extra as isize * 4) + .cast::() + .cast_mut() + }) + } + + fn extra_mut(&mut self, arena: &SectionArena) -> &mut InputSectionExtras { + if self.extra == 0 { + let ptr = arena.insert_extra(InputSectionExtras::default()); + let field = std::ptr::addr_of!(self.extra).cast::(); + let bytes = ptr as isize - field as isize; + debug_assert_eq!(bytes % 4, 0); + self.extra = i32::try_from(bytes / 4).expect("input-section extras are too far away"); + debug_assert_ne!(self.extra, 0); + } + // SAFETY: extra_mut has exclusive access to the section and hence to + // its uniquely owned extras record. + unsafe { &mut *self.extra_ptr().unwrap() } + } + + #[inline] + pub fn icf_leader(&self) -> Option { + self.is_icf_removed().then(|| self.icf_leader_in_round()) + } + + #[inline] + pub(crate) fn icf_leader_in_round(&self) -> SectionRef { + SectionRef::decode(self.offset()) + } + + #[inline] + pub fn set_icf_leader(&self, leader: SectionRef) { + self.set_offset(leader.encode()); + } + + #[inline] + pub fn exidx(&self) -> Option { + self.extra_ptr() + // SAFETY: a published extras pointer remains valid with its arena. + .and_then(|extra| unsafe { (*extra).exidx }) + } + + pub fn set_exidx(&mut self, exidx: u32, arena: &SectionArena) { + self.extra_mut(arena).exidx = Some(exidx); + } + + #[inline] + pub fn r_deltas(&self) -> &[RelocDelta] { + self.extra_ptr() + // SAFETY: a published extras pointer remains valid with its arena. + .map_or(&[], |extra| unsafe { &(*extra).r_deltas }) + } + + pub fn set_r_deltas(&mut self, deltas: Box<[RelocDelta]>, arena: &SectionArena) { + debug_assert!(!deltas.is_empty()); + self.extra_mut(arena).r_deltas = deltas; + } + + /// Reports a relocation whose value doesn't fit in the field. + #[inline(always)] + pub fn check_range( + &self, + ctx: &Context, + rel_idx: usize, + val: i64, + lo: i64, + hi: i64, + ) { + if val < lo || hi <= val { + self.report_out_of_range::(ctx, rel_idx, val, lo, hi); + } + } + + #[cold] + fn report_out_of_range( + &self, + ctx: &Context, + rel_idx: usize, + val: i64, + lo: i64, + hi: i64, + ) { + let file = &ctx.objs[self.file.index()]; + let rel = self.rels::(file).at(rel_idx); + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + error!( + ctx, + "{}: relocation {} against {} out of range: {val} is not in [{lo}, {hi})", + self.display(file), + rel.type_name::(), + sym + ); + } + + /// For a relocation in a non-allocated section, finds the section + /// fragment it refers to, if it refers to a mergeable section. + #[inline] + pub fn fragment(&self, ctx: &Context, rel: &ElfRel) -> Option<(FragmentRef, i64)> { + debug_assert!(!self.is_alloc()); + let file = &ctx.objs[self.file.index()]; + self.fragment_with_file::(file, rel) + } + + /// Like [`Self::fragment`], using an owner the caller already loaded. + #[inline] + pub fn fragment_with_file( + &self, + file: &ObjectFile, + rel: &ElfRel, + ) -> Option<(FragmentRef, i64)> { + debug_assert!(!self.is_alloc()); + let sym_idx = rel.r_sym as usize; + if sym_idx >= file.base.elf_syms.len() { + return None; + } + let st_shndx = file.base.elf_syms.st_shndx_in::(sym_idx); + if matches!(st_shndx as u32, SHN_UNDEF | SHN_ABS | SHN_COMMON) { + return None; + } + let shndx = file.shndx_from(sym_idx, st_shndx); + let m = file.mergeable_section(shndx)?; + let esym = file.base.elf_syms.at_in::(sym_idx); + let addend = self.rel_addend::(rel); + if esym.st_type() == STT_SECTION { + let (frag, offset) = m.fragment(esym.st_value.wrapping_add(addend as u64))?; + Some(( + FragmentRef { + section: m.parent, + entry: frag, + }, + offset, + )) + } else { + let (frag, offset) = m.fragment(esym.st_value)?; + Some(( + FragmentRef { + section: m.parent, + entry: frag, + }, + offset + addend, + )) + } + } + + /// Returns a tombstone value if `sym` refers to a dead debug section. + /// + /// Linkers don't remove debug info for functions eliminated by COMDAT + /// deduplication or ICF, since that would require parsing the whole + /// debug section. Instead, dead debug records get a tombstone value + /// that debuggers know to skip. + #[inline(always)] + pub fn tombstone( + &self, + ctx: &Context, + sym: &Symbol, + frag: Option, + ) -> Option { + let file = &ctx.objs[self.file.index()]; + self.tombstone_with_file(ctx, file, sym, frag) + } + + /// Like [`Self::tombstone`], using an owner the caller already loaded. + #[inline(always)] + pub fn tombstone_with_file( + &self, + ctx: &Context, + file: &ObjectFile, + sym: &Symbol, + frag: Option, + ) -> Option { + if frag.is_some() { + return None; + } + + let isec = sym.input_section_ref(); + let discarded = sym.file().is_none() && sym.name().is_empty() && !sym.is_fragment_dummy(); + let discarded = discarded && std::ptr::eq(sym, &ctx.symbols[SymbolId::DISCARDED_COMDAT]); + + match isec { + None if !discarded => return None, + Some(isec) if isec.is_alive() => return None, + _ => {} + } + + let name: &[u8] = self.name(file); + if !name.starts_with(b".debug_") { + return None; + } + + // A section folded by ICF keeps real values in .debug_line so that + // users can set breakpoints inside the merged section. + if let Some(isec) = isec { + if isec.is_icf_removed() && name == b".debug_line" { + return None; + } + } + + // 0 is an invalid value in most debug info sections, so we use it + // as a tombstone. .debug_loc and .debug_ranges reserve 0 as the + // terminator marker, so those use 1. + Some(if name == b".debug_loc" || name == b".debug_ranges" { + 1 + } else { + 0 + }) + } + + /// Checks whether the symbol a relocation refers to is unresolved and + /// records the error if so. Returns true if the relocation must be + /// skipped. + #[inline(always)] + pub fn record_undef_error(&self, ctx: &Context, rel: &ElfRel) -> bool { + let file = &ctx.objs[self.file.index()]; + self.record_undef_error_with_file(ctx, file, rel) + } + + /// Like [`Self::record_undef_error`], using an owner the caller already + /// loaded. + #[inline(always)] + pub fn record_undef_error_with_file( + &self, + ctx: &Context, + file: &ObjectFile, + rel: &ElfRel, + ) -> bool { + // A relocation may refer to a linker-synthesized symbol for a + // section fragment, which is always resolved. + let sym_idx = rel.r_sym as usize; + if sym_idx >= file.base.elf_syms.len() { + return false; + } + let sym_id = file.base.symbols[sym_idx]; + let sym = &ctx.symbols[sym_id]; + + // A global symbol in a discarded COMDAT group should resolve to the + // corresponding symbol in the prevailing group. If it doesn't, the + // object files violate the One Definition Rule. + if sym.file().is_none() && sym_id != SymbolId::DISCARDED_COMDAT { + self.report_discarded_comdat(ctx, file, rel, sym); + return true; + } + + // A non-weak undefined symbol must be promoted to an imported + // symbol or resolved to a definition. Otherwise, we need to report + // an error or a warning. + // + // Every ELF file has an absolute local symbol as its first symbol. + // Referring to that symbol is always valid. + let st_info = file.base.elf_syms.st_info_in::(sym_idx); + let is_undef = file.base.elf_syms.st_shndx_in::(sym_idx) as u32 == SHN_UNDEF + && (st_info >> 4) as u32 != STB_WEAK + && sym.sym_idx != 0; + + if is_undef && sym.is_undef() { + match ctx.args.unresolved_symbols { + UnresolvedKind::Error if !sym.is_imported() => { + self.record_undefined_reference(ctx, file, rel, sym_id); + return true; + } + UnresolvedKind::Warn => self.record_undefined_reference(ctx, file, rel, sym_id), + _ => {} + } + } + false + } + + #[cold] + fn report_discarded_comdat( + &self, + ctx: &Context, + file: &ObjectFile, + rel: &ElfRel, + sym: &Symbol, + ) { + let mut msg = format!( + "{}: {} refers to a discarded COMDAT section probably due to an ODR violation", + self.display(file), + sym + ); + if let Some(owner) = find_comdat_owner(ctx, file, rel.r_sym as usize) { + msg += &format!( + "\n>>> prevailing definition is in {}", + ctx.objs[owner.index()] + ); + } + error!(ctx, "{msg}"); + } + + #[cold] + fn record_undefined_reference( + &self, + ctx: &Context, + file: &ObjectFile, + rel: &ElfRel, + sym_id: SymbolId, + ) { + let mut msg = String::new(); + match file.base.source_name(&ctx.symbols) { + Some(source) => msg += &format!(">>> referenced by {}\n", util::display(source)), + None => msg += &format!(">>> referenced by {}\n", self.display(file)), + } + msg += &format!(">>> {file}"); + if let Some(func) = self.func_name(ctx, rel.r_offset) { + msg += &format!(":({func})"); + } + msg.push('\n'); + ctx.undef_errors + .lock() + .unwrap() + .entry(sym_id) + .or_default() + .push(msg); + } + + /// Copies the section to `buf` and applies relocations. + pub fn write_to(&self, ctx: &Context, buf: &mut [u8]) { + let file = &ctx.objs[self.file.index()]; + if self.sh_type(file) == SHT_NOBITS || self.sh_size == 0 { + return; + } + let buf = &mut buf[..self.sh_size as usize]; + let input_size = file.base.shdrs.sh_offset_and_size(self.shndx as usize).1 as usize; + + // On RISC-V and LoongArch, relaxation may have removed bytes from + // the middle of the section, so a relaxed section is copied piecewise. + if self.r_deltas().is_empty() { + self.copy_contents_to::(&ctx.diag, file, self.name(file), input_size, buf); + } else { + let contents = self.original_contents(file); + let deltas = self.r_deltas(); + buf[..deltas[0].offset as usize] + .copy_from_slice(&contents[..deltas[0].offset as usize]); + + for (i, d) in deltas.iter().enumerate() { + let end = deltas + .get(i + 1) + .map_or(contents.len(), |n| n.offset as usize); + let removed = removed_bytes(deltas, i) as usize; + let src_start = d.offset as usize + removed; + let dst_start = (d.offset as i64 + removed as i64 - d.delta) as usize; + let len = end - src_start; + buf[dst_start..dst_start + len].copy_from_slice(&contents[src_start..end]); + } + } + + if !ctx.args.relocatable { + if self.is_alloc() { + E::apply_reloc_alloc(ctx, self, buf); + } else { + E::apply_reloc_nonalloc(ctx, self, buf); + } + } + } +} + +impl Drop for InputSection { + fn drop(&mut self) { + if let Some(extra) = self.extra_ptr() { + // SAFETY: this section uniquely owns its initialized extras + // record. Its arena storage is released after all sections drop. + unsafe { std::ptr::drop_in_place(extra) }; + } + } +} + +impl InputSection { + /// The number of bytes relaxation removed at the location of + /// relocation `rel`, and the number removed before it. + #[inline] + pub fn removed_at(&self, rel: &ElfRel) -> (i64, i64) { + let deltas = self.r_deltas(); + let k = deltas.partition_point(|d| d.offset < rel.r_offset); + let removed = if deltas.get(k).is_some_and(|d| d.offset == rel.r_offset) { + removed_bytes(deltas, k) + } else { + 0 + }; + let before = if k > 0 { deltas[k - 1].delta } else { 0 }; + (removed, before) + } +} + +/// The number of bytes removed at delta `i`. +pub fn removed_bytes(deltas: &[RelocDelta], i: usize) -> i64 { + if i == 0 { + deltas[0].delta + } else { + deltas[i].delta - deltas[i - 1].delta + } +} + +/// The total number of bytes removed before `offset`. +pub fn r_delta(isec: &InputSection, offset: u64) -> i64 { + let deltas = isec.r_deltas(); + let i = deltas.partition_point(|d| d.offset < offset); + if i == 0 { + 0 + } else { + deltas[i - 1].delta + } +} + +/// Finds the object owning the prevailing COMDAT group for a symbol in a +/// discarded group. Used only on an error path. +fn find_comdat_owner( + ctx: &Context, + file: &ObjectFile, + sym_idx: usize, +) -> Option { + let esym = &file.base.elf_syms.at(sym_idx); + if esym.is_undef() || esym.is_abs() || esym.is_common() { + return None; + } + let shndx = file.shndx_at(sym_idx); + for group in &file.comdat_groups { + if group.is_owner() || !file.comdat_members(group).any(|m| m == shndx as u32) { + continue; + } + for (i, other) in ctx.objs.iter().enumerate() { + for other_group in &other.comdat_groups { + if other_group.is_owner() && other_group.signature() == group.signature() { + return Some(ObjId(i as u32)); + } + } + } + } + None +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Action { + None, + Error, + Canonical, + Plt, +} + +fn do_action( + ctx: &Context, + action: Action, + isec: &InputSection, + sym: &Symbol, + rel: &ElfRel, +) { + match action { + Action::None => {} + Action::Error => error!( + ctx, + "{}: {} relocation at offset 0x{:x} against symbol `{}' can not be used; recompile with -fPIC", + isec.display(&ctx.objs[isec.file.index()]), + rel.type_name::(), + rel.r_offset, + sym + ), + Action::Canonical => sym.add_flags(NEEDS_CANONICAL), + Action::Plt => sym.add_flags(NEEDS_PLT), + } +} + +fn output_type(ctx: &Context) -> usize { + if ctx.args.shared { + 0 + } else if ctx.args.pie { + 1 + } else { + 2 + } +} + +fn sym_type(sym: &Symbol) -> usize { + if sym.is_absolute() { + 0 + } else if !sym.is_imported() { + 1 + } else if sym.ty() != STT_FUNC { + 2 + } else { + 3 + } +} + +/// Handles a PC-relative relocation such as R_X86_64_PC32. These can't be +/// promoted to dynamic relocations because dynamic linkers generally don't +/// support PC-relative ones. +pub fn scan_pcrel(ctx: &Context, isec: &InputSection, sym: &Symbol, rel: &ElfRel) { + use Action::*; + const TABLE: [[Action; 4]; 3] = [ + // Absolute Local Imported data Imported code + [Error, None, Error, Plt], // Shared object + [Error, None, Canonical, Canonical], // Position-independent exec + [None, None, Canonical, Canonical], // Position-dependent exec + ]; + do_action(ctx, TABLE[output_type(ctx)][sym_type(sym)], isec, sym, rel); +} + +/// Handles an absolute relocation smaller than the pointer size, such as +/// R_X86_64_32. Dynamic linkers don't support dynamic relocations of that +/// size, so it's an error if the value isn't known at link time. +pub fn scan_absrel(ctx: &Context, isec: &InputSection, sym: &Symbol, rel: &ElfRel) { + use Action::*; + const TABLE: [[Action; 4]; 3] = [ + // Absolute Local Imported data Imported code + [None, Error, Error, Error], // Shared object + [None, Error, Error, Error], // Position-independent exec + [None, None, Canonical, Canonical], // Position-dependent exec + ]; + do_action(ctx, TABLE[output_type(ctx)][sym_type(sym)], isec, sym, rel); +} + +pub fn scan_tlsdesc(ctx: &Context, sym: &Symbol) { + if ctx.args.is_static || (ctx.args.relax && sym.is_tprel_linktime_const(ctx)) { + // Relax TLSDESC to Local Exec: the TP-relative offset is + // materialized directly, so no dynamic relocation is needed. + // + // TLSDESC relocations must always be relaxed for statically-linked + // executables even with -no-relax, because a static executable + // doesn't contain the trampoline function TLSDESC needs. + } else if ctx.args.relax && sym.is_tprel_runtime_const(ctx) { + // The TP-relative offset is known at process startup, so relax to + // code that reads it from the GOT and adds TP. + sym.add_flags(NEEDS_GOTTP); + } else { + sym.add_flags(NEEDS_TLSDESC); + } +} + +pub fn check_tlsle(ctx: &Context, isec: &InputSection, sym: &Symbol, rel: &ElfRel) { + if ctx.args.shared { + error!( + ctx, + "{}: relocation {} against `{}` can not be used when making a shared object; recompile with -fPIC", + isec.display(&ctx.objs[isec.file.index()]), + rel.type_name::(), + sym + ); + } +} + +/// A CIE record in an input `.eh_frame` section. +/// +/// Compilers emit nearly identical CIEs in every object file, so the +/// linker deduplicates them; and since FDEs of dead functions must go, the +/// linker has to understand the record structure rather than copying +/// `.eh_frame` as opaque bytes. +#[derive(Clone, Copy, Debug)] +pub(crate) enum RelocationSpan { + Input(&'static [u8]), + SideTable(u32), +} + +impl RelocationSpan { + #[inline] + fn rels<'a, E: Layout>(self, file: &'a ObjectFile) -> Rels<'a, E> { + match self { + RelocationSpan::Input(data) => Rels::new(data), + RelocationSpan::SideTable(relsec_idx) => file.relocations::(Some(relsec_idx)), + } + } +} + +#[derive(Debug)] +pub struct CieRecord { + /// The `.eh_frame` input section containing the record. + pub section: u32, + /// That section's contents, which FDEs consult through their CIE. + pub contents: &'static [u8], + pub input_offset: u32, + pub output_offset: u32, + /// Index of the first relocation applying to the record. + pub rel_idx: u32, + /// The relocation table shared by this CIE and its FDEs. + pub(crate) relocations: RelocationSpan, + pub icf_idx: u32, + /// The size of the initial_location and address_range fields of FDEs + /// associated with this CIE: 4 or 8. + pub fde_ptr_size: u8, + /// Whether this is the representative of its equivalence class. + pub is_leader: bool, +} + +impl CieRecord { + #[inline] + pub fn size(&self) -> usize { + record_size::(self.contents, self.input_offset) + } + + #[inline] + pub fn contents(&self) -> &'static [u8] { + let start = self.input_offset as usize; + &self.contents[start..start + self.size::()] + } + + #[inline] + pub fn rels<'a, E: Layout>(&self, file: &'a ObjectFile) -> Rels<'a, E> { + rels_in( + self.relocations.rels::(file), + self.rel_idx, + self.input_offset as usize + self.size::(), + ) + } +} + +/// The size of the `.eh_frame` record at `offset`: its length field +/// plus the field itself. +#[inline] +fn record_size(contents: &[u8], offset: u32) -> usize { + E::Endian::read_u32(&contents[offset as usize..]) as usize + 4 +} + +/// The relocations of a `.eh_frame` record: those from index `begin` +/// that apply before `end`. +#[inline] +fn rels_in(rels: Rels<'_, E>, begin: u32, end: usize) -> Rels<'_, E> { + let begin = begin as usize; + let rest = rels.slice(begin..rels.len()); + let count = rest + .iter() + .take_while(|r| (r.r_offset as usize) < end) + .count(); + rels.slice(begin..begin + count) +} + +/// An FDE record in an input `.eh_frame` section. +#[derive(Debug)] +pub struct FdeRecord { + pub input_offset: u32, + pub output_offset: u32, + pub rel_idx: u32, + pub cie_idx: u16, + pub is_alive: AtomicBool, + pub(crate) is_last: bool, +} + +impl FdeRecord { + #[inline] + pub fn new(input_offset: u32, rel_idx: u32) -> FdeRecord { + FdeRecord { + input_offset, + output_offset: 0, + rel_idx, + cie_idx: 0, + is_alive: AtomicBool::new(true), + is_last: false, + } + } + + #[inline] + pub fn is_alive(&self) -> bool { + self.is_alive.load(Ordering::Relaxed) + } + + #[inline] + pub fn kill(&self) { + self.is_alive.store(false, Ordering::Relaxed); + } + + #[inline] + fn cie<'a>(&self, file: &'a ObjectFile) -> &'a CieRecord { + &file.cies[self.cie_idx as usize] + } + + #[inline] + pub fn size(&self, file: &ObjectFile) -> usize { + self.size_with::(&file.cies) + } + + #[inline] + pub(crate) fn size_with(&self, cies: &[CieRecord]) -> usize { + record_size::(cies[self.cie_idx as usize].contents, self.input_offset) + } + + #[inline] + pub fn contents(&self, file: &ObjectFile) -> &'static [u8] { + let start = self.input_offset as usize; + &self.cie(file).contents[start..start + self.size::(file)] + } + + #[inline] + pub fn rels<'a, E: Layout>(&self, file: &'a ObjectFile) -> Rels<'a, E> { + let cie = self.cie(file); + let end = self.input_offset as usize + record_size::(cie.contents, self.input_offset); + rels_in(cie.relocations.rels::(file), self.rel_idx, end) + } +} + +/// A function descriptor entry read from an input `.sframe` section. The +/// FRE block carries no relocations and is copied verbatim; only the +/// function start is relocated. +#[derive(Debug)] +pub struct SFrameFde { + /// The function's input section, for liveness. + pub section: u32, + /// The symbol the func_start relocation refers to. + pub sym: SymbolId, + pub addend: i64, + pub fre: &'static [u8], + pub func_size: u32, + pub num_fres: u32, +} + +/// A section fragment: the unit of deduplication in a mergeable section. +#[derive(Debug)] +pub struct SectionFragment { + pub p2align: AtomicU8, + /// The offset in the output section. Fragments are laid out in runs + /// that are processed in parallel; each fragment belongs to exactly + /// one run. + offset: AtomicU64, + pub is_alive: AtomicBool, + /// Whether the fragment must be placed within 4 GiB of the start of + /// the output section. + pub is_32bit: AtomicBool, +} + +impl SectionFragment { + #[inline] + pub fn new(is_alive: bool) -> SectionFragment { + SectionFragment { + p2align: AtomicU8::new(0), + offset: AtomicU64::new(0), + is_alive: AtomicBool::new(is_alive), + is_32bit: AtomicBool::new(false), + } + } + + #[inline] + pub fn offset(&self) -> u64 { + self.offset.load(Ordering::Relaxed) + } + + #[inline] + pub fn set_offset(&self, offset: u64) { + self.offset.store(offset, Ordering::Relaxed); + } + + #[inline] + pub fn is_alive(&self) -> bool { + self.is_alive.load(Ordering::Relaxed) + } + + #[inline] + pub fn set_alive(&self) { + self.is_alive.store(true, Ordering::Relaxed); + } + + pub fn is_32bit(&self) -> bool { + self.is_32bit.load(Ordering::Relaxed) + } + + pub fn set_32bit(&self) { + self.is_32bit.store(true, Ordering::Relaxed); + } +} + +/// An input section with the SHF_MERGE flag, split into fragments. +/// +/// Mergeable sections typically contain string literals or fixed-size +/// constants. The linker splits them into pieces, deduplicates the pieces +/// across all input files, and emits each unique piece once. +#[derive(Debug)] +pub struct MergeableSection { + pub parent: MergedSectionId, + pub p2align: u8, + pub shndx: u32, + input_offset: u32, + /// The fragment of each piece, parallel to `frag_offsets`. + pub fragments: Vec, + frag_offsets: Vec, + hashes: Vec, +} + +impl MergeableSection { + /// Refers to an input section in its stable dense slot. The section + /// itself is dead from now on; its contents live on as fragments. + #[inline] + fn new(parent: MergedSectionId, input_offset: u32, section: &InputSection) -> MergeableSection { + section.kill(); + MergeableSection { + parent, + p2align: section.p2align(), + shndx: section.shndx, + input_offset, + fragments: Vec::new(), + frag_offsets: Vec::new(), + hashes: Vec::new(), + } + } + + /// Splits the section contents into pieces and computes their hashes. + pub fn split_contents( + &mut self, + diag: &Diagnostics, + file: &dyn fmt::Display, + section: &InputSection, + name: &BStr, + parent: &MergedSection, + sketch: &mut HyperLogLog, + ) { + let data = section.contents(); + if data.len() > u32::MAX as usize { + fatal!( + diag, + "{}: mergeable section too large", + format_args!("{file}:({name})") + ); + } + let entsize = parent.hdr.shdr.sh_entsize as usize; + + if parent.hdr.shdr.sh_flags & SHF_STRINGS as u64 != 0 { + let mut pos = 0; + while pos < data.len() { + self.frag_offsets.push(pos as u32); + let Some(end) = find_null(data, pos, entsize) else { + fatal!( + diag, + "{}: string is not null terminated", + format_args!("{file}:({name})") + ); + }; + pos = end + entsize; + } + } else { + if !data.len().is_multiple_of(entsize) { + fatal!( + diag, + "{}: section size is not multiple of sh_entsize", + format_args!("{file}:({name})") + ); + } + self.frag_offsets = (0..data.len()).step_by(entsize).map(|p| p as u32).collect(); + } + + // Compute hashes for section pieces + self.hashes.reserve(self.frag_offsets.len()); + for i in 0..self.frag_offsets.len() { + let hash = xxhash_rust::xxh3::xxh3_64(self.contents(section, i)); + self.hashes.push(hash); + sketch.insert(hash); + } + } + + /// Inserts the pieces into the parent section's fragment map. + pub fn resolve_contents( + &mut self, + section: &InputSection, + parent: &crate::chunks::merged::MergedSection, + gc_sections: bool, + ) { + let n = self.frag_offsets.len(); + self.fragments.reserve(n); + + // The hash table is typically much larger than the cache, so each + // insertion stalls on a cache miss for its first probe. We know all + // hashes upfront, so prefetch the bucket a few insertions ahead. + const LOOKAHEAD: usize = 8; + + for i in 0..n { + if let Some(&hash) = self.hashes.get(i + LOOKAHEAD) { + parent.map.prefetch(hash); + } + let id = parent.insert( + self.contents(section, i), + self.hashes[i], + self.p2align, + gc_sections, + ); + self.fragments.push(id); + } + + // Reclaim memory as we'll never use this vector again. + self.hashes = Vec::new(); + } + + /// Finds the fragment containing `offset` and the offset within it. + #[inline] + pub fn fragment(&self, offset: u64) -> Option<(EntryId, i64)> { + let idx = self + .frag_offsets + .partition_point(|&o| o as u64 <= offset) + .checked_sub(1)?; + Some(( + self.fragments[idx], + offset as i64 - self.frag_offsets[idx] as i64, + )) + } + + #[inline] + fn contents(&self, section: &InputSection, i: usize) -> &'static [u8] { + let contents = section.contents(); + let start = self.frag_offsets[i] as usize; + match self.frag_offsets.get(i + 1) { + Some(&end) => &contents[start..end as usize], + None => &contents[start..], + } + } +} + +#[inline] +fn find_null(data: &[u8], pos: usize, entsize: usize) -> Option { + if entsize == 1 { + return memchr::memchr(0, &data[pos..]).map(|i| pos + i); + } + let mut p = pos; + while p + entsize <= data.len() { + if data[p..p + entsize].iter().all(|&b| b == 0) { + return Some(p); + } + p += entsize; + } + None +} + +/// ObjectFile needs a lookup table indexed by ELF section number. The table +/// lives outside the arena, so a regular section is stored as its 31-bit arena +/// index. The high bit distinguishes mergeable sections, which are stored as +/// indices into `mergeable`. +#[derive(Debug)] +pub struct SectionList { + indices: Vec, + mergeable: Vec, + arena_base: NonNull, +} + +// SAFETY: a SectionList owns the distinct arena objects named by its table. +// Shared access yields only shared references, and mutation requires an +// exclusive borrow of the list. +unsafe impl Send for SectionList {} +unsafe impl Sync for SectionList {} + +/// A sparsely-backed address range for input sections. Allocation is +/// thread-safe and monotonic; individual allocations are not freed. Large +/// links fill the beginning densely, so transparent huge pages reduce +/// address-translation overhead without populating unused pages. +pub struct SectionArena { + data: NonNull, + offset: std::sync::atomic::AtomicUsize, + size: usize, + id: u64, +} + +const SECTION_ARENA_BLOCK_SIZE: usize = 64 * 1024; +const SECTION_ARENA_BLOCK_ALIGNMENT: usize = 64; +const MAX_LOCAL_SECTION_ALLOC: usize = SECTION_ARENA_BLOCK_SIZE / 4; + +#[derive(Clone, Copy)] +struct LocalSectionBlock { + arena_id: u64, + position: usize, + end: usize, +} + +thread_local! { + static LOCAL_SECTION_BLOCK: Cell = const { + Cell::new(LocalSectionBlock { + arena_id: 0, + position: 0, + end: 0, + }) + }; +} + +static NEXT_SECTION_ARENA_ID: AtomicU64 = AtomicU64::new(1); + +// SAFETY: allocation returns disjoint ranges, and the arena never accesses +// their contents itself. +unsafe impl Send for SectionArena {} +unsafe impl Sync for SectionArena {} + +impl SectionArena { + // The mapping is much larger than ordinary links need. A smaller + // reservation is used on 32-bit hosts, where address space is limited. + const SIZE: usize = if usize::BITS == 64 { + 1usize << 33 + } else { + 1usize << 28 + }; + + pub fn new() -> SectionArena { + let flags = libc::MAP_ANONYMOUS | libc::MAP_PRIVATE; + #[cfg(any(target_os = "android", target_os = "linux"))] + let flags = flags | libc::MAP_NORESERVE; + + // SAFETY: this creates private anonymous storage. SectionList makes + // typed references only to elements it has initialized. + let data = unsafe { + libc::mmap( + std::ptr::null_mut(), + Self::SIZE, + libc::PROT_READ | libc::PROT_WRITE, + flags, + -1, + 0, + ) + }; + if data == libc::MAP_FAILED { + panic!( + "mmap of {} bytes for input sections failed: {}", + Self::SIZE, + std::io::Error::last_os_error() + ); + } + + #[cfg(any(target_os = "android", target_os = "linux"))] + // SAFETY: the range is the fresh mapping; the advice is only a hint. + unsafe { + libc::madvise(data, Self::SIZE, libc::MADV_HUGEPAGE); + } + + SectionArena { + data: NonNull::new(data.cast()).expect("mmap returned a null address"), + // Leave the first slots unused so that a base-relative index is + // never zero. + offset: std::sync::atomic::AtomicUsize::new(8), + size: Self::SIZE, + id: NEXT_SECTION_ARENA_ID.fetch_add(1, Ordering::Relaxed), + } + } + + fn allocate_global(&self, size: usize, alignment: usize) -> usize { + debug_assert!(alignment.is_power_of_two()); + let mut old = self.offset.load(Ordering::Relaxed); + loop { + let begin = old + .checked_add(alignment - 1) + .map(|value| value & !(alignment - 1)) + .expect("input-section arena is full"); + let end = begin + .checked_add(size) + .expect("input-section arena is full"); + assert!(end <= self.size, "input-section arena is full"); + match self + .offset + .compare_exchange_weak(old, end, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return begin, + Err(value) => old = value, + } + } + } + + fn allocate_offset(&self, size: usize, alignment: usize) -> usize { + if size <= MAX_LOCAL_SECTION_ALLOC && alignment <= SECTION_ARENA_BLOCK_ALIGNMENT { + // Most files need a small section block. Reserve such allocations + // from a thread-local block to avoid contending on the global bump + // pointer. + return LOCAL_SECTION_BLOCK.with(|slot| { + let mut local = slot.get(); + if local.arena_id == self.id { + let begin = (local.position + alignment - 1) & !(alignment - 1); + if begin <= local.end && size <= local.end - begin { + local.position = begin + size; + slot.set(local); + return begin; + } + } + + let begin = + self.allocate_global(SECTION_ARENA_BLOCK_SIZE, SECTION_ARENA_BLOCK_ALIGNMENT); + slot.set(LocalSectionBlock { + arena_id: self.id, + position: begin + size, + end: begin + SECTION_ARENA_BLOCK_SIZE, + }); + begin + }); + } + + self.allocate_global(size, alignment) + } + + fn insert(&self, section: InputSection) -> u32 { + let size = std::mem::size_of::(); + let begin = self.allocate_offset(size, std::mem::align_of::()); + debug_assert!(begin > 0 && begin < Self::SIZE); + debug_assert_eq!(begin % 4, 0); + debug_assert_eq!(begin % std::mem::align_of::(), 0); + + // SAFETY: the atomic bump pointer assigned this object a disjoint, + // properly aligned range within the mapping. + unsafe { + self.data + .as_ptr() + .add(begin) + .cast::() + .write(section) + }; + u32::try_from(begin / 4).expect("input-section arena is too large") + } + + #[inline] + fn input_ptr(&self, id: InputSectionId) -> *mut InputSection { + debug_assert_ne!(id, InputSectionId::NONE); + // SAFETY: every nonzero InputSectionId was created from an initialized + // allocation within this arena. + unsafe { + self.data + .as_ptr() + .add(id.0 as usize * 4) + .cast::() + } + } + + #[inline] + pub(crate) fn section(&self, id: InputSectionId) -> &InputSection { + // SAFETY: InputSectionId values remain live until their SectionList is + // dropped, after all output-section users. + unsafe { &*self.input_ptr(id) } + } + + #[inline] + pub(crate) fn section_mut(&mut self, id: InputSectionId) -> &mut InputSection { + // SAFETY: an exclusive arena borrow gives exclusive access to the + // section named by this id. + unsafe { &mut *self.input_ptr(id) } + } + + fn insert_extra(&self, extra: InputSectionExtras) -> *mut InputSectionExtras { + let begin = self.allocate_offset( + std::mem::size_of::(), + std::mem::align_of::(), + ); + // SAFETY: the arena assigned this record a disjoint, properly aligned + // range which remains live until after all InputSections are dropped. + let ptr = unsafe { self.data.as_ptr().add(begin).cast::() }; + unsafe { ptr.write(extra) }; + ptr + } +} + +impl Default for SectionArena { + fn default() -> Self { + Self::new() + } +} + +impl Drop for SectionArena { + fn drop(&mut self) { + // SAFETY: SectionLists have already dropped their initialized + // elements; this releases the mapping that supplied their storage. + unsafe { libc::munmap(self.data.as_ptr().cast(), self.size) }; + } +} + +const MERGEABLE_SECTION: u32 = 1 << 31; +const SECTION_INDEX_MASK: u32 = !MERGEABLE_SECTION; + +impl Default for SectionList { + fn default() -> Self { + SectionList { + indices: Vec::new(), + mergeable: Vec::new(), + arena_base: NonNull::dangling(), + } + } +} + +impl SectionList { + /// A list for `nsections` section indices, none with a section yet. + #[inline] + pub fn new(nsections: usize, additional: usize, arena: &SectionArena) -> SectionList { + let mut indices = vec![0; nsections]; + indices.reserve(additional); + SectionList { + indices, + mergeable: Vec::new(), + arena_base: arena.data, + } + } + + #[inline] + fn input_ptr(&self, index: u32) -> *mut InputSection { + debug_assert!(index != 0 && index & MERGEABLE_SECTION == 0); + // SAFETY: regular indices are offsets in four-byte units into this + // SectionList's arena mapping. + unsafe { self.arena_base.as_ptr().add(index as usize * 4).cast() } + } + + /// The number of section indices. + #[inline] + pub fn len(&self) -> usize { + self.indices.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.indices.is_empty() + } + + /// Adds the section for `shndx`, which has none yet. + #[inline] + pub fn insert(&mut self, shndx: usize, section: InputSection, arena: &SectionArena) { + debug_assert_eq!(self.indices[shndx], 0); + debug_assert_eq!(self.arena_base, arena.data); + self.indices[shndx] = arena.insert(section); + } + + /// Adds a section the linker made up, under a new section index. + pub fn push(&mut self, section: InputSection, arena: &SectionArena) { + self.indices.push(0); + self.insert(self.indices.len() - 1, section, arena); + } + + #[inline] + pub fn section(&self, shndx: usize) -> Option<&InputSection> { + let value = *self.indices.get(shndx)?; + if value == 0 { + None + } else if value & MERGEABLE_SECTION != 0 { + let m = &self.mergeable[((value & SECTION_INDEX_MASK) - 1) as usize]; + // SAFETY: the mergeable section retains the arena index of its + // stable input section. + Some(unsafe { &*self.input_ptr(m.input_offset) }) + } else { + // SAFETY: a nonzero regular table entry is a live arena index. + Some(unsafe { &*self.input_ptr(value) }) + } + } + + #[inline] + pub fn section_mut(&mut self, shndx: usize) -> Option<&mut InputSection> { + let value = *self.indices.get(shndx)?; + if value == 0 { + None + } else if value & MERGEABLE_SECTION != 0 { + let input_offset = + self.mergeable[((value & SECTION_INDEX_MASK) - 1) as usize].input_offset; + // SAFETY: an exclusive SectionList borrow gives exclusive access + // to each input section it owns. + Some(unsafe { &mut *self.input_ptr(input_offset) }) + } else { + // SAFETY: an exclusive SectionList borrow gives exclusive access + // to each input section it owns. + Some(unsafe { &mut *self.input_ptr(value) }) + } + } + + #[inline] + pub fn mergeable(&self, shndx: usize) -> Option<&MergeableSection> { + let value = *self.indices.get(shndx)?; + (value & MERGEABLE_SECTION != 0) + .then(|| &self.mergeable[((value & SECTION_INDEX_MASK) - 1) as usize]) + } + + #[inline] + pub fn mergeable_mut(&mut self, shndx: usize) -> Option<&mut MergeableSection> { + let value = *self.indices.get(shndx)?; + (value & MERGEABLE_SECTION != 0) + .then(|| &mut self.mergeable[((value & SECTION_INDEX_MASK) - 1) as usize]) + } + + /// Returns a regular section before it is converted to a mergeable one. + pub fn regular_section_mut(&mut self, shndx: usize) -> Option<&mut InputSection> { + let value = self.indices[shndx]; + (value != 0 && value & MERGEABLE_SECTION == 0) + // SAFETY: an exclusive SectionList borrow gives exclusive access + // to the regular section named by this table entry. + .then(|| unsafe { &mut *self.input_ptr(value) }) + } + + /// Installs mergeable metadata while leaving the input section in its + /// stable arena slot. + pub fn set_mergeable(&mut self, shndx: usize, parent: MergedSectionId) { + let value = self.indices[shndx]; + debug_assert!(value != 0 && value & MERGEABLE_SECTION == 0); + debug_assert!(self.mergeable.len() < SECTION_INDEX_MASK as usize); + // SAFETY: `value` is the regular arena index currently in this slot. + let input = unsafe { &*self.input_ptr(value) }; + self.mergeable + .push(MergeableSection::new(parent, value, input)); + self.indices[shndx] = MERGEABLE_SECTION | self.mergeable.len() as u32; + } + + /// Returns mergeable metadata together with its stable input section. + pub fn mergeable_with_section_mut( + &mut self, + shndx: usize, + ) -> Option<(&mut MergeableSection, &InputSection)> { + let value = *self.indices.get(shndx)?; + if value & MERGEABLE_SECTION == 0 { + return None; + } + let mergeable_idx = ((value & SECTION_INDEX_MASK) - 1) as usize; + let input_offset = self.mergeable[mergeable_idx].input_offset; + let input = self.input_ptr(input_offset); + // SAFETY: mergeable metadata and its arena-allocated input section + // occupy disjoint storage and both belong to this SectionList. + Some((&mut self.mergeable[mergeable_idx], unsafe { &*input })) + } + + /// The regular sections in section order. + pub fn regular(&self) -> impl Iterator { + self.indices + .iter() + .copied() + .filter(|&index| index != 0 && index & MERGEABLE_SECTION == 0) + .map(|index| { + // SAFETY: every yielded regular index names a distinct, + // initialized section owned by this SectionList. + unsafe { &*self.input_ptr(index) } + }) + } + + pub fn regular_mut(&mut self) -> impl Iterator { + let base = self.arena_base; + self.indices + .iter() + .copied() + .filter(|&index| index != 0 && index & MERGEABLE_SECTION == 0) + .map(move |index| { + // SAFETY: regular table entries are distinct, and the + // iterator holds an exclusive borrow of this SectionList. + unsafe { &mut *base.as_ptr().add(index as usize * 4).cast() } + }) + } + + /// The regular sections and their compact arena pointers, in section order. + pub fn regular_ids_mut(&mut self) -> impl Iterator { + let base = self.arena_base; + self.indices + .iter() + .copied() + .filter(|&index| index != 0 && index & MERGEABLE_SECTION == 0) + .map(move |index| { + // SAFETY: regular table entries are distinct, and the + // iterator holds an exclusive borrow of this SectionList. + let section = unsafe { &mut *base.as_ptr().add(index as usize * 4).cast() }; + (InputSectionId(index), section) + }) + } + + pub fn mergeable_sections(&self) -> impl Iterator { + self.mergeable.iter() + } + + pub fn mergeable_sections_mut(&mut self) -> impl Iterator { + self.mergeable.iter_mut() + } + + /// The mergeable sections together with their stable input sections. + pub fn mergeable_sections_with_inputs_mut( + &mut self, + ) -> impl Iterator { + let base = self.arena_base; + self.mergeable.iter_mut().map(move |m| { + // SAFETY: each mergeable record retains the arena index of its + // initialized input section, which is disjoint from this vector. + let input = unsafe { + &*base + .as_ptr() + .add(m.input_offset as usize * 4) + .cast::() + }; + (m, input) + }) + } +} + +impl Drop for SectionList { + fn drop(&mut self) { + for &value in &self.indices { + let index = if value == 0 { + continue; + } else if value & MERGEABLE_SECTION != 0 { + self.mergeable[((value & SECTION_INDEX_MASK) - 1) as usize].input_offset + } else { + value + }; + // SAFETY: each initialized input section appears exactly once in + // the table, either directly or through its mergeable metadata. + unsafe { std::ptr::drop_in_place(self.input_ptr(index)) }; + } + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..dd0518d06 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,30 @@ +//! A high-performance ELF linker. + +pub mod arch; +pub mod archive; +pub mod args; +pub mod chunks; +pub mod context; +pub mod diagnostics; +pub mod driver; +pub mod elf; +pub mod filetype; +pub mod gc_sections; +pub mod gdb_index; +pub mod icf; +pub mod input_files; +pub mod input_sections; +pub mod linker_script; +pub mod lto; +pub mod mapfile; +pub mod mapped_file; +pub mod output_file; +pub mod passes; +pub mod reader; +pub mod relax; +pub mod relocatable; +pub mod subprocess; +pub mod symbol; +pub mod thunks; +pub mod tls; +pub mod util; diff --git a/src/linker_script.rs b/src/linker_script.rs new file mode 100644 index 000000000..5e9fb33b0 --- /dev/null +++ b/src/linker_script.rs @@ -0,0 +1,559 @@ +//! A parser for the subset of the GNU linker script language that mold +//! supports: `GROUP`/`INPUT` file lists as found in glibc's `libc.so`, +//! `OUTPUT_FORMAT`, symbol assignments, version scripts and dynamic +//! lists. + +use crate::arch::Arch; +use crate::args::{DefsymValue, ReaderContext}; +use crate::context::Context; +use crate::elf::*; +use crate::mapped_file::{must_open_file, open_file, MappedFile}; +use crate::reader; +use crate::util::{self, path_clean}; +use crate::{fatal, warn}; + +/// A version script pattern. +#[derive(Clone, Debug)] +pub struct VersionPattern { + pub pattern: &'static [u8], + pub source: String, + pub ver_str: &'static [u8], + pub ver_idx: u16, + pub is_cpp: bool, +} + +/// A dynamic list pattern. +#[derive(Clone, Debug)] +pub struct DynamicPattern { + pub pattern: &'static [u8], + pub source: String, + pub is_cpp: bool, +} + +pub struct Script<'a, E: Arch> { + ctx: &'a mut Context, + rctx: &'a mut ReaderContext, + mf: &'static MappedFile, + tokens: Vec<&'static [u8]>, +} + +fn get_line(input: &[u8], pos: usize) -> (usize, &[u8]) { + let start = input[..pos] + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |i| i + 1); + let end = input[pos..] + .iter() + .position(|&b| b == b'\n') + .map_or(input.len(), |i| pos + i); + (start, &input[start..end]) +} + +/// Position of a token's bytes within the script. +fn offset_of(input: &[u8], tok: &[u8]) -> usize { + tok.as_ptr() as usize - input.as_ptr() as usize +} + +fn unquote(s: &[u8]) -> &[u8] { + match s.strip_prefix(b"\"") { + Some(rest) => rest.strip_suffix(b"\"").unwrap_or(rest), + None => s, + } +} + +/// Reports a syntax error, pointing at the offending token. +fn syntax_error(ctx: &Context, mf: &MappedFile, tok: &[u8], msg: &str) -> ! { + let input = mf.data(); + let pos = offset_of(input, tok).min(input.len().saturating_sub(1)); + let (line_start, line) = get_line(input, pos); + let lineno = input[..line_start].iter().filter(|&&b| b == b'\n').count() + 1; + let label = format!("{}:{}: ", mf.name, lineno); + let indent = "mold: fatal: ".len() + label.len(); + let column = pos - line_start; + fatal!( + ctx, + "{label}{}\n{}^ {msg}", + util::display(line), + " ".repeat(indent + column) + ); +} + +fn tokenize(ctx: &Context, mf: &'static MappedFile) -> Vec<&'static [u8]> { + let mut tokens = Vec::new(); + let mut input = mf.data(); + + while let Some(&c) = input.first() { + if c.is_ascii_whitespace() { + input = &input[1..]; + continue; + } + + if input.starts_with(b"/*") { + let Some(pos) = input[2..].windows(2).position(|w| w == b"*/") else { + syntax_error(ctx, mf, input, "unclosed comment"); + }; + input = &input[pos + 4..]; + continue; + } + + if c == b'#' { + match input.iter().position(|&b| b == b'\n') { + Some(pos) => input = &input[pos + 1..], + None => break, + } + continue; + } + + if c == b'"' { + let Some(pos) = input[1..].iter().position(|&b| b == b'"') else { + syntax_error(ctx, mf, input, "unclosed string literal"); + }; + tokens.push(&input[..pos + 2]); + input = &input[pos + 2..]; + continue; + } + + let is_word_char = |b: u8| b.is_ascii_alphanumeric() || b"_.$/\\~=+[]*?-!^:".contains(&b); + let len = match input.iter().position(|&b| !is_word_char(b)) { + Some(0) => 1, + Some(pos) => pos, + None => input.len(), + }; + tokens.push(&input[..len]); + input = &input[len..]; + } + tokens +} + +fn is_in_sysroot(ctx: &Context, path: &str) -> bool { + let mut sysroot = ctx.args.sysroot.clone(); + if sysroot.starts_with('/') && !ctx.args.chroot.is_empty() { + sysroot = format!("{}/{}", ctx.args.chroot, path_clean(&sysroot)); + } + let path = std::path::Path::new(path); + let sysroot = std::path::Path::new(&sysroot); + let (Ok(path), Ok(sysroot)) = (path.canonicalize(), sysroot.canonicalize()) else { + return false; + }; + path != sysroot && path.starts_with(&sysroot) +} + +/// Opens a file named by the script `mf`, resolving relative names +/// against the script's directory and the library search paths. +fn resolve_path( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, + tok: &'static [u8], + check_target: bool, +) -> &'static MappedFile { + let s = String::from_utf8_lossy(unquote(tok)).into_owned(); + let diag = &ctx.diag; + let chroot = &ctx.args.chroot; + + let open = |path: &str| -> Option<&'static MappedFile> { + let mf = open_file(diag, chroot, path)?; + if check_target { + if let Some(target) = + crate::filetype::get_machine_type(diag, &ctx.args.plugin, mf, || None) + { + if target != E::NAME { + warn!( + ctx, + "{path}: skipping incompatible file: {target} (e_machine {})", + E::E_MACHINE + ); + return None; + } + } + } + Some(mf) + }; + + // GNU ld prepends the sysroot if a pathname starts with '/' and the + // script being processed is in the sysroot. We do the same. + if s.starts_with('/') && is_in_sysroot(ctx, &mf.name) { + let path = format!("{}{s}", ctx.args.sysroot); + return must_open_file(diag, chroot, &path); + } + + if let Some(rest) = s.strip_prefix('=') { + let path = if ctx.args.sysroot.is_empty() { + rest.to_string() + } else { + format!("{}{rest}", ctx.args.sysroot) + }; + return must_open_file(diag, chroot, &path); + } + + if let Some(lib) = s.strip_prefix("-l") { + return reader::find_library(ctx, rctx, lib); + } + + if !s.starts_with('/') { + let path = path_clean(&format!("{}/../{s}", mf.name)); + if let Some(mf) = open(&path) { + return mf; + } + } + + if let Some(mf) = open(&s) { + return mf; + } + + for dir in &ctx.args.library_paths { + if let Some(mf) = open(&format!("{dir}/{s}")) { + return mf; + } + } + + syntax_error(ctx, mf, tok, &format!("library not found: {s}")); +} + +/// The target a script produces output for: the one `OUTPUT_FORMAT` +/// names, or else that of the first file it names. +pub fn output_target( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, +) -> Option<&'static str> { + let tokens = tokenize(ctx, mf); + let mut tok: &[&'static [u8]] = &tokens; + + if tok.len() >= 3 && tok[0] == b"OUTPUT_FORMAT" && tok[1] == b"(" { + if tok[2] == b"elf64-x86-64" { + return Some("x86_64"); + } + if tok[2] == b"elf32-i386" { + return Some("i386"); + } + } + + if tok.len() >= 3 && (tok[0] == b"INPUT" || tok[0] == b"GROUP") && tok[1] == b"(" { + if tok.len() >= 5 && tok[2] == b"AS_NEEDED" && tok[3] == b"(" { + tok = &tok[2..]; + } + let named = resolve_path(ctx, rctx, mf, tok[2], false); + return reader::get_machine_type(ctx, rctx, named); + } + None +} + +impl<'a, E: Arch> Script<'a, E> { + pub fn new( + ctx: &'a mut Context, + rctx: &'a mut ReaderContext, + mf: &'static MappedFile, + ) -> Self { + let tokens = tokenize(ctx, mf); + Script { + ctx, + rctx, + mf, + tokens, + } + } + + fn error(&self, tok: &[u8], msg: &str) -> ! { + syntax_error(self.ctx, self.mf, tok, msg) + } + + fn skip<'t>(&self, tok: &'t [&'static [u8]], expected: &str) -> &'t [&'static [u8]] { + match tok.first() { + None => fatal!( + self.ctx, + "{}: expected '{expected}', but got EOF", + self.mf.name + ), + Some(t) if *t == expected.as_bytes() => &tok[1..], + Some(t) => self.error(t, &format!("expected '{expected}'")), + } + } + + /// Version scripts and dynamic lists take a quoted name literally, + /// while an unquoted one is a glob pattern. Glob metacharacters in a + /// quoted name are escaped so the matcher treats them as literals. + fn unquote_pattern(&self, tok: &'static [u8]) -> &'static [u8] { + if !tok.starts_with(b"\"") { + return tok; + } + let s = unquote(tok); + if !s.iter().any(|&c| matches!(c, b'*' | b'?' | b'[' | b'\\')) { + return s; + } + let mut out = Vec::with_capacity(s.len() * 2); + for &c in s { + if matches!(c, b'*' | b'?' | b'[' | b'\\') { + out.push(b'\\'); + } + out.push(c); + } + util::leak_bytes(out) + } + + fn read_output_format<'t>(&self, tok: &'t [&'static [u8]]) -> &'t [&'static [u8]] { + let tok = self.skip(tok, "("); + match tok.iter().position(|t| *t == b")") { + Some(pos) => &tok[pos + 1..], + None => fatal!(self.ctx, "{}: expected ')', but got EOF", self.mf.name), + } + } + + fn read_group<'t>(&mut self, mut tok: &'t [&'static [u8]]) -> &'t [&'static [u8]] { + tok = self.skip(tok, "("); + + while let Some(&t) = tok.first() { + if t == b")" { + break; + } + if t == b"AS_NEEDED" { + let orig = self.rctx.as_needed; + self.rctx.as_needed = true; + tok = self.read_group(&tok[1..]); + self.rctx.as_needed = orig; + continue; + } + + let mf = resolve_path(self.ctx, self.rctx, self.mf, t, true); + let mut child = self.rctx.next_child(); + reader::read_file(self.ctx, &mut child, mf); + tok = &tok[1..]; + } + + if tok.is_empty() { + fatal!(self.ctx, "{}: expected ')', but got EOF", self.mf.name); + } + &tok[1..] + } + + pub fn parse_linker_script(&mut self) { + let tokens = self.tokens.clone(); + let mut tok: &[&'static [u8]] = &tokens; + + while let Some(&t) = tok.first() { + if t == b"OUTPUT_FORMAT" { + tok = self.read_output_format(&tok[1..]); + } else if t == b"INPUT" || t == b"GROUP" { + tok = self.read_group(&tok[1..]); + } else if t == b"VERSION" { + tok = self.skip(&tok[1..], "{"); + tok = self.read_version_script(tok); + tok = self.skip(tok, "}"); + } else if tok.len() > 3 && tok[1] == b"=" && tok[3] == b";" { + let name = String::from_utf8_lossy(unquote(tok[0])).into_owned(); + let value = String::from_utf8_lossy(unquote(tok[2])).into_owned(); + self.ctx + .args + .defsyms + .push((name, DefsymValue::Symbol(value))); + tok = &tok[4..]; + } else if t == b";" { + tok = &tok[1..]; + } else { + self.error(t, "unknown linker script token"); + } + } + } + + fn read_version_script_commands<'t>( + &mut self, + mut tok: &'t [&'static [u8]], + ver_str: &'static [u8], + ver_idx: u16, + mut is_global: bool, + is_cpp: bool, + ) -> &'t [&'static [u8]] { + while let Some(&t) = tok.first() { + if t == b"}" { + break; + } + + if let Some(rest) = read_label(tok, b"global") { + is_global = true; + tok = rest; + continue; + } + if let Some(rest) = read_label(tok, b"local") { + is_global = false; + tok = rest; + continue; + } + + if t == b"extern" { + tok = &tok[1..]; + if tok.first() == Some(&&b"\"C\""[..]) { + tok = self.skip(&tok[1..], "{"); + tok = + self.read_version_script_commands(tok, ver_str, ver_idx, is_global, false); + } else { + tok = self.skip(tok, "\"C++\""); + tok = self.skip(tok, "{"); + tok = self.read_version_script_commands(tok, ver_str, ver_idx, is_global, true); + } + tok = self.skip(tok, "}"); + tok = self.skip(tok, ";"); + continue; + } + + if t == b"*" { + self.ctx.default_version = if is_global { + ver_idx + } else { + VER_NDX_LOCAL as u16 + }; + } else { + let pattern = self.unquote_pattern(t); + self.ctx.version_patterns.push(VersionPattern { + pattern, + source: self.mf.name.clone(), + ver_str, + ver_idx: if is_global { + ver_idx + } else { + VER_NDX_LOCAL as u16 + }, + is_cpp, + }); + } + + tok = &tok[1..]; + if tok.first() == Some(&&b"}"[..]) { + break; + } + tok = self.skip(tok, ";"); + } + tok + } + + fn read_version_script<'t>(&mut self, mut tok: &'t [&'static [u8]]) -> &'t [&'static [u8]] { + let mut next_ver = + VER_NDX_LAST_RESERVED as u16 + self.ctx.args.version_definitions.len() as u16 + 1; + + while let Some(&t) = tok.first() { + if t == b"}" { + break; + } + + let (ver_str, ver_idx): (&'static [u8], u16) = if t == b"{" { + (b"global", VER_NDX_GLOBAL as u16) + } else { + let idx = next_ver; + next_ver += 1; + self.ctx + .args + .version_definitions + .push(String::from_utf8_lossy(t).into_owned()); + tok = &tok[1..]; + (t, idx) + }; + + tok = self.skip(tok, "{"); + tok = self.read_version_script_commands(tok, ver_str, ver_idx, true, false); + tok = self.skip(tok, "}"); + if let Some(&t) = tok.first() { + if t != b";" { + // A parent version, e.g. `} VER_1.0;` + tok = &tok[1..]; + } + } + tok = self.skip(tok, ";"); + } + tok + } + + pub fn parse_version_script(&mut self) { + let tokens = split_labels(&self.tokens); + let tok = self.read_version_script(&tokens); + if let Some(&t) = tok.first() { + self.error(t, "trailing garbage token"); + } + } + + fn read_dynamic_list_commands<'t>( + &mut self, + mut tok: &'t [&'static [u8]], + result: &mut Vec, + is_cpp: bool, + ) -> &'t [&'static [u8]] { + while let Some(&t) = tok.first() { + if t == b"}" { + break; + } + + if t == b"extern" { + tok = &tok[1..]; + if tok.first() == Some(&&b"\"C\""[..]) { + tok = self.skip(&tok[1..], "{"); + tok = self.read_dynamic_list_commands(tok, result, false); + } else { + tok = self.skip(tok, "\"C++\""); + tok = self.skip(tok, "{"); + tok = self.read_dynamic_list_commands(tok, result, true); + } + tok = self.skip(tok, "}"); + tok = self.skip(tok, ";"); + continue; + } + + result.push(DynamicPattern { + pattern: self.unquote_pattern(t), + source: self.mf.name.clone(), + is_cpp, + }); + tok = self.skip(&tok[1..], ";"); + } + tok + } + + pub fn parse_dynamic_list(&mut self) -> Vec { + let tokens = self.tokens.clone(); + let mut result = Vec::new(); + + let tok = self.skip(&tokens, "{"); + let tok = self.read_dynamic_list_commands(tok, &mut result, false); + let tok = self.skip(tok, "}"); + let tok = self.skip(tok, ";"); + if let Some(&t) = tok.first() { + self.error(t, "trailing garbage token"); + } + result + } +} + +/// The tokenizer keeps colons inside tokens because of the C++ scope +/// operator, so `local:*` is a single token. Splits such labels off. +fn split_labels(tokens: &[&'static [u8]]) -> Vec<&'static [u8]> { + let mut out = Vec::with_capacity(tokens.len()); + for &tok in tokens { + let glued = [&b"global:"[..], b"local:"].into_iter().find(|label| { + tok.starts_with(label) && tok.len() > label.len() && tok[label.len()] != b':' + }); + match glued { + Some(label) => { + let colon = label.len() - 1; + out.extend([&tok[..colon], &tok[colon..colon + 1], &tok[colon + 1..]]); + } + None => out.push(tok), + } + } + out +} + +/// Matches a `global:` or `local:` label. +fn read_label<'t>(tok: &'t [&'static [u8]], label: &[u8]) -> Option<&'t [&'static [u8]]> { + let first = *tok.first()?; + let with_colon = [label, b":"].concat(); + if first == with_colon.as_slice() { + return Some(&tok[1..]); + } + if first == label && tok.get(1) == Some(&&b":"[..]) { + return Some(&tok[2..]); + } + None +} + +pub fn parse_dynamic_list(ctx: &mut Context, path: &str) -> Vec { + let mf = must_open_file(&ctx.diag, &ctx.args.chroot.clone(), path); + let mut rctx = ReaderContext::default(); + Script::new(ctx, &mut rctx, mf).parse_dynamic_list() +} diff --git a/src/lto.rs b/src/lto.rs new file mode 100644 index 000000000..b13457847 --- /dev/null +++ b/src/lto.rs @@ -0,0 +1,921 @@ +//! Link-time optimization through the linker plugin interface. +//! +//! With `-flto`, compilers emit files holding their intermediate +//! representation instead of machine code: GCC wraps its IR in an ELF +//! file, LLVM writes bitcode. Such files can't be linked directly. The +//! compiler provides a linker plugin (`-plugin`) instead, and the link +//! goes like this: +//! +//! 1. Symbols are read from ELF objects as usual and from IR objects +//! through the plugin, and resolved together. +//! 2. Once the set of input files is known, the plugin is told how every +//! IR symbol was resolved and compiles all IR objects into a few ELF +//! objects. +//! 3. The IR objects are dropped, the compiled objects take their place, +//! and linking continues as usual. +//! +//! The plugin API (https://gcc.gnu.org/wiki/whopr/driver) is a set of C +//! callbacks: `onload` receives a table of linker-provided functions and +//! registers the plugin's hooks, and results are "returned" by calling +//! back into the linker. The plugin isn't thread-safe and keeps global +//! state, so this module does too. + +use std::ffi::{c_char, c_int, c_uint, c_void, CStr, CString}; +use std::fs::File; +use std::os::unix::io::AsRawFd; +use std::os::unix::process::CommandExt; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::{Mutex, OnceLock}; + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::args::VERSION; +use crate::context::Context; +use crate::diagnostics::Diagnostics; +use crate::elf::*; +use crate::input_files::{FileId, ObjId, ObjectFile}; +use crate::mapped_file::{must_open_file, MappedFile}; +use crate::symbol::SymbolId; +use crate::util::leak_bytes; +use crate::{fatal, out, warn}; + +// Status codes. +const LDPS_OK: c_int = 0; +const LDPS_NO_SYMS: c_int = 1; +const LDPS_BAD_HANDLE: c_int = 2; + +// Transfer vector tags. +const LDPT_NULL: c_int = 0; +const LDPT_LINKER_OUTPUT: c_int = 3; +const LDPT_OPTION: c_int = 4; +const LDPT_REGISTER_CLAIM_FILE_HOOK: c_int = 5; +const LDPT_REGISTER_ALL_SYMBOLS_READ_HOOK: c_int = 6; +const LDPT_REGISTER_CLEANUP_HOOK: c_int = 7; +const LDPT_ADD_SYMBOLS: c_int = 8; +const LDPT_GET_SYMBOLS: c_int = 9; +const LDPT_ADD_INPUT_FILE: c_int = 10; +const LDPT_MESSAGE: c_int = 11; +const LDPT_GET_INPUT_FILE: c_int = 12; +const LDPT_RELEASE_INPUT_FILE: c_int = 13; +const LDPT_ADD_INPUT_LIBRARY: c_int = 14; +const LDPT_OUTPUT_NAME: c_int = 15; +const LDPT_SET_EXTRA_LIBRARY_PATH: c_int = 16; +const LDPT_GET_VIEW: c_int = 18; +const LDPT_GET_INPUT_SECTION_COUNT: c_int = 19; +const LDPT_GET_INPUT_SECTION_TYPE: c_int = 20; +const LDPT_GET_INPUT_SECTION_NAME: c_int = 21; +const LDPT_GET_INPUT_SECTION_CONTENTS: c_int = 22; +const LDPT_UPDATE_SECTION_ORDER: c_int = 23; +const LDPT_ALLOW_SECTION_ORDERING: c_int = 24; +const LDPT_GET_SYMBOLS_V2: c_int = 25; +const LDPT_ALLOW_UNIQUE_SEGMENT_FOR_SECTIONS: c_int = 26; +const LDPT_UNIQUE_SEGMENT_FOR_SECTIONS: c_int = 27; +const LDPT_GET_SYMBOLS_V3: c_int = 28; +const LDPT_GET_INPUT_SECTION_ALIGNMENT: c_int = 29; +const LDPT_GET_INPUT_SECTION_SIZE: c_int = 30; +const LDPT_REGISTER_NEW_INPUT_HOOK: c_int = 31; +const LDPT_GET_WRAP_SYMBOLS: c_int = 32; +const LDPT_ADD_SYMBOLS_V2: c_int = 33; +const LDPT_GET_API_VERSION: c_int = 34; + +// Output file types. +const LDPO_EXEC: c_int = 1; +const LDPO_DYN: c_int = 2; +const LDPO_PIE: c_int = 3; + +// Symbol kinds. +const LDPK_DEF: u8 = 0; +const LDPK_WEAKDEF: u8 = 1; +const LDPK_WEAKUNDEF: u8 = 3; +const LDPK_COMMON: u8 = 4; + +// Symbol types. +const LDST_FUNCTION: u8 = 1; +const LDST_VARIABLE: u8 = 2; + +// Symbol visibilities. +const LDPV_PROTECTED: i32 = 1; +const LDPV_INTERNAL: i32 = 2; +const LDPV_HIDDEN: i32 = 3; + +// Symbol resolutions. +const LDPR_UNDEF: i32 = 1; +const LDPR_PREVAILING_DEF: i32 = 2; +const LDPR_PREVAILING_DEF_IRONLY: i32 = 3; +const LDPR_PREEMPTED_REG: i32 = 4; +const LDPR_PREEMPTED_IR: i32 = 5; +const LDPR_RESOLVED_IR: i32 = 6; +const LDPR_RESOLVED_EXEC: i32 = 7; +const LDPR_RESOLVED_DYN: i32 = 8; +const LDPR_PREVAILING_DEF_IRONLY_EXP: i32 = 9; + +// Message levels. +const LDPL_INFO: c_int = 0; +const LDPL_WARNING: c_int = 1; + +// Linker API versions. +const LAPI_V0: c_int = 0; +const LAPI_V1: c_int = 1; + +#[repr(C)] +union TagData { + val: c_int, + ptr: *const c_void, +} + +/// An entry of the transfer vector handed to `onload`. +#[repr(C)] +struct TagValue { + tag: c_int, + data: TagData, +} + +impl TagValue { + fn int(tag: c_int, val: c_int) -> TagValue { + TagValue { + tag, + data: TagData { val }, + } + } + + fn ptr(tag: c_int, ptr: *const c_void) -> TagValue { + TagValue { + tag, + data: TagData { ptr }, + } + } +} + +#[repr(C)] +struct PluginInputFile { + name: *const c_char, + fd: c_int, + offset: u64, + filesize: u64, + handle: *mut c_void, +} + +#[repr(C)] +struct PluginSection { + handle: *const c_void, + shndx: u32, +} + +#[cfg(target_endian = "little")] +#[repr(C)] +#[derive(Clone, Copy)] +struct SymbolKinds { + def: u8, + symbol_type: u8, + section_kind: u8, + padding: u8, +} + +#[cfg(target_endian = "big")] +#[repr(C)] +#[derive(Clone, Copy)] +struct SymbolKinds { + padding: u8, + section_kind: u8, + symbol_type: u8, + def: u8, +} + +#[repr(C)] +struct PluginSymbol { + name: *mut c_char, + version: *mut c_char, + kinds: SymbolKinds, + visibility: i32, + size: u64, + comdat_key: *mut c_char, + resolution: i32, +} + +type OnloadFn = unsafe extern "C" fn(*const TagValue) -> c_int; +type ClaimFileHandler = unsafe extern "C" fn(*const PluginInputFile, *mut c_int) -> c_int; +type Hook = unsafe extern "C" fn() -> c_int; +type NewInputHandler = unsafe extern "C" fn(*const PluginInputFile) -> c_int; + +extern "C" { + /// The printf-like diagnostics callback, defined in `csrc/lto-message.c`. + fn mold_lto_message(level: c_int, fmt: *const c_char, ...) -> c_int; +} + +/// The hooks the plugin registers during `onload`. +#[derive(Default)] +struct Hooks { + claim_file: Option, + all_symbols_read: Option, + cleanup: Option, + /// Whether the plugin speaks the GCC linker API v1, and hence + /// supports `get_symbols_v3`. + gcc_api_v1: bool, +} + +/// A symbol the plugin reported for the file being claimed. +struct ClaimedSymbol { + name: Vec, + comdat_key: Option>, + def: u8, + symbol_type: u8, + visibility: i32, + size: u64, +} + +impl ClaimedSymbol { + unsafe fn from_plugin(sym: &PluginSymbol) -> ClaimedSymbol { + let bytes = |p: *const c_char| CStr::from_ptr(p).to_bytes().to_vec(); + ClaimedSymbol { + name: bytes(sym.name), + comdat_key: (!sym.comdat_key.is_null()).then(|| bytes(sym.comdat_key)), + def: sym.kinds.def, + symbol_type: sym.kinds.symbol_type, + visibility: sym.visibility, + size: sym.size, + } + } + + /// An IR symbol as an ELF symbol; definitions are absolute, since an + /// IR object has no sections. + fn to_elf_sym(&self) -> ElfSym { + let mut esym = ElfSym { + st_size: self.size, + ..ElfSym::default() + }; + esym.st_shndx = match self.def { + LDPK_DEF | LDPK_WEAKDEF => SHN_ABS as u16, + LDPK_COMMON => SHN_COMMON as u16, + _ => SHN_UNDEF as u16, + }; + if matches!(self.def, LDPK_WEAKDEF | LDPK_WEAKUNDEF) { + esym.set_bind(STB_WEAK); + } + match self.symbol_type { + LDST_FUNCTION => esym.set_type(STT_FUNC), + LDST_VARIABLE => esym.set_type(STT_OBJECT), + _ => {} + } + match self.visibility { + LDPV_PROTECTED => esym.set_visibility(STV_PROTECTED), + LDPV_INTERNAL => esym.set_visibility(STV_INTERNAL), + LDPV_HIDDEN => esym.set_visibility(STV_HIDDEN), + _ => {} + } + esym + } +} + +static LOADED: OnceLock<()> = OnceLock::new(); +static HOOKS: Mutex = Mutex::new(Hooks { + claim_file: None, + all_symbols_read: None, + cleanup: None, + gcc_api_v1: false, +}); + +/// Claims are serialized: the plugin hands back the symbols of the file +/// being claimed through this buffer. +static CLAIM_LOCK: Mutex<()> = Mutex::new(()); +static CLAIMED_SYMBOLS: Mutex> = Mutex::new(Vec::new()); + +/// The linker's state, for callbacks. `DIAG` is set once the plugin is +/// loaded; `CONTEXT` only while the plugin is compiling, when nothing +/// else touches the context. +static DIAG: AtomicPtr = AtomicPtr::new(ptr::null_mut()); +static CONTEXT: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +fn diag() -> &'static Diagnostics { + // SAFETY: set in `load_plugin` to the diagnostics of the context, + // which outlives the plugin. + unsafe { &*DIAG.load(Ordering::Acquire) } +} + +/// Reports a message the plugin formatted. +/// +/// # Safety +/// +/// `msg` must point to a NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn mold_lto_report(level: c_int, msg: *const c_char) { + let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); + match level { + LDPL_INFO => out!(diag(), "{msg}"), + LDPL_WARNING => warn!(diag(), "{msg}"), + _ => fatal!(diag(), "{msg}"), + } +} + +unsafe extern "C" fn register_claim_file_hook(f: ClaimFileHandler) -> c_int { + HOOKS.lock().unwrap().claim_file = Some(f); + LDPS_OK +} + +unsafe extern "C" fn register_all_symbols_read_hook(f: Hook) -> c_int { + HOOKS.lock().unwrap().all_symbols_read = Some(f); + LDPS_OK +} + +unsafe extern "C" fn register_cleanup_hook(f: Hook) -> c_int { + HOOKS.lock().unwrap().cleanup = Some(f); + LDPS_OK +} + +unsafe extern "C" fn add_symbols( + _handle: *mut c_void, + nsyms: c_int, + psyms: *const PluginSymbol, +) -> c_int { + let syms = std::slice::from_raw_parts(psyms, nsyms as usize); + *CLAIMED_SYMBOLS.lock().unwrap() = syms.iter().map(|s| ClaimedSymbol::from_plugin(s)).collect(); + LDPS_OK +} + +/// Receives an object file the plugin compiled. +unsafe extern "C" fn add_input_file(path: *const c_char) -> c_int { + let ctx = &mut *(CONTEXT.load(Ordering::Acquire) as *mut Context); + let path = CStr::from_ptr(path).to_string_lossy().into_owned(); + let mf = must_open_file(&ctx.diag, "", &path); + mf.set_dependency(false); + + let mut file = ObjectFile::new::(&ctx.diag, mf, String::new()); + file.is_lto_output = true; + file.base.set_reachable(true); + file.base.priority = ctx.file_by_priority.len() as u32; + file.register_global_symbols::(&ctx.args, &mut ctx.symbol_bin()); + ctx.file_by_priority + .push(Some(FileId::Obj(ObjId(ctx.objs.len() as u32)))); + ctx.objs.push(Box::new(file)); + LDPS_OK +} + +unsafe extern "C" fn get_input_file(_handle: *const c_void, _file: *mut PluginInputFile) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn release_input_file(_handle: *const c_void) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn add_input_library(_path: *const c_char) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn set_extra_library_path(_path: *const c_char) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_view(handle: *const c_void, view: *mut *const c_void) -> c_int { + let mf = &*(handle as *const MappedFile); + *view = mf.data().as_ptr() as *const c_void; + LDPS_OK +} + +unsafe extern "C" fn get_input_section_count(_handle: *const c_void, _count: *mut c_int) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_input_section_type(_section: PluginSection, _ty: *mut c_int) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_input_section_name( + _section: PluginSection, + _name: *mut *mut c_char, +) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_input_section_contents( + _section: PluginSection, + _contents: *mut *const c_char, + _len: *mut usize, +) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn update_section_order(_sections: *const PluginSection, _num: c_int) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn allow_section_ordering() -> c_int { + LDPS_OK +} + +unsafe extern "C" fn allow_unique_segment_for_sections() -> c_int { + LDPS_OK +} + +unsafe extern "C" fn unique_segment_for_sections( + _name: *const c_char, + _flags: u64, + _align: u64, + _sections: *const PluginSection, + _num: c_int, +) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_input_section_alignment( + _section: PluginSection, + _align: *mut c_int, +) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_input_section_size(_section: PluginSection, _size: *mut u64) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn register_new_input_hook(_f: NewInputHandler) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_wrap_symbols(_num: *mut u64, _syms: *mut *const *const c_char) -> c_int { + LDPS_OK +} + +unsafe extern "C" fn get_symbols_v1( + _handle: *const c_void, + _nsyms: c_int, + _psyms: *mut PluginSymbol, +) -> c_int { + unreachable!("the v1 get_symbols API is never offered to the plugin") +} + +unsafe extern "C" fn get_symbols_v2( + handle: *const c_void, + nsyms: c_int, + psyms: *mut PluginSymbol, +) -> c_int { + get_symbols::(handle, nsyms, psyms, true) +} + +unsafe extern "C" fn get_symbols_v3( + handle: *const c_void, + nsyms: c_int, + psyms: *mut PluginSymbol, +) -> c_int { + get_symbols::(handle, nsyms, psyms, false) +} + +/// Tells the plugin how the symbols of an IR object were resolved. The +/// backend uses this to decide what it must keep: a definition that no +/// regular object refers to may be inlined away, for example. +unsafe fn get_symbols( + handle: *const c_void, + nsyms: c_int, + psyms: *mut PluginSymbol, + is_v2: bool, +) -> c_int { + let ctx = &*(CONTEXT.load(Ordering::Acquire) as *const Context); + let psyms = std::slice::from_raw_parts_mut(psyms, nsyms as usize); + let handle = handle as *const MappedFile; + let Some((fi, file)) = ctx + .objs + .iter() + .enumerate() + .find(|(_, f)| f.base.mf.is_some_and(|mf| ptr::eq(mf, handle))) + else { + return LDPS_BAD_HANDLE; + }; + + // An archive member that wasn't extracted contributes nothing. + if !file.base.is_reachable() { + for psym in psyms { + psym.resolution = LDPR_PREEMPTED_REG; + } + return LDPS_NO_SYMS; + } + + let this = FileId::Obj(ObjId(fi as u32)); + for (i, psym) in psyms.iter_mut().enumerate() { + let esym = &file.base.elf_syms.at(i + 1); + let sym = &ctx.symbols[file.base.symbols[i + 1]]; + psym.resolution = match sym.file() { + None => LDPR_UNDEF, + Some(owner) if owner == this => { + if sym.referenced_by_regular_obj() { + LDPR_PREVAILING_DEF + } else if sym.is_exported() { + if is_v2 { + LDPR_PREVAILING_DEF + } else { + LDPR_PREVAILING_DEF_IRONLY_EXP + } + } else { + LDPR_PREVAILING_DEF_IRONLY + } + } + Some(FileId::Dso(_)) => LDPR_RESOLVED_DYN, + Some(FileId::Obj(owner)) => { + let in_ir = ctx.objs[owner.index()].is_lto_input && !sym.is_wrapped(); + match (in_ir, esym.is_undef()) { + (true, true) => LDPR_RESOLVED_IR, + (true, false) => LDPR_PREEMPTED_IR, + (false, true) => LDPR_RESOLVED_EXEC, + (false, false) => LDPR_PREEMPTED_REG, + } + } + }; + } + LDPS_OK +} + +unsafe extern "C" fn get_api_version( + _plugin_identifier: *const c_char, + _plugin_version: c_uint, + minimal_api_supported: c_int, + maximal_api_supported: c_int, + linker_identifier: *mut *const c_char, + linker_version: *mut *const c_char, +) -> c_int { + if LAPI_V1 < minimal_api_supported { + fatal!(diag(), "LTO plugin does not support V0 or V1 API"); + } + static LINKER_VERSION: OnceLock = OnceLock::new(); + *linker_identifier = c"mold".as_ptr(); + *linker_version = LINKER_VERSION + .get_or_init(|| CString::new(VERSION).unwrap()) + .as_ptr(); + if LAPI_V1 <= maximal_api_supported { + HOOKS.lock().unwrap().gcc_api_v1 = true; + return LAPI_V1; + } + LAPI_V0 +} + +fn dlerror_string() -> String { + // SAFETY: dlerror returns a static string or null. + unsafe { + let err = libc::dlerror(); + if err.is_null() { + String::new() + } else { + CStr::from_ptr(err).to_string_lossy().into_owned() + } + } +} + +/// Loads the plugin and hands it the transfer vector, once. +fn load_plugin(ctx: &Context) { + LOADED.get_or_init(|| { + DIAG.store( + std::sync::Arc::as_ptr(&ctx.diag) as *mut Diagnostics, + Ordering::Release, + ); + + let path = CString::new(ctx.args.plugin.as_str()).unwrap(); + // SAFETY: plain dlopen/dlsym calls. + let onload: OnloadFn = unsafe { + let handle = libc::dlopen(path.as_ptr(), libc::RTLD_NOW | libc::RTLD_LOCAL); + if handle.is_null() { + fatal!(ctx, "could not open plugin file: {}", dlerror_string()); + } + let onload = libc::dlsym(handle, c"onload".as_ptr()); + if onload.is_null() { + fatal!( + ctx, + "failed to load plugin {}: {}", + ctx.args.plugin, + dlerror_string() + ); + } + std::mem::transmute::<*mut c_void, OnloadFn>(onload) + }; + + // Strings in the transfer vector must outlive the plugin. + let cstr = |s: &str| CString::new(s).unwrap().into_raw() as *const c_void; + let func = |f: usize| f as *const c_void; + let output = if ctx.args.shared { + LDPO_DYN + } else if ctx.args.pie { + LDPO_PIE + } else { + LDPO_EXEC + }; + + let mut tv = vec![ + TagValue::ptr(LDPT_MESSAGE, func(mold_lto_message as *const () as usize)), + TagValue::int(LDPT_LINKER_OUTPUT, output), + ]; + for opt in &ctx.args.plugin_opt { + tv.push(TagValue::ptr(LDPT_OPTION, cstr(opt))); + } + tv.extend([ + TagValue::ptr( + LDPT_REGISTER_CLAIM_FILE_HOOK, + func(register_claim_file_hook as *const () as usize), + ), + TagValue::ptr( + LDPT_REGISTER_ALL_SYMBOLS_READ_HOOK, + func(register_all_symbols_read_hook as *const () as usize), + ), + TagValue::ptr( + LDPT_REGISTER_CLEANUP_HOOK, + func(register_cleanup_hook as *const () as usize), + ), + TagValue::ptr(LDPT_ADD_SYMBOLS, func(add_symbols as *const () as usize)), + TagValue::ptr(LDPT_GET_SYMBOLS, func(get_symbols_v1 as *const () as usize)), + TagValue::ptr( + LDPT_ADD_INPUT_FILE, + func(add_input_file:: as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_FILE, + func(get_input_file as *const () as usize), + ), + TagValue::ptr( + LDPT_RELEASE_INPUT_FILE, + func(release_input_file as *const () as usize), + ), + TagValue::ptr( + LDPT_ADD_INPUT_LIBRARY, + func(add_input_library as *const () as usize), + ), + TagValue::ptr(LDPT_OUTPUT_NAME, cstr(&ctx.args.output)), + TagValue::ptr( + LDPT_SET_EXTRA_LIBRARY_PATH, + func(set_extra_library_path as *const () as usize), + ), + TagValue::ptr(LDPT_GET_VIEW, func(get_view as *const () as usize)), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_COUNT, + func(get_input_section_count as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_TYPE, + func(get_input_section_type as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_NAME, + func(get_input_section_name as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_CONTENTS, + func(get_input_section_contents as *const () as usize), + ), + TagValue::ptr( + LDPT_UPDATE_SECTION_ORDER, + func(update_section_order as *const () as usize), + ), + TagValue::ptr( + LDPT_ALLOW_SECTION_ORDERING, + func(allow_section_ordering as *const () as usize), + ), + TagValue::ptr(LDPT_ADD_SYMBOLS_V2, func(add_symbols as *const () as usize)), + TagValue::ptr( + LDPT_GET_SYMBOLS_V2, + func(get_symbols_v2:: as *const () as usize), + ), + TagValue::ptr( + LDPT_ALLOW_UNIQUE_SEGMENT_FOR_SECTIONS, + func(allow_unique_segment_for_sections as *const () as usize), + ), + TagValue::ptr( + LDPT_UNIQUE_SEGMENT_FOR_SECTIONS, + func(unique_segment_for_sections as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_SYMBOLS_V3, + func(get_symbols_v3:: as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_ALIGNMENT, + func(get_input_section_alignment as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_INPUT_SECTION_SIZE, + func(get_input_section_size as *const () as usize), + ), + TagValue::ptr( + LDPT_REGISTER_NEW_INPUT_HOOK, + func(register_new_input_hook as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_WRAP_SYMBOLS, + func(get_wrap_symbols as *const () as usize), + ), + TagValue::ptr( + LDPT_GET_API_VERSION, + func(get_api_version as *const () as usize), + ), + TagValue::int(LDPT_NULL, 0), + ]); + + // SAFETY: the transfer vector is terminated by LDPT_NULL. + let status = unsafe { onload(tv.as_ptr()) }; + if status != LDPS_OK { + fatal!(ctx, "LTO plugin's onload failed: {status}"); + } + }); +} + +/// Whether the plugin looks like LLVM's rather than GCC's. +fn is_llvm(ctx: &Context) -> bool { + ctx.args.plugin.contains("LLVMgold.") +} + +/// Whether the plugin supports `get_symbols_v3`: any LLVM, and GCC 12 or +/// newer. +fn supports_v3_api(ctx: &Context) -> bool { + HOOKS.lock().unwrap().gcc_api_v1 || is_llvm(ctx) +} + +/// Describes a file to the plugin, which reads it through a descriptor, +/// at an offset for an archive member. The descriptor stays open as long +/// as the returned file does. +fn plugin_input_file( + ctx: &Context, + mf: &'static MappedFile, +) -> (PluginInputFile, File) { + let container = mf.parent.unwrap_or(mf); + let file = File::open(&container.name) + .unwrap_or_else(|e| fatal!(ctx, "cannot open {}: {e}", container.name)); + let input = PluginInputFile { + name: CString::new(container.name.as_str()).unwrap().into_raw(), + fd: file.as_raw_fd(), + offset: mf.offset() as u64, + filesize: mf.size() as u64, + handle: mf as *const MappedFile as *mut c_void, + }; + (input, file) +} + +/// Reads the symbols of an IR object through the plugin. Returns `None` +/// for an archive member the plugin declines. +pub fn read_lto_object( + ctx: &Context, + mf: &'static MappedFile, + archive_name: String, +) -> Option { + if ctx.args.plugin.is_empty() { + fatal!( + ctx, + "{}: unable to handle this LTO object file because the -plugin option was not provided. \ + Please make sure you added -flto not only when creating object files but also when linking \ + the final executable.", + mf.name + ); + } + load_plugin(ctx); + let Some(claim_file) = HOOKS.lock().unwrap().claim_file else { + fatal!(ctx, "LTO plugin did not register a claim_file hook"); + }; + + // Input files are read in parallel, but claims are serialized: the + // plugin returns a file's symbols through a global buffer. + let _claim = CLAIM_LOCK.lock().unwrap(); + let (input, file) = plugin_input_file(ctx, mf); + let mut claimed: c_int = 0; + // SAFETY: `input` describes an open file. + unsafe { claim_file(&input, &mut claimed) }; + drop(file); + + if claimed == 0 { + if mf.parent.is_none() && mf.thin_parent.is_none() { + fatal!( + ctx, + "{}: not claimed by the LTO plugin; please make sure you are using the same compiler of the \ + same version for all object files", + mf.name + ); + } + return None; + } + + // A null symbol followed by the plugin's symbols, with their names in + // a string table of our own. + let symbols = std::mem::take(&mut *CLAIMED_SYMBOLS.lock().unwrap()); + let mut strtab = vec![0u8]; + let mut elf_syms = vec![ElfSym::default()]; + let mut comdat_keys = vec![None]; + for sym in &symbols { + let mut esym = sym.to_elf_sym(); + esym.st_name = strtab.len() as u32; + strtab.extend_from_slice(&sym.name); + strtab.push(0); + elf_syms.push(esym); + comdat_keys.push(sym.comdat_key.as_ref().map(|key| leak_bytes(key.clone()))); + } + Some(ObjectFile::lto_input::( + mf, + archive_name, + SymTable::from_records(RecordLayout::of::(), &elf_syms), + leak_bytes(strtab), + comdat_keys, + )) +} + +/// Restarts the linker with the archive members that turned out to be +/// unneeded excluded, for plugins without `get_symbols_v3`, which +/// otherwise provides no way to withdraw a file that was claimed. +fn restart_process(ctx: &Context) -> ! { + let mut args: Vec = ctx.cmdline_args.clone(); + for file in &ctx.objs { + if file.is_lto_input && !file.base.is_reachable() { + args.push(format!( + "--:ignore-ir-file={}", + file.base.mf.unwrap().identifier() + )); + } + } + args.push("--:lto-pass2".to_string()); + + let _ = std::io::Write::flush(&mut std::io::stdout()); + let _ = std::io::Write::flush(&mut std::io::stderr()); + let err = std::process::Command::new(crate::util::self_path()) + .args(&args[1..]) + .exec(); + eprintln!("mold: execv failed: {err}"); + std::process::exit(1); +} + +/// Has the plugin compile the IR objects. The resulting objects are added +/// to the context. +pub fn run_plugin(ctx: &mut Context) { + let _t = ctx.timer("run_lto_plugin"); + load_plugin(ctx); + + if !ctx.args.lto_pass2 && !supports_v3_api(ctx) { + restart_process(ctx); + } + + // Definitions in IR objects that regular objects refer to must survive + // the optimization. + let referenced: Vec = { + let ctx: &Context = ctx; + ctx.objs + .par_iter() + .filter(|file| !file.is_lto_input) + .flat_map_iter(|file| { + file.base.global_symbols().iter().copied().filter(|&id| { + matches!(ctx.symbols[id].file(), Some(FileId::Obj(owner)) if ctx.objs[owner.index()].is_lto_input) + }) + }) + .collect() + }; + for id in referenced { + ctx.symbols[id].set_referenced_by_regular_obj(true); + } + + // Wrapped symbols are referred to by name from regular objects, and + // -u symbols must be kept. + let mut names: Vec = Vec::new(); + for name in &ctx.args.wrap { + names.extend([ + name.clone(), + format!("__wrap_{name}"), + format!("__real_{name}"), + ]); + } + names.extend(ctx.args.undefined.iter().cloned()); + for name in names { + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].set_referenced_by_regular_obj(true); + } + + // Objects with .gnu.offload_lto_.* sections carry code for + // accelerators, which the backend needs as well. + let claim_file = HOOKS + .lock() + .unwrap() + .claim_file + .expect("the plugin registered a claim_file hook"); + for fi in 0..ctx.objs.len() { + let file = &ctx.objs[fi]; + if file.base.is_reachable() && !file.is_lto_input && file.is_gcc_offload_obj { + let (input, _file) = plugin_input_file(ctx, file.base.mf.unwrap()); + let mut claimed: c_int = 0; + // SAFETY: `input` describes an open file. + unsafe { claim_file(&input, &mut claimed) }; + } + } + + // The hook asks for symbol resolutions, compiles, and returns the + // objects through add_input_file. + let all_symbols_read = HOOKS + .lock() + .unwrap() + .all_symbols_read + .expect("the plugin registered an all_symbols_read hook"); + CONTEXT.store(ctx as *mut Context as *mut c_void, Ordering::Release); + // SAFETY: the callbacks are the only users of the context until the + // hook returns. + let status = unsafe { all_symbols_read() }; + CONTEXT.store(ptr::null_mut(), Ordering::Release); + if status != LDPS_OK { + fatal!(ctx, "LTO: all_symbols_read_hook returns {status}"); + } +} + +/// Lets the plugin remove its temporary files. +pub fn cleanup() { + let cleanup = HOOKS.lock().unwrap().cleanup; + if let Some(cleanup) = cleanup { + // SAFETY: a hook registered by the plugin. + unsafe { cleanup() }; + } +} diff --git a/src/mapfile.rs b/src/mapfile.rs new file mode 100644 index 000000000..d87569f24 --- /dev/null +++ b/src/mapfile.rs @@ -0,0 +1,98 @@ +//! `--print-map` / `-Map`: a listing of output sections, their input +//! sections and symbols. + +use std::collections::HashMap; +use std::io::Write; + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::ChunkId; +use crate::context::Context; +use crate::elf::*; +use crate::fatal; +use crate::input_files::FileId; +use crate::input_sections::SectionRef; +use crate::symbol::SymbolId; + +/// Maps each input section to the symbols defined in it, sorted by value. +fn section_symbols(ctx: &Context) -> HashMap> { + let mut map: HashMap> = HashMap::new(); + for (fi, file) in ctx.objs.iter().enumerate() { + let file_id = FileId::Obj(crate::input_files::ObjId(fi as u32)); + for &id in &file.base.symbols { + let sym = &ctx.symbols[id]; + if sym.file() == Some(file_id) && sym.ty() != STT_SECTION { + if let Some(r) = sym.input_section() { + map.entry(r).or_default().push(id); + } + } + } + } + for syms in map.values_mut() { + syms.sort_by_key(|&id| ctx.symbols[id].value); + } + map +} + +pub fn print_map(ctx: &Context) { + let _t = ctx.timer("print_map"); + let map = section_symbols(ctx); + + let mut out = String::from(" VMA Size Align Out In Symbol\n"); + for &id in &ctx.chunks { + let hdr = ctx.chunk_header(id); + out.push_str(&format!( + "{:>18}{:>11}{:>6} {}\n", + format!("{:#x}", hdr.shdr.sh_addr), + hdr.shdr.sh_size, + hdr.shdr.sh_addralign, + hdr.name + )); + + let ChunkId::Output(osec_id) = id else { + continue; + }; + let osec = &ctx.output_sections[osec_id.index()]; + let lines: Vec = osec + .members + .par_iter() + .map(|&member| { + let isec = ctx.input_section(member); + let r = isec.section_ref(); + let addr = if osec.hdr.is_alloc() { + osec.hdr.shdr.sh_addr + isec.offset() + } else { + 0 + }; + let mut s = format!( + "{:>18}{:>11}{:>6} {}\n", + format!("{addr:#x}"), + isec.sh_size, + 1u64 << isec.p2align(), + isec.display(&ctx.objs[isec.file.index()]) + ); + if let Some(syms) = map.get(&r) { + for &id in syms { + let sym = &ctx.symbols[id]; + s.push_str(&format!( + "{:>18} 0 0 {sym}\n", + format!("{:#x}", sym.addr(ctx)) + )); + } + } + s + }) + .collect(); + for line in lines { + out.push_str(&line); + } + } + + if !ctx.args.map.is_empty() && ctx.args.map != "-" { + std::fs::write(&ctx.args.map, out) + .unwrap_or_else(|e| fatal!(ctx, "--print-map: cannot open {}: {e}", ctx.args.map)); + } else { + let _ = std::io::stdout().write_all(out.as_bytes()); + } +} diff --git a/src/mapped_file.rs b/src/mapped_file.rs new file mode 100644 index 000000000..5a6e8ce1b --- /dev/null +++ b/src/mapped_file.rs @@ -0,0 +1,269 @@ +//! Input file access. +//! +//! Every input file is read once and kept in memory for the whole link, +//! because sections, symbol names and relocation tables of object files +//! are used until the output is written. Files are therefore leaked with a +//! `'static` lifetime rather than tracked with reference counts, which +//! keeps lifetimes out of every data structure that refers to file +//! contents. + +use std::fs::File; +use std::io::Read; +use std::ops::Range; +use std::path::Path; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; + +use rayon::prelude::*; + +use crate::diagnostics::{errno_string, Diagnostics}; +use crate::fatal; +use crate::util; + +/// The files that are memory-mapped, for [`drop_mappings`]. +static MAPPED: Mutex> = Mutex::new(Vec::new()); + +/// Drops the page table entries of the mapped input files, in parallel. +/// This makes process exit faster, as the kernel otherwise reclaims them +/// in a single thread on exit. File contents stay in the page cache. +pub fn drop_mappings() { + let files = std::mem::take(&mut *MAPPED.lock().unwrap()); + files.par_iter().for_each(|mf| { + // SAFETY: the range is a whole mapping of the file that lives for + // the rest of the process; MADV_DONTNEED only discards the pages, + // which are faulted in again if they are ever read. + unsafe { + libc::madvise( + mf.data().as_ptr() as *mut libc::c_void, + mf.data().len(), + libc::MADV_DONTNEED, + ) + }; + }); +} + +/// Files up to this size are read into memory rather than mmap'ed. mmap(2) +/// takes the process's address space lock, so with tens of thousands of +/// input files the calls serialize; read(2) doesn't take that lock but +/// copies the data, which only pays off for small files. +const READ_THRESHOLD: u64 = 32 * 1024; + +/// A byte range whose allocation is deliberately leaked with the input file. +/// Keeping the pointer rather than a permanent shared slice lets the linker +/// modify private relocation records in place, as C++ mold does. +#[derive(Clone, Copy, Debug)] +pub(crate) struct MappedBytes { + ptr: NonNull, + len: usize, +} + +impl MappedBytes { + fn empty() -> MappedBytes { + MappedBytes { + ptr: NonNull::dangling(), + len: 0, + } + } + + fn from_mut(data: &mut [u8]) -> MappedBytes { + MappedBytes { + ptr: NonNull::new(data.as_mut_ptr()).unwrap_or_else(NonNull::dangling), + len: data.len(), + } + } + + fn slice(self, start: usize, size: usize) -> MappedBytes { + assert!(start <= self.len && size <= self.len - start); + MappedBytes { + // SAFETY: the checked range lies in the same allocation. + ptr: unsafe { NonNull::new_unchecked(self.ptr.as_ptr().add(start)) }, + len: size, + } + } +} + +/// An input file, or a slice of one for an archive member. +#[derive(Debug)] +pub struct MappedFile { + pub name: String, + pub(crate) data: MappedBytes, + + /// False if the file was found by searching library paths (`-l`), + /// which affects how a shared library's soname defaults. + pub given_fullpath: bool, + + /// The archive this file is a member of. + pub parent: Option<&'static MappedFile>, + + /// The thin archive this file is a member of. + pub thin_parent: Option<&'static MappedFile>, + + /// Whether the file should be listed in a `--dependency-file`. + pub is_dependency: AtomicBool, +} + +// The bytes are normally read concurrently. Mutable access is restricted to +// disjoint relocation ranges owned by file-parallel passes. +unsafe impl Send for MappedFile {} +unsafe impl Sync for MappedFile {} + +impl MappedFile { + /// Opens a file, returning `None` if it doesn't exist. + pub fn open(diag: &Diagnostics, path: &str) -> Option<&'static MappedFile> { + let file = match File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, + Err(e) => fatal!(diag, "opening {path} failed: {e}"), + }; + + let metadata = file + .metadata() + .unwrap_or_else(|e| fatal!(diag, "{path}: fstat failed: {e}")); + let size = metadata.len(); + + let mut is_mmapped = false; + let data = if size == 0 { + MappedBytes::empty() + } else if size <= READ_THRESHOLD && metadata.is_file() { + let mut buf = Vec::with_capacity(size as usize); + (&file) + .take(size) + .read_to_end(&mut buf) + .unwrap_or_else(|e| fatal!(diag, "{path}: read failed: {e}")); + if buf.len() as u64 != size { + fatal!(diag, "{path}: file is shorter than its reported size"); + } + MappedBytes::from_mut(Vec::leak(buf)) + } else { + // C++ mold maps inputs MAP_PRIVATE with write permission so it + // can redirect ordinary relocation records in place without + // changing the input file. + // + // SAFETY: the mapping is private and lives for the rest of the + // process. Input files are not expected to change while the + // linker runs. + let map_len = usize::try_from(size) + .unwrap_or_else(|_| fatal!(diag, "{path}: file is too large to map")); + let map = unsafe { memmap2::MmapOptions::new().len(map_len).map_copy(&file) } + .unwrap_or_else(|e| fatal!(diag, "{path}: mmap failed: {e}")); + is_mmapped = true; + MappedBytes::from_mut(Box::leak(Box::new(map)).as_mut()) + }; + + let mf = util::leak(MappedFile { + name: path.to_string(), + data, + given_fullpath: true, + parent: None, + thin_parent: None, + is_dependency: AtomicBool::new(true), + }); + if is_mmapped { + MAPPED.lock().unwrap().push(mf); + } + Some(mf) + } + + /// Opens a file that must exist. + pub fn must_open(diag: &Diagnostics, path: &str) -> &'static MappedFile { + MappedFile::open(diag, path) + .unwrap_or_else(|| fatal!(diag, "cannot open {path}: {}", errno_string())) + } + + /// Returns a view of a member of this archive. + pub fn slice(&'static self, name: String, start: usize, size: usize) -> &'static MappedFile { + util::leak(MappedFile { + name, + data: self.data.slice(start, size), + given_fullpath: true, + parent: Some(self), + thin_parent: None, + is_dependency: AtomicBool::new(true), + }) + } + + pub fn size(&self) -> usize { + self.data.len + } + + #[inline] + pub fn data(&self) -> &[u8] { + // SAFETY: the allocation is leaked and the range was checked when + // this file or archive member was created. Mutable accesses are to + // relocation ranges during exclusive linker phases. + unsafe { std::slice::from_raw_parts(self.data.ptr.as_ptr(), self.data.len) } + } + + /// Returns a mutable raw pointer to a relocation range. + /// + /// # Safety + /// + /// The caller must have exclusive access to `range` while dereferencing + /// the pointer, and no shared reference to an overlapping range may be + /// used during that time. + pub(crate) unsafe fn data_mut_ptr(&self, range: Range) -> *mut [u8] { + assert!( + range.start <= range.end && range.start <= self.data.len && range.end <= self.data.len + ); + // SAFETY: the range is in bounds. Dereferencing remains the caller's + // responsibility under the contract above. + std::ptr::slice_from_raw_parts_mut( + unsafe { self.data.ptr.as_ptr().add(range.start) }, + range.end - range.start, + ) + } + + /// The offset of this slice within its top-level archive file. + pub fn offset(&self) -> usize { + match self.parent { + Some(parent) => { + let base = parent.data.ptr.as_ptr() as usize; + self.data.ptr.as_ptr() as usize - base + parent.offset() + } + None => 0, + } + } + + /// Returns a string that uniquely identifies the file, including files + /// inside archives. + pub fn identifier(&self) -> String { + if let Some(parent) = self.parent { + // Archive members may share a name, so use the offset instead. + return format!("{}:{}", parent.name, self.offset()); + } + if let Some(thin_parent) = self.thin_parent { + // A thin archive member's file name is unique within the archive. + return format!("{}:{}", thin_parent.name, self.name); + } + self.name.clone() + } + + pub fn is_dependency(&self) -> bool { + self.is_dependency.load(Ordering::Relaxed) + } + + pub fn set_dependency(&self, value: bool) { + self.is_dependency.store(value, Ordering::Relaxed); + } +} + +/// Opens an input file, applying `--chroot` to absolute paths. +pub fn open_file(diag: &Diagnostics, chroot: &str, path: &str) -> Option<&'static MappedFile> { + if path.starts_with('/') && !chroot.is_empty() { + let path = format!("{chroot}/{}", util::path_clean(path)); + return MappedFile::open(diag, &path); + } + MappedFile::open(diag, path) +} + +/// Opens an input file that must exist, applying `--chroot` to absolute paths. +pub fn must_open_file(diag: &Diagnostics, chroot: &str, path: &str) -> &'static MappedFile { + open_file(diag, chroot, path) + .unwrap_or_else(|| fatal!(diag, "cannot open {path}: {}", errno_string())) +} + +/// Whether a path refers to something that is not a directory. +pub fn is_file(path: &str) -> bool { + Path::new(path).is_file() +} diff --git a/src/output_file.rs b/src/output_file.rs new file mode 100644 index 000000000..cae7fdc5f --- /dev/null +++ b/src/output_file.rs @@ -0,0 +1,392 @@ +//! The output file and helpers for writing to it from many threads. + +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use memmap2::{MmapMut, MmapOptions}; + +use crate::diagnostics::Diagnostics; +use crate::fatal; + +/// The temporary file being written, removed on a fatal error. +static TMPFILE: Mutex> = Mutex::new(None); + +/// Removes a partially written output file. +pub fn cleanup() { + if let Ok(mut guard) = TMPFILE.lock() { + if let Some(path) = guard.take() { + let _ = std::fs::remove_file(path); + } + } +} + +enum Storage { + /// A mapping of the file, possibly larger than the file itself; `len` + /// is the file's size. + Mmap { + map: MmapMut, + len: usize, + }, + Memory(Vec), +} + +/// An output file, either memory-mapped or buffered in memory. +pub struct OutputFile { + path: String, + tmp_path: Option, + /// The file being written, if it is a real file. + file: Option, + storage: Storage, + perm: u32, +} + +/// Calling fallocate speeds up later linking passes on ext4 by taking disk +/// block allocation out of the page-fault handler. On tmpfs, it instead +/// makes things much slower: the kernel allocates and zeroes every page of +/// the range inside the syscall, on one thread, which takes ~1.8 s for a +/// 5 GiB file. +fn preallocate(file: &File, size: u64) { + // SAFETY: fstatfs and fallocate only inspect and act on a valid open + // descriptor; the statfs buffer is fully written before it is read. + unsafe { + let mut fs: libc::statfs = std::mem::zeroed(); + if libc::fstatfs(file.as_raw_fd(), &mut fs) == 0 && fs.f_type != libc::TMPFS_MAGIC { + libc::fallocate(file.as_raw_fd(), 0, 0, size as libc::off_t); + } + } +} + +fn map_file(file: &File, size: u64) -> Storage { + if size == 0 { + return Storage::Memory(Vec::new()); + } + // We map the file with twice as much address space as its size, so + // that extend() can grow the file into the mapping in place. Touching + // the mapping beyond the end of the file is not allowed, but growing + // the file with ftruncate makes the tail of the mapping accessible + // without any further mmap call. + // + // SAFETY: the file is private to this process until it's closed. + let map = unsafe { MmapOptions::new().len(size as usize * 2).map_mut(file) } + // If the address space is too tight, map just the file. + .or_else(|_| unsafe { MmapMut::map_mut(file) }); + match map { + Ok(mut map) => { + // Without this, every 4 KiB page of the output takes its own + // page fault when it is first written, and the faults of the + // many copying threads serialize on the file's page cache. With + // it, the kernel backs the mapping with large folios and the + // number of faults drops by an order of magnitude. + // SAFETY: the range is the mapping; the advice is only a hint. + unsafe { + libc::madvise( + map.as_mut_ptr() as *mut libc::c_void, + map.len(), + libc::MADV_HUGEPAGE, + ) + }; + Storage::Mmap { + map, + len: size as usize, + } + } + Err(_) => Storage::Memory(vec![0; size as usize]), + } +} + +impl OutputFile { + /// Opens an output file of the given size. + /// + /// A regular file is written through a memory mapping of a temporary + /// file that is renamed into place on close, so that a failed link + /// doesn't leave a truncated output behind and a running executable + /// isn't modified underneath the kernel. Anything else — a device, a + /// pipe, or standard output — is assembled in memory and written out + /// at the end. + pub fn open( + diag: &Diagnostics, + path: &str, + size: u64, + perm: u32, + overwrite_in_place: bool, + ) -> OutputFile { + let is_special = path == "-" || std::fs::metadata(path).is_ok_and(|m| !m.is_file()); + if is_special { + return OutputFile { + path: path.to_string(), + tmp_path: None, + file: None, + storage: Storage::Memory(vec![0; size as usize]), + perm, + }; + } + + let dir = Path::new(path) + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let name = Path::new(path) + .file_name() + .map_or(String::new(), |n| n.to_string_lossy().into_owned()); + let tmp = dir.join(format!(".{name}.{}", std::process::id())); + + // On Linux, writing to an existing file is much faster than + // creating a fresh one, so when it's safe to do so the previous + // output is renamed to the temporary name and reused. Every byte + // of it is overwritten before the rename back. + let reuse_existing = || -> Option { + if !overwrite_in_place || std::fs::rename(path, &tmp).is_err() { + return None; + } + match OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(perm) + .open(&tmp) + { + Ok(file) => Some(file), + Err(_) => { + let _ = std::fs::remove_file(&tmp); + None + } + } + }; + let file = reuse_existing().unwrap_or_else(|| { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .mode(perm) + .open(&tmp) + .unwrap_or_else(|e| fatal!(diag, "cannot open {}: {e}", tmp.display())) + }); + *TMPFILE.lock().unwrap() = Some(tmp.clone()); + + file.set_permissions(std::fs::Permissions::from_mode(perm & !umask())) + .unwrap_or_else(|e| fatal!(diag, "{}: fchmod failed: {e}", tmp.display())); + file.set_len(size) + .unwrap_or_else(|e| fatal!(diag, "{}: ftruncate failed: {e}", tmp.display())); + preallocate(&file, size); + + let storage = map_file(&file, size); + OutputFile { + path: path.to_string(), + tmp_path: Some(tmp), + file: Some(file), + storage, + perm, + } + } + + /// Opens a file that is written in place under an exclusive lock, for + /// a separate debug file that a debugger may wait on. The file is + /// made unusable right away so that a stale one isn't picked up by + /// accident; [`Self::resize`] gives it its size. + pub fn open_locked(diag: &Diagnostics, path: &str, perm: u32) -> OutputFile { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(perm) + .open(path) + .unwrap_or_else(|e| fatal!(diag, "cannot open {path}: {e}")); + // SAFETY: flock on a valid descriptor. + unsafe { + libc::flock(file.as_raw_fd(), libc::LOCK_EX); + } + file.write_all(&[0; 256]) + .unwrap_or_else(|e| fatal!(diag, "{path}: write failed: {e}")); + OutputFile { + path: path.to_string(), + tmp_path: None, + file: Some(file), + storage: Storage::Memory(Vec::new()), + perm, + } + } + + /// Sets the size of a file opened with [`Self::open_locked`]. + pub fn resize(&mut self, diag: &Diagnostics, size: u64) { + let file = self + .file + .as_ref() + .expect("resizing an output file that isn't a file"); + file.set_len(size) + .unwrap_or_else(|e| fatal!(diag, "{}: ftruncate failed: {e}", self.path)); + self.storage = map_file(file, size); + } + + /// Whether the buffer is a mapping of the file rather than memory. + pub fn is_mmapped(&self) -> bool { + matches!(self.storage, Storage::Mmap { .. }) + } + + pub fn buf(&mut self) -> &mut [u8] { + match &mut self.storage { + Storage::Mmap { map, len } => &mut map[..*len], + Storage::Memory(vec) => &mut vec[..], + } + } + + pub fn len(&self) -> usize { + match &self.storage { + Storage::Mmap { len, .. } => *len, + Storage::Memory(vec) => vec.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Extends the file so that the caller can fill the appended data + /// through the tail of the mapping, for sections whose size is known + /// only after everything else has been written. This is called at + /// most once per output file. + pub fn extend(&mut self, diag: &Diagnostics, size: usize) { + let new_len = self.len() + size; + match (&mut self.storage, &self.file) { + (Storage::Memory(vec), _) => vec.resize(new_len, 0), + (Storage::Mmap { map, len }, Some(file)) => { + file.set_len(new_len as u64) + .unwrap_or_else(|e| fatal!(diag, "{}: ftruncate failed: {e}", self.path)); + preallocate(file, new_len as u64); + if new_len <= map.len() { + *len = new_len; + } else { + // The appended data does not fit in the mapping. Map the + // grown file again, moving the buffer. + self.storage = map_file(file, new_len as u64); + } + } + (Storage::Mmap { .. }, None) => unreachable!("a mapping always has a file"), + } + } + + /// Finishes writing and moves the file into place. The mapping is + /// released without waiting for the data to reach the disk. + pub fn close(self, diag: &Diagnostics) { + match self.storage { + Storage::Mmap { map, .. } => drop(map), + Storage::Memory(vec) => { + if self.path == "-" { + let mut stdout = std::io::stdout().lock(); + stdout + .write_all(&vec) + .and_then(|()| stdout.flush()) + .unwrap_or_else(|e| fatal!(diag, "write failed: {e}")); + // Close the descriptor too: the parent process may + // already have exited, and a program that then runs + // the output would fail with ETXTBSY while this + // process still holds it open for writing. + // SAFETY: nothing else writes to stdout after this. + unsafe { + libc::close(libc::STDOUT_FILENO); + } + return; + } + let mut file = match self.file { + Some(file) => file, + None => OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(self.perm) + .open(&self.path) + .unwrap_or_else(|e| fatal!(diag, "cannot open {}: {e}", self.path)), + }; + file.write_all(&vec) + .unwrap_or_else(|e| fatal!(diag, "{}: write failed: {e}", self.path)); + } + } + if let Some(tmp) = self.tmp_path { + // If an output file already exists, open a file and then remove + // it. This is the fastest way to unlink a file, as it does not + // make the system immediately release disk blocks occupied by + // the file. The descriptor is kept until the process exits. + if let Ok(old) = File::open(&self.path) { + let _ = std::fs::remove_file(&self.path); + std::mem::forget(old); + } + std::fs::rename(&tmp, &self.path).unwrap_or_else(|e| { + fatal!( + diag, + "cannot rename {} to {}: {e}", + tmp.display(), + self.path + ) + }); + *TMPFILE.lock().unwrap() = None; + } + } +} + +fn umask() -> u32 { + // SAFETY: umask is thread-safe in the sense that it just swaps a + // process-wide value; restoring it immediately keeps it unchanged. + unsafe { + let mask = libc::umask(0); + libc::umask(mask); + mask as u32 + } +} + +/// Splits a buffer into consecutive slices at the given ascending +/// offsets. The first slice starts at `offsets[0]`; bytes before it are +/// not returned. The last slice extends to the end of the buffer. +pub fn split_at_offsets<'a, T>(buf: &'a mut [T], offsets: &[u64]) -> Vec<&'a mut [T]> { + let mut slices = Vec::with_capacity(offsets.len()); + let mut rest = buf; + let mut pos = 0usize; + for (i, &start) in offsets.iter().enumerate() { + let start = start as usize; + let end = offsets.get(i + 1).map_or(rest.len() + pos, |&e| e as usize); + debug_assert!(pos <= start && start <= end); + let (_, tail) = std::mem::take(&mut rest).split_at_mut(start - pos); + let (slice, tail) = tail.split_at_mut(end - start); + slices.push(slice); + rest = tail; + pos = end; + } + slices +} + +/// A byte range of the output file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Range { + pub offset: u64, + pub size: u64, +} + +/// Borrows several disjoint ranges of a buffer mutably at once. +pub fn split_ranges<'a>(buf: &'a mut [u8], ranges: &[Range]) -> Vec<&'a mut [u8]> { + let mut order: Vec = (0..ranges.len()).collect(); + order.sort_by_key(|&i| ranges[i].offset); + + let mut result: Vec> = (0..ranges.len()).map(|_| None).collect(); + let mut rest = buf; + let mut pos = 0u64; + for i in order { + let r = ranges[i]; + assert!(r.offset >= pos, "overlapping output ranges"); + let (_, tail) = std::mem::take(&mut rest).split_at_mut((r.offset - pos) as usize); + let (slice, tail) = tail.split_at_mut(r.size as usize); + result[i] = Some(slice); + rest = tail; + pos = r.offset + r.size; + } + result.into_iter().map(|s| s.unwrap()).collect() +} + +/// Creates a file with the given contents. +pub fn write_file(diag: &Diagnostics, path: &str, contents: &[u8]) { + let mut file = File::create(path).unwrap_or_else(|e| fatal!(diag, "cannot open {path}: {e}")); + file.write_all(contents) + .unwrap_or_else(|e| fatal!(diag, "{path}: write failed: {e}")); +} diff --git a/src/passes.rs b/src/passes.rs new file mode 100644 index 000000000..59d7f5ece --- /dev/null +++ b/src/passes.rs @@ -0,0 +1,4422 @@ +//! The passes of a link, in roughly the order the driver runs them. + +use std::cell::UnsafeCell; +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use bstr::BStr; +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::args::{ + BsymbolicKind, BuildIdKind, CetReportKind, DefsymValue, SectionOrderKind, SeparateCodeKind, + ShuffleSectionsKind, UnresolvedKind, +}; +use crate::chunks::dynamic::{DynamicSection, RelrDynSection}; +use crate::chunks::eh_frame::{EhFrameHdrSection, EhFrameRelocSection}; +use crate::chunks::misc::{ + self, BuildIdSection, GnuDebuglinkSection, InterpSection, NotePropertySection, + RelroPaddingSection, +}; +use crate::chunks::output_section::OutputSection; +use crate::chunks::symtab::{ + self, GnuHashSection, HashSection, ShstrtabSection, SymtabShndxSection, +}; +use crate::chunks::version::VerdefSection; +use crate::chunks::{ + self, ChunkHeader, ChunkId, GdbIndexSection, OutputEhdr, OutputPhdr, OutputSectionId, + OutputShdr, +}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::{ + resolved_symbol_rank, symbol_resolution_rank, ComdatGroupRef, FileId, ObjId, ObjectFile, + SymbolEditor, SymbolResolver, +}; +use crate::input_sections::{InputSectionId, SectionRef}; +use crate::linker_script::VersionPattern; +use crate::mapped_file::MappedFile; +use crate::output_file::OutputFile; +use crate::symbol::{ + is_c_identifier, Bins, Symbol, SymbolId, NEEDS_CANONICAL, NEEDS_GOT, NEEDS_GOTTP, NEEDS_PLT, + NEEDS_PPC_OPD, NEEDS_TLSDESC, NEEDS_TLSGD, +}; +use crate::util::glob::Glob; +use crate::util::{align_down, align_to, leak_bytes, path_filename}; +use crate::{error, fatal, out, warn}; + +pub fn apply_exclude_libs(ctx: &mut Context) { + let _t = ctx.timer("apply_exclude_libs"); + let set = &ctx.args.exclude_libs; + if set.is_empty() { + return; + } + for file in &mut ctx.objs { + if !file.archive_name.is_empty() + && (set.contains(&path_filename(&file.archive_name)) || set.contains("ALL")) + { + file.exclude_libs = true; + } + } +} + +fn has_debug_info_section(ctx: &Context) -> bool { + ctx.objs.iter().any(|f| !f.debug_info_sections.is_empty()) +} + +pub fn create_synthetic_sections(ctx: &mut Context) { + let mut chunks = Vec::new(); + + if !ctx.args.oformat_binary { + let find = |name: &str| { + ctx.args + .section_order + .iter() + .any(|o| o.kind == SectionOrderKind::Section && o.name == name) + }; + let ehdr_flags = if ctx.args.section_order.is_empty() || find("EHDR") { + SHF_ALLOC as u64 + } else { + 0 + }; + let phdr_flags = if ctx.args.section_order.is_empty() || find("PHDR") { + SHF_ALLOC as u64 + } else { + 0 + }; + ctx.ehdr = Some(OutputEhdr::new::(ehdr_flags)); + chunks.push(ChunkId::Ehdr); + ctx.phdr = Some(OutputPhdr::new::(phdr_flags)); + chunks.push(ChunkId::Phdr); + if ctx.args.z_sectionheader { + ctx.shdr = Some(OutputShdr::new::()); + chunks.push(ChunkId::Shdr); + } + } + + chunks.push(ChunkId::Got); + if !E::IS_SPARC { + chunks.push(ChunkId::GotPlt); + } + chunks.push(ChunkId::RelDyn); + chunks.push(ChunkId::RelPlt); + if ctx.args.pack_dyn_relocs_relr { + ctx.relrdyn = Some(RelrDynSection::new::(&ctx.args)); + chunks.push(ChunkId::RelrDyn); + } + chunks.push(ChunkId::Strtab); + chunks.push(ChunkId::Plt); + chunks.push(ChunkId::PltGot); + chunks.push(ChunkId::Symtab); + chunks.push(ChunkId::Dynsym); + chunks.push(ChunkId::Dynstr); + chunks.push(ChunkId::EhFrame); + chunks.push(ChunkId::SFrame); + chunks.push(ChunkId::Copyrel); + chunks.push(ChunkId::CopyrelRelro); + + if ctx.shdr.is_some() { + ctx.shstrtab = Some(ShstrtabSection::new()); + chunks.push(ChunkId::Shstrtab); + } + if !ctx.args.dynamic_linker.is_empty() { + ctx.interp = Some(InterpSection::new()); + chunks.push(ChunkId::Interp); + } + if ctx.args.build_id.kind != BuildIdKind::None { + ctx.buildid = Some(BuildIdSection::new()); + chunks.push(ChunkId::BuildId); + } + if ctx.args.eh_frame_hdr { + ctx.eh_frame_hdr = Some(EhFrameHdrSection::new()); + chunks.push(ChunkId::EhFrameHdr); + } + if ctx.args.gdb_index && has_debug_info_section(ctx) { + ctx.gdb_index = Some(GdbIndexSection::new()); + chunks.push(ChunkId::GdbIndex); + } + if ctx.args.z_relro && ctx.args.section_order.is_empty() { + ctx.relro_padding = Some(RelroPaddingSection::new()); + chunks.push(ChunkId::RelroPadding); + } + if ctx.args.hash_style_sysv { + ctx.hash = Some(HashSection::new::()); + chunks.push(ChunkId::Hash); + } + if ctx.args.hash_style_gnu { + ctx.gnu_hash = Some(GnuHashSection::new::()); + chunks.push(ChunkId::GnuHash); + } + if !ctx.args.version_definitions.is_empty() { + ctx.verdef = Some(VerdefSection::new()); + chunks.push(ChunkId::Verdef); + } + if ctx.args.emit_relocs { + ctx.eh_frame_reloc = Some(EhFrameRelocSection::new::()); + chunks.push(ChunkId::EhFrameReloc); + } + if !ctx.args.separate_debug_file.is_empty() { + ctx.gnu_debuglink = Some(GnuDebuglinkSection::new()); + chunks.push(ChunkId::GnuDebuglink); + } + + if ctx.args.shared || !ctx.dsos.is_empty() || ctx.args.pie { + ctx.dynamic = Some(DynamicSection::new::(&ctx.args)); + chunks.push(ChunkId::Dynamic); + // .dynamic refers to .dynsym and .dynstr, so they must exist. + ctx.dynstr.add_string(b""); + if ctx.dynsym.symbols.is_empty() { + ctx.dynsym.symbols.push(None); + } + } + + chunks.push(ChunkId::Versym); + chunks.push(ChunkId::Verneed); + chunks.push(ChunkId::NotePackage); + + if !ctx.args.oformat_binary { + let shdr = ElfShdr { + sh_type: SHT_PROGBITS, + sh_flags: (SHF_MERGE | SHF_STRINGS) as u64, + ..ElfShdr::default() + }; + let merged = RwLock::new(std::mem::take(&mut ctx.merged_sections)); + ctx.comment = crate::chunks::merged::MergedSection::get_instance( + &ctx.args, + &merged, + BStr::new(b".comment"), + &shdr, + ); + ctx.merged_sections = merged.into_inner().unwrap(); + } + + if E::IS_X86 { + ctx.note_property = Some(NotePropertySection::new::()); + chunks.push(ChunkId::NoteProperty); + } + if E::IS_RISCV { + ctx.riscv_attributes = Some(crate::chunks::misc::RiscvAttributesSection::new()); + chunks.push(ChunkId::RiscvAttributes); + } + if E::FAMILY == Family::Ppc64V2 { + ctx.ppc64_save_restore = Some(crate::chunks::misc::Ppc64SaveRestoreSection::new()); + chunks.push(ChunkId::Ppc64SaveRestore); + } + if E::FAMILY == Family::Ppc64V1 { + ctx.ppc64_opd = Some(crate::chunks::opd::Ppc64OpdSection::new()); + chunks.push(ChunkId::Ppc64Opd); + } + + ctx.chunks.extend(chunks); +} + +/// Marks the files that the given files depend on reachable, recursively. +/// Marks the files reachable from `roots` live. A file found live is +/// visited as a task of its own: there are at most thousands of files, +/// so the tasks are cheap, and the pool stays busy rather than draining +/// between rounds of a search whose frontier is often small. +fn mark_live_files(ctx: &Context, roots: Vec) { + fn visit<'s, E: Arch>(ctx: &'s Context, id: FileId, scope: &rayon::Scope<'s>) { + for found in mark_live_file(ctx, id) { + scope.spawn(move |scope| visit(ctx, found, scope)); + } + } + rayon::scope(|scope| { + for id in roots { + scope.spawn(move |scope| visit(ctx, id, scope)); + } + }); +} + +fn mark_live_file(ctx: &Context, id: FileId) -> Vec { + let mut found = Vec::new(); + match id { + FileId::Obj(obj_id) => { + let file = &ctx.objs[obj_id.index()]; + debug_assert!(file.base.is_reachable()); + for i in file.base.first_global..file.base.elf_syms.len() { + let esym = &file.base.elf_syms.at_in::(i); + let sym = &ctx.symbols[file.base.symbols[i]]; + + if !esym.is_undef() && file.exclude_libs { + sym.merge_visibility(STV_HIDDEN); + } else { + sym.merge_visibility(esym.st_visibility()); + } + if sym.is_traced() { + crate::input_files::print_trace_symbol(&ctx.diag, file, esym, sym); + } + + if let Some(target) = sym.file() { + let undef_ref = esym.is_undef() && (!esym.is_weak() || target.is_dso()); + let common_ref = esym.is_common() && !sym.is_common(); + if (undef_ref || common_ref) && ctx.file(target).mark_reachable() { + found.push(target); + if sym.is_traced() { + out!( + ctx, + "trace-symbol: {file} keeps {} for {sym}", + ctx.file_display(target) + ); + } + } + } + } + } + FileId::Dso(dso_id) => { + let file = &ctx.dsos[dso_id.index()]; + for i in 0..file.base.elf_syms.len() { + let esym = &file.base.elf_syms.at_in::(i); + let sym = &ctx.symbols[file.base.symbols[i]]; + if sym.is_traced() { + crate::input_files::print_trace_symbol(&ctx.diag, file, esym, sym); + } + // Undefined symbols in a DSO are followed only for + // --no-allow-shlib-undefined. + if esym.is_undef() && !esym.is_weak() { + if let Some(target) = sym.file() { + if (!target.is_dso() || !ctx.args.allow_shlib_undefined) + && ctx.file(target).mark_reachable() + { + found.push(target); + if sym.is_traced() { + out!( + ctx, + "trace-symbol: {file} keeps {} for {sym}", + ctx.file_display(target) + ); + } + } + } + } + } + } + } + found +} + +fn mark_live_objects(ctx: &mut Context) { + // Symbols named on the command line pull in their files, and keep + // their sections under --gc-sections; so does the entry point. + let names: Vec = ctx + .args + .undefined + .iter() + .chain(&ctx.args.require_defined) + .cloned() + .collect(); + for name in names { + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].set_gc_root(true); + if let Some(file) = ctx.symbols[id].file() { + ctx.file(file).set_reachable(true); + } + } + let entry = ctx.syms.entry; + ctx.symbols[entry].set_gc_root(true); + + if !ctx.args.undefined_glob.is_empty() { + let roots: Vec = { + let Context { + objs, + symbols, + args, + .. + } = &*ctx; + objs.par_iter() + .enumerate() + .filter(|(_, file)| !file.base.is_reachable()) + .filter_map(|(i, file)| { + file.base.global_symbols().iter().copied().find(|&id| { + let sym = &symbols[id]; + sym.file() == Some(FileId::Obj(ObjId(i as u32))) + && args.undefined_glob.find(sym.name()) != -1 + }) + }) + .collect() + }; + for id in roots { + let file = ctx.symbols[id].file().unwrap(); + ctx.file(file).set_reachable(true); + ctx.symbols[id].set_gc_root(true); + } + } + + let mut roots: Vec = Vec::new(); + for (i, file) in ctx.objs.iter().enumerate() { + if !file.base.as_needed { + file.base.set_reachable(true); + } + if file.base.is_reachable() { + roots.push(FileId::Obj(ObjId(i as u32))); + } + } + for (i, file) in ctx.dsos.iter().enumerate() { + if !file.base.as_needed { + file.base.set_reachable(true); + } + if file.base.is_reachable() { + roots.push(FileId::Dso(crate::input_files::DsoId(i as u32))); + } + } + mark_live_files(ctx, roots); +} + +/// A default-versioned symbol `foo@@VER` can be referred to as `foo` or +/// `foo@VER`. References to the latter were resolved to a forwarding +/// alias; redirect them to the real symbol. +fn resolve_default_symver(ctx: &mut Context) { + let Context { + objs, + dsos, + symbols, + .. + } = ctx; + objs.par_iter_mut().for_each(|file| { + let start = file.base.first_global.min(file.base.symbols.len()); + for id in &mut file.base.symbols[start..] { + if symbols[*id].is_versioned_default() { + *id = symbols[*id].symbol_origin().unwrap(); + } + } + }); + dsos.par_iter_mut().for_each(|file| { + for id in &mut file.base.symbols { + if symbols[*id].is_versioned_default() { + *id = symbols[*id].symbol_origin().unwrap(); + } + } + }); +} + +fn clear_symbol(sym: &mut Symbol) { + sym.clear_file(); + sym.clear_origin(); + sym.value = 0; + sym.sym_idx = u32::MAX; + sym.set_esym(&ElfSym::default()); + sym.ver_idx = VER_NDX_UNSPECIFIED as u16; + sym.set_weak(false); + sym.set_imported(false); + sym.set_exported(false); + sym.set_versioned_default(false); +} + +/// Resets the resolution of every global symbol. Local symbols belong to +/// their files and aren't resolved, so they're left alone. +fn clear_symbols(ctx: &mut Context) { + ctx.symbols.par_for_each_global_mut(|sym| { + if sym.file().is_some() { + clear_symbol(sym); + } + }); +} + +/// Creates a symbol for each global symbol name recorded while reading +/// files and fills in the files' symbol references. +/// Interns the global symbols every input file refers to. +/// +/// The keys are recorded by owner and index, then interned all at once: +/// objects own their global symbol slots, and a shared library owns two +/// sets, its symbols and their default-version aliases. +pub fn gather_symbols(ctx: &mut Context) { + let _t = ctx.timer("gather_symbols"); + let bins = ctx.take_symbol_bins(); + let Context { symbols, .. } = ctx; + + // Each hash shard is populated by one thread and writes directly to the + // stable slots recorded while files were parsed, so no synchronization + // or final scatter pass is needed. + symbols.gather_symbol_slots(bins); +} + +fn current_rank(ctx: &Context, sym: &Symbol) -> u64 { + match sym.file() { + None => 7 << 32, + Some(file) => { + let base = ctx.file(file); + resolved_symbol_rank(sym, file.is_dso(), !base.is_reachable(), base.priority) + } + } +} + +/// Resolves every global symbol in place. Files run in parallel, taking +/// each symbol's lock before comparing and updating its definition. +fn resolve_symbols_pass(ctx: &mut Context, files: &[FileId], only_reachable: bool) { + let _t = ctx.timer("resolve_symbols_pass"); + let Context { + objs, + dsos, + symbols, + default_version, + .. + } = ctx; + let resolver = SymbolResolver::new(symbols.as_mut_slice(), objs, dsos, *default_version); + files.par_iter().for_each(|&id| match id { + FileId::Obj(id) => { + let file = &objs[id.index()]; + if !only_reachable || file.base.is_reachable() { + file.resolve_symbols::(&resolver, id); + } + } + FileId::Dso(id) => { + let file = &dsos[id.index()]; + if !only_reachable || file.base.is_reachable() { + file.resolve_symbols::(&resolver, id); + } + } + }); +} + +/// Resolves the rare hidden-symbol retry while ignoring DSO definitions. +fn resolve_skip_dso_symbols_pass(ctx: &mut Context) { + let _t = ctx.timer("resolve_symbols_pass"); + let Context { + objs, + dsos, + symbols, + default_version, + .. + } = ctx; + let resolver = SymbolResolver::new(symbols.as_mut_slice(), objs, dsos, *default_version); + objs.par_iter().enumerate().for_each(|(i, file)| { + file.resolve_skip_dso_symbols::(&resolver, ObjId(i as u32)); + }); +} + +/// An exceptional COMDAT signature whose Symbol slot is filled by gathering. +/// The owning ObjectFile and its group vector stay in place through the +/// gather, just as they do for the files' ordinary SymbolSlots. +struct PendingComdatOwner { + group: NonNull, + priority: u32, + is_lto_output: bool, +} + +// SAFETY: every pending record names a distinct group, and it is consumed only +// after gathering has stopped writing the group's signature slot. +unsafe impl Send for PendingComdatOwner {} + +/// A packed COMDAT signature word that receives the interned SymbolId before +/// ownership is selected. +#[derive(Clone, Copy)] +struct ComdatSymbolSlot(NonNull); + +// SAFETY: each exceptional COMDAT signature has one pending slot, and one +// symbol-map shard writes it before the groups are examined again. +unsafe impl Send for ComdatSymbolSlot {} +unsafe impl Sync for ComdatSymbolSlot {} + +impl ComdatSymbolSlot { + fn new(group: &mut ComdatGroupRef) -> ComdatSymbolSlot { + ComdatSymbolSlot(NonNull::from(group.signature_word_mut())) + } + + fn assign(self, id: SymbolId) { + assert!(id.0 < 1 << 31); + // SAFETY: guaranteed by the construction and synchronization rules + // documented on ComdatSymbolSlot. + unsafe { self.0.write(id.0) }; + } +} + +/// COMDAT signature keys and owners collected by each Rayon worker. +struct ComdatWorkBins(Vec, Vec)>>); + +// SAFETY: a Rayon worker has a unique index and executes at most one closure +// at a time. The only caller does not invoke nested parallel work while using +// its bin, and the bins are consumed after the traversal has joined. +unsafe impl Sync for ComdatWorkBins {} + +impl ComdatWorkBins { + fn new() -> ComdatWorkBins { + ComdatWorkBins( + (0..=rayon::current_num_threads()) + .map(|_| UnsafeCell::new((Bins::new(), Vec::new()))) + .collect(), + ) + } + + #[inline] + fn with_local( + &self, + f: impl FnOnce(&mut Bins, &mut Vec), + ) { + let fallback = self.0.len() - 1; + let i = rayon::current_thread_index() + .unwrap_or(fallback) + .min(fallback); + // SAFETY: the Sync invariant above gives this worker exclusive access + // to its indexed buffers for the duration of this non-nested closure. + let (bins, pending) = unsafe { &mut *self.0[i].get() }; + f(bins, pending); + } + + fn into_parts(self) -> (Vec>, Vec>) { + self.0.into_iter().map(UnsafeCell::into_inner).unzip() + } +} + +/// Selects COMDAT groups and constructs input sections. +fn parse_input_sections(ctx: &mut Context) { + let _t = ctx.timer("parse_input_sections"); + + // Symbol resolution is clear while COMDAT groups are selected, so sym_idx + // can temporarily hold the winning file priority. + let record_owner = |sym: &Symbol, priority: u32| { + // SAFETY: sym_idx is aligned for u32. During this parallel phase it is + // accessed only through this atomic reference, and the phase is joined + // before ordinary accesses resume. + let owner = unsafe { AtomicU32::from_ptr(std::ptr::addr_of!(sym.sym_idx).cast_mut()) }; + let mut old = owner.load(Ordering::Relaxed); + while priority < old { + match owner.compare_exchange_weak(old, priority, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(actual) => old = actual, + } + } + }; + + // Read COMDAT metadata and choose an owner among reachable regular objects. + // Ordinary global signatures already refer to the files' symbols; record + // the other signatures for interning while each file's metadata is hot. + let t = ctx.timer("read_section_metadata"); + let (bins, pending): (Vec>, Vec>) = { + let Context { + objs, + symbols, + diag, + .. + } = ctx; + let work = ComdatWorkBins::new(); + objs.par_iter_mut().for_each(|file| { + work.with_local(|bins, pending| { + if file.base.is_reachable() { + if file.base.mf.is_some() && !file.is_lto_input && !file.sections_parsed { + file.read_section_metadata::(diag); + } + let priority = file.base.priority; + let is_lto_output = file.is_lto_output; + for group in &mut file.comdat_groups { + if group.signature() != SymbolId::DISCARDED_COMDAT { + let sym = &symbols[group.signature()]; + if !sym.comdat_claimed_by_ir() || is_lto_output { + record_owner(sym, priority); + } + } + } + let signatures = &file.pending_comdat_signatures; + let groups = &mut file.comdat_groups; + for signature in signatures { + let group = &mut groups[signature.group_idx as usize]; + bins.record( + signature.key, + signature.name_len as usize, + ComdatSymbolSlot::new(group), + ); + pending.push(PendingComdatOwner { + group: NonNull::from(group), + priority, + is_lto_output, + }); + } + } + }); + }); + work.into_parts() + }; + + drop(t); + + // Intern the signature symbols all at once, as for the files' own symbols. + let t = ctx.timer("comdat_signatures"); + { + let Context { objs, symbols, .. } = ctx; + symbols.gather(bins, ComdatSymbolSlot::assign); + + // Signatures just interned could not participate in the metadata + // traversal above. Record them now. + let symbols: &crate::symbol::SymbolTable = symbols; + pending.into_par_iter().flatten().for_each(|pending| { + // SAFETY: each pending pointer came from a distinct group that + // stays in place, and symbol gathering has joined before this + // parallel traversal begins. + let group = unsafe { &*pending.group.as_ptr() }; + let sym = &symbols[group.signature()]; + if !sym.comdat_claimed_by_ir() || pending.is_lto_output { + record_owner(sym, pending.priority); + } + }); + objs.par_iter_mut() + .for_each(|file| file.pending_comdat_signatures = Vec::new()); + } + + // IR objects name their COMDAT groups per symbol. + for fi in 0..ctx.objs.len() { + for i in 0..ctx.objs[fi].lto_comdat_keys.len() { + if let (Some(key), None) = ( + ctx.objs[fi].lto_comdat_keys[i], + ctx.objs[fi].lto_comdat_signatures[i], + ) { + let sig = ctx.symbols.intern(key); + ctx.objs[fi].lto_comdat_signatures[i] = Some(sig); + } + } + } + + drop(t); + + // Choose an owner for each group: the reachable object with the + // lowest priority. A group claimed by an IR file belongs to the LTO + // result, so only an LTO-generated object may own it. + let t = ctx.timer("comdat_owners"); + + // Set each group's result after all reachable files have recorded their + // claims. + let symbols = &ctx.symbols; + ctx.objs.par_iter_mut().for_each(|file| { + if file.base.is_reachable() { + let priority = file.base.priority; + for group in &mut file.comdat_groups { + group.set_owner(symbols[group.signature()].sym_idx == priority); + } + } + }); + + // An IR file may claim a group only if no reachable regular object + // already has. Its claim is permanent: the LTO result provides the + // group's definitions, so a regular object extracted after LTO must + // not win the group and resurrect a copy of them. + for fi in 0..ctx.objs.len() { + let file = &ctx.objs[fi]; + if !file.base.is_reachable() || file.lto_comdat_signatures.is_empty() { + continue; + } + let priority = file.base.priority; + let mut discarded = Vec::with_capacity(file.lto_comdat_signatures.len()); + for &sig in &file.lto_comdat_signatures { + let Some(sig) = sig else { + discarded.push(false); + continue; + }; + if ctx.symbols[sig].sym_idx == u32::MAX { + ctx.symbols[sig].sym_idx = priority; + ctx.symbols[sig].set_comdat_claimed_by_ir(true); + } + discarded.push(ctx.symbols[sig].sym_idx != priority); + } + ctx.objs[fi].lto_comdat_discarded = discarded; + } + + drop(t); + + // LTO can change archive extraction and therefore the winning COMDAT + // group. Construct the losing copies too so one can become the winner + // after LTO. + let keep_discarded_comdat = ctx + .objs + .iter() + .any(|f| f.base.is_reachable() && (f.is_lto_input || f.is_gcc_offload_obj)); + + let t = ctx.timer("parse_sections"); + let maximum: usize = ctx + .objs + .iter() + .filter(|file| { + file.base.is_reachable() + && file.base.mf.is_some() + && !file.is_lto_input + && !file.sections_parsed + }) + .map(|file| { + if file.base.elf_syms.is_empty() { + 0 + } else { + file.base.first_global.max(1) + } + }) + .sum(); + { + let Context { + objs, + symbols, + section_arena, + diag, + args, + .. + } = ctx; + symbols.with_parallel_appender(maximum, |allocator| { + objs.par_iter_mut().enumerate().for_each(|(i, file)| { + if file.base.is_reachable() + && file.base.mf.is_some() + && !file.is_lto_input + && !file.sections_parsed + { + file.parse_sections::( + diag, + args, + ObjId(i as u32), + section_arena, + allocator, + keep_discarded_comdat, + ); + } + }); + }); + } + drop(t); + + // Apply the selection to all group members. + let _t = ctx.timer("comdat_members"); + ctx.objs.par_iter().for_each(|file| { + if !file.base.is_reachable() { + return; + } + for group in &file.comdat_groups { + for member in file.comdat_members(group) { + if let Some(isec) = file.section(member as usize) { + if group.is_owner() { + isec.revive(); + } else { + file.kill_section(member as usize); + } + } + } + } + }); +} + +pub fn resolve_symbols(ctx: &mut Context) { + let _t = ctx.timer("resolve_symbols"); + gather_symbols(ctx); + + let files: Vec = (0..ctx.objs.len()) + .map(|i| FileId::Obj(ObjId(i as u32))) + .chain((0..ctx.dsos.len()).map(|i| FileId::Dso(crate::input_files::DsoId(i as u32)))) + .collect(); + + // Call resolve_symbols() to find the most appropriate file for each + // symbol. And then mark reachable objects to decide which files to + // include into an output. + resolve_symbols_pass(ctx, &files, false); + let t = ctx.timer("resolve_default_symver"); + resolve_default_symver(ctx); + drop(t); + let t = ctx.timer("mark_live_objects"); + mark_live_objects(ctx); + drop(t); + + // Symbols with hidden visibility need to be resolved within the + // output file. If a hidden symbol was resolved to a DSO, we resolve + // that symbol again, skipping DSOs. Its new definition may be in an + // archive member that is not reachable yet; making it reachable can + // change other symbols' visibility, so we repeat until no hidden + // symbol resolves to a DSO. This should be rare. + loop { + let hidden = ctx.symbols.par_find_globals(|sym| { + matches!(sym.file(), Some(FileId::Dso(dso)) + if ctx.dsos[dso.index()].base.is_reachable() + && sym.visibility() == STV_HIDDEN) + }); + if hidden.is_empty() { + break; + } + for &id in &hidden { + let sym = &mut ctx.symbols[id]; + sym.set_skip_dso(true); + clear_symbol(sym); + } + resolve_skip_dso_symbols_pass(ctx); + + let mut roots = Vec::new(); + for &id in &hidden { + if let Some(file) = ctx.symbols[id].file() { + if !ctx.file(file).is_reachable() { + ctx.file(file).set_reachable(true); + roots.push(file); + } + } + } + mark_live_files(ctx, roots); + } + + // Now that we know the exact set of input files that are to be + // included in the output file, we want to redo symbol resolution. + // This is because symbols defined by object files in archive files + // may have risen as a result of mark_live_objects(). + // + // To redo symbol resolution, we want to clear the state first. + let t = ctx.timer("clear_symbols"); + clear_symbols(ctx); + drop(t); + + // Parse input sections after archive extraction, so COMDAT selection + // considers only reachable objects. This must happen before final symbol + // resolution; otherwise discarding a group could leave a dangling reference. + parse_input_sections(ctx); + + // Redo symbol resolution + resolve_symbols_pass(ctx, &files, true); +} + +/// Drops files that didn't make it into the link and renumbers the rest. +pub fn remove_unreachable_files(ctx: &mut Context) { + remove_objects(ctx, |file| !file.base.is_reachable()); + remove_unreachable_dsos(ctx); +} + +/// Drops the object files `remove` selects and renumbers the rest. +fn remove_objects(ctx: &mut Context, remove: impl Fn(&ObjectFile) -> bool) { + if !ctx.objs.iter().any(|file| remove(file)) { + return; + } + let objs = std::mem::take(&mut ctx.objs); + let mut obj_map: Vec> = vec![None; objs.len()]; + let mut removed = Vec::new(); + for (i, file) in objs.into_iter().enumerate() { + if remove(&file) { + removed.push(file); + } else { + obj_map[i] = Some(ObjId(ctx.objs.len() as u32)); + ctx.objs.push(file); + } + } + // C++ InputFiles are arena-owned, so erasing a pointer from ctx.objs does + // not destroy the file. Keep the boxes until Context teardown too. + ctx.discarded_objs.extend(removed); + + ctx.symbols.as_mut_slice().par_iter_mut().for_each(|sym| { + if let Some(FileId::Obj(o)) = sym.file() { + match obj_map[o.index()] { + Some(new) => sym.set_file(FileId::Obj(new)), + None => clear_symbol(sym), + } + } + }); + for slot in &mut ctx.file_by_priority { + if let Some(FileId::Obj(o)) = *slot { + *slot = obj_map[o.index()].map(FileId::Obj); + } + } + ctx.internal_obj = ctx.internal_obj.and_then(|id| obj_map[id.index()]); + + // Section references inside sections refer to their own file. + ctx.objs.par_iter_mut().enumerate().for_each(|(i, file)| { + let n = file.sections.len(); + for shndx in 0..n { + if let Some(isec) = file.section_mut(shndx) { + isec.file = ObjId(i as u32); + } + } + }); +} + +/// Whether the link involves the LTO plugin. +pub fn has_lto_obj(ctx: &Context) -> bool { + ctx.objs + .iter() + .any(|file| file.base.is_reachable() && (file.is_lto_input || file.is_gcc_offload_obj)) +} + +/// Runs link-time optimization: the plugin compiles the IR objects into +/// ELF objects, which then take their place. +pub fn do_lto(ctx: &mut Context) { + let _t = ctx.timer("do_lto"); + + // The compiler backend needs to know how symbols are resolved, so + // visibility and import/export bits are computed early. + apply_version_script(ctx); + parse_symbol_version(ctx); + compute_import_export(ctx); + + // If several IR objects define the same symbol, the backend would + // pick one at random rather than complain. + if !ctx.args.allow_multiple_definition { + check_duplicate_symbols(ctx); + } + + crate::lto::run_plugin(ctx); + + // Redo name resolution without the IR objects. Archive members that + // were extracted for them may no longer be needed, so their + // reachability is decided afresh too. + clear_symbols(ctx); + for file in &ctx.objs { + if file.is_lto_input || file.base.as_needed { + file.base.set_reachable(false); + } + } + let ir_files = ctx + .objs + .iter() + .filter(|file| file.is_lto_input) + .filter_map(|file| Some((file.base.priority, file.base.mf?))); + ctx.lto_input_files.extend(ir_files); + remove_objects(ctx, |file| file.is_lto_input); + resolve_symbols(ctx); +} + +pub fn parse_eh_frame_sections(ctx: &mut Context) { + let _t = ctx.timer("parse_eh_frame_sections"); + let Context { objs, diag, .. } = ctx; + objs.par_iter_mut() + .for_each(|file| file.parse_ehframe::(diag)); +} + +pub fn parse_sframe_sections(ctx: &mut Context) { + if !E::SUPPORTS_SFRAME { + return; + } + let _t = ctx.timer("parse_sframe_sections"); + let Context { objs, diag, .. } = ctx; + objs.par_iter_mut() + .for_each(|file| file.parse_sframe::(diag)); +} + +/// Registers direct, stable member borrows with their merged sections for a +/// parallel resolution phase. +fn merged_resolve_members( + objs: &mut [Box], + count: usize, +) -> Vec>> { + let mut members: Vec>> = + (0..count).map(|_| Vec::new()).collect(); + for file in objs { + let filename = file.base.filename.as_str(); + let archive_name = file.archive_name.as_str(); + let shstrtab = file.base.shstrtab; + let num_elf_sections = file.num_elf_sections; + for (mergeable, input) in file.sections.mergeable_sections_with_inputs_mut() { + let parent = mergeable.parent.index(); + members[parent].push(crate::chunks::merged::ResolveMember { + mergeable, + section: input, + filename, + archive_name, + name: input.name_in(shstrtab, num_elf_sections), + }); + } + } + members +} + +pub fn create_merged_sections(ctx: &mut Context) { + let _t = ctx.timer("create_merged_sections"); + + let t = ctx.timer("convert_mergeable_sections"); + { + let Context { + objs, + merged_sections, + args, + diag, + .. + } = ctx; + let merged = RwLock::new(std::mem::take(merged_sections)); + objs.par_iter_mut() + .for_each(|file| file.convert_mergeable_sections::(args, &merged, diag)); + *merged_sections = merged.into_inner().unwrap(); + } + drop(t); + + // Register each mergeable section with its merged section. + let t = ctx.timer("register_members"); + for (fi, file) in ctx.objs.iter().enumerate() { + for m in file.mergeable_sections() { + ctx.merged_sections[m.parent.index()] + .members + .push(SectionRef { + file: ObjId(fi as u32), + shndx: m.shndx, + }); + } + } + drop(t); + + let t = ctx.timer("resolve"); + let mut members = merged_resolve_members(&mut ctx.objs, ctx.merged_sections.len()); + crate::chunks::merged::resolve_sections::( + &mut ctx.merged_sections, + &mut members, + crate::chunks::merged::ResolveOptions { + allocated_only: true, + gc_sections: ctx.args.gc_sections, + diag: &ctx.diag, + comment: ctx.comment, + cmdline_args: &ctx.cmdline_args, + timers: &ctx.timers, + }, + ); + drop(t); + + let _t = ctx.timer("reattach_section_pieces"); + let Context { + objs, + symbols, + merged_sections, + diag, + .. + } = ctx; + + // Arena allocations cannot be reclaimed, so grow the symbol table only + // once. num_frag_syms, counted when the sections were parsed, may include + // references to mergeable sections that were not converted; the extra + // symbols stay unused. + let counts: Vec = objs + .iter() + .map(|file| file.num_fragment_dummies()) + .collect(); + let base = symbols.len(); + // SAFETY: the new slots are split among the files exactly, and each file + // initializes every slot in its slice. The editor covers only the + // disjoint, initialized prefix. + unsafe { + symbols.add_many_with_existing(counts.iter().sum(), |existing, slots| { + let editor = SymbolEditor::new(existing); + let mut rest = slots; + let mut slices = Vec::with_capacity(counts.len()); + let mut offset = base; + for &count in &counts { + let (head, tail) = rest.split_at_mut(count); + slices.push((SymbolId(offset as u32), head)); + rest = tail; + offset += count; + } + objs.par_iter_mut().enumerate().zip(slices).for_each( + |((i, file), (base_id, slots))| { + file.reattach_section_symbols::( + diag, + ObjId(i as u32), + &editor, + merged_sections, + ); + file.reattach_fragment_relocations::( + diag, + ObjId(i as u32), + merged_sections, + base_id, + slots, + ); + }, + ); + }); + } +} + +pub fn convert_common_symbols(ctx: &mut Context) { + let _t = ctx.timer("convert_common_symbols"); + let default_version = ctx.default_version; + for fi in 0..ctx.objs.len() { + let Context { + objs, + symbols, + diag, + args, + section_arena, + .. + } = ctx; + objs[fi].convert_common_symbols::( + diag, + args, + ObjId(fi as u32), + symbols, + default_version, + section_arena, + ); + } +} + +fn has_ctors_and_init_array(ctx: &Context) -> bool { + ctx.objs.iter().any(|f| f.has_ctors) && ctx.objs.iter().any(|f| f.has_init_array) +} + +fn canonicalize_type(name: &[u8], ty: u32) -> u32 { + // Some old assemblers don't recognize these section names and create + // them as SHT_PROGBITS. + if ty == SHT_PROGBITS { + if name == b".init_array" || name.starts_with(b".init_array.") { + return SHT_INIT_ARRAY; + } + if name == b".fini_array" || name.starts_with(b".fini_array.") { + return SHT_FINI_ARRAY; + } + } + // The x86-64 psABI defines SHT_X86_64_UNWIND for .eh_frame, but the + // section may come as either type. Use SHT_PROGBITS consistently. + if E::FAMILY == Family::X86_64 && ty == SHT_X86_64_UNWIND { + return SHT_PROGBITS; + } + ty +} + +fn output_name( + args: &crate::args::Args, + name: &'static BStr, + flags: u64, +) -> &'static BStr { + if args.relocatable && !args.relocatable_merge_sections { + return name; + } + if !args.unique.is_empty() && args.unique.find(name) != -1 { + return name; + } + if flags & SHF_MERGE as u64 != 0 { + return name; + } + if E::FAMILY == Family::Arm32 { + if name.starts_with(b".ARM.exidx") { + return BStr::new(b".ARM.exidx"); + } + if name.starts_with(b".ARM.extab") { + return BStr::new(b".ARM.extab"); + } + } + + if args.z_keep_text_section_prefix { + for prefix in [ + ".text.hot.", + ".text.unknown.", + ".text.unlikely.", + ".text.startup.", + ".text.exit.", + ] { + let stem = &prefix[..prefix.len() - 1]; + if name == stem.as_bytes() || name.starts_with(prefix.as_bytes()) { + return BStr::new(stem.as_bytes()); + } + } + } + + const PREFIXES: &[&str] = &[ + ".text.", + ".data.rel.ro.", + ".data.", + ".rodata.", + ".bss.rel.ro.", + ".bss.", + ".init_array.", + ".fini_array.", + ".tbss.", + ".tdata.", + ".gcc_except_table.", + ".ctors.", + ".dtors.", + ".gnu.warning.", + ".openbsd.randomdata.", + ".sdata.", + ".sbss.", + ".srodata.", + ".gnu.build.attributes.", + ]; + for prefix in PREFIXES { + let stem = &prefix[..prefix.len() - 1]; + if name == stem.as_bytes() || name.starts_with(prefix.as_bytes()) { + return BStr::new(stem.as_bytes()); + } + } + name +} + +fn output_section_key( + args: &crate::args::Args, + isec: &crate::input_sections::InputSection, + name: &'static BStr, + sh_type: u32, + ctors_in_init_array: bool, +) -> (&'static BStr, u32) { + // .ctors/.dtors are merged into .init_array/.fini_array if those + // exist, except for the relocation-free sentinel sections in CRT + // files, whose values 0 and -1 would crash the program. + if ctors_in_init_array && isec.has_relocations() { + if name == b".ctors" || name.starts_with(b".ctors.") { + return (BStr::new(b".init_array"), SHT_INIT_ARRAY); + } + if name == b".dtors" || name.starts_with(b".dtors.") { + return (BStr::new(b".fini_array"), SHT_FINI_ARRAY); + } + } + let name = output_name::(args, name, isec.sh_flags); + (name, canonicalize_type::(name, sh_type)) +} + +type OutputSectionKey = (&'static BStr, u32); + +/// One file's contribution to an output section while members are gathered. +struct OutputSectionFileMembers { + members: Vec, + sh_flags: u64, + p2align: u8, +} + +/// Stable scratch storage corresponding to C++ `OutputSection::members_vec`. +struct OutputSectionBuilder { + section: OutputSectionId, + files: Box<[UnsafeCell]>, +} + +type OutputSectionShared = ( + HashMap>, + Vec, +); + +// SAFETY: the file-parallel traversal gives each task a distinct slot. The +// slots are read only after that traversal joins. +unsafe impl Sync for OutputSectionBuilder {} + +impl OutputSectionBuilder { + fn new(section: OutputSectionId, num_files: usize) -> OutputSectionBuilder { + OutputSectionBuilder { + section, + files: (0..num_files) + .map(|_| { + UnsafeCell::new(OutputSectionFileMembers { + members: Vec::new(), + sh_flags: 0, + p2align: 0, + }) + }) + .collect(), + } + } + + /// Returns the slot owned by the task processing `file`. + /// + /// # Safety + /// + /// Only one task may call this for a given file index, and all calls must + /// finish before the slots are read. + #[inline] + unsafe fn with_file_mut( + &self, + file: usize, + f: impl FnOnce(&mut OutputSectionFileMembers) -> R, + ) -> R { + unsafe { f(&mut *self.files[file].get()) } + } +} + +/// The output-section map cached by each Rayon worker. This is the equivalent +/// of C++ mold's `enumerable_thread_specific`. +struct OutputSectionCaches(Vec>>>); + +// SAFETY: a Rayon worker has a unique index and executes at most one closure +// at a time. The only caller does not invoke nested parallel work while using +// its cache, and the caches are dropped after the parallel traversal joins. +unsafe impl Sync for OutputSectionCaches {} + +impl OutputSectionCaches { + fn new() -> OutputSectionCaches { + OutputSectionCaches( + (0..=rayon::current_num_threads()) + .map(|_| UnsafeCell::new(HashMap::new())) + .collect(), + ) + } + + #[inline] + fn with_local( + &self, + f: impl FnOnce(&mut HashMap>) -> R, + ) -> R { + let fallback = self.0.len() - 1; + let i = rayon::current_thread_index() + .unwrap_or(fallback) + .min(fallback); + // SAFETY: the Sync invariant above gives this worker exclusive access + // to its indexed map for the duration of this non-nested closure. + f(unsafe { &mut *self.0[i].get() }) + } +} + +/// PT_GNU_RELRO makes pages that need dynamic relocations at load time +/// read-only afterwards. Sections such as `.init_array`, `.got` and +/// `.dynamic` need relocations but not writability at runtime. +fn is_relro(osec: &OutputSection) -> bool { + let name = osec.hdr.name; + let ty = osec.hdr.shdr.sh_type; + let flags = osec.hdr.shdr.sh_flags; + name == b".toc" + || name.ends_with(b".rel.ro") + || name.ends_with(b".rel.ro.hot") + || name.ends_with(b".rel.ro.unlikely") + || matches!(ty, SHT_INIT_ARRAY | SHT_FINI_ARRAY | SHT_PREINIT_ARRAY) + || flags & SHF_TLS as u64 != 0 +} + +/// Creates output sections for input sections. +pub fn create_output_sections(ctx: &mut Context) { + let _t = ctx.timer("create_output_sections"); + let ctors_in_init_array = has_ctors_and_init_array(ctx); + let first_new = ctx.output_sections.len(); + + // Instantiate output sections and assign input sections to them. The + // output sections and the map from keys to them are shared under a + // lock; each worker keeps a cache of the map to avoid lock contention. + // It makes a noticeable difference if we have millions of input + // sections. + let num_files = ctx.objs.len(); + let shared: Mutex = + Mutex::new((HashMap::new(), std::mem::take(&mut ctx.output_sections))); + let caches = OutputSectionCaches::new(); + + // Instantiate output sections and assign input sections to them + { + let Context { objs, args, .. } = ctx; + objs.par_iter_mut().enumerate().for_each(|(fi, file)| { + caches.with_local(|cache| { + let shstrtab = file.base.shstrtab; + let num_elf_sections = file.num_elf_sections; + let shdrs = &file.base.shdrs; + for (member, isec) in file + .sections + .regular_ids_mut() + .filter(|(_, isec)| isec.is_alive()) + { + let name = isec.name_in(shstrtab, num_elf_sections); + let sh_type = isec.sh_type_from(shdrs); + let sh_flags = isec.sh_flags + & !(SHF_MERGE | SHF_STRINGS | SHF_COMPRESSED | SHF_GNU_RETAIN) as u64; + + if args.relocatable && sh_flags & SHF_GROUP as u64 != 0 { + // COMDAT group members keep their own output sections + // in a relocatable output. + let mut osec = OutputSection::new(name, sh_type); + osec.hdr.shdr.sh_flags = sh_flags; + osec.hdr.shdr.sh_addralign = 1 << isec.p2align(); + osec.hdr.is_relro = is_relro(&osec); + osec.members.push(member); + let mut shared = shared.lock().unwrap(); + shared.1.push(osec); + isec.output_section = Some(OutputSectionId::new(shared.1.len() as u32 - 1)); + continue; + } + + let key = + output_section_key::(args, isec, name, sh_type, ctors_in_init_array); + let builder = match cache.entry(key) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + let mut shared = shared.lock().unwrap(); + let (map, sections) = &mut *shared; + let builder = map + .entry(key) + .or_insert_with(|| { + sections.push(OutputSection::new(key.0, key.1)); + let id = OutputSectionId::new(sections.len() as u32 - 1); + Arc::new(OutputSectionBuilder::new(id, num_files)) + }) + .clone(); + entry.insert(builder) + } + }; + isec.output_section = Some(builder.section); + + // SAFETY: this closure is the only task for file fi; + // flattening starts after the parallel traversal joins. + unsafe { + builder.with_file_mut(fi, |file| { + file.members.push(member); + file.sh_flags |= sh_flags & !(SHF_GROUP as u64); + file.p2align = file.p2align.max(isec.p2align()); + }); + } + } + }) + }); + } + let (map, sections) = shared.into_inner().unwrap(); + ctx.output_sections = sections; + + // Flatten members_vec into members and compute the section alignment. + // Both are done in parallel over the files; an output section such as + // .text has a million members. + let builders: Vec> = map.into_values().collect(); + drop(caches); + let flattened: Vec<(OutputSectionId, Vec, u64, u8)> = builders + .par_iter() + .map(|builder| { + let parts: Vec<&OutputSectionFileMembers> = builder + .files + .iter() + // SAFETY: the file traversal has joined, so the slots are no + // longer being mutated. + .map(|file| unsafe { &*file.get() }) + .collect(); + + let n = parts.iter().map(|g| g.members.len()).sum(); + let mut members = vec![InputSectionId::NONE; n]; + let mut rest = members.as_mut_slice(); + let mut slices = Vec::with_capacity(parts.len()); + for g in &parts { + let (head, tail) = rest.split_at_mut(g.members.len()); + slices.push(head); + rest = tail; + } + parts.par_iter().zip(slices).for_each(|(g, slice)| { + slice.copy_from_slice(&g.members); + }); + + let sh_flags = parts.iter().fold(0, |flags, g| flags | g.sh_flags); + let p2align = parts.iter().map(|g| g.p2align).max().unwrap_or(0); + (builder.section, members, sh_flags, p2align) + }) + .collect(); + for (id, members, sh_flags, p2align) in flattened { + let osec = &mut ctx.output_sections[id.index()]; + osec.members = members; + osec.hdr.shdr.sh_flags = sh_flags; + osec.hdr.shdr.sh_addralign = 1 << p2align; + osec.hdr.is_relro = is_relro(osec); + } + + // Output sections are created in an arbitrary order; sort them to + // make the output deterministic. + let mut chunks: Vec = (first_new..ctx.output_sections.len()) + .map(|i| ChunkId::Output(OutputSectionId::new(i as u32))) + .chain( + (0..ctx.merged_sections.len()) + .map(|i| ChunkId::Merged(crate::chunks::merged::MergedSectionId(i as u32))), + ) + .collect(); + chunks.sort_by_cached_key(|&id| { + let hdr = ctx.chunk_header(id); + (hdr.name.to_vec(), hdr.shdr.sh_type, hdr.shdr.sh_flags) + }); + ctx.chunks.extend(chunks); +} + +/// Creates the object file that holds linker-synthesized symbols. +pub fn create_internal_file(ctx: &mut Context) { + let mut obj = ObjectFile::internal(); + obj.base.priority = 0; + + ctx.internal_esyms = vec![ElfSym::default()]; + let dummy = ctx.symbols.add(Symbol::new(BStr::new(b""))); + obj.base.symbols.push(dummy); + obj.base.first_global = 1; + + let add = |ctx: &mut Context, obj: &mut ObjectFile, name: &str| { + let id = ctx.get_symbol(name.as_bytes()); + obj.base.symbols.push(id); + // The real value is set by fix_synthetic_symbols; a distinctive + // dummy makes accidental early uses easier to spot. + ctx.symbols[id].value = 0xdeadbeef; + let mut esym = ElfSym { + st_shndx: SHN_ABS as u16, + ..ElfSym::default() + }; + esym.set_type(STT_NOTYPE); + esym.set_bind(STB_GLOBAL); + esym.set_visibility(STV_DEFAULT); + ctx.internal_esyms.push(esym); + }; + + for (name, _) in ctx.args.defsyms.clone() { + add(ctx, &mut obj, &name); + } + for order in ctx.args.section_order.clone() { + if order.kind == SectionOrderKind::Symbol { + add(ctx, &mut obj, &order.name); + } + } + + obj.base.elf_syms = SymTable::from_records(RecordLayout::of::(), &ctx.internal_esyms); + ctx.objs.push(Box::new(obj)); + let id = ObjId(ctx.objs.len() as u32 - 1); + ctx.internal_obj = Some(id); + if ctx.file_by_priority.is_empty() { + ctx.file_by_priority.push(None); + } + ctx.file_by_priority[0] = Some(FileId::Obj(id)); +} + +fn start_stop_name(ctx: &Context, id: ChunkId) -> Option { + let hdr = ctx.chunk_header(id); + if !hdr.is_alloc() || hdr.name.is_empty() { + return None; + } + if is_c_identifier(hdr.name) { + return Some(String::from_utf8_lossy(hdr.name).into_owned()); + } + if ctx.args.start_stop { + let s = String::from_utf8_lossy(hdr.name).into_owned(); + let s = s.strip_prefix('.').unwrap_or(&s); + return Some( + s.chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(), + ); + } + None +} + +/// Resolves the internal object's symbols; its definitions win ties by +/// virtue of its priority 0. +fn resolve_internal_symbols(ctx: &mut Context) { + let id = ctx.internal_obj.unwrap(); + let obj = &ctx.objs[id.index()]; + for i in obj.base.first_global..obj.base.elf_syms.len() { + let esym = obj.base.elf_syms.at_in::(i); + let sym_id = obj.base.symbols[i]; + let rank = symbol_resolution_rank(&esym, false, false, 0); + if rank < current_rank(ctx, &ctx.symbols[sym_id]) { + let sym = &mut ctx.symbols[sym_id]; + sym.set_file(FileId::Obj(id)); + sym.clear_origin(); + sym.value = esym.st_value; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.ver_idx = ctx.default_version; + sym.set_weak(esym.is_weak()); + sym.set_versioned_default(false); + } + } +} + +pub fn add_synthetic_symbols(ctx: &mut Context) { + let obj_id = ctx.internal_obj.unwrap(); + + fn add(ctx: &mut Context, name: &str, ty: u32) -> SymbolId { + let mut esym = ElfSym { + st_shndx: SHN_ABS as u16, + ..ElfSym::default() + }; + esym.set_type(ty); + esym.set_bind(STB_GLOBAL); + esym.set_visibility(STV_HIDDEN); + ctx.internal_esyms.push(esym); + let id = ctx.get_symbol(name.as_bytes()); + ctx.symbols[id].value = 0xdeadbeef; + let obj = ctx.internal_obj.unwrap(); + ctx.objs[obj.index()].base.symbols.push(id); + id + } + + let s = |ctx: &mut Context, name: &str| add(ctx, name, STT_NOTYPE); + + ctx.syms.ehdr_start = Some(s(ctx, "__ehdr_start")); + ctx.syms.init_array_start = Some(s(ctx, "__init_array_start")); + ctx.syms.init_array_end = Some(s(ctx, "__init_array_end")); + ctx.syms.fini_array_start = Some(s(ctx, "__fini_array_start")); + ctx.syms.fini_array_end = Some(s(ctx, "__fini_array_end")); + ctx.syms.preinit_array_start = Some(s(ctx, "__preinit_array_start")); + ctx.syms.preinit_array_end = Some(s(ctx, "__preinit_array_end")); + ctx.syms.dynamic = Some(s(ctx, "_DYNAMIC")); + ctx.syms.global_offset_table = Some(s(ctx, "_GLOBAL_OFFSET_TABLE_")); + ctx.syms.procedure_linkage_table = Some(s(ctx, "_PROCEDURE_LINKAGE_TABLE_")); + ctx.syms.bss_start = Some(s(ctx, "__bss_start")); + ctx.syms.end_ = Some(s(ctx, "_end")); + ctx.syms.etext_ = Some(s(ctx, "_etext")); + ctx.syms.edata_ = Some(s(ctx, "_edata")); + ctx.syms.executable_start = Some(s(ctx, "__executable_start")); + ctx.syms.rel_iplt_start = Some(s( + ctx, + if E::IS_RELA { + "__rela_iplt_start" + } else { + "__rel_iplt_start" + }, + )); + ctx.syms.rel_iplt_end = Some(s( + ctx, + if E::IS_RELA { + "__rela_iplt_end" + } else { + "__rel_iplt_end" + }, + )); + + if ctx.args.eh_frame_hdr { + ctx.syms.gnu_eh_frame_hdr = Some(s(ctx, "__GNU_EH_FRAME_HDR")); + } + + for (name, slot) in [("end", 0), ("etext", 1), ("edata", 2), ("__dso_handle", 3)] { + let id = ctx.get_symbol(name.as_bytes()); + if ctx.symbols[id].file().is_none() { + let id = s(ctx, name); + match slot { + 0 => ctx.syms.end = Some(id), + 1 => ctx.syms.etext = Some(id), + 2 => ctx.syms.edata = Some(id), + _ => ctx.syms.dso_handle = Some(id), + } + } + } + + if E::SUPPORTS_TLSDESC { + ctx.syms.tls_module_base = Some(add(ctx, "_TLS_MODULE_BASE_", STT_TLS)); + } + if E::IS_RISCV { + let id = s(ctx, "__global_pointer$"); + ctx.syms.global_pointer = Some(id); + if ctx.dynamic.is_some() && !ctx.args.shared { + ctx.symbols[id].set_exported(true); + } + } + if E::FAMILY == Family::Arm32 { + ctx.syms.exidx_start = Some(s(ctx, "__exidx_start")); + ctx.syms.exidx_end = Some(s(ctx, "__exidx_end")); + } + if E::IS_PPC64 { + ctx.syms.toc = Some(s(ctx, ".TOC.")); + } + if E::FAMILY == Family::Ppc64V2 { + for &(label, _) in crate::arch::ppc64v2::SAVE_RESTORE_INSNS { + if !label.is_empty() { + s(ctx, label); + } + } + } + if E::FAMILY == Family::Ppc32 { + ctx.syms.sda_base = Some(s(ctx, "_SDA_BASE_")); + } + if E::IS_SPARC { + // TLS_GD_CALL and TLS_LDM_CALL relocations call it implicitly. + ctx.syms.tls_get_addr = Some(ctx.get_symbol(b"__tls_get_addr")); + } + + let add_start_stop = |ctx: &mut Context, name: String| { + let id = s(ctx, &name); + if ctx.args.z_start_stop_visibility_protected { + ctx.symbols[id].set_exported(true); + } + }; + for id in ctx.chunks.clone() { + if let Some(name) = start_stop_name(ctx, id) { + add_start_stop(ctx, format!("__start_{name}")); + add_start_stop(ctx, format!("__stop_{name}")); + if ctx.args.physical_image_base.is_some() { + add_start_stop(ctx, format!("__phys_start_{name}")); + add_start_stop(ctx, format!("__phys_stop_{name}")); + } + } + } + + ctx.objs[obj_id.index()].base.elf_syms = + SymTable::from_records(RecordLayout::of::(), &ctx.internal_esyms); + resolve_internal_symbols(ctx); + + // Make all synthetic symbols relative by associating them with a + // dummy output section. + let syms = ctx.objs[obj_id.index()].base.symbols.clone(); + for id in &syms { + let sym = &mut ctx.symbols[*id]; + if sym.file() == Some(FileId::Obj(obj_id)) { + sym.set_output_chunk(ChunkId::Symtab); + sym.set_imported(false); + } + } + + // --defsym + for (i, (name, value)) in ctx.args.defsyms.clone().iter().enumerate() { + let sym1 = ctx.get_symbol(name.as_bytes()); + match value { + DefsymValue::Symbol(target) => { + let sym2 = ctx.get_symbol(target.as_bytes()); + if ctx.symbols[sym2].file().is_none() { + error!(ctx, "--defsym: undefined symbol: {}", ctx.symbols[sym2]); + continue; + } + let sym2_esym = ctx.symbols[sym2].esym(ctx); + let obj = &mut ctx.objs[obj_id.index()]; + let mut esym = obj.base.elf_syms.at_in::(i + 1); + esym.set_type(sym2_esym.st_type()); + if E::FAMILY == Family::Ppc64V2 { + esym.st_other = (esym.st_other & 0x1f) | (sym2_esym.st_other & 0xe0); + } + obj.base.elf_syms.set_in::(i + 1, esym); + ctx.internal_esyms[i + 1] = esym; + if ctx.symbols[sym1].file() == Some(FileId::Obj(obj_id)) { + ctx.symbols[sym1].set_esym(&esym); + } + if ctx.symbols[sym2].is_absolute() { + ctx.symbols[sym1].clear_origin(); + } + } + DefsymValue::Addr(_) => ctx.symbols[sym1].clear_origin(), + } + } +} + +pub fn apply_section_align(ctx: &mut Context) { + if ctx.args.section_align.is_empty() { + return; + } + for osec in &mut ctx.output_sections { + if let Some(&align) = ctx + .args + .section_align + .get(&*String::from_utf8_lossy(osec.hdr.name)) + { + osec.hdr.shdr.sh_addralign = align; + } + } +} + +pub fn check_cet_errors(ctx: &Context) { + let warning = ctx.args.z_cet_report == CetReportKind::Warning; + let has_feature = |file: &ObjectFile, feature: u32| { + file.gnu_properties + .get(&GNU_PROPERTY_X86_FEATURE_1_AND) + .is_some_and(|v| v & feature != 0) + }; + for (i, file) in ctx.objs.iter().enumerate() { + if ctx.is_internal(ObjId(i as u32)) { + continue; + } + for (feature, name) in [ + (GNU_PROPERTY_X86_FEATURE_1_IBT, "IBT"), + (GNU_PROPERTY_X86_FEATURE_1_SHSTK, "SHSTK"), + ] { + if !has_feature(file, feature) { + if warning { + warn!( + ctx, + "{file}: -cet-report=warning: missing GNU_PROPERTY_X86_FEATURE_1_{name}" + ); + } else { + error!( + ctx, + "{file}: -cet-report=error: missing GNU_PROPERTY_X86_FEATURE_1_{name}" + ); + } + } + } + } +} + +pub fn print_dependencies(ctx: &Context) { + out!( + ctx, + "# This is an output of the mold linker's --print-dependencies option.\n\ + #\n\ + # Each line consists of 4 fields, , , and\n\ + # , separated by tab characters. It indicates that depends\n\ + # on to use . is either \"u\" or \"w\" for\n\ + # regular undefined or weak undefined, respectively.\n\ + #\n\ + # If you want to obtain dependency information per function granularity,\n\ + # compile source files with the -ffunction-sections compiler flag." + ); + + let println = |src: &dyn std::fmt::Display, sym: &Symbol, esym: &ElfSym| { + let kind = if esym.is_weak() { 'w' } else { 'u' }; + match sym.input_section() { + Some(sec) => out!(ctx, "{src}\t{}\t{kind}\t{sym}", ctx.section_display(sec)), + None => out!( + ctx, + "{src}\t{}\t{kind}\t{sym}", + ctx.file_display(sym.file().unwrap()) + ), + } + }; + + for (fi, file) in ctx.objs.iter().enumerate() { + for isec in file.input_sections() { + let mut visited: HashSet = HashSet::new(); + for r in isec.rels::(file) { + if r.r_type == R_NONE || file.base.elf_syms.len() <= r.r_sym as usize { + continue; + } + let esym = &file.base.elf_syms.at_in::(r.r_sym as usize); + let id = file.base.symbols[r.r_sym as usize]; + let sym = &ctx.symbols[id]; + if esym.is_undef() + && sym.file().is_some() + && sym.file() != Some(FileId::Obj(ObjId(fi as u32))) + && visited.insert(id) + { + println(&isec.display(file), sym, esym); + } + } + } + } + for (fi, file) in ctx.dsos.iter().enumerate() { + for i in 0..file.base.elf_syms.len() { + let esym = &file.base.elf_syms.at_in::(i); + let sym = &ctx.symbols[file.base.symbols[i]]; + if esym.is_undef() + && sym.file().is_some() + && sym.file() != Some(FileId::Dso(crate::input_files::DsoId(fi as u32))) + { + println(file, sym, esym); + } + } + } +} + +fn create_response_file(ctx: &Context) -> String { + let cwd = std::env::current_dir() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut out = format!("-C {}\n", cwd.strip_prefix('/').unwrap_or(&cwd)); + if cwd != "/" { + out.push_str("--chroot .."); + let depth = cwd.matches('/').count(); + for _ in 1..depth { + out.push_str("/.."); + } + out.push('\n'); + } + for arg in &ctx.cmdline_args[1..] { + if arg != "-repro" && arg != "--repro" { + out.push_str(arg); + out.push('\n'); + } + } + out +} + +pub fn write_repro_file(ctx: &Context) { + let _t = ctx.timer("write_repro_file"); + let path = format!("{}.repro.tar", ctx.args.output); + let basedir = format!("{}.repro", path_filename(&ctx.args.output)); + let mut tar = crate::util::tar::TarWriter::open(&path, &basedir) + .unwrap_or_else(|e| fatal!(ctx, "cannot open {path}: {e}")); + + let write = |tar: &mut crate::util::tar::TarWriter, name: &str, data: &[u8]| { + tar.append(name, data) + .unwrap_or_else(|e| fatal!(ctx, "{path}: write failed: {e}")); + }; + write( + &mut tar, + "response.txt", + create_response_file(ctx).as_bytes(), + ); + write( + &mut tar, + "version.txt", + format!("{}\n", crate::args::VERSION).as_bytes(), + ); + + let mut seen: HashSet = HashSet::new(); + let files: Vec<&crate::mapped_file::MappedFile> = ctx + .objs + .iter() + .filter_map(|f| f.base.mf) + .chain(ctx.dsos.iter().filter_map(|f| f.base.mf)) + .collect(); + for mf in files { + let top = mf.parent.unwrap_or(mf); + if seen.insert(top.name.clone()) { + let abs = std::fs::canonicalize(&top.name) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or(top.name.clone()); + write(&mut tar, &abs, top.data()); + } + } +} + +pub fn check_duplicate_symbols(ctx: &Context) { + let _t = ctx.timer("check_duplicate_symbols"); + ctx.objs.par_iter().enumerate().for_each(|(fi, file)| { + if !file.base.is_reachable() { + return; + } + let file_id = FileId::Obj(ObjId(fi as u32)); + for i in file.base.first_global..file.base.elf_syms.len() { + let esym = &file.base.elf_syms.at_in::(i); + let sym = &ctx.symbols[file.base.symbols[i]]; + + // Skip if our symbol is undefined or weak. + let Some(owner) = sym.file() else { continue }; + if owner == file_id + || ctx.internal_obj.map(FileId::Obj) == Some(owner) + || esym.is_undef() + || esym.is_common() + || esym.st_bind() == STB_WEAK + { + continue; + } + // Skip if our symbol is in a dead section, most likely due to + // COMDAT deduplication. + if !esym.is_abs() { + match file.symbol_section(i) { + Some(isec) if isec.is_alive() => {} + _ => continue, + } + } + if file.is_lto_input && file.lto_comdat_discarded[i] { + continue; + } + // The LTO backend sorts out conflicts between IR and regular + // objects itself; only IR-vs-IR duplicates are caught here. + if let FileId::Obj(o) = owner { + if ctx.objs[o.index()].is_lto_input != file.is_lto_input { + continue; + } + } + error!( + ctx, + "duplicate symbol: {file}: {}: {sym}", + ctx.file_display(owner) + ); + } + }); + ctx.checkpoint(); +} + +/// Exporting both `foo@@VER` and `foo@VER` would leave the loader to pick +/// between two definitions of the same versioned name. +pub fn check_symbol_version_conflicts(ctx: &Context) { + if ctx.dynamic.is_none() || ctx.args.allow_multiple_definition { + return; + } + let _t = ctx.timer("check_symbol_version_conflicts"); + for &id in ctx.dynsym.symbols.iter().skip(1).flatten() { + let sym = &ctx.symbols[id]; + let Some(FileId::Obj(obj)) = sym.file() else { + continue; + }; + if sym.is_weak() + || sym.ver_idx as u32 == VER_NDX_UNSPECIFIED + || sym.ver_idx as u32 & VERSYM_HIDDEN == 0 + { + continue; + } + let Some(id2) = ctx.symbols.lookup(sym.name()) else { + continue; + }; + if id2 == id { + continue; + } + let sym2 = &ctx.symbols[id2]; + if let Some(FileId::Obj(_)) = sym2.file() { + if !sym2.is_weak() && sym2.ver_idx as u32 == (sym.ver_idx as u32 & !VERSYM_HIDDEN) { + let file = &ctx.objs[obj.index()]; + error!( + ctx, + "duplicate symbol: {file}: {}: {}", + ctx.file_display(sym2.file().unwrap()), + crate::util::display(file.base.symbol_name_in::(sym.sym_idx as usize)) + ); + } + } + } + ctx.checkpoint(); +} + +/// Converts allocated data sections containing only zeros into BSS. +pub fn convert_zero_to_bss(ctx: &mut Context) { + let _t = ctx.timer("convert_zero_to_bss"); + ctx.objs.par_iter_mut().for_each(|file| { + if !file.base.is_reachable() { + return; + } + let sections: Vec = file + .input_sections() + .filter(|isec| { + let flags = isec.sh_flags as u32; + isec.is_alive() + && isec.sh_type(file) == SHT_PROGBITS + && flags & SHF_ALLOC != 0 + && flags & SHF_WRITE != 0 + && flags & SHF_EXECINSTR == 0 + && isec.rels::(file).is_empty() + && !isec.contents().is_empty() + && isec.contents().iter().all(|&b| b == 0) + }) + .map(|isec| isec.shndx) + .collect(); + for shndx in sections { + let isec = file.section_mut(shndx as usize).unwrap(); + isec.set_nobits(); + isec.clear_contents(); + } + }); +} + +fn has_dso_definition(ctx: &Context, id: SymbolId) -> bool { + ctx.dsos.iter().any(|dso| { + dso.base + .symbols + .iter() + .zip(dso.base.elf_syms.iter()) + .any(|(&s, esym)| s == id && !esym.is_undef()) + }) +} + +/// With --no-allow-shlib-undefined, reports unresolved symbols in shared +/// libraries, which the dynamic linker would otherwise report at runtime. +pub fn check_shlib_undefined(ctx: &mut Context) { + let _t = ctx.timer("check_shlib_undefined"); + + // Skip the test unless we have the complete set of shared libraries: + // a missing one might define the symbol. + let complete = ctx.dsos.iter().all(|dso| { + dso.dt_needed::(&ctx.diag) + .iter() + .all(|needed| ctx.dso_sonames.contains(&*String::from_utf8_lossy(needed))) + }); + + if complete { + ctx.dsos.par_iter().for_each(|file| { + for i in 0..file.base.elf_syms.len() { + let esym = &file.base.elf_syms.at_in::(i); + let id = file.base.symbols[i]; + let sym = &ctx.symbols[id]; + let is_sparc_register = E::IS_SPARC && esym.st_type() == STT_SPARC_REGISTER; + let defined = match sym.file() { + Some(_) => sym.visibility() != STV_HIDDEN, + None => ctx.dropped_dso_definitions.contains(&id), + }; + if esym.is_undef() + && !esym.is_weak() + && !is_sparc_register + && !defined + && !has_dso_definition(ctx, id) + { + error!( + ctx, + "{file}: --no-allow-shlib-undefined: undefined symbol: {sym}" + ); + } + } + }); + } + + // DSOs not referenced by any object were kept only for this pass. + for file in &ctx.dsos { + file.base.set_reachable(!file.base.as_needed); + } + let Context { + objs, + dsos, + symbols, + .. + } = ctx; + objs.par_iter().for_each(|file| { + for &id in file.base.global_symbols() { + if let Some(FileId::Dso(dso)) = symbols[id].file() { + dsos[dso.index()].base.set_reachable(true); + } + } + }); + remove_unreachable_dsos(ctx); +} + +/// Drops DSOs that are no longer needed, renumbering the rest. +pub fn remove_unreachable_dsos(ctx: &mut Context) { + if ctx.dsos.iter().all(|d| d.base.is_reachable()) { + return; + } + let dsos = std::mem::take(&mut ctx.dsos); + let mut map: Vec> = vec![None; dsos.len()]; + let mut removed = Vec::new(); + for (i, file) in dsos.into_iter().enumerate() { + if file.base.is_reachable() { + map[i] = Some(crate::input_files::DsoId(ctx.dsos.len() as u32)); + ctx.dsos.push(file); + } else { + removed.push(file); + } + } + // SharedFiles have the same arena lifetime in C++ mold. + ctx.discarded_dsos.extend(removed); + let dropped: Vec = ctx + .symbols + .as_mut_slice() + .par_iter_mut() + .enumerate() + .filter_map(|(i, sym)| { + let Some(FileId::Dso(d)) = sym.file() else { + return None; + }; + match map[d.index()] { + Some(new) => { + sym.set_file(FileId::Dso(new)); + None + } + None => { + clear_symbol(sym); + Some(SymbolId(i as u32)) + } + } + }) + .collect(); + ctx.dropped_dso_definitions.extend(dropped); + for slot in &mut ctx.file_by_priority { + if let Some(FileId::Dso(d)) = *slot { + *slot = map[d.index()].map(FileId::Dso); + } + } +} + +pub fn check_symbol_types(ctx: &Context) { + let _t = ctx.timer("check_symbol_types"); + let canonicalize = |ty: u32| match ty { + STT_GNU_IFUNC => STT_FUNC, + STT_COMMON => STT_OBJECT, + ty => ty, + }; + let check = |file: &dyn std::fmt::Display, file_id: FileId, sym: &Symbol, esym2: &ElfSym| { + let esym1 = &sym.esym(ctx); + if let Some(owner) = sym.file() { + if owner != file_id + && esym1.st_type() != STT_NOTYPE + && esym2.st_type() != STT_NOTYPE + && canonicalize(esym1.st_type()) != canonicalize(esym2.st_type()) + { + warn!( + ctx, + "symbol type mismatch: {sym}\n>>> defined in {} as {}\n>>> defined in {file} as {}", + ctx.file_display(owner), + stt_to_string(esym1.st_type()), + stt_to_string(esym2.st_type()) + ); + } + } + }; + + ctx.objs.par_iter().enumerate().for_each(|(fi, file)| { + let id = FileId::Obj(ObjId(fi as u32)); + for i in file.base.first_global..file.base.elf_syms.len() { + let sym = &ctx.symbols[file.base.symbols[i]]; + if sym.file().is_some() && sym.file() != Some(id) { + check(file, id, sym, &file.base.elf_syms.at_in::(i)); + } + } + }); + ctx.dsos.par_iter().enumerate().for_each(|(fi, file)| { + let id = FileId::Dso(crate::input_files::DsoId(fi as u32)); + for i in 0..file.base.elf_syms.len() { + let sym = &ctx.symbols[file.base.symbols[i]]; + if sym.file().is_some() && sym.file() != Some(id) { + check(file, id, sym, &file.base.elf_syms.at_in::(i)); + } + let id2 = file.symbols2[i]; + if id2 != SymbolId::NONE { + let sym = &ctx.symbols[id2]; + if sym.file().is_some() && sym.file() != Some(id) { + check(file, id, sym, &file.base.elf_syms.at_in::(i)); + } + } + } + }); +} + +/// The `.N` suffix of an init/fini section name, or 65536. +fn init_fini_priority(name: &[u8]) -> i64 { + numeric_suffix(name).unwrap_or(65536) +} + +fn numeric_suffix(name: &[u8]) -> Option { + let pos = name.iter().rposition(|&b| b == b'.')?; + let digits = &name[pos + 1..]; + if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) { + return None; + } + std::str::from_utf8(digits).ok()?.parse().ok() +} + +fn ctor_dtor_priority(ctx: &Context, id: InputSectionId) -> i64 { + // crtbegin.o and crtend.o contain marker symbols such as __CTOR_LIST__ + // and must be at the beginning or end of the section. + let isec = ctx.input_section(id); + let file = &ctx.objs[isec.file.index()]; + let filename = &file.base.filename; + if filename.contains("crtbegin") { + return -2; + } + if filename.contains("crtend") { + return 65536; + } + numeric_suffix(isec.name(file)).unwrap_or(-1) +} + +pub fn sort_init_fini(ctx: &mut Context) { + let _t = ctx.timer("sort_init_fini"); + for i in 0..ctx.output_sections.len() { + let name = ctx.output_sections[i].hdr.name; + if name != b".init_array" && name != b".preinit_array" && name != b".fini_array" { + continue; + } + let mut members = std::mem::take(&mut ctx.output_sections[i].members); + if ctx.args.shuffle_sections == ShuffleSectionsKind::Reverse { + members.reverse(); + } + let mut entries: Vec<(InputSectionId, i64)> = members + .iter() + .map(|&id| { + let isec = ctx.input_section(id); + let name = isec.name(&ctx.objs[isec.file.index()]); + let prio = if name.starts_with(b".ctors") || name.starts_with(b".dtors") { + 65535 - ctor_dtor_priority(ctx, id) + } else { + init_fini_priority(name) + }; + (id, prio) + }) + .collect(); + entries.sort_by_key(|e| e.1); + ctx.output_sections[i].members = entries.into_iter().map(|e| e.0).collect(); + } +} + +pub fn sort_ctor_dtor(ctx: &mut Context) { + let _t = ctx.timer("sort_ctor_dtor"); + for i in 0..ctx.output_sections.len() { + let name = ctx.output_sections[i].hdr.name; + if name != b".ctors" && name != b".dtors" { + continue; + } + let mut members = std::mem::take(&mut ctx.output_sections[i].members); + if ctx.args.shuffle_sections != ShuffleSectionsKind::Reverse { + members.reverse(); + } + let mut entries: Vec<(InputSectionId, i64)> = members + .iter() + .map(|&id| (id, ctor_dtor_priority(ctx, id))) + .collect(); + entries.sort_by_key(|e| e.1); + ctx.output_sections[i].members = entries.into_iter().map(|e| e.0).collect(); + } +} + +/// Places DWARF32 input sections before DWARF64 ones in large debug +/// sections, so that 32-bit references between debug sections don't +/// overflow until the DWARF32 part alone exceeds 4 GiB. +pub fn sort_debug_info_sections(ctx: &mut Context) { + let _t = ctx.timer("sort_debug_info_sections"); + let is_in_test = std::env::var("MOLD_DEBUG").is_ok_and(|v| !v.is_empty()); + + let vec1: Vec = (0..ctx.output_sections.len()) + .map(|i| OutputSectionId::new(i as u32)) + .filter(|&id| { + let osec = &ctx.output_sections[id.index()]; + !osec.hdr.is_alloc() + && osec.hdr.name.starts_with(b".debug_") + && (osec.hdr.shdr.sh_size >= u32::MAX as u64 || is_in_test) + }) + .collect(); + let vec2: Vec = (0..ctx.merged_sections.len()) + .map(|i| crate::chunks::merged::MergedSectionId(i as u32)) + .filter(|&id| { + let msec = &ctx.merged_sections[id.index()]; + !msec.is_alloc() + && msec.hdr.name.starts_with(b".debug_") + && (msec.hdr.shdr.sh_size >= u32::MAX as u64 || is_in_test) + }) + .collect(); + if vec1.is_empty() && vec2.is_empty() { + return; + } + + { + let Context { objs, diag, .. } = ctx; + objs.par_iter_mut().for_each(|file| { + file.is_dwarf32 = !file.debug_info_sections.is_empty() && file.is_dwarf32::(diag); + }); + } + + let has_dwarf32 = ctx.objs.iter().any(|f| f.is_dwarf32); + let has_dwarf64 = ctx + .objs + .iter() + .any(|f| !f.is_dwarf32 && !f.debug_info_sections.is_empty()); + if !has_dwarf32 || !has_dwarf64 { + return; + } + + for id in vec1 { + let section_arena = &ctx.section_arena; + let objs = &ctx.objs; + let osec = &mut ctx.output_sections[id.index()]; + let (a, b): (Vec, Vec) = osec + .members + .iter() + .partition(|&&m| objs[section_arena.section(m).file.index()].is_dwarf32); + osec.members = a.into_iter().chain(b).collect(); + chunks::compute_section_size(ctx, ChunkId::Output(id)); + } + + for id in &vec2 { + let msec = &ctx.merged_sections[id.index()]; + for file in ctx.objs.iter().filter(|f| f.is_dwarf32) { + for m in file.mergeable_sections().filter(|m| m.parent == *id) { + for &frag in &m.fragments { + msec.fragments.get(frag).set_32bit(); + } + } + } + } + for id in vec2 { + chunks::compute_section_size(ctx, ChunkId::Merged(id)); + } +} + +/// `.ctors`/`.dtors` are executed in the opposite order to +/// `.init_array`/`.fini_array`, so their contents are reversed when they +/// are placed there. +pub fn fixup_ctors_in_init_array(ctx: &mut Context) { + let _t = ctx.timer("fixup_ctors_in_init_array"); + let word = E::WORD_SIZE; + + for (osec_name, prefix) in [ + (&b".init_array"[..], &b".ctors"[..]), + (&b".fini_array"[..], &b".dtors"[..]), + ] { + let Some(ChunkId::Output(id)) = ctx.find_chunk_by_name(osec_name) else { + continue; + }; + let members = ctx.output_sections[id.index()].members.clone(); + for m in members { + let section_ref = ctx.input_section(m).section_ref(); + if !ctx + .input_section(m) + .name(&ctx.objs[section_ref.file.index()]) + .starts_with(prefix) + { + continue; + } + let diag = &ctx.diag; + let file = &ctx.objs[section_ref.file.index()]; + let isec = ctx.input_section(m); + if !isec.sh_size.is_multiple_of(word as u64) { + fatal!(diag, "{}: section corrupted", isec.display(file)); + } + let mut contents = isec.contents().to_vec(); + let n = contents.len() / word; + for i in 0..n / 2 { + let (a, b) = (i * word, (n - 1 - i) * word); + for k in 0..word { + contents.swap(a + k, b + k); + } + } + let size = isec.sh_size; + let mut rels: Vec = isec.rels::(file).iter().collect(); + for r in &mut rels { + r.r_offset = size - r.r_offset - word as u64; + } + rels.sort_by_key(|r| r.r_offset); + let file = &mut ctx.objs[section_ref.file.index()]; + file.section_mut(section_ref.shndx as usize) + .unwrap() + .set_contents(leak_bytes(contents)); + file.rels_mut::(section_ref.shndx).set_all(&rels); + } + } +} + +/// Shuffles members with a xorshift generator and Fisher-Yates, which +/// are stable across platforms unlike the standard library's. +fn shuffle(vec: &mut [InputSectionId], mut seed: u64) { + if vec.is_empty() { + return; + } + let mut rand = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for i in 0..vec.len() - 1 { + let j = i + (rand() % (vec.len() - i) as u64) as usize; + vec.swap(i, j); + } +} + +pub fn shuffle_sections(ctx: &mut Context) { + let _t = ctx.timer("shuffle_sections"); + let is_eligible = |osec: &OutputSection| { + let name = osec.hdr.name; + osec.hdr.is_alloc() + && name != b".init" + && name != b".fini" + && name != b".ctors" + && name != b".dtors" + && name != b".init_array" + && name != b".preinit_array" + && name != b".fini_array" + }; + let kind = ctx.args.shuffle_sections; + let seed = ctx.args.shuffle_sections_seed; + ctx.output_sections + .par_iter_mut() + .filter(|o| is_eligible(o)) + .for_each(|osec| match kind { + ShuffleSectionsKind::Shuffle => { + let s = seed.wrapping_add(xxhash_rust::xxh3::xxh3_64(osec.hdr.name)); + shuffle(&mut osec.members, s); + } + ShuffleSectionsKind::Reverse => osec.members.reverse(), + ShuffleSectionsKind::None => {} + }); +} + +pub fn add_dynamic_strings(ctx: &mut Context) { + for i in 0..ctx.dsos.len() { + let audit = ctx.dsos[i].dt_audit::(&ctx.diag); + if !audit.is_empty() { + if !ctx.args.depaudit.is_empty() { + ctx.args.depaudit.push(':'); + } + ctx.args.depaudit.push_str(&String::from_utf8_lossy(audit)); + } + } + let mut strings: Vec> = ctx + .dsos + .iter() + .map(|d| d.soname.clone().into_bytes()) + .collect(); + strings.extend(ctx.args.auxiliary.iter().map(|s| s.clone().into_bytes())); + strings.extend(ctx.args.filter.iter().map(|s| s.clone().into_bytes())); + strings.push(ctx.args.audit.clone().into_bytes()); + strings.push(ctx.args.depaudit.clone().into_bytes()); + strings.push(ctx.args.rpaths.clone().into_bytes()); + strings.push(ctx.args.soname.clone().into_bytes()); + for s in strings { + if !s.is_empty() { + ctx.dynstr.add_string(&s); + } + } +} + +pub fn compute_section_sizes(ctx: &mut Context) { + let _t = ctx.timer("compute_section_sizes"); + + // On a target with range extension thunks, an executable section gets + // its thunks and its layout at the same time. + let needs_thunks = |ctx: &Context, id: ChunkId| match id { + ChunkId::Output(osec) => { + E::NEEDS_THUNK + && !ctx.args.relocatable + && ctx.output_sections[osec.index()].hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0 + } + _ => false, + }; + + // create_range_extension_thunks is not thread-safe + for id in ctx.chunks.clone() { + if let ChunkId::Output(osec) = id { + if needs_thunks(ctx, id) { + crate::thunks::create_range_extension_thunks(ctx, osec); + } + } + } + + // Merged sections without SHF_ALLOC are resolved lazily here. C++ mold + // processes all chunks in one parallel loop, so resolve their independent + // parent sections concurrently too. + { + let mut members = merged_resolve_members(&mut ctx.objs, ctx.merged_sections.len()); + crate::chunks::merged::resolve_sections::( + &mut ctx.merged_sections, + &mut members, + crate::chunks::merged::ResolveOptions { + allocated_only: false, + gc_sections: ctx.args.gc_sections, + diag: &ctx.diag, + comment: ctx.comment, + cmdline_args: &ctx.cmdline_args, + timers: &ctx.timers, + }, + ); + } + + // Output sections are laid out in parallel with each other, and each + // in parallel over its members; the synthesized chunks follow. + let sizes: Vec<(OutputSectionId, u64)> = { + let ctx: &Context = ctx; + ctx.chunks + .par_iter() + .filter_map(|&id| match id { + ChunkId::Output(osec) if !needs_thunks(ctx, id) => { + Some((osec, chunks::output_section::layout(ctx, osec))) + } + _ => None, + }) + .collect() + }; + for (osec, size) in sizes { + ctx.output_sections[osec.index()].hdr.shdr.sh_size = size; + } + + // Every merged section now owns all the mutable state needed for its + // layout, just as each C++ Chunk does in the parallel chunk loop. + ctx.merged_sections + .par_iter_mut() + .for_each(crate::chunks::merged::layout); + + for id in ctx.chunks.clone() { + match id { + ChunkId::Output(_) => {} + ChunkId::Merged(_) => {} + _ => chunks::compute_section_size(ctx, id), + } + } +} + +/// Attaches every unresolved symbol to a file: a shared library import, +/// or an absolute symbol with value 0. Errors for referenced ones are +/// reported by relocation scanning, so that only actually used symbols +/// are reported. +pub fn claim_unresolved_symbols(ctx: &mut Context) { + let _t = ctx.timer("claim_unresolved_symbols"); + + // Nearly all references are to defined symbols, which are filtered + // out first. + let candidates: Vec<(usize, usize)> = { + let ctx_ref: &Context = ctx; + ctx_ref + .objs + .par_iter() + .enumerate() + .flat_map_iter(|(fi, file)| { + let internal = ctx_ref.is_internal(ObjId(fi as u32)); + (file.base.first_global..file.base.elf_syms.len()) + .filter(move |&i| { + !internal + && file.base.elf_syms.at_in::(i).is_undef() + && ctx_ref.symbols[file.base.symbols[i]].file().is_none() + }) + .map(move |i| (fi, i)) + }) + .collect() + }; + + for (fi, i) in candidates { + let file = &ctx.objs[fi]; + let esym = file.base.elf_syms.at_in::(i); + let id = file.base.symbols[i]; + let priority = file.base.priority; + let file_id = FileId::Obj(ObjId(fi as u32)); + + { + let sym = &ctx.symbols[id]; + if let Some(owner) = sym.file() { + if !sym.is_undef() || ctx.file(owner).priority <= priority { + continue; + } + } + } + + let claim = |ctx: &mut Context, is_imported: bool| { + let sym = &ctx.symbols[id]; + if sym.is_traced() { + out!( + ctx, + "trace-symbol: {}: unresolved{} symbol {sym}", + ctx.objs[fi], + if esym.is_weak() { " weak" } else { "" } + ); + } + let default_version = ctx.default_version; + let is_rust = ctx.objs[fi].is_rust_obj; + let sym = &mut ctx.symbols[id]; + sym.set_file(file_id); + sym.clear_origin(); + sym.value = 0; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + sym.set_rust(is_rust); + sym.set_weak(false); + sym.set_imported(is_imported); + sym.set_exported(false); + sym.ver_idx = if is_imported { + VER_NDX_UNSPECIFIED as u16 + } else { + default_version + }; + }; + + let visibility = ctx.symbols[id].visibility(); + if esym.is_undef_weak() { + // Weak undefined symbols become dynamic symbols only in a DSO + // by default: an executable might need a copy relocation for a + // data symbol, whose size is unknown for an unclaimed symbol. + // Otherwise they become absolute symbols with value 0. + claim( + ctx, + ctx.args.z_dynamic_undefined_weak && visibility != STV_HIDDEN, + ); + continue; + } + // Undefined symbols in a shared object are promoted to dynamic + // symbols unless `-z defs` is given. + if ctx.args.shared + && visibility != STV_HIDDEN + && ctx.args.unresolved_symbols != UnresolvedKind::Error + { + claim(ctx, true); + continue; + } + claim(ctx, false); + } +} + +pub fn scan_relocations(ctx: &mut Context) { + let _t = ctx.timer("scan_relocations"); + + { + let ctx_ref: &Context = ctx; + ctx_ref + .objs + .par_iter() + .for_each(|file| file.scan_relocations(ctx_ref)); + } + ctx.checkpoint(); + + // Word-size absolute relocations are handled separately since they + // can be promoted to dynamic relocations. + let results: Vec<( + OutputSectionId, + Vec, + Vec, + )> = { + let ctx_ref: &Context = ctx; + (0..ctx_ref.output_sections.len()) + .into_par_iter() + .map(|i| OutputSectionId::new(i as u32)) + .filter(|&id| ctx_ref.output_sections[id.index()].hdr.is_alloc()) + .map(|id| { + let (abs_rels, offsets) = + crate::chunks::output_section::scan_abs_relocations(ctx_ref, id); + (id, abs_rels, offsets) + }) + .collect() + }; + for (id, abs_rels, offsets) in results { + let osec = &mut ctx.output_sections[id.index()]; + osec.abs_rels = abs_rels; + osec.dynrel_offsets = offsets; + } + ctx.checkpoint(); + + // Gather the dynamic symbols. + let syms: Vec = { + let ctx_ref: &Context = ctx; + let objs: Vec> = ctx_ref + .objs + .par_iter() + .enumerate() + .map(|(i, file)| { + let id = FileId::Obj(ObjId(i as u32)); + file.base + .symbols + .iter() + .copied() + .filter(|&s| { + let sym = &ctx_ref.symbols[s]; + sym.file() == Some(id) + && (sym.flags() != 0 || sym.is_imported() || sym.is_exported()) + }) + .collect() + }) + .collect(); + let dsos: Vec> = ctx_ref + .dsos + .par_iter() + .enumerate() + .map(|(i, file)| { + let id = FileId::Dso(crate::input_files::DsoId(i as u32)); + file.base + .symbols + .iter() + .copied() + .filter(|&s| { + let sym = &ctx_ref.symbols[s]; + sym.file() == Some(id) + && (sym.flags() != 0 || sym.is_imported() || sym.is_exported()) + }) + .collect() + }) + .collect(); + let mut syms = Vec::with_capacity(objs.iter().chain(&dsos).map(Vec::len).sum()); + syms.extend(objs.into_iter().chain(dsos).flatten()); + syms + }; + + if ctx.needs_tlsld.load(Ordering::Relaxed) { + crate::chunks::got::got::add_tlsld(ctx); + } + + // Every dynamic symbol gets its auxiliary record. The loop below + // assigns table entries in order and so runs on one thread; the + // records are allocated beforehand in the side arena. + { + let mut ids = syms.clone(); + ids.par_sort_unstable(); + ids.dedup(); + ctx.symbols.allocate_aux(&ids); + } + + // Assign entries in the various tables to each dynamic symbol. + for id in syms { + let flags = ctx.symbols[id].flags(); + let (is_imported, is_exported, ty) = { + let sym = &ctx.symbols[id]; + (sym.is_imported(), sym.is_exported(), sym.ty()) + }; + + if is_imported || is_exported { + symtab::dynsym::add_symbol(ctx, id); + } + if flags & NEEDS_GOT != 0 { + crate::chunks::got::got::add_got_symbol(ctx, id); + } + if flags & NEEDS_CANONICAL != 0 && ty == STT_FUNC { + // A canonical PLT must be visible from DSOs, and can't use + // .plt.got because .plt.got and .got would then refer to each + // other in an infinite loop. + let sym = &mut ctx.symbols[id]; + sym.set_canonical(true); + sym.set_exported(true); + crate::chunks::got::plt::add_symbol(ctx, id); + } else if flags & NEEDS_PLT != 0 { + if flags & NEEDS_GOT != 0 { + crate::chunks::got::pltgot::add_symbol(ctx, id); + } else { + crate::chunks::got::plt::add_symbol(ctx, id); + } + } + if flags & NEEDS_GOTTP != 0 { + crate::chunks::got::got::add_gottp_symbol(ctx, id); + } + if flags & NEEDS_TLSGD != 0 { + crate::chunks::got::got::add_tlsgd_symbol(ctx, id); + } + if flags & NEEDS_TLSDESC != 0 { + crate::chunks::got::got::add_tlsdesc_symbol(ctx, id); + } + if flags & NEEDS_CANONICAL != 0 && ty != STT_FUNC { + let relro = ctx.args.z_relro + && match ctx.symbols[id].file() { + Some(FileId::Dso(dso)) => { + ctx.dsos[dso.index()].is_readonly::(&ctx.symbols[id]) + } + _ => false, + }; + misc::copyrel::add_symbol(ctx, relro, id); + } + if E::FAMILY == Family::Ppc64V1 && flags & NEEDS_PPC_OPD != 0 { + crate::chunks::opd::add_symbol(ctx, id); + } + ctx.symbols[id].clear_flags(); + } + + if ctx.has_textrel.load(Ordering::Relaxed) && ctx.args.warn_textrel { + warn!(ctx, "creating a DT_TEXTREL in an output file"); + } +} + +/// An imported symbol is weak in `.dynsym` only if all references to it +/// are weak. +pub fn compute_imported_symbol_weakness(ctx: &mut Context) { + let _t = ctx.timer("compute_imported_symbol_weakness"); + let strong: Vec = ctx + .objs + .par_iter() + .flat_map_iter(|file| { + (file.base.first_global..file.base.elf_syms.len()).filter_map(|i| { + let esym = &file.base.elf_syms.at_in::(i); + let id = file.base.symbols[i]; + (esym.is_undef() + && !esym.is_weak() + && matches!(ctx.symbols[id].file(), Some(FileId::Dso(_)))) + .then_some(id) + }) + }) + .collect(); + for id in strong { + ctx.symbols[id].set_weak(false); + } +} + +/// Reports all undefined symbols, grouped by symbol. +pub fn report_undef_errors(ctx: &Context) { + const MAX_ERRORS: usize = 3; + if ctx.args.unresolved_symbols == UnresolvedKind::Ignore { + return; + } + let mut errors: Vec<(SymbolId, Vec)> = + std::mem::take(&mut *ctx.undef_errors.lock().unwrap()) + .into_iter() + .collect(); + errors.sort_by_key(|e| e.0); + + for (id, messages) in errors { + let sym = &ctx.symbols[id]; + let mut msg = format!("undefined symbol: {sym}\n"); + for m in messages.iter().take(MAX_ERRORS) { + msg.push_str(m); + } + if messages.len() > MAX_ERRORS { + msg.push_str(&format!( + ">>> referenced {} more times\n", + messages.len() - MAX_ERRORS + )); + } + msg.pop(); + if ctx.args.unresolved_symbols == UnresolvedKind::Error { + error!(ctx, "{msg}"); + } else { + warn!(ctx, "{msg}"); + } + } + ctx.checkpoint(); +} + +pub fn create_reloc_sections(ctx: &mut Context) { + let _t = ctx.timer("create_reloc_sections"); + let ids: Vec = ctx + .chunks + .iter() + .filter_map(|c| c.as_output_section()) + .collect(); + let secs: Vec = { + let ctx_ref: &Context = ctx; + ids.par_iter() + .map(|&id| misc::reloc::new(ctx_ref, id)) + .collect() + }; + for (id, sec) in ids.into_iter().zip(secs) { + ctx.reloc_sections.push(sec); + let idx = ctx.reloc_sections.len() as u32 - 1; + ctx.output_sections[id.index()].reloc_sec = Some(idx); + ctx.chunks.push(ChunkId::Reloc(idx)); + } +} + +pub fn sort_dynsyms(ctx: &mut Context) { + let _t = ctx.timer("sort_dynsyms"); + if ctx.dynsym.symbols.is_empty() { + return; + } + let mut syms: Vec = ctx.dynsym.symbols[1..].iter().flatten().copied().collect(); + + // In any symtab, local symbols must precede global symbols. + let (locals, mut globals): (Vec, Vec) = + syms.iter().partition(|&&id| ctx.symbols[id].is_local(ctx)); + let num_locals = locals.len(); + + // .gnu.hash imposes more restrictions on the order of the symbols in + // .dynsym. + if let Some(gnu_hash) = &mut ctx.gnu_hash { + let (unexported, mut exported): (Vec, Vec) = globals + .iter() + .partition(|&&id| !ctx.symbols[id].is_exported()); + + // Count the number of exported symbols to compute the size of .gnu.hash. + let num_exported = exported.len() as u32; + let num_buckets = num_exported / GnuHashSection::LOAD_FACTOR + 1; + let symbols = &mut ctx.symbols; + // SAFETY: .dynsym contains each symbol at most once, and exported is a + // subset of it. Every dynamic symbol already has an auxiliary record. + unsafe { + symbols.par_for_each_aux_mut(&exported, |_, sym, aux| { + aux.djb_hash = symtab::djb_hash(sym.name()); + }); + } + exported.par_sort_unstable_by(|&a, &b| { + let a = &symbols[a]; + let b = &symbols[b]; + (a.aux(symbols).unwrap().djb_hash % num_buckets, a.name()) + .cmp(&(b.aux(symbols).unwrap().djb_hash % num_buckets, b.name())) + }); + gnu_hash.num_buckets = num_buckets; + gnu_hash.num_exported = num_exported; + globals = unexported.into_iter().chain(exported).collect(); + } + + syms = locals.into_iter().chain(globals).collect(); + + // Compute .dynstr size + ctx.dynsym.dynstr_offset = ctx.dynstr.hdr.shdr.sh_size; + // SAFETY: .dynsym contains each symbol at most once. Every dynamic symbol + // already has an auxiliary record. + let size = unsafe { + ctx.symbols.par_sum_aux_mut(&syms, |i, sym, aux| { + aux.dynsym_idx = Some(i as u32 + 1); + sym.name().len() as u64 + 1 + }) + }; + ctx.dynstr.hdr.shdr.sh_size += size; + ctx.dynsym.symbols = std::iter::once(None) + .chain(syms.into_iter().map(Some)) + .collect(); + + // ELF's symbol table sh_info holds the offset of the first global symbol. + ctx.dynsym.hdr.shdr.sh_info = num_locals as u32 + 1; +} + +pub fn create_output_symtab(ctx: &mut Context) { + let _t = ctx.timer("compute_symtab_size"); + if E::NEEDS_THUNK { + let mut n = 0; + for osec in &mut ctx.output_sections { + for thunk in &mut osec.thunks { + thunk.name = format!("thunk{n}"); + n += 1; + } + } + } + for id in ctx.chunks.clone() { + chunks::compute_symtab_size(ctx, id); + } + + let obj_plans: Vec = { + let ctx_ref: &Context = ctx; + ctx_ref + .objs + .par_iter() + .enumerate() + .map(|(i, f)| f.plan_symtab(ctx_ref, ObjId(i as u32))) + .collect() + }; + let dso_plans: Vec = { + let ctx_ref: &Context = ctx; + ctx_ref + .dsos + .par_iter() + .enumerate() + .map(|(i, f)| f.plan_symtab(ctx_ref, crate::input_files::DsoId(i as u32))) + .collect() + }; + for (file, plan) in ctx.objs.iter_mut().zip(obj_plans) { + file.base.apply_symtab_plan(plan); + } + for (file, plan) in ctx.dsos.iter_mut().zip(dso_plans) { + file.base.apply_symtab_plan(plan); + } +} + +pub fn apply_version_script(ctx: &mut Context) { + let _t = ctx.timer("apply_version_script"); + + // Assign versions to symbols specified with `extern "C++"` or + // wildcard patterns first. + let mut matcher = Glob::new(); + let mut cpp_matcher = Glob::new(); + + // The "local:" label has a special meaning in the version script. + // It can appear in any VERSION clause, and it hides matched symbols + // unless other non-local patterns match to them. In other words, + // "local:" has lower precedence than other version definitions. + // + // If two or more non-local patterns match to the same symbol, the + // last one takes precedence. + let mut patterns: Vec = ctx.version_patterns.clone(); + let (local, other): (Vec, Vec) = patterns + .drain(..) + .partition(|p| p.ver_idx as u32 == VER_NDX_LOCAL); + patterns = local.into_iter().chain(other).collect(); + + let has_wildcard = |s: &[u8]| s.iter().any(|&c| matches!(c, b'*' | b'?' | b'[')); + for (i, v) in patterns.iter().enumerate() { + if v.is_cpp { + if !cpp_matcher.add(v.pattern, i as i64) { + fatal!( + ctx, + "invalid version pattern: {}", + crate::util::display(v.pattern) + ); + } + } else if has_wildcard(v.pattern) && !matcher.add(v.pattern, i as i64) { + fatal!( + ctx, + "invalid version pattern: {}", + crate::util::display(v.pattern) + ); + } + } + + if !matcher.is_empty() || !cpp_matcher.is_empty() { + ctx.symbols.par_for_each_global_mut(|sym| { + if !matches!(sym.file(), Some(FileId::Obj(_))) { + return; + } + + let mut m = matcher.find(sym.name()); + + // Match non-mangled symbols against the C++ pattern as well. + // Weird, but required to match other linkers' behavior. + if !cpp_matcher.is_empty() { + let demangled = crate::util::demangle::demangle_cpp(sym.name()); + let name: &[u8] = demangled.as_deref().map_or(sym.name(), str::as_bytes); + m = m.max(cpp_matcher.find(name)); + } + + if m != -1 { + sym.ver_idx = patterns[m as usize].ver_idx; + } + }); + } + + // Next, assign versions to symbols specified by exact name. + // In other words, exact matches have higher precedence over + // wildcard or `extern "C++"` patterns. + for v in &patterns { + if !v.is_cpp && !has_wildcard(v.pattern) { + let id = ctx.get_symbol(v.pattern); + let sym = &ctx.symbols[id]; + if sym.file().is_none() && !ctx.args.undefined_version { + warn!( + ctx, + "{}: cannot assign version `{}` to symbol `{sym}`: symbol not found", + v.source, + crate::util::display(v.ver_str) + ); + } + if matches!(sym.file(), Some(FileId::Obj(_))) { + ctx.symbols[id].ver_idx = v.ver_idx; + } + } + } +} + +pub fn parse_symbol_version(ctx: &mut Context) { + if !ctx.args.shared { + return; + } + let _t = ctx.timer("parse_symbol_version"); + + let verdefs: HashMap, u16> = ctx + .args + .version_definitions + .iter() + .enumerate() + .map(|(i, v)| { + ( + v.clone().into_bytes(), + i as u16 + VER_NDX_LAST_RESERVED as u16 + 1, + ) + }) + .collect(); + + for fi in 0..ctx.objs.len() { + if ctx.is_internal(ObjId(fi as u32)) { + continue; + } + let file_id = FileId::Obj(ObjId(fi as u32)); + for i in ctx.objs[fi].base.first_global..ctx.objs[fi].base.elf_syms.len() { + let file = &ctx.objs[fi]; + if !file.has_symver[i - file.base.first_global] { + continue; + } + let id = file.base.symbols[i]; + if ctx.symbols[id].file() != Some(file_id) { + continue; + } + let name = file.base.symbol_name_in::(i); + let at = crate::util::find_byte(b'@', name).unwrap(); + let mut ver = &name[at + 1..]; + let mut is_default = false; + if let Some(rest) = ver.strip_prefix(b"@") { + is_default = true; + ver = rest; + } + + // `foo@@` is the unversioned default; export it globally. + if ver.is_empty() { + ctx.symbols[id].ver_idx = VER_NDX_GLOBAL as u16; + continue; + } + let Some(&ver_idx) = verdefs.get(ver) else { + error!( + ctx, + "{}: symbol {} has undefined version {}", + ctx.objs[fi], + ctx.symbols[id], + crate::util::display(ver) + ); + continue; + }; + let ver_idx = if is_default { + ver_idx + } else { + ver_idx | VERSYM_HIDDEN as u16 + }; + ctx.symbols[id].ver_idx = ver_idx; + + // If both `foo` and `foo@VERSION` are defined, the versioned + // one hides `foo`; likewise the default one takes precedence. + let sym_name = ctx.symbols[id].name(); + if let Some(id2) = ctx.symbols.lookup(sym_name) { + if id2 != id && ctx.symbols[id2].file() == Some(file_id) { + let sym2_idx = ctx.symbols[id2].sym_idx as usize; + let file = &ctx.objs[fi]; + if !file.has_symver[sym2_idx - file.base.first_global] { + let v2 = ctx.symbols[id2].ver_idx as u32; + if v2 == ctx.default_version as u32 + || (v2 & !VERSYM_HIDDEN) == (ver_idx as u32 & !VERSYM_HIDDEN) + { + ctx.symbols[id2].ver_idx = VER_NDX_LOCAL as u16; + } + } + } + } + } + } +} + +fn should_export(ctx: &Context, sym: &Symbol) -> bool { + if sym.visibility() == STV_HIDDEN { + return false; + } + match sym.ver_idx as u32 { + VER_NDX_UNSPECIFIED => { + if ctx.args.dynamic_list_data { + let ty = sym.ty(); + if ty != STT_FUNC && ty != STT_GNU_IFUNC { + return true; + } + } + if ctx.args.shared { + return match sym.file() { + Some(FileId::Obj(id)) => !ctx.objs[id.index()].exclude_libs, + _ => true, + }; + } + ctx.args.export_dynamic + } + VER_NDX_LOCAL => false, + _ => true, + } +} + +fn is_protected(ctx: &Context, sym: &Symbol) -> bool { + if sym.visibility() == STV_PROTECTED { + return true; + } + match ctx.args.bsymbolic { + BsymbolicKind::All => true, + BsymbolicKind::None => false, + BsymbolicKind::Functions => sym.ty() == STT_FUNC, + BsymbolicKind::NonWeak => !sym.is_weak(), + BsymbolicKind::NonWeakFunctions => !sym.is_weak() && sym.ty() == STT_FUNC, + } +} + +pub fn compute_import_export(ctx: &mut Context) { + let _t = ctx.timer("compute_import_export"); + + // An executable exports the symbols that DSOs reference, unless a + // version script marks them local. + let mut exports: Vec = Vec::new(); + if !ctx.args.shared { + let symbols = &ctx.symbols; + exports = ctx + .dsos + .par_iter() + .flat_map_iter(|file| { + file.base.symbols.iter().copied().filter(|&id| { + let sym = &symbols[id]; + matches!(sym.file(), Some(FileId::Obj(_))) + && sym.visibility() != STV_HIDDEN + && sym.ver_idx as u32 != VER_NDX_LOCAL + }) + }) + .collect(); + } + for id in exports { + ctx.symbols[id].set_exported(true); + } + + // Export symbols that are neither hidden nor local, and mark imported + // symbols as such. + let updates: Vec<(SymbolId, bool, bool)> = { + let ctx_ref: &Context = ctx; + ctx_ref + .objs + .par_iter() + .enumerate() + .flat_map_iter(|(fi, file)| { + let file_id = FileId::Obj(ObjId(fi as u32)); + file.base + .global_symbols() + .iter() + .copied() + .filter_map(move |id| { + let sym = &ctx_ref.symbols[id]; + if let Some(FileId::Dso(_)) = sym.file() { + return Some((id, true, false)); + } + if sym.file() == Some(file_id) && should_export(ctx_ref, sym) { + let imported = ctx_ref.args.shared && !is_protected(ctx_ref, sym); + return Some((id, imported, true)); + } + None + }) + }) + .collect() + }; + for (id, imported, exported) in updates { + let sym = &mut ctx.symbols[id]; + if imported { + sym.set_imported(true); + } + if exported { + sym.set_exported(true); + } + } + + // --dynamic-list and friends. For an executable, matched symbols are + // exported; for a shared object, matched symbols are imported if + // exported, so that they are interposable, and the rest are bound + // locally. + let handle_match = |ctx: &mut Context, id: SymbolId| { + let shared = ctx.args.shared; + let sym = &mut ctx.symbols[id]; + if shared { + if sym.is_exported() { + sym.set_imported(true); + } + } else if matches!(sym.file(), Some(FileId::Obj(_))) && sym.visibility() != STV_HIDDEN { + sym.set_exported(true); + } + }; + + let mut matcher = Glob::new(); + let mut cpp_matcher = Glob::new(); + for p in ctx.dynamic_list_patterns.clone() { + if p.is_cpp { + if !cpp_matcher.add(p.pattern, 1) { + fatal!( + ctx, + "{}: invalid dynamic list entry: {}", + p.source, + crate::util::display(p.pattern) + ); + } + continue; + } + if p.pattern.iter().any(|&c| matches!(c, b'*' | b'?' | b'[')) { + if !matcher.add(p.pattern, 1) { + fatal!( + ctx, + "{}: invalid dynamic list entry: {}", + p.source, + crate::util::display(p.pattern) + ); + } + continue; + } + let id = ctx.get_symbol(p.pattern); + handle_match(ctx, id); + } + + if !matcher.is_empty() || !cpp_matcher.is_empty() { + let shared = ctx.args.shared; + let symbols = &ctx.symbols; + let matched: Vec = symbols + .global_ids() + .collect::>() + .par_iter() + .copied() + .filter(|&id| { + let sym = &symbols[id]; + if !matches!(sym.file(), Some(FileId::Obj(_))) || (shared && !sym.is_exported()) { + return false; + } + if matcher.find(sym.name()) != -1 { + return true; + } + if !cpp_matcher.is_empty() { + let demangled = crate::util::demangle::demangle_cpp(sym.name()); + let name: &[u8] = demangled.as_deref().map_or(sym.name(), str::as_bytes); + return cpp_matcher.find(name) != -1; + } + false + }) + .collect(); + for id in matched { + handle_match(ctx, id); + } + } +} + +/// Computes the "address-taken" bit of each input section, for ICF. +/// +/// Merging two identical objects changes pointer equality, so ICF folds +/// only sections whose addresses are never taken. For functions, direct +/// calls use dedicated relocation types, so pointer-taking references can +/// be told apart. For data, LLVM's `.llvm_addrsig` lists the symbols whose +/// addresses are taken; without it, all data is assumed address-taken. +pub fn compute_address_significance(ctx: &mut Context) { + let _t = ctx.timer("compute_address_significance"); + let ctx_ref: &Context = ctx; + + ctx_ref.objs.par_iter().for_each(|file| { + if let Some(sec) = &file.llvm_addrsig { + let mut p = sec.contents(); + while !p.is_empty() { + let idx = crate::util::read_uleb(&mut p) as usize; + let sym = &ctx_ref.symbols[file.base.symbols[idx]]; + if let Some(r) = sym.input_section() { + ctx_ref.section(r).set_address_taken(); + } + } + return; + } + for isec in file.input_sections() { + if !isec.is_alive() || !isec.is_alloc() { + continue; + } + if isec.sh_flags & SHF_EXECINSTR as u64 == 0 { + isec.set_address_taken(); + } + for r in isec.rels::(file) { + if !r.is_func_call::() { + let sym = &ctx_ref.symbols[file.base.symbols[r.r_sym as usize]]; + if let Some(dst) = sym.input_section_ref() { + if dst.sh_flags & SHF_EXECINSTR as u64 != 0 { + dst.set_address_taken(); + } + } + } + } + } + }); + + let mark = |id: SymbolId| { + if let Some(r) = ctx_ref.symbols[id].input_section() { + ctx_ref.section(r).set_address_taken(); + } + }; + // Some symbols' addresses leak into the dynamic section. + mark(ctx_ref.syms.entry); + mark(ctx_ref.syms.init); + mark(ctx_ref.syms.fini); + // Exported symbols are conservatively considered address-taken. + for &id in ctx_ref.dynsym.symbols.iter().flatten() { + if ctx_ref.symbols[id].is_exported() { + mark(id); + } + } +} + +/// Sorts chunks into the standard layout: headers, then read-only data +/// needed by the loader, code, TLS, RELRO data, `.got`, writable data, +/// BSS, non-allocated sections, and the section header last. +fn sort_output_sections_regular(ctx: &mut Context) { + let rank1 = |ctx: &Context, id: ChunkId| -> i64 { + let hdr = ctx.chunk_header(id); + let ty = hdr.shdr.sh_type; + let flags = hdr.shdr.sh_flags; + match id { + ChunkId::Ehdr => return 0, + ChunkId::Phdr => return 1, + ChunkId::Interp => return 2, + _ if ty == SHT_NOTE && flags & SHF_ALLOC as u64 != 0 => return 3, + ChunkId::Hash => return 4, + ChunkId::GnuHash => return 5, + ChunkId::Dynsym => return 6, + ChunkId::Dynstr => return 7, + ChunkId::Versym => return 8, + ChunkId::Verneed => return 9, + ChunkId::RelDyn => return 10, + ChunkId::RelPlt => return 11, + ChunkId::Shdr => return i32::MAX as i64 - 1, + ChunkId::GdbIndex => return i32::MAX as i64, + _ => {} + } + let alloc = flags & SHF_ALLOC as u64 != 0; + let writable = flags & SHF_WRITE as u64 != 0; + let exec = flags & SHF_EXECINSTR as u64 != 0; + let tls = flags & SHF_TLS as u64 != 0; + let relro = hdr.is_relro; + let is_bss = ty == SHT_NOBITS; + (1 << 10) + | ((!alloc as i64) << 9) + | ((writable as i64) << 8) + | ((exec as i64) << 7) + | ((!tls as i64) << 6) + | ((!relro as i64) << 5) + | ((is_bss as i64) << 4) + }; + let rank2 = |ctx: &Context, id: ChunkId| -> i64 { + let hdr = ctx.chunk_header(id); + if hdr.shdr.sh_type == SHT_NOTE { + return -(hdr.shdr.sh_addralign as i64); + } + match id { + ChunkId::Got => 2, + _ if hdr.name == b".toc" => 3, + ChunkId::RelroPadding => i64::MAX, + _ => 0, + } + }; + let mut chunks = std::mem::take(&mut ctx.chunks); + chunks.sort_by_cached_key(|&id| { + ( + rank1(ctx, id), + rank2(ctx, id), + ctx.chunk_header(id).name.to_vec(), + ) + }); + ctx.chunks = chunks; +} + +fn section_order_group(ctx: &Context, id: ChunkId) -> &'static str { + let hdr = ctx.chunk_header(id); + if hdr.shdr.sh_type == SHT_NOBITS { + "BSS" + } else if hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0 { + "TEXT" + } else if hdr.shdr.sh_flags & SHF_WRITE as u64 != 0 { + "DATA" + } else { + "RODATA" + } +} + +fn sort_output_sections_by_order(ctx: &mut Context) { + let rank = + |ctx: &Context, id: ChunkId| -> i64 { + let hdr = ctx.chunk_header(id); + let flags = hdr.shdr.sh_flags; + match id { + ChunkId::Ehdr if flags & SHF_ALLOC as u64 == 0 => return -2, + ChunkId::Phdr if flags & SHF_ALLOC as u64 == 0 => return -1, + ChunkId::Shdr => return i32::MAX as i64, + _ if flags & SHF_ALLOC as u64 == 0 => return i32::MAX as i64 - 1, + _ => {} + } + let name = hdr.name; + if let Some(i) = ctx + .args + .section_order + .iter() + .position(|o| o.kind == SectionOrderKind::Section && o.name.as_bytes() == name) + { + return i as i64; + } + let group = section_order_group(ctx, id); + if let Some(i) = ctx.args.section_order.iter().position(|o| { + o.kind == SectionOrderKind::Group && o.name.eq_ignore_ascii_case(group) + }) { + return i as i64; + } + error!( + ctx, + "--section-order: missing section specification for {}", hdr.name + ); + 0 + }; + for id in ctx.chunks.clone() { + let r = rank(ctx, id); + ctx.chunk_header_mut(id).sect_order = r; + } + let mut chunks = std::mem::take(&mut ctx.chunks); + chunks.sort_by_key(|&id| ctx.chunk_header(id).sect_order); + ctx.chunks = chunks; +} + +pub fn sort_output_sections(ctx: &mut Context) { + if ctx.args.section_order.is_empty() { + sort_output_sections_regular(ctx); + } else { + sort_output_sections_by_order(ctx); + } +} + +fn tls_segment_alignment(ctx: &Context) -> u64 { + ctx.chunks + .iter() + .map(|&id| ctx.chunk_header(id)) + .filter(|h| h.shdr.sh_flags & SHF_TLS as u64 != 0) + .map(|h| h.shdr.sh_addralign) + .max() + .unwrap_or(1) + .max(1) +} + +/// Assigns virtual addresses. Sections with different memory protection +/// must be in different pages, and a section's file offset must be +/// congruent to its address modulo the page size; sections are packed as +/// tightly as those constraints allow. +fn set_virtual_addresses_regular(ctx: &mut Context) { + const RELRO: u64 = 1 << 32; + let flags_of = |ctx: &Context, id: ChunkId| -> u64 { + let flags = chunks::to_phdr_flags(ctx, id) as u64; + if ctx.args.z_relro && ctx.chunk_header(id).is_relro { + flags | RELRO + } else { + flags + } + }; + let is_tls = + |ctx: &Context, id: ChunkId| ctx.chunk_header(id).shdr.sh_flags & SHF_TLS as u64 != 0; + let is_tbss = |ctx: &Context, id: ChunkId| { + let shdr = &ctx.chunk_header(id).shdr; + shdr.sh_flags & SHF_TLS as u64 != 0 && shdr.sh_type == SHT_NOBITS + }; + + let chunks = ctx.chunks.clone(); + let mut addr = ctx.args.image_base; + let page_size = ctx.page_size; + let mut i = 0; + while i < chunks.len() { + let id = chunks[i]; + if !ctx.chunk_header(id).is_alloc() { + i += 1; + continue; + } + + if id == ChunkId::RelroPadding { + let hdr = ctx.chunk_header_mut(id); + hdr.shdr.sh_addr = addr; + hdr.shdr.sh_size = align_to(addr, page_size) - addr; + addr += page_size; + i += 1; + continue; + } + + // --section-start + let name = String::from_utf8_lossy(ctx.chunk_header(id).name).into_owned(); + if let Some(&start) = ctx.args.section_start.get(&name) { + addr = start; + let hdr = ctx.chunk_header_mut(id); + hdr.shdr.sh_addr = addr; + addr += hdr.shdr.sh_size; + i += 1; + continue; + } + + // Sections with different memory attributes go to different pages. + if i > 0 && chunks[i - 1] != ChunkId::RelroPadding { + let flags1 = flags_of(ctx, chunks[i - 1]); + let flags2 = flags_of(ctx, id); + if !ctx.args.nmagic && flags1 != flags2 { + match ctx.args.z_separate_code { + SeparateCodeKind::SeparateLoadableSegments => addr = align_to(addr, page_size), + SeparateCodeKind::SeparateCode + if (flags1 & PF_X as u64) != (flags2 & PF_X as u64) => + { + addr = align_to(addr, page_size) + } + _ => { + if !addr.is_multiple_of(page_size) { + addr += page_size; + } + } + } + } + } + + // The first TLS section is aligned to the PT_TLS segment's alignment. + if is_tls(ctx, id) && (i == 0 || !is_tls(ctx, chunks[i - 1])) { + addr = align_to(addr, tls_segment_alignment(ctx)); + } + + // TLS BSS sections overlap the following sections: a TLS segment is + // an initialization image and its BSS part is never read. + if is_tbss(ctx, id) { + let mut addr2 = addr; + loop { + let hdr = ctx.chunk_header_mut(chunks[i]); + addr2 = align_to(addr2, hdr.shdr.sh_addralign); + hdr.shdr.sh_addr = addr2; + addr2 += hdr.shdr.sh_size; + if i + 2 == chunks.len() || !is_tbss(ctx, chunks[i + 1]) { + break; + } + i += 1; + } + i += 1; + continue; + } + + let hdr = ctx.chunk_header_mut(id); + addr = align_to(addr, hdr.shdr.sh_addralign); + hdr.shdr.sh_addr = addr; + addr += hdr.shdr.sh_size; + i += 1; + } +} + +fn set_virtual_addresses_by_order(ctx: &mut Context) { + let vec: Vec = ctx + .chunks + .iter() + .copied() + .filter(|&c| ctx.chunk_header(c).is_alloc()) + .collect(); + let mut addr = ctx.args.image_base; + let page_size = ctx.page_size; + let mut i = 0; + + for (j, ord) in ctx.args.section_order.clone().iter().enumerate() { + match ord.kind { + SectionOrderKind::Section | SectionOrderKind::Group => { + while i < vec.len() && ctx.chunk_header(vec[i]).sect_order == j as i64 { + if i != 0 { + let flags1 = chunks::to_phdr_flags(ctx, vec[i - 1]); + let flags2 = chunks::to_phdr_flags(ctx, vec[i]); + if flags1 != flags2 { + match ctx.args.z_separate_code { + SeparateCodeKind::SeparateLoadableSegments => { + addr = align_to(addr, page_size) + } + SeparateCodeKind::SeparateCode + if (flags1 & PF_X) != (flags2 & PF_X) => + { + addr = align_to(addr, page_size) + } + _ => {} + } + } + } + let hdr = ctx.chunk_header_mut(vec[i]); + addr = align_to(addr, hdr.shdr.sh_addralign); + hdr.shdr.sh_addr = addr; + addr += hdr.shdr.sh_size; + i += 1; + } + } + SectionOrderKind::Addr => { + if addr != ctx.args.image_base && ord.value < addr { + error!( + ctx, + "--section-order: address goes backward: requested {:#x} < current {addr:#x} (at token '{}')", + ord.value, + ord.token + ); + } + addr = ord.value; + } + SectionOrderKind::Align => addr = align_to(addr, ord.value), + SectionOrderKind::Symbol => { + let id = ctx.get_symbol(ord.name.as_bytes()); + ctx.symbols[id].value = addr; + } + } + } +} + +/// The smallest N >= val with N % align == skew % align. +fn align_with_skew(val: u64, align: u64, skew: u64) -> u64 { + val + (skew.wrapping_sub(val) & (align - 1)) +} + +fn set_file_offsets(ctx: &mut Context) -> u64 { + let chunks = ctx.chunks.clone(); + let page_size = ctx.page_size; + let mut fileoff = 0u64; + let mut i = 0; + + while i < chunks.len() { + let first = ctx.chunk_header(chunks[i]).shdr; + if first.sh_flags & SHF_ALLOC as u64 == 0 { + fileoff = align_to(fileoff, first.sh_addralign); + ctx.chunk_header_mut(chunks[i]).shdr.sh_offset = fileoff; + fileoff += first.sh_size; + i += 1; + continue; + } + if first.sh_type == SHT_NOBITS { + ctx.chunk_header_mut(chunks[i]).shdr.sh_offset = fileoff; + i += 1; + continue; + } + + if first.sh_addralign > page_size { + fileoff = align_to(fileoff, first.sh_addralign); + } else { + fileoff = align_with_skew(fileoff, page_size, first.sh_addr); + } + + // Allocated sections contiguous in memory get contiguous file + // offsets. + loop { + let shdr = ctx.chunk_header(chunks[i]).shdr; + ctx.chunk_header_mut(chunks[i]).shdr.sh_offset = fileoff + shdr.sh_addr - first.sh_addr; + i += 1; + if i >= chunks.len() { + break; + } + let next = ctx.chunk_header(chunks[i]).shdr; + if next.sh_flags & SHF_ALLOC as u64 == 0 || next.sh_type == SHT_NOBITS { + break; + } + // With --section-start, addresses may not increase monotonically. + if next.sh_addr < first.sh_addr { + break; + } + let prev = ctx.chunk_header(chunks[i - 1]).shdr; + // A section with a larger alignment needs offset % align == vaddr % align. + if next.sh_addralign > page_size && next.sh_addralign > prev.sh_addralign { + break; + } + // Don't allocate disk space for a large gap (--section-start). + let gap = next.sh_addr - prev.sh_addr - prev.sh_size; + if gap >= page_size { + break; + } + } + + let last = ctx.chunk_header(chunks[i - 1]).shdr; + fileoff = last.sh_offset + last.sh_size; + + while i < chunks.len() { + let shdr = ctx.chunk_header(chunks[i]).shdr; + if shdr.sh_flags & SHF_ALLOC as u64 == 0 || shdr.sh_type != SHT_NOBITS { + break; + } + ctx.chunk_header_mut(chunks[i]).shdr.sh_offset = fileoff; + i += 1; + } + } + fileoff +} + +/// Sets aside debug sections for `--separate-debug-file`. +pub fn separate_debug_sections(ctx: &mut Context) { + let is_debug = |ctx: &Context, id: ChunkId| { + let hdr = ctx.chunk_header(id); + !hdr.is_alloc() + && (matches!(id, ChunkId::GdbIndex | ChunkId::Symtab | ChunkId::Strtab) + || hdr.name.starts_with(b".debug_")) + }; + let (debug, rest): (Vec, Vec) = + ctx.chunks.iter().partition(|&&id| is_debug(ctx, id)); + ctx.chunks = rest; + ctx.debug_chunks = debug; +} + +pub fn compute_section_headers(ctx: &mut Context) { + for id in ctx.chunks.clone() { + chunks::update_shdr(ctx, id); + } + + // Remove empty chunks. + let chunks = std::mem::take(&mut ctx.chunks); + ctx.chunks = chunks + .into_iter() + .filter(|&id| { + matches!( + id, + ChunkId::Output(_) | ChunkId::GdbIndex | ChunkId::Placeholder(_) + ) || ctx.chunk_header(id).shdr.sh_size != 0 + }) + .collect(); + + // Assign section indices. + let mut shndx = 1u32; + for id in ctx.chunks.clone() { + if !id.is_header() { + ctx.chunk_header_mut(id).shndx = shndx; + shndx += 1; + } + } + + if shndx >= SHN_LORESERVE && ctx.chunks.contains(&ChunkId::Symtab) && ctx.symtab_shndx.is_none() + { + let mut sec = SymtabShndxSection::new(); + sec.hdr.shndx = shndx; + shndx += 1; + sec.hdr.shdr.sh_link = ctx.symtab.hdr.shndx; + ctx.symtab_shndx = Some(sec); + ctx.chunks.push(ChunkId::SymtabShndx); + } + + if let Some(shdr) = &mut ctx.shdr { + shdr.hdr.shdr.sh_size = shndx as u64 * ElfShdr::size::() as u64; + } + + // Some section headers refer to other sections by index, so recompute + // them now that indices are known. + for id in ctx.chunks.clone() { + chunks::update_shdr(ctx, id); + } + + if let Some(symtab_shndx) = &mut ctx.symtab_shndx { + let n = ctx.symtab.hdr.shdr.sh_size / ElfSym::size::() as u64; + symtab_shndx.hdr.shdr.sh_size = n * 4; + } +} + +/// Assigns addresses and file offsets, repeating until the program +/// header's size, which depends on the layout, converges. +pub fn set_osec_offsets(ctx: &mut Context) -> u64 { + let _t = ctx.timer("set_osec_offsets"); + loop { + if ctx.args.section_order.is_empty() { + set_virtual_addresses_regular(ctx); + } else { + set_virtual_addresses_by_order(ctx); + } + + if ctx.args.pack_dyn_relocs_android { + let before = ctx.reldyn.hdr.shdr.sh_size; + crate::chunks::dynamic::reldyn::update_shdr(ctx); + if before != ctx.reldyn.hdr.shdr.sh_size { + continue; + } + } + ctx.checkpoint(); + + let fileoff = set_file_offsets(ctx); + if ctx.phdr.is_some() { + let before = ctx.phdr.as_ref().unwrap().hdr.shdr.sh_size; + chunks::update_phdr(ctx); + if before < ctx.phdr.as_ref().unwrap().hdr.shdr.sh_size { + continue; + } + } + return fileoff; + } +} + +fn num_irelative_relocs(ctx: &Context) -> u64 { + let mut n = ctx.num_ifunc_dynrels.load(Ordering::Relaxed) as u64; + n += ctx + .got + .got_syms + .iter() + .filter(|&&id| ctx.symbols[id].is_ifunc()) + .count() as u64; + n +} + +fn to_paddr(ctx: &Context, vaddr: u64) -> u64 { + if let Some(phdr) = &ctx.phdr { + for p in &phdr.phdrs { + if p.p_type == PT_LOAD && p.p_vaddr <= vaddr && vaddr < p.p_vaddr + p.p_memsz { + return p.p_paddr + (vaddr - p.p_vaddr); + } + } + } + 0 +} + +pub fn fix_synthetic_symbols(ctx: &mut Context) { + fn start( + ctx: &mut Context, + sym: Option, + chunk: Option, + bias: i64, + ) { + if let (Some(sym), Some(chunk)) = (sym, chunk) { + let addr = ctx.chunk_header(chunk).shdr.sh_addr; + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(chunk); + s.value = addr.wrapping_add(bias as u64); + } + } + fn stop( + ctx: &mut Context, + sym: Option, + chunk: Option, + bias: i64, + ) { + if let (Some(sym), Some(chunk)) = (sym, chunk) { + let shdr = ctx.chunk_header(chunk).shdr; + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(chunk); + s.value = (shdr.sh_addr + shdr.sh_size).wrapping_add(bias as u64); + } + } + + let sections: Vec = ctx + .chunks + .iter() + .copied() + .filter(|&c| !c.is_header() && ctx.chunk_header(c).is_alloc()) + .collect(); + let find = |ctx: &Context, name: &[u8]| { + sections + .iter() + .copied() + .find(|&c| ctx.chunk_header(c).name == name) + }; + let first = sections.first().copied(); + + if let Some(bss) = find(ctx, b".bss") { + start(ctx, ctx.syms.bss_start, Some(bss), 0); + } + + if let Some(ehdr) = &ctx.ehdr { + if ehdr.hdr.is_alloc() { + let addr = ehdr.hdr.shdr.sh_addr; + for sym in [ctx.syms.ehdr_start, ctx.syms.executable_start] { + if let (Some(sym), Some(first)) = (sym, first) { + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(first); + s.value = addr; + } + } + } + } + + if let (Some(sym), Some(first)) = (ctx.syms.dso_handle, first) { + let addr = ctx.chunk_header(first).shdr.sh_addr; + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(first); + s.value = addr; + } + + // __rel_iplt_start/end are needed in a statically linked + // non-relocatable executable, which has no .dynamic to find IFUNC + // relocations by. + if ctx.chunks.contains(&ChunkId::RelDyn) && ctx.args.is_static && !ctx.args.pie { + let n = num_irelative_relocs(ctx) as i64 * ElfRel::size::() as i64; + stop(ctx, ctx.syms.rel_iplt_start, Some(ChunkId::RelDyn), -n); + stop(ctx, ctx.syms.rel_iplt_end, Some(ChunkId::RelDyn), 0); + } else { + for sym in [ctx.syms.rel_iplt_start, ctx.syms.rel_iplt_end] + .into_iter() + .flatten() + { + ctx.symbols[sym].clear_origin(); + } + } + + for &chunk in §ions { + match ctx.chunk_header(chunk).shdr.sh_type { + SHT_INIT_ARRAY => { + start(ctx, ctx.syms.init_array_start, Some(chunk), 0); + stop(ctx, ctx.syms.init_array_end, Some(chunk), 0); + } + SHT_PREINIT_ARRAY => { + start(ctx, ctx.syms.preinit_array_start, Some(chunk), 0); + stop(ctx, ctx.syms.preinit_array_end, Some(chunk), 0); + } + SHT_FINI_ARRAY => { + start(ctx, ctx.syms.fini_array_start, Some(chunk), 0); + stop(ctx, ctx.syms.fini_array_end, Some(chunk), 0); + } + _ => {} + } + } + + for &chunk in §ions { + let shdr = ctx.chunk_header(chunk).shdr; + if shdr.sh_flags & SHF_ALLOC as u64 != 0 { + stop(ctx, ctx.syms.end_, Some(chunk), 0); + stop(ctx, ctx.syms.end, Some(chunk), 0); + } + if shdr.sh_flags & SHF_EXECINSTR as u64 != 0 { + stop(ctx, ctx.syms.etext_, Some(chunk), 0); + stop(ctx, ctx.syms.etext, Some(chunk), 0); + } + if shdr.sh_type != SHT_NOBITS && shdr.sh_flags & SHF_ALLOC as u64 != 0 { + stop(ctx, ctx.syms.edata_, Some(chunk), 0); + stop(ctx, ctx.syms.edata, Some(chunk), 0); + } + } + + let dynamic = ctx.dynamic.as_ref().map(|_| ChunkId::Dynamic); + start(ctx, ctx.syms.dynamic, dynamic, 0); + + // _GLOBAL_OFFSET_TABLE_ is the start of .got.plt on x86 for + // compatibility, and of .got elsewhere. + let got = if E::IS_X86 { + ChunkId::GotPlt + } else { + ChunkId::Got + }; + start(ctx, ctx.syms.global_offset_table, Some(got), 0); + start(ctx, ctx.syms.procedure_linkage_table, Some(ChunkId::Plt), 0); + + if let (Some(sym), Some(first)) = (ctx.syms.tls_module_base, first) { + let dtp = ctx.dtp_addr; + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(first); + s.value = dtp; + } + + let eh_frame_hdr = ctx.eh_frame_hdr.as_ref().map(|_| ChunkId::EhFrameHdr); + start(ctx, ctx.syms.gnu_eh_frame_hdr, eh_frame_hdr, 0); + + if let Some(sym) = ctx.syms.global_pointer { + match find(ctx, b".sdata") { + Some(c) => start(ctx, Some(sym), Some(c), 0x800), + None => start(ctx, Some(sym), first, 0), + } + } + if ctx.syms.exidx_start.is_some() { + if let Some(c) = find(ctx, b".ARM.exidx") { + start(ctx, ctx.syms.exidx_start, Some(c), 0); + stop(ctx, ctx.syms.exidx_end, Some(c), 0); + } + } + if E::IS_PPC64 { + if let Some(c) = find(ctx, b".got").or_else(|| find(ctx, b".toc")) { + start(ctx, ctx.syms.toc, Some(c), 0x8000); + } else if let (Some(sym), Some(first)) = (ctx.syms.toc, first) { + let s = &mut ctx.symbols[sym]; + s.set_output_chunk(first); + s.value = 0; + } + } + + // The register save and restore routines, unless an input defines them. + if E::FAMILY == Family::Ppc64V2 { + for (i, &(label, _)) in crate::arch::ppc64v2::SAVE_RESTORE_INSNS.iter().enumerate() { + if label.is_empty() { + continue; + } + let sym = ctx.get_symbol(label.as_bytes()); + if ctx.symbols[sym].file() == ctx.internal_obj.map(FileId::Obj) { + start( + ctx, + Some(sym), + Some(ChunkId::Ppc64SaveRestore), + (i * 4) as i64, + ); + } + } + } + + for &chunk in §ions { + if let Some(name) = start_stop_name(ctx, chunk) { + let s = ctx.get_symbol(format!("__start_{name}").as_bytes()); + start(ctx, Some(s), Some(chunk), 0); + let e = ctx.get_symbol(format!("__stop_{name}").as_bytes()); + stop(ctx, Some(e), Some(chunk), 0); + + if ctx.args.physical_image_base.is_some() { + let shdr = ctx.chunk_header(chunk).shdr; + let paddr = to_paddr(ctx, shdr.sh_addr); + let x = ctx.get_symbol(format!("__phys_start_{name}").as_bytes()); + ctx.symbols[x].set_output_chunk(chunk); + ctx.symbols[x].value = paddr; + let y = ctx.get_symbol(format!("__phys_stop_{name}").as_bytes()); + ctx.symbols[y].set_output_chunk(chunk); + ctx.symbols[y].value = paddr + shdr.sh_size; + } + } + } + + // --defsym + for (name, value) in ctx.args.defsyms.clone() { + let sym = ctx.get_symbol(name.as_bytes()); + match value { + DefsymValue::Addr(addr) => { + let s = &mut ctx.symbols[sym]; + s.clear_origin(); + s.value = addr; + } + DefsymValue::Symbol(target) => { + let sym2 = ctx.get_symbol(target.as_bytes()); + let (value, origin, vis) = { + let s2 = &ctx.symbols[sym2]; + (s2.value, s2.origin_state(), s2.visibility()) + }; + let s = &mut ctx.symbols[sym]; + s.value = value; + s.set_origin_state(origin); + s.set_visibility(vis); + } + } + } + + // --section-order symbols + for ord in ctx.args.section_order.clone() { + if ord.kind == SectionOrderKind::Symbol { + let sym = ctx.get_symbol(ord.name.as_bytes()); + if let Some(first) = first { + ctx.symbols[sym].set_output_chunk(first); + } + } + } +} + +pub fn compress_debug_sections(ctx: &mut Context) { + let _t = ctx.timer("compress_debug_sections"); + let targets: Vec<(usize, ChunkId)> = ctx + .chunks + .iter() + .enumerate() + .filter(|&(_, &id)| { + let hdr = ctx.chunk_header(id); + !hdr.is_alloc() && hdr.shdr.sh_size != 0 && hdr.name.starts_with(b".debug_") + }) + .map(|(i, &id)| (i, id)) + .collect(); + let compressed: Vec = { + let ctx_ref: &Context = ctx; + targets + .par_iter() + .map(|&(_, id)| misc::compressed::new(ctx_ref, id)) + .collect() + }; + for ((i, _), sec) in targets.into_iter().zip(compressed) { + ctx.compressed_sections.push(sec); + ctx.chunks[i] = ChunkId::Compressed(ctx.compressed_sections.len() as u32 - 1); + } +} + +/// Computes the build ID from the output file contents. +/// Writes the build ID. A hash-based ID is a hash of the output file: +/// BLAKE3 is a cryptographic hash function just like SHA256; we use it +/// instead of SHA256 because it's faster. The file is hashed in 4 MiB +/// shards in parallel, and the ID is the hash of the shards' hashes. +pub fn write_build_id(ctx: &mut Context, buf: &mut [u8], is_mmapped: bool) { + let _t = ctx.timer("write_build_id"); + let contents: Vec = match ctx.args.build_id.kind { + BuildIdKind::Hex => ctx.args.build_id.value.clone(), + BuildIdKind::Hash => { + const SHARD: usize = 4 * 1024 * 1024; + let hashes: Vec<[u8; 32]> = buf + .par_chunks_mut(SHARD) + .enumerate() + .map(|(i, shard)| { + let hash = *blake3::hash(shard).as_bytes(); + // Make the kernel page out the file contents we've just + // written so that the subsequent close(2) call will + // become quicker. + if i > 0 && is_mmapped { + // SAFETY: the shard is part of the output mapping; the + // advice only drops the process's mapping of pages + // that are backed by the file. + unsafe { + libc::madvise( + shard.as_mut_ptr() as *mut libc::c_void, + shard.len(), + libc::MADV_DONTNEED, + ) + }; + } + hash + }) + .collect(); + let digest = *blake3::hash(hashes.as_flattened()).as_bytes(); + digest[..ctx.args.build_id.size()].to_vec() + } + BuildIdKind::Uuid => { + let mut bytes = [0u8; 16]; + crate::util::random_bytes(&mut bytes); + // UUIDv4 as defined by RFC 4122 + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + bytes.to_vec() + } + BuildIdKind::None => unreachable!(), + }; + ctx.buildid.as_mut().unwrap().contents = contents; + let hdr = ctx.buildid.as_ref().unwrap().hdr.shdr; + misc::build_id::copy_buf( + ctx, + &mut buf[hdr.sh_offset as usize..(hdr.sh_offset + hdr.sh_size) as usize], + ); +} + +/// Writes `.gnu_debuglink` with a CRC32 that the separate debug file will +/// be adjusted to match. +pub fn write_gnu_debuglink(ctx: &mut Context, buf: &mut [u8]) { + let _t = ctx.timer("write_gnu_debuglink"); + let crc = match &ctx.buildid { + Some(buildid) => crc32fast::hash(&buildid.contents), + None => { + const SHARD: usize = 4 * 1024 * 1024; + let hashes: Vec = buf + .par_chunks(SHARD) + .flat_map_iter(|s| xxhash_rust::xxh3::xxh3_64(s).to_le_bytes()) + .collect(); + crc32fast::hash(&hashes) + } + }; + ctx.gnu_debuglink.as_mut().unwrap().crc32 = crc; + let hdr = ctx.gnu_debuglink.as_ref().unwrap().hdr.shdr; + misc::gnu_debuglink::copy_buf( + ctx, + &mut buf[hdr.sh_offset as usize..(hdr.sh_offset + hdr.sh_size) as usize], + ); +} + +/// The CRC32 of a large buffer, computed in parallel. +fn crc32_parallel(buf: &[u8]) -> u32 { + const SHARD: usize = 1024 * 1024; + buf.par_chunks(SHARD) + .map(|shard| { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(shard); + hasher + }) + .reduce(crc32fast::Hasher::new, |mut a, b| { + a.combine(&b); + a + }) + .finalize() +} + +/// Four bytes that, appended to data whose CRC32 is `current`, make its +/// CRC32 `desired`. ELF files ignore trailing bytes, so this is how a +/// debug file is given the checksum its executable recorded for it. +fn crc32_solve(current: u32, desired: u32) -> [u8; 4] { + const POLY: u32 = 0xedb8_8320; + let mut x = !desired; + // Each iteration multiplies x by the inverse of x modulo the polynomial. + for _ in 0..32 { + x = x.rotate_left(1); + x ^= (x & 1) * (POLY << 1); + } + (x ^ !current).to_le_bytes() +} + +/// Writes the debug sections set aside by `separate_debug_sections` to a +/// separate file, whose name and checksum the main output recorded in +/// `.gnu_debuglink`. +pub fn write_separate_debug_file(ctx: &mut Context) { + let _t = ctx.timer("write_separate_debug_file"); + + let path = ctx.args.separate_debug_file.clone(); + let mut output = OutputFile::open_locked(&ctx.diag, &path, 0o666); + + // The main output is complete; writing the debug file can go on in + // the background. + if ctx.args.detach { + crate::subprocess::notify_parent(); + } + + // A debug file has the same sections as the main file, but all except + // the debug sections are empty, like bss. Replace them with + // placeholders that keep their addresses and indices. + let num_chunks = ctx.chunks.len(); + let debug_chunks = std::mem::take(&mut ctx.debug_chunks); + ctx.chunks.extend(debug_chunks); + for i in 0..num_chunks { + let id = ctx.chunks[i]; + if id.is_header() + || id == ChunkId::Shstrtab + || ctx.chunk_header(id).shdr.sh_type == SHT_NOTE + { + continue; + } + let hdr = ctx.chunk_header(id); + let mut placeholder = ChunkHeader::with_name(hdr.name, SHT_NOBITS, hdr.shdr.sh_flags); + placeholder.shdr = ElfShdr { + sh_type: SHT_NOBITS, + ..hdr.shdr + }; + placeholder.shndx = hdr.shndx; + ctx.placeholders.push(placeholder); + ctx.chunks[i] = ChunkId::Placeholder(ctx.placeholders.len() as u32 - 1); + } + + let new_chunks = ctx.chunks[num_chunks..].to_vec(); + for id in new_chunks { + chunks::compute_section_size(ctx, id); + } + sort_debug_info_sections(ctx); + if ctx.args.compress_debug_sections != ELFCOMPRESS_NONE { + compress_debug_sections(ctx); + } + compute_section_headers(ctx); + + let page_size = ctx.page_size; + let mut fileoff = 0; + for id in ctx.chunks.clone() { + let shdr = &mut ctx.chunk_header_mut(id).shdr; + if shdr.sh_type == SHT_NOBITS { + shdr.sh_offset = fileoff; + } else if shdr.sh_flags & SHF_ALLOC as u64 != 0 { + fileoff = align_with_skew(fileoff, page_size, shdr.sh_addr); + shdr.sh_offset = fileoff; + fileoff += shdr.sh_size; + } else { + fileoff = align_to(fileoff, shdr.sh_addralign); + shdr.sh_offset = fileoff; + fileoff += shdr.sh_size; + } + } + + // The program header keeps its size, since the placeholders' addresses + // were laid out around it. + if let Some(n) = ctx.phdr.as_ref().map(|p| p.phdrs.len()) { + chunks::update_phdr(ctx); + let phdr = ctx.phdr.as_mut().unwrap(); + phdr.phdrs.resize(n, ElfPhdr::default()); + phdr.hdr.shdr.sh_size = (n * ElfPhdr::size::()) as u64; + } + + output.resize(&ctx.diag, fileoff); + crate::driver::copy_chunks(ctx, output.buf()); + + if ctx.gdb_index.is_some() { + crate::gdb_index::build_tables_now(ctx); + crate::gdb_index::write(ctx, &mut output); + } + + let trailer = crc32_solve( + crc32_parallel(output.buf()), + ctx.gnu_debuglink.as_ref().unwrap().crc32, + ); + let len = output.len(); + output.extend(&ctx.diag, trailer.len()); + output.buf()[len..].copy_from_slice(&trailer); + output.close(&ctx.diag); +} + +/// Writes Makefile-style dependency rules, like the compiler's -M. +pub fn write_dependency_file(ctx: &Context) { + // Dependencies are listed in command line order, which is the order + // of file priorities. + let mut files: Vec<(u32, &MappedFile)> = ctx + .objs + .iter() + .filter_map(|f| Some((f.base.priority, f.base.mf?))) + .chain( + ctx.dsos + .iter() + .filter_map(|f| Some((f.base.priority, f.base.mf?))), + ) + .chain(ctx.lto_input_files.iter().copied()) + .collect(); + files.sort_by_key(|&(priority, _)| priority); + + let mut deps: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for (_, mf) in files { + let top = mf.parent.unwrap_or(mf); + if top.is_dependency() { + let path = crate::util::path_clean(&top.name); + if seen.insert(path.clone()) { + deps.push(path); + } + } + } + + let mut out = format!("{}:", ctx.args.output); + for d in &deps { + out.push(' '); + out.push_str(d); + } + out.push('\n'); + for d in &deps { + out.push_str(&format!("\n{d}:\n")); + } + + let path = &ctx.args.dependency_file; + if path == "-" { + let _ = std::io::stdout().write_all(out.as_bytes()); + } else { + std::fs::write(path, out) + .unwrap_or_else(|e| fatal!(ctx, "--dependency-file: cannot open {path}: {e}")); + } +} + +pub fn show_stats(ctx: &Context) { + let print = |name: &str, value: usize| out!(ctx, "{name:>20}={value}"); + print("num_objs", ctx.objs.len()); + print("num_dsos", ctx.dsos.len()); + print("output_chunks", ctx.chunks.len()); + print( + "input_sections", + ctx.objs.iter().map(|f| f.sections.len()).sum(), + ); + print( + "comdats", + ctx.objs.iter().map(|f| f.comdat_groups.len()).sum(), + ); + print("num_cies", ctx.objs.iter().map(|f| f.cies.len()).sum()); + print("num_fdes", ctx.objs.iter().map(|f| f.fdes.len()).sum()); + print( + "merged_strings", + ctx.merged_sections + .iter() + .map(crate::chunks::merged::num_fragments) + .sum(), + ); +} + +/// Whether `--section-order` names a section that doesn't exist. +pub fn align_down_to_page(ctx: &Context, addr: u64) -> u64 { + align_down(addr, ctx.page_size) +} diff --git a/src/reader.rs b/src/reader.rs new file mode 100644 index 000000000..10b1b06ec --- /dev/null +++ b/src/reader.rs @@ -0,0 +1,452 @@ +//! Reading input files. +//! +//! Reading is I/O- and CPU-intensive and a large program has tens of +//! thousands of input files, so files are read in parallel. The command +//! line is inherently sequential, though: options such as `--as-needed` +//! apply to the files after them, and a file's priority for symbol +//! resolution is its position. The argument parser records the reader +//! state and position for each file, files are read in any order, and +//! the results are sorted back into command line order. + +use std::cell::UnsafeCell; + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::archive; +use crate::args::{ReaderContext, ReaderJob}; +use crate::context::Context; +use crate::filetype::{self, FileType}; +use crate::input_files::{DsoId, FileId, ObjId, ObjectFile, SharedFile}; +use crate::linker_script::Script; +use crate::mapped_file::{must_open_file, open_file, MappedFile}; +use crate::{fatal, out, warn}; + +/// A file that has been read, with its command line position. +enum Loaded { + Obj(Vec, Box), + Dso(Vec, Box), +} + +/// A concurrent vector implemented as one bin per Rayon worker. Writers on +/// different workers never contend, and ordering is restored from ReaderJob +/// positions after all input has been read. +struct WorkerBins { + bins: Vec>>, +} + +// SAFETY: a Rayon worker has a unique index and executes at most one closure +// at a time. Callers do not invoke nested parallel work while pushing, and the +// bins are consumed only after their scoped traversals have joined. +unsafe impl Sync for WorkerBins {} + +impl WorkerBins { + fn new() -> WorkerBins { + WorkerBins { + bins: (0..=rayon::current_num_threads()) + .map(|_| UnsafeCell::new(Vec::new())) + .collect(), + } + } + + fn push(&self, value: T) { + let fallback = self.bins.len() - 1; + let i = rayon::current_thread_index() + .unwrap_or(fallback) + .min(fallback); + // SAFETY: the Sync invariant above gives this worker exclusive access + // to its indexed vector for the duration of this non-nested closure. + unsafe { &mut *self.bins[i].get() }.push(value); + } + + fn into_vec(self) -> Vec { + self.bins + .into_iter() + .flat_map(UnsafeCell::into_inner) + .collect() + } +} + +fn get_file_type(ctx: &Context, mf: &MappedFile) -> FileType { + filetype::get_file_type(&ctx.args.plugin, mf) +} + +/// Returns the target a file was compiled for. +pub fn get_machine_type( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, +) -> Option<&'static str> { + match get_file_type(ctx, mf) { + FileType::Text => crate::linker_script::output_target(ctx, rctx, mf), + _ => filetype::get_machine_type(&ctx.diag, &ctx.args.plugin, mf, || None), + } +} + +fn new_object_file( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, + archive_name: &str, +) -> ObjectFile { + let target = filetype::get_machine_type(&ctx.diag, &ctx.args.plugin, mf, || None); + match target { + None => fatal!(ctx, "{}: unknown machine type", mf.name), + Some(t) if t != ctx.args.emulation => { + fatal!( + ctx, + "{}: incompatible file type: {} is expected but got {t}", + mf.name, + ctx.args.emulation + ) + } + _ => {} + } + let mut file = ObjectFile::new::(&ctx.diag, mf, archive_name.to_string()); + file.base.as_needed = rctx.in_lib || (!archive_name.is_empty() && !rctx.whole_archive); + file.register_global_symbols::(&ctx.args, &mut ctx.symbol_bin()); + file +} + +fn new_shared_file( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, +) -> SharedFile { + if rctx.is_static { + fatal!( + ctx, + "{}: attempted static link of a dynamic object", + mf.name + ); + } + let target = filetype::get_machine_type(&ctx.diag, &ctx.args.plugin, mf, || None); + match target { + None => fatal!(ctx, "{}: unknown machine type", mf.name), + Some(t) if t != ctx.args.emulation => { + fatal!( + ctx, + "{}: incompatible file type: {} is expected but got {t}", + mf.name, + ctx.args.emulation + ) + } + _ => {} + } + let mut file = SharedFile::new::(&ctx.diag, mf); + file.base.as_needed = rctx.as_needed; + file +} + +/// Reads an IR object through the LTO plugin. An object listed by +/// `--:ignore-ir-file` is an archive member a previous pass found +/// unneeded. +fn new_lto_object( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, + archive_name: &str, +) -> Option { + if ctx.args.ignore_ir_file.contains(&mf.identifier()) { + return None; + } + let mut file = crate::lto::read_lto_object(ctx, mf, archive_name.to_string())?; + file.base.as_needed = rctx.in_lib || (!archive_name.is_empty() && !rctx.whole_archive); + file.register_global_symbols::(&ctx.args, &mut ctx.symbol_bin()); + Some(file) +} + +/// Reads a member of an archive. +fn read_archive_member( + ctx: &Context, + rctx: &ReaderContext, + mf: &'static MappedFile, + archive_name: &str, +) -> Option { + match get_file_type(ctx, mf) { + FileType::ElfObj => { + let file = new_object_file(ctx, rctx, mf, archive_name); + Some(Loaded::Obj(rctx.pos.clone(), Box::new(file))) + } + FileType::GccLtoObj | FileType::LlvmBitcode => { + let file = new_lto_object(ctx, rctx, mf, archive_name)?; + Some(Loaded::Obj(rctx.pos.clone(), Box::new(file))) + } + FileType::ElfDso => { + warn!( + ctx, + "{archive_name}({}): shared object file in an archive is ignored", mf.name + ); + None + } + _ => None, + } +} + +/// Reads a file at `rctx.pos`. Containers (archives and linker scripts) +/// are expanded in place, recursively. +pub fn read_file(ctx: &mut Context, rctx: &mut ReaderContext, mf: &'static MappedFile) { + match get_file_type(ctx, mf) { + FileType::ElfObj => { + let file = new_object_file(ctx, rctx, mf, ""); + push_loaded(ctx, Loaded::Obj(rctx.pos.clone(), Box::new(file))); + } + FileType::ElfDso => { + let file = new_shared_file(ctx, rctx, mf); + push_loaded(ctx, Loaded::Dso(rctx.pos.clone(), Box::new(file))); + } + FileType::Ar | FileType::ThinAr => { + for child in archive::read_archive_members(&ctx.diag, mf) { + let child_rctx = rctx.next_child(); + if let Some(loaded) = read_archive_member(ctx, &child_rctx, child, &mf.name) { + push_loaded(ctx, loaded); + } + } + } + FileType::Text => Script::new(ctx, rctx, mf).parse_linker_script(), + FileType::GccLtoObj | FileType::LlvmBitcode => { + if let Some(file) = new_lto_object(ctx, rctx, mf, "") { + push_loaded(ctx, Loaded::Obj(rctx.pos.clone(), Box::new(file))); + } + } + _ => fatal!(ctx, "{}: unknown file type", mf.name), + } +} + +fn push_loaded(ctx: &mut Context, loaded: Loaded) { + match loaded { + Loaded::Obj(pos, file) => { + ctx.objs.push(file); + ctx.pending_files + .push((pos, FileId::Obj(ObjId(ctx.objs.len() as u32 - 1)))); + } + Loaded::Dso(pos, file) => { + ctx.dsos.push(file); + ctx.pending_files + .push((pos, FileId::Dso(DsoId(ctx.dsos.len() as u32 - 1)))); + } + } +} + +/// Deduces the target from the first recognizable input file. +pub fn detect_machine_type(ctx: &mut Context, jobs: &[ReaderJob]) -> String { + for job in jobs { + if job.is_lib { + continue; + } + if let Some(mf) = open_file(&ctx.diag, &ctx.args.chroot, &job.name) { + if get_file_type(ctx, mf) != FileType::Text { + if let Some(target) = + filetype::get_machine_type(&ctx.diag, &ctx.args.plugin, mf, || None) + { + return target.to_string(); + } + } + } + } + for job in jobs { + if job.is_lib { + continue; + } + if let Some(mf) = open_file(&ctx.diag, &ctx.args.chroot, &job.name) { + if get_file_type(ctx, mf) == FileType::Text { + if let Some(target) = crate::linker_script::output_target(ctx, &job.rctx, mf) { + return target.to_string(); + } + } + } + } + fatal!(ctx, "-m option is missing"); +} + +fn open_library( + ctx: &Context, + rctx: &ReaderContext, + path: &str, +) -> Option<&'static MappedFile> { + let mf = open_file(&ctx.diag, &ctx.args.chroot, path)?; + if let Some(target) = get_machine_type(ctx, rctx, mf) { + if target != E::NAME { + warn!( + ctx, + "{path}: skipping incompatible file: {target} (e_machine {})", + E::E_MACHINE + ); + return None; + } + } + Some(mf) +} + +/// Finds a library given to `-l`. +pub fn find_library( + ctx: &Context, + rctx: &ReaderContext, + name: &str, +) -> &'static MappedFile { + if let Some(name) = name.strip_prefix(':') { + for dir in &ctx.args.library_paths { + if let Some(mf) = open_library(ctx, rctx, &format!("{dir}/{name}")) { + return mf; + } + } + fatal!(ctx, "library not found: :{name}"); + } + + for dir in &ctx.args.library_paths { + let stem = format!("{dir}/lib{name}"); + if !rctx.is_static { + if let Some(mf) = open_library(ctx, rctx, &format!("{stem}.so")) { + return mf; + } + } + if let Some(mf) = open_library(ctx, rctx, &format!("{stem}.a")) { + return mf; + } + } + fatal!(ctx, "library not found: {name}"); +} + +/// Reads all input files. +pub fn read_input_files(ctx: &mut Context, jobs: Vec) { + let _t = ctx.timer("read_input_files"); + + // Open files and expand archives into one job per member so that + // members are read in parallel too. Linker scripts can modify the + // context, so they are collected and parsed afterwards, one at a time + // and in command line order. + let scripts = WorkerBins::new(); + let loaded = WorkerBins::new(); + + let ctx_ref: &Context = ctx; + rayon::scope(|scope| { + jobs.into_par_iter().for_each(|job| { + let mut rctx = job.rctx.clone(); + let mf = if job.is_lib { + let mf = find_library(ctx_ref, &rctx, &job.name); + crate::util::leak(MappedFile { + name: mf.name.clone(), + data: mf.data, + given_fullpath: false, + parent: mf.parent, + thin_parent: mf.thin_parent, + is_dependency: std::sync::atomic::AtomicBool::new(true), + }) + } else { + must_open_file(&ctx_ref.diag, &ctx_ref.args.chroot, &job.name) + }; + + match get_file_type(ctx_ref, mf) { + FileType::Ar => { + for child in archive::read_fat_archive_members(&ctx_ref.diag, mf) { + let child_rctx = rctx.next_child(); + let archive_name = mf.name.clone(); + let loaded = &loaded; + scope.spawn(move |_| { + if let Some(file) = + read_archive_member(ctx_ref, &child_rctx, child, &archive_name) + { + loaded.push(file); + } + }); + } + } + FileType::ThinAr => { + for path in archive::get_thin_archive_member_paths(&ctx_ref.diag, mf) { + let child_rctx = rctx.next_child(); + let archive_name = mf.name.clone(); + let loaded = &loaded; + scope.spawn(move |_| { + let child = must_open_file(&ctx_ref.diag, &ctx_ref.args.chroot, &path); + let child = crate::util::leak(MappedFile { + name: child.name.clone(), + data: child.data, + given_fullpath: true, + parent: None, + thin_parent: Some(mf), + is_dependency: std::sync::atomic::AtomicBool::new(true), + }); + if let Some(file) = + read_archive_member(ctx_ref, &child_rctx, child, &archive_name) + { + loaded.push(file); + } + }); + } + } + FileType::Text => { + let mut job = job; + job.mf = Some(mf); + scripts.push(job); + } + FileType::ElfObj => { + let file = new_object_file(ctx_ref, &rctx, mf, ""); + loaded.push(Loaded::Obj(rctx.pos, Box::new(file))); + } + FileType::ElfDso => { + let file = new_shared_file(ctx_ref, &rctx, mf); + loaded.push(Loaded::Dso(rctx.pos, Box::new(file))); + } + FileType::GccLtoObj | FileType::LlvmBitcode => { + if let Some(file) = new_lto_object(ctx_ref, &rctx, mf, "") { + loaded.push(Loaded::Obj(rctx.pos, Box::new(file))); + } + } + _ => fatal!(ctx_ref, "{}: unknown file type", mf.name), + } + }); + }); + + for l in loaded.into_vec() { + push_loaded(ctx, l); + } + + // Parse linker scripts and read the files they name. + let mut scripts = scripts.into_vec(); + scripts.sort_by(|a, b| a.rctx.pos.cmp(&b.rctx.pos)); + for job in scripts { + let mut rctx = job.rctx.clone(); + Script::new(ctx, &mut rctx, job.mf.unwrap()).parse_linker_script(); + } + + // Sort the files into command line order and assign priorities. + let mut pending = std::mem::take(&mut ctx.pending_files); + pending.sort_by(|a, b| a.0.cmp(&b.0)); + + let objs = std::mem::take(&mut ctx.objs); + let dsos = std::mem::take(&mut ctx.dsos); + let mut objs: Vec>> = objs.into_iter().map(Some).collect(); + let mut dsos: Vec>> = dsos.into_iter().map(Some).collect(); + + // Priority 0 is reserved for the internal object file. + ctx.file_by_priority.push(None); + for (_, id) in pending { + let priority = ctx.file_by_priority.len() as u32; + match id { + FileId::Obj(idx) => { + let mut file = objs[idx.index()].take().unwrap(); + file.base.priority = priority; + if ctx.args.trace { + out!(ctx, "trace: {file}"); + } + ctx.objs.push(file); + ctx.file_by_priority + .push(Some(FileId::Obj(ObjId(ctx.objs.len() as u32 - 1)))); + } + FileId::Dso(idx) => { + let mut file = dsos[idx.index()].take().unwrap(); + file.base.priority = priority; + if ctx.args.trace { + out!(ctx, "trace: {file}"); + } + ctx.dsos.push(file); + ctx.file_by_priority + .push(Some(FileId::Dso(DsoId(ctx.dsos.len() as u32 - 1)))); + } + } + } + + if ctx.objs.is_empty() && ctx.dsos.is_empty() { + fatal!(ctx, "no input files"); + } +} diff --git a/src/relax.rs b/src/relax.rs new file mode 100644 index 000000000..b88ac2704 --- /dev/null +++ b/src/relax.rs @@ -0,0 +1,113 @@ +//! Linker relaxation that shrinks sections, for RISC-V and LoongArch. +//! +//! RISC instructions have room for only small immediates, so a branch +//! that may need to reach far is emitted as an instruction pair: RISC-V's +//! AUIPC+JALR reaches ±2 GiB where JAL alone reaches ±1 MiB. Most targets +//! have the compiler emit the short form and let the linker redirect +//! out-of-range branches through thunks. RISC-V and LoongArch do the +//! opposite: the compiler emits the long form, and the linker replaces it +//! with a shorter one when the target turns out to be close enough. +//! +//! Deleting an instruction from the middle of a section shifts everything +//! after it, so sections are no longer copied as a unit: `r_deltas` +//! records how far the bytes at each point moved, relocation offsets are +//! adjusted through it, and symbol values are adjusted in place. Sections +//! only ever shrink, so there is no oscillation to worry about. +//! +//! Since even in-section branches may cross deleted bytes, all branches +//! of these targets are expressed with relocations, which is why their +//! object files have so many. + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::{self, ChunkId}; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::{r_delta, InputSection, RelocDelta, SectionRef}; +use crate::symbol::Symbol; + +/// The distance between a relocated place and a symbol. +pub fn compute_distance( + ctx: &Context, + sym: &Symbol, + isec: &InputSection, + rel: &ElfRel, +) -> i64 { + // An absolute symbol counts as infinitely far away, since shrinking + // can increase the distance to it. Branching to an absolute address + // is extremely rare in real code anyway. + if sym.is_absolute() { + return i64::MAX; + } + let s = sym.addr(ctx) as i64; + let p = (isec.addr(ctx) + rel.r_offset) as i64; + s.wrapping_add(rel.r_addend).wrapping_sub(p) +} + +/// Finds the relaxable relocations of all executable sections, records +/// the bytes they save, and shrinks the sections accordingly. +/// +/// Relaxing one branch may bring another within reach, but such cases +/// are negligible (mold's own RISC-V build ends up ~0.04% larger than GNU +/// ld's), so relocations are scanned only once. +pub fn shrink_sections(ctx: &mut Context) { + let _t = ctx.timer("shrink_sections"); + + let shrunk: Vec<(SectionRef, Vec)> = { + let ctx: &Context = ctx; + ctx.objs + .par_iter() + .flat_map_iter(|file| { + file.input_sections() + .filter(|isec| isec.is_alive() && isec.sh_flags & SHF_EXECINSTR as u64 != 0) + .filter_map(|isec| { + let deltas = E::shrink_section(ctx, isec); + (!deltas.is_empty()).then_some(( + SectionRef { + file: isec.file, + shndx: isec.shndx, + }, + deltas, + )) + }) + .collect::>() + }) + .collect() + }; + for (r, deltas) in shrunk { + let isec = ctx.section_mut(r); + isec.sh_size -= deltas.last().unwrap().delta as u64; + ctx.set_r_deltas(r, deltas.into_boxed_slice()); + } + + // Symbols past removed bytes move with their sections. + for fi in 0..ctx.objs.len() { + let file_id = crate::input_files::FileId::Obj(crate::input_files::ObjId(fi as u32)); + for i in 0..ctx.objs[fi].base.symbols.len() { + let id = ctx.objs[fi].base.symbols[i]; + let sym = &ctx.symbols[id]; + if sym.file() != Some(file_id) { + continue; + } + let Some(isec) = sym.input_section_ref() else { + continue; + }; + if isec.sh_flags & SHF_EXECINSTR as u64 == 0 { + continue; + } + let delta = r_delta(isec, sym.value); + if delta != 0 { + ctx.symbols[id].value -= delta as u64; + } + } + } + + for id in ctx.chunks.clone() { + if let ChunkId::Output(osec) = id { + if ctx.output_sections[osec.index()].hdr.shdr.sh_flags & SHF_EXECINSTR as u64 != 0 { + chunks::compute_section_size(ctx, id); + } + } + } +} diff --git a/src/relocatable.rs b/src/relocatable.rs new file mode 100644 index 000000000..e513d8157 --- /dev/null +++ b/src/relocatable.rs @@ -0,0 +1,166 @@ +//! `-r` / `--relocatable`: combining object files into one object file. +//! +//! Regular sections are copied and merged by name, the symbol and string +//! tables are merged, COMDAT groups are uniquified, and relocations are +//! copied with their symbol indices rewritten. This matches what GNU ld +//! produces, which some programs (notably GHC's runtime linker) depend on. + +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::eh_frame::EhFrameRelocSection; +use crate::chunks::misc::{ComdatGroupSection, NotePropertySection}; +use crate::chunks::sframe::SFrameRelocSection; +use crate::chunks::symtab::ShstrtabSection; +use crate::chunks::{self, ChunkId, OutputEhdr, OutputShdr}; +use crate::context::Context; +use crate::elf::*; +use crate::input_files::FileId; +use crate::output_file::OutputFile; +use crate::passes; +use crate::util::align_to; + +fn create_synthetic_sections(ctx: &mut Context) { + ctx.ehdr = Some(OutputEhdr::new::(0)); + ctx.shdr = Some(OutputShdr::new::()); + ctx.eh_frame_reloc = Some(EhFrameRelocSection::new::()); + ctx.sframe_reloc = Some(SFrameRelocSection::new::()); + ctx.shstrtab = Some(ShstrtabSection::new()); + ctx.chunks.extend([ + ChunkId::Ehdr, + ChunkId::Shdr, + ChunkId::EhFrame, + ChunkId::EhFrameReloc, + ChunkId::SFrame, + ChunkId::SFrameReloc, + ChunkId::Strtab, + ChunkId::Symtab, + ChunkId::Shstrtab, + ]); + if E::IS_X86 { + ctx.note_property = Some(NotePropertySection::new::()); + ctx.chunks.push(ChunkId::NoteProperty); + } +} + +/// Propagates input COMDAT groups that survived uniquification as output +/// groups. +fn create_comdat_group_sections(ctx: &mut Context) { + let _t = ctx.timer("create_comdat_group_sections"); + let mut sections = Vec::new(); + for file in &ctx.objs { + for group in &file.comdat_groups { + if !group.is_owner() { + continue; + } + let sym = + file.base.symbols[file.base.shdrs.at(group.sect_idx as usize).sh_info as usize]; + let mut members = Vec::new(); + for j in file.comdat_members(group) { + let shdr = &file.base.shdrs.at(j as usize); + let is_reloc = shdr.sh_type == if E::IS_RELA { SHT_RELA } else { SHT_REL } + || shdr.sh_type == SHT_CREL; + let target = if is_reloc { shdr.sh_info } else { j }; + let Some(isec) = file.section(target as usize) else { + continue; + }; + let Some(osec) = isec.output_section else { + continue; + }; + if is_reloc { + if let Some(reloc) = ctx.output_sections[osec.index()].reloc_sec { + members.push(ChunkId::Reloc(reloc)); + } + } else { + members.push(ChunkId::Output(osec)); + } + } + sections.push(ComdatGroupSection::new(sym, members)); + } + } + for sec in sections { + ctx.comdat_group_sections.push(sec); + ctx.chunks.push(ChunkId::ComdatGroup( + ctx.comdat_group_sections.len() as u32 - 1, + )); + } +} + +/// Unresolved symbols are propagated to the output as undefined symbols +/// belonging to some input file. +fn claim_unresolved_symbols(ctx: &mut Context) { + let _t = ctx.timer("r_claim_unresolved_symbols"); + let candidates: Vec<(usize, usize)> = ctx + .objs + .par_iter() + .enumerate() + .flat_map_iter(|(fi, file)| { + (file.base.first_global..file.base.elf_syms.len()) + .filter(move |&i| file.base.elf_syms.at(i).is_undef()) + .map(move |i| (fi, i)) + }) + .collect(); + for (fi, i) in candidates { + let file = &ctx.objs[fi]; + let id = file.base.symbols[i]; + let esym = file.base.elf_syms.at(i); + let priority = file.base.priority; + let sym = &ctx.symbols[id]; + if let Some(owner) = sym.file() { + if !sym.is_undef() || ctx.file(owner).priority <= priority { + continue; + } + } + let sym = &mut ctx.symbols[id]; + sym.set_file(FileId::Obj(crate::input_files::ObjId(fi as u32))); + sym.clear_origin(); + sym.value = 0; + sym.sym_idx = i as u32; + sym.set_esym(&esym); + } +} + +/// Assigns file offsets; addresses stay zero. +fn set_osec_offsets(ctx: &mut Context) -> u64 { + let mut offset = 0; + for id in ctx.chunks.clone() { + let hdr = ctx.chunk_header_mut(id); + offset = align_to(offset, hdr.shdr.sh_addralign); + hdr.shdr.sh_offset = offset; + offset += hdr.shdr.sh_size; + } + offset +} + +pub fn combine_objects(ctx: &mut Context) { + passes::create_output_sections(ctx); + create_synthetic_sections(ctx); + claim_unresolved_symbols(ctx); + passes::compute_section_sizes(ctx); + passes::sort_output_sections(ctx); + passes::create_output_symtab(ctx); + chunks::eh_frame::construct(ctx); + chunks::sframe::construct(ctx); + passes::create_reloc_sections(ctx); + create_comdat_group_sections(ctx); + passes::compute_section_headers(ctx); + + let filesize = set_osec_offsets(ctx); + let mut output = OutputFile::open(&ctx.diag, &ctx.args.output, filesize, 0o666, false); + crate::driver::copy_chunks(ctx, output.buf()); + output.close(&ctx.diag); + ctx.checkpoint(); + + if ctx.args.print_map { + crate::mapfile::print_map(ctx); + } + if ctx.args.stats { + passes::show_stats(ctx); + } + if ctx.args.perf { + ctx.timers.print(); + } + if ctx.args.quick_exit { + crate::diagnostics::exit_after_cleanup(0); + } +} diff --git a/src/subprocess.rs b/src/subprocess.rs new file mode 100644 index 000000000..b073ca7dc --- /dev/null +++ b/src/subprocess.rs @@ -0,0 +1,127 @@ +//! Process management: forking a child to hide exit latency, signal +//! handling for disk-full errors, and the `-run` subcommand. + +use std::sync::atomic::{AtomicI32, Ordering}; + +use crate::diagnostics::Diagnostics; +use crate::fatal; + +static PIPE_WRITE_FD: AtomicI32 = AtomicI32::new(-1); + +/// Exiting a process with a large heap is slow, so the parent forks a +/// child to do the actual work and exits as soon as the child reports +/// that the output has been written. +pub fn fork_child() { + let mut pipefd = [0i32; 2]; + // SAFETY: plain libc calls with valid arguments. + unsafe { + if libc::pipe(pipefd.as_mut_ptr()) == -1 { + eprintln!("mold: pipe failed"); + std::process::exit(1); + } + let pid = libc::fork(); + if pid == -1 { + eprintln!("mold: fork failed"); + std::process::exit(1); + } + if pid > 0 { + // Parent + libc::close(pipefd[1]); + let mut buf = [0u8; 1]; + if libc::read(pipefd[0], buf.as_mut_ptr() as *mut libc::c_void, 1) == 1 { + libc::_exit(0); + } + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + if libc::WIFEXITED(status) { + libc::_exit(libc::WEXITSTATUS(status)); + } + if libc::WIFSIGNALED(status) { + libc::raise(libc::WTERMSIG(status)); + } + libc::_exit(1); + } + // Child + libc::close(pipefd[0]); + } + PIPE_WRITE_FD.store(pipefd[1], Ordering::Relaxed); +} + +/// Tells the parent that the output is complete. +pub fn notify_parent() { + let fd = PIPE_WRITE_FD.swap(-1, Ordering::Relaxed); + if fd == -1 { + return; + } + let buf = [1u8]; + // SAFETY: fd is a valid pipe write end. + unsafe { + libc::write(fd, buf.as_ptr() as *const libc::c_void, 1); + } +} + +extern "C" fn on_signal(signo: libc::c_int) { + // A write to a memory-mapped output file fails with SIGBUS if the + // disk fills up; explain that rather than crashing silently. + if signo == libc::SIGBUS { + let msg = b"mold: failed to write to an output file. Disk full?\n"; + // SAFETY: write is async-signal-safe. + unsafe { + libc::write( + libc::STDERR_FILENO, + msg.as_ptr() as *const libc::c_void, + msg.len(), + ); + } + } + crate::output_file::cleanup(); + // SAFETY: restoring the default handler and re-raising. + unsafe { + libc::signal(signo, libc::SIG_DFL); + libc::raise(signo); + } +} + +pub fn install_signal_handler() { + // SAFETY: installing a handler that only calls async-signal-safe + // functions. + unsafe { + libc::signal(libc::SIGSEGV, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGBUS, on_signal as *const () as libc::sighandler_t); + } +} + +/// `mold -run COMMAND ARGS...` runs a command with mold interposed as the +/// linker, which requires the `mold-wrapper.so` preload library. +pub fn process_run_subcommand(diag: &Diagnostics, argv: &[String]) -> ! { + if argv.len() < 3 { + fatal!(diag, "-run: argument missing"); + } + let self_path = crate::util::self_path(); + let candidates = [ + self_path.parent().map(|p| p.join("mold-wrapper.so")), + Some(std::path::PathBuf::from( + "/usr/local/lib/mold/mold-wrapper.so", + )), + self_path + .parent() + .map(|p| p.join("../lib/mold/mold-wrapper.so")), + ]; + let Some(dso) = candidates.into_iter().flatten().find(|p| p.is_file()) else { + fatal!(diag, "mold-wrapper.so is missing"); + }; + + std::env::set_var("LD_PRELOAD", &dso); + std::env::set_var("MOLD_PATH", &self_path); + + use std::os::unix::process::CommandExt; + let cmd = crate::util::path_filename(&argv[2]); + let err = if cmd == "ld" || cmd == "ld.lld" || cmd == "ld.gold" { + std::process::Command::new(&self_path) + .args(&argv[3..]) + .exec() + } else { + std::process::Command::new(&argv[2]).args(&argv[3..]).exec() + }; + fatal!(diag, "mold -run failed: {}: {err}", argv[2]); +} diff --git a/src/symbol.rs b/src/symbol.rs new file mode 100644 index 000000000..e5ddce33a --- /dev/null +++ b/src/symbol.rs @@ -0,0 +1,1922 @@ +//! Symbols and the global symbol table. +//! +//! There is one [`Symbol`] per unique global symbol name plus one per local +//! symbol of each object file. All of them live in a single arena, the +//! [`SymbolTable`], and are referred to by [`SymbolId`]. + +use std::fmt; +use std::hash::{BuildHasherDefault, Hash, Hasher}; +use std::marker::PhantomData; +use std::mem::MaybeUninit; +use std::ops::{Deref, DerefMut, Index, IndexMut, Range}; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering}; + +use bstr::BStr; +use hashbrown::{Equivalent, HashMap}; +use rayon::prelude::*; + +use crate::arch::Arch; +use crate::chunks::ChunkId; +use crate::context::Context; +use crate::diagnostics::demangle_enabled; +use crate::elf::*; +use crate::input_files::FileId; +use crate::input_sections::{FragmentRef, InputSection, SectionRef}; +use crate::util::demangle::{demangle_cpp, demangle_rust}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SymbolId(pub u32); + +impl SymbolId { + /// A placeholder for local symbols in discarded COMDAT sections. + pub const DISCARDED_COMDAT: SymbolId = SymbolId(0); + + /// No symbol. Used where an optional id must remain four bytes. + pub const NONE: SymbolId = SymbolId(u32::MAX); + + #[inline] + pub fn index(self) -> usize { + self.0 as usize + } +} + +/// The payload describing what a symbol's value is relative to. Its kind +/// lives in a byte of [`Symbol`]'s padding so that the payload remains eight +/// bytes without restricting any of the contained ids. +#[repr(C)] +#[derive(Clone, Copy)] +pub union Origin { + raw: u64, + fragment: FragmentRef, + chunk: ChunkId, + symbol: SymbolId, +} + +impl Origin { + fn none() -> Origin { + Origin { raw: 0 } + } + + fn section(section: &InputSection) -> Origin { + Origin { + raw: section as *const InputSection as usize as u64, + } + } + + fn fragment(fragment: FragmentRef) -> Origin { + let mut origin = Origin::none(); + origin.fragment = fragment; + origin + } + + fn chunk(chunk: ChunkId) -> Origin { + let mut origin = Origin::none(); + origin.chunk = chunk; + origin + } + + fn symbol(symbol: SymbolId) -> Origin { + let mut origin = Origin::none(); + origin.symbol = symbol; + origin + } +} + +impl fmt::Debug for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Origin") + } +} + +const _: () = assert!(std::mem::size_of::() == 8); + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum OriginKind { + /// An absolute symbol, or an undefined one. + #[default] + None, + Section, + Fragment, + Chunk, + /// A default-versioned alias (`foo@VERSION`) forwarding to `foo`. + Symbol, +} + +#[derive(Clone, Copy)] +pub(crate) struct OriginState { + payload: Origin, + kind: OriginKind, +} + +/// Symbol flags set while scanning relocations. +pub const NEEDS_GOT: u8 = 1 << 0; +pub const NEEDS_PLT: u8 = 1 << 1; +/// A canonical PLT entry or a copy relocation. +pub const NEEDS_CANONICAL: u8 = 1 << 2; +pub const NEEDS_GOTTP: u8 = 1 << 3; +pub const NEEDS_TLSGD: u8 = 1 << 4; +pub const NEEDS_TLSDESC: u8 = 1 << 5; +pub const NEEDS_PPC_OPD: u8 = 1 << 6; + +/// Flags for [`Symbol::addr`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AddrFlags { + /// Request an address other than the PLT entry. + pub no_plt: bool, + /// Request an address other than the OPD entry (PPC64 ELFv1 only). + pub no_opd: bool, +} + +impl AddrFlags { + pub const NO_PLT: AddrFlags = AddrFlags { + no_plt: true, + no_opd: false, + }; + pub const NO_OPD: AddrFlags = AddrFlags { + no_plt: false, + no_opd: true, + }; +} + +/// Table indices for dynamic symbols. Most symbols never need them, so +/// they are allocated on demand. +#[derive(Debug, Default)] +pub struct SymbolAux { + pub got_idx: Option, + pub gottp_idx: Option, + pub tlsgd_idx: Option, + pub tlsdesc_idx: Option, + pub plt_idx: Option, + pub pltgot_idx: Option, + pub dynsym_idx: Option, + pub opd_idx: Option, + pub djb_hash: u32, + /// Addresses of range extension thunks, in ascending order. + pub thunk_addrs: Vec, +} + +/// A nullable 32-bit file reference. C++ mold's arena pointers have the +/// same compact representation; the high bit distinguishes shared files +/// from object files and the all-ones value represents no file. +#[derive(Clone, Copy, Debug)] +struct SymbolFile(u32); + +impl SymbolFile { + const DSO: u32 = 1 << 31; + const NONE: u32 = u32::MAX; + + #[inline] + fn none() -> SymbolFile { + SymbolFile(Self::NONE) + } + + #[inline] + fn some(file: FileId) -> SymbolFile { + let raw = match file { + FileId::Obj(id) => { + debug_assert!(id.0 < Self::DSO); + id.0 + } + FileId::Dso(id) => { + debug_assert!(id.0 < Self::DSO - 1); + id.0 | Self::DSO + } + }; + SymbolFile(raw) + } + + #[inline] + fn get(self) -> Option { + match self.0 { + Self::NONE => None, + raw if raw & Self::DSO != 0 => { + Some(FileId::Dso(crate::input_files::DsoId(raw & !Self::DSO))) + } + raw => Some(FileId::Obj(crate::input_files::ObjId(raw))), + } + } +} + +#[derive(Debug)] +pub struct Symbol { + // The name bytes live in the surrounding map entry or the owner file. + // Rust's symbol map is separate from its symbol array, so retain a thin + // pointer rather than a 16-byte slice. + name_ptr: usize, + name_len: u32, + + /// Serializes the parallel updates made while resolving definitions. + /// The byte also holds the resolution-only `skip_dso` bit; both fit in + /// the structure's existing padding. + mu: AtomicU8, + + /// The file that defines the symbol, if any. If several files define + /// it, the one with the strongest definition owns it. + file: SymbolFile, + origin: Origin, + + /// The symbol value: its address if absolute, otherwise an offset + /// relative to `origin`. + pub value: u64, + + /// Index of the symbol's entry in the owner file's symbol table. The + /// entry itself is read from the file (see [`Self::esym`]); only its + /// type and binding, and the section index that tells an undefined + /// or common symbol, are kept here, being consulted constantly. + pub sym_idx: u32, + st_info: u8, + + pub ver_idx: u16, + pub visibility: AtomicU8, + pub flags: AtomicU8, + aux_idx: u32, + + /// The symbol's boolean attributes, packed; the accessors below name + /// them. + bits: u16, +} + +const _: () = assert!(std::mem::size_of::() == 48); + +const SYMBOL_LOCKED: u8 = 1 << 0; +const SYMBOL_SKIP_DSO: u8 = 1 << 1; + +const NO_AUX: u32 = u32::MAX; +const WRITE_TO_SYMTAB: u8 = 1 << 7; +const NEEDS_MASK: u8 = !WRITE_TO_SYMTAB; + +const VISIBILITY_MASK: u8 = 0b11; +const ORIGIN_KIND_SHIFT: u32 = 2; +const ORIGIN_KIND_MASK: u8 = 0b111 << ORIGIN_KIND_SHIFT; +const SYMBOL_STATE_SHIFT: u32 = 5; +const SYMBOL_STATE_MASK: u8 = 0b11 << SYMBOL_STATE_SHIFT; +const SYMBOL_UNDEFINED: u8 = 0; +const SYMBOL_COMMON: u8 = 1; +const SYMBOL_DEFINED: u8 = 2; + +const WEAK: u16 = 1 << 0; + +/// A symbol that may resolve to a definition in another ELF file at +/// runtime is imported; one that other files may use at runtime is +/// exported. Both can be true: a symbol exported from a DSO is usually +/// also imported by it, since a definition elsewhere may interpose it. +const IMPORTED: u16 = 1 << 1; +const EXPORTED: u16 = 1 << 2; + +/// Whether the symbol's address is that of its PLT entry. C guarantees +/// that function pointers compare equal process-wide, so when a +/// position-dependent executable takes the address of an imported +/// function it uses its own PLT entry as the address, and the DSO +/// defining the function must use the same address. +const CANONICAL: u16 = 1 << 3; + +/// Whether the symbol's data is copied from a DSO into the executable's +/// BSS at load time, so that non-PIC code can address it. +const COPYREL: u16 = 1 << 4; +const COPYREL_READONLY: u16 = 1 << 5; + +const TRACED: u16 = 1 << 6; +const WRAPPED: u16 = 1 << 7; + +/// For symbols with a default version, `foo@@VERSION`. +const VERSIONED_DEFAULT: u16 = 1 << 8; + +const GC_ROOT: u16 = 1 << 10; + +/// For LTO: referenced by a regular (non-IR) object. +const REFERENCED_BY_REGULAR_OBJ: u16 = 1 << 11; + +/// For LTO: the signature of a COMDAT group claimed by an IR file. +const COMDAT_CLAIMED_BY_IR: u16 = 1 << 12; + +/// A dummy symbol standing in for a relocation into a section fragment. +const FRAGMENT_DUMMY: u16 = 1 << 13; + +/// Whether the symbol comes from a Rust object, which decides how a +/// legacy-mangled name is demangled. +const RUST: u16 = 1 << 14; + +/// Defines a getter and a setter for each of the packed attributes. +macro_rules! symbol_bits { + ($($get:ident, $set:ident: $bit:ident;)*) => { + impl Symbol { + $( + #[inline] + pub fn $get(&self) -> bool { + self.bits & $bit != 0 + } + + #[inline] + pub fn $set(&mut self, on: bool) { + if on { + self.bits |= $bit; + } else { + self.bits &= !$bit; + } + } + )* + } + }; +} + +symbol_bits! { + is_weak, set_weak: WEAK; + is_imported, set_imported: IMPORTED; + is_exported, set_exported: EXPORTED; + is_canonical, set_canonical: CANONICAL; + has_copyrel, set_copyrel: COPYREL; + is_copyrel_readonly, set_copyrel_readonly: COPYREL_READONLY; + is_traced, set_traced: TRACED; + is_wrapped, set_wrapped: WRAPPED; + is_versioned_default, set_versioned_default: VERSIONED_DEFAULT; + gc_root, set_gc_root: GC_ROOT; + referenced_by_regular_obj, set_referenced_by_regular_obj: REFERENCED_BY_REGULAR_OBJ; + comdat_claimed_by_ir, set_comdat_claimed_by_ir: COMDAT_CLAIMED_BY_IR; + is_fragment_dummy, set_fragment_dummy: FRAGMENT_DUMMY; + is_rust, set_rust: RUST; +} + +impl Symbol { + #[inline] + pub fn new(name: &'static BStr) -> Symbol { + let name_len = u32::try_from(name.len()).expect("symbol name is larger than 4 GiB"); + Symbol { + name_ptr: name.as_ptr() as usize, + name_len, + mu: AtomicU8::new(0), + file: SymbolFile::none(), + origin: Origin::none(), + value: 0, + sym_idx: u32::MAX, + st_info: 0, + ver_idx: VER_NDX_UNSPECIFIED as u16, + visibility: AtomicU8::new(STV_DEFAULT as u8), + flags: AtomicU8::new(0), + aux_idx: NO_AUX, + bits: 0, + } + } + + /// The name storage outlives the link. Only [`Self::new`] sets the + /// pointer, and it accepts a static string. + #[inline] + pub fn name(&self) -> &'static BStr { + // SAFETY: `name_ptr` and `name_len` came from the same static slice + // in `new` and are never modified. + BStr::new(unsafe { + std::slice::from_raw_parts(self.name_ptr as *const u8, self.name_len as usize) + }) + } + + #[inline] + pub fn file(&self) -> Option { + self.file.get() + } + + #[inline] + pub fn set_file(&mut self, file: FileId) { + self.file = SymbolFile::some(file); + } + + #[inline] + pub fn clear_file(&mut self) { + self.file = SymbolFile::none(); + } + + /// Prevents a definition in a DSO from resolving the symbol. + #[inline] + pub fn skip_dso(&self) -> bool { + self.mu.load(Ordering::Relaxed) & SYMBOL_SKIP_DSO != 0 + } + + #[inline] + pub fn set_skip_dso(&self, on: bool) { + if on { + self.mu.fetch_or(SYMBOL_SKIP_DSO, Ordering::Relaxed); + } else { + self.mu.fetch_and(!SYMBOL_SKIP_DSO, Ordering::Relaxed); + } + } + + /// Reads `skip_dso` without forming a reference to the rest of a Symbol + /// being edited by another resolver worker. + /// + /// # Safety + /// + /// `ptr` must point to a live Symbol. + #[inline] + pub(crate) unsafe fn skip_dso_at(ptr: *const Symbol) -> bool { + let mu = unsafe { std::ptr::addr_of!((*ptr).mu) }; + unsafe { &*mu }.load(Ordering::Relaxed) & SYMBOL_SKIP_DSO != 0 + } + + /// Runs `f` while holding this symbol's resolution lock. + /// + /// # Safety + /// + /// `ptr` must come from an exclusively borrowed symbol table that stays + /// in place for the call. All concurrent access to the pointed-to symbol + /// must use this function. + #[inline(always)] + pub(crate) unsafe fn with_resolution_lock( + ptr: *mut Symbol, + f: impl FnOnce(&mut Symbol) -> R, + ) -> R { + let mu = unsafe { std::ptr::addr_of!((*ptr).mu) }; + let mut unlocked = 0; + loop { + match unsafe { &*mu }.compare_exchange_weak( + unlocked, + unlocked | SYMBOL_LOCKED, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(mut actual) => { + while actual & SYMBOL_LOCKED != 0 { + std::hint::spin_loop(); + actual = unsafe { &*mu }.load(Ordering::Relaxed); + } + unlocked = actual; + } + } + } + + struct Guard(*const AtomicU8, u8); + impl Drop for Guard { + #[inline(always)] + fn drop(&mut self) { + unsafe { &*self.0 }.store(self.1, Ordering::Release); + } + } + + let _guard = Guard(mu, unlocked); + unsafe { f(&mut *ptr) } + } + + #[inline] + fn set_visibility_bits(&self, mask: u8, value: u8) { + let mut cur = self.visibility.load(Ordering::Relaxed); + loop { + let new = (cur & !mask) | (value & mask); + match self.visibility.compare_exchange_weak( + cur, + new, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return, + Err(actual) => cur = actual, + } + } + } + + #[inline] + fn origin_kind(&self) -> OriginKind { + match (self.visibility.load(Ordering::Relaxed) & ORIGIN_KIND_MASK) >> ORIGIN_KIND_SHIFT { + 0 => OriginKind::None, + 1 => OriginKind::Section, + 2 => OriginKind::Fragment, + 3 => OriginKind::Chunk, + 4 => OriginKind::Symbol, + _ => unreachable!(), + } + } + + #[inline] + fn set_origin_kind(&mut self, kind: OriginKind) { + let bits = self.visibility.get_mut(); + *bits = (*bits & !ORIGIN_KIND_MASK) | ((kind as u8) << ORIGIN_KIND_SHIFT); + } + + #[inline] + fn symbol_state(&self) -> u8 { + (self.visibility.load(Ordering::Relaxed) & SYMBOL_STATE_MASK) >> SYMBOL_STATE_SHIFT + } + + #[inline] + fn set_symbol_state(&mut self, state: u8) { + let bits = self.visibility.get_mut(); + *bits = (*bits & !SYMBOL_STATE_MASK) | (state << SYMBOL_STATE_SHIFT); + } + + #[inline] + pub fn visibility(&self) -> u32 { + (self.visibility.load(Ordering::Relaxed) & VISIBILITY_MASK) as u32 + } + + #[inline] + pub fn set_visibility(&self, v: u32) { + self.set_visibility_bits(VISIBILITY_MASK, v as u8); + } + + /// Narrows the visibility to the most restrictive of the current one + /// and `vis`: a hidden reference makes a symbol hidden. + #[inline] + pub fn merge_visibility(&self, vis: u32) { + let vis = if vis == STV_INTERNAL { STV_HIDDEN } else { vis } as u8; + let rank = |v: u8| match v as u32 { + STV_HIDDEN => 1, + STV_PROTECTED => 2, + _ => 3, + }; + let mut cur = self.visibility.load(Ordering::Relaxed); + while rank(vis) < rank(cur & VISIBILITY_MASK) { + let new = (cur & !VISIBILITY_MASK) | vis; + match self.visibility.compare_exchange_weak( + cur, + new, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => cur = actual, + } + } + } + + #[inline] + pub fn flags(&self) -> u8 { + self.flags.load(Ordering::Relaxed) & NEEDS_MASK + } + + #[inline] + pub fn add_flags(&self, flags: u8) { + debug_assert_eq!(flags & WRITE_TO_SYMTAB, 0); + self.flags.fetch_or(flags, Ordering::Relaxed); + } + + #[inline] + pub fn write_to_symtab(&self) -> bool { + self.flags.load(Ordering::Relaxed) & WRITE_TO_SYMTAB != 0 + } + + #[inline] + pub fn set_write_to_symtab(&self) { + self.flags.fetch_or(WRITE_TO_SYMTAB, Ordering::Relaxed); + } + + #[inline] + pub fn clear_flags(&self) { + self.flags.fetch_and(WRITE_TO_SYMTAB, Ordering::Relaxed); + } + + /// Marks the symbol, returning true if it wasn't marked yet. Once the + /// NEEDS_* flags have been turned into GOT and PLT entries, the field + /// is free to serve as a scratch mark, which thunk creation uses. + #[inline] + pub fn mark(&self) -> bool { + // A relaxed load + branch (assuming miss) takes only around 20 cycles, + // while an atomic RMW can easily take hundreds on x86. We note that it's + // common that another thread beat us in marking, so doing an optimistic + // early test tends to improve performance in the ~20% ballpark. + if self.flags.load(Ordering::Relaxed) & NEEDS_MASK != 0 { + return false; + } + self.flags.fetch_or(1, Ordering::Relaxed) & NEEDS_MASK == 0 + } + + #[inline] + pub fn is_marked(&self) -> bool { + self.flags.load(Ordering::Relaxed) & NEEDS_MASK != 0 + } + + #[inline] + pub fn unmark(&self) { + self.clear_flags(); + } + + pub fn aux<'a>(&self, symbols: &'a SymbolTable) -> Option<&'a SymbolAux> { + (self.aux_idx != NO_AUX).then(|| &symbols.aux[self.aux_idx as usize]) + } + + pub fn got_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.got_idx) + } + + pub fn gottp_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.gottp_idx) + } + + pub fn tlsgd_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.tlsgd_idx) + } + + pub fn tlsdesc_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.tlsdesc_idx) + } + + pub fn plt_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.plt_idx) + } + + pub fn pltgot_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.pltgot_idx) + } + + pub fn dynsym_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.dynsym_idx) + } + + pub fn opd_idx(&self, symbols: &SymbolTable) -> Option { + self.aux(symbols).and_then(|a| a.opd_idx) + } + + #[inline] + pub fn has_plt(&self, symbols: &SymbolTable) -> bool { + self.plt_idx(symbols).is_some() || self.pltgot_idx(symbols).is_some() + } + + #[inline] + pub fn has_got(&self, symbols: &SymbolTable) -> bool { + self.got_idx(symbols).is_some() + } + + #[inline] + pub fn has_gottp(&self, symbols: &SymbolTable) -> bool { + self.gottp_idx(symbols).is_some() + } + + #[inline] + pub fn has_tlsgd(&self, symbols: &SymbolTable) -> bool { + self.tlsgd_idx(symbols).is_some() + } + + #[inline] + pub fn has_tlsdesc(&self, symbols: &SymbolTable) -> bool { + self.tlsdesc_idx(symbols).is_some() + } + + pub fn has_opd(&self, symbols: &SymbolTable) -> bool { + self.opd_idx(symbols).is_some() + } + + #[inline] + pub fn input_section(&self) -> Option { + self.input_section_ref().map(|section| SectionRef { + file: section.file, + shndx: section.shndx, + }) + } + + /// The input section itself, stored directly as in C++'s tagged origin. + #[inline] + pub fn input_section_ref(&self) -> Option<&InputSection> { + if self.origin_kind() == OriginKind::Section { + // SAFETY: input sections have stable arena addresses, and + // mergeable conversion leaves them in those slots. + Some(unsafe { &*(self.origin.raw as usize as *const InputSection) }) + } else { + None + } + } + + #[inline] + pub fn fragment(&self) -> Option { + if self.origin_kind() == OriginKind::Fragment { + // SAFETY: the kind records the union field last written. + Some(unsafe { self.origin.fragment }) + } else { + None + } + } + + pub fn output_chunk(&self) -> Option { + if self.origin_kind() == OriginKind::Chunk { + // SAFETY: the kind records the union field last written. + Some(unsafe { self.origin.chunk }) + } else { + None + } + } + + pub fn symbol_origin(&self) -> Option { + if self.origin_kind() == OriginKind::Symbol { + // SAFETY: the kind records the union field last written. + Some(unsafe { self.origin.symbol }) + } else { + None + } + } + + #[inline] + pub fn clear_origin(&mut self) { + self.origin = Origin::none(); + self.set_origin_kind(OriginKind::None); + } + + #[inline] + pub fn set_input_section(&mut self, section: &InputSection) { + self.origin = Origin::section(section); + self.set_origin_kind(OriginKind::Section); + } + + #[inline] + pub fn set_fragment(&mut self, fragment: FragmentRef) { + self.origin = Origin::fragment(fragment); + self.set_origin_kind(OriginKind::Fragment); + } + + #[inline] + pub fn set_output_chunk(&mut self, chunk: ChunkId) { + self.origin = Origin::chunk(chunk); + self.set_origin_kind(OriginKind::Chunk); + } + + #[inline] + pub fn set_symbol_origin(&mut self, symbol: SymbolId) { + self.origin = Origin::symbol(symbol); + self.set_origin_kind(OriginKind::Symbol); + } + + #[inline] + pub(crate) fn origin_state(&self) -> OriginState { + OriginState { + payload: self.origin, + kind: self.origin_kind(), + } + } + + #[inline] + pub(crate) fn set_origin_state(&mut self, state: OriginState) { + self.origin = state.payload; + self.set_origin_kind(state.kind); + } + + /// The symbol's entry in the owner file's symbol table; a blank one + /// for a symbol no file defines. + #[inline] + pub fn esym(&self, ctx: &Context) -> ElfSym { + match self.file() { + Some(file) => ctx.file(file).elf_syms.at_in::(self.sym_idx as usize), + None => ElfSym::default(), + } + } + + /// Records the file's entry the symbol now refers to (see + /// [`Self::esym`]). + #[inline] + pub fn set_esym(&mut self, esym: &ElfSym) { + self.st_info = esym.st_info; + let state = if esym.st_shndx as u32 == SHN_UNDEF { + SYMBOL_UNDEFINED + } else if esym.st_shndx as u32 == SHN_COMMON { + SYMBOL_COMMON + } else { + SYMBOL_DEFINED + }; + self.set_symbol_state(state); + } + + #[inline] + pub fn st_type(&self) -> u32 { + (self.st_info & 0xf) as u32 + } + + #[inline] + pub fn st_bind(&self) -> u32 { + (self.st_info >> 4) as u32 + } + + #[inline] + pub fn is_undef(&self) -> bool { + self.symbol_state() == SYMBOL_UNDEFINED + } + + #[inline] + pub fn is_common(&self) -> bool { + self.symbol_state() == SYMBOL_COMMON + } + + #[inline] + pub fn is_undef_weak(&self) -> bool { + self.is_undef() && self.st_bind() == STB_WEAK + } + + /// The symbol type, treating an IFUNC defined in a DSO as a plain + /// function since the resolver runs inside that DSO. + #[inline] + pub fn ty(&self) -> u32 { + let ty = self.st_type(); + if ty == STT_GNU_IFUNC && matches!(self.file(), Some(FileId::Dso(_))) { + STT_FUNC + } else { + ty + } + } + + #[inline] + pub fn is_ifunc(&self) -> bool { + self.ty() == STT_GNU_IFUNC + } + + /// An unresolved weak symbol acts as an absolute symbol at address 0. + #[inline] + pub fn is_remaining_undef_weak(&self) -> bool { + !self.is_imported() && self.is_undef_weak() + } + + #[inline] + pub fn is_absolute(&self) -> bool { + if self.is_remaining_undef_weak() { + return true; + } + !self.is_imported() && self.origin_kind() == OriginKind::None + } + + #[inline] + pub fn is_relative(&self) -> bool { + !self.is_absolute() + } + + /// Whether the symbol appears as a local symbol in the output symbol + /// table. A global symbol may be demoted by visibility or by a version + /// script; not being exported to the dynamic symbol table is not + /// enough. + #[inline] + pub fn is_local(&self, ctx: &Context) -> bool { + if self.st_bind() == STB_LOCAL { + return true; + } + if ctx.args.relocatable { + return false; + } + let vis = self.visibility(); + vis == STV_HIDDEN || vis == STV_INTERNAL || self.ver_idx as u32 == VER_NDX_LOCAL + } + + /// An IFUNC in a position-dependent executable occupies two GOT slots: + /// the PLT address, used as the symbol's address, and the resolved one. + pub fn is_pde_ifunc(&self, ctx: &Context) -> bool { + self.is_ifunc() && !ctx.args.pic && !E::IS_PPC64 + } + + /// Whether the PC-relative address is known at link time. + pub fn is_pcrel_linktime_const(&self, ctx: &Context) -> bool { + !self.is_imported() && !self.is_ifunc() && (self.is_relative() || !ctx.args.pic) + } + + /// Whether the thread-pointer-relative address is known at link time. + pub fn is_tprel_linktime_const(&self, ctx: &Context) -> bool { + debug_assert_eq!(self.ty(), STT_TLS); + !ctx.args.shared && !self.is_imported() + } + + /// Whether the thread-pointer-relative address is known at load time, + /// i.e. unless we are creating a dlopen'able DSO. + pub fn is_tprel_runtime_const(&self, ctx: &Context) -> bool { + debug_assert_eq!(self.ty(), STT_TLS); + !(ctx.args.shared && ctx.args.z_dlopen) + } + + /// The symbol's address, taking PLT, copy relocations and section + /// fragments into account. + #[inline] + pub fn addr(&self, ctx: &Context) -> u64 { + self.addr_with(ctx, AddrFlags::default()) + } + + #[inline] + pub fn addr_with(&self, ctx: &Context, flags: AddrFlags) -> u64 { + if let Some(frag_ref) = self.fragment() { + let frag = ctx.fragment(frag_ref); + if !frag.is_alive() { + // A non-alloc section (typically debug info) refers to a + // piece of an alloc section that was garbage-collected. + return 0; + } + return ctx.fragment_addr(frag_ref) + self.value; + } + + if self.has_copyrel() { + let chunk = if self.is_copyrel_readonly() { + &ctx.copyrel_relro + } else { + &ctx.copyrel + }; + return chunk.hdr.shdr.sh_addr + self.value; + } + + if E::FAMILY == crate::arch::Family::Ppc64V1 && !flags.no_opd && self.has_opd(&ctx.symbols) + { + return self.opd_addr(ctx); + } + + if !flags.no_plt && self.has_plt(&ctx.symbols) { + debug_assert!(self.is_imported() || self.is_ifunc()); + return self.plt_addr(ctx); + } + + match self.input_section_ref() { + Some(isec) => { + if !isec.is_alive() { + if let Some(leader) = isec.icf_leader() { + return ctx.section(leader).addr(ctx) + self.value; + } + + if isec.name(&ctx.objs[isec.file.index()]) == b".eh_frame" { + // .eh_frame contents are parsed and reconstructed by + // the linker, so a pointer into an input .eh_frame + // isn't meaningful; but CRT files define symbols at + // the very beginning and end of the section. + let name = self.name(); + let eh_frame = &ctx.eh_frame.hdr.shdr; + if name.starts_with(b"__EH_FRAME_BEGIN__") + || name.starts_with(b"__EH_FRAME_LIST__") + || name.starts_with(b".eh_frame_seg") + || self.st_type() == STT_SECTION + { + return eh_frame.sh_addr; + } + if name.starts_with(b"__FRAME_END__") + || name.starts_with(b"__EH_FRAME_LIST_END__") + { + return eh_frame.sh_addr + eh_frame.sh_size; + } + // ARM object files contain "$d" local symbols at + // the beginning of data sections. Their values are + // not significant for .eh_frame. + if name == b"$d" || name.starts_with(b"$d.") { + return eh_frame.sh_addr; + } + crate::fatal!( + ctx, + "symbol referring to .eh_frame is not supported: {} {}", + self, + ctx.file_display(self.file().unwrap()) + ); + } + + // A relocation refers to a local symbol in a COMDAT + // section that was discarded. This violates the spec, + // which allows only global symbols to refer to COMDAT + // members, but .eh_frame tends to do it. + return 0; + } + isec.addr(ctx) + self.value + } + // Synthetic symbols hold their final address in `value`, as do + // absolute ones. + None => self.value, + } + } + + #[inline] + pub fn got_addr(&self, ctx: &Context) -> u64 { + ctx.got.hdr.shdr.sh_addr + self.got_idx(&ctx.symbols).unwrap() as u64 * E::WORD_SIZE as u64 + } + + pub fn gotplt_addr(&self, ctx: &Context) -> u64 { + ctx.gotplt.hdr.shdr.sh_addr + + crate::chunks::got::gotplt::header_size::() + + self.plt_idx(&ctx.symbols).unwrap() as u64 + * crate::chunks::got::gotplt::entry_size::() + } + + pub fn gottp_addr(&self, ctx: &Context) -> u64 { + ctx.got.hdr.shdr.sh_addr + + self.gottp_idx(&ctx.symbols).unwrap() as u64 * E::WORD_SIZE as u64 + } + + pub fn tlsgd_addr(&self, ctx: &Context) -> u64 { + ctx.got.hdr.shdr.sh_addr + + self.tlsgd_idx(&ctx.symbols).unwrap() as u64 * E::WORD_SIZE as u64 + } + + pub fn tlsdesc_addr(&self, ctx: &Context) -> u64 { + ctx.got.hdr.shdr.sh_addr + + self.tlsdesc_idx(&ctx.symbols).unwrap() as u64 * E::WORD_SIZE as u64 + } + + #[inline] + pub fn plt_addr(&self, ctx: &Context) -> u64 { + if let Some(idx) = self.plt_idx(&ctx.symbols) { + return ctx.plt.hdr.shdr.sh_addr + crate::chunks::got::plt::entry_offset::(idx); + } + ctx.pltgot.hdr.shdr.sh_addr + self.pltgot_idx(&ctx.symbols).unwrap() as u64 * E::PLTGOT_SIZE + } + + pub fn opd_addr(&self, ctx: &Context) -> u64 { + ctx.opd_addr(self.opd_idx(&ctx.symbols).unwrap()) + } + + /// The GOT slot a PLT entry should load its target from: for an IFUNC + /// in a position-dependent executable that is the second of its two + /// slots. + pub fn got_pltgot_addr(&self, ctx: &Context) -> u64 { + if self.is_pde_ifunc(ctx) { + self.got_addr(ctx) + E::WORD_SIZE as u64 + } else { + self.got_addr(ctx) + } + } + + /// Finds a thunk within branch range of `p`. + pub fn thunk_addr(&self, ctx: &Context, p: u64) -> u64 { + let distance = E::branch_distance() as u64; + let addrs: &[u64] = self.aux(&ctx.symbols).map_or(&[], |a| &a.thunk_addrs); + let lo = p.saturating_sub(distance); + let i = addrs.partition_point(|&a| a < lo); + if let Some(&addr) = addrs.get(i) { + let disp = addr as i64 - p as i64; + if -(distance as i64) <= disp && disp < distance as i64 { + return addr; + } + } + crate::fatal!(ctx, "range extension thunk out of range: {}", self); + } + + /// The symbol's index in the output symbol table. + pub fn output_sym_idx(&self, ctx: &Context) -> u32 { + let file = ctx.file(self.file().unwrap()); + let i = file.output_sym_indices[self.sym_idx as usize]; + debug_assert!(i >= 0); + if self.is_local(ctx) { + file.local_symtab_idx + i as u32 + } else { + file.global_symtab_idx + i as u32 + } + } + + /// The version string of a symbol defined in a DSO. + pub fn version(&self, ctx: &Context) -> &'static [u8] { + if let Some(FileId::Dso(id)) = self.file() { + let dso = &ctx.dsos[id.index()]; + if let Some(&ver) = dso.version_strings.get(self.ver_idx as usize) { + return ver; + } + } + b"" + } + + /// Returns the demangled name if demangling is possible. + pub fn demangled(&self) -> Option { + if self.is_rust() || self.name().starts_with(b"_R") { + demangle_rust(self.name()) + } else { + demangle_cpp(self.name()) + } + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if demangle_enabled() { + if let Some(name) = self.demangled() { + return f.write_str(&name); + } + } + write!(f, "{}", self.name()) + } +} + +/// The length of the symbol name in a key, which may carry a version +/// suffix (`foo@VER`). +#[inline] +pub fn name_len(key: &[u8]) -> usize { + crate::util::find_byte(b'@', key).unwrap_or(key.len()) +} + +/// A key with its hash. Recording a key already computed its hash; it is +/// stored in the map key so that the map does not scan the string again. +#[derive(Clone, Copy, Debug)] +struct Key { + hash: u64, + key: &'static [u8], +} + +impl PartialEq for Key { + fn eq(&self, other: &Self) -> bool { + self.key == other.key + } +} + +impl Eq for Key {} + +impl Hash for Key { + fn hash(&self, state: &mut H) { + state.write_u64(self.hash); + } +} + +/// A key for lookups, which needn't outlive the lookup. +struct Query<'a> { + hash: u64, + key: &'a [u8], +} + +impl Hash for Query<'_> { + fn hash(&self, state: &mut H) { + state.write_u64(self.hash); + } +} + +impl Equivalent for Query<'_> { + fn equivalent(&self, key: &Key) -> bool { + self.key == key.key + } +} + +/// Passes a key's precomputed hash through. +#[derive(Default)] +struct PassThroughHasher(u64); + +impl Hasher for PassThroughHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, _: &[u8]) { + unreachable!("keys hash by their precomputed hash") + } + + fn write_u64(&mut self, hash: u64) { + self.0 = hash; + } +} + +type ShardMap = HashMap>; + +const NUM_SHARDS: usize = 64; + +/// The hash of a symbol table key, which the sharded table and its bins +/// share. +pub fn hash_key(key: &[u8]) -> u64 { + xxhash_rust::xxh3::xxh3_64(key) +} + +fn shard_of(hash: u64) -> usize { + (hash % NUM_SHARDS as u64) as usize +} + +/// A key recorded for interning, with the slot that receives its symbol. +/// A slot is an owner and an index whose meaning is up to the recorder. +#[derive(Clone, Copy, Debug)] +struct Pending { + key: Key, + name_len: u32, + slot: S, +} + +/// The keys one task recorded for interning, grouped by shard. +#[derive(Debug)] +pub struct Bins(Vec>>); + +impl Default for Bins { + fn default() -> Self { + Self::new() + } +} + +impl Bins { + pub fn new() -> Bins { + Bins((0..NUM_SHARDS).map(|_| Vec::new()).collect()) + } + + /// Records `key`, whose symbol is named by its first `name_len` bytes. + #[inline] + pub fn record(&mut self, key: &'static [u8], name_len: usize, slot: S) { + self.record_hashed(key, hash_key(key), name_len, slot); + } + + /// Records a key whose hash was computed along with the key, as the + /// files' keys are hashed while the files are read. + #[inline] + pub fn record_hashed(&mut self, key: &'static [u8], hash: u64, name_len: usize, slot: S) { + debug_assert_eq!(hash, hash_key(key)); + self.0[shard_of(hash)].push(Pending { + key: Key { hash, key }, + name_len: name_len as u32, + slot, + }); + } +} + +/// A stable slot in an input file that receives an interned symbol id. +/// +/// Files allocate their complete symbol-id arrays before recording keys and +/// keep them alive until `gather`, just as C++ keeps an ArenaPtr slot stable. +#[derive(Clone, Copy, Debug)] +pub(crate) struct SymbolSlot(NonNull); + +// SAFETY: each slot is recorded exactly once and written by the one shard +// that owns its key. The owner file remains alive until gathering finishes. +unsafe impl Send for SymbolSlot {} +unsafe impl Sync for SymbolSlot {} + +impl SymbolSlot { + #[inline] + pub(crate) fn new(slot: &mut SymbolId) -> SymbolSlot { + SymbolSlot(NonNull::from(slot)) + } + + #[inline] + fn assign(self, id: SymbolId) { + // SAFETY: guaranteed by the construction and synchronization rules + // documented on SymbolSlot. + unsafe { self.0.write(id) }; + } +} + +/// A thread-safe bump allocator over the unused tail of a symbol table. +/// C++ mold's arena lets each file allocate its local symbols while that +/// file is being parsed; this provides the same stable, disjoint ranges in +/// the central Rust arena. +pub struct ParallelSymbolAllocator<'a> { + slots: AtomicPtr>, + first: usize, + next: AtomicUsize, + maximum: usize, + marker: PhantomData<&'a mut [MaybeUninit]>, +} + +impl ParallelSymbolAllocator<'_> { + /// Allocates `n` consecutive symbols and passes their id and storage to + /// `init`. + /// + /// # Safety + /// + /// `init` must initialize every element of the slice it is given. + pub unsafe fn allocate( + &self, + n: usize, + init: impl FnOnce(SymbolId, &mut [MaybeUninit]), + ) -> SymbolId { + let offset = self.next.fetch_add(n, Ordering::Relaxed); + let end = offset.checked_add(n).expect("too many symbols"); + assert!(end <= self.maximum); + let base_id = SymbolId((self.first + offset) as u32); + let ptr = self.slots.load(Ordering::Relaxed); + // SAFETY: the table reserved `maximum` slots before publishing the + // pointer, and the atomic bump pointer gives this call an exclusive + // range within them. + let slots = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), n) }; + init(base_id, slots); + base_id + } +} + +/// A sparsely-backed address range for symbols. Allocation is monotonic and +/// individual symbols are not freed. Reserving the complete range up front +/// keeps symbol addresses stable and lets the kernel back the densely filled +/// prefix with transparent huge pages, as C++ mold's `ArenaResource` does. +struct SymbolArena { + data: NonNull, + len: usize, + size: usize, +} + +// SAFETY: initialized symbols follow their own Send and Sync bounds. Growing +// the initialized prefix requires an exclusive borrow of the arena. +unsafe impl Send for SymbolArena {} +unsafe impl Sync for SymbolArena {} + +impl SymbolArena { + // C++ mold reserves 8 GiB on 64-bit hosts and a smaller range where + // address space is limited. The mapping is sparse, so unused pages consume + // neither physical memory nor page-table entries. + const SIZE: usize = if usize::BITS == 64 { + 1usize << 33 + } else { + 1usize << 28 + }; + + fn new() -> SymbolArena { + let flags = libc::MAP_ANONYMOUS | libc::MAP_PRIVATE; + #[cfg(any(target_os = "android", target_os = "linux"))] + let flags = flags | libc::MAP_NORESERVE; + + // SAFETY: this creates private anonymous storage. No typed access is + // made until an element has been initialized below. + let data = unsafe { + libc::mmap( + std::ptr::null_mut(), + Self::SIZE, + libc::PROT_READ | libc::PROT_WRITE, + flags, + -1, + 0, + ) + }; + if data == libc::MAP_FAILED { + panic!( + "mmap of {} bytes for symbols failed: {}", + Self::SIZE, + std::io::Error::last_os_error() + ); + } + + // Large links fill the beginning of the arena densely. Transparent + // huge pages reduce address-translation overhead without populating + // unused pages. + #[cfg(any(target_os = "android", target_os = "linux"))] + // SAFETY: the range is the fresh mapping; the advice is only a hint. + unsafe { + libc::madvise(data, Self::SIZE, libc::MADV_HUGEPAGE); + } + + SymbolArena { + data: NonNull::new(data.cast()).expect("mmap returned a null address"), + len: 0, + size: Self::SIZE, + } + } + + #[inline] + fn capacity(&self) -> usize { + self.size / std::mem::size_of::() + } + + #[inline] + fn reserve(&self, additional: usize) { + let end = self.len.checked_add(additional).expect("too many symbols"); + assert!(end <= self.capacity(), "symbol arena is full"); + } + + #[inline] + fn push(&mut self, symbol: Symbol) { + self.reserve(1); + // SAFETY: reserve proved this is the first uninitialized slot. + unsafe { self.data.as_ptr().add(self.len).write(symbol) }; + self.len += 1; + } + + #[inline] + fn as_mut_ptr(&mut self) -> *mut Symbol { + self.data.as_ptr() + } + + /// # Safety + /// + /// Every element added to the initialized prefix must have been written. + #[inline] + unsafe fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.capacity()); + self.len = len; + } +} + +impl Deref for SymbolArena { + type Target = [Symbol]; + + fn deref(&self) -> &[Symbol] { + // SAFETY: `len` covers exactly the initialized prefix. + unsafe { std::slice::from_raw_parts(self.data.as_ptr(), self.len) } + } +} + +impl DerefMut for SymbolArena { + fn deref_mut(&mut self) -> &mut [Symbol] { + // SAFETY: an exclusive arena borrow gives exclusive access to the + // initialized prefix. + unsafe { std::slice::from_raw_parts_mut(self.data.as_ptr(), self.len) } + } +} + +impl fmt::Debug for SymbolArena { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Drop for SymbolArena { + fn drop(&mut self) { + // SAFETY: the prefix contains initialized symbols, and the complete + // address range is the mapping created in `new`. + unsafe { + std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut( + self.data.as_ptr(), + self.len, + )); + libc::munmap(self.data.as_ptr().cast(), self.size); + } + } +} + +/// Mutable access to the stable symbol-arena blocks owned by map shards. +struct SymbolBlockPtr(*mut Symbol); + +// SAFETY: `SymbolTable::par_for_each_global_mut` gives each parallel task the +// non-overlapping blocks owned by one shard while holding the arena exclusively. +unsafe impl Sync for SymbolBlockPtr {} + +/// A pointer to the auxiliary arena during a parallel scatter to unique symbols. +struct SymbolAuxPtr(*mut SymbolAux); + +// SAFETY: the two methods using this wrapper require distinct symbol ids, so +// their parallel tasks access non-overlapping auxiliary records. +unsafe impl Sync for SymbolAuxPtr {} + +impl SymbolAuxPtr { + /// Applies `f` to the record at `index`. + /// + /// # Safety + /// The caller must have exclusive access to this record. + unsafe fn with_mut(&self, index: usize, f: impl FnOnce(&mut SymbolAux) -> R) -> R { + // SAFETY: the caller supplies a valid, exclusively owned index. + f(unsafe { &mut *self.0.add(index) }) + } +} + +impl SymbolBlockPtr { + /// Applies `f` to the symbols in non-overlapping arena ranges. + /// + /// # Safety + /// The ranges must be initialized and exclusively owned by this task. + unsafe fn for_each(&self, ranges: &[Range], f: &(impl Fn(&mut Symbol) + Sync)) { + for range in ranges { + let len = (range.end - range.start) as usize; + // SAFETY: guaranteed by the caller for every shard block. + let symbols = + unsafe { std::slice::from_raw_parts_mut(self.0.add(range.start as usize), len) }; + symbols.iter_mut().for_each(&f); + } + } +} + +/// The arena of all symbols, and the index of global ones by name. +/// +/// The index is a map from strings to symbols, built in two phases. In +/// the first phase, which may run in parallel, [`Bins::record`] records a +/// key and a slot that needs the key's symbol. [`SymbolTable::gather`] +/// then deduplicates the keys, finds or creates one symbol for each key, +/// and hands its id to every recorded slot. +/// +/// Keys whose hashes fall into different shards never interact, and each +/// shard is processed by exactly one thread during `gather`, so unlike +/// with a concurrent hash table, no synchronization is needed, and each +/// key is hashed only once, when recorded. +/// +/// [`SymbolTable::intern`] handles keys that arrive outside the two-phase +/// pattern, one at a time. +#[derive(Debug)] +pub struct SymbolTable { + symbols: SymbolArena, + aux: Vec, + shards: Vec, + + /// The arena blocks of named symbols owned by each map shard. The other + /// symbols are files' local symbols. + globals: Vec>>, +} + +impl Default for SymbolTable { + fn default() -> Self { + Self::new() + } +} + +impl SymbolTable { + pub fn new() -> SymbolTable { + let mut table = SymbolTable { + symbols: SymbolArena::new(), + aux: Vec::new(), + shards: (0..NUM_SHARDS).map(|_| ShardMap::default()).collect(), + globals: (0..NUM_SHARDS).map(|_| Vec::new()).collect(), + }; + let placeholder = table.add(Symbol::new(BStr::new(b""))); + debug_assert_eq!(placeholder, SymbolId::DISCARDED_COMDAT); + table + } + + /// Returns the global symbol for `key`, creating it if necessary. The + /// key may carry a version suffix (`foo@VER`), which is not part of + /// the symbol's name. + pub fn intern(&mut self, key: &'static [u8]) -> SymbolId { + self.intern_with_name(key, &key[..name_len(key)]) + } + + /// Interns `key` for a symbol named `name`, a prefix of the key. + pub fn intern_with_name(&mut self, key: &'static [u8], name: &'static [u8]) -> SymbolId { + debug_assert!(key.starts_with(name)); + let hash = hash_key(key); + let shard_idx = shard_of(hash); + let shard = &mut self.shards[shard_idx]; + if let Some(&id) = shard.get(&Key { hash, key }) { + return id; + } + let id = SymbolId(self.symbols.len() as u32); + self.symbols.push(Symbol::new(BStr::new(name))); + shard.insert(Key { hash, key }, id); + self.note_globals(shard_idx, id.0..id.0 + 1); + id + } + + /// Records that the symbols in `range` are globals owned by `shard`. + fn note_globals(&mut self, shard: usize, range: Range) { + let globals = &mut self.globals[shard]; + match globals.last_mut() { + Some(last) if last.end == range.start => last.end = range.end, + _ => globals.push(range), + } + } + + /// Makes room for `additional` more symbols, so that the table isn't + /// copied when it grows. + pub fn reserve(&mut self, additional: usize) { + self.symbols.reserve(additional); + } + + /// Returns the auxiliary record for `id`, allocating it in the side + /// arena if this symbol has none. C++ mold likewise keeps this rarely + /// used state outside `Symbol` and refers to it with an arena index. + pub fn aux_mut(&mut self, id: SymbolId) -> &mut SymbolAux { + let mut index = self.symbols[id.index()].aux_idx; + if index == NO_AUX { + index = self.aux.len() as u32; + assert_ne!(index, NO_AUX, "too many symbol auxiliary records"); + self.aux.push(SymbolAux::default()); + self.symbols[id.index()].aux_idx = index; + } + &mut self.aux[index as usize] + } + + /// Allocates auxiliary records for the sorted, unique symbol ids that + /// do not already have one. + pub fn allocate_aux(&mut self, ids: &[SymbolId]) { + let ids: Vec = ids + .iter() + .copied() + .filter(|id| self.symbols[id.index()].aux_idx == NO_AUX) + .collect(); + let first = self.aux.len(); + let records: Vec = ids.par_iter().map(|_| SymbolAux::default()).collect(); + self.aux.extend(records); + assert!(self.aux.len() < NO_AUX as usize); + for (i, id) in ids.into_iter().enumerate() { + self.symbols[id.index()].aux_idx = (first + i) as u32; + } + } + + /// Records a range extension thunk address. Thunks are created in + /// address order. + pub fn add_thunk_addr(&mut self, id: SymbolId, addr: u64) { + let addrs = &mut self.aux_mut(id).thunk_addrs; + debug_assert!(addrs.last().is_none_or(|&last| last < addr)); + addrs.push(addr); + } + + pub fn lookup(&self, key: &[u8]) -> Option { + let hash = hash_key(key); + self.shards[shard_of(hash)] + .get(&Query { hash, key }) + .copied() + } + + /// Interns the keys recorded in `bins` all at once and hands each slot + /// its symbol through `assign`. + pub fn gather( + &mut self, + bins: Vec>, + assign: impl Fn(S, SymbolId) + Sync, + ) { + // C++ mold gives each shard stable arena blocks and constructs a new + // symbol as soon as its key is inserted. Reserve the backing vector + // once, then hand out the same 256-slot blocks from an atomic bump + // pointer. At most one partial block per shard is left unused. + const BLOCK_SIZE: usize = 256; + let count: usize = bins.iter().flat_map(|bin| &bin.0).map(Vec::len).sum(); + let first = self.symbols.len(); + self.symbols + .reserve(count.saturating_add(NUM_SHARDS * (BLOCK_SIZE - 1))); + let capacity = self.symbols.capacity(); + let storage = AtomicPtr::new(self.symbols.as_mut_ptr()); + let next = AtomicUsize::new(first); + + let ranges: Vec>> = self + .shards + .par_iter_mut() + .enumerate() + .map(|(i, shard)| { + let count: usize = bins.iter().map(|bin| bin.0[i].len()).sum(); + shard.reserve(count); + let symbols = storage.load(Ordering::Relaxed); + let mut blocks: Vec<(usize, usize)> = Vec::new(); + for p in bins.iter().flat_map(|bin| &bin.0[i]) { + let id = match shard.entry(p.key) { + hashbrown::hash_map::Entry::Occupied(entry) => *entry.get(), + hashbrown::hash_map::Entry::Vacant(entry) => { + if blocks.last().is_none_or(|&(_, used)| used == BLOCK_SIZE) { + let start = next.fetch_add(BLOCK_SIZE, Ordering::Relaxed); + let end = start.checked_add(BLOCK_SIZE).expect("too many symbols"); + assert!(end <= capacity && end < u32::MAX as usize); + blocks.push((start, 0)); + } + let (start, used) = blocks.last_mut().unwrap(); + let index = *start + *used; + *used += 1; + let id = SymbolId(index as u32); + // SAFETY: reserve keeps `symbols` stable, and the + // atomic bump pointer gives this shard exclusive + // ownership of the slot. + unsafe { + symbols.add(index).write(Symbol::new(BStr::new( + &p.key.key[..p.name_len as usize], + ))); + } + entry.insert(id); + id + } + }; + assign(p.slot, id); + } + + for &(start, used) in &blocks { + for index in start + used..start + BLOCK_SIZE { + // SAFETY: these are the unused slots in this shard's + // exclusive block. Initializing them makes the whole + // vector prefix valid while global scans skip them. + unsafe { + symbols.add(index).write(Symbol::new(BStr::new(b""))); + } + } + } + blocks + .into_iter() + .map(|(start, used)| start as u32..(start + used) as u32) + .collect() + }) + .collect(); + + let len = next.load(Ordering::Relaxed); + // SAFETY: every allocated block, including its unused tail, was + // initialized by its owning shard. + unsafe { self.symbols.set_len(len) }; + for (shard, shard_ranges) in ranges.into_iter().enumerate() { + for range in shard_ranges { + self.note_globals(shard, range); + } + } + } + + /// Gathers keys recorded against stable input-file symbol slots. + pub(crate) fn gather_symbol_slots(&mut self, bins: Vec>) { + self.gather(bins, SymbolSlot::assign); + } + + /// Adds a symbol that is not indexed by name, such as a local symbol. + pub fn add(&mut self, sym: Symbol) -> SymbolId { + let id = SymbolId(self.symbols.len() as u32); + self.symbols.push(sym); + id + } + + /// Adds `n` symbols at once, with `init` filling in the new tail of + /// the table, typically from many threads at once. The first of them + /// gets the id returned. + /// + /// # Safety + /// + /// `init` must initialize every element of the slice it is given. + pub unsafe fn add_many( + &mut self, + n: usize, + init: impl FnOnce(&mut [MaybeUninit]), + ) -> SymbolId { + // SAFETY: `init` has the same obligation for the new slots and does + // not access the existing ones. + unsafe { self.add_many_with_existing(n, |_, slots| init(slots)) } + } + + /// Adds `n` symbols while also exposing the initialized prefix. The two + /// slices are disjoint, so a file-parallel pass can update old symbols + /// and construct new ones together, as C++ mold's arena permits. + /// + /// # Safety + /// + /// `init` must initialize every element of the second slice. + pub unsafe fn add_many_with_existing( + &mut self, + n: usize, + init: impl FnOnce(&mut [Symbol], &mut [MaybeUninit]), + ) -> SymbolId { + let first = SymbolId(self.symbols.len() as u32); + self.symbols.reserve(n); + let ptr = self.symbols.as_mut_ptr(); + // SAFETY: `first` is the initialized length and reserve made room for + // `n` further elements. The ranges do not overlap. + let existing = unsafe { std::slice::from_raw_parts_mut(ptr, first.index()) }; + let slots = unsafe { + std::slice::from_raw_parts_mut(ptr.add(first.index()).cast::>(), n) + }; + init(existing, slots); + // SAFETY: the caller promises that init initialized all n elements. + unsafe { self.symbols.set_len(first.index() + n) }; + first + } + + /// Lets parallel workers append up to `maximum` symbols without moving + /// the table. The final length contains exactly the ranges they allocate. + pub fn with_parallel_appender( + &mut self, + maximum: usize, + init: impl FnOnce(&ParallelSymbolAllocator<'_>), + ) { + let first = self.symbols.len(); + let end = first.checked_add(maximum).expect("too many symbols"); + assert!(end <= u32::MAX as usize); + self.symbols.reserve(maximum); + + let allocator = ParallelSymbolAllocator { + // SAFETY: reserve made the entire tail available, even though it + // is outside the vector's initialized length. + slots: AtomicPtr::new(unsafe { + self.symbols + .as_mut_ptr() + .add(first) + .cast::>() + }), + first, + next: AtomicUsize::new(0), + maximum, + marker: PhantomData, + }; + init(&allocator); + + let added = allocator.next.load(Ordering::Relaxed); + debug_assert!(added <= maximum); + // SAFETY: every allocated range was initialized before `init` + // returned, and the bump pointer leaves no gaps between ranges. + unsafe { self.symbols.set_len(first + added) }; + } + + pub fn len(&self) -> usize { + self.symbols.len() + } + + pub fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + pub fn ids(&self) -> impl Iterator { + (0..self.symbols.len() as u32).map(SymbolId) + } + + pub fn iter(&self) -> impl Iterator { + self.symbols + .iter() + .enumerate() + .map(|(i, s)| (SymbolId(i as u32), s)) + } + + pub fn iter_mut(&mut self) -> impl Iterator { + self.symbols + .iter_mut() + .enumerate() + .map(|(i, s)| (SymbolId(i as u32), s)) + } + + pub fn as_slice(&self) -> &[Symbol] { + &self.symbols + } + + pub fn as_mut_slice(&mut self) -> &mut [Symbol] { + &mut self.symbols + } + + /// The ids of all named (global) symbols. + pub fn global_ids(&self) -> impl Iterator + '_ { + self.globals + .iter() + .flatten() + .flat_map(|range| range.clone().map(SymbolId)) + } + + /// The id ranges of the named (global) symbols. + pub fn global_ranges(&self) -> impl Iterator> { + self.globals.iter().flatten() + } + + /// Applies `f` to all named symbols in parallel, one task per map shard, + /// as C++ mold's `ShardedMap::parallel_for_each` does. + pub fn par_for_each_global_mut(&mut self, f: impl Fn(&mut Symbol) + Send + Sync) { + let symbols = SymbolBlockPtr(self.symbols.as_mut_ptr()); + self.globals.par_iter().for_each(|ranges| { + // SAFETY: The arena is exclusively borrowed, every global range + // belongs to exactly one shard, and shard blocks never overlap. + unsafe { symbols.for_each(ranges, &f) }; + }); + } + + /// Applies `f` in parallel to symbols and their auxiliary records. + /// + /// # Safety + /// + /// `ids` must contain no duplicates, since each invocation receives mutable + /// access to the corresponding auxiliary record. + pub(crate) unsafe fn par_for_each_aux_mut( + &mut self, + ids: &[SymbolId], + f: impl Fn(usize, &Symbol, &mut SymbolAux) + Send + Sync, + ) { + let symbols = &self.symbols; + let aux = SymbolAuxPtr(self.aux.as_mut_ptr()); + let aux_len = self.aux.len(); + ids.par_iter().enumerate().for_each(|(i, &id)| { + let sym = &symbols[id.index()]; + debug_assert_ne!(sym.aux_idx, NO_AUX); + debug_assert!((sym.aux_idx as usize) < aux_len); + // SAFETY: the caller guarantees that ids, and hence aux_idx values, + // are distinct, and the exclusive table borrow keeps the arena fixed. + unsafe { aux.with_mut(sym.aux_idx as usize, |record| f(i, sym, record)) }; + }); + } + + /// Applies `f` as above and sums its per-symbol results in parallel. + /// + /// # Safety + /// + /// `ids` must contain no duplicates, since each invocation receives mutable + /// access to the corresponding auxiliary record. + pub(crate) unsafe fn par_sum_aux_mut( + &mut self, + ids: &[SymbolId], + f: impl Fn(usize, &Symbol, &mut SymbolAux) -> u64 + Send + Sync, + ) -> u64 { + let symbols = &self.symbols; + let aux = SymbolAuxPtr(self.aux.as_mut_ptr()); + let aux_len = self.aux.len(); + ids.par_iter() + .enumerate() + .map(|(i, &id)| { + let sym = &symbols[id.index()]; + debug_assert_ne!(sym.aux_idx, NO_AUX); + debug_assert!((sym.aux_idx as usize) < aux_len); + // SAFETY: the caller guarantees that ids, and hence aux_idx + // values, are distinct, and the exclusive table borrow keeps + // the arena fixed. + unsafe { aux.with_mut(sym.aux_idx as usize, |record| f(i, sym, record)) } + }) + .sum() + } + + /// Finds named symbols satisfying `predicate` in deterministic shard + /// order, while examining all of the symbol-map blocks in parallel. + pub fn par_find_globals( + &self, + predicate: impl Fn(&Symbol) -> bool + Send + Sync, + ) -> Vec { + self.globals + .par_iter() + .map(|ranges| { + ranges + .iter() + .flat_map(|range| { + self.symbols[range.start as usize..range.end as usize] + .iter() + .enumerate() + .filter_map(|(i, sym)| { + predicate(sym).then_some(SymbolId(range.start + i as u32)) + }) + }) + .collect::>() + }) + .collect::>() + .into_iter() + .flatten() + .collect() + } + + /// Splits the table into two disjoint mutable views: the symbol at `id` + /// and everything else, for the rare cases where one symbol is updated + /// from another's state. + pub fn get2_mut(&mut self, a: SymbolId, b: SymbolId) -> (&mut Symbol, &mut Symbol) { + assert_ne!(a, b); + let (a, b) = (a.index(), b.index()); + if a < b { + let (lo, hi) = self.symbols.split_at_mut(b); + (&mut lo[a], &mut hi[0]) + } else { + let (lo, hi) = self.symbols.split_at_mut(a); + (&mut hi[0], &mut lo[b]) + } + } +} + +impl Index for SymbolTable { + type Output = Symbol; + + fn index(&self, id: SymbolId) -> &Symbol { + &self.symbols[id.index()] + } +} + +impl IndexMut for SymbolTable { + fn index_mut(&mut self, id: SymbolId) -> &mut Symbol { + &mut self.symbols[id.index()] + } +} + +/// Whether a string is a valid C identifier, which decides whether a +/// section gets `__start_`/`__stop_` symbols. +pub fn is_c_identifier(s: &[u8]) -> bool { + let is_alpha = |c: u8| c == b'_' || c.is_ascii_alphabetic(); + match s.split_first() { + Some((&first, rest)) => { + is_alpha(first) && rest.iter().all(|&c| is_alpha(c) || c.is_ascii_digit()) + } + None => false, + } +} diff --git a/src/thunks.rs b/src/thunks.rs new file mode 100644 index 000000000..191c6e684 --- /dev/null +++ b/src/thunks.rs @@ -0,0 +1,424 @@ +//! Range extension thunks. +//! +//! RISC branch instructions have small immediates: ARM32 branches reach +//! ±16 MiB and ARM64 ones ±128 MiB, unlike x86-64's 32-bit displacements. +//! A call whose target is further away is redirected to a thunk, a +//! linker-synthesized code sequence between input sections that loads the +//! full address into a register and jumps there. +//! +//! Thunks are created in two passes. Before addresses are known, every +//! out-of-section call is pessimistically assumed to need a thunk; once +//! the layout is fixed, the entries that turned out to be unneeded are +//! removed. Sections only shrink in the second pass, so no existing +//! reference to a thunk goes out of range because of it. + +use std::cell::UnsafeCell; + +use rayon::prelude::*; + +use crate::arch::{Arch, Family}; +use crate::chunks::output_section::OutputSection; +use crate::chunks::{ChunkId, OutputSectionId}; +use crate::context::Context; +use crate::elf::*; +use crate::input_sections::InputSection; +use crate::symbol::{AddrFlags, Symbol, SymbolId}; +use crate::util::align_to; + +/// A block of branch stubs placed between input sections. +#[derive(Debug)] +pub struct Thunk { + /// Offset within the output section. + pub offset: u64, + pub symbols: Vec, + /// Offset of each stub within the thunk; the last entry is the size. + pub offsets: Vec, + pub name: String, +} + +impl Thunk { + pub fn size(&self) -> u64 { + self.offsets.last().copied().unwrap_or(0) + } + + pub fn addr(&self, osec: &OutputSection) -> u64 { + osec.hdr.shdr.sh_addr + self.offset + } + + /// The entry offsets of a thunk with fixed-size entries. + pub fn fixed_offsets(&self) -> Vec { + let layout = E::THUNK.expect("target without thunks"); + (0..=self.symbols.len()) + .map(|i| layout.header_size + i as u64 * layout.entry_size) + .collect() + } +} + +/// A section offset that hasn't been assigned yet. +const UNPLACED: u64 = u64::MAX; + +/// Thunks are created per batch of code of this size. +fn batch_size() -> u64 { + let mib = match E::FAMILY { + Family::Arm64 => 32, + Family::Arm32 => 8, + _ => 16, + }; + mib << 20 +} + +/// A thunk is assumed to be smaller than half a batch. +fn max_thunk_size() -> u64 { + batch_size::() / 2 +} + +/// Power10 prefixed instructions must not cross a 64-byte boundary. +/// Aligning each thunk to 8 bytes guarantees that. +const THUNK_ALIGN: u64 = 8; + +/// The thunk symbols collected by each Rayon worker. This is the equivalent +/// of C++ mold's `enumerable_thread_specific>`. +struct ThunkSymbolBins(Vec>>); + +// SAFETY: a Rayon worker has a unique index and executes at most one closure +// at a time. The only caller does not invoke nested parallel work while using +// its bin; the final merge happens after the parallel traversal has joined. +unsafe impl Sync for ThunkSymbolBins {} + +impl ThunkSymbolBins { + fn new() -> ThunkSymbolBins { + ThunkSymbolBins( + (0..=rayon::current_num_threads()) + .map(|_| UnsafeCell::new(Vec::new())) + .collect(), + ) + } + + #[inline] + fn push(&self, sym: SymbolId) { + let fallback = self.0.len() - 1; + let i = rayon::current_thread_index() + .unwrap_or(fallback) + .min(fallback); + // SAFETY: the Sync invariant above gives this worker exclusive access + // to its indexed vector for the duration of this non-nested closure. + unsafe { &mut *self.0[i].get() }.push(sym); + } + + fn into_vec(self) -> Vec { + let mut bins: Vec> = self.0.into_iter().map(UnsafeCell::into_inner).collect(); + let mut symbols = Vec::with_capacity(bins.iter().map(Vec::len).sum()); + for bin in &mut bins { + symbols.append(bin); + } + symbols + } +} + +/// Whether a call needs a thunk. On the first pass, before addresses are +/// known, every call out of the section is assumed to need one. +#[inline(always)] +fn requires_thunk( + ctx: &Context, + isec: &InputSection, + rel: &ElfRel, + sym: &Symbol, + first_pass: bool, +) -> bool { + if !rel.is_func_call::() { + return false; + } + + if first_pass { + // On the first pass, we pessimistically assume that all out-of-section + // relocations are out of range. + match sym.input_section_ref() { + Some(target) if target.output_section == isec.output_section => { + // If the target section is in the same output section but + // hasn't got any address yet, that's unreachable. + if target.offset() == UNPLACED { + return true; + } + } + _ => return true, + } + + // Even if the target is the same section, we branch to its PLT + // if it has one. So a symbol with a PLT is also considered an + // out-of-section reference. + if sym.has_plt(&ctx.symbols) { + return true; + } + } + + if E::always_needs_thunk(ctx, sym, rel) { + return true; + } + + let s = sym.addr_with(ctx, AddrFlags::NO_OPD) as i64; + let a = isec.rel_addend::(rel); + let p = (isec.addr(ctx) + rel.r_offset) as i64; + let val = s.wrapping_add(a).wrapping_sub(p); + val < -E::branch_distance() || E::branch_distance() <= val +} + +/// The executable output sections, in output order. +fn executable_sections(ctx: &Context) -> Vec { + ctx.chunks + .iter() + .filter_map(|&id| match id { + ChunkId::Output(osec) + if ctx.output_sections[osec.index()].hdr.shdr.sh_flags & SHF_EXECINSTR as u64 + != 0 => + { + Some(osec) + } + _ => None, + }) + .collect() +} + +/// Lays out an executable output section, inserting a thunk after each +/// batch of input sections. +/// +/// Progress is tracked with four indices into the members, A <= B <= C +/// <= D. The sections between B and C are the current batch; A is the +/// first section that can still reach the batch, and D the last section +/// such that a thunk placed after it is reachable from the whole batch. +/// +/// ................................ ............ +/// A B C D +/// ^ a thunk for the current batch goes just before D +/// <---> the current batch, smaller than the batch size +/// <--------> smaller than the branch distance +/// <--------> smaller than the branch distance +/// <-------------> reachable from the current batch +pub fn create_range_extension_thunks(ctx: &mut Context, id: OutputSectionId) { + let members = std::mem::take(&mut ctx.output_sections[id.index()].members); + if members.is_empty() { + return; + } + + // Initialize input sections with a dummy offset so that we can + // distinguish sections whose addresses have been assigned from those + // whose addresses have not. + members + .par_iter() + .for_each(|&member| ctx.input_section(member).set_offset(UNPLACED)); + + let distance = E::branch_distance() as u64; + let batch = batch_size::(); + let max_thunk = max_thunk_size::(); + let n = members.len(); + + let mut thunks: Vec = Vec::new(); + let (mut a, mut b, mut d) = (0usize, 0usize, 0usize); + let mut offset = 0u64; + // The first thunk that is still reachable from the current batch. + let mut t = 0usize; + + while b < n { + // Move D forward as far as a thunk placed after D is reachable + // from B. + while d < n { + let sec = ctx.input_section(members[d]); + let (p2align, size) = (sec.p2align(), sec.sh_size); + if b != d { + let thunk_end = + align_to(align_to(offset, 1 << p2align) + size, THUNK_ALIGN) + max_thunk; + if thunk_end > ctx.input_section(members[b]).offset() + distance { + break; + } + } + offset = align_to(offset, 1 << p2align); + sec.set_offset(offset); + offset += size; + d += 1; + } + + // The batch ends at the first section that ends beyond B plus the + // batch size. Section ends increase, so binary search; starting + // from B + 1 guarantees progress. + let b_offset = ctx.input_section(members[b]).offset(); + let c = b + + 1 + + members[b + 1..d].partition_point(|&member| { + let sec = ctx.input_section(member); + sec.offset() + sec.sh_size < b_offset + batch + }); + + // The first section within branch range of C. + let c_offset = if c == d { + offset + } else { + ctx.input_section(members[c]).offset() + }; + a += members[a..b].partition_point(|&member| { + ctx.input_section(member).offset() < c_offset.saturating_sub(distance) + }); + + // Thunks before A are out of range now. + while t < thunks.len() && thunks[t].offset < ctx.input_section(members[a]).offset() { + for &sym in &thunks[t].symbols { + ctx.symbols[sym].unmark(); + } + t += 1; + } + + // Create a thunk at D for the calls of the batch. A symbol + // already covered by a reachable thunk is marked and skipped. + offset = align_to(offset, THUNK_ALIGN); + let symbol_bins = ThunkSymbolBins::new(); + { + let ctx: &Context = ctx; + members[b..c].par_iter().for_each(|&member| { + let isec = ctx.input_section(member); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if !rel.is_func_call::() { + continue; + } + let id = file.base.symbols[rel.r_sym as usize]; + let sym = &ctx.symbols[id]; + if requires_thunk(ctx, isec, &rel, sym, true) && sym.mark() { + symbol_bins.push(id); + } + } + }); + } + let mut thunk = Thunk { + offset, + symbols: symbol_bins.into_vec(), + offsets: Vec::new(), + name: String::new(), + }; + thunk.offsets = thunk.fixed_offsets::(); + debug_assert!(thunk.size() < max_thunk); + offset += thunk.size(); + thunks.push(thunk); + + b = c; + } + + for thunk in &thunks[t..] { + for &sym in &thunk.symbols { + ctx.symbols[sym].unmark(); + } + } + + // Sort the symbols for deterministic output. + { + let ctx: &Context = ctx; + thunks.par_iter_mut().for_each(|thunk| { + thunk.symbols.sort_by_key(|&id| { + let sym = &ctx.symbols[id]; + (sym.file().map_or(0, |f| ctx.file(f).priority), sym.sym_idx) + }); + }); + } + + let osec = &mut ctx.output_sections[id.index()]; + osec.hdr.shdr.sh_size = offset; + osec.thunks = thunks; + osec.members = members; +} + +/// Now that addresses are known, drops the thunk entries for calls that +/// turned out to be in range, and shrinks the sections accordingly. +pub fn remove_redundant_thunks(ctx: &mut Context) { + let _t = ctx.timer("remove_redundant_thunks"); + let sections = executable_sections(ctx); + + // Mark the symbols that really need thunks. + { + let ctx: &Context = ctx; + for &id in §ions { + ctx.output_sections[id.index()] + .members + .par_iter() + .for_each(|&m| { + let isec = ctx.input_section(m); + let file = &ctx.objs[isec.file.index()]; + for rel in isec.rels::(file) { + if !rel.is_func_call::() { + continue; + } + let sym = &ctx.symbols[file.base.symbols[rel.r_sym as usize]]; + if !sym.is_marked() && requires_thunk(ctx, isec, &rel, sym, false) { + sym.mark(); + } + } + }); + } + } + + for &id in §ions { + let mut thunks = std::mem::take(&mut ctx.output_sections[id.index()].thunks); + { + let ctx: &Context = ctx; + let osec = &ctx.output_sections[id.index()]; + thunks.par_iter_mut().for_each(|thunk| { + thunk.symbols.retain(|&sym| ctx.symbols[sym].is_marked()); + thunk.offsets = E::thunk_offsets(ctx, thunk, thunk.addr(osec)); + }); + } + + // Lay the members and thunks out again, in their existing order. + let members = ctx.output_sections[id.index()].members.clone(); + let (mut mi, mut ti) = (0, 0); + let mut offset = 0; + while mi < members.len() || ti < thunks.len() { + let member_first = mi < members.len() + && (ti >= thunks.len() + || ctx.input_section(members[mi]).offset() < thunks[ti].offset); + if member_first { + let sec = ctx.input_section(members[mi]); + offset = align_to(offset, 1 << sec.p2align()); + sec.set_offset(offset); + offset += sec.sh_size; + mi += 1; + } else { + offset = align_to(offset, THUNK_ALIGN); + thunks[ti].offset = offset; + offset += thunks[ti].size(); + ti += 1; + } + } + let osec = &mut ctx.output_sections[id.index()]; + osec.hdr.shdr.sh_size = offset; + osec.thunks = thunks; + } +} + +/// Records with each symbol the addresses of its thunk entries, so that +/// applying a branch relocation can find one in range quickly. +pub fn gather_thunk_addresses(ctx: &mut Context) { + let _t = ctx.timer("gather_thunk_addresses"); + let mut sections = executable_sections(ctx); + sections.sort_by_key(|id| ctx.output_sections[id.index()].hdr.shdr.sh_addr); + + for id in sections { + let osec = &ctx.output_sections[id.index()]; + let entries: Vec<(SymbolId, u64)> = osec + .thunks + .iter() + .flat_map(|thunk| { + let base = thunk.addr(osec); + thunk + .symbols + .iter() + .enumerate() + .map(move |(i, &sym)| (sym, base + thunk.offsets[i])) + }) + .collect(); + for (sym, addr) in entries { + ctx.symbols.add_thunk_addr(sym, addr); + } + } +} + +/// Writes a thunk's stubs into the output section buffer. +pub fn copy_buf(ctx: &Context, osec: &OutputSection, thunk: &Thunk, buf: &mut [u8]) { + debug_assert_eq!(buf.len(), thunk.size() as usize); + E::write_thunk(ctx, thunk, thunk.addr(osec), buf); +} diff --git a/src/tls.rs b/src/tls.rs new file mode 100644 index 000000000..89268b4a6 --- /dev/null +++ b/src/tls.rs @@ -0,0 +1,45 @@ +//! Thread-local storage layout constants. +//! +//! Each thread has its own copy of the thread-local variables, initialized +//! from the PT_TLS segment's template image. A register, the thread +//! pointer (TP), refers to the thread's copy, and where exactly TP points +//! relative to the copy is decided by each psABI. The dynamic thread +//! pointer (DTP) is the base `__tls_get_addr` returns for offset 0. + +use crate::arch::{Arch, Family}; +use crate::elf::{ElfPhdr, PT_TLS}; +use crate::util::{align_down, align_to}; + +/// The TP value the main executable's TLS block would have if the +/// template image were the block itself. +pub fn tp_addr(phdr: &ElfPhdr) -> u64 { + debug_assert_eq!(phdr.p_type, PT_TLS); + match E::FAMILY { + // x86, SPARC and s390x put TP past the end of the TLS block, so + // variables are accessed with negative offsets. + Family::X86_64 | Family::I386 | Family::Sparc64 | Family::S390x => { + align_to(phdr.p_vaddr + phdr.p_memsz, phdr.p_align) + } + // ARM and SH4 prepend two words to the template image. + Family::Arm64 | Family::Arm32 | Family::Sh4 => { + align_down(phdr.p_vaddr - E::WORD_SIZE as u64 * 2, phdr.p_align) + } + // PowerPC and m68k put TP 0x7000 bytes past the start of the block + // to maximize the reach of 16-bit signed displacements. + Family::Ppc32 | Family::Ppc64V1 | Family::Ppc64V2 | Family::M68k => phdr.p_vaddr + 0x7000, + // RISC-V and LoongArch use the start of the block. + Family::RiscV | Family::LoongArch => phdr.p_vaddr, + } +} + +/// The address `__tls_get_addr` returns for offset 0. +pub fn dtp_addr(phdr: &ElfPhdr) -> u64 { + debug_assert_eq!(phdr.p_type, PT_TLS); + match E::FAMILY { + // PowerPC and m68k bias DTP by 0x8000, RISC-V by 0x800, so that + // signed displacements cover the whole block. + Family::Ppc32 | Family::Ppc64V1 | Family::Ppc64V2 | Family::M68k => phdr.p_vaddr + 0x8000, + Family::RiscV => phdr.p_vaddr + 0x800, + _ => phdr.p_vaddr, + } +} diff --git a/src/util/compress.rs b/src/util/compress.rs new file mode 100644 index 000000000..bc4431464 --- /dev/null +++ b/src/util/compress.rs @@ -0,0 +1,172 @@ +//! Multi-threaded zlib and zstd compression for `--compress-debug-sections`. +//! +//! zlib-compressed data can be merged by concatenation as long as each +//! piece ends with a sync flush, so the input is split into shards, +//! compressed in parallel, and concatenated under one header and trailer. +//! zstd frames concatenate the same way. Resetting the dictionary at shard +//! boundaries costs a little compression ratio, which is negligible with +//! large shards. + +use flate2::{Compress, Compression, FlushCompress, FlushDecompress}; +use rayon::prelude::*; + +const SHARD_SIZE: usize = 1024 * 1024; + +pub enum Compressor { + Zlib { shards: Vec>, checksum: u32 }, + Zstd { shards: Vec> }, +} + +fn adler32(data: &[u8]) -> u32 { + let mut a: u32 = 1; + let mut b: u32 = 0; + for chunk in data.chunks(5552) { + for &byte in chunk { + a += byte as u32; + b += a; + } + a %= 65521; + b %= 65521; + } + (b << 16) | a +} + +/// Combines two Adler-32 checksums, where `len2` is the length of the +/// second input. +fn adler32_combine(adler1: u32, adler2: u32, len2: u64) -> u32 { + const BASE: u64 = 65521; + let rem = len2 % BASE; + let mut sum1 = (adler1 & 0xffff) as u64; + let mut sum2 = rem * sum1 % BASE; + sum1 += (adler2 & 0xffff) as u64 + BASE - 1; + sum2 += ((adler1 >> 16) & 0xffff) as u64 + ((adler2 >> 16) & 0xffff) as u64 + BASE - rem; + if sum1 >= BASE { + sum1 -= BASE; + } + if sum1 >= BASE { + sum1 -= BASE; + } + if sum2 >= BASE << 1 { + sum2 -= BASE << 1; + } + if sum2 >= BASE { + sum2 -= BASE; + } + (sum1 | (sum2 << 16)) as u32 +} + +/// Compresses a shard as a raw deflate stream ending with a sync flush, +/// so that the stream ends on a byte boundary and can be concatenated. +fn zlib_compress(input: &[u8], level: u32) -> Vec { + let mut compress = Compress::new(Compression::new(level), false); + let mut out = Vec::with_capacity(input.len() / 2 + 64); + loop { + let before = compress.total_in() as usize; + let status = compress + .compress_vec(&input[before..], &mut out, FlushCompress::Sync) + .expect("deflate failed"); + let consumed = compress.total_in() as usize; + if consumed == input.len() && status == flate2::Status::Ok && out.len() < out.capacity() { + break; + } + if status == flate2::Status::StreamEnd { + break; + } + out.reserve(out.capacity().max(1024)); + } + out +} + +impl Compressor { + pub fn zlib(input: &[u8], level: u32) -> Compressor { + let inputs: Vec<&[u8]> = input.chunks(SHARD_SIZE).collect(); + let (shards, adlers): (Vec>, Vec) = inputs + .par_iter() + .map(|shard| (zlib_compress(shard, level), adler32(shard))) + .unzip(); + + let mut checksum = adlers.first().copied().unwrap_or(1); + for (adler, shard) in adlers.iter().zip(&inputs).skip(1) { + checksum = adler32_combine(checksum, *adler, shard.len() as u64); + } + Compressor::Zlib { shards, checksum } + } + + pub fn zstd(input: &[u8], level: i32) -> Compressor { + let shards = input + .par_chunks(SHARD_SIZE) + .map(|shard| zstd::bulk::compress(shard, level).expect("zstd compression failed")) + .collect(); + Compressor::Zstd { shards } + } + + pub fn compressed_size(&self) -> usize { + match self { + // The header and the trailer add 8 bytes. + Compressor::Zlib { shards, .. } => 8 + shards.iter().map(Vec::len).sum::(), + Compressor::Zstd { shards } => shards.iter().map(Vec::len).sum(), + } + } + + pub fn write_to(&self, buf: &mut [u8]) { + match self { + Compressor::Zlib { shards, checksum } => { + buf[0] = 0x78; + buf[1] = 0x9c; + let mut pos = 2; + for shard in shards { + buf[pos..pos + shard.len()].copy_from_slice(shard); + pos += shard.len(); + } + // An empty final block, then the Adler-32 checksum. + buf[pos] = 3; + buf[pos + 1] = 0; + buf[pos + 2..pos + 6].copy_from_slice(&checksum.to_be_bytes()); + } + Compressor::Zstd { shards } => { + let mut pos = 0; + for shard in shards { + buf[pos..pos + shard.len()].copy_from_slice(shard); + pos += shard.len(); + } + } + } + } +} + +/// Decompresses a zlib stream into a buffer of known size. +pub fn zlib_decompress(input: &[u8], out: &mut [u8]) -> Result<(), String> { + let mut decompress = flate2::Decompress::new(true); + loop { + let in_pos = decompress.total_in() as usize; + let out_pos = decompress.total_out() as usize; + if out_pos >= out.len() { + return Ok(()); + } + let status = decompress + .decompress( + &input[in_pos.min(input.len())..], + &mut out[out_pos..], + FlushDecompress::None, + ) + .map_err(|e| e.to_string())?; + match status { + flate2::Status::StreamEnd => break, + flate2::Status::BufError if decompress.total_in() as usize >= input.len() => break, + _ => {} + } + } + if (decompress.total_out() as usize) < out.len() { + return Err("premature end of input".to_string()); + } + Ok(()) +} + +/// Decompresses a zstd stream into a buffer of known size. +pub fn zstd_decompress(input: &[u8], out: &mut [u8]) -> Result<(), String> { + let n = zstd::bulk::decompress_to_buffer(input, out).map_err(|e| e.to_string())?; + if n < out.len() { + return Err("premature end of input".to_string()); + } + Ok(()) +} diff --git a/src/util/concurrent_map.rs b/src/util/concurrent_map.rs new file mode 100644 index 000000000..90b170c31 --- /dev/null +++ b/src/util/concurrent_map.rs @@ -0,0 +1,496 @@ +//! A fast concurrent hash map. +//! +//! Unlike ordinary hash tables, this implementation just aborts if it +//! becomes full. So you need to give a correct estimation of the final +//! size before using it (see [`crate::util::hyperloglog`]). We use this +//! hash map to uniquify pieces of data in mergeable sections. +//! +//! We've implemented this ourselves because the performance of the +//! concurrent hash map is critical for our linker. +//! +//! The map is an open-addressing table. Insertion is lock-free: a thread +//! claims an empty bucket with a compare-and-swap on its key pointer, +//! initializes the value, and then publishes the key; a thread that finds +//! a claimed bucket spins until the key appears. Probing stays within a +//! shard of the table, so the entries of a shard can be laid out +//! independently of the others. Keys are byte strings that live for the +//! whole link, such as string literals inside input files, hashed once by +//! the caller. + +use rayon::prelude::*; +use std::cell::UnsafeCell; +use std::mem::MaybeUninit; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; + +pub const NUM_SHARDS: usize = 64; + +/// `MIN_NBUCKETS` is chosen so that even the smallest map has `MAX_RETRY` +/// buckets per shard; probing is confined to a shard, and a probe that +/// visits `MAX_RETRY` distinct occupied slots aborts. +const MIN_NBUCKETS: usize = 16384; +const MAX_RETRY: usize = 256; + +/// The key pointer of a bucket that a thread has claimed but not yet +/// published: -1 marks the slot as claimed until its value has been +/// initialized. +const CLAIMED: *mut u8 = usize::MAX as *mut u8; + +/// A bucket of the table. In order to avoid unnecessary cache-line false +/// sharing, we want to make this object aligned to a reasonably large +/// power-of-two address. +#[repr(C, align(32))] +struct Entry { + key: AtomicPtr, + keylen: UnsafeCell, + value: UnsafeCell>, +} + +/// The index of a bucket, which identifies an entry for the life of +/// the map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct EntryId(u32); + +impl EntryId { + pub fn raw(self) -> u32 { + self.0 + } +} + +pub struct ConcurrentMap { + entries: *mut Entry, + nbuckets: usize, +} + +// SAFETY: entries are shared between threads only through atomics and +// values that are initialized before they become reachable. +unsafe impl Send for ConcurrentMap {} +unsafe impl Sync for ConcurrentMap {} + +impl Default for ConcurrentMap { + /// A map without buckets, to be replaced before use. + fn default() -> Self { + ConcurrentMap { + entries: ptr::null_mut(), + nbuckets: 0, + } + } +} + +impl ConcurrentMap { + /// A map with room for about `nkeys` distinct keys. + pub fn with_capacity(nkeys: usize) -> Self { + let nbuckets = nkeys.next_power_of_two().max(MIN_NBUCKETS); + let bufsize = Self::bufsize(nbuckets); + + // Allocate a zero-initialized buffer: zeroed memory is a table of + // empty buckets. mmap is faster than malloc + memset. + // SAFETY: an anonymous private mapping, checked for failure below. + let entries = unsafe { + libc::mmap( + ptr::null_mut(), + bufsize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_ANONYMOUS | libc::MAP_PRIVATE, + -1, + 0, + ) + }; + if entries == libc::MAP_FAILED { + panic!( + "mmap of {bufsize} bytes failed: {}", + std::io::Error::last_os_error() + ); + } + // SAFETY: the range is the fresh mapping; the advice is only a hint. + unsafe { libc::madvise(entries, bufsize, libc::MADV_HUGEPAGE) }; + ConcurrentMap { + entries: entries as *mut Entry, + nbuckets, + } + } + + fn bufsize(nbuckets: usize) -> usize { + std::mem::size_of::>() + .checked_mul(nbuckets) + .expect("table size overflow") + } + + pub fn nbuckets(&self) -> usize { + self.nbuckets + } + + /// The number of entries, counted. + pub fn len(&self) -> usize { + (0..self.nbuckets) + .filter(|&idx| self.key_at(idx).is_some()) + .count() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn entry(&self, idx: usize) -> &Entry { + debug_assert!(idx < self.nbuckets); + // SAFETY: idx is within the allocation. + unsafe { &*self.entries.add(idx) } + } + + /// The published key of a bucket, if any. + fn key_at(&self, idx: usize) -> Option<&'static [u8]> { + let ent = self.entry(idx); + let key = ent.key.load(Ordering::Acquire); + if key.is_null() || key == CLAIMED { + return None; + } + // SAFETY: a published key is a live 'static slice whose length was + // written before the key pointer. + Some(unsafe { std::slice::from_raw_parts(key, *ent.keylen.get() as usize) }) + } + + fn value_at(&self, idx: usize) -> &T { + // SAFETY: callers only ask for values of published entries, which + // were initialized before publication. + unsafe { (*self.entry(idx).value.get()).assume_init_ref() } + } + + /// The probe sequence for a hash: the buckets of one shard, starting + /// from the hash's home bucket and wrapping around within the shard. + fn probe(&self, hash: u64) -> impl Iterator { + let begin = hash as usize & (self.nbuckets - 1); + let mask = self.nbuckets / NUM_SHARDS - 1; + (0..MAX_RETRY).map(move |i| (begin & !mask) | ((begin + i) & mask)) + } + + /// Inserts `key` unless it is present, initializing the value with + /// `init`. Returns the entry, its value and whether it was inserted. + pub fn insert_with( + &self, + key: &'static [u8], + hash: u64, + init: impl FnOnce() -> T, + ) -> (EntryId, &T, bool) { + self.insert_entry( + key.as_ptr(), + hash, + |ptr, len| { + // SAFETY: published keys remain live for the link. + unsafe { std::slice::from_raw_parts(ptr, len as usize) == key } + }, + || (key.len() as u32, init()), + ) + } + + /// Inserts a NUL-terminated key without storing its length at the call + /// site. `initialize` computes the length and value only if this call + /// claims a new entry. + /// + /// # Safety + /// + /// `key` must point to a NUL-terminated byte string that remains live for + /// the lifetime of the map. Its length must fit in u32. + pub unsafe fn insert_cstr_with( + &self, + key: *const u8, + hash: u64, + initialize: impl FnOnce(*const u8) -> (u32, T), + ) -> (EntryId, &T, bool) { + self.insert_entry( + key, + hash, + |ptr, _| { + // SAFETY: guaranteed for `key` by the caller and for `ptr` by + // the entry publication invariant. + unsafe { libc::strcmp(key.cast(), ptr.cast()) == 0 } + }, + || initialize(key), + ) + } + + fn insert_entry( + &self, + key: *const u8, + hash: u64, + equals: impl Fn(*const u8, u32) -> bool, + initialize: impl FnOnce() -> (u32, T), + ) -> (EntryId, &T, bool) { + assert!(self.nbuckets > 0, "the map hasn't been sized"); + let mut initialize = Some(initialize); + + for idx in self.probe(hash) { + let ent = self.entry(idx); + + // Avoid an atomic update when the slot is already occupied. + let mut ptr = ent.key.load(Ordering::Acquire); + if !ptr.is_null() && ptr != CLAIMED { + // SAFETY: a published pointer and length name a live key. + if equals(ptr, unsafe { *ent.keylen.get() }) { + return (EntryId(idx as u32), self.value_at(idx), false); + } + continue; + } + + if ptr.is_null() { + match ent.key.compare_exchange( + ptr::null_mut(), + CLAIMED, + Ordering::Acquire, + Ordering::Acquire, + ) { + Ok(_) => { + // SAFETY: the claim gives this thread exclusive + // access to the bucket until the key is published. + unsafe { + let (keylen, value) = initialize.take().expect("initialized once")(); + (*ent.value.get()).write(value); + *ent.keylen.get() = keylen; + } + ent.key.store(key as *mut u8, Ordering::Release); + return (EntryId(idx as u32), self.value_at(idx), true); + } + Err(current) => ptr = current, + } + } + + // Wait for the thread that claimed the slot to publish its key. + while ptr == CLAIMED { + std::hint::spin_loop(); + ptr = ent.key.load(Ordering::Acquire); + } + // SAFETY: the claiming thread published the pointer and length. + if equals(ptr, unsafe { *ent.keylen.get() }) { + return (EntryId(idx as u32), self.value_at(idx), false); + } + } + panic!("the concurrent map is full"); + } + + /// Prefetches the bucket where a key with the given hash would be + /// probed first. Useful when a caller knows the hashes of upcoming + /// insertions, as probes into a large table miss the cache almost + /// every time. + pub fn prefetch(&self, hash: u64) { + #[cfg(target_arch = "x86_64")] + if self.nbuckets > 0 { + let idx = hash as usize & (self.nbuckets - 1); + // SAFETY: prefetching is a hint that never faults; the address is + // within the table anyway. + unsafe { + use std::arch::x86_64::{_mm_prefetch, _MM_HINT_T0}; + _mm_prefetch(self.entries.add(idx) as *const i8, _MM_HINT_T0); + } + } + } + + /// Looks up a key. + pub fn get(&self, key: &[u8], hash: u64) -> Option<(EntryId, &T)> { + if self.nbuckets == 0 { + return None; + } + for idx in self.probe(hash) { + match self.key_at(idx) { + None => return None, + Some(existing) if existing == key => { + return Some((EntryId(idx as u32), self.value_at(idx))) + } + Some(_) => {} + } + } + None + } + + pub fn value(&self, id: EntryId) -> &T { + self.value_at(id.0 as usize) + } + + pub fn key(&self, id: EntryId) -> &'static [u8] { + self.key_at(id.0 as usize).expect("an occupied bucket") + } + + /// The entries in bucket order. + pub fn iter(&self) -> impl Iterator { + (0..self.nbuckets).filter_map(move |idx| { + self.key_at(idx) + .map(|key| (key, EntryId(idx as u32), self.value_at(idx))) + }) + } + + /// Returns a list of the entries of a shard sorted in a deterministic + /// order. + /// + /// Linear probing fills the same set of buckets whatever the order + /// keys were inserted in, but which of two colliding keys got the + /// earlier bucket depends on it, so each run of adjacent occupied + /// buckets is sorted by key. + pub fn sorted_entries(&self, shard: usize) -> Vec { + if self.nbuckets == 0 { + return Vec::new(); + } + let shard_size = self.nbuckets / NUM_SHARDS; + let begin = shard * shard_size; + let mut end = begin + shard_size; + let occupied = |idx: usize| self.key_at(idx).is_some(); + + // Since the shard is circular, we need to handle the last entries + // as if they were next to the first entries. + let mut vec: Vec = Vec::new(); + while begin < end && occupied(end - 1) { + end -= 1; + vec.push(EntryId(end as u32)); + } + + let sort_run = |run: &mut [EntryId]| { + run.sort_by(|&a, &b| { + let (ka, kb) = (self.key(a), self.key(b)); + ka.len().cmp(&kb.len()).then_with(|| ka.cmp(kb)) + }); + }; + + // Find entries contiguous in the buckets and sort them. + let mut last = 0; + let mut i = begin; + while i < end { + while i < end && occupied(i) { + vec.push(EntryId(i as u32)); + i += 1; + } + sort_run(&mut vec[last..]); + last = vec.len(); + while i < end && !occupied(i) { + i += 1; + } + } + vec + } + + /// Returns all map entries in deterministic order. + pub fn sorted_entries_all(&self) -> Vec + where + T: Send + Sync, + { + (0..NUM_SHARDS) + .into_par_iter() + .map(|shard| self.sorted_entries(shard)) + .collect::>() + .into_iter() + .flatten() + .collect() + } + + /// Freezes the map for exclusive, mutable access to its values. + pub fn freeze(self) -> FrozenMap { + FrozenMap { map: self } + } +} + +impl Drop for ConcurrentMap { + fn drop(&mut self) { + if self.entries.is_null() { + return; + } + if std::mem::needs_drop::() { + for idx in 0..self.nbuckets { + if self.key_at(idx).is_some() { + // SAFETY: the value of a published entry is initialized + // and dropped exactly once, here. + unsafe { (*self.entry(idx).value.get()).assume_init_drop() }; + } + } + } + // SAFETY: mapped in with_capacity with the same size. + unsafe { + libc::munmap( + self.entries as *mut libc::c_void, + Self::bufsize(self.nbuckets), + ) + }; + } +} + +impl std::fmt::Debug for ConcurrentMap { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ConcurrentMap({} entries)", self.len()) + } +} + +/// A map after all insertions, whose values can be updated from a +/// unique reference. +#[derive(Debug)] +pub struct FrozenMap { + map: ConcurrentMap, +} + +impl Default for FrozenMap { + fn default() -> Self { + FrozenMap { + map: ConcurrentMap::default(), + } + } +} + +impl FrozenMap { + pub fn get(&self, id: EntryId) -> &T { + self.map.value(id) + } + + pub fn get_mut(&mut self, id: EntryId) -> &mut T { + // SAFETY: the unique reference to the map makes the value's + // UnsafeCell exclusively ours; the entry is occupied. + unsafe { (*self.map.entry(id.0 as usize).value.get()).assume_init_mut() } + } + + /// Returns the address of an occupied value for disjoint parallel updates. + pub fn value_mut_ptr(&self, id: EntryId) -> *mut T { + self.map.entry(id.0 as usize).value.get().cast() + } + + pub fn key(&self, id: EntryId) -> &'static [u8] { + self.map.key(id) + } + + pub fn iter(&self) -> impl Iterator { + self.map.iter() + } + + pub fn sorted_entries(&self, shard: usize) -> Vec { + self.map.sorted_entries(shard) + } + + pub fn len(&self) -> usize { + self.map.len() + } + + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn insert_and_lookup() { + let map: ConcurrentMap = ConcurrentMap::with_capacity(100); + let keys: Vec<&'static [u8]> = (0..1000) + .map(|i| &*Box::leak(format!("key{i}").into_bytes().into_boxed_slice())) + .collect(); + let hash = |k: &[u8]| xxhash_rust::xxh3::xxh3_64(k); + for (i, &k) in keys.iter().enumerate() { + let (_, v, inserted) = map.insert_with(k, hash(k), || i as u32); + assert!(inserted); + assert_eq!(*v, i as u32); + } + for (i, &k) in keys.iter().enumerate() { + let (_, v, inserted) = map.insert_with(k, hash(k), || 99999); + assert!(!inserted); + assert_eq!(*v, i as u32); + assert_eq!(map.get(k, hash(k)).map(|(_, v)| *v), Some(i as u32)); + } + assert_eq!(map.len(), 1000); + assert_eq!(map.iter().count(), 1000); + let sorted: usize = (0..NUM_SHARDS).map(|s| map.sorted_entries(s).len()).sum(); + assert_eq!(sorted, 1000); + } +} diff --git a/src/util/demangle.rs b/src/util/demangle.rs new file mode 100644 index 000000000..4ad714072 --- /dev/null +++ b/src/util/demangle.rs @@ -0,0 +1,64 @@ +//! Symbol name demangling for diagnostics. + +/// Demangles an Itanium C++ ABI symbol name, if `name` is one. +pub fn demangle_cpp(name: &[u8]) -> Option { + if !name.starts_with(b"_Z") { + return None; + } + let sym = cpp_demangle::Symbol::new(name).ok()?; + let options = cpp_demangle::DemangleOptions::default(); + sym.demangle(&options).ok() +} + +/// Demangles a Rust symbol name in either the legacy or the v0 scheme. +/// Crate disambiguator hashes are omitted, as rustc's own tools do. +pub fn demangle_rust(name: &[u8]) -> Option { + let name = std::str::from_utf8(name).ok()?; + if let Ok(sym) = rustc_demangle::try_demangle(name) { + return Some(format!("{sym:#}")); + } + + // A legacy name may be followed by characters that aren't part of the + // mangling, as in `_ZN2ns7versionEv`, which is `ns::version` and a + // trailing `v`. Demangle the path and keep the rest as it is. + let end = legacy_path_end(name)?; + let sym = rustc_demangle::try_demangle(&name[..end]).ok()?; + Some(format!("{sym:#}{}", &name[end..])) +} + +/// The end of the legacy-mangled path `_ZN...E` at the start +/// of `name`, if there is one. +fn legacy_path_end(name: &str) -> Option { + let bytes = name.as_bytes(); + let mut pos = name.strip_prefix("_ZN").map(|_| 3)?; + loop { + if bytes.get(pos) == Some(&b'E') { + return Some(pos + 1); + } + let digits = bytes[pos..] + .iter() + .take_while(|c| c.is_ascii_digit()) + .count(); + let len: usize = name[pos..pos + digits].parse().ok()?; + pos += digits + len; + if len == 0 || pos > bytes.len() { + return None; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rust_legacy_with_suffix() { + assert_eq!( + demangle_rust(b"_ZN2ns7versionEv").as_deref(), + Some("ns::versionv") + ); + assert_eq!(demangle_rust(b"_ZN3foo3barE").as_deref(), Some("foo::bar")); + assert_eq!(demangle_rust(b"_ZN3foo").as_deref(), None); + assert_eq!(demangle_rust(b"main"), None); + } +} diff --git a/src/util/glob.rs b/src/util/glob.rs new file mode 100644 index 000000000..def7541ff --- /dev/null +++ b/src/util/glob.rs @@ -0,0 +1,711 @@ +//! Glob matching for symbol name patterns in version scripts and dynamic +//! lists. +//! +//! Exact, prefix and suffix patterns are matched directly, simple +//! substring patterns are combined into an Aho-Corasick automaton, and the +//! remaining patterns use the non-recursive algorithm described at +//! . If there are many such patterns, a +//! bit-parallel NFA matches them all at once. + +use std::collections::VecDeque; +use std::sync::OnceLock; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Str, + Star, + Question, + Bracket, +} + +#[derive(Clone, Debug)] +struct Token { + kind: Kind, + str: Vec, + chars: Box<[bool; 256]>, +} + +impl Token { + fn new(kind: Kind) -> Self { + Token { + kind, + str: Vec::new(), + chars: Box::new([false; 256]), + } + } +} + +#[derive(Debug)] +struct Pattern { + tokens: Vec, + value: i64, +} + +impl Pattern { + fn compile(mut pat: &[u8], value: i64) -> Option { + let mut tokens: Vec = Vec::new(); + + while let Some((&c, rest)) = pat.split_first() { + pat = rest; + match c { + b'[' => { + // Bracket expressions: [abc], [a-z], [!a-z] and [^a-z]. + // Both `!` and `^` negate; `!` is the POSIX convention + // and `^` is kept for backward compatibility. + let mut tok = Token::new(Kind::Bracket); + let mut negate = false; + let mut closed = false; + + if let Some((&b'!' | &b'^', rest)) = pat.split_first() { + negate = true; + pat = rest; + } + + while let Some((&c, rest)) = pat.split_first() { + if c == b']' { + pat = rest; + closed = true; + break; + } + + if c == b'\\' { + pat = rest; + if pat.is_empty() { + return None; + } + } + + if pat.len() >= 3 && pat[1] == b'-' { + let start = pat[0]; + let mut end = pat[2]; + pat = &pat[3..]; + if end == b'\\' { + end = *pat.first()?; + pat = &pat[1..]; + } + if end < start { + return None; + } + for i in start..=end { + tok.chars[i as usize] = true; + } + } else { + tok.chars[pat[0] as usize] = true; + pat = &pat[1..]; + } + } + + if !closed { + return None; + } + if negate { + for flag in tok.chars.iter_mut() { + *flag = !*flag; + } + } + tokens.push(tok); + } + b'?' => tokens.push(Token::new(Kind::Question)), + b'*' => { + if tokens.last().is_none_or(|t| t.kind != Kind::Star) { + tokens.push(Token::new(Kind::Star)); + } + } + b'\\' => { + let (&escaped, rest) = pat.split_first()?; + pat = rest; + push_char(&mut tokens, escaped); + } + _ => push_char(&mut tokens, c), + } + } + Some(Pattern { tokens, value }) + } + + fn matches(&self, s: &[u8]) -> bool { + if let Some(last) = self.tokens.last() { + if last.kind == Kind::Str && !s.ends_with(&last.str) { + return false; + } + } + + let mut x = 0; + let mut y = 0; + let mut next: Option<(usize, usize)> = None; + + while x < s.len() || y < self.tokens.len() { + if y < self.tokens.len() { + let tok = &self.tokens[y]; + match tok.kind { + Kind::Str => { + if s[x..].starts_with(&tok.str) { + x += tok.str.len(); + y += 1; + continue; + } + } + Kind::Star => { + next = Some((x + 1, y)); + y += 1; + if let Some(tok) = self.tokens.get(y).filter(|t| t.kind == Kind::Str) { + let Some(pos) = find(&s[x..], &tok.str) else { + return false; + }; + let pos = x + pos; + next = Some((pos + 1, y - 1)); + x = pos + tok.str.len(); + y += 1; + } + continue; + } + Kind::Question => { + if x < s.len() { + x += 1; + y += 1; + continue; + } + } + Kind::Bracket => { + if x < s.len() && tok.chars[s[x] as usize] { + x += 1; + y += 1; + continue; + } + } + } + } + + // Retry the last star after assigning one more input byte to it. + match next { + Some((nx, ny)) if nx <= s.len() => { + x = nx; + y = ny; + } + _ => return false, + } + } + true + } +} + +fn push_char(tokens: &mut Vec, c: u8) { + if tokens.last().is_none_or(|t| t.kind != Kind::Str) { + tokens.push(Token::new(Kind::Str)); + } + tokens.last_mut().unwrap().str.push(c); +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() { + return Some(0); + } + + let end = haystack.len().checked_sub(needle.len())? + 1; + let mut pos = 0; + while pos < end { + pos += memchr::memchr(needle[0], &haystack[pos..end])?; + if haystack[pos..].starts_with(needle) { + return Some(pos); + } + pos += 1; + } + None +} + +/// A bit-parallel NFA that matches many glob patterns at once. Each NFA +/// state is one bit, so a byte of input advances all patterns with a few +/// word operations. +#[derive(Debug, Default)] +struct Nfa { + initial_states: Vec, + star_states: Vec, + accept_states: Vec, + char_masks: Vec, + values: Vec, +} + +impl Nfa { + fn compile(patterns: &[Pattern]) -> Nfa { + let mut num_states = 0; + for pattern in patterns { + num_states += 1; + for tok in &pattern.tokens { + match tok.kind { + Kind::Str => num_states += tok.str.len(), + Kind::Star => {} + _ => num_states += 1, + } + } + } + + let num_words = num_states.div_ceil(64); + let mut nfa = Nfa { + initial_states: vec![0; num_words], + star_states: vec![0; num_words], + accept_states: vec![0; num_words], + char_masks: vec![0; 256 * num_words], + values: vec![-1; num_states], + }; + + let set_bit = |vec: &mut [u64], pos: usize| vec[pos / 64] |= 1 << (pos % 64); + let mut state = 0; + + for pattern in patterns { + set_bit(&mut nfa.initial_states, state); + for tok in &pattern.tokens { + match tok.kind { + Kind::Str => { + for &c in &tok.str { + state += 1; + nfa.char_masks[c as usize * num_words + state / 64] |= + 1 << (state % 64); + } + } + Kind::Star => set_bit(&mut nfa.star_states, state), + Kind::Question => { + state += 1; + for c in 0..256 { + nfa.char_masks[c * num_words + state / 64] |= 1 << (state % 64); + } + } + Kind::Bracket => { + state += 1; + for c in 0..256 { + if tok.chars[c] { + nfa.char_masks[c * num_words + state / 64] |= 1 << (state % 64); + } + } + } + } + } + set_bit(&mut nfa.accept_states, state); + nfa.values[state] = pattern.value; + state += 1; + } + nfa + } + + fn is_empty(&self) -> bool { + self.initial_states.is_empty() + } + + fn matches(&self, s: &[u8]) -> i64 { + let num_words = self.initial_states.len(); + let mut states = self.initial_states.clone(); + + for &c in s { + let mask = &self.char_masks[c as usize * num_words..][..num_words]; + let mut carry = 0; + for i in 0..num_words { + let old = states[i]; + let next = (old << 1) | carry; + states[i] = (old & self.star_states[i]) | (next & mask[i]); + carry = old >> 63; + } + } + + let mut value = -1; + for (i, &state) in states.iter().enumerate().take(num_words) { + let mut word = state & self.accept_states[i]; + while word != 0 { + let bit = word.trailing_zeros() as usize; + value = value.max(self.values[i * 64 + bit]); + word &= word - 1; + } + } + value + } +} + +/// An Aho-Corasick automaton for patterns that are plain substring +/// searches, such as `*foo*`, `foo*` or `*foo`. Anchors are represented by +/// a NUL byte at the beginning or end of the pattern. +#[derive(Debug, Default)] +struct AhoCorasick { + root_children: Vec, + nodes: Vec, +} + +#[derive(Debug)] +struct TrieNode { + value: i64, + suffix_link: i32, + first_child: i32, + next_sibling: i32, + ch: u8, +} + +impl Default for TrieNode { + fn default() -> Self { + TrieNode { + value: -1, + suffix_link: -1, + first_child: -1, + next_sibling: -1, + ch: 0, + } + } +} + +impl AhoCorasick { + fn can_handle(pat: &[u8]) -> bool { + let pat = pat.strip_prefix(b"*").unwrap_or(pat); + let pat = pat.strip_suffix(b"*").unwrap_or(pat); + !pat.is_empty() && !pat.iter().any(|&c| matches!(c, b'*' | b'?' | b'[' | b'\\')) + } + + fn find_child(&self, node: i32, ch: u8) -> i32 { + if node == 0 { + return self.root_children[ch as usize]; + } + let mut i = self.nodes[node as usize].first_child; + while i != -1 { + if self.nodes[i as usize].ch == ch { + return i; + } + i = self.nodes[i as usize].next_sibling; + } + -1 + } + + fn add_child(&mut self, node: i32, ch: u8) -> i32 { + let child = self.find_child(node, ch); + if child != -1 { + return child; + } + let child = self.nodes.len() as i32; + let sibling = self.nodes[node as usize].first_child; + self.nodes.push(TrieNode { + next_sibling: sibling, + ch, + ..TrieNode::default() + }); + self.nodes[node as usize].first_child = child; + if node == 0 { + self.root_children[ch as usize] = child; + } + child + } + + fn add(&mut self, pat: &[u8], value: i64) { + debug_assert!(Self::can_handle(pat)); + if self.nodes.is_empty() { + self.root_children = vec![-1; 256]; + self.nodes.push(TrieNode::default()); + } + + // "foo" is handled as "\0foo\0", "*foo" as "foo\0", "foo*" as + // "\0foo" and "*foo*" as "foo": NUL marks the beginning or end of + // the input. + let mut idx = 0; + if !pat.starts_with(b"*") { + idx = self.add_child(idx, 0); + } + for &c in pat { + if c != b'*' { + idx = self.add_child(idx, c); + } + } + if !pat.ends_with(b"*") { + idx = self.add_child(idx, 0); + } + let node = &mut self.nodes[idx as usize]; + node.value = node.value.max(value); + } + + fn compile(&mut self) { + if self.nodes.is_empty() { + return; + } + + // Suffix links refer to nodes at a smaller depth, so build them + // breadth-first. + let mut queue = VecDeque::new(); + let mut child = self.nodes[0].first_child; + while child != -1 { + self.nodes[child as usize].suffix_link = 0; + queue.push_back(child); + child = self.nodes[child as usize].next_sibling; + } + + while let Some(idx) = queue.pop_front() { + let mut child = self.nodes[idx as usize].first_child; + while child != -1 { + let ch = self.nodes[child as usize].ch; + let mut suffix = self.nodes[idx as usize].suffix_link; + while suffix != 0 && self.find_child(suffix, ch) == -1 { + suffix = self.nodes[suffix as usize].suffix_link; + } + let next = self.find_child(suffix, ch); + if next != -1 { + suffix = next; + } + self.nodes[child as usize].suffix_link = suffix; + let suffix_value = self.nodes[suffix as usize].value; + let node = &mut self.nodes[child as usize]; + node.value = node.value.max(suffix_value); + queue.push_back(child); + child = self.nodes[child as usize].next_sibling; + } + } + } + + fn find(&self, s: &[u8]) -> i64 { + if self.nodes.is_empty() { + return -1; + } + + let mut idx = 0; + let mut value = -1; + let mut walk = |c: u8| { + let mut j = idx; + while j != -1 { + let child = self.find_child(j, c); + if child != -1 { + idx = child; + value = value.max(self.nodes[child as usize].value); + return; + } + j = self.nodes[j as usize].suffix_link; + } + idx = 0; + }; + + walk(0); + for &c in s { + walk(c); + } + walk(0); + value + } +} + +#[derive(Debug)] +struct Literal { + pat: Vec, + value: i64, +} + +/// A set of glob patterns, each associated with a value. Looking up a +/// string returns the largest value among the matching patterns, or -1. +#[derive(Debug, Default)] +pub struct Glob { + match_all: i64, + exacts: Vec, + prefixes: Vec, + suffixes: Vec, + patterns: Vec, + aho_corasick: AhoCorasick, + compiled: OnceLock, + is_empty: bool, +} + +#[derive(Debug)] +struct Compiled { + exacts: Vec, + nfa: Nfa, + patterns: Vec, + aho_corasick: AhoCorasick, +} + +fn is_literal(pat: &[u8]) -> bool { + !pat.iter().any(|&c| matches!(c, b'*' | b'?' | b'[' | b'\\')) +} + +impl Glob { + pub fn new() -> Self { + Glob { + match_all: -1, + is_empty: true, + ..Default::default() + } + } + + pub fn is_empty(&self) -> bool { + self.is_empty + } + + /// Adds a pattern. Returns false if the pattern is malformed. + pub fn add(&mut self, pat: &[u8], value: i64) -> bool { + debug_assert!(value >= 0); + debug_assert!(self.compiled.get().is_none()); + self.is_empty = false; + + // Match-all, exact, prefix and suffix patterns are handled with + // plain string comparisons, which are much cheaper than the + // general matchers. + if pat == b"*" { + self.match_all = self.match_all.max(value); + return true; + } + if is_literal(pat) { + self.exacts.push(Literal { + pat: pat.to_vec(), + value, + }); + return true; + } + if let Some(prefix) = pat.strip_suffix(b"*").filter(|p| is_literal(p)) { + self.prefixes.push(Literal { + pat: prefix.to_vec(), + value, + }); + return true; + } + if let Some(suffix) = pat.strip_prefix(b"*").filter(|p| is_literal(p)) { + self.suffixes.push(Literal { + pat: suffix.to_vec(), + value, + }); + return true; + } + if AhoCorasick::can_handle(pat) { + self.aho_corasick.add(pat, value); + return true; + } + match Pattern::compile(pat, value) { + Some(pattern) => { + self.patterns.push(pattern); + true + } + None => false, + } + } + + fn compiled(&self) -> &Compiled { + self.compiled.get_or_init(|| { + // If the same name was added more than once, keep only the + // entry with the largest value, as find() returns the largest + // match. + let mut exacts: Vec = self + .exacts + .iter() + .map(|l| Literal { + pat: l.pat.clone(), + value: l.value, + }) + .collect(); + exacts.sort_by(|a, b| a.pat.cmp(&b.pat).then(b.value.cmp(&a.value))); + exacts.dedup_by(|a, b| a.pat == b.pat); + + let mut patterns: Vec = self + .patterns + .iter() + .map(|p| Pattern { + tokens: p.tokens.clone(), + value: p.value, + }) + .collect(); + let nfa = if patterns.len() >= 64 { + let nfa = Nfa::compile(&patterns); + patterns.clear(); + nfa + } else { + Nfa::default() + }; + + let mut aho_corasick = AhoCorasick { + root_children: self.aho_corasick.root_children.clone(), + nodes: self + .aho_corasick + .nodes + .iter() + .map(|n| TrieNode { + value: n.value, + suffix_link: n.suffix_link, + first_child: n.first_child, + next_sibling: n.next_sibling, + ch: n.ch, + }) + .collect(), + }; + aho_corasick.compile(); + + Compiled { + exacts, + nfa, + patterns, + aho_corasick, + } + }) + } + + /// Returns the largest value of a matching pattern, or -1 if none match. + pub fn find(&self, s: &[u8]) -> i64 { + let compiled = self.compiled(); + let mut value = self.match_all; + + if let Ok(i) = compiled + .exacts + .binary_search_by(|l| l.pat.as_slice().cmp(s)) + { + value = value.max(compiled.exacts[i].value); + } + for p in &self.prefixes { + if value < p.value && s.starts_with(&p.pat) { + value = p.value; + } + } + for p in &self.suffixes { + if value < p.value && s.ends_with(&p.pat) { + value = p.value; + } + } + if !compiled.nfa.is_empty() { + value = value.max(compiled.nfa.matches(s)); + } + for p in &compiled.patterns { + if value < p.value && p.matches(s) { + value = p.value; + } + } + value.max(compiled.aho_corasick.find(s)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn glob(pats: &[&str]) -> Glob { + let mut g = Glob::new(); + for (i, p) in pats.iter().enumerate() { + assert!(g.add(p.as_bytes(), i as i64)); + } + g + } + + #[test] + fn literals() { + let g = glob(&["foo", "bar*", "*baz", "*"]); + assert_eq!(g.find(b"foo"), 3); + assert_eq!(g.find(b"barx"), 3); + assert_eq!(g.find(b"xbaz"), 3); + let g = glob(&["foo", "bar*", "*baz"]); + assert_eq!(g.find(b"foo"), 0); + assert_eq!(g.find(b"barx"), 1); + assert_eq!(g.find(b"xbaz"), 2); + assert_eq!(g.find(b"nothing"), -1); + } + + #[test] + fn complex() { + let g = glob(&["a*b*c", "x?z", "[a-c]d", "*mid*", "[!x]y"]); + assert_eq!(g.find(b"axxbyyc"), 0); + assert_eq!(g.find(b"xyz"), 1); + assert_eq!(g.find(b"bd"), 2); + assert_eq!(g.find(b"dd"), -1); + assert_eq!(g.find(b"leftmidright"), 3); + assert_eq!(g.find(b"ay"), 4); + assert_eq!(g.find(b"xy"), -1); + } + + #[test] + fn many_patterns_use_nfa() { + let pats: Vec = (0..100).map(|i| format!("p{i}*q?r")).collect(); + let refs: Vec<&str> = pats.iter().map(String::as_str).collect(); + let g = glob(&refs); + assert_eq!(g.find(b"p42xxqyr"), 42); + assert_eq!(g.find(b"p42xxqr"), -1); + } +} diff --git a/src/util/hyperloglog.rs b/src/util/hyperloglog.rs new file mode 100644 index 000000000..24b3f1de9 --- /dev/null +++ b/src/util/hyperloglog.rs @@ -0,0 +1,72 @@ +//! Cardinality estimation, for sizing hash tables before filling them. + +/// Estimates the number of distinct values in a stream of hashes. +/// +/// Each hash selects a register by its low bits; the register keeps the +/// longest run of leading zeros seen, plus one. The estimate is a scaled +/// harmonic mean of the registers. +#[derive(Clone)] +pub struct HyperLogLog { + registers: [u8; Self::NUM_REGISTERS], +} + +impl HyperLogLog { + const NUM_REGISTERS: usize = 2048; + const ALPHA: f64 = 0.79402; + + #[inline] + pub fn insert(&mut self, hash: u64) { + let register = &mut self.registers[hash as usize & (Self::NUM_REGISTERS - 1)]; + *register = (*register).max(hash.leading_zeros() as u8 + 1); + } + + /// Combines two estimators, as if every value had been inserted into + /// one of them. + #[inline] + pub fn merged(mut self, other: &HyperLogLog) -> HyperLogLog { + for (a, b) in self.registers.iter_mut().zip(&other.registers) { + *a = (*a).max(*b); + } + self + } + + #[inline] + pub fn cardinality(&self) -> u64 { + let z: f64 = self + .registers + .iter() + .map(|&r| 2f64.powi(-i32::from(r))) + .sum(); + (Self::ALPHA * (Self::NUM_REGISTERS * Self::NUM_REGISTERS) as f64 / z) as u64 + } +} + +impl Default for HyperLogLog { + fn default() -> Self { + HyperLogLog { + registers: [0; Self::NUM_REGISTERS], + } + } +} + +impl Extend for HyperLogLog { + fn extend>(&mut self, hashes: I) { + for hash in hashes { + self.insert(hash); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn estimates_within_a_few_percent() { + let mut hll = HyperLogLog::default(); + hll.extend((0..100_000u64).map(|i| xxhash_rust::xxh3::xxh3_64(&i.to_le_bytes()))); + let n = hll.cardinality(); + // The scale constant follows mold, whose estimates run about 10% high. + assert!((95_000..125_000).contains(&n), "estimate {n}"); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs new file mode 100644 index 000000000..8b964410a --- /dev/null +++ b/src/util/mod.rs @@ -0,0 +1,302 @@ +//! Small helpers shared across the linker. + +pub mod compress; +pub mod concurrent_map; +pub mod demangle; +pub mod glob; +pub mod hyperloglog; +pub mod tar; +pub mod timer; + +/// Rounds `value` up to a multiple of `align`, which must be zero or a power +/// of two. Zero means "no alignment". +#[inline] +pub fn align_to(value: u64, align: u64) -> u64 { + if align == 0 { + return value; + } + debug_assert!(align.is_power_of_two()); + (value + align - 1) & !(align - 1) +} + +/// Rounds `value` down to a multiple of `align`, which must be a power of two. +pub fn align_down(value: u64, align: u64) -> u64 { + debug_assert!(align.is_power_of_two()); + value & !(align - 1) +} + +/// Returns bit `pos` of `value`. +pub fn bit(value: u64, pos: u32) -> u64 { + (value >> pos) & 1 +} + +/// Returns bits `hi..=lo` of `value`, shifted down to start at bit zero. +#[inline] +pub fn bits(value: u64, hi: u32, lo: u32) -> u64 { + (value >> lo) & ((1u64 << (hi - lo + 1)) - 1) +} + +/// Sign-extends the low `n` bits of `value`. +pub fn sign_extend(value: u64, n: u32) -> i64 { + ((value << (64 - n)) as i64) >> (64 - n) +} + +/// Whether `value` is representable as a signed `n`-bit integer. +pub fn is_int(value: i64, n: u32) -> bool { + sign_extend(value as u64, n) == value +} + +/// Whether `value` is representable as an unsigned `n`-bit integer. +pub fn is_uint(value: u64, n: u32) -> bool { + n >= 64 || value >> n == 0 +} + +/// Writes a NUL-terminated string and returns the number of bytes written. +pub fn write_cstr(buf: &mut [u8], s: &[u8]) -> usize { + buf[..s.len()].copy_from_slice(s); + buf[s.len()] = 0; + s.len() + 1 +} + +/// Returns the NUL-terminated string starting at `offset` in a string table. +/// The result excludes the terminator. A missing terminator yields the rest +/// of the table. +#[inline] +pub fn cstr_at(table: &[u8], offset: usize) -> &[u8] { + let rest = table.get(offset..).unwrap_or(&[]); + if rest.is_empty() { + return rest; + } + + // ELF string tables normally end in NUL, so strlen cannot read past the + // table. Keep the bounded path for malformed tables without a terminator. + let end = unsafe { + if table.last() == Some(&0) { + libc::strlen(rest.as_ptr().cast()) + } else { + libc::strnlen(rest.as_ptr().cast(), rest.len()) + } + }; + &rest[..end] +} + +/// Returns the position of `byte` in `data`. +#[inline] +pub fn find_byte(byte: u8, data: &[u8]) -> Option { + // SAFETY: memchr reads at most data.len() bytes from the slice. A + // non-null result therefore points into the same allocation. + let ptr = unsafe { libc::memchr(data.as_ptr().cast(), byte.into(), data.len()) }.cast::(); + if ptr.is_null() { + None + } else { + Some(unsafe { ptr.offset_from(data.as_ptr()) as usize }) + } +} + +/// Appends `value` in unsigned LEB128 encoding. +pub fn encode_uleb(out: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +/// Appends `value` in signed LEB128 encoding. +pub fn encode_sleb(out: &mut Vec, mut value: i64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + let negative = byte & 0x40 != 0; + if (value == 0 && !negative) || (value == -1 && negative) { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +/// The number of bytes `value` occupies in unsigned LEB128 encoding. +pub fn uleb_size(value: u64) -> usize { + let bits = 64 - value.leading_zeros() as usize; + bits.div_ceil(7).max(1) +} + +/// Overwrites an existing unsigned LEB128 value in place, keeping its length. +pub fn overwrite_uleb(buf: &mut [u8], mut value: u64) { + let mut i = 0; + while buf[i] & 0x80 != 0 { + buf[i] = 0x80 | (value & 0x7f) as u8; + value >>= 7; + i += 1; + } + buf[i] = (value & 0x7f) as u8; +} + +/// Reads an unsigned LEB128 value, advancing `bytes` past it. +#[inline] +pub fn read_uleb(bytes: &mut &[u8]) -> u64 { + let mut value = 0; + let mut shift = 0; + loop { + let (&byte, rest) = bytes.split_first().expect("truncated LEB128"); + *bytes = rest; + if shift < 64 { + value |= ((byte & 0x7f) as u64) << shift; + } + shift += 7; + if byte & 0x80 == 0 { + return value; + } + } +} + +/// Reads a signed LEB128 value, advancing `bytes` past it. +#[inline] +pub fn read_sleb(bytes: &mut &[u8]) -> i64 { + let mut value = 0u64; + let mut shift = 0; + loop { + let (&byte, rest) = bytes.split_first().expect("truncated LEB128"); + *bytes = rest; + if shift < 64 { + value |= ((byte & 0x7f) as u64) << shift; + } + shift += 7; + if byte & 0x80 == 0 { + return if shift < 64 { + sign_extend(value, shift) + } else { + value as i64 + }; + } + } +} + +/// Removes consecutive duplicates from a sorted vector. +pub fn dedup_sorted(vec: &mut Vec) { + vec.dedup(); +} + +/// Fills `buf` with random bytes from the operating system. +pub fn random_bytes(buf: &mut [u8]) { + use std::io::Read; + let mut file = std::fs::File::open("/dev/urandom").expect("cannot open /dev/urandom"); + file.read_exact(buf).expect("cannot read /dev/urandom"); +} + +/// Leaks a value for the rest of the process's lifetime. +/// +/// Input files, symbol names and a few other objects must outlive every +/// data structure of a link, and the process exits as soon as the link is +/// done, so never freeing them is both simplest and cheapest. +pub fn leak(value: T) -> &'static T { + Box::leak(Box::new(value)) +} + +/// Leaks a byte string for the rest of the process's lifetime. +pub fn leak_bytes(bytes: Vec) -> &'static [u8] { + Vec::leak(bytes) +} + +/// Leaks a string for the rest of the process's lifetime. +pub fn leak_str(s: String) -> &'static str { + String::leak(s) +} + +/// Returns the path of the running executable. +pub fn self_path() -> std::path::PathBuf { + std::fs::read_link("/proc/self/exe").expect("cannot read /proc/self/exe") +} + +/// Normalizes a path lexically, resolving `.` and `..` components without +/// consulting the file system. +pub fn path_clean(path: &str) -> String { + use std::path::{Component, Path, PathBuf}; + let mut out = PathBuf::new(); + for component in Path::new(path).components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if !matches!( + out.components().next_back(), + Some(Component::RootDir) | None + ) { + out.pop(); + } else if out.components().next_back().is_none() { + out.push(".."); + } + } + other => out.push(other), + } + } + let s = out.to_string_lossy().into_owned(); + if s.is_empty() { + ".".to_string() + } else { + s + } +} + +/// Returns the directory part of a path, as `dirname(1)` would. +pub fn path_dirname(path: &str) -> String { + match std::path::Path::new(path).parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.to_string_lossy().into_owned(), + _ => ".".to_string(), + } +} + +/// Returns the file name part of a path. +pub fn path_filename(path: &str) -> String { + std::path::Path::new(path) + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Formats a byte string for diagnostics, replacing invalid UTF-8. +pub fn display(bytes: &[u8]) -> std::borrow::Cow<'_, str> { + String::from_utf8_lossy(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leb128_roundtrip() { + for &value in &[0u64, 1, 127, 128, 300, u64::MAX] { + let mut buf = Vec::new(); + encode_uleb(&mut buf, value); + assert_eq!(buf.len(), uleb_size(value)); + let mut slice = buf.as_slice(); + assert_eq!(read_uleb(&mut slice), value); + assert!(slice.is_empty()); + } + for &value in &[0i64, -1, 63, 64, -64, -65, i64::MIN, i64::MAX] { + let mut buf = Vec::new(); + encode_sleb(&mut buf, value); + let mut slice = buf.as_slice(); + assert_eq!(read_sleb(&mut slice), value); + } + } + + #[test] + fn sign_extension() { + assert_eq!(sign_extend(0xff, 8), -1); + assert_eq!(sign_extend(0x7f, 8), 127); + assert!(is_int(-128, 8)); + assert!(!is_int(128, 8)); + } + + #[test] + fn clean_paths() { + assert_eq!(path_clean("a/./b/../c"), "a/c"); + assert_eq!(path_clean("/a/../.."), "/"); + assert_eq!(path_clean("../a"), "../a"); + } +} diff --git a/src/util/tar.rs b/src/util/tar.rs new file mode 100644 index 000000000..c5bfde08e --- /dev/null +++ b/src/util/tar.rs @@ -0,0 +1,75 @@ +//! A minimal tar writer, used by `--repro` to bundle input files. + +use std::fs::File; +use std::io::{self, Seek, SeekFrom, Write}; + +use super::{align_to, path_clean}; + +const BLOCK_SIZE: u64 = 512; + +/// A tar file consists of one or more ustar headers each followed by file +/// data. The `name` field of a ustar header is only 100 bytes long, so a +/// PAX header carrying the full path is emitted before every entry. +pub struct TarWriter { + out: File, + basedir: String, +} + +fn ustar_header(name: &[u8], mode: &[u8], size: u64, typeflag: u8) -> [u8; 512] { + let mut hdr = [0u8; 512]; + hdr[..name.len()].copy_from_slice(name); + hdr[100..100 + mode.len()].copy_from_slice(mode); + let size = format!("{size:011o}"); + hdr[124..124 + size.len()].copy_from_slice(size.as_bytes()); + hdr[148..156].fill(b' '); + hdr[156] = typeflag; + hdr[257..262].copy_from_slice(b"ustar"); + hdr[263..265].copy_from_slice(b"00"); + + let checksum: u32 = hdr.iter().map(|&b| b as u32).sum(); + let checksum = format!("{checksum:06o}\0"); + hdr[148..148 + checksum.len()].copy_from_slice(checksum.as_bytes()); + hdr +} + +/// Builds a PAX extended header record for a path: "N path=...\n" where N +/// is the length of the whole record including N itself. +fn encode_path(basedir: &str, path: &str) -> String { + let path = path_clean(&format!("{basedir}/{path}")); + let len = " path=\n".len() + path.len(); + let total = len + len.to_string().len(); + let total = len + total.to_string().len(); + format!("{total} path={path}\n") +} + +impl TarWriter { + pub fn open(output_path: &str, basedir: &str) -> io::Result { + Ok(TarWriter { + out: File::create(output_path)?, + basedir: basedir.to_string(), + }) + } + + pub fn append(&mut self, path: &str, data: &[u8]) -> io::Result<()> { + let attr = encode_path(&self.basedir, path); + self.out + .write_all(&ustar_header(b"/", b"", attr.len() as u64, b'x'))?; + self.out.write_all(attr.as_bytes())?; + self.pad()?; + + self.out + .write_all(&ustar_header(b"", b"0000664", data.len() as u64, b'0'))?; + self.out.write_all(data)?; + self.pad()?; + + // A tar file must end with two empty blocks. + let pos = self.out.stream_position()?; + self.out.set_len(pos + BLOCK_SIZE * 2) + } + + fn pad(&mut self) -> io::Result<()> { + let pos = self.out.stream_position()?; + self.out.seek(SeekFrom::Start(align_to(pos, BLOCK_SIZE)))?; + Ok(()) + } +} diff --git a/src/util/timer.rs b/src/util/timer.rs new file mode 100644 index 000000000..ce28ab4c6 --- /dev/null +++ b/src/util/timer.rs @@ -0,0 +1,166 @@ +//! Wall-clock and CPU time accounting for `--perf`. + +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +#[derive(Debug)] +struct Record { + name: String, + start: Instant, + end: Option, + user_start: f64, + sys_start: f64, + user: f64, + sys: f64, + parent: Option, + children: Vec, +} + +/// Collects timing records for the passes of a link. Cloning shares the +/// underlying records. +#[derive(Clone, Debug, Default)] +pub struct Timers { + records: Arc>>, +} + +/// A running timer, stopped when dropped. +pub struct Timer { + timers: Timers, + index: usize, + stopped: bool, +} + +/// A clonable handle for starting timers beneath a running timer. +#[derive(Clone)] +pub struct TimerHandle { + timers: Timers, + parent: usize, +} + +fn rusage() -> (f64, f64) { + // SAFETY: `usage` is a valid, writable rusage struct. + let mut usage: libc::rusage = unsafe { std::mem::zeroed() }; + unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut usage) }; + let to_secs = |t: libc::timeval| t.tv_sec as f64 + t.tv_usec as f64 / 1_000_000.0; + (to_secs(usage.ru_utime), to_secs(usage.ru_stime)) +} + +impl Timers { + pub fn new() -> Self { + Timers::default() + } + + /// Starts a timer, nested in the timer still running that started last. + pub fn start(&self, name: &str) -> Timer { + let records = self.records.lock().unwrap(); + let parent = records.iter().rposition(|r| r.end.is_none()); + drop(records); + self.start_child(name, parent) + } + + fn start_child(&self, name: &str, parent: Option) -> Timer { + let (user, sys) = rusage(); + let mut records = self.records.lock().unwrap(); + let index = records.len(); + records.push(Record { + name: name.to_string(), + start: Instant::now(), + end: None, + user_start: user, + sys_start: sys, + user: 0.0, + sys: 0.0, + parent, + children: Vec::new(), + }); + if let Some(parent) = parent { + records[parent].children.push(index); + } + Timer { + timers: self.clone(), + index, + stopped: false, + } + } + + fn stop(&self, index: usize) { + let (user, sys) = rusage(); + let mut records = self.records.lock().unwrap(); + let record = &mut records[index]; + if record.end.is_none() { + record.end = Some(Instant::now()); + record.user = user - record.user_start; + record.sys = sys - record.sys_start; + } + } + + pub fn print(&self) { + let mut records = self.records.lock().unwrap(); + let now = Instant::now(); + for r in records.iter_mut() { + if r.end.is_none() { + r.end = Some(now); + } + } + + fn print_rec(records: &[Record], i: usize, indent: usize) { + let r = &records[i]; + let real = r.end.unwrap().duration_since(r.start).as_secs_f64(); + println!( + " {:8.3} {:8.3} {:8.3} {}{}", + r.user, + r.sys, + real, + " ".repeat(indent * 2), + r.name + ); + let mut children = r.children.clone(); + children.sort_by_key(|&c| records[c].start); + for child in children { + print_rec(records, child, indent + 1); + } + } + + println!(" User System Real Name"); + for i in 0..records.len() { + if records[i].parent.is_none() { + print_rec(&records, i, 0); + } + } + } +} + +impl Timer { + /// Starts a timer nested in this one. Timers started from parallel + /// tasks name their parent this way, since which timer started last + /// says nothing about nesting then. + pub fn child(&self, name: &str) -> Timer { + self.timers.start_child(name, Some(self.index)) + } + + pub fn handle(&self) -> TimerHandle { + TimerHandle { + timers: self.timers.clone(), + parent: self.index, + } + } + + pub fn stop(&mut self) { + if !self.stopped { + self.stopped = true; + self.timers.stop(self.index); + } + } +} + +impl TimerHandle { + pub fn child(&self, name: &str) -> Timer { + self.timers.start_child(name, Some(self.parent)) + } +} + +impl Drop for Timer { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/targets/arm32/Cargo.toml b/targets/arm32/Cargo.toml new file mode 100644 index 000000000..76cc8e9de --- /dev/null +++ b/targets/arm32/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-arm32" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for arm32" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/arm32/src/lib.rs b/targets/arm32/src/lib.rs new file mode 100644 index 000000000..5c4cfb1e0 --- /dev/null +++ b/targets/arm32/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for arm32. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/arm32be/Cargo.toml b/targets/arm32be/Cargo.toml new file mode 100644 index 000000000..93aff99cc --- /dev/null +++ b/targets/arm32be/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-arm32be" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for arm32be" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/arm32be/src/lib.rs b/targets/arm32be/src/lib.rs new file mode 100644 index 000000000..627693dcf --- /dev/null +++ b/targets/arm32be/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for arm32be. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/arm64/Cargo.toml b/targets/arm64/Cargo.toml new file mode 100644 index 000000000..0b497bd68 --- /dev/null +++ b/targets/arm64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-arm64" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for arm64" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/arm64/src/lib.rs b/targets/arm64/src/lib.rs new file mode 100644 index 000000000..7d9bc271b --- /dev/null +++ b/targets/arm64/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for arm64. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/arm64be/Cargo.toml b/targets/arm64be/Cargo.toml new file mode 100644 index 000000000..98827d5b6 --- /dev/null +++ b/targets/arm64be/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-arm64be" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for arm64be" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/arm64be/src/lib.rs b/targets/arm64be/src/lib.rs new file mode 100644 index 000000000..96acc54a2 --- /dev/null +++ b/targets/arm64be/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for arm64be. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/i386/Cargo.toml b/targets/i386/Cargo.toml new file mode 100644 index 000000000..3da30532d --- /dev/null +++ b/targets/i386/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-i386" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for i386" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/i386/src/lib.rs b/targets/i386/src/lib.rs new file mode 100644 index 000000000..c9ff9f191 --- /dev/null +++ b/targets/i386/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for i386. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/loongarch32/Cargo.toml b/targets/loongarch32/Cargo.toml new file mode 100644 index 000000000..4b3852a92 --- /dev/null +++ b/targets/loongarch32/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-loongarch32" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for loongarch32" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/loongarch32/src/lib.rs b/targets/loongarch32/src/lib.rs new file mode 100644 index 000000000..2996b64b5 --- /dev/null +++ b/targets/loongarch32/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for loongarch32. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/loongarch64/Cargo.toml b/targets/loongarch64/Cargo.toml new file mode 100644 index 000000000..eb4c14a63 --- /dev/null +++ b/targets/loongarch64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-loongarch64" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for loongarch64" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/loongarch64/src/lib.rs b/targets/loongarch64/src/lib.rs new file mode 100644 index 000000000..9ff6dc39e --- /dev/null +++ b/targets/loongarch64/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for loongarch64. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/m68k/Cargo.toml b/targets/m68k/Cargo.toml new file mode 100644 index 000000000..8a219068d --- /dev/null +++ b/targets/m68k/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-m68k" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for m68k" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/m68k/src/lib.rs b/targets/m68k/src/lib.rs new file mode 100644 index 000000000..b41f7d941 --- /dev/null +++ b/targets/m68k/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for m68k. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/ppc32/Cargo.toml b/targets/ppc32/Cargo.toml new file mode 100644 index 000000000..21a48252d --- /dev/null +++ b/targets/ppc32/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-ppc32" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for ppc32" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/ppc32/src/lib.rs b/targets/ppc32/src/lib.rs new file mode 100644 index 000000000..806726c98 --- /dev/null +++ b/targets/ppc32/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for ppc32. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/ppc64v1/Cargo.toml b/targets/ppc64v1/Cargo.toml new file mode 100644 index 000000000..3c77b1a32 --- /dev/null +++ b/targets/ppc64v1/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-ppc64v1" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for ppc64v1" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/ppc64v1/src/lib.rs b/targets/ppc64v1/src/lib.rs new file mode 100644 index 000000000..e21a9d786 --- /dev/null +++ b/targets/ppc64v1/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for ppc64v1. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/ppc64v2/Cargo.toml b/targets/ppc64v2/Cargo.toml new file mode 100644 index 000000000..00f601832 --- /dev/null +++ b/targets/ppc64v2/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-ppc64v2" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for ppc64v2" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/ppc64v2/src/lib.rs b/targets/ppc64v2/src/lib.rs new file mode 100644 index 000000000..86b54d54e --- /dev/null +++ b/targets/ppc64v2/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for ppc64v2. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/riscv32/Cargo.toml b/targets/riscv32/Cargo.toml new file mode 100644 index 000000000..2f3ed5e01 --- /dev/null +++ b/targets/riscv32/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-riscv32" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for riscv32" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/riscv32/src/lib.rs b/targets/riscv32/src/lib.rs new file mode 100644 index 000000000..35c92505b --- /dev/null +++ b/targets/riscv32/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for riscv32. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/riscv32be/Cargo.toml b/targets/riscv32be/Cargo.toml new file mode 100644 index 000000000..594706403 --- /dev/null +++ b/targets/riscv32be/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-riscv32be" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for riscv32be" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/riscv32be/src/lib.rs b/targets/riscv32be/src/lib.rs new file mode 100644 index 000000000..9258eefee --- /dev/null +++ b/targets/riscv32be/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for riscv32be. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/riscv64/Cargo.toml b/targets/riscv64/Cargo.toml new file mode 100644 index 000000000..8385fdf3d --- /dev/null +++ b/targets/riscv64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-riscv64" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for riscv64" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/riscv64/src/lib.rs b/targets/riscv64/src/lib.rs new file mode 100644 index 000000000..5bd7971ae --- /dev/null +++ b/targets/riscv64/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for riscv64. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/riscv64be/Cargo.toml b/targets/riscv64be/Cargo.toml new file mode 100644 index 000000000..040808fa5 --- /dev/null +++ b/targets/riscv64be/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-riscv64be" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for riscv64be" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/riscv64be/src/lib.rs b/targets/riscv64be/src/lib.rs new file mode 100644 index 000000000..7123138de --- /dev/null +++ b/targets/riscv64be/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for riscv64be. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/s390x/Cargo.toml b/targets/s390x/Cargo.toml new file mode 100644 index 000000000..764d87ec6 --- /dev/null +++ b/targets/s390x/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-s390x" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for s390x" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/s390x/src/lib.rs b/targets/s390x/src/lib.rs new file mode 100644 index 000000000..6a0c90aa0 --- /dev/null +++ b/targets/s390x/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for s390x. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/sh4/Cargo.toml b/targets/sh4/Cargo.toml new file mode 100644 index 000000000..02e449699 --- /dev/null +++ b/targets/sh4/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-sh4" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for sh4" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/sh4/src/lib.rs b/targets/sh4/src/lib.rs new file mode 100644 index 000000000..33045b3f3 --- /dev/null +++ b/targets/sh4/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for sh4. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/sh4be/Cargo.toml b/targets/sh4be/Cargo.toml new file mode 100644 index 000000000..f4a1c3cbd --- /dev/null +++ b/targets/sh4be/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-sh4be" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for sh4be" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/sh4be/src/lib.rs b/targets/sh4be/src/lib.rs new file mode 100644 index 000000000..92ccad71e --- /dev/null +++ b/targets/sh4be/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for sh4be. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/sparc64/Cargo.toml b/targets/sparc64/Cargo.toml new file mode 100644 index 000000000..68294af01 --- /dev/null +++ b/targets/sparc64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-sparc64" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for sparc64" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/sparc64/src/lib.rs b/targets/sparc64/src/lib.rs new file mode 100644 index 000000000..2a08c3bb0 --- /dev/null +++ b/targets/sparc64/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for sparc64. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/targets/x86_64/Cargo.toml b/targets/x86_64/Cargo.toml new file mode 100644 index 000000000..08ac54a7f --- /dev/null +++ b/targets/x86_64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mold-target-x86_64" +version = "2.42.0" +edition = "2021" +description = "The mold linker instantiated for x86_64" +license = "MIT" + +[dependencies] +mold = { path = "../.." } diff --git a/targets/x86_64/src/lib.rs b/targets/x86_64/src/lib.rs new file mode 100644 index 000000000..72d5755dd --- /dev/null +++ b/targets/x86_64/src/lib.rs @@ -0,0 +1,10 @@ +//! The linker instantiated for x86_64. Every target has a crate like this +//! one, so that the compiler can build the targets in parallel. + +use mold::diagnostics::Diagnostics; + +/// Links for this target, or reports the target the inputs are actually +/// for. +pub fn link(cmdline: &[String], diag: &Diagnostics) -> Result { + mold::driver::link::(cmdline, diag) +} diff --git a/test-runner/Cargo.toml b/test-runner/Cargo.toml new file mode 100644 index 000000000..6899e3036 --- /dev/null +++ b/test-runner/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mold-test-runner" +version = "2.42.0" +edition = "2021" +description = "Runner for mold's shell test suite" +license = "MIT" +publish = false + +[dependencies] +libc = "0.2" diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs new file mode 100644 index 000000000..cb5003fb9 --- /dev/null +++ b/test-runner/src/main.rs @@ -0,0 +1,761 @@ +//! Runs mold's upstream shell tests in parallel. +//! +//! The tests themselves deliberately remain shell scripts so that this port +//! exercises exactly the same inputs and toolchains as C++ mold. This binary +//! owns test discovery, target selection, scheduling, timeouts and reporting. + +use std::collections::BTreeMap; +use std::env; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc}; +use std::thread; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::os::unix::process::CommandExt; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); + +#[derive(Clone, Copy)] +struct TargetSpec { + machine: &'static str, + triple: &'static str, + qemu: &'static str, +} + +// Keep this list in the same order as mold's test/CMakeLists.txt. +const TARGETS: &[TargetSpec] = &[ + TargetSpec { + machine: "x86_64", + triple: "x86_64-linux-gnu", + qemu: "qemu-x86_64", + }, + TargetSpec { + machine: "i686", + triple: "i686-linux-gnu", + qemu: "qemu-i386", + }, + TargetSpec { + machine: "aarch64", + triple: "aarch64-linux-gnu", + qemu: "qemu-aarch64", + }, + TargetSpec { + machine: "aarch64_be", + triple: "aarch64_be-linux-gnu", + qemu: "qemu-aarch64_be", + }, + TargetSpec { + machine: "arm", + triple: "arm-linux-gnueabihf", + qemu: "qemu-arm", + }, + TargetSpec { + machine: "armeb", + triple: "armeb-linux-gnueabihf", + qemu: "qemu-armeb", + }, + TargetSpec { + machine: "riscv64", + triple: "riscv64-linux-gnu", + qemu: "qemu-riscv64", + }, + TargetSpec { + machine: "riscv32", + triple: "riscv32-linux-gnu", + qemu: "qemu-riscv32", + }, + TargetSpec { + machine: "ppc", + triple: "powerpc-linux-gnu", + qemu: "qemu-ppc", + }, + TargetSpec { + machine: "ppc64", + triple: "powerpc64-linux-gnu", + qemu: "qemu-ppc64", + }, + TargetSpec { + machine: "ppc64le", + triple: "powerpc64le-linux-gnu", + qemu: "qemu-ppc64le", + }, + TargetSpec { + machine: "sparc64", + triple: "sparc64-linux-gnu", + qemu: "qemu-sparc64", + }, + TargetSpec { + machine: "s390x", + triple: "s390x-linux-gnu", + qemu: "qemu-s390x", + }, + TargetSpec { + machine: "sh4", + triple: "sh4-linux-gnu", + qemu: "qemu-sh4", + }, + TargetSpec { + machine: "sh4aeb", + triple: "sh4aeb-linux-gnu", + qemu: "qemu-sh4eb", + }, + TargetSpec { + machine: "m68k", + triple: "m68k-linux-gnu", + qemu: "qemu-m68k", + }, + TargetSpec { + machine: "loongarch64", + triple: "loongarch64-linux-gnu", + qemu: "qemu-loongarch64", + }, +]; + +#[derive(Clone, Debug)] +struct Target { + machine: String, + triple: Option, + cpu: Option, + label: String, +} + +impl Target { + fn native(machine: String) -> Target { + Target { + label: machine.clone(), + machine, + triple: None, + cpu: None, + } + } + + fn cross(machine: String, triple: String) -> Target { + Target { + label: machine.clone(), + machine, + triple: Some(triple), + cpu: None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + Native, + All, + Triple, +} + +struct Options { + jobs: usize, + mode: Mode, + triple: Option, + cpu: Option, + patterns: Vec, + timeout: Duration, + list: bool, +} + +#[derive(Clone)] +struct TestJob { + target: Arc, + script: PathBuf, + name: String, + log: PathBuf, + status_file: PathBuf, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Outcome { + Pass, + Skip, + Fail, + Timeout, +} + +impl Outcome { + fn status(self) -> &'static str { + match self { + Outcome::Pass => "pass", + Outcome::Skip => "skip", + Outcome::Fail | Outcome::Timeout => "fail", + } + } +} + +struct TestResult { + target: Arc, + name: String, + log: PathBuf, + outcome: Outcome, +} + +#[derive(Default)] +struct Counts { + pass: usize, + skip: usize, + fail: usize, +} + +impl Counts { + fn add(&mut self, outcome: Outcome) { + match outcome { + Outcome::Pass => self.pass += 1, + Outcome::Skip => self.skip += 1, + Outcome::Fail | Outcome::Timeout => self.fail += 1, + } + } + + fn merge(&mut self, other: &Counts) { + self.pass += other.pass; + self.skip += other.skip; + self.fail += other.fail; + } +} + +fn usage() -> ! { + eprintln!( + "Usage: run-tests.sh [-j N] [--native | --all | --triple TRIPLE] \ + [--cpu CPU] [--timeout SECONDS] [--list] [pattern ...]" + ); + std::process::exit(2); +} + +fn parse_usize(value: Option) -> usize { + value + .and_then(|s| s.parse().ok()) + .filter(|&n| n != 0) + .unwrap_or_else(|| usage()) +} + +fn parse_options() -> Options { + let mut jobs = thread::available_parallelism().map_or(1, usize::from); + let mut mode = Mode::Native; + let mut mode_was_set = false; + let mut triple = None; + let mut cpu = None; + let mut patterns = Vec::new(); + let mut timeout = DEFAULT_TIMEOUT; + let mut list = false; + + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "-j" | "--jobs" => jobs = parse_usize(args.next()), + "--native" => { + mode = Mode::Native; + mode_was_set = true; + triple = None; + } + "--all" => { + mode = Mode::All; + mode_was_set = true; + triple = None; + } + "--triple" => { + mode = Mode::Triple; + mode_was_set = true; + triple = args.next(); + if triple.is_none() { + usage(); + } + } + "--cpu" => { + cpu = args.next(); + if cpu.is_none() { + usage(); + } + } + "--timeout" => timeout = Duration::from_secs(parse_usize(args.next()) as u64), + "--list" => list = true, + "-h" | "--help" => usage(), + _ if arg.starts_with("-j") && arg.len() > 2 => { + jobs = parse_usize(Some(arg[2..].to_owned())) + } + _ if arg.starts_with('-') => usage(), + _ => patterns.push(arg), + } + } + + // Preserve the old runner's TRIPLE/CPU interface for callers that set + // the cross target in the environment. + if !mode_was_set { + if let Some(value) = env::var_os("TRIPLE").filter(|s| !s.is_empty()) { + mode = Mode::Triple; + triple = Some(value.to_string_lossy().into_owned()); + } + if cpu.is_none() { + cpu = env::var_os("CPU") + .filter(|s| !s.is_empty()) + .map(|s| s.to_string_lossy().into_owned()); + } + } + + Options { + jobs, + mode, + triple, + cpu, + patterns, + timeout, + list, + } +} + +fn canonical_machine(machine: &str) -> String { + let machine = machine.trim(); + if machine == "amd64" { + return "x86_64".to_owned(); + } + if machine.len() == 4 && machine.starts_with('i') && machine.ends_with("86") { + return "i686".to_owned(); + } + if machine.starts_with("armeb") { + return "armeb".to_owned(); + } + if machine.starts_with("arm") { + return "arm".to_owned(); + } + match machine { + "powerpc" => "ppc".to_owned(), + "powerpc64" => "ppc64".to_owned(), + "powerpc64le" => "ppc64le".to_owned(), + _ => machine.to_owned(), + } +} + +fn machine_from_triple(triple: &str) -> String { + canonical_machine(triple.split('-').next().unwrap_or(triple)) +} + +fn native_machine() -> String { + if let Some(machine) = env::var_os("MACHINE").filter(|s| !s.is_empty()) { + return canonical_machine(&machine.to_string_lossy()); + } + + if let Ok(output) = Command::new("cc").arg("-dumpmachine").output() { + if output.status.success() { + let triple = String::from_utf8_lossy(&output.stdout); + if !triple.trim().is_empty() { + return machine_from_triple(&triple); + } + } + } + canonical_machine(env::consts::ARCH) +} + +fn command_exists(command: &str) -> bool { + let path = Path::new(command); + if path.components().count() > 1 { + return path.is_file(); + } + env::var_os("PATH") + .into_iter() + .flat_map(|paths| env::split_paths(&paths).collect::>()) + .any(|dir| dir.join(command).is_file()) +} + +fn supports_power10() -> bool { + if !command_exists("powerpc64le-linux-gnu-gcc") || !command_exists("qemu-ppc64le") { + return false; + } + + let compiler = Command::new("powerpc64le-linux-gnu-gcc") + .args(["-mcpu=power10", "-E", "-x", "c", "-"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + let qemu = Command::new("qemu-ppc64le") + .args(["-cpu", "help"]) + .output() + .is_ok_and(|output| String::from_utf8_lossy(&output.stdout).contains("power10_v2.0")); + compiler && qemu +} + +fn all_targets(native: &str) -> (Vec, Vec<&'static TargetSpec>) { + let mut targets = Vec::new(); + let mut unavailable = Vec::new(); + + for spec in TARGETS { + if spec.machine == native { + targets.push(Target::native(native.to_owned())); + } else if command_exists(spec.qemu) && command_exists(&format!("{}-gcc", spec.triple)) { + targets.push(Target::cross( + spec.machine.to_owned(), + spec.triple.to_owned(), + )); + } else { + unavailable.push(spec); + } + } + + if native == "ppc64le" + && fs::read_to_string("/proc/cpuinfo").is_ok_and(|s| s.contains("POWER10")) + { + let mut target = Target::native(native.to_owned()); + target.cpu = Some("power10".to_owned()); + target.label = "ppc64le-power10".to_owned(); + targets.push(target); + } else if supports_power10() { + let mut target = Target::cross("ppc64le".to_owned(), "powerpc64le-linux-gnu".to_owned()); + target.cpu = Some("power10".to_owned()); + target.label = "ppc64le-power10".to_owned(); + targets.push(target); + } + + (targets, unavailable) +} + +fn selected_targets(options: &Options) -> (Vec, Vec<&'static TargetSpec>) { + let native = native_machine(); + match options.mode { + Mode::Native => (vec![Target::native(native)], Vec::new()), + Mode::All => all_targets(&native), + Mode::Triple => { + let triple = options.triple.clone().unwrap_or_else(|| usage()); + let machine = machine_from_triple(&triple); + let mut target = Target::cross(machine, triple); + target.cpu.clone_from(&options.cpu); + if let Some(cpu) = &target.cpu { + target.label = format!("{}-{cpu}", target.machine); + } + (vec![target], Vec::new()) + } + } +} + +fn matches_target(name: &str, machine: &str) -> bool { + !name.starts_with("arch-") || name.starts_with(&format!("arch-{machine}-")) +} + +fn matches_patterns(name: &str, patterns: &[String]) -> bool { + patterns.is_empty() || patterns.iter().any(|pattern| name.contains(pattern)) +} + +fn discover_scripts(test_dir: &Path) -> io::Result> { + let mut scripts = Vec::new(); + for entry in fs::read_dir(test_dir)? { + let path = entry?.path(); + if path.extension() != Some(OsStr::new("sh")) { + continue; + } + let name = path.file_stem().unwrap().to_string_lossy().into_owned(); + scripts.push((name, path)); + } + scripts.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(scripts) +} + +fn clear_results(dir: &Path) -> io::Result<()> { + fs::create_dir_all(dir)?; + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if matches!( + path.extension().and_then(OsStr::to_str), + Some("log" | "status") + ) { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn make_jobs( + root: &Path, + targets: Vec, + patterns: &[String], + clean: bool, +) -> io::Result> { + let scripts = discover_scripts(&root.join("test"))?; + let mut jobs = Vec::new(); + + for target in targets { + let target = Arc::new(target); + let result_dir = root.join("out/test/results").join(&target.label); + if clean { + clear_results(&result_dir)?; + } + for (name, script) in &scripts { + if matches_target(name, &target.machine) && matches_patterns(name, patterns) { + jobs.push(TestJob { + target: Arc::clone(&target), + script: script.clone(), + name: name.clone(), + log: result_dir.join(format!("{name}.log")), + status_file: result_dir.join(format!("{name}.status")), + }); + } + } + } + Ok(jobs) +} + +fn log_says_skipped(path: &Path) -> bool { + fs::read(path).is_ok_and(|bytes| { + bytes + .split(|&byte| byte == b'\n') + .any(|line| line.strip_suffix(b"\r").unwrap_or(line) == b"skipped") + }) +} + +fn run_job(root: &Path, job: &TestJob, timeout: Duration) -> TestResult { + let outcome = match File::create(&job.log) { + Err(err) => { + eprintln!("{}: cannot create {}: {err}", job.name, job.log.display()); + Outcome::Fail + } + Ok(log) => { + let stderr = match log.try_clone() { + Ok(file) => file, + Err(err) => { + eprintln!("{}: cannot clone {}: {err}", job.name, job.log.display()); + return TestResult { + target: Arc::clone(&job.target), + name: job.name.clone(), + log: job.log.clone(), + outcome: Outcome::Fail, + }; + } + }; + let mut command = Command::new("bash"); + command + .arg(&job.script) + .current_dir(root) + .env("MACHINE", &job.target.machine) + .stdout(Stdio::from(log)) + .stderr(Stdio::from(stderr)); + match &job.target.triple { + Some(triple) => { + command.env("TRIPLE", triple); + } + None => { + command.env_remove("TRIPLE"); + } + } + match &job.target.cpu { + Some(cpu) => { + command.env("CPU", cpu); + } + None => { + command.env_remove("CPU"); + } + } + + // Give every test its own process group, so a timeout also kills + // compiler or QEMU children rather than leaving them behind. + #[cfg(unix)] + command.process_group(0); + + match command.spawn() { + Err(err) => { + eprintln!("{}: cannot run {}: {err}", job.name, job.script.display()); + Outcome::Fail + } + Ok(mut child) => { + let start = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => { + break if !status.success() { + Outcome::Fail + } else if log_says_skipped(&job.log) { + Outcome::Skip + } else { + Outcome::Pass + }; + } + Ok(None) if start.elapsed() < timeout => { + thread::sleep(Duration::from_millis(20)) + } + Ok(None) => { + #[cfg(unix)] + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + #[cfg(not(unix))] + let _ = child.kill(); + let _ = child.wait(); + break Outcome::Timeout; + } + Err(err) => { + eprintln!("{}: cannot wait for test: {err}", job.name); + break Outcome::Fail; + } + } + } + } + } + } + }; + + if let Err(err) = fs::write(&job.status_file, format!("{}\n", outcome.status())) { + eprintln!( + "{}: cannot write {}: {err}", + job.name, + job.status_file.display() + ); + } + TestResult { + target: Arc::clone(&job.target), + name: job.name.clone(), + log: job.log.clone(), + outcome, + } +} + +fn run_jobs(root: &Path, jobs: Vec, options: &Options) -> Vec { + if jobs.is_empty() { + return Vec::new(); + } + let jobs = Arc::new(jobs); + let next = Arc::new(AtomicUsize::new(0)); + let (sender, receiver) = mpsc::channel(); + let workers = options.jobs.min(jobs.len()); + + thread::scope(|scope| { + for _ in 0..workers { + let jobs = Arc::clone(&jobs); + let next = Arc::clone(&next); + let sender = sender.clone(); + scope.spawn(move || loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(job) = jobs.get(index) else { + break; + }; + if sender.send(run_job(root, job, options.timeout)).is_err() { + break; + } + }); + } + drop(sender); + + let mut results = Vec::with_capacity(jobs.len()); + while let Ok(result) = receiver.recv() { + if matches!(result.outcome, Outcome::Fail | Outcome::Timeout) { + eprintln!( + "FAIL {}:{}{} ({})", + result.target.label, + result.name, + if result.outcome == Outcome::Timeout { + " [timeout]" + } else { + "" + }, + result.log.display() + ); + } + results.push(result); + if results.len().is_multiple_of(100) || results.len() == jobs.len() { + eprintln!("completed {}/{}", results.len(), jobs.len()); + } + } + results + }) +} + +fn print_inventory(jobs: &[TestJob], unavailable: &[&TargetSpec]) { + let mut counts = BTreeMap::new(); + for job in jobs { + *counts.entry(job.target.label.as_str()).or_insert(0usize) += 1; + } + for (target, count) in counts { + println!("{target}: tests={count}"); + } + println!("total: tests={}", jobs.len()); + if !unavailable.is_empty() { + let targets = unavailable + .iter() + .map(|target| target.machine) + .collect::>() + .join(", "); + println!("unavailable: {targets}"); + } +} + +fn print_summary(results: &[TestResult]) -> bool { + let mut by_target: BTreeMap<&str, Counts> = BTreeMap::new(); + for result in results { + by_target + .entry(&result.target.label) + .or_default() + .add(result.outcome); + } + + let mut total = Counts::default(); + for (target, counts) in &by_target { + println!( + "{target}: pass={} skip={} fail={}", + counts.pass, counts.skip, counts.fail + ); + total.merge(counts); + } + if by_target.len() > 1 { + println!( + "total: pass={} skip={} fail={}", + total.pass, total.skip, total.fail + ); + } else { + println!( + "pass={} skip={} fail={}", + total.pass, total.skip, total.fail + ); + } + total.fail == 0 +} + +fn main() { + let options = parse_options(); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let (targets, unavailable) = selected_targets(&options); + let jobs = make_jobs(root, targets, &options.patterns, !options.list).unwrap_or_else(|err| { + eprintln!("mold-test-runner: {err}"); + std::process::exit(1); + }); + + if options.list { + print_inventory(&jobs, &unavailable); + return; + } + if options.mode == Mode::All && !unavailable.is_empty() { + let targets = unavailable + .iter() + .map(|target| target.machine) + .collect::>() + .join(", "); + eprintln!("skipping targets without both compiler and QEMU: {targets}"); + } + + let results = run_jobs(root, jobs, &options); + if !print_summary(&results) { + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonicalizes_machine_names() { + assert_eq!(canonical_machine("amd64"), "x86_64"); + assert_eq!(canonical_machine("i386"), "i686"); + assert_eq!(canonical_machine("armv7l"), "arm"); + assert_eq!(canonical_machine("aarch64"), "aarch64"); + assert_eq!(canonical_machine("powerpc64le"), "ppc64le"); + } + + #[test] + fn selects_generic_and_target_tests() { + assert!(matches_target("gc-sections", "aarch64")); + assert!(matches_target("arch-aarch64-reloc", "aarch64")); + assert!(!matches_target("arch-x86_64-reloc", "aarch64")); + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 000000000..5e7183d05 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,191 @@ +option(MOLD_ENABLE_QEMU_TESTS "Enable tests on non-native targets" ON) +set(MACHINE ${CMAKE_HOST_SYSTEM_PROCESSOR}) + +if(EXISTS "/proc/cpuinfo") + file(READ "/proc/cpuinfo" CPUINFO) +endif() + +# CMAKE_HOST_SYSTEM_PROCESSOR returns "aarch64" on ARM32 userland +# with ARM64 kernel. Here, we run `cc -dumpmachine` to determine the +# compiler's default target. +execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpmachine + RESULT_VARIABLE EXIT_CODE + OUTPUT_VARIABLE ARCH + ERROR_QUIET) +if(NOT EXIT_CODE AND ARCH MATCHES "([^-]+).*") + set(MACHINE ${CMAKE_MATCH_1}) +endif() + +if(${MACHINE} MATCHES "amd64") + set(MACHINE x86_64) +elseif(${MACHINE} MATCHES "i.86") + set(MACHINE i686) +elseif(${MACHINE} MATCHES "arm.*") + set(MACHINE arm) +elseif(${MACHINE} STREQUAL "powerpc64") + set(MACHINE ppc64) +elseif(${MACHINE} STREQUAL "powerpc64le") + set(MACHINE ppc64le) +endif() + +if(MOLD_ENABLE_QEMU_TESTS) + list(APPEND QEMU_ARCHS + x86_64 i386 arm armeb aarch64 aarch64_be ppc ppc64 ppc64le sparc64 + sh4 sh4eb s390x riscv64 riscv32 m68k loongarch64) + + LIST(APPEND TRIPLES + x86_64-linux-gnu + i686-linux-gnu + aarch64-linux-gnu + aarch64_be-linux-gnu + arm-linux-gnueabihf + armeb-linux-gnueabihf + riscv64-linux-gnu + powerpc-linux-gnu + powerpc64-linux-gnu + powerpc64le-linux-gnu + sparc64-linux-gnu + s390x-linux-gnu + sh4-linux-gnu + sh4aeb-linux-gnu + riscv32-linux-gnu + m68k-linux-gnu + loongarch64-linux-gnu) + + foreach(ARCH IN LISTS QEMU_ARCHS) + find_program(HAS_qemu-${ARCH} qemu-${ARCH}) + endforeach() + + foreach(TRIPLE IN LISTS TRIPLES) + find_program(HAS_${TRIPLE}-gcc ${TRIPLE}-gcc) + endforeach() +endif() + +function(add_target ARCH TRIPLE) + set(CPU ${ARGV2}) + + if(${ARCH} STREQUAL ${MACHINE}) + set(IS_NATIVE 1) + endif() + + file(GLOB ALL_TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS + "*.sh") + + list(FILTER ALL_TESTS EXCLUDE REGEX "^arch-") + + file(GLOB TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS + "arch-${ARCH}-*.sh") + + list(APPEND TESTS ${ALL_TESTS}) + + foreach(TEST IN LISTS TESTS) + if(CPU) + string(REGEX REPLACE "\\.sh$" "" TESTNAME "${ARCH}_${CPU}-${TEST}") + else() + string(REGEX REPLACE "\\.sh$" "" TESTNAME "${ARCH}-${TEST}") + endif() + + add_test(NAME ${TESTNAME} + COMMAND ${CMAKE_CURRENT_LIST_DIR}/${TEST} + WORKING_DIRECTORY ${mold_BINARY_DIR}) + + set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT + "MACHINE=${MACHINE};CPU=${CPU}") + + if(IS_NATIVE) + set_property(TEST ${TESTNAME} APPEND PROPERTY SKIP_REGULAR_EXPRESSION + "skipped") + else() + set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT + "TRIPLE=${TRIPLE}") + endif() + endforeach() +endfunction() + +if(${MACHINE} STREQUAL "x86_64" OR (HAS_qemu-x86_64 AND HAS_x86_64-linux-gnu-gcc)) + add_target(x86_64 x86_64-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "i686" OR (HAS_qemu-i386 AND HAS_i686-linux-gnu-gcc)) + add_target(i686 i686-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "aarch64" OR (HAS_qemu-aarch64 AND HAS_aarch64-linux-gnu-gcc)) + add_target(aarch64 aarch64-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "aarch64_be" OR (HAS_qemu-aarch64_be AND HAS_aarch64_be-linux-gnu-gcc)) + add_target(aarch64_be aarch64_be-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "arm" OR (HAS_qemu-arm AND HAS_arm-linux-gnueabihf-gcc)) + add_target(arm arm-linux-gnueabihf) +endif() + +if(${MACHINE} STREQUAL "armeb" OR (HAS_qemu-armeb AND HAS_armeb-linux-gnueabihf-gcc)) + add_target(armeb armeb-linux-gnueabihf) +endif() + +if(${MACHINE} STREQUAL "riscv64" OR (HAS_qemu-riscv64 AND HAS_riscv64-linux-gnu-gcc)) + add_target(riscv64 riscv64-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "riscv32" OR (HAS_qemu-riscv32 AND HAS_riscv32-linux-gnu-gcc)) + add_target(riscv32 riscv32-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "ppc" OR (HAS_qemu-ppc AND HAS_powerpc-linux-gnu-gcc)) + add_target(ppc powerpc-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "ppc64" OR (HAS_qemu-ppc64 AND HAS_powerpc64-linux-gnu-gcc)) + add_target(ppc64 powerpc64-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "ppc64le" OR (HAS_qemu-ppc64le AND HAS_powerpc64le-linux-gnu-gcc)) + add_target(ppc64le powerpc64le-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "ppc64le" AND "${CPUINFO}" MATCHES "POWER10") + add_target(ppc64le powerpc64le-linux-gnu power10) +elseif(HAS_qemu-ppc64le AND HAS_powerpc64le-linux-gnu-gcc) + file(WRITE "${CMAKE_BINARY_DIR}/empty.c" "") + + execute_process( + COMMAND powerpc64le-linux-gnu-gcc -mcpu=power10 -E + "${CMAKE_BINARY_DIR}/empty.c" + RESULT_VARIABLE GCC_EXIT_CODE + OUTPUT_QUIET ERROR_QUIET) + + execute_process( + COMMAND qemu-ppc64le -cpu help + OUTPUT_VARIABLE QEMU_CPUS) + + if("${GCC_EXIT_CODE}" EQUAL "0" AND "${QEMU_CPUS}" MATCHES power10_v2.0) + add_target(ppc64le powerpc64le-linux-gnu power10) + endif() +endif() + +if(${MACHINE} STREQUAL "sparc64" OR (HAS_qemu-sparc64 AND HAS_sparc64-linux-gnu-gcc)) + add_target(sparc64 sparc64-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "s390x" OR (HAS_qemu-s390x AND HAS_s390x-linux-gnu-gcc)) + add_target(s390x s390x-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "sh4" OR (HAS_qemu-sh4 AND HAS_sh4-linux-gnu-gcc)) + add_target(sh4 sh4-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "sh4aeb" OR (HAS_qemu-sh4eb AND HAS_sh4aeb-linux-gnu-gcc)) + add_target(sh4aeb sh4aeb-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "m68k" OR (HAS_qemu-m68k AND HAS_m68k-linux-gnu-gcc)) + add_target(m68k m68k-linux-gnu) +endif() + +if(${MACHINE} STREQUAL "loongarch64" OR (HAS_qemu-loongarch64 AND HAS_loongarch64-linux-gnu-gcc)) + add_target(loongarch64 loongarch64-linux-gnu) +endif() diff --git a/test/abs-error.sh b/test/abs-error.sh new file mode 100755 index 000000000..79e8e6cac --- /dev/null +++ b/test/abs-error.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = aarch64 ] && skip +[ $MACHINE = ppc64 ] && skip +[ $MACHINE = ppc64le ] && skip +[ $MACHINE = s390x ] && skip +[[ $MACHINE = loongarch* ]] && skip + +cat < +extern char foo; +int main() { printf("foo=%p\n", &foo); } +EOF + +not $CC -B. -o $t/exe -pie $t/a.o $t/b.o -Wl,-z,text |& + grep 'recompile with -fPIC' diff --git a/test/absolute-symbols.sh b/test/absolute-symbols.sh new file mode 100755 index 000000000..9ac6a3cb9 --- /dev/null +++ b/test/absolute-symbols.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# This test crashes only on qemu-sparc64 running on GitHub Actions, +# even though it works on a local x86-64 machine and on an actual +# SPARC machine. +[ $MACHINE = sparc64 ] && skip + +cat < +#include +#include +#include + +void handler(int signum, siginfo_t *info, void *ptr) { + printf("ip=%p\n", info->si_addr); + exit(0); +} + +extern volatile int foo; + +int main() { + struct sigaction act; + act.sa_flags = SA_SIGINFO | SA_RESETHAND; + act.sa_sigaction = handler; + sigemptyset(&act.sa_mask); + sigaction(SIGSEGV, &act, 0); + foo = 5; +} +EOF + +$CC -B. -o $t/exe -no-pie $t/a.o $t/b.o +$QEMU $t/exe | grep '^ip=0xa0000.$' diff --git a/test/allow-multiple-definition.sh b/test/allow-multiple-definition.sh new file mode 100755 index 000000000..ccff25511 --- /dev/null +++ b/test/allow-multiple-definition.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc - +echo 'int main() { return 1; }' | $CC -c -o $t/b.o -xc - + +not $CC -B. -o $t/exe $t/a.o $t/b.o 2> /dev/null +$CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-allow-multiple-definition +$CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-z,muldefs diff --git a/test/ar-alignment.sh b/test/ar-alignment.sh new file mode 100755 index 000000000..f63144484 --- /dev/null +++ b/test/ar-alignment.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <> $t/a.o + +cat < + +int two(); +int three(); + +int main() { + printf("%d\n", two() + three()); +} +EOF + +rm -f $t/d.a +ar rcs $t/d.a $t/a.o $t/b.o + +$CC -B. -o $t/exe $t/c.o $t/d.a diff --git a/test/arch-aarch64-emit-relocs-relax.sh b/test/arch-aarch64-emit-relocs-relax.sh new file mode 100755 index 000000000..8c145523e --- /dev/null +++ b/test/arch-aarch64-emit-relocs-relax.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# When mold relaxes an instruction sequence, the relocations attached to the +# original instructions no longer match the relaxed code. With --emit-relocs, +# mold must rewrite the emitted relocations so that their types describe the +# relaxed instructions. + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.text +.globl _start +_start: + # GOT-load relaxation: ADRP+LDR => ADRP+ADD + # R_AARCH64_ADR_GOT_PAGE => R_AARCH64_ADR_PREL_PG_HI21 + # R_AARCH64_LD64_GOT_LO12_NC => R_AARCH64_ADD_ABS_LO12_NC + adrp x0, :got:gvar + ldr x0, [x0, #:got_lo12:gvar] + + # ADRP+ADD => NOP+ADR (gvar is within +-1 MiB) + # R_AARCH64_ADR_PREL_PG_HI21 => R_AARCH64_NONE + # R_AARCH64_ADD_ABS_LO12_NC => R_AARCH64_ADR_PREL_LO21 + adrp x1, gvar + add x1, x1, #:lo12:gvar + + # Both sequences must materialize the same address. + cmp x0, x1 + bne 1f + + # ... and it must point at gvar, whose value is 42. + ldr w2, [x0] + cmp w2, #42 + bne 1f + + mov x0, #0 + b 2f +1: + mov x0, #1 +2: + mov x8, #93 // __NR_exit + svc #0 + +.data +.hidden gvar +gvar: + .word 42 +EOF + +$CC -B. -o $t/exe $t/a.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax +$QEMU $t/exe + +$OBJDUMP -dr $t/exe > $t/exe.objdump + +# Relaxation took place. +grep -Fw nop $t/exe.objdump +grep -Fw adr $t/exe.objdump +grep -Fw add $t/exe.objdump + +# The relocations were rewritten to match the relaxed instructions. +grep -Fw R_AARCH64_ADR_PREL_PG_HI21 $t/exe.objdump +grep -Fw R_AARCH64_ADD_ABS_LO12_NC $t/exe.objdump +grep -Fw R_AARCH64_ADR_PREL_LO21 $t/exe.objdump + +# The stale pre-relaxation relocations are gone. +not grep -Fw R_AARCH64_ADR_GOT_PAGE $t/exe.objdump +not grep -Fw R_AARCH64_LD64_GOT_LO12_NC $t/exe.objdump + +# Each rewritten relocation must sit on the relaxed instruction it describes. +awk ' + $3 == "adrp" || $3 == "add" || $3 == "adr" || $3 == "nop" { insn[$1] = $3 } + $2 == "R_AARCH64_ADR_PREL_PG_HI21" { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } } + $2 == "R_AARCH64_ADD_ABS_LO12_NC" { if (insn[$1] != "add") { bad=1; print "bad offset: " $0 } } + $2 == "R_AARCH64_ADR_PREL_LO21" { if (insn[$1] != "adr") { bad=1; print "bad offset: " $0 } } + END { exit bad+0 } +' $t/exe.objdump diff --git a/test/arch-aarch64-emit-relocs-tlsdesc-relax.sh b/test/arch-aarch64-emit-relocs-tlsdesc-relax.sh new file mode 100755 index 000000000..b53713eee --- /dev/null +++ b/test/arch-aarch64-emit-relocs-tlsdesc-relax.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec +# (LE) or initial-exec (IE) sequence. The four-instruction sequence is rewritten +# in place: the ADRP and LDR become NOPs, and the ADD and BLR become the two +# instructions that materialize the address. With --emit-relocs, the emitted +# relocations must be rewritten to match whichever relaxation was applied. + +cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC +_Thread_local char foo[4] = "foo"; +_Thread_local char bar[4] = "bar"; +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC -mtls-dialect=desc -O2 +extern _Thread_local char foo[4]; +extern _Thread_local char bar[4]; +char *get_foo() { return foo; } +char *get_bar() { return bar; } +EOF + +cat < +char *get_foo(); +char *get_bar(); +int main() { printf("%s %s\n", get_foo(), get_bar()); } +EOF + +# +# Local TLS in an executable: TLSDESC => LE +# +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--emit-relocs,--relax +$QEMU $t/exe1 | grep -F 'foo bar' + +$OBJDUMP -dr $t/exe1 > $t/exe1.objdump + +# The ADD/BLR became MOVZ/MOVK that build the TP-relative offset. +grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/exe1.objdump +grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G0_NC $t/exe1.objdump + +# No TLSDESC relocation survives. +not grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/exe1.objdump +not grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/exe1.objdump +not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe1.objdump +not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe1.objdump + +# The G1 relocation sits on a movz and the G0_NC on a movk, not on the leading +# adrp/ldr (which are now nops). +awk ' + $3 == "movz" || $3 == "movk" || $3 == "nop" { insn[$1] = $3 } + $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G1" { if (insn[$1] != "movz") { bad=1; print "bad offset: " $0 } } + $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G0_NC" { if (insn[$1] != "movk") { bad=1; print "bad offset: " $0 } } + END { exit bad+0 } +' $t/exe1.objdump + +# +# TLS defined in a shared library: TLSDESC => IE +# +$CC -B. -shared -o $t/d.so $t/a.o -Wl,--relax +$CC -B. -o $t/exe2 $t/b.o $t/c.o $t/d.so -Wl,--emit-relocs,--relax -Wl,-rpath=$t +$QEMU $t/exe2 | grep -F 'foo bar' + +$OBJDUMP -dr $t/exe2 > $t/exe2.objdump + +# The ADD/BLR became ADRP/LDR that load the address from the GOT. +grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/exe2.objdump +grep -Fw R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC $t/exe2.objdump +not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe2.objdump +not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe2.objdump + +awk ' + $3 == "adrp" || $3 == "ldr" || $3 == "nop" { insn[$1] = $3 } + $2 == "R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21" { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } } + $2 == "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC" { if (insn[$1] != "ldr") { bad=1; print "bad offset: " $0 } } + END { exit bad+0 } +' $t/exe2.objdump + +# +# Relocatable output: no relaxation happens and TLSDESC is not downgraded, so +# every relocation must be passed through unchanged rather than rewritten as if +# it had been relaxed to LE/IE. +# +$CC -B. -o $t/reloc.o $t/b.o -r +$OBJDUMP -r $t/reloc.o > $t/reloc.objdump +grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/reloc.objdump +grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/reloc.objdump +grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/reloc.objdump +grep -Fw R_AARCH64_TLSDESC_CALL $t/reloc.objdump +not grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/reloc.objdump +not grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/reloc.objdump diff --git a/test/arch-aarch64-long-thunk.sh b/test/arch-aarch64-long-thunk.sh new file mode 100755 index 000000000..0caad2835 --- /dev/null +++ b/test/arch-aarch64-long-thunk.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/a.c +#include + +void fn3(); +void fn4(); + +__attribute__((section(SEC1))) void fn1() { printf(" fn1"); fn3(); } +__attribute__((section(SEC1))) void fn2() { printf(" fn2"); fn4(); } + +int main() { + printf(" main"); + fn1(); + printf("\n"); +} +EOF + +cat < $t/b.c +#include + +void fn1(); +void fn2(); + +__attribute__((section(SEC2))) void fn3() { printf(" fn3"); fn2(); } +__attribute__((section(SEC2))) void fn4() { printf(" fn4"); } +EOF + +# .high is placed billions of bytes away from the rest of the program. +# Functions there cannot have unwind tables because an FDE refers to its +# function with a 32-bit PC-relative relocation. We put fn1 and fn2 in +# either section to test calls in both directions. +flags="-fno-asynchronous-unwind-tables -fno-unwind-tables" +starts="-Wl,--section-start=.low=0x10000000,--section-start=.high=0x100000000" + +$CC -c -o $t/c.o $t/a.c $flags -DSEC1='".low"' -DSEC2='".high"' +$CC -c -o $t/d.o $t/b.c $flags -DSEC1='".low"' -DSEC2='".high"' +$CC -B. -o $t/exe1 $t/c.o $t/d.o $starts +$QEMU $t/exe1 | grep 'main fn1 fn3 fn2 fn4' + +$CC -c -o $t/e.o $t/a.c $flags -DSEC1='".high"' -DSEC2='".low"' +$CC -c -o $t/f.o $t/b.c $flags -DSEC1='".high"' -DSEC2='".low"' +$CC -B. -o $t/exe2 $t/e.o $t/f.o $starts +$QEMU $t/exe2 | grep 'main fn1 fn3 fn2 fn4' + +flags="$flags -fno-PIC -mcmodel=large" +starts="-Wl,--section-start=.low=0x10000000,--section-start=.high=0x400000000" + +$CC -c -o $t/g.o $t/a.c $flags -DSEC1='".low"' -DSEC2='".high"' +$CC -c -o $t/h.o $t/b.c $flags -DSEC1='".low"' -DSEC2='".high"' +$CC -B. -o $t/exe3 $t/g.o $t/h.o -pie $starts +$QEMU $t/exe3 | grep 'main fn1 fn3 fn2 fn4' + +$CC -c -o $t/i.o $t/a.c $flags -DSEC1='".high"' -DSEC2='".low"' +$CC -c -o $t/j.o $t/b.c $flags -DSEC1='".high"' -DSEC2='".low"' +$CC -B. -o $t/exe4 $t/i.o $t/j.o -pie $starts +$QEMU $t/exe4 | grep 'main fn1 fn3 fn2 fn4' diff --git a/test/arch-aarch64-pack-dyn-relocs.sh b/test/arch-aarch64-pack-dyn-relocs.sh new file mode 100755 index 000000000..a7a8a34e1 --- /dev/null +++ b/test/arch-aarch64-pack-dyn-relocs.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = aarch64 ] || skip +command -v timeout >/dev/null || skip + +cat < /dev/null || skip +.section .custom_rodata_a, "a", %progbits +.hidden A +.global A +A: + .space 8 + +.section .custom_rodata_b_huge, "a", %progbits +.balign 16384 +.hidden B +.global B +B: + .space 8 + +.section .data, "aw", %progbits +.rept 7439 + .quad A +.endr + +.global P_1 +P_1: + .quad A + +.global P_2 +P_2: + .quad B +EOF + +timeout 5s ./mold -m aarch64elf -shared -o $t/b.so $t/a.o \ + --pack-dyn-relocs=android -z max-page-size=16384 diff --git a/test/arch-aarch64-range-extension-thunk-disassembly.sh b/test/arch-aarch64-range-extension-thunk-disassembly.sh new file mode 100755 index 000000000..7cbb78a2d --- /dev/null +++ b/test/arch-aarch64-range-extension-thunk-disassembly.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void fn1(); +void fn2(); + +__attribute__((section(".low"))) void fn1() { fn2(); } +__attribute__((section(".high"))) void fn2() { fn1(); } + +int main() { + fn1(); +} +EOF + +$CC -B. -o $t/exe $t/a.o \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 + +$OBJDUMP -dr $t/exe | grep -E ':' diff --git a/test/arch-aarch64-variant-pcs.sh b/test/arch-aarch64-variant-pcs.sh new file mode 100755 index 000000000..077faa2e7 --- /dev/null +++ b/test/arch-aarch64-variant-pcs.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < /dev/null || skip +.global foo +.type foo, %function +.variant_pcs foo +foo: + ret +EOF + +$CC -B. -shared -o $t/b.so $t/a.o +readelf -W --dyn-syms $t/b.so | grep foo | grep -F '[VARIANT_PCS]' + +cat < /dev/null || skip +#include +extern char foo; +int main() { printf("foo=%p\n", &foo); } +EOF + +$CC -o $t/exe -pie $t/a.o $t/b.o >& /dev/null && skip + +not $CC -B. -o $t/exe -pie $t/a.o $t/b.o |& + grep 'recompile with -fPIC' diff --git a/test/arch-arm-exidx-sentinel.sh b/test/arch-arm-exidx-sentinel.sh new file mode 100755 index 000000000..bbc2ed6a3 --- /dev/null +++ b/test/arch-arm-exidx-sentinel.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +extern char _etext[]; +int main() { + printf("%p\n", _etext); +} +EOF + +$CXX -B. -o $t/exe $t/a.o -no-pie +readelf --unwind $t/exe | grep "$($QEMU $t/exe) .*cantunwind" diff --git a/test/arch-arm-range-extension-thunk-disassembly.sh b/test/arch-arm-range-extension-thunk-disassembly.sh new file mode 100755 index 000000000..f53114d6b --- /dev/null +++ b/test/arch-arm-range-extension-thunk-disassembly.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void fn1(); +void fn2(); + +__attribute__((section(".low"))) void fn1() { fn2(); } +__attribute__((section(".high"))) void fn2() { fn1(); } + +int main() { + fn1(); +} +EOF + +$CC -B. -o $t/exe $t/a.o \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 + +$OBJDUMP -dr $t/exe | grep -E -A7 ':' > $t/log + +grep -E 'bx\s+pc' $t/log +grep -E 'add\s+pc, ip, pc' $t/log diff --git a/test/arch-arm-range-extension-thunk.sh b/test/arch-arm-range-extension-thunk.sh new file mode 100755 index 000000000..a8aef1833 --- /dev/null +++ b/test/arch-arm-range-extension-thunk.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -mthumb || skip + +cat < $t/a.c +#include + +void fn3(); +void fn4(); + +__attribute__((section(".low"))) void fn1() { printf(" fn1"); fn3(); } +__attribute__((section(".low"))) void fn2() { printf(" fn2"); fn4(); } + +int main() { + printf(" main"); + fn1(); + printf("\n"); +} +EOF + +cat < $t/b.c +#include + +void fn1(); +void fn2(); + +__attribute__((section(".high"))) void fn3() { printf(" fn3"); fn2(); } +__attribute__((section(".high"))) void fn4() { printf(" fn4"); } +EOF + +$CC -c -o $t/c.o $t/a.c -O0 -mthumb +$CC -c -o $t/d.o $t/b.c -O0 -marm + +$CC -B. -o $t/exe $t/c.o $t/d.o \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 +$QEMU $t/exe | grep 'main fn1 fn3 fn2 fn4' + +$CC -c -o $t/e.o $t/a.c -O2 -mthumb +$CC -c -o $t/f.o $t/b.c -O2 -marm + +$CC -B. -o $t/exe $t/e.o $t/f.o \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 +$QEMU $t/exe | grep 'main fn1 fn3 fn2 fn4' diff --git a/test/arch-arm-target1.sh b/test/arch-arm-target1.sh new file mode 100755 index 000000000..0ae82e8b7 --- /dev/null +++ b/test/arch-arm-target1.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +extern char *foo; +char bar[] = "Hello world"; +int main() { printf("%s\n", foo); } +EOF + +$CC -B. -o $t/exe -pie $t/a.o $t/b.o +$QEMU $t/exe | grep 'Hello world' diff --git a/test/arch-arm-thm-jump19.sh b/test/arch-arm-thm-jump19.sh new file mode 100755 index 000000000..20d965161 --- /dev/null +++ b/test/arch-arm-thm-jump19.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <' diff --git a/test/arch-arm-thm-jump8.sh b/test/arch-arm-thm-jump8.sh new file mode 100755 index 000000000..d215be839 --- /dev/null +++ b/test/arch-arm-thm-jump8.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <' diff --git a/test/arch-arm-thumb-interwork.sh b/test/arch-arm-thumb-interwork.sh new file mode 100755 index 000000000..0c3bb0120 --- /dev/null +++ b/test/arch-arm-thumb-interwork.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -mthumb || skip + +cat < +int bar(); +int foo() { + printf(" foo"); + bar(); +} +EOF + +cat < +int bar() { + printf(" bar\n"); +} + +int foo(); + +int main() { + printf("main"); + foo(); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 'main foo bar' diff --git a/test/arch-arm-thumb-notype.sh b/test/arch-arm-thumb-notype.sh new file mode 100755 index 000000000..d938f8166 --- /dev/null +++ b/test/arch-arm-thumb-notype.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -mthumb || skip + +# Only function symbols indicate in their value whether they are Thumb or +# ARM code. A branch to a symbol of another type, such as a plain label in +# hand-written assembly, must not switch the instruction set. + +cat < +void foo(); +__attribute__((noinline)) void bar() { foo(); } +int main() { + foo(); + bar(); + printf("OK\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep OK + +$OBJDUMP -d $t/exe > $t/log +grep -E '\bbl\b.*' $t/log +grep -E '\bb\.w\b.*' $t/log diff --git a/test/arch-arm-tlsdesc.sh b/test/arch-arm-tlsdesc.sh new file mode 100755 index 000000000..0b17a9bb2 --- /dev/null +++ b/test/arch-arm-tlsdesc.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +is_musl && skip +test_cflags -mthumb || skip + +cat < $t/a.c +extern _Thread_local int foo; + +__attribute__((section(".low"))) +int get_foo() { + int y = foo; + return y; +} + +static _Thread_local int bar = 5; + +__attribute__((section(".high"))) +int get_bar() { + return bar; +} +EOF + +cat < $t/b.c +#include + +_Thread_local int foo; + +int get_foo(); +int get_bar(); + +int main() { + foo = 42; + printf("%d %d\n", get_foo(), get_bar()); + return 0; +} +EOF + +$GCC -fPIC -mtls-dialect=gnu2 -c -o $t/c.o $t/a.c -marm +$GCC -fPIC -mtls-dialect=gnu2 -c -o $t/d.o $t/b.c -marm + +$CC -B. -o $t/exe1 $t/c.o $t/d.o +$QEMU $t/exe1 | grep '42 5' + +$CC -B. -o $t/exe2 $t/c.o $t/d.o -Wl,-no-relax +$QEMU $t/exe2 | grep '42 5' + +$CC -B. -o $t/exe3 $t/c.o $t/d.o -Wl,-no-relax \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 +$QEMU $t/exe3 | grep '42 5' + +$GCC -B. -shared -o $t/c.so $t/c.o -Wl,-z,nodlopen +$CC -B. -o $t/exe4 $t/c.so $t/d.o +$QEMU $t/exe4 | grep '42 5' + + +$GCC -fPIC -mtls-dialect=gnu2 -c -o $t/e.o $t/a.c -mthumb +$GCC -fPIC -mtls-dialect=gnu2 -c -o $t/f.o $t/b.c -mthumb + +$CC -B. -o $t/exe5 $t/e.o $t/f.o +$QEMU $t/exe5 | grep '42 5' + +$CC -B. -o $t/exe6 $t/e.o $t/f.o -Wl,-no-relax +$QEMU $t/exe6 | grep '42 5' + +$CC -B. -o $t/exe7 $t/e.o $t/f.o -Wl,-no-relax \ + -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000 +$QEMU $t/exe7 | grep '42 5' + +$GCC -B. -shared -o $t/e.so $t/e.o -Wl,-z,nodlopen +$CC -B. -o $t/exe8 $t/e.so $t/f.o +$QEMU $t/exe8 | grep '42 5' diff --git a/test/arch-armeb-be32.sh b/test/arch-armeb-be32.sh new file mode 100755 index 000000000..00318ff6c --- /dev/null +++ b/test/arch-armeb-be32.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +extern char target; +extern unsigned char x8, xpc8; +extern unsigned short x16, xpc16; + +int main() { + printf("%d %d %d %d\n", x8, x16, + (signed char)xpc8 == (char *)&target - (char *)&xpc8 - 1, + (short)xpc16 == (char *)&target - (char *)&xpc16 - 512); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/a.o $t/b.o $t/c.o +$QEMU $t/exe | grep '^99 488 1 1$' + +$OBJDUMP -s -j .nonalloc $t/exe | grep -w 63e801 diff --git a/test/arch-i686-emit-relocs-relax.sh b/test/arch-i686-emit-relocs-relax.sh new file mode 100755 index 000000000..1112482a8 --- /dev/null +++ b/test/arch-i686-emit-relocs-relax.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# mold relaxes a GOT load of a non-preemptible symbol from +# mov foo@GOT(%ebx), %reg => lea foo@GOTOFF(%ebx), %reg +# With --emit-relocs the R_386_GOT32X relocation must be rewritten to +# R_386_GOTOFF to match the lea (GNU ld does the same). + +cat <<'EOF' | $CC -o $t/a.o -c -xc - +int foo = 42; +EOF + +cat <<'EOF' | $CC -fPIC -O2 -o $t/b.o -c -xc - +extern int foo; +int get(void) { return foo; } +EOF + +cat <<'EOF' | $CC -o $t/c.o -c -xc - +extern int get(void); +int main(void) { return get() != 42; } // exits 0 iff foo resolved to 42 +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o -no-pie -Wl,--emit-relocs +$QEMU $t/exe + +$OBJDUMP -dr $t/exe > $t/exe.objdump + +# The GOT load of foo was relaxed and its relocation rewritten to GOTOFF. The +# compiler only emits GOT32X for the extern access, so a GOTOFF naming foo can +# only come from the relaxation. +grep -E $'R_386_GOTOFF[ \t]+foo' $t/exe.objdump +not grep -E $'R_386_GOT32X[ \t]+foo' $t/exe.objdump diff --git a/test/arch-i686-tls-module-base.sh b/test/arch-i686-tls-module-base.sh new file mode 100755 index 000000000..16e4fac61 --- /dev/null +++ b/test/arch-i686-tls-module-base.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -fPIC -o $t/a.o -c -xassembler - +.globl get_foo +.type get_foo, @function +get_foo: + push %ebx + call __x86.get_pc_thunk.bx +1: + addl $_GLOBAL_OFFSET_TABLE_ - 1b, %ebx + lea _TLS_MODULE_BASE_@TLSDESC(%ebx), %eax + call *_TLS_MODULE_BASE_@TLSCALL(%eax) + lea foo@dtpoff(%eax), %eax + mov %gs:(%eax), %eax + pop %ebx + ret +.section .tdata, "awT", @progbits +foo: +.long 20 +.section .note.GNU-stack, "", @progbits +EOF + +cat < + +int get_foo(); +extern _Thread_local int bar; + +int main() { + printf("%d %d\n", get_foo(), bar); +} +EOF + +$CC -o $t/exe1 $t/a.o $t/b.o $t/c.o -pie +$QEMU $t/exe1 | grep '^20 3$' + +$CC -o $t/exe2 $t/a.o $t/b.o $t/c.o -Wl,-no-relax -pie +$QEMU $t/exe2 | grep '^20 3$' + +$CC -o $t/d.so $t/a.o -shared +$CC -o $t/exe3 $t/b.o $t/c.o $t/d.so -pie +$QEMU $t/exe3 | grep '^20 3$' diff --git a/test/arch-i686-tlsdesc.sh b/test/arch-i686-tlsdesc.sh new file mode 100755 index 000000000..3cf2f1ce3 --- /dev/null +++ b/test/arch-i686-tlsdesc.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_tlsdesc || skip + +cat <<'EOF' | $GCC -c -o $t/a.o -xassembler - +.globl get_foo +.type get_foo, @function +get_foo: + pushl %ebx + call __x86.get_pc_thunk.bx + addl $_GLOBAL_OFFSET_TABLE_, %ebx + subl $8, %esp + leal foo@TLSDESC(%ebx), %ebx + movl %ebx, %eax + call *foo@TLSCALL(%eax) + movl %gs:(%eax), %eax + addl $8, %esp + popl %ebx + ret +EOF + +cat < + +_Thread_local int foo; + +int get_foo(); + +int main() { + foo = 42; + printf("%d\n", get_foo()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o +$QEMU $t/exe1 | grep 42 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-no-relax +$QEMU $t/exe2 | grep 42 + +$CC -B. -shared -o $t/c.so $t/a.o +$CC -B. -o $t/exe3 $t/b.o $t/c.so +$QEMU $t/exe3 | grep 42 + +$CC -B. -shared -o $t/c.so $t/a.o -Wl,-no-relax +$CC -B. -o $t/exe4 $t/b.o $t/c.so -Wl,-no-relax +$QEMU $t/exe4 | grep 42 diff --git a/test/arch-loongarch64-emit-relocs-relax.sh b/test/arch-loongarch64-emit-relocs-relax.sh new file mode 100755 index 000000000..ca3ce1f91 --- /dev/null +++ b/test/arch-loongarch64-emit-relocs-relax.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# When mold relaxes an instruction sequence, the relocations attached to the +# original instructions no longer match the relaxed code. With --emit-relocs, +# mold must rewrite them so that the emitted relocations describe the relaxed +# instructions, both in type and in offset. + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.text +.globl _start +_start: + # pcaddu18i + jirl => bl (R_LARCH_CALL36 => R_LARCH_B26) + .reloc ., R_LARCH_RELAX + pcaddu18i $ra, %call36(get_zero) + jirl $ra, $ra, 0 + + # pcalau12i + addi.d => pcaddi (R_LARCH_PCALA_* => R_LARCH_PCREL20_S2) + .reloc ., R_LARCH_RELAX + pcalau12i $t0, %pc_hi20(v) + .reloc ., R_LARCH_RELAX + addi.d $t0, $t0, %pc_lo12(v) + ld.w $t0, $t0, 0 + or $a0, $a0, $t0 + + # pcalau12i + ld.d => pcaddi (R_LARCH_GOT_PC_* => R_LARCH_PCREL20_S2) + .reloc ., R_LARCH_RELAX + pcalau12i $t1, %got_pc_hi20(v) + .reloc ., R_LARCH_RELAX + ld.d $t1, $t1, %got_pc_lo12(v) + ld.w $t1, $t1, 0 + or $a0, $a0, $t1 + + li.w $a7, 93 # __NR_exit + syscall 0 + +get_zero: + li.w $a0, 0 + ret + +.section .rodata,"a" +.p2align 2 +.hidden v +v: + .word 0 +EOF + +$CC -B. -o $t/exe $t/a.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax +$QEMU $t/exe + +$OBJDUMP -dr $t/exe > $t/exe.objdump + +# Relaxation took place. +grep -Fw pcaddi $t/exe.objdump +grep -Fw bl $t/exe.objdump + +# The relocations were rewritten to match the relaxed instructions. The dead +# high-part relocations are turned into R_LARCH_NONE. +grep -Fw R_LARCH_PCREL20_S2 $t/exe.objdump +grep -Fw R_LARCH_B26 $t/exe.objdump +grep -Fw R_LARCH_NONE $t/exe.objdump + +# The stale pre-relaxation relocations are gone. +not grep -Fw R_LARCH_PCALA_HI20 $t/exe.objdump +not grep -Fw R_LARCH_PCALA_LO12 $t/exe.objdump +not grep -Fw R_LARCH_GOT_PC_HI20 $t/exe.objdump +not grep -Fw R_LARCH_GOT_PC_LO12 $t/exe.objdump +not grep -Fw R_LARCH_CALL36 $t/exe.objdump + +# Each rewritten relocation must sit at the address of the relaxed instruction +# it describes, not at its original pre-relaxation offset. This checks that the +# relocation offsets were adjusted: R_LARCH_PCREL20_S2 must land on a pcaddi +# and R_LARCH_B26 on a b/bl. +awk ' + $3 == "pcaddi" { insn[$1] = $3 } + $3 == "b" || $3 == "bl" { insn[$1] = $3 } + $2 == "R_LARCH_PCREL20_S2" { if (insn[$1] != "pcaddi") { bad=1; print "bad offset: " $0 } } + $2 == "R_LARCH_B26" { if (insn[$1] !~ /^bl?$/) { bad=1; print "bad offset: " $0 } } + END { exit bad+0 } +' $t/exe.objdump diff --git a/test/arch-loongarch64-emit-relocs-tlsdesc-relax.sh b/test/arch-loongarch64-emit-relocs-tlsdesc-relax.sh new file mode 100755 index 000000000..e6a997167 --- /dev/null +++ b/test/arch-loongarch64-emit-relocs-tlsdesc-relax.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec +# (LE) or initial-exec (IE) sequence, or keeps it as TLSDESC but folds its +# first two instructions into a single pcaddi. With --emit-relocs, the emitted +# relocations must be rewritten to match whichever relaxation was applied. + +cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC +_Thread_local char foo[4] = "foo"; +_Thread_local char padding[100000] = "pad"; +_Thread_local char bar[4] = "bar"; +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC -mtls-dialect=desc -O2 +extern _Thread_local char foo[4]; +extern _Thread_local char bar[4]; +char *get_foo() { return foo; } +char *get_bar() { return bar; } +EOF + +cat < +char *get_foo(); +char *get_bar(); +int main() { printf("%s %s\n", get_foo(), get_bar()); } +EOF + +# +# Local TLS in an executable: TLSDESC => LE +# +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--emit-relocs,--relax +{ $QEMU $t/exe1; true; } |& grep -q 'unexpected reloc type' && skip +$QEMU $t/exe1 | grep -F 'foo bar' + +$OBJDUMP -dr $t/exe1 > $t/exe1.objdump + +# foo has a small TP offset, so its sequence relaxes to a single instruction +# with only a low-part relocation. bar has a large offset, so it keeps both +# the high- and low-part relocations. +grep -E $'R_LARCH_TLS_LE_LO12[ \t]+foo' $t/exe1.objdump +not grep -E $'R_LARCH_TLS_LE_HI20[ \t]+foo' $t/exe1.objdump +grep -E $'R_LARCH_TLS_LE_HI20[ \t]+bar' $t/exe1.objdump +grep -E $'R_LARCH_TLS_LE_LO12[ \t]+bar' $t/exe1.objdump + +# No TLSDESC relocation survives. +not grep -Fw R_LARCH_TLS_DESC_PC_HI20 $t/exe1.objdump +not grep -Fw R_LARCH_TLS_DESC_PC_LO12 $t/exe1.objdump +not grep -Fw R_LARCH_TLS_DESC_LD $t/exe1.objdump +not grep -Fw R_LARCH_TLS_DESC_CALL $t/exe1.objdump + +# +# TLS defined in a shared library: TLSDESC => IE +# +$CC -B. -shared -o $t/d.so $t/a.o -Wl,--relax +$CC -B. -o $t/exe2 $t/b.o $t/c.o $t/d.so -Wl,--emit-relocs,--relax -Wl,-rpath=$t +$QEMU $t/exe2 | grep -F 'foo bar' + +$OBJDUMP -dr $t/exe2 > $t/exe2.objdump +grep -Fw R_LARCH_TLS_IE_PC_HI20 $t/exe2.objdump +grep -Fw R_LARCH_TLS_IE_PC_LO12 $t/exe2.objdump +not grep -Fw R_LARCH_TLS_DESC_LD $t/exe2.objdump +not grep -Fw R_LARCH_TLS_DESC_CALL $t/exe2.objdump + +# +# Shared library output: TLSDESC is kept, but pcalau12i+addi.d => pcaddi, so +# the low-part relocation becomes the pcaddi's TLS_DESC_PCREL20_S2 while the +# ld.d/jirl relocations are left unchanged. +# +$CC -B. -shared -o $t/e.so $t/a.o $t/b.o -Wl,--emit-relocs,--relax +$OBJDUMP -dr $t/e.so > $t/e.objdump +grep -Fw R_LARCH_TLS_DESC_PCREL20_S2 $t/e.objdump +grep -Fw R_LARCH_TLS_DESC_LD $t/e.objdump +grep -Fw R_LARCH_TLS_DESC_CALL $t/e.objdump +not grep -Fw R_LARCH_TLS_DESC_PC_HI20 $t/e.objdump +not grep -Fw R_LARCH_TLS_DESC_PC_LO12 $t/e.objdump + +# +# Relocatable output: no relaxation happens and TLSDESC is not downgraded, so +# every relocation must be passed through unchanged rather than rewritten as +# if it had been relaxed to LE/IE. +# +$CC -B. -o $t/reloc.o $t/b.o -r +$OBJDUMP -r $t/reloc.o > $t/reloc.objdump +grep -Fw R_LARCH_TLS_DESC_PC_HI20 $t/reloc.objdump +grep -Fw R_LARCH_TLS_DESC_PC_LO12 $t/reloc.objdump +grep -Fw R_LARCH_TLS_DESC_LD $t/reloc.objdump +grep -Fw R_LARCH_TLS_DESC_CALL $t/reloc.objdump +not grep -Fw R_LARCH_TLS_LE_HI20 $t/reloc.objdump +not grep -Fw R_LARCH_TLS_LE_LO12 $t/reloc.objdump diff --git a/test/arch-loongarch64-mcmodel-extreme.sh b/test/arch-loongarch64-mcmodel-extreme.sh new file mode 100755 index 000000000..ec474bffd --- /dev/null +++ b/test/arch-loongarch64-mcmodel-extreme.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +char msg[] = "Hello world\n"; +int main() { printf(msg); } +EOF + +$CC -B. -o $t/exe1 $t/a.o +$QEMU $t/exe1 | grep 'Hello world' diff --git a/test/arch-loongarch64-relax-call36.sh b/test/arch-loongarch64-relax-call36.sh new file mode 100755 index 000000000..368478a91 --- /dev/null +++ b/test/arch-loongarch64-relax-call36.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.globl foo, bar +.space 0x100000 +foo: + move $s0, $ra + .reloc ., R_LARCH_CALL36, print + .reloc ., R_LARCH_RELAX + pcaddu18i $t0, 0 + jirl $ra, $t0, 0 + move $ra, $s0 + ret +bar: + .reloc ., R_LARCH_CALL36, print + .reloc ., R_LARCH_RELAX + pcaddu18i $t0, 0 + jirl $zero, $t0, 0 +.space 0x100000 +EOF + +cat < + +void foo(); +void bar(); + +void print() { + printf("foo"); +} + +int main() { + foo(); + bar(); + printf("\n"); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o -Wl,--no-relax +$QEMU $t/exe1 | grep foofoo + +$OBJDUMP -d $t/exe1 > $t/exe1.objdump +grep -A2 ':' $t/exe1.objdump | grep -w pcaddu18i +grep -A2 ':' $t/exe1.objdump | grep -w pcaddu18i + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,--relax +$QEMU $t/exe2 | grep foofoo + +$OBJDUMP -d $t/exe2 > $t/exe2.objdump +grep -A2 ':' $t/exe2.objdump | grep -w bl +grep -A2 ':' $t/exe2.objdump | grep -w b diff --git a/test/arch-loongarch64-relax-got-load.sh b/test/arch-loongarch64-relax-got-load.sh new file mode 100755 index 000000000..dd5252ba1 --- /dev/null +++ b/test/arch-loongarch64-relax-got-load.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int get_foo(); +int main() { printf("%d\n", get_foo()); } +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -pie -Wl,--no-relax +$QEMU $t/exe1 | grep '^3$' +$OBJDUMP -d $t/exe1 | grep -A2 ':' | grep -Fw pcalau12i +$OBJDUMP -d $t/exe1 | grep -A2 ':' | grep -Fw ld.d + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o -pie -Wl,--relax +$QEMU $t/exe2 | grep '^3$' +$OBJDUMP -d $t/exe2 | grep -A1 ':' | grep -Fw pcaddi + +$CC -B. -o $t/exe3 $t/a.o $t/b.o $t/c.o -pie -Wl,--relax \ + -Wl,-Ttext=0x1000000,-Tdata=0x2000000 + +$QEMU $t/exe3 | grep '^3$' +$OBJDUMP -d $t/exe3 | grep -A2 ':' | grep -Fw pcalau12i +$OBJDUMP -d $t/exe3 | grep -A2 ':' | grep -Fw addi.d diff --git a/test/arch-loongarch64-relax-got-load2.sh b/test/arch-loongarch64-relax-got-load2.sh new file mode 100755 index 000000000..f6e20080c --- /dev/null +++ b/test/arch-loongarch64-relax-got-load2.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# pcaddi takes a 20-bit immediate that is shifted left by 2 to form +# a PC-relative offset. Therefore, relaxing a GOT load (pcalau12i + +# ld.d) to pcaddi is valid only when the symbol's address is 4-byte +# aligned. Here, `foo` is placed at an odd address, so the relaxation +# must not happen. + +cat < +int get_foo(); +int main() { printf("%d\n", get_foo()); } +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o -pie -Wl,--relax +$QEMU $t/exe | grep '^66$' +$OBJDUMP -d $t/exe | grep -A2 ':' | grep -Fw pcalau12i diff --git a/test/arch-loongarch64-relax-pcala-addi.sh b/test/arch-loongarch64-relax-pcala-addi.sh new file mode 100755 index 000000000..138ec9ab8 --- /dev/null +++ b/test/arch-loongarch64-relax-pcala-addi.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.globl get_sym1, get_sym2, get_sym3 +get_sym1: + la.pcrel $a0, sym1 + ret +get_sym2: + la.pcrel $a0, sym2 + ret +get_sym3: + la.pcrel $a0, sym3 + ret +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -xassembler - +.globl sym1, sym2, sym3 +sym1: + li.d $a0, 1 + ret +.space 1024 * 1024 +sym2: + li.d $a0, 2 + ret +.space 1024 * 1024 +sym3: + li.d $a0, 3 + ret +EOF + +cat < + +int (*get_sym1())(); +int (*get_sym2())(); +int (*get_sym3())(); + +int main() { + printf("%d %d %d\n", get_sym1()(), get_sym2()(), get_sym3()()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--no-relax +$QEMU $t/exe1 | grep '^1 2 3$' + +$OBJDUMP -d $t/exe1 > $t/exe1.objdump +grep -A1 ':' $t/exe1.objdump | grep pcalau12i +grep -A1 ':' $t/exe1.objdump | grep pcalau12i +grep -A1 ':' $t/exe1.objdump | grep pcalau12i + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o -Wl,--relax +$QEMU $t/exe2 | grep '^1 2 3$' + +$OBJDUMP -d $t/exe2 > $t/exe2.objdump +grep -A1 ':' $t/exe2.objdump | grep pcaddi +grep -A1 ':' $t/exe2.objdump | grep pcaddi +grep -A1 ':' $t/exe2.objdump | grep pcalau12i diff --git a/test/arch-loongarch64-relax-tlsdesc.sh b/test/arch-loongarch64-relax-tlsdesc.sh new file mode 100755 index 000000000..b6861d381 --- /dev/null +++ b/test/arch-loongarch64-relax-tlsdesc.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC +_Thread_local char foo[4] = "foo"; +_Thread_local char padding[100000] = "padding"; +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC +_Thread_local char bar[4] = "bar"; +EOF + +cat <<'EOF' | $CC -o $t/c.o -c -xc - -fPIC -mtls-dialect=desc -O2 +extern _Thread_local char foo[4]; +extern _Thread_local char bar[4]; + +char *get_foo() { return foo; } +char *get_bar() { return bar; } +EOF + +cat < +char *get_foo(); +char *get_bar(); + +int main() { + printf("%s %s\n", get_foo(), get_bar()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--no-relax +{ $QEMU $t/exe1; true; } |& grep 'unexpected reloc type' && skip +$QEMU $t/exe1 | grep 'foo bar' + +$OBJDUMP -d $t/exe1 > $t/exe1.objdump +grep -A6 ':' $t/exe1.objdump | grep -F pcalau12i +grep -A6 ':' $t/exe1.objdump | grep -F pcalau12i + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--relax +$QEMU $t/exe2 | grep 'foo bar' + +$OBJDUMP -d $t/exe2 > $t/exe2.objdump +grep -A6 ':' $t/exe2.objdump | grep -F li.w +grep -A6 ':' $t/exe2.objdump | grep -F lu12i.w diff --git a/test/arch-ppc64-elfv2-error.sh b/test/arch-ppc64-elfv2-error.sh new file mode 100755 index 000000000..7c54c4380 --- /dev/null +++ b/test/arch-ppc64-elfv2-error.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# We support ELFv2 only for little-endian PPC64. A big-endian ELFv2 file +# must be rejected instead of silently linked as ELFv1. +# https://github.com/rui314/mold/issues/1498 + +[ $MACHINE = ppc64 ] || skip + +cat < /dev/null || skip +int main() {} +EOF + +readelf -h $t/a.o | grep -q abiv2 || skip + +! ./mold -m elf64ppc -o $t/exe $t/a.o 2> $t/log +grep 'unknown machine type' $t/log diff --git a/test/arch-ppc64le-save-restore-fprs.sh b/test/arch-ppc64le-save-restore-fprs.sh new file mode 100755 index 000000000..a8aac935f --- /dev/null +++ b/test/arch-ppc64le-save-restore-fprs.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# The linker has to synthesize the _savefpr_N, _restfpr_N, _savevr_N and +# _restvr_N routines that GCC calls from function prologues and epilogues +# with -Os, just like the _savegpr0_N ones. + +cat < $t/log1 +grep -A1 '<_savefpr_14>:' $t/log1 | grep -F 'stfd f14,-144(r1)' +grep -A2 '<_restfpr_29>:' $t/log1 | grep -F 'lfd f29,-24(r1)' +grep -A2 '<_savevr_20>:' $t/log1 | grep -F 'stvx v20,r12,r0' +grep -A2 '<_restvr_31>:' $t/log1 | grep -F 'lvx v31,r12,r0' + +# A function that keeps many floating-point values live across calls +# makes GCC use the routines with -Os. +cat < +extern double out[20]; +void f(double); +double g(double x) { return x * 2; } + +int main() { + f(1); + double sum = 0; + for (int i = 0; i < 20; i++) + sum += out[i]; + printf("%g\n", sum); +} +EOF + +$CC -B. -o $t/exe2 $t/c.o $t/d.o +$QEMU $t/exe2 | grep '^420$' diff --git a/test/arch-ppc64le-save-restore-gprs.sh b/test/arch-ppc64le-save-restore-gprs.sh new file mode 100755 index 000000000..234d9fdef --- /dev/null +++ b/test/arch-ppc64le-save-restore-gprs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o +$OBJDUMP -d $t/exe | grep '<_savegpr0_14>' diff --git a/test/arch-riscv64-attributes.sh b/test/arch-riscv64-attributes.sh new file mode 100755 index 000000000..2eca08a73 --- /dev/null +++ b/test/arch-riscv64-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +not grep -F .riscv.attributes $t/log +not grep -F RISCV_ATTR $t/log diff --git a/test/arch-riscv64-emit-relocs-relax.sh b/test/arch-riscv64-emit-relocs-relax.sh new file mode 100755 index 000000000..12d812128 --- /dev/null +++ b/test/arch-riscv64-emit-relocs-relax.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# When mold relaxes instructions, the relocations attached to the relaxed code +# must be rewritten for --emit-relocs. A shortened call still references its +# symbol, so R_RISCV_CALL_PLT becomes R_RISCV_JAL or R_RISCV_RVC_JUMP. A +# relaxation that folds an address into the instruction as a link-time +# constant (lui/addi or an AUIPC+LD GOT load) leaves the instruction without a +# relocation, so we emit R_RISCV_NONE. + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.text +.globl _start +_start: + # CALL_PLT => jal (near target) + .reloc ., R_RISCV_RELAX + call func + + # lui + addi of a small absolute symbol => lui is deleted (HI20 => NONE), + # while the low-part relocation stays. + lui t0, %hi(small) + .reloc ., R_RISCV_RELAX + addi t0, t0, %lo(small) + + # GOT load of an absolute symbol => the value is materialized directly + # (GOT_HI20 and its paired PCREL_LO12_I both => NONE). +got_anchor: + .reloc ., R_RISCV_GOT_HI20, small + .reloc ., R_RISCV_RELAX + auipc t1, 0 + .reloc ., R_RISCV_PCREL_LO12_I, got_anchor + .reloc ., R_RISCV_RELAX + ld t1, 0(t1) + + # CALL with rd=zero (tail call) to a near target => c.j (=> RVC_JUMP) + .reloc ., R_RISCV_RELAX + tail do_exit + +func: + ret + +do_exit: + li a0, 0 + li a7, 93 # __NR_exit + ecall +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -xassembler - +.globl small +small = 0xba +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax +$QEMU $t/exe + +$OBJDUMP -dr $t/exe > $t/exe.objdump + +# A shortened call keeps a relocation that names the symbol. +grep -Fw R_RISCV_JAL $t/exe.objdump +grep -Fw R_RISCV_RVC_JUMP $t/exe.objdump + +# Address-materializing relaxations drop their relocations. +grep -Fw R_RISCV_NONE $t/exe.objdump + +# The stale pre-relaxation relocations are gone. +not grep -Fw R_RISCV_CALL_PLT $t/exe.objdump +not grep -Fw R_RISCV_GOT_HI20 $t/exe.objdump +not grep -Fw R_RISCV_HI20 $t/exe.objdump + +# The low-part relocation of the lui/addi pair survives (the addi is kept). +grep -Fw R_RISCV_LO12_I $t/exe.objdump + +# Each rewritten call relocation must sit on the relaxed instruction it +# describes: R_RISCV_JAL on a jal and R_RISCV_RVC_JUMP on a c.j. This checks +# that the relocation offsets were adjusted for the deleted bytes. +awk ' + $3 == "jal" { insn[$1] = "jal" } + $3 == "j" || $3 == "c.j" { insn[$1] = "cj" } + $2 == "R_RISCV_JAL" { if (insn[$1] != "jal") { bad=1; print "bad offset: " $0 } } + $2 == "R_RISCV_RVC_JUMP" { if (insn[$1] != "cj") { bad=1; print "bad offset: " $0 } } + END { exit bad+0 } +' $t/exe.objdump + +# TLS local-exec: `lui + add` that materializes TP + %tprel_hi are deleted for +# a variable within 2 KiB of TP (TPREL_HI20 and TPREL_ADD => NONE), while the +# access relative to TP keeps its low-part relocation. +cat <<'EOF' | $CC -o $t/tls.o -c -xc - -O2 -fno-section-anchors +__thread int tv; +int main(void) { return tv; } // tv is zero-initialized, so exits 0 +EOF + +$CC -B. -o $t/tls $t/tls.o -static -Wl,--emit-relocs,--relax +$QEMU $t/tls + +$OBJDUMP -dr $t/tls > $t/tls.objdump +grep -E $'R_RISCV_TPREL_LO12_[IS][ \t]+tv' $t/tls.objdump +not grep -E $'R_RISCV_TPREL_HI20[ \t]+tv' $t/tls.objdump +not grep -E $'R_RISCV_TPREL_ADD[ \t]+tv' $t/tls.objdump diff --git a/test/arch-riscv64-global-pointer-dso.sh b/test/arch-riscv64-global-pointer-dso.sh new file mode 100755 index 000000000..e4d5ada13 --- /dev/null +++ b/test/arch-riscv64-global-pointer-dso.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/b.so $t/c.o $t/d.o +$QEMU $t/exe | grep 'Hello world' diff --git a/test/arch-riscv64-global-pointer.sh b/test/arch-riscv64-global-pointer.sh new file mode 100755 index 000000000..d7f6bc21a --- /dev/null +++ b/test/arch-riscv64-global-pointer.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o -fno-PIE +readelf -W --dyn-syms $t/exe1 | grep -F '__global_pointer$' + +$CC -B. -o $t/exe2 $t/a.o -fPIE +readelf -W --dyn-syms $t/exe2 | grep -F '__global_pointer$' + +cat < +int hello() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/c.so $t/b.o -shared +readelf -W --dyn-syms $t/c.so | not grep -F '__global_pointer$' diff --git a/test/arch-riscv64-obj-compatible.sh b/test/arch-riscv64-obj-compatible.sh new file mode 100755 index 000000000..9cb690669 --- /dev/null +++ b/test/arch-riscv64-obj-compatible.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +set -x + +cat < +#include +extern char x1, x2; +int main() { + printf("%lu %lu %lu\n", + (uintptr_t)&x1 % 32, + (uintptr_t)&x2 % 32, + &x2 - &x1); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o +$QEMU $t/exe | grep '0 0 32' diff --git a/test/arch-riscv64-relax-align2.sh b/test/arch-riscv64-relax-align2.sh new file mode 100755 index 000000000..45bc590a8 --- /dev/null +++ b/test/arch-riscv64-relax-align2.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# The assembler emits (alignment - minimum instruction size) bytes of NOPs +# for an alignment directive, so the addend of R_RISCV_ALIGN is itself a +# power of two for `.balign 4` in RVC code and for `.balign 8` in non-RVC +# code. Make sure that the linker infers the right alignment from them. + +cat < +#include +extern char x1, x2; +int main() { + printf("%lu %lu\n", (uintptr_t)&x1 % 4, (uintptr_t)&x2 % 8); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o +$QEMU $t/exe | grep '^0 0$' diff --git a/test/arch-riscv64-relax-got.sh b/test/arch-riscv64-relax-got.sh new file mode 100755 index 000000000..fbca44d7d --- /dev/null +++ b/test/arch-riscv64-relax-got.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int get_sym1(); +int get_sym2(); +int get_sym3(); +int get_sym4(); +int get_sym5(); + +int main() { + printf("%x %x %x %x %x\n", + get_sym1(), get_sym2(), get_sym3(), get_sym4(), get_sym5()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--no-relax +$QEMU $t/exe1 | grep -E '^0 ba beef 11beef deadbeef$' + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o +$QEMU $t/exe2 | grep -E '^0 ba beef 11beef deadbeef$' + +$OBJDUMP -d $t/exe2 | grep -A2 ':' | grep -E $'li[ \t]+a0,186$' diff --git a/test/arch-riscv64-relax-hi20.sh b/test/arch-riscv64-relax-hi20.sh new file mode 100755 index 000000000..8a083ffe7 --- /dev/null +++ b/test/arch-riscv64-relax-hi20.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int get_foo(); +int get_foo2(); +int get_bar(); +int get_bar2(); +int get_baz(); + +int main() { + printf("%x %x %x %x %x\n", + get_foo(), get_foo2(), get_bar(), get_bar2(), get_baz()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--no-relax +$QEMU $t/exe1 | grep 'f00 10000f00 0 1ffff 11beef' + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o +$QEMU $t/exe2 | grep 'f00 10000f00 0 1ffff 11beef' + +readelf --syms $t/exe1 > $t/log1 +grep -E ' 10 NOTYPE .* get_foo$' $t/log1 +grep -E ' 10 NOTYPE .* get_foo2$' $t/log1 +grep -E ' 10 NOTYPE .* get_bar$' $t/log1 +grep -E ' 10 NOTYPE .* get_bar2$' $t/log1 +grep -E ' 10 NOTYPE .* get_baz$' $t/log1 + +readelf --syms $t/exe2 > $t/log2 +grep -E ' 8 NOTYPE .* get_foo$' $t/log2 +grep -E ' 10 NOTYPE .* get_foo2$' $t/log2 +grep -E ' 6 NOTYPE .* get_bar$' $t/log2 +grep -E ' 10 NOTYPE .* get_bar2$' $t/log2 +grep -E ' 10 NOTYPE .* get_baz$' $t/log2 diff --git a/test/arch-riscv64-relax-j.sh b/test/arch-riscv64-relax-j.sh new file mode 100755 index 000000000..df4731923 --- /dev/null +++ b/test/arch-riscv64-relax-j.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <' + + +cat <' diff --git a/test/arch-riscv64-reloc-overflow.sh b/test/arch-riscv64-reloc-overflow.sh new file mode 100755 index 000000000..fc54c4426 --- /dev/null +++ b/test/arch-riscv64-reloc-overflow.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int get_foo(); +int get_bar(); +int main() { printf("%x %x\n", get_foo(), get_bar()); } +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o + +readelf --syms $t/a.o | grep -E ' 10 FUNC .* get_foo$' +readelf --syms $t/a.o | grep -E ' 10 FUNC .* get_bar$' + +readelf --syms $t/exe | grep -E ' 8 FUNC .* get_foo$' +readelf --syms $t/exe | grep -E ' 8 FUNC .* get_bar$' diff --git a/test/arch-riscv64-variant-cc.sh b/test/arch-riscv64-variant-cc.sh new file mode 100755 index 000000000..76e57b5fd --- /dev/null +++ b/test/arch-riscv64-variant-cc.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < /dev/null || skip +.global foo +.type foo, %function +.variant_cc foo +foo: + ret +EOF + +$CC -B. -shared -o $t/b.so $t/a.o +readelf -W --dyn-syms $t/b.so | grep foo | grep -F '[VARIANT_CC]' + +cat < +long foo(); +int main() { printf("%ld\n", foo()); } +EOF + +$CC -B. -static -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep '^0$' diff --git a/test/arch-s390x-emit-relocs-relax.sh b/test/arch-s390x-emit-relocs-relax.sh new file mode 100755 index 000000000..3fd4e8114 --- /dev/null +++ b/test/arch-s390x-emit-relocs-relax.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# mold relaxes a GOT-loading LGRL into an address-materializing LARL when the +# symbol's address is a link-time constant. With --emit-relocs the R_390_GOTENT +# relocation must be rewritten to R_390_PC32DBL to match the LARL; otherwise it +# would describe a GOT slot that the relaxation removed. + +cat <<'EOF' | $CC -o $t/a.o -c -xc - +int foo = 42; +EOF + +cat <<'EOF' | $CC -fPIC -O2 -o $t/b.o -c -xc - +extern int foo; +int get(void) { return foo; } +EOF + +cat <<'EOF' | $CC -o $t/c.o -c -xc - +extern int get(void); +int main(void) { return get() != 42; } // exits 0 iff foo resolved to 42 +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o -static -Wl,--emit-relocs +$QEMU $t/exe + +$OBJDUMP -dr $t/exe > $t/exe.objdump + +# The GOT load was relaxed to a direct larl of foo. +grep -E 'larl.*' $t/exe.objdump + +# Its relocation was rewritten to match the larl, and the stale GOTENT is gone. +grep -E $'R_390_PC32DBL[ \t]+foo' $t/exe.objdump +not grep -E $'R_390_GOTENT[ \t]+foo' $t/exe.objdump diff --git a/test/arch-s390x-got.sh b/test/arch-s390x-got.sh new file mode 100755 index 000000000..63df31a86 --- /dev/null +++ b/test/arch-s390x-got.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# GOT[0] must be set to the link-time address of .dynamic on s390x. + +cat < + +extern char _DYNAMIC; +extern void *got[]; + +int main() { + printf("%d %p %p\n", &_DYNAMIC == got[0], &_DYNAMIC, got[0]); +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,-defsym=got=_GLOBAL_OFFSET_TABLE_ -no-pie +$QEMU $t/exe | grep -E '^1' diff --git a/test/arch-x86_64-address-equality.sh b/test/arch-x86_64-address-equality.sh new file mode 100755 index 000000000..a260eaafd --- /dev/null +++ b/test/arch-x86_64-address-equality.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +void *foo(); +void bar(); +int main() { printf("%d %p %p\n", foo() == bar, foo(), bar); } +EOF + +cat <& /dev/null || skip +{ sde64 -help; true; } | grep 'Diamond Rapids' || skip + +cat < + +void foo(); +void *get_foo_addr(); + +int main() { + printf("%d %p %p\n", foo == get_foo_addr(), foo, get_foo_addr()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/d.o +sde64 -dmr -- $t/exe1 | grep -E '^1 ' +$OBJDUMP -d $t/exe1 | grep -A1 ':' | grep -w lea + +$CC -B. -o $t/exe2 $t/a.o $t/c.so $t/d.o +sde64 -dmr -- $t/exe2 | grep -E '^1 ' +$OBJDUMP -d $t/exe2 | grep -A1 ':' | grep -w mov diff --git a/test/arch-x86_64-apx-gottpoff.sh b/test/arch-x86_64-apx-gottpoff.sh new file mode 100755 index 000000000..f50056c6b --- /dev/null +++ b/test/arch-x86_64-apx-gottpoff.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# We use Intel SDE to run programs compiled for APX +command -v sde64 >& /dev/null || skip +{ sde64 -help; true; } | grep 'Diamond Rapids' || skip + +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - || skip +.globl get_foo +get_foo: + mov foo@gottpoff(%rip), %r16 + mov %fs:(%r16), %eax + ret +EOF + +$OBJDUMP -r $t/a.o | grep -w R_X86_64_CODE_4_GOTTPOFF + +cat < + +_Thread_local int foo = 3; +int get_foo(); + +int main() { + printf("%d %d\n", foo, get_foo()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o +sde64 -dmr -- $t/exe1 | grep -E '^3 3$' + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-no-relax +sde64 -dmr -- $t/exe2 | grep -E '^3 3$' diff --git a/test/arch-x86_64-apx-gottpoff2.sh b/test/arch-x86_64-apx-gottpoff2.sh new file mode 100755 index 000000000..4a0249310 --- /dev/null +++ b/test/arch-x86_64-apx-gottpoff2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# We use Intel SDE to run programs compiled for APX +command -v sde64 >& /dev/null || skip +{ sde64 -help; true; } | grep 'Diamond Rapids' || skip + +cat < + +_Thread_local int foo = 3; +int get_foo(); + +int main() { + printf("%d %d\n", foo, get_foo()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o +sde64 -dmr -- $t/exe1 | grep -E '^3 3$' diff --git a/test/arch-x86_64-apx-tlsdesc.sh b/test/arch-x86_64-apx-tlsdesc.sh new file mode 100755 index 000000000..58a67c1d9 --- /dev/null +++ b/test/arch-x86_64-apx-tlsdesc.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_tlsdesc || skip + +# We use Intel SDE to run programs compiled for APX +command -v sde64 >& /dev/null || skip +{ sde64 -help; true; } | grep 'Diamond Rapids' || skip + +cat < + +_Thread_local int foo; + +int get_foo(); + +int main() { + foo = 42; + printf("%d\n", get_foo()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o +sde64 -dmr -- $t/exe1 | grep 42 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-no-relax +sde64 -dmr -- $t/exe2 | grep 42 + +$CC -shared -o $t/c.so $t/a.o +$CC -B. -o $t/exe3 $t/b.o $t/c.so +sde64 -dmr -- $t/exe3 | grep 42 + +$CC -shared -o $t/c.so $t/a.o -Wl,-no-relax +$CC -B. -o $t/exe4 $t/b.o $t/c.so -Wl,-no-relax +sde64 -dmr -- $t/exe4 | grep 42 diff --git a/test/arch-x86_64-comdat-early-elimination.sh b/test/arch-x86_64-comdat-early-elimination.sh new file mode 100755 index 000000000..74fd16254 --- /dev/null +++ b/test/arch-x86_64-comdat-early-elimination.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = x86_64 ] || skip + +cat <<'EOF' | $CC -o $t/a.o -c -x assembler - +.section .text.foo,"axG",@progbits,foo,comdat +.globl foo +.type foo, @function +foo: + .cfi_startproc + ret + .cfi_endproc +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -x assembler - +.section .text.foo,"axG",@progbits,foo,comdat +.globl foo +.type foo, @function +foo: + .cfi_startproc +.Ldead: + ret + .cfi_endproc + +.section .debug_addr,"",@progbits + .quad .Ldead + 26 + +.section .debug_ranges,"",@progbits + .quad .Ldead + 26 +EOF + +cat <<'EOF' | $CC -o $t/c.o -c -x assembler - +.text +.globl _start +_start: + call foo + ret +EOF + +$CC -B. -nostdlib -Wl,-e,_start -o $t/exe $t/a.o $t/b.o $t/c.o + +# The FDE and debug relocations in b.o refer to a discarded COMDAT member. +test "$(readelf -wf $t/exe | grep -c 'FDE cie=')" = 1 +$OBJDUMP -s -j .debug_addr $t/exe | grep '0000 00000000 00000000' +$OBJDUMP -s -j .debug_ranges $t/exe | grep '0000 01000000 00000000' diff --git a/test/arch-x86_64-comdat-signatures.sh b/test/arch-x86_64-comdat-signatures.sh new file mode 100755 index 000000000..74cb50f0c --- /dev/null +++ b/test/arch-x86_64-comdat-signatures.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = x86_64 ] || skip + +cat <<'EOF' | $CC -o $t/a.o -c -x assembler - +.section .text.local,"axG",@progbits,localsig,comdat +.local localsig +localsig: + .byte 0x11 + +.section .text.section,"axG",@progbits,.text.section,comdat + .byte 0x22 + +.section .text.version,"axG",@progbits,"versig@V1",comdat +.globl "versig@V1" +"versig@V1": + .byte 0x33 +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -x assembler - +.section .text.local,"axG",@progbits,localsig,comdat +.local localsig +localsig: + .byte 0x44 + +.section .text.section,"axG",@progbits,.text.section,comdat + .byte 0x55 + +.section .text.version,"axG",@progbits,"versig@V1",comdat +.globl "versig@V1" +"versig@V1": + .byte 0x66 +EOF + +cat <<'EOF' | $CC -o $t/c.o -c -x assembler - +.text +.globl _start +_start: + ret +EOF + +$CC -B. -nostdlib -Wl,-e,_start -o $t/exe $t/c.o $t/a.o $t/b.o +$OBJDUMP -s -j .text $t/exe | grep '112233' diff --git a/test/arch-x86_64-eh-frame-overflow.sh b/test/arch-x86_64-eh-frame-overflow.sh new file mode 100755 index 000000000..2d8c77754 --- /dev/null +++ b/test/arch-x86_64-eh-frame-overflow.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# An FDE refers to its function with a 32-bit PC-relative relocation, so +# a function that is more than 2 GiB away from .eh_frame cannot be +# described. That must be reported rather than silently wrapped around. + +cat < lea foo(%rip), %rax +# so R_X86_64_REX_GOTPCRELX is rewritten to R_X86_64_PC32. +# +cat <<'EOF' | $CC -o $t/a.o -c -xassembler - +.text +.globl _start +_start: + movq foo@GOTPCREL(%rip), %rax + movl (%rax), %eax + cmpl $42, %eax # exit(0) iff the relaxed lea resolved to foo + jne 1f + xorl %edi, %edi + jmp 2f +1: + movl $1, %edi +2: + movl $60, %eax # __NR_exit + syscall +.data +.hidden foo +foo: + .long 42 +EOF + +$CC -B. -o $t/exe1 $t/a.o -static -nostdlib -Wl,-e,_start,--emit-relocs +$QEMU $t/exe1 + +$OBJDUMP -dr $t/exe1 > $t/exe1.objdump + +# The GOT load was relaxed to a PC-relative lea of foo. +grep -E 'lea.*' $t/exe1.objdump + +# The relocation was rewritten to match the lea, and it sits on the lea +# (objdump prints a relocation right after the instruction it applies to). +grep -Fw R_X86_64_PC32 $t/exe1.objdump +not grep -Fw R_X86_64_REX_GOTPCRELX $t/exe1.objdump +awk ' + /R_X86_64_PC32/ { if (prev !~ /lea/) { bad=1; print "not on lea: " prev }; next } + { prev = $0 } + END { exit bad+0 } +' $t/exe1.objdump + +# +# TLS general-dynamic of a local symbol is relaxed to local-exec: +# lea gd@tlsgd(%rip),%rdi; call __tls_get_addr => mov %fs:0,%rax; ... +# The whole sequence is rewritten, so R_X86_64_TLSGD is kept as a marker. +# +cat <<'EOF' | $CC -fPIC -mtls-dialect=gnu -o $t/b.o -c -xc - +__thread int gd; +int main(void) { return gd; } // gd is zero-initialized, so exits 0 +EOF + +$CC -B. -o $t/exe2 $t/b.o -no-pie -Wl,--emit-relocs +$QEMU $t/exe2 + +$OBJDUMP -dr $t/exe2 > $t/exe2.objdump + +# The general-dynamic sequence was relaxed to a local-exec access via %fs. +grep -F %fs:0x0 $t/exe2.objdump + +# The TLS relocation is kept and still names the symbol. +grep -E $'R_X86_64_TLSGD[ \t]+gd' $t/exe2.objdump diff --git a/test/arch-x86_64-empty-arg.sh b/test/arch-x86_64-empty-arg.sh new file mode 100755 index 000000000..5f20dd482 --- /dev/null +++ b/test/arch-x86_64-empty-arg.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +nm -D mold | grep __msan_init && skip + +not ./mold -m elf_x86_64 '' |& grep 'cannot open :' diff --git a/test/arch-x86_64-empty-mergeable-section.sh b/test/arch-x86_64-empty-mergeable-section.sh new file mode 100755 index 000000000..6cc199899 --- /dev/null +++ b/test/arch-x86_64-empty-mergeable-section.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep '.rodata.str1.1 .* AMS ' $t/log +not grep '.rodata.str1.1 .* AM ' $t/log diff --git a/test/arch-x86_64-emulation-deduction.sh b/test/arch-x86_64-emulation-deduction.sh new file mode 100755 index 000000000..5748cc3f3 --- /dev/null +++ b/test/arch-x86_64-emulation-deduction.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <& /dev/null +readelf --segments -W $t/exe | grep 'GNU_STACK.* RW ' + +$CC -B. -o $t/exe $t/a.o -Wl,-z,execstack-if-needed +readelf --segments -W $t/exe | grep 'GNU_STACK.* RWE ' diff --git a/test/arch-x86_64-function-multiversion.sh b/test/arch-x86_64-function-multiversion.sh new file mode 100755 index 000000000..d33bf763d --- /dev/null +++ b/test/arch-x86_64-function-multiversion.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip +[ "$(uname)" = FreeBSD ] && skip + +cat < + +class Hello { +public: + __attribute__((target("default"))) void say() { std::cout << "Hello\n"; } + __attribute__((target("popcnt"))) void say() { std::cout << "Howdy\n"; } +}; + +void hello() { + Hello().say(); +} +EOF + +$CXX -B. -shared -o $t/b.so $t/a.o + +cat <& /dev/null || skip +$OBJDUMP -d $t/exe | grep -A1 '<__x86.get_pc_thunk.bx>:' | + grep -F 'puts$plt' diff --git a/test/arch-x86_64-gnu-retain.sh b/test/arch-x86_64-gnu-retain.sh new file mode 100755 index 000000000..d172b4db0 --- /dev/null +++ b/test/arch-x86_64-gnu-retain.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo '.section foo,"R"' | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip + +cat < + +extern char foo[5000L * 1000 * 1000]; +extern char bar[1000 * 1000]; + +int main() { + printf("%d %d\n", foo[0], bar[0]); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o + +$QEMU $t/exe diff --git a/test/arch-x86_64-icf-personality.sh b/test/arch-x86_64-icf-personality.sh new file mode 100755 index 000000000..d72928191 --- /dev/null +++ b/test/arch-x86_64-icf-personality.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# ICF must not merge functions whose FDEs point to different CIEs. +# fn1 and fn2 below are identical except for the personality routine +# recorded in their CIEs, so only fn1 and fn3 are mergeable. + +cat < + +extern char fn1, fn2, fn3; + +int main() { + printf("%d %d\n", (long)&fn1 == (long)&fn2, (long)&fn1 == (long)&fn3); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-icf=all +$QEMU $t/exe | grep '0 1' diff --git a/test/arch-x86_64-ifunc-alias.sh b/test/arch-x86_64-ifunc-alias.sh new file mode 100755 index 000000000..c41e21c04 --- /dev/null +++ b/test/arch-x86_64-ifunc-alias.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip +test_cflags -static || skip + +cat < +#include + +__attribute__((target("default"))) +int foo() { + return 0; +} + +__attribute__((target("ssse3,avx2"))) +int foo() { + return 1; +} + +int (*p)() = foo; + +int main() { + int val = foo(); + assert(val == p()); +} +EOF + +$CXX -B. -o $t/exe $t/a.o -static +$QEMU $t/exe diff --git a/test/arch-x86_64-ifunc-export.sh b/test/arch-x86_64-ifunc-export.sh new file mode 100755 index 000000000..68efd24f2 --- /dev/null +++ b/test/arch-x86_64-ifunc-export.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < +struct MyClass { + MyClass(T data) { myfunc(data); } +}; + +template struct MyClass; +EOF + +$CXX -shared -o $t/b.so $t/a.o + +cat < +struct MyClass { + MyClass(T data) { myfunc(data); } +}; + +extern template struct MyClass; + +int main() { + MyClass obj(3); +} +EOF + +$CXX -B. -o $t/exe $t/c.o $t/b.so -fno-PIE +$QEMU $t/exe |& grep 'circular dependency' || skip +$QEMU $t/exe diff --git a/test/arch-x86_64-incompatible-libs-linker-script.sh b/test/arch-x86_64-incompatible-libs-linker-script.sh new file mode 100755 index 000000000..5fb466f4c --- /dev/null +++ b/test/arch-x86_64-incompatible-libs-linker-script.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -m32 || skip + +mkdir -p $t/foo + +echo 'char hello[] = "Hello world";' | $CC -shared -o $t/libbar.so -m32 -xc - +echo 'char hello[] = "Hello world";' | $CC -shared -o $t/foo/libbar.so -xc - + +cat < +extern char hello[]; +int main() { + printf("%s\n", hello); +} +EOF + +cat < $t/b.script +INPUT(libbar.so) +EOF + +cd $t + +$CC -B$OLDPWD -o exe1 -Lfoo a.o b.script +LD_LIBRARY_PATH=. $QEMU ./exe1 | grep 'Hello world' + +$CC -B$OLDPWD -o exe2 -Lfoo b.script a.o +LD_LIBRARY_PATH=. $QEMU ./exe2 | grep 'Hello world' diff --git a/test/arch-x86_64-incompatible-libs-linker-script2.sh b/test/arch-x86_64-incompatible-libs-linker-script2.sh new file mode 100755 index 000000000..77c5fcba8 --- /dev/null +++ b/test/arch-x86_64-incompatible-libs-linker-script2.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +nm mold | grep '__tsan_init' && skip +test_cflags -m32 || skip + +mkdir -p $t/foo + +cat < +extern char hello[]; +int main() { + printf("%s\n", hello); +} +EOF + +cat < $t/d.script +INPUT(a.o) +EOF + +cd $t + +$OLDPWD/ld -o e.o -r -Lfoo d.script c.o +$OLDPWD/ld -o f.o -r -Lfoo c.o d.script diff --git a/test/arch-x86_64-incompatible-libs.sh b/test/arch-x86_64-incompatible-libs.sh new file mode 100755 index 000000000..bd6d672b7 --- /dev/null +++ b/test/arch-x86_64-incompatible-libs.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -m32 || skip + +cat < + +extern char hello[]; + +int main() { + printf("%s\n", hello); +} +EOF + +mkdir -p $t/script +echo 'OUTPUT_FORMAT(elf32-i386)' > $t/script/libfoo.so + +$CC -B. -o $t/exe -L$t/script -L$t/lib32 -L$t/lib64 \ + $t/e.o -lfoo -Wl,-rpath $t/lib64 >& $t/log + +grep 'script/libfoo.so: skipping incompatible file' $t/log +grep 'lib32/libfoo.so: skipping incompatible file' $t/log +grep 'lib32/libfoo.a: skipping incompatible file' $t/log +$QEMU $t/exe | grep 'Hello world' diff --git a/test/arch-x86_64-incompatible-libs2.sh b/test/arch-x86_64-incompatible-libs2.sh new file mode 100755 index 000000000..2de13aa66 --- /dev/null +++ b/test/arch-x86_64-incompatible-libs2.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -m32 || skip +nm mold | grep '__tsan_init' && skip + +cat < + +extern char hello[]; + +int main() { + printf("%s\n", hello); +} +EOF + +mkdir -p $t/script +echo 'GROUP(libfoo.so)' > $t/script/libfoo.so + +$CC -B. -o $t/exe -L$t/lib32 -L$t/lib64 -lfoo $t/e.o -Wl,-rpath $t/lib64 |& + grep 'lib32/libfoo.so: skipping incompatible file' + +$QEMU $t/exe | grep 'Hello world' diff --git a/test/arch-x86_64-incompatible-obj.sh b/test/arch-x86_64-incompatible-obj.sh new file mode 100755 index 000000000..a0c1dbb84 --- /dev/null +++ b/test/arch-x86_64-incompatible-obj.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -m32 || skip + +cat <& $t/log +grep "$t/b.o: incompatible file type: x86_64 is expected but got i386" $t/log diff --git a/test/arch-x86_64-init-array-readonly.sh b/test/arch-x86_64-init-array-readonly.sh new file mode 100755 index 000000000..915a4a2d3 --- /dev/null +++ b/test/arch-x86_64-init-array-readonly.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void init1() { printf("init1 "); } +void init2() { printf("init2 "); } + +int main() { + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o +$QEMU $t/exe | grep 'init1 init2' diff --git a/test/arch-x86_64-init-array.sh b/test/arch-x86_64-init-array.sh new file mode 100755 index 000000000..4d92f9327 --- /dev/null +++ b/test/arch-x86_64-init-array.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void init1() { printf("init1 "); } +void init2() { printf("init2 "); } +void fini1() { printf("fini1\n"); } +void fini2() { printf("fini2 "); } + +int main() { + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 'init1 init2 fini2 fini1' diff --git a/test/arch-x86_64-isa-level.sh b/test/arch-x86_64-isa-level.sh new file mode 100755 index 000000000..4f0f3b7eb --- /dev/null +++ b/test/arch-x86_64-isa-level.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +char arr1[0xc0000000]; +extern char arr2[0xc0000000]; +int main() { + printf("%d %lx\n", (void *)arr1 < (void *)arr2, arr2 - arr1); +} +EOF + +cat <& /dev/null || skip +echo 'int main() {}' | $CLANG -B. -flto -o /dev/null -xc - >& /dev/null || skip + +cat <<'EOF' > $t/a.ll +$mix = comdat any +@mix_lto = global i32 1, comdat($mix) +EOF + +cat <<'EOF' > $t/b.s +.globl keep_me +.section .text.keep_me,"axG",@progbits,mix,comdat +keep_me: + ret + +.text +.globl main +main: + call keep_me + xor %eax, %eax + ret +EOF + +$CLANG -S -emit-llvm -flto -o $t/a.bc $t/a.ll +$CC -o $t/b.o -c -x assembler $t/b.s +$CLANG -B. -o $t/exe -flto $t/a.bc $t/b.o + +$QEMU $t/exe diff --git a/test/arch-x86_64-mergeable-records.sh b/test/arch-x86_64-mergeable-records.sh new file mode 100755 index 000000000..aaabc6a29 --- /dev/null +++ b/test/arch-x86_64-mergeable-records.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static || skip + +cat <<'EOF' | $CC -o $t/a.o -c -x assembler - + .text + .globl main +main: + sub $8, %rsp + + mov $.L.str1, %rdi + xor %rax, %rax + call printf + + mov $.L.str1+1, %rdi + xor %rax, %rax + call printf + + mov $str2+2, %rdi + xor %rax, %rax + call printf + + mov $.L.str3+3, %rdi + xor %rax, %rax + call printf + + mov $.rodata.cst8+16, %rdi + xor %rax, %rax + call printf + + xor %rax, %rax + add $8, %rsp + ret + + .section .rodata.cst8, "aM", @progbits, 8 + .align 8 +.L.str1: + .ascii "abcdef\n\0" +.globl str2 +str2: + .ascii "ghijkl\n\0" +.L.str3: + .ascii "mnopqr\n\0" +EOF + +$CC -B. -static -no-pie -o $t/exe $t/a.o + +$QEMU $t/exe | grep '^abcdef$' +$QEMU $t/exe | grep '^bcdef$' +$QEMU $t/exe | grep '^ijkl$' +$QEMU $t/exe | grep '^pqr$' +$QEMU $t/exe | grep '^mnopqr$' diff --git a/test/arch-x86_64-mergeable-strings-nonalloc.sh b/test/arch-x86_64-mergeable-strings-nonalloc.sh new file mode 100755 index 000000000..ad180f210 --- /dev/null +++ b/test/arch-x86_64-mergeable-strings-nonalloc.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -o $t/a.o -c -xc - +int main() {} +EOF + +cat <<'EOF' | $CC -o $t/b.o -c -x assembler - +.section .foo, "", @progbits +.quad .L1 - 1 +.quad .L2 - 1 + +.section .bar, "MS", @progbits, 1 +.L1: + .string "abc" +.L2: + .string "xyz" +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o + +readelf -x .foo $t/exe | grep -F '03000000 00000000 ffffffff ffffffff' +readelf -x .bar $t/exe | grep -F 'xyz.abc.' diff --git a/test/arch-x86_64-mergeable-strings.sh b/test/arch-x86_64-mergeable-strings.sh new file mode 100755 index 000000000..9db1795fc --- /dev/null +++ b/test/arch-x86_64-mergeable-strings.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static || skip + +cat <<'EOF' | $CC -o $t/a.o -c -x assembler - + .text + .globl main +main: + sub $8, %rsp + mov $.L.str+3, %rdi + xor %rax, %rax + call printf + mov $.rodata.str1.1+16, %rdi + xor %rax, %rax + call printf + xor %rax, %rax + add $8, %rsp + ret + + .section .rodata.str1.1, "aMS", @progbits, 1 + .string "bar" +.L.str: +foo: + .string "xyzHello" + .string "foo world\n" +EOF + +$CC -B. -static -no-pie -o $t/exe $t/a.o +$QEMU $t/exe | grep 'Hello world' + +readelf -sW $t/exe | grep -E '[0-9] foo$' diff --git a/test/arch-x86_64-note-property.sh b/test/arch-x86_64-note-property.sh new file mode 100755 index 000000000..59f1b63e1 --- /dev/null +++ b/test/arch-x86_64-note-property.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip +test_cflags -fcf-protection=branch || skip + +cat < $t/log1 + +grep -E '.note.bar\s+NOTE.+000008 00 A 0 0 4' $t/log1 +grep -E '.note.baz\s+NOTE.+000008 00 A 0 0 8' $t/log1 +grep -E '.note.nonalloc\s+NOTE.+000008 00 0 0 1' $t/log1 + +readelf --segments $t/exe > $t/log2 +grep -F '01 .note.baz .note.foo .note.bar' $t/log2 +not grep 'NOTE.*0x0000000000000000 0x0000000000000000' $t/log2 diff --git a/test/arch-x86_64-note2.sh b/test/arch-x86_64-note2.sh new file mode 100755 index 000000000..6ca1f656b --- /dev/null +++ b/test/arch-x86_64-note2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Binutils 2.32 injects their own .note.gnu.property section interfering with the tests +test_cflags -Xassembler -mx86-used-note=no && CFLAGS="-Xassembler -mx86-used-note=no" || CFLAGS="" + +cat < +#include + +void preinit() { write(STDOUT_FILENO, "preinit ", 8); } +void init() { write(STDOUT_FILENO, "init ", 5); } +void fini() { write(STDOUT_FILENO, "fini\n", 5); } +int main() {} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 'preinit init fini' diff --git a/test/arch-x86_64-relax.sh b/test/arch-x86_64-relax.sh new file mode 100755 index 000000000..d0c9a8ee8 --- /dev/null +++ b/test/arch-x86_64-relax.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Skip if target is not x86-64 +echo ret | cc -c -o /dev/null -xassembler -Wa,-mrelax-relocations=yes - 2> /dev/null || skip + +cat <:/,/<.*>:/p' > $t/log + +grep -E 'lea \s*0x.+\(%rip\),%rax .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rcx .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rdx .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rbx .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rbp .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rsi .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%rdi .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r8 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r9 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r10 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r11 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r12 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r13 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r14 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r15 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r16 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r17 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r18 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r19 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r20 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r21 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r22 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r23 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r24 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r25 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r26 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r27 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r28 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r29 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r30 .*' $t/log +grep -E 'lea \s*0x.+\(%rip\),%r31 .*' $t/log + +grep -E 'call.*' $t/log +grep -E 'jmp.*' $t/log diff --git a/test/arch-x86_64-reloc-overflow.sh b/test/arch-x86_64-reloc-overflow.sh new file mode 100755 index 000000000..d6173943e --- /dev/null +++ b/test/arch-x86_64-reloc-overflow.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int print(int x) { + printf("%d\n", x); + return 0; +} + +int print64(long x) { + printf("%ld\n", x); + return 0; +} +EOF + +$CC -shared -o $t/c.so $t/a.o $t/b.o -Wl,-z,noexecstack + +# Absolute symbol +cat <<'EOF' > $t/d.s +.globl abs_sym +.set abs_sym, 42 + +.globl main +main: + sub $8, %rsp + lea abs_sym, %edi + call print + add $8, %rsp + ret +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s -no-pie +$QEMU $t/exe | grep '^42$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^42$' + +# GOT +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov ext_var@GOTPCREL(%rip), %rdi + mov (%rdi), %edi + call print + add $8, %rsp + ret +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s -no-pie +$QEMU $t/exe | grep '^56$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^56$' + +# Copyrel +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov ext_var(%rip), %edi + call print + add $8, %rsp + ret +EOF + +$CC -c -o $t/d.o $t/d.s +$CC -B. -o $t/exe $t/c.so $t/d.o -no-pie +$QEMU $t/exe | grep '^56$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^56$' + +# Copyrel +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov foo(%rip), %rdi + mov (%rdi), %edi + call print + add $8, %rsp + ret + +.data +foo: + .quad ext_var +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s -no-pie +$QEMU $t/exe | grep '^56$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^56$' + +# PLT +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov $76, %edi + call print@PLT + add $8, %rsp + ret +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s -no-pie +$QEMU $t/exe | grep '^76$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^76$' + +# PLT +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov $76, %edi + lea print(%rip), %rax + call *%rax + add $8, %rsp + ret +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s -no-pie +$QEMU $t/exe | grep '^76$' +$CC -B. -o $t/exe $t/c.so $t/d.s -pie +$QEMU $t/exe | grep '^76$' + +# SIZE32 +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + mov $foo+2@SIZE, %edi + call print@PLT + add $8, %rsp + ret + +.data +.globl foo +.type foo, %object +.size foo, 24 +foo: +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s +$QEMU $t/exe | grep '^26$' + +# SIZE64 +cat <<'EOF' > $t/d.s +.globl main +main: + sub $8, %rsp + movabs $foo+5@SIZE, %rdi + call print64@PLT + add $8, %rsp + ret + +.data +.globl foo +.type foo, %object +.size foo, 56 +foo: +EOF + +$CC -B. -o $t/exe $t/c.so $t/d.s +$QEMU $t/exe | grep '^61$' + +# GOTPCREL64 +cat <<'EOF' > $t/e.c +extern long ext_var; +static long arr[50000] = {1, 2, 3}; +void print64(long); + +int main() { + print64(ext_var * 1000000 + arr[2]); +} +EOF + +$CC -c -o $t/e.o $t/e.c -mcmodel=large -fPIC +$CC -B. -o $t/exe $t/c.so $t/e.o +$QEMU $t/exe | grep '^56000003$' + +# R_X86_64_32 against non-alloc section +cat <<'EOF' > $t/f.s +.globl main +main: + sub $8, %rsp + add $8, %rsp + ret + +.section .foo, "", @progbits +.zero 16 +foo: +.quad bar + +.section .bar, "", @progbits +.zero 24 +bar: +.quad foo +EOF + +$CC -c -o $t/f.o $t/f.s +$CC -B. -o $t/exe $t/f.o +readelf -x .foo -x .bar $t/exe > $t/log + +grep -F '0x00000010 00000000 00000000 10000000 00000000' $t/log +grep -F '0x00000010 18000000 00000000' $t/log diff --git a/test/arch-x86_64-section-alignment.sh b/test/arch-x86_64-section-alignment.sh new file mode 100755 index 000000000..50cec9917 --- /dev/null +++ b/test/arch-x86_64-section-alignment.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -c -o $t/a.o -xc - +#include +#include + +__attribute__((aligned(8192))) int foo = 1; + +typedef struct { + uint8_t e_ident[16]; + uint16_t e_type; + uint16_t e_machine; + uint32_t e_version; + uint64_t e_entry; + uint64_t e_phoff; + uint64_t e_shoff; + uint32_t e_flags; + uint16_t e_ehsize; + uint16_t e_phentsize; + uint16_t e_phnum; + uint16_t e_shentsize; + uint16_t e_shnum; + uint16_t e_shstrndx; +} Ehdr; + +char __ehdr_start; + +int main() { + Ehdr *e = (Ehdr *)&__ehdr_start; + printf("%lu %lu %lu\n", e->e_phoff % 8, e->e_shoff % 8, (uint64_t)&foo % 8192); +} +EOF + +$CC -B. -o $t/exe $t/a.o +$QEMU $t/exe | grep '^0 0 0$' diff --git a/test/arch-x86_64-section-name.sh b/test/arch-x86_64-section-name.sh new file mode 100755 index 000000000..20785e1f9 --- /dev/null +++ b/test/arch-x86_64-section-name.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' | $CC -o $t/a.o -c -x assembler - +.globl _start +.text +_start: + ret + +.section .text.hot +.ascii ".text.hot " +.section .text.hot.foo +.ascii ".text.hot.foo " + +.section .text.unknown +.ascii ".text.unknown " +.section .text.unknown.foo +.ascii ".text.unknown.foo " + +.section .text.unlikely +.ascii ".text.unlikely " +.section .text.unlikely.foo +.ascii ".text.unlikely.foo " + +.section .text.startup +.ascii ".text.startup " +.section .text.startup.foo +.ascii ".text.startup.foo " + +.section .text.exit +.ascii ".text.exit " +.section .text.exit.foo +.ascii ".text.exit.foo " + +.section .text +.ascii ".text " +.section .text.foo +.ascii ".text.foo " + +.section .data.rel.ro +.ascii ".data.rel.ro " +.section .data.rel.ro.foo +.ascii ".data.rel.ro.foo " + +.section .data +.ascii ".data " +.section .data.foo +.ascii ".data.foo " + +.section .rodata +.ascii ".rodata " +.section .rodata.foo +.ascii ".rodata.foo " + +.section .srodata +.ascii ".srodata " +.section .srodata.foo +.ascii ".srodata.foo " +EOF + +./mold -o $t/exe $t/a.o -z keep-text-section-prefix + +readelf -p .text.hot $t/exe | grep -F '.text.hot .text.hot.foo' +readelf -p .text.unknown $t/exe | grep -F '.text.unknown .text.unknown.foo' +readelf -p .text.unlikely $t/exe | grep -F '.text.unlikely .text.unlikely.foo' +readelf -p .text.startup $t/exe | grep -F '.text.startup .text.startup.foo' +readelf -p .text.exit $t/exe | grep -F '.text.exit .text.exit.foo' +readelf -p .text $t/exe | grep -F '.text .text.foo' +readelf -p .data.rel.ro $t/exe | grep -F '.data.rel.ro .data.rel.ro.foo' +readelf -p .data $t/exe | grep -F '.data .data.foo' +readelf -p .rodata $t/exe | grep -F '.rodata .rodata.foo' +readelf -p .srodata $t/exe | grep -F '.srodata .srodata.foo' + +./mold -o $t/exe $t/a.o +readelf --sections $t/exe | not grep -F .text.hot + +./mold -o $t/exe $t/a.o -z nokeep-text-section-prefix +readelf --sections $t/exe | not grep -F .text.hot diff --git a/test/arch-x86_64-sframe-unwind.sh b/test/arch-x86_64-sframe-unwind.sh new file mode 100755 index 000000000..f5355b153 --- /dev/null +++ b/test/arch-x86_64-sframe-unwind.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Verify that the .sframe section mold produces is not just well-formed but +# actually usable for stack unwinding. We link a program that, using only +# its own merged .sframe (read via libsframe), walks a known call chain and +# checks that each recovered frame maps to the expected function. + +# Skip unless the assembler emits SFrame Version 3. +cat </dev/null || skip +int main() { return 0; } +EOF +readelf --sframe=.sframe $t/probe.o 2>/dev/null | grep -q SFRAME_VERSION_3 || skip + +# Skip unless libsframe and its header are available. +echo 'int main(){return 0;}' | \ + $CC -o $t/probe -xc - -lsframe -include sframe-api.h 2>/dev/null || skip + +cat <<'EOF' | $CC -O0 -g -fno-omit-frame-pointer -Wa,--gsframe -B. -o $t/exe -xc - -lsframe +#define _GNU_SOURCE +#include +#include +#include +#include + +#ifndef PT_GNU_SFRAME +#define PT_GNU_SFRAME 0x6474e554 +#endif + +static uintptr_t sec_addr; +static size_t sec_size; + +static int find_cb(struct dl_phdr_info *info, size_t sz, void *u) { + (void)sz; (void)u; + for (int i = 0; i < info->dlpi_phnum; i++) { + const ElfW(Phdr) *p = &info->dlpi_phdr[i]; + if (p->p_type == PT_GNU_SFRAME) { + sec_addr = info->dlpi_addr + p->p_vaddr; + sec_size = p->p_memsz; + return 1; + } + } + return 0; +} + +static sframe_decoder_ctx *dctx; +static int32_t fixed_ra; + +#define MAXF 4096 +static uintptr_t fstart[MAXF], fend[MAXF]; +static int nfde; + +static int find_fde(uintptr_t pc) { + for (int i = 0; i < nfde; i++) + if (fstart[i] <= pc && pc < fend[i]) + return i; + return -1; +} + +// One SFrame unwind step per loop iteration; records the start address of +// the function each frame belongs to. +static int unwind(uintptr_t pc, uintptr_t sp, uintptr_t fp, + uintptr_t *out, int max) { + int n = 0; + while (n < max) { + int idx = find_fde(pc); + if (idx < 0) + break; + out[n++] = fstart[idx]; + + uint32_t num_fres, func_size; + int64_t spo; + unsigned char fi, fi2; + uint8_t rep; + if (sframe_decoder_get_funcdesc_v3(dctx, idx, &num_fres, &func_size, + &spo, &fi, &fi2, &rep)) + break; + uint32_t fde_type = SFRAME_V3_FDE_TYPE(fi2); + + int64_t off = pc - fstart[idx]; + sframe_frame_row_entry fre, cur; + int have = 0; + for (uint32_t k = 0; k < num_fres; k++) { + if (sframe_decoder_get_fre(dctx, idx, k, &cur)) + break; + if ((int64_t)cur.fre_start_addr <= off) { fre = cur; have = 1; } + else break; + } + if (!have) + break; + + int e = 0; + int base = sframe_fre_get_base_reg_id(&fre, &e); + int32_t cfa_off = sframe_fre_get_cfa_offset(dctx, &fre, fde_type, &e); + uintptr_t cfa = (base == SFRAME_BASE_REG_SP ? sp : fp) + cfa_off; + uintptr_t ra = *(uintptr_t *)(cfa + fixed_ra); + + e = 0; + int32_t fp_off = sframe_fre_get_fp_offset(dctx, &fre, fde_type, &e); + uintptr_t next_fp = (e == 0) ? *(uintptr_t *)(cfa + fp_off) : fp; + + if (!ra) + break; + sp = cfa; + fp = next_fp; + pc = ra; + } + return n; +} + +static uintptr_t chain[16]; +static int chain_n; + +__attribute__((noinline)) static int level3(int x) { + uintptr_t pc, sp, fp; + asm volatile("leaq 1f(%%rip), %0\n1:\n\t" + "movq %%rsp, %1\n\t" + "movq %%rbp, %2" + : "=r"(pc), "=r"(sp), "=r"(fp)); + chain_n = unwind(pc, sp, fp, chain, 16); + return x + 1; +} + +__attribute__((noinline)) static int level2(int x) { return level3(x) + x; } +__attribute__((noinline)) static int level1(int x) { return level2(x) + x; } + +int main(void) { + if (!dl_iterate_phdr(find_cb, NULL) || !sec_addr) + return 1; + + int err = 0; + dctx = sframe_decode((const char *)sec_addr, sec_size, &err); + if (!dctx) + return 1; + fixed_ra = sframe_decoder_get_fixed_ra_offset(dctx); + + uint32_t num = sframe_decoder_get_num_fidx(dctx); + for (uint32_t i = 0; i < num && nfde < MAXF; i++) { + uint32_t num_fres, func_size; + int64_t spo; + unsigned char fi, fi2; + uint8_t rep; + if (sframe_decoder_get_funcdesc_v3(dctx, i, &num_fres, &func_size, + &spo, &fi, &fi2, &rep)) + continue; + int e = 0; + uint32_t fo = sframe_decoder_get_offsetof_fde_start_addr(dctx, i, &e); + fstart[nfde] = sec_addr + fo + spo; // PCREL: relative to the field + fend[nfde] = fstart[nfde] + func_size; + nfde++; + } + + level1(0); + + uintptr_t want[] = { (uintptr_t)&level3, (uintptr_t)&level2, + (uintptr_t)&level1, (uintptr_t)&main }; + int n = sizeof(want) / sizeof(want[0]); + if (chain_n < n) + return 1; + for (int i = 0; i < n; i++) + if (chain[i] != want[i]) + return 1; + return 0; +} +EOF + +$QEMU $t/exe diff --git a/test/arch-x86_64-tbss-only.sh b/test/arch-x86_64-tbss-only.sh new file mode 100755 index 000000000..9f32a7c66 --- /dev/null +++ b/test/arch-x86_64-tbss-only.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Test if grep supports backreferences +echo abab | grep -E '(ab)\1' || skip + +cat < + +static _Thread_local int x1 = 1; +static _Thread_local int x2; +extern _Thread_local int x3; +extern _Thread_local int x4; +int get_x5(); +int get_x6(); + +int main() { + x2 = 2; + + printf("%d %d %d %d %d %d\n", x1, x2, x3, x4, get_x5(), get_x6()); + return 0; +} +EOF + +cat < + +__attribute__((tls_model("global-dynamic"))) static _Thread_local int x1 = 1; +__attribute__((tls_model("global-dynamic"))) _Thread_local int x2 = 2; +__attribute__((tls_model("global-dynamic"))) _Thread_local int x3; + +int foo() { + x3 = 3; + + printf("%d %d %d\n", x1, x2, x3); + return 0; +} +EOF + +cat < + +extern _Thread_local char x[1024000]; +extern _Thread_local char y[1024000]; + +int main() { + x[0] = 3; + x[1023] = 5; + printf("%d %d %d %d %d %d\n", x[0], x[1], x[1023], y[0], y[1], y[1023]); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep '^3 0 5 0 0 0$' diff --git a/test/arch-x86_64-tls-ld-mcmodel-large.sh b/test/arch-x86_64-tls-ld-mcmodel-large.sh new file mode 100755 index 000000000..96ce68c52 --- /dev/null +++ b/test/arch-x86_64-tls-ld-mcmodel-large.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +extern _Thread_local int foo; +static _Thread_local int bar; + +int *get_foo_addr() { return &foo; } +int *get_bar_addr() { return &bar; } + +int main() { + bar = 5; + + printf("%d %d %d %d\n", *get_foo_addr(), *get_bar_addr(), foo, bar); + return 0; +} +EOF + +cat < + +int get_foo(); +extern _Thread_local int bar; + +int main() { + printf("%d %d\n", get_foo(), bar); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o +$QEMU $t/exe1 | grep '^20 3$' + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o -Wl,-no-relax +$QEMU $t/exe2 | grep '^20 3$' + +$CC -B. -o $t/d.so $t/a.o -shared +$CC -B. -o $t/exe3 $t/b.o $t/c.o $t/d.so +$QEMU $t/exe3 | grep '^20 3$' diff --git a/test/arch-x86_64-tlsdesc.sh b/test/arch-x86_64-tlsdesc.sh new file mode 100755 index 000000000..addd1d5c0 --- /dev/null +++ b/test/arch-x86_64-tlsdesc.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_tlsdesc || skip + +cat < + +_Thread_local int foo; + +int get_foo(); + +int main() { + foo = 42; + printf("%d\n", get_foo()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o +$QEMU $t/exe1 | grep 42 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-no-relax +$QEMU $t/exe2 | grep 42 + +$CC -B. -shared -o $t/c.so $t/a.o +$CC -B. -o $t/exe3 $t/b.o $t/c.so +$QEMU $t/exe3 | grep 42 + +$CC -B. -shared -o $t/c.so $t/a.o -Wl,-no-relax +$CC -B. -o $t/exe4 $t/b.o $t/c.so -Wl,-no-relax +$QEMU $t/exe4 | grep 42 diff --git a/test/arch-x86_64-unique.sh b/test/arch-x86_64-unique.sh new file mode 100755 index 000000000..2ae744824 --- /dev/null +++ b/test/arch-x86_64-unique.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <& $t/log +grep 'relocation against symbol `main'\'' in read-only section' $t/log +grep 'creating a DT_TEXTREL in an output file' $t/log diff --git a/test/arch-x86_64-warn-textrel.sh b/test/arch-x86_64-warn-textrel.sh new file mode 100755 index 000000000..5964f1366 --- /dev/null +++ b/test/arch-x86_64-warn-textrel.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Skip if libc is musl +is_musl && skip + +cat <<'EOF' | $CC -c -o $t/a.o -x assembler - +.globl fn +fn: + movabs main, %rax + ret +EOF + +cat <& $t/log +grep 'relocation against symbol `main'\'' in read-only section' $t/log +grep 'creating a DT_TEXTREL in an output file' $t/log diff --git a/test/arch-x86_64-z-dynamic-undefined-weak.sh b/test/arch-x86_64-z-dynamic-undefined-weak.sh new file mode 100755 index 000000000..b4c16793b --- /dev/null +++ b/test/arch-x86_64-z-dynamic-undefined-weak.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ "$(uname)" = FreeBSD ] && skip + +cat < +__attribute__((weak)) extern int foo; +int main() { printf("%p\n", &foo); } +EOF + +not $CC -B. -o $t/exe3 $t/b.o -no-pie -Wl,-z,dynamic-undefined-weak |& + grep 'cannot create a copy relocation for foo' diff --git a/test/arch-x86_64-z-ibt.sh b/test/arch-x86_64-z-ibt.sh new file mode 100755 index 000000000..0d07ff5b7 --- /dev/null +++ b/test/arch-x86_64-z-ibt.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip + +cat < +void hello() { printf("Hello"); } +void world() { printf("world"); } +EOF + +$CC -B. -o $t/b.so -shared $t/a.o -Wl,-z,ibtplt + +cat < + +void hello(); +void world(); + +int main() { + hello(); + printf(" "); + world(); + printf("\n"); +} +EOF + +$CC -B. -o $t/exe $t/c.o $t/b.so -Wl,-z,ibtplt +$QEMU $t/exe | grep 'Hello world' diff --git a/test/arch-x86_64-z-rewrite-endbr.sh b/test/arch-x86_64-z-rewrite-endbr.sh new file mode 100755 index 000000000..a2e18bc3d --- /dev/null +++ b/test/arch-x86_64-z-rewrite-endbr.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -fcf-protection || skip + +cat < $t/log1 + +grep -A1 ':' $t/log1 | grep endbr64 +grep -A1 ':' $t/log1 | grep endbr64 +grep -A1 '
:' $t/log1 | grep endbr64 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-z,rewrite-endbr +$OBJDUMP -dr $t/exe2 > $t/log2 + +grep -A1 ':' $t/log2 | grep nop +grep -A1 ':' $t/log2 | grep nop +grep -A1 '
:' $t/log2 | grep endbr64 diff --git a/test/arch-x86_64-z-rewrite-endbr2.sh b/test/arch-x86_64-z-rewrite-endbr2.sh new file mode 100755 index 000000000..74e26dfb7 --- /dev/null +++ b/test/arch-x86_64-z-rewrite-endbr2.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -fcf-protection || skip + +cat < $t/log1 + +grep -A1 ':' $t/log1 | grep endbr64 +grep -A1 ':' $t/log1 | grep endbr64 +grep -A1 '
:' $t/log1 | grep endbr64 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-z,rewrite-endbr +$OBJDUMP -dr $t/exe2 > $t/log2 + +grep -A1 ':' $t/log2 | grep nop +grep -A1 ':' $t/log2 | grep nop +grep -A1 '
:' $t/log2 | grep endbr64 diff --git a/test/arch-x86_64-z-rewrite-endbr3.sh b/test/arch-x86_64-z-rewrite-endbr3.sh new file mode 100755 index 000000000..cf3d693ab --- /dev/null +++ b/test/arch-x86_64-z-rewrite-endbr3.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -fcf-protection || skip +[ "$QEMU" == '' ] || skip + +# Check if Intel SDE CPU emulator is available +command -v sde64 >& /dev/null || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,-z,rewrite-endbr +sde64 -cet 1 -- $t/exe | grep 'Hello world' diff --git a/test/arch-x86_64-z-shstk.sh b/test/arch-x86_64-z-shstk.sh new file mode 100755 index 000000000..34e29b609 --- /dev/null +++ b/test/arch-x86_64-z-shstk.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip + +cat < + +int fn1(); + +int fn2() { + return 3; +} + +void *ptr = fn2; + +int main() { + printf("%d\n", fn1()); +} +EOF + +$CC -B. -pie -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 3 + +readelf --dynamic $t/exe | grep -F '(TEXTREL)' +readelf --dynamic $t/exe | grep '\(FLAGS\).*TEXTREL' diff --git a/test/as-needed-dso.sh b/test/as-needed-dso.sh new file mode 100755 index 000000000..0c5714e24 --- /dev/null +++ b/test/as-needed-dso.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log2 +grep libbar $t/log2 +not grep libfoo $t/log2 diff --git a/test/as-needed-dso2.sh b/test/as-needed-dso2.sh new file mode 100755 index 000000000..245b946c0 --- /dev/null +++ b/test/as-needed-dso2.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log2 + +not grep libfoo $t/log2 +not grep libbar $t/log2 +grep libbaz $t/log2 diff --git a/test/as-needed-gc.sh b/test/as-needed-gc.sh new file mode 100755 index 000000000..7eae91a3b --- /dev/null +++ b/test/as-needed-gc.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Create a shared library +cat < $t/log1 +not grep -F 'Shared library: [libfoo.so]' $t/log1 + +# Link with --as-needed but without --gc-sections (should keep libfoo.so because dead() is not GC'ed) +$CC -B. -o $t/exe2 $t/prog.o -Wl,--as-needed -Wl,--no-gc-sections $t/libfoo.so +readelf --dynamic $t/exe2 > $t/log2 +grep -F 'Shared library: [libfoo.so]' $t/log2 diff --git a/test/as-needed-weak.sh b/test/as-needed-weak.sh new file mode 100755 index 000000000..32c2637a7 --- /dev/null +++ b/test/as-needed-weak.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log1 +grep -F 'Shared library: [libfoo.so]' $t/log1 +grep -F 'Shared library: [libbar.so]' $t/log1 + +$CC -B. -o $t/exe2 $t/a.o -Wl,-as-needed -L$t -lbar -lfoo + +readelf --dynamic $t/exe2 > $t/log2 +grep -F 'Shared library: [libfoo.so]' $t/log2 +not grep -F 'Shared library: [libbar.so]' $t/log2 diff --git a/test/as-needed.sh b/test/as-needed.sh new file mode 100755 index 000000000..555a7e10d --- /dev/null +++ b/test/as-needed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -F 'Shared library: [libfoo.so]' $t/log +grep -F 'Shared library: [libbar.so]' $t/log + +$CC -B. -o $t/exe $t/a.o -Wl,--as-needed $t/b.so $t/c.so + +readelf --dynamic $t/exe > $t/log +grep -F 'Shared library: [libfoo.so]' $t/log +not grep -F 'Shared library: [libbar.so]' $t/log diff --git a/test/audit.sh b/test/audit.sh new file mode 100755 index 000000000..25d20c0cf --- /dev/null +++ b/test/audit.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -F 'Auxiliary library: [foo]' $t/log +grep -F 'Auxiliary library: [bar]' $t/log + +not ./mold -o exe $t/a.o -f bar |& grep 'auxiliary may not be used without -shared' + +# -fuse-ld is ignored +./mold -o exe $t/a.o -fuse-ld=mold diff --git a/test/bno-symbolic.sh b/test/bno-symbolic.sh new file mode 100755 index 000000000..cb3fb301a --- /dev/null +++ b/test/bno-symbolic.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# GCC produces buggy code for this test case on s390x. +# https://sourceware.org/bugzilla/show_bug.cgi?id=29655 +[ $MACHINE = s390x ] && $CC -v |& grep -E '^gcc version 1[0-5]\.' && skip + +cat < + +extern int foo; +int get_foo(); +void *bar(); + +int main() { + foo = 3; + printf("%d %d %d\n", foo, get_foo(), bar == bar()); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep '3 3 1' diff --git a/test/bsymbolic-functions.sh b/test/bsymbolic-functions.sh new file mode 100755 index 000000000..098c58044 --- /dev/null +++ b/test/bsymbolic-functions.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int foo = 3; +int x = 5; +int get_foo(); +void *bar() { return &x; } + +int main() { + printf("%d %d %d\n", foo, get_foo(), bar == bar()); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep '3 3 0' diff --git a/test/bsymbolic-non-weak-functions.sh b/test/bsymbolic-non-weak-functions.sh new file mode 100755 index 000000000..eefd939c6 --- /dev/null +++ b/test/bsymbolic-non-weak-functions.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int foo = 3; +int bar = 3; +int baz = 3; + +int get_foo1() { return 7; } +int get_bar1() { return 7; } +int get_baz1() { return 7; } + +int get_foo2(); +int get_bar2(); +int get_baz2(); + +int main() { + printf("%d %d %d %d %d %d\n", foo, bar, baz, + get_foo2(), get_bar2(), get_baz2()); +} +EOF + +$CC -B. -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep '^3 3 3 3 3 7$' diff --git a/test/bsymbolic-non-weak.sh b/test/bsymbolic-non-weak.sh new file mode 100755 index 000000000..795f35dc6 --- /dev/null +++ b/test/bsymbolic-non-weak.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int foo = 3; +int bar = 3; +int baz = 3; + +int get_foo1() { return 7; } +int get_bar1() { return 7; } +int get_baz1() { return 7; } + +int get_foo2(); +int get_bar2(); +int get_baz2(); + +int main() { + printf("%d %d %d %d %d %d\n", foo, bar, baz, + get_foo2(), get_bar2(), get_baz2()); +} +EOF + +$CC -B. -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep '^3 3 3 3 4 7$' diff --git a/test/bsymbolic.sh b/test/bsymbolic.sh new file mode 100755 index 000000000..651103967 --- /dev/null +++ b/test/bsymbolic.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int foo = 3; +int get_foo(); + +int main() { + printf("%d %d\n", foo, get_foo()); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep '3 4' diff --git a/test/build-id.sh b/test/build-id.sh new file mode 100755 index 000000000..27a7a9c92 --- /dev/null +++ b/test/build-id.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc - + +$CC -B. -o $t/exe1 $t/a.o -Wl,-build-id +readelf -n $t/exe1 | grep 'GNU.*0x00000014.*NT_GNU_BUILD_ID' + +$CC -B. -o $t/exe2 $t/a.o -Wl,-build-id=uuid +readelf -nW $t/exe2 | grep -E 'Build ID: ............4...[89abcdef]' + +$CC -B. -o $t/exe3 $t/a.o -Wl,-build-id=md5 +readelf -n $t/exe3 | grep 'GNU.*0x00000010.*NT_GNU_BUILD_ID' + +$CC -B. -o $t/exe4 $t/a.o -Wl,-build-id=sha1 +readelf -n $t/exe4 | grep 'GNU.*0x00000014.*NT_GNU_BUILD_ID' + +$CC -B. -o $t/exe5 $t/a.o -Wl,-build-id=sha256 +readelf -n $t/exe5 | grep 'GNU.*0x00000020.*NT_GNU_BUILD_ID' + +$CC -B. -o $t/exe6 $t/a.o -Wl,-build-id=fast +readelf -n $t/exe6 | grep 'GNU.*0x00000020.*NT_GNU_BUILD_ID' + +$CC -B. -o $t/exe7 $t/a.o -Wl,-build-id=0xdeadbeefdeadbeef +readelf -n $t/exe7 | grep 'Build ID: deadbeefdeadbeef' diff --git a/test/canonical-plt.sh b/test/canonical-plt.sh new file mode 100755 index 000000000..f8dad8e3d --- /dev/null +++ b/test/canonical-plt.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# GCC produces buggy code for this test case on s390x. +# https://sourceware.org/bugzilla/show_bug.cgi?id=29655 +[ $MACHINE = s390x ] && $CC -v |& grep -E '^gcc version 1[0-5]\.' && skip + +cat < + +void *foo(); +void *bar(); +void *baz(); + +int main() { + printf("%d %d %d\n", foo == foo(), bar == bar(), bar == baz()); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/a.so $t/b.o $t/c.o +$QEMU $t/exe | grep '^1 1 1$' diff --git a/test/cmdline.sh b/test/cmdline.sh new file mode 100755 index 000000000..fcff22220 --- /dev/null +++ b/test/cmdline.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +not ./mold -zfoo |& grep 'unknown command line option: -zfoo' +not ./mold -z foo |& grep 'unknown command line option: -z foo' +not ./mold -abcdefg |& grep 'unknown command line option: -abcdefg' +not ./mold --abcdefg |& grep 'unknown command line option: --abcdefg' diff --git a/test/color-diagnostics.sh b/test/color-diagnostics.sh new file mode 100755 index 000000000..c3d1ab0b8 --- /dev/null +++ b/test/color-diagnostics.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <&1; [ $? = 1 ]; } | + grep -F ">>> prevailing definition is in $t/a.o" diff --git a/test/comment.sh b/test/comment.sh new file mode 100755 index 000000000..202d952f9 --- /dev/null +++ b/test/comment.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int foo; +int bar; +extern int baz; +__attribute__((weak)) int two(); + +int main() { + printf("%d %d %d %d\n", foo, bar, baz, two ? two() : -1); +} +EOF + +cat < + +int bar; + +int main() { + printf("%d\n", bar); +} +EOF + +cat < + +int foo; +int bar = 5; +int baz; + +int main() { + printf("%d %d %d\n", foo, bar, baz); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep '0 5 42' + +readelf --sections $t/exe > $t/log +grep '.common .*NOBITS' $t/log diff --git a/test/common.inc b/test/common.inc new file mode 100644 index 000000000..e5f639efb --- /dev/null +++ b/test/common.inc @@ -0,0 +1,150 @@ +# -*- mode: sh -*- + +# Make sure all commands print out messages in English +export LC_ALL=C + +canonical_name() { + case $1 in + i?86) echo i686 ;; + amd64) echo x86_64 ;; + armeb*) echo armeb ;; + arm*) echo arm ;; + powerpc) echo ppc ;; + powerpc64) echo ppc64 ;; + powerpc64le) echo ppc64le ;; + *) echo $1 + esac +} + +if [ -z "$MACHINE" ]; then + MACHINE=$(canonical_name $(uname -m)) +fi + +# Set tool names +if [ -z "$TRIPLE" ]; then + TESTDIR=out/test/$MACHINE + CC="${TEST_CC:-cc}" + CXX="${TEST_CXX:-c++}" + GCC="${TEST_GCC:-gcc}" + GXX="${TEST_GXX:-g++}" + OBJDUMP=objdump + OBJCOPY=objcopy + STRIP=strip + QEMU= +elif [ "$TRIPLE" = powerpc64le-linux-gnu -a "$CPU" = power10 ]; then + MACHINE=ppc64le + TESTDIR=out/test/ppc64le-power10 + CC="${TEST_CC:-$TRIPLE-gcc} -mcpu=power10" + CXX="${TEST_CXX:-$TRIPLE-g++} -mcpu=power10" + GCC="${TEST_GCC:-$TRIPLE-gcc} -mcpu=power10" + GXX="${TEST_GXX:-$TRIPLE-g++} -mcpu=power10" + OBJDUMP="$TRIPLE-objdump" + OBJCOPY="$TRIPLE-objcopy" + STRIP="$TRIPLE-strip" + QEMU="qemu-ppc64le -L /usr/$TRIPLE -cpu power10" +else + MACHINE=$(canonical_name $(echo $TRIPLE | sed 's/-.*//')) + TESTDIR=out/test/$MACHINE + CC="${TEST_CC:-$TRIPLE-gcc}" + CXX="${TEST_CXX:-$TRIPLE-g++}" + GCC="${TEST_GCC:-$TRIPLE-gcc}" + GXX="${TEST_GXX:-$TRIPLE-g++}" + OBJDUMP="$TRIPLE-objdump" + OBJCOPY="$TRIPLE-objcopy" + STRIP="$TRIPLE-strip" + + case $MACHINE in + i686) QEMU="qemu-i386 -L /usr/$TRIPLE" ;; + sh4aeb) QEMU="qemu-sh4eb -L /usr/$TRIPLE" ;; + *) QEMU="qemu-$MACHINE -L /usr/$TRIPLE" ;; + esac +fi + +testname=$(basename "$0" .sh) +t=$TESTDIR/$testname +mkdir -p $t + +case $MACHINE in +x86_64 | i686 | arm | armeb) + tlsdesc_opt=-mtls-dialect=gnu2 ;; +aarch64 | riscv* | loongarch*) + tlsdesc_opt=-mtls-dialect=desc ;; +esac + +# We want to use GNU's binutils even on BSDs. `pkg install binutils` +# installs GNU binutils under /usr/local/bin. +if [ "$(uname)" = FreeBSD ]; then + export PATH="/usr/local/bin:$PATH" +fi + +# Common functions +not() { + if "$@"; then return 1; else return 0; fi +} + +test_cflags() { + echo 'int main() {}' | $CC -B. "$@" -o /dev/null -xc - >& /dev/null +} + +test_cxxflags() { + echo 'int main() {}' | $CXX -B. "$@" -o /dev/null -xc++ - >& /dev/null +} + +is_musl() { + { ldd --version; true; } |& grep musl > /dev/null +} + +supports_ifunc() { + ! is_musl && + echo 'void x() __attribute__((ifunc("y"))); void *y() { return 0; }' | + $CC -c -o /dev/null -xc - >& /dev/null +} + +supports_tlsdesc() { + # musl's tlsdesc on arm32 seems to be broken + [[ $MACHINE = arm* ]] && is_musl && return 1 + + # FreeBSD's loader doesn't seem to support TLSDESC relocs in an executable + [ "$(uname)" = FreeBSD ] && return 1 + + [ "$tlsdesc_opt" = '' ] && return 1 + + # TLSDESC may not be supported on old systems. Compile a DSO with + # it to see if it is actually supported. + echo '_Thread_local int x; int y() { return x; }' | + $CC -shared -fPIC -xc -o $t/tlsdesc.so $tlsdesc_opt - 2> /dev/null || + return 1 + echo 'int y(); int main() { y(); }' | $CC -xc -c -o $t/tlsdesc.o - + $CC -o $t/tlsdesc $t/tlsdesc.o $t/tlsdesc.so + $QEMU $t/tlsdesc 2> /dev/null +} + +on_qemu() { + [ "$QEMU" != '' ] || grep -w qemu /proc/cpuinfo >& /dev/null +} + +skip() { + echo skipped + trap - EXIT + exit 0 +} + +on_error() { + code=$? + echo "command failed: $1: $BASH_COMMAND" + trap - EXIT + exit $code +} + +on_exit() { + echo OK + exit 0 +} + +trap 'on_error $LINENO' ERR +trap on_exit EXIT + +# Print out the startup message +echo -n "Testing $testname ... " +set -o pipefail +set -x diff --git a/test/compress-debug-sections-zlib-level.sh b/test/compress-debug-sections-zlib-level.sh new file mode 100755 index 000000000..442d950fe --- /dev/null +++ b/test/compress-debug-sections-zlib-level.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +#include +#include +int main() { printf("Hello world\n"); } +EOF + +# Level 0 stores data uncompressed; level 9 compresses hardest. +# The level-0 executable must be larger than the level-9 one. +$CC -B. -o $t/exe0 $t/a.o -Wl,--compress-debug-sections=zlib:0 +$CC -B. -o $t/exe9 $t/a.o -Wl,--compress-debug-sections=zlib:9 +readelf -WS $t/exe0 | grep '\.debug_info .* [Cx] ' +readelf -WS $t/exe9 | grep '\.debug_info .* [Cx] ' +[ $(wc -c < $t/exe0) -gt $(wc -c < $t/exe9) ] + +# Out-of-range level should fail with a descriptive error +not $CC -B. -o $t/exe-1 $t/a.o -Wl,--compress-debug-sections=zlib:-1 |& + grep 'zlib level must be between 0 and 9' +not $CC -B. -o $t/exe10 $t/a.o -Wl,--compress-debug-sections=zlib:10 |& + grep 'zlib level must be between 0 and 9' diff --git a/test/compress-debug-sections-zstd-level.sh b/test/compress-debug-sections-zstd-level.sh new file mode 100755 index 000000000..1fd128953 --- /dev/null +++ b/test/compress-debug-sections-zstd-level.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { printf("Hello world\n"); } +EOF + +# Test zstd:1 (lowest level) +$CC -B. -o $t/exe1 $t/a.o -Wl,--compress-debug-sections=zstd:1 +readelf -WS $t/exe1 | grep '\.debug_info .* [Cx] ' + +# Test zstd:22 (highest level) +$CC -B. -o $t/exe22 $t/a.o -Wl,--compress-debug-sections=zstd:22 +readelf -WS $t/exe22 | grep '\.debug_info .* [Cx] ' + +# Out-of-range level should fail with a descriptive error +not $CC -B. -o $t/exe0 $t/a.o -Wl,--compress-debug-sections=zstd:0 |& + grep 'zstd level must be between 1 and 22' +not $CC -B. -o $t/exe23 $t/a.o -Wl,--compress-debug-sections=zstd:23 |& + grep 'zstd level must be between 1 and 22' diff --git a/test/compress-debug-sections-zstd.sh b/test/compress-debug-sections-zstd.sh new file mode 100755 index 000000000..423002210 --- /dev/null +++ b/test/compress-debug-sections-zstd.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# arm-linux-gnueabihf-objcopy crashes on x86-64 +[[ $MACHINE = arm* ]] && skip +[ $MACHINE = riscv32 ] && skip + +command -v zstdcat >& /dev/null || skip + +cat < + +int main() { + printf("Hello world\n"); + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,--compress-debug-sections=zstd +$OBJCOPY --dump-section .debug_info=$t/debug_info $t/exe +dd if=$t/debug_info of=$t/debug_info.zstd bs=24 skip=1 status=none +zstdcat $t/debug_info.zstd > /dev/null diff --git a/test/compress-debug-sections.sh b/test/compress-debug-sections.sh new file mode 100755 index 000000000..2dfb09acf --- /dev/null +++ b/test/compress-debug-sections.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { printf("Hello world\n"); } +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,--compress-debug-sections=zlib + +readelf -WS $t/exe > $t/log +grep '\.debug_info .* [Cx] ' $t/log +grep '\.debug_str .* MS[Cx] ' $t/log diff --git a/test/compressed-debug-info.sh b/test/compressed-debug-info.sh new file mode 100755 index 000000000..34c277e54 --- /dev/null +++ b/test/compressed-debug-info.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +command -v dwarfdump >& /dev/null || skip + +cat < /dev/null +readelf --sections $t/exe | grep -F .debug_info diff --git a/test/copyrel-alignment.sh b/test/copyrel-alignment.sh new file mode 100755 index 000000000..ce600ebe6 --- /dev/null +++ b/test/copyrel-alignment.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = ppc64 ] && skip +[ $MACHINE = ppc64le ] && skip +[[ $MACHINE = loongarch* ]] && skip + +cat < +extern int foo; +int main() { printf("%d %p\n", foo, &foo); } +EOF + +$CC -B. -o $t/exe1 $t/d.o $t/a.so -no-pie +$QEMU $t/exe1 > /dev/null +readelf -W --sections $t/exe1 | grep '\.copyrel.* 32$' + +$CC -B. -o $t/exe2 $t/d.o $t/b.so -no-pie +$QEMU $t/exe2 > /dev/null +readelf -W --sections $t/exe2 | grep '\.copyrel.* 8$' + +$CC -B. -o $t/exe3 $t/d.o $t/c.so -no-pie +$QEMU $t/exe3 > /dev/null +readelf -W --sections $t/exe3 | grep '\.copyrel.* 256$' diff --git a/test/copyrel-norelro.sh b/test/copyrel-norelro.sh new file mode 100755 index 000000000..0992a64fd --- /dev/null +++ b/test/copyrel-norelro.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[[ $MACHINE = ppc64* ]] && skip +[[ $MACHINE = loongarch* ]] && skip + +cat < + +extern char msg[100]; + +int main() { + printf("%s\n", msg); +} +EOF + +cat < +#include +#include + +extern const char readonly[100]; +extern char readwrite[100]; + +static int segv = 0; +static jmp_buf buf; + +void handler(int sig) { + segv = 1; + longjmp(buf, 1); +} + +int main() { + signal(SIGSEGV, handler); + + readwrite[0] = 5; + int x = segv; + + if (setjmp(buf) == 0) + *(char *)readonly = 5; + int y = segv; + + printf("sigsegv %d %d\n", x, y); +} +EOF + +cat < +#include +#include + +extern char readonly[100]; +extern char readwrite[100]; + +static int segv = 0; +static jmp_buf buf; + +void handler(int sig) { + segv = 1; + longjmp(buf, 1); +} + +int main() { + signal(SIGSEGV, handler); + + readwrite[0] = 5; + int x = segv; + + if (setjmp(buf) == 0) + *(char *)readonly = 5; + int y = segv; + + printf("sigsegv %d %d\n", x, y); +} +EOF + +cat < + +extern int foo; +extern int *get_bar(); + +int main() { + printf("%d %d %d\n", foo, *get_bar(), &foo == get_bar()); + return 0; +} +EOF + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +clang "${clang_args[@]}" -B. -o $t/exe $t/a.o +$QEMU $t/exe | grep 'Hello world' +readelf --debug-dump=info $t/exe | grep -E 'DW_AT_name.*: main$' diff --git a/test/ctors-in-init-array.sh b/test/ctors-in-init-array.sh new file mode 100755 index 000000000..c771eeca0 --- /dev/null +++ b/test/ctors-in-init-array.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +static void ctor1() { printf("ctor1 "); } +static void ctor2() { printf("ctor2 "); } +static void ctor3() { printf("ctor3 "); } +static void ctor4() { printf("ctor4 "); } + +static void dtor1() { printf("dtor1 "); } +static void dtor2() { printf("dtor2 "); } +static void dtor3() { printf("dtor3 "); } +static void dtor4() { printf("dtor4 "); } + +__attribute__((aligned(sizeof(void *)), section(".ctors.65435"))) +void (*ctors65435[])() = { ctor1 }; + +__attribute__((aligned(sizeof(void *)), section(".ctors.65433"))) +void (*ctors65433[])() = { ctor2 }; + +__attribute__((aligned(sizeof(void *)), section(".ctors"))) +void (*ctors[])() = { ctor4, ctor3 }; + +__attribute__((aligned(sizeof(void *)), section(".dtors"))) +void (*dtors[])() = { dtor1, dtor2 }; + +__attribute__((aligned(sizeof(void *)), section(".dtors.65433"))) +void (*dtors65433[])() = { dtor3 }; + +__attribute__((aligned(sizeof(void *)), section(".dtors.65435"))) +void (*dtors65435[])() = { dtor4 }; +EOF + +cat < + +__attribute__((constructor(101))) +static void init1() { printf("init1 "); } + +__attribute__((constructor)) +static void init2() { printf("init2 "); } + +__attribute__((destructor(101))) +static void fini1() { printf("fini1 "); } + +__attribute__((destructor)) +static void fini2() { printf("fini2 "); } + +int main() {} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 'ctor1 init1 ctor2 ctor3 ctor4 init2 fini2 dtor1 dtor2 dtor3 fini1 dtor4' diff --git a/test/dead-debug-sections.sh b/test/dead-debug-sections.sh new file mode 100755 index 000000000..2b54e3f52 --- /dev/null +++ b/test/dead-debug-sections.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +command -v dwarfdump >& /dev/null || skip + +cat < +const char *msg; +int main() { printf("%s\n", msg); } +EOF + +$CXX -o $t/exe $t/a.o $t/b.o $t/c.o -g +$QEMU $t/exe | grep 'Hello world' + +dwarfdump $t/exe > /dev/null diff --git a/test/debug-macro-section.sh b/test/debug-macro-section.sh new file mode 100755 index 000000000..6a862f935 --- /dev/null +++ b/test/debug-macro-section.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/a.h +#define A 23 +#define B 99 +EOF + +cat < $t/b.ver + +$CC -B. -o $t/c.so -shared $t/a.o -Wl,--default-symver -Wl,--version-script=$t/b.ver +readelf --dyn-syms $t/c.so | grep -F ' foo@@c.so' diff --git a/test/default-symver.sh b/test/default-symver.sh new file mode 100755 index 000000000..342eda79f --- /dev/null +++ b/test/default-symver.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +extern void live_func(); +void dead_func() { + printf("OK\n"); +} + +int main() { + live_func(); +} +EOF + +$CC -B. -flto -o $t/exe $t/a.o -Wl,-defsym,live_func=dead_func + +$QEMU $t/exe | grep "^OK$" diff --git a/test/defsym-missing-symbol.sh b/test/defsym-missing-symbol.sh new file mode 100755 index 000000000..4989d63ae --- /dev/null +++ b/test/defsym-missing-symbol.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +extern char foo; +extern char bar; +void baz(); + +void print() { + printf("Hello %p %p\n", &foo, &bar); +} + +int main() { + baz(); +} +EOF + +$CC -B. -o $t/exe $t/a.o -pie -Wl,-defsym=foo=16 \ + -Wl,-defsym=bar=0x2000 -Wl,-defsym=baz=print + +$QEMU $t/exe | grep '^Hello 0x10 0x2000$' diff --git a/test/defsym2.sh b/test/defsym2.sh new file mode 100755 index 000000000..57e87b5d6 --- /dev/null +++ b/test/defsym2.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < as core::iter::iterator::Iterator>::rposition::::{closure#0}' diff --git a/test/demangle.sh b/test/demangle.sh new file mode 100755 index 000000000..a20e1dbfd --- /dev/null +++ b/test/demangle.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +unsigned la_version(unsigned v) { + fprintf(stderr, "version=%d\n", v); + return 0; +} + +void foo() {} +EOF + +cat <<'EOF' | $CC -B. -shared -o $t/b.so -xc - -fPIC -Wl,--audit=$t/a.so +void foo(); +void bar() { foo(); } +EOF + +cat <<'EOF' | $CC -c -o $t/c.o -xc - +void bar(); +int main() { bar(); } +EOF + +$CC -B. -o $t/exe1 $t/c.o $t/b.so -Wl,--allow-shlib-undefined +readelf --dynamic $t/exe1 | grep 'Dependency audit library:..*/a.so' + +$CC -B. -o $t/exe2 $t/c.o $t/b.so -Wl,--depaudit=foo -Wl,--allow-shlib-undefined +readelf --dynamic $t/exe2 | grep 'Dependency audit library:..*foo:.*/a.so' diff --git a/test/dependency-file-lto.sh b/test/dependency-file-lto.sh new file mode 100755 index 000000000..9d1e3514d --- /dev/null +++ b/test/dependency-file-lto.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -flto -fno-fat-lto-objects || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -flto -o $t/exe $t/a.o -Wl,-dependency-file=$t/dep + +grep '/exe:.*/a.o ' $t/dep +grep '/a.o:$' $t/dep +not grep '^/tmp' $t/dep diff --git a/test/dependency-file-response-file.sh b/test/dependency-file-response-file.sh new file mode 100755 index 000000000..c7910929a --- /dev/null +++ b/test/dependency-file-response-file.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +echo "$t/a.o -Wl,-dependency-file=$t/dep" > $t/rsp + +$CC -B. -o $t/exe @$t/rsp + +grep '/exe:.*/a.o ' $t/dep +grep '/a.o:$' $t/dep +not grep '^/tmp' $t/dep diff --git a/test/dependency-file.sh b/test/dependency-file.sh new file mode 100755 index 000000000..46d2b0726 --- /dev/null +++ b/test/dependency-file.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,-dependency-file=$t/dep + +grep "dependency-file/exe:.*/a.o " $t/dep +grep ".*/a.o:$" $t/dep diff --git a/test/disable-new-dtags.sh b/test/disable-new-dtags.sh new file mode 100755 index 000000000..9fafe0bfc --- /dev/null +++ b/test/disable-new-dtags.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -F _start $t/log +grep -F foo $t/log +not grep -F .Lbar $t/log + +./mold -o $t/exe $t/a.o --discard-none +readelf --symbols $t/exe > $t/log +grep -F _start $t/log +grep -F foo $t/log +grep -F .Lbar $t/log + +./mold -o $t/exe $t/a.o --discard-locals +readelf --symbols $t/exe > $t/log +grep -F _start $t/log +grep -F foo $t/log +not grep -F .Lbar $t/log + +./mold -o $t/exe $t/a.o --discard-all +readelf --symbols $t/exe > $t/log +grep -F _start $t/log +not grep -F foo $t/log +not grep -F .Lbar $t/log + +./mold -o $t/exe $t/a.o --strip-all +readelf --symbols $t/exe > $t/log +not grep -F _start $t/log +not grep -F foo $t/log +not grep -F .Lbar $t/log diff --git a/test/dso-undef.sh b/test/dso-undef.sh new file mode 100755 index 000000000..1845e78a0 --- /dev/null +++ b/test/dso-undef.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void init() { + printf("init\n"); +} + +void fini() { + printf("fini\n"); +} + +void keep() {} +EOF + +$CC -B. -o $t/c.so -shared $t/b.o +$CC -B. -o $t/d.so -shared $t/b.o -Wl,-init,init -Wl,-fini,fini + +$CC -B. -o $t/exe1 $t/a.o $t/c.so +$CC -B. -o $t/exe2 $t/a.o $t/d.so + +$QEMU $t/exe1 > $t/log1 +$QEMU $t/exe2 > $t/log2 + +not grep init $t/log1 +not grep fini $t/log1 + +grep init $t/log2 +grep fini $t/log2 diff --git a/test/dt-needed.sh b/test/dt-needed.sh new file mode 100755 index 000000000..6f4921f9e --- /dev/null +++ b/test/dt-needed.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log + +grep 'duplicate symbol:.* foo' $t/log diff --git a/test/duplicate-error-gc-sections.sh b/test/duplicate-error-gc-sections.sh new file mode 100755 index 000000000..a7f807eea --- /dev/null +++ b/test/duplicate-error-gc-sections.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +nm mold | grep '__tsan_init' && skip + +cat < $t/log +grep -F '(DEBUG)' $t/log + +cat < $t/log +grep -w foo $t/log +not grep -w bar $t/log diff --git a/test/dynamic-list.sh b/test/dynamic-list.sh new file mode 100755 index 000000000..6e084d116 --- /dev/null +++ b/test/dynamic-list.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +not grep ' foo' $t/log +not grep ' bar' $t/log + +cat < $t/dyn +{ foo; bar; }; +EOF + +$CC -B. -o $t/exe1 $t/a.o -Wl,-dynamic-list=$t/dyn + +readelf --dyn-syms $t/exe1 > $t/log1 +grep ' foo' $t/log1 +grep ' bar' $t/log1 + +$CC -B. -o $t/exe2 $t/a.o -Wl,--export-dynamic-symbol-list=$t/dyn + +readelf --dyn-syms $t/exe2 > $t/log2 +grep ' foo' $t/log2 +grep ' bar' $t/log2 + +$CC -B. -o $t/exe3 $t/a.o -Wl,--export-dynamic-symbol=foo,--export-dynamic-symbol=bar + +readelf --dyn-syms $t/exe3 > $t/log3 +grep ' foo' $t/log3 +grep ' bar' $t/log3 diff --git a/test/dynamic-list2.sh b/test/dynamic-list2.sh new file mode 100755 index 000000000..d215dd87e --- /dev/null +++ b/test/dynamic-list2.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +not grep ' foo' $t/log +not grep ' bar' $t/log + +cat < $t/dyn +{ foo; extern "C++" { "baz(int)"; }; }; +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o -Wl,-dynamic-list=$t/dyn + +readelf --dyn-syms $t/exe1 > $t/log1 +grep ' foo' $t/log1 +not grep ' bar' $t/log1 +grep ' _Z3bazi' $t/log1 + +$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,--export-dynamic-symbol-list=$t/dyn + +readelf --dyn-syms $t/exe2 > $t/log2 +grep ' foo' $t/log2 +not grep ' bar' $t/log2 +grep ' _Z3bazi' $t/log2 diff --git a/test/dynamic-list3.sh b/test/dynamic-list3.sh new file mode 100755 index 000000000..e0613b30f --- /dev/null +++ b/test/dynamic-list3.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <<'EOF' > $t/dyn +{ + xyz; + foo*bar*[abc]x; +}; +EOF + +cat < $t/log1 +grep ' xyz' $t/log1 +not grep ' foobarzx' $t/log1 +grep ' foobarcx' $t/log1 +grep ' foo123bar456bx' $t/log1 +not grep ' foo123bar456c' $t/log1 +not grep ' foo123bar456x' $t/log1 + +$CC -B. -Wl,--export-dynamic-symbol-list=$t/dyn -o $t/exe2 $t/b.o + +readelf --dyn-syms $t/exe2 > $t/log2 +grep ' xyz' $t/log2 +not grep ' foobarzx' $t/log2 +grep ' foobarcx' $t/log2 +grep ' foo123bar456bx' $t/log2 +not grep ' foo123bar456c' $t/log2 +not grep ' foo123bar456x' $t/log2 diff --git a/test/dynamic-list4.sh b/test/dynamic-list4.sh new file mode 100755 index 000000000..061e05537 --- /dev/null +++ b/test/dynamic-list4.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void foo() { printf("foo1 "); } +void bar() { printf("bar1 "); } +void baz() { printf("baz1 "); } + +void print() { + foo(); + bar(); + baz(); + printf("\n"); +} +EOF + +cat < $t/dyn +{ foo; bar; }; +EOF + +$CC -B. -shared -o $t/b.so $t/a.o -Wl,--dynamic-list=$t/dyn + +cat < +void foo() { printf("foo2 "); } +void bar() { printf("bar2 "); } +void baz() { printf("baz2 "); } +EOF + +$CC -B. -shared -o $t/d.so $t/c.o + +cat < +void print(); +int main() { print(); } +EOF + +$CC -B. -o $t/exe1 $t/e.o -Wl,-push-state,-no-as-needed $t/b.so -Wl,-pop-state +$QEMU $t/exe1 | grep 'foo1 bar1 baz1' + +$CC -B. -o $t/exe2 $t/e.o -Wl,-push-state,-no-as-needed $t/d.so $t/b.so -Wl,-pop-state +$QEMU $t/exe2 | grep 'foo2 bar2 baz1' diff --git a/test/dynamic.sh b/test/dynamic.sh new file mode 100755 index 000000000..d111e20be --- /dev/null +++ b/test/dynamic.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo '.globl main; main:' | $CC -o $t/a.o -c -x assembler - + +$CC -B. -o $t/exe $t/a.o + +readelf --dynamic $t/exe | grep -E 'Shared library:.*\blibc\b' + +readelf -W --dyn-syms --use-dynamic $t/exe | + grep -E 'FUNC\s+GLOBAL\s+DEFAULT.*UND\s+__libc_start' diff --git a/test/eh-frame-empty-fde.sh b/test/eh-frame-empty-fde.sh new file mode 100755 index 000000000..907e8a4f1 --- /dev/null +++ b/test/eh-frame-empty-fde.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Compilers may emit an FDE covering a zero-length address range, e.g. +# for a function fragment that GCC's hot/cold splitting has entirely +# optimized away. Such an FDE begins at the same address as the FDE of +# the next real function fragment and can shadow it in .eh_frame_hdr's +# binary search table, so that the unwinder misses an existing exception +# handler and aborts the process. +# +# The zero-length FDEs below collapse onto the beginning of run(), which +# catches an exception and throws another one. Looking up run()'s frame +# during unwinding may then find a zero-length FDE instead of the real +# one, in which case an exception cannot propagate to main() and the +# process aborts. Which entry a search returns among the ones with the +# same address depends on how the sort orders them, so we create enough +# zero-length FDEs that a search practically never finds the real one. +# run() is alone in its file so that its fragments are the first ones +# after the zero-length fragments in the output. + +cat < + +void maybe_throw(int x); +void rethrower(); + +__attribute__((noinline)) int run(int x) { + try { + maybe_throw(x); + return 0; + } catch (const std::exception &) { + rethrower(); + return 1; + } +} +EOF + +cat < + +__attribute__((noinline)) void maybe_throw(int x) { + if (x < 0) + throw std::runtime_error("first"); +} + +__attribute__((noinline)) void rethrower() { + throw std::runtime_error("second"); +} +EOF + +cat < +#include + +int run(int x); + +int main() { + try { + run(-1); + } catch (const std::exception &e) { + std::printf("outer caught %s\n", e.what()); + } +} +EOF + +$CXX -B. -o $t/exe $t/a.o $t/b.o $t/c.o $t/d.o +$QEMU $t/exe | grep 'outer caught' diff --git a/test/emit-relocs-cpp.sh b/test/emit-relocs-cpp.sh new file mode 100755 index 000000000..97a3e4159 --- /dev/null +++ b/test/emit-relocs-cpp.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[[ $MACHINE = arm* ]] && skip + +cat < +int main() { printf("Hello world\n"); } +EOF + +$CXX -B. -o $t/exe $t/a.o -Wl,-emit-relocs +$QEMU $t/exe | grep 'Hello world' + +readelf -SW $t/exe | grep -E 'rela?\.text' +readelf -SW $t/exe | grep -E 'rela?\.eh_frame' diff --git a/test/emit-relocs-dead-sections.sh b/test/emit-relocs-dead-sections.sh new file mode 100755 index 000000000..d92a4bce2 --- /dev/null +++ b/test/emit-relocs-dead-sections.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = m68k ] && skip +[ $MACHINE = sh4 ] && skip + +cat < +struct Foo { + Foo() { std::cout << "Hello world\n"; } +}; + +Foo x; +EOF + +cat < +struct Foo { + Foo() { std::cout << "Hello world\n"; } +}; + +Foo y; +int main() {} +EOF + +$CXX -B. -o $t/exe $t/a.o $t/b.o -Wl,-emit-relocs +$QEMU $t/exe | grep 'Hello world' diff --git a/test/emit-relocs-local-sym.sh b/test/emit-relocs-local-sym.sh new file mode 100755 index 000000000..3b65156ca --- /dev/null +++ b/test/emit-relocs-local-sym.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# A relocation against a named local symbol must refer to the local's entry +# in the output .symtab with --emit-relocs. We once computed its symbol index +# against the global part of the symbol table, so the emitted relocation +# referred to an arbitrary unrelated global symbol. +# https://github.com/rui314/mold/issues/1631 +# +# Assemblers usually redirect relocations against local symbols to section +# symbols, so we assemble `str` as a global symbol and localize it with +# objcopy afterwards. + +echo | $CC -c -o $t/a.o -xc - +if readelf -h $t/a.o | grep -q ELF64; then + reloc=BFD_RELOC_64 +else + reloc=BFD_RELOC_32 +fi + +cat < +int main() { + puts("Hello world"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,-emit-relocs +$QEMU $t/exe | grep 'Hello world' + +readelf -S $t/exe | grep -E 'rela?\.text' diff --git a/test/empty-default-symver.sh b/test/empty-default-symver.sh new file mode 100755 index 000000000..0227ae5b1 --- /dev/null +++ b/test/empty-default-symver.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# A symbol exported with the default version but an empty (base) version +# name, i.e. `foo@@`, must be exported as a plain unversioned global even +# when a version script hides everything else with `local: *`. The explicit +# `@@` is an explicit export that takes precedence over the `local: *` +# wildcard, just as GNU ld does. Ceph's librados depends on this: its +# LIBRADOS_C_API_BASE_DEFAULT macro emits `.symver _fn, fn@@`. + +cat < $t/b.ver + +$CC -B. -shared -o $t/c.so $t/a.o \ + -Wl,--version-script=$t/b.ver -Wl,--exclude-libs,ALL + +readelf --dyn-syms $t/c.so > $t/log +grep -Eq ' FUNC +GLOBAL +DEFAULT +.*[0-9]+ foo$' $t/log diff --git a/test/empty-dso.sh b/test/empty-dso.sh new file mode 100755 index 000000000..aeaa4b622 --- /dev/null +++ b/test/empty-dso.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +rm -f $t/b.script +touch $t/b.script + +$CC -B. -o $t/exe $t/a.o -Wl,--version-script,$t/b.script +$QEMU $t/exe | grep 'Hello world' diff --git a/test/empty-input.sh b/test/empty-input.sh new file mode 100755 index 000000000..67b97d3b0 --- /dev/null +++ b/test/empty-input.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +rm -f $t/a.o +touch $t/a.o +not $CC -B. -o $t/exe $t/a.o &> $t/log +grep 'unknown file type' $t/log diff --git a/test/empty-version.sh b/test/empty-version.sh new file mode 100755 index 000000000..db6cb02eb --- /dev/null +++ b/test/empty-version.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep "Entry point address:.*0x1000$" $t/log + +$CC -B. -o $t/exe2 -Wl,-e,bar $t/a.o $t/b.o +readelf -e $t/exe2 > $t/log +grep "Entry point address:.*0x2000$" $t/log diff --git a/test/exception-multiple-ehframe.sh b/test/exception-multiple-ehframe.sh new file mode 100755 index 000000000..4652b6e19 --- /dev/null +++ b/test/exception-multiple-ehframe.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = sh4aeb ] && skip +nm mold | grep '__tsan_init' && skip +command -v perl > /dev/null || skip + +[ $MACHINE = sh4 ] && skip + +cat < + +int foo(); +int bar(); + +int main() { + printf("%d %d\n", foo(), bar()); +} +EOF + +$CXX -B. -o $t/exe1 $t/d.o $t/c.o +$QEMU $t/exe1 | grep '^1 3$' diff --git a/test/exception.sh b/test/exception.sh new file mode 100755 index 000000000..b10471625 --- /dev/null +++ b/test/exception.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +static= +test_cxxflags -static && static=-static + +# I don't know why, but we need -pthread on m68k +static="$static -pthread" + +cat < $t/a.cc +int main() { + try { + throw 0; + } catch (int x) { + return x; + } + return 1; +} +EOF + +$CXX -c -o $t/b.o $t/a.cc -fPIC +$CXX -c -o $t/c.o $t/a.cc -fno-PIC + +$CXX -B. -o $t/exe1 $t/b.o $static +$QEMU $t/exe1 + +$CXX -B. -o $t/exe2 $t/c.o -no-pie $static +$QEMU $t/exe2 + +$CXX -B. -o $t/exe3 $t/b.o -pie +$QEMU $t/exe3 + +$CXX -B. -o $t/exe4 $t/c.o -no-pie +$QEMU $t/exe4 + +$CXX -B. -o $t/exe5 $t/b.o -pie -Wl,--gc-sections +$QEMU $t/exe5 + +$CXX -B. -o $t/exe6 $t/c.o -no-pie $static -Wl,--gc-sections +$QEMU $t/exe6 + +if [ $MACHINE = x86_64 ]; then + $CXX -c -o $t/d.o $t/a.cc -mcmodel=large -fPIC + + $CXX -B. -o $t/exe7 $t/d.o $static + $QEMU $t/exe7 + + $CXX -B. -o $t/exe8 $t/d.o -pie + $QEMU $t/exe8 +fi + +if [ $MACHINE = x86_64 -o $MACHINE = aarch64 ]; then + $CXX -c -o $t/e.o $t/a.cc -mcmodel=large -fno-PIC + + $CXX -B. -o $t/exe9 $t/e.o -no-pie $static + $QEMU $t/exe9 + + $CXX -B. -o $t/exe10 $t/e.o -no-pie + $QEMU $t/exe10 +fi + +# riscv64-linux-gnu-strip crashes for some reason +if [ $MACHINE != riscv32 ]; then + $CXX -B. -o $t/exe11 $t/b.o -pie + $STRIP $t/exe11 + $QEMU $t/exe11 + + $CXX -B. -o $t/exe12 $t/c.o -no-pie + $STRIP $t/exe12 + $QEMU $t/exe12 +fi diff --git a/test/exclude-libs.sh b/test/exclude-libs.sh new file mode 100755 index 000000000..507d09258 --- /dev/null +++ b/test/exclude-libs.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -F foo $t/log +grep -F bar $t/log +grep -F baz $t/log + +$CC -B. -shared -o $t/f.so $t/e.o $t/c.a $t/d.a -Wl,-exclude-libs=c.a +readelf --dyn-syms $t/f.so > $t/log +not grep -F foo $t/log +grep -F bar $t/log +grep -F baz $t/log + +$CC -B. -shared -o $t/f.so $t/e.o $t/c.a $t/d.a -Wl,-exclude-libs=c.a -Wl,-exclude-libs=d.a +readelf --dyn-syms $t/f.so > $t/log +not grep -F foo $t/log +not grep -F bar $t/log +grep -F baz $t/log + +$CC -B. -shared -o $t/f.so $t/e.o $t/c.a $t/d.a -Wl,-exclude-libs=c.a:d.a +readelf --dyn-syms $t/f.so > $t/log +not grep -F foo $t/log +not grep -F bar $t/log +grep -F baz $t/log + +$CC -B. -shared -o $t/f.so $t/e.o $t/c.a $t/d.a -Wl,-exclude-libs=ALL +readelf --dyn-syms $t/f.so > $t/log +not grep -F foo $t/log +not grep -F bar $t/log +grep -F baz $t/log diff --git a/test/exclude-libs2.sh b/test/exclude-libs2.sh new file mode 100755 index 000000000..97430604d --- /dev/null +++ b/test/exclude-libs2.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,--execute-only +$QEMU $t/exe | grep 'Hello world' +readelf -W --segments $t/exe | grep -E 'LOAD\s.*[0-9a-f] E 0x' diff --git a/test/export-dynamic.sh b/test/export-dynamic.sh new file mode 100755 index 000000000..6311a40b7 --- /dev/null +++ b/test/export-dynamic.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -E 'NOTYPE\s+GLOBAL DEFAULT\s+[0-9]+ bar' $t/log +grep -E 'NOTYPE\s+GLOBAL DEFAULT\s+[0-9]+ _start' $t/log diff --git a/test/export-from-exe.sh b/test/export-from-exe.sh new file mode 100755 index 000000000..c9332a953 --- /dev/null +++ b/test/export-from-exe.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < /dev/null + +not $CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-warn-common -Wl,-fatal-warnings 2> /dev/null diff --git a/test/filler.sh b/test/filler.sh new file mode 100755 index 000000000..687e0e8f4 --- /dev/null +++ b/test/filler.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static || skip + +cat < + +__attribute__((aligned(512))) +char hello[] = "Hello"; + +__attribute__((aligned(512))) +char world[] = "world"; + +int main() { + printf("%s %s\n", hello, world); +} +EOF + +$CC -B. -static -Wl,--filler,0xfe -o $t/exe1 $t/a.o +sed -i -e 's/--filler 0xfe/--filler 0x00/' $t/exe1 +od -x $t/exe1 > $t/txt1 + +$CC -B. -static -Wl,--filler,0x00 -o $t/exe2 $t/a.o +od -x $t/exe2 > $t/txt2 + +diff -q $t/txt1 $t/txt2 diff --git a/test/filter.sh b/test/filter.sh new file mode 100755 index 000000000..94f06c0f4 --- /dev/null +++ b/test/filter.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +grep -F 'Filter library: [foo]' $t/log +grep -F 'Filter library: [bar]' $t/log diff --git a/test/func-addr.sh b/test/func-addr.sh new file mode 100755 index 000000000..f8731e530 --- /dev/null +++ b/test/func-addr.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +typedef void Func(); + +void fn(); +Func *const ptr = fn; + +int main() { + printf("%d\n", fn == ptr); +} +EOF + +$CC -B. -o $t/exe -no-pie $t/b.o $t/a.so +$QEMU $t/exe | grep 1 diff --git a/test/gc-sections-start-stop.sh b/test/gc-sections-start-stop.sh new file mode 100755 index 000000000..a28abc9c6 --- /dev/null +++ b/test/gc-sections-start-stop.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# A section whose name is a valid C identifier should be GC'd if +# neither its contents nor its __start_/__stop_ marker symbols are +# referenced from a live section. +cat < $t/log + +not grep dead_data $t/log +grep live_data $t/log +grep __start_bar $t/log +grep __stop_bar $t/log diff --git a/test/gc-sections.sh b/test/gc-sections.sh new file mode 100755 index 000000000..a3ea256a4 --- /dev/null +++ b/test/gc-sections.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int two() { return 2; } + +int live_var1 = 1; +int live_var2 = two(); +int dead_var1 = 3; +int dead_var2 = 4; + +void live_fn1() {} +void live_fn2() { live_fn1(); } +void dead_fn1() {} +void dead_fn2() { dead_fn1(); } + +int main() { + printf("%d %d\n", live_var1, live_var2); + live_fn2(); +} +EOF + +$CXX -B. -o $t/exe1 $t/a.o +readelf --symbols $t/exe1 > $t/log1 +$QEMU $t/exe1 | grep '1 2' + +grep live_fn1 $t/log1 +grep live_fn2 $t/log1 +grep dead_fn1 $t/log1 +grep dead_fn2 $t/log1 +grep live_var1 $t/log1 +grep live_var2 $t/log1 +grep dead_var1 $t/log1 +grep dead_var2 $t/log1 + +$CXX -B. -o $t/exe2 $t/a.o -Wl,-gc-sections +readelf --symbols $t/exe2 > $t/log2 +$QEMU $t/exe2 | grep '1 2' + +grep live_fn1 $t/log2 +grep live_fn2 $t/log2 +not grep dead_fn1 $t/log2 +not grep dead_fn2 $t/log2 +grep live_var1 $t/log2 +grep live_var2 $t/log2 +not grep dead_var1 $t/log2 +not grep dead_var2 $t/log2 diff --git a/test/gdb-index-compress-output.sh b/test/gdb-index-compress-output.sh new file mode 100755 index 000000000..ba2fe6737 --- /dev/null +++ b/test/gdb-index-compress-output.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip + +cat < + +void trap() {} + +static void hello() { + printf("Hello world\n"); + trap(); +} + +void greet() { + hello(); +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o -Wl,--gdb-index -Wl,--compress-debug-sections=zlib-gabi +readelf -WS $t/b.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log + +grep 'hello () at .*:7' $t/log +grep 'greet () at .*:11' $t/log +grep 'main () at .*:4' $t/log diff --git a/test/gdb-index-deterministic.sh b/test/gdb-index-deterministic.sh new file mode 100755 index 000000000..7792f4ab9 --- /dev/null +++ b/test/gdb-index-deterministic.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +test_cflags -gdwarf-5 -g || skip + +# Generate many translation units that all reference common types and +# names so the per-name CU index list contains entries from multiple +# threads. This stresses the parallel writer in --gdb-index that fills +# in (type, cu) tuples for each name, which must produce a deterministic +# output regardless of thread scheduling. +objs= +for i in $(seq 1 24); do + cat < $t/u$i.c +#include +struct s$i { int x; }; +int fn$i(struct s$i *p) { return printf("%d\n", p->x); } +EOF + $CC -c -o $t/u$i.o $t/u$i.c -fPIC -g -ggnu-pubnames + objs="$objs $t/u$i.o" +done + +$CC -B. -shared -o $t/out1.so -Wl,--gdb-index $objs +$CC -B. -shared -o $t/out2.so -Wl,--gdb-index $objs +$CC -B. -shared -o $t/out3.so -Wl,--gdb-index $objs +$CC -B. -shared -o $t/out4.so -Wl,--gdb-index $objs +$CC -B. -shared -o $t/out5.so -Wl,--gdb-index $objs + +cmp $t/out1.so $t/out2.so +cmp $t/out1.so $t/out3.so +cmp $t/out1.so $t/out4.so +cmp $t/out1.so $t/out5.so diff --git a/test/gdb-index-dwarf2.sh b/test/gdb-index-dwarf2.sh new file mode 100755 index 000000000..7d5bc6691 --- /dev/null +++ b/test/gdb-index-dwarf2.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +test_cflags -gdwarf-2 -g || skip + +cat < + +void trap() {} + +void hello2() { + printf("Hello world\n"); + trap(); +} +EOF + +$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index +readelf -WS $t/c.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log + +grep 'hello2 () at .*:7' $t/log +grep 'hello () at .*:4' $t/log +grep 'greet () at .*:8' $t/log +grep 'main () at .*:4' $t/log diff --git a/test/gdb-index-dwarf3.sh b/test/gdb-index-dwarf3.sh new file mode 100755 index 000000000..b9684ef7b --- /dev/null +++ b/test/gdb-index-dwarf3.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +test_cflags -gdwarf-3 || skip + +cat < + +void trap() {} + +void hello2() { + printf("Hello world\n"); + trap(); +} +EOF + +$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index +readelf -WS $t/c.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log + +grep 'hello2 () at .*:7' $t/log +grep 'hello () at .*:4' $t/log +grep 'greet () at .*:8' $t/log +grep 'main () at .*:4' $t/log diff --git a/test/gdb-index-dwarf4.sh b/test/gdb-index-dwarf4.sh new file mode 100755 index 000000000..d39488007 --- /dev/null +++ b/test/gdb-index-dwarf4.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +test_cflags -gdwarf-4 -g || skip + +cat < + +void trap() {} + +void hello2() { + printf("Hello world\n"); + trap(); +} +EOF + +$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index +readelf -WS $t/c.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log + +grep 'hello2 () at .*:7' $t/log +grep 'hello () at .*:4' $t/log +grep 'greet () at .*:8' $t/log +grep 'main () at .*:4' $t/log diff --git a/test/gdb-index-dwarf5-types.sh b/test/gdb-index-dwarf5-types.sh new file mode 100755 index 000000000..7a9e9180b --- /dev/null +++ b/test/gdb-index-dwarf5-types.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# OneTBB isn't tsan-clean +nm mold | grep '__tsan_init' && skip + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +command -v ${GCC%% *} >& /dev/null || skip + +# Regression test for https://github.com/rui314/mold/issues/1634. +cat < $t/shared.h +struct Point { int x; int y; }; +struct Shape { struct Point a; struct Point b; double area; }; +int use_shape(struct Shape *s); +int make_shape(struct Shape *s); +EOF + +cat < $t/a.c +#include "shared.h" +int use_shape(struct Shape *s) { return s->a.x + s->b.y; } +EOF + +cat < $t/b.c +#include "shared.h" +int make_shape(struct Shape *s) { + s->a.x = 1; + s->b.y = 2; + s->area = 3.0; + return use_shape(s); +} +int main() { struct Shape s; return make_shape(&s); } +EOF + +$GCC -g -ggnu-pubnames -gdwarf-5 -fdebug-types-section -c $t/a.c -o $t/a.o || skip +$GCC -g -ggnu-pubnames -gdwarf-5 -fdebug-types-section -c $t/b.c -o $t/b.o + +# Clang accepts -fdebug-types-section but does not emit type units. +readelf --debug-dump=info $t/a.o | grep -F DW_UT_type || skip + +# Old GDB versions choke on DWARF 5 type units no matter how the program +# is linked; GDB 10 even crashes. Skip if GDB cannot debug an executable +# that doesn't contain a .gdb_index section. +$GCC -B. $t/a.o $t/b.o -o $t/exe-noindex +DEBUGINFOD_URLS= gdb $t/exe-noindex -nx -batch -ex 'ptype struct Shape' \ + -ex quit >& $t/log-noindex || skip +grep -Fq 'type = struct Shape {' $t/log-noindex || skip + +check_index() { + readelf --debug-dump=gdb_index $1 > $2 || true + grep -F 'Version 9' $2 || return 1 + + # Binutils 2.38 predates version 9 and cannot decode the tables. Newer + # readelf versions should show both type units. + if grep -q '^TU table:' $2; then + grep -A4 '^TU table:' $2 | grep -E '^\[ *0\]' || return 1 + grep -A4 '^TU table:' $2 | grep -E '^\[ *1\]' || return 1 + fi +} + +$GCC -B. -Wl,--gdb-index $t/a.o $t/b.o -o $t/exe +check_index $t/exe $t/index + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'ptype struct Shape' -ex quit >& $t/log +grep -F 'type = struct Shape {' $t/log +grep -F 'struct Point a;' $t/log +grep -F 'double area;' $t/log + +# A partial link combines all pubnames sets and their relocations into one +# section. Make sure each set is still matched to its CU or TU contribution. +./mold -r -o $t/ab.o $t/a.o $t/b.o +$GCC -B. -Wl,--gdb-index $t/ab.o -o $t/exe-r +check_index $t/exe-r $t/index-r + +# Exercise the 64-bit DWARF type-unit header when the compiler supports it. +if $GCC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -fdebug-types-section \ + -c $t/a.c -o $t/a64.o 2> /dev/null; then + $GCC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -fdebug-types-section \ + -c $t/b.c -o $t/b64.o + $GCC -B. -Wl,--gdb-index $t/a64.o $t/b64.o -o $t/exe64 + check_index $t/exe64 $t/index64 +fi diff --git a/test/gdb-index-dwarf5.sh b/test/gdb-index-dwarf5.sh new file mode 100755 index 000000000..ae524ad69 --- /dev/null +++ b/test/gdb-index-dwarf5.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +test_cflags -gdwarf-5 -g || skip + +cat < $t/a.c +void fn3(); + +static void fn2() { + fn3(); +} + +void fn1() { + fn2(); +} +EOF + +cat < $t/b.c +void fn5(); + +static void fn4() { + fn5(); +} + +void fn3() { + fn4(); +} +EOF + +cat < $t/c.c +void fn7(); + +static void fn6() { + fn7(); +} + +void fn5() { + fn6(); +} +EOF + +cat < $t/d.c +#include +void trap() {} + +static void fn8() { + printf("Hello world\n"); + trap(); +} + +void fn7() { + fn8(); +} +EOF + +$CC -c -o $t/a.o $t/a.c -fPIC -g -ggnu-pubnames -gdwarf-5 -ffunction-sections +$CC -c -o $t/b.o $t/b.c -fPIC -g -ggnu-pubnames -gdwarf-4 -ffunction-sections +$CC -c -o $t/c.o $t/c.c -fPIC -g -ggnu-pubnames -gdwarf-5 +$CC -c -o $t/d.o $t/d.c -fPIC -g -ggnu-pubnames -gdwarf-5 -ffunction-sections + +$CC -B. -shared -o $t/e.so $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--gdb-index +readelf -WS $t/e.so 2> /dev/null | grep -F .gdb_index +readelf --debug-dump=gdb_index $t/e.so 2> /dev/null | grep 'fn1: .* \[global, function\]' +readelf --debug-dump=gdb_index $t/e.so 2> /dev/null | grep 'char: .* \[static, type\]' + +cat < /dev/null | grep -F .gdb_index +readelf --debug-dump=gdb_index $t/exe 2> /dev/null | grep 'main: .* \[global, function\]' + +$QEMU $t/exe | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log + +grep 'fn8 () at .*/d.c:6' $t/log +grep 'fn7 () at .*/d.c:10' $t/log +grep 'fn6 () at .*/c.c:4' $t/log +grep 'fn5 () at .*/c.c:8' $t/log +grep 'fn4 () at .*/b.c:4' $t/log +grep 'fn3 () at .*/b.c:8' $t/log +grep 'fn2 () at .*/a.c:4' $t/log +grep 'fn1 () at .*/a.c:8' $t/log diff --git a/test/gdb-index-dwarf64.sh b/test/gdb-index-dwarf64.sh new file mode 100755 index 000000000..28953c56f --- /dev/null +++ b/test/gdb-index-dwarf64.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip + +test_cflags -gdwarf-5 -g -gdwarf64 || skip + +cat < $t/a.c +void fn3(); + +static void fn2() { + fn3(); +} + +void fn1() { + fn2(); +} +EOF + +cat < $t/b.c +void fn5(); + +static void fn4() { + fn5(); +} + +void fn3() { + fn4(); +} +EOF + +cat < $t/c.c +void fn7(); + +static void fn6() { + fn7(); +} + +void fn5() { + fn6(); +} +EOF + +cat < $t/d.c +#include +void trap() {} + +static void fn8() { + printf("Hello world\n"); + trap(); +} + +void fn7() { + fn8(); +} +EOF + +$CC -c -o $t/a.o $t/a.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -ffunction-sections +$CC -c -o $t/b.o $t/b.c -fPIC -g -ggnu-pubnames -gdwarf-4 -gdwarf64 -ffunction-sections +$CC -c -o $t/c.o $t/c.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 +$CC -c -o $t/d.o $t/d.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -ffunction-sections + +$CC -B. -shared -o $t/e.so $t/a.o $t/b.o $t/c.o $t/d.o + +$CC -B. -shared -o $t/f.so $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--gdb-index +readelf -WS $t/f.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe2 | grep 'Hello world' + +DEBUGINFOD_URLS= gdb $t/exe2 -nx -batch -ex 'b main' -ex r -ex 'b trap' \ + -ex c -ex bt -ex quit >& $t/log2 + +grep 'fn8 () at .*/d.c:6' $t/log2 +grep 'fn7 () at .*/d.c:10' $t/log2 +grep 'fn6 () at .*/c.c:4' $t/log2 +grep 'fn5 () at .*/c.c:8' $t/log2 +grep 'fn4 () at .*/b.c:4' $t/log2 +grep 'fn3 () at .*/b.c:8' $t/log2 +grep 'fn2 () at .*/a.c:4' $t/log2 +grep 'fn1 () at .*/a.c:8' $t/log2 diff --git a/test/gdb-index-empty.sh b/test/gdb-index-empty.sh new file mode 100755 index 000000000..ad3f9e78e --- /dev/null +++ b/test/gdb-index-empty.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo 'void _start() {}' | $CC -c -o $t/a.o -xc - +./mold -o $t/exe $t/a.o -gdb-index +readelf -WS $t/exe | not grep -F .gdb_index diff --git a/test/gdb-index-rnglistx.sh b/test/gdb-index-rnglistx.sh new file mode 100755 index 000000000..16a82f3ae --- /dev/null +++ b/test/gdb-index-rnglistx.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Regression test for https://github.com/rui314/mold/issues/1578. +# When DWARF 5 .debug_rnglists' offset table contains entries for both +# the CU root and child DIEs, mold must read only the entry indexed by +# the CU's DW_FORM_rnglistx. Otherwise child DIE ranges leak into the +# .gdb_index address table, overlapping the CU's enclosing range. + +[ $MACHINE = $(uname -m) ] || skip +echo 'int main() {}' | clang++ -gdwarf-5 -g -o /dev/null -xc++ - 2> /dev/null || skip + +cat < $t/a.cc +void may_throw(int x) { + if (x < 0) + throw x; +} + +struct Guard { + int v; + ~Guard() { may_throw(v); } +}; + +int main() { + { + Guard g{1}; + may_throw(2); + } +} +EOF + +clang++ -B. -O0 -ggdb3 -ggnu-pubnames -gdwarf-5 \ + -Wl,--gdb-index $t/a.cc -o $t/exe + +readelf --debug-dump=gdb_index $t/exe | + sed -n '/^Address table:/,/^$/p' | + grep -E '^[0-9a-f]+ [0-9a-f]+' | + sort | + awk 'NR > 1 && $1 < prev { exit 1 } { prev = $2 }' diff --git a/test/gdb-index-split-dwarf.sh b/test/gdb-index-split-dwarf.sh new file mode 100755 index 000000000..fce750f9c --- /dev/null +++ b/test/gdb-index-split-dwarf.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +on_qemu && skip +[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip +command -v gdb >& /dev/null || skip +test_cflags -gdwarf-5 -g || skip + +cat < $t/a.c +void fn3(); + +static void fn2() { + fn3(); +} + +void fn1() { + fn2(); +} +EOF + +cat < $t/b.c +void fn5(); + +static void fn4() { + fn5(); +} + +void fn3() { + fn4(); +} +EOF + +cat < $t/c.c +void fn7(); + +static void fn6() { + fn7(); +} + +void fn5() { + fn6(); +} +EOF + +cat < $t/d.c +#include +void trap() {} + +static void fn8() { + printf("Hello world\n"); + trap(); +} + +void fn7() { + fn8(); +} +EOF + +$CC -c -o $t/a.o $t/a.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gsplit-dwarf +$CC -c -o $t/b.o $t/b.c -fPIC -g -ggnu-pubnames -gdwarf-4 -gsplit-dwarf +$CC -c -o $t/c.o $t/c.c -fPIC -g -ggnu-pubnames -gdwarf-5 +$CC -c -o $t/d.o $t/d.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gsplit-dwarf + +$CC -B. -shared -o $t/e.so $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--gdb-index +readelf -WS $t/e.so 2> /dev/null | grep -F .gdb_index + +cat < /dev/null | grep -F .gdb_index + +$QEMU $t/exe | grep 'Hello world' diff --git a/test/glibc-2.22-bug.sh b/test/glibc-2.22-bug.sh new file mode 100755 index 000000000..af4117bbb --- /dev/null +++ b/test/glibc-2.22-bug.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + + +# glibc 2.22 or prior have a bug that ld-linux.so.2 crashes on dlopen() +# if .rela.dyn and .rela.plt are not contiguous in a given DSO. +# This test verifies that these sections are contiguous in mold's output. + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/b.so -shared $t/a.o +readelf -W --sections $t/b.so | grep -E -A1 '\.rela?\.dyn' | + grep -E '\.rela?\.plt' diff --git a/test/global-offset-table.sh b/test/global-offset-table.sh new file mode 100755 index 000000000..7221dc7f6 --- /dev/null +++ b/test/global-offset-table.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = ppc64le ] && skip + +cat < + +extern char foo; + +int main() { + printf("%lx\n", (unsigned long)&foo); +} +EOF + +$CC -B. -no-pie -o $t/exe $t/a.o -Wl,-defsym=foo=_GLOBAL_OFFSET_TABLE_ + +$QEMU $t/exe > /dev/null +GOT_ADDR=$($QEMU $t/exe) + +# _GLOBAL_OFFSET_TABLE_ refers the end of .got only on x86. +# We assume .got is followed by .gotplt. +if [ $MACHINE = x86_64 -o $MACHINE = i686 ]; then + readelf -WS $t/exe | grep "\.got\.plt .*$GOT_ADDR " +else + readelf -WS $t/exe | grep "\.got .*$GOT_ADDR " +fi diff --git a/test/gnu-hash.sh b/test/gnu-hash.sh new file mode 100755 index 000000000..9c870f33a --- /dev/null +++ b/test/gnu-hash.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < /dev/null + __attribute__((retain)) int foo() {} +int bar() {} +int main() {} +EOF + +# Older versions of GCC does not support __attribute__((retain)) +readelf -WS $t/a.o | grep '\.text\.foo.*AXR' || skip + +$CC -B. -o $t/exe $t/a.o -Wl,-gc-sections +nm $t/exe > $t/log +grep foo $t/log +not grep bar $t/log diff --git a/test/gnu-unique.sh b/test/gnu-unique.sh new file mode 100755 index 000000000..f5c7aff1e --- /dev/null +++ b/test/gnu-unique.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +command -v $GXX >& /dev/null || skip + +cat < + +inline int foo = 5; + +int main() { + printf("foo=%d\n", foo); +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o -no-pie +$QEMU $t/exe | grep 'foo=5' diff --git a/test/gnu-warning.sh b/test/gnu-warning.sh new file mode 100755 index 000000000..f7e1bc8c9 --- /dev/null +++ b/test/gnu-warning.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +__attribute__((section(".gnu.warning.foo"))) +static const char foo[] = "foo is deprecated"; + +__attribute__((section(".gnu.warning.bar"))) +const char bar[] = "bar is deprecated"; + +int main() { + printf("Hello world\n"); +} +EOF + +# Make sure that we do not copy .gnu.warning.* sections. +$CC -B. -o $t/exe $t/a.o -no-pie +$QEMU $t/exe | grep 'Hello world' diff --git a/test/hash-style-sysv.sh b/test/hash-style-sysv.sh new file mode 100755 index 000000000..498668341 --- /dev/null +++ b/test/hash-style-sysv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +void hello() { + printf("Hello world\n"); +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o -Wl,--hash-style=sysv + +cat < $t/log +not grep -F ' .hash' $t/log +not grep -F ' .gnu.hash' $t/log diff --git a/test/hello-dynamic.sh b/test/hello-dynamic.sh new file mode 100755 index 000000000..4151a55cb --- /dev/null +++ b/test/hello-dynamic.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -no-pie +$QEMU $t/exe | grep 'Hello world' diff --git a/test/hello-static.sh b/test/hello-static.sh new file mode 100755 index 000000000..ac7130248 --- /dev/null +++ b/test/hello-static.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -static +$QEMU $t/exe | grep 'Hello world' diff --git a/test/help.sh b/test/help.sh new file mode 100755 index 000000000..a0677391a --- /dev/null +++ b/test/help.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +./mold --help | grep Usage diff --git a/test/hidden-archive.sh b/test/hidden-archive.sh new file mode 100755 index 000000000..9773d09b4 --- /dev/null +++ b/test/hidden-archive.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log +not grep -w foo $t/log +grep -w bar $t/log diff --git a/test/hwasan.sh b/test/hwasan.sh new file mode 100755 index 000000000..faf765583 --- /dev/null +++ b/test/hwasan.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Verify that mold can link object files compiled with Clang's HWASAN +# instrumentation. On aarch64 such objects contain relocations like +# R_AARCH64_MOVW_PREL_G3 that mold previously rejected +# (https://github.com/rui314/mold/issues/1592). +[ "$CC" = cc ] || skip + +# Disable HWASAN's global-variable tagging. It is orthogonal to the +# relocations exercised here, and old Clang implements it by baking a tag +# into the high bits of a 32-bit relocation's addend, producing objects +# that no linker (mold, GNU ld or lld) can resolve. +cat < /dev/null || skip +#include +int main() { printf("Hello world\n"); } +EOF + +# Skip unless this toolchain can produce a HWASAN executable at all (e.g. +# the sanitizer runtime may not be installed). We probe with Clang's +# default linker so that a mold-specific failure below is still reported +# as a test failure rather than silently skipped. +clang -fsanitize=hwaddress -o $t/exe $t/a.o 2> /dev/null || skip + +clang -fsanitize=hwaddress -B. -o $t/exe $t/a.o diff --git a/test/icf-alignment-bug.sh b/test/icf-alignment-bug.sh new file mode 100755 index 000000000..dcc7a4860 --- /dev/null +++ b/test/icf-alignment-bug.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# This test is only applicable to x86 and x86_64 because they use the lowest +# bit of member function pointers for virtuality. +case $MACHINE in + x86_64 | i686) ;; + *) skip ;; +esac + +# We use assembly to define functions to have full control over alignment. +# We swap the order so that regular_func (alignment 1) comes first and +# is more likely to be chosen as the leader by ICF. +# We make pre_func size 2 to try to make regular_func start at an odd address. +cat < +#include + +struct Foo { + void member_func(); +}; + +extern "C" void regular_func(); + +int main() { + void (Foo::*ptr)() = &Foo::member_func; + union { + void (Foo::*mem_ptr)(); + struct { + void* ptr; + ptrdiff_t adj; + } parts; + } u; + u.mem_ptr = ptr; + + std::cout << "member_func address: " << u.parts.ptr << std::endl; + std::cout << "regular_func address: " << (void*)®ular_func << std::endl; + + if (u.parts.ptr != (void*)®ular_func) { + std::cerr << "FAIL: functions were not folded!" << std::endl; + return 2; + } + + if (reinterpret_cast(u.parts.ptr) & 1) { + std::cerr << "FAIL: member function pointer has odd address (treated as virtual)" << std::endl; + return 1; + } + return 0; +} +EOF + +$CXX -B. -o $t/exe $t/main.o $t/fns.o -Wl,--icf=all +$QEMU $t/exe diff --git a/test/icf-gcc-except-table.sh b/test/icf-gcc-except-table.sh new file mode 100755 index 000000000..e666f7ec8 --- /dev/null +++ b/test/icf-gcc-except-table.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = arm ] && skip +[ $MACHINE = armeb ] && skip +[[ $MACHINE = riscv* ]] && skip +[[ $MACHINE = loongarch* ]] && skip + +cat < + +template +struct X { + static void raise() { throw std::logic_error("foo"); } +}; + +int main() { + X().raise(); + X().raise(); +} +EOF + +$CXX -B. -o $t/exe $t/a.o -Wl,-icf=safe,--print-icf-sections |& + grep -E 'removing .*/a.o:\(.gcc_except_table' diff --git a/test/icf-lsda.sh b/test/icf-lsda.sh new file mode 100755 index 000000000..b3645e36e --- /dev/null +++ b/test/icf-lsda.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# On 32-bit ARM, exception tables are .ARM.extab sections tied to +# functions via .ARM.exidx instead of .eh_frame FDE references, and +# ICF does not yet take them into account. +[[ $MACHINE = arm* ]] && skip + +# f1 and f2 compile to identical machine code, but their exception +# tables catch different types. ICF must not merge them; if it does, +# f2 runs with f1's exception table and B escapes uncaught. + +cat < + +struct A {}; +struct B {}; + +int which; + +__attribute__((noinline)) void g() { + if (which) + throw B(); + throw A(); +} + +__attribute__((noinline)) void f1() { + try { + g(); + } catch (A &) { + } +} + +__attribute__((noinline)) void f2() { + try { + g(); + } catch (B &) { + } +} + +int main() { + which = 1; + f2(); + which = 0; + f1(); + std::puts("OK"); +} +EOF + +$CXX -B. -o $t/exe $t/a.o -Wl,--icf=all +$QEMU $t/exe | grep OK diff --git a/test/icf-preemption.sh b/test/icf-preemption.sh new file mode 100755 index 000000000..8c06cefe5 --- /dev/null +++ b/test/icf-preemption.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# On PPC64V1, function pointers refer function descriptors in .opd +# instead of directly referring .text section. +[ $MACHINE = ppc64 ] && skip + +cat < $t/out 2>&1 + +# Verify that caller_local and caller_global were NOT folded. +# If they were folded, one of them would be removed. +not grep -q "removing identical section.*caller_local" $t/out +not grep -q "removing identical section.*caller_global" $t/out + +# Also check with nm to be sure they have different addresses +nm $t/libtest.so | grep -E 'caller_local|caller_global' | awk '{print $1}' | uniq | wc -l | grep -q 2 diff --git a/test/icf-safe.sh b/test/icf-safe.sh new file mode 100755 index 000000000..a49a1b89a --- /dev/null +++ b/test/icf-safe.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# GCC 11 or older uses R_390_PLT32DBL to take an address of a function +# instead of R_390_PC32DBL +if [ $MACHINE = s390x ]; then + echo 'void *foo() { return foo; }' | $CC -c -o $t/a.o -xc - + readelf -r $t/a.o | grep R_390_PLT32DBL && skip +fi + +cat < + +int foo1(); +int foo2(); +int foo3(); + +int main() { + printf("%d %d\n", foo1 == foo2, foo2 == foo3); +} +EOF + +$CC -B. -o $t/exe1 -Wl,-icf=safe $t/a.o $t/b.o +$QEMU $t/exe1 | grep '^0 0$' + +cat < + +int bar() { + return 5; +} + +int foo1(int x) { + return bar() + x; +} + +int foo2(int x) { + return bar() + x; +} + +int foo3() { + bar(); + return 5; +} + +int main() { + printf("%d %d\n", (long)foo1 == (long)foo2, (long)foo1 == (long)foo3); + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o -Wl,-icf=all +$QEMU $t/exe | grep '1 0' diff --git a/test/ifunc-address-equality-exported.sh b/test/ifunc-address-equality-exported.sh new file mode 100755 index 000000000..9cdf87e3c --- /dev/null +++ b/test/ifunc-address-equality-exported.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Clang miscompiles the test code, so skip it if Clang. +# https://github.com/llvm/llvm-project/issues/111338 +$CC --version | grep clang && skip + +supports_ifunc || skip + +cat < + +typedef void Func(); + +__attribute__((ifunc("resolve_foo"))) void foo(void); +void real_foo(void) { printf("foo "); } +Func *resolve_foo() { return real_foo; } + +Func *get_foo(); + +int main() { + printf("%p %p\n", foo, get_foo()); + foo(); + printf("\n"); +} +EOF + +$CC -B. -o $t/exe $t/c.o $t/b.so -no-pie +$QEMU $t/exe | grep -E '^(\S+) \1' diff --git a/test/ifunc-address-equality.sh b/test/ifunc-address-equality.sh new file mode 100755 index 000000000..01c408563 --- /dev/null +++ b/test/ifunc-address-equality.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +typedef void Func(); + +__attribute__((ifunc("resolve_foo"))) void foo(void); +void real_foo(void) { printf("foo "); } +Func *resolve_foo() { return real_foo; } + +__attribute__((ifunc("resolve_bar"))) void bar(void); +void real_bar(void) { printf("bar "); } +Func *resolve_bar() { return real_bar; } +EOF + +cat < + +typedef void Func(); + +void foo(); +void bar(); +Func *get_foo(); +Func *get_bar(); + +int main() { + printf("%p %p %p %p\n", foo, get_foo(), bar, get_bar()); + foo(); + bar(); + printf("\n"); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -no-pie +$QEMU $t/exe1 | grep -E '^(\S+) \1 (\S+) \2' + +readelf --dynamic $t/exe1 | not grep TEXTREL diff --git a/test/ifunc-alias.sh b/test/ifunc-alias.sh new file mode 100755 index 000000000..1c1b6039d --- /dev/null +++ b/test/ifunc-alias.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +void foo() {} +int bar() __attribute__((ifunc("resolve_bar"))); +void *resolve_bar() { return foo; } +void *bar2 = bar; + +int main() { + printf("%p %p\n", bar, bar2); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o -pie +$QEMU $t/exe1 | grep -E '^(\S+) \1$' + +$CC -B. -o $t/exe2 $t/a.o -no-pie +$QEMU $t/exe2 | grep -E '^(\S+) \1$' diff --git a/test/ifunc-dlopen.sh b/test/ifunc-dlopen.sh new file mode 100755 index 000000000..527326430 --- /dev/null +++ b/test/ifunc-dlopen.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < +#include + +typedef void Func(); +void foo(void); + +int main() { + void *handle = dlopen(NULL, RTLD_NOW); + Func *p = dlsym(handle, "foo"); + + foo(); + p(); + printf("%p %p\n", foo, p); +} +EOF + +cat < + +__attribute__((ifunc("resolve_foo"))) +void foo(void); + +static void real_foo(void) { + printf("foo "); +} + +typedef void Func(); + +static Func *resolve_foo(void) { + return real_foo; +} +EOF + +$CC -B. -o $t/c.so $t/b.o -shared +$CC -B. -o $t/exe $t/a.o $t/c.so -no-pie -ldl +$QEMU $t/exe | grep 'foo foo' diff --git a/test/ifunc-dso.sh b/test/ifunc-dso.sh new file mode 100755 index 000000000..afcd77d6d --- /dev/null +++ b/test/ifunc-dso.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +__attribute__((ifunc("resolve_foobar"))) +void foobar(void); + +static void real_foobar(void) { + printf("Hello world\n"); +} + +typedef void Func(); + +static Func *resolve_foobar(void) { + return real_foobar; +} +EOF + +$CC -B. -o $t/c.so $t/b.o -shared +readelf -W --dyn-syms $t/c.so | grep -E '(IFUNC|: 10).*foobar' + +$CC -B. -o $t/exe $t/a.o $t/c.so +$QEMU $t/exe | grep 'Hello world' diff --git a/test/ifunc-dynamic.sh b/test/ifunc-dynamic.sh new file mode 100755 index 000000000..98bbaadd5 --- /dev/null +++ b/test/ifunc-dynamic.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +__attribute__((ifunc("resolve_foobar"))) +static void foobar(void); + +static void real_foobar(void) { + printf("Hello world\n"); +} + +typedef void Func(); + +static Func *resolve_foobar(void) { + return real_foobar; +} + +int main() { + foobar(); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o -Wl,-z,lazy +$QEMU $t/exe1 | grep 'Hello world' + +$CC -B. -o $t/exe2 $t/a.o -Wl,-z,now +$QEMU $t/exe2 | grep 'Hello world' diff --git a/test/ifunc-export.sh b/test/ifunc-export.sh new file mode 100755 index 000000000..48e668a1b --- /dev/null +++ b/test/ifunc-export.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +__attribute__((ifunc("resolve_foobar"))) +void foobar(void); + +void real_foobar(void) { + printf("Hello world\n"); +} + +typedef void Func(); + +Func *resolve_foobar(void) { + return real_foobar; +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o +readelf --dyn-syms $t/b.so | grep -E '(IFUNC|: 10)\s+GLOBAL DEFAULT.* foobar' diff --git a/test/ifunc-funcptr.sh b/test/ifunc-funcptr.sh new file mode 100755 index 000000000..6df4b280a --- /dev/null +++ b/test/ifunc-funcptr.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +typedef int Fn(); +Fn *get_foo(); + +int main() { + Fn *f = get_foo(); + printf("%d\n", f()); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -pie +$QEMU $t/exe1 | grep '^3$' + +$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o -no-pie +$QEMU $t/exe2 | grep '^3$' diff --git a/test/ifunc-noplt.sh b/test/ifunc-noplt.sh new file mode 100755 index 000000000..4bc6b2928 --- /dev/null +++ b/test/ifunc-noplt.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +supports_ifunc || skip + +cat < + +__attribute__((ifunc("resolve_foo"))) +void foo(void); + +void hello(void) { + printf("Hello world\n"); +} + +typedef void Fn(); + +Fn *resolve_foo(void) { + return hello; +} + +int main() { + foo(); +} +EOF + +$CC -B. -o $t/exe1 $t/a.o -pie +$QEMU $t/exe1 | grep 'Hello world' + +$CC -B. -o $t/exe2 $t/a.o -no-pie +$QEMU $t/exe2 | grep 'Hello world' diff --git a/test/ifunc-static-pie.sh b/test/ifunc-static-pie.sh new file mode 100755 index 000000000..bdb96b970 --- /dev/null +++ b/test/ifunc-static-pie.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static-pie || skip +supports_ifunc || skip + +cat < + +void foo() __attribute__((ifunc("resolve_foo"))); + +void hello() { + printf("Hello world\n"); +} + +void *resolve_foo() { + return hello; +} + +int main() { + foo(); + return 0; +} +EOF + +$CC -B. -o $t/exe2 $t/a.o -static-pie +$QEMU $t/exe2 | grep 'Hello world' diff --git a/test/ifunc-static.sh b/test/ifunc-static.sh new file mode 100755 index 000000000..c96e201b4 --- /dev/null +++ b/test/ifunc-static.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static || skip +supports_ifunc || skip + +cat < + +void foo() __attribute__((ifunc("resolve_foo"))); + +void hello() { + printf("Hello world\n"); +} + +void *resolve_foo() { + return hello; +} + +int main() { + foo(); + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o -static +$QEMU $t/exe | grep 'Hello world' diff --git a/test/image-base.sh b/test/image-base.sh new file mode 100755 index 000000000..c0622bc4b --- /dev/null +++ b/test/image-base.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +int main() { + printf("Hello world\n"); + return 0; +} +EOF + +$CC -B. -no-pie -o $t/exe1 $t/a.o -Wl,--image-base=0x8000000 +$QEMU $t/exe1 | grep 'Hello world' +readelf -W --sections $t/exe1 | grep -E '.interp\s+PROGBITS\s+0*8000...\b' + +cat < +__attribute__((constructor(10000))) void init4() { printf("1"); } +EOF + +cat <<'EOF' | $CC -c -o $t/b.o -xc - +#include +__attribute__((constructor(1000))) void init3() { printf("2"); } +EOF + +cat <<'EOF' | $CC -c -o $t/c.o -xc - +#include +__attribute__((constructor)) void init1() { printf("3"); } +EOF + +cat <<'EOF' | $CC -c -o $t/d.o -xc - +#include +__attribute__((constructor)) void init2() { printf("4"); } +EOF + +cat <<'EOF' | $CC -c -o $t/e.o -xc - +#include +__attribute__((destructor(10000))) void fini4() { printf("5"); } +EOF + +cat <<'EOF' | $CC -c -o $t/f.o -xc - +#include +__attribute__((destructor(1000))) void fini3() { printf("6"); } +EOF + +cat <<'EOF' | $CC -c -o $t/g.o -xc - +#include +__attribute__((destructor)) void fini1() { printf("7"); } +EOF + +cat <<'EOF' | $CC -c -o $t/h.o -xc - +#include +__attribute__((destructor)) void fini2() { printf("8"); } +EOF + +cat < + +void foo() { + printf("Hello world\n"); +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o -Wl,-z,initfirst +readelf --dynamic $t/b.so | grep 'Flags:.*INITFIRST' diff --git a/test/interpose.sh b/test/interpose.sh new file mode 100755 index 000000000..78aa25a5a --- /dev/null +++ b/test/interpose.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < + +void foo() { + printf("Hello world\n"); +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o -Wl,-z,interpose +readelf --dynamic $t/b.so | grep 'Flags:.*INTERPOSE' diff --git a/test/invalid-version-script.sh b/test/invalid-version-script.sh new file mode 100755 index 000000000..b81309cdc --- /dev/null +++ b/test/invalid-version-script.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo 'int main() {}' | $CC -c -o $t/a.o -xc - + +echo 'VER1 { foo[12; };' > $t/b.ver + +not $CC -B. -shared -o $t/c.so -Wl,-version-script,$t/b.ver $t/a.o |& + grep 'invalid version pattern' diff --git a/test/issue646.sh b/test/issue646.sh new file mode 100755 index 000000000..d7e86af94 --- /dev/null +++ b/test/issue646.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = sh4aeb ] && skip + +cat < +#include + +class Foo : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +static void do_throw() { + throw Foo("exception"); +} + +int main() { + try { + do_throw(); + } catch (const Foo &e) { + std::cout << "error: " << e.what() << std::endl; + } +} +EOF + +$CXX -B. -o $t/exe $t/a.o +$QEMU $t/exe | grep 'error: exception' diff --git a/test/large-alignment-dso.sh b/test/large-alignment-dso.sh new file mode 100755 index 000000000..859fa1fb9 --- /dev/null +++ b/test/large-alignment-dso.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = i686 ] && skip + +cat < +#include + +void hello() __attribute__((aligned(32768), section(".hello"))); +void world() __attribute__((aligned(32768), section(".world"))); + +void hello() { + printf("Hello"); +} + +void world() { + printf(" world"); +} + +void greet() { + hello(); + world(); +} +EOF + +$CC -B. -o $t/b.so $t/a.o -shared + +cat < +#include + +void hello() __attribute__((aligned(32768), section(".hello"))); +void world() __attribute__((aligned(32768), section(".world"))); + +void hello() { + printf("Hello"); +} + +void world() { + printf(" world"); +} + +int main() { + hello(); + world(); + + // Linux kernel may ignore a riduculously large alignment requirement, + // but we still want to verify that an executable with a large + // alignment requirement can still run. + // + // printf(" %lu %lu\n", + // (unsigned long)((uintptr_t)hello % 32768), + // (unsigned long)((uintptr_t)world % 32768)); +} +EOF + +$CC -B. -o $t/exe $t/a.o +$QEMU $t/exe | grep 'Hello world' diff --git a/test/large-max-page-size-strip.sh b/test/large-max-page-size-strip.sh new file mode 100755 index 000000000..9483fff69 --- /dev/null +++ b/test/large-max-page-size-strip.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# ARM32's strip command crashes on the output of this test for some reason. +[[ $MACHINE = arm* ]] && skip + +strip=$STRIP +command -v $strip >& /dev/null || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -pie -Wl,-zmax-page-size=0x200000 +$strip $t/exe +$QEMU $t/exe | grep 'Hello world' diff --git a/test/large-max-page-size.sh b/test/large-max-page-size.sh new file mode 100755 index 000000000..dbfe2ced2 --- /dev/null +++ b/test/large-max-page-size.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -pie -Wl,-zmax-page-size=0x200000 +$QEMU $t/exe | grep 'Hello world' diff --git a/test/large-text.sh b/test/large-text.sh new file mode 100755 index 000000000..5a2d79976 --- /dev/null +++ b/test/large-text.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { printf("Hello world\n"); } +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep 'Hello world' diff --git a/test/library.sh b/test/library.sh new file mode 100755 index 000000000..87f21a982 --- /dev/null +++ b/test/library.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +void hello() { + printf("Hello world\n"); +} +EOF + +$CC -B. -shared -o $t/libfoobar.so $t/a.o + +cat < $t/script +bar = foo; +EOF + +$CC -B. -o $t/b.so -shared $t/script $t/a.o +readelf -sW $t/b.so | grep 'FUNC .* bar' + +cat < +int bar(); +int main() { + printf("%d\n", bar()); + return 0; +} +EOF + +$CC -B. -o $t/exe $t/c.o $t/b.so +$QEMU $t/exe | grep 42 diff --git a/test/linker-script-error.sh b/test/linker-script-error.sh new file mode 100755 index 000000000..7708e35bd --- /dev/null +++ b/test/linker-script-error.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/b.script + +not $CC -B. -o $t/exe $t/a.o $t/b.script |& grep 'unclosed comment' diff --git a/test/linker-script-group-as-needed.sh b/test/linker-script-group-as-needed.sh new file mode 100755 index 000000000..913845ba0 --- /dev/null +++ b/test/linker-script-group-as-needed.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +cat < $t/libscript.a +GROUP(AS_NEEDED("$t/libB.so")) +EOF + +$CC -B. -o $t/exe -L$t -lscript $t/a.o +$QEMU $t/exe | grep 'Hello world' + +$CC -B. -o $t/exe -L$t -l:libscript.a $t/a.o +$QEMU $t/exe | grep 'Hello world' diff --git a/test/linker-script-relocatable.sh b/test/linker-script-relocatable.sh new file mode 100755 index 000000000..7e8ab94ae --- /dev/null +++ b/test/linker-script-relocatable.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# OneTBB isn't tsan-clean +nm mold | grep '__tsan_init' && skip + +cat < +void hello() { printf("Hello world\n"); } +EOF + +cat < $t/c.script + +./mold --relocatable -o $t/d.o $t/c.script + +$CC -B. -o $t/exe $t/d.o +$QEMU $t/exe | grep Hello diff --git a/test/linker-script-version.sh b/test/linker-script-version.sh new file mode 100755 index 000000000..3f952917b --- /dev/null +++ b/test/linker-script-version.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Symbol versions must be defined in the command line order even +# though input files are read in parallel. + +cat < $t/a.script +VERSION { + ver_a { global: foo; }; +} +EOF + +cat < $t/b.script +VERSION { + ver_b { global: bar; }; +} +EOF + +cat < $t/log +grep -o 'Name: ver_[ab]' $t/log > $t/order + +diff $t/order - < +int main() { + printf("Hello world\n"); +} +EOF + +cat < $t/script +GROUP("$t/a.o") +EOF + +$CC -B. -o $t/exe $t/script +$QEMU $t/exe | grep 'Hello world' + +$CC -B. -o $t/exe -Wl,-T,$t/script +$QEMU $t/exe | grep 'Hello world' + +$CC -B. -o $t/exe -Wl,--script,$t/script +$QEMU $t/exe | grep 'Hello world' diff --git a/test/linker-script2.sh b/test/linker-script2.sh new file mode 100755 index 000000000..56cfa88a9 --- /dev/null +++ b/test/linker-script2.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/b.script +INPUT(-lfoo) +EOF + +$CC -B. -o $t/exe -L$t/foo/bar $t/b.script diff --git a/test/linker-script3.sh b/test/linker-script3.sh new file mode 100755 index 000000000..d123e73a4 --- /dev/null +++ b/test/linker-script3.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +mkdir -p $t/foo + +cat < $t/b.script +INPUT(a.o) +EOF + +$CC -B. -o $t/exe -L$t/foo $t/b.script diff --git a/test/linker-script4.sh b/test/linker-script4.sh new file mode 100755 index 000000000..30d156457 --- /dev/null +++ b/test/linker-script4.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +echo 'VERSION { ver_x { global: *; }; };' > $t/a.script + +cat < $t/b.s +.globl foo, bar, baz +foo: + nop +bar: + nop +baz: + nop +EOF + +$CC -B. -shared -o $t/c.so $t/a.script $t/b.s +readelf --version-info $t/c.so > $t/log + +grep -F 'Rev: 1 Flags: none Index: 2 Cnt: 1 Name: ver_x' $t/log diff --git a/test/linker-script5.sh b/test/linker-script5.sh new file mode 100755 index 000000000..a02fc02e1 --- /dev/null +++ b/test/linker-script5.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +mkdir -p $t/foo + +cat < $t/foo/b.script +INPUT(a.o) +EOF + +$CC -B. -o $t/exe $t/foo/b.script diff --git a/test/linker-script6.sh b/test/linker-script6.sh new file mode 100755 index 000000000..b1304c4aa --- /dev/null +++ b/test/linker-script6.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +mkdir -p $t/foo + +cat < $t/foo/b.script + +INPUT(a.o) +EOF + +$CC -B. -o $t/exe $t/foo/b.script diff --git a/test/long-symbol-name.sh b/test/long-symbol-name.sh new file mode 100755 index 000000000..14b0232bf --- /dev/null +++ b/test/long-symbol-name.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +suffix=$(printf 'x%.0s' {1..300}) +plain_name=plain_$suffix +versioned_name=versioned_$suffix +defsym_name=defsym_$suffix + +cat < $t/a.ver +$CC -B. -shared -o $t/b.so $t/a.o -Wl,--version-script=$t/a.ver + +readelf --wide --dyn-syms $t/b.so > $t/log +grep -F "$plain_name" $t/log +grep -F "$versioned_name@@VER" $t/log + +$CC -B. -shared -o $t/c.so $t/a.o -Wl,--version-script=$t/a.ver \ + -Wl,-defsym=$defsym_name=42 +readelf --wide --symbols $t/c.so | grep -F "$defsym_name" diff --git a/test/lto-archive.sh b/test/lto-archive.sh new file mode 100755 index 000000000..9a0e2d92e --- /dev/null +++ b/test/lto-archive.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ "$CC" = cc ] || skip +test_cflags -flto || skip + +cat < +void hello() { + printf("Hello world\n"); +} +EOF + +cat < +void howdy() { + printf("Hello world\n"); +} +EOF + +rm -f $t/c.a +ar rc $t/c.a $t/a.o $t/b.o + +cat < $t/log +grep hello $t/log +not grep howdy $t/log diff --git a/test/lto-archive2.sh b/test/lto-archive2.sh new file mode 100755 index 000000000..5a0bcc05e --- /dev/null +++ b/test/lto-archive2.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -flto=auto || skip + +echo | $CC -o $t/a.o -c -flto=auto -xc - + +rm -f $t/b.a +ar rc $t/b.a $t/a.o + +cat <& /dev/null || skip +echo 'int foo() { return 0; }' | + $GXX -flto -fno-fat-lto-objects -c -o /dev/null -xc++ - >& /dev/null || skip + +cat <<'EOF' > $t/conv.h +template +__attribute__((noinline)) +int convert(T x) { + return (int)x + 1; +} +EOF + +cat < when both +# members are extracted. +ar rcs $t/libutil.a $t/a.o $t/b.o + +cat <<'EOF' | $GXX -O2 -flto -fno-fat-lto-objects -c -o $t/mid.o -xc++ - +int do_a(int); +int do_b(int); +int use_a(int x) { return do_a(x); } +int use_b(int x) { return do_b(x); } +EOF + +ar rcs $t/libmid.a $t/mid.o + +# main() calls only use_b(), so LTO eliminates use_a() and with it the only +# reference to do_a(). a.o is not re-extracted after LTO even though it won +# the COMDAT group before LTO. +cat <<'EOF' | $GXX -O2 -flto -fno-fat-lto-objects -c -o $t/main.o -xc++ - +int use_b(int); +int main() { return use_b(42) == 53 ? 0 : 1; } +EOF + +$GXX -B. -O2 -flto -o $t/exe $t/main.o $t/libmid.a $t/libutil.a +$QEMU $t/exe + +# Same but with the losing member linked directly instead of via an archive. +ar rcs $t/libutil2.a $t/a.o +$GXX -B. -O2 -flto -o $t/exe2 $t/main.o $t/libmid.a $t/libutil2.a $t/b.o +$QEMU $t/exe2 diff --git a/test/lto-comdat-archive2.sh b/test/lto-comdat-archive2.sh new file mode 100755 index 000000000..d8c933194 --- /dev/null +++ b/test/lto-comdat-archive2.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Regression test for https://github.com/rui314/mold/issues/1637. +# In the pre-LTO round, the only reachable file holding the COMDAT group for +# `cnt` is an IR file, so mold reports the IR definition as prevailing, and +# GCC emits `cnt` into the LTO result as a plain global with no COMDAT group. +# In the post-LTO round, a.o becomes reachable because the LTO-generated code +# calls __divti3. a.o must not win the COMDAT group it carries: its copy of +# `cnt` would collide with the definition mold asked GCC to emit. + +[ $MACHINE = $(uname -m) ] || skip + +echo 'int main() {}' | $GXX -B. -flto -o /dev/null -xc++ - >& /dev/null || skip +echo 'int foo() { return 0; }' | + $GXX -flto -fno-fat-lto-objects -c -o /dev/null -xc++ - >& /dev/null || skip +# The scheme below needs a target that lowers 128-bit division to a +# __divti3 call. Some lower it inline (e.g. POWER10), and 32-bit targets +# don't support __int128 at all. +echo 'volatile long long x, y; __int128 f() { return (__int128)x / y; }' | + $GXX -O2 -S -o - -xc++ - 2> /dev/null | grep -q __divti3 || skip + +# GCC emits the function-local static of an inline function as an +# STB_GNU_UNIQUE symbol in a COMDAT group. The non-weak binding is what turns +# a surviving second copy into a duplicate symbol error. +cat <<'EOF' > $t/uniq.h +inline int bump() { + static int cnt; + return ++cnt; +} +EOF + +# Non-LTO archive member. Its only entry point is __divti3, which nothing +# references until the LTO backend expands main()'s 128-bit division to a +# libcall, so the member is extracted only in the post-LTO round. The archive +# precedes libgcc on the command line, so this copy shadows libgcc's. +# __udivti3 would not work: libstdc++.so carries an undefined reference to +# it, which would extract the member before LTO. +cat <<'EOF' | $GXX -O2 -I$t -c -o $t/a.o -xc++ - +#include "uniq.h" +// The casts keep the division 64-bit so that it doesn't recurse. +extern "C" __int128 __divti3(__int128 a, __int128 b) { + return (long long)a / (long long)b + bump(); +} +EOF + +readelf -sW $t/a.o | grep -q 'UNIQUE.*_ZZ4bumpvE3cnt' || skip + +ar rcs $t/libutil.a $t/a.o + +# The 128-bit division compiles to a __divti3 call. Both bump() calls must +# increment the same cnt, so a / b is 21 / 7 plus the second increment. +# A wrong link either fails with a duplicate symbol error or leaves two +# copies of cnt, in which case a / b evaluates to 4. +cat <<'EOF' | $GXX -O2 -I$t -flto -fno-fat-lto-objects -c -o $t/main.o -xc++ - +#include "uniq.h" +volatile long long lo = 21, d = 7; +int main() { + bump(); + __int128 a = lo, b = d; + return a / b == 5 ? 0 : 1; +} +EOF + +$GXX -B. -O2 -flto -o $t/exe $t/main.o $t/libutil.a +$t/exe diff --git a/test/lto-comdat-archive3.sh b/test/lto-comdat-archive3.sh new file mode 100755 index 000000000..13b6bf08c --- /dev/null +++ b/test/lto-comdat-archive3.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Clang variant of lto-comdat-archive2.sh. Unlike GCC, LLVM keeps claimed +# COMDAT groups in its LTO output, so the LTO-generated file itself must be +# allowed to win the group while the archive member extracted after LTO must +# still lose it. Clang emits cnt as a weak symbol, so a wrong link does not +# necessarily fail; the exit code check below catches two live copies too. + +[ $MACHINE = $(uname -m) ] || skip + +CLANG="${TEST_CLANG:-clang}" +$CLANG --version >& /dev/null || skip +echo 'int main() {}' | $CLANG -B. -flto -o /dev/null -xc - >& /dev/null || skip +echo '__int128 x;' | $CLANG -c -o /dev/null -xc - >& /dev/null || skip + +cat <<'EOF' > $t/uniq.h +inline int bump() { + static int cnt; + return ++cnt; +} +EOF + +# See lto-comdat-archive2.sh for why __divti3. +cat <<'EOF' | $CLANG -O2 -I$t -c -o $t/a.o -xc++ - +#include "uniq.h" +// The casts keep the division 64-bit so that it doesn't recurse. +extern "C" __int128 __divti3(__int128 a, __int128 b) { + return (long long)a / (long long)b + bump(); +} +EOF + +ar rcs $t/libutil.a $t/a.o + +cat <<'EOF' | $CLANG -O2 -I$t -flto -c -o $t/main.o -xc++ - +#include "uniq.h" +volatile long long lo = 21, d = 7; +int main() { + bump(); + __int128 a = lo, b = d; + return a / b == 5 ? 0 : 1; +} +EOF + +$CLANG -B. -O2 -flto -o $t/exe $t/main.o $t/libutil.a +$t/exe diff --git a/test/lto-comdat.sh b/test/lto-comdat.sh new file mode 100755 index 000000000..88b7432d9 --- /dev/null +++ b/test/lto-comdat.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = $(uname -m) ] || skip +echo 'int main() {}' | clang -B. -flto -o /dev/null -xc - || skip + +echo '' | clang -S -emit-llvm -flto -o $t/a.ll -xc - || skip + +cat <<'EOF' >> $t/a.ll +$foo = comdat any +@foo = global i32 42, comdat($foo) +EOF + +cp $t/a.ll $t/b.ll + +clang -S -emit-llvm -flto -o $t/a.bc $t/a.ll +clang -S -emit-llvm -flto -o $t/b.bc $t/b.ll + +cat <<'EOF' | clang -o $t/c.o -c -flto -xc - +#include +extern int foo; +int main() { printf("%d\n", foo); } +EOF + +clang -B. -o $t/exe -flto $t/a.bc $t/b.bc $t/c.o +$QEMU $t/exe | grep 42 diff --git a/test/lto-dso.sh b/test/lto-dso.sh new file mode 100755 index 000000000..1f4684e4c --- /dev/null +++ b/test/lto-dso.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -flto || skip + +cat <& /dev/null || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$GCC -B. -o $t/exe1 -flto $t/a.o +$QEMU $t/exe1 | grep 'Hello world' + +# Test that LTO is used for FAT LTO objects +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$GCC -B. -o $t/exe2 $t/b.o --verbose |& grep -- -fwpa + +# Test FAT objects if -fno-use-linker-plugin is used + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$GCC -B. -o $t/exe3 -flto -fno-use-linker-plugin $t/c.o +$QEMU $t/exe3 | grep 'Hello world' diff --git a/test/lto-llvm.sh b/test/lto-llvm.sh new file mode 100755 index 000000000..4f312e20c --- /dev/null +++ b/test/lto-llvm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = $(uname -m) ] || skip + +echo 'int main() {}' | clang -B. -flto -o /dev/null -xc - >& /dev/null || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +clang -B. -o $t/exe -flto $t/a.o +$t/exe | grep 'Hello world' diff --git a/test/lto-llvm2.sh b/test/lto-llvm2.sh new file mode 100755 index 000000000..4c558875e --- /dev/null +++ b/test/lto-llvm2.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = $(uname -m) ] || skip + +echo 'int main() {}' | clang -B. -flto -o /dev/null -xc - >& /dev/null || skip + +cat <& /dev/null || skip +$GCC --version >& /dev/null || skip + +echo 'int main() {}' | $CLANG -B. -flto -o /dev/null -xc - >& /dev/null || skip +echo 'int foo() { return 0; }' | + $GCC -flto -ffat-lto-objects -c -o /dev/null -xc - >& /dev/null || skip + +cat <<'EOF' | $CLANG -flto -c -o $t/main.o -xc - +int foo(void); +int main(void) { return foo(); } +EOF + +cat <<'EOF' | $GCC -flto -ffat-lto-objects -c -o $t/foo.o -xc - +int foo(void) { return 0; } +EOF + +$CLANG -B. -flto -o $t/exe $t/main.o $t/foo.o +$QEMU $t/exe diff --git a/test/lto-mixed-gcc-slim-archive.sh b/test/lto-mixed-gcc-slim-archive.sh new file mode 100755 index 000000000..1e53439f4 --- /dev/null +++ b/test/lto-mixed-gcc-slim-archive.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# Regression test: an unreferenced GCC slim LTO object inside an archive should +# not abort a mixed LLVM-LTO link. + +[ $MACHINE = $(uname -m) ] || skip + +CLANG="${TEST_CLANG:-clang}" +$CLANG --version >& /dev/null || skip +$GCC --version >& /dev/null || skip + +echo 'int main() {}' | $CLANG -B. -flto -o /dev/null -xc - >& /dev/null || skip +echo 'int foo() { return 0; }' | + $GCC -flto -fno-fat-lto-objects -c -o /dev/null -xc - >& /dev/null || skip + +cat <<'EOF' | $CLANG -flto -c -o $t/main.o -xc - +int main(void) { return 0; } +EOF + +cat <<'EOF' | $GCC -flto -fno-fat-lto-objects -c -o $t/foo.c.o -xc - +int foo(void) { return 7; } +EOF + +ar rcs $t/libfoo.a $t/foo.c.o +$CLANG -B. -flto -o $t/exe $t/main.o $t/libfoo.a + +$QEMU $t/exe diff --git a/test/lto-no-plugin.sh b/test/lto-no-plugin.sh new file mode 100755 index 000000000..98cbf3bd3 --- /dev/null +++ b/test/lto-no-plugin.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = $(uname -m) ] || skip + +echo 'int main() {}' | clang -B. -flto -o /dev/null -xc - >& /dev/null || skip + +echo 'int main() {}' | clang -c -o $t/a.o -xc - +echo 'void foo() {}' | clang -c -o $t/b.o -xc - -flto + +not ./mold -o /dev/null $t/a.o $t/b.o |& + grep "b.o: unable to handle this LTO object file" diff --git a/test/lto-nostdlib.sh b/test/lto-nostdlib.sh new file mode 100755 index 000000000..55f3286ed --- /dev/null +++ b/test/lto-nostdlib.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -flto || skip + +cat < $t/a.o + +rm -f $t/b.a +ar rc $t/b.a $t/a.o + +cat < $t/b.script +{ + global: foo; + local: *; +}; +EOF + +$CC -B. -shared -o $t/c.so -flto $t/a.o -Wl,-version-script=$t/b.script + +if [ $MACHINE = ppc64 ]; then + # On PPC64V1, function symbol refers a function descriptor in .opd + nm -D $t/c.so | grep 'D foo' + nm -D $t/c.so | not grep 'D bar' +else + nm -D $t/c.so | grep 'T foo' + nm -D $t/c.so | not grep 'T bar' +fi diff --git a/test/main-in-dso.sh b/test/main-in-dso.sh new file mode 100755 index 000000000..1a598ce45 --- /dev/null +++ b/test/main-in-dso.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ $MACHINE = ppc64 ] && skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -shared -o $t/b.so $t/a.o + +$CC -o $t/c.o -c -xc /dev/null -fPIC +$CC -B. -o $t/exe1 $t/c.o $t/b.so -pie +$QEMU $t/exe1 | grep 'Hello world' + +$CC -o $t/c.o -c -xc /dev/null -fno-PIC +$CC -B. -o $t/exe2 $t/c.o $t/b.so -no-pie +$QEMU $t/exe2 | grep 'Hello world' diff --git a/test/many-input-sections.sh b/test/many-input-sections.sh new file mode 100755 index 000000000..43cc5ace7 --- /dev/null +++ b/test/many-input-sections.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +seq 1 100000 | sed 's/.*/.section .data.&,"aw"\n.word 0\n/g' | + $CC -c -xassembler -o $t/a.o - + +cat <<'EOF' | $CC -c -xc -o $t/b.o - +#include + +int main() { + printf("Hello\n"); + return 0; +} +EOF + +$CC -B. -o $t/exe $t/a.o $t/b.o +$QEMU $t/exe | grep Hello diff --git a/test/many-input-sections2.sh b/test/many-input-sections2.sh new file mode 100755 index 000000000..6ad0c9041 --- /dev/null +++ b/test/many-input-sections2.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +# OneTBB isn't tsan-clean +nm mold | grep '__tsan_init' && skip + +echo 'foo = 0x1000' > $t/a.s +seq 1 100000 | sed 's/.*/.section .data.&,"aw"\n.globl x&\nx&: .word 0\n/g' >> $t/a.s +$CC -c -xassembler -o $t/a.o $t/a.s + +./mold --relocatable -o $t/b.o $t/a.o + +readelf -WS $t/b.o > $t/log1 +grep -F .data.100000 $t/log1 + +readelf -Ws $t/b.o > $t/log2 +grep -F 'GLOBAL DEFAULT 100000' $t/log2 +grep -F 'ABS foo' $t/log2 +not grep -F 'ABS x68966' $t/log2 diff --git a/test/many-output-sections.sh b/test/many-output-sections.sh new file mode 100755 index 000000000..628d0010a --- /dev/null +++ b/test/many-output-sections.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +seq 1 100000 | sed 's/.*/.globl foo&\n.section .foo&,"aw"\nfoo&:.word 0\n/g' | + $CC -c -xassembler -o $t/a.o - + +cat <<'EOF' | $CC -c -xc -o $t/b.o - -fPIC +extern int foo100000; +int bar() { return foo100000; } +EOF + +not $CC -B. -shared -o $t/c.so $t/a.o $t/b.o |& grep -F 'too many output sections' diff --git a/test/mcmodel-large.sh b/test/mcmodel-large.sh new file mode 100755 index 000000000..728752529 --- /dev/null +++ b/test/mcmodel-large.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +test_cflags -static -mcmodel=large || skip + +cat < +int main() { + printf("Hello world\n"); +} +EOF + +$CC -B. -o $t/exe $t/a.o -static -no-pie +$QEMU $t/exe | grep 'Hello world' diff --git a/test/mergeable-strings.sh b/test/mergeable-strings.sh new file mode 100755 index 000000000..26b9b4045 --- /dev/null +++ b/test/mergeable-strings.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < +#include + +char *cstr1 = "foo"; +wchar_t *wide1 = L"foo"; +char16_t *utf16_1 = u"foo"; +char32_t *utf32_1 = U"foo"; +EOF + +cat < +#include +#include + +extern char *cstr1; +extern wchar_t *wide1; +extern char16_t *utf16_1; +extern char32_t *utf32_1; + +char *cstr2 = "foo"; +wchar_t *wide2 = L"foo"; +char16_t *utf16_2 = u"foo"; +char32_t *utf32_2 = U"foo"; + +int main() { + printf("%p %p %p %p %p %p %p %p\n", + cstr1, cstr2, wide1, wide2, utf16_1, utf16_2, utf32_1, utf32_2); +} +EOF + +# String merging is an optional feature, so test it with the default +# linker first to verify that it does work on this system. +$CC -o $t/exe1 $t/a.o $t/b.o -no-pie + +if $QEMU $t/exe1 | grep -E '^(\S+) \1 (\S+) \2 (\S+) \3 (\S+) \4$'; then + $CC -B. -o $t/exe2 $t/a.o $t/b.o -no-pie + $QEMU $t/exe2 | grep -E '^(\S+) \1 (\S+) \2 (\S+) \3 (\S+) \4$' +fi diff --git a/test/missing-but-ok.sh b/test/missing-but-ok.sh new file mode 100755 index 000000000..c2d01b9fe --- /dev/null +++ b/test/missing-but-ok.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat < $t/log + +grep 'undefined symbol: foo' $t/log +grep '>>> .*a\.o' $t/log diff --git a/test/mold-wrapper.sh b/test/mold-wrapper.sh new file mode 100755 index 000000000..964d3a707 --- /dev/null +++ b/test/mold-wrapper.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +[ "$CC" = cc ] || skip + +ldd mold-wrapper.so | grep libasan && skip + +nm mold | grep '__[at]san_init' && skip + +cat <<'EOF' > $t/a.sh +#!/usr/bin/env bash +echo "$0" "$@" $FOO +EOF + +chmod 755 $t/a.sh + +cat <<'EOF' | $CC -xc -o $t/exe - +#define _GNU_SOURCE 1 + +#include +#include +#include +#include + +extern char **environ; + +int main(int argc, char **argv) { + if (!strcmp(argv[1], "execl")) { + execl("/usr/bin/ld", "/usr/bin/ld", "execl", (char *)0); + perror("execl"); + return 1; + } + + if (!strcmp(argv[1], "execlp")) { + execlp("/usr/bin/ld", "/usr/bin/ld", "execlp", (char *)0); + perror("execl"); + return 1; + } + + if (!strcmp(argv[1], "execle")) { + execle("/usr/bin/ld", "/usr/bin/ld", "execle", (char *)0, environ); + perror("execl"); + return 1; + } + + if (!strcmp(argv[1], "execv")) { + execv("/usr/bin/ld", (char *[]){"/usr/bin/ld", "execv", (char *)0}); + perror("execl"); + return 1; + } + + if (!strcmp(argv[1], "execvp")) { + execvp("/usr/bin/ld", (char *[]){"/usr/bin/ld", "execvp", (char *)0}); + perror("execl"); + return 1; + } + + if (!strcmp(argv[1], "execvpe")) { + char *env[] = {"FOO=bar", NULL}; + execvpe("/usr/bin/ld", (char *[]){"/usr/bin/ld", "execvpe", (char *)0}, env); + perror("execl"); + return 1; + } + + fprintf(stderr, "unreachable: %s\n", argv[1]); + return 1; +} +EOF + +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execl | grep 'a.sh execl' +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execlp | grep 'a.sh execlp' +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execle | grep 'a.sh execle' +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execv | grep 'a.sh execv' +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execvp | grep 'a.sh execvp' +LD_PRELOAD=`pwd`/mold-wrapper.so MOLD_PATH=$t/a.sh $t/exe execvpe | grep 'a.sh execvpe bar' diff --git a/test/mold-wrapper2.sh b/test/mold-wrapper2.sh new file mode 100755 index 000000000..2d1073859 --- /dev/null +++ b/test/mold-wrapper2.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +ldd mold-wrapper.so | grep libasan && skip +nm mold | grep '__[at]san_init' && skip + +./mold -run bash -c 'echo $LD_PRELOAD' | grep -F mold-wrapper.so diff --git a/test/nmagic.sh b/test/nmagic.sh new file mode 100755 index 000000000..8797cf2d1 --- /dev/null +++ b/test/nmagic.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +. $(dirname $0)/common.inc + +cat <