Bump mold version to 2.42.1

Initialize reachability when creating object, LTO, and shared inputs so
mandatory objects outrank unextracted archive members during the first
symbol resolution pass.

Process all --undefined-glob matches regardless of file reachability so
--gc-sections retains every matching definition. Cover archive ordering,
directly linked objects, and multiple matches in one object.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Delete this message and describe the bug here. Note that we only fix bugs in the most recent release of mold, so please make sure that you are using the most recent version.

.github/workflows/build-all.yml

@@ -3,12 +3,27 @@ name: Build all tarballs
 on:
   schedule:
     - cron: '0 0 * * *'
-  release:
-    types: [ published ]
   workflow_dispatch:
 
 jobs:
-  build-tarballs:
+  check-changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      has_changes: ${{ github.event_name == 'workflow_dispatch' || steps.check.outputs.has_changes }}
+    steps:
+    - name: Check for new commits since last successful run
+      id: check
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        last=$(gh run list -R ${{ github.repository }} -w build-all.yml -b main -s success -L1 --json createdAt -q '.[0].createdAt')
+        if [ -z "$last" ] || [ "$last" = "null" ] || [ "$(gh api repos/${{ github.repository }}/commits?sha=main\&since=$last\&per_page=1 | jq length)" -gt 0 ]; then
+          echo "has_changes=true" >> "$GITHUB_OUTPUT"
+        fi
+
+  build-linux:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
     strategy:
       matrix:
         include:
@@ -28,7 +43,7 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install Podman
       run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
@@ -44,8 +59,35 @@ jobs:
       run: ./dist.sh ${{ matrix.target }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: ${{ matrix.target }}
         path: dist/mold-*.tar.gz
         compression-level: 0
+
+  build-windows:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
+    runs-on: windows-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-windows-x86_64.zip
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*

.github/workflows/build-native.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install Podman
       run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
@@ -37,7 +37,7 @@ jobs:
       run: ./dist.sh ${{ matrix.target }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: ${{ matrix.target }}
         path: dist/mold-*.tar.gz

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ on:
 env:
   UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
 jobs:
-  build-sanitizers:
+  build-asan-tsan:
     strategy:
       matrix:
         target:
@@ -13,8 +13,9 @@ jobs:
         - '-DMOLD_USE_ASAN=On'
         - '-DMOLD_USE_TSAN=On'
     runs-on: ubuntu-24.04
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - uses: rui314/setup-mold@staging
     - run: sudo ./install-build-deps.sh
     - name: build
@@ -26,7 +27,7 @@ jobs:
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: test-results-clang
@@ -34,10 +35,23 @@ jobs:
           build
           !build/CMakeFiles
 
+  build-msan:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v6
+    - uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - run: .github/workflows/run-msan.sh
+
   build-multi-archs:
     runs-on: ubuntu-24.04
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: install-build-deps
       run: |
         sudo ./install-build-deps.sh
@@ -51,7 +65,7 @@ jobs:
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: test-results-multi-archs
@@ -65,16 +79,17 @@ jobs:
         distro:
         - alpine
         - archlinux
-        - debian:11 # GCC 10 and CMake 3.18 - the minimum supported by mold
+        - debian:12
         - fedora
         - gentoo/stage3
         - opensuse/tumbleweed
         - ubuntu:22.04
         - ubuntu:25.04
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     container: ${{ matrix.distro }}
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - run: ./install-build-deps.sh
     - name: build
       run: |
@@ -83,19 +98,24 @@ jobs:
         cmake ..
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
+    - name: sanitize distro name
+      if: failure()
+      id: sanitize
+      run: echo "name=$(echo '${{ matrix.distro }}' | tr '/:' '--')" >> "$GITHUB_OUTPUT"
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
-        name: test-results-${{ matrix.distro }}
+        name: test-results-${{ steps.sanitize.outputs.name }}
         path: |
           build
           !build/CMakeFiles
 
   build-macos:
     runs-on: macos-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: build
       run: |
         mkdir build
@@ -105,8 +125,9 @@ jobs:
 
   build-windows:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: build
       run: |
         mkdir build
@@ -116,8 +137,9 @@ jobs:
 
   build-msys:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Setup MSYS2
       uses: msys2/setup-msys2@v2
       with:
@@ -134,8 +156,9 @@ jobs:
 
   build-freebsd:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Build and test
       uses: vmactions/freebsd-vm@v1
       with:

.github/workflows/install-extras.sh

@@ -5,7 +5,7 @@ apt-get install -y wget xz-utils
 
 # Install a 32-bit RISC-V toolchain
 mkdir /rv32
-wget -O- -q https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.01.20/riscv32-glibc-ubuntu-22.04-gcc-nightly-2025.01.20-nightly.tar.xz | tar -C /rv32 --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.06.13/riscv32-glibc-ubuntu-24.04-gcc-nightly-2025.06.13-nightly.tar.xz | tar -C /rv32 --strip-components=1 --xz -xf -
 
 ln -sf /rv32/sysroot /usr/riscv32-linux-gnu
 echo '/rv32/bin/riscv32-unknown-linux-gnu-gcc -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-gcc
@@ -18,7 +18,7 @@ done
 
 # Install a LoongArch toolchain
 mkdir /larch
-wget -O- -q https://github.com/loongson/build-tools/releases/download/2025.02.21/x86_64-cross-tools-loongarch64-binutils_2.44-gcc_14.2.0-glibc_2.41.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/x86_64-cross-tools-loongarch64-binutils_2.44-gcc_15.1.0-glibc_2.41.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
 
 cp -r /larch/loongarch64-unknown-linux-gnu/lib/* /larch/target/lib64
 ln -sf /larch/target /usr/loongarch64-linux-gnu
@@ -27,25 +27,39 @@ for i in gcc g++ objdump objcopy strip; do
   ln -sf /larch/bin/loongarch64-unknown-linux-gnu-$i /usr/bin/loongarch64-linux-gnu-$i
 done
 
-wget -O /usr/local/bin/qemu-loongarch64 -q https://github.com/loongson/build-tools/releases/download/2024.11.01/qemu-loongarch64
+wget -O /usr/local/bin/qemu-loongarch64 --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/qemu-loongarch64
 chmod 755 /usr/local/bin/qemu-loongarch64
 
+# Install ARM32 big-endian toolchain
+mkdir /armeb
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/armebv7-eabihf/tarballs/armebv7-eabihf--glibc--stable-2024.05-1.tar.xz | tar -C /armeb --strip-components=1 --xz -xf -
+
+ln -sf /armeb/armeb-buildroot-linux-gnueabihf/sysroot /usr/armeb-linux-gnueabihf
+echo '/armeb/bin/armeb-linux-gcc -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-gcc
+echo '/armeb/bin/armeb-linux-g++ -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-g++
+chmod 755 /usr/bin/armeb-linux-gnueabihf-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /armeb/bin/armeb-linux-$i /usr/bin/armeb-linux-gnueabihf-$i
+done
+
 # Install ARM64 big-endian toolchain
 mkdir /aarch64be
-wget -O- -q https://sources.buildroot.net/toolchain-external-arm-aarch64-be/arm-gnu-toolchain-13.3.rel1-x86_64-aarch64_be-none-linux-gnu.tar.xz | tar -C /aarch64be --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64be/tarballs/aarch64be--glibc--stable-2024.05-1.tar.xz | tar -C /aarch64be --strip-components=1 --xz -xf -
 
-ln -sf /aarch64be/aarch64_be-none-linux-gnu/libc /usr/aarch64_be-linux-gnu
-echo '/aarch64be/bin/aarch64_be-none-linux-gnu-gcc -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-gcc
-echo '/aarch64be/bin/aarch64_be-none-linux-gnu-g++ -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-g++
+ln -sf /aarch64be/aarch64_be-buildroot-linux-gnu/sysroot /usr/aarch64_be-linux-gnu
+echo '/aarch64be/bin/aarch64_be-linux-gcc -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-gcc
+echo '/aarch64be/bin/aarch64_be-linux-g++ -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-g++
 chmod 755 /usr/bin/aarch64_be-linux-gnu-{gcc,g++}
 
 for i in objdump objcopy strip; do
-  ln -sf /aarch64be/bin/aarch64_be-none-linux-gnu-$i /usr/bin/aarch64_be-linux-gnu-$i
+  ln -sf /aarch64be/bin/aarch64_be-linux-$i /usr/bin/aarch64_be-linux-gnu-$i
 done
 
 # Install SH4 big-endian toolchain
 mkdir /sh4aeb
-wget -O- -q https://toolchains.bootlin.com/downloads/releases/toolchains/sh-sh4aeb/tarballs/sh-sh4aeb--glibc--stable-2024.05-1.tar.xz | tar -C /sh4aeb --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/sh-sh4aeb/tarballs/sh-sh4aeb--glibc--stable-2024.05-1.tar.xz | tar -C /sh4aeb --strip-components=1 --xz -xf -
+
 ln -sf /sh4aeb/sh4aeb-buildroot-linux-gnu/sysroot /usr/sh4aeb-linux-gnu
 echo '/sh4aeb/bin/sh4aeb-linux-gcc -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-gcc
 echo '/sh4aeb/bin/sh4aeb-linux-g++ -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-g++
@@ -57,5 +71,5 @@ done
 
 # Install Intel SDE CPU emulator for CET-related tests
 mkdir /sde
-wget -O- -q https://downloadmirror.intel.com/843185/sde-external-9.48.0-2024-11-25-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://downloadmirror.intel.com/850782/sde-external-9.53.0-2025-03-16-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
 ln -s /sde/sde64 /usr/bin

.github/workflows/release-assets.yml

@@ -0,0 +1,104 @@
+name: Build & attach tarballs on tag push
+
+on:
+  push:
+    tags: ['v[0-9]*']
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+env:
+  TAG: ${{ github.ref_name }}
+
+jobs:
+  build-linux:
+    strategy:
+      matrix:
+        include:
+        - { target: x86_64,      os: ubuntu-24.04 }
+        - { target: aarch64,     os: ubuntu-24.04-arm }
+        - { target: arm,         os: ubuntu-24.04-arm }
+        - { target: riscv64,     os: ubuntu-24.04 }
+        - { target: ppc64le,     os: ubuntu-24.04 }
+        - { target: s390x,       os: ubuntu-24.04 }
+        - { target: loongarch64, os: ubuntu-24.04 }
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install Podman
+      run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
+
+    - name: Login to GitHub Container Registry
+      uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - run: ./dist.sh ${{ matrix.target }}
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: ${{ matrix.target }}
+        path: dist/mold-*.tar.gz
+
+  build-windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-x86_64-windows.zip
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*
+
+  publish:
+    needs: [build-linux, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - uses: actions/download-artifact@v8
+      with:
+        path: dist
+        pattern: '*'
+        merge-multiple: true
+
+    - name: Ensure release exists (create if missing)
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -e
+        if gh release view "$TAG" >& /dev/null; then
+          echo "Existing release found for $TAG"
+        else
+          echo "Creating draft release for $TAG"
+          gh release create "$TAG" --draft \
+            --title "$TAG" \
+            --notes "Automated draft – fill the changelog later."
+        fi
+
+    - name: Upload tarballs
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        gh release upload "$TAG" dist/* --clobber
+        gh release view "$TAG" --json assets --jq '.assets[].name'

.github/workflows/run-msan.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e -x
+cd "$(dirname $0)"/../..
+
+if [ "$GITHUB_REPOSITORY" = '' ]; then
+  image=mold-msan
+  image_build="podman build -t $image -"
+else
+  # If this script is running on GitHub Actions, we want to cache
+  # the created container image in GitHub's container repostiory.
+  image=ghcr.io/$GITHUB_REPOSITORY/mold-msan
+  image_build="podman build -t $image --output=type=registry --layers --cache-to $image --cache-from $image -"
+fi
+
+cat <<EOF | $image_build
+FROM mirror.gcr.io/library/ubuntu:24.04
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc g++ wget ca-certificates make cmake ninja-build python3 clang-20 llvm-20-dev libclang-20-dev libclang-rt-20-dev libtbb-dev mold && \
+  rm -rf /var/lib/apt/lists
+RUN mkdir /llvm && \
+  cd /llvm && \
+  wget -O- --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-21.1.1.tar.gz | tar xzf - --strip-components=1 && \
+  cmake -S ./runtimes -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi;libunwind' -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DLLVM_USE_SANITIZER=MemoryWithOrigins && \
+  cmake --build build -j\$(nproc) && \
+  cmake --install build --prefix /msan && \
+  rm -rf /llvm
+EOF
+
+# Build mold and run its tests
+podman run -it --rm --pull=never -v "$(pwd):/mold:ro" --security-opt seccomp=unconfined $image bash -c "
+set -e
+mkdir /build
+cd /build
+cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DMOLD_USE_MSAN=ON -DMOLD_STDLIB_PREFIX=/msan -DMOLD_USE_MIMALLOC=0 -DMOLD_USE_SYSTEM_TBB=1 -DMOLD_USE_MOLD=1 /mold
+cmake --build . -j\$(nproc)
+setarch -R ctest --output-on-failure -j\$(nproc)
+"

.github/workflows/update-manpage.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install ronn
       run: sudo apt-get update && sudo apt-get install -y ronn

CMakeLists.txt

@@ -44,7 +44,7 @@
 # features and behave exactly the same.
 
 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.37.1)
+project(mold VERSION 2.42.1)
 
 include(CMakeDependentOption)
 include(CheckSymbolExists)
@@ -62,7 +62,11 @@ endif()
 # Build mold itself using mold if -DMOLD_USE_MOLD=ON
 option(MOLD_USE_MOLD "Use mold to build mold" OFF)
 if(MOLD_USE_MOLD)
-  target_link_options(mold PRIVATE -fuse-ld=mold -Wl,--gdb-index)
+  target_link_options(mold PRIVATE -fuse-ld=mold)
+
+  if(CMAKE_BUILD_TYPE MATCHES "Deb")
+    target_link_options(mold PRIVATE -Wl,--gdb-index)
+  endif()
 
   if(CMAKE_BUILD_TYPE MATCHES "^Rel")
     target_link_options(mold PRIVATE -Wl,--gc-sections,--icf=safe)
@@ -113,6 +117,22 @@ if(MOLD_USE_TSAN)
   target_link_options(mold PRIVATE -fsanitize=thread)
 endif()
 
+# Enable MemorySanitizer if -DMOLD_USE_MSAN=ON
+option(MOLD_USE_MSAN "Build mold with MemorySanitizer" OFF)
+if(MOLD_USE_MSAN)
+  set(MOLD_STDLIB_PREFIX /usr CACHE FILEPATH "prefix of instrumented stdlib")
+  target_compile_options(mold PRIVATE
+    -g -Og -fsanitize=memory -fsanitize-memory-track-origins
+    -nostdinc++ -isystem ${MOLD_STDLIB_PREFIX}/include/c++/v1
+    -DENABLE_MSAN_UNPOISON
+  )
+  target_link_options(mold PRIVATE
+    -fsanitize=memory
+    -nostdlib++ -L ${MOLD_STDLIB_PREFIX}/lib -lc++
+    -Wl,-rpath,${MOLD_STDLIB_PREFIX}/lib
+  )
+endif()
+
 # Statically-link libstdc++ if -DMOLD_MOSTLY_STATIC=ON.
 #
 # This option is intended to be used by `./dist.sh` script to create a
@@ -129,10 +149,9 @@ find_package(ZLIB QUIET)
 if(ZLIB_FOUND AND NOT MOLD_MOSTLY_STATIC)
   target_link_libraries(mold PRIVATE ZLIB::ZLIB)
 else()
-  set(ZLIB_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_SHARED OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_TESTING OFF CACHE INTERNAL "")
   add_subdirectory(third-party/zlib EXCLUDE_FROM_ALL)
-  target_include_directories(zlibstatic INTERFACE third-party/zlib
-    $<TARGET_PROPERTY:zlibstatic,BINARY_DIR>)
   target_link_libraries(mold PRIVATE zlibstatic)
 endif()
 
@@ -160,13 +179,13 @@ check_include_file(zstd.h HAVE_ZSTD_H)
 if(HAVE_ZSTD_H AND NOT MOLD_MOSTLY_STATIC)
   target_link_libraries(mold PRIVATE zstd)
 else()
+  set(ZSTD_BUILD_PROGRAMS OFF)
+  set(ZSTD_BUILD_CONTRIB OFF)
+  set(ZSTD_BUILD_TESTS OFF)
+  set(ZSTD_MULTITHREAD_SUPPORT OFF)
+  set(ZSTD_BUILD_SHARED OFF)
+  set(ZSTD_BUILD_STATIC ON)
   add_subdirectory(third-party/zstd/build/cmake EXCLUDE_FROM_ALL)
-  target_compile_definitions(libzstd_static PRIVATE
-    ZSTD_BUILD_STATIC=1
-    ZSTD_BUILD_SHARED=0
-    ZSTD_BUILD_PROGRAMS=0
-    ZSTD_MULTITHREAD_SUPPORT=0
-    ZSTD_BUILD_TESTS=0)
   target_include_directories(mold PUBLIC third-party/zstd/lib)
   target_link_libraries(mold PRIVATE libzstd_static)
 endif()
@@ -189,14 +208,19 @@ cmake_dependent_option(
 # By default, we build a bundled mimalloc and statically-link it to
 # mold. If you want to dynamically link to the system's
 # libmimalloc.so, pass -DMOLD_USE_SYSTEM_MIMALLOC=ON.
+#
+# We require mimalloc v3 or newer. mimalloc v2 doesn't use transparent
+# huge pages for the allocation path that serves mold's largest data
+# structures, which makes linking large programs significantly slower.
 if(MOLD_USE_MIMALLOC)
   if(MOLD_USE_SYSTEM_MIMALLOC)
-    find_package(mimalloc REQUIRED)
+    find_package(mimalloc 3 REQUIRED)
     target_link_libraries(mold PRIVATE mimalloc)
   else()
     function(mold_add_mimalloc)
       set(MI_BUILD_STATIC ON CACHE INTERNAL "")
       set(MI_BUILD_TESTS OFF CACHE INTERNAL "")
+      set(MI_NO_OPT_ARCH ON CACHE INTERNAL "")
       add_subdirectory(third-party/mimalloc EXCLUDE_FROM_ALL)
       target_compile_definitions(mimalloc-static PRIVATE MI_USE_ENVIRON=0)
       target_link_libraries(mold PRIVATE mimalloc-static)
@@ -213,7 +237,7 @@ endif()
 # to mold. If you want to link to the system's libtbb2.so, pass
 # -DMOLD_USE_SYSTEM_TBB=ON.
 option(MOLD_USE_SYSTEM_TBB "Use system or vendored TBB" OFF)
-if(MOLD_USE_SYSTEM_TBB)
+if(MOLD_USE_SYSTEM_TBB OR BLAKE3_USE_TBB)
   find_package(TBB REQUIRED)
   target_link_libraries(mold PRIVATE TBB::tbb)
 else()
@@ -285,22 +309,25 @@ endif()
 check_symbol_exists(madvise sys/mman.h HAVE_MADVISE)
 check_symbol_exists(uname sys/utsname.h HAVE_UNAME)
 
+# glibc declares fallocate only under _GNU_SOURCE. C++ compilers
+# predefine it on Linux, but this check runs the C compiler, which
+# does not.
+set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+check_symbol_exists(fallocate fcntl.h HAVE_FALLOCATE)
+unset(CMAKE_REQUIRED_DEFINITIONS)
+
 # Create a .cc file containing the current git hash for `mold --version`.
 add_custom_target(git_hash
   COMMAND ${CMAKE_COMMAND}
     -DSOURCE_DIR=${CMAKE_SOURCE_DIR}
-    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/git-hash.cc
+    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/mold-git-hash.h
     -P ${CMAKE_SOURCE_DIR}/lib/update-git-hash.cmake
   DEPENDS lib/update-git-hash.cmake
-  BYPRODUCTS git-hash.cc
+  BYPRODUCTS mold-git-hash.h
   VERBATIM)
 
 add_dependencies(mold git_hash)
 
-# Create config.h file
-configure_file(lib/config.h.in config.h)
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
 # Almost all functions are template in mold which take a target type
 # (e.g. X86_64) as its type parameter. Since we suport more than 10
 # targets, compiling a single source file for all the targets is very
@@ -309,16 +336,38 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # As a workaround, we create a .cc file for each target and spawn many
 # compiler instances. This is hacky but greatly reduces compile time
 # on a multicore machine.
-list(APPEND MOLD_ELF_TARGETS
-  X86_64 I386 ARM64LE ARM64BE ARM32 RV32LE RV32BE RV64LE RV64BE PPC32
-  PPC64V1 PPC64V2 S390X SPARC64 M68K SH4LE SH4BE LOONGARCH32 LOONGARCH64)
+#
+# You can build mold for a specific set of targets by passing, e.g.,
+# -DMOLD_TARGETS='X86_64;ARM64LE', though this is strongly discouraged
+# for build reproducibility. Use this option only if you build mold
+# frequently for your personal use; otherwise, always build mold with all
+# targets enabled. We provide this flag in the trust that you will not
+# abuse it, so please don’t betray that trust.
+set(MOLD_TARGETS
+  X86_64 I386 ARM64LE ARM64BE ARM32LE ARM32BE RV32LE RV32BE RV64LE
+  RV64BE PPC32 PPC64V1 PPC64V2 S390X SPARC64 M68K SH4LE SH4BE
+  LOONGARCH32 LOONGARCH64
+  CACHE STRING "List of supported targets")
 
-list(APPEND MOLD_ELF_TEMPLATE_FILES
+list(GET MOLD_TARGETS 0 MOLD_FIRST_TARGET)
+
+list(APPEND MOLD_TEMPLATE_FILES
+  src/arch-arm32.cc
   src/arch-arm64.cc
+  src/arch-i386.cc
   src/arch-loongarch.cc
+  src/arch-m68k.cc
+  src/arch-ppc32.cc
+  src/arch-ppc64v1.cc
+  src/arch-ppc64v2.cc
   src/arch-riscv.cc
+  src/arch-s390x.cc
   src/arch-sh4.cc
+  src/arch-sparc64.cc
+  src/arch-x86-64.cc
+  src/archive-file.cc
   src/cmdline.cc
+  src/error.cc
   src/filetype.cc
   src/gc-sections.cc
   src/gdb-index.cc
@@ -337,18 +386,18 @@ list(APPEND MOLD_ELF_TEMPLATE_FILES
   )
 
 if(WIN32 AND NOT MINGW)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-win32.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-win32.cc)
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-unix.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-unix.cc)
 endif()
 
 if(WIN32)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
     src/output-file-win32.cc
     src/subprocess-win32.cc
     )
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
     src/output-file-unix.cc
     src/subprocess-unix.cc
   )
@@ -365,54 +414,50 @@ function(mold_instantiate_templates SOURCE TARGET)
   target_sources(mold PRIVATE ${PATH})
 endfunction()
 
-foreach (SOURCE IN LISTS MOLD_ELF_TEMPLATE_FILES)
-  foreach(TARGET IN LISTS MOLD_ELF_TARGETS)
+foreach (SOURCE IN LISTS MOLD_TEMPLATE_FILES)
+  foreach(TARGET IN LISTS MOLD_TARGETS)
     mold_instantiate_templates(${SOURCE} ${TARGET})
+    set(HAVE_TARGET_${TARGET} 1)
   endforeach()
 endforeach()
 
 # Add other non-template source files.
 target_sources(mold PRIVATE
-  git-hash.cc
+  lib/aho-corasick.cc
+  lib/arena.cc
   lib/compress.cc
   lib/crc32.cc
   lib/demangle.cc
   lib/filepath.cc
   lib/glob.cc
   lib/hyperloglog.cc
-  lib/mimalloc.cc
-  lib/multi-glob.cc
   lib/perf.cc
   lib/random.cc
   lib/tar.cc
-  src/arch-arm32.cc
-  src/arch-i386.cc
-  src/arch-m68k.cc
-  src/arch-ppc32.cc
-  src/arch-ppc64v1.cc
-  src/arch-ppc64v2.cc
-  src/arch-s390x.cc
-  src/arch-sparc64.cc
-  src/arch-x86-64.cc
-  src/config.cc
   src/elf.cc
+  src/entry.cc
   third-party/rust-demangle/rust-demangle.c
   )
 
 if(WIN32)
   target_sources(mold PRIVATE
-    lib/jobs-win32.cc
-    lib/mapped-file-win32.cc
-    lib/signal-win32.cc
+    src/jobs-win32.cc
+    src/mapped-file-win32.cc
+    src/signal-win32.cc
     )
 else()
   target_sources(mold PRIVATE
-    lib/jobs-unix.cc
-    lib/mapped-file-unix.cc
-    lib/signal-unix.cc
+    src/jobs-unix.cc
+    src/mapped-file-unix.cc
+    src/signal-unix.cc
     )
 endif()
 
+# Create config.h file
+configure_file(lib/config.h.in config.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+# Test configs
 include(CTest)
 
 if(BUILD_TESTING)
@@ -448,10 +493,12 @@ if(NOT CMAKE_SKIP_INSTALL_RULES)
       file(CREATE_LINK \${OLD_REL} \$ENV{DESTDIR}\${NEW_ABS} SYMBOLIC)")
   endfunction()
 
-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
-    ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  if(NOT WIN32)
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
+      ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  endif()
 endif()

README.md

@@ -1,30 +1,21 @@
 # mold: A Modern Linker
 
-mold is a faster drop-in replacement for existing Unix linkers. It is several
-times quicker than the LLVM lld linker, the second-fastest open-source linker,
-which I initially developed a few years ago. mold aims to enhance developer
-productivity by minimizing build time, particularly in rapid
-debug-edit-rebuild cycles.
+mold is a high-performance drop-in replacement for existing Unix linkers,
+designed to speed up builds. In our August 2026 benchmarks, it links 4.9x
+faster than [LLVM lld](https://lld.llvm.org) and 1.9x faster than
+[wild](https://github.com/wild-linker/wild) at the median; see
+[Benchmark](#benchmark) for the full results.
 
-Here is a performance comparison of GNU ld, GNU gold, LLVM lld, and
-mold when linking final debuginfo-enabled executables for major large
-programs on a simulated 16-core, 32-thread machine.
+mold is written by the original developer of LLVM lld, the linker that
+Android, Chrome, FreeBSD, PlayStation, Nintendo Switch, and other production
+systems are built with. mold started as an effort to build an even faster
+linker from scratch, free of the architectural limits its author had run into
+while optimizing lld. It has been in production use since 2021, and today it
+is the default linker of many large open-source projects and is used
+internally by many companies.
 
-![Link speed comparison](docs/chart.svg)
-
-| Program (linker output size)  | GNU ld | GNU gold | LLVM lld | mold
-|-------------------------------|--------|----------|----------|------
-| MySQL 8.3 (0.47 GiB)          | 10.84s | 7.47s    | 1.64s    | 0.46s
-| Clang 19 (1.56 GiB)           | 42.07s | 33.13s   | 5.20s    | 1.35s
-| Chromium 124 (1.35 GiB)       | N/A    | 27.40s   | 6.10s    | 1.52s
-
-mold is so fast that it is only 2x _slower_ than the `cp` command on the same
-machine. If you find that mold is not faster than other linkers, feel
-free to [file a bug report](https://github.com/rui314/mold/issues).
-
-mold supports x86-64, i386, ARM64, ARM32, 64-bit/32-bit little/big-endian
-RISC-V, 32-bit PowerPC, 64-bit big-endian PowerPC ELFv1, 64-bit little-endian
-PowerPC ELFv2, s390x, 64-bit/32-bit LoongArch, SPARC64, m68k, and SH-4.
+mold supports x86-64, i386, ARM 32/64, RISC-V 32/64, PowerPC 32/64, s390x,
+LoongArch 32/64, SPARC64, m68k, and SH-4.
 
 ## Why does linking speed matter?
 
@@ -38,30 +29,124 @@ speed up this process, saving you time and preventing distractions while
 waiting for a lengthy build to finish. The difference is most noticeable
 during rapid debug-edit-rebuild cycles.
 
+## Benchmark
+
+Here is a performance comparison of lld, wild, and mold when linking nine
+large programs on two machines:
+
+- AMD Ryzen Threadripper 7980X (64 cores) running Ubuntu 24.04
+- Apple M1 Ultra (16 performance cores and 4 efficiency cores) running Fedora
+  Asahi Remix 42; the benchmark is restricted to the performance cores
+
+The Threadripper represents many-core workstation and server processors, and
+the M1 Ultra represents high-performance desktop processors.
+
+All three linkers were built from source in release configuration as of
+2026-08-28 and ran with their default options. Times are wall-clock times as
+seen by the caller, the median of three runs after a warm-up. The ARM64 rows
+are the ARM64 builds of the same programs. The benchmark suite, including all
+linker inputs, is available on [Zenodo](https://zenodo.org/records/21882261).
+
+**AMD Ryzen Threadripper 7980X, debug builds**
+
+| Program (output size)             | lld    | wild  | mold  | wild/mold | lld/mold
+|-----------------------------------|--------|-------|-------|-----------|---------
+| Blender 5.2 (2.46 GiB)            | 4.72s  | 1.98s | 0.86s | 2.3x      | 5.5x
+| Chromium 145 (4.51 GiB)           | 16.64s | 3.98s | 1.65s | 2.4x      | 10.1x
+| Chromium 145 ARM64 (4.76 GiB)     | 20.91s | N/A   | 1.83s | N/A       | 11.4x
+| Clang 21 (4.19 GiB)               | 6.19s  | 3.71s | 1.34s | 2.8x      | 4.6x
+| ClickHouse 26.1 (5.58 GiB)        | 6.62s  | 4.40s | 0.98s | 4.5x      | 6.7x
+| Firefox 149 (2.37 GiB)            | 5.11s  | N/A   | 0.78s | N/A       | 6.5x
+| Firefox 149 ARM64 (2.43 GiB)      | 6.40s  | N/A   | 0.93s | N/A       | 6.9x
+| Godot 4.6 (1.01 GiB)              | 1.77s  | 1.08s | 0.46s | 2.3x      | 3.8x
+| LibreOffice 26.2 (0.98 GiB)       | 3.46s  | 1.41s | 0.44s | 3.2x      | 7.9x
+| PyTorch 2.9 (3.51 GiB)            | 4.35s  | 2.48s | 0.80s | 3.1x      | 5.4x
+| TensorFlow 2.21 (9.55 GiB)        | 50.73s | N/A   | 3.15s | N/A       | 16.1x
+
+**AMD Ryzen Threadripper 7980X, release builds**
+
+| Program (output size)             | lld   | wild  | mold  | wild/mold | lld/mold
+|-----------------------------------|-------|-------|-------|-----------|---------
+| Blender 5.2 (0.24 GiB)            | 0.85s | 0.30s | 0.20s | 1.5x      | 4.2x
+| Chromium 145 (0.58 GiB)           | 6.48s | N/A   | 0.64s | N/A       | 10.2x
+| Chromium 145 ARM64 (0.60 GiB)     | 7.91s | N/A   | 0.73s | N/A       | 10.8x
+| Clang 21 (0.21 GiB)               | 0.53s | 0.23s | 0.11s | 2.2x      | 5.0x
+| ClickHouse 26.1 (1.23 GiB)        | 3.18s | 2.07s | 0.41s | 5.0x      | 7.7x
+| Firefox 149 (0.22 GiB)            | 1.01s | 0.41s | 0.21s | 2.0x      | 4.9x
+| Godot 4.6 (0.15 GiB)              | 0.44s | 0.21s | 0.08s | 2.7x      | 5.8x
+| LibreOffice 26.2 (0.19 GiB)       | 1.13s | 0.57s | 0.19s | 3.0x      | 6.0x
+| PyTorch 2.9 (0.31 GiB)            | 0.68s | 0.32s | 0.15s | 2.2x      | 4.7x
+| TensorFlow 2.21 (0.73 GiB)        | 9.62s | N/A   | 0.70s | N/A       | 13.7x
+
+**Apple M1 Ultra, debug builds**
+
+| Program (output size)             | lld    | wild  | mold  | wild/mold | lld/mold
+|-----------------------------------|--------|-------|-------|-----------|---------
+| Blender 5.2 (2.46 GiB)            | 3.21s  | 1.81s | 1.56s | 1.2x      | 2.1x
+| Chromium 145 (4.51 GiB)           | 9.54s  | 3.49s | 2.22s | 1.6x      | 4.3x
+| Chromium 145 ARM64 (4.76 GiB)     | 12.67s | N/A   | 2.31s | N/A       | 5.5x
+| Clang 21 (4.19 GiB)               | 4.40s  | 2.78s | 2.96s | 0.9x      | 1.5x
+| ClickHouse 26.1 (5.58 GiB)        | 4.92s  | 3.50s | 1.71s | 2.1x      | 2.9x
+| Firefox 149 (2.37 GiB)            | 3.21s  | N/A   | 1.12s | N/A       | 2.9x
+| Firefox 149 ARM64 (2.43 GiB)      | 4.13s  | N/A   | 1.12s | N/A       | 3.7x
+| Godot 4.6 (1.01 GiB)              | 1.12s  | 0.81s | 0.62s | 1.3x      | 1.8x
+| LibreOffice 26.2 (0.98 GiB)       | 2.08s  | 0.94s | 0.63s | 1.5x      | 3.3x
+| PyTorch 2.9 (3.51 GiB)            | 3.07s  | 2.10s | 1.45s | 1.4x      | 2.1x
+| TensorFlow 2.21 (9.55 GiB)        | 43.68s | N/A   | 4.43s | N/A       | 9.9x
+
+**Apple M1 Ultra, release builds**
+
+| Program (output size)             | lld   | wild  | mold  | wild/mold | lld/mold
+|-----------------------------------|-------|-------|-------|-----------|---------
+| Blender 5.2 (0.24 GiB)            | 0.56s | 0.20s | 0.25s | 0.8x      | 2.3x
+| Chromium 145 (0.58 GiB)           | 4.25s | N/A   | 0.78s | N/A       | 5.5x
+| Chromium 145 ARM64 (0.60 GiB)     | 5.17s | N/A   | 0.91s | N/A       | 5.7x
+| Clang 21 (0.21 GiB)               | 0.30s | 0.15s | 0.14s | 1.0x      | 2.1x
+| ClickHouse 26.1 (1.23 GiB)        | 1.94s | 0.95s | 0.55s | 1.7x      | 3.5x
+| Firefox 149 (0.22 GiB)            | 0.56s | 0.23s | 0.20s | 1.1x      | 2.7x
+| Godot 4.6 (0.15 GiB)              | 0.26s | 0.11s | 0.11s | 1.0x      | 2.3x
+| LibreOffice 26.2 (0.19 GiB)       | 0.62s | 0.30s | 0.23s | 1.3x      | 2.7x
+| PyTorch 2.9 (0.31 GiB)            | 0.43s | 0.22s | 0.17s | 1.3x      | 2.5x
+| TensorFlow 2.21 (0.73 GiB)        | 8.07s | N/A   | 0.68s | N/A       | 11.9x
+
+N/A indicates that the linker cannot link that program. wild's Chromium
+release links are also marked N/A because wild does not implement `--icf=all`
+and links without identical code folding.
+
+## Why is mold so fast?
+
+mold owes its speed to pervasive parallelism and to efficient data structures
+and algorithms. For details, read our paper "mold: A Massively Parallel
+Linker" (ASPLOS 2027), available at https://arxiv.org/abs/2608.23228.
+
 ## Installation
 
 Binary packages for the following systems are currently available:
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/mold.svg)](https://repology.org/project/mold/versions)
 
+Prebuilt binaries for Linux on x86-64, ARM64, ARM32, RISC-V, PPC64LE, s390x,
+and LoongArch are also attached to each
+[GitHub release](https://github.com/rui314/mold/releases).
+
 ## How to Build
 
 mold is written in C++20, so if you build mold yourself, you will need a
 recent version of a C++ compiler and a C++ standard library. We recommend GCC
-10.2 or Clang 12.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).
+10.2 or Clang 16.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).
 
 ### Install Dependencies
 
 To install build dependencies, run `./install-build-deps.sh` in this
 directory. It will detect your Linux distribution and attempt to install the
-necessary packages. You may need to run it as root.
+necessary packages.
 
 ### Compile mold
 
 ```shell
 git clone --branch stable https://github.com/rui314/mold.git
 cd mold
-./install-build-deps.sh
+sudo ./install-build-deps.sh
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ -B build
 cmake --build build -j$(nproc)
 sudo cmake --build build --target install
@@ -120,7 +205,7 @@ accept an absolute path as an argument for `-fuse-ld`.
 Create `.cargo/config.toml` in your project directory with the following:
 
 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 linker = "clang"
 rustflags = ["-C", "link-arg=-fuse-ld=/path/to/mold"]
 ```
@@ -131,7 +216,7 @@ example above, we use `clang` as a linker driver since it always accepts the
 may be able to remove the `linker = "clang"` line.
 
 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 rustflags = ["-C", "link-arg=-fuse-ld=mold"]
 ```
 
@@ -219,39 +304,26 @@ is available online. You can read the same manual by running `man mold`.
 
 </details>
 
-## Why is mold so fast?
+## Stability
 
-One reason is that it utilizes faster algorithms and more efficient data
-structures compared to other linkers. Another reason is that mold is highly
-parallelized.
-
-Here is a side-by-side comparison of per-core CPU usage for lld (left) and
-mold (right), linking the same program, a Chromium executable.
-
-![CPU usage comparison in htop animation](docs/htop.gif)
-
-As you can see, mold uses all available cores throughout its execution and
-finishes quickly. In contrast, lld fails to utilize available cores most of
-the time. In this demo, the maximum parallelism is artificially capped at 16,
-so that the bars fit in the GIF.
-
-For details, please see the [design notes](docs/design.md).
+mold has been developed in the open since 2020 and has more than 140
+contributors. Its test suite, which covers every linker feature, runs in CI for
+all supported target CPU architectures, natively or under QEMU, and under ASAN
+and TSAN. Before each release, we try to build all of Gentoo Linux's roughly
+19,000 packages with mold, using GNU ld as a control, to find regressions before
+they reach a release.
 
 ## Sponsors
 
-It is taken for granted nowadays that compiler toolchains can be easily
-installed and used for free, and people may not think too much about the
-individuals behind these "free tools". mold supports many projects, but it
-is essentially a one-person project. This situation is similar to the one
-depicted in the following xkcd illustration.
+mold is free to use, but keeping it maintained is continuous work: supporting
+new architectures and toolchain features, keeping up with the projects that
+depend on it, and making it faster. That work is funded by sponsors. If mold
+saves you or your company time, please consider becoming a
+[GitHub sponsor](https://github.com/sponsors/rui314).
 
-[![xkcd 2347](https://imgs.xkcd.com/comics/dependency.png)](https://xkcd.com/2347)
-
-If you think that the "Nebraska guy" should be rewarded, please consider
-becoming our [GitHub sponsor](https://github.com/sponsors/rui314)!
-
-We thank everyone who sponsors our project. In particular, we'd like to acknowledge
-the following people and organizations who have sponsored $128/month or more:
+We thank everyone who sponsors the project. In particular, we'd like to
+acknowledge the following people and organizations who have sponsored
+$128/month or more:
 
 ### Corporate sponsors
 

dist.sh

@@ -1,40 +1,45 @@
 #!/bin/bash
 #
-# This script creates a mold binary distribution. The output is written in
-# `dist` directory as `mold-$version-$arch-linux.tar.gz` (e.g.,
-# `mold-1.0.3-x86_64-linux.tar.gz`).
+# This script creates a mold binary distribution. The output is written to
+# the `dist` directory as `mold-$version-$arch-linux.tar.gz` (e.g.
+# `mold-2.40.0-x86_64-linux.tar.gz`).
 #
-# The mold executable created by this script is statically linked to
-# libstdc++ but dynamically linked to libc, libm, libz, and librt, as
-# these libraries almost always exist on any Linux system. We can't
-# statically link libc because doing so would disable dlopen(), which is
-# necessary to open the LTO linker plugin.
-#
-# This script aims to produce reproducible outputs. That means if you run
-# the script twice on the same git commit, it should produce bit-by-bit
-# identical binary files. This property is crucial as a countermeasure
-# against supply chain attacks. With this, you can verify that the binary
-# files distributed on the GitHub release pages are created from the
-# commit with release tags by rebuilding the binaries yourself.
+# This script aims to produce reproducible outputs. That means each time
+# it's run on the same git commit, it generates a bit-for-bit identical
+# binary file regardless of when or where it's executed. This property
+# serves as a strong safeguard against supply chain attacks. With a
+# reproducible build, anyone can independently verify that the binary
+# files published on our GitHub release page were built from the git
+# commit tagged for release by rebuilding the binaries themselves.
 #
 # Debian provides snapshot.debian.org to host all historical binary
-# packages. We use it to construct Podman images pinned to a
-# particular timestamp.
+# packages. We use it to construct a container image pinned to a
+# particular timestamp. snapshot.debian.org is known to be very slow,
+# but that shouldn't be a big problem for us because we only need that
+# site the first time.
 #
-# We aim to use a reasonably old Debian version because we'll dynamically
-# link glibc to mold, and a binary linked against a newer version of glibc
-# won't work on a system with an older version of glibc.
+# The mold executable created by this script is statically linked to
+# libstdc++, but dynamically linked to glibc, libm and a few other
+# libraries, as these libraries are almost always available on any Linux
+# system. We can't statically link glibc because doing so would disable
+# dlopen(), which is required to load the LTO linker plugin.
 #
-# We want to build mold with Clang rather than GCC because mold's
+# We use a reasonably old Debian version for the build environment because
+# a binary dynamically linked against a newer version of glibc won't work
+# on a system with an older version of glibc.
+#
+# We prefer to build mold with Clang rather than GCC because mold's
 # Identical Code Folding works best with the LLVM address significance
-# table (.llvm_addrsig). Building a release binary with GCC yields a
-# slightly larger binary than Clang's.
+# table (.llvm_addrsig). Building a release binary with GCC produces a
+# slightly larger binary than with Clang.
 #
 # We need a recent version of Clang to build mold. If it's not available
 # via apt-get, we'll build it ourselves.
 #
-# You may need to install qemu-user-static package to build non-native
-# binaries.
+# This script can be used to create non-native binaries (e.g., building
+# aarch64 binary on x86-64) because Podman automatically runs everything
+# under QEMU if the container image is not native. To use this script for
+# non-native builds, you may need to install the qemu-user-static package.
 
 set -e -x
 cd "$(dirname $0)"
@@ -74,21 +79,34 @@ fi
 case $arch in
 x86_64)
   # Debian 9 (Stretch) released in June 2017.
-  # We use a Google-provided mirror (gcr.io) of the official Docker hub
-  # (docker.io) because docker.io has a strict rate limit policy.
+  #
+  # We use a Google-provided mirror (gcr.io) instead of the official Docker
+  # Hub (docker.io) because docker.io has a strict rate limit policy.
+  #
+  # The toolchain in Debian 9 is too old to build mold, so we rebuild it
+  # from source. We download source archives from official sites and build
+  # them locally, rather than downloading pre-built binaries from somewhere
+  # else, to avoid relying on unverifiable third-party binary blobs. Each
+  # archive is checked against the SHA-256 hash recorded below before it is
+  # unpacked, so this git commit alone determines every input of the build.
+  # A compromised network or mirror can only make the build fail, never
+  # change its output. Podman caches the result of each RUN command, so
+  # rebuilding is done only once per host.
   cat <<EOF | $image_build
 FROM mirror.gcr.io/library/debian:stretch@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends wget file make gcc g++ zlib1g-dev libssl-dev && \
+  apt-get install -y --no-install-recommends wget xz-utils file make gcc g++ git zlib1g-dev libssl-dev ca-certificates && \
   rm -rf /var/lib/apt/lists
 
 # Build CMake 3.27
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
+  wget --progress=dot:mega https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz && \
+  echo '08f71a106036bf051f692760ef9558c0577c42ac39e96ba097e7662bd4158d8e cmake-3.27.7.tar.gz' | sha256sum -c && \
+  tar xf cmake-3.27.7.tar.gz --strip-components=1 && \
   ./bootstrap --parallel=\$(nproc) && \
   make -j\$(nproc) && \
   make install && \
@@ -97,11 +115,19 @@ RUN mkdir /build && \
 # Build GCC 14
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
+  wget --progress=dot:mega https://ftp.gnu.org/gnu/gcc/gcc-14.2.0/gcc-14.2.0.tar.xz && \
+  echo 'a7b39bc69cbf9e25826c5a60ab26477001f7c08d85cec04bc0e29cabed6f3cc9 gcc-14.2.0.tar.xz' | sha256sum -c && \
+  tar xf gcc-14.2.0.tar.xz --strip-components=1 && \
   mkdir gmp mpc mpfr && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
+  wget --progress=dot:mega https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.xz && \
+  echo 'a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898 gmp-6.3.0.tar.xz' | sha256sum -c && \
+  tar xf gmp-6.3.0.tar.xz --strip-components=1 -C gmp && \
+  wget --progress=dot:mega https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz && \
+  echo 'ab642492f5cf882b74aa0cb730cd410a81edcdbec895183ce930e706c1c759b8 mpc-1.3.1.tar.gz' | sha256sum -c && \
+  tar xf mpc-1.3.1.tar.gz --strip-components=1 -C mpc && \
+  wget --progress=dot:mega https://ftp.gnu.org/gnu/mpfr/mpfr-4.2.1.tar.xz && \
+  echo '277807353a6726978996945af13e52829e3abd7a9a5b7fb2793894e18f1fcbb2 mpfr-4.2.1.tar.xz' | sha256sum -c && \
+  tar xf mpfr-4.2.1.tar.xz --strip-components=1 -C mpfr && \
   ./configure --prefix=/usr --enable-languages=c,c++ --disable-bootstrap --disable-multilib && \
   make -j\$(nproc) && \
   make install && \
@@ -111,7 +137,9 @@ RUN mkdir /build && \
 # Build GNU binutils 2.43
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
+  wget --progress=dot:mega https://ftp.gnu.org/gnu/binutils/binutils-2.43.tar.xz && \
+  echo 'b53606f443ac8f01d1d5fc9c39497f2af322d99e14cea5c0b4b124d630379365 binutils-2.43.tar.xz' | sha256sum -c && \
+  tar xf binutils-2.43.tar.xz --strip-components=1 && \
   ./configure --prefix=/usr && \
   make -j\$(nproc) && \
   make install && \
@@ -120,16 +148,20 @@ RUN mkdir /build && \
 # Build Python 3.12.7
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz | tar xzf - --strip-components=1 && \
+  wget --progress=dot:mega https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tar.xz && \
+  echo '24887b92e2afd4a2ac602419ad4b596372f67ac9b077190f459aba390faf5550 Python-3.12.7.tar.xz' | sha256sum -c && \
+  tar xf Python-3.12.7.tar.xz --strip-components=1 && \
   ./configure && \
   make -j\$(nproc) && \
   make install && \
   rm -rf /build
 
-# Build LLVM 19
+# Build LLVM 20
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-19.1.7.tar.gz | tar xzf - --strip-components=1 && \
+  wget --progress=dot:mega https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.3/llvm-project-20.1.3.src.tar.xz && \
+  echo 'b6183c41281ee3f23da7fda790c6d4f5877aed103d1e759763b1008bdd0e2c50 llvm-project-20.1.3.src.tar.xz' | sha256sum -c && \
+  tar xf llvm-project-20.1.3.src.tar.xz --strip-components=1 && \
   mkdir b && \
   cd b && \
   cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang ../llvm && \
@@ -141,16 +173,16 @@ EOF
 aarch64 | arm | ppc64le | s390x)
   # Debian 11 (Bullseye) released in August 2021
   #
-  # We don't want to build Clang for these targets with Qemu becuase
-  # that'd take extremely long time. Also I believe old build machines
-  # are usually x86-64.
+  # We don't want to build Clang for these targets on QEMU becuase it
+  # would take an extremely long time. Also, I believe old Linux boxes
+  # are typically x86-64.
   cat <<EOF | $image_build
 FROM mirror.gcr.io/library/debian:bullseye-20240904@sha256:8ccc486c29a3ad02ad5af7f1156e2152dff3ba5634eec9be375269ef123457d8
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake git && \
   ln -sf /usr/bin/clang-16 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-16 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -163,19 +195,20 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^URIs/d' -e 's/^# http/URIs: http/' /etc/apt/sources.list.d/debian.sources && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake git && \
   ln -sf /usr/bin/clang-18 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-18 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
 EOF
   ;;
 loongarch64)
-  # LoongArch build is not reproducible yet
   cat <<EOF | $image_build
-FROM mirror.gcr.io/loongarch64/debian:sid
+FROM mirror.gcr.io/loongarch64/debian:sid@sha256:0356df4e494bbb86bb469377a00789a5b42bbf67d5ff649a3f9721b745cbef77
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake && \
+RUN echo 'deb http://deb.debian.org/debian sid main' > /etc/apt/sources.list && \
+  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake git && \
   ln -sf /usr/bin/clang-19 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-19 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -189,20 +222,34 @@ esac
 version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' CMakeLists.txt)
 dest=mold-$version-$arch-linux
 
-# Source tarballs available on GitHub don't contain .git history.
+# Source tarballs available on GitHub don't contain .git directory.
 # Clone the repo if missing.
 [ -d .git ] || git clone --branch v$version --depth 1 --bare https://github.com/rui314/mold .git
 
 # We use the timestamp of the last Git commit as the file timestamp
 # for build artifacts.
-timestamp="$(git log -1 --format=%ci)"
+timestamp=$(git log -1 --format=%ct)
+
+# `uname -m` in an ARM32 container running on an ARM64 host reports it
+# not as ARM32 but as ARM64. That confuses BLAKE3's cmake script and
+# erroneously enables NEON SIMD instructions. `setarch` can be used to
+# change the output of `uname -m`.
+setarch=
+[ $arch = arm ] && setarch='setarch linux32'
 
-# Build mold in a container.
 mkdir -p dist
 
+# Build mold in a container.
+#
+# SOURCE_DATE_EPOCH is a standardized environment variable that allows
+# build artifacts to appear as if they were built at a specific time.
+# We use it to control how the compiler expands the C/C++ __DATE__ and
+# __TIME__ macros.
 podman run --arch $arch -it --rm --userns=host --pids-limit=-1 --network=none \
-  -v "$(pwd):/mold:ro" -v "$(pwd)/dist:/dist" $image bash -c "
+  --pull=never -v "$(pwd):/mold:ro" -v "$(pwd)/dist:/dist" $image \
+   $setarch bash -c "
 set -e
+export SOURCE_DATE_EPOCH=$timestamp
 mkdir /build
 cd /build
 cmake -DCMAKE_BUILD_TYPE=Release -DMOLD_MOSTLY_STATIC=1 -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ /mold
@@ -212,7 +259,7 @@ cmake -DMOLD_USE_MOLD=1 .
 cmake --build . -j\$(nproc)
 ctest --output-on-failure -j\$(nproc)
 cmake --install . --prefix $dest --strip
-find $dest -print | xargs touch --no-dereference --date='$timestamp'
+find $dest -print | xargs touch --no-dereference --date=@$timestamp
 find $dest -print | sort | tar -cf - --no-recursion --files-from=- | gzip -9nc > /dist/$dest.tar.gz
 cp mold /dist
 sha256sum /dist/$dest.tar.gz

docs/chart.svg

@@ -1 +0,0 @@
-<svg version="1.1" viewBox="0.0 0.0 600.0 371.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" width="600" height="371" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><path fill="#ffffff" d="M0 0L600.0 0L600.0 371.0L0 371.0L0 0Z" fill-rule="nonzero"/><path stroke="#333333" stroke-width="1.0" stroke-linecap="butt" d="M82.5 275.5L581.5 275.5" fill-rule="nonzero"/><path stroke="#cccccc" stroke-width="1.0" stroke-linecap="butt" d="M82.5 231.5L581.5 231.5" fill-rule="nonzero"/><path stroke="#cccccc" stroke-width="1.0" stroke-linecap="butt" d="M82.5 186.5L581.5 186.5" fill-rule="nonzero"/><path stroke="#cccccc" stroke-width="1.0" stroke-linecap="butt" d="M82.5 142.5L581.5 142.5" fill-rule="nonzero"/><path stroke="#cccccc" stroke-width="1.0" stroke-linecap="butt" d="M82.5 98.5L581.5 98.5" fill-rule="nonzero"/><path stroke="#cccccc" stroke-width="1.0" stroke-linecap="butt" d="M82.5 53.5L581.5 53.5" fill-rule="nonzero"/><clipPath id="id_0"><path d="M82.55 53.916668L581.45 53.916668L581.45 275.45L82.55 275.45L82.55 53.916668Z" clip-rule="nonzero"/></clipPath><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M158.0 275.0L136.0 275.0L136.0 229.0C136.0 227.89543 136.89543 227.0 138.0 227.0L156.0 227.0C157.10457 227.0 158.0 227.89543 158.0 229.0Z" fill-rule="nonzero"/><path fill="#4285f4" clip-path="url(#id_0)" d="M158.0 275.0L136.0 275.0L136.0 229.0C136.0 227.89543 136.89543 227.0 138.0 227.0L156.0 227.0C157.10457 227.0 158.0 227.89543 158.0 229.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M307.0 275.0L285.0 275.0L285.0 91.0C285.0 89.89543 285.89542 89.0 287.0 89.0L305.0 89.0C306.10458 89.0 307.0 89.89543 307.0 91.0Z" fill-rule="nonzero"/><path fill="#4285f4" clip-path="url(#id_0)" d="M307.0 275.0L285.0 275.0L285.0 91.0C285.0 89.89543 285.89542 89.0 287.0 89.0L305.0 89.0C306.10458 89.0 307.0 89.89543 307.0 91.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M182.0 275.0L160.0 275.0L160.0 244.0C160.0 242.89543 160.89543 242.0 162.0 242.0L180.0 242.0C181.10457 242.0 182.0 242.89543 182.0 244.0Z" fill-rule="nonzero"/><path fill="#ea4335" clip-path="url(#id_0)" d="M182.0 275.0L160.0 275.0L160.0 244.0C160.0 242.89543 160.89543 242.0 162.0 242.0L180.0 242.0C181.10457 242.0 182.0 242.89543 182.0 244.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M331.0 275.0L309.0 275.0L309.0 131.0C309.0 129.89543 309.89542 129.0 311.0 129.0L329.0 129.0C330.10458 129.0 331.0 129.89543 331.0 131.0Z" fill-rule="nonzero"/><path fill="#ea4335" clip-path="url(#id_0)" d="M331.0 275.0L309.0 275.0L309.0 131.0C309.0 129.89543 309.89542 129.0 311.0 129.0L329.0 129.0C330.10458 129.0 331.0 129.89543 331.0 131.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M480.0 275.0L458.0 275.0L458.0 156.0C458.0 154.89543 458.89542 154.0 460.0 154.0L478.0 154.0C479.10458 154.0 480.0 154.89543 480.0 156.0Z" fill-rule="nonzero"/><path fill="#ea4335" clip-path="url(#id_0)" d="M480.0 275.0L458.0 275.0L458.0 156.0C458.0 154.89543 458.89542 154.0 460.0 154.0L478.0 154.0C479.10458 154.0 480.0 154.89543 480.0 156.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M206.0 275.0L184.0 275.0L184.0 270.0C184.0 268.89542 184.89543 268.0 186.0 268.0L204.0 268.0C205.10457 268.0 206.0 268.89542 206.0 270.0Z" fill-rule="nonzero"/><path fill="#fbbc04" clip-path="url(#id_0)" d="M206.0 275.0L184.0 275.0L184.0 270.0C184.0 268.89542 184.89543 268.0 186.0 268.0L204.0 268.0C205.10457 268.0 206.0 268.89542 206.0 270.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M355.0 275.0L333.0 275.0L333.0 254.0C333.0 252.89543 333.89542 252.0 335.0 252.0L353.0 252.0C354.10458 252.0 355.0 252.89543 355.0 254.0Z" fill-rule="nonzero"/><path fill="#fbbc04" clip-path="url(#id_0)" d="M355.0 275.0L333.0 275.0L333.0 254.0C333.0 252.89543 333.89542 252.0 335.0 252.0L353.0 252.0C354.10458 252.0 355.0 252.89543 355.0 254.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroke-opacity="0.0" clip-path="url(#id_0)" d="M504.0 275.0L482.0 275.0L482.0 250.0C482.0 248.89543 482.89542 248.0 484.0 248.0L502.0 248.0C503.10458 248.0 504.0 248.89543 504.0 250.0Z" fill-rule="nonzero"/><path fill="#fbbc04" clip-path="url(#id_0)" d="M504.0 275.0L482.0 275.0L482.0 250.0C482.0 248.89543 482.89542 248.0 484.0 248.0L502.0 248.0C503.10458 248.0 504.0 248.89543 504.0 250.0Z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" stroLine truncated

docs/memory-sanitizer.md

@@ -0,0 +1,179 @@
+# Instrumenting mold with MemorySanitizer
+
+## Introduction
+
+Per <https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo>:
+
+> If you want MemorySanitizer to work properly and not produce any false
+> positives, you must ensure that all the code in your program and in
+> libraries it uses is instrumented (i.e. built with `-fsanitize=memory`).
+> In particular, you would need to link against an MSan-instrumented C++
+> standard library. We recommend to use [libc++](https://libcxx.llvm.org/)
+> for that purpose.
+
+## Building instrumented libc++
+
+Build an MSan-instrumented libc++ from source:
+
+```sh
+cd ~
+git clone https://github.com/llvm/llvm-project
+cd llvm-project
+cmake -S ./runtimes -B build-libcxx -G Ninja  \
+    -DCMAKE_BUILD_TYPE=Release                \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+    -DCMAKE_C_COMPILER=clang                  \
+    -DCMAKE_CXX_COMPILER=clang++              \
+    -DLLVM_USE_SANITIZER=MemoryWithOrigins
+cmake --build build-libcxx -- cxx cxxabi
+export LIBCXX="$HOME/llvm-project/build-libcxx" # for subsequent build steps
+```
+
+Upon success, `./build-libcxx/{include,lib}` will contain the resulting
+headers and shared libraries.
+
+## Linking mold against instrumented libc++
+
+Use the `MOLD_USE_MSAN` and `MOLD_STDLIB_PREFIX` cmake variables to link
+mold against the instrumented build of libc++:
+
+```sh
+cd ~
+git clone https://github.com/rui314/mold.git
+cd mold
+cmake -B build -G Ninja            \
+    -DCMAKE_BUILD_TYPE=Debug       \
+    -DCMAKE_C_COMPILER=clang       \
+    -DCMAKE_CXX_COMPILER=clang++   \
+    -DMOLD_USE_MSAN=ON             \
+    -DMOLD_STDLIB_PREFIX="$LIBCXX" \
+    -DMOLD_USE_MIMALLOC=OFF        \
+    -DMOLD_USE_SYSTEM_TBB=ON
+cmake --build build
+```
+
+Most of mold's tests (except those for `-flto`) should work at this
+point. Run them like normal:
+
+```sh
+ctest --test-dir build
+```
+
+Any resulting MemorySanitizer errors should be visible in
+`./build/Temporary/Testing/LastTest.log`.
+
+## Building instrumented LTO plugin (experimental)
+
+Exercising `-flto` with MemorySanitizer (and without false positives)
+requires instrumenting the transitive dependencies of `lto-unix.cc`:
+
+- `libiberty.a`
+- `liblto_plugin.so`
+- `LLVMgold.so`
+
+> [!NOTE]
+> This is more involved and time-consuming than building only libc++ from
+> source. The cost-benefit ratio of this additional instrumentation and
+> test coverage may be unfavorable in many cases.
+
+> [!IMPORTANT]
+> The following steps are experimental and unlikely to work exactly as-is
+> under CI runners and individual developer environments. Consider the
+> following a starting point rather than a complete HOWTO.
+
+Build an MSan-instrumented GNU libiberty:
+
+```sh
+cd ~
+git clone [email protected]:gcc-mirror/gcc.git
+cd gcc/libiberty
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `./libiberty.a` into the build toolchain being
+tested with mold. If the system image is ephemeral or disposable (e.g. a
+short-lived VM or container), a quick-and-dirty install could look like:
+
+```sh
+sudo cp /usr/lib/libiberty.a{,.bak}
+sudo cp ./libiberty.a /usr/lib/
+```
+
+Build an MSan-instrumented GCC LTO plugin library:
+
+```sh
+cd ~/gcc/lto-plugin # use already-cloned repo from earlier step
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure --with-libiberty=/usr/lib
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `liblto_plugin.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+export GCC_VERSION="$(gcc -dumpversion)"
+sudo cp /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/liblto_plugin.so{,.bak}
+sudo cp .libs/liblto_plugin.so /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/
+```
+
+Build an MSan-instrumented LLVM LTO plugin library:
+
+```sh
+cd ~/llvm-project # use already-cloned repo from earlier step
+
+CF="-nostdinc++ -isystem $LIBCXX/include/c++/v1"
+INCDIR="$(find /usr -name plugin-api.h -type f | head -1 | xargs dirname)"
+
+function configure
+{
+    cmake -S ./llvm -B build-plugin -G Ninja   \
+        -DLLVM_ENABLE_PROJECTS=clang           \
+        -DLLVM_TARGETS_TO_BUILD=X86            \
+        -DCMAKE_BUILD_TYPE=Release             \
+        -DCMAKE_C_COMPILER=clang               \
+        -DCMAKE_CXX_COMPILER=clang++           \
+        -DCMAKE_C_FLAGS="$CF"                  \
+        -DCMAKE_CXX_FLAGS="$CF"                \
+        -DCMAKE_EXE_LINKER_FLAGS="$1"          \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON   \
+        -DLLVM_BINUTILS_INCDIR="$INCDIR"       \
+        -DLLVM_USE_SANITIZER=MemoryWithOrigins
+}
+
+# workaround for linker issues: configure twice with different LDFLAGS
+configure "-nostdlib++ -L $LIBCXX/lib -Wl,--rpath=$LIBCXX/lib"
+configure "-nostdlib++ -L $LIBCXX/lib -lc++ -Wl,--rpath=$LIBCXX/lib"
+
+cmake --build build-plugin -- LLVMgold.so
+```
+
+Install the resulting `LLVMgold.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+sudo cp /usr/lib/LLVMgold.so{,.bak}
+sudo cp ./build-plugin/lib/LLVMgold.so /usr/lib/
+```
+
+mold's tests for `-flto` should now (mostly) work, though there do seem
+to be issues around `gcc` inferring `-flto` when not explicitly specified.
+One workaround is to use `clang` as `TEST_CC`:
+
+```sh
+cd ~/mold # use already-cloned and built repo from earlier step
+TEST_CC=clang TEST_CXX=clang++ ctest --test-dir build
+```
+
+## References
+
+- <https://github.com/google/sanitizers/wiki/MemorySanitizerBootstrappingClang>
+- <https://llvm.org/docs/GoldPlugin.html#lto-how-to-build>

docs/mold.1

@@ -1,6 +1,6 @@
 .\" generated with Ronn-NG/v0.9.1
 .\" http://github.com/apjanke/ronn-ng/tree/0.9.1
-.TH "MOLD" "1" "February 2025" ""
+.TH "MOLD" "1" "August 2026" ""
 .SH "NAME"
 \fBmold\fR \- a modern linker
 .SH "SYNOPSIS"
@@ -70,6 +70,8 @@ Permit or do not permit mold to create a debug info file in the background\.
 .TP
 \fB\-\-fork\fR, \fB\-\-no\-fork\fR
 Spawn a child process and let it do the actual linking\. When linking a large program, the OS kernel can take a few hundred milliseconds to terminate a \fBmold\fR process\. \fB\-\-fork\fR hides that latency\. By default, it does fork\.
+.IP
+Note that \fB\-\-fork\fR also hides the resource usage statistics reported by time(2), since it doesn't call waitpid(2) on the child process\. If you need those statistics, pass \fB\-\-no\-fork\fR\.
 .TP
 \fB\-\-perf\fR
 Print performance statistics\.
@@ -130,6 +132,17 @@ Use multiple threads\. By default, \fBmold\fR uses as many threads as the number
 \fB\-\-quick\-exit\fR, \fB\-\-no\-quick\-exit\fR
 Use or do not use \fBquick_exit\fR to exit\.
 .TP
+\fB\-\-zero\-to\-bss\fR
+Convert all\-zero data sections into BSS\.
+.IP
+When this option is enabled, \fBmold\fR scans input data sections that are not of type \fBSHT_NOBITS\fR and checks whether their contents consist solely of zero bytes\. Such sections are then converted into BSS (\fBSHT_NOBITS\fR) sections\. This reduces the size of the output file, since BSS sections occupy no space in the file image\.
+.IP
+This behavior is especially useful for user\-defined sections created with \fB__attribute__((section("\.sectname")))\fR that contain uninitialized global variables\. GCC and Clang do not automatically mark such sections as BSS even if their contents are entirely zero, and instead emit them as regular data sections\.
+.IP
+For example, consider \fB__attribute__((section("\.sectname"))) int vec[256];\fR\.
+.IP
+By default, this results in a \fB\.sectname\fR section of type \fBSHT_PROGBITS\fR filled with zeros\. With \fB\-\-zero\-to\-bss\fR, the linker will recognize it as empty data and convert it to a \fBSHT_NOBITS\fR section, reducing the output file size without changing runtime semantics\.
+.TP
 \fB\-z rewrite\-endbr\fR, \fB\-z norewrite\-endbr\fR
 As a security measure, some CPU instruction sets have recently gained a feature to protect control flow integrity by disallowing indirect branches by default\. If the feature is enabled, the instruction that is executed immediately after an indirect branch must be an branch target marker instruction, or a CPU\-level fault will raise\. The marker instruction is also known as "landing pad" instruction, to which indirect branches can land\. This feature makes ROP attacks harder to conduct\.
 .IP
@@ -139,7 +152,9 @@ By default, the compiler always emits a landing pad at the beginning of each glo
 .IP
 If \fB\-\-rewrite\-endbr\fR is given, mold conducts a whole program analysis to identify functions whose addresses are actually taken and rewrites landing pads with no\-ops for non\-address\-taken functions, reducing the attack surface\.
 .IP
-This feature is currently available only on x86\-64\.
+This feature is currently available only on x86\-64 and arm64\.
+.IP
+On arm64, the landing pad is \fBbti c\fR\. Note that code compiled with \fB\-mbranch\-protection=standard\fR (BTI + pointer authentication) starts functions with \fBpaciasp\fR, which doubles as a landing pad, so there is no separate \fBbti c\fR to rewrite and this option has no effect on such code\.
 .SH "GNU\-COMPATIBLE OPTIONS"
 .TP
 \fB\-\-help\fR
@@ -180,7 +195,10 @@ Omit \fB\.debug_*\fR sections from the output file\.
 Read linker script from \fIfile\fR\.
 .TP
 \fB\-X\fR, \fB\-\-discard\-locals\fR
-Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\.
+Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\. This is the default; use \fB\-\-discard\-none\fR to keep temporary local symbols\.
+.TP
+\fB\-\-discard\-none\fR
+Keep all local symbols in the symbol table, including temporary ones starting with \fB\.L\fR, except ones in mergeable sections\. This restores the default behavior of GNU ld\. \fBmold\fR and \fBld\.lld\fR discard temporary local symbols by default\.
 .TP
 \fB\-e\fR \fIsymbol\fR, \fB\-\-entry\fR=\fIsymbol\fR:
 
@@ -245,6 +263,9 @@ Alias for \fB\-\-section\-start=\.data=\fR\fIaddress\fR\.
 \fB\-\-Ttext\fR=\fIaddress\fR
 Alias for \fB\-\-section\-start=\.text=\fR\fIaddress\fR\.
 .TP
+\fB\-\-Ttext\-segment\fR=\fIaddress\fR
+Set the address of the first byte of the text segment\.
+.TP
 \fB\-\-allow\-multiple\-definition\fR
 Normally, the linker reports an error if there are more than one definition of a symbol\. This option changes the default behavior so that it doesn't report an error for duplicate definitions and instead use the first definition\.
 .TP
@@ -269,8 +290,8 @@ Synonym for \fB\-\-build\-id=sha256\fR\.
 \fB\-\-no\-build\-id\fR
 Synonym for \fB\-\-build\-id=none\fR\.
 .TP
-\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzstd\fR | \fBnone\fR ]
-Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\.
+\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzlib:\fR\fIN\fR | \fBzstd\fR | \fBzstd:\fR\fIN\fR | \fBnone\fR ]
+Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\. \fBzlib:\fR\fIN\fR selects the zlib compression level, where \fIN\fR is between 0 and 9 (0 means no compression), and \fBzstd:\fR\fIN\fR selects the zstd compression level, where \fIN\fR is between 1 and 22\. Higher levels achieve better compression ratios but are slower\. \fBzlib\fR is equivalent to \fBzlib:1\fR and \fBzstd\fR is equivalent to \fBzstd:3\fR\.
 .TP
 \fB\-\-defsym\fR=\fIsymbol\fR=\fIvalue\fR
 Define \fIsymbol\fR as an alias for \fIvalue\fR\.
@@ -358,10 +379,16 @@ Embed a specified string into the \fB\.note\.package\fR section\. This option is
 .IP
 An argument to this option is treated as percent\-encoded and decoded before being inserted into the section, allowing you to avoid the use of the comma (\fB,\fR) character in the argument\. This is useful because the compiler replaces all occurrences of commas in \fB\-Wl,\fR with spaces before forwarding them to the linker\. Note that \fBmold\fR always interprets the argument as percent\-encoded, so you also need to escape all occurrences of \fB%\fR as \fB%25\fR\.
 .TP
-\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBnone\fR ]
+\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBandroid\fR | \fBandroid+relr\fR | \fBnone\fR ]
+Compress dynamic relocations in the output file\.
+.IP
 If \fBrelr\fR is specified, all \fBR_*_RELATIVE\fR relocations are put into \fB\.relr\.dyn\fR section instead of \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section\. Since \fB\.relr\.dyn\fR section uses a space\-efficient encoding scheme, specifying this flag can reduce the size of the output\. This is typically most effective for position\-independent executable\.
 .IP
-Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2022, only ChromeOS, Android and Fuchsia support it\.
+If \fBandroid\fR is specified, all dynamic relocations are encoded in Android's packed relocation format (APS2) and stored in the \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section, whose section type is changed to \fBSHT_ANDROID_REL\fR or \fBSHT_ANDROID_RELA\fR\. The dynamic table is updated to use \fBDT_ANDROID_REL\fR / \fBDT_ANDROID_RELA\fR and their \fBSZ\fR variants in place of the standard tags\.
+.IP
+\fBandroid+relr\fR combines both: \fBR_*_RELATIVE\fR relocations go to \fB\.relr\.dyn\fR and the rest are APS2\-encoded\. This is the configuration used by most Android system binaries\.
+.IP
+Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2026, glibc (since 2\.36), musl (since 1\.2\.4), Android, ChromeOS, and Fuchsia all support it\. The APS2 format, on the other hand, is only handled by Android's bionic loader; the encoder itself is target\-agnostic, but the produced binary can only be loaded on platforms where bionic (or a compatible loader) handles \fBDT_ANDROID_REL(A)\fR\.
 .TP
 \fB\-\-pie\fR, \fB\-\-pic\-executable\fR, \fB\-\-no\-pie\fR, \fB\-\-no\-pic\-executable\fR
 Create a position\-independent executable\.
@@ -427,6 +454,9 @@ How to handle undefined symbols\.
 \fB\-\-version\-script\fR=\fIfile\fR
 Read version script from \fIfile\fR\. If \fIfile\fR does not exist in the current directory, it is searched from library search paths for the sake of compatibility with GNU ld\.
 .TP
+\fB\-w\fR, \fB\-\-no\-warnings\fR
+Suppress warnings and cancel \fB\-\-fatal\-warnings\fR\.
+.TP
 \fB\-\-warn\-common\fR, \fB\-\-no\-warn\-common\fR
 Warn about common symbols\.
 .TP
@@ -483,9 +513,9 @@ By default, \fBmold\fR generates a relro segment\. \fB\-z norelro\fR disables th
 \fB\-z separate\-loadable\-segments\fR, \fB\-z separate\-code\fR, \fB\-z noseparate\-code\fR
 If one memory page contains multiple segments, the page protection bits are set in such a way that the needed attributes (writable or executable) are satisfied for all segments\. This usually happens at a boundary of two segments with two different attributes\.
 .IP
-\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\. This is the default\.
+\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\.
 .IP
-\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\.
+\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\. This is the default\.
 .IP
 \fBnoseparate\-code\fR does not add any paddings between segments\.
 .TP

docs/mold.md

@@ -159,6 +159,10 @@ but as `-o magic`.
   program, the OS kernel can take a few hundred milliseconds to terminate a
   `mold` process. `--fork` hides that latency. By default, it does fork.
 
+  Note that `--fork` also hides the resource usage statistics reported by
+  time(2), since it doesn't call waitpid(2) on the child process. If you
+  need those statistics, pass `--no-fork`.
+
 * `--perf`:
   Print performance statistics.
 
@@ -267,6 +271,28 @@ but as `-o magic`.
 * `--quick-exit`, `--no-quick-exit`:
   Use or do not use `quick_exit` to exit.
 
+* `--zero-to-bss`:
+  Convert all-zero data sections into BSS.
+
+  When this option is enabled, `mold` scans input data sections that are not
+  of type `SHT_NOBITS` and checks whether their contents consist solely of
+  zero bytes. Such sections are then converted into BSS (`SHT_NOBITS`)
+  sections. This reduces the size of the output file, since BSS sections
+  occupy no space in the file image.
+
+  This behavior is especially useful for user-defined sections created with
+  `__attribute__((section(".sectname")))` that contain uninitialized global
+  variables. GCC and Clang do not automatically mark such sections as BSS
+  even if their contents are entirely zero, and instead emit them as regular
+  data sections.
+
+  For example, consider `__attribute__((section(".sectname"))) int vec[256];`.
+
+  By default, this results in a `.sectname` section of type `SHT_PROGBITS`
+  filled with zeros. With `--zero-to-bss`, the linker will recognize it as
+  empty data and convert it to a `SHT_NOBITS` section, reducing the output
+  file size without changing runtime semantics.
+
 * `-z rewrite-endbr`, `-z norewrite-endbr`:
   As a security measure, some CPU instruction sets have recently gained a
   feature to protect control flow integrity by disallowing indirect branches
@@ -292,7 +318,12 @@ but as `-o magic`.
   landing pads with no-ops for non-address-taken functions, reducing the
   attack surface.
 
-  This feature is currently available only on x86-64.
+  This feature is currently available only on x86-64 and arm64.
+
+  On arm64, the landing pad is `bti c`. Note that code compiled with
+  `-mbranch-protection=standard` (BTI + pointer authentication) starts
+  functions with `paciasp`, which doubles as a landing pad, so there is no
+  separate `bti c` to rewrite and this option has no effect on such code.
 
 ## GNU-COMPATIBLE OPTIONS
 
@@ -349,7 +380,14 @@ but as `-o magic`.
   Discard temporary local symbols to reduce the sizes of the symbol table and
   the string table. Temporary local symbols are local symbols starting with
   `.L`. Compilers usually generate such symbols for unnamed program elements
-  such as string literals or floating-point literals.
+  such as string literals or floating-point literals. This is the default;
+  use `--discard-none` to keep temporary local symbols.
+
+* `--discard-none`:
+  Keep all local symbols in the symbol table, including temporary ones
+  starting with `.L`, except ones in mergeable sections. This restores the
+  default behavior of GNU ld. `mold` and `ld.lld` discard temporary local
+  symbols by default.
 
 * `-e` _symbol_, `--entry`=_symbol_:
 
@@ -432,6 +470,9 @@ but as `-o magic`.
 * `--Ttext`=_address_:
   Alias for `--section-start=.text=`_address_.
 
+* `--Ttext-segment`=_address_:
+  Set the address of the first byte of the text segment.
+
 * `--allow-multiple-definition`:
   Normally, the linker reports an error if there are more than one definition
   of a symbol. This option changes the default behavior so that it doesn't
@@ -480,9 +521,14 @@ but as `-o magic`.
 * `--no-build-id`:
   Synonym for `--build-id=none`.
 
-* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zstd` | `none` ]:
+* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zlib:`_N_ | `zstd` | `zstd:`_N_ | `none` ]:
   Compress DWARF debug info (`.debug_*` sections) using the zlib or zstd
-  compression algorithm. `zlib-gabi` is an alias for `zlib`.
+  compression algorithm. `zlib-gabi` is an alias for `zlib`. `zlib:`_N_
+  selects the zlib compression level, where _N_ is between 0 and 9 (0
+  means no compression), and `zstd:`_N_ selects the zstd compression
+  level, where _N_ is between 1 and 22. Higher levels achieve better
+  compression ratios but are slower. `zlib` is equivalent to `zlib:1`
+  and `zstd` is equivalent to `zstd:3`.
 
 * `--defsym`=_symbol_=_value_:
   Define _symbol_ as an alias for _value_.
@@ -632,16 +678,32 @@ but as `-o magic`.
   argument as percent-encoded, so you also need to escape all occurrences
   of `%` as `%25`.
 
-* `--pack-dyn-relocs`=[ `relr` | `none` ]:
+* `--pack-dyn-relocs`=[ `relr` | `android` | `android+relr` | `none` ]:
+  Compress dynamic relocations in the output file.
+
   If `relr` is specified, all `R_*_RELATIVE` relocations are put into
   `.relr.dyn` section instead of `.rel.dyn` or `.rela.dyn` section. Since
   `.relr.dyn` section uses a space-efficient encoding scheme, specifying this
   flag can reduce the size of the output. This is typically most effective for
   position-independent executable.
 
+  If `android` is specified, all dynamic relocations are encoded in Android's
+  packed relocation format (APS2) and stored in the `.rel.dyn` or `.rela.dyn`
+  section, whose section type is changed to `SHT_ANDROID_REL` or
+  `SHT_ANDROID_RELA`. The dynamic table is updated to use `DT_ANDROID_REL`
+  / `DT_ANDROID_RELA` and their `SZ` variants in place of the standard tags.
+
+  `android+relr` combines both: `R_*_RELATIVE` relocations go to `.relr.dyn`
+  and the rest are APS2-encoded. This is the configuration used by most
+  Android system binaries.
+
   Note that a runtime loader has to support `.relr.dyn` to run executables or
-  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2022, only
-  ChromeOS, Android and Fuchsia support it.
+  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2026, glibc
+  (since 2.36), musl (since 1.2.4), Android, ChromeOS, and Fuchsia all
+  support it. The APS2 format, on the other hand, is only handled by
+  Android's bionic loader; the encoder itself is target-agnostic, but the
+  produced binary can only be loaded on platforms where bionic (or a
+  compatible loader) handles `DT_ANDROID_REL(A)`.
 
 * `--pie`, `--pic-executable`, `--no-pie`, `--no-pic-executable`:
   Create a position-independent executable.
@@ -731,6 +793,9 @@ but as `-o magic`.
   directory, it is searched from library search paths for the sake of
   compatibility with GNU ld.
 
+* `-w`, `--no-warnings`:
+  Suppress warnings and cancel `--fatal-warnings`.
+
 * `--warn-common`, `--no-warn-common`:
   Warn about common symbols.
 
@@ -828,10 +893,10 @@ but as `-o magic`.
   segments with two different attributes.
 
   `separate-loadable-segments` adds paddings between segments with different
-  attributes so that they do not share the same page. This is the default.
+  attributes so that they do not share the same page.
 
   `separate-code` adds paddings only between executable and non-executable
-  segments.
+  segments. This is the default.
 
   `noseparate-code` does not add any paddings between segments.
 

install-build-deps.sh

@@ -8,24 +8,24 @@ set -e
 set -x
 
 case "$ID" in
-ubuntu | pop | linuxmint | debian | raspbian | neon)
+ubuntu | pop | linuxmint | debian | raspbian | neon | zorin)
   apt-get update
   apt-get install -y cmake gcc g++ clang gdb
   ;;
-fedora | amzn | rhel | centos)
-  dnf install -y gcc-g++ cmake glibc-static libstdc++-static diffutils util-linux tar
+fedora | fedora-* | amzn | rhel | centos)
+  dnf install -y gcc-c++ cmake glibc-static libstdc++-static diffutils util-linux tar
   ;;
-rocky)
-  dnf install -y gcc-g++ cmake diffutils util-linux
+rocky | ol)
+  dnf install -y gcc-c++ cmake diffutils util-linux
   ;;
 opensuse-*)
-  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux
+  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux gawk
   ;;
 gentoo)
   emerge-webrsync
   FEATURES='getbinpkg binpkg-request-signature' emerge dev-build/cmake
   ;;
-arch | archarm | artix | endeavouros)
+arch | archarm | artix | endeavouros | manjaro | cachyos)
   pacman -Sy --needed --noconfirm base-devel cmake util-linux
   ;;
 void)
@@ -42,6 +42,10 @@ clear-linux-os)
 almalinux)
   dnf install -y gcc-toolset-13-gcc-c++ gcc-toolset-13-libstdc++-devel cmake diffutils
   ;;
+altlinux)
+  apt-get update
+  apt-get install -y gcc-c++ make cmake ctest diffutils util-linux
+  ;;
 freebsd)
   pkg update
   pkg install -y cmake bash binutils gcc

lib/aho-corasick.cc

@@ -0,0 +1,146 @@
+// This file implements the Aho-Corasick algorithm to search multiple
+// strings within an input string simultaneously. It is essentially a
+// trie with additional links. For details, see
+// https://en.wikipedia.org/wiki/Aho-Corasick_algorithm.
+//
+// We use it for simple glob patterns in version scripts or dynamic
+// list files. Here are some examples of glob patterns:
+//
+//    qt_private_api_tag*
+//    *16QAccessibleCache*
+//    *32QAbstractFileIconProviderPrivate*
+//    *17QPixmapIconEngine*
+//
+// Aho-Corasick can do only substring search, so it cannot handle
+// complex glob patterns such as `*foo*bar*`. We handle such patterns
+// with the Glob class.
+
+#include "lib.h"
+
+#include <queue>
+
+namespace mold {
+
+bool AhoCorasick::can_handle(std::string_view pat) {
+  if (pat.starts_with('*'))
+    pat.remove_prefix(1);
+  if (pat.ends_with('*'))
+    pat.remove_suffix(1);
+  return !pat.empty() && pat.find_first_of("*?[\\") == pat.npos;
+}
+
+i32 AhoCorasick::find_child(i32 node, u8 ch) const {
+  if (node == 0)
+    return root_children[ch];
+
+  for (i32 i = nodes[node].first_child; i != -1; i = nodes[i].next_sibling)
+    if (nodes[i].ch == ch)
+      return i;
+  return -1;
+}
+
+i32 AhoCorasick::add_child(i32 node, u8 ch) {
+  if (i32 child = find_child(node, ch); child != -1)
+    return child;
+
+  i32 child = nodes.size();
+  i32 sibling = nodes[node].first_child;
+  nodes.emplace_back();
+  nodes[child].next_sibling = sibling;
+  nodes[child].ch = ch;
+  nodes[node].first_child = child;
+
+  if (node == 0)
+    root_children[ch] = child;
+  return child;
+}
+
+i64 AhoCorasick::find(std::string_view str) {
+  if (nodes.empty())
+    return -1;
+
+  i32 idx = 0;
+  i64 val = -1;
+
+  auto walk = [&](u8 c) {
+    for (i32 j = idx; j != -1; j = nodes[j].suffix_link) {
+      if (i32 child = find_child(j, c); child != -1) {
+        idx = child;
+        val = std::max(val, nodes[child].value);
+        return;
+      }
+    }
+    idx = 0;
+  };
+
+  walk('\0');
+  for (u8 c : str) {
+    walk(c);
+    if (val == max_value)
+      return val;
+  }
+  walk('\0');
+  return val;
+}
+
+bool AhoCorasick::add(std::string_view pat, i64 val) {
+  assert(can_handle(pat));
+  max_value = std::max(max_value, val);
+  if (nodes.empty()) {
+    root_children.fill(-1);
+    nodes.emplace_back();
+  }
+  i32 idx = 0;
+
+  auto walk = [&](u8 c) { idx = add_child(idx, c); };
+
+  // We handle "foo" as if "\0foo\0", "*foo" as if "foo\0", "foo*" as
+  // if "\0foo", and "*foo*" as if "foo". Aho-Corasick can do only
+  // substring matching, so we use \0 as a beginning/end-of-string
+  // markers.
+  if (!pat.starts_with('*'))
+    walk('\0');
+  for (u8 c : pat)
+    if (c != '*')
+      walk(c);
+  if (!pat.ends_with('*'))
+    walk('\0');
+
+  nodes[idx].value = std::max(nodes[idx].value, val);
+  return true;
+}
+
+void AhoCorasick::compile() {
+  if (nodes.empty())
+    return;
+
+  // A failure link may refer to any node at the previous depth, so failure
+  // links must be constructed breadth-first.
+  std::queue<i32> queue;
+  for (i32 child = nodes[0].first_child; child != -1;
+       child = nodes[child].next_sibling) {
+    nodes[child].suffix_link = 0;
+    queue.push(child);
+  }
+
+  while (!queue.empty()) {
+    i32 idx = queue.front();
+    queue.pop();
+
+    for (i32 child = nodes[idx].first_child; child != -1;
+         child = nodes[child].next_sibling) {
+      i32 suffix = nodes[idx].suffix_link;
+      while (suffix != 0 && find_child(suffix, nodes[child].ch) == -1)
+        suffix = nodes[suffix].suffix_link;
+
+      if (i32 next = find_child(suffix, nodes[child].ch); next != -1)
+        suffix = next;
+
+      nodes[child].suffix_link = suffix;
+      nodes[child].value = std::max(nodes[child].value, nodes[suffix].value);
+      queue.push(child);
+    }
+  }
+}
+
+} // namespace mold

lib/arena.cc

@@ -0,0 +1,134 @@
+#include "lib.h"
+
+#ifdef _WIN32
+# include <windows.h>
+#endif
+
+namespace mold {
+
+static constexpr u32 BLOCK_SIZE = 1024 * 1024;
+static constexpr u32 BLOCK_ALIGNMENT = 64;
+static constexpr u32 MAX_LOCAL_ALLOC = BLOCK_SIZE / 4;
+static constexpr u32 NUM_ARENA_SLOTS = 10;
+
+// Reserved address space is made accessible in chunks of this size.
+static constexpr u64 COMMIT_CHUNK = 64 * 1024 * 1024;
+
+static std::atomic<u32> next_arena_slot;
+static thread_local u32 positions[NUM_ARENA_SLOTS];
+static thread_local u32 ends[NUM_ARENA_SLOTS];
+
+[[noreturn]]
+static void memory_error(std::string_view action, u64 size) {
+  std::cerr << "mold: cannot " << action << ' ' << size
+            << " bytes of virtual memory\n";
+  std::exit(1);
+}
+
+ArenaResource::ArenaResource() : arena_slot(next_arena_slot++) {
+  assert(arena_slot < NUM_ARENA_SLOTS);
+
+  // Reserve address space without making it accessible. An inaccessible
+  // mapping is not charged against the system's commit limit even where
+  // overcommit is disabled, and fork() does not charge it either. Pages
+  // are made accessible by commit() as the arena grows.
+#ifdef _WIN32
+  data = (u8 *)VirtualAlloc(nullptr, (SIZE_T)SIZE, MEM_RESERVE, PAGE_NOACCESS);
+  if (!data)
+    memory_error("reserve", SIZE);
+#else
+  data = (u8 *)mmap(nullptr, SIZE, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (data == MAP_FAILED)
+    memory_error("reserve", SIZE);
+#endif
+
+#ifdef MADV_HUGEPAGE
+  // Large links fill the beginning of the arena densely. Transparent huge
+  // pages reduce address-translation overhead without populating unused pages.
+  madvise(data, SIZE, MADV_HUGEPAGE);
+#endif
+}
+
+ArenaResource::~ArenaResource() {
+#ifdef _WIN32
+  VirtualFree(data, 0, MEM_RELEASE);
+#else
+  munmap(data, SIZE);
+#endif
+}
+
+// Makes the reserved range up to `end` accessible. Memory is committed in
+// whole chunks so that the arena grows with few system calls.
+void ArenaResource::commit(u64 end) {
+  std::scoped_lock lock(commit_mu);
+  u64 cur = committed.load(std::memory_order_relaxed);
+  if (end <= cur)
+    return;
+
+  u64 next = align_to(end, COMMIT_CHUNK);
+#ifdef _WIN32
+  if (!VirtualAlloc(data + cur, (SIZE_T)(next - cur), MEM_COMMIT, PAGE_READWRITE))
+    memory_error("commit", next - cur);
+#else
+  if (mprotect(data + cur, next - cur, PROT_READ | PROT_WRITE) == -1)
+    memory_error("commit", next - cur);
+#endif
+  committed.store(next, std::memory_order_release);
+}
+
+void *ArenaResource::allocate(u64 size, u64 alignment) {
+  assert(std::has_single_bit(alignment));
+
+  // ArenaPtr and base-relative indices both encode offsets in four-byte units.
+  alignment = std::max<u64>(alignment, 4);
+
+  // Standard containers make many small allocations. Reserve them from a
+  // thread-local block to avoid contending on the global bump pointer.
+  if (size <= MAX_LOCAL_ALLOC && alignment <= BLOCK_ALIGNMENT) {
+    // Offsets are from the beginning of the arena in four-byte units.
+    u32 &pos = positions[arena_slot];
+    u32 &end = ends[arena_slot];
+    u32 size4 = align_to(size, 4) / 4;
+
+    if (pos) {
+      u32 begin = align_to(pos, alignment / 4);
+      if (begin <= end && size4 <= end - begin) {
+        pos = begin + size4;
+        return data + (u64)begin * 4;
+      }
+    }
+
+    u8 *begin = (u8 *)allocate_global(BLOCK_SIZE, BLOCK_ALIGNMENT);
+    u32 begin4 = (begin - data) / 4;
+    pos = begin4 + size4;
+    end = begin4 + BLOCK_SIZE / 4;
+    return begin;
+  }
+
+  return allocate_global(size, alignment);
+}
+
+void *ArenaResource::allocate_global(u64 size, u64 alignment) {
+  u64 old = offset.load(std::memory_order_relaxed);
+  u64 begin, end;
+
+  for (;;) {
+    u64 padding = -old & (alignment - 1);
+    if (old > SIZE || padding > SIZE - old || size > SIZE - old - padding) {
+      std::cerr << "mold: cannot allocate more than " << SIZE
+                << " bytes for linker data structures on this host\n";
+      std::exit(1);
+    }
+
+    begin = old + padding;
+    end = begin + size;
+    if (offset.compare_exchange_weak(old, end, std::memory_order_relaxed))
+      break;
+  }
+
+  if (end > committed.load(std::memory_order_acquire))
+    commit(end);
+  return data + begin;
+}
+
+} // namespace mold

lib/atomics.h

@@ -0,0 +1,64 @@
+// This is the same as std::atomic except that the default memory
+// order is relaxed instead of sequential consistency.
+
+#pragma once
+
+#include <atomic>
+
+namespace mold {
+
+template <typename T>
+struct Atomic : std::atomic<T> {
+  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
+
+  using std::atomic<T>::atomic;
+
+  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
+
+  Atomic<T> &operator=(const Atomic<T> &other) {
+    store(other.load());
+    return *this;
+  }
+
+  void operator=(T val) { store(val); }
+  operator T() const { return load(); }
+
+  void store(T val, std::memory_order order = relaxed) {
+    std::atomic<T>::store(val, order);
+  }
+
+  T load(std::memory_order order = relaxed) const {
+    return std::atomic<T>::load(order);
+  }
+
+  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
+
+  T fetch_or(T val, std::memory_order order = relaxed) {
+    return std::atomic<T>::fetch_or(val, order);
+  }
+
+  T fetch_and(T val, std::memory_order order = relaxed) {
+    return std::atomic<T>::fetch_and(val, order);
+  }
+
+  T operator|=(T val) {
+    // Avoid exclusive cache-line ownership if all requested bits are set.
+    T old = load();
+    return (old & val) == val ? old : fetch_or(val);
+  }
+
+  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
+  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
+  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
+  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
+
+  bool test_and_set() {
+    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
+    // while an atomic RMW can easily take hundreds on x86. We note that it's
+    // common that another thread beat us in marking, so doing an optimistic
+    // early test tends to improve performance in the ~20% ballpark.
+    return load() || exchange(true);
+  }
+};
+
+} // namespace mold

lib/common.h

@@ -1,979 +0,0 @@
-#pragma once
-
-#include "config.h"
-#include "integers.h"
-
-#include <array>
-#include <atomic>
-#include <bit>
-#include <bitset>
-#include <cassert>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <fcntl.h>
-#include <filesystem>
-#include <iostream>
-#include <mutex>
-#include <optional>
-#include <span>
-#include <sstream>
-#include <string>
-#include <string_view>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <tbb/concurrent_vector.h>
-#include <tbb/enumerable_thread_specific.h>
-#include <tbb/parallel_for.h>
-#include <vector>
-
-#ifdef _WIN32
-# include <io.h>
-#else
-# include <sys/mman.h>
-# include <unistd.h>
-#endif
-
-#define XXH_INLINE_ALL 1
-#include "../third-party/xxhash/xxhash.h"
-
-#ifdef NDEBUG
-# define unreachable() __builtin_unreachable()
-#else
-# define unreachable() assert(0 && "unreachable")
-#endif
-
-inline uint64_t hash_string(std::string_view str) {
-  return XXH3_64bits(str.data(), str.size());
-}
-
-class HashCmp {
-public:
-  static size_t hash(const std::string_view &k) {
-    return hash_string(k);
-  }
-
-  static bool equal(const std::string_view &k1, const std::string_view &k2) {
-    return k1 == k2;
-  }
-};
-
-namespace mold {
-
-namespace ranges = std::ranges;
-using namespace std::literals::string_literals;
-using namespace std::literals::string_view_literals;
-
-inline char *output_tmpfile;
-
-inline u8 *output_buffer_start = nullptr;
-inline u8 *output_buffer_end = nullptr;
-
-extern std::string mold_git_hash;
-
-std::string errno_string();
-void cleanup();
-void install_signal_handler();
-
-inline u64 combine_hash(u64 a, u64 b) {
-  return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
-}
-
-//
-// Error output
-//
-
-// Some C++ stdlibs don't support std::osyncstream even though
-// it's is in the C++20 standard. So we implement it ourselves.
-class SyncStream {
-public:
-  SyncStream(std::ostream &out) : out(out) {}
-
-  ~SyncStream() {
-    emit();
-  }
-
-  template <typename T> SyncStream &operator<<(T &&val) {
-    ss << std::forward<T>(val);
-    return *this;
-  }
-
-  void emit() {
-    if (emitted)
-      return;
-
-    std::scoped_lock lock(mu);
-    out << ss.str() << '\n';
-    emitted = true;
-  }
-
-private:
-  std::ostream &out;
-  std::stringstream ss;
-  bool emitted = false;
-  static inline std::mutex mu;
-};
-
-template <typename Context>
-class Out {
-public:
-  Out(Context &ctx) {}
-
-  template <typename T> Out &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cout};
-};
-
-static std::string_view fatal_mono = "mold: fatal: ";
-static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
-static std::string_view error_mono = "mold: error: ";
-static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
-static std::string_view warning_mono = "mold: warning: ";
-static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
-
-template <typename Context>
-class Fatal {
-public:
-  Fatal(Context &ctx) {
-    out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
-  }
-
-  [[noreturn]] ~Fatal() {
-    out.emit();
-    cleanup();
-    _exit(1);
-  }
-
-  template <typename T> Fatal &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Error {
-public:
-  Error(Context &ctx) {
-    if (ctx.arg.noinhibit_exec) {
-      out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    } else {
-      out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    }
-  }
-
-  template <typename T> Error &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Warn {
-public:
-  Warn(Context &ctx) {
-    if (ctx.arg.suppress_warnings)
-      return;
-
-    out.emplace(std::cerr);
-
-    if (ctx.arg.fatal_warnings) {
-      *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    } else {
-      *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    }
-  }
-
-  template <typename T> Warn &operator<<(T &&val) {
-    if (out)
-      *out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  std::optional<SyncStream> out;
-};
-
-//
-// Atomics
-//
-
-// This is the same as std::atomic except that the default memory
-// order is relaxed instead of sequential consistency.
-template <typename T>
-struct Atomic : std::atomic<T> {
-  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
-
-  using std::atomic<T>::atomic;
-
-  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
-
-  Atomic<T> &operator=(const Atomic<T> &other) {
-    store(other.load());
-    return *this;
-  }
-
-  void operator=(T val) { store(val); }
-  operator T() const { return load(); }
-
-  void store(T val, std::memory_order order = relaxed) {
-    std::atomic<T>::store(val, order);
-  }
-
-  T load(std::memory_order order = relaxed) const {
-    return std::atomic<T>::load(order);
-  }
-
-  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
-  T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
-  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
-  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
-  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
-  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
-
-  bool test_and_set() {
-    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
-    // while an atomic RMW can easily take hundreds on x86. We note that it's
-    // common that another thread beat us in marking, so doing an optimistic
-    // early test tends to improve performance in the ~20% ballpark.
-    return load() || exchange(true);
-  }
-};
-
-//
-// mimalloc.cc
-//
-
-void set_mimalloc_options();
-
-//
-// perf.cc
-//
-
-// Counter is used to collect statistics numbers.
-class Counter {
-public:
-  Counter(std::string_view name, i64 value = 0) : name(name), values(value) {
-    static std::mutex mu;
-    std::scoped_lock lock(mu);
-    instances.push_back(this);
-  }
-
-  Counter &operator++(int) {
-    if (enabled) [[unlikely]]
-      values.local()++;
-    return *this;
-  }
-
-  Counter &operator+=(int delta) {
-    if (enabled) [[unlikely]]
-      values.local() += delta;
-    return *this;
-  }
-
-  static void print();
-
-  static inline bool enabled = false;
-
-private:
-  i64 get_value();
-
-  std::string_view name;
-  tbb::enumerable_thread_specific<i64> values;
-
-  static inline std::vector<Counter *> instances;
-};
-
-// Timer and TimeRecord records elapsed time (wall clock time)
-// used by each pass of the linker.
-struct TimerRecord {
-  TimerRecord(std::string name, TimerRecord *parent = nullptr);
-  void stop();
-
-  std::string name;
-  TimerRecord *parent;
-  tbb::concurrent_vector<TimerRecord *> children;
-  i64 start;
-  i64 end;
-  i64 user;
-  i64 sys;
-  bool stopped = false;
-};
-
-void
-print_timer_records(tbb::concurrent_vector<std::unique_ptr<TimerRecord>> &);
-
-template <typename Context>
-class Timer {
-public:
-  Timer(Context &ctx, std::string name, Timer *parent = nullptr) {
-    record = new TimerRecord(name, parent ? parent->record : nullptr);
-    ctx.timer_records.emplace_back(record);
-  }
-
-  Timer(const Timer &) = delete;
-
-  ~Timer() {
-    record->stop();
-  }
-
-  void stop() {
-    record->stop();
-  }
-
-private:
-  TimerRecord *record;
-};
-
-//
-// Utility functions
-//
-
-// Some C++ libraries haven't implemented std::has_single_bit yet.
-inline bool has_single_bit(u64 val) {
-  return std::popcount(val) == 1;
-}
-
-// Some C++ libraries haven't implemented std::bit_ceil yet.
-inline u64 bit_ceil(u64 val) {
-  if (has_single_bit(val))
-    return val;
-  return 1LL << (64 - std::countl_zero(val));
-}
-
-inline u64 align_to(u64 val, u64 align) {
-  if (align == 0)
-    return val;
-  assert(has_single_bit(align));
-  return (val + align - 1) & ~(align - 1);
-}
-
-inline u64 align_down(u64 val, u64 align) {
-  assert(has_single_bit(align));
-  return val & ~(align - 1);
-}
-
-inline u64 bit(u64 val, i64 pos) {
-  return (val >> pos) & 1;
-};
-
-// Returns [hi:lo] bits of val.
-inline u64 bits(u64 val, u64 hi, u64 lo) {
-  return (val >> lo) & ((1LL << (hi - lo + 1)) - 1);
-}
-
-// Cast val to a signed N bit integer.
-// For example, sign_extend(x, 32) == (i32)x for any integer x.
-inline i64 sign_extend(u64 val, i64 n) {
-  return (i64)(val << (64 - n)) >> (64 - n);
-}
-
-inline bool is_int(u64 val, i64 n) {
-  return sign_extend(val, n) == val;
-}
-
-template <typename T, typename Compare = std::less<T>>
-void update_minimum(std::atomic<T> &atomic, u64 new_val, Compare cmp = {}) {
-  T old_val = atomic.load(std::memory_order_relaxed);
-  while (cmp(new_val, old_val) &&
-         !atomic.compare_exchange_weak(old_val, new_val,
-                                       std::memory_order_relaxed));
-}
-
-template <typename T, typename Compare = std::less<T>>
-void update_maximum(std::atomic<T> &atomic, u64 new_val, Compare cmp = {}) {
-  T old_val = atomic.load(std::memory_order_relaxed);
-  while (cmp(old_val, new_val) &&
-         !atomic.compare_exchange_weak(old_val, new_val,
-                                       std::memory_order_relaxed));
-}
-
-template <typename T>
-inline void append(std::vector<T> &x, const auto &y) {
-  x.insert(x.end(), y.begin(), y.end());
-}
-
-template <typename T>
-inline std::vector<T> flatten(std::vector<std::vector<T>> &vec) {
-  i64 size = 0;
-  for (std::vector<T> &v : vec)
-    size += v.size();
-
-  std::vector<T> ret;
-  ret.reserve(size);
-  for (std::vector<T> &v : vec)
-    append(ret, v);
-  return ret;
-}
-
-template <typename T>
-inline void remove_duplicates(std::vector<T> &vec) {
-  vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
-}
-
-inline i64 write_string(void *buf, std::string_view str) {
-  memcpy(buf, str.data(), str.size());
-  *((u8 *)buf + str.size()) = '\0';
-  return str.size() + 1;
-}
-
-template <typename T>
-inline void write_vector(void *buf, const std::vector<T> &vec) {
-  if (!vec.empty())
-    memcpy(buf, vec.data(), vec.size() * sizeof(T));
-}
-
-inline void encode_uleb(std::vector<u8> &vec, u64 val) {
-  do {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-    vec.push_back(val ? (byte | 0x80) : byte);
-  } while (val);
-}
-
-inline void encode_sleb(std::vector<u8> &vec, i64 val) {
-  for (;;) {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-
-    bool neg = (byte & 0x40);
-    if ((val == 0 && !neg) || (val == -1 && neg)) {
-      vec.push_back(byte);
-      break;
-    }
-    vec.push_back(byte | 0x80);
-  }
-}
-
-inline i64 write_uleb(u8 *buf, u64 val) {
-  i64 i = 0;
-  do {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-    buf[i++] = val ? (byte | 0x80) : byte;
-  } while (val);
-  return i;
-}
-
-inline u64 read_uleb(u8 **buf) {
-  u64 val = 0;
-  u8 shift = 0;
-  u8 byte;
-  do {
-    byte = *(*buf)++;
-    val |= (byte & 0x7f) << shift;
-    shift += 7;
-  } while (byte & 0x80);
-  return val;
-}
-
-inline u64 read_uleb(u8 *buf) {
-  u8 *tmp = buf;
-  return read_uleb(&tmp);
-}
-
-inline i64 read_sleb(u8 **buf) {
-  u64 val = 0;
-  u8 shift = 0;
-  u8 byte;
-  do {
-    byte = *(*buf)++;
-    val |= (byte & 0x7f) << shift;
-    shift += 7;
-  } while (byte & 0x80);
-  return sign_extend(val, shift);
-}
-
-inline i64 read_sleb(u8 *buf) {
-  u8 *tmp = buf;
-  return read_sleb(&tmp);
-}
-
-inline u64 read_uleb(std::string_view *str) {
-  u8 *start = (u8 *)str->data();
-  u8 *ptr = start;
-  u64 val = read_uleb(&ptr);
-  *str = str->substr(ptr - start);
-  return val;
-}
-
-inline u64 read_uleb(std::string_view str) {
-  std::string_view tmp = str;
-  return read_uleb(&tmp);
-}
-
-inline i64 uleb_size(u64 val) {
-  for (int i = 1; i < 9; i++)
-    if (val < (1LL << (7 * i)))
-      return i;
-  return 9;
-}
-
-inline void overwrite_uleb(u8 *loc, u64 val) {
-  while (*loc & 0b1000'0000) {
-    *loc++ = 0b1000'0000 | (val & 0b0111'1111);
-    val >>= 7;
-  }
-  *loc = val & 0b0111'1111;
-}
-
-static inline void pause() {
-#if defined(__x86_64__)
-  asm volatile("pause");
-#elif defined(__aarch64__)
-  asm volatile("yield");
-#elif defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_8A__)
-  asm volatile("yield");
-#endif
-}
-
-//
-// Concurrent Map
-//
-
-// This is an implementation of a fast concurrent hash map. Unlike
-// ordinary hash tables, this impl just aborts if it becomes full.
-// So you need to give a correct estimation of the final size before
-// using it. We use this hash map to uniquify pieces of data in
-// mergeable sections.
-//
-// We've implemented this ourselves because the performance of
-// conrurent hash map is critical for our linker.
-template <typename T>
-class ConcurrentMap {
-public:
-  ConcurrentMap() = default;
-
-  ConcurrentMap(i64 nbuckets) {
-    resize(nbuckets);
-  }
-
-  ~ConcurrentMap() {
-    if (entries) {
-#ifdef _WIN32
-      _aligned_free(entries);
-#else
-      munmap(entries, sizeof(Entry) * nbuckets);
-#endif
-    }
-  }
-
-  // In order to avoid unnecessary cache-line false sharing, we want
-  // to make this object to be aligned to a reasonably large
-  // power-of-two address.
-  struct alignas(32) Entry {
-    Atomic<const char *> key;
-    u32 keylen;
-    T value;
-  };
-
-  void resize(i64 nbuckets) {
-    assert(!entries);
-    this->nbuckets = std::max<i64>(MIN_NBUCKETS, bit_ceil(nbuckets));
-    i64 bufsize = sizeof(Entry) * this->nbuckets;
-
-    // Allocate a zero-initialized buffer. We use mmap() if available
-    // because it's faster than malloc() and memset().
-#ifdef _WIN32
-    entries = (Entry *)_aligned_malloc(bufsize, alignof(Entry));
-    memset((void *)entries, 0, bufsize);
-#else
-    entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
-                            MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-#endif
-  }
-
-  std::pair<T *, bool> insert(std::string_view key, u64 hash, const T &val) {
-    assert(has_single_bit(nbuckets));
-
-    u64 begin = hash & (nbuckets - 1);
-    u64 mask = nbuckets / NUM_SHARDS - 1;
-
-    for (i64 i = 0; i < MAX_RETRY; i++) {
-      u64 idx = (begin & ~mask) | ((begin + i) & mask);
-      Entry &ent = entries[idx];
-
-      // It seems avoiding compare-and-swap is faster overall at least
-      // on my Zen4 machine, so do it.
-      if (const char *ptr = ent.key.load(std::memory_order_acquire);
-          ptr != nullptr && ptr != (char *)-1) {
-        if (key == std::string_view(ptr, ent.keylen))
-          return {&ent.value, false};
-        continue;
-      }
-
-      // Otherwise, use CAS to atomically claim the ownership of the slot.
-      const char *ptr = nullptr;
-      bool claimed = ent.key.compare_exchange_strong(ptr, (char *)-1,
-                                                     std::memory_order_acquire);
-
-      // If we successfully claimed the ownership of the slot,
-      // copy values to it.
-      if (claimed) {
-        new (&ent.value) T(val);
-        ent.keylen = key.size();
-        ent.key.store(key.data(), std::memory_order_release);
-        return {&ent.value, true};
-      }
-
-      // If someone is copying values to the slot, do busy wait.
-      while (ptr == (char *)-1) {
-        pause();
-        ptr = ent.key.load(std::memory_order_acquire);
-      }
-
-      // If the same key is already present, this is the slot we are
-      // looking for.
-      if (key == std::string_view(ptr, ent.keylen))
-        return {&ent.value, false};
-    }
-
-    std::cerr << "ConcurrentMap is full\n";
-    abort();
-  }
-
-  i64 get_idx(T *value) const {
-    uintptr_t addr = (uintptr_t)value - (uintptr_t)value % sizeof(Entry);
-    return (Entry *)addr - entries;
-  }
-
-  // Return a list of map entries sorted in a deterministic order.
-  std::vector<Entry *> get_sorted_entries(i64 shard_idx) {
-    if (nbuckets == 0)
-      return {};
-
-    i64 shard_size = nbuckets / NUM_SHARDS;
-    i64 begin = shard_idx * shard_size;
-    i64 end = begin + shard_size;
-
-    i64 sz = 0;
-    for (i64 i = begin; i < end; i++)
-      if (entries[i].key)
-        sz++;
-
-    std::vector<Entry *> vec;
-    vec.reserve(sz);
-
-    // Since the shard is circular, we need to handle the last entries
-    // as if they were next to the first entries.
-    while (entries[end - 1].key)
-      vec.push_back(entries + --end);
-
-    // Find entries contiguous in the buckets and sort them.
-    i64 last = 0;
-    for (i64 i = begin; i < end;) {
-      while (i < end && entries[i].key)
-        vec.push_back(entries + i++);
-
-      std::sort(vec.begin() + last, vec.end(), [](Entry *a, Entry *b) {
-        if (a->keylen != b->keylen)
-          return a->keylen < b->keylen;
-        return memcmp(a->key, b->key, a->keylen) < 0;
-      });
-
-      last = vec.size();
-
-      while (i < end && !entries[i].key)
-        i++;
-    }
-    return vec;
-  }
-
-  std::vector<Entry *> get_sorted_entries_all() {
-    std::vector<std::vector<Entry *>> vec(NUM_SHARDS);
-    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
-      vec[i] = get_sorted_entries(i);
-    });
-    return flatten(vec);
-  }
-
-  static constexpr i64 MIN_NBUCKETS = 4096;
-  static constexpr i64 NUM_SHARDS = 16;
-  static constexpr i64 MAX_RETRY = 256;
-
-  Entry *entries = nullptr;
-  u64 nbuckets = 0;
-};
-
-//
-// random.cc
-//
-
-void get_random_bytes(u8 *buf, i64 size);
-
-//
-// hyperloglog.cc
-//
-
-class HyperLogLog {
-public:
-  void insert(u64 hash) {
-    update_maximum(buckets[hash & (NBUCKETS - 1)], std::countl_zero(hash) + 1);
-  }
-
-  i64 get_cardinality() const;
-
-  void merge(const HyperLogLog &other) {
-    for (i64 i = 0; i < NBUCKETS; i++)
-      update_maximum(buckets[i], other.buckets[i]);
-  }
-
-private:
-  static constexpr i64 NBUCKETS = 2048;
-  static constexpr double ALPHA = 0.79402;
-
-  Atomic<u8> buckets[NBUCKETS];
-};
-
-//
-// glob.cc
-//
-
-class Glob {
-  typedef enum { STRING, STAR, QUESTION, BRACKET } Kind;
-
-  struct Element {
-    Element(Kind k) : kind(k) {}
-    Kind kind;
-    std::string str;
-    std::bitset<256> bitset;
-  };
-
-public:
-  static std::optional<Glob> compile(std::string_view pat);
-  bool match(std::string_view str);
-
-private:
-  Glob(std::vector<Element> &&vec) : elements(vec) {}
-  static bool do_match(std::string_view str, std::span<Element> elements);
-
-  std::vector<Element> elements;
-};
-
-//
-// multi-glob.cc
-//
-
-class MultiGlob {
-public:
-  bool add(std::string_view pat, i64 val);
-  bool empty() const { return strings.empty(); }
-  std::optional<i64> find(std::string_view str);
-
-private:
-  struct TrieNode {
-    i64 value = -1;
-    TrieNode *suffix_link = nullptr;
-    std::unique_ptr<TrieNode> children[256];
-  };
-
-  void compile();
-  void fix_suffix_links(TrieNode &node);
-  void fix_values();
-  i64 find_aho_corasick(std::string_view str);
-
-  std::vector<std::string> strings;
-  std::unique_ptr<TrieNode> root;
-  std::vector<std::pair<Glob, i64>> globs;
-  std::once_flag once;
-  bool is_compiled = false;
-  bool prefix_match = false;
-};
-
-//
-// filepath.cc
-//
-
-inline std::filesystem::path path_dirname(std::string_view path) {
-  return std::filesystem::path(path).parent_path();
-}
-
-inline std::string path_filename(std::string_view path) {
-  return std::filesystem::path(path).filename().string();
-}
-
-inline std::string path_clean(std::string_view path) {
-  return std::filesystem::path(path).lexically_normal().string();
-}
-
-std::string get_self_path();
-
-//
-// demangle.cc
-//
-
-std::optional<std::string_view> demangle_cpp(std::string_view name);
-std::optional<std::string_view> demangle_rust(std::string_view name);
-
-//
-// jbos.cc
-//
-
-void acquire_global_lock();
-void release_global_lock();
-
-//
-// crc32.cc
-//
-
-u32 compute_crc32(u32 crc, u8 *buf, i64 len);
-std::vector<u8> crc32_solve(u32 current, u32 desired);
-
-//
-// compress.cc
-//
-
-class Compressor {
-public:
-  virtual void write_to(u8 *buf) = 0;
-  virtual ~Compressor() {}
-  i64 compressed_size = 0;
-};
-
-class ZlibCompressor : public Compressor {
-public:
-  ZlibCompressor(u8 *buf, i64 size);
-  void write_to(u8 *buf) override;
-
-private:
-  std::vector<std::vector<u8>> shards;
-  u64 checksum = 0;
-};
-
-class ZstdCompressor : public Compressor {
-public:
-  ZstdCompressor(u8 *buf, i64 size);
-  void write_to(u8 *buf) override;
-
-private:
-  std::vector<std::vector<u8>> shards;
-};
-
-//
-// tar.cc
-//
-
-// TarFile is a class to create a tar file.
-//
-// If you pass `--repro` to mold, mold collects all input files and
-// put them into `<output-file-path>.repro.tar`, so that it is easy to
-// run the same command with the same command line arguments.
-class TarWriter {
-public:
-  static std::unique_ptr<TarWriter>
-  open(std::string output_path, std::string basedir);
-
-  ~TarWriter();
-  void append(std::string path, std::string_view data);
-
-private:
-  TarWriter(FILE *out, std::string basedir) : out(out), basedir(basedir) {}
-
-  FILE *out = nullptr;
-  std::string basedir;
-};
-
-//
-// Memory-mapped file
-//
-
-// MappedFile represents an mmap'ed input file.
-// mold uses mmap-IO only.
-class MappedFile {
-public:
-  ~MappedFile() { unmap(); }
-  void unmap();
-  void close_fd();
-  void reopen_fd(const std::string &path);
-
-  template <typename Context>
-  MappedFile *slice(Context &ctx, std::string name, u64 start, u64 size) {
-    MappedFile *mf = new MappedFile;
-    mf->name = name;
-    mf->data = data + start;
-    mf->size = size;
-    mf->parent = this;
-
-    ctx.mf_pool.emplace_back(mf);
-    return mf;
-  }
-
-  std::string_view get_contents() {
-    return std::string_view((char *)data, size);
-  }
-
-  i64 get_offset() const {
-    return parent ? (data - parent->data + parent->get_offset()) : 0;
-  }
-
-  // Returns a string that uniquely identify a file that is possibly
-  // in an archive.
-  std::string get_identifier() const {
-    if (parent) {
-      // We use the file offset within an archive as an identifier
-      // because archive members may have the same name.
-      return parent->name + ":" + std::to_string(get_offset());
-    }
-
-    if (thin_parent) {
-      // If this is a thin archive member, the filename part is
-      // guaranteed to be unique.
-      return thin_parent->name + ":" + name;
-    }
-    return name;
-  }
-
-  std::string name;
-  u8 *data = nullptr;
-  i64 size = 0;
-  bool given_fullpath = true;
-  MappedFile *parent = nullptr;
-  MappedFile *thin_parent = nullptr;
-
-  // For --dependency-file
-  bool is_dependency = true;
-
-#ifdef _WIN32
-  HANDLE fd = INVALID_HANDLE_VALUE;
-#else
-  int fd = -1;
-#endif
-};
-
-MappedFile *open_file_impl(const std::string &path, std::string &error);
-
-template <typename Context>
-MappedFile *open_file(Context &ctx, std::string path) {
-  if (path.starts_with('/') && !ctx.arg.chroot.empty())
-    path = ctx.arg.chroot + "/" + path_clean(path);
-
-  std::string error;
-  MappedFile *mf = open_file_impl(path, error);
-  if (!error.empty())
-    Fatal(ctx) << error;
-
-  if (mf)
-    ctx.mf_pool.emplace_back(mf);
-  return mf;
-}
-
-template <typename Context>
-MappedFile *must_open_file(Context &ctx, std::string path) {
-  MappedFile *mf = open_file(ctx, path);
-  if (!mf)
-    Fatal(ctx) << "cannot open " << path << ": " << errno_string();
-  return mf;
-}
-
-} // namespace mold

lib/compress.cc

@@ -14,7 +14,7 @@
 // a little bit. However, if a shard size is large enough, that loss
 // is negligible in practice.
 
-#include "common.h"
+#include "lib.h"
 
 #include <tbb/parallel_for_each.h>
 #include <zlib.h>
@@ -30,42 +30,41 @@ namespace mold {
 
 static constexpr i64 SHARD_SIZE = 1024 * 1024;
 
-static std::vector<std::string_view> split(std::string_view input) {
-  std::vector<std::string_view> shards;
-
-  while (input.size() >= SHARD_SIZE) {
-    shards.push_back(input.substr(0, SHARD_SIZE));
-    input = input.substr(SHARD_SIZE);
-  }
-  if (!input.empty())
-    shards.push_back(input);
-  return shards;
+Compressor::~Compressor() {
+  for (std::span<u8> shard : shards)
+    delete[] shard.data();
 }
 
-static std::vector<u8> zlib_compress(std::string_view input) {
-  // Initialize zlib stream. Since debug info is generally compressed
-  // pretty well with lower compression levels, we chose compression
-  // level 1.
-  z_stream strm;
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
+static std::vector<std::span<u8>> split(std::span<u8> input) {
+  std::vector<std::span<u8>> vec;
+  while (!input.empty()) {
+    i64 sz = std::min<i64>(SHARD_SIZE, input.size());
+    vec.push_back(input.subspan(0, sz));
+    input = input.subspan(sz);
+  }
+  return vec;
+}
 
-  CHECK(deflateInit2(&strm, 1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));
+static std::span<u8> zlib_compress(std::span<u8> input, int level) {
+  // Initialize zlib stream. Since debug info is generally compressed
+  // pretty well with lower compression levels, the default level is 1.
+  z_stream strm = {};
+  CHECK(deflateInit2(&strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));
 
   // Set an input buffer
   strm.avail_in = input.size();
-  strm.next_in = (u8 *)input.data();
+  strm.next_in = input.data();
 
   // Set an output buffer. deflateBound() returns an upper bound
   // on the compression size. +16 for Z_SYNC_FLUSH.
-  std::vector<u8> buf(deflateBound(&strm, strm.avail_in) + 16);
+  i64 bufsize = deflateBound(&strm, strm.avail_in) + 16;
+  u8 *buf = new u8[bufsize];
 
   // Compress data. It writes all compressed bytes except the last
   // partial byte, so up to 7 bits can be held to be written to the
   // buffer.
-  strm.avail_out = buf.size();
-  strm.next_out = buf.data();
+  strm.avail_out = bufsize;
+  strm.next_out = buf;
   CHECK(deflate(&strm, Z_BLOCK));
 
   // This is a workaround for libbacktrace before 2022-04-06.
@@ -84,29 +83,26 @@ static std::vector<u8> zlib_compress(std::string_view input) {
   // that the bit position is 7 in the next byte.
   //
   // https://github.com/ianlancetaylor/libbacktrace/pull/87
-  int nbits;
+  int nbits = 0;
   deflatePending(&strm, Z_NULL, &nbits);
   if (nbits == 5)
     CHECK(deflatePrime(&strm, 10, 2));
   CHECK(deflate(&strm, Z_SYNC_FLUSH));
 
-  assert(strm.avail_out > 0);
-  buf.resize(buf.size() - strm.avail_out);
-  buf.shrink_to_fit();
   deflateEnd(&strm);
-  return buf;
+  return {buf, (size_t)(bufsize - strm.avail_out)};
 }
 
-ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
-  std::vector<u64> adlers(inputs.size());
+ZlibCompressor::ZlibCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
+  std::vector<u32> adlers(inputs.size());
   shards.resize(inputs.size());
 
   // Compress each shard
   tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    adlers[i] = adler32(1, (u8 *)inputs[i].data(), inputs[i].size());
-    shards[i] = zlib_compress(inputs[i]);
+    std::span<u8> in = inputs[i];
+    adlers[i] = adler32(1, in.data(), in.size());
+    shards[i] = zlib_compress(in, level);
   });
 
   // Combine checksums
@@ -116,7 +112,7 @@ ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
 
   // Comput the total size
   compressed_size = 8; // the header and the trailer
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
     compressed_size += shard.size();
 }
 
@@ -127,7 +123,7 @@ void ZlibCompressor::write_to(u8 *buf) {
 
   // Copy compressed data
   std::vector<i64> offsets(shards.size());
-  offsets[0] = 2; // +2 for header
+  offsets[0] = 2; // +2 for the header
   for (i64 i = 1; i < shards.size(); i++)
     offsets[i] = offsets[i - 1] + shards[i - 1].size();
 
@@ -139,35 +135,28 @@ void ZlibCompressor::write_to(u8 *buf) {
   u8 *end = buf + compressed_size;
   end[-6] = 3;
   end[-5] = 0;
-
-  // Write a checksum
   *(ub32 *)(end - 4) = checksum;
 }
 
-static std::vector<u8> zstd_compress(std::string_view input) {
-  std::vector<u8> buf(ZSTD_COMPRESSBOUND(input.size()));
-  constexpr int LEVEL = 3; // compression level; must be between 1 to 22
-
-  size_t sz = ZSTD_compress(buf.data(), buf.size(), input.data(), input.size(),
-                            LEVEL);
+static std::span<u8> zstd_compress(std::span<u8> input, int level) {
+  i64 bufsize = ZSTD_COMPRESSBOUND(input.size());
+  u8 *buf = new u8[bufsize];
+  size_t sz = ZSTD_compress(buf, bufsize, input.data(), input.size(), level);
   assert(!ZSTD_isError(sz));
-  buf.resize(sz);
-  buf.shrink_to_fit();
-  return buf;
+  return {buf, sz};
 }
 
-ZstdCompressor::ZstdCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
+ZstdCompressor::ZstdCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
   shards.resize(inputs.size());
 
   // Compress each shard
   tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    shards[i] = zstd_compress(inputs[i]);
+    shards[i] = zstd_compress(inputs[i], level);
   });
 
   compressed_size = 0;
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
     compressed_size += shard.size();
 }
 

lib/config.h.in

@@ -1,7 +1,30 @@
 #define MOLD_VERSION "@mold_VERSION@"
 #define MOLD_LIBDIR "@CMAKE_INSTALL_FULL_LIBDIR@"
+#define MOLD_FIRST_TARGET @MOLD_FIRST_TARGET@
 
-#cmakedefine HAVE_MADVISE 1
-#cmakedefine HAVE_UNAME 1
-#cmakedefine MOLD_USE_MIMALLOC 1
-#cmakedefine MOLD_USE_SYSTEM_MIMALLOC 1
+#cmakedefine01 HAVE_FALLOCATE
+#cmakedefine01 HAVE_MADVISE
+#cmakedefine01 HAVE_UNAME
+#cmakedefine01 MOLD_USE_MIMALLOC
+#cmakedefine01 MOLD_USE_SYSTEM_MIMALLOC
+
+#cmakedefine01 HAVE_TARGET_X86_64
+#cmakedefine01 HAVE_TARGET_I386
+#cmakedefine01 HAVE_TARGET_ARM64LE
+#cmakedefine01 HAVE_TARGET_ARM64BE
+#cmakedefine01 HAVE_TARGET_ARM32LE
+#cmakedefine01 HAVE_TARGET_ARM32BE
+#cmakedefine01 HAVE_TARGET_RV32LE
+#cmakedefine01 HAVE_TARGET_RV32BE
+#cmakedefine01 HAVE_TARGET_RV64LE
+#cmakedefine01 HAVE_TARGET_RV64BE
+#cmakedefine01 HAVE_TARGET_PPC32
+#cmakedefine01 HAVE_TARGET_PPC64V1
+#cmakedefine01 HAVE_TARGET_PPC64V2
+#cmakedefine01 HAVE_TARGET_S390X
+#cmakedefine01 HAVE_TARGET_SPARC64
+#cmakedefine01 HAVE_TARGET_M68K
+#cmakedefine01 HAVE_TARGET_SH4LE
+#cmakedefine01 HAVE_TARGET_SH4BE
+#cmakedefine01 HAVE_TARGET_LOONGARCH64
+#cmakedefine01 HAVE_TARGET_LOONGARCH32

lib/crc32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <tbb/parallel_for_each.h>
 #include <zlib.h>

lib/demangle.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <cstdlib>
 

lib/filepath.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <filesystem>
 #include <sys/stat.h>

lib/gentoo-gpkg-workaround.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Work around a deadlock in portage's binary package extraction.
+#
+# tar_stream_reader.close() joins its stdin-feeder thread and then waits
+# for the external decompressor to exit without draining the
+# decompressor's remaining stdout. tarfile stops reading at the tar
+# end-of-archive marker, so trailing archive padding is left in the
+# pipe. If that residue exceeds the pipe buffer, the decompressor
+# blocks writing forever and close() never returns, freezing the whole
+# emerge. Small pipe buffers make this reachable in practice: the
+# kernel shrinks new pipes to one page once a user exceeds
+# fs.pipe-user-pages-soft, which our 80 parallel containers do.
+#
+# The fix is to drain the decompressor's stdout to EOF before joining
+# and waiting. This script patches the installed gpkg.py at container
+# image build time and fails loudly if the code no longer matches, so
+# a portage upgrade that changes (or fixes) this code is noticed.
+
+import sys
+import portage.gpkg
+
+path = portage.gpkg.__file__
+src = open(path).read()
+
+old = """        if self.proc is not None:
+            self.thread.join()
+"""
+
+new = """        if self.proc is not None:
+            # Drain remaining decompressor output before joining and
+            # waiting; otherwise the decompressor can block forever on
+            # a full stdout pipe. Injected by mold's lib/gentoo-test.sh.
+            try:
+                while self.read_io.read(65536):
+                    pass
+            except Exception:
+                pass
+            self.thread.join()
+"""
+
+if src.count(old) != 1:
+    sys.exit(f"gpkg workaround: expected pattern not found in {path}; "
+             "portage changed, update lib/gentoo-gpkg-workaround.py")
+
+with open(path, "w") as f:
+    f.write(src.replace(old, new))
+
+import py_compile
+py_compile.compile(path, doraise=True)
+print(f"gpkg deadlock workaround applied to {path}")

lib/gentoo-test.sh

@@ -2,10 +2,25 @@
 #
 # This test script takes a Gentoo package name and tries to build it
 # using mold in a Podman environment. We chose Gentoo Linux as a test
-# target, because its source-based package allows us to build programs
+# target, because its source-based packages allow us to build programs
 # locally and run their test suites without any hassle.
 #
-# You can get a complete list of Gentoo packages availalbe for testing
+# We first build a package with mold acting as /usr/bin/ld. Only if
+# that fails, we build the same package again with GNU ld to tell
+# whether the failure is mold's fault. The package's own test suite
+# runs in both cases; FEATURES=test is scoped to just the package
+# under test via package.env, because applying it to the entire
+# dependency graph makes a large fraction of the tree unresolvable.
+# Results are classified into three directories:
+#
+#   success: the package built fine with mold (or, for a control
+#            build, with GNU ld)
+#   failure: the package built with GNU ld but not with mold, which
+#            means it hit a mold bug
+#   broken:  the package didn't build even with GNU ld, so the
+#            failure is not mold's fault
+#
+# You can get a complete list of Gentoo packages available for testing
 # with the following command:
 #
 # podman run --rm mold-gentoo emerge --color n -s '' | \
@@ -20,50 +35,152 @@ fi
 
 set -x
 
-# Create a Podman image
-if ! podman image ls mold-gentoo | grep -q mold-gentoo; then
-  set -e
-  cat <<EOF | podman build -t mold-gentoo -
-FROM docker.io/gentoo/stage3
-RUN emerge-webrsync
-RUN echo 'USE="X ssl elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary -perl"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
-    echo 'FEATURES="\${FEATURE} noclean nostrip ccache -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
-    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
-    FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache xeyes dev-build/cmake dev-vcs/git && \
-    rm -rf /var/tmp/portage
-EOF
-  set +e
+# Test results are keyed by the commit hash embedded in the mold
+# binary, so refuse to run with a binary that doesn't have one.
+git_hash=$(./dist/mold --version | perl -ne '/\(([0-9a-f]{40})\b/ and print $1;')
+
+if [ "$git_hash" = "" ]; then
+  echo "Cannot get a git hash from ./dist/mold --version. Run this script at the mold source root, with dist/mold built from a git checkout."
+  exit 1
 fi
 
-git_hash=$(./dist/mold --version | perl -ne '/\((\w+)/; print $1;')
+cache_dir=/var/cache/mold-gentoo
+
+# Create a Podman image. The lock prevents parallel invocations of
+# this script from racing to build the same image.
+#
+# A few notes on the container configuration:
+#
+# - We accept ~amd64 (testing) packages to test the newest version of
+#   everything. The stage3 image is built for the stable profile, so
+#   we update the whole system to ~amd64 once here; without that,
+#   half-upgraded packages (e.g. perl and its virtuals) cause
+#   unsolvable dependency conflicts in every container.
+#
+# - EMERGE_DEFAULT_OPTS makes emerge automatically apply the USE flag
+#   adjustments it would otherwise just print and give up on. An
+#   ephemeral container has no /etc/portage/package.use accumulated by
+#   an administrator, so without this, thousands of packages are
+#   unbuildable out of the box.
+#
+# - freetype and harfbuzz mutually depend on each other, so we
+#   pre-install them with the documented two-step bootstrap. During
+#   that step, pillow is built without truetype support to break
+#   another dependency cycle (docutils -> pillow -> harfbuzz -> glib
+#   -> docutils). The override is removed afterwards; once the cycle
+#   participants are installed, they satisfy buildtime dependencies
+#   and the cycle can no longer form.
+#
+# - qtbase's test suite requires icu (REQUIRED_USE), which autounmask
+#   doesn't resolve on its own, hence the package.use entry.
+#
+# - gcc doesn't auto-select a newly installed major version, so we
+#   switch to the newest slot explicitly after the world update.
+#
+# - lib/gentoo-gpkg-workaround.py patches a deadlock in portage's
+#   binary package extraction. See that file for details.
+#
+# - static-libs is not in USE because it contradicts REQUIRED_USE
+#   constraints in some packages (e.g. lvm2 forbids static-libs with
+#   udev), and building static archives doesn't exercise the linker
+#   anyway. Where a dependency genuinely needs it, emerge re-enables
+#   it per package via autounmask.
+#
+# - /etc/portage/env/test.conf allows marking a single package for
+#   FEATURES=test through package.env, which is how we scope test
+#   suites to just the package under test.
+#
+# - The image build compiles a couple hundred packages from source,
+#   because the Gentoo binhost only carries stable versions built with
+#   default USE flags, which rarely match our ~amd64/USE combination.
+#   To pay that cost only once, the build publishes what it compiles
+#   to the GNU ld binary package pool and reuses it, so a rebuilt
+#   image only compiles packages whose version has changed since the
+#   last build. The toolchain step still uses the binhost, where our
+#   configuration does match (e.g. clang). Since it verifies binhost
+#   package signatures and our own pool is unsigned, it runs with a
+#   private PKGDIR so the two never mix.
+(
+  flock 9
+  if ! podman image exists mold-gentoo; then
+    cat <<EOF | podman build -v $cache_dir/distfiles:/var/cache/distfiles -v $cache_dir/ccache:/ccache -v $cache_dir/binpkgs-ld:/var/cache/binpkgs -v `pwd`/lib:/mold-lib:ro -t mold-gentoo - || exit 1
+FROM docker.io/gentoo/stage3
+RUN emerge-webrsync
+RUN echo 'USE="X ssl elogind -systemd truetype jpeg jpeg2k tiff zstd -perl udev lcms alsa"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
+    echo 'FEATURES="nostrip ccache binpkg-multi-instance -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
+    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
+    echo 'MAKEOPTS="-j$(nproc) --load-average=100"' >> /etc/portage/make.conf && \
+    echo 'EMERGE_DEFAULT_OPTS="--autounmask-continue=y --autounmask-license=n --autounmask-backtrack=y --backtrack=100 --jobs=16 --load-average=100"' >> /etc/portage/make.conf && \
+    mkdir -p /etc/portage/env /etc/portage/package.use && \
+    echo 'FEATURES="test"' > /etc/portage/env/test.conf && \
+    printf 'media-libs/freetype harfbuzz\ndev-qt/qtbase icu\n' > /etc/portage/package.use/mold && \
+    echo 'dev-python/pillow -truetype' > /etc/portage/package.use/cycle-break && \
+    FEATURES=buildpkg emerge --usepkg --update --deep --newuse @world && \
+    gcc-config \$(gcc-config -l | tail -n 1 | awk '{print \$2}') && env-update && \
+    USE=-harfbuzz FEATURES=buildpkg emerge --oneshot media-libs/freetype && \
+    FEATURES=buildpkg emerge --usepkg --oneshot media-libs/freetype media-libs/harfbuzz && \
+    rm /etc/portage/package.use/cycle-break && \
+    PKGDIR=/var/tmp/binpkgs FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache dev-build/cmake dev-vcs/git && \
+    rm -rf /var/tmp/portage /var/tmp/binpkgs
+RUN python3 /mold-lib/gentoo-gpkg-workaround.py
+EOF
+  fi
+) 9> /tmp/mold-gentoo-image.lock || exit 1
 
 if [ "$package" = dev-libs/concurrencykit ]; then
   echo "Skipping known broken package: $package"
   exit 0
 fi
 
-# Build a given package in Podman
+# Build a given package in Podman.
+#
+# Build artifacts are shared between containers through host
+# directories bind-mounted into every container:
+#
+#   /var/cache/mold-gentoo/ccache: compiler cache
+#   /var/cache/mold-gentoo/distfiles: source tarballs, so that each
+#     distfile is downloaded only once across all containers and runs
+#   /var/cache/mold-gentoo/binpkgs-{mold,ld}: binary packages, so that each
+#     dependency is built only once instead of once per container that
+#     needs it
+#
+# Every emerge runs with FEATURES=buildpkg to publish what it built to
+# the binary package pool, and dependencies are installed with
+# --usepkg from the pool when an exact version and USE match exists.
+# The package under test itself is always built from source. In the
+# mold run, dependencies are still linked with mold and their binaries
+# are still exercised by the test suites of everything built on top of
+# them; the pool just memoizes that work across containers. The mold
+# and GNU ld runs use separate pools so that a control build can never
+# install mold-linked binaries.
 cmd1='(cd /usr/bin; ln -sf /mold/dist/mold $(realpath ld))'
-cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
-cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
+cmd2="echo '$package test.conf' > /etc/portage/package.env"
+cmd3="FEATURES=buildpkg emerge --usepkg --onlydeps $package"
+cmd4="FEATURES=buildpkg emerge $package"
 filename=`echo "$package" | sed 's!/!_!g'`
-podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 3h"
+podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v $cache_dir/ccache:/ccache -v $cache_dir/distfiles:/var/cache/distfiles"
+run="mold-gentoo timeout -v -k 15s 3h chrt --idle 0 nice -n 19 bash -c"
 dir=gentoo/$git_hash
 
-mkdir -p "$dir"/success "$dir"/failure
+mkdir -p "$dir"/success "$dir"/failure "$dir"/broken
 
-$podman nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
+# Build the package with mold. If this succeeds, we don't need a
+# control build.
+$podman -v $cache_dir/binpkgs-mold:/var/cache/binpkgs $run "$cmd1 && $cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".mold
 if [ $? = 0 ]; then
   mv "$dir"/"$filename".mold "$dir"/success
-else
-  mv "$dir"/"$filename".mold "$dir"/failure
+  exit 0
 fi
 
-$podman nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
+# Build the package again with GNU ld to tell whether the failure is
+# mold's fault.
+$podman -v $cache_dir/binpkgs-ld:/var/cache/binpkgs $run "$cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".ld
 if [ $? = 0 ]; then
+  mv "$dir"/"$filename".mold "$dir"/failure
   mv "$dir"/"$filename".ld "$dir"/success
 else
-  mv "$dir"/"$filename".ld "$dir"/failure
+  mv "$dir"/"$filename".mold "$dir"/broken
+  mv "$dir"/"$filename".ld "$dir"/broken
 fi

lib/glob.cc

@@ -1,15 +1,21 @@
-#include "common.h"
+// This file implements the glob matcher used for symbol name patterns.
+// Exact, prefix and suffix patterns are matched directly, and simple
+// substring patterns are combined into an Aho-Corasick matcher. The
+// remaining patterns are matched with the non-recursive algorithm described
+// at https://research.swtch.com/glob. If there are many such patterns, a
+// bit-parallel NFA matches them together.
 
-#include <cstring>
+#include "lib.h"
 
 namespace mold {
 
-std::optional<Glob> Glob::compile(std::string_view pat) {
-  std::vector<Element> vec;
+std::optional<Glob::Pattern> Glob::Pattern::compile(std::string_view pat,
+                                                    i64 value) {
+  std::vector<Token> tokens;
 
   while (!pat.empty()) {
     u8 c = pat[0];
-    pat = pat.substr(1);
+    pat.remove_prefix(1);
 
     switch (c) {
     case '[': {
@@ -18,27 +24,30 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
       // [abc]: a, b or c
       // [$\]!]: $, ] or !
       // [a-czg-i]: a, b, c, z, g, h, or i
-      // [^a-z]: Any character except lowercase letters
-      vec.emplace_back(BRACKET);
-      std::bitset<256> &bitset = vec.back().bitset;
-
+      // [!a-z]: Any character except lowercase letters
+      //
+      // Both `!` and `^` are accepted as negation markers. `!` is the
+      // POSIX/shell convention used by other linkers. `^` was mold's
+      // original syntax and is kept for backward compatibility.
+      tokens.emplace_back(BRACKET);
+      std::bitset<256> &chars = tokens.back().chars;
       bool negate = false;
-      if (!pat.empty() && pat[0] == '^') {
-        negate = true;
-        pat = pat.substr(1);
-      }
-
       bool closed = false;
 
+      if (!pat.empty() && (pat[0] == '!' || pat[0] == '^')) {
+        negate = true;
+        pat.remove_prefix(1);
+      }
+
       while (!pat.empty()) {
         if (pat[0] == ']') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           closed = true;
           break;
         }
 
         if (pat[0] == '\\') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           if (pat.empty())
             return {};
         }
@@ -46,23 +55,23 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
         if (pat.size() >= 3 && pat[1] == '-') {
           u8 start = pat[0];
           u8 end = pat[2];
-          pat = pat.substr(3);
+          pat.remove_prefix(3);
 
           if (end == '\\') {
             if (pat.empty())
               return {};
             end = pat[0];
-            pat = pat.substr(1);
+            pat.remove_prefix(1);
           }
 
           if (end < start)
             return {};
 
           for (i64 i = start; i <= end; i++)
-            bitset[i] = true;
+            chars[i] = true;
         } else {
-          bitset[(u8)pat[0]] = true;
-          pat = pat.substr(1);
+          chars[(u8)pat[0]] = true;
+          pat.remove_prefix(1);
         }
       }
 
@@ -70,87 +79,289 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
         return {};
 
       if (negate)
-        bitset.flip();
+        chars.flip();
       break;
     }
     case '?':
-      vec.emplace_back(QUESTION);
+      tokens.emplace_back(QUESTION);
       break;
     case '*':
-      vec.emplace_back(STAR);
+      if (tokens.empty() || tokens.back().kind != STAR)
+        tokens.emplace_back(STAR);
       break;
     case '\\':
       if (pat.empty())
         return {};
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.emplace_back(STRING);
-      vec.back().str += pat[0];
-      pat = pat.substr(1);
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += pat[0];
+      pat.remove_prefix(1);
       break;
     default:
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.emplace_back(STRING);
-      vec.back().str += c;
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += c;
       break;
     }
   }
-
-  return {Glob{std::move(vec)}};
+  return Pattern(std::move(tokens), value);
 }
 
-bool Glob::match(std::string_view str) {
-  return do_match(str, elements);
-}
+bool Glob::Pattern::match(std::string_view str) const {
+  if (!tokens.empty() && tokens.back().kind == STRING &&
+      !str.ends_with(tokens.back().str))
+    return false;
 
-bool Glob::do_match(std::string_view str, std::span<Element> elements) {
-  while (!elements.empty()) {
-    Element &e = elements[0];
-    elements = elements.subspan(1);
+  i64 x = 0;
+  i64 y = 0;
+  i64 next_x = -1;
+  i64 next_y = -1;
 
-    switch (e.kind) {
-    case STRING:
-      if (!str.starts_with(e.str))
-        return false;
-      str = str.substr(e.str.size());
-      break;
-    case STAR:
-      if (elements.empty())
-        return true;
+  while (x < str.size() || y < tokens.size()) {
+    if (y < tokens.size()) {
+      const Token &tok = tokens[y];
 
-      // Patterns like "*foo*bar*" should be much more common than more
-      // complex ones like "*foo*[abc]*" or "*foo**?bar*", so we optimize
-      // the former case here.
-      if (elements[0].kind == STRING) {
-        for (;;) {
-          size_t pos = str.find(elements[0].str);
+      switch (tok.kind) {
+      case STRING:
+        if (str.substr(x).starts_with(tok.str)) {
+          x += tok.str.size();
+          y++;
+          continue;
+        }
+        break;
+      case STAR:
+        next_y = y++;
+        next_x = x + 1;
+
+        if (y < tokens.size() && tokens[y].kind == STRING) {
+          const std::string &s = tokens[y].str;
+          i64 pos = str.find(s, x);
           if (pos == str.npos)
             return false;
-          if (do_match(str.substr(pos + elements[0].str.size()),
-                       elements.subspan(1)))
-            return true;
-          str = str.substr(pos + 1);
+          next_x = pos + 1;
+          x = pos + s.size();
+          y++;
         }
+        continue;
+      case QUESTION:
+        if (x < str.size()) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
+      case BRACKET:
+        if (x < str.size() && tok.chars[(u8)str[x]]) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
       }
+    }
 
-      // Other cases are handled here.
-      for (i64 j = 0; j < str.size(); j++)
-        if (do_match(str.substr(j), elements))
-          return true;
-      return false;
-    case QUESTION:
-      if (str.empty())
-        return false;
-      str = str.substr(1);
-      break;
-    case BRACKET:
-      if (str.empty() || !e.bitset[str[0]])
-        return false;
-      str = str.substr(1);
-      break;
+    // Retry the last star after assigning one more input byte to it.
+    if (next_x != -1 && next_x <= str.size()) {
+      x = next_x;
+      y = next_y;
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+void Glob::Nfa::compile(std::span<const Pattern> patterns) {
+  i64 num_states = 0;
+
+  for (const Pattern &pattern : patterns) {
+    num_states++;
+
+    for (const Pattern::Token &tok : pattern.tokens) {
+      if (tok.kind == Pattern::STRING)
+        num_states += tok.str.size();
+      else if (tok.kind != Pattern::STAR)
+        num_states++;
     }
   }
 
-  return str.empty();
+  i64 num_words = (num_states + 63) / 64;
+  initial_states.resize(num_words);
+  star_states.resize(num_words);
+  accept_states.resize(num_words);
+  char_masks.resize(256 * num_words);
+  values.resize(num_states, -1);
+
+  auto set_bit = [](std::vector<u64> &vec, i64 pos) {
+    vec[pos / 64] |= 1ULL << (pos % 64);
+  };
+
+  i64 state = 0;
+
+  for (const Pattern &pattern : patterns) {
+    set_bit(initial_states, state);
+
+    for (const Pattern::Token &tok : pattern.tokens) {
+      switch (tok.kind) {
+      case Pattern::STRING:
+        for (u8 c : tok.str) {
+          state++;
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        }
+        break;
+      case Pattern::STAR:
+        set_bit(star_states, state);
+        break;
+      case Pattern::QUESTION:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      case Pattern::BRACKET:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          if (tok.chars[c])
+            char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      }
+    }
+
+    set_bit(accept_states, state);
+    values[state] = pattern.value;
+    state++;
+  }
+}
+
+i64 Glob::Nfa::match(std::string_view str) const {
+  static thread_local std::vector<u64> states;
+  states.assign(initial_states.begin(), initial_states.end());
+
+  i64 num_words = states.size();
+
+  for (u8 c : str) {
+    const u64 *mask = char_masks.data() + c * num_words;
+    u64 carry = 0;
+
+    for (i64 i = 0; i < num_words; i++) {
+      u64 old = states[i];
+      u64 next = (old << 1) | carry;
+      states[i] = (old & star_states[i]) | (next & mask[i]);
+      carry = old >> 63;
+    }
+  }
+
+  i64 value = -1;
+
+  for (i64 i = 0; i < num_words; i++) {
+    u64 word = states[i] & accept_states[i];
+
+    while (word) {
+      i64 bit = std::countr_zero(word);
+      value = std::max(value, values[i * 64 + bit]);
+      word &= word - 1;
+    }
+  }
+  return value;
+}
+
+static bool is_literal(std::string_view pat) {
+  return pat.find_first_of("*?[\\") == pat.npos;
+}
+
+bool Glob::add(std::string_view pat, i64 val) {
+  assert(val >= 0);
+  assert(!is_compiled);
+  is_empty = false;
+  max_value = std::max(max_value, val);
+
+  // Match-all, exact, prefix and suffix patterns are handled with
+  // plain string comparisons instead of the matchers below, which
+  // have to scan the entire input string on every query.
+  if (pat == "*") {
+    match_all = std::max(match_all, val);
+    return true;
+  }
+
+  if (is_literal(pat)) {
+    exacts.push_back({std::string(pat), val});
+    return true;
+  }
+
+  if (pat.ends_with('*') && is_literal(pat.substr(0, pat.size() - 1))) {
+    prefixes.push_back({std::string(pat.substr(0, pat.size() - 1)), val});
+    return true;
+  }
+
+  if (pat.starts_with('*') && is_literal(pat.substr(1))) {
+    suffixes.push_back({std::string(pat.substr(1)), val});
+    return true;
+  }
+
+  // If the pattern requires only a single substring search, the
+  // Aho-Corasick algorithm is even faster than our glob matcher.
+  if (aho_corasick.can_handle(pat))
+    return aho_corasick.add(pat, val);
+
+  if (std::optional<Pattern> pattern = Pattern::compile(pat, val)) {
+    patterns.push_back(std::move(*pattern));
+    return true;
+  }
+  return false;
+}
+
+i64 Glob::find(std::string_view str) {
+  std::call_once(once, [&] {
+    // If the same name was added more than once, keep only the entry
+    // with the largest value, as find() returns the largest match.
+    // Sorting by (name, negated value) places that entry first in
+    // each run of duplicates, which is the one unique() keeps.
+    ranges::sort(exacts, {}, [](const LiteralPattern &p) {
+      return std::pair<std::string_view, i64>(p.pat, -p.value);
+    });
+
+    auto dup = ranges::unique(exacts, {}, &LiteralPattern::pat);
+    exacts.erase(dup.begin(), dup.end());
+
+    if (patterns.size() >= 64) {
+      nfa.compile(patterns);
+      patterns.clear();
+    }
+
+    aho_corasick.compile();
+    is_compiled = true;
+  });
+
+  i64 val = match_all;
+  if (val == max_value)
+    return val;
+
+  auto it = ranges::lower_bound(exacts, str, {}, &LiteralPattern::pat);
+  if (it != exacts.end() && it->pat == str)
+    val = std::max(val, it->value);
+
+  for (const LiteralPattern &p : prefixes)
+    if (val < p.value && str.starts_with(p.pat))
+      val = p.value;
+
+  for (const LiteralPattern &p : suffixes)
+    if (val < p.value && str.ends_with(p.pat))
+      val = p.value;
+
+  if (val == max_value)
+    return val;
+
+  val = std::max(val, aho_corasick.find(str));
+  if (val == max_value)
+    return val;
+
+  if (!nfa.empty())
+    val = std::max(val, nfa.match(str));
+
+  for (const Pattern &p : patterns)
+    if (val < p.value && p.match(str))
+      val = p.value;
+
+  return val;
 }
 
 } // namespace mold

lib/hyperloglog.cc

@@ -4,13 +4,18 @@
 // For more info, read
 // https://engineering.fb.com/2018/12/13/data-infrastructure/hyperloglog
 
-#include "common.h"
+#include "lib.h"
 
 #include <cmath>
 
 namespace mold {
 
 i64 HyperLogLog::get_cardinality() const {
+  u8 buckets[NBUCKETS] = {};
+  for (const Sketch &sketch : sketches)
+    for (i64 i = 0; i < NBUCKETS; i++)
+      buckets[i] = std::max(buckets[i], sketch.buckets[i]);
+
   double z = 0;
   for (i64 val : buckets)
     z += std::ldexp(1.0, -val);

lib/integers.h

@@ -5,24 +5,30 @@
 //
 // 1. mold is always a cross linker and should not depend on what host it
 //    is running on. For example, users should be able to run mold on a
-//    big-endian SPARC machine to create a little-endian RV64 binary.
+//    little-endian x86 machine to create a big-endian s390x binary.
 //
 // 2. Even though data members in all ELF data strucutres are naturally
 //    aligned, they are not guaranteed to be aligned on memory because of
-//    archive files. Archive files (.a files) align each member only to a
+//    archive files. Archive files (.a files) align each file only to a
 //    2 byte boundary, so anything larger than 2 bytes may be misaligned
 //    in an mmap'ed memory. Misaligned access is an undefined behavior in
 //    C/C++, so we shouldn't cast an arbitrary pointer to a uint32_t, for
 //    example, to read a 32 bit value.
 //
-// The data types defined in this file don't depend on host byte order and
-// don't do unaligned access. Note that modern compilers are smart enough
-// to recognize shift and bitwise OR patterns and compile them into a
-// single load instruction.
+// The data types defined in this file are independent of the host byte
+// order and are designed to avoid unaligned access.
+//
+// Note that in C/C++, memcpy is a portable and efficient way to access
+// unaligned data, as it is typically treated as an intrinsic. Compilers
+// can easily optimize memcpy calls in this file into a single load or
+// store instruction.
 
 #pragma once
 
+#include <bit>
 #include <cstdint>
+#include <cstring>
+#include <type_traits>
 
 #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
 # if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
@@ -36,93 +42,70 @@
 
 namespace mold {
 
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-
-typedef int8_t i8;
-typedef int16_t i16;
-typedef int32_t i32;
-typedef int64_t i64;
-
 template <typename T, bool is_le, int size = sizeof(T)>
 class Integer {
 public:
   constexpr Integer() = default;
 
-  constexpr Integer(T x) requires (is_le && size == 2)
-    : buf{(u8)x, (u8)(x >> 8)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 3)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 4)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16), (u8)(x >> 24)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 8)
-    : buf{(u8)x,         (u8)(x >> 8),  (u8)(x >> 16), (u8)(x >> 24),
-          (u8)(x >> 32), (u8)(x >> 40), (u8)(x >> 48), (u8)(x >> 56)} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 2)
-    : buf{(u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 3)
-    : buf{(u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 4)
-    : buf{(u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 8)
-    : buf{(u8)(x >> 56), (u8)(x >> 48), (u8)(x >> 40), (u8)(x >> 32),
-          (u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8),  (u8)x} {}
-
-  operator T() const {
-    if constexpr (is_le) {
-      if constexpr (size == 2)
-        return buf[1] << 8 | buf[0];
-      else if constexpr (size == 3)
-        return buf[2] << 16 | buf[1] << 8 | buf[0];
-      else if constexpr (size == 4)
-        return buf[3] << 24 | buf[2] << 16 | buf[1] << 8 | buf[0];
-      else
-        return (u64)buf[7] << 56 | (u64)buf[6] << 48 |
-               (u64)buf[5] << 40 | (u64)buf[4] << 32 |
-               (u64)buf[3] << 24 | (u64)buf[2] << 16 |
-               (u64)buf[1] << 8  | (u64)buf[0];
+  constexpr Integer(T v) {
+    if (std::is_constant_evaluated() || size == 3) {
+      for (int i = 0; i < size; i++)
+        buf[is_le ? i : (size - i - 1)] = v >> (i * 8);
     } else {
-      if constexpr (size == 2)
-        return buf[0] << 8 | buf[1];
-      else if constexpr (size == 3)
-        return buf[0] << 16 | buf[1] << 8 | buf[2];
-      else if constexpr (size == 4)
-        return buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3];
-      else
-        return (u64)buf[0] << 56 | (u64)buf[1] << 48 |
-               (u64)buf[2] << 40 | (u64)buf[3] << 32 |
-               (u64)buf[4] << 24 | (u64)buf[5] << 16 |
-               (u64)buf[6] << 8  | (u64)buf[7];
+      if (!is_native)
+        v = bswap(v);
+      memcpy(buf, &v, size);
     }
   }
 
-  Integer &operator=(T x) {
-    new (this) Integer(x);
-    return *this;
+  operator T() const {
+    if constexpr (size == 3) {
+      if constexpr (is_le)
+        return buf[2] << 16 | buf[1] << 8 | buf[0];
+      return buf[0] << 16 | buf[1] << 8 | buf[2];
+    } else {
+      T v;
+      memcpy(&v, buf, size);
+      return is_native ? v : bswap(v);
+    }
   }
 
+  Integer &operator=(T v)  { new (this) Integer(v); return *this; }
   Integer &operator++()    { return *this = *this + 1; }
-  Integer operator++(int)  { return ++*this - 1; }
+  Integer operator++(int)  { auto x = *this; ++*this; return x; }
   Integer &operator--()    { return *this = *this - 1; }
-  Integer operator--(int)  { return --*this + 1; }
-  Integer &operator+=(T x) { return *this = *this + x; }
-  Integer &operator-=(T x) { return *this = *this - x; }
-  Integer &operator&=(T x) { return *this = *this & x; }
-  Integer &operator|=(T x) { return *this = *this | x; }
+  Integer operator--(int)  { auto x = *this; --*this; return x; }
+  Integer &operator+=(T v) { return *this = *this + v; }
+  Integer &operator-=(T v) { return *this = *this - v; }
+  Integer &operator&=(T v) { return *this = *this & v; }
+  Integer &operator|=(T v) { return *this = *this | v; }
 
 private:
-  u8 buf[size];
+  static constexpr bool is_native =
+    (std::endian::native == (is_le ? std::endian::little : std::endian::big));
+
+  static T bswap(T v) {
+    switch (size) {
+    case 2: return __builtin_bswap16(v);
+    case 4: return __builtin_bswap32(v);
+    case 8: return __builtin_bswap64(v);
+    default: __builtin_unreachable();
+    }
+  }
+
+  uint8_t buf[size];
 };
 
+using i8 = int8_t;
+using i16 = int16_t;
+using i32 = int32_t;
+using i64 = int64_t;
+
+using u8 = uint8_t;
+using u16 = uint16_t;
+using u32 = uint32_t;
+using u64 = uint64_t;
+
 using il16 = Integer<i16, true>;
 using il32 = Integer<i32, true>;
 using il64 = Integer<i64, true>;

lib/mapped-file-unix.cc

@@ -1,51 +0,0 @@
-#include "common.h"
-
-namespace mold {
-
-MappedFile *open_file_impl(const std::string &path, std::string &error) {
-  i64 fd = ::open(path.c_str(), O_RDONLY);
-  if (fd == -1) {
-    if (errno != ENOENT)
-      error = "opening " + path + " failed: " + errno_string();
-    return nullptr;
-  }
-
-  struct stat st;
-  if (fstat(fd, &st) == -1)
-    error = path + ": fstat failed: " + errno_string();
-
-  MappedFile *mf = new MappedFile;
-  mf->name = path;
-  mf->size = st.st_size;
-
-  if (st.st_size > 0) {
-    mf->data = (u8 *)mmap(nullptr, st.st_size, PROT_READ | PROT_WRITE,
-                          MAP_PRIVATE, fd, 0);
-    if (mf->data == MAP_FAILED)
-      error = path + ": mmap failed: " + errno_string();
-  }
-
-  close(fd);
-  return mf;
-}
-
-void MappedFile::unmap() {
-  if (size == 0 || parent || !data)
-    return;
-  munmap(data, size);
-  data = nullptr;
-}
-
-void MappedFile::close_fd() {
-  if (fd == -1)
-    return;
-  close(fd);
-  fd = -1;
-}
-
-void MappedFile::reopen_fd(const std::string &path) {
-  if (fd == -1)
-    fd = open(path.c_str(), O_RDONLY);
-}
-
-} // namespace mold

lib/mimalloc.cc

@@ -1,24 +0,0 @@
-#include "common.h"
-
-// Including mimalloc-new-delete.h overrides new/delete operators.
-// We need it only when we are using mimalloc as a dynamic library.
-#if MOLD_USE_SYSTEM_MIMALLOC
-# include <mimalloc-new-delete.h>
-#endif
-
-// Silence mimalloc warning messages that users can just ignore.
-#if MOLD_USE_MIMALLOC
-# include <mimalloc.h>
-
-namespace mold {
-void set_mimalloc_options() {
-  mi_option_disable(mi_option_verbose);
-  mi_option_disable(mi_option_show_errors);
-}
-}
-
-#else
-namespace mold {
-void set_mimalloc_options() {}
-}
-#endif

lib/multi-glob.cc

@@ -1,186 +0,0 @@
-// This file implements the Aho-Corasick algorithm to match multiple
-// glob patterns to symbol strings as quickly as possible.
-//
-// Here are some examples of glob patterns:
-//
-//    qt_private_api_tag*
-//    *16QAccessibleCache*
-//    *32QAbstractFileIconProviderPrivate*
-//    *17QPixmapIconEngine*
-//
-// `*` is a wildcard that matches any substring. We sometimes have
-// hundreds of glob patterns and have to match them against millions
-// of symbol strings.
-//
-// Aho-Corasick cannot handle complex patterns such as `*foo*bar*`.
-// We handle such patterns with the Glob class. Glob is relatively
-// slow, but complex patterns are rare in practice, so it should be
-// OK.
-
-#include "common.h"
-
-#include <queue>
-#include <regex>
-
-namespace mold {
-
-std::optional<i64> MultiGlob::find(std::string_view str) {
-  std::call_once(once, [&] { compile(); });
-  i64 val = -1;
-
-  // Match against simple glob patterns
-  if (root)
-    val = find_aho_corasick(str);
-
-  // Match against complex glob patterns
-  for (std::pair<Glob, i64> &glob : globs)
-    if (glob.first.match(str))
-      val = std::max(val, glob.second);
-
-  if (val == -1)
-    return {};
-  return val;
-}
-
-i64 MultiGlob::find_aho_corasick(std::string_view str) {
-  TrieNode *node = root.get();
-  i64 val = -1;
-
-  auto walk = [&](u8 c) {
-    for (;;) {
-      if (node->children[c]) {
-        node = node->children[c].get();
-        val = std::max(val, node->value);
-        return;
-      }
-
-      if (!node->suffix_link)
-        return;
-      node = node->suffix_link;
-    }
-  };
-
-  walk('\0');
-
-  for (u8 c : str) {
-    if (prefix_match && node == root.get())
-      return val;
-    walk(c);
-  }
-
-  walk('\0');
-  return val;
-}
-
-static bool is_simple_pattern(std::string_view pat) {
-  static std::regex re(R"(\*?[^*[?]+\*?)", std::regex_constants::optimize);
-  return std::regex_match(pat.begin(), pat.end(), re);
-}
-
-static std::string handle_stars(std::string_view pat) {
-  std::string str(pat);
-
-  // Convert "foo" -> "\0foo\0", "*foo" -> "foo\0", "foo*" -> "\0foo"
-  // and "*foo*" -> "foo". Aho-Corasick can do only substring matching,
-  // so we use \0 as beginning/end-of-string markers.
-  if (str.starts_with('*') && str.ends_with('*'))
-    return str.substr(1, str.size() - 2);
-  if (str.starts_with('*'))
-    return str.substr(1) + "\0"s;
-  if (str.ends_with('*'))
-    return "\0"s + str.substr(0, str.size() - 1);
-  return "\0"s + str + "\0"s;
-}
-
-bool MultiGlob::add(std::string_view pat, i64 val) {
-  assert(!is_compiled);
-  assert(!pat.empty());
-
-  strings.emplace_back(pat);
-
-  // Complex glob pattern
-  if (!is_simple_pattern(pat)) {
-    if (std::optional<Glob> glob = Glob::compile(pat)) {
-      globs.emplace_back(std::move(*glob), val);
-      return true;
-    }
-    return false;
-  }
-
-  // Simple glob pattern
-  if (!root)
-    root.reset(new TrieNode);
-  TrieNode *node = root.get();
-
-  for (u8 c : handle_stars(pat)) {
-    if (!node->children[c])
-      node->children[c].reset(new TrieNode);
-    node = node->children[c].get();
-  }
-
-  node->value = std::max(node->value, val);
-  return true;
-}
-
-void MultiGlob::compile() {
-  is_compiled = true;
-  if (root) {
-    fix_suffix_links(*root);
-    fix_values();
-
-    // If no pattern starts with '*', set prefix_match to true.
-    // We'll use this flag for optimization.
-    prefix_match = true;
-    for (i64 i = 1; i < 256; i++) {
-      if (root->children[i]) {
-        prefix_match = false;
-        break;
-      }
-    }
-  }
-}
-
-void MultiGlob::fix_suffix_links(TrieNode &node) {
-  for (i64 i = 0; i < 256; i++) {
-    if (!node.children[i])
-      continue;
-
-    TrieNode &child = *node.children[i];
-
-    TrieNode *cur = node.suffix_link;
-    for (;;) {
-      if (!cur) {
-        child.suffix_link = root.get();
-        break;
-      }
-
-      if (cur->children[i]) {
-        child.suffix_link = cur->children[i].get();
-        break;
-      }
-
-      cur = cur->suffix_link;
-    }
-
-    fix_suffix_links(child);
-  }
-}
-
-void MultiGlob::fix_values() {
-  std::queue<TrieNode *> queue;
-  queue.push(root.get());
-
-  do {
-    TrieNode *node = queue.front();
-    queue.pop();
-
-    for (std::unique_ptr<TrieNode> &child : node->children) {
-      if (!child)
-        continue;
-      child->value = std::max(child->value, child->suffix_link->value);
-      queue.push(child.get());
-    }
-  } while (!queue.empty());
-}
-
-} // namespace mold

lib/perf.cc

@@ -1,11 +1,13 @@
-#include "common.h"
+#include "lib.h"
 
 #include <functional>
 #include <iomanip>
 #include <ios>
 #include <tbb/concurrent_vector.h>
 
-#ifndef _WIN32
+#ifdef _WIN32
+#include <windows.h>
+#else
 #include <sys/resource.h>
 #include <sys/time.h>
 #endif

lib/random.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <random>
 

lib/tar.cc

@@ -1,6 +1,6 @@
 // This file contains functions to create a tar file.
 
-#include "common.h"
+#include "lib.h"
 
 #ifdef _WIN32
 # define ftruncate _chsize_s
@@ -86,6 +86,9 @@ void TarWriter::append(std::string path, std::string_view data) {
   UstarHeader pax = {};
 
   std::string attr = encode_path(basedir, path);
+
+  if (attr.size() >= 0100'000'000'000)
+    unreachable();
   snprintf(pax.size, sizeof(pax.size), "%011zo", attr.size());
   pax.name[0] = '/';
   pax.typeflag[0] = 'x';
@@ -99,6 +102,8 @@ void TarWriter::append(std::string path, std::string_view data) {
   // Write Ustar header
   UstarHeader ustar = {};
   memcpy(ustar.mode, "0000664", 8);
+  if (data.size() >= 0100'000'000'000)
+    unreachable();
   snprintf(ustar.size, sizeof(ustar.size), "%011zo", data.size());
   finalize(ustar);
   fwrite(&ustar, sizeof(ustar), 1, out);

lib/update-git-hash.cmake

@@ -1,27 +1,26 @@
-# Get a git hash value. We do not want to use git command here
-# because we don't want to make git a build-time dependency.
-if(EXISTS "${SOURCE_DIR}/.git/HEAD")
-  file(READ "${SOURCE_DIR}/.git/HEAD" HASH)
-  string(STRIP "${HASH}" HASH)
+# Embed the current git commit hash into the mold executable. We ask git
+# instead of parsing files under .git because the repository format varies
+# (e.g. packed refs, reftable). When building from a source tarball, there's
+# no .git, and the hash is simply omitted.
+if(EXISTS "${SOURCE_DIR}/.git")
+  execute_process(
+    COMMAND git -C "${SOURCE_DIR}" rev-parse HEAD
+    OUTPUT_VARIABLE HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+    RESULT_VARIABLE EXIT_CODE)
 
-  if(HASH MATCHES "^ref: (.*)")
-    set(HEAD "${CMAKE_MATCH_1}")
-    if(EXISTS "${SOURCE_DIR}/.git/${HEAD}")
-      file(READ "${SOURCE_DIR}/.git/${HEAD}" HASH)
-      string(STRIP "${HASH}" HASH)
-    else()
-      file(READ "${SOURCE_DIR}/.git/packed-refs" PACKED_REFS)
-      string(REGEX REPLACE ".*\n([0-9a-f]+) ${HEAD}\n.*" "\\1" HASH "\n${PACKED_REFS}")
-    endif()
+  if(NOT EXIT_CODE EQUAL 0)
+    set(HASH "")
   endif()
 endif()
 
 # Create new file contents and update a given file if necessary.
-set(NEW_CONTENTS "#include <string>
-namespace mold {
-std::string mold_git_hash = \"${HASH}\";
-}
-")
+if("${HASH}" STREQUAL "")
+  set(NEW_CONTENTS "")
+else()
+  set(NEW_CONTENTS "#define MOLD_GIT_HASH \"${HASH}\"\n")
+endif()
 
 if(EXISTS "${OUTPUT_FILE}")
   file(READ "${OUTPUT_FILE}" OLD_CONTENTS)

src/arch-arm32.cc

@@ -31,18 +31,21 @@
 //
 // https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst
 
+#if MOLD_ARM32LE || MOLD_ARM32BE
+
 #include "mold.h"
 
 #include <tbb/parallel_for.h>
+#include <tbb/parallel_for_each.h>
 
 namespace mold {
 
-using E = ARM32;
+using E = MOLD_TARGET;
 
 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
-  ul32 *arm = (ul32 *)loc;
-  ul16 *thm = (ul16 *)loc;
+  U32<E> *arm = (U32<E> *)loc;
+  U16<E> *thm = (U16<E> *)loc;
 
   switch (rel.r_type) {
   case R_ARM_ABS32:
@@ -59,7 +62,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   case R_ARM_TLS_GOTDESC:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
-    return (il32)*arm;
+    return (I32<E>)*arm;
   case R_ARM_THM_JUMP8:
     return sign_extend(thm[0], 8) << 1;
   case R_ARM_THM_JUMP11:
@@ -125,7 +128,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
 static void write_arm_mov(u8 *loc, u32 val) {
   u32 imm12 = bits(val, 11, 0);
   u32 imm4 = bits(val, 15, 12);
-  *(ul32 *)loc = (*(ul32 *)loc & 0xfff0'f000) | (imm4 << 16) | imm12;
+  *(U32<E> *)loc = (*(U32<E> *)loc & 0xfff0'f000) | (imm4 << 16) | imm12;
 }
 
 static void write_thm_b21(u8 *loc, u32 val) {
@@ -135,7 +138,7 @@ static void write_thm_b21(u8 *loc, u32 val) {
   u32 imm6 = bits(val, 17, 12);
   u32 imm11 = bits(val, 11, 1);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1011'1100'0000) | (S << 10) | imm6;
   buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
 }
@@ -149,7 +152,7 @@ static void write_thm_b25(u8 *loc, u32 val) {
   u32 imm10 = bits(val, 21, 12);
   u32 imm11 = bits(val, 11, 1);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1000'0000'0000) | (S << 10) | imm10;
   buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
 }
@@ -160,7 +163,7 @@ static void write_thm_mov(u8 *loc, u32 val) {
   u32 imm3 = bits(val, 10, 8);
   u32 imm8 = bits(val, 7, 0);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1011'1111'0000) | (i << 10) | imm4;
   buf[1] = (buf[1] & 0b1000'1111'0000'0000) | (imm3 << 12) | imm8;
 }
@@ -184,13 +187,13 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
   case R_ARM_TLS_GOTDESC:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
     break;
   case R_ARM_THM_JUMP8:
-    *(ul16 *)loc = (*(ul16 *)loc & 0xff00) | bits(val, 8, 1);
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xff00) | bits(val, 8, 1);
     break;
   case R_ARM_THM_JUMP11:
-    *(ul16 *)loc = (*(ul16 *)loc & 0xf800) | bits(val, 11, 1);
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xf800) | bits(val, 11, 1);
     break;
   case R_ARM_THM_CALL:
   case R_ARM_THM_JUMP24:
@@ -200,7 +203,7 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PLT32:
-    *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
     break;
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVW_ABS_NC:
@@ -209,7 +212,7 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
     write_arm_mov(loc, val);
     break;
   case R_ARM_PREL31:
-    *(ul32 *)loc = (*(ul32 *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
     break;
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVW_ABS_NC:
@@ -236,7 +239,7 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
+  *(U32<E> *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
 }
 
 static constexpr ul32 plt_entry[] = {
@@ -249,42 +252,34 @@ static constexpr ul32 plt_entry[] = {
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
 
   switch (rel.r_type) {
   case R_NONE:
     break;
   case R_ARM_ABS32:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
     break;
   case R_ARM_REL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
   }
 }
 
-static Thunk<E> &get_reachable_thunk(OutputSection<E> &osec, u64 addr) {
-  auto it = std::upper_bound(osec.thunks.begin(), osec.thunks.end(), addr,
-                             [](u64 addr, std::unique_ptr<Thunk<E>> &thunk) {
-    return addr < thunk->get_addr();
-  });
-  return **it;
-}
-
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
@@ -294,13 +289,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     if (rel.r_type == R_NONE || rel.r_type == R_ARM_V4BX)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
     u64 A = get_addend(*this, rel);
     u64 P = get_addr() + rel.r_offset;
-    u64 T = S & 1;
+    u64 T = is_thumb_func(ctx, sym);
     u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
     u64 GOT = ctx.got->shdr.sh_addr;
 
@@ -312,7 +310,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     auto get_arm_thunk_addr   = [&] { return sym.get_thunk_addr(ctx, P) + 4; };
 
     auto get_tlsdesc_trampoline_addr = [&] {
-      return get_reachable_thunk(*output_section, P).get_addr();
+      auto it = ranges::upper_bound(output_section->thunks, P, {},
+                                    [](std::unique_ptr<Thunk<E>> &thunk) {
+        return thunk->get_addr();
+      });
+      return (*it)->get_addr();
     };
 
     switch (rel.r_type) {
@@ -320,78 +322,79 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_ARM_TARGET1:
       break;
     case R_ARM_REL32:
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
       break;
     case R_ARM_THM_CALL: {
       if (sym.is_remaining_undef_weak()) {
         // On ARM, calling an weak undefined symbol jumps to the
         // next instruction.
-        *(ul32 *)loc = 0x8000'f3af; // NOP.W
+        *(U32<E> *)loc = 0x8000'f3af; // NOP.W
         break;
       }
 
       // THM_CALL relocation refers to either BL or BLX instruction.
-      // They are different in only one bit. We need to use BL if
-      // the jump target is Thumb. Otherwise, use BLX.
+      // They are different in only one bit. We need to use BLX if the
+      // jump target is an ARM function. Otherwise, use BL.
       i64 val1 = S + A - P;
       i64 val2 = align_to(S + A - P, 4);
+      bool arm = is_arm_func(ctx, sym);
 
-      if (T && is_int(val1, 25)) {
-        *(ul16 *)(loc + 2) |= 0x1000;  // BL
+      if (!arm && is_int(val1, 25)) {
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
         write_thm_b25(loc, val1);
-      } else if (!T && is_int(val2, 25)) {
-        *(ul16 *)(loc + 2) &= ~0x1000; // BLX
+      } else if (arm && is_int(val2, 25)) {
+        *(U16<E> *)(loc + 2) &= ~0x1000; // BLX
         write_thm_b25(loc, val2);
       } else {
-        *(ul16 *)(loc + 2) |= 0x1000;  // BL
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
         write_thm_b25(loc, get_thumb_thunk_addr() + A - P);
       }
       break;
     }
     case R_ARM_BASE_PREL:
-      *(ul32 *)loc = GOT + A - P;
+      *(U32<E> *)loc = GOT + A - P;
       break;
     case R_ARM_GOTOFF32:
-      *(ul32 *)loc = ((S + A) | T) - GOT;
+      *(U32<E> *)loc = ((S + A) | T) - GOT;
       break;
     case R_ARM_GOT_PREL:
     case R_ARM_TARGET2:
-      *(ul32 *)loc = GOT + G + A - P;
+      *(U32<E> *)loc = GOT + G + A - P;
       break;
     case R_ARM_GOT_BREL:
-      *(ul32 *)loc = G + A;
+      *(U32<E> *)loc = G + A;
       break;
     case R_ARM_CALL: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
         break;
       }
 
       // Just like THM_CALL, ARM_CALL relocation refers to either BL or
       // BLX instruction. We may need to rewrite BL → BLX or BLX → BL.
-      bool is_bl = ((*(ul32 *)loc & 0xff00'0000) == 0xeb00'0000);
-      bool is_blx = ((*(ul32 *)loc & 0xfe00'0000) == 0xfa00'0000);
+      bool is_bl = ((*(U32<E> *)loc & 0xff00'0000) == 0xeb00'0000);
+      bool is_blx = ((*(U32<E> *)loc & 0xfe00'0000) == 0xfa00'0000);
       if (!is_bl && !is_blx)
         Fatal(ctx) << *this << ": R_ARM_CALL refers to neither BL nor BLX";
 
       i64 val = S + A - P;
       if (is_int(val, 26)) {
         if (T) {
-          *(ul32 *)loc = 0xfa00'0000; // BLX
-          *(ul32 *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
+          *(U32<E> *)loc = 0xfa00'0000; // BLX
+          *(U32<E> *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
         } else {
-          *(ul32 *)loc = 0xeb00'0000; // BL
-          *(ul32 *)loc |= bits(val, 25, 2);
+          *(U32<E> *)loc = 0xeb00'0000; // BL
+          *(U32<E> *)loc |= bits(val, 25, 2);
         }
       } else {
-        *(ul32 *)loc = 0xeb00'0000; // BL
-        *(ul32 *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // BL
+        *(U32<E> *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
       }
       break;
     }
     case R_ARM_JUMP24: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
         break;
       }
 
@@ -404,26 +407,26 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       i64 val = S + A - P;
       if (T || !is_int(val, 26))
         val = get_arm_thunk_addr() + A - P;
-      *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+      *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
       break;
     }
     case R_ARM_PLT32:
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
       } else {
         u64 val = (T ? get_arm_thunk_addr() : S) + A - P;
-        *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+        *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
       }
       break;
     case R_ARM_THM_JUMP8:
       check(S + A - P, -(1 << 8), 1 << 8);
-      *(ul16 *)loc &= 0xff00;
-      *(ul16 *)loc |= bits(S + A - P, 8, 1);
+      *(U16<E> *)loc &= 0xff00;
+      *(U16<E> *)loc |= bits(S + A - P, 8, 1);
       break;
     case R_ARM_THM_JUMP11:
       check(S + A - P, -(1 << 11), 1 << 11);
-      *(ul16 *)loc &= 0xf800;
-      *(ul16 *)loc |= bits(S + A - P, 11, 1);
+      *(U16<E> *)loc &= 0xf800;
+      *(U16<E> *)loc |= bits(S + A - P, 11, 1);
       break;
     case R_ARM_THM_JUMP19:
       check(S + A - P, -(1 << 20), 1 << 20);
@@ -431,14 +434,14 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_ARM_THM_JUMP24: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0x8000'f3af; // NOP
+        *(U32<E> *)loc = 0x8000'f3af; // NOP
         break;
       }
 
       // Just like R_ARM_JUMP24, we need to jump to a thunk if we need to
       // switch processor mode.
       i64 val = S + A - P;
-      if (!T || !is_int(val, 25))
+      if (is_arm_func(ctx, sym) || !is_int(val, 25))
         val = get_thumb_thunk_addr() + A - P;
       write_thm_b25(loc, val);
       break;
@@ -454,8 +457,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_ARM_PREL31:
       check(S + A - P, -(1LL << 30), 1LL << 30);
-      *(ul32 *)loc &= 0x8000'0000;
-      *(ul32 *)loc |= (S + A - P) & 0x7fff'ffff;
+      *(U32<E> *)loc &= 0x8000'0000;
+      *(U32<E> *)loc |= (S + A - P) & 0x7fff'ffff;
       break;
     case R_ARM_THM_MOVW_ABS_NC:
       write_thm_mov(loc, (S + A) | T);
@@ -473,19 +476,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_thm_mov(loc, (S + A) >> 16);
       break;
     case R_ARM_TLS_GD32:
-      *(ul32 *)loc = sym.get_tlsgd_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_tlsgd_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LDM32:
-      *(ul32 *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
+      *(U32<E> *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LDO32:
-      *(ul32 *)loc = S + A - ctx.dtp_addr;
+      *(U32<E> *)loc = S + A - ctx.dtp_addr;
       break;
     case R_ARM_TLS_IE32:
-      *(ul32 *)loc = sym.get_gottp_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_gottp_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LE32:
-      *(ul32 *)loc = S + A - ctx.tp_addr;
+      *(U32<E> *)loc = S + A - ctx.tp_addr;
       break;
     case R_ARM_TLS_GOTDESC:
       // ARM32 TLSDESC uses the following code sequence to materialize
@@ -514,35 +517,35 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       //  .L2: .word   foo(gottpoff) + . - .L1
       if (sym.has_tlsdesc(ctx)) {
         // A is odd if the corresponding TLS_CALL is Thumb.
-        *(ul32 *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
+        *(U32<E> *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
       } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
+        *(U32<E> *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
       } else {
-        *(ul32 *)loc = S - ctx.tp_addr;
+        *(U32<E> *)loc = S - ctx.tp_addr;
       }
       break;
     case R_ARM_TLS_CALL:
       if (sym.has_tlsdesc(ctx)) {
-        *(ul32 *)loc = 0xeb00'0000; // bl 0
-        *(ul32 *)loc |= bits(get_tlsdesc_trampoline_addr() - P - 8, 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // bl 0
+        *(U32<E> *)loc |= bits(get_tlsdesc_trampoline_addr() - P - 8, 25, 2);
       } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
+        *(U32<E> *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
       } else {
-        *(ul32 *)loc = 0xe320'f000; // nop
+        *(U32<E> *)loc = 0xe320'f000; // nop
       }
       break;
     case R_ARM_THM_TLS_CALL:
       if (sym.has_tlsdesc(ctx)) {
         u64 val = align_to(get_tlsdesc_trampoline_addr() - P - 4, 4);
         write_thm_b25(loc, val);
-        *(ul16 *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
+        *(U16<E> *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
       } else if (sym.has_gottp(ctx)) {
         // Since `ldr r0, [pc, r0]` is not representable in Thumb,
         // we use two instructions instead.
-        *(ul16 *)loc = 0x4478;         // add r0, pc
-        *(ul16 *)(loc + 2) = 0x6800;   // ldr r0, [r0]
+        *(U16<E> *)loc = 0x4478;         // add r0, pc
+        *(U16<E> *)(loc + 2) = 0x6800;   // ldr r0, [r0]
       } else {
-        *(ul32 *)loc = 0x8000'f3af;    // nop.w
+        *(U32<E> *)loc = 0x8000'f3af;    // nop.w
       }
       break;
     default:
@@ -553,14 +556,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -573,22 +573,22 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     switch (rel.r_type) {
     case R_ARM_ABS32:
       if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
       else
-        *(ul32 *)loc = S + A;
+        *(U32<E> *)loc = S + A;
       break;
     case R_ARM_TLS_LDO32:
       if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
       else
-        *(ul32 *)loc = S + A - ctx.dtp_addr;
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
       break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -602,7 +602,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -695,27 +695,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_hdr_size == sizeof(hdr));
   static_assert(E::thunk_size == sizeof(entry));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  memcpy(buf, hdr, sizeof(hdr));
-  buf += sizeof(hdr);
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  memcpy(base, hdr, sizeof(hdr));
 
-  u64 P = output_section.shdr.sh_addr + offset + sizeof(hdr);
-
-  for (Symbol<E> *sym : symbols) {
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
     memcpy(buf, entry, sizeof(entry));
-    *(ul32 *)(buf + 12) = sym->get_addr(ctx) - P - 16;
-
-    buf += sizeof(entry);
-    P += sizeof(entry);
+    *(U32<E> *)(buf + 12) = S - P - 16;
   }
 }
 
 template <>
 u64 get_eflags(Context<E> &ctx) {
-  return EF_ARM_EABI_VER5;
+  if constexpr (E::is_le)
+    return EF_ARM_EABI_VER5;
+  else
+    return EF_ARM_EABI_VER5 | EF_ARM_BE8;
 }
 
-void create_arm_exidx_section(Context<E> &ctx) {
+template <>
+void create_arm_exidx_section<E>(Context<E> &ctx) {
   for (i64 i = 0; i < ctx.chunks.size(); i++) {
     OutputSection<E> *osec = ctx.chunks[i]->to_osec();
 
@@ -726,34 +727,26 @@ void create_arm_exidx_section(Context<E> &ctx) {
       ctx.chunk_pool.emplace_back(sec);
 
       for (InputSection<E> *isec : osec->members)
-        isec->is_alive = false;
+        isec->kill();
       break;
     }
   }
 }
 
-void Arm32ExidxSection::compute_section_size(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::compute_section_size(Context<E> &ctx) {
   output_section.compute_section_size(ctx);
   this->shdr.sh_size = output_section.shdr.sh_size + 8; // +8 for sentinel
 }
 
-void Arm32ExidxSection::update_shdr(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::update_shdr(Context<E> &ctx) {
   // .ARM.exidx's sh_link should be set to the .text section index.
   // Runtime doesn't care about it, but the binutils's strip command does.
   if (Chunk<E> *chunk = find_chunk(ctx, ".text"))
     this->shdr.sh_link = chunk->shndx;
 }
 
-void Arm32ExidxSection::remove_duplicate_entries(Context<E> &ctx) {
-  this->shdr.sh_size = get_contents(ctx).size();
-}
-
-void Arm32ExidxSection::copy_buf(Context<E> &ctx) {
-  std::vector<u8> contents = get_contents(ctx);
-  assert(this->shdr.sh_size == contents.size());
-  write_vector(ctx.buf + this->shdr.sh_offset, contents);
-}
-
 // Returns the end of the text segment
 static u64 get_text_end(Context<E> &ctx) {
   u64 ret = 0;
@@ -772,7 +765,8 @@ static u64 get_text_end(Context<E> &ctx) {
 // likely that it's due to some historical reason.
 //
 // This function returns contents of .ARM.exidx.
-std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
+template <>
+std::vector<u8> Arm32ExidxSection<E>::get_contents(Context<E> &ctx) {
   // .ARM.exidx records consists of a signed 31-bit relative address
   // and a 32-bit value. The relative address indicates the start
   // address of a function that the record covers. The value is one of
@@ -788,8 +782,8 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
   const u32 CANTUNWIND = 1;
 
   struct Entry {
-    ul32 addr;
-    ul32 val;
+    U32<E> addr;
+    U32<E> val;
   };
 
   // We reserve one extra slot for the sentinel
@@ -820,19 +814,13 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
       ent[i].val = 0x7fff'ffff & (ent[i].val + offset);
   });
 
-  std::sort(ent, ent + num_entries, [](const Entry &a, const Entry &b) {
-    return a.addr < b.addr;
-  });
+  ranges::sort(ent, ent + num_entries, {}, &Entry::addr);
 
   // Remove duplicate adjacent entries. That is, if two adjacent functions
   // have the same compact unwind info or are both CANTUNWIND, we can
   // merge them into a single address range.
-  auto it = std::unique(ent, ent + num_entries,
-                        [](const Entry &a, const Entry &b) {
-    return a.val == b.val;
-  });
-
-  num_entries = it - ent;
+  auto tail = ranges::unique(ent, ent + num_entries, {}, &Entry::val);
+  num_entries -= tail.size();
   buf.resize(num_entries * sizeof(Entry));
 
   // Make addresses relative to themselves.
@@ -846,4 +834,84 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
   return buf;
 }
 
+template <>
+void Arm32ExidxSection<E>::remove_duplicate_entries(Context<E> &ctx) {
+  this->shdr.sh_size = get_contents(ctx).size();
+}
+
+template <>
+void Arm32ExidxSection<E>::copy_buf(Context<E> &ctx) {
+  std::vector<u8> contents = get_contents(ctx);
+  assert(this->shdr.sh_size == contents.size());
+  write_vector(ctx.buf + this->shdr.sh_offset, contents);
+}
+
+// Even though using ARM32 in big-endian mode is very rare, the processor
+// technically supports both little- and big-endian modes. There are two
+// variants of big-endian mode: BE32 and BE8. In BE32, instructions and
+// data are encoded in big-endian. In BE8, instructions are encoded in
+// little-endian, and only data is in big-endian. BE8 is the de facto
+// standard for ARMv6 or later. We support only BE8.
+//
+// A tricky thing is that instructions in an object file are always
+// big-endian if the file is compiled for big-endian mode. In other words,
+// the compiler always emit code in BE32 if -mbig-endian is specified. It
+// is the linker's responsibility to rewrite instructions from big-endian
+// to little-endian for an BE8 output. This function does that.
+//
+// The text section may contain a mix of 32-bit ARM instructions, 16-bit
+// Thumb instructions, and data. We need to distinguish them to swap 4
+// bytes, 2 bytes, or not swap bytes, respectively. The beginning of ARM
+// code, Thumb code, and data is labeled with a mapping symbol of $a, $t,
+// and $d, respectively. We use mapping symbols to determine what to do
+// with the text section.
+//
+// This function is called after we copy the input section contents to the
+// output file. We rewrite instructions in the output buffer in place.
+#if MOLD_ARM32BE
+void arm32be_swap_bytes(Context<E> &ctx) {
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    // Collect mapping symbols
+    std::vector<Symbol<E> *> syms;
+
+    for (Symbol<E> *sym : file->get_local_syms())
+      if (InputSection<E> *isec = sym->get_input_section())
+        if (isec->is_alive() && (isec->shdr().sh_flags & SHF_EXECINSTR))
+          if (std::string_view x = sym->name();
+              x == "$a" || x.starts_with("$a.") ||
+              x == "$t" || x.starts_with("$t.") ||
+              x == "$d" || x.starts_with("$d."))
+            syms.push_back(sym);
+
+    // Group mapping symbols by input section and sort by address
+    ranges::stable_sort(syms, {}, [](const Symbol<E> *sym) {
+      return std::tuple{(uintptr_t)sym->get_input_section(), sym->value};
+    });
+
+    // Swap bytes
+    for (i64 i = 0; i < syms.size(); i++) {
+      Symbol<E> &sym = *syms[i];
+      if (sym.name().starts_with("$d"))
+        continue;
+
+      InputSection<E> &isec = *sym.get_input_section();
+      u8 *base = ctx.buf + isec.output_section->shdr.sh_offset + isec.offset;
+      u8 *start = base + sym.value;
+      u8 *end;
+
+      if (i + 1 < syms.size() && syms[i + 1]->get_input_section() == &isec)
+        end = base + syms[i + 1]->value;
+      else
+        end = base + isec.sh_size;
+
+      i64 sz = sym.name().starts_with("$a") ? 4 : 2;
+      for (u8 *p = start; p < end; p += sz)
+        std::reverse(p, p + sz);
+    }
+  });
+}
+#endif
+
 } // namespace mold
+
+#endif

src/arch-arm64.cc

@@ -104,9 +104,14 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -115,10 +120,11 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(U64<E> *)loc = val;
     break;
   case R_AARCH64_PREL32:
-    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(U32<E> *)loc = val - P;
     break;
   case R_AARCH64_PREL64:
-    *(U64<E> *)loc = val - this->shdr.sh_addr - offset;
+    *(U64<E> *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -145,14 +151,17 @@ static bool is_add(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -165,6 +174,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_AARCH64_ABS64:
       break;
@@ -208,6 +222,30 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_AARCH64_MOVW_UABS_G3:
       *(ul32 *)loc |= bits(S + A, 63, 48) << 5;
       break;
+    case R_AARCH64_MOVW_PREL_G0:
+      check(S + A - P, -(1 << 15), 1 << 15);
+      write_movn_movz(loc, S + A - P);
+      break;
+    case R_AARCH64_MOVW_PREL_G0_NC:
+      *(ul32 *)loc |= bits(S + A - P, 15, 0) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G1:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
+      write_movn_movz(loc, (i64)(S + A - P) >> 16);
+      break;
+    case R_AARCH64_MOVW_PREL_G1_NC:
+      *(ul32 *)loc |= bits(S + A - P, 31, 16) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G2:
+      check(S + A - P, -(1LL << 47), 1LL << 47);
+      write_movn_movz(loc, (i64)(S + A - P) >> 32);
+      break;
+    case R_AARCH64_MOVW_PREL_G2_NC:
+      *(ul32 *)loc |= bits(S + A - P, 47, 32) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G3:
+      *(ul32 *)loc |= bits(S + A - P, 63, 48) << 5;
+      break;
     case R_AARCH64_ADR_GOT_PAGE:
       if (sym.has_got(ctx)) {
         i64 val = page(G + GOT + A) - page(P);
@@ -222,6 +260,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         u32 reg = bits(*(ul32 *)loc, 4, 0);
         *(ul32 *)(loc + 4) = 0x9100'0000 | (reg << 5) | reg; // ADD
         *(ul32 *)(loc + 4) |= bits(S + A, 11, 0) << 10;
+
+        rewrite(i, R_AARCH64_ADR_PREL_PG_HI21);
+        rewrite(i + 1, R_AARCH64_ADD_ABS_LO12_NC);
         i++;
       }
       break;
@@ -248,6 +289,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             *(ul32 *)loc = 0xd503'201f;              // nop
             *(ul32 *)(loc + 4) = 0x1000'0000 | reg1; // adr
             write_adr(loc + 4, val);
+            rewrite(i, R_NONE);
+            rewrite(i + 1, R_AARCH64_ADR_PREL_LO21);
             i++;
             break;
           }
@@ -283,6 +326,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check(S + A - P, -(1LL << 31), 1LL << 31);
       *(U32<E> *)loc = S + A - P;
       break;
+    case R_AARCH64_GOTPCREL32:
+      check(G + GOT + A - P, -(1LL << 31), 1LL << 31);
+      *(U32<E> *)loc = G + GOT + A - P;
+      break;
     case R_AARCH64_CONDBR19:
     case R_AARCH64_LD_PREL_LO19:
       check(S + A - P, -(1LL << 20), 1LL << 20);
@@ -397,13 +444,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         write_adrp(loc, val);
       } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
       }
       break;
     case R_AARCH64_TLSDESC_LD64_LO12:
-      if (sym.has_tlsdesc(ctx))
+      if (sym.has_tlsdesc(ctx)) {
         *(ul32 *)loc |= bits(sym.get_tlsdesc_addr(ctx) + A, 11, 3) << 10;
-      else
+      } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
+      }
       break;
     case R_AARCH64_TLSDESC_ADD_LO12:
       if (sym.has_tlsdesc(ctx)) {
@@ -411,9 +461,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x9000'0000; // adrp x0, 0
         write_adrp(loc, page(sym.get_gottp_addr(ctx) + A) - page(P));
+        rewrite(i, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21);
       } else {
         *(ul32 *)loc = 0xd2a0'0000; // movz x0, 0, lsl #16
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 32, 16) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G1);
       }
       break;
     case R_AARCH64_TLSDESC_CALL:
@@ -422,9 +474,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0xf940'0000; // ldr x0, [x0, 0]
         *(ul32 *)loc |= bits(sym.get_gottp_addr(ctx) + A, 11, 3) << 10;
+        rewrite(i, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC);
       } else {
         *(ul32 *)loc = 0xf280'0000; // movk x0, 0
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 15, 0) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G0_NC);
       }
       break;
     default:
@@ -435,14 +489,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -453,7 +504,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     u64 A = frag ? frag_addend : (i64)rel.r_addend;
 
     auto check = [&](i64 val, i64 lo, i64 hi) {
-      check_range(ctx, val, i, lo, hi);
+      check_range(ctx, i, val, lo, hi);
     };
 
     switch (rel.r_type) {
@@ -467,12 +518,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       check(S + A, 0, 1LL << 32);
       *(U32<E> *)loc = S + A;
       break;
+    case R_AARCH64_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -486,8 +543,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -525,6 +582,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       break;
     case R_AARCH64_LD64_GOT_LO12_NC:
     case R_AARCH64_LD64_GOTPAGE_LO15:
+    case R_AARCH64_GOTPCREL32:
       sym.flags |= NEEDS_GOT;
       break;
     case R_AARCH64_CALL26:
@@ -568,6 +626,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_AARCH64_MOVW_UABS_G1_NC:
     case R_AARCH64_MOVW_UABS_G2:
     case R_AARCH64_MOVW_UABS_G2_NC:
+    case R_AARCH64_MOVW_PREL_G0:
+    case R_AARCH64_MOVW_PREL_G0_NC:
+    case R_AARCH64_MOVW_PREL_G1:
+    case R_AARCH64_MOVW_PREL_G1_NC:
+    case R_AARCH64_MOVW_PREL_G2:
+    case R_AARCH64_MOVW_PREL_G2_NC:
+    case R_AARCH64_MOVW_PREL_G3:
     case R_AARCH64_PREL16:
     case R_AARCH64_PREL32:
     case R_AARCH64_PREL64:
@@ -587,31 +652,78 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
   }
 }
 
+// The size of a thunk entry varies on ARM64 depending on the distance to
+// the branch target. This function computes the size of each thunk entry.
 template <>
-void Thunk<E>::copy_buf(Context<E> &ctx) {
-  constexpr ul32 insn[] = {
-    0x9000'0010, // adrp x16, 0   # R_AARCH64_ADR_PREL_PG_HI21
-    0x9100'0210, // add  x16, x16 # R_AARCH64_ADD_ABS_LO12_NC
-    0xd61f'0200, // br   x16
-    0xd420'7d00, // brk
+void Thunk<E>::shrink_size(Context<E> &ctx) {
+  offsets.clear();
+  offsets.push_back(0);
+  i64 off = 0;
+
+  // The distance between S and P is only reduced by shrink_size(), but
+  // page(S) – page(P) may still increase by one page due to address
+  // changes, so we add a safety margin.
+  //
+  // For example, page(0x1200) – page(0x1000) is 0, whereas
+  // page(0x1100) – page(0xfff) is 0x1000, even though the latter
+  // distance is shorter than the former.
+  auto is_small = [](i64 prel) {
+    return is_int(prel + 0x1000, 33) && is_int(prel - 0x1000, 33);
   };
 
-  static_assert(E::thunk_size == sizeof(insn));
-
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
-
   for (Symbol<E> *sym : symbols) {
     u64 S = sym->get_addr(ctx);
-    memcpy(buf, insn, E::thunk_size);
-    write_adrp(buf, page(S) - page(P));
-    *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
+    u64 P = get_addr() + off;
+    i64 prel = page(S) - page(P);
+    off += is_small(prel) ? 12 : 24;
+    offsets.push_back(off);
   }
 }
 
+template <>
+void Thunk<E>::copy_buf(Context<E> &ctx) {
+  // Short thunk with a 33 bit displacement
+  constexpr ul32 insn1[] = {
+    0x9000'0010, // adrp x16, 0
+    0x9100'0210, // add  x16, x16
+    0xd61f'0200, // br   x16
+  };
+
+  // Long thunk with a 64 bit displacement
+  constexpr ul32 insn2[] = {
+    0x1000'0010, // adr  x16, 0
+    0xd2a0'0011, // movz x17, 0, lsl #16
+    0xf2c0'0011, // movk x17, 0, lsl #32
+    0xf2e0'0011, // movk x17, 0, lsl #48
+    0x8b11'0210, // add  x16, x16, x17
+    0xd61f'0200, // br   x16
+  };
+
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
+
+    if (offsets[i + 1] - offsets[i] == sizeof(insn1)) {
+      i64 prel = page(S) - page(P);
+      assert(is_int(prel, 33));
+      memcpy(buf, insn1, sizeof(insn1));
+      write_adrp(buf, prel);
+      *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
+    } else {
+      memcpy(buf, insn2, sizeof(insn2));
+      write_adr(buf, bits(S - P, 15, 0));
+      *(ul32 *)(buf + 4) |= bits(S - P, 31, 16) << 5;
+      *(ul32 *)(buf + 8) |= bits(S - P, 47, 32) << 5;
+      *(ul32 *)(buf + 12) |= bits(S - P, 63, 48) << 5;
+    }
+  }
+}
+
+template class Thunk<E>;
+
 } // namespace mold
 
 #endif

src/arch-i386.cc

@@ -33,6 +33,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/i386.pdf
 
+#if MOLD_I386
+
 #include "mold.h"
 
 namespace mold {
@@ -44,10 +46,10 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   switch (rel.r_type) {
   case R_386_8:
   case R_386_PC8:
-    return *loc;
+    return *(i8 *)loc;
   case R_386_16:
   case R_386_PC16:
-    return *(ul16 *)loc;
+    return *(il16 *)loc;
   case R_386_32:
   case R_386_PC32:
   case R_386_GOT32:
@@ -63,7 +65,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   case R_386_TLS_LDO_32:
   case R_386_SIZE32:
   case R_386_TLS_GOTDESC:
-    return *(ul32 *)loc;
+    return *(il32 *)loc;
   default:
     return 0;
   }
@@ -172,8 +174,8 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
 
   switch (rel.r_type) {
@@ -278,14 +280,17 @@ static u32 relax_tlsdesc_to_le(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -333,6 +338,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         loc[-2] = insn >> 8;
         loc[-1] = insn;
         *(ul32 *)loc = S + A - GOT;
+        if (ctx.arg.emit_relocs)
+          rel.r_type = R_386_GOTOFF;
       }
       break;
     case R_386_GOTOFF:
@@ -434,14 +441,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -453,7 +457,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     u64 GOT = ctx.got->shdr.sh_addr;
 
     auto check = [&](i64 val, i64 lo, i64 hi) {
-      check_range(ctx, val, i, lo, hi);
+      check_range(ctx, i, val, lo, hi);
     };
 
     switch (rel.r_type) {
@@ -500,7 +504,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       unreachable();
     }
-  }
+  });
 }
 
 template <>
@@ -514,8 +518,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -596,3 +600,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-loongarch.cc

@@ -66,7 +66,7 @@ static u64 higher(u64 val, u64 pc) {
   // LU52I.D simply set bits to [51:31] and to [63:53], respectively.
   //
   // Compensating all the sign-extensions is a bit complicated. The
-  // psABI gave the following formula.
+  // psABI gives the following formula.
   val = val + 0x8000'0000 + ((val & 0x800) ? (0x1000 - 0x1'0000'0000) : 0);
   return page(val) - page(pc - 8);
 }
@@ -132,8 +132,8 @@ static void set_rj(u8 *loc, u32 rj) {
 //   ld.d      $t0, $t0, 0    # R_LARCH_GOT_PC_LO12, R_LARCH_RELAX
 static bool is_relaxable_got_load(Context<E> &ctx, InputSection<E> &isec, i64 i) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  Symbol<E> &sym = *isec.file.symbols[rels[i].r_sym];
-  u8 *buf = (u8 *)isec.contents.data();
+  Symbol<E> &sym = *isec.file->symbols[rels[i].r_sym];
+  u8 *buf = isec.contents;
 
   if (ctx.arg.relax &&
       sym.is_pcrel_linktime_const(ctx) &&
@@ -219,9 +219,14 @@ void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -257,7 +262,8 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(ul64 *)loc -= val;
     break;
   case R_LARCH_32_PCREL:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ul32 *)loc = val - P;
     break;
   case R_LARCH_64_PCREL:
     *(ul64 *)loc = val - this->shdr.sh_addr - offset;
@@ -269,13 +275,13 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
 
     if (rel.r_type == R_NONE || rel.r_type == R_LARCH_RELAX ||
         rel.r_type == R_LARCH_MARK_LA || rel.r_type == R_LARCH_MARK_PCREL ||
@@ -294,7 +300,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         r_delta = deltas[k - 1].delta;
     }
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -332,6 +341,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
                    << " for relocation " << rel;
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_LARCH_32:
       assert(E::is_64);
@@ -346,7 +360,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_d5k16(loc, (S + A - P) >> 2);
       break;
     case R_LARCH_B26:
-      check_branch(S + A - P, -(1 << 27), 1 << 27);
+      if (!sym.is_remaining_undef_weak())
+        check_branch(S + A - P, -(1 << 27), 1 << 27);
       write_d10k16(loc, (S + A - P) >> 2);
       break;
     case R_LARCH_ABS_LO12:
@@ -375,10 +390,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (removed_bytes == 0) {
         write_j20(loc, hi20(S + A, P));
       } else {
-        // Rewrite pcalau12i + addi.d with pcaddi
+        // Rewrite pcalau12i + addi.d with pcaddi. The high part vanishes and
+        // the low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -417,10 +435,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         }
         write_j20(loc, hi20(GOT + G + A, P));
       } else {
-        // Rewrite pcalau12i + ld.d with pcaddi
+        // Rewrite pcalau12i + ld.d with pcaddi. The high part vanishes and the
+        // low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -518,6 +539,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul64 *)loc -= S + A;
       break;
     case R_LARCH_32_PCREL:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
       *(ul32 *)loc = S + A - P;
       break;
     case R_LARCH_64_PCREL:
@@ -526,7 +548,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_LARCH_CALL36:
       if (removed_bytes == 0) {
         i64 val = S + A - P;
-        check_branch(val, -(1LL << 37) - 0x20000, (1LL << 37) - 0x20000);
+        if (!sym.is_remaining_undef_weak())
+          check_branch(val, -(1LL << 37) - 0x20000, (1LL << 37) - 0x20000);
         write_j20(loc, (val + 0x20000) >> 18);
         write_k16(loc + 4, val >> 2);
       } else {
@@ -537,6 +560,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x5400'0000; // BL
         write_d10k16(loc, (S + A - P) >> 2);
+        rewrite(i, R_LARCH_B26);
       }
       break;
     case R_LARCH_ADD_ULEB128:
@@ -590,30 +614,49 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       //   jirl      $ra, $ra, 0
       if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
         write_j20(loc, hi20(sym.get_tlsdesc_addr(ctx) + A, P));
+
+      // pcalau12i + addi.d => pcaddi when TLSDESC is kept; both deleted when it
+      // is relaxed to IE/LE. Either way the high part loses its relocation.
+      // The folded pcaddi's relocation is emitted from the LO12 slot below.
+      if (!sym.has_tlsdesc(ctx) || removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_DESC_PC_LO12:
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0) {
-        i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
-        if (is_int(dist, 22)) {
-          *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
-          write_j20(loc, dist >> 2);
-        } else {
-          write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0) {
+          i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
+          if (is_int(dist, 22)) {
+            *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
+            write_j20(loc, dist >> 2);
+            rewrite(i, R_LARCH_TLS_DESC_PCREL20_S2);
+          } else {
+            write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+          }
         }
+      } else {
+        rewrite(i, R_NONE);
       }
       break;
     case R_LARCH_TLS_DESC_LD:
-      if (sym.has_tlsdesc(ctx) || removed_bytes == 4) {
-        // Do nothing
+      // TLSDESC is relaxed to IE or LE. The ld.d slot holds the first
+      // instruction of the resulting sequence.
+      if (sym.has_tlsdesc(ctx)) {
+        // Do nothing (TLSDESC kept)
+      } else if (removed_bytes == 4) {
+        // Small TP offset: the instruction was deleted.
+        rewrite(i, R_NONE);
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x1a00'0004; // pcalau12i $a0, 0
         write_j20(loc, hi20(sym.get_gottp_addr(ctx) + A, P));
+        rewrite(i, R_LARCH_TLS_IE_PC_HI20);
       } else {
         *(ul32 *)loc = 0x1400'0004; // lu12i.w   $a0, 0
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+        rewrite(i, R_LARCH_TLS_LE_HI20);
       }
       break;
     case R_LARCH_TLS_DESC_CALL:
+      // The jirl slot holds the second instruction of the IE or LE sequence.
       if (sym.has_tlsdesc(ctx)) {
         // Do nothing
       } else if (sym.has_gottp(ctx)) {
@@ -622,6 +665,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x2880'0084; // ld.w $a0, $a0, 0
         write_k12(loc, sym.get_gottp_addr(ctx) + A);
+        rewrite(i, R_LARCH_TLS_IE_PC_LO12);
       } else {
         i64 val = S + A - ctx.tp_addr;
         if (0 <= val && val < 0x1000)
@@ -629,11 +673,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x0280'0084; // addi.w $a0, $a0, 0
         write_k12(loc, val);
+        rewrite(i, R_LARCH_TLS_LE_LO12);
       }
       break;
     case R_LARCH_TLS_LE_HI20_R:
+      // lu12i.w + add.d + addi.d => addi.d when the variable is within 2 KiB
+      // of TP, in which case the lu12i.w loses its relocation.
       if (removed_bytes == 0)
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_LE_LO12_R: {
       i64 val = S + A - ctx.tp_addr;
@@ -645,8 +694,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         set_rj(loc, 2);
       break;
     }
-    case R_LARCH_64:
     case R_LARCH_TLS_LE_ADD_R:
+      // add.d that materializes TP + offset; removed together with the
+      // lu12i.w when the variable is within 2 KiB of TP.
+      if (removed_bytes)
+        rewrite(i, R_NONE);
+      break;
+    case R_LARCH_64:
       break;
     default:
       unreachable();
@@ -656,19 +710,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE)
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
-    if (!sym.file) {
+    if (!sym.file && &sym != discarded_comdat_sym<E>) {
       record_undef_error(ctx, rel);
-      continue;
+      return;
     }
 
     SectionFragment<E> *frag;
@@ -741,7 +792,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -761,7 +812,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -854,13 +905,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 template <>
 void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
+  std::vector<RelocDelta> deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &r = rels[i];
-    Symbol<E> &sym = *isec.file.symbols[r.r_sym];
+    Symbol<E> &sym = *isec.file->symbols[r.r_sym];
 
     auto remove = [&](i64 d) {
       r_delta += d;
@@ -980,7 +1031,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
       //   pcaddi    $t0, <offset>
       if (is_relaxable_got_load(ctx, isec, i))
         if (i64 dist = compute_distance(ctx, sym, isec, r);
-            is_int(dist, 22))
+            is_int(dist, 22) && (dist & 3) == 0)
           remove(4);
       break;
     case R_LARCH_TLS_DESC_PC_HI20:
@@ -1006,6 +1057,12 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
     }
   }
 
+  if (deltas.empty())
+    return;
+
+  RelocDelta *p = ctx.arena.template allocate<RelocDelta>(deltas.size());
+  ranges::copy(deltas, p);
+  isec.extra.r_deltas = {p, deltas.size()};
   isec.sh_size -= r_delta;
 }
 

src/arch-m68k.cc

@@ -14,6 +14,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/m68k.pdf
 
+#if MOLD_M68K
+
 #include "mold.h"
 
 namespace mold {
@@ -56,8 +58,8 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
 
   switch (rel.r_type) {
@@ -83,7 +85,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -208,14 +213,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -232,11 +234,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(ub32 *)loc = S + A;
       break;
+    case R_68K_TLS_LDO32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -249,7 +257,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       Error(ctx) << sym << ": GNU ifunc symbol is not supported on m68k";
@@ -310,3 +318,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc32.cc

@@ -40,6 +40,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc32.pdf
 
+#if MOLD_PPC32
+
 #include "mold.h"
 
 namespace mold {
@@ -126,8 +128,8 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
 
   switch (rel.r_type) {
@@ -148,14 +150,17 @@ template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
 
-  u64 GOT2 = file.extra.got2 ? file.extra.got2->get_addr() : 0;
+  u64 GOT2 = file->extra.got2 ? file->extra.got2->get_addr() : 0;
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -282,14 +287,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -306,11 +308,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(ub32 *)loc = S + A;
       break;
+    case R_PPC_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -324,7 +332,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -416,27 +424,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(plt_entry));
   static_assert(E::thunk_size == sizeof(local_thunk));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
       i64 val = got - P - 8;
       memcpy(buf, plt_entry, sizeof(plt_entry));
       *(ub32 *)(buf + 16) |= higha(val);
       *(ub32 *)(buf + 20) |= lo(val);
     } else {
-      i64 val = sym->get_addr(ctx) - P - 8;
+      i64 val = sym.get_addr(ctx) - P - 8;
       memcpy(buf, local_thunk, sizeof(local_thunk));
       *(ub32 *)(buf + 16) |= higha(val);
       *(ub32 *)(buf + 20) |= lo(val);
     }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
   }
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc64v1.cc

@@ -45,6 +45,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v1.pdf
 
+#if MOLD_PPC64V1
+
 #include "mold.h"
 
 #include <algorithm>
@@ -129,9 +131,14 @@ template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {}
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -140,10 +147,11 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(ub64 *)loc = val;
     break;
   case R_PPC64_REL32:
-    *(ub32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ub32 *)loc = val - P;
     break;
   case R_PPC64_REL64:
-    *(ub64 *)loc = val - this->shdr.sh_addr - offset;
+    *(ub64 *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -159,7 +167,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -217,6 +228,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_REL16_LO:
       *(ub16 *)loc = lo(S + A - P);
       break;
+    case R_PPC64_GOT16:
+      *(ub16 *)loc = G - TOC;
+      break;
     case R_PPC64_PLT16_HA:
       *(ub16 *)loc = ha(G + GOT - TOC);
       break;
@@ -280,14 +294,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -298,7 +309,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     u64 A = frag ? frag_addend : (i64)rel.r_addend;
 
     auto check = [&](i64 val, i64 lo, i64 hi) {
-      check_range(ctx, val, i, lo, hi);
+      check_range(ctx, i, val, lo, hi);
     };
 
     switch (rel.r_type) {
@@ -319,7 +330,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -333,7 +344,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT | NEEDS_PPC_OPD;
@@ -351,6 +362,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       if (sym.is_imported)
         sym.flags |= NEEDS_PLT;
       break;
+    case R_PPC64_GOT16:
     case R_PPC64_PLT16_HA:
       sym.flags |= NEEDS_GOT;
       break;
@@ -452,49 +464,46 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(plt_thunk));
   static_assert(E::thunk_size == sizeof(local_thunk));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  u64 TOC = ctx.extra.TOC->value;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_got(ctx)) {
-      i64 val = sym->get_got_addr(ctx) - ctx.extra.TOC->value;
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_got(ctx)) {
       memcpy(buf, pltgot_thunk, sizeof(pltgot_thunk));
+      i64 val = sym.get_got_addr(ctx) - TOC;
       *(ub32 *)(buf + 4) |= higha(val);
       *(ub32 *)(buf + 8) |= lo(val);
-    } else if(sym->has_plt(ctx)) {
-      i64 val = sym->get_gotplt_addr(ctx) - ctx.extra.TOC->value;
+    } else if(sym.has_plt(ctx)) {
       memcpy(buf, plt_thunk, sizeof(plt_thunk));
+      i64 val = sym.get_gotplt_addr(ctx) - TOC;
       *(ub32 *)(buf + 4) |= higha(val);
       *(ub32 *)(buf + 8) |= lo(val);
     } else {
-      i64 val = sym->get_addr(ctx, NO_OPD) - ctx.extra.TOC->value;
       memcpy(buf, local_thunk, sizeof(local_thunk));
+      i64 val = sym.get_addr(ctx, NO_OPD) - TOC;
       *(ub32 *)buf |= higha(val);
       *(ub32 *)(buf + 4) |= lo(val);
     }
-
-    buf += E::thunk_size;
   }
 }
 
 static InputSection<E> *get_opd_section(ObjectFile<E> &file) {
-  for (std::unique_ptr<InputSection<E>> &isec : file.sections)
+  for (InputSection<E> *isec : file.sections)
     if (isec && isec->name() == ".opd")
-      return isec.get();
+      return isec;
   return nullptr;
 }
 
 static ElfRel<E> *
-get_relocation_at(Context<E> &ctx, InputSection<E> &isec, i64 offset) {
+get_relocation_at(Context<E> &ctx, InputSection<E> &isec, u64 offset) {
   std::span<ElfRel<E>> rels = isec.get_rels(ctx);
-
-  auto it = std::lower_bound(rels.begin(), rels.end(), offset,
-                             [](const ElfRel<E> &r, i64 offset) {
-    return r.r_offset < offset;
+  auto it = ranges::lower_bound(rels, offset, {}, [](const ElfRel<E> &r) {
+    return (u64)r.r_offset;
   });
-
-  if (it == rels.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  if (it == rels.end() || it->r_offset != offset)
     return nullptr;
   return &*it;
 }
@@ -509,9 +518,7 @@ struct OpdSymbol {
 static Symbol<E> *
 get_opd_sym_at(std::span<OpdSymbol> syms, u64 offset) {
   auto it = ranges::lower_bound(syms, offset, {}, &OpdSymbol::r_offset);
-  if (it == syms.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  if (it == syms.end() || it->r_offset != offset)
     return nullptr;
   return it->sym;
 }
@@ -562,7 +569,7 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
     InputSection<E> *opd = get_opd_section(*file);
     if (!opd)
       return;
-    opd->is_alive = false;
+    opd->kill();
 
     // Move symbols from .opd to .text.
     std::vector<OpdSymbol> opd_syms;
@@ -593,8 +600,8 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
     ranges::stable_sort(opd_syms, {}, &OpdSymbol::r_offset);
 
     // Rewrite relocations so that they directly refer to .opd.
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (!isec || !isec->is_alive || isec.get() == opd)
+    for (InputSection<E> *isec : file->sections) {
+      if (!isec || !isec->is_alive() || isec == opd)
         continue;
 
       for (ElfRel<E> &r : isec->get_rels(ctx)) {
@@ -618,11 +625,10 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
 // refers to the function's .opd entry. This function marks such symbols
 // with NEEDS_PPC_OPD.
 void ppc64v1_scan_symbols(Context<E> &ctx) {
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (Symbol<E> *sym : file->symbols)
-      if (sym->file == file && sym->is_exported)
-        if (u32 ty = sym->get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
-          sym->flags |= NEEDS_PPC_OPD;
+  ctx.symbol_map.parallel_for_each([](Symbol<E> &sym) {
+    if (sym.file && !sym.file->is_dso && sym.is_exported)
+      if (u32 ty = sym.get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
+        sym.flags |= NEEDS_PPC_OPD;
   });
 
   // Functions referenced by the ELF header also have to have .opd entries.
@@ -637,36 +643,55 @@ void ppc64v1_scan_symbols(Context<E> &ctx) {
 }
 
 void PPC64OpdSection::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_opd_idx(ctx, symbols.size());
+  sym->aux->opd_idx = symbols.size();
   symbols.push_back(sym);
   this->shdr.sh_size += ENTRY_SIZE;
 }
 
-i64 PPC64OpdSection::get_reldyn_size(Context<E> &ctx) const {
-  if (ctx.arg.pic)
-    return symbols.size() * 2;
-  return 0;
+i64 PPC64OpdSection::get_num_dynrels(Context<E> &ctx) const {
+  return ctx.arg.pic ? symbols.size() * 2 : 0;
+}
+
+std::vector<u64> PPC64OpdSection::get_relr_offsets(Context<E> &ctx) {
+  if (!ctx.arg.pic)
+    return {};
+
+  std::vector<u64> offsets;
+  offsets.reserve(symbols.size() * 2);
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx) - this->shdr.sh_addr;
+    offsets.push_back(loc);
+    offsets.push_back(loc + 8);
+  }
+  return offsets;
+}
+
+void PPC64OpdSection::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  if (!ctx.arg.pic)
+    return;
+
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx);
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        loc % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc, E::R_RELATIVE, 0, sym->get_addr(ctx, NO_PLT | NO_OPD));
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        (loc + 8) % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
+  }
 }
 
 void PPC64OpdSection::copy_buf(Context<E> &ctx) {
   ub64 *buf = (ub64 *)(ctx.buf + this->shdr.sh_offset);
 
-  ElfRel<E> *rel = nullptr;
-  if (ctx.arg.pic)
-    rel = (ElfRel<E> *)(ctx.buf + ctx.reldyn->shdr.sh_offset + reldyn_offset);
-
   for (Symbol<E> *sym : symbols) {
     u64 addr = sym->get_addr(ctx, NO_PLT | NO_OPD);
     *buf++ = addr;
     *buf++ = ctx.extra.TOC->value;
     *buf++ = 0;
-
-    if (ctx.arg.pic) {
-      u64 loc = sym->get_opd_addr(ctx);
-      *rel++ = ElfRel<E>(loc, E::R_RELATIVE, 0, addr);
-      *rel++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
-    }
   }
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc64v2.cc

@@ -80,6 +80,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v2.pdf
 
+#if MOLD_PPC64V2
+
 #include "mold.h"
 
 namespace mold {
@@ -150,9 +152,14 @@ template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {}
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -161,10 +168,11 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(ul64 *)loc = val;
     break;
   case R_PPC64_REL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ul32 *)loc = val - P;
     break;
   case R_PPC64_REL64:
-    *(ul64 *)loc = val - this->shdr.sh_addr - offset;
+    *(ul64 *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -191,7 +199,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -204,6 +215,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     auto r2save_thunk_addr = [&] { return sym.get_thunk_addr(ctx, P); };
     auto no_r2save_thunk_addr = [&] { return sym.get_thunk_addr(ctx, P) + 8; };
 
+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
     switch (rel.r_type) {
     case R_PPC64_TOC16_HA:
       *(ul16 *)loc = ha(S + A - TOC);
@@ -212,6 +227,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc = lo(S + A - TOC);
       break;
     case R_PPC64_TOC16_DS:
+      check(S + A - TOC, -(1 << 15), 1 << 15);
+      *(ul16 *)loc |= (S + A - TOC) & 0xfffc;
+      break;
     case R_PPC64_TOC16_LO_DS:
       *(ul16 *)loc |= (S + A - TOC) & 0xfffc;
       break;
@@ -243,6 +261,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         *(ul32 *)loc |= bits(val, 25, 2) << 2;
       }
       break;
+    case R_PPC64_REL14:
+      *(ul32 *)loc |= bits(S + A - P, 15, 2) << 2;
+      break;
     case R_PPC64_REL32:
       *(ul32 *)loc = S + A - P;
       break;
@@ -255,6 +276,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_REL16_LO:
       *(ul16 *)loc = lo(S + A - P);
       break;
+    case R_PPC64_GOT16:
+      *(ul16 *)loc = G - TOC;
+      break;
+    case R_PPC64_GOT16_LO:
+      *(ul16 *)loc = lo(G - TOC);
+      break;
+    case R_PPC64_GOT16_HI:
+      *(ul16 *)loc = hi(G - TOC);
+      break;
+    case R_PPC64_GOT16_HA:
+      *(ul16 *)loc = ha(G - TOC);
+      break;
     case R_PPC64_PLT16_HA:
       *(ul16 *)loc = ha(G + GOT - TOC);
       break;
@@ -344,14 +377,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -383,7 +413,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -397,7 +427,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -416,6 +446,10 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_PLT;
       ctx.extra.is_power10 = true;
       break;
+    case R_PPC64_GOT16:
+    case R_PPC64_GOT16_LO:
+    case R_PPC64_GOT16_HI:
+    case R_PPC64_GOT16_HA:
     case R_PPC64_PLT16_HA:
     case R_PPC64_PLT_PCREL34:
     case R_PPC64_PLT_PCREL34_NOTOC:
@@ -431,12 +465,11 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       ctx.needs_tlsld = true;
       break;
     case R_PPC64_TPREL16_HA:
-    case R_PPC64_TPREL16_LO:
-    case R_PPC64_TPREL16_LO_DS:
     case R_PPC64_TPREL34:
       check_tlsle(ctx, sym, rel);
       break;
     case R_PPC64_ADDR64:
+    case R_PPC64_REL14:
     case R_PPC64_REL32:
     case R_PPC64_REL64:
     case R_PPC64_TOC16_HA:
@@ -456,6 +489,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_PPC64_GOT_TPREL16_LO_DS:
     case R_PPC64_GOT_TLSGD16_LO:
     case R_PPC64_GOT_TLSLD16_LO:
+    case R_PPC64_TPREL16_LO:
+    case R_PPC64_TPREL16_LO_DS:
     case R_PPC64_TLS:
     case R_PPC64_TLSGD:
     case R_PPC64_TLSLD:
@@ -518,15 +553,16 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(local_thunk));
   static_assert(E::thunk_size == sizeof(local_thunk_power10));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
   u64 TOC = ctx.extra.TOC->value;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
 
+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
       if (ctx.extra.is_power10) {
         memcpy(buf, plt_thunk_power10, E::thunk_size);
         write34(buf + 8, got - P - 8);
@@ -536,7 +572,7 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
         *(ul32 *)(buf + 12) |= lo(got - TOC);
       }
     } else {
-      u64 S = sym->get_addr(ctx);
+      u64 S = sym.get_addr(ctx);
       if (ctx.extra.is_power10) {
         memcpy(buf, local_thunk_power10, E::thunk_size);
         write34(buf + 8, S - P - 8);
@@ -546,15 +582,13 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
         *(ul32 *)(buf + 12) |= lo(S - TOC);
       }
     }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
   }
 }
 
 // GCC may emit references to the following functions in function prologue
-// and epiilogue if -Os is specified. For some reason, these functions are
-// not in libgcc.a and expected to be synthesized by the linker.
+// and epilogue if -Os is specified. For some reason, these functions are
+// not in libgcc.a and expected to be synthesized by the linker. There are
+// variants for general-purpose, floating-point and vector registers.
 const std::vector<std::pair<std::string_view, u32>>
 ppc64_save_restore_insns = {
   { "_savegpr0_14", 0xf9c1ff70 }, // std r14,-144(r1)
@@ -644,6 +678,106 @@ ppc64_save_restore_insns = {
   { "_restgpr1_30", 0xebccfff0 }, // ld r30,-16(r12)
   { "_restgpr1_31", 0xebecfff8 }, // ld r31,-8(r12)
   { "",             0x4e800020 }, // blr
+
+  { "_savefpr_14",  0xd9c1ff70 }, // stfd f14,-144(r1)
+  { "_savefpr_15",  0xd9e1ff78 }, // stfd f15,-136(r1)
+  { "_savefpr_16",  0xda01ff80 }, // stfd f16,-128(r1)
+  { "_savefpr_17",  0xda21ff88 }, // stfd f17,-120(r1)
+  { "_savefpr_18",  0xda41ff90 }, // stfd f18,-112(r1)
+  { "_savefpr_19",  0xda61ff98 }, // stfd f19,-104(r1)
+  { "_savefpr_20",  0xda81ffa0 }, // stfd f20,-96(r1)
+  { "_savefpr_21",  0xdaa1ffa8 }, // stfd f21,-88(r1)
+  { "_savefpr_22",  0xdac1ffb0 }, // stfd f22,-80(r1)
+  { "_savefpr_23",  0xdae1ffb8 }, // stfd f23,-72(r1)
+  { "_savefpr_24",  0xdb01ffc0 }, // stfd f24,-64(r1)
+  { "_savefpr_25",  0xdb21ffc8 }, // stfd f25,-56(r1)
+  { "_savefpr_26",  0xdb41ffd0 }, // stfd f26,-48(r1)
+  { "_savefpr_27",  0xdb61ffd8 }, // stfd f27,-40(r1)
+  { "_savefpr_28",  0xdb81ffe0 }, // stfd f28,-32(r1)
+  { "_savefpr_29",  0xdba1ffe8 }, // stfd f29,-24(r1)
+  { "_savefpr_30",  0xdbc1fff0 }, // stfd f30,-16(r1)
+  { "_savefpr_31",  0xdbe1fff8 }, // stfd f31,-8(r1)
+  { "",             0xf8010010 }, // std r0,16(r1)
+  { "",             0x4e800020 }, // blr
+
+  { "_restfpr_14",  0xc9c1ff70 }, // lfd f14,-144(r1)
+  { "_restfpr_15",  0xc9e1ff78 }, // lfd f15,-136(r1)
+  { "_restfpr_16",  0xca01ff80 }, // lfd f16,-128(r1)
+  { "_restfpr_17",  0xca21ff88 }, // lfd f17,-120(r1)
+  { "_restfpr_18",  0xca41ff90 }, // lfd f18,-112(r1)
+  { "_restfpr_19",  0xca61ff98 }, // lfd f19,-104(r1)
+  { "_restfpr_20",  0xca81ffa0 }, // lfd f20,-96(r1)
+  { "_restfpr_21",  0xcaa1ffa8 }, // lfd f21,-88(r1)
+  { "_restfpr_22",  0xcac1ffb0 }, // lfd f22,-80(r1)
+  { "_restfpr_23",  0xcae1ffb8 }, // lfd f23,-72(r1)
+  { "_restfpr_24",  0xcb01ffc0 }, // lfd f24,-64(r1)
+  { "_restfpr_25",  0xcb21ffc8 }, // lfd f25,-56(r1)
+  { "_restfpr_26",  0xcb41ffd0 }, // lfd f26,-48(r1)
+  { "_restfpr_27",  0xcb61ffd8 }, // lfd f27,-40(r1)
+  { "_restfpr_28",  0xcb81ffe0 }, // lfd f28,-32(r1)
+  { "_restfpr_29",  0xe8010010 }, // ld r0,16(r1)
+  { "",             0xcba1ffe8 }, // lfd f29,-24(r1)
+  { "",             0x7c0803a6 }, // mtlr r0
+  { "",             0xcbc1fff0 }, // lfd f30,-16(r1)
+  { "",             0xcbe1fff8 }, // lfd f31,-8(r1)
+  { "",             0x4e800020 }, // blr
+  { "_restfpr_30",  0xcbc1fff0 }, // lfd f30,-16(r1)
+  { "_restfpr_31",  0xe8010010 }, // ld r0,16(r1)
+  { "",             0xcbe1fff8 }, // lfd f31,-8(r1)
+  { "",             0x7c0803a6 }, // mtlr r0
+  { "",             0x4e800020 }, // blr
+
+  { "_savevr_20",   0x3980ff40 }, // li r12,-192
+  { "",             0x7e8c01ce }, // stvx v20,r12,r0
+  { "_savevr_21",   0x3980ff50 }, // li r12,-176
+  { "",             0x7eac01ce }, // stvx v21,r12,r0
+  { "_savevr_22",   0x3980ff60 }, // li r12,-160
+  { "",             0x7ecc01ce }, // stvx v22,r12,r0
+  { "_savevr_23",   0x3980ff70 }, // li r12,-144
+  { "",             0x7eec01ce }, // stvx v23,r12,r0
+  { "_savevr_24",   0x3980ff80 }, // li r12,-128
+  { "",             0x7f0c01ce }, // stvx v24,r12,r0
+  { "_savevr_25",   0x3980ff90 }, // li r12,-112
+  { "",             0x7f2c01ce }, // stvx v25,r12,r0
+  { "_savevr_26",   0x3980ffa0 }, // li r12,-96
+  { "",             0x7f4c01ce }, // stvx v26,r12,r0
+  { "_savevr_27",   0x3980ffb0 }, // li r12,-80
+  { "",             0x7f6c01ce }, // stvx v27,r12,r0
+  { "_savevr_28",   0x3980ffc0 }, // li r12,-64
+  { "",             0x7f8c01ce }, // stvx v28,r12,r0
+  { "_savevr_29",   0x3980ffd0 }, // li r12,-48
+  { "",             0x7fac01ce }, // stvx v29,r12,r0
+  { "_savevr_30",   0x3980ffe0 }, // li r12,-32
+  { "",             0x7fcc01ce }, // stvx v30,r12,r0
+  { "_savevr_31",   0x3980fff0 }, // li r12,-16
+  { "",             0x7fec01ce }, // stvx v31,r12,r0
+  { "",             0x4e800020 }, // blr
+
+  { "_restvr_20",   0x3980ff40 }, // li r12,-192
+  { "",             0x7e8c00ce }, // lvx v20,r12,r0
+  { "_restvr_21",   0x3980ff50 }, // li r12,-176
+  { "",             0x7eac00ce }, // lvx v21,r12,r0
+  { "_restvr_22",   0x3980ff60 }, // li r12,-160
+  { "",             0x7ecc00ce }, // lvx v22,r12,r0
+  { "_restvr_23",   0x3980ff70 }, // li r12,-144
+  { "",             0x7eec00ce }, // lvx v23,r12,r0
+  { "_restvr_24",   0x3980ff80 }, // li r12,-128
+  { "",             0x7f0c00ce }, // lvx v24,r12,r0
+  { "_restvr_25",   0x3980ff90 }, // li r12,-112
+  { "",             0x7f2c00ce }, // lvx v25,r12,r0
+  { "_restvr_26",   0x3980ffa0 }, // li r12,-96
+  { "",             0x7f4c00ce }, // lvx v26,r12,r0
+  { "_restvr_27",   0x3980ffb0 }, // li r12,-80
+  { "",             0x7f6c00ce }, // lvx v27,r12,r0
+  { "_restvr_28",   0x3980ffc0 }, // li r12,-64
+  { "",             0x7f8c00ce }, // lvx v28,r12,r0
+  { "_restvr_29",   0x3980ffd0 }, // li r12,-48
+  { "",             0x7fac00ce }, // lvx v29,r12,r0
+  { "_restvr_30",   0x3980ffe0 }, // li r12,-32
+  { "",             0x7fcc00ce }, // lvx v30,r12,r0
+  { "_restvr_31",   0x3980fff0 }, // li r12,-16
+  { "",             0x7fec00ce }, // lvx v31,r12,r0
+  { "",             0x4e800020 }, // blr
 };
 
 void PPC64SaveRestoreSection::copy_buf(Context<E> &ctx) {
@@ -658,3 +792,5 @@ u64 get_eflags(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-riscv.cc

@@ -158,9 +158,14 @@ void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -193,7 +198,8 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(U32<E> *)loc = val;
     break;
   case R_RISCV_32_PCREL:
-    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(U32<E> *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -252,26 +258,26 @@ find_paired_reloc(Context<E> &ctx, InputSection<E> &isec,
 //   ld    t0, 0(t0)  # R_RISCV_PCREL_LO12_I(.L0), R_RISCV_RELAX
 static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
                              std::span<const ElfRel<E>> rels, i64 i) {
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
   return i + 3 < rels.size() &&
          rels[i].r_type == R_RISCV_GOT_HI20 &&
          rels[i + 1].r_type == R_RISCV_RELAX &&
          rels[i + 2].r_type == R_RISCV_PCREL_LO12_I &&
          rels[i + 3].r_type == R_RISCV_RELAX &&
          rels[i].r_offset == rels[i + 2].r_offset - 4 &&
-         rels[i].r_offset == isec.file.symbols[rels[i + 2].r_sym]->value &&
+         rels[i].r_offset == isec.file->symbols[rels[i + 2].r_sym]->value &&
          get_rd(buf + rels[i].r_offset) == get_rd(buf + rels[i + 2].r_offset);
 }
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE || rel.r_type == R_RISCV_RELAX)
       continue;
 
@@ -287,7 +293,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         r_delta = deltas[k - 1].delta;
     }
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -301,10 +310,20 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto utype = [&](i64 val) {
+      check(val, -(1LL << 31) - 0x800, (1LL << 31) - 0x800);
+      write_utype(loc, val);
+    };
+
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_RISCV_32:
-      assert(E::is_64);
-      *(U32<E> *)loc = S + A;
+      if (E::is_64)
+        *(U32<E> *)loc = S + A;
       break;
     case R_RISCV_64:
       break;
@@ -321,28 +340,26 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       i64 val = S + A - P;
       i64 rd = get_rd(buf + rel.r_offset + 4);
 
-      // Calling an undefined weak symbol does not make sense.
-      // We make such call into an infinite loop. This should
-      // help debugging of a faulty program.
-      if (sym.esym().is_undef_weak())
-        val = 0;
-
       if (removed_bytes == 4) {
         // auipc + jalr -> jal
         *(ul32 *)loc = (rd << 7) | 0b1101111;
         write_jtype(loc, val);
+        rewrite(i, R_RISCV_JAL);
       } else if (removed_bytes == 6 && rd == 0) {
         // auipc + jalr -> c.j
         *(ul16 *)loc = 0b101'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else if (removed_bytes == 6 && rd == 1) {
         // auipc + jalr -> c.jal
         assert(!E::is_64);
         *(ul16 *)loc = 0b001'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else {
         assert(removed_bytes == 0);
-        check(val, -(1LL << 31), 1LL << 31);
+        if (!sym.is_remaining_undef_weak())
+          check(val, -(1LL << 31) - 0x800, (1LL << 31) - 0x800);
         write_utype(loc, val);
         write_itype(loc + 4, val);
       }
@@ -359,11 +376,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         // c.li <rd>, val
         *(ul16 *)loc = 0b010'0'00000'00000'01 | (rd << 7);
         write_citype(loc, sym.get_addr(ctx));
+        // The value is materialized directly, so neither this nor the paired
+        // PCREL_LO12 (the load) needs a relocation anymore.
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else if (removed_bytes == 4) {
         // addi <rd>, zero, val
         *(ul32 *)loc = 0b0010011 | (rd << 7);
         write_itype(loc, sym.get_addr(ctx));
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else {
         assert(removed_bytes == 0);
@@ -372,31 +395,31 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         if (ctx.arg.relax && sym.is_pcrel_linktime_const(ctx) &&
             is_got_load_pair(ctx, *this, rels, i) && is_int(val, 32)) {
           // auipc <rd>, %hi20(val)
-          write_utype(loc, val);
+          utype(val);
 
           // addi <rd>, <rd>, %lo12(val)
           *(ul32 *)(loc + 4) = 0b0010011 | (rd << 15) | (rd << 7);
           write_itype(loc + 4, val);
           i += 3;
         } else {
-          write_utype(loc, G + GOT + A - P);
+          utype(G + GOT + A - P);
         }
       }
       break;
     }
     case R_RISCV_TLS_GOT_HI20:
-      write_utype(loc, sym.get_gottp_addr(ctx) + A - P);
+      utype(sym.get_gottp_addr(ctx) + A - P);
       break;
     case R_RISCV_TLS_GD_HI20:
-      write_utype(loc, sym.get_tlsgd_addr(ctx) + A - P);
+      utype(sym.get_tlsgd_addr(ctx) + A - P);
       break;
     case R_RISCV_PCREL_HI20:
-      write_utype(loc, S + A - P);
+      utype(S + A - P);
       break;
     case R_RISCV_PCREL_LO12_I:
     case R_RISCV_PCREL_LO12_S: {
       const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
-      Symbol<E> &sym2 = *file.symbols[rel2.r_sym];
+      Symbol<E> &sym2 = *file->symbols[rel2.r_sym];
 
       auto write =
         (rel.r_type == R_RISCV_PCREL_LO12_I) ? write_itype : write_stype;
@@ -423,15 +446,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     }
     case R_RISCV_HI20:
+      // lui (+ addi) => an instruction holding a link-time constant. The lui
+      // may be compressed to c.lui (removed 2 bytes) or deleted outright
+      // (removed 4 bytes); either way it no longer needs a relocation.
       if (removed_bytes == 2) {
         // Rewrite LUI with C.LUI
         i64 rd = get_rd(buf + rel.r_offset);
         *(ul16 *)loc = 0b011'0'00000'00000'01 | (rd << 7);
         write_citype(loc, (S + A + 0x800) >> 12);
       } else if (removed_bytes == 0) {
-        check(S + A, -(1LL << 31), 1LL << 31);
-        write_utype(loc, S + A);
+        utype(S + A);
       }
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_LO12_I:
     case R_RISCV_LO12_S:
@@ -448,14 +475,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_RISCV_TPREL_HI20:
       assert(removed_bytes == 0 || removed_bytes == 4);
+      // lui + add => deleted; the variable is accessed relative to tp directly.
       if (removed_bytes == 0)
-        write_utype(loc, S + A - ctx.tp_addr);
+        utype(S + A - ctx.tp_addr);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_ADD:
       // This relocation just annotates an ADD instruction that can be
       // removed when a TPREL is relaxed. No value is needed to be
       // written.
       assert(removed_bytes == 0 || removed_bytes == 4);
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_LO12_I:
     case R_RISCV_TPREL_LO12_S: {
@@ -473,7 +505,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
     case R_RISCV_TLSDESC_HI20:
       // RISC-V TLSDESC uses the following code sequence to materialize
-      // a TP-relative address in x0.
+      // a TP-relative address in a0.
       //
       //   .L0:
       //   auipc  tX, 0
@@ -510,18 +542,25 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       // If the code-shrinking relaxation is disabled, we may leave
       // original useless instructions instead of deleting them, but we
       // accept that because relaxations are enabled by default.
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
-        write_utype(loc, sym.get_tlsdesc_addr(ctx) + A - P);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0)
+          utype(sym.get_tlsdesc_addr(ctx) + A - P);
+      } else {
+        rewrite(i, R_NONE);
+      }
       break;
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12:
     case R_RISCV_TLSDESC_CALL: {
+      const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
+      Symbol<E> &sym2 = *file->symbols[rel2.r_sym];
+
+      if (!sym2.has_tlsdesc(ctx))
+        rewrite(i, R_NONE);
+
       if (removed_bytes == 4)
         break;
 
-      const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
-      Symbol<E> &sym2 = *file.symbols[rel2.r_sym];
-
       u64 S = sym2.get_addr(ctx);
       u64 A = rel2.r_addend;
       u64 P = get_addr() + rel2.r_offset - get_r_delta(*this, rel2.r_offset);
@@ -538,10 +577,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           write_itype(loc, sym2.get_tlsdesc_addr(ctx) + A - P);
         } else if (sym2.has_gottp(ctx)) {
           *(ul32 *)loc = 0x517; // auipc a0,<hi20>
-          write_utype(loc, sym2.get_gottp_addr(ctx) + A - P);
+          utype(sym2.get_gottp_addr(ctx) + A - P);
         } else {
           *(ul32 *)loc = 0x537; // lui a0,<hi20>
-          write_utype(loc, S + A - ctx.tp_addr);
+          utype(S + A - ctx.tp_addr);
         }
         break;
       case R_RISCV_TLSDESC_CALL:
@@ -564,7 +603,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     }
     case R_RISCV_ADD8:
-      loc += S + A;
+      *loc += S + A;
       break;
     case R_RISCV_ADD16:
       *(U16<E> *)loc += S + A;
@@ -576,7 +615,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(U64<E> *)loc += S + A;
       break;
     case R_RISCV_SUB8:
-      loc -= S + A;
+      *loc -= S + A;
       break;
     case R_RISCV_SUB16:
       *(U16<E> *)loc -= S + A;
@@ -632,6 +671,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_RISCV_32_PCREL:
       *(U32<E> *)loc = S + A - P;
       break;
+    case R_RISCV_GOT32_PCREL:
+      *(U32<E> *)loc = G + GOT + A - P;
+      break;
     case R_RISCV_SET_ULEB128:
       overwrite_uleb(loc, S + A);
       break;
@@ -646,14 +688,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -673,6 +712,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U64<E> *)loc = S + A;
       break;
+    case R_RISCV_TLS_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
+    case R_RISCV_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     case R_RISCV_ADD8:
       *loc += S + A;
       break;
@@ -723,7 +774,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -737,7 +788,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -757,6 +808,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_PLT;
       break;
     case R_RISCV_GOT_HI20:
+    case R_RISCV_GOT32_PCREL:
       sym.flags |= NEEDS_GOT;
       break;
     case R_RISCV_TLS_GOT_HI20:
@@ -773,9 +825,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       scan_pcrel(ctx, sym, rel);
       break;
     case R_RISCV_TPREL_HI20:
-    case R_RISCV_TPREL_LO12_I:
-    case R_RISCV_TPREL_LO12_S:
-    case R_RISCV_TPREL_ADD:
       check_tlsle(ctx, sym, rel);
       break;
     case R_RISCV_64:
@@ -785,6 +834,9 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_RISCV_PCREL_LO12_S:
     case R_RISCV_LO12_I:
     case R_RISCV_LO12_S:
+    case R_RISCV_TPREL_LO12_I:
+    case R_RISCV_TPREL_LO12_S:
+    case R_RISCV_TPREL_ADD:
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12:
     case R_RISCV_TLSDESC_CALL:
@@ -843,17 +895,17 @@ u64 get_eflags(Context<E> &ctx) {
 template <>
 void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
+  std::vector<RelocDelta> deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   // True if we can use 2-byte instructions. This is usually true on
   // Unix because RV64GC is generally considered the baseline hardware.
-  bool use_rvc = isec.file.get_eflags() & EF_RISCV_RVC;
+  bool use_rvc = isec.file->get_eflags() & EF_RISCV_RVC;
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &r = rels[i];
-    Symbol<E> &sym = *isec.file.symbols[r.r_sym];
+    Symbol<E> &sym = *isec.file->symbols[r.r_sym];
 
     auto remove = [&](i64 d) {
       r_delta += d;
@@ -867,9 +919,14 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
     // follows the NOPs is aligned to a specified alignment boundary.
     if (r.r_type == R_RISCV_ALIGN) {
       // The total bytes of NOPs is stored to r_addend, so the next
-      // instruction is r_addend away.
+      // instruction is r_addend away. The alignment itself is not recorded
+      // anywhere; it is the smallest power of two greater than r_addend,
+      // because the assembler emits as many NOP bytes as the worst case
+      // requires, which is the alignment minus the minimum instruction
+      // size. For example, `.balign 4` yields r_addend 2 in RVC code and
+      // `.balign 8` yields 4 in non-RVC code.
       u64 P = isec.get_addr() + r.r_offset - r_delta;
-      u64 desired = align_to(P, bit_ceil(r.r_addend));
+      u64 desired = align_to(P, bit_ceil(r.r_addend + 1));
       u64 actual = P + r.r_addend;
       if (desired != actual)
         remove(actual - desired);
@@ -979,7 +1036,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12: {
       const ElfRel<E> &rel2 = find_paired_reloc(ctx, isec, rels, sym, i);
-      Symbol<E> &sym2 = *isec.file.symbols[rel2.r_sym];
+      Symbol<E> &sym2 = *isec.file->symbols[rel2.r_sym];
 
       if (r.r_type == R_RISCV_TLSDESC_LOAD_LO12) {
         if (!sym2.has_tlsdesc(ctx))
@@ -996,6 +1053,12 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
     }
   }
 
+  if (deltas.empty())
+    return;
+
+  RelocDelta *p = ctx.arena.template allocate<RelocDelta>(deltas.size());
+  ranges::copy(deltas, p);
+  isec.extra.r_deltas = {p, deltas.size()};
   isec.sh_size -= r_delta;
 }
 

src/arch-s390x.cc

@@ -35,6 +35,8 @@
 //
 // https://github.com/IBM/s390x-abi/releases/download/v1.6.1/lzsabi_s390x.pdf
 
+#if MOLD_S390X
+
 #include "mold.h"
 
 namespace mold {
@@ -48,18 +50,24 @@ static void write_mid20(u8 *loc, u64 val) {
 template <>
 void write_plt_header(Context<E> &ctx, u8 *buf) {
   static u8 insn[] = {
-    // Compute PLT_INDEX
+    // Compute the offset into .rela.plt. This is equivalent to
+    // (%r0 - %r1 - 48 - 14) * 3/2 where %r0 is the PLT entry address
+    // plus 14, %r1 is the start address of .plt, and 48 is the size
+    // of this PLT header. We multiply by 3/2 because each PLT entry
+    // is 16 bytes, whereas each .rela.plt entry is 24 bytes.
     0xb9, 0x09, 0x00, 0x01,             // sgr   %r0, %r1
     0xa7, 0x0b, 0xff, 0xc2,             // aghi  %r0, -62
     0xeb, 0x10, 0x00, 0x01, 0x00, 0x0c, // srlg  %r1, %r0, 1
     0xb9, 0x08, 0x00, 0x01,             // agr   %r0, %r1
+    // Store the computed value to 56(%r15) and .got.plt[1] to 48(%15)
+    // where %r15 is the stack pointer.
     0xe3, 0x00, 0xf0, 0x38, 0x00, 0x24, // stg   %r0, 56(%r15)
-    // Branch to _dl_runtime_resolve
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_OFFSET
     0xd2, 0x07, 0xf0, 0x30, 0x10, 0x08, // mvc   48(8, %r15), 8(%r1)
+    // Branch to _dl_runtime_resolve
     0xe3, 0x10, 0x10, 0x10, 0x00, 0x04, // lg    %r1, 16(%r1)
     0x07, 0xf1,                         // br    %r1
-    0x07, 0x00, 0x07, 0x00,             // nopr; nopr
+    0x00, 0x00, 0x00, 0x00,             // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -73,7 +81,7 @@ void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_ENTRY_OFFSET
     0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
     0x0d, 0x01,                         // basr  %r0, %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -86,7 +94,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOT_ENTRY_OFFSET
     0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
     0x07, 0xf1,                         // br    %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -94,15 +102,21 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
     break;
   case R_390_PC32:
-    *(ub32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ub32 *)loc = val - P;
     break;
   case R_390_64:
     *(ub64 *)loc = val;
@@ -114,14 +128,17 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -151,7 +168,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_390_12:
       check(S + A, 0, 1 << 12);
-      *(ul16 *)loc |= bits(S + A, 11, 0);
+      *(ub16 *)loc |= bits(S + A, 11, 0);
       break;
     case R_390_16:
       check(S + A, 0, 1 << 16);
@@ -169,7 +186,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_390_PC12DBL:
     case R_390_PLT12DBL:
       check_dbl(S + A - P, -(1 << 12), 1 << 12);
-      *(ul16 *)loc |= bits(S + A - P, 12, 1);
+      *(ub16 *)loc |= bits(S + A - P, 12, 1);
       break;
     case R_390_PC16:
       check(S + A - P, -(1 << 15), 1 << 15);
@@ -194,14 +211,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ub32 *)loc |= bits(S + A - P, 24, 1);
       break;
     case R_390_PC32DBL:
-    case R_390_PLT32DBL:
       check_dbl(S + A - P, -(1LL << 32), 1LL << 32);
       *(ub32 *)loc = (S + A - P) >> 1;
       break;
+    case R_390_PLT32DBL:
+      if (!sym.is_remaining_undef_weak())
+        check_dbl(S + A - P, -(1LL << 32), 1LL << 32);
+      *(ub32 *)loc = (S + A - P) >> 1;
+      break;
     case R_390_GOT12:
     case R_390_GOTPLT12:
       check(G + A, 0, 1 << 12);
-      *(ul16 *)loc |= bits(G + A, 11, 0);
+      *(ub16 *)loc |= bits(G + A, 11, 0);
       break;
     case R_390_GOT16:
     case R_390_GOTPLT16:
@@ -254,6 +275,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             is_int(val, 33)) {
           *(ub16 *)(loc - 2) = 0xc000 | (op & 0x00f0);
           *(ub32 *)loc = val >> 1;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_390_PC32DBL;
           break;
         }
       }
@@ -334,14 +357,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -352,7 +372,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     u64 A = frag ? frag_addend : (i64)rel.r_addend;
 
     auto check = [&](i64 val, i64 lo, i64 hi) {
-      check_range(ctx, val, i, lo, hi);
+      check_range(ctx, i, val, lo, hi);
     };
 
     switch (rel.r_type) {
@@ -375,7 +395,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -389,7 +409,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -483,3 +503,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-sh4.cc

@@ -47,9 +47,10 @@
 //    with a piece of code to store the address of .got to %r12.
 //    We can use the register in our PLT for position-independent output.
 //
-//  - Even though it uses the RELA-type relocations, relocation addends
-//    are stored not to the r_addend field but to the relocated section
-//    contents for some reason. Therefore, it's effectively REL.
+//  - Even though it uses RELA-type relocations, object files store
+//    addends not in the r_addend field but in the relocated section
+//    contents. Dynamic relocations, however, follow the usual RELA
+//    convention.
 //
 //  - It looks like the ecosystem has bit-rotted. Some tests, especially
 //    one using C++ exceptions, don't pass even with GNU ld.
@@ -66,8 +67,8 @@ namespace mold {
 
 using E = MOLD_TARGET;
 
-// Even though SH-4 uses RELA-type relocations, addends are stored to
-// relocated places for some reason.
+// Even though SH-4 uses RELA-type relocations, addends are stored in
+// the relocated places for some reason.
 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   switch (rel.r_type) {
@@ -137,7 +138,7 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
       0x5221, //    mov.l   @(4, r2), r2
       0x402b, //    jmp     @r0
       0xe000, //    mov     #0, r0
-      0x0009, //    nop
+      0xfffd, //    (illegal)
       0, 0,   // 1: .long GOTPLT
     };
 
@@ -151,32 +152,36 @@ template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   if (ctx.arg.pic) {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x00ce, //    mov.l   @(r0, r12), r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   } else {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x6002, //    mov.l   @r0, r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx);
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx);
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   }
 }
 
@@ -210,8 +215,8 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
 
   switch (rel.r_type) {
@@ -237,7 +242,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -285,14 +293,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -309,11 +314,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U32<E> *)loc = S + A;
       break;
+    case R_SH_TLS_LDO_32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -326,7 +337,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       Error(ctx) << sym << ": GNU ifunc symbol is not supported on sh4";

src/arch-sparc64.cc

@@ -53,6 +53,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/sparc.pdf
 
+#if MOLD_SPARC64
+
 #include "mold.h"
 
 namespace mold {
@@ -76,26 +78,77 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
   memset(buf, 0, E::plt_hdr_size);
 }
 
+// Returns the byte offset within .plt of the data pointer for a large SPARC
+// PLT entry. See write_plt_entry below for the block layout this assumes.
+i64 sparc_plt_ptr_offset(Context<E> &ctx, i64 pltidx) {
+  i64 i = pltidx - sparc_num_small_plt;
+  i64 block = i / 160;
+  i64 nlarge = (i64)ctx.plt->symbols.size() - sparc_num_small_plt;
+  i64 nstub = std::min<i64>(160, nlarge - block * 160);
+  return 0x100000 + block * 5120 + nstub * 24 + (i % 160) * 8;
+}
+
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  static ub32 insn[] = {
-    0x0300'0000, // sethi (. - .PLT0), %g1
-    0x3068'0000, // ba,a  %xcc, .PLT1
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-  };
+  // SPARC uses two PLT entry formats. A "small" entry branches directly to
+  // the resolver stub (.PLT1) with a BPcc instruction whose reach is only
+  // ±1 MiB. Once the PLT grows past that (0x100000 bytes), we switch to a
+  // "large" format: rather than branching to the resolver, a large entry
+  // loads a 64-bit value from a nearby data pointer and jumps to (that value
+  // + its own address).
+  //
+  // Large entries are grouped into blocks of 160: each block is 160 code
+  // stubs followed by 160 8-byte data pointers, one per stub. The stubs must
+  // sit on a fixed grid, 24 bytes apart, because that is how the loader finds
+  // them: on a lazy bind the stub jumps to the resolver (.PLT0) with only its
+  // own address, and the loader derives which symbol to resolve from that
+  // address. Nothing may sit between two stubs, so each stub's pointer lives
+  // in the block's pointer region, which it reaches with a signed 13-bit ldx
+  // offset (see to_plt_offset). This layout is dictated by the loader; we
+  // cannot rearrange or simplify it.
+  i64 idx = sym.get_plt_idx(ctx);
 
-  u64 plt0 = ctx.plt->shdr.sh_addr;
-  u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
-  u64 entry = sym.get_plt_addr(ctx);
+  if (idx < sparc_num_small_plt) {
+    static ub32 insn[] = {
+      0x0300'0000, // sethi (. - .PLT0), %g1
+      0x3068'0000, // ba,a  %xcc, .PLT1
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+    };
 
-  memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)buf |= bits(entry - plt0, 21, 0);
-  *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+    u64 plt0 = ctx.plt->shdr.sh_addr;
+    u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
+    u64 entry = sym.get_plt_addr(ctx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)buf |= bits(entry - plt0, 21, 0);
+    *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+  } else {
+    static ub32 insn[] = {
+      0x8a10'000f, // mov  %o7, %g5
+      0x4000'0002, // call . + 8
+      0x0100'0000, // nop
+      0xc25b'e000, // ldx  [ %o7 + .ptr ], %g1
+      0x83c3'c001, // jmpl %o7 + %g1, %g1
+      0x9e10'0005, // mov  %g5, %o7
+    };
+
+    u64 plt = ctx.plt->shdr.sh_addr;
+    u64 call = sym.get_plt_addr(ctx) + 4;
+    i64 ptroff = sparc_plt_ptr_offset(ctx, idx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)(buf + 12) |= bits(plt + ptroff - call, 12, 0);
+
+    // The data pointer initially holds (.PLT0 - call) so that the first call
+    // jumps to .PLT0, where the loader's lazy resolver lives. The resolver
+    // later overwrites it with (target - call).
+    *(ub64 *)(ctx.buf + ctx.plt->shdr.sh_offset + ptroff) = plt - call;
+  }
 }
 
 template <>
@@ -116,9 +169,14 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
@@ -128,7 +186,8 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
     *(ub64 *)loc = val;
     break;
   case R_SPARC_DISP32:
-    *(ub32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ub32 *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -139,12 +198,17 @@ template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
 
-  for (i64 i = 0; i < rels.size(); i++) {
+  // We iterate over relocations in reverse order so that it is easy
+  // to swap instructions for R_SPARC_TLS_GD_CALL.
+  for (i64 i = (i64)rels.size() - 1; i >= 0; i--) {
     const ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -157,6 +221,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto rs1 = [&] { return *(ub32 *)loc & (0b11111 << 14); };
+    auto rs2 = [&] { return *(ub32 *)loc & 0b11111; };
+    auto rd  = [&] { return *(ub32 *)loc & (0b11111 << 25); };
+
     switch (rel.r_type) {
     case R_SPARC_5:
       check(S + A, 0, 1 << 5);
@@ -172,7 +240,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_SPARC_8:
       check(S + A, 0, 1 << 8);
-      *(u8 *)loc = S + A;
+      *loc = S + A;
       break;
     case R_SPARC_10:
       check(S + A, 0, 1 << 10);
@@ -206,13 +274,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ub32 *)loc = S + A;
       break;
     case R_SPARC_PLT64:
-    case R_SPARC_UA64:
     case R_SPARC_REGISTER:
       *(ub64 *)loc = S + A;
       break;
     case R_SPARC_DISP8:
       check(S + A - P, -(1 << 7), 1 << 7);
-      *(u8 *)loc = S + A - P;
+      *loc = S + A - P;
       break;
     case R_SPARC_DISP16:
       check(S + A - P, -(1 << 15), 1 << 15);
@@ -242,7 +309,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_SPARC_WDISP30:
     case R_SPARC_WPLT30:
-      check(S + A - P, -(1LL << 31), 1LL << 31);
+      if (!sym.is_remaining_undef_weak())
+        check(S + A - P, -(1LL << 31), 1LL << 31);
       *(ub32 *)loc |= bits(S + A - P, 31, 2);
       break;
     case R_SPARC_HI22:
@@ -297,12 +365,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_SPARC_GOTDATA_OP:
       if (sym.is_absolute()) {
-        // ldx [ %g2 + %g1 ], %g1  →  nop
-        *(ub32 *)loc = 0x0100'0000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  mov %rs2, %rd
+        *(ub32 *)loc = 0x8010'0000 | rs2() | rd();
       } else if (sym.is_pcrel_linktime_const(ctx)) {
-        // ldx [ %g2 + %g1 ], %g1  →  add %g2, %g1, %g1
-        *(ub32 *)loc &= 0b00'11111'000000'11111'1'11111111'11111;
-        *(ub32 *)loc |= 0b10'00000'000000'00000'0'00000000'00000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  add %rs1, %rs2, %rd
+        *(ub32 *)loc = 0x8000'0000 | rs1() | rs2() | rd();
       }
       break;
     case R_SPARC_PC10:
@@ -357,12 +424,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (sym.has_tlsgd(ctx)) {
         *(ub32 *)loc |= bits(sym.get_tlsgd_addr(ctx) + A - GOT, 9, 0);
       } else if (sym.has_gottp(ctx)) {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8010'2000 | (rd << 25) | (rd << 14); // or  %reg, $0, %reg
+        // add %rs1, %rs2, %rd → or  %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8010'2000 | rs1() | rd();
         *(ub32 *)loc |= bits(sym.get_gottp_addr(ctx) + A - GOT, 9, 0);
       } else {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8018'2000 | (rd << 25) | (rd << 14); // xor %reg, $0, %reg
+        // add %rs1, %rs2, %rd → xor %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8018'2000 | rs1() | rd();
         *(ub32 *)loc |= bits(S + A - ctx.tp_addr, 9, 0) | 0b1'1100'0000'0000;
       }
       break;
@@ -370,11 +437,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (sym.has_tlsgd(ctx)) {
         // do nothing
       } else if (sym.has_gottp(ctx)) {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0xd05d'c000 | rs2; // ldx [ %l7 + %reg ], %o0
+        // add %rs1, %rs2, %rd → ldx [ %rs1 + %rs2 ], %rd
+        *(ub32 *)loc = 0xc058'0000 | rs1() | rs2() | rd();
       } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9001'c000 | rs2; // add %g7, %reg, %o0
+        // add %rs1, %rs2, %rd → add %g7, %rs2, %rd
+        *(ub32 *)loc = 0x8001'c000 | rs2() | rd();
       }
       break;
     case R_SPARC_TLS_GD_CALL:
@@ -382,9 +449,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         u64 addr = ctx.extra.tls_get_addr->get_addr(ctx);
         *(ub32 *)loc |= bits(addr + A - P, 31, 2);
       } else if (sym.has_gottp(ctx)) {
-        *(ub32 *)loc = 0x9001'c008; // add %g7, %o0, %o0
+        // When we rewrite a branch instruction with a non-branch one,
+        // we need to swap the instruction and the following one so that
+        // the original execution order, which is inverted due to the
+        // branch delay slot, is preserved.
+        //
+        // Since we apply relocations from the end to the beginning,
+        // the instruction at loc + 4 is already complete.
+        memcpy(loc, loc + 4, 4);
+        *(ub32 *)(loc + 4) = 0x9001'c008; // add %g7, %o0, %o0
       } else {
-        *(ub32 *)loc = 0x0100'0000; // nop
+        *(ub32 *)loc = 0x0100'0000; // call → nop
       }
       break;
     case R_SPARC_TLS_LDM_HI22:
@@ -400,12 +475,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         *(ub32 *)loc |= bits(ctx.tp_addr - ctx.tls_begin, 9, 0);
       break;
     case R_SPARC_TLS_LDM_ADD:
-      if (ctx.got->has_tlsld(ctx)) {
-        // do nothing
-      } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9021'c000 | rs2; // sub %g7, %reg, %o0
-      }
+      if (!ctx.got->has_tlsld(ctx))
+        *(ub32 *)loc = 0x8021'c000 | rs2() | rd(); // sub %g7, %rs2, %rd
       break;
     case R_SPARC_TLS_LDM_CALL:
       if (ctx.got->has_tlsld(ctx)) {
@@ -437,6 +508,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ub32 *)loc = sym.esym().st_size + A;
       break;
     case R_SPARC_64:
+    case R_SPARC_UA64:
     case R_SPARC_TLS_LDO_ADD:
     case R_SPARC_TLS_IE_LD:
     case R_SPARC_TLS_IE_LDX:
@@ -450,14 +522,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -468,7 +537,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     u64 A = frag ? frag_addend : (i64)rel.r_addend;
 
     auto check = [&](i64 val, i64 lo, i64 hi) {
-      check_range(ctx, val, i, lo, hi);
+      check_range(ctx, i, val, lo, hi);
     };
 
     switch (rel.r_type) {
@@ -493,13 +562,14 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>
 void InputSection<E>::scan_relocations(Context<E> &ctx) {
   assert(shdr().sh_flags & SHF_ALLOC);
   std::span<const ElfRel<E>> rels = get_rels(ctx);
+  bool needs_tlsgd = false;
 
   // Scan relocations
   for (i64 i = 0; i < rels.size(); i++) {
@@ -507,7 +577,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -526,7 +596,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_SPARC_REGISTER:
     case R_SPARC_UA16:
     case R_SPARC_UA32:
-    case R_SPARC_UA64:
     case R_SPARC_PC_HM10:
     case R_SPARC_OLO10:
     case R_SPARC_LOX10:
@@ -584,6 +653,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_GOTTP;
       } else {
         sym.flags |= NEEDS_TLSGD;
+        needs_tlsgd = true;
       }
       break;
     case R_SPARC_TLS_LDM_HI22:
@@ -597,23 +667,21 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_SPARC_TLS_IE_HI22:
       sym.flags |= NEEDS_GOTTP;
       break;
-    case R_SPARC_TLS_GD_CALL:
-    case R_SPARC_TLS_LDM_CALL:
-      if (Symbol<E> *sym = ctx.extra.tls_get_addr; sym->is_imported)
-        sym->flags |= NEEDS_PLT;
-      break;
     case R_SPARC_TLS_LE_HIX22:
     case R_SPARC_TLS_LE_LOX10:
       check_tlsle(ctx, sym, rel);
       break;
     case R_SPARC_64:
+    case R_SPARC_UA64:
     case R_SPARC_GOTDATA_OP_LOX10:
     case R_SPARC_GOTDATA_OP:
     case R_SPARC_GOTDATA_LOX10:
     case R_SPARC_TLS_GD_LO10:
     case R_SPARC_TLS_GD_ADD:
+    case R_SPARC_TLS_GD_CALL:
     case R_SPARC_TLS_LDM_LO10:
     case R_SPARC_TLS_LDM_ADD:
+    case R_SPARC_TLS_LDM_CALL:
     case R_SPARC_TLS_LDO_HIX22:
     case R_SPARC_TLS_LDO_LOX10:
     case R_SPARC_TLS_LDO_ADD:
@@ -627,6 +695,14 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       Error(ctx) << *this << ": unknown relocation: " << rel;
     }
   }
+
+  // TLS_GD_CALL and TLS_LDM_CALL relocations implicitly refer to
+  // __tls_get_addr, which may be dynamically linked from libc.so.
+  Symbol<E> &sym = *ctx.extra.tls_get_addr;
+  if (sym.is_imported && (needs_tlsgd || ctx.needs_tlsld))
+    sym.flags |= NEEDS_PLT;
 }
 
 } // namespace mold
+
+#endif

src/arch-x86-64.cc

@@ -25,8 +25,9 @@
 //
 // https://gitlab.com/x86-psABIs/x86-64-ABI
 
-#include "mold.h"
+#if MOLD_X86_64
 
+#include "mold.h"
 #include <tbb/parallel_for_each.h>
 
 namespace mold {
@@ -103,24 +104,31 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 }
 
 template <>
-void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
-                                       u64 offset, u64 val) {
+void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, InputSection<E> &isec,
+                                       const ElfRel<E> &rel, u64 offset, u64 val) {
   u8 *loc = ctx.buf + this->shdr.sh_offset + offset;
+  u64 P = this->shdr.sh_addr + offset;
+
+  auto check = [&](i64 val, i64 lo, i64 hi) {
+    check_range(ctx, isec, rel, val, lo, hi);
+  };
 
   switch (rel.r_type) {
   case R_NONE:
     break;
   case R_X86_64_32:
+    check(val, 0, 1LL << 32);
     *(ul32 *)loc = val;
     break;
   case R_X86_64_64:
     *(ul64 *)loc = val;
     break;
   case R_X86_64_PC32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    check(val - P, -(1LL << 31), 1LL << 31);
+    *(ul32 *)loc = val - P;
     break;
   case R_X86_64_PC64:
-    *(ul64 *)loc = val - this->shdr.sh_addr - offset;
+    *(ul64 *)loc = val - P;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -432,14 +440,17 @@ static void relax_ld_to_le(u8 *loc, const ElfRel<E> &rel, i64 tls_size) {
 // scan_relocations().
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -488,9 +499,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc = S + A - P;
       break;
     case R_X86_64_PC32:
-    case R_X86_64_PLT32:
       write32s(S + A - P);
       break;
+    case R_X86_64_PLT32:
+      if (!sym.is_remaining_undef_weak())
+        check(S + A - P, -(1LL << 31), 1LL << 31);
+      *(ul32 *)loc = S + A - P;
+      break;
     case R_X86_64_PC64:
       *(ul64 *)loc = S + A - P;
       break;
@@ -527,6 +542,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           loc[-2] = insn >> 8;
           loc[-1] = insn;
           *(ul32 *)loc = S + A - P;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_X86_64_PC32;
           break;
         }
       }
@@ -656,14 +673,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 // scan_relocations.
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
     SectionFragment<E> *frag;
@@ -740,7 +754,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 // Linker has to create data structures in an output file to apply
@@ -759,8 +773,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
       continue;
 
-    Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    Symbol<E> &sym = *file->symbols[rel.r_sym];
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -858,95 +872,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
   }
 }
 
-// Intel CET is a relatively new CPU feature to enhance security by
-// protecting control flow integrity. If the feature is enabled, indirect
-// branches (i.e. branch instructions that take a register instead of an
-// immediate) must land on a "landing pad" instruction, or a CPU-level fault
-// will raise. That prevents an attacker to branch to a middle of a random
-// function, making ROP or JOP much harder to conduct.
-//
-// On x86-64, the landing pad instruction is ENDBR64. That is actually a
-// repurposed NOP instruction to provide binary compatibility with older
-// hardware that doesn't support CET.
-//
-// The problem here is that the compiler always emits a landing pad at the
-// beginning fo a global function because it doesn't know whether or not the
-// function's address is taken in other translation units. As a result, the
-// resulting binary contains more landing pads than necessary.
-//
-// This function rewrites a landing pad with a nop if the function's address
-// was not actually taken. We can do what the compiler cannot because we
-// know about all translation units.
-void rewrite_endbr(Context<E> &ctx) {
-  Timer t(ctx, "rewrite_endbr");
-
-  constexpr u8 endbr64[] = {0xf3, 0x0f, 0x1e, 0xfa};
-  constexpr u8 nop[] = {0x0f, 0x1f, 0x40, 0x00};
-
-  // Rewrite all endbr64 instructions referred to by function symbols with
-  // NOPs. We handle only global symbols because the compiler doesn't emit
-  // an endbr64 for a file-scoped function in the first place if its address
-  // is not taken within the file.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (Symbol<E> *sym : file->get_global_syms()) {
-      if (sym->file == file && sym->esym().st_type == STT_FUNC) {
-        if (InputSection<E> *isec = sym->get_input_section();
-            isec && (isec->shdr().sh_flags & SHF_EXECINSTR)) {
-          if (OutputSection<E> *osec = isec->output_section) {
-            u8 *buf = ctx.buf + osec->shdr.sh_offset + isec->offset + sym->value;
-            if (memcmp(buf, endbr64, 4) == 0)
-              memcpy(buf, nop, 4);
-          }
-        }
-      }
-    }
-  });
-
-  auto write_back = [&](InputSection<E> *isec, i64 offset) {
-    // If isec has an endbr64 at a given offset, copy that instruction to
-    // the output buffer, possibly overwriting a nop written in the above
-    // loop.
-    if (isec && isec->output_section &&
-        (isec->shdr().sh_flags & SHF_EXECINSTR) &&
-        0 <= offset && offset <= isec->contents.size() - 4 &&
-        memcmp(isec->contents.data() + offset, endbr64, 4) == 0)
-      memcpy(ctx.buf + isec->output_section->shdr.sh_offset + isec->offset + offset,
-             endbr64, 4);
-  };
-
-  // Write back endbr64 instructions if they are referred to by address-taking
-  // relocations.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC)) {
-        for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
-          if (!is_func_call_rel(rel)) {
-            Symbol<E> *sym = file->symbols[rel.r_sym];
-            if (sym->esym().st_type == STT_SECTION)
-              write_back(sym->get_input_section(), rel.r_addend);
-            else
-              write_back(sym->get_input_section(), sym->value);
-          }
-        }
-      }
-    }
-  });
-
-  // We record addresses of some symbols in the ELF header, .dynamic or in
-  // .dynsym. We need to retain endbr64s for such symbols.
-  auto keep = [&](Symbol<E> *sym) {
-    if (sym)
-      write_back(sym->get_input_section(), sym->value);
-  };
-
-  keep(ctx.arg.entry);
-  keep(ctx.arg.init);
-  keep(ctx.arg.fini);
-
-  if (ctx.dynsym)
-    for (Symbol<E> *sym : ctx.dynsym->symbols)
-      if (sym && sym->is_exported)
-        keep(sym);
-}
-
 } // namespace mold
+
+#endif

src/archive-file.cc

@@ -23,12 +23,11 @@
 // see the contents of libc.a by running `ar t
 // /usr/lib/x86_64-linux-gnu/libc.a`.
 
-#pragma once
-
-#include "common.h"
+#include "mold.h"
 
 namespace mold {
 
+namespace {
 struct ArHdr {
   char ar_name[16];
   char ar_date[12];
@@ -74,13 +73,16 @@ struct ArHdr {
     return {ar_name, sizeof(ar_name)};
   }
 };
+}
 
-template <typename Context>
-std::vector<MappedFile *>
-read_thin_archive_members(Context &ctx, MappedFile *mf) {
+// Returns the paths of the members of a thin archive, which are stored
+// outside of the archive file, without opening them.
+template <typename E>
+std::vector<std::string>
+get_thin_archive_member_paths(Context<E> &ctx, MappedFile *mf) {
   u8 *begin = mf->data;
   u8 *data = begin + 8;
-  std::vector<MappedFile *> vec;
+  std::vector<std::string> vec;
   std::string_view strtab;
 
   while (data < begin + mf->size) {
@@ -111,20 +113,34 @@ read_thin_archive_members(Context &ctx, MappedFile *mf) {
     std::string name = hdr.read_name(strtab, body);
 
     // Skip if symbol table
-    if (name == "__.SYMDEF" || name == "__.SYMDEF SORTED")
+    if (name == "__.SYMDEF" || name == "__.SYMDEF SORTED") {
+      data = body + size;
       continue;
+    }
 
-    std::string path = name.starts_with('/') ?
-      name : (path_dirname(mf->name) / name).string();
-    vec.push_back(must_open_file(ctx, path));
-    vec.back()->thin_parent = mf;
+    if (name.starts_with('/'))
+      vec.push_back(name);
+    else
+      vec.push_back((path_dirname(mf->name) / name).string());
     data = body;
   }
   return vec;
 }
 
-template <typename Context>
-std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_thin_archive_members(Context<E> &ctx, MappedFile *mf) {
+  std::vector<MappedFile *> vec;
+  for (std::string &path : get_thin_archive_member_paths(ctx, mf)) {
+    vec.push_back(must_open_file(ctx, path));
+    vec.back()->thin_parent = mf;
+  }
+  return vec;
+}
+
+template <typename E>
+std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &ctx, MappedFile *mf) {
   u8 *begin = mf->data;
   u8 *data = begin + 8;
   std::vector<MappedFile *> vec;
@@ -161,13 +177,30 @@ std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf)
   return vec;
 }
 
-template <typename Context>
-std::vector<MappedFile *> read_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_archive_members(Context<E> &ctx, MappedFile *mf) {
   std::string_view str = mf->get_contents();
+
   if (str.starts_with("!<arch>\n"))
     return read_fat_archive_members(ctx, mf);
+
   assert(str.starts_with("!<thin>\n"));
   return read_thin_archive_members(ctx, mf);
 }
 
+using E = MOLD_TARGET;
+
+template std::vector<std::string>
+get_thin_archive_member_paths(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_thin_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_archive_members(Context<E> &, MappedFile *);
+
 } // namespace mold

src/config.cc

@@ -1,12 +0,0 @@
-#include "mold.h"
-
-namespace mold {
-
-std::string get_mold_version() {
-  if (mold_git_hash.empty())
-    return "mold "s + MOLD_VERSION + " (compatible with GNU ld)";
-  return "mold "s + MOLD_VERSION + " (" + mold_git_hash +
-         "; compatible with GNU ld)";
-}
-
-} // namespace mold

src/elf.cc

@@ -1,11 +1,12 @@
 #include "elf.h"
+#include <sstream>
 
 namespace mold {
 
 static std::string unknown_type(u32 r_type) {
-  char buf[50];
-  snprintf(buf, sizeof(buf), "unknown (0x%x)", r_type);
-  return buf;
+  std::stringstream ss;
+  ss << "unknown (0x" << std::hex << r_type << ")";
+  return ss.str();
 }
 
 #define CASE(x) case x: return #x
@@ -160,6 +161,7 @@ std::string rel_to_string<ARM64LE>(u32 r_type) {
   CASE(R_AARCH64_LD64_GOT_LO12_NC);
   CASE(R_AARCH64_LD64_GOTPAGE_LO15);
   CASE(R_AARCH64_PLT32);
+  CASE(R_AARCH64_GOTPCREL32);
   CASE(R_AARCH64_TLSGD_ADR_PREL21);
   CASE(R_AARCH64_TLSGD_ADR_PAGE21);
   CASE(R_AARCH64_TLSGD_ADD_LO12_NC);
@@ -232,7 +234,7 @@ std::string rel_to_string<ARM64BE>(u32 r_type) {
 }
 
 template <>
-std::string rel_to_string<ARM32>(u32 r_type) {
+std::string rel_to_string<ARM32LE>(u32 r_type) {
   switch (r_type) {
   CASE(R_ARM_NONE);
   CASE(R_ARM_PC24);
@@ -373,6 +375,11 @@ std::string rel_to_string<ARM32>(u32 r_type) {
   return unknown_type(r_type);
 }
 
+template <>
+std::string rel_to_string<ARM32BE>(u32 r_type) {
+  return rel_to_string<ARM32LE>(r_type);
+}
+
 template <>
 std::string rel_to_string<RV64LE>(u32 r_type) {
   switch (r_type) {
@@ -388,6 +395,7 @@ std::string rel_to_string<RV64LE>(u32 r_type) {
   CASE(R_RISCV_TLS_DTPREL64);
   CASE(R_RISCV_TLS_TPREL32);
   CASE(R_RISCV_TLS_TPREL64);
+  CASE(R_RISCV_TLSDESC);
   CASE(R_RISCV_BRANCH);
   CASE(R_RISCV_JAL);
   CASE(R_RISCV_CALL);
@@ -422,6 +430,7 @@ std::string rel_to_string<RV64LE>(u32 r_type) {
   CASE(R_RISCV_SET8);
   CASE(R_RISCV_SET16);
   CASE(R_RISCV_SET32);
+  CASE(R_RISCV_GOT32_PCREL);
   CASE(R_RISCV_32_PCREL);
   CASE(R_RISCV_IRELATIVE);
   CASE(R_RISCV_PLT32);
@@ -744,6 +753,8 @@ std::string rel_to_string<SPARC64>(u32 r_type) {
   CASE(R_SPARC_GOTDATA_OP_HIX22);
   CASE(R_SPARC_GOTDATA_OP_LOX10);
   CASE(R_SPARC_GOTDATA_OP);
+  CASE(R_SPARC_SIZE32);
+  CASE(R_SPARC_JMP_IREL);
   CASE(R_SPARC_IRELATIVE);
   }
   return unknown_type(r_type);

src/elf.h

@@ -13,7 +13,8 @@ struct X86_64;
 struct I386;
 struct ARM64LE;
 struct ARM64BE;
-struct ARM32;
+struct ARM32LE;
+struct ARM32BE;
 struct RV64LE;
 struct RV64BE;
 struct RV32LE;
@@ -80,7 +81,11 @@ enum : u32 {
   SHT_RELR = 19,
   SHT_CREL = 0x40000014,
   SHT_LOOS = 0x60000000,
+  SHT_ANDROID_REL = 0x60000001,
+  SHT_ANDROID_RELA = 0x60000002,
   SHT_LLVM_ADDRSIG = 0x6fff4c03,
+  SHT_ANDROID_RELR = 0x6fffff00,
+  SHT_GNU_SFRAME = 0x6ffffff4,
   SHT_GNU_HASH = 0x6ffffff6,
   SHT_GNU_VERDEF = 0x6ffffffd,
   SHT_GNU_VERNEED = 0x6ffffffe,
@@ -165,6 +170,7 @@ enum : u32 {
   VER_NDX_GLOBAL = 1,
   VER_NDX_LAST_RESERVED = 1,
   VER_NDX_UNSPECIFIED = 0xffff,
+  VERSYM_HIDDEN = 0x8000,
 };
 
 enum : u32 {
@@ -173,10 +179,6 @@ enum : u32 {
   VER_FLG_INFO = 4,
 };
 
-enum : u32 {
-  VERSYM_HIDDEN = 0x8000,
-};
-
 enum : u32 {
   PT_NULL = 0,
   PT_LOAD = 1,
@@ -190,6 +192,7 @@ enum : u32 {
   PT_GNU_STACK = 0x6474e551,
   PT_GNU_RELRO = 0x6474e552,
   PT_GNU_PROPERTY = 0x6474e553,
+  PT_GNU_SFRAME = 0x6474e554,
   PT_OPENBSD_RANDOMIZE = 0x65a3dbe6,
   PT_ARM_EXIDX = 0x70000001,
   PT_RISCV_ATTRIBUTES = 0x70000003,
@@ -284,6 +287,13 @@ enum : u32 {
   DT_RELRSZ = 35,
   DT_RELR = 36,
   DT_RELRENT = 37,
+  DT_ANDROID_REL = 0x6000000f,
+  DT_ANDROID_RELSZ = 0x60000010,
+  DT_ANDROID_RELA = 0x60000011,
+  DT_ANDROID_RELASZ = 0x60000012,
+  DT_ANDROID_RELR = 0x6fffe000,
+  DT_ANDROID_RELRSZ = 0x6fffe001,
+  DT_ANDROID_RELRENT = 0x6fffe003,
   DT_GNU_HASH = 0x6ffffef5,
   DT_DEPAUDIT = 0x6ffffefb,
   DT_AUDIT = 0x6ffffefc,
@@ -352,14 +362,20 @@ enum : u32 {
 };
 
 enum : u32 {
+  ELFCOMPRESS_NONE = 0,
   ELFCOMPRESS_ZLIB = 1,
   ELFCOMPRESS_ZSTD = 2,
 };
 
 enum : u32 {
-  EF_ARM_ABI_FLOAT_SOFT = 0x00000200,
-  EF_ARM_ABI_FLOAT_HARD = 0x00000400,
-  EF_ARM_EABI_VER5 = 0x05000000,
+  EF_ARM_ABI_FLOAT_SOFT = 0x200,
+  EF_ARM_ABI_FLOAT_HARD = 0x400,
+  EF_ARM_BE8 = 0x800000,
+  EF_ARM_EABI_VER5 = 0x5000000,
+};
+
+enum : u32 {
+  EF_PPC64_ABI = 3,
 };
 
 enum : u32 {
@@ -554,6 +570,7 @@ enum : u32 {
   R_AARCH64_LD64_GOT_LO12_NC = 0x138,
   R_AARCH64_LD64_GOTPAGE_LO15 = 0x139,
   R_AARCH64_PLT32 = 0x13a,
+  R_AARCH64_GOTPCREL32 = 0x13b,
   R_AARCH64_TLSGD_ADR_PREL21 = 0x200,
   R_AARCH64_TLSGD_ADR_PAGE21 = 0x201,
   R_AARCH64_TLSGD_ADD_LO12_NC = 0x202,
@@ -804,6 +821,7 @@ enum : u32 {
   R_RISCV_SET8 = 54,
   R_RISCV_SET16 = 55,
   R_RISCV_SET32 = 56,
+  R_RISCV_GOT32_PCREL = 41,
   R_RISCV_32_PCREL = 57,
   R_RISCV_IRELATIVE = 58,
   R_RISCV_PLT32 = 59,
@@ -1411,9 +1429,11 @@ enum : u32 {
 
 enum : u32 {
   DW_UT_compile = 0x01,
+  DW_UT_type = 0x02,
   DW_UT_partial = 0x03,
   DW_UT_skeleton = 0x04,
   DW_UT_split_compile = 0x05,
+  DW_UT_split_type = 0x06,
 };
 
 enum : u32 {
@@ -1488,6 +1508,52 @@ template <typename E> using U64 = std::conditional_t<E::is_le, ul64, ub64>;
 template <typename E> using Word = std::conditional_t<E::is_64, U64<E>, U32<E>>;
 template <typename E> using SWord = std::conditional_t<E::is_64, I64<E>, I32<E>>;
 
+//
+// SFrame is a simple unwind information format used as a lightweight
+// alternative to .eh_frame. A .sframe section consists of a header, an
+// array of Function Descriptor Entries (FDEs) sorted by PC, and a blob
+// of Frame Row Entries (FREs). mold understands SFrame Version 3.
+//
+// https://sourceware.org/binutils/docs/sframe-spec.html
+//
+
+static constexpr u16 SFRAME_MAGIC = 0xdee2;
+
+static constexpr u8 SFRAME_F_FDE_SORTED = 0x1;
+static constexpr u8 SFRAME_F_FRAME_POINTER = 0x2;
+static constexpr u8 SFRAME_F_FDE_FUNC_START_PCREL = 0x4;
+
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_BIG = 1;
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_LITTLE = 2;
+static constexpr u8 SFRAME_ABI_AMD64_ENDIAN_LITTLE = 3;
+static constexpr u8 SFRAME_ABI_S390X_ENDIAN_BIG = 4;
+
+template <typename E>
+struct SFrameHeader {
+  U16<E> magic;
+  u8 version;
+  u8 flags;
+  u8 abi_arch;
+  i8 cfa_fixed_fp_offset;
+  i8 cfa_fixed_ra_offset;
+  u8 auxhdr_len;
+  U32<E> num_fdes;
+  U32<E> num_fres;
+  U32<E> fre_len;
+  U32<E> fdeoff;
+  U32<E> freoff;
+};
+
+// The index part of an SFrame Version 3 FDE. The func_start_offset field
+// is PC-relative (relative to its own address) when the section flag
+// SFRAME_F_FDE_FUNC_START_PCREL is set, which is how mold always emits it.
+template <typename E>
+struct SFrameFdeIdx {
+  I64<E> func_start_offset;
+  U32<E> func_size;
+  U32<E> func_start_fre_off;
+};
+
 template <typename E> requires E::is_64
 struct ElfSym<E> {
   bool is_undef() const { return st_shndx == SHN_UNDEF; }
@@ -1770,47 +1836,21 @@ struct ElfRel<SPARC64> {
   ib64 r_addend;
 };
 
-template <>
-struct ElfRel<SH4LE> {
-  ElfRel() = default;
-
-  // Addend is ignored except for base relocations because even though
-  // SH4 is RELA, r_addend is ignored in most cases and works as if it
-  // were REL.
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_type(type), r_sym(sym), r_addend(sym ? 0 : addend) {}
-
-  ul32 r_offset;
-  u8 r_type;
-  ul24 r_sym;
-  il32 r_addend;
-};
-
-template <>
-struct ElfRel<SH4BE> {
-  ElfRel() = default;
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_sym(sym), r_type(type), r_addend(sym ? 0 : addend) {}
-
-  ub32 r_offset;
-  ub24 r_sym;
-  u8 r_type;
-  ib32 r_addend;
-};
-
 //
 // Machine descriptions
 //
 
 template <typename E> concept supports_ifunc = requires { E::R_IRELATIVE; };
 template <typename E> concept supports_tlsdesc = requires { E::R_TLSDESC; };
+template <typename E> concept supports_sframe = requires { E::sframe_abi; };
 template <typename E> concept needs_thunk = requires { E::thunk_size; };
 
 template <typename E> concept is_x86_64 = std::same_as<E, X86_64>;
 template <typename E> concept is_i386 = std::same_as<E, I386>;
 template <typename E> concept is_arm64le = std::same_as<E, ARM64LE>;
 template <typename E> concept is_arm64be = std::same_as<E, ARM64BE>;
-template <typename E> concept is_arm32 = std::same_as<E, ARM32>;
+template <typename E> concept is_arm32le = std::same_as<E, ARM32LE>;
+template <typename E> concept is_arm32be = std::same_as<E, ARM32BE>;
 template <typename E> concept is_rv64le = std::same_as<E, RV64LE>;
 template <typename E> concept is_rv64be = std::same_as<E, RV64BE>;
 template <typename E> concept is_rv32le = std::same_as<E, RV32LE>;
@@ -1827,6 +1867,7 @@ template <typename E> concept is_loongarch64 = std::same_as<E, LOONGARCH64>;
 template <typename E> concept is_loongarch32 = std::same_as<E, LOONGARCH32>;
 
 template <typename E> concept is_x86 = is_x86_64<E> || is_i386<E>;
+template <typename E> concept is_arm32 = is_arm32le<E> || is_arm32be<E>;
 template <typename E> concept is_arm64 = is_arm64le<E> || is_arm64be<E>;
 template <typename E> concept is_arm = is_arm64<E> || is_arm32<E>;
 template <typename E> concept is_rv64 = is_rv64le<E> || is_rv64be<E>;
@@ -1845,10 +1886,11 @@ struct X86_64 {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_X86_64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AMD64_ENDIAN_LITTLE;
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u8 trap[] = { 0xcc }; // int3
 
   static constexpr u32 R_COPY = R_X86_64_COPY;
   static constexpr u32 R_GLOB_DAT = R_X86_64_GLOB_DAT;
@@ -1860,6 +1902,7 @@ struct X86_64 {
   static constexpr u32 R_TPOFF = R_X86_64_TPOFF64;
   static constexpr u32 R_DTPMOD = R_X86_64_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_X86_64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_X86_64_PC64;
   static constexpr u32 R_FUNCALL[] = { R_X86_64_PLT32, R_X86_64_PLTOFF64 };
 };
 
@@ -1873,7 +1916,7 @@ struct I386 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u8 trap[] = { 0xcc }; // int3
 
   static constexpr u32 R_COPY = R_386_COPY;
   static constexpr u32 R_GLOB_DAT = R_386_GLOB_DAT;
@@ -1895,12 +1938,13 @@ struct ARM64LE {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 65536;
   static constexpr u32 e_machine = EM_AARCH64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_LITTLE;
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk
+  static constexpr u32 thunk_size = 24;
+  static constexpr u8 trap[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk
 
   static constexpr u32 R_COPY = R_AARCH64_COPY;
   static constexpr u32 R_GLOB_DAT = R_AARCH64_GLOB_DAT;
@@ -1912,15 +1956,17 @@ struct ARM64LE {
   static constexpr u32 R_TPOFF = R_AARCH64_TLS_TPREL64;
   static constexpr u32 R_DTPMOD = R_AARCH64_TLS_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_AARCH64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_AARCH64_PREL64;
   static constexpr u32 R_FUNCALL[] = { R_AARCH64_JUMP26, R_AARCH64_CALL26 };
 };
 
 struct ARM64BE : ARM64LE {
   static constexpr std::string_view name = "arm64be";
   static constexpr bool is_le = false;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_BIG;
 };
 
-struct ARM32 {
+struct ARM32LE {
   static constexpr std::string_view name = "arm32";
   static constexpr bool is_64 = false;
   static constexpr bool is_le = true;
@@ -1932,7 +1978,7 @@ struct ARM32 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 16;
   static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0xff, 0xde }; // udf
+  static constexpr u8 trap[] = { 0xff, 0xde }; // udf
 
   static constexpr u32 R_COPY = R_ARM_COPY;
   static constexpr u32 R_GLOB_DAT = R_ARM_GLOB_DAT;
@@ -1950,6 +1996,11 @@ struct ARM32 {
   };
 };
 
+struct ARM32BE : ARM32LE {
+  static constexpr std::string_view name = "arm32be";
+  static constexpr bool is_le = false;
+};
+
 struct RV64LE {
   static constexpr std::string_view name = "riscv64";
   static constexpr bool is_64 = true;
@@ -1960,7 +2011,7 @@ struct RV64LE {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_64;
@@ -1990,7 +2041,7 @@ struct RV32LE {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_32;
@@ -2022,7 +2073,7 @@ struct PPC32 {
   static constexpr u32 pltgot_size = 36;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 36;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
 
   static constexpr u32 R_COPY = R_PPC_COPY;
   static constexpr u32 R_GLOB_DAT = R_PPC_GLOB_DAT;
@@ -2064,7 +2115,7 @@ struct PPC64V1 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 28;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
 };
 
 struct PPC64V2 : PPC64 {
@@ -2075,7 +2126,7 @@ struct PPC64V2 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 24;
-  static constexpr u8 filler[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
+  static constexpr u8 trap[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
 };
 
 struct S390X {
@@ -2085,10 +2136,11 @@ struct S390X {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_S390X;
+  static constexpr u8 sframe_abi = SFRAME_ABI_S390X_ENDIAN_BIG;
   static constexpr u32 plt_hdr_size = 48;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x07, 0x00 }; // nop
+  static constexpr u8 trap[] = { 0x00, 0x00 }; // invalid
 
   static constexpr u32 R_COPY = R_390_COPY;
   static constexpr u32 R_GLOB_DAT = R_390_GLOB_DAT;
@@ -2099,6 +2151,7 @@ struct S390X {
   static constexpr u32 R_DTPOFF = R_390_TLS_DTPOFF;
   static constexpr u32 R_TPOFF = R_390_TLS_TPOFF;
   static constexpr u32 R_DTPMOD = R_390_TLS_DTPMOD;
+  static constexpr u32 R_SFRAME = R_390_PC64;
   static constexpr u32 R_FUNCALL[] = { R_390_PLT32DBL };
 };
 
@@ -2112,7 +2165,7 @@ struct SPARC64 {
   static constexpr u32 plt_hdr_size = 128;
   static constexpr u32 plt_size = 32;
   static constexpr u32 pltgot_size = 32;
-  static constexpr u8 filler[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5
+  static constexpr u8 trap[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5
 
   static constexpr u32 R_COPY = R_SPARC_COPY;
   static constexpr u32 R_GLOB_DAT = R_SPARC_GLOB_DAT;
@@ -2136,7 +2189,7 @@ struct M68K {
   static constexpr u32 plt_hdr_size = 18;
   static constexpr u32 plt_size = 14;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0x4a, 0xfc }; // illegal
+  static constexpr u8 trap[] = { 0x4a, 0xfc }; // illegal
 
   static constexpr u32 R_COPY = R_68K_COPY;
   static constexpr u32 R_GLOB_DAT = R_68K_GLOB_DAT;
@@ -2157,9 +2210,9 @@ struct SH4LE {
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_SH;
   static constexpr u32 plt_hdr_size = 16;
-  static constexpr u32 plt_size = 16;
+  static constexpr u32 plt_size = 20;
   static constexpr u32 pltgot_size = 12;
-  static constexpr u8 filler[] = { 0x00, 0x90 }; // nop
+  static constexpr u8 trap[] = { 0xfd, 0xff }; // illegal
 
   static constexpr u32 R_COPY = R_SH_COPY;
   static constexpr u32 R_GLOB_DAT = R_SH_GLOB_DAT;
@@ -2175,7 +2228,7 @@ struct SH4LE {
 struct SH4BE : SH4LE {
   static constexpr std::string_view name = "sh4be";
   static constexpr bool is_le = false;
-  static constexpr u8 filler[] = { 0x90, 0x00 }; // nop
+  static constexpr u8 trap[] = { 0xff, 0xfd }; // illegal
 };
 
 struct LOONGARCH64 {
@@ -2188,7 +2241,7 @@ struct LOONGARCH64 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_64;
@@ -2213,7 +2266,7 @@ struct LOONGARCH32 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_32;

src/entry.cc

@@ -0,0 +1,43 @@
+#include "config.h"
+#include "mold.h"
+#include "mold-git-hash.h"
+
+#if MOLD_USE_MIMALLOC
+# include <mimalloc.h>
+#endif
+
+#if MOLD_USE_SYSTEM_MIMALLOC
+// Including mimalloc-new-delete.h overrides the new/delete operators.
+// We need it only when using mimalloc as a dynamic library.
+// This header should be included in only one source file, so we do
+// it in this file.
+# include <mimalloc-new-delete.h>
+#endif
+
+#ifdef __linux__
+# include <sys/prctl.h>
+#endif
+
+std::string mold::mold_version =
+#ifdef MOLD_GIT_HASH
+  "mold " MOLD_VERSION " (" MOLD_GIT_HASH "; compatible with GNU ld)";
+#else
+  "mold " MOLD_VERSION " (compatible with GNU ld)";
+#endif
+
+int main(int argc, char **argv) {
+#if MOLD_USE_MIMALLOC
+  // Silence mimalloc warnings that users can ignore
+  mi_option_disable(mi_option_verbose);
+  mi_option_disable(mi_option_show_errors);
+#endif
+
+#if defined(__linux__) && defined(PR_SET_THP_DISABLE)
+  // A parent process may have disabled transparent huge pages, and the
+  // flag is inherited. Huge pages make mold considerably faster on
+  // large links, so re-enable them for this process.
+  prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
+#endif
+
+  return mold::mold_main<mold::MOLD_FIRST_TARGET>(argc, argv);
+}

src/error.cc

@@ -0,0 +1,55 @@
+#include "mold.h"
+
+namespace mold {
+
+static std::string_view fatal_mono = "mold: fatal: ";
+static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
+static std::string_view error_mono = "mold: error: ";
+static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
+static std::string_view warning_mono = "mold: warning: ";
+static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
+
+template <typename E>
+Fatal<E>::Fatal(Context<E> &ctx) {
+  out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
+}
+
+template <typename E>
+[[noreturn]] Fatal<E>::~Fatal() {
+  out.emit();
+  cleanup();
+  _exit(1);
+}
+
+template <typename E>
+Error<E>::Error(Context<E> &ctx) {
+  if (ctx.arg.noinhibit_exec) {
+    out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  } else {
+    out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  }
+}
+
+template <typename E>
+Warn<E>::Warn(Context<E> &ctx) {
+  if (ctx.arg.suppress_warnings)
+    return;
+
+  out.emplace(std::cerr);
+
+  if (ctx.arg.fatal_warnings) {
+    *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  } else {
+    *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  }
+}
+
+using E = MOLD_TARGET;
+
+template class Fatal<E>;
+template class Error<E>;
+template class Warn<E>;
+
+} // namespace mold

src/filetype.cc

@@ -1,5 +1,4 @@
 #include "mold.h"
-#include "../lib/archive-file.h"
 
 namespace mold {
 
@@ -14,11 +13,16 @@ static bool is_text_file(MappedFile *mf) {
 }
 
 template <typename E>
-static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
+static bool is_gcc_lto_obj(MappedFile *mf, bool has_gcc_plugin) {
   const char *data = mf->get_contents().data();
   ElfEhdr<E> &ehdr = *(ElfEhdr<E> *)data;
   ElfShdr<E> *sh_begin = (ElfShdr<E> *)(data + ehdr.e_shoff);
-  std::span<ElfShdr<E>> shdrs{(ElfShdr<E> *)(data + ehdr.e_shoff), ehdr.e_shnum};
+
+  // e_shnum is a 16-bit field. If an object file contains more than 65279
+  // sections, e_shnum is zero and the actual number is stored to the first
+  // section header's sh_size field.
+  i64 num_sections = (ehdr.e_shnum == 0) ? sh_begin->sh_size : ehdr.e_shnum;
+  std::span<ElfShdr<E>> shdrs{sh_begin, (size_t)num_sections};
 
   // e_shstrndx is a 16-bit field. If .shstrtab's section index is
   // too large, the actual number is stored to sh_link field.
@@ -31,7 +35,7 @@ static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
     // the LTO linker plugin is available and falls back as regular
     // objects otherwise. GCC FAT LTO object can be identified by the
     // presence of `.gcc.lto_.symtab` section.
-    if (has_plugin) {
+    if (has_gcc_plugin) {
       std::string_view name = data + shdrs[shstrtab_idx].sh_offset + sec.sh_name;
       if (name.starts_with(".gnu.lto_.symtab."))
         return true;
@@ -69,11 +73,15 @@ static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
 template <typename E>
 FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
   std::string_view data = mf->get_contents();
-  bool has_plugin = !ctx.arg.plugin.empty();
-
   if (data.empty())
     return FileType::EMPTY;
 
+  // GCC FAT LTO objects can be linked as regular ELF objects. If the active
+  // plugin is LLVM's, treat them as regular objects so that we can fall back
+  // to native code instead of routing them through GCC LTO handling.
+  bool has_gcc_plugin = !ctx.arg.plugin.empty() &&
+                        ctx.arg.plugin.find("LLVMgold.") == std::string_view::npos;
+
   if (data.starts_with("\177ELF")) {
     u8 byte_order = ((ElfEhdr<I386> *)data.data())->e_ident[EI_DATA];
 
@@ -82,10 +90,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
 
       if (ehdr.e_type == ET_REL) {
         if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<I386>(mf, has_plugin))
+          if (is_gcc_lto_obj<I386>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         } else {
-          if (is_gcc_lto_obj<X86_64>(mf, has_plugin))
+          if (is_gcc_lto_obj<X86_64>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         }
         return FileType::ELF_OBJ;
@@ -98,10 +106,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
 
       if (ehdr.e_type == ET_REL) {
         if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<M68K>(mf, has_plugin))
+          if (is_gcc_lto_obj<M68K>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         } else {
-          if (is_gcc_lto_obj<SPARC64>(mf, has_plugin))
+          if (is_gcc_lto_obj<SPARC64>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         }
         return FileType::ELF_OBJ;
@@ -130,16 +138,16 @@ static std::string_view get_elf_type(u8 *buf) {
   bool is_le = (buf[EI_DATA] == ELFDATA2LSB);
   bool is_64 = (buf[EI_CLASS] == ELFCLASS64);
 
-  auto *ehdr_le = (ElfEhdr<I386> *)buf;
-  auto *ehdr_be = (ElfEhdr<M68K> *)buf;
+  auto *ehdr64_le = (ElfEhdr<ARM64LE> *)buf;
+  auto *ehdr64_be = (ElfEhdr<ARM64BE> *)buf;
 
-  switch (is_le ? ehdr_le->e_machine : ehdr_be->e_machine) {
+  switch (is_le ? ehdr64_le->e_machine : ehdr64_be->e_machine) {
   case EM_386:
     return I386::name;
   case EM_X86_64:
     return X86_64::name;
   case EM_ARM:
-    return ARM32::name;
+    return is_le ? ARM32LE::name : ARM32BE::name;
   case EM_AARCH64:
     return is_le ? ARM64LE::name : ARM64BE::name;
   case EM_RISCV:
@@ -149,7 +157,18 @@ static std::string_view get_elf_type(u8 *buf) {
   case EM_PPC:
     return PPC32::name;
   case EM_PPC64:
-    return is_le ? PPC64V2::name : PPC64V1::name;
+    // ELFv1 is big-endian and ELFv2 is little-endian by convention, but
+    // the correspondence is not a rule; musl for example uses ELFv2 on
+    // big-endian too. We support only the usual combinations, so treat
+    // the others as unrecognizable rather than silently linking them
+    // against the wrong ABI.
+    if (!is_le)
+      if (u32 abi = (ehdr64_be->e_flags & EF_PPC64_ABI); abi == 0 || abi == 1)
+        return PPC64V1::name;
+    if (is_le)
+      if (u32 abi = (ehdr64_le->e_flags & EF_PPC64_ABI); abi == 0 || abi == 2)
+        return PPC64V2::name;
+    return "";
   case EM_S390X:
     return S390X::name;
   case EM_SPARC64:

src/gc-sections.cc

@@ -4,6 +4,8 @@
 
 #include "mold.h"
 
+#include <fstream>
+#include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/parallel_for_each.h>
 
@@ -27,24 +29,46 @@ static bool should_keep(const InputSection<E> &isec) {
          name.starts_with(".ctors") ||
          name.starts_with(".dtors") ||
          name.starts_with(".init") ||
-         name.starts_with(".fini") ||
-         is_c_identifier(name);
+         name.starts_with(".fini");
+}
+
+// Sections whose names are valid C identifiers can be referenced via
+// __start_<name>/__stop_<name> symbols, which the linker synthesizes.
+// Such sections must be kept alive only if such a marker symbol is
+// referenced from a live section. This map lets us find all sections
+// of a given name when we encounter such a reference during marking.
+template <typename E>
+using StartStopMap =
+  tbb::concurrent_unordered_map<std::string_view,
+                                tbb::concurrent_vector<InputSection<E> *>>;
+
+template <typename E>
+static StartStopMap<E> build_start_stop_map(Context<E> &ctx) {
+  StartStopMap<E> map;
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    for (InputSection<E> *isec : file->sections)
+      if (isec && isec->is_alive() && (isec->shdr().sh_flags & SHF_ALLOC))
+        if (std::string_view name = isec->name();
+            is_c_identifier(name))
+          map[name].push_back(isec);
+  });
+  return map;
 }
 
 template <typename E>
 static bool mark_section(InputSection<E> *isec) {
-  return isec && isec->is_alive && !isec->is_visited.test_and_set();
+  return isec && isec->is_alive() && isec->visit();
 }
 
 template <typename E>
-static tbb::concurrent_vector<InputSection<E> *>
+static std::vector<InputSection<E> *>
 collect_root_set(Context<E> &ctx) {
   Timer t(ctx, "collect_root_set");
-  tbb::concurrent_vector<InputSection<E> *> rootset;
+  tbb::enumerable_thread_specific<std::vector<InputSection<E> *>> rootset;
 
   auto enqueue_section = [&](InputSection<E> *isec) {
     if (mark_section(isec))
-      rootset.push_back(isec);
+      rootset.local().push_back(isec);
   };
 
   auto enqueue_symbol = [&](Symbol<E> *sym) {
@@ -58,20 +82,21 @@ collect_root_set(Context<E> &ctx) {
 
   // Add sections that are not subject to garbage collection.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (!isec || !isec->is_alive)
+    for (InputSection<E> *isec : file->sections) {
+      if (!isec || !isec->is_alive())
         continue;
 
       // --gc-sections discards only SHF_ALLOC sections. If you want to
       // reduce the amount of non-memory-mapped segments, you should
       // use `strip` command, compile without debug info or use
       // --strip-all linker option.
-      u32 flags = isec->shdr().sh_flags;
-      if (!(flags & SHF_ALLOC))
-        isec->is_visited = true;
+      if (!(isec->shdr().sh_flags & SHF_ALLOC)) {
+        isec->set_visited();
+        continue;
+      }
 
       if (should_keep(*isec))
-        enqueue_section(isec.get());
+        enqueue_section(isec);
     }
   });
 
@@ -91,20 +116,32 @@ collect_root_set(Context<E> &ctx) {
         enqueue_symbol(file->symbols[rel.r_sym]);
   });
 
-  return rootset;
+  std::vector<std::vector<InputSection<E> *>> vec;
+  for (std::vector<InputSection<E> *> &v : rootset)
+    vec.push_back(std::move(v));
+  return flatten(vec);
+}
+
+static std::string_view start_stop_name(std::string_view sym) {
+  if (sym.starts_with("__start_"))
+    return sym.substr(8);
+  if (sym.starts_with("__stop_"))
+    return sym.substr(7);
+  return "";
 }
 
 template <typename E>
-static void visit(Context<E> &ctx, InputSection<E> *isec,
-                  tbb::feeder<InputSection<E> *> &feeder, i64 depth) {
-  assert(isec->is_visited);
+static void visit_section(Context<E> &ctx, InputSection<E> *isec,
+                          tbb::feeder<InputSection<E> *> &feeder, i64 depth,
+                          const StartStopMap<E> &start_stop_map) {
+  assert(isec->is_visited());
 
   // Mark a section alive. For better performacne, we don't call
   // `feeder.add` too often.
   auto mark = [&](InputSection<E> *sec) {
     if (mark_section(sec)) {
       if (depth < 3)
-        visit(ctx, sec, feeder, depth + 1);
+        visit_section(ctx, sec, feeder, depth + 1, start_stop_map);
       else
         feeder.add(sec);
     }
@@ -114,17 +151,45 @@ static void visit(Context<E> &ctx, InputSection<E> *isec,
   // describing how to handle exceptions for that function.
   // We want to keep associated .eh_frame records.
   for (FdeRecord<E> &fde : isec->get_fdes())
-    for (const ElfRel<E> &rel : fde.get_rels(isec->file).subspan(1))
-      if (Symbol<E> *sym = isec->file.symbols[rel.r_sym])
+    for (const ElfRel<E> &rel : fde.get_rels(*isec->file).subspan(1))
+      if (Symbol<E> *sym = isec->file->symbols[rel.r_sym])
         mark(sym->get_input_section());
 
   for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
     // Symbol can refer to either a section fragment or an input section.
-    Symbol<E> &sym = *isec->file.symbols[rel.r_sym];
-    if (SectionFragment<E> *frag = sym.get_frag())
+    Symbol<E> &sym = *isec->file->symbols[rel.r_sym];
+
+    if (sym.file && sym.file->is_dso) {
+      sym.file->is_reachable.test_and_set();
+      continue;
+    }
+
+    if (SectionFragment<E> *frag = sym.get_frag()) {
       frag->is_alive = true;
-    else
-      mark(sym.get_input_section());
+      continue;
+    }
+
+    mark(sym.get_input_section());
+
+    // A reference to __start_<name> or __stop_<name> keeps every
+    // section named <name> alive, mirroring how those symbols are
+    // defined. A single such reference can keep an enormous number of
+    // sections alive, so we spread the fanout over threads instead
+    // of marking the sections one by one.
+    if (std::string_view name = start_stop_name(sym.name());
+        !name.empty()) {
+      if (auto it = start_stop_map.find(name);
+          it != start_stop_map.end()) {
+        // Mark and visit the sections with a nested parallel loop.
+        // As in mark() below, a section added to a feeder has
+        // already been marked, so a feeder's loop body must visit
+        // it unconditionally.
+        tbb::parallel_for_each(it->second, [&](InputSection<E> *isec) {
+          if (mark_section(isec))
+            visit_section(ctx, isec, feeder, 0, start_stop_map);
+        });
+      }
+    }
   }
 
   if constexpr (is_arm32<E>)
@@ -134,12 +199,13 @@ static void visit(Context<E> &ctx, InputSection<E> *isec,
 // Mark all reachable sections
 template <typename E>
 static void mark(Context<E> &ctx,
-                 tbb::concurrent_vector<InputSection<E> *> &rootset) {
+                 std::vector<InputSection<E> *> &rootset,
+                 const StartStopMap<E> &start_stop_map) {
   Timer t(ctx, "mark");
 
   tbb::parallel_for_each(rootset, [&](InputSection<E> *isec,
                                       tbb::feeder<InputSection<E> *> &feeder) {
-    visit(ctx, isec, feeder, 0);
+    visit_section(ctx, isec, feeder, 0, start_stop_map);
   });
 }
 
@@ -147,26 +213,65 @@ static void mark(Context<E> &ctx,
 template <typename E>
 static void sweep(Context<E> &ctx) {
   Timer t(ctx, "sweep");
-  static Counter counter("garbage_sections");
 
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (isec && isec->is_alive && !isec->is_visited) {
-        if (ctx.arg.print_gc_sections)
-          Out(ctx) << "removing unused section " << *isec;
+  std::vector<std::vector<InputSection<E> *>> sections(ctx.objs.size());
+
+  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
+    ObjectFile<E> &file = *ctx.objs[i];
+
+    for (InputSection<E> *isec : file.sections) {
+      if (isec && isec->is_alive() && !isec->is_visited()) {
         isec->kill();
-        counter++;
+        sections[i].push_back(isec);
       }
     }
   });
+
+  std::string &path = ctx.arg.print_gc_sections;
+
+  if (!path.empty()) {
+    std::ostream *out = &std::cout;
+    std::ofstream file;
+
+    if (path != "-") {
+      file.open(path);
+      if (file.fail())
+        Fatal(ctx) << "--print-gc-sections: cannot open " << path << ": "
+                   << errno_string();
+      out = &file;
+    }
+
+    i64 saved_bytes = 0;
+    for (std::span<InputSection<E> *> vec : sections)
+      for (InputSection<E> *isec : vec)
+        *out << "removing unused section " << *isec << '\n';
+
+    *out << "GC saved " << saved_bytes << " bytes\n";
+  }
 }
 
 template <typename E>
 void gc_sections(Context<E> &ctx) {
   Timer t(ctx, "gc");
-  tbb::concurrent_vector<InputSection<E> *> rootset = collect_root_set(ctx);
-  mark(ctx, rootset);
+
+  for (SharedFile<E> *file : ctx.dsos)
+    if (file->as_needed)
+      file->is_reachable = false;
+
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
+  std::vector<InputSection<E> *> rootset = collect_root_set(ctx);
+  StartStopMap<E> start_stop_map = build_start_stop_map(ctx);
+  mark(ctx, rootset, start_stop_map);
   sweep(ctx);
+
+  std::erase_if(ctx.dsos, [](SharedFile<E> *file) { return !file->is_reachable; });
 }
 
 using E = MOLD_TARGET;

src/icf.cc

@@ -67,8 +67,7 @@
 #include "mold.h"
 #include "../lib/siphash.h"
 
-#include <array>
-#include <cstdio>
+#include <fstream>
 #include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
@@ -76,24 +75,14 @@
 #include <tbb/parallel_for_each.h>
 #include <tbb/parallel_sort.h>
 
-static constexpr int64_t HASH_SIZE = 16;
-
-using Digest = std::array<uint8_t, HASH_SIZE>;
-
-namespace std {
-template <> struct hash<Digest> {
-  size_t operator()(const Digest &k) const {
-    static_assert(sizeof(size_t) <= HASH_SIZE);
-    size_t val;
-    memcpy(&val, k.data(), sizeof(size_t));
-    return val;
-  }
-};
-}
-
 namespace mold {
 
-static u8 hmac_key[16];
+struct Digest {
+  u64 hi;
+  u64 lo;
+};
+
+static u8 siphash_key[16];
 
 template <typename E>
 static void uniquify_cies(Context<E> &ctx) {
@@ -129,112 +118,25 @@ static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
     return false;
 
   if (shdr.sh_flags & SHF_EXECINSTR)
-    return (ctx.arg.icf_all || !isec.address_taken) &&
+    return (ctx.arg.icf_all || !isec.is_address_taken()) &&
            name != ".init" && name != ".fini";
 
+  // .gcc_except_table contains a compiler-generated table. Pointer
+  // equality for the section is not significant because only the C++
+  // exception handling code will use the table at runtime.
+  if (name == ".gcc_except_table" ||
+      name.starts_with(".gcc_except_table."))
+    return true;
+
   bool is_readonly = !(shdr.sh_flags & SHF_WRITE);
-  bool is_relro = isec.name().starts_with(".data.rel.ro");
-  return (ctx.arg.ignore_data_address_equality || !isec.address_taken) &&
+  bool is_relro = name.starts_with(".data.rel.ro");
+  return (ctx.arg.ignore_data_address_equality || !isec.is_address_taken()) &&
          (is_readonly || is_relro);
 }
 
-template <typename E>
-static bool is_leaf(Context<E> &ctx, InputSection<E> &isec) {
-  if (!isec.get_rels(ctx).empty())
-    return false;
-
-  for (FdeRecord<E> &fde : isec.get_fdes())
-    if (fde.get_rels(isec.file).size() > 1)
-      return false;
-
-  return true;
-}
-
-template <typename E>
-struct LeafHasher {
-  size_t operator()(InputSection<E> *isec) const {
-    u64 h = hash_string(isec->contents);
-    for (FdeRecord<E> &fde : isec->get_fdes()) {
-      u64 h2 = hash_string(fde.get_contents(isec->file).substr(8));
-      h = combine_hash(h, h2);
-    }
-    return h;
-  }
-};
-
-template <typename E>
-struct LeafEq {
-  bool operator()(InputSection<E> *a, InputSection<E> *b) const {
-    if (a->contents != b->contents)
-      return false;
-
-    std::span<FdeRecord<E>> x = a->get_fdes();
-    std::span<FdeRecord<E>> y = b->get_fdes();
-
-    if (x.size() != y.size())
-      return false;
-
-    for (i64 i = 0; i < x.size(); i++)
-      if (x[i].get_contents(a->file).substr(8) !=
-          y[i].get_contents(b->file).substr(8))
-        return false;
-    return true;
-  }
-};
-
-// Early merge of leaf nodes, which can be processed without constructing the
-// entire graph. This reduces the vertex count and improves memory efficiency.
-template <typename E>
-static void merge_leaf_nodes(Context<E> &ctx) {
-  Timer t(ctx, "merge_leaf_nodes");
-
-  static Counter eligible("icf_eligibles");
-  static Counter non_eligible("icf_non_eligibles");
-  static Counter leaf("icf_leaf_nodes");
-
-  tbb::concurrent_unordered_map<InputSection<E> *, Atomic<InputSection<E> *>,
-                                LeafHasher<E>, LeafEq<E>> map;
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (!isec || !isec->is_alive)
-        continue;
-
-      if (!is_eligible(ctx, *isec)) {
-        non_eligible++;
-        continue;
-      }
-
-      if (is_leaf(ctx, *isec)) {
-        leaf++;
-        isec->icf_leaf = true;
-        auto [it, inserted] = map.insert({isec.get(), isec.get()});
-        if (!inserted) {
-          InputSection<E> *isec2 = it->second.load();
-          while (isec->get_priority() < isec2->get_priority() &&
-                 !it->second.compare_exchange_strong(isec2, isec.get()));
-        }
-      } else {
-        eligible++;
-        isec->icf_eligible = true;
-      }
-    }
-  });
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (isec && isec->is_alive && isec->icf_leaf) {
-        auto it = map.find(isec.get());
-        assert(it != map.end());
-        isec->leader = it->second;
-      }
-    }
-  });
-}
-
 template <typename E>
 static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
-  SipHash13_128 hasher(hmac_key);
+  SipHash13_128 hasher(siphash_key);
 
   auto hash = [&](auto val) {
     hasher.update((u8 *)&val, sizeof(val));
@@ -248,7 +150,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
   auto hash_symbol = [&](Symbol<E> &sym) {
     InputSection<E> *isec = sym.get_input_section();
 
-    if (!sym.file) {
+    if (!sym.file || sym.is_imported) {
       hash('1');
       hash((u64)&sym);
     } else if (SectionFragment<E> *frag = sym.get_frag()) {
@@ -256,83 +158,207 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
       hash((u64)frag);
     } else if (!isec) {
       hash('3');
-    } else if (isec->leader) {
+    } else if (isec->icf_idx != -1) {
       hash('4');
-      hash((u64)isec->leader);
-    } else if (isec->icf_eligible) {
-      hash('5');
     } else {
-      hash('6');
+      hash('5');
       hash((u64)isec);
     }
     hash(sym.value);
   };
 
-  hash_string(isec.contents);
+  hash_string(isec.get_contents());
   hash(isec.shdr().sh_flags);
-  hash(isec.get_fdes().size());
+  std::span<FdeRecord<E>> fdes = isec.get_fdes();
+  hash(fdes.size());
   hash(isec.get_rels(ctx).size());
 
-  for (FdeRecord<E> &fde : isec.get_fdes()) {
-    hash(isec.file.cies[fde.cie_idx].icf_idx);
+  for (FdeRecord<E> &fde : fdes) {
+    hash(isec.file->cies[fde.cie_idx].icf_idx);
 
     // Bytes 0 to 4 contain the length of this record, and
     // bytes 4 to 8 contain an offset to CIE.
-    hash_string(fde.get_contents(isec.file).substr(8));
+    hash_string(fde.get_contents(*isec.file).substr(8));
 
-    hash(fde.get_rels(isec.file).size());
+    hash(fde.get_rels(*isec.file).size());
 
-    for (const ElfRel<E> &rel : fde.get_rels(isec.file).subspan(1)) {
-      hash_symbol(*isec.file.symbols[rel.r_sym]);
+    for (const ElfRel<E> &rel : fde.get_rels(*isec.file).subspan(1)) {
+      hash_symbol(*isec.file->symbols[rel.r_sym]);
       hash(rel.r_type);
       hash(rel.r_offset - fde.input_offset);
-      hash(get_addend(isec.file.cies[fde.cie_idx].input_section, rel));
+      hash(get_addend(isec.file->cies[fde.cie_idx].input_section, rel));
     }
   }
 
-  for (i64 i = 0; i < isec.get_rels(ctx).size(); i++) {
-    const ElfRel<E> &rel = isec.get_rels(ctx)[i];
+  for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
     hash(rel.r_offset);
     hash(rel.r_type);
     hash(get_addend(isec, rel));
-    hash_symbol(*isec.file.symbols[rel.r_sym]);
+    hash_symbol(*isec.file->symbols[rel.r_sym]);
   }
 
   Digest digest;
-  hasher.finish(digest.data());
+  hasher.finish(&digest);
   return digest;
 }
 
+// A concurrent hash map from digest to section. We use it to count the
+// number of distinct digests and to elect the leader section of each
+// digest's equivalence class.
+//
+// The number of distinct digests cannot exceed the number of digests,
+// so we can allocate a large enough table upfront and never need to
+// grow it.
+//
+// The map is reused across propagation rounds. Instead of clearing the
+// table at the start of each round, we stamp each slot with the round
+// number in which it was written; a slot stamped with an earlier round
+// is treated as vacant.
+//
+// Each slot consists of two 64-bit words and a section pointer. The
+// first word packs the round number, a busy bit, and 48 bits of the
+// digest; the second word holds another 64 bits. An inserter claims a
+// vacant slot by installing the first word with compare-and-swap with
+// the busy bit set, writes the second word and the pointer, and then
+// rewrites the first word with the busy bit cleared to publish the
+// slot. Since the slot index is derived from digest bits that the
+// first word doesn't contain, a successful match effectively compares
+// an entire 128-bit digest, so the map is exact under the same
+// hash-collision assumption the surrounding algorithm is built on.
+//
+// Of all sections inserted with the same digest in the same round, the
+// slot ends up pointing to the one with the lowest priority, which ICF
+// uses as the leader of the digest's equivalence class.
+template <typename E>
+class DigestMap {
+public:
+  DigestMap(i64 n) :
+    mask(bit_ceil(n * 2) - 1),
+    slots(mask + 1) {}
+
+  void next_round() {
+    if (++round == 1 << 15) {
+      // The round number wrapped around, making stale slot stamps
+      // ambiguous, so reset the table. In practice, ICF converges long
+      // before this point.
+      for (Slot &slot : slots)
+        slot.hi = 0;
+      round = 1;
+    }
+  }
+
+  // Returns true if the digest was not in the table.
+  bool insert(const Digest &digest, InputSection<E> *isec) {
+    constexpr u64 busy_bit = 1LL << 48;
+    u64 tag = digest.hi >> 16;
+    u64 value = (round << 49) | tag;
+
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask) {
+      Slot &slot = slots[i];
+      u64 x = slot.hi.load(std::memory_order_acquire);
+
+      // If the slot was last written in an earlier round, it's vacant;
+      // try to claim it.
+      while ((x >> 49) != round) {
+        if (slot.hi.compare_exchange_weak(x, value | busy_bit,
+                                          std::memory_order_acquire)) {
+          slot.lo = digest.lo;
+          slot.leader = isec;
+          slot.hi.store(value, std::memory_order_release);
+          return true;
+        }
+      }
+
+      // The slot is occupied. If it holds a different digest, try the
+      // next slot.
+      if ((x & 0xffff'ffff'ffff) != tag)
+        continue;
+
+      // The tags match; compare the digest bits in the second word,
+      // waiting for the writer to publish them if the slot is still
+      // being claimed.
+      while (x & busy_bit)
+        x = slot.hi.load(std::memory_order_acquire);
+      if (slot.lo != digest.lo)
+        continue;
+
+      // The digest is already in the table; keep the slot pointing to
+      // the lowest-priority section.
+      InputSection<E> *cur = slot.leader;
+      while (isec->get_priority() < cur->get_priority() &&
+             !slot.leader.compare_exchange_strong(cur, isec));
+      return false;
+    }
+  }
+
+  // Returns the section associated with a digest. The digest must have
+  // been inserted in the current round, and no insertion may be running
+  // concurrently.
+  InputSection<E> *find(const Digest &digest) {
+    u64 value = (round << 49) | (digest.hi >> 16);
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask)
+      if (slots[i].hi == value && slots[i].lo == digest.lo)
+        return slots[i].leader;
+  }
+
+private:
+  struct Slot {
+    Atomic<u64> hi;
+    Atomic<u64> lo;
+    Atomic<InputSection<E> *> leader;
+  };
+
+  // Time begins in round 1 so that all-zero slots, the initial state
+  // of the table, read as vacant.
+  u64 round = 1;
+  u64 mask;
+  std::vector<Slot> slots;
+};
+
 template <typename E>
 static std::vector<InputSection<E> *> gather_sections(Context<E> &ctx) {
   Timer t(ctx, "gather_sections");
 
-  // Count the number of input sections for each input file.
-  std::vector<i64> num_sections(ctx.objs.size());
+  static Counter eligible("icf_eligibles");
+  static Counter non_eligible("icf_non_eligibles");
+
+  // Count the number of eligible input sections for each input file
+  // and turn the counts into starting indices with a prefix sum.
+  std::vector<i64> indices(ctx.objs.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        num_sections[i]++;
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (!isec)
+        continue;
+
+      assert(isec->icf_idx == -1);
+      if (!isec->is_alive())
+        continue;
+
+      if (is_eligible(ctx, *isec)) {
+        eligible++;
+        isec->icf_idx = 0;
+        indices[i + 1]++;
+      } else {
+        non_eligible++;
+      }
+    }
   });
 
-  std::vector<i64> section_indices(ctx.objs.size());
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
-    section_indices[i + 1] = section_indices[i] + num_sections[i];
+  for (i64 i = 1; i < indices.size(); i++)
+    indices[i] += indices[i - 1];
 
-  std::vector<InputSection<E> *> sections(
-    section_indices.back() + num_sections.back());
+  std::vector<InputSection<E> *> sections(indices.back());
 
   // Fill `sections` contents.
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    i64 idx = section_indices[i];
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        sections[idx++] = isec.get();
-  });
-
-  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-    sections[i]->icf_idx = i;
+    i64 idx = indices[i];
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (isec && isec->icf_idx != -1) {
+        isec->icf_idx = idx;
+        sections[idx++] = isec;
+      }
+    }
   });
 
   return sections;
@@ -353,6 +379,13 @@ compute_digests(Context<E> &ctx, std::span<InputSection<E> *> sections) {
 // Build a graph, treating every function as a vertex and every function call
 // as an edge. See the description at the top for a more detailed formulation.
 // We use u32 indices here to improve cache locality.
+//
+// Relocations in a section's FDEs are edges too, because compute_digest
+// hashes eligible relocation targets without identity, and every such
+// target must be represented as an edge to remain distinguishable. In
+// particular, an FDE's reference to an LSDA is an edge; without it, two
+// identical functions whose exception tables catch different types would
+// be folded into one.
 template <typename E>
 static void gather_edges(Context<E> &ctx,
                          std::span<InputSection<E> *> sections,
@@ -360,136 +393,134 @@ static void gather_edges(Context<E> &ctx,
                          std::vector<u32> &edge_indices) {
   Timer t(ctx, "gather_edges");
 
-  if (sections.empty())
-    return;
-
-  std::vector<i64> num_edges(sections.size());
-  edge_indices.resize(sections.size());
+  // Count the number of outgoing edges for each vertex and turn the
+  // counts into starting indices with a prefix sum. The extra entry at
+  // the end makes edge_indices[i + 1] valid for every vertex, so that
+  // vertex i's edges are edge_indices[i] to edge_indices[i + 1].
+  edge_indices.resize(sections.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
-    assert(isec.icf_eligible);
+    assert(isec.icf_idx != -1);
 
-    for (i64 j = 0; j < isec.get_rels(ctx).size(); j++) {
-      const ElfRel<E> &rel = isec.get_rels(ctx)[j];
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (!sym.get_frag())
-        if (InputSection<E> *isec = sym.get_input_section())
-          if (isec->icf_eligible)
-            num_edges[i]++;
-    }
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(*isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
+          if (isec->icf_idx != -1)
+            edge_indices[i + 1]++;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
+        if (isec->icf_idx != -1)
+          edge_indices[i + 1]++;
   });
 
-  for (i64 i = 0; i < num_edges.size() - 1; i++)
-    edge_indices[i + 1] = edge_indices[i] + num_edges[i];
+  for (i64 i = 1; i < edge_indices.size(); i++)
+    edge_indices[i] += edge_indices[i - 1];
 
-  edges.resize(edge_indices.back() + num_edges.back());
+  edges.resize(edge_indices.back());
 
-  tbb::parallel_for((i64)0, (i64)num_edges.size(), [&](i64 i) {
+  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
     i64 idx = edge_indices[i];
 
-    for (ElfRel<E> &rel : isec.get_rels(ctx)) {
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (InputSection<E> *isec = sym.get_input_section())
-        if (isec->icf_eligible)
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(*isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
+          if (isec->icf_idx != -1)
+            edges[idx++] = isec->icf_idx;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
+        if (isec->icf_idx != -1)
           edges[idx++] = isec->icf_idx;
-  }
   });
 }
 
-template <typename E>
-static i64 propagate(std::span<std::vector<Digest>> digests,
-                     std::span<u32> edges, std::span<u32> edge_indices,
-                     bool &slot, std::span<u8> converged,
-                     tbb::affinity_partitioner &ap) {
-  static Counter round("icf_round");
-  round++;
-
-  i64 num_digests = digests[0].size();
-  tbb::enumerable_thread_specific<i64> changed;
-
-  tbb::parallel_for((i64)0, num_digests, [&](i64 i) {
-    if (converged[i])
-      return;
-
-    SipHash13_128 hasher(hmac_key);
-    hasher.update(digests[2][i].data(), HASH_SIZE);
+// Compute the next-round digest of each vertex by hashing its current
+// digest and the current digests of the vertices it refers to. A
+// vertex's digest after the nth round is therefore a hash of its
+// unfolding into a tree of depth n.
+static void propagate(std::vector<Digest> &cur, std::vector<Digest> &next,
+                      std::span<u32> edges, std::span<u32> edge_indices) {
+  tbb::parallel_for((i64)0, (i64)cur.size(), [&](i64 i) {
+    SipHash13_128 hasher(siphash_key);
+    hasher.update(&cur[i], sizeof(Digest));
 
     i64 begin = edge_indices[i];
-    i64 end = (i + 1 == num_digests) ? edges.size() : edge_indices[i + 1];
-
+    i64 end = edge_indices[i + 1];
     for (i64 j : edges.subspan(begin, end - begin))
-      hasher.update(digests[slot][j].data(), HASH_SIZE);
-
-    hasher.finish(digests[!slot][i].data());
-
-    if (digests[slot][i] == digests[!slot][i]) {
-      // This node has converged. Skip further iterations as it will
-      // yield the same hash.
-      converged[i] = true;
-    } else {
-      changed.local()++;
-    }
-  }, ap);
-
-  slot = !slot;
-  return changed.combine(std::plus());
+      hasher.update(&cur[j], sizeof(Digest));
+    hasher.finish(&next[i]);
+  });
+  std::swap(cur, next);
 }
 
 template <typename E>
 static i64 count_num_classes(std::span<Digest> digests,
-                             tbb::affinity_partitioner &ap) {
-  std::vector<Digest> vec(digests.begin(), digests.end());
-  tbb::parallel_sort(vec);
+                             std::span<InputSection<E> *> sections,
+                             DigestMap<E> &map) {
+  map.next_round();
 
   tbb::enumerable_thread_specific<i64> num_classes;
-  tbb::parallel_for((i64)0, (i64)vec.size() - 1, [&](i64 i) {
-    if (vec[i] != vec[i + 1])
+  tbb::parallel_for((i64)0, (i64)digests.size(), [&](i64 i) {
+    if (map.insert(digests[i], sections[i]))
       num_classes.local()++;
-  }, ap);
+  });
+
+  static Counter counter("icf_round");
+  counter++;
   return num_classes.combine(std::plus());
 }
 
 template <typename E>
-static void print_icf_sections(Context<E> &ctx) {
-  tbb::concurrent_vector<InputSection<E> *> leaders;
+static void
+print_icf_sections(Context<E> &ctx, std::span<InputSection<E> *> sections) {
+  tbb::concurrent_vector<InputSection<E> *> leader_sections;
   tbb::concurrent_unordered_multimap<InputSection<E> *, InputSection<E> *> map;
 
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (isec && isec->is_alive && isec->leader) {
-        if (isec.get() == isec->leader)
-          leaders.push_back(isec.get());
-        else
-          map.insert({isec->leader, isec.get()});
-      }
-    }
+  tbb::parallel_for_each(sections, [&](InputSection<E> *isec) {
+    if (isec == isec->icf_leader)
+      leader_sections.push_back(isec);
+    else
+      map.insert({isec->icf_leader, isec});
   });
 
-  tbb::parallel_sort(leaders.begin(), leaders.end(),
+  tbb::parallel_sort(leader_sections.begin(), leader_sections.end(),
                      [](InputSection<E> *a, InputSection<E> *b) {
                        return a->get_priority() < b->get_priority();
                      });
 
-  i64 saved_bytes = 0;
+  std::ostream *out = &std::cout;
+  std::ofstream file;
 
-  for (InputSection<E> *leader : leaders) {
-    auto [begin, end] = map.equal_range(leader);
-    if (begin == end)
-      continue;
-
-    Out(ctx) << "selected section " << *leader;
-
-    i64 n = 0;
-    for (auto it = begin; it != end; it++) {
-      Out(ctx) << "  removing identical section " << *it->second;
-      n++;
-    }
-    saved_bytes += leader->contents.size() * n;
+  std::string &path = ctx.arg.print_icf_sections;
+  if (path != "-") {
+    file.open(path);
+    if (file.fail())
+      Fatal(ctx) << "--print-icf-sections: cannot open " << path << ": "
+                 << errno_string();
+    out = &file;
   }
 
-  Out(ctx) << "ICF saved " << saved_bytes << " bytes";
+  i64 saved_bytes = 0;
+
+  for (InputSection<E> *leader : leader_sections) {
+    auto [begin, end] = map.equal_range(leader);
+    if (begin != end) {
+      *out << "selected section " << *leader << '\n';
+      for (auto it = begin; it != end; it++) {
+        *out << "  removing identical section " << *it->second << '\n';
+        saved_bytes += leader->get_contents().size();
+      }
+    }
+  }
+
+  *out << "ICF saved " << saved_bytes << " bytes\n";
 }
 
 template <typename E>
@@ -498,106 +529,76 @@ void icf_sections(Context<E> &ctx) {
   if (ctx.objs.empty())
     return;
 
-  get_random_bytes(hmac_key, sizeof(hmac_key));
-
+  get_random_bytes(siphash_key, sizeof(siphash_key));
   uniquify_cies(ctx);
-  merge_leaf_nodes(ctx);
 
   // Prepare for the propagation rounds.
   std::vector<InputSection<E> *> sections = gather_sections(ctx);
 
-  // We allocate 3 arrays to store hashes for each vertex.
-  //
-  // Index 0 and 1 are used for tree hashes from the previous
-  // iteration and the current iteration. They switch roles every
-  // iteration. See `slot` below.
-  //
-  // Index 2 stores the initial, single-vertex hash. This is combined
-  // with hashes from the connected vertices to form the tree hash
-  // described above.
-  std::vector<std::vector<Digest>> digests(3);
-  digests[0] = compute_digests<E>(ctx, sections);
-  digests[1].resize(digests[0].size());
-  digests[2] = digests[0];
+  // `digests` holds the current digest of each vertex
+  std::vector<Digest> digests = compute_digests<E>(ctx, sections);
 
   std::vector<u32> edges;
   std::vector<u32> edge_indices;
   gather_edges<E>(ctx, sections, edges, edge_indices);
 
-  std::vector<u8> converged(digests[0].size());
-  bool slot = 0;
+  // The digest map is used to count the number of distinct digests in
+  // the loop below. As a side effect, it records the lowest-priority
+  // section for each digest, which the grouping step after the loop
+  // uses as the leader of each equivalence class.
+  DigestMap<E> map(digests.size());
 
   // Execute the propagation rounds until convergence is obtained.
+  //
+  // The number of distinct digests can only monotonically increase as
+  // the rounds hash ever-deeper trees, so once two consecutive rounds
+  // yield the same count, the partition of sections into equivalence
+  // classes has stopped changing and will remain unchanged for further
+  // iterations (proof omitted for brevity). Note that individual
+  // digests may well still be changing at that point; sections that
+  // have a cycle in downstream (i.e. recursive functions and functions
+  // that call them) never settle on a digest. That doesn't matter
+  // because sections in the same class change their digests in
+  // lockstep, keeping the partition intact.
   {
     Timer t(ctx, "propagate");
-    tbb::affinity_partitioner ap;
-
-    // A cheap test that the graph hasn't converged yet.
-    // The loop after this one uses a strict condition, but it's expensive
-    // as it requires sorting the entire hash collection.
-    //
-    // For nodes that have a cycle in downstream (i.e. recursive
-    // functions and functions that calls recursive functions) will always
-    // change with the iterations. Nodes that doesn't (i.e. non-recursive
-    // functions) will stop changing as soon as the propagation depth reaches
-    // the call tree depth.
-    // Here, we test whether we have reached sufficient depth for the latter,
-    // which is a necessary (but not sufficient) condition for convergence.
-    i64 num_changed = -1;
-    for (;;) {
-      i64 n = propagate<E>(digests, edges, edge_indices, slot, converged, ap);
-      if (n == num_changed)
-        break;
-      num_changed = n;
-    }
-
-    // Run the pass until the unique number of hashes stop increasing, at which
-    // point we have achieved convergence (proof omitted for brevity).
     i64 num_classes = -1;
-    for (;;) {
-      // count_num_classes requires sorting which is O(n log n), so do a little
-      // more work beforehand to amortize that log factor.
-      for (i64 i = 0; i < 10; i++)
-        propagate<E>(digests, edges, edge_indices, slot, converged, ap);
+    std::vector<Digest> scratch(digests.size());
 
-      i64 n = count_num_classes<E>(digests[slot], ap);
-      if (n == num_classes)
+    for (;;) {
+      // count_num_classes is as expensive as propagate, so we propagate
+      // a few times before counting the number of distinct groups.
+      propagate(digests, scratch, edges, edge_indices);
+      propagate(digests, scratch, edges, edge_indices);
+      propagate(digests, scratch, edges, edge_indices);
+
+      i64 m = count_num_classes<E>(digests, sections, map);
+      if (m == num_classes)
         break;
-      num_classes = n;
+      num_classes = m;
     }
   }
 
-  // Group sections by hash values.
+  // Group sections by digest. The final counting round has already
+  // elected a leader for each digest; look it up.
   {
     Timer t(ctx, "group");
-
-    auto *map =
-      new tbb::concurrent_unordered_map<Digest, Atomic<InputSection<E> *>>;
-
-    std::span<Digest> digest = digests[slot];
-
     tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      InputSection<E> *isec = sections[i];
-      auto [it, inserted] = map->insert({digest[i], isec});
-      if (!inserted) {
-        InputSection<E> *isec2 = it->second.load();
-        while (isec->get_priority() < isec2->get_priority() &&
-               !it->second.compare_exchange_strong(isec2, isec));
-      }
+      sections[i]->icf_leader = map.find(digests[i]);
     });
-
-    tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      auto it = map->find(digest[i]);
-      assert(it != map->end());
-      sections[i]->leader = it->second;
-    });
-
-    // Since free'ing the map is slow, postpone it.
-    ctx.on_exit.push_back([=] { delete map; });
   }
 
-  if (ctx.arg.print_icf_sections)
-    print_icf_sections(ctx);
+  if (!ctx.arg.print_icf_sections.empty())
+    print_icf_sections<E>(ctx, sections);
+
+  // Update alignment of leaders.
+  {
+    Timer t(ctx, "update_alignment");
+    tbb::parallel_for_each(sections, [&](InputSection<E> *isec) {
+      if (isec != isec->icf_leader)
+        update_maximum(isec->icf_leader->p2align, isec->p2align);
+    });
+  }
 
   // Eliminate duplicate sections.
   // Symbols pointing to eliminated sections will be redirected on the fly when
@@ -605,12 +606,13 @@ void icf_sections(Context<E> &ctx) {
   {
     Timer t(ctx, "sweep");
     static Counter eliminated("icf_eliminated");
-    tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-      for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-        if (isec && isec->is_alive && isec->icf_removed()) {
-          isec->kill();
-          eliminated++;
-        }
+    tbb::parallel_for_each(sections, [&](InputSection<E> *isec) {
+      if (isec != isec->icf_leader) {
+        isec->set_icf_removed();
+        isec->kill();
+        eliminated++;
+      } else {
+        isec->offset = -1;
       }
     });
   }

src/input-sections.cc

@@ -31,13 +31,20 @@ bool cie_equals(const CieRecord<E> &a, const CieRecord<E> &b) {
 }
 
 template <typename E>
-InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
-  : file(file), shndx(shndx) {
-  if (shndx < file.elf_sections.size())
-    contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};
+InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx,
+                              std::string_view section_name)
+  : file(&file), shndx(shndx) {
+  if (shndx < file.elf_sections.size()) {
+    std::string_view name = section_name;
+    if (name.empty())
+      name = file.shstrtab.data() + file.elf_sections[shndx].sh_name;
+    namelen = std::min<i64>(name.size(), UINT16_MAX);
+
+    contents = file.mf->data + shdr().sh_offset;
+  }
 
   if (shdr().sh_flags & SHF_COMPRESSED) {
-    ElfChdr<E> &chdr = *(ElfChdr<E> *)&contents[0];
+    ElfChdr<E> &chdr = *(ElfChdr<E> *)contents;
     sh_size = chdr.ch_size;
     p2align = to_p2align(chdr.ch_addralign);
   } else {
@@ -59,48 +66,73 @@ InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
 
 template <typename E>
 void InputSection<E>::uncompress(Context<E> &ctx) {
-  if (!(shdr().sh_flags & SHF_COMPRESSED) || uncompressed)
+  if (!(shdr().sh_flags & SHF_COMPRESSED) || is_uncompressed())
     return;
 
   u8 *buf = new u8[sh_size];
-  copy_contents(ctx, buf);
-  contents = std::string_view((char *)buf, sh_size);
+  copy_contents_to(ctx, buf, sh_size);
+  contents = buf;
   ctx.string_pool.emplace_back(buf);
-  uncompressed = true;
+  set_uncompressed();
 }
 
 template <typename E>
-void InputSection<E>::copy_contents(Context<E> &ctx, u8 *buf) {
-  if (!(shdr().sh_flags & SHF_COMPRESSED) || uncompressed) {
-    memcpy(buf, contents.data(), contents.size());
+void InputSection<E>::copy_contents_to(Context<E> &ctx, u8 *buf, i64 sz) {
+  if (!(shdr().sh_flags & SHF_COMPRESSED) || is_uncompressed()) {
+    memcpy(buf, contents, sz);
     return;
   }
 
-  if (contents.size() < sizeof(ElfChdr<E>))
+  std::string_view view = get_contents();
+  if (view.size() < sizeof(ElfChdr<E>))
     Fatal(ctx) << *this << ": corrupted compressed section";
 
-  ElfChdr<E> &hdr = *(ElfChdr<E> *)&contents[0];
-  std::string_view data = contents.substr(sizeof(ElfChdr<E>));
+  ElfChdr<E> &hdr = *(ElfChdr<E> *)contents;
+  std::string_view data = view.substr(sizeof(ElfChdr<E>));
 
   switch (hdr.ch_type) {
   case ELFCOMPRESS_ZLIB: {
-    unsigned long size = sh_size;
-    if (::uncompress(buf, &size, (u8 *)data.data(), data.size()) != Z_OK)
-      Fatal(ctx) << *this << ": uncompress failed";
-    assert(size == sh_size);
+    z_stream s = {};
+    inflateInit(&s);
+    s.next_in = (u8 *)data.data();
+    s.avail_in = data.size();
+    s.next_out = buf;
+    s.avail_out = sz;
+
+    int r;
+    while (s.total_out < sz && (r = inflate(&s, Z_NO_FLUSH)) == Z_OK);
+    if (s.total_out < sz && r != Z_STREAM_END)
+      Fatal(ctx) << *this << ": uncompress failed: " << s.msg;
+    inflateEnd(&s);
+    msan_unpoison(buf, sz);
     break;
   }
-  case ELFCOMPRESS_ZSTD:
-    if (ZSTD_decompress(buf, sh_size, (u8 *)data.data(), data.size()) != sh_size)
-      Fatal(ctx) << *this << ": ZSTD_decompress failed";
+  case ELFCOMPRESS_ZSTD: {
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+    ZSTD_inBuffer in = { data.data(), data.size() };
+    ZSTD_outBuffer out = { buf, (size_t)sz };
+
+    while (out.pos < out.size) {
+      size_t r = ZSTD_decompressStream(dctx, &out, &in);
+      if (ZSTD_isError(r))
+        Fatal(ctx) << *this << ": uncompress failed: " << ZSTD_getErrorName(r);
+      if (r == 0 && out.pos < out.size)
+        Fatal(ctx) << *this << ": uncompress failed: premature end of input";
+    }
+    ZSTD_freeDCtx(dctx);
+    msan_unpoison(buf, sz);
     break;
+  }
   default:
     Fatal(ctx) << *this << ": unsupported compression type: 0x"
                << std::hex << hdr.ch_type;
   }
 }
 
-typedef enum : u8 { NONE, ERROR, COPYREL, PLT, CPLT } Action;
+// wingdi.h defines ERROR as a macro, so undefine it before use
+#undef ERROR
+
+typedef enum : u8 { NONE, ERROR, CANONICAL, PLT } Action;
 
 template <typename E>
 static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
@@ -113,23 +145,13 @@ static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
                << std::hex << rel.r_offset << " against symbol `"
                << sym << "' can not be used; recompile with -fPIC";
     break;
-  case COPYREL:
-    // Create a copy relocation
-    if (!sym.file->is_dso) {
-      assert(sym.esym().is_undef_weak());
-      Error(ctx) << isec << ": cannot create a copy relocation for "
-                 << sym <<"; recompile with -fPIE or -fPIC";
-    }
-    sym.flags |= NEEDS_COPYREL;
+  case CANONICAL:
+    sym.flags |= NEEDS_CANONICAL;
     break;
   case PLT:
     // Create a PLT entry
     sym.flags |= NEEDS_PLT;
     break;
-  case CPLT:
-    // Create a canonical PLT entry
-    sym.flags |= NEEDS_CPLT;
-    break;
   }
 }
 
@@ -161,9 +183,9 @@ void InputSection<E>::scan_pcrel(Context<E> &ctx, Symbol<E> &sym,
   // linker generally does not support PC-relative relocations.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  ERROR,    NONE,    ERROR,         PLT    },  // Shared object
-    {  ERROR,    NONE,    COPYREL,       CPLT   },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT   },  // Position-dependent exec
+    {  ERROR,    NONE,    ERROR,         PLT       }, // Shared object
+    {  ERROR,    NONE,    CANONICAL,     CANONICAL }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -180,9 +202,9 @@ void InputSection<E>::scan_absrel(Context<E> &ctx, Symbol<E> &sym,
   // resolved at link-time.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Shared object
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT  },  // Position-dependent exec
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Shared object
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -234,23 +256,26 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 
     if (deltas.empty()) {
       // If a section is not relaxed, we can copy it as a one big chunk.
-      copy_contents(ctx, buf);
+      copy_contents_to(ctx, buf, sh_size);
     } else {
       // A relaxed section is copied piece-wise.
-      memcpy(buf, contents.data(), deltas[0].offset);
+      memcpy(buf, contents, deltas[0].offset);
 
+      i64 input_size = get_contents().size();
       for (i64 i = 0; i < deltas.size(); i++) {
         i64 offset = deltas[i].offset;
         i64 delta = deltas[i].delta;
-        i64 end = (i + 1 == deltas.size()) ? contents.size() : deltas[i + 1].offset;
+        i64 end = input_size;
+        if (i + 1 < deltas.size())
+          end = deltas[i + 1].offset;
         i64 removed_bytes = get_removed_bytes(deltas, i);
         memcpy(buf + offset + removed_bytes - delta,
-               contents.data() + offset + removed_bytes,
+               contents + offset + removed_bytes,
                end - offset - removed_bytes);
       }
     }
   } else {
-    copy_contents(ctx, buf);
+    copy_contents_to(ctx, buf, sh_size);
   }
 
   // Apply relocations
@@ -266,49 +291,72 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 template <typename E>
 std::string_view
 InputSection<E>::get_func_name(Context<E> &ctx, i64 offset) const {
-  for (Symbol<E> *sym : file.symbols) {
-    if (sym->file == &file) {
-      const ElfSym<E> &esym = sym->esym();
-      if (esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
-          esym.st_value <= offset && offset < esym.st_value + esym.st_size) {
-        if (ctx.arg.demangle)
-          return demangle(*sym);
-        return sym->name();
-      }
-    }
-  }
+  for (Symbol<E> *sym : file->symbols)
+    if (sym->file == file)
+      if (const ElfSym<E> &esym = sym->esym();
+          esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
+          esym.st_value <= offset && offset < esym.st_value + esym.st_size)
+        return ctx.arg.demangle ? demangle(*sym) : sym->name();
   return "";
 }
 
+// Find the prevailing group having the same signature as the discarded group
+// that contains esym. This is called only on an error path, so a linear scan is
+// sufficient.
+template <typename E>
+static ObjectFile<E> *
+find_comdat_owner(Context<E> &ctx, ObjectFile<E> &file, const ElfSym<E> &esym) {
+  if (esym.is_undef() || esym.is_abs() || esym.is_common())
+    return nullptr;
+
+  i64 shndx = file.get_shndx(esym);
+  for (ComdatGroupRef<E> &ref : file.comdat_groups) {
+    if (ref.is_owner ||
+        !ranges::any_of(ref.members(file), [&](u32 x) { return x == shndx; }))
+      continue;
+
+    Symbol<E> *sig = ref.signature(ctx);
+    for (ObjectFile<E> *file : ctx.objs)
+      for (ComdatGroupRef<E> &ref : file->comdat_groups)
+        if (ref.is_owner && ref.signature(ctx) == sig)
+          return file;
+  }
+  return nullptr;
+}
+
 // Test if the symbol a given relocation refers to has already been resolved.
 // If not, record that error and returns true.
 template <typename E>
 bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel) {
   // If a relocation refers to a linker-synthesized symbol for a
   // section fragment, it's always been resolved.
-  if (file.elf_syms.size() <= rel.r_sym)
+  if (file->elf_syms.size() <= rel.r_sym)
     return false;
 
-  Symbol<E> &sym = *file.symbols[rel.r_sym];
-  const ElfSym<E> &esym = file.elf_syms[rel.r_sym];
+  Symbol<E> &sym = *file->symbols[rel.r_sym];
+  const ElfSym<E> &esym = file->elf_syms[rel.r_sym];
 
-  // If a symbol is defined in a comdat group, and the comdat group is
-  // discarded, the symbol may not have an owner. It is technically an
-  // violation of the One Definition Rule, so it is a programmer's fault.
-  if (!sym.file) {
-    Error(ctx) << *this << ": " << sym << " refers to a discarded COMDAT section"
-               << " probably due to an ODR violation";
+  // A global symbol in a discarded COMDAT group should resolve to the
+  // corresponding symbol in the prevailing group. If it does not, the
+  // object files violate the One Definition Rule.
+  if (!sym.file && &sym != discarded_comdat_sym<E>) {
+    std::stringstream ss;
+    ss << *this << ": " << sym << " refers to a discarded COMDAT section"
+       << " probably due to an ODR violation";
+    if (ObjectFile<E> *owner = find_comdat_owner(ctx, *file, esym))
+      ss << '\n' << ">>> prevailing definition is in " << *owner;
+    Error(ctx) << ss.str();
     return true;
   }
 
   auto record = [&] {
     std::stringstream ss;
-    if (std::string_view source = file.get_source_name(); !source.empty())
+    if (std::string_view source = file->get_source_name(); !source.empty())
       ss << ">>> referenced by " << source << "\n";
     else
       ss << ">>> referenced by " << *this << "\n";
 
-    ss << ">>>               " << file;
+    ss << ">>>               " << *file;
     if (std::string_view func = get_func_name(ctx, rel.r_offset); !func.empty())
       ss << ":(" << func << ")";
     ss << '\n';
@@ -342,12 +390,9 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
 
 template <typename E>
 MergeableSection<E>::MergeableSection(Context<E> &ctx, MergedSection<E> &parent,
-                                      std::unique_ptr<InputSection<E>> &isec)
-  : parent(parent), p2align(isec->p2align), section(std::move(isec)) {
-  section->uncompress(ctx);
-
-  std::scoped_lock lock(parent.mu);
-  parent.members.push_back(this);
+                                      InputSection<E> *isec)
+  : parent(parent), p2align(isec->p2align), input_section(isec) {
+  input_section->uncompress(ctx);
 }
 
 static size_t find_null(std::string_view data, i64 pos, i64 entsize) {
@@ -380,42 +425,40 @@ static size_t find_null(std::string_view data, i64 pos, i64 entsize) {
 // We do not support mergeable sections that have relocations.
 template <typename E>
 void MergeableSection<E>::split_contents(Context<E> &ctx) {
-  std::string_view data = section->contents;
+  std::string_view data = input_section->get_contents();
   if (data.size() > UINT32_MAX)
-    Fatal(ctx) << *section << ": mergeable section too large";
+    Fatal(ctx) << *input_section << ": mergeable section too large";
 
   i64 entsize = parent.shdr.sh_entsize;
+  HyperLogLog::Sketch &sketch = parent.estimator.local();
 
-  // Split sections
+  auto add_fragment = [&](i64 pos, i64 size) {
+    frag_offsets.push_back(pos);
+    u64 hash = hash_string(data.substr(pos, size));
+    hashes.push_back(hash);
+    sketch.insert(hash);
+  };
+
+  // Hash each fragment while its contents are still in the cache.
   if (parent.shdr.sh_flags & SHF_STRINGS) {
     for (i64 pos = 0; pos < data.size();) {
-      frag_offsets.push_back(pos);
       size_t end = find_null(data, pos, entsize);
       if (end == data.npos)
-        Fatal(ctx) << *section << ": string is not null terminated";
+        Fatal(ctx) << *input_section << ": string is not null terminated";
+      add_fragment(pos, end + entsize - pos);
       pos = end + entsize;
     }
   } else {
     if (data.size() % entsize)
-      Fatal(ctx) << *section << ": section size is not multiple of sh_entsize";
+      Fatal(ctx) << *input_section
+                 << ": section size is not multiple of sh_entsize";
     frag_offsets.reserve(data.size() / entsize);
+    hashes.reserve(data.size() / entsize);
 
     for (i64 pos = 0; pos < data.size(); pos += entsize)
-      frag_offsets.push_back(pos);
+      add_fragment(pos, entsize);
   }
 
-  // Compute hashes for section pieces
-  HyperLogLog estimator;
-  hashes.reserve(frag_offsets.size());
-
-  for (i64 i = 0; i < frag_offsets.size(); i++) {
-    u64 hash = hash_string(get_contents(i));
-    hashes.push_back(hash);
-    estimator.insert(hash);
-  }
-
-  parent.estimator.merge(estimator);
-
   static Counter counter("string_fragments");
   counter += frag_offsets.size();
 }
@@ -423,8 +466,20 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
 template <typename E>
 void MergeableSection<E>::resolve_contents(Context<E> &ctx) {
   fragments.reserve(frag_offsets.size());
-  for (i64 i = 0; i < frag_offsets.size(); i++)
-    fragments.push_back(parent.insert(ctx, get_contents(i), hashes[i], p2align));
+
+  // The hash table is typically much larger than the cache, so each
+  // insertion stalls on a cache miss for its first probe. We know all
+  // hashes upfront, so prefetch the bucket a few insertions ahead.
+  constexpr i64 lookahead = 8;
+
+  for (i64 i = 0; i < frag_offsets.size(); i++) {
+    if (i + lookahead < frag_offsets.size())
+      parent.map.prefetch(hashes[i + lookahead]);
+
+    SectionFragment<E> *frag =
+      parent.insert(ctx, get_contents(i), hashes[i], p2align);
+    fragments.push_back(parent.map.get_idx(frag));
+  }
 
   // Reclaim memory as we'll never use this vector again
   hashes.clear();
@@ -433,6 +488,10 @@ void MergeableSection<E>::resolve_contents(Context<E> &ctx) {
 
 using E = MOLD_TARGET;
 
+// InputSections are allocated from the arena, which does not run
+// destructors, so the class must stay trivially destructible.
+static_assert(std::is_trivially_destructible_v<InputSection<E>>);
+
 template bool cie_equals(const CieRecord<E> &, const CieRecord<E> &);
 template class InputSection<E>;
 template class MergeableSection<E>;

src/jobs-unix.cc

@@ -9,7 +9,7 @@
 // mold processes to just 1 for each user. It is intended to be used as
 // `MOLD_JOBS=1 ninja` or `MOLD_JOBS=1 make -j$(nproc)`.
 
-#include "common.h"
+#include "mold.h"
 
 #include <cstdlib>
 #include <fcntl.h>

src/linker-script.cc

@@ -111,6 +111,27 @@ static std::string_view unquote(std::string_view s) {
   return s;
 }
 
+// Version scripts and dynamic lists take a quoted name literally, while
+// an unquoted one is a glob pattern. We escape glob metacharacters in a
+// quoted name so that the pattern matcher treats them as literals.
+template <typename E>
+std::string_view Script<E>::unquote_pattern(std::string_view tok) {
+  if (!tok.starts_with('"'))
+    return tok;
+
+  std::string_view str = unquote(tok);
+  if (str.find_first_of("*?[\\") == str.npos)
+    return str;
+
+  std::string out;
+  for (char c : str) {
+    if (c == '*' || c == '?' || c == '[' || c == '\\')
+      out += '\\';
+    out += c;
+  }
+  return save_string(ctx, out);
+}
+
 template <typename E>
 std::span<std::string_view>
 Script<E>::read_output_format(std::span<std::string_view> tok) {
@@ -124,7 +145,11 @@ Script<E>::read_output_format(std::span<std::string_view> tok) {
 
 template <typename E>
 static bool is_in_sysroot(Context<E> &ctx, std::string path) {
-  std::string rel = std::filesystem::relative(path, ctx.arg.sysroot).string();
+  std::string sysroot = ctx.arg.sysroot;
+  if (sysroot.starts_with('/') && !ctx.arg.chroot.empty())
+    sysroot = ctx.arg.chroot + "/" + path_clean(sysroot);
+
+  std::string rel = std::filesystem::relative(path, sysroot).string();
   return rel != "." && !rel.starts_with("../");
 }
 
@@ -196,7 +221,8 @@ Script<E>::read_group(std::span<std::string_view> tok) {
     }
 
     MappedFile *mf = resolve_path(tok[0], true);
-    read_file(ctx, rctx, mf);
+    ReaderContext rctx2 = rctx.next_child();
+    read_file(ctx, rctx2, mf);
     tok = tok.subspan(1);
   }
 
@@ -245,9 +271,12 @@ std::string_view Script<E>::get_script_output_type() {
   }
 
   if (tok.size() >= 3 && (tok[0] == "INPUT" || tok[0] == "GROUP") &&
-      tok[1] == "(")
+      tok[1] == "(") {
+    if (tok.size() >= 5 && tok[2] == "AS_NEEDED" && tok[3] == "(")
+      tok = tok.subspan(2);
     if (MappedFile *mf = resolve_path(tok[2], false))
       return get_machine_type(ctx, rctx, mf);
+  }
   return "";
 }
 
@@ -261,6 +290,13 @@ static bool read_label(std::span<std::string_view> &tok, std::string label) {
     tok = tok.subspan(2);
     return true;
   }
+
+  // The tokenizer keeps a colon in a token because of the C++ scope
+  // operator, so `local:*` is a single token. Split the pattern off.
+  if (tok.size() >= 1 && tok[0].starts_with(label + ":")) {
+    tok[0] = tok[0].substr(label.size() + 1);
+    return true;
+  }
   return false;
 }
 
@@ -301,11 +337,11 @@ Script<E>::read_version_script_commands(std::span<std::string_view> tok,
     if (tok[0] == "*") {
       ctx.default_version = (is_global ? ver_idx : (u32)VER_NDX_LOCAL);
     } else if (is_global) {
-      ctx.version_patterns.push_back({unquote(tok[0]), mf->name, ver_str,
-                                      ver_idx, is_cpp});
+      ctx.version_patterns.push_back({unquote_pattern(tok[0]), mf->name,
+                                      ver_str, ver_idx, is_cpp});
     } else {
-      ctx.version_patterns.push_back({unquote(tok[0]), mf->name, ver_str,
-                                      VER_NDX_LOCAL, is_cpp});
+      ctx.version_patterns.push_back({unquote_pattern(tok[0]), mf->name,
+                                      ver_str, VER_NDX_LOCAL, is_cpp});
     }
 
     tok = tok.subspan(1);
@@ -379,7 +415,7 @@ Script<E>::read_dynamic_list_commands(std::span<std::string_view> tok,
       continue;
     }
 
-    result.push_back({unquote(tok[0]), "", is_cpp});
+    result.push_back({unquote_pattern(tok[0]), "", is_cpp});
     tok = skip(tok.subspan(1), ";");
   }
   return tok;

src/lto-unix.cc

@@ -29,8 +29,8 @@
 //
 //  4. Continue the rest of the linking process as usual.
 //
-// When gcc or clang inovkes ld, they pass `-plugin linker-plugin.so` to
-// the linker. The given .so file provides a way to call the compiler
+// When gcc or clang inovkes ld, they pass `-plugin /path/to/linker-plugin.so`
+// to the linker. The given .so file provides a way to call the compiler
 // backend.
 //
 // The linker plugin API is documented at
@@ -179,13 +179,18 @@ static PluginStatus add_input_file(const char *path) {
   MappedFile *mf = must_open_file(ctx, path);
   mf->is_dependency = false;
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, "");
+  ObjectFile<E> *file = ctx.arena.template make<ObjectFile<E>>(ctx, mf, "");
   ctx.obj_pool.emplace_back(file);
   lto_objects<E>.push_back(file);
 
   file->priority = file_priority++;
+  file->is_lto_output = true;
   file->is_reachable = true;
-  file->parse(ctx);
+  file->parse_symbols(ctx);
+
+  // parse_symbols() only registers global symbols. Create their shared
+  // Symbol objects and fill in the file's pointers before resolving.
+  gather_symbols(ctx);
   file->resolve_symbols(ctx);
   return LDPS_OK;
 }
@@ -280,7 +285,7 @@ template <typename E>
 static PluginStatus
 get_symbols(const void *handle, int nsyms, PluginSymbol *psyms, bool is_v2) {
   ObjectFile<E> &file = *(ObjectFile<E> *)handle;
-  assert(file.is_lto_obj);
+  assert(file.is_lto_input);
 
   // If file is an archive member which was not chose to be included in
   // to the final result, we need to make the plugin to ignore all
@@ -307,7 +312,7 @@ get_symbols(const void *handle, int nsyms, PluginSymbol *psyms, bool is_v2) {
     if (sym.file->is_dso)
       return LDPR_RESOLVED_DYN;
 
-    if (((ObjectFile<E> *)sym.file)->is_lto_obj && !sym.is_wrapped)
+    if (sym.file->to_obj()->is_lto_input && !sym.is_wrapped)
       return esym.is_undef() ? LDPR_RESOLVED_IR : LDPR_PREEMPTED_IR;
     return esym.is_undef() ? LDPR_RESOLVED_EXEC : LDPR_PREEMPTED_REG;
   };
@@ -342,8 +347,8 @@ static void restart_process(Context<E> &ctx) {
   for (std::string_view arg : ctx.cmdline_args)
     args.push_back(strdup(std::string(arg).c_str()));
 
-  for (std::unique_ptr<ObjectFile<E>> &file : ctx.obj_pool)
-    if (file->is_lto_obj && !file->is_reachable)
+  for (ArenaObjectPtr<ObjectFile<E>> &file : ctx.obj_pool)
+    if (file->is_lto_input && !file->is_reachable)
       args.push_back(strdup(("--:ignore-ir-file=" +
                              file->mf->get_identifier()).c_str()));
 
@@ -425,7 +430,8 @@ get_api_version(const char *plugin_identifier,
   if (LAPI_V1 < minimal_api_supported)
     Fatal(*gctx<E>) << "LTO plugin does not support V0 or V1 API";
 
-  std::string version = get_mold_version() + "\0"s;
+  // The plugin reads the string after this function has returned
+  static std::string version = mold_version + "\0"s;
 
   *linker_identifier = "mold";
   *linker_version = version.data();
@@ -440,74 +446,71 @@ get_api_version(const char *plugin_identifier,
 // dlopen the linker plugin file
 template <typename E>
 static void load_lto_plugin(Context<E> &ctx) {
-  static std::once_flag flag;
+  if (phase != 0)
+    return;
+  phase = 1;
+  gctx<E> = &ctx;
 
-  std::call_once(flag, [&] {
-    assert(phase == 0);
-    phase = 1;
-    gctx<E> = &ctx;
+  void *handle = dlopen(ctx.arg.plugin.c_str(), RTLD_NOW | RTLD_LOCAL);
+  if (!handle)
+    Fatal(ctx) << "could not open plugin file: " << dlerror();
 
-    void *handle = dlopen(ctx.arg.plugin.c_str(), RTLD_NOW | RTLD_LOCAL);
-    if (!handle)
-      Fatal(ctx) << "could not open plugin file: " << dlerror();
+  OnloadFn *onload = (OnloadFn *)dlsym(handle, "onload");
+  if (!onload)
+    Fatal(ctx) << "failed to load plugin " << ctx.arg.plugin << ": "
+               << dlerror();
 
-    OnloadFn *onload = (OnloadFn *)dlsym(handle, "onload");
-    if (!onload)
-      Fatal(ctx) << "failed to load plugin " << ctx.arg.plugin << ": "
-                 << dlerror();
+  auto save = [&](std::string_view str) {
+    return save_string(ctx, std::string(str).c_str()).data();
+  };
 
-    auto save = [&](std::string_view str) {
-      return save_string(ctx, std::string(str).c_str()).data();
-    };
+  std::vector<PluginTagValue> tv;
+  tv.emplace_back(LDPT_MESSAGE, message<E>);
 
-    std::vector<PluginTagValue> tv;
-    tv.emplace_back(LDPT_MESSAGE, message<E>);
+  if (ctx.arg.shared)
+    tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_DYN);
+  else if (ctx.arg.pie)
+    tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_PIE);
+  else
+    tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_EXEC);
 
-    if (ctx.arg.shared)
-      tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_DYN);
-    else if (ctx.arg.pie)
-      tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_PIE);
-    else
-      tv.emplace_back(LDPT_LINKER_OUTPUT, LDPO_EXEC);
+  for (std::string_view opt : ctx.arg.plugin_opt)
+    tv.emplace_back(LDPT_OPTION, save(opt));
 
-    for (std::string_view opt : ctx.arg.plugin_opt)
-      tv.emplace_back(LDPT_OPTION, save(opt));
+  tv.emplace_back(LDPT_REGISTER_CLAIM_FILE_HOOK, register_claim_file_hook<E>);
+  tv.emplace_back(LDPT_REGISTER_ALL_SYMBOLS_READ_HOOK,
+                  register_all_symbols_read_hook<E>);
+  tv.emplace_back(LDPT_REGISTER_CLEANUP_HOOK, register_cleanup_hook<E>);
+  tv.emplace_back(LDPT_ADD_SYMBOLS, add_symbols);
+  tv.emplace_back(LDPT_GET_SYMBOLS, get_symbols_v1);
+  tv.emplace_back(LDPT_ADD_INPUT_FILE, add_input_file<E>);
+  tv.emplace_back(LDPT_GET_INPUT_FILE, get_input_file);
+  tv.emplace_back(LDPT_RELEASE_INPUT_FILE, release_input_file<E>);
+  tv.emplace_back(LDPT_ADD_INPUT_LIBRARY, add_input_library);
+  tv.emplace_back(LDPT_OUTPUT_NAME, save(ctx.arg.output));
+  tv.emplace_back(LDPT_SET_EXTRA_LIBRARY_PATH, set_extra_library_path);
+  tv.emplace_back(LDPT_GET_VIEW, get_view<E>);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_COUNT, get_input_section_count);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_TYPE, get_input_section_type);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_NAME, get_input_section_name);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_CONTENTS, get_input_section_contents);
+  tv.emplace_back(LDPT_UPDATE_SECTION_ORDER, update_section_order);
+  tv.emplace_back(LDPT_ALLOW_SECTION_ORDERING, allow_section_ordering);
+  tv.emplace_back(LDPT_ADD_SYMBOLS_V2, add_symbols);
+  tv.emplace_back(LDPT_GET_SYMBOLS_V2, get_symbols_v2<E>);
+  tv.emplace_back(LDPT_ALLOW_UNIQUE_SEGMENT_FOR_SECTIONS,
+                  allow_unique_segment_for_sections);
+  tv.emplace_back(LDPT_UNIQUE_SEGMENT_FOR_SECTIONS, unique_segment_for_sections);
+  tv.emplace_back(LDPT_GET_SYMBOLS_V3, get_symbols_v3<E>);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_ALIGNMENT, get_input_section_alignment);
+  tv.emplace_back(LDPT_GET_INPUT_SECTION_SIZE, get_input_section_size);
+  tv.emplace_back(LDPT_REGISTER_NEW_INPUT_HOOK, register_new_input_hook<E>);
+  tv.emplace_back(LDPT_GET_WRAP_SYMBOLS, get_wrap_symbols);
+  tv.emplace_back(LDPT_GET_API_VERSION, get_api_version<E>);
+  tv.emplace_back(LDPT_NULL, 0);
 
-    tv.emplace_back(LDPT_REGISTER_CLAIM_FILE_HOOK, register_claim_file_hook<E>);
-    tv.emplace_back(LDPT_REGISTER_ALL_SYMBOLS_READ_HOOK,
-                    register_all_symbols_read_hook<E>);
-    tv.emplace_back(LDPT_REGISTER_CLEANUP_HOOK, register_cleanup_hook<E>);
-    tv.emplace_back(LDPT_ADD_SYMBOLS, add_symbols);
-    tv.emplace_back(LDPT_GET_SYMBOLS, get_symbols_v1);
-    tv.emplace_back(LDPT_ADD_INPUT_FILE, add_input_file<E>);
-    tv.emplace_back(LDPT_GET_INPUT_FILE, get_input_file);
-    tv.emplace_back(LDPT_RELEASE_INPUT_FILE, release_input_file<E>);
-    tv.emplace_back(LDPT_ADD_INPUT_LIBRARY, add_input_library);
-    tv.emplace_back(LDPT_OUTPUT_NAME, save(ctx.arg.output));
-    tv.emplace_back(LDPT_SET_EXTRA_LIBRARY_PATH, set_extra_library_path);
-    tv.emplace_back(LDPT_GET_VIEW, get_view<E>);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_COUNT, get_input_section_count);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_TYPE, get_input_section_type);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_NAME, get_input_section_name);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_CONTENTS, get_input_section_contents);
-    tv.emplace_back(LDPT_UPDATE_SECTION_ORDER, update_section_order);
-    tv.emplace_back(LDPT_ALLOW_SECTION_ORDERING, allow_section_ordering);
-    tv.emplace_back(LDPT_ADD_SYMBOLS_V2, add_symbols);
-    tv.emplace_back(LDPT_GET_SYMBOLS_V2, get_symbols_v2<E>);
-    tv.emplace_back(LDPT_ALLOW_UNIQUE_SEGMENT_FOR_SECTIONS,
-                    allow_unique_segment_for_sections);
-    tv.emplace_back(LDPT_UNIQUE_SEGMENT_FOR_SECTIONS, unique_segment_for_sections);
-    tv.emplace_back(LDPT_GET_SYMBOLS_V3, get_symbols_v3<E>);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_ALIGNMENT, get_input_section_alignment);
-    tv.emplace_back(LDPT_GET_INPUT_SECTION_SIZE, get_input_section_size);
-    tv.emplace_back(LDPT_REGISTER_NEW_INPUT_HOOK, register_new_input_hook<E>);
-    tv.emplace_back(LDPT_GET_WRAP_SYMBOLS, get_wrap_symbols);
-    tv.emplace_back(LDPT_GET_API_VERSION, get_api_version<E>);
-    tv.emplace_back(LDPT_NULL, 0);
-
-    [[maybe_unused]] PluginStatus status = onload(tv.data());
-    assert(status == LDPS_OK);
-  });
+  [[maybe_unused]] PluginStatus status = onload(tv.data());
+  assert(status == LDPS_OK);
 }
 
 template <typename E>
@@ -536,8 +539,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
   }
 
   switch (psym.symbol_type) {
-  case LDST_UNKNOWN:
-    break;
   case LDST_FUNCTION:
     esym.st_type = STT_FUNC;
     break;
@@ -547,8 +548,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
   };
 
   switch (psym.visibility) {
-  case LDPV_DEFAULT:
-    break;
   case LDPV_PROTECTED:
     esym.st_visibility = STV_PROTECTED;
     break;
@@ -600,27 +599,23 @@ create_plugin_input_file(Context<E> &ctx, MappedFile *mf) {
 template <typename E>
 ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
   if (ctx.arg.plugin.empty())
-    Fatal(ctx) << mf->name << ": don't know how to handle this LTO object file "
-               << "because no -plugin option was given. Please make sure you "
-               << "added -flto not only for creating object files but also for "
-               << "creating the final executable.";
+    Fatal(ctx) << mf->name << ": unable to handle this LTO object file because "
+               << "the -plugin option was not provided. Please make sure you "
+               << "added -flto not only when creating object files but also "
+               << "when linking the final executable.";
 
   load_lto_plugin(ctx);
 
-  // V0 API's claim_file is not thread-safe.
-  static std::mutex mu;
-  std::unique_lock lock(mu, std::defer_lock);
-  if (!is_gcc_linker_api_v1)
-    lock.lock();
-
   // Create mold's object instance
-  ObjectFile<E> *obj = new ObjectFile<E>;
+  ObjectFile<E> *obj =
+    ctx.arena.template make<ObjectFile<E>>(ctx);
   ctx.obj_pool.emplace_back(obj);
 
   obj->filename = mf->name;
-  obj->symbols.push_back(new Symbol<E>);
+  Symbol<E> *dummy = ctx.arena.template make<Symbol<E>>();
+  obj->symbols.emplace_back(dummy);
   obj->first_global = 1;
-  obj->is_lto_obj = true;
+  obj->is_lto_input = true;
   obj->mf = mf;
   obj->archive_name = mf->parent ? mf->parent->name : "";
 
@@ -633,17 +628,18 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
   // claim_file_hook() calls add_symbols() which initializes `plugin_symbols`
   int claimed = false;
   claim_file_hook(&file, &claimed);
-  if (!claimed)
-    Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
-               << " please make sure you are using the same compiler of the"
-               << " same version for all object files";
 
-  // It looks like GCC doesn't need fd after claim_file_hook() while
-  // LLVM needs it and takes the ownership of fd. To prevent "too many
-  // open files" issue, we close fd only for GCC. This is ugly, though.
-  if (!is_llvm(ctx)) {
-    MappedFile *mf2 = mf->parent ? mf->parent : mf;
-    mf2->close_fd();
+  if (mf->parent)
+    mf->parent->close_fd();
+  else
+    mf->close_fd();
+
+  if (!claimed) {
+    if (!mf->parent && !mf->thin_parent)
+      Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
+                 << " please make sure you are using the same compiler of the"
+                 << " same version for all object files";
+    return nullptr;
   }
 
   // Create a symbol strtab
@@ -654,7 +650,8 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
 
   // Initialize esyms
   obj->lto_elf_syms.resize(plugin_symbols.size() + 1);
-  obj->lto_comdat_groups.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_signatures.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_discarded.resize(plugin_symbols.size() + 1);
   i64 strtab_offset = 1;
 
   for (i64 i = 0; i < plugin_symbols.size(); i++) {
@@ -670,13 +667,15 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
     // section. We handle such symbols differently than comdat symbols in
     // a regular file because, unlike regular object files, IR files don't
     // have input sections.
-    if (psym.comdat_key)
-      obj->lto_comdat_groups[i + 1] = insert_comdat_group(ctx, psym.comdat_key);
+    if (psym.comdat_key) {
+      std::string_view key = save_string(ctx, psym.comdat_key);
+      obj->lto_comdat_signatures[i + 1] = get_symbol(ctx, key);
+    }
   }
 
   obj->symbol_strtab = save_string(ctx, strtab);
   obj->elf_syms = obj->lto_elf_syms;
-  obj->initialize_symbols(ctx);
+  obj->register_global_symbols(ctx);
   plugin_symbols.clear();
   return obj;
 }
@@ -695,10 +694,10 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
 
   // Set `referenced_by_regular_obj` bit.
   tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-    if (!file->is_lto_obj) {
+    if (!file->is_lto_input) {
       for (Symbol<E> *sym : file->get_global_syms()) {
         if (sym->file && !sym->file->is_dso &&
-            ((ObjectFile<E> *)sym->file)->is_lto_obj) {
+            sym->file->to_obj()->is_lto_input) {
           std::scoped_lock lock(sym->mu);
           sym->referenced_by_regular_obj = true;
         }
@@ -726,7 +725,7 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
   // given to the LTO backend. Such sections contains code and data for
   // peripherails (typically GPUs).
   for (ObjectFile<E> *file : ctx.objs) {
-    if (file->is_reachable && !file->is_lto_obj && file->is_gcc_offload_obj) {
+    if (file->is_reachable && !file->is_lto_input && file->is_gcc_offload_obj) {
       PluginInputFile pfile = create_plugin_input_file(ctx, file->mf);
       int claimed = false;
       claim_file_hook(&pfile, &claimed);

src/main.cc

@@ -1,122 +1,141 @@
 #include "mold.h"
-#include "../lib/archive-file.h"
+#include "config.h"
 
 #include <cstring>
+#include <filesystem>
 #include <functional>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <tbb/global_control.h>
+#include <tbb/parallel_for_each.h>
+#include <tbb/parallel_sort.h>
 #include <unordered_set>
 
-#ifdef _WIN32
-# include <direct.h>
-# define chdir _chdir
-#else
-# include <unistd.h>
-#endif
-
-#ifdef MOLD_X86_64
-int main(int argc, char **argv) {
-  mold::set_mimalloc_options();
-  return mold::mold_main<mold::X86_64>(argc, argv);
-}
-#endif
-
 namespace mold {
 
 template <typename E>
 static void
 check_file_compatibility(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   std::string_view target = get_machine_type(ctx, rctx, mf);
+  if (target.empty())
+    Fatal(ctx) << mf->name << ": unknown machine type";
   if (target != ctx.arg.emulation)
     Fatal(ctx) << mf->name << ": incompatible file type: "
                << ctx.arg.emulation << " is expected but got " << target;
 }
 
 template <typename E>
-static ObjectFile<E> *new_object_file(Context<E> &ctx, ReaderContext &rctx,
-                                      MappedFile *mf, std::string archive_name) {
+static void new_object_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_objs");
   count++;
 
   check_file_compatibility(ctx, rctx, mf);
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, archive_name);
+  ObjectFile<E> *file =
+    ctx.arena.template make<ObjectFile<E>>(ctx, mf, archive_name);
   ctx.obj_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
-  file->is_reachable =
-    !rctx.in_lib && (archive_name.empty() || rctx.whole_archive);
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
+  file->is_reachable = !file->as_needed;
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse_symbols(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
+}
+
+// IR files for LTO are not read in place. We only record them here,
+// and read_input_files() hands them to the LTO plugin once all input
+// files have been found. We do this because LTO object file reading
+// is order-dependent.
+template <typename E>
+static void defer_lto_obj(Context<E> &ctx, ReaderContext &rctx,
+                          MappedFile *mf, std::string archive_name) {
+  ReaderJob job;
+  job.rctx = rctx;
+  job.mf = mf;
+  job.archive_name = archive_name;
+  ctx.lto_jobs.push_back(std::move(job));
 }
 
 template <typename E>
-static ObjectFile<E> *new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
-                                  MappedFile *mf, std::string archive_name) {
+static void new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
+                        MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_lto_objs");
   count++;
 
   if (ctx.arg.ignore_ir_file.count(mf->get_identifier()))
-    return nullptr;
+    return;
 
   ObjectFile<E> *file = read_lto_object(ctx, mf);
-  file->priority = ctx.file_priority++;
-  file->archive_name = archive_name;
-  file->is_reachable =
-    !rctx.in_lib && (archive_name.empty() || rctx.whole_archive);
+  if (!file)
+    return;
 
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->archive_name = archive_name;
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
+  file->is_reachable = !file->as_needed;
+
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
 template <typename E>
-static SharedFile<E> *
-new_shared_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
+static void new_shared_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf) {
+  if (rctx.static_)
+    Fatal(ctx) << mf->name << ": attempted static link of a dynamic object";
+
   check_file_compatibility(ctx, rctx, mf);
 
-  SharedFile<E> *file = new SharedFile<E>(ctx, mf);
+  SharedFile<E> *file = ctx.arena.template make<SharedFile<E>>(ctx, mf);
   ctx.dso_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
-  file->is_reachable = !rctx.as_needed;
+  file->as_needed = rctx.as_needed;
+  file->is_reachable = !file->as_needed;
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
+// Reads a file inside an archive.
+template <typename E>
+static void read_archive_member(Context<E> &ctx, ReaderContext &rctx,
+                                MappedFile *mf, std::string archive_name) {
+  switch (get_file_type(ctx, mf)) {
+  case FileType::ELF_OBJ:
+    new_object_file(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::GCC_LTO_OBJ:
+  case FileType::LLVM_BITCODE:
+    defer_lto_obj(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::ELF_DSO:
+    Warn(ctx) << archive_name << "(" << mf->name
+              << "): shared object file in an archive is ignored";
+    break;
+  default:
+    break;
+  }
+}
+
+// Reads the given file, which is located at rctx.pos in the command
+// line. If the file is a container, i.e. an archive file or a linker
+// script, the files in it are read in place, recursively. IR files
+// for LTO are only recorded; see defer_lto_obj().
+//
+// read_input_files() reads top-level files with this function too but
+// overrides the container cases to read archive members in parallel.
 template <typename E>
 void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   switch (get_file_type(ctx, mf)) {
   case FileType::ELF_OBJ:
-    ctx.objs.push_back(new_object_file(ctx, rctx, mf, ""));
+    new_object_file(ctx, rctx, mf, "");
     return;
   case FileType::ELF_DSO:
-    ctx.dsos.push_back(new_shared_file(ctx, rctx, mf));
+    new_shared_file(ctx, rctx, mf);
     return;
   case FileType::AR:
   case FileType::THIN_AR:
     for (MappedFile *child : read_archive_members(ctx, mf)) {
-      switch (get_file_type(ctx, child)) {
-      case FileType::ELF_OBJ:
-        ctx.objs.push_back(new_object_file(ctx, rctx, child, mf->name));
-        break;
-      case FileType::GCC_LTO_OBJ:
-      case FileType::LLVM_BITCODE:
-        if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, child, mf->name))
-          ctx.objs.push_back(file);
-        break;
-      case FileType::ELF_DSO:
-        Warn(ctx) << mf->name << "(" << child->name
-                  << "): shared object file in an archive is ignored";
-        break;
-      default:
-        break;
-      }
+      ReaderContext rctx2 = rctx.next_child();
+      read_archive_member(ctx, rctx2, child, mf->name);
     }
     return;
   case FileType::TEXT:
@@ -124,8 +143,7 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
     return;
   case FileType::GCC_LTO_OBJ:
   case FileType::LLVM_BITCODE:
-    if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, mf, ""))
-      ctx.objs.push_back(file);
+    defer_lto_obj(ctx, rctx, mf, "");
     return;
   default:
     Fatal(ctx) << mf->name << ": unknown file type";
@@ -134,35 +152,23 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
 
 template <typename E>
 static std::string_view
-detect_machine_type(Context<E> &ctx, std::vector<std::string> args) {
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+detect_machine_type(Context<E> &ctx, std::span<ReaderJob> jobs) {
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) != FileType::TEXT)
-          if (std::string_view target = get_machine_type(ctx, rctx, mf);
+          if (std::string_view target = get_machine_type(ctx, job.rctx, mf);
               !target.empty())
             return target;
-    }
-  }
 
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) == FileType::TEXT)
           if (std::string_view target =
-              Script(ctx, rctx, mf).get_script_output_type();
+              Script(ctx, job.rctx, mf).get_script_output_type();
               !target.empty())
             return target;
-    }
-  }
 
   Fatal(ctx) << "-m option is missing";
 }
@@ -204,72 +210,169 @@ MappedFile *find_library(Context<E> &ctx, ReaderContext &rctx, std::string name)
   Fatal(ctx) << "library not found: " << name;
 }
 
+// Reads all input files.
+//
+// Reading input files is I/O- and CPU-intensive, and a large program
+// can easily consist of tens of thousands of them, so we want to read
+// files in parallel. The command line, on the other hand, is
+// inherently sequential: options such as --as-needed or
+// --whole-archive apply to the files after them, and a file's
+// priority for symbol resolution is its position in the command line.
+//
+// We reconcile the two as follows: the command line parser has
+// already recorded the reader state and the position for each file
+// argument in its ReaderJob. We open and read files in parallel and
+// then sort the files we've found back into the command line order to
+// assign priorities.
 template <typename E>
-static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
+static void read_input_files(Context<E> &ctx, std::vector<ReaderJob> &jobs) {
   Timer t(ctx, "read_input_files");
 
-  ReaderContext rctx;
-  std::vector<ReaderContext> stack;
-  std::unordered_set<std::string_view> visited;
+  // Open and read files in parallel. Archive files are expanded into
+  // one job per member so that members are read in parallel too.
+  //
+  // Linker scripts are one exception to the parallelism: they can
+  // modify the context, e.g. by defining symbol versions, so we only
+  // collect them here and parse them after this loop, one at a time
+  // and in the command line order, to keep their effects
+  // deterministic. Scripts given as input files are rare and small,
+  // such as the GROUP file that glibc installs as libc.so, so the
+  // lost parallelism doesn't matter.
+  //
+  // IR files for LTO are the other exception. This loop only records
+  // them, and we hand them to the LTO plugin after the scripts have
+  // been parsed, also in the command line order.
+  //
+  // Parsing scripts late assumes that no script directive affects how
+  // command line arguments after the script are read. That holds for
+  // the directives we currently support: a script can only add input
+  // files, whose positions order them correctly, and define symbol
+  // versions or symbols, which are not used until after this
+  // function. If we add a directive that doesn't satisfy this, such
+  // as SEARCH_DIR, which affects how -l arguments after it are
+  // resolved, this scheme needs to be revisited.
+  tbb::concurrent_vector<ReaderJob> scripts;
 
-  tbb::task_group tg;
-  rctx.tg = &tg;
+  tbb::parallel_for_each(jobs, [&](ReaderJob &job,
+                                   tbb::feeder<ReaderJob> &feeder) {
+    ReaderContext &rctx = job.rctx;
 
-  while (!args.empty()) {
-    std::string_view arg = args[0];
-    args = args.subspan(1);
-
-    if (arg == "--as-needed") {
-      rctx.as_needed = true;
-    } else if (arg == "--no-as-needed") {
-      rctx.as_needed = false;
-    } else if (arg == "--whole-archive") {
-      rctx.whole_archive = true;
-    } else if (arg == "--no-whole-archive") {
-      rctx.whole_archive = false;
-    } else if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (arg == "--start-lib") {
-      rctx.in_lib = true;
-    } else if (arg == "--end-lib") {
-      rctx.in_lib = false;
-    } else if (arg == "--push-state") {
-      stack.push_back(rctx);
-    } else if (arg == "--pop-state") {
-      if (stack.empty())
-        Fatal(ctx) << "no state pushed before popping";
-      rctx = stack.back();
-      stack.pop_back();
-    } else if (arg.starts_with("-l")) {
-      arg = arg.substr(2);
-      if (visited.contains(arg))
-        continue;
-      visited.insert(arg);
-
-      MappedFile *mf = find_library(ctx, rctx, std::string(arg));
-      mf->given_fullpath = false;
-      read_file(ctx, rctx, mf);
-    } else {
-      read_file(ctx, rctx, must_open_file(ctx, std::string(arg)));
+    // An archive member is enqueued by the job that read its archive
+    // file. A thin archive's members are opened here rather than by
+    // that job, so that the files of a large archive are opened in
+    // parallel.
+    if (!job.archive_name.empty()) {
+      MappedFile *mf = job.mf;
+      if (!mf) {
+        mf = must_open_file(ctx, job.name);
+        mf->thin_parent = job.thin_parent;
+      }
+      read_archive_member(ctx, rctx, mf, job.archive_name);
+      return;
     }
+
+    // Everything else is a command line argument that we need to open.
+    MappedFile *mf;
+    if (job.is_lib) {
+      mf = find_library(ctx, rctx, job.name);
+      mf->given_fullpath = false;
+    } else {
+      mf = must_open_file(ctx, job.name);
+    }
+
+    switch (get_file_type(ctx, mf)) {
+    case FileType::AR:
+      for (MappedFile *child : read_fat_archive_members(ctx, mf)) {
+        ReaderJob job2;
+        job2.rctx = rctx.next_child();
+        job2.mf = child;
+        job2.archive_name = mf->name;
+        feeder.add(std::move(job2));
+      }
+      break;
+    case FileType::THIN_AR:
+      for (std::string &path : get_thin_archive_member_paths(ctx, mf)) {
+        ReaderJob job2;
+        job2.rctx = rctx.next_child();
+        job2.name = std::move(path);
+        job2.archive_name = mf->name;
+        job2.thin_parent = mf;
+        feeder.add(std::move(job2));
+      }
+      break;
+    case FileType::TEXT:
+      job.mf = mf;
+      scripts.push_back(std::move(job));
+      break;
+    default:
+      read_file(ctx, rctx, mf);
+    }
+  });
+
+  // Parse linker scripts and read the files they name.
+  ranges::sort(scripts, {}, [](const ReaderJob &job) {
+    return job.rctx.pos;
+  });
+
+  for (ReaderJob &job : scripts)
+    Script(ctx, job.rctx, job.mf).parse_linker_script();
+
+  // Hand IR files to the LTO plugin, in the command line order. The
+  // plugin keeps global state, so claims can't run in parallel anyway,
+  // and LLVM's LTO assumes that the module holding the prevailing copy
+  // of a COMDAT group arrives before the modules holding the other
+  // copies, as it does with a serial linker. Otherwise the losing
+  // copies' internal symbols become undefined references in the LTO
+  // result.
+  ranges::sort(ctx.lto_jobs, {}, [](const ReaderJob &job) {
+    return job.rctx.pos;
+  });
+
+  for (ReaderJob &job : ctx.lto_jobs)
+    new_lto_obj(ctx, job.rctx, job.mf, job.archive_name);
+
+  // Sort the files into the command line order and assign priorities.
+  tbb::parallel_sort(ctx.unsorted_input_files, [](auto &a, auto &b) {
+    return a.first < b.first;
+  });
+
+  for (auto &pair : ctx.unsorted_input_files) {
+    InputFile<E> *file = pair.second;
+    file->priority = ctx.file_priority++;
+    if (ctx.arg.trace)
+      Out(ctx) << "trace: " << *file;
+
+    if (file->is_dso)
+      ctx.dsos.push_back(file->to_dso());
+    else
+      ctx.objs.push_back(file->to_obj());
   }
 
-  if (ctx.objs.empty())
-    Fatal(ctx) << "no input files";
+  ctx.unsorted_input_files.clear();
 
-  tg.wait();
+  if (ctx.objs.empty() && ctx.dsos.empty())
+    Fatal(ctx) << "no input files";
 }
 
 template <typename E>
 static bool has_lto_obj(Context<E> &ctx) {
   for (ObjectFile<E> *file : ctx.objs)
-    if (file->is_reachable && (file->is_lto_obj || file->is_gcc_offload_obj))
+    if (file->is_reachable && (file->is_lto_input || file->is_gcc_offload_obj))
       return true;
   return false;
 }
 
+template <typename E>
+static i64 get_thread_count(Context<E> &ctx) {
+  if (ctx.arg.thread_count.has_value())
+    return *ctx.arg.thread_count;
+
+  // mold doesn't scale well with too many threads, so limit it to 32.
+  int n = tbb::global_control::active_value(
+    tbb::global_control::max_allowed_parallelism);
+  return std::min(n, 32);
+}
+
 template <typename E>
 int mold_main(int argc, char **argv) {
   Context<E> ctx;
@@ -278,36 +381,39 @@ int mold_main(int argc, char **argv) {
   if (argc >= 2 && (argv[1] == "-run"sv || argv[1] == "--run"sv))
     process_run_subcommand(ctx, argc, argv);
 
+  // parse_nonpositional_args() may chdir(2) for -C. If we end up
+  // restarting in redo_main(), we need to re-enter from the original
+  // directory so relative paths (e.g. response files) still resolve.
+  std::error_code ec;
+  std::filesystem::path orig_cwd = std::filesystem::current_path(ec);
+
   // Parse non-positional command line options
   ctx.cmdline_args = expand_response_files(ctx, argv);
-  std::vector<std::string> file_args = parse_nonpositional_args(ctx);
+  std::vector<ReaderJob> jobs = parse_nonpositional_args(ctx);
 
   // If no -m option is given, deduce it from input files.
   if (ctx.arg.emulation.empty())
-    ctx.arg.emulation = detect_machine_type(ctx, file_args);
+    ctx.arg.emulation = detect_machine_type(ctx, jobs);
 
-  // Redo if -m is not x86-64.
-  if constexpr (is_x86_64<E>)
-    if (ctx.arg.emulation != X86_64::name)
-      return redo_main(ctx, argc, argv);
+  // Redo if -m does not match with our speculation.
+  if (ctx.arg.emulation != E::name) {
+    if (!ec)
+      std::filesystem::current_path(orig_cwd, ec);
+    return redo_main<E>(ctx.arg.emulation, argc, argv);
+  }
 
   Timer t_all(ctx, "all");
 
   install_signal_handler();
 
-  if (!ctx.arg.directory.empty())
-    if (chdir(ctx.arg.directory.c_str()) == -1)
-      Fatal(ctx) << "chdir failed: " << ctx.arg.directory
-                 << ": " << errno_string();
-
   // Fork a subprocess unless --no-fork is given.
   if (ctx.arg.fork)
-    fork_child();
+    fork_child<E>();
 
   acquire_global_lock();
 
-  tbb::global_control tbb_cont(tbb::global_control::max_allowed_parallelism,
-                               ctx.arg.thread_count);
+  ctx.global_limit.emplace(tbb::global_control::max_allowed_parallelism,
+                           get_thread_count(ctx));
 
   // Handle --wrap options if any.
   for (std::string_view name : ctx.arg.wrap)
@@ -322,7 +428,7 @@ int mold_main(int argc, char **argv) {
     get_symbol(ctx, arg)->is_traced = true;
 
   // Parse input files
-  read_input_files(ctx, file_args);
+  read_input_files(ctx, jobs);
 
   // Uniquify shared object files by soname
   {
@@ -360,9 +466,25 @@ int mold_main(int argc, char **argv) {
   std::erase_if(ctx.objs, [](InputFile<E> *file) { return !file->is_reachable; });
   std::erase_if(ctx.dsos, [](InputFile<E> *file) { return !file->is_reachable; });
 
+  // Building .gdb_index is split into three stages because the required data
+  // becomes available at different points in the link. Compilation units and
+  // public names depend only on input sections, so read them now in a
+  // low-priority arena while foreground passes continue.
+  bool create_gdb_index = ctx.arg.gdb_index && !ctx.arg.relocatable;
+  tbb::task_arena gdb_input_arena(tbb::task_arena::automatic, 1,
+                                  tbb::task_arena::priority::low);
+  tbb::task_group gdb_task;
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] {
+      gdb_task.run([&] { read_gdb_index_inputs(ctx); });
+    });
+
   // Parse .eh_frame section contents.
   parse_eh_frame_sections(ctx);
 
+  // Parse .sframe section contents.
+  parse_sframe_sections(ctx);
+
   // Split mergeable section contents into section pieces.
   create_merged_sections(ctx);
 
@@ -374,6 +496,21 @@ int mold_main(int argc, char **argv) {
     return 0;
   }
 
+  // Non-allocated merged sections are independent of symbol and relocation
+  // processing. Resolve them while the remaining foreground passes run.
+  tbb::task_arena merge_arena(tbb::task_arena::automatic, 1,
+                              tbb::task_arena::priority::low);
+  tbb::task_group merge_task;
+  merge_arena.execute([&] {
+    merge_task.run([&] {
+      tbb::parallel_for_each(ctx.merged_sections,
+                             [&](ArenaObjectPtr<MergedSection<E>> &sec) {
+        if (!(sec->shdr.sh_flags & SHF_ALLOC))
+          sec->resolve(ctx);
+      });
+    });
+  });
+
   // Create .bss sections for common symbols.
   convert_common_symbols(ctx);
 
@@ -390,6 +527,11 @@ int mold_main(int argc, char **argv) {
   if (!ctx.arg.allow_multiple_definition)
     check_duplicate_symbols(ctx);
 
+  // Handle --zero-to-bss, which converts data sections containing only
+  // zeros into BSS.
+  if (ctx.arg.zero_to_bss)
+    convert_zero_to_bss(ctx);
+
   // Set "address-taken" bits for input sections.
   if (ctx.arg.icf)
     compute_address_significance(ctx);
@@ -489,31 +631,55 @@ int mold_main(int argc, char **argv) {
   // .got.plt, .dynsym, .dynstr, etc.
   scan_relocations(ctx);
 
-  // Compute the is_weak bit for each imported symbol.
-  compute_imported_symbol_weakness(ctx);
+  // Now that we know all exported symbols, make sure that no versioned
+  // name is defined twice.
+  check_symbol_version_conflicts(ctx);
+
+  merge_arena.execute([&] { merge_task.wait(); });
 
   // Sort sections by section attributes so that we'll have to
   // create as few segments as possible.
   sort_output_sections(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  // Handle --separate-debug-file.
+  if (ctx.gnu_debuglink)
     separate_debug_sections(ctx);
 
   // Compute sizes of output sections while assigning offsets
   // within an output section to input sections.
   compute_section_sizes(ctx);
 
-  // If --packed_dyn_relocs=relr was given, base relocations are stored
-  // to a .relr.dyn section in a compressed form. Construct a compressed
-  // relocations now so that we can fix section sizes and file layout.
+  // RELR is encoded independently for each output chunk using offsets
+  // relative to that chunk.
   if (ctx.arg.pack_dyn_relocs_relr)
-    construct_relr(ctx);
+    ctx.reldyn->construct_relr(ctx);
 
   // Reserve a space for dynamic symbol strings in .dynstr and sort
   // .dynsym contents if necessary. Beyond this point, no symbol will
   // be added to .dynsym.
   sort_dynsyms(ctx);
 
+  // sort_debug_info_sections may uncompress the same .debug_info sections.
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] { gdb_task.wait(); });
+
+  // Sort .debug_info contents so that DWARF32 debug info precedes that of
+  // DWARF64. This is to mitigate the possibility of a relocation overflow.
+  sort_debug_info_sections(ctx);
+
+  // Type vectors identify compilation units by their order in the output
+  // .debug_info section. That order is now fixed, so build the table while the
+  // remaining layout passes continue. This stage stops scaling after about 12
+  // workers, so limit its arena to avoid competing with foreground work.
+  i64 gdb_table_workers =
+    std::clamp<i64>(get_thread_count(ctx) * 3 / 8, 1, 12);
+  tbb::task_arena gdb_table_arena(gdb_table_workers, 1,
+                                  tbb::task_arena::priority::low);
+  if (ctx.gdb_index && !ctx.gnu_debuglink)
+    gdb_table_arena.execute([&] {
+      gdb_task.run([&] { build_gdb_index_tables(ctx); });
+    });
+
   // Print reports about undefined symbols, if needed.
   if (ctx.arg.unresolved_symbols == UNRESOLVED_ERROR)
     report_undef_errors(ctx);
@@ -531,18 +697,16 @@ int mold_main(int argc, char **argv) {
   // Here, we construct output .eh_frame contents.
   ctx.eh_frame->construct(ctx);
 
+  // .sframe is likewise parsed and reconstructed by the linker. Build
+  // the merged, PC-sorted output .sframe.
+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
   // If --emit-relocs is given, we'll copy relocation sections from input
   // files to an output file.
   if (ctx.arg.emit_relocs)
     create_reloc_sections(ctx);
 
-  // We've created range extension thunks with a pessimistive assumption
-  // that all out-of-section references are out of range. Now that we are
-  // able to assign addresses to all SHF_ALLOC output sections, we can
-  // eliminate excessive thunks.
-  if constexpr (needs_thunk<E>)
-    remove_redundant_thunks(ctx);
-
   // Compute .symtab and .strtab sizes for each file.
   if (!ctx.arg.strip_all)
     create_output_symtab(ctx);
@@ -562,6 +726,14 @@ int mold_main(int argc, char **argv) {
     filesize = set_osec_offsets(ctx);
   }
 
+  // We've created range extension thunks with a pessimistive assumption
+  // that all out-of-section references are out of range. Now that we know
+  // the addresses of all sections,, we can eliminate excessive thunks.
+  if constexpr (needs_thunk<E>) {
+    remove_redundant_thunks(ctx);
+    filesize = set_osec_offsets(ctx);
+  }
+
   if constexpr (is_arm32<E>) {
     if (ctx.extra.exidx) {
       ctx.extra.exidx->remove_duplicate_entries(ctx);
@@ -578,18 +750,29 @@ int mold_main(int argc, char **argv) {
   // GOT or PLT addresses have a correct final value.
 
   // If --compress-debug-sections is given, compress .debug_* sections
-  // using zlib.
-  if (ctx.arg.compress_debug_sections != COMPRESS_NONE) {
+  // using zlib or zstd.
+  if (ctx.arg.compress_debug_sections != ELFCOMPRESS_NONE) {
     compress_debug_sections(ctx);
     filesize = set_osec_offsets(ctx);
   }
 
-  // At this point, both memory and file layouts are fixed.
-
   // Gather thunk symbols and attach them to themselves.
   if constexpr (needs_thunk<E>)
     gather_thunk_addresses(ctx);
 
+  // Re-finalize layout. fix_synthetic_symbols above may have changed
+  // addends for dynamic relocations referencing synthetic symbols, which
+  // can shift the encoded size of .rela.dyn under --pack-dyn-relocs=android
+  // because Android's packed format encodes addends in variable-length
+  // SLEB128. Other modes, including ordinary RELR, encode nothing whose
+  // size depends on addends, so they do not need this pass.
+  if (ctx.arg.pack_dyn_relocs_android)
+    filesize = set_osec_offsets(ctx);
+
+  // At this point, both memory and file layouts are fixed.
+
+  ctx.reldyn->update_shdr(ctx);
+
   t_before_copy.stop();
 
   // Create an output file
@@ -601,7 +784,10 @@ int mold_main(int argc, char **argv) {
   // Copy input sections to the output file and apply relocations.
   copy_chunks(ctx);
 
-  if constexpr (is_x86_64<E>)
+  if constexpr (is_arm32be<E>)
+    arm32be_swap_bytes(ctx);
+
+  if constexpr (is_x86_64<E> || is_arm64<E>)
     if (ctx.arg.z_rewrite_endbr)
       rewrite_endbr(ctx);
 
@@ -609,11 +795,12 @@ int mold_main(int argc, char **argv) {
   // so we sort them.
   sort_reldyn(ctx);
 
-  // .gdb_index's contents cannot be constructed before applying
-  // relocations to other debug sections. We have relocated debug
-  // sections now, so write the .gdb_index section.
-  if (ctx.gdb_index && ctx.arg.separate_debug_file.empty())
+  // The final stage reads address ranges, which requires relocated debug
+  // sections. We have applied the relocations now, so finish the index.
+  if (ctx.gdb_index && !ctx.gnu_debuglink) {
+    gdb_table_arena.execute([&] { gdb_task.wait(); });
     write_gdb_index(ctx);
+  }
 
   // .note.gnu.build-id section contains a cryptographic hash of the
   // entire output file. Now that we wrote everything except build-id,
@@ -621,7 +808,7 @@ int mold_main(int argc, char **argv) {
   if (ctx.buildid)
     write_build_id(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
     write_gnu_debuglink(ctx);
 
   t_copy.stop();
@@ -642,7 +829,7 @@ int mold_main(int argc, char **argv) {
   if (ctx.arg.print_map)
     print_map(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
     write_separate_debug_file(ctx);
 
   // Show stats numbers
@@ -655,14 +842,22 @@ int mold_main(int argc, char **argv) {
   std::cout << std::flush;
   std::cerr << std::flush;
 
-  notify_parent();
+  notify_parent<E>();
   release_global_lock();
 
+#if HAVE_MADVISE
+  // Dropping page table entries here in parallel makes process exit
+  // faster, as the kernel otherwise reclaims them in a single thread
+  // on exit. File contents stay in the page cache.
+  tbb::parallel_for_each(ctx.mf_pool, [](std::unique_ptr<MappedFile> &mf) {
+    if (mf->is_mmapped && !mf->parent && mf->data && mf->size)
+      madvise(mf->data, mf->size, MADV_DONTNEED);
+  });
+#endif
+
   if (ctx.arg.quick_exit)
     _exit(0);
 
-  for (std::function<void()> &fn : ctx.on_exit)
-    fn();
   ctx.checkpoint();
   return 0;
 }
@@ -670,5 +865,6 @@ int mold_main(int argc, char **argv) {
 using E = MOLD_TARGET;
 
 template int mold_main<E>(int, char **);
+template void read_file(Context<E> &, ReaderContext &, MappedFile *);
 
 } // namespace mold

src/mapfile.cc

@@ -19,14 +19,12 @@ static Map<E> get_map(Context<E> &ctx) {
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     for (Symbol<E> *sym : file->symbols) {
-      if (sym->file != file || sym->get_type() == STT_SECTION)
-        continue;
-
-      if (InputSection<E> *isec = sym->get_input_section()) {
-        assert(file == &isec->file);
-        typename Map<E>::accessor acc;
-        map.insert(acc, {isec, {}});
-        acc->second.push_back(sym);
+      if (sym->file == file && sym->get_type() != STT_SECTION) {
+        if (InputSection<E> *isec = sym->get_input_section()) {
+          typename Map<E>::accessor acc;
+          map.insert(acc, {isec, {}});
+          acc->second.push_back(sym);
+        }
       }
     }
   });
@@ -48,10 +46,11 @@ void print_map(Context<E> &ctx) {
   std::ostream *out = &std::cout;
   std::ofstream file;
 
-  if (!ctx.arg.Map.empty()) {
+  if (!ctx.arg.Map.empty() && ctx.arg.Map != "-") {
     file.open(ctx.arg.Map);
-    if (!file.is_open())
-      Fatal(ctx) << "cannot open " << ctx.arg.Map << ": " << errno_string();
+    if (file.fail())
+      Fatal(ctx) << "--print-map: cannot open " << ctx.arg.Map << ": "
+                 << errno_string();
     out = &file;
   }
 
@@ -72,7 +71,7 @@ void print_map(Context<E> &ctx) {
     if (!osec)
       continue;
 
-    std::span<InputSection<E> *> members = osec->members;
+    std::span<ArenaPtr<InputSection<E>>> members = osec->members;
     std::vector<std::string> bufs(members.size());
 
     tbb::parallel_for((i64)0, (i64)members.size(), [&](i64 i) {

src/mapped-file-unix.cc

@@ -0,0 +1,83 @@
+#include "mold.h"
+
+namespace mold {
+
+// Files up to this size are read into malloc'ed memory rather than
+// mmap'ed. mmap(2) takes the process's address space lock, so with tens
+// of thousands of input files, the calls serialize at a few microseconds
+// each no matter how many threads make them. read(2) takes no such lock,
+// but copying costs memory bandwidth in proportion to the file size, so
+// large files are still mmap'ed. On a Chromium link, files below this
+// threshold are 64% of the inputs by count but 16% by size.
+//
+// The buffers add up to hundreds of megabytes on such a link, so they
+// need to be backed by huge pages; with 4 KiB pages, faulting them in
+// costs more than the mmap calls did. mimalloc does that by default.
+static constexpr i64 READ_THRESHOLD = 32 * 1024;
+
+MappedFile *open_file_impl(const std::string &path, std::string &error) {
+  i64 fd = ::open(path.c_str(), O_RDONLY);
+  if (fd == -1) {
+    if (errno != ENOENT)
+      error = "opening " + path + " failed: " + errno_string();
+    return nullptr;
+  }
+
+  struct stat st;
+  if (fstat(fd, &st) == -1)
+    error = path + ": fstat failed: " + errno_string();
+
+  MappedFile *mf = new MappedFile;
+  mf->name = path;
+  mf->size = st.st_size;
+
+  if (st.st_size > 0 && st.st_size <= READ_THRESHOLD && S_ISREG(st.st_mode)) {
+    u8 *buf = (u8 *)malloc(st.st_size);
+    for (i64 pos = 0; pos < st.st_size;) {
+      i64 n = pread(fd, buf + pos, st.st_size - pos, pos);
+      if (n > 0) {
+        pos += n;
+      } else if (n == 0) {
+        error = path + ": file is shorter than its reported size";
+        break;
+      } else if (errno != EINTR) {
+        error = path + ": read failed: " + errno_string();
+        break;
+      }
+    }
+    mf->data = buf;
+  } else if (st.st_size > 0) {
+    mf->data = (u8 *)mmap(nullptr, st.st_size, PROT_READ | PROT_WRITE,
+                          MAP_PRIVATE, fd, 0);
+    if (mf->data == MAP_FAILED)
+      error = path + ": mmap failed: " + errno_string();
+    mf->is_mmapped = true;
+  }
+
+  close(fd);
+  return mf;
+}
+
+void MappedFile::unmap() {
+  if (size == 0 || parent || !data)
+    return;
+  if (is_mmapped)
+    munmap(data, size);
+  else
+    free(data);
+  data = nullptr;
+}
+
+void MappedFile::close_fd() {
+  if (fd == -1)
+    return;
+  close(fd);
+  fd = -1;
+}
+
+void MappedFile::reopen_fd(const std::string &path) {
+  if (fd == -1)
+    fd = open(path.c_str(), O_RDONLY);
+}
+
+} // namespace mold

src/mapped-file-win32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 namespace mold {
 
@@ -48,6 +48,7 @@ MappedFile *open_file_impl(const std::string &path, std::string &error) {
       error = path + ": MapViewOfFile failed: " + errno_string();
       return nullptr;
     }
+    mf->is_mmapped = true;
   }
 
   return mf;

src/output-file-unix.cc

@@ -1,3 +1,4 @@
+#include "config.h"
 #include "mold.h"
 
 #include <fcntl.h>
@@ -6,6 +7,11 @@
 #include <sys/mman.h>
 #include <system_error>
 
+#ifdef __linux__
+# include <sys/vfs.h>
+# include <linux/magic.h>
+#endif
+
 namespace mold {
 
 static u32 get_umask() {
@@ -40,8 +46,7 @@ public:
   MemoryMappedOutputFile(Context<E> &ctx, std::string path, i64 filesize, int perm)
     : OutputFile<E>(path, filesize, true) {
     std::string pid = std::to_string(getpid());
-    std::string tmpfile =
-      path_dirname(path) / ("." + path_filename(path) + "." + pid);
+    std::string tmpfile = path_dirname(path) / ("."s + path_filename(path) + "." + pid);
 
     this->fd = open_or_create_file(ctx, path, tmpfile, perm);
 
@@ -53,14 +58,34 @@ public:
 
     output_tmpfile = (char *)save_string(ctx, tmpfile).data();
 
-#ifdef __linux__
-    fallocate(this->fd, 0, 0, filesize);
+#if HAVE_FALLOCATE
+    // Calling fallocate speeds up later linking passes on ext4 by
+    // taking disk block allocation out of the page-fault handler.
+    // On tmpfs, it instead makes things much slower: the kernel
+    // allocates and zeroes every page of the range inside the
+    // syscall, on one thread, which takes ~1.8 s for a 5 GiB file.
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, 0, filesize);
 #endif
 
-    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+    // We map the file with twice as much address space as its size, so
+    // that extend() can grow the file into the mapping in place.
+    // Touching the mapping beyond the end of the file is not allowed,
+    // but growing the file with ftruncate makes the tail of the
+    // mapping accessible without any further mmap call.
+    vasize = filesize * 2;
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                            MAP_SHARED, this->fd, 0);
-    if (this->buf == MAP_FAILED)
-      Fatal(ctx) << path << ": mmap failed: " << errno_string();
+
+    if (this->buf == MAP_FAILED) {
+      // If the address space is too tight, map just the file.
+      vasize = filesize;
+      this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << path << ": mmap failed: " << errno_string();
+    }
 
     mold::output_buffer_start = this->buf;
     mold::output_buffer_end = this->buf + filesize;
@@ -71,20 +96,51 @@ public:
       ::close(fd2);
   }
 
+  // Extend the file so that the caller can fill the appended data
+  // through the tail of the mapping. This is called at most once per
+  // output file.
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    i64 mapsize = this->filesize;
+
+    if (ftruncate(this->fd, mapsize + size) == -1)
+      Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+#if HAVE_FALLOCATE
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, mapsize, size);
+#endif
+
+    if (mapsize + size > vasize) {
+      // The appended data does not fit in the mapping. Map the grown
+      // file again, moving the buffer.
+      munmap(this->buf, vasize);
+      vasize = mapsize + size;
+
+      this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+      ctx.buf = this->buf;
+      mold::output_buffer_start = this->buf;
+    }
+
+    this->filesize += size;
+    mold::output_buffer_end = this->buf + mapsize + size;
+
+#ifdef MADV_HUGEPAGE
+    madvise(this->buf, mapsize + size, MADV_HUGEPAGE);
+#endif
+    return this->buf + mapsize;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     if (!this->is_unmapped)
-      munmap(this->buf, this->filesize);
-
-    if (this->buf2.empty()) {
-      ::close(this->fd);
-    } else {
-      FILE *out = fdopen(this->fd, "w");
-      fseek(out, 0, SEEK_END);
-      fwrite(&this->buf2[0], this->buf2.size(), 1, out);
-      fclose(out);
-    }
+      munmap(this->buf, vasize);
+    ::close(this->fd);
 
     // If an output file already exists, open a file and then remove it.
     // This is the fastest way to unlink a file, as it does not make the
@@ -100,6 +156,9 @@ public:
 
 private:
   int fd2 = -1;
+
+  // Size of the file mapping, which may extend past the end of the file.
+  i64 vasize = 0;
 };
 
 template <typename E>
@@ -122,14 +181,7 @@ OutputFile<E>::open(Context<E> &ctx, std::string path, i64 filesize, int perm) {
 
 #ifdef MADV_HUGEPAGE
   // Enable transparent huge page for an output memory-mapped file.
-  // On Linux, it has an effect only on tmpfs mounted with `huge=advise`,
-  // but it can make the linker ~10% faster. You can try it by creating
-  // a tmpfs with the following commands
-  //
-  //  $ mkdir tmp
-  //  $ sudo mount -t tmpfs -o size=2G,huge=advise none tmp
-  //
-  // and then specifying a path under the directory as an output file.
+  // Linking a Chromium debug build is ~20% faster with this madvise call.
   madvise(file->buf, filesize, MADV_HUGEPAGE);
 #endif
 
@@ -161,10 +213,20 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
   if (ftruncate(this->fd, filesize) == -1)
     Fatal(ctx) << "ftruncate failed: " << errno_string();
 
-  this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+  // As in MemoryMappedOutputFile, we map the file with twice as much
+  // address space as its size so that extend() can grow the file into
+  // the mapping in place.
+  vasize = filesize * 2;
+  this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                          MAP_SHARED, this->fd, 0);
-  if (this->buf == MAP_FAILED)
-    Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+  if (this->buf == MAP_FAILED) {
+    vasize = filesize;
+    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+  }
 
   this->filesize = filesize;
   mold::output_buffer_start = this->buf;
@@ -172,17 +234,36 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
 }
 
 template <typename E>
-void LockingOutputFile<E>::close(Context<E> &ctx) {
-  if (!this->is_unmapped)
-    munmap(this->buf, this->filesize);
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  i64 mapsize = this->filesize;
 
-  if (!this->buf2.empty()) {
-    FILE *out = fdopen(this->fd, "w");
-    fseek(out, 0, SEEK_END);
-    fwrite(&this->buf2[0], this->buf2.size(), 1, out);
-    fclose(out);
+  if (ftruncate(this->fd, mapsize + size) == -1)
+    Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+  if (mapsize + size > vasize) {
+    // The appended data does not fit in the mapping. Map the grown
+    // file again, moving the buffer.
+    munmap(this->buf, vasize);
+    vasize = mapsize + size;
+
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
   }
 
+  this->filesize += size;
+  mold::output_buffer_end = this->buf + mapsize + size;
+  return this->buf + mapsize;
+}
+
+template <typename E>
+void LockingOutputFile<E>::close(Context<E> &ctx) {
+  if (!this->is_unmapped)
+    munmap(this->buf, vasize);
   ::close(this->fd);
 }
 

src/output-file-win32.cc

@@ -39,22 +39,36 @@ public:
       CloseHandle(handle);
   }
 
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    UnmapViewOfFile(this->buf);
+
+    HANDLE map = CreateFileMapping(handle, nullptr, PAGE_READWRITE, 0,
+                                   this->filesize + size, nullptr);
+    if (!map)
+      Fatal(ctx) << this->path << ": CreateFileMapping failed: "
+                 << GetLastError();
+
+    this->buf = (u8 *)MapViewOfFile(map, FILE_MAP_WRITE, 0, 0,
+                                    this->filesize + size);
+    if (!this->buf)
+      Fatal(ctx) << this->path << ": MapViewOfFile failed: " << GetLastError();
+
+    CloseHandle(map);
+
+    u8 *space = this->buf + this->filesize;
+    memset(space, 0, size);
+
+    this->filesize += size;
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
+    mold::output_buffer_end = this->buf + this->filesize;
+    return space;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     UnmapViewOfFile(this->buf);
-
-    if (!this->buf2.empty()) {
-      if (SetFilePointer(handle, 0, nullptr, FILE_END) == INVALID_SET_FILE_POINTER)
-        Fatal(ctx) << this->path << ": SetFilePointer failed: "
-                   << GetLastError();
-
-      DWORD written;
-      if (!WriteFile(handle, this->buf2.data(), this->buf2.size(), &written,
-                     nullptr))
-        Fatal(ctx) << this->path << ": WriteFile failed: " << GetLastError();
-    }
-
     CloseHandle(handle);
     handle = INVALID_HANDLE_VALUE;
   }
@@ -106,6 +120,11 @@ LockingOutputFile<E>::LockingOutputFile(Context<E> &ctx, std::string path,
 template <typename E>
 void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {}
 
+template <typename E>
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  return nullptr;
+}
+
 template <typename E>
 void LockingOutputFile<E>::close(Context<E> &ctx) {}
 

src/relocatable.cc

@@ -50,6 +50,8 @@ static void r_create_synthetic_sections(Context<E> &ctx) {
   ctx.shdr = push(new OutputShdr<E>);
   ctx.eh_frame = push(new EhFrameSection<E>);
   ctx.eh_frame_reloc = push(new EhFrameRelocSection<E>);
+  ctx.sframe = push(new SFrameSection<E>);
+  ctx.sframe_reloc = push(new SFrameRelocSection<E>);
   ctx.strtab = push(new StrtabSection<E>);
   ctx.symtab = push(new SymtabSection<E>);
   ctx.shstrtab = push(new ShstrtabSection<E>);
@@ -74,16 +76,17 @@ static void create_comdat_group_sections(Context<E> &ctx) {
     ObjectFile<E> &file = *ctx.objs[i];
 
     for (ComdatGroupRef<E> &ref : file.comdat_groups) {
-      if (ref.group->owner != file.priority)
+      if (!ref.is_owner)
         continue;
 
       Symbol<E> *sym = file.symbols[file.elf_sections[ref.sect_idx].sh_info];
       assert(sym);
 
       std::vector<Chunk<E> *> members;
-      for (u32 j : ref.members) {
+      for (u32 j : ref.members(file)) {
         const ElfShdr<E> &shdr = file.elf_sections[j];
-        if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL)) {
+        if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL) ||
+            shdr.sh_type == SHT_CREL) {
           InputSection<E> &isec = *file.sections[shdr.sh_info];
           members.push_back(isec.output_section->reloc_sec.get());
         } else {
@@ -125,7 +128,7 @@ static void r_claim_unresolved_symbols(Context<E> &ctx) {
         continue;
 
       sym.file = file;
-      sym.origin = 0;
+      sym.origin = nullptr;
       sym.value = 0;
       sym.sym_idx = i;
     }
@@ -161,6 +164,9 @@ void combine_objects(Context<E> &ctx) {
 
   ctx.eh_frame->construct(ctx);
 
+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
   create_reloc_sections(ctx);
 
   create_comdat_group_sections(ctx);

src/shrink-sections.cc

@@ -89,8 +89,8 @@ void shrink_sections(Context<E> &ctx) {
   // only ~0.04% larger than that of GNU ld), so we don't bother to handle
   // them. We scan relocations only once here.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
-      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
+    for (InputSection<E> *isec : file->sections)
+      if (isec && isec->is_alive() && (isec->shdr().sh_flags & SHF_EXECINSTR))
         shrink_section(ctx, *isec);
   });
 
@@ -122,10 +122,6 @@ i64 compute_distance(Context<E> &ctx, Symbol<E> &sym,
   if (sym.is_absolute())
     return INT64_MAX;
 
-  // Likewise, relocations against weak undefined symbols won't be relaxed.
-  if (sym.esym().is_undef_weak())
-    return INT64_MAX;
-
   // Compute a distance between the relocated place and the symbol.
   i64 S = sym.get_addr(ctx);
   i64 A = rel.r_addend;

src/signal-unix.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 #include <csignal>
 #include <tbb/version.h>
@@ -28,8 +28,7 @@ void cleanup() {
 // for the first time.
 //
 // If a disk becomes full as a result of a write to an mmap'ed memory
-// region, the failure of the write is reported as a SIGBUS or structured
-// exeption with code EXCEPTION_IN_PAGE_ERROR on Windows. This
+// region, the failure of the write is reported as a SIGBUS. This
 // signal handler catches that signal and prints out a user-friendly
 // error message. Without this, it is very hard to realize that the
 // disk might be full.
@@ -74,7 +73,7 @@ void install_signal_handler() {
   sigaction(SIGBUS, &action, NULL);
 
   // OneTBB 2021.9.0 has the interface version 12090.
-  if (TBB_runtime_interface_version() < 12090) {
+  if (TBB_runtime_interface_version() <= 12090) {
     sigabrt_msg = "mold: aborted\n"
       "mold: mold with libtbb version 2021.9.0 or older is known to be unstable "
       "under heavy load. Your libtbb version is " +

src/signal-win32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 #include <windows.h>
 

src/subprocess-unix.cc

@@ -1,3 +1,4 @@
+#include "config.h"
 #include "mold.h"
 
 #include <csignal>
@@ -10,12 +11,12 @@
 
 namespace mold {
 
-#ifdef MOLD_X86_64
 static int pipe_write_fd = -1;
 
 // Exiting from a program with large memory usage is slow --
 // it may take a few hundred milliseconds. To hide the latency,
 // we fork a child and let it do the actual linking work.
+template <typename E>
 void fork_child() {
   int pipefd[2];
   if (pipe(pipefd) == -1) {
@@ -52,6 +53,7 @@ void fork_child() {
   pipe_write_fd = pipefd[1];
 }
 
+template <typename E>
 void notify_parent() {
   if (pipe_write_fd == -1)
     return;
@@ -61,7 +63,6 @@ void notify_parent() {
   assert(n == 1);
   pipe_write_fd = -1;
 }
-#endif
 
 template <typename E>
 static std::string find_dso(Context<E> &ctx, std::filesystem::path self) {
@@ -104,7 +105,7 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
   setenv("LD_PRELOAD", dso_path.c_str(), 1);
   setenv("MOLD_PATH", self.c_str(), 1);
 
-  // If ld, ld.lld or ld.gold is specified, run mold itself
+  // If ld, ld.lld or ld.gold is specified, run mold instead
   if (std::string cmd = path_filename(argv[2]);
       cmd == "ld" || cmd == "ld.lld" || cmd == "ld.gold") {
     std::vector<char *> args;
@@ -123,6 +124,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
 
 using E = MOLD_TARGET;
 
+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);
 
 } // namespace mold

src/subprocess-win32.cc

@@ -2,10 +2,11 @@
 
 namespace mold {
 
-#ifdef MOLD_X86_64
+template <typename E>
 void fork_child() {}
+
+template <typename E>
 void notify_parent() {}
-#endif
 
 template <typename E>
 [[noreturn]]
@@ -15,6 +16,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
 
 using E = MOLD_TARGET;
 
+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);
 
 } // namespace mold

src/thunks.cc

@@ -11,17 +11,12 @@
 //
 // The function in this file creates thunks.
 //
-// Note that although thunks play an important role in an executable, they
-// don't take up too much space in it. For example, among the clang-16's
-// text segment whose size is ~300 MiB on ARM64, thunks in total occupy
-// only ~30 KiB or 0.01%. Of course the number depends on an ISA; we would
-// need more thunks on ARM32 whose branch range is shorter than ARM64.
-// That said, the total size of thunks still isn't that much. Therefore,
-// we don't need to try too hard to reduce thunk size to the absolute
-// minimum.
+// Thunk size varies widely across programs. In an ARM64 build of Clang 16,
+// thunks occupy about 30 KiB (0.01%) of a ~300 MiB text section, compared
+// with about 12.5 MiB (2.5%) of a ~500 MiB text section in TensorFlow.
 
-#if MOLD_ARM32 || MOLD_ARM64LE || MOLD_ARM64BE || MOLD_PPC32 || \
-    MOLD_PPC64V1 || MOLD_PPC64V2
+#if MOLD_ARM32LE || MOLD_ARM32BE || MOLD_ARM64LE || MOLD_ARM64BE || \
+    MOLD_PPC32 || MOLD_PPC64V1 || MOLD_PPC64V2
 
 #include "mold.h"
 
@@ -32,97 +27,104 @@ namespace mold {
 
 using E = MOLD_TARGET;
 
-// We create thunks for each 25.6/3.2/6.4 MiB code block for
+// We create thunks for each 32/8/16 MiB code block for
 // ARM64/ARM32/PPC, respectively.
-static constexpr i64 batch_size = branch_distance<E> / 5;
+static constexpr i64 batch_size =
+  (is_arm64<E> ? 32 : is_arm32<E> ? 8 : 16) * 1024 * 1024;
 
-// We assume that a single thunk group is smaller than 1 MiB.
-static constexpr i64 max_thunk_size = 1024 * 1024;
+// We assume that a single thunk group is smaller than 16/4/8 MiB
+// for ARM64/ARM32/PPC, respectively.
+static constexpr i64 max_thunk_size = batch_size / 2;
 
-// We align thunks to 16 byte boundaries because many processor vendors
-// recommend we align branch targets to 16 byte boundaries for performance
-// reasons.
-static constexpr i64 thunk_align = 16;
+// Power10 prefixed instructions must not cross a 64-byte boundary.
+// Aligning each thunk group to a 8-byte boundary guarantees that.
+static constexpr i64 thunk_align = 8;
 
 template <typename E>
-static bool is_reachable(Context<E> &ctx, bool first_pass, InputSection<E> &isec,
-                         Symbol<E> &sym, const ElfRel<E> &rel) {
-  // If the target section is in the same output section but
-  // hasn't got any address yet, that's unreacahble.
-  InputSection<E> *isec2 = sym.get_input_section();
-  if (isec2 && isec.output_section == isec2->output_section &&
-      isec2->offset == -1)
+static bool
+requires_thunk(Context<E> &ctx, InputSection<E> &isec, const ElfRel<E> &rel,
+               bool first_pass) {
+  if (!is_func_call_rel(rel))
     return false;
+  Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
 
-  // We don't know about the final file layout on the first pass, so
-  // we assume pessimistically that all out-of-section relocations are
-  // out-of-range. Excessive thunks will be removed on the second pass.
   if (first_pass) {
+    // On the first pass, we pessimistically assume that all out-of-section
+    // relocations are out of range.
+    InputSection<E> *isec2 = sym.get_input_section();
     if (!isec2 || isec.output_section != isec2->output_section)
-      return false;
+      return true;
+
+    // If the target section is in the same output section but
+    // hasn't got any address yet, that's unreacahble.
+    if (isec2->offset == -1)
+      return true;
 
     // Even if the target is the same section, we branch to its PLT
     // if it has one. So a symbol with a PLT is also considered an
     // out-of-section reference.
     if (sym.has_plt(ctx))
-      return false;
+      return true;
   }
 
+  // Thumb and ARM B instructions cannot be converted to BX, so we
+  // always have to make them jump to a thunk to switch processor mode
+  // even if their destinations are reachable.
+  if constexpr (is_arm32<E>)
+    if ((rel.r_type == R_ARM_JUMP24 && is_thumb_func(ctx, sym)) ||
+        (rel.r_type == R_ARM_PLT32 && is_thumb_func(ctx, sym)) ||
+        (rel.r_type == R_ARM_THM_JUMP24 && is_arm_func(ctx, sym)))
+      return true;
+
+  // On PowerPC, all PLT calls go through range extension thunks.
+  if constexpr (is_ppc<E>)
+    if (sym.has_plt(ctx))
+      return true;
+
+  // PowerPC before Power9 lacks PC-relative load/store instructions.
+  // Functions compiled for Power9 or earlier assume that r2 points to
+  // GOT+0x8000, while those for Power10 uses r2 as a scratch register.
+  // We need a thunk to recompute r2 for interworking.
+  if constexpr (is_ppc64v2<E>)
+    if ((rel.r_type == R_PPC64_REL24 && !sym.esym().ppc64_preserves_r2()) ||
+        (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().ppc64_uses_toc()))
+      return true;
+
   // Compute a distance between the relocated place and the symbol
   // and check if they are within reach.
   i64 S = sym.get_addr(ctx, NO_OPD);
   i64 A = get_addend(isec, rel);
   i64 P = isec.get_addr() + rel.r_offset;
   i64 val = S + A - P;
-  return -branch_distance<E> <= val && val < branch_distance<E>;
-}
-
-template <typename E>
-static bool needs_shim(Context<E> &ctx, Symbol<E> &sym, const ElfRel<E> &rel) {
-  // Thumb and ARM B instructions cannot be converted to BX, so we
-  // always have to make them jump to a thunk to switch processor mode
-  // even if their destinations are within their ranges.
-  if constexpr (is_arm32<E>) {
-    bool is_thumb = sym.get_addr(ctx) & 1;
-    return (rel.r_type == R_ARM_THM_JUMP24 && !is_thumb) ||
-           (rel.r_type == R_ARM_JUMP24 && is_thumb) ||
-           (rel.r_type == R_ARM_PLT32 && is_thumb);
-  }
-
-  // On PowerPC, all PLT calls go through range extension thunks.
-  if constexpr (is_ppc32<E> || is_ppc64v1<E>)
-    return sym.has_plt(ctx);
-
-  // PowerPC before Power9 lacks PC-relative load/store instructions.
-  // Functions compiled for Power9 or earlier assume that r2 points to
-  // GOT+0x8000, while those for Power10 uses r2 as a scratch register.
-  // We need to a thunk to recompute r2 for interworking.
-  if constexpr (is_ppc64v2<E>)
-    return sym.has_plt(ctx) ||
-           (rel.r_type == R_PPC64_REL24 && !sym.esym().ppc64_preserves_r2()) ||
-           (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().ppc64_uses_toc());
-  return false;
+  return val < -branch_distance<E> || branch_distance<E> <= val;
 }
 
 template <>
-void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
-                                                     bool first_pass) {
-  std::span<InputSection<E> *> m = members;
+void Thunk<E>::compute_size() {
+  offsets.clear();
+  for (i64 i = 0; i <= symbols.size(); i++)
+    offsets.push_back(E::thunk_hdr_size + i * E::thunk_size);
+}
+
+template <>
+void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
+  std::span<ArenaPtr<InputSection<E>>> m = members;
   if (m.empty())
     return;
 
   // Initialize input sections with a dummy offset so that we can
-  // distinguish sections that have got an address with the one who
-  // haven't.
-  for (InputSection<E> *isec : m)
+  // distinguish sections whose addresses have been assigned from those
+  // whose addresses have not.
+  tbb::parallel_for_each(m, [](InputSection<E> *isec) {
     isec->offset = -1;
+  });
   thunks.clear();
 
   // We create thunks from the beginning of the section to the end.
   // We manage progress using four offsets which increase monotonically.
   // The locations they point to are always A <= B <= C <= D.
   //
-  // Input sections between B and C are in the current batch.
+  // Input sections between B and C are the current batch.
   //
   // A is the input section with the smallest address than can reach
   // from the current batch.
@@ -161,17 +163,21 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
       d++;
     }
 
-    // Move C forward so that C is apart from B by BATCH_SIZE. We want
-    // to make sure that there's at least one section between B and C
-    // to ensure progress.
-    c = b + 1;
-    while (c < d && m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
-      c++;
+    // Find the end of the current batch. Section end addresses are sorted,
+    // so use binary search. Starting from B + 1 guarantees progress.
+    std::span<ArenaPtr<InputSection<E>>> range = m.subspan(b + 1, d - b - 1);
+    c = ranges::lower_bound(range, m[b]->offset + batch_size, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset + isec->sh_size;
+                            }) - m.begin();
 
-    // Move A forward so that A is reachable from C.
+    // Find the first section that is within branch range of C.
     i64 c_offset = (c == d) ? offset : m[c]->offset;
-    while (a < b && m[a]->offset + branch_distance<E> < c_offset)
-      a++;
+    range = m.subspan(a, b - a);
+    a = ranges::lower_bound(range, c_offset - branch_distance<E>, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset;
+                            }) - m.begin();
 
     // Erase references to out-of-range thunks.
     for (; t < thunks.size() && thunks[t]->offset < m[a]->offset; t++)
@@ -181,44 +187,28 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
     // Create a new thunk and place it at D.
     offset = align_to(offset, thunk_align);
     thunks.emplace_back(std::make_unique<Thunk<E>>(*this, offset));
+
     Thunk<E> &thunk = *thunks.back();
+    tbb::enumerable_thread_specific<std::vector<Symbol<E> *>> symbols;
 
     // Scan relocations between B and C to collect symbols that need
     // entries in the new thunk.
-    std::mutex mu;
-
     tbb::parallel_for(b, c, [&](i64 i) {
       InputSection<E> &isec = *m[i];
-
-      for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
-        if (!is_func_call_rel(rel))
-          continue;
-
-        // Skip if the symbol is undefined. apply_reloc() will report an error.
-        Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-        if (!sym.file)
-          continue;
-
-        // Skip if we can directly branch to the destination.
-        if (is_reachable(ctx, first_pass, isec, sym, rel) &&
-            !needs_shim(ctx, sym, rel))
-          continue;
-
-        // Add the symbol to the current thunk if it's not added already.
-        if (!sym.flags.test_and_set()) {
-          std::scoped_lock lock(mu);
-          thunk.symbols.push_back(&sym);
-        }
-      }
+      for (const ElfRel<E> &rel : isec.get_rels(ctx))
+        if (requires_thunk(ctx, isec, rel, true))
+          if (Symbol<E> &sym = *isec.file->symbols[rel.r_sym];
+              !sym.flags.test_and_set())
+            symbols.local().push_back(&sym);
     });
 
-    // Sort symbols added to the thunk to make the output deterministic.
-    ranges::sort(thunk.symbols, {}, [](Symbol<E> *x) {
-      return std::tuple{x->file->priority, x->sym_idx};
-    });
+    // Add symbols to the thunk
+    for (std::vector<Symbol<E> *> &vec : symbols)
+      append(thunk.symbols, vec);
 
     // Now that we know the number of symbols in the thunk, we can compute
     // the thunk's size.
+    thunk.compute_size();
     assert(thunk.size() < max_thunk_size);
     offset += thunk.size();
 
@@ -226,22 +216,89 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
     b = c;
   }
 
+  // Clear marks for thunks that are still reachable from the last batch.
   for (; t < thunks.size(); t++)
     for (Symbol<E> *sym : thunks[t]->symbols)
       sym->flags = 0;
 
+  // Sort symbols for deterministic output.
+  tbb::parallel_for_each(thunks, [](std::unique_ptr<Thunk<E>> &thunk) {
+    ranges::sort(thunk->symbols, {}, [](Symbol<E> *sym) {
+      return std::tuple{sym->file->priority, sym->sym_idx};
+    });
+  });
+
   this->shdr.sh_size = offset;
 }
 
+// create_range_extension_thunks() creates thunks with a pessimistic
+// assumption that all out-of-section references are out of range.
+// After computing output section addresses, we revisit all thunks to
+// remove unneeded entries from them.
+//
+// We create more thunks than necessary and then eliminate some of
+// them later, instead of just creating thunks at this stage. This is
+// because we can safely shrink sections after assigning addresses to
+// them without worrying about making existing references to thunks go
+// out of range. On the other hand, if we insert thunks after
+// assigning addresses to sections, references to thunks could become
+// out of range due to the new extra gaps for thunks. Thus, the
+// creation of thunks is a two-pass process.
 template <>
 void remove_redundant_thunks(Context<E> &ctx) {
   Timer t(ctx, "remove_redundant_thunks");
-  set_osec_offsets(ctx);
 
+  // Gather output executable sections
+  std::vector<OutputSection<E> *> sections;
   for (Chunk<E> *chunk : ctx.chunks)
     if (OutputSection<E> *osec = chunk->to_osec())
       if (osec->shdr.sh_flags & SHF_EXECINSTR)
-        osec->create_range_extension_thunks(ctx, false);
+        sections.push_back(osec);
+
+  // Mark all symbols that actually need range extension thunks
+  for (OutputSection<E> *osec : sections) {
+    tbb::parallel_for_each(osec->members, [&](InputSection<E> *isec) {
+      for (const ElfRel<E> &rel : isec->get_rels(ctx))
+        if (is_func_call_rel(rel))
+          if (Symbol<E> *sym = isec->file->symbols[rel.r_sym];
+              !sym->flags &&
+              requires_thunk(ctx, *isec, rel, false))
+            sym->flags.test_and_set();
+    });
+  }
+
+  // Remove symbols from thunks if they don't actually need range
+  // extension thunks
+  for (OutputSection<E> *osec : sections) {
+    tbb::parallel_for_each(osec->thunks,
+                           [&](std::unique_ptr<Thunk<E>> &thunk) {
+      std::erase_if(thunk->symbols,
+                    [&](Symbol<E> *sym) { return !sym->flags; });
+      thunk->shrink_size(ctx);
+    });
+  }
+
+  // Recompute section sizes
+  tbb::parallel_for_each(sections, [&](OutputSection<E> *osec) {
+    std::span<ArenaPtr<InputSection<E>>> m = osec->members;
+    std::span<std::unique_ptr<Thunk<E>>> t = osec->thunks;
+    i64 offset = 0;
+
+    while (!m.empty() || !t.empty()) {
+      if (!m.empty() && (t.empty() || m[0]->offset < t[0]->offset)) {
+        offset = align_to(offset, 1 << m[0]->p2align);
+        m[0]->offset = offset;
+        offset += m[0]->sh_size;
+        m = m.subspan(1);
+      } else {
+        offset = align_to(offset, thunk_align);
+        t[0]->offset = offset;
+        offset += t[0]->size();
+        t = t.subspan(1);
+      }
+    }
+    osec->shdr.sh_size = offset;
+  });
 }
 
 // When applying relocations, we want to know the address in a reachable
@@ -259,7 +316,8 @@ void gather_thunk_addresses(Context<E> &ctx) {
   std::vector<OutputSection<E> *> sections;
   for (Chunk<E> *chunk : ctx.chunks)
     if (OutputSection<E> *osec = chunk->to_osec())
-      sections.push_back(osec);
+      if (osec->shdr.sh_flags & SHF_EXECINSTR)
+        sections.push_back(osec);
 
   ranges::stable_sort(sections, {}, [](OutputSection<E> *x) {
     return x->shdr.sh_addr;
@@ -269,8 +327,8 @@ void gather_thunk_addresses(Context<E> &ctx) {
     for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks) {
       for (i64 i = 0; i < thunk->symbols.size(); i++) {
         Symbol<E> &sym = *thunk->symbols[i];
-        sym.add_aux(ctx);
-        ctx.symbol_aux[sym.aux_idx].thunk_addrs.push_back(thunk->get_addr(i));
+        u64 addr = thunk->get_addr() + thunk->offsets[i];
+        sym.add_thunk_addr(ctx, addr);
       }
     }
   }

test/CMakeLists.txt

@@ -30,8 +30,8 @@ endif()
 
 if(MOLD_ENABLE_QEMU_TESTS)
   list(APPEND QEMU_ARCHS
-    x86_64 i386 arm aarch64 aarch64_be ppc ppc64 ppc64le sparc64 sh4 sh4eb
-    s390x riscv64 riscv32 m68k loongarch64)
+    x86_64 i386 arm armeb aarch64 aarch64_be ppc ppc64 ppc64le sparc64
+    sh4 sh4eb s390x riscv64 riscv32 m68k loongarch64)
 
   LIST(APPEND TRIPLES
     x86_64-linux-gnu
@@ -39,6 +39,7 @@ if(MOLD_ENABLE_QEMU_TESTS)
     aarch64-linux-gnu
     aarch64_be-linux-gnu
     arm-linux-gnueabihf
+    armeb-linux-gnueabihf
     riscv64-linux-gnu
     powerpc-linux-gnu
     powerpc64-linux-gnu
@@ -85,7 +86,7 @@ function(add_target ARCH TRIPLE)
     endif()
 
     add_test(NAME ${TESTNAME}
-      COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
+      COMMAND ${CMAKE_CURRENT_LIST_DIR}/${TEST}
       WORKING_DIRECTORY ${mold_BINARY_DIR})
 
     set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT
@@ -121,6 +122,10 @@ if(${MACHINE} STREQUAL "arm" OR (HAS_qemu-arm AND HAS_arm-linux-gnueabihf-gcc))
   add_target(arm arm-linux-gnueabihf)
 endif()
 
+if(${MACHINE} STREQUAL "armeb" OR (HAS_qemu-armeb AND HAS_armeb-linux-gnueabihf-gcc))
+  add_target(armeb armeb-linux-gnueabihf)
+endif()
+
 if(${MACHINE} STREQUAL "riscv64" OR (HAS_qemu-riscv64 AND HAS_riscv64-linux-gnu-gcc))
   add_target(riscv64 riscv64-linux-gnu)
 endif()

test/abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 [ $MACHINE = aarch64 ] && skip

test/absolute-symbols.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # This test crashes only on qemu-sparc64 running on GitHub Actions,

test/allow-multiple-definition.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc -

test/ar-alignment.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-aarch64-emit-relocs-relax.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# When mold relaxes an instruction sequence, the relocations attached to the
+# original instructions no longer match the relaxed code. With --emit-relocs,
+# mold must rewrite the emitted relocations so that their types describe the
+# relaxed instructions.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xassembler -
+.text
+.globl _start
+_start:
+  # GOT-load relaxation: ADRP+LDR => ADRP+ADD
+  #   R_AARCH64_ADR_GOT_PAGE     => R_AARCH64_ADR_PREL_PG_HI21
+  #   R_AARCH64_LD64_GOT_LO12_NC => R_AARCH64_ADD_ABS_LO12_NC
+  adrp x0, :got:gvar
+  ldr  x0, [x0, #:got_lo12:gvar]
+
+  # ADRP+ADD => NOP+ADR (gvar is within +-1 MiB)
+  #   R_AARCH64_ADR_PREL_PG_HI21 => R_AARCH64_NONE
+  #   R_AARCH64_ADD_ABS_LO12_NC  => R_AARCH64_ADR_PREL_LO21
+  adrp x1, gvar
+  add  x1, x1, #:lo12:gvar
+
+  # Both sequences must materialize the same address.
+  cmp  x0, x1
+  bne  1f
+
+  # ... and it must point at gvar, whose value is 42.
+  ldr  w2, [x0]
+  cmp  w2, #42
+  bne  1f
+
+  mov  x0, #0
+  b    2f
+1:
+  mov  x0, #1
+2:
+  mov  x8, #93           // __NR_exit
+  svc  #0
+
+.data
+.hidden gvar
+gvar:
+  .word 42
+EOF
+
+$CC -B. -o $t/exe $t/a.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax
+$QEMU $t/exe
+
+$OBJDUMP -dr $t/exe > $t/exe.objdump
+
+# Relaxation took place.
+grep -Fw nop $t/exe.objdump
+grep -Fw adr $t/exe.objdump
+grep -Fw add $t/exe.objdump
+
+# The relocations were rewritten to match the relaxed instructions.
+grep -Fw R_AARCH64_ADR_PREL_PG_HI21 $t/exe.objdump
+grep -Fw R_AARCH64_ADD_ABS_LO12_NC $t/exe.objdump
+grep -Fw R_AARCH64_ADR_PREL_LO21 $t/exe.objdump
+
+# The stale pre-relaxation relocations are gone.
+not grep -Fw R_AARCH64_ADR_GOT_PAGE $t/exe.objdump
+not grep -Fw R_AARCH64_LD64_GOT_LO12_NC $t/exe.objdump
+
+# Each rewritten relocation must sit on the relaxed instruction it describes.
+awk '
+  $3 == "adrp" || $3 == "add" || $3 == "adr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_ADR_PREL_PG_HI21" { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADD_ABS_LO12_NC"  { if (insn[$1] != "add")  { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADR_PREL_LO21"    { if (insn[$1] != "adr")  { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe.objdump

test/arch-aarch64-emit-relocs-tlsdesc-relax.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec
+# (LE) or initial-exec (IE) sequence. The four-instruction sequence is rewritten
+# in place: the ADRP and LDR become NOPs, and the ADD and BLR become the two
+# instructions that materialize the address. With --emit-relocs, the emitted
+# relocations must be rewritten to match whichever relaxation was applied.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC
+_Thread_local char foo[4] = "foo";
+_Thread_local char bar[4] = "bar";
+EOF
+
+cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC -mtls-dialect=desc -O2
+extern _Thread_local char foo[4];
+extern _Thread_local char bar[4];
+char *get_foo() { return foo; }
+char *get_bar() { return bar; }
+EOF
+
+cat <<EOF | $CC -o $t/c.o -c -xc - -mtls-dialect=desc
+#include <stdio.h>
+char *get_foo();
+char *get_bar();
+int main() { printf("%s %s\n", get_foo(), get_bar()); }
+EOF
+
+#
+# Local TLS in an executable: TLSDESC => LE
+#
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--emit-relocs,--relax
+$QEMU $t/exe1 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe1 > $t/exe1.objdump
+
+# The ADD/BLR became MOVZ/MOVK that build the TP-relative offset.
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/exe1.objdump
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G0_NC $t/exe1.objdump
+
+# No TLSDESC relocation survives.
+not grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe1.objdump
+
+# The G1 relocation sits on a movz and the G0_NC on a movk, not on the leading
+# adrp/ldr (which are now nops).
+awk '
+  $3 == "movz" || $3 == "movk" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G1"    { if (insn[$1] != "movz") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G0_NC" { if (insn[$1] != "movk") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe1.objdump
+
+#
+# TLS defined in a shared library: TLSDESC => IE
+#
+$CC -B. -shared -o $t/d.so $t/a.o -Wl,--relax
+$CC -B. -o $t/exe2 $t/b.o $t/c.o $t/d.so -Wl,--emit-relocs,--relax -Wl,-rpath=$t
+$QEMU $t/exe2 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe2 > $t/exe2.objdump
+
+# The ADD/BLR became ADRP/LDR that load the address from the GOT.
+grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/exe2.objdump
+grep -Fw R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe2.objdump
+
+awk '
+  $3 == "adrp" || $3 == "ldr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21"  { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC" { if (insn[$1] != "ldr") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe2.objdump
+
+#
+# Relocatable output: no relaxation happens and TLSDESC is not downgraded, so
+# every relocation must be passed through unchanged rather than rewritten as if
+# it had been relaxed to LE/IE.
+#
+$CC -B. -o $t/reloc.o $t/b.o -r
+$OBJDUMP -r $t/reloc.o > $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_CALL $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/reloc.objdump

test/arch-aarch64-long-thunk.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+cat <<EOF > $t/a.c
+#include <stdio.h>
+
+void fn3();
+void fn4();
+
+__attribute__((section(SEC1))) void fn1() { printf(" fn1"); fn3(); }
+__attribute__((section(SEC1))) void fn2() { printf(" fn2"); fn4(); }
+
+int main() {
+  printf(" main");
+  fn1();
+  printf("\n");
+}
+EOF
+
+cat <<EOF > $t/b.c
+#include <stdio.h>
+
+void fn1();
+void fn2();
+
+__attribute__((section(SEC2))) void fn3() { printf(" fn3"); fn2(); }
+__attribute__((section(SEC2))) void fn4() { printf(" fn4"); }
+EOF
+
+# .high is placed billions of bytes away from the rest of the program.
+# Functions there cannot have unwind tables because an FDE refers to its
+# function with a 32-bit PC-relative relocation. We put fn1 and fn2 in
+# either section to test calls in both directions.
+flags="-fno-asynchronous-unwind-tables -fno-unwind-tables"
+starts="-Wl,--section-start=.low=0x10000000,--section-start=.high=0x100000000"
+
+$CC -c -o $t/c.o $t/a.c $flags -DSEC1='".low"' -DSEC2='".high"'
+$CC -c -o $t/d.o $t/b.c $flags -DSEC1='".low"' -DSEC2='".high"'
+$CC -B. -o $t/exe1 $t/c.o $t/d.o $starts
+$QEMU $t/exe1 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -c -o $t/e.o $t/a.c $flags -DSEC1='".high"' -DSEC2='".low"'
+$CC -c -o $t/f.o $t/b.c $flags -DSEC1='".high"' -DSEC2='".low"'
+$CC -B. -o $t/exe2 $t/e.o $t/f.o $starts
+$QEMU $t/exe2 | grep 'main fn1 fn3 fn2 fn4'
+
+flags="$flags -fno-PIC -mcmodel=large"
+starts="-Wl,--section-start=.low=0x10000000,--section-start=.high=0x400000000"
+
+$CC -c -o $t/g.o $t/a.c $flags -DSEC1='".low"' -DSEC2='".high"'
+$CC -c -o $t/h.o $t/b.c $flags -DSEC1='".low"' -DSEC2='".high"'
+$CC -B. -o $t/exe3 $t/g.o $t/h.o -pie $starts
+$QEMU $t/exe3 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -c -o $t/i.o $t/a.c $flags -DSEC1='".high"' -DSEC2='".low"'
+$CC -c -o $t/j.o $t/b.c $flags -DSEC1='".high"' -DSEC2='".low"'
+$CC -B. -o $t/exe4 $t/i.o $t/j.o -pie $starts
+$QEMU $t/exe4 | grep 'main fn1 fn3 fn2 fn4'

test/arch-aarch64-pack-dyn-relocs.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = aarch64 ] || skip
+command -v timeout >/dev/null || skip
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip
+.section .custom_rodata_a, "a", %progbits
+.hidden A
+.global A
+A:
+  .space 8
+
+.section .custom_rodata_b_huge, "a", %progbits
+.balign 16384
+.hidden B
+.global B
+B:
+  .space 8
+
+.section .data, "aw", %progbits
+.rept 7439
+  .quad A
+.endr
+
+.global P_1
+P_1:
+  .quad A
+
+.global P_2
+P_2:
+  .quad B
+EOF
+
+timeout 5s ./mold -m aarch64elf -shared -o $t/b.so $t/a.o \
+  --pack-dyn-relocs=android -z max-page-size=16384

test/arch-aarch64-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-aarch64-variant-pcs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip

test/arch-arm-abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -fPIC -c -o $t/a.o -xassembler - -mthumb

test/arch-arm-exidx-sentinel.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CXX -o $t/a.o -c -xc++ - -fno-PIC

test/arch-arm-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-arm-range-extension-thunk.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 test_cflags -mthumb || skip