Update mold.1 (automated commit)

A weak TLS variable is usually guarded by a separate weak non-TLS
symbol indicating whether the TLS definition is available. A typical
use looks like this:

  extern char foo_used __attribute__((weak));
  extern __thread int foo __attribute__((weak));

  if (&foo_used)
    foo = 1;

If foo remains undefined, its access is unreachable and there is no
meaningful value to apply to its TLS relocations. Leave those
relocations unapplied.

This prevents AArch64 from rejecting an otherwise unreachable TLS
access when an R_AARCH64_TLSLE_ADD_TPREL_HI12 relocation is out of range.

https://github.com/rui314/mold/issues/1626

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Delete this message and describe the bug here. Note that we only fix bugs in the most recent release of mold, so please make sure that you are using the most recent version.

.github/workflows/build-all.yml

@@ -3,12 +3,27 @@ name: Build all tarballs
 on:
   schedule:
     - cron: '0 0 * * *'
-  release:
-    types: [ published ]
   workflow_dispatch:
 
 jobs:
-  build-tarballs:
+  check-changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      has_changes: ${{ github.event_name == 'workflow_dispatch' || steps.check.outputs.has_changes }}
+    steps:
+    - name: Check for new commits since last successful run
+      id: check
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        last=$(gh run list -R ${{ github.repository }} -w build-all.yml -b main -s success -L1 --json createdAt -q '.[0].createdAt')
+        if [ -z "$last" ] || [ "$last" = "null" ] || [ "$(gh api repos/${{ github.repository }}/commits?sha=main\&since=$last\&per_page=1 | jq length)" -gt 0 ]; then
+          echo "has_changes=true" >> "$GITHUB_OUTPUT"
+        fi
+
+  build-linux:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
     strategy:
       matrix:
         include:
@@ -28,7 +43,7 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install Podman
       run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
@@ -44,8 +59,35 @@ jobs:
       run: ./dist.sh ${{ matrix.target }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: ${{ matrix.target }}
         path: dist/mold-*.tar.gz
         compression-level: 0
+
+  build-windows:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
+    runs-on: windows-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-windows-x86_64.zip
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*

.github/workflows/build-native.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install Podman
       run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
@@ -37,7 +37,7 @@ jobs:
       run: ./dist.sh ${{ matrix.target }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: ${{ matrix.target }}
         path: dist/mold-*.tar.gz

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ on:
 env:
   UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
 jobs:
-  build-sanitizers:
+  build-asan-tsan:
     strategy:
       matrix:
         target:
@@ -13,8 +13,9 @@ jobs:
         - '-DMOLD_USE_ASAN=On'
         - '-DMOLD_USE_TSAN=On'
     runs-on: ubuntu-24.04
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - uses: rui314/setup-mold@staging
     - run: sudo ./install-build-deps.sh
     - name: build
@@ -26,7 +27,7 @@ jobs:
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: test-results-clang
@@ -34,10 +35,23 @@ jobs:
           build
           !build/CMakeFiles
 
+  build-msan:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v6
+    - uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - run: .github/workflows/run-msan.sh
+
   build-multi-archs:
     runs-on: ubuntu-24.04
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: install-build-deps
       run: |
         sudo ./install-build-deps.sh
@@ -51,7 +65,7 @@ jobs:
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
         name: test-results-multi-archs
@@ -72,9 +86,10 @@ jobs:
         - ubuntu:22.04
         - ubuntu:25.04
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     container: ${{ matrix.distro }}
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - run: ./install-build-deps.sh
     - name: build
       run: |
@@ -83,19 +98,24 @@ jobs:
         cmake ..
         cmake --build . -j$(nproc)
     - run: cd build; ctest --output-on-failure -j$(nproc)
+    - name: sanitize distro name
+      if: failure()
+      id: sanitize
+      run: echo "name=$(echo '${{ matrix.distro }}' | tr '/:' '--')" >> "$GITHUB_OUTPUT"
     - name: archive test results
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: failure()
       with:
-        name: test-results-${{ matrix.distro }}
+        name: test-results-${{ steps.sanitize.outputs.name }}
         path: |
           build
           !build/CMakeFiles
 
   build-macos:
     runs-on: macos-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: build
       run: |
         mkdir build
@@ -105,8 +125,9 @@ jobs:
 
   build-windows:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: build
       run: |
         mkdir build
@@ -116,8 +137,9 @@ jobs:
 
   build-msys:
     runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Setup MSYS2
       uses: msys2/setup-msys2@v2
       with:
@@ -134,8 +156,9 @@ jobs:
 
   build-freebsd:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Build and test
       uses: vmactions/freebsd-vm@v1
       with:

.github/workflows/install-extras.sh

@@ -5,7 +5,7 @@ apt-get install -y wget xz-utils
 
 # Install a 32-bit RISC-V toolchain
 mkdir /rv32
-wget -O- -q https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.01.20/riscv32-glibc-ubuntu-22.04-gcc-nightly-2025.01.20-nightly.tar.xz | tar -C /rv32 --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.06.13/riscv32-glibc-ubuntu-24.04-gcc-nightly-2025.06.13-nightly.tar.xz | tar -C /rv32 --strip-components=1 --xz -xf -
 
 ln -sf /rv32/sysroot /usr/riscv32-linux-gnu
 echo '/rv32/bin/riscv32-unknown-linux-gnu-gcc -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-gcc
@@ -18,7 +18,7 @@ done
 
 # Install a LoongArch toolchain
 mkdir /larch
-wget -O- -q https://github.com/loongson/build-tools/releases/download/2025.02.21/x86_64-cross-tools-loongarch64-binutils_2.44-gcc_14.2.0-glibc_2.41.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/x86_64-cross-tools-loongarch64-binutils_2.44-gcc_15.1.0-glibc_2.41.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
 
 cp -r /larch/loongarch64-unknown-linux-gnu/lib/* /larch/target/lib64
 ln -sf /larch/target /usr/loongarch64-linux-gnu
@@ -27,25 +27,39 @@ for i in gcc g++ objdump objcopy strip; do
   ln -sf /larch/bin/loongarch64-unknown-linux-gnu-$i /usr/bin/loongarch64-linux-gnu-$i
 done
 
-wget -O /usr/local/bin/qemu-loongarch64 -q https://github.com/loongson/build-tools/releases/download/2024.11.01/qemu-loongarch64
+wget -O /usr/local/bin/qemu-loongarch64 --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/qemu-loongarch64
 chmod 755 /usr/local/bin/qemu-loongarch64
 
+# Install ARM32 big-endian toolchain
+mkdir /armeb
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/armebv7-eabihf/tarballs/armebv7-eabihf--glibc--stable-2024.05-1.tar.xz | tar -C /armeb --strip-components=1 --xz -xf -
+
+ln -sf /armeb/armeb-buildroot-linux-gnueabihf/sysroot /usr/armeb-linux-gnueabihf
+echo '/armeb/bin/armeb-linux-gcc -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-gcc
+echo '/armeb/bin/armeb-linux-g++ -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-g++
+chmod 755 /usr/bin/armeb-linux-gnueabihf-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /armeb/bin/armeb-linux-$i /usr/bin/armeb-linux-gnueabihf-$i
+done
+
 # Install ARM64 big-endian toolchain
 mkdir /aarch64be
-wget -O- -q https://sources.buildroot.net/toolchain-external-arm-aarch64-be/arm-gnu-toolchain-13.3.rel1-x86_64-aarch64_be-none-linux-gnu.tar.xz | tar -C /aarch64be --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64be/tarballs/aarch64be--glibc--stable-2024.05-1.tar.xz | tar -C /aarch64be --strip-components=1 --xz -xf -
 
-ln -sf /aarch64be/aarch64_be-none-linux-gnu/libc /usr/aarch64_be-linux-gnu
-echo '/aarch64be/bin/aarch64_be-none-linux-gnu-gcc -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-gcc
-echo '/aarch64be/bin/aarch64_be-none-linux-gnu-g++ -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-g++
+ln -sf /aarch64be/aarch64_be-buildroot-linux-gnu/sysroot /usr/aarch64_be-linux-gnu
+echo '/aarch64be/bin/aarch64_be-linux-gcc -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-gcc
+echo '/aarch64be/bin/aarch64_be-linux-g++ -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-g++
 chmod 755 /usr/bin/aarch64_be-linux-gnu-{gcc,g++}
 
 for i in objdump objcopy strip; do
-  ln -sf /aarch64be/bin/aarch64_be-none-linux-gnu-$i /usr/bin/aarch64_be-linux-gnu-$i
+  ln -sf /aarch64be/bin/aarch64_be-linux-$i /usr/bin/aarch64_be-linux-gnu-$i
 done
 
 # Install SH4 big-endian toolchain
 mkdir /sh4aeb
-wget -O- -q https://toolchains.bootlin.com/downloads/releases/toolchains/sh-sh4aeb/tarballs/sh-sh4aeb--glibc--stable-2024.05-1.tar.xz | tar -C /sh4aeb --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/sh-sh4aeb/tarballs/sh-sh4aeb--glibc--stable-2024.05-1.tar.xz | tar -C /sh4aeb --strip-components=1 --xz -xf -
+
 ln -sf /sh4aeb/sh4aeb-buildroot-linux-gnu/sysroot /usr/sh4aeb-linux-gnu
 echo '/sh4aeb/bin/sh4aeb-linux-gcc -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-gcc
 echo '/sh4aeb/bin/sh4aeb-linux-g++ -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-g++
@@ -57,5 +71,5 @@ done
 
 # Install Intel SDE CPU emulator for CET-related tests
 mkdir /sde
-wget -O- -q https://downloadmirror.intel.com/843185/sde-external-9.48.0-2024-11-25-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
+wget -O- --progress=dot:mega https://downloadmirror.intel.com/850782/sde-external-9.53.0-2025-03-16-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
 ln -s /sde/sde64 /usr/bin

.github/workflows/release-assets.yml

@@ -0,0 +1,104 @@
+name: Build & attach tarballs on tag push
+
+on:
+  push:
+    tags: ['v[0-9]*']
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+env:
+  TAG: ${{ github.ref_name }}
+
+jobs:
+  build-linux:
+    strategy:
+      matrix:
+        include:
+        - { target: x86_64,      os: ubuntu-24.04 }
+        - { target: aarch64,     os: ubuntu-24.04-arm }
+        - { target: arm,         os: ubuntu-24.04-arm }
+        - { target: riscv64,     os: ubuntu-24.04 }
+        - { target: ppc64le,     os: ubuntu-24.04 }
+        - { target: s390x,       os: ubuntu-24.04 }
+        - { target: loongarch64, os: ubuntu-24.04 }
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install Podman
+      run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
+
+    - name: Login to GitHub Container Registry
+      uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - run: ./dist.sh ${{ matrix.target }}
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: ${{ matrix.target }}
+        path: dist/mold-*.tar.gz
+
+  build-windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-x86_64-windows.zip
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*
+
+  publish:
+    needs: [build-linux, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - uses: actions/download-artifact@v8
+      with:
+        path: dist
+        pattern: '*'
+        merge-multiple: true
+
+    - name: Ensure release exists (create if missing)
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -e
+        if gh release view "$TAG" >& /dev/null; then
+          echo "Existing release found for $TAG"
+        else
+          echo "Creating draft release for $TAG"
+          gh release create "$TAG" --draft \
+            --title "$TAG" \
+            --notes "Automated draft – fill the changelog later."
+        fi
+
+    - name: Upload tarballs
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        gh release upload "$TAG" dist/* --clobber
+        gh release view "$TAG" --json assets --jq '.assets[].name'

.github/workflows/run-msan.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e -x
+cd "$(dirname $0)"/../..
+
+if [ "$GITHUB_REPOSITORY" = '' ]; then
+  image=mold-msan
+  image_build="podman build -t $image -"
+else
+  # If this script is running on GitHub Actions, we want to cache
+  # the created container image in GitHub's container repostiory.
+  image=ghcr.io/$GITHUB_REPOSITORY/mold-msan
+  image_build="podman build -t $image --output=type=registry --layers --cache-to $image --cache-from $image -"
+fi
+
+cat <<EOF | $image_build
+FROM mirror.gcr.io/library/ubuntu:24.04
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc g++ wget ca-certificates make cmake ninja-build python3 clang-20 llvm-20-dev libclang-20-dev libclang-rt-20-dev libtbb-dev mold && \
+  rm -rf /var/lib/apt/lists
+RUN mkdir /llvm && \
+  cd /llvm && \
+  wget -O- --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-21.1.1.tar.gz | tar xzf - --strip-components=1 && \
+  cmake -S ./runtimes -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi;libunwind' -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DLLVM_USE_SANITIZER=MemoryWithOrigins && \
+  cmake --build build -j\$(nproc) && \
+  cmake --install build --prefix /msan && \
+  rm -rf /llvm
+EOF
+
+# Build mold and run its tests
+podman run -it --rm --pull=never -v "$(pwd):/mold:ro" --security-opt seccomp=unconfined $image bash -c "
+set -e
+mkdir /build
+cd /build
+cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DMOLD_USE_MSAN=ON -DMOLD_STDLIB_PREFIX=/msan -DMOLD_USE_MIMALLOC=0 -DMOLD_USE_SYSTEM_TBB=1 -DMOLD_USE_MOLD=1 /mold
+cmake --build . -j\$(nproc)
+setarch -R ctest --output-on-failure -j\$(nproc)
+"

.github/workflows/update-manpage.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Check out repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install ronn
       run: sudo apt-get update && sudo apt-get install -y ronn

CMakeLists.txt

@@ -44,7 +44,7 @@
 # features and behave exactly the same.
 
 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.37.1)
+project(mold VERSION 2.42.0)
 
 include(CMakeDependentOption)
 include(CheckSymbolExists)
@@ -62,7 +62,11 @@ endif()
 # Build mold itself using mold if -DMOLD_USE_MOLD=ON
 option(MOLD_USE_MOLD "Use mold to build mold" OFF)
 if(MOLD_USE_MOLD)
-  target_link_options(mold PRIVATE -fuse-ld=mold -Wl,--gdb-index)
+  target_link_options(mold PRIVATE -fuse-ld=mold)
+
+  if(CMAKE_BUILD_TYPE MATCHES "Deb")
+    target_link_options(mold PRIVATE -Wl,--gdb-index)
+  endif()
 
   if(CMAKE_BUILD_TYPE MATCHES "^Rel")
     target_link_options(mold PRIVATE -Wl,--gc-sections,--icf=safe)
@@ -113,6 +117,22 @@ if(MOLD_USE_TSAN)
   target_link_options(mold PRIVATE -fsanitize=thread)
 endif()
 
+# Enable MemorySanitizer if -DMOLD_USE_MSAN=ON
+option(MOLD_USE_MSAN "Build mold with MemorySanitizer" OFF)
+if(MOLD_USE_MSAN)
+  set(MOLD_STDLIB_PREFIX /usr CACHE FILEPATH "prefix of instrumented stdlib")
+  target_compile_options(mold PRIVATE
+    -g -Og -fsanitize=memory -fsanitize-memory-track-origins
+    -nostdinc++ -isystem ${MOLD_STDLIB_PREFIX}/include/c++/v1
+    -DENABLE_MSAN_UNPOISON
+  )
+  target_link_options(mold PRIVATE
+    -fsanitize=memory
+    -nostdlib++ -L ${MOLD_STDLIB_PREFIX}/lib -lc++
+    -Wl,-rpath,${MOLD_STDLIB_PREFIX}/lib
+  )
+endif()
+
 # Statically-link libstdc++ if -DMOLD_MOSTLY_STATIC=ON.
 #
 # This option is intended to be used by `./dist.sh` script to create a
@@ -129,10 +149,9 @@ find_package(ZLIB QUIET)
 if(ZLIB_FOUND AND NOT MOLD_MOSTLY_STATIC)
   target_link_libraries(mold PRIVATE ZLIB::ZLIB)
 else()
-  set(ZLIB_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_SHARED OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_TESTING OFF CACHE INTERNAL "")
   add_subdirectory(third-party/zlib EXCLUDE_FROM_ALL)
-  target_include_directories(zlibstatic INTERFACE third-party/zlib
-    $<TARGET_PROPERTY:zlibstatic,BINARY_DIR>)
   target_link_libraries(mold PRIVATE zlibstatic)
 endif()
 
@@ -160,13 +179,13 @@ check_include_file(zstd.h HAVE_ZSTD_H)
 if(HAVE_ZSTD_H AND NOT MOLD_MOSTLY_STATIC)
   target_link_libraries(mold PRIVATE zstd)
 else()
+  set(ZSTD_BUILD_PROGRAMS OFF)
+  set(ZSTD_BUILD_CONTRIB OFF)
+  set(ZSTD_BUILD_TESTS OFF)
+  set(ZSTD_MULTITHREAD_SUPPORT OFF)
+  set(ZSTD_BUILD_SHARED OFF)
+  set(ZSTD_BUILD_STATIC ON)
   add_subdirectory(third-party/zstd/build/cmake EXCLUDE_FROM_ALL)
-  target_compile_definitions(libzstd_static PRIVATE
-    ZSTD_BUILD_STATIC=1
-    ZSTD_BUILD_SHARED=0
-    ZSTD_BUILD_PROGRAMS=0
-    ZSTD_MULTITHREAD_SUPPORT=0
-    ZSTD_BUILD_TESTS=0)
   target_include_directories(mold PUBLIC third-party/zstd/lib)
   target_link_libraries(mold PRIVATE libzstd_static)
 endif()
@@ -189,14 +208,19 @@ cmake_dependent_option(
 # By default, we build a bundled mimalloc and statically-link it to
 # mold. If you want to dynamically link to the system's
 # libmimalloc.so, pass -DMOLD_USE_SYSTEM_MIMALLOC=ON.
+#
+# We require mimalloc v3 or newer. mimalloc v2 doesn't use transparent
+# huge pages for the allocation path that serves mold's largest data
+# structures, which makes linking large programs significantly slower.
 if(MOLD_USE_MIMALLOC)
   if(MOLD_USE_SYSTEM_MIMALLOC)
-    find_package(mimalloc REQUIRED)
+    find_package(mimalloc 3 REQUIRED)
     target_link_libraries(mold PRIVATE mimalloc)
   else()
     function(mold_add_mimalloc)
       set(MI_BUILD_STATIC ON CACHE INTERNAL "")
       set(MI_BUILD_TESTS OFF CACHE INTERNAL "")
+      set(MI_NO_OPT_ARCH ON CACHE INTERNAL "")
       add_subdirectory(third-party/mimalloc EXCLUDE_FROM_ALL)
       target_compile_definitions(mimalloc-static PRIVATE MI_USE_ENVIRON=0)
       target_link_libraries(mold PRIVATE mimalloc-static)
@@ -213,7 +237,7 @@ endif()
 # to mold. If you want to link to the system's libtbb2.so, pass
 # -DMOLD_USE_SYSTEM_TBB=ON.
 option(MOLD_USE_SYSTEM_TBB "Use system or vendored TBB" OFF)
-if(MOLD_USE_SYSTEM_TBB)
+if(MOLD_USE_SYSTEM_TBB OR BLAKE3_USE_TBB)
   find_package(TBB REQUIRED)
   target_link_libraries(mold PRIVATE TBB::tbb)
 else()
@@ -285,22 +309,25 @@ endif()
 check_symbol_exists(madvise sys/mman.h HAVE_MADVISE)
 check_symbol_exists(uname sys/utsname.h HAVE_UNAME)
 
+# glibc declares fallocate only under _GNU_SOURCE. C++ compilers
+# predefine it on Linux, but this check runs the C compiler, which
+# does not.
+set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+check_symbol_exists(fallocate fcntl.h HAVE_FALLOCATE)
+unset(CMAKE_REQUIRED_DEFINITIONS)
+
 # Create a .cc file containing the current git hash for `mold --version`.
 add_custom_target(git_hash
   COMMAND ${CMAKE_COMMAND}
     -DSOURCE_DIR=${CMAKE_SOURCE_DIR}
-    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/git-hash.cc
+    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/mold-git-hash.h
     -P ${CMAKE_SOURCE_DIR}/lib/update-git-hash.cmake
   DEPENDS lib/update-git-hash.cmake
-  BYPRODUCTS git-hash.cc
+  BYPRODUCTS mold-git-hash.h
   VERBATIM)
 
 add_dependencies(mold git_hash)
 
-# Create config.h file
-configure_file(lib/config.h.in config.h)
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
 # Almost all functions are template in mold which take a target type
 # (e.g. X86_64) as its type parameter. Since we suport more than 10
 # targets, compiling a single source file for all the targets is very
@@ -309,16 +336,38 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # As a workaround, we create a .cc file for each target and spawn many
 # compiler instances. This is hacky but greatly reduces compile time
 # on a multicore machine.
-list(APPEND MOLD_ELF_TARGETS
-  X86_64 I386 ARM64LE ARM64BE ARM32 RV32LE RV32BE RV64LE RV64BE PPC32
-  PPC64V1 PPC64V2 S390X SPARC64 M68K SH4LE SH4BE LOONGARCH32 LOONGARCH64)
+#
+# You can build mold for a specific set of targets by passing, e.g.,
+# -DMOLD_TARGETS='X86_64;ARM64LE', though this is strongly discouraged
+# for build reproducibility. Use this option only if you build mold
+# frequently for your personal use; otherwise, always build mold with all
+# targets enabled. We provide this flag in the trust that you will not
+# abuse it, so please don’t betray that trust.
+set(MOLD_TARGETS
+  X86_64 I386 ARM64LE ARM64BE ARM32LE ARM32BE RV32LE RV32BE RV64LE
+  RV64BE PPC32 PPC64V1 PPC64V2 S390X SPARC64 M68K SH4LE SH4BE
+  LOONGARCH32 LOONGARCH64
+  CACHE STRING "List of supported targets")
 
-list(APPEND MOLD_ELF_TEMPLATE_FILES
+list(GET MOLD_TARGETS 0 MOLD_FIRST_TARGET)
+
+list(APPEND MOLD_TEMPLATE_FILES
+  src/arch-arm32.cc
   src/arch-arm64.cc
+  src/arch-i386.cc
   src/arch-loongarch.cc
+  src/arch-m68k.cc
+  src/arch-ppc32.cc
+  src/arch-ppc64v1.cc
+  src/arch-ppc64v2.cc
   src/arch-riscv.cc
+  src/arch-s390x.cc
   src/arch-sh4.cc
+  src/arch-sparc64.cc
+  src/arch-x86-64.cc
+  src/archive-file.cc
   src/cmdline.cc
+  src/error.cc
   src/filetype.cc
   src/gc-sections.cc
   src/gdb-index.cc
@@ -337,18 +386,18 @@ list(APPEND MOLD_ELF_TEMPLATE_FILES
   )
 
 if(WIN32 AND NOT MINGW)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-win32.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-win32.cc)
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-unix.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-unix.cc)
 endif()
 
 if(WIN32)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
     src/output-file-win32.cc
     src/subprocess-win32.cc
     )
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
     src/output-file-unix.cc
     src/subprocess-unix.cc
   )
@@ -365,54 +414,50 @@ function(mold_instantiate_templates SOURCE TARGET)
   target_sources(mold PRIVATE ${PATH})
 endfunction()
 
-foreach (SOURCE IN LISTS MOLD_ELF_TEMPLATE_FILES)
-  foreach(TARGET IN LISTS MOLD_ELF_TARGETS)
+foreach (SOURCE IN LISTS MOLD_TEMPLATE_FILES)
+  foreach(TARGET IN LISTS MOLD_TARGETS)
     mold_instantiate_templates(${SOURCE} ${TARGET})
+    set(HAVE_TARGET_${TARGET} 1)
   endforeach()
 endforeach()
 
 # Add other non-template source files.
 target_sources(mold PRIVATE
-  git-hash.cc
+  lib/aho-corasick.cc
+  lib/arena.cc
   lib/compress.cc
   lib/crc32.cc
   lib/demangle.cc
   lib/filepath.cc
   lib/glob.cc
   lib/hyperloglog.cc
-  lib/mimalloc.cc
-  lib/multi-glob.cc
   lib/perf.cc
   lib/random.cc
   lib/tar.cc
-  src/arch-arm32.cc
-  src/arch-i386.cc
-  src/arch-m68k.cc
-  src/arch-ppc32.cc
-  src/arch-ppc64v1.cc
-  src/arch-ppc64v2.cc
-  src/arch-s390x.cc
-  src/arch-sparc64.cc
-  src/arch-x86-64.cc
-  src/config.cc
   src/elf.cc
+  src/entry.cc
   third-party/rust-demangle/rust-demangle.c
   )
 
 if(WIN32)
   target_sources(mold PRIVATE
-    lib/jobs-win32.cc
-    lib/mapped-file-win32.cc
-    lib/signal-win32.cc
+    src/jobs-win32.cc
+    src/mapped-file-win32.cc
+    src/signal-win32.cc
     )
 else()
   target_sources(mold PRIVATE
-    lib/jobs-unix.cc
-    lib/mapped-file-unix.cc
-    lib/signal-unix.cc
+    src/jobs-unix.cc
+    src/mapped-file-unix.cc
+    src/signal-unix.cc
     )
 endif()
 
+# Create config.h file
+configure_file(lib/config.h.in config.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+# Test configs
 include(CTest)
 
 if(BUILD_TESTING)
@@ -448,10 +493,12 @@ if(NOT CMAKE_SKIP_INSTALL_RULES)
       file(CREATE_LINK \${OLD_REL} \$ENV{DESTDIR}\${NEW_ABS} SYMBOLIC)")
   endfunction()
 
-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
-    ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  if(NOT WIN32)
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
+      ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  endif()
 endif()

README.md

@@ -48,20 +48,20 @@ Binary packages for the following systems are currently available:
 
 mold is written in C++20, so if you build mold yourself, you will need a
 recent version of a C++ compiler and a C++ standard library. We recommend GCC
-10.2 or Clang 12.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).
+10.2 or Clang 16.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).
 
 ### Install Dependencies
 
 To install build dependencies, run `./install-build-deps.sh` in this
 directory. It will detect your Linux distribution and attempt to install the
-necessary packages. You may need to run it as root.
+necessary packages.
 
 ### Compile mold
 
 ```shell
 git clone --branch stable https://github.com/rui314/mold.git
 cd mold
-./install-build-deps.sh
+sudo ./install-build-deps.sh
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ -B build
 cmake --build build -j$(nproc)
 sudo cmake --build build --target install
@@ -120,7 +120,7 @@ accept an absolute path as an argument for `-fuse-ld`.
 Create `.cargo/config.toml` in your project directory with the following:
 
 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 linker = "clang"
 rustflags = ["-C", "link-arg=-fuse-ld=/path/to/mold"]
 ```
@@ -131,7 +131,7 @@ example above, we use `clang` as a linker driver since it always accepts the
 may be able to remove the `linker = "clang"` line.
 
 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 rustflags = ["-C", "link-arg=-fuse-ld=mold"]
 ```
 

dist.sh

@@ -1,40 +1,45 @@
 #!/bin/bash
 #
-# This script creates a mold binary distribution. The output is written in
-# `dist` directory as `mold-$version-$arch-linux.tar.gz` (e.g.,
-# `mold-1.0.3-x86_64-linux.tar.gz`).
+# This script creates a mold binary distribution. The output is written to
+# the `dist` directory as `mold-$version-$arch-linux.tar.gz` (e.g.
+# `mold-2.40.0-x86_64-linux.tar.gz`).
 #
-# The mold executable created by this script is statically linked to
-# libstdc++ but dynamically linked to libc, libm, libz, and librt, as
-# these libraries almost always exist on any Linux system. We can't
-# statically link libc because doing so would disable dlopen(), which is
-# necessary to open the LTO linker plugin.
-#
-# This script aims to produce reproducible outputs. That means if you run
-# the script twice on the same git commit, it should produce bit-by-bit
-# identical binary files. This property is crucial as a countermeasure
-# against supply chain attacks. With this, you can verify that the binary
-# files distributed on the GitHub release pages are created from the
-# commit with release tags by rebuilding the binaries yourself.
+# This script aims to produce reproducible outputs. That means each time
+# it's run on the same git commit, it generates a bit-for-bit identical
+# binary file regardless of when or where it's executed. This property
+# serves as a strong safeguard against supply chain attacks. With a
+# reproducible build, anyone can independently verify that the binary
+# files published on our GitHub release page were built from the git
+# commit tagged for release by rebuilding the binaries themselves.
 #
 # Debian provides snapshot.debian.org to host all historical binary
-# packages. We use it to construct Podman images pinned to a
-# particular timestamp.
+# packages. We use it to construct a container image pinned to a
+# particular timestamp. snapshot.debian.org is known to be very slow,
+# but that shouldn't be a big problem for us because we only need that
+# site the first time.
 #
-# We aim to use a reasonably old Debian version because we'll dynamically
-# link glibc to mold, and a binary linked against a newer version of glibc
-# won't work on a system with an older version of glibc.
+# The mold executable created by this script is statically linked to
+# libstdc++, but dynamically linked to glibc, libm and a few other
+# libraries, as these libraries are almost always available on any Linux
+# system. We can't statically link glibc because doing so would disable
+# dlopen(), which is required to load the LTO linker plugin.
 #
-# We want to build mold with Clang rather than GCC because mold's
+# We use a reasonably old Debian version for the build environment because
+# a binary dynamically linked against a newer version of glibc won't work
+# on a system with an older version of glibc.
+#
+# We prefer to build mold with Clang rather than GCC because mold's
 # Identical Code Folding works best with the LLVM address significance
-# table (.llvm_addrsig). Building a release binary with GCC yields a
-# slightly larger binary than Clang's.
+# table (.llvm_addrsig). Building a release binary with GCC produces a
+# slightly larger binary than with Clang.
 #
 # We need a recent version of Clang to build mold. If it's not available
 # via apt-get, we'll build it ourselves.
 #
-# You may need to install qemu-user-static package to build non-native
-# binaries.
+# This script can be used to create non-native binaries (e.g., building
+# aarch64 binary on x86-64) because Podman automatically runs everything
+# under QEMU if the container image is not native. To use this script for
+# non-native builds, you may need to install the qemu-user-static package.
 
 set -e -x
 cd "$(dirname $0)"
@@ -74,21 +79,29 @@ fi
 case $arch in
 x86_64)
   # Debian 9 (Stretch) released in June 2017.
-  # We use a Google-provided mirror (gcr.io) of the official Docker hub
-  # (docker.io) because docker.io has a strict rate limit policy.
+  #
+  # We use a Google-provided mirror (gcr.io) instead of the official Docker
+  # Hub (docker.io) because docker.io has a strict rate limit policy.
+  #
+  # The toolchain in Debian 9 is too old to build mold, so we rebuild it
+  # from source. We download source archives from official sites and build
+  # them locally, rather than downloading pre-built binaries from somewhere
+  # else, to avoid relying on unverifiable third-party binary blobs. Podman
+  # caches the result of each RUN command, so rebuilding is done only once
+  # per host.
   cat <<EOF | $image_build
 FROM mirror.gcr.io/library/debian:stretch@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends wget file make gcc g++ zlib1g-dev libssl-dev && \
+  apt-get install -y --no-install-recommends wget file make gcc g++ git zlib1g-dev libssl-dev ca-certificates && \
   rm -rf /var/lib/apt/lists
 
 # Build CMake 3.27
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
   ./bootstrap --parallel=\$(nproc) && \
   make -j\$(nproc) && \
   make install && \
@@ -97,11 +110,11 @@ RUN mkdir /build && \
 # Build GCC 14
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
   mkdir gmp mpc mpfr && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
   ./configure --prefix=/usr --enable-languages=c,c++ --disable-bootstrap --disable-multilib && \
   make -j\$(nproc) && \
   make install && \
@@ -111,7 +124,7 @@ RUN mkdir /build && \
 # Build GNU binutils 2.43
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://ftpmirror.gnu.org/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
   ./configure --prefix=/usr && \
   make -j\$(nproc) && \
   make install && \
@@ -120,16 +133,16 @@ RUN mkdir /build && \
 # Build Python 3.12.7
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz | tar xzf - --strip-components=1 && \
   ./configure && \
   make -j\$(nproc) && \
   make install && \
   rm -rf /build
 
-# Build LLVM 19
+# Build LLVM 20
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --no-check-certificate --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-19.1.7.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.3.tar.gz | tar xzf - --strip-components=1 && \
   mkdir b && \
   cd b && \
   cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang ../llvm && \
@@ -141,16 +154,16 @@ EOF
 aarch64 | arm | ppc64le | s390x)
   # Debian 11 (Bullseye) released in August 2021
   #
-  # We don't want to build Clang for these targets with Qemu becuase
-  # that'd take extremely long time. Also I believe old build machines
-  # are usually x86-64.
+  # We don't want to build Clang for these targets on QEMU becuase it
+  # would take an extremely long time. Also, I believe old Linux boxes
+  # are typically x86-64.
   cat <<EOF | $image_build
 FROM mirror.gcr.io/library/debian:bullseye-20240904@sha256:8ccc486c29a3ad02ad5af7f1156e2152dff3ba5634eec9be375269ef123457d8
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake git && \
   ln -sf /usr/bin/clang-16 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-16 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -163,19 +176,20 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^URIs/d' -e 's/^# http/URIs: http/' /etc/apt/sources.list.d/debian.sources && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake git && \
   ln -sf /usr/bin/clang-18 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-18 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
 EOF
   ;;
 loongarch64)
-  # LoongArch build is not reproducible yet
   cat <<EOF | $image_build
-FROM mirror.gcr.io/loongarch64/debian:sid
+FROM mirror.gcr.io/loongarch64/debian:sid@sha256:0356df4e494bbb86bb469377a00789a5b42bbf67d5ff649a3f9721b745cbef77
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake && \
+RUN echo 'deb http://deb.debian.org/debian sid main' > /etc/apt/sources.list && \
+  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake git && \
   ln -sf /usr/bin/clang-19 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-19 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -189,20 +203,34 @@ esac
 version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' CMakeLists.txt)
 dest=mold-$version-$arch-linux
 
-# Source tarballs available on GitHub don't contain .git history.
+# Source tarballs available on GitHub don't contain .git directory.
 # Clone the repo if missing.
 [ -d .git ] || git clone --branch v$version --depth 1 --bare https://github.com/rui314/mold .git
 
 # We use the timestamp of the last Git commit as the file timestamp
 # for build artifacts.
-timestamp="$(git log -1 --format=%ci)"
+timestamp=$(git log -1 --format=%ct)
+
+# `uname -m` in an ARM32 container running on an ARM64 host reports it
+# not as ARM32 but as ARM64. That confuses BLAKE3's cmake script and
+# erroneously enables NEON SIMD instructions. `setarch` can be used to
+# change the output of `uname -m`.
+setarch=
+[ $arch = arm ] && setarch='setarch linux32'
 
-# Build mold in a container.
 mkdir -p dist
 
+# Build mold in a container.
+#
+# SOURCE_DATE_EPOCH is a standardized environment variable that allows
+# build artifacts to appear as if they were built at a specific time.
+# We use it to control how the compiler expands the C/C++ __DATE__ and
+# __TIME__ macros.
 podman run --arch $arch -it --rm --userns=host --pids-limit=-1 --network=none \
-  -v "$(pwd):/mold:ro" -v "$(pwd)/dist:/dist" $image bash -c "
+  --pull=never -v "$(pwd):/mold:ro" -v "$(pwd)/dist:/dist" $image \
+   $setarch bash -c "
 set -e
+export SOURCE_DATE_EPOCH=$timestamp
 mkdir /build
 cd /build
 cmake -DCMAKE_BUILD_TYPE=Release -DMOLD_MOSTLY_STATIC=1 -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ /mold
@@ -212,7 +240,7 @@ cmake -DMOLD_USE_MOLD=1 .
 cmake --build . -j\$(nproc)
 ctest --output-on-failure -j\$(nproc)
 cmake --install . --prefix $dest --strip
-find $dest -print | xargs touch --no-dereference --date='$timestamp'
+find $dest -print | xargs touch --no-dereference --date=@$timestamp
 find $dest -print | sort | tar -cf - --no-recursion --files-from=- | gzip -9nc > /dist/$dest.tar.gz
 cp mold /dist
 sha256sum /dist/$dest.tar.gz

docs/memory-sanitizer.md

@@ -0,0 +1,179 @@
+# Instrumenting mold with MemorySanitizer
+
+## Introduction
+
+Per <https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo>:
+
+> If you want MemorySanitizer to work properly and not produce any false
+> positives, you must ensure that all the code in your program and in
+> libraries it uses is instrumented (i.e. built with `-fsanitize=memory`).
+> In particular, you would need to link against an MSan-instrumented C++
+> standard library. We recommend to use [libc++](https://libcxx.llvm.org/)
+> for that purpose.
+
+## Building instrumented libc++
+
+Build an MSan-instrumented libc++ from source:
+
+```sh
+cd ~
+git clone https://github.com/llvm/llvm-project
+cd llvm-project
+cmake -S ./runtimes -B build-libcxx -G Ninja  \
+    -DCMAKE_BUILD_TYPE=Release                \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+    -DCMAKE_C_COMPILER=clang                  \
+    -DCMAKE_CXX_COMPILER=clang++              \
+    -DLLVM_USE_SANITIZER=MemoryWithOrigins
+cmake --build build-libcxx -- cxx cxxabi
+export LIBCXX="$HOME/llvm-project/build-libcxx" # for subsequent build steps
+```
+
+Upon success, `./build-libcxx/{include,lib}` will contain the resulting
+headers and shared libraries.
+
+## Linking mold against instrumented libc++
+
+Use the `MOLD_USE_MSAN` and `MOLD_STDLIB_PREFIX` cmake variables to link
+mold against the instrumented build of libc++:
+
+```sh
+cd ~
+git clone https://github.com/rui314/mold.git
+cd mold
+cmake -B build -G Ninja            \
+    -DCMAKE_BUILD_TYPE=Debug       \
+    -DCMAKE_C_COMPILER=clang       \
+    -DCMAKE_CXX_COMPILER=clang++   \
+    -DMOLD_USE_MSAN=ON             \
+    -DMOLD_STDLIB_PREFIX="$LIBCXX" \
+    -DMOLD_USE_MIMALLOC=OFF        \
+    -DMOLD_USE_SYSTEM_TBB=ON
+cmake --build build
+```
+
+Most of mold's tests (except those for `-flto`) should work at this
+point. Run them like normal:
+
+```sh
+ctest --test-dir build
+```
+
+Any resulting MemorySanitizer errors should be visible in
+`./build/Temporary/Testing/LastTest.log`.
+
+## Building instrumented LTO plugin (experimental)
+
+Exercising `-flto` with MemorySanitizer (and without false positives)
+requires instrumenting the transitive dependencies of `lto-unix.cc`:
+
+- `libiberty.a`
+- `liblto_plugin.so`
+- `LLVMgold.so`
+
+> [!NOTE]
+> This is more involved and time-consuming than building only libc++ from
+> source. The cost-benefit ratio of this additional instrumentation and
+> test coverage may be unfavorable in many cases.
+
+> [!IMPORTANT]
+> The following steps are experimental and unlikely to work exactly as-is
+> under CI runners and individual developer environments. Consider the
+> following a starting point rather than a complete HOWTO.
+
+Build an MSan-instrumented GNU libiberty:
+
+```sh
+cd ~
+git clone [email protected]:gcc-mirror/gcc.git
+cd gcc/libiberty
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `./libiberty.a` into the build toolchain being
+tested with mold. If the system image is ephemeral or disposable (e.g. a
+short-lived VM or container), a quick-and-dirty install could look like:
+
+```sh
+sudo cp /usr/lib/libiberty.a{,.bak}
+sudo cp ./libiberty.a /usr/lib/
+```
+
+Build an MSan-instrumented GCC LTO plugin library:
+
+```sh
+cd ~/gcc/lto-plugin # use already-cloned repo from earlier step
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure --with-libiberty=/usr/lib
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `liblto_plugin.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+export GCC_VERSION="$(gcc -dumpversion)"
+sudo cp /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/liblto_plugin.so{,.bak}
+sudo cp .libs/liblto_plugin.so /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/
+```
+
+Build an MSan-instrumented LLVM LTO plugin library:
+
+```sh
+cd ~/llvm-project # use already-cloned repo from earlier step
+
+CF="-nostdinc++ -isystem $LIBCXX/include/c++/v1"
+INCDIR="$(find /usr -name plugin-api.h -type f | head -1 | xargs dirname)"
+
+function configure
+{
+    cmake -S ./llvm -B build-plugin -G Ninja   \
+        -DLLVM_ENABLE_PROJECTS=clang           \
+        -DLLVM_TARGETS_TO_BUILD=X86            \
+        -DCMAKE_BUILD_TYPE=Release             \
+        -DCMAKE_C_COMPILER=clang               \
+        -DCMAKE_CXX_COMPILER=clang++           \
+        -DCMAKE_C_FLAGS="$CF"                  \
+        -DCMAKE_CXX_FLAGS="$CF"                \
+        -DCMAKE_EXE_LINKER_FLAGS="$1"          \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON   \
+        -DLLVM_BINUTILS_INCDIR="$INCDIR"       \
+        -DLLVM_USE_SANITIZER=MemoryWithOrigins
+}
+
+# workaround for linker issues: configure twice with different LDFLAGS
+configure "-nostdlib++ -L $LIBCXX/lib -Wl,--rpath=$LIBCXX/lib"
+configure "-nostdlib++ -L $LIBCXX/lib -lc++ -Wl,--rpath=$LIBCXX/lib"
+
+cmake --build build-plugin -- LLVMgold.so
+```
+
+Install the resulting `LLVMgold.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+sudo cp /usr/lib/LLVMgold.so{,.bak}
+sudo cp ./build-plugin/lib/LLVMgold.so /usr/lib/
+```
+
+mold's tests for `-flto` should now (mostly) work, though there do seem
+to be issues around `gcc` inferring `-flto` when not explicitly specified.
+One workaround is to use `clang` as `TEST_CC`:
+
+```sh
+cd ~/mold # use already-cloned and built repo from earlier step
+TEST_CC=clang TEST_CXX=clang++ ctest --test-dir build
+```
+
+## References
+
+- <https://github.com/google/sanitizers/wiki/MemorySanitizerBootstrappingClang>
+- <https://llvm.org/docs/GoldPlugin.html#lto-how-to-build>

docs/mold.1

@@ -1,6 +1,6 @@
 .\" generated with Ronn-NG/v0.9.1
 .\" http://github.com/apjanke/ronn-ng/tree/0.9.1
-.TH "MOLD" "1" "February 2025" ""
+.TH "MOLD" "1" "August 2026" ""
 .SH "NAME"
 \fBmold\fR \- a modern linker
 .SH "SYNOPSIS"
@@ -70,6 +70,8 @@ Permit or do not permit mold to create a debug info file in the background\.
 .TP
 \fB\-\-fork\fR, \fB\-\-no\-fork\fR
 Spawn a child process and let it do the actual linking\. When linking a large program, the OS kernel can take a few hundred milliseconds to terminate a \fBmold\fR process\. \fB\-\-fork\fR hides that latency\. By default, it does fork\.
+.IP
+Note that \fB\-\-fork\fR also hides the resource usage statistics reported by time(2), since it doesn't call waitpid(2) on the child process\. If you need those statistics, pass \fB\-\-no\-fork\fR\.
 .TP
 \fB\-\-perf\fR
 Print performance statistics\.
@@ -130,6 +132,17 @@ Use multiple threads\. By default, \fBmold\fR uses as many threads as the number
 \fB\-\-quick\-exit\fR, \fB\-\-no\-quick\-exit\fR
 Use or do not use \fBquick_exit\fR to exit\.
 .TP
+\fB\-\-zero\-to\-bss\fR
+Convert all\-zero data sections into BSS\.
+.IP
+When this option is enabled, \fBmold\fR scans input data sections that are not of type \fBSHT_NOBITS\fR and checks whether their contents consist solely of zero bytes\. Such sections are then converted into BSS (\fBSHT_NOBITS\fR) sections\. This reduces the size of the output file, since BSS sections occupy no space in the file image\.
+.IP
+This behavior is especially useful for user\-defined sections created with \fB__attribute__((section("\.sectname")))\fR that contain uninitialized global variables\. GCC and Clang do not automatically mark such sections as BSS even if their contents are entirely zero, and instead emit them as regular data sections\.
+.IP
+For example, consider \fB__attribute__((section("\.sectname"))) int vec[256];\fR\.
+.IP
+By default, this results in a \fB\.sectname\fR section of type \fBSHT_PROGBITS\fR filled with zeros\. With \fB\-\-zero\-to\-bss\fR, the linker will recognize it as empty data and convert it to a \fBSHT_NOBITS\fR section, reducing the output file size without changing runtime semantics\.
+.TP
 \fB\-z rewrite\-endbr\fR, \fB\-z norewrite\-endbr\fR
 As a security measure, some CPU instruction sets have recently gained a feature to protect control flow integrity by disallowing indirect branches by default\. If the feature is enabled, the instruction that is executed immediately after an indirect branch must be an branch target marker instruction, or a CPU\-level fault will raise\. The marker instruction is also known as "landing pad" instruction, to which indirect branches can land\. This feature makes ROP attacks harder to conduct\.
 .IP
@@ -180,7 +193,10 @@ Omit \fB\.debug_*\fR sections from the output file\.
 Read linker script from \fIfile\fR\.
 .TP
 \fB\-X\fR, \fB\-\-discard\-locals\fR
-Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\.
+Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\. This is the default; use \fB\-\-discard\-none\fR to keep temporary local symbols\.
+.TP
+\fB\-\-discard\-none\fR
+Keep all local symbols in the symbol table, including temporary ones starting with \fB\.L\fR, except ones in mergeable sections\. This restores the default behavior of GNU ld\. \fBmold\fR and \fBld\.lld\fR discard temporary local symbols by default\.
 .TP
 \fB\-e\fR \fIsymbol\fR, \fB\-\-entry\fR=\fIsymbol\fR:
 
@@ -245,6 +261,9 @@ Alias for \fB\-\-section\-start=\.data=\fR\fIaddress\fR\.
 \fB\-\-Ttext\fR=\fIaddress\fR
 Alias for \fB\-\-section\-start=\.text=\fR\fIaddress\fR\.
 .TP
+\fB\-\-Ttext\-segment\fR=\fIaddress\fR
+Set the address of the first byte of the text segment\.
+.TP
 \fB\-\-allow\-multiple\-definition\fR
 Normally, the linker reports an error if there are more than one definition of a symbol\. This option changes the default behavior so that it doesn't report an error for duplicate definitions and instead use the first definition\.
 .TP
@@ -269,8 +288,8 @@ Synonym for \fB\-\-build\-id=sha256\fR\.
 \fB\-\-no\-build\-id\fR
 Synonym for \fB\-\-build\-id=none\fR\.
 .TP
-\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzstd\fR | \fBnone\fR ]
-Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\.
+\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzlib:\fR\fIN\fR | \fBzstd\fR | \fBzstd:\fR\fIN\fR | \fBnone\fR ]
+Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\. \fBzlib:\fR\fIN\fR selects the zlib compression level, where \fIN\fR is between 0 and 9 (0 means no compression), and \fBzstd:\fR\fIN\fR selects the zstd compression level, where \fIN\fR is between 1 and 22\. Higher levels achieve better compression ratios but are slower\. \fBzlib\fR is equivalent to \fBzlib:1\fR and \fBzstd\fR is equivalent to \fBzstd:3\fR\.
 .TP
 \fB\-\-defsym\fR=\fIsymbol\fR=\fIvalue\fR
 Define \fIsymbol\fR as an alias for \fIvalue\fR\.
@@ -358,10 +377,16 @@ Embed a specified string into the \fB\.note\.package\fR section\. This option is
 .IP
 An argument to this option is treated as percent\-encoded and decoded before being inserted into the section, allowing you to avoid the use of the comma (\fB,\fR) character in the argument\. This is useful because the compiler replaces all occurrences of commas in \fB\-Wl,\fR with spaces before forwarding them to the linker\. Note that \fBmold\fR always interprets the argument as percent\-encoded, so you also need to escape all occurrences of \fB%\fR as \fB%25\fR\.
 .TP
-\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBnone\fR ]
+\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBandroid\fR | \fBandroid+relr\fR | \fBnone\fR ]
+Compress dynamic relocations in the output file\.
+.IP
 If \fBrelr\fR is specified, all \fBR_*_RELATIVE\fR relocations are put into \fB\.relr\.dyn\fR section instead of \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section\. Since \fB\.relr\.dyn\fR section uses a space\-efficient encoding scheme, specifying this flag can reduce the size of the output\. This is typically most effective for position\-independent executable\.
 .IP
-Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2022, only ChromeOS, Android and Fuchsia support it\.
+If \fBandroid\fR is specified, all dynamic relocations are encoded in Android's packed relocation format (APS2) and stored in the \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section, whose section type is changed to \fBSHT_ANDROID_REL\fR or \fBSHT_ANDROID_RELA\fR\. The dynamic table is updated to use \fBDT_ANDROID_REL\fR / \fBDT_ANDROID_RELA\fR and their \fBSZ\fR variants in place of the standard tags\.
+.IP
+\fBandroid+relr\fR combines both: \fBR_*_RELATIVE\fR relocations go to \fB\.relr\.dyn\fR and the rest are APS2\-encoded\. This is the configuration used by most Android system binaries\.
+.IP
+Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2026, glibc (since 2\.36), musl (since 1\.2\.4), Android, ChromeOS, and Fuchsia all support it\. The APS2 format, on the other hand, is only handled by Android's bionic loader; the encoder itself is target\-agnostic, but the produced binary can only be loaded on platforms where bionic (or a compatible loader) handles \fBDT_ANDROID_REL(A)\fR\.
 .TP
 \fB\-\-pie\fR, \fB\-\-pic\-executable\fR, \fB\-\-no\-pie\fR, \fB\-\-no\-pic\-executable\fR
 Create a position\-independent executable\.
@@ -427,6 +452,9 @@ How to handle undefined symbols\.
 \fB\-\-version\-script\fR=\fIfile\fR
 Read version script from \fIfile\fR\. If \fIfile\fR does not exist in the current directory, it is searched from library search paths for the sake of compatibility with GNU ld\.
 .TP
+\fB\-w\fR, \fB\-\-no\-warnings\fR
+Suppress warnings and cancel \fB\-\-fatal\-warnings\fR\.
+.TP
 \fB\-\-warn\-common\fR, \fB\-\-no\-warn\-common\fR
 Warn about common symbols\.
 .TP
@@ -483,9 +511,9 @@ By default, \fBmold\fR generates a relro segment\. \fB\-z norelro\fR disables th
 \fB\-z separate\-loadable\-segments\fR, \fB\-z separate\-code\fR, \fB\-z noseparate\-code\fR
 If one memory page contains multiple segments, the page protection bits are set in such a way that the needed attributes (writable or executable) are satisfied for all segments\. This usually happens at a boundary of two segments with two different attributes\.
 .IP
-\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\. This is the default\.
+\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\.
 .IP
-\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\.
+\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\. This is the default\.
 .IP
 \fBnoseparate\-code\fR does not add any paddings between segments\.
 .TP

docs/mold.md

@@ -159,6 +159,10 @@ but as `-o magic`.
   program, the OS kernel can take a few hundred milliseconds to terminate a
   `mold` process. `--fork` hides that latency. By default, it does fork.
 
+  Note that `--fork` also hides the resource usage statistics reported by
+  time(2), since it doesn't call waitpid(2) on the child process. If you
+  need those statistics, pass `--no-fork`.
+
 * `--perf`:
   Print performance statistics.
 
@@ -267,6 +271,28 @@ but as `-o magic`.
 * `--quick-exit`, `--no-quick-exit`:
   Use or do not use `quick_exit` to exit.
 
+* `--zero-to-bss`:
+  Convert all-zero data sections into BSS.
+
+  When this option is enabled, `mold` scans input data sections that are not
+  of type `SHT_NOBITS` and checks whether their contents consist solely of
+  zero bytes. Such sections are then converted into BSS (`SHT_NOBITS`)
+  sections. This reduces the size of the output file, since BSS sections
+  occupy no space in the file image.
+
+  This behavior is especially useful for user-defined sections created with
+  `__attribute__((section(".sectname")))` that contain uninitialized global
+  variables. GCC and Clang do not automatically mark such sections as BSS
+  even if their contents are entirely zero, and instead emit them as regular
+  data sections.
+
+  For example, consider `__attribute__((section(".sectname"))) int vec[256];`.
+
+  By default, this results in a `.sectname` section of type `SHT_PROGBITS`
+  filled with zeros. With `--zero-to-bss`, the linker will recognize it as
+  empty data and convert it to a `SHT_NOBITS` section, reducing the output
+  file size without changing runtime semantics.
+
 * `-z rewrite-endbr`, `-z norewrite-endbr`:
   As a security measure, some CPU instruction sets have recently gained a
   feature to protect control flow integrity by disallowing indirect branches
@@ -349,7 +375,14 @@ but as `-o magic`.
   Discard temporary local symbols to reduce the sizes of the symbol table and
   the string table. Temporary local symbols are local symbols starting with
   `.L`. Compilers usually generate such symbols for unnamed program elements
-  such as string literals or floating-point literals.
+  such as string literals or floating-point literals. This is the default;
+  use `--discard-none` to keep temporary local symbols.
+
+* `--discard-none`:
+  Keep all local symbols in the symbol table, including temporary ones
+  starting with `.L`, except ones in mergeable sections. This restores the
+  default behavior of GNU ld. `mold` and `ld.lld` discard temporary local
+  symbols by default.
 
 * `-e` _symbol_, `--entry`=_symbol_:
 
@@ -432,6 +465,9 @@ but as `-o magic`.
 * `--Ttext`=_address_:
   Alias for `--section-start=.text=`_address_.
 
+* `--Ttext-segment`=_address_:
+  Set the address of the first byte of the text segment.
+
 * `--allow-multiple-definition`:
   Normally, the linker reports an error if there are more than one definition
   of a symbol. This option changes the default behavior so that it doesn't
@@ -480,9 +516,14 @@ but as `-o magic`.
 * `--no-build-id`:
   Synonym for `--build-id=none`.
 
-* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zstd` | `none` ]:
+* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zlib:`_N_ | `zstd` | `zstd:`_N_ | `none` ]:
   Compress DWARF debug info (`.debug_*` sections) using the zlib or zstd
-  compression algorithm. `zlib-gabi` is an alias for `zlib`.
+  compression algorithm. `zlib-gabi` is an alias for `zlib`. `zlib:`_N_
+  selects the zlib compression level, where _N_ is between 0 and 9 (0
+  means no compression), and `zstd:`_N_ selects the zstd compression
+  level, where _N_ is between 1 and 22. Higher levels achieve better
+  compression ratios but are slower. `zlib` is equivalent to `zlib:1`
+  and `zstd` is equivalent to `zstd:3`.
 
 * `--defsym`=_symbol_=_value_:
   Define _symbol_ as an alias for _value_.
@@ -632,16 +673,32 @@ but as `-o magic`.
   argument as percent-encoded, so you also need to escape all occurrences
   of `%` as `%25`.
 
-* `--pack-dyn-relocs`=[ `relr` | `none` ]:
+* `--pack-dyn-relocs`=[ `relr` | `android` | `android+relr` | `none` ]:
+  Compress dynamic relocations in the output file.
+
   If `relr` is specified, all `R_*_RELATIVE` relocations are put into
   `.relr.dyn` section instead of `.rel.dyn` or `.rela.dyn` section. Since
   `.relr.dyn` section uses a space-efficient encoding scheme, specifying this
   flag can reduce the size of the output. This is typically most effective for
   position-independent executable.
 
+  If `android` is specified, all dynamic relocations are encoded in Android's
+  packed relocation format (APS2) and stored in the `.rel.dyn` or `.rela.dyn`
+  section, whose section type is changed to `SHT_ANDROID_REL` or
+  `SHT_ANDROID_RELA`. The dynamic table is updated to use `DT_ANDROID_REL`
+  / `DT_ANDROID_RELA` and their `SZ` variants in place of the standard tags.
+
+  `android+relr` combines both: `R_*_RELATIVE` relocations go to `.relr.dyn`
+  and the rest are APS2-encoded. This is the configuration used by most
+  Android system binaries.
+
   Note that a runtime loader has to support `.relr.dyn` to run executables or
-  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2022, only
-  ChromeOS, Android and Fuchsia support it.
+  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2026, glibc
+  (since 2.36), musl (since 1.2.4), Android, ChromeOS, and Fuchsia all
+  support it. The APS2 format, on the other hand, is only handled by
+  Android's bionic loader; the encoder itself is target-agnostic, but the
+  produced binary can only be loaded on platforms where bionic (or a
+  compatible loader) handles `DT_ANDROID_REL(A)`.
 
 * `--pie`, `--pic-executable`, `--no-pie`, `--no-pic-executable`:
   Create a position-independent executable.
@@ -731,6 +788,9 @@ but as `-o magic`.
   directory, it is searched from library search paths for the sake of
   compatibility with GNU ld.
 
+* `-w`, `--no-warnings`:
+  Suppress warnings and cancel `--fatal-warnings`.
+
 * `--warn-common`, `--no-warn-common`:
   Warn about common symbols.
 
@@ -828,10 +888,10 @@ but as `-o magic`.
   segments with two different attributes.
 
   `separate-loadable-segments` adds paddings between segments with different
-  attributes so that they do not share the same page. This is the default.
+  attributes so that they do not share the same page.
 
   `separate-code` adds paddings only between executable and non-executable
-  segments.
+  segments. This is the default.
 
   `noseparate-code` does not add any paddings between segments.
 

install-build-deps.sh

@@ -8,24 +8,24 @@ set -e
 set -x
 
 case "$ID" in
-ubuntu | pop | linuxmint | debian | raspbian | neon)
+ubuntu | pop | linuxmint | debian | raspbian | neon | zorin)
   apt-get update
   apt-get install -y cmake gcc g++ clang gdb
   ;;
-fedora | amzn | rhel | centos)
-  dnf install -y gcc-g++ cmake glibc-static libstdc++-static diffutils util-linux tar
+fedora | fedora-* | amzn | rhel | centos)
+  dnf install -y gcc-c++ cmake glibc-static libstdc++-static diffutils util-linux tar
   ;;
-rocky)
-  dnf install -y gcc-g++ cmake diffutils util-linux
+rocky | ol)
+  dnf install -y gcc-c++ cmake diffutils util-linux
   ;;
 opensuse-*)
-  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux
+  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux gawk
   ;;
 gentoo)
   emerge-webrsync
   FEATURES='getbinpkg binpkg-request-signature' emerge dev-build/cmake
   ;;
-arch | archarm | artix | endeavouros)
+arch | archarm | artix | endeavouros | manjaro | cachyos)
   pacman -Sy --needed --noconfirm base-devel cmake util-linux
   ;;
 void)
@@ -42,6 +42,10 @@ clear-linux-os)
 almalinux)
   dnf install -y gcc-toolset-13-gcc-c++ gcc-toolset-13-libstdc++-devel cmake diffutils
   ;;
+altlinux)
+  apt-get update
+  apt-get install -y gcc-c++ make cmake ctest diffutils util-linux
+  ;;
 freebsd)
   pkg update
   pkg install -y cmake bash binutils gcc

lib/aho-corasick.cc

@@ -0,0 +1,142 @@
+// This file implements the Aho-Corasick algorithm to search multiple
+// strings within an input string simultaneously. It is essentially a
+// trie with additional links. For details, see
+// https://en.wikipedia.org/wiki/Aho-Corasick_algorithm.
+//
+// We use it for simple glob patterns in version scripts or dynamic
+// list files. Here are some examples of glob patterns:
+//
+//    qt_private_api_tag*
+//    *16QAccessibleCache*
+//    *32QAbstractFileIconProviderPrivate*
+//    *17QPixmapIconEngine*
+//
+// Aho-Corasick can do only substring search, so it cannot handle
+// complex glob patterns such as `*foo*bar*`. We handle such patterns
+// with the Glob class.
+
+#include "lib.h"
+
+#include <queue>
+
+namespace mold {
+
+bool AhoCorasick::can_handle(std::string_view pat) {
+  if (pat.starts_with('*'))
+    pat.remove_prefix(1);
+  if (pat.ends_with('*'))
+    pat.remove_suffix(1);
+  return !pat.empty() && pat.find_first_of("*?[\\") == pat.npos;
+}
+
+i32 AhoCorasick::find_child(i32 node, u8 ch) const {
+  if (node == 0)
+    return root_children[ch];
+
+  for (i32 i = nodes[node].first_child; i != -1; i = nodes[i].next_sibling)
+    if (nodes[i].ch == ch)
+      return i;
+  return -1;
+}
+
+i32 AhoCorasick::add_child(i32 node, u8 ch) {
+  if (i32 child = find_child(node, ch); child != -1)
+    return child;
+
+  i32 child = nodes.size();
+  i32 sibling = nodes[node].first_child;
+  nodes.emplace_back();
+  nodes[child].next_sibling = sibling;
+  nodes[child].ch = ch;
+  nodes[node].first_child = child;
+
+  if (node == 0)
+    root_children[ch] = child;
+  return child;
+}
+
+i64 AhoCorasick::find(std::string_view str) {
+  if (nodes.empty())
+    return -1;
+
+  i32 idx = 0;
+  i64 val = -1;
+
+  auto walk = [&](u8 c) {
+    for (i32 j = idx; j != -1; j = nodes[j].suffix_link) {
+      if (i32 child = find_child(j, c); child != -1) {
+        idx = child;
+        val = std::max(val, nodes[child].value);
+        return;
+      }
+    }
+    idx = 0;
+  };
+
+  walk('\0');
+  for (u8 c : str)
+    walk(c);
+  walk('\0');
+  return val;
+}
+
+bool AhoCorasick::add(std::string_view pat, i64 val) {
+  assert(can_handle(pat));
+  if (nodes.empty()) {
+    root_children.fill(-1);
+    nodes.emplace_back();
+  }
+  i32 idx = 0;
+
+  auto walk = [&](u8 c) { idx = add_child(idx, c); };
+
+  // We handle "foo" as if "\0foo\0", "*foo" as if "foo\0", "foo*" as
+  // if "\0foo", and "*foo*" as if "foo". Aho-Corasick can do only
+  // substring matching, so we use \0 as a beginning/end-of-string
+  // markers.
+  if (!pat.starts_with('*'))
+    walk('\0');
+  for (u8 c : pat)
+    if (c != '*')
+      walk(c);
+  if (!pat.ends_with('*'))
+    walk('\0');
+
+  nodes[idx].value = std::max(nodes[idx].value, val);
+  return true;
+}
+
+void AhoCorasick::compile() {
+  if (nodes.empty())
+    return;
+
+  // A failure link may refer to any node at the previous depth, so failure
+  // links must be constructed breadth-first.
+  std::queue<i32> queue;
+  for (i32 child = nodes[0].first_child; child != -1;
+       child = nodes[child].next_sibling) {
+    nodes[child].suffix_link = 0;
+    queue.push(child);
+  }
+
+  while (!queue.empty()) {
+    i32 idx = queue.front();
+    queue.pop();
+
+    for (i32 child = nodes[idx].first_child; child != -1;
+         child = nodes[child].next_sibling) {
+      i32 suffix = nodes[idx].suffix_link;
+      while (suffix != 0 && find_child(suffix, nodes[child].ch) == -1)
+        suffix = nodes[suffix].suffix_link;
+
+      if (i32 next = find_child(suffix, nodes[child].ch); next != -1)
+        suffix = next;
+
+      nodes[child].suffix_link = suffix;
+      nodes[child].value = std::max(nodes[child].value, nodes[suffix].value);
+      queue.push(child);
+    }
+  }
+}
+
+} // namespace mold

lib/arena.cc

@@ -0,0 +1,77 @@
+#include "lib.h"
+
+#ifdef _WIN32
+# include <windows.h>
+#endif
+
+namespace mold {
+
+[[noreturn]]
+static void memory_error(std::string_view action, u64 size) {
+  std::cerr << "mold: cannot " << action << ' ' << size
+            << " bytes of virtual memory\n";
+  std::exit(1);
+}
+
+ArenaResource::ArenaResource() {
+#ifdef _WIN32
+  data = (u8 *)VirtualAlloc(nullptr, (SIZE_T)SIZE, MEM_RESERVE, PAGE_NOACCESS);
+  if (!data)
+    memory_error("reserve", SIZE);
+#else
+  int flags = MAP_ANONYMOUS | MAP_PRIVATE;
+# ifdef MAP_NORESERVE
+  // The arena is much larger than most links need. Do not reserve swap for
+  // pages that may never be touched.
+  flags |= MAP_NORESERVE;
+# endif
+  data = (u8 *)mmap(nullptr, SIZE, PROT_READ | PROT_WRITE, flags, -1, 0);
+  if (data == MAP_FAILED)
+    memory_error("reserve", SIZE);
+#endif
+
+#ifdef MADV_HUGEPAGE
+  // Large links fill the beginning of the arena densely. Transparent huge
+  // pages reduce address-translation overhead without populating unused pages.
+  madvise(data, SIZE, MADV_HUGEPAGE);
+#endif
+}
+
+ArenaResource::~ArenaResource() {
+#ifdef _WIN32
+  VirtualFree(data, 0, MEM_RELEASE);
+#else
+  munmap(data, SIZE);
+#endif
+}
+
+void *ArenaResource::allocate(u64 size, u64 alignment) {
+  assert(std::has_single_bit(alignment));
+
+  // ArenaPtr and base-relative indices both encode offsets in four-byte units.
+  alignment = std::max<u64>(alignment, 4);
+  u64 old = offset.load(std::memory_order_relaxed);
+
+  for (;;) {
+    u64 padding = -old & (alignment - 1);
+    if (old > SIZE || padding > SIZE - old || size > SIZE - old - padding) {
+      std::cerr << "mold: cannot allocate more than " << SIZE
+                << " bytes for linker data structures on this host\n";
+      std::exit(1);
+    }
+
+    u64 begin = old + padding;
+    u64 end = begin + size;
+    if (offset.compare_exchange_weak(old, end, std::memory_order_relaxed)) {
+#ifdef _WIN32
+      // VirtualAlloc reserves and commits address space separately.
+      if (size &&
+          !VirtualAlloc(data + begin, (SIZE_T)size, MEM_COMMIT, PAGE_READWRITE))
+        memory_error("commit", size);
+#endif
+      return data + begin;
+    }
+  }
+}
+
+} // namespace mold

lib/atomics.h

@@ -0,0 +1,50 @@
+// This is the same as std::atomic except that the default memory
+// order is relaxed instead of sequential consistency.
+
+#pragma once
+
+#include <atomic>
+
+namespace mold {
+
+template <typename T>
+struct Atomic : std::atomic<T> {
+  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
+
+  using std::atomic<T>::atomic;
+
+  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
+
+  Atomic<T> &operator=(const Atomic<T> &other) {
+    store(other.load());
+    return *this;
+  }
+
+  void operator=(T val) { store(val); }
+  operator T() const { return load(); }
+
+  void store(T val, std::memory_order order = relaxed) {
+    std::atomic<T>::store(val, order);
+  }
+
+  T load(std::memory_order order = relaxed) const {
+    return std::atomic<T>::load(order);
+  }
+
+  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
+  T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
+  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
+  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
+  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
+  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
+
+  bool test_and_set() {
+    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
+    // while an atomic RMW can easily take hundreds on x86. We note that it's
+    // common that another thread beat us in marking, so doing an optimistic
+    // early test tends to improve performance in the ~20% ballpark.
+    return load() || exchange(true);
+  }
+};
+
+} // namespace mold

lib/common.h

@@ -1,979 +0,0 @@
-#pragma once
-
-#include "config.h"
-#include "integers.h"
-
-#include <array>
-#include <atomic>
-#include <bit>
-#include <bitset>
-#include <cassert>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <fcntl.h>
-#include <filesystem>
-#include <iostream>
-#include <mutex>
-#include <optional>
-#include <span>
-#include <sstream>
-#include <string>
-#include <string_view>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <tbb/concurrent_vector.h>
-#include <tbb/enumerable_thread_specific.h>
-#include <tbb/parallel_for.h>
-#include <vector>
-
-#ifdef _WIN32
-# include <io.h>
-#else
-# include <sys/mman.h>
-# include <unistd.h>
-#endif
-
-#define XXH_INLINE_ALL 1
-#include "../third-party/xxhash/xxhash.h"
-
-#ifdef NDEBUG
-# define unreachable() __builtin_unreachable()
-#else
-# define unreachable() assert(0 && "unreachable")
-#endif
-
-inline uint64_t hash_string(std::string_view str) {
-  return XXH3_64bits(str.data(), str.size());
-}
-
-class HashCmp {
-public:
-  static size_t hash(const std::string_view &k) {
-    return hash_string(k);
-  }
-
-  static bool equal(const std::string_view &k1, const std::string_view &k2) {
-    return k1 == k2;
-  }
-};
-
-namespace mold {
-
-namespace ranges = std::ranges;
-using namespace std::literals::string_literals;
-using namespace std::literals::string_view_literals;
-
-inline char *output_tmpfile;
-
-inline u8 *output_buffer_start = nullptr;
-inline u8 *output_buffer_end = nullptr;
-
-extern std::string mold_git_hash;
-
-std::string errno_string();
-void cleanup();
-void install_signal_handler();
-
-inline u64 combine_hash(u64 a, u64 b) {
-  return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
-}
-
-//
-// Error output
-//
-
-// Some C++ stdlibs don't support std::osyncstream even though
-// it's is in the C++20 standard. So we implement it ourselves.
-class SyncStream {
-public:
-  SyncStream(std::ostream &out) : out(out) {}
-
-  ~SyncStream() {
-    emit();
-  }
-
-  template <typename T> SyncStream &operator<<(T &&val) {
-    ss << std::forward<T>(val);
-    return *this;
-  }
-
-  void emit() {
-    if (emitted)
-      return;
-
-    std::scoped_lock lock(mu);
-    out << ss.str() << '\n';
-    emitted = true;
-  }
-
-private:
-  std::ostream &out;
-  std::stringstream ss;
-  bool emitted = false;
-  static inline std::mutex mu;
-};
-
-template <typename Context>
-class Out {
-public:
-  Out(Context &ctx) {}
-
-  template <typename T> Out &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cout};
-};
-
-static std::string_view fatal_mono = "mold: fatal: ";
-static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
-static std::string_view error_mono = "mold: error: ";
-static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
-static std::string_view warning_mono = "mold: warning: ";
-static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
-
-template <typename Context>
-class Fatal {
-public:
-  Fatal(Context &ctx) {
-    out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
-  }
-
-  [[noreturn]] ~Fatal() {
-    out.emit();
-    cleanup();
-    _exit(1);
-  }
-
-  template <typename T> Fatal &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Error {
-public:
-  Error(Context &ctx) {
-    if (ctx.arg.noinhibit_exec) {
-      out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    } else {
-      out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    }
-  }
-
-  template <typename T> Error &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Warn {
-public:
-  Warn(Context &ctx) {
-    if (ctx.arg.suppress_warnings)
-      return;
-
-    out.emplace(std::cerr);
-
-    if (ctx.arg.fatal_warnings) {
-      *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    } else {
-      *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    }
-  }
-
-  template <typename T> Warn &operator<<(T &&val) {
-    if (out)
-      *out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  std::optional<SyncStream> out;
-};
-
-//
-// Atomics
-//
-
-// This is the same as std::atomic except that the default memory
-// order is relaxed instead of sequential consistency.
-template <typename T>
-struct Atomic : std::atomic<T> {
-  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
-
-  using std::atomic<T>::atomic;
-
-  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
-
-  Atomic<T> &operator=(const Atomic<T> &other) {
-    store(other.load());
-    return *this;
-  }
-
-  void operator=(T val) { store(val); }
-  operator T() const { return load(); }
-
-  void store(T val, std::memory_order order = relaxed) {
-    std::atomic<T>::store(val, order);
-  }
-
-  T load(std::memory_order order = relaxed) const {
-    return std::atomic<T>::load(order);
-  }
-
-  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
-  T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
-  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
-  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
-  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
-  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
-
-  bool test_and_set() {
-    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
-    // while an atomic RMW can easily take hundreds on x86. We note that it's
-    // common that another thread beat us in marking, so doing an optimistic
-    // early test tends to improve performance in the ~20% ballpark.
-    return load() || exchange(true);
-  }
-};
-
-//
-// mimalloc.cc
-//
-
-void set_mimalloc_options();
-
-//
-// perf.cc
-//
-
-// Counter is used to collect statistics numbers.
-class Counter {
-public:
-  Counter(std::string_view name, i64 value = 0) : name(name), values(value) {
-    static std::mutex mu;
-    std::scoped_lock lock(mu);
-    instances.push_back(this);
-  }
-
-  Counter &operator++(int) {
-    if (enabled) [[unlikely]]
-      values.local()++;
-    return *this;
-  }
-
-  Counter &operator+=(int delta) {
-    if (enabled) [[unlikely]]
-      values.local() += delta;
-    return *this;
-  }
-
-  static void print();
-
-  static inline bool enabled = false;
-
-private:
-  i64 get_value();
-
-  std::string_view name;
-  tbb::enumerable_thread_specific<i64> values;
-
-  static inline std::vector<Counter *> instances;
-};
-
-// Timer and TimeRecord records elapsed time (wall clock time)
-// used by each pass of the linker.
-struct TimerRecord {
-  TimerRecord(std::string name, TimerRecord *parent = nullptr);
-  void stop();
-
-  std::string name;
-  TimerRecord *parent;
-  tbb::concurrent_vector<TimerRecord *> children;
-  i64 start;
-  i64 end;
-  i64 user;
-  i64 sys;
-  bool stopped = false;
-};
-
-void
-print_timer_records(tbb::concurrent_vector<std::unique_ptr<TimerRecord>> &);
-
-template <typename Context>
-class Timer {
-public:
-  Timer(Context &ctx, std::string name, Timer *parent = nullptr) {
-    record = new TimerRecord(name, parent ? parent->record : nullptr);
-    ctx.timer_records.emplace_back(record);
-  }
-
-  Timer(const Timer &) = delete;
-
-  ~Timer() {
-    record->stop();
-  }
-
-  void stop() {
-    record->stop();
-  }
-
-private:
-  TimerRecord *record;
-};
-
-//
-// Utility functions
-//
-
-// Some C++ libraries haven't implemented std::has_single_bit yet.
-inline bool has_single_bit(u64 val) {
-  return std::popcount(val) == 1;
-}
-
-// Some C++ libraries haven't implemented std::bit_ceil yet.
-inline u64 bit_ceil(u64 val) {
-  if (has_single_bit(val))
-    return val;
-  return 1LL << (64 - std::countl_zero(val));
-}
-
-inline u64 align_to(u64 val, u64 align) {
-  if (align == 0)
-    return val;
-  assert(has_single_bit(align));
-  return (val + align - 1) & ~(align - 1);
-}
-
-inline u64 align_down(u64 val, u64 align) {
-  assert(has_single_bit(align));
-  return val & ~(align - 1);
-}
-
-inline u64 bit(u64 val, i64 pos) {
-  return (val >> pos) & 1;
-};
-
-// Returns [hi:lo] bits of val.
-inline u64 bits(u64 val, u64 hi, u64 lo) {
-  return (val >> lo) & ((1LL << (hi - lo + 1)) - 1);
-}
-
-// Cast val to a signed N bit integer.
-// For example, sign_extend(x, 32) == (i32)x for any integer x.
-inline i64 sign_extend(u64 val, i64 n) {
-  return (i64)(val << (64 - n)) >> (64 - n);
-}
-
-inline bool is_int(u64 val, i64 n) {
-  return sign_extend(val, n) == val;
-}
-
-template <typename T, typename Compare = std::less<T>>
-void update_minimum(std::atomic<T> &atomic, u64 new_val, Compare cmp = {}) {
-  T old_val = atomic.load(std::memory_order_relaxed);
-  while (cmp(new_val, old_val) &&
-         !atomic.compare_exchange_weak(old_val, new_val,
-                                       std::memory_order_relaxed));
-}
-
-template <typename T, typename Compare = std::less<T>>
-void update_maximum(std::atomic<T> &atomic, u64 new_val, Compare cmp = {}) {
-  T old_val = atomic.load(std::memory_order_relaxed);
-  while (cmp(old_val, new_val) &&
-         !atomic.compare_exchange_weak(old_val, new_val,
-                                       std::memory_order_relaxed));
-}
-
-template <typename T>
-inline void append(std::vector<T> &x, const auto &y) {
-  x.insert(x.end(), y.begin(), y.end());
-}
-
-template <typename T>
-inline std::vector<T> flatten(std::vector<std::vector<T>> &vec) {
-  i64 size = 0;
-  for (std::vector<T> &v : vec)
-    size += v.size();
-
-  std::vector<T> ret;
-  ret.reserve(size);
-  for (std::vector<T> &v : vec)
-    append(ret, v);
-  return ret;
-}
-
-template <typename T>
-inline void remove_duplicates(std::vector<T> &vec) {
-  vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
-}
-
-inline i64 write_string(void *buf, std::string_view str) {
-  memcpy(buf, str.data(), str.size());
-  *((u8 *)buf + str.size()) = '\0';
-  return str.size() + 1;
-}
-
-template <typename T>
-inline void write_vector(void *buf, const std::vector<T> &vec) {
-  if (!vec.empty())
-    memcpy(buf, vec.data(), vec.size() * sizeof(T));
-}
-
-inline void encode_uleb(std::vector<u8> &vec, u64 val) {
-  do {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-    vec.push_back(val ? (byte | 0x80) : byte);
-  } while (val);
-}
-
-inline void encode_sleb(std::vector<u8> &vec, i64 val) {
-  for (;;) {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-
-    bool neg = (byte & 0x40);
-    if ((val == 0 && !neg) || (val == -1 && neg)) {
-      vec.push_back(byte);
-      break;
-    }
-    vec.push_back(byte | 0x80);
-  }
-}
-
-inline i64 write_uleb(u8 *buf, u64 val) {
-  i64 i = 0;
-  do {
-    u8 byte = val & 0x7f;
-    val >>= 7;
-    buf[i++] = val ? (byte | 0x80) : byte;
-  } while (val);
-  return i;
-}
-
-inline u64 read_uleb(u8 **buf) {
-  u64 val = 0;
-  u8 shift = 0;
-  u8 byte;
-  do {
-    byte = *(*buf)++;
-    val |= (byte & 0x7f) << shift;
-    shift += 7;
-  } while (byte & 0x80);
-  return val;
-}
-
-inline u64 read_uleb(u8 *buf) {
-  u8 *tmp = buf;
-  return read_uleb(&tmp);
-}
-
-inline i64 read_sleb(u8 **buf) {
-  u64 val = 0;
-  u8 shift = 0;
-  u8 byte;
-  do {
-    byte = *(*buf)++;
-    val |= (byte & 0x7f) << shift;
-    shift += 7;
-  } while (byte & 0x80);
-  return sign_extend(val, shift);
-}
-
-inline i64 read_sleb(u8 *buf) {
-  u8 *tmp = buf;
-  return read_sleb(&tmp);
-}
-
-inline u64 read_uleb(std::string_view *str) {
-  u8 *start = (u8 *)str->data();
-  u8 *ptr = start;
-  u64 val = read_uleb(&ptr);
-  *str = str->substr(ptr - start);
-  return val;
-}
-
-inline u64 read_uleb(std::string_view str) {
-  std::string_view tmp = str;
-  return read_uleb(&tmp);
-}
-
-inline i64 uleb_size(u64 val) {
-  for (int i = 1; i < 9; i++)
-    if (val < (1LL << (7 * i)))
-      return i;
-  return 9;
-}
-
-inline void overwrite_uleb(u8 *loc, u64 val) {
-  while (*loc & 0b1000'0000) {
-    *loc++ = 0b1000'0000 | (val & 0b0111'1111);
-    val >>= 7;
-  }
-  *loc = val & 0b0111'1111;
-}
-
-static inline void pause() {
-#if defined(__x86_64__)
-  asm volatile("pause");
-#elif defined(__aarch64__)
-  asm volatile("yield");
-#elif defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_8A__)
-  asm volatile("yield");
-#endif
-}
-
-//
-// Concurrent Map
-//
-
-// This is an implementation of a fast concurrent hash map. Unlike
-// ordinary hash tables, this impl just aborts if it becomes full.
-// So you need to give a correct estimation of the final size before
-// using it. We use this hash map to uniquify pieces of data in
-// mergeable sections.
-//
-// We've implemented this ourselves because the performance of
-// conrurent hash map is critical for our linker.
-template <typename T>
-class ConcurrentMap {
-public:
-  ConcurrentMap() = default;
-
-  ConcurrentMap(i64 nbuckets) {
-    resize(nbuckets);
-  }
-
-  ~ConcurrentMap() {
-    if (entries) {
-#ifdef _WIN32
-      _aligned_free(entries);
-#else
-      munmap(entries, sizeof(Entry) * nbuckets);
-#endif
-    }
-  }
-
-  // In order to avoid unnecessary cache-line false sharing, we want
-  // to make this object to be aligned to a reasonably large
-  // power-of-two address.
-  struct alignas(32) Entry {
-    Atomic<const char *> key;
-    u32 keylen;
-    T value;
-  };
-
-  void resize(i64 nbuckets) {
-    assert(!entries);
-    this->nbuckets = std::max<i64>(MIN_NBUCKETS, bit_ceil(nbuckets));
-    i64 bufsize = sizeof(Entry) * this->nbuckets;
-
-    // Allocate a zero-initialized buffer. We use mmap() if available
-    // because it's faster than malloc() and memset().
-#ifdef _WIN32
-    entries = (Entry *)_aligned_malloc(bufsize, alignof(Entry));
-    memset((void *)entries, 0, bufsize);
-#else
-    entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
-                            MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-#endif
-  }
-
-  std::pair<T *, bool> insert(std::string_view key, u64 hash, const T &val) {
-    assert(has_single_bit(nbuckets));
-
-    u64 begin = hash & (nbuckets - 1);
-    u64 mask = nbuckets / NUM_SHARDS - 1;
-
-    for (i64 i = 0; i < MAX_RETRY; i++) {
-      u64 idx = (begin & ~mask) | ((begin + i) & mask);
-      Entry &ent = entries[idx];
-
-      // It seems avoiding compare-and-swap is faster overall at least
-      // on my Zen4 machine, so do it.
-      if (const char *ptr = ent.key.load(std::memory_order_acquire);
-          ptr != nullptr && ptr != (char *)-1) {
-        if (key == std::string_view(ptr, ent.keylen))
-          return {&ent.value, false};
-        continue;
-      }
-
-      // Otherwise, use CAS to atomically claim the ownership of the slot.
-      const char *ptr = nullptr;
-      bool claimed = ent.key.compare_exchange_strong(ptr, (char *)-1,
-                                                     std::memory_order_acquire);
-
-      // If we successfully claimed the ownership of the slot,
-      // copy values to it.
-      if (claimed) {
-        new (&ent.value) T(val);
-        ent.keylen = key.size();
-        ent.key.store(key.data(), std::memory_order_release);
-        return {&ent.value, true};
-      }
-
-      // If someone is copying values to the slot, do busy wait.
-      while (ptr == (char *)-1) {
-        pause();
-        ptr = ent.key.load(std::memory_order_acquire);
-      }
-
-      // If the same key is already present, this is the slot we are
-      // looking for.
-      if (key == std::string_view(ptr, ent.keylen))
-        return {&ent.value, false};
-    }
-
-    std::cerr << "ConcurrentMap is full\n";
-    abort();
-  }
-
-  i64 get_idx(T *value) const {
-    uintptr_t addr = (uintptr_t)value - (uintptr_t)value % sizeof(Entry);
-    return (Entry *)addr - entries;
-  }
-
-  // Return a list of map entries sorted in a deterministic order.
-  std::vector<Entry *> get_sorted_entries(i64 shard_idx) {
-    if (nbuckets == 0)
-      return {};
-
-    i64 shard_size = nbuckets / NUM_SHARDS;
-    i64 begin = shard_idx * shard_size;
-    i64 end = begin + shard_size;
-
-    i64 sz = 0;
-    for (i64 i = begin; i < end; i++)
-      if (entries[i].key)
-        sz++;
-
-    std::vector<Entry *> vec;
-    vec.reserve(sz);
-
-    // Since the shard is circular, we need to handle the last entries
-    // as if they were next to the first entries.
-    while (entries[end - 1].key)
-      vec.push_back(entries + --end);
-
-    // Find entries contiguous in the buckets and sort them.
-    i64 last = 0;
-    for (i64 i = begin; i < end;) {
-      while (i < end && entries[i].key)
-        vec.push_back(entries + i++);
-
-      std::sort(vec.begin() + last, vec.end(), [](Entry *a, Entry *b) {
-        if (a->keylen != b->keylen)
-          return a->keylen < b->keylen;
-        return memcmp(a->key, b->key, a->keylen) < 0;
-      });
-
-      last = vec.size();
-
-      while (i < end && !entries[i].key)
-        i++;
-    }
-    return vec;
-  }
-
-  std::vector<Entry *> get_sorted_entries_all() {
-    std::vector<std::vector<Entry *>> vec(NUM_SHARDS);
-    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
-      vec[i] = get_sorted_entries(i);
-    });
-    return flatten(vec);
-  }
-
-  static constexpr i64 MIN_NBUCKETS = 4096;
-  static constexpr i64 NUM_SHARDS = 16;
-  static constexpr i64 MAX_RETRY = 256;
-
-  Entry *entries = nullptr;
-  u64 nbuckets = 0;
-};
-
-//
-// random.cc
-//
-
-void get_random_bytes(u8 *buf, i64 size);
-
-//
-// hyperloglog.cc
-//
-
-class HyperLogLog {
-public:
-  void insert(u64 hash) {
-    update_maximum(buckets[hash & (NBUCKETS - 1)], std::countl_zero(hash) + 1);
-  }
-
-  i64 get_cardinality() const;
-
-  void merge(const HyperLogLog &other) {
-    for (i64 i = 0; i < NBUCKETS; i++)
-      update_maximum(buckets[i], other.buckets[i]);
-  }
-
-private:
-  static constexpr i64 NBUCKETS = 2048;
-  static constexpr double ALPHA = 0.79402;
-
-  Atomic<u8> buckets[NBUCKETS];
-};
-
-//
-// glob.cc
-//
-
-class Glob {
-  typedef enum { STRING, STAR, QUESTION, BRACKET } Kind;
-
-  struct Element {
-    Element(Kind k) : kind(k) {}
-    Kind kind;
-    std::string str;
-    std::bitset<256> bitset;
-  };
-
-public:
-  static std::optional<Glob> compile(std::string_view pat);
-  bool match(std::string_view str);
-
-private:
-  Glob(std::vector<Element> &&vec) : elements(vec) {}
-  static bool do_match(std::string_view str, std::span<Element> elements);
-
-  std::vector<Element> elements;
-};
-
-//
-// multi-glob.cc
-//
-
-class MultiGlob {
-public:
-  bool add(std::string_view pat, i64 val);
-  bool empty() const { return strings.empty(); }
-  std::optional<i64> find(std::string_view str);
-
-private:
-  struct TrieNode {
-    i64 value = -1;
-    TrieNode *suffix_link = nullptr;
-    std::unique_ptr<TrieNode> children[256];
-  };
-
-  void compile();
-  void fix_suffix_links(TrieNode &node);
-  void fix_values();
-  i64 find_aho_corasick(std::string_view str);
-
-  std::vector<std::string> strings;
-  std::unique_ptr<TrieNode> root;
-  std::vector<std::pair<Glob, i64>> globs;
-  std::once_flag once;
-  bool is_compiled = false;
-  bool prefix_match = false;
-};
-
-//
-// filepath.cc
-//
-
-inline std::filesystem::path path_dirname(std::string_view path) {
-  return std::filesystem::path(path).parent_path();
-}
-
-inline std::string path_filename(std::string_view path) {
-  return std::filesystem::path(path).filename().string();
-}
-
-inline std::string path_clean(std::string_view path) {
-  return std::filesystem::path(path).lexically_normal().string();
-}
-
-std::string get_self_path();
-
-//
-// demangle.cc
-//
-
-std::optional<std::string_view> demangle_cpp(std::string_view name);
-std::optional<std::string_view> demangle_rust(std::string_view name);
-
-//
-// jbos.cc
-//
-
-void acquire_global_lock();
-void release_global_lock();
-
-//
-// crc32.cc
-//
-
-u32 compute_crc32(u32 crc, u8 *buf, i64 len);
-std::vector<u8> crc32_solve(u32 current, u32 desired);
-
-//
-// compress.cc
-//
-
-class Compressor {
-public:
-  virtual void write_to(u8 *buf) = 0;
-  virtual ~Compressor() {}
-  i64 compressed_size = 0;
-};
-
-class ZlibCompressor : public Compressor {
-public:
-  ZlibCompressor(u8 *buf, i64 size);
-  void write_to(u8 *buf) override;
-
-private:
-  std::vector<std::vector<u8>> shards;
-  u64 checksum = 0;
-};
-
-class ZstdCompressor : public Compressor {
-public:
-  ZstdCompressor(u8 *buf, i64 size);
-  void write_to(u8 *buf) override;
-
-private:
-  std::vector<std::vector<u8>> shards;
-};
-
-//
-// tar.cc
-//
-
-// TarFile is a class to create a tar file.
-//
-// If you pass `--repro` to mold, mold collects all input files and
-// put them into `<output-file-path>.repro.tar`, so that it is easy to
-// run the same command with the same command line arguments.
-class TarWriter {
-public:
-  static std::unique_ptr<TarWriter>
-  open(std::string output_path, std::string basedir);
-
-  ~TarWriter();
-  void append(std::string path, std::string_view data);
-
-private:
-  TarWriter(FILE *out, std::string basedir) : out(out), basedir(basedir) {}
-
-  FILE *out = nullptr;
-  std::string basedir;
-};
-
-//
-// Memory-mapped file
-//
-
-// MappedFile represents an mmap'ed input file.
-// mold uses mmap-IO only.
-class MappedFile {
-public:
-  ~MappedFile() { unmap(); }
-  void unmap();
-  void close_fd();
-  void reopen_fd(const std::string &path);
-
-  template <typename Context>
-  MappedFile *slice(Context &ctx, std::string name, u64 start, u64 size) {
-    MappedFile *mf = new MappedFile;
-    mf->name = name;
-    mf->data = data + start;
-    mf->size = size;
-    mf->parent = this;
-
-    ctx.mf_pool.emplace_back(mf);
-    return mf;
-  }
-
-  std::string_view get_contents() {
-    return std::string_view((char *)data, size);
-  }
-
-  i64 get_offset() const {
-    return parent ? (data - parent->data + parent->get_offset()) : 0;
-  }
-
-  // Returns a string that uniquely identify a file that is possibly
-  // in an archive.
-  std::string get_identifier() const {
-    if (parent) {
-      // We use the file offset within an archive as an identifier
-      // because archive members may have the same name.
-      return parent->name + ":" + std::to_string(get_offset());
-    }
-
-    if (thin_parent) {
-      // If this is a thin archive member, the filename part is
-      // guaranteed to be unique.
-      return thin_parent->name + ":" + name;
-    }
-    return name;
-  }
-
-  std::string name;
-  u8 *data = nullptr;
-  i64 size = 0;
-  bool given_fullpath = true;
-  MappedFile *parent = nullptr;
-  MappedFile *thin_parent = nullptr;
-
-  // For --dependency-file
-  bool is_dependency = true;
-
-#ifdef _WIN32
-  HANDLE fd = INVALID_HANDLE_VALUE;
-#else
-  int fd = -1;
-#endif
-};
-
-MappedFile *open_file_impl(const std::string &path, std::string &error);
-
-template <typename Context>
-MappedFile *open_file(Context &ctx, std::string path) {
-  if (path.starts_with('/') && !ctx.arg.chroot.empty())
-    path = ctx.arg.chroot + "/" + path_clean(path);
-
-  std::string error;
-  MappedFile *mf = open_file_impl(path, error);
-  if (!error.empty())
-    Fatal(ctx) << error;
-
-  if (mf)
-    ctx.mf_pool.emplace_back(mf);
-  return mf;
-}
-
-template <typename Context>
-MappedFile *must_open_file(Context &ctx, std::string path) {
-  MappedFile *mf = open_file(ctx, path);
-  if (!mf)
-    Fatal(ctx) << "cannot open " << path << ": " << errno_string();
-  return mf;
-}
-
-} // namespace mold

lib/compress.cc

@@ -14,7 +14,7 @@
 // a little bit. However, if a shard size is large enough, that loss
 // is negligible in practice.
 
-#include "common.h"
+#include "lib.h"
 
 #include <tbb/parallel_for_each.h>
 #include <zlib.h>
@@ -30,42 +30,41 @@ namespace mold {
 
 static constexpr i64 SHARD_SIZE = 1024 * 1024;
 
-static std::vector<std::string_view> split(std::string_view input) {
-  std::vector<std::string_view> shards;
-
-  while (input.size() >= SHARD_SIZE) {
-    shards.push_back(input.substr(0, SHARD_SIZE));
-    input = input.substr(SHARD_SIZE);
-  }
-  if (!input.empty())
-    shards.push_back(input);
-  return shards;
+Compressor::~Compressor() {
+  for (std::span<u8> shard : shards)
+    delete[] shard.data();
 }
 
-static std::vector<u8> zlib_compress(std::string_view input) {
-  // Initialize zlib stream. Since debug info is generally compressed
-  // pretty well with lower compression levels, we chose compression
-  // level 1.
-  z_stream strm;
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
+static std::vector<std::span<u8>> split(std::span<u8> input) {
+  std::vector<std::span<u8>> vec;
+  while (!input.empty()) {
+    i64 sz = std::min<i64>(SHARD_SIZE, input.size());
+    vec.push_back(input.subspan(0, sz));
+    input = input.subspan(sz);
+  }
+  return vec;
+}
 
-  CHECK(deflateInit2(&strm, 1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));
+static std::span<u8> zlib_compress(std::span<u8> input, int level) {
+  // Initialize zlib stream. Since debug info is generally compressed
+  // pretty well with lower compression levels, the default level is 1.
+  z_stream strm = {};
+  CHECK(deflateInit2(&strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));
 
   // Set an input buffer
   strm.avail_in = input.size();
-  strm.next_in = (u8 *)input.data();
+  strm.next_in = input.data();
 
   // Set an output buffer. deflateBound() returns an upper bound
   // on the compression size. +16 for Z_SYNC_FLUSH.
-  std::vector<u8> buf(deflateBound(&strm, strm.avail_in) + 16);
+  i64 bufsize = deflateBound(&strm, strm.avail_in) + 16;
+  u8 *buf = new u8[bufsize];
 
   // Compress data. It writes all compressed bytes except the last
   // partial byte, so up to 7 bits can be held to be written to the
   // buffer.
-  strm.avail_out = buf.size();
-  strm.next_out = buf.data();
+  strm.avail_out = bufsize;
+  strm.next_out = buf;
   CHECK(deflate(&strm, Z_BLOCK));
 
   // This is a workaround for libbacktrace before 2022-04-06.
@@ -84,29 +83,26 @@ static std::vector<u8> zlib_compress(std::string_view input) {
   // that the bit position is 7 in the next byte.
   //
   // https://github.com/ianlancetaylor/libbacktrace/pull/87
-  int nbits;
+  int nbits = 0;
   deflatePending(&strm, Z_NULL, &nbits);
   if (nbits == 5)
     CHECK(deflatePrime(&strm, 10, 2));
   CHECK(deflate(&strm, Z_SYNC_FLUSH));
 
-  assert(strm.avail_out > 0);
-  buf.resize(buf.size() - strm.avail_out);
-  buf.shrink_to_fit();
   deflateEnd(&strm);
-  return buf;
+  return {buf, (size_t)(bufsize - strm.avail_out)};
 }
 
-ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
-  std::vector<u64> adlers(inputs.size());
+ZlibCompressor::ZlibCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
+  std::vector<u32> adlers(inputs.size());
   shards.resize(inputs.size());
 
   // Compress each shard
   tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    adlers[i] = adler32(1, (u8 *)inputs[i].data(), inputs[i].size());
-    shards[i] = zlib_compress(inputs[i]);
+    std::span<u8> in = inputs[i];
+    adlers[i] = adler32(1, in.data(), in.size());
+    shards[i] = zlib_compress(in, level);
   });
 
   // Combine checksums
@@ -116,7 +112,7 @@ ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
 
   // Comput the total size
   compressed_size = 8; // the header and the trailer
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
     compressed_size += shard.size();
 }
 
@@ -127,7 +123,7 @@ void ZlibCompressor::write_to(u8 *buf) {
 
   // Copy compressed data
   std::vector<i64> offsets(shards.size());
-  offsets[0] = 2; // +2 for header
+  offsets[0] = 2; // +2 for the header
   for (i64 i = 1; i < shards.size(); i++)
     offsets[i] = offsets[i - 1] + shards[i - 1].size();
 
@@ -139,35 +135,28 @@ void ZlibCompressor::write_to(u8 *buf) {
   u8 *end = buf + compressed_size;
   end[-6] = 3;
   end[-5] = 0;
-
-  // Write a checksum
   *(ub32 *)(end - 4) = checksum;
 }
 
-static std::vector<u8> zstd_compress(std::string_view input) {
-  std::vector<u8> buf(ZSTD_COMPRESSBOUND(input.size()));
-  constexpr int LEVEL = 3; // compression level; must be between 1 to 22
-
-  size_t sz = ZSTD_compress(buf.data(), buf.size(), input.data(), input.size(),
-                            LEVEL);
+static std::span<u8> zstd_compress(std::span<u8> input, int level) {
+  i64 bufsize = ZSTD_COMPRESSBOUND(input.size());
+  u8 *buf = new u8[bufsize];
+  size_t sz = ZSTD_compress(buf, bufsize, input.data(), input.size(), level);
   assert(!ZSTD_isError(sz));
-  buf.resize(sz);
-  buf.shrink_to_fit();
-  return buf;
+  return {buf, sz};
 }
 
-ZstdCompressor::ZstdCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
+ZstdCompressor::ZstdCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
   shards.resize(inputs.size());
 
   // Compress each shard
   tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    shards[i] = zstd_compress(inputs[i]);
+    shards[i] = zstd_compress(inputs[i], level);
   });
 
   compressed_size = 0;
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
     compressed_size += shard.size();
 }
 

lib/config.h.in

@@ -1,7 +1,30 @@
 #define MOLD_VERSION "@mold_VERSION@"
 #define MOLD_LIBDIR "@CMAKE_INSTALL_FULL_LIBDIR@"
+#define MOLD_FIRST_TARGET @MOLD_FIRST_TARGET@
 
-#cmakedefine HAVE_MADVISE 1
-#cmakedefine HAVE_UNAME 1
-#cmakedefine MOLD_USE_MIMALLOC 1
-#cmakedefine MOLD_USE_SYSTEM_MIMALLOC 1
+#cmakedefine01 HAVE_FALLOCATE
+#cmakedefine01 HAVE_MADVISE
+#cmakedefine01 HAVE_UNAME
+#cmakedefine01 MOLD_USE_MIMALLOC
+#cmakedefine01 MOLD_USE_SYSTEM_MIMALLOC
+
+#cmakedefine01 HAVE_TARGET_X86_64
+#cmakedefine01 HAVE_TARGET_I386
+#cmakedefine01 HAVE_TARGET_ARM64LE
+#cmakedefine01 HAVE_TARGET_ARM64BE
+#cmakedefine01 HAVE_TARGET_ARM32LE
+#cmakedefine01 HAVE_TARGET_ARM32BE
+#cmakedefine01 HAVE_TARGET_RV32LE
+#cmakedefine01 HAVE_TARGET_RV32BE
+#cmakedefine01 HAVE_TARGET_RV64LE
+#cmakedefine01 HAVE_TARGET_RV64BE
+#cmakedefine01 HAVE_TARGET_PPC32
+#cmakedefine01 HAVE_TARGET_PPC64V1
+#cmakedefine01 HAVE_TARGET_PPC64V2
+#cmakedefine01 HAVE_TARGET_S390X
+#cmakedefine01 HAVE_TARGET_SPARC64
+#cmakedefine01 HAVE_TARGET_M68K
+#cmakedefine01 HAVE_TARGET_SH4LE
+#cmakedefine01 HAVE_TARGET_SH4BE
+#cmakedefine01 HAVE_TARGET_LOONGARCH64
+#cmakedefine01 HAVE_TARGET_LOONGARCH32

lib/crc32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <tbb/parallel_for_each.h>
 #include <zlib.h>

lib/demangle.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <cstdlib>
 

lib/filepath.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <filesystem>
 #include <sys/stat.h>

lib/gentoo-gpkg-workaround.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Work around a deadlock in portage's binary package extraction.
+#
+# tar_stream_reader.close() joins its stdin-feeder thread and then waits
+# for the external decompressor to exit without draining the
+# decompressor's remaining stdout. tarfile stops reading at the tar
+# end-of-archive marker, so trailing archive padding is left in the
+# pipe. If that residue exceeds the pipe buffer, the decompressor
+# blocks writing forever and close() never returns, freezing the whole
+# emerge. Small pipe buffers make this reachable in practice: the
+# kernel shrinks new pipes to one page once a user exceeds
+# fs.pipe-user-pages-soft, which our 80 parallel containers do.
+#
+# The fix is to drain the decompressor's stdout to EOF before joining
+# and waiting. This script patches the installed gpkg.py at container
+# image build time and fails loudly if the code no longer matches, so
+# a portage upgrade that changes (or fixes) this code is noticed.
+
+import sys
+import portage.gpkg
+
+path = portage.gpkg.__file__
+src = open(path).read()
+
+old = """        if self.proc is not None:
+            self.thread.join()
+"""
+
+new = """        if self.proc is not None:
+            # Drain remaining decompressor output before joining and
+            # waiting; otherwise the decompressor can block forever on
+            # a full stdout pipe. Injected by mold's lib/gentoo-test.sh.
+            try:
+                while self.read_io.read(65536):
+                    pass
+            except Exception:
+                pass
+            self.thread.join()
+"""
+
+if src.count(old) != 1:
+    sys.exit(f"gpkg workaround: expected pattern not found in {path}; "
+             "portage changed, update lib/gentoo-gpkg-workaround.py")
+
+with open(path, "w") as f:
+    f.write(src.replace(old, new))
+
+import py_compile
+py_compile.compile(path, doraise=True)
+print(f"gpkg deadlock workaround applied to {path}")

lib/gentoo-test.sh

@@ -2,10 +2,25 @@
 #
 # This test script takes a Gentoo package name and tries to build it
 # using mold in a Podman environment. We chose Gentoo Linux as a test
-# target, because its source-based package allows us to build programs
+# target, because its source-based packages allow us to build programs
 # locally and run their test suites without any hassle.
 #
-# You can get a complete list of Gentoo packages availalbe for testing
+# We first build a package with mold acting as /usr/bin/ld. Only if
+# that fails, we build the same package again with GNU ld to tell
+# whether the failure is mold's fault. The package's own test suite
+# runs in both cases; FEATURES=test is scoped to just the package
+# under test via package.env, because applying it to the entire
+# dependency graph makes a large fraction of the tree unresolvable.
+# Results are classified into three directories:
+#
+#   success: the package built fine with mold (or, for a control
+#            build, with GNU ld)
+#   failure: the package built with GNU ld but not with mold, which
+#            means it hit a mold bug
+#   broken:  the package didn't build even with GNU ld, so the
+#            failure is not mold's fault
+#
+# You can get a complete list of Gentoo packages available for testing
 # with the following command:
 #
 # podman run --rm mold-gentoo emerge --color n -s '' | \
@@ -20,50 +35,152 @@ fi
 
 set -x
 
-# Create a Podman image
-if ! podman image ls mold-gentoo | grep -q mold-gentoo; then
-  set -e
-  cat <<EOF | podman build -t mold-gentoo -
-FROM docker.io/gentoo/stage3
-RUN emerge-webrsync
-RUN echo 'USE="X ssl elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary -perl"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
-    echo 'FEATURES="\${FEATURE} noclean nostrip ccache -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
-    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
-    FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache xeyes dev-build/cmake dev-vcs/git && \
-    rm -rf /var/tmp/portage
-EOF
-  set +e
+# Test results are keyed by the commit hash embedded in the mold
+# binary, so refuse to run with a binary that doesn't have one.
+git_hash=$(./dist/mold --version | perl -ne '/\(([0-9a-f]{40})\b/ and print $1;')
+
+if [ "$git_hash" = "" ]; then
+  echo "Cannot get a git hash from ./dist/mold --version. Run this script at the mold source root, with dist/mold built from a git checkout."
+  exit 1
 fi
 
-git_hash=$(./dist/mold --version | perl -ne '/\((\w+)/; print $1;')
+cache_dir=/var/cache/mold-gentoo
+
+# Create a Podman image. The lock prevents parallel invocations of
+# this script from racing to build the same image.
+#
+# A few notes on the container configuration:
+#
+# - We accept ~amd64 (testing) packages to test the newest version of
+#   everything. The stage3 image is built for the stable profile, so
+#   we update the whole system to ~amd64 once here; without that,
+#   half-upgraded packages (e.g. perl and its virtuals) cause
+#   unsolvable dependency conflicts in every container.
+#
+# - EMERGE_DEFAULT_OPTS makes emerge automatically apply the USE flag
+#   adjustments it would otherwise just print and give up on. An
+#   ephemeral container has no /etc/portage/package.use accumulated by
+#   an administrator, so without this, thousands of packages are
+#   unbuildable out of the box.
+#
+# - freetype and harfbuzz mutually depend on each other, so we
+#   pre-install them with the documented two-step bootstrap. During
+#   that step, pillow is built without truetype support to break
+#   another dependency cycle (docutils -> pillow -> harfbuzz -> glib
+#   -> docutils). The override is removed afterwards; once the cycle
+#   participants are installed, they satisfy buildtime dependencies
+#   and the cycle can no longer form.
+#
+# - qtbase's test suite requires icu (REQUIRED_USE), which autounmask
+#   doesn't resolve on its own, hence the package.use entry.
+#
+# - gcc doesn't auto-select a newly installed major version, so we
+#   switch to the newest slot explicitly after the world update.
+#
+# - lib/gentoo-gpkg-workaround.py patches a deadlock in portage's
+#   binary package extraction. See that file for details.
+#
+# - static-libs is not in USE because it contradicts REQUIRED_USE
+#   constraints in some packages (e.g. lvm2 forbids static-libs with
+#   udev), and building static archives doesn't exercise the linker
+#   anyway. Where a dependency genuinely needs it, emerge re-enables
+#   it per package via autounmask.
+#
+# - /etc/portage/env/test.conf allows marking a single package for
+#   FEATURES=test through package.env, which is how we scope test
+#   suites to just the package under test.
+#
+# - The image build compiles a couple hundred packages from source,
+#   because the Gentoo binhost only carries stable versions built with
+#   default USE flags, which rarely match our ~amd64/USE combination.
+#   To pay that cost only once, the build publishes what it compiles
+#   to the GNU ld binary package pool and reuses it, so a rebuilt
+#   image only compiles packages whose version has changed since the
+#   last build. The toolchain step still uses the binhost, where our
+#   configuration does match (e.g. clang). Since it verifies binhost
+#   package signatures and our own pool is unsigned, it runs with a
+#   private PKGDIR so the two never mix.
+(
+  flock 9
+  if ! podman image exists mold-gentoo; then
+    cat <<EOF | podman build -v $cache_dir/distfiles:/var/cache/distfiles -v $cache_dir/ccache:/ccache -v $cache_dir/binpkgs-ld:/var/cache/binpkgs -v `pwd`/lib:/mold-lib:ro -t mold-gentoo - || exit 1
+FROM docker.io/gentoo/stage3
+RUN emerge-webrsync
+RUN echo 'USE="X ssl elogind -systemd truetype jpeg jpeg2k tiff zstd -perl udev lcms alsa"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
+    echo 'FEATURES="nostrip ccache binpkg-multi-instance -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
+    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
+    echo 'MAKEOPTS="-j$(nproc) --load-average=100"' >> /etc/portage/make.conf && \
+    echo 'EMERGE_DEFAULT_OPTS="--autounmask-continue=y --autounmask-license=n --autounmask-backtrack=y --backtrack=100 --jobs=16 --load-average=100"' >> /etc/portage/make.conf && \
+    mkdir -p /etc/portage/env /etc/portage/package.use && \
+    echo 'FEATURES="test"' > /etc/portage/env/test.conf && \
+    printf 'media-libs/freetype harfbuzz\ndev-qt/qtbase icu\n' > /etc/portage/package.use/mold && \
+    echo 'dev-python/pillow -truetype' > /etc/portage/package.use/cycle-break && \
+    FEATURES=buildpkg emerge --usepkg --update --deep --newuse @world && \
+    gcc-config \$(gcc-config -l | tail -n 1 | awk '{print \$2}') && env-update && \
+    USE=-harfbuzz FEATURES=buildpkg emerge --oneshot media-libs/freetype && \
+    FEATURES=buildpkg emerge --usepkg --oneshot media-libs/freetype media-libs/harfbuzz && \
+    rm /etc/portage/package.use/cycle-break && \
+    PKGDIR=/var/tmp/binpkgs FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache dev-build/cmake dev-vcs/git && \
+    rm -rf /var/tmp/portage /var/tmp/binpkgs
+RUN python3 /mold-lib/gentoo-gpkg-workaround.py
+EOF
+  fi
+) 9> /tmp/mold-gentoo-image.lock || exit 1
 
 if [ "$package" = dev-libs/concurrencykit ]; then
   echo "Skipping known broken package: $package"
   exit 0
 fi
 
-# Build a given package in Podman
+# Build a given package in Podman.
+#
+# Build artifacts are shared between containers through host
+# directories bind-mounted into every container:
+#
+#   /var/cache/mold-gentoo/ccache: compiler cache
+#   /var/cache/mold-gentoo/distfiles: source tarballs, so that each
+#     distfile is downloaded only once across all containers and runs
+#   /var/cache/mold-gentoo/binpkgs-{mold,ld}: binary packages, so that each
+#     dependency is built only once instead of once per container that
+#     needs it
+#
+# Every emerge runs with FEATURES=buildpkg to publish what it built to
+# the binary package pool, and dependencies are installed with
+# --usepkg from the pool when an exact version and USE match exists.
+# The package under test itself is always built from source. In the
+# mold run, dependencies are still linked with mold and their binaries
+# are still exercised by the test suites of everything built on top of
+# them; the pool just memoizes that work across containers. The mold
+# and GNU ld runs use separate pools so that a control build can never
+# install mold-linked binaries.
 cmd1='(cd /usr/bin; ln -sf /mold/dist/mold $(realpath ld))'
-cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
-cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
+cmd2="echo '$package test.conf' > /etc/portage/package.env"
+cmd3="FEATURES=buildpkg emerge --usepkg --onlydeps $package"
+cmd4="FEATURES=buildpkg emerge $package"
 filename=`echo "$package" | sed 's!/!_!g'`
-podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 3h"
+podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v $cache_dir/ccache:/ccache -v $cache_dir/distfiles:/var/cache/distfiles"
+run="mold-gentoo timeout -v -k 15s 3h chrt --idle 0 nice -n 19 bash -c"
 dir=gentoo/$git_hash
 
-mkdir -p "$dir"/success "$dir"/failure
+mkdir -p "$dir"/success "$dir"/failure "$dir"/broken
 
-$podman nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
+# Build the package with mold. If this succeeds, we don't need a
+# control build.
+$podman -v $cache_dir/binpkgs-mold:/var/cache/binpkgs $run "$cmd1 && $cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".mold
 if [ $? = 0 ]; then
   mv "$dir"/"$filename".mold "$dir"/success
-else
-  mv "$dir"/"$filename".mold "$dir"/failure
+  exit 0
 fi
 
-$podman nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
+# Build the package again with GNU ld to tell whether the failure is
+# mold's fault.
+$podman -v $cache_dir/binpkgs-ld:/var/cache/binpkgs $run "$cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".ld
 if [ $? = 0 ]; then
+  mv "$dir"/"$filename".mold "$dir"/failure
   mv "$dir"/"$filename".ld "$dir"/success
 else
-  mv "$dir"/"$filename".ld "$dir"/failure
+  mv "$dir"/"$filename".mold "$dir"/broken
+  mv "$dir"/"$filename".ld "$dir"/broken
 fi

lib/glob.cc

@@ -1,15 +1,21 @@
-#include "common.h"
+// This file implements the glob matcher used for symbol name patterns.
+// Exact, prefix and suffix patterns are matched directly, and simple
+// substring patterns are combined into an Aho-Corasick matcher. The
+// remaining patterns are matched with the non-recursive algorithm described
+// at https://research.swtch.com/glob. If there are many such patterns, a
+// bit-parallel NFA matches them together.
 
-#include <cstring>
+#include "lib.h"
 
 namespace mold {
 
-std::optional<Glob> Glob::compile(std::string_view pat) {
-  std::vector<Element> vec;
+std::optional<Glob::Pattern> Glob::Pattern::compile(std::string_view pat,
+                                                    i64 value) {
+  std::vector<Token> tokens;
 
   while (!pat.empty()) {
     u8 c = pat[0];
-    pat = pat.substr(1);
+    pat.remove_prefix(1);
 
     switch (c) {
     case '[': {
@@ -18,27 +24,30 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
       // [abc]: a, b or c
       // [$\]!]: $, ] or !
       // [a-czg-i]: a, b, c, z, g, h, or i
-      // [^a-z]: Any character except lowercase letters
-      vec.emplace_back(BRACKET);
-      std::bitset<256> &bitset = vec.back().bitset;
-
+      // [!a-z]: Any character except lowercase letters
+      //
+      // Both `!` and `^` are accepted as negation markers. `!` is the
+      // POSIX/shell convention used by other linkers. `^` was mold's
+      // original syntax and is kept for backward compatibility.
+      tokens.emplace_back(BRACKET);
+      std::bitset<256> &chars = tokens.back().chars;
       bool negate = false;
-      if (!pat.empty() && pat[0] == '^') {
-        negate = true;
-        pat = pat.substr(1);
-      }
-
       bool closed = false;
 
+      if (!pat.empty() && (pat[0] == '!' || pat[0] == '^')) {
+        negate = true;
+        pat.remove_prefix(1);
+      }
+
       while (!pat.empty()) {
         if (pat[0] == ']') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           closed = true;
           break;
         }
 
         if (pat[0] == '\\') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           if (pat.empty())
             return {};
         }
@@ -46,23 +55,23 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
         if (pat.size() >= 3 && pat[1] == '-') {
           u8 start = pat[0];
           u8 end = pat[2];
-          pat = pat.substr(3);
+          pat.remove_prefix(3);
 
           if (end == '\\') {
             if (pat.empty())
               return {};
             end = pat[0];
-            pat = pat.substr(1);
+            pat.remove_prefix(1);
           }
 
           if (end < start)
             return {};
 
           for (i64 i = start; i <= end; i++)
-            bitset[i] = true;
+            chars[i] = true;
         } else {
-          bitset[(u8)pat[0]] = true;
-          pat = pat.substr(1);
+          chars[(u8)pat[0]] = true;
+          pat.remove_prefix(1);
         }
       }
 
@@ -70,87 +79,279 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
         return {};
 
       if (negate)
-        bitset.flip();
+        chars.flip();
       break;
     }
     case '?':
-      vec.emplace_back(QUESTION);
+      tokens.emplace_back(QUESTION);
       break;
     case '*':
-      vec.emplace_back(STAR);
+      if (tokens.empty() || tokens.back().kind != STAR)
+        tokens.emplace_back(STAR);
       break;
     case '\\':
       if (pat.empty())
         return {};
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.emplace_back(STRING);
-      vec.back().str += pat[0];
-      pat = pat.substr(1);
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += pat[0];
+      pat.remove_prefix(1);
       break;
     default:
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.emplace_back(STRING);
-      vec.back().str += c;
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += c;
       break;
     }
   }
-
-  return {Glob{std::move(vec)}};
+  return Pattern(std::move(tokens), value);
 }
 
-bool Glob::match(std::string_view str) {
-  return do_match(str, elements);
-}
+bool Glob::Pattern::match(std::string_view str) const {
+  if (!tokens.empty() && tokens.back().kind == STRING &&
+      !str.ends_with(tokens.back().str))
+    return false;
 
-bool Glob::do_match(std::string_view str, std::span<Element> elements) {
-  while (!elements.empty()) {
-    Element &e = elements[0];
-    elements = elements.subspan(1);
+  i64 x = 0;
+  i64 y = 0;
+  i64 next_x = -1;
+  i64 next_y = -1;
 
-    switch (e.kind) {
-    case STRING:
-      if (!str.starts_with(e.str))
-        return false;
-      str = str.substr(e.str.size());
-      break;
-    case STAR:
-      if (elements.empty())
-        return true;
+  while (x < str.size() || y < tokens.size()) {
+    if (y < tokens.size()) {
+      const Token &tok = tokens[y];
 
-      // Patterns like "*foo*bar*" should be much more common than more
-      // complex ones like "*foo*[abc]*" or "*foo**?bar*", so we optimize
-      // the former case here.
-      if (elements[0].kind == STRING) {
-        for (;;) {
-          size_t pos = str.find(elements[0].str);
+      switch (tok.kind) {
+      case STRING:
+        if (str.substr(x).starts_with(tok.str)) {
+          x += tok.str.size();
+          y++;
+          continue;
+        }
+        break;
+      case STAR:
+        next_y = y++;
+        next_x = x + 1;
+
+        if (y < tokens.size() && tokens[y].kind == STRING) {
+          const std::string &s = tokens[y].str;
+          i64 pos = str.find(s, x);
           if (pos == str.npos)
             return false;
-          if (do_match(str.substr(pos + elements[0].str.size()),
-                       elements.subspan(1)))
-            return true;
-          str = str.substr(pos + 1);
+          next_x = pos + 1;
+          x = pos + s.size();
+          y++;
         }
+        continue;
+      case QUESTION:
+        if (x < str.size()) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
+      case BRACKET:
+        if (x < str.size() && tok.chars[(u8)str[x]]) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
       }
+    }
 
-      // Other cases are handled here.
-      for (i64 j = 0; j < str.size(); j++)
-        if (do_match(str.substr(j), elements))
-          return true;
-      return false;
-    case QUESTION:
-      if (str.empty())
-        return false;
-      str = str.substr(1);
-      break;
-    case BRACKET:
-      if (str.empty() || !e.bitset[str[0]])
-        return false;
-      str = str.substr(1);
-      break;
+    // Retry the last star after assigning one more input byte to it.
+    if (next_x != -1 && next_x <= str.size()) {
+      x = next_x;
+      y = next_y;
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+void Glob::Nfa::compile(std::span<const Pattern> patterns) {
+  i64 num_states = 0;
+
+  for (const Pattern &pattern : patterns) {
+    num_states++;
+
+    for (const Pattern::Token &tok : pattern.tokens) {
+      if (tok.kind == Pattern::STRING)
+        num_states += tok.str.size();
+      else if (tok.kind != Pattern::STAR)
+        num_states++;
     }
   }
 
-  return str.empty();
+  i64 num_words = (num_states + 63) / 64;
+  initial_states.resize(num_words);
+  star_states.resize(num_words);
+  accept_states.resize(num_words);
+  char_masks.resize(256 * num_words);
+  values.resize(num_states, -1);
+
+  auto set_bit = [](std::vector<u64> &vec, i64 pos) {
+    vec[pos / 64] |= 1ULL << (pos % 64);
+  };
+
+  i64 state = 0;
+
+  for (const Pattern &pattern : patterns) {
+    set_bit(initial_states, state);
+
+    for (const Pattern::Token &tok : pattern.tokens) {
+      switch (tok.kind) {
+      case Pattern::STRING:
+        for (u8 c : tok.str) {
+          state++;
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        }
+        break;
+      case Pattern::STAR:
+        set_bit(star_states, state);
+        break;
+      case Pattern::QUESTION:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      case Pattern::BRACKET:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          if (tok.chars[c])
+            char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      }
+    }
+
+    set_bit(accept_states, state);
+    values[state] = pattern.value;
+    state++;
+  }
+}
+
+i64 Glob::Nfa::match(std::string_view str) const {
+  static thread_local std::vector<u64> states;
+  states.assign(initial_states.begin(), initial_states.end());
+
+  i64 num_words = states.size();
+
+  for (u8 c : str) {
+    const u64 *mask = char_masks.data() + c * num_words;
+    u64 carry = 0;
+
+    for (i64 i = 0; i < num_words; i++) {
+      u64 old = states[i];
+      u64 next = (old << 1) | carry;
+      states[i] = (old & star_states[i]) | (next & mask[i]);
+      carry = old >> 63;
+    }
+  }
+
+  i64 value = -1;
+
+  for (i64 i = 0; i < num_words; i++) {
+    u64 word = states[i] & accept_states[i];
+
+    while (word) {
+      i64 bit = std::countr_zero(word);
+      value = std::max(value, values[i * 64 + bit]);
+      word &= word - 1;
+    }
+  }
+  return value;
+}
+
+static bool is_literal(std::string_view pat) {
+  return pat.find_first_of("*?[\\") == pat.npos;
+}
+
+bool Glob::add(std::string_view pat, i64 val) {
+  assert(val >= 0);
+  assert(!is_compiled);
+  is_empty = false;
+
+  // Match-all, exact, prefix and suffix patterns are handled with
+  // plain string comparisons instead of the matchers below, which
+  // have to scan the entire input string on every query.
+  if (pat == "*") {
+    match_all = std::max(match_all, val);
+    return true;
+  }
+
+  if (is_literal(pat)) {
+    exacts.push_back({std::string(pat), val});
+    return true;
+  }
+
+  if (pat.ends_with('*') && is_literal(pat.substr(0, pat.size() - 1))) {
+    prefixes.push_back({std::string(pat.substr(0, pat.size() - 1)), val});
+    return true;
+  }
+
+  if (pat.starts_with('*') && is_literal(pat.substr(1))) {
+    suffixes.push_back({std::string(pat.substr(1)), val});
+    return true;
+  }
+
+  // If the pattern requires only a single substring search, the
+  // Aho-Corasick algorithm is even faster than our glob matcher.
+  if (aho_corasick.can_handle(pat))
+    return aho_corasick.add(pat, val);
+
+  if (std::optional<Pattern> pattern = Pattern::compile(pat, val)) {
+    patterns.push_back(std::move(*pattern));
+    return true;
+  }
+  return false;
+}
+
+i64 Glob::find(std::string_view str) {
+  std::call_once(once, [&] {
+    // If the same name was added more than once, keep only the entry
+    // with the largest value, as find() returns the largest match.
+    // Sorting by (name, negated value) places that entry first in
+    // each run of duplicates, which is the one unique() keeps.
+    ranges::sort(exacts, {}, [](const LiteralPattern &p) {
+      return std::pair<std::string_view, i64>(p.pat, -p.value);
+    });
+
+    auto dup = ranges::unique(exacts, {}, &LiteralPattern::pat);
+    exacts.erase(dup.begin(), dup.end());
+
+    if (patterns.size() >= 64) {
+      nfa.compile(patterns);
+      patterns.clear();
+    }
+
+    aho_corasick.compile();
+    is_compiled = true;
+  });
+
+  i64 val = match_all;
+
+  auto it = ranges::lower_bound(exacts, str, {}, &LiteralPattern::pat);
+  if (it != exacts.end() && it->pat == str)
+    val = std::max(val, it->value);
+
+  for (const LiteralPattern &p : prefixes)
+    if (val < p.value && str.starts_with(p.pat))
+      val = p.value;
+
+  for (const LiteralPattern &p : suffixes)
+    if (val < p.value && str.ends_with(p.pat))
+      val = p.value;
+
+  if (!nfa.empty())
+    val = std::max(val, nfa.match(str));
+
+  for (const Pattern &p : patterns)
+    if (val < p.value && p.match(str))
+      val = p.value;
+
+  return std::max(val, aho_corasick.find(str));
 }
 
 } // namespace mold

lib/hyperloglog.cc

@@ -4,13 +4,18 @@
 // For more info, read
 // https://engineering.fb.com/2018/12/13/data-infrastructure/hyperloglog
 
-#include "common.h"
+#include "lib.h"
 
 #include <cmath>
 
 namespace mold {
 
 i64 HyperLogLog::get_cardinality() const {
+  u8 buckets[NBUCKETS] = {};
+  for (const Sketch &sketch : sketches)
+    for (i64 i = 0; i < NBUCKETS; i++)
+      buckets[i] = std::max(buckets[i], sketch.buckets[i]);
+
   double z = 0;
   for (i64 val : buckets)
     z += std::ldexp(1.0, -val);

lib/integers.h

@@ -5,24 +5,30 @@
 //
 // 1. mold is always a cross linker and should not depend on what host it
 //    is running on. For example, users should be able to run mold on a
-//    big-endian SPARC machine to create a little-endian RV64 binary.
+//    little-endian x86 machine to create a big-endian s390x binary.
 //
 // 2. Even though data members in all ELF data strucutres are naturally
 //    aligned, they are not guaranteed to be aligned on memory because of
-//    archive files. Archive files (.a files) align each member only to a
+//    archive files. Archive files (.a files) align each file only to a
 //    2 byte boundary, so anything larger than 2 bytes may be misaligned
 //    in an mmap'ed memory. Misaligned access is an undefined behavior in
 //    C/C++, so we shouldn't cast an arbitrary pointer to a uint32_t, for
 //    example, to read a 32 bit value.
 //
-// The data types defined in this file don't depend on host byte order and
-// don't do unaligned access. Note that modern compilers are smart enough
-// to recognize shift and bitwise OR patterns and compile them into a
-// single load instruction.
+// The data types defined in this file are independent of the host byte
+// order and are designed to avoid unaligned access.
+//
+// Note that in C/C++, memcpy is a portable and efficient way to access
+// unaligned data, as it is typically treated as an intrinsic. Compilers
+// can easily optimize memcpy calls in this file into a single load or
+// store instruction.
 
 #pragma once
 
+#include <bit>
 #include <cstdint>
+#include <cstring>
+#include <type_traits>
 
 #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
 # if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
@@ -36,93 +42,70 @@
 
 namespace mold {
 
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-
-typedef int8_t i8;
-typedef int16_t i16;
-typedef int32_t i32;
-typedef int64_t i64;
-
 template <typename T, bool is_le, int size = sizeof(T)>
 class Integer {
 public:
   constexpr Integer() = default;
 
-  constexpr Integer(T x) requires (is_le && size == 2)
-    : buf{(u8)x, (u8)(x >> 8)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 3)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 4)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16), (u8)(x >> 24)} {}
-
-  constexpr Integer(T x) requires (is_le && size == 8)
-    : buf{(u8)x,         (u8)(x >> 8),  (u8)(x >> 16), (u8)(x >> 24),
-          (u8)(x >> 32), (u8)(x >> 40), (u8)(x >> 48), (u8)(x >> 56)} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 2)
-    : buf{(u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 3)
-    : buf{(u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 4)
-    : buf{(u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (!is_le && size == 8)
-    : buf{(u8)(x >> 56), (u8)(x >> 48), (u8)(x >> 40), (u8)(x >> 32),
-          (u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8),  (u8)x} {}
-
-  operator T() const {
-    if constexpr (is_le) {
-      if constexpr (size == 2)
-        return buf[1] << 8 | buf[0];
-      else if constexpr (size == 3)
-        return buf[2] << 16 | buf[1] << 8 | buf[0];
-      else if constexpr (size == 4)
-        return buf[3] << 24 | buf[2] << 16 | buf[1] << 8 | buf[0];
-      else
-        return (u64)buf[7] << 56 | (u64)buf[6] << 48 |
-               (u64)buf[5] << 40 | (u64)buf[4] << 32 |
-               (u64)buf[3] << 24 | (u64)buf[2] << 16 |
-               (u64)buf[1] << 8  | (u64)buf[0];
+  constexpr Integer(T v) {
+    if (std::is_constant_evaluated() || size == 3) {
+      for (int i = 0; i < size; i++)
+        buf[is_le ? i : (size - i - 1)] = v >> (i * 8);
     } else {
-      if constexpr (size == 2)
-        return buf[0] << 8 | buf[1];
-      else if constexpr (size == 3)
-        return buf[0] << 16 | buf[1] << 8 | buf[2];
-      else if constexpr (size == 4)
-        return buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3];
-      else
-        return (u64)buf[0] << 56 | (u64)buf[1] << 48 |
-               (u64)buf[2] << 40 | (u64)buf[3] << 32 |
-               (u64)buf[4] << 24 | (u64)buf[5] << 16 |
-               (u64)buf[6] << 8  | (u64)buf[7];
+      if (!is_native)
+        v = bswap(v);
+      memcpy(buf, &v, size);
     }
   }
 
-  Integer &operator=(T x) {
-    new (this) Integer(x);
-    return *this;
+  operator T() const {
+    if constexpr (size == 3) {
+      if constexpr (is_le)
+        return buf[2] << 16 | buf[1] << 8 | buf[0];
+      return buf[0] << 16 | buf[1] << 8 | buf[2];
+    } else {
+      T v;
+      memcpy(&v, buf, size);
+      return is_native ? v : bswap(v);
+    }
   }
 
+  Integer &operator=(T v)  { new (this) Integer(v); return *this; }
   Integer &operator++()    { return *this = *this + 1; }
-  Integer operator++(int)  { return ++*this - 1; }
+  Integer operator++(int)  { auto x = *this; ++*this; return x; }
   Integer &operator--()    { return *this = *this - 1; }
-  Integer operator--(int)  { return --*this + 1; }
-  Integer &operator+=(T x) { return *this = *this + x; }
-  Integer &operator-=(T x) { return *this = *this - x; }
-  Integer &operator&=(T x) { return *this = *this & x; }
-  Integer &operator|=(T x) { return *this = *this | x; }
+  Integer operator--(int)  { auto x = *this; --*this; return x; }
+  Integer &operator+=(T v) { return *this = *this + v; }
+  Integer &operator-=(T v) { return *this = *this - v; }
+  Integer &operator&=(T v) { return *this = *this & v; }
+  Integer &operator|=(T v) { return *this = *this | v; }
 
 private:
-  u8 buf[size];
+  static constexpr bool is_native =
+    (std::endian::native == (is_le ? std::endian::little : std::endian::big));
+
+  static T bswap(T v) {
+    switch (size) {
+    case 2: return __builtin_bswap16(v);
+    case 4: return __builtin_bswap32(v);
+    case 8: return __builtin_bswap64(v);
+    default: __builtin_unreachable();
+    }
+  }
+
+  uint8_t buf[size];
 };
 
+using i8 = int8_t;
+using i16 = int16_t;
+using i32 = int32_t;
+using i64 = int64_t;
+
+using u8 = uint8_t;
+using u16 = uint16_t;
+using u32 = uint32_t;
+using u64 = uint64_t;
+
 using il16 = Integer<i16, true>;
 using il32 = Integer<i32, true>;
 using il64 = Integer<i64, true>;

lib/mimalloc.cc

@@ -1,24 +0,0 @@
-#include "common.h"
-
-// Including mimalloc-new-delete.h overrides new/delete operators.
-// We need it only when we are using mimalloc as a dynamic library.
-#if MOLD_USE_SYSTEM_MIMALLOC
-# include <mimalloc-new-delete.h>
-#endif
-
-// Silence mimalloc warning messages that users can just ignore.
-#if MOLD_USE_MIMALLOC
-# include <mimalloc.h>
-
-namespace mold {
-void set_mimalloc_options() {
-  mi_option_disable(mi_option_verbose);
-  mi_option_disable(mi_option_show_errors);
-}
-}
-
-#else
-namespace mold {
-void set_mimalloc_options() {}
-}
-#endif

lib/multi-glob.cc

@@ -1,186 +0,0 @@
-// This file implements the Aho-Corasick algorithm to match multiple
-// glob patterns to symbol strings as quickly as possible.
-//
-// Here are some examples of glob patterns:
-//
-//    qt_private_api_tag*
-//    *16QAccessibleCache*
-//    *32QAbstractFileIconProviderPrivate*
-//    *17QPixmapIconEngine*
-//
-// `*` is a wildcard that matches any substring. We sometimes have
-// hundreds of glob patterns and have to match them against millions
-// of symbol strings.
-//
-// Aho-Corasick cannot handle complex patterns such as `*foo*bar*`.
-// We handle such patterns with the Glob class. Glob is relatively
-// slow, but complex patterns are rare in practice, so it should be
-// OK.
-
-#include "common.h"
-
-#include <queue>
-#include <regex>
-
-namespace mold {
-
-std::optional<i64> MultiGlob::find(std::string_view str) {
-  std::call_once(once, [&] { compile(); });
-  i64 val = -1;
-
-  // Match against simple glob patterns
-  if (root)
-    val = find_aho_corasick(str);
-
-  // Match against complex glob patterns
-  for (std::pair<Glob, i64> &glob : globs)
-    if (glob.first.match(str))
-      val = std::max(val, glob.second);
-
-  if (val == -1)
-    return {};
-  return val;
-}
-
-i64 MultiGlob::find_aho_corasick(std::string_view str) {
-  TrieNode *node = root.get();
-  i64 val = -1;
-
-  auto walk = [&](u8 c) {
-    for (;;) {
-      if (node->children[c]) {
-        node = node->children[c].get();
-        val = std::max(val, node->value);
-        return;
-      }
-
-      if (!node->suffix_link)
-        return;
-      node = node->suffix_link;
-    }
-  };
-
-  walk('\0');
-
-  for (u8 c : str) {
-    if (prefix_match && node == root.get())
-      return val;
-    walk(c);
-  }
-
-  walk('\0');
-  return val;
-}
-
-static bool is_simple_pattern(std::string_view pat) {
-  static std::regex re(R"(\*?[^*[?]+\*?)", std::regex_constants::optimize);
-  return std::regex_match(pat.begin(), pat.end(), re);
-}
-
-static std::string handle_stars(std::string_view pat) {
-  std::string str(pat);
-
-  // Convert "foo" -> "\0foo\0", "*foo" -> "foo\0", "foo*" -> "\0foo"
-  // and "*foo*" -> "foo". Aho-Corasick can do only substring matching,
-  // so we use \0 as beginning/end-of-string markers.
-  if (str.starts_with('*') && str.ends_with('*'))
-    return str.substr(1, str.size() - 2);
-  if (str.starts_with('*'))
-    return str.substr(1) + "\0"s;
-  if (str.ends_with('*'))
-    return "\0"s + str.substr(0, str.size() - 1);
-  return "\0"s + str + "\0"s;
-}
-
-bool MultiGlob::add(std::string_view pat, i64 val) {
-  assert(!is_compiled);
-  assert(!pat.empty());
-
-  strings.emplace_back(pat);
-
-  // Complex glob pattern
-  if (!is_simple_pattern(pat)) {
-    if (std::optional<Glob> glob = Glob::compile(pat)) {
-      globs.emplace_back(std::move(*glob), val);
-      return true;
-    }
-    return false;
-  }
-
-  // Simple glob pattern
-  if (!root)
-    root.reset(new TrieNode);
-  TrieNode *node = root.get();
-
-  for (u8 c : handle_stars(pat)) {
-    if (!node->children[c])
-      node->children[c].reset(new TrieNode);
-    node = node->children[c].get();
-  }
-
-  node->value = std::max(node->value, val);
-  return true;
-}
-
-void MultiGlob::compile() {
-  is_compiled = true;
-  if (root) {
-    fix_suffix_links(*root);
-    fix_values();
-
-    // If no pattern starts with '*', set prefix_match to true.
-    // We'll use this flag for optimization.
-    prefix_match = true;
-    for (i64 i = 1; i < 256; i++) {
-      if (root->children[i]) {
-        prefix_match = false;
-        break;
-      }
-    }
-  }
-}
-
-void MultiGlob::fix_suffix_links(TrieNode &node) {
-  for (i64 i = 0; i < 256; i++) {
-    if (!node.children[i])
-      continue;
-
-    TrieNode &child = *node.children[i];
-
-    TrieNode *cur = node.suffix_link;
-    for (;;) {
-      if (!cur) {
-        child.suffix_link = root.get();
-        break;
-      }
-
-      if (cur->children[i]) {
-        child.suffix_link = cur->children[i].get();
-        break;
-      }
-
-      cur = cur->suffix_link;
-    }
-
-    fix_suffix_links(child);
-  }
-}
-
-void MultiGlob::fix_values() {
-  std::queue<TrieNode *> queue;
-  queue.push(root.get());
-
-  do {
-    TrieNode *node = queue.front();
-    queue.pop();
-
-    for (std::unique_ptr<TrieNode> &child : node->children) {
-      if (!child)
-        continue;
-      child->value = std::max(child->value, child->suffix_link->value);
-      queue.push(child.get());
-    }
-  } while (!queue.empty());
-}
-
-} // namespace mold

lib/perf.cc

@@ -1,11 +1,13 @@
-#include "common.h"
+#include "lib.h"
 
 #include <functional>
 #include <iomanip>
 #include <ios>
 #include <tbb/concurrent_vector.h>
 
-#ifndef _WIN32
+#ifdef _WIN32
+#include <windows.h>
+#else
 #include <sys/resource.h>
 #include <sys/time.h>
 #endif

lib/random.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"
 
 #include <random>
 

lib/tar.cc

@@ -1,6 +1,6 @@
 // This file contains functions to create a tar file.
 
-#include "common.h"
+#include "lib.h"
 
 #ifdef _WIN32
 # define ftruncate _chsize_s
@@ -86,6 +86,9 @@ void TarWriter::append(std::string path, std::string_view data) {
   UstarHeader pax = {};
 
   std::string attr = encode_path(basedir, path);
+
+  if (attr.size() >= 0100'000'000'000)
+    unreachable();
   snprintf(pax.size, sizeof(pax.size), "%011zo", attr.size());
   pax.name[0] = '/';
   pax.typeflag[0] = 'x';
@@ -99,6 +102,8 @@ void TarWriter::append(std::string path, std::string_view data) {
   // Write Ustar header
   UstarHeader ustar = {};
   memcpy(ustar.mode, "0000664", 8);
+  if (data.size() >= 0100'000'000'000)
+    unreachable();
   snprintf(ustar.size, sizeof(ustar.size), "%011zo", data.size());
   finalize(ustar);
   fwrite(&ustar, sizeof(ustar), 1, out);

lib/update-git-hash.cmake

@@ -1,27 +1,26 @@
-# Get a git hash value. We do not want to use git command here
-# because we don't want to make git a build-time dependency.
-if(EXISTS "${SOURCE_DIR}/.git/HEAD")
-  file(READ "${SOURCE_DIR}/.git/HEAD" HASH)
-  string(STRIP "${HASH}" HASH)
+# Embed the current git commit hash into the mold executable. We ask git
+# instead of parsing files under .git because the repository format varies
+# (e.g. packed refs, reftable). When building from a source tarball, there's
+# no .git, and the hash is simply omitted.
+if(EXISTS "${SOURCE_DIR}/.git")
+  execute_process(
+    COMMAND git -C "${SOURCE_DIR}" rev-parse HEAD
+    OUTPUT_VARIABLE HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+    RESULT_VARIABLE EXIT_CODE)
 
-  if(HASH MATCHES "^ref: (.*)")
-    set(HEAD "${CMAKE_MATCH_1}")
-    if(EXISTS "${SOURCE_DIR}/.git/${HEAD}")
-      file(READ "${SOURCE_DIR}/.git/${HEAD}" HASH)
-      string(STRIP "${HASH}" HASH)
-    else()
-      file(READ "${SOURCE_DIR}/.git/packed-refs" PACKED_REFS)
-      string(REGEX REPLACE ".*\n([0-9a-f]+) ${HEAD}\n.*" "\\1" HASH "\n${PACKED_REFS}")
-    endif()
+  if(NOT EXIT_CODE EQUAL 0)
+    set(HASH "")
   endif()
 endif()
 
 # Create new file contents and update a given file if necessary.
-set(NEW_CONTENTS "#include <string>
-namespace mold {
-std::string mold_git_hash = \"${HASH}\";
-}
-")
+if("${HASH}" STREQUAL "")
+  set(NEW_CONTENTS "")
+else()
+  set(NEW_CONTENTS "#define MOLD_GIT_HASH \"${HASH}\"\n")
+endif()
 
 if(EXISTS "${OUTPUT_FILE}")
   file(READ "${OUTPUT_FILE}" OLD_CONTENTS)

src/arch-arm32.cc

@@ -31,18 +31,21 @@
 //
 // https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst
 
+#if MOLD_ARM32LE || MOLD_ARM32BE
+
 #include "mold.h"
 
 #include <tbb/parallel_for.h>
+#include <tbb/parallel_for_each.h>
 
 namespace mold {
 
-using E = ARM32;
+using E = MOLD_TARGET;
 
 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
-  ul32 *arm = (ul32 *)loc;
-  ul16 *thm = (ul16 *)loc;
+  U32<E> *arm = (U32<E> *)loc;
+  U16<E> *thm = (U16<E> *)loc;
 
   switch (rel.r_type) {
   case R_ARM_ABS32:
@@ -59,7 +62,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   case R_ARM_TLS_GOTDESC:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
-    return (il32)*arm;
+    return (I32<E>)*arm;
   case R_ARM_THM_JUMP8:
     return sign_extend(thm[0], 8) << 1;
   case R_ARM_THM_JUMP11:
@@ -125,7 +128,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
 static void write_arm_mov(u8 *loc, u32 val) {
   u32 imm12 = bits(val, 11, 0);
   u32 imm4 = bits(val, 15, 12);
-  *(ul32 *)loc = (*(ul32 *)loc & 0xfff0'f000) | (imm4 << 16) | imm12;
+  *(U32<E> *)loc = (*(U32<E> *)loc & 0xfff0'f000) | (imm4 << 16) | imm12;
 }
 
 static void write_thm_b21(u8 *loc, u32 val) {
@@ -135,7 +138,7 @@ static void write_thm_b21(u8 *loc, u32 val) {
   u32 imm6 = bits(val, 17, 12);
   u32 imm11 = bits(val, 11, 1);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1011'1100'0000) | (S << 10) | imm6;
   buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
 }
@@ -149,7 +152,7 @@ static void write_thm_b25(u8 *loc, u32 val) {
   u32 imm10 = bits(val, 21, 12);
   u32 imm11 = bits(val, 11, 1);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1000'0000'0000) | (S << 10) | imm10;
   buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
 }
@@ -160,7 +163,7 @@ static void write_thm_mov(u8 *loc, u32 val) {
   u32 imm3 = bits(val, 10, 8);
   u32 imm8 = bits(val, 7, 0);
 
-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
   buf[0] = (buf[0] & 0b1111'1011'1111'0000) | (i << 10) | imm4;
   buf[1] = (buf[1] & 0b1000'1111'0000'0000) | (imm3 << 12) | imm8;
 }
@@ -184,13 +187,13 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
   case R_ARM_TLS_GOTDESC:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
     break;
   case R_ARM_THM_JUMP8:
-    *(ul16 *)loc = (*(ul16 *)loc & 0xff00) | bits(val, 8, 1);
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xff00) | bits(val, 8, 1);
     break;
   case R_ARM_THM_JUMP11:
-    *(ul16 *)loc = (*(ul16 *)loc & 0xf800) | bits(val, 11, 1);
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xf800) | bits(val, 11, 1);
     break;
   case R_ARM_THM_CALL:
   case R_ARM_THM_JUMP24:
@@ -200,7 +203,7 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PLT32:
-    *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
     break;
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVW_ABS_NC:
@@ -209,7 +212,7 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
     write_arm_mov(loc, val);
     break;
   case R_ARM_PREL31:
-    *(ul32 *)loc = (*(ul32 *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
     break;
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVW_ABS_NC:
@@ -236,7 +239,7 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
+  *(U32<E> *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
 }
 
 static constexpr ul32 plt_entry[] = {
@@ -249,13 +252,13 @@ static constexpr ul32 plt_entry[] = {
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 template <>
@@ -267,24 +270,16 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
   case R_NONE:
     break;
   case R_ARM_ABS32:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
     break;
   case R_ARM_REL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
     break;
   default:
     Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
   }
 }
 
-static Thunk<E> &get_reachable_thunk(OutputSection<E> &osec, u64 addr) {
-  auto it = std::upper_bound(osec.thunks.begin(), osec.thunks.end(), addr,
-                             [](u64 addr, std::unique_ptr<Thunk<E>> &thunk) {
-    return addr < thunk->get_addr();
-  });
-  return **it;
-}
-
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
@@ -295,6 +290,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -312,7 +310,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     auto get_arm_thunk_addr   = [&] { return sym.get_thunk_addr(ctx, P) + 4; };
 
     auto get_tlsdesc_trampoline_addr = [&] {
-      return get_reachable_thunk(*output_section, P).get_addr();
+      auto it = ranges::upper_bound(output_section->thunks, P, {},
+                                    [](std::unique_ptr<Thunk<E>> &thunk) {
+        return thunk->get_addr();
+      });
+      return (*it)->get_addr();
     };
 
     switch (rel.r_type) {
@@ -320,13 +322,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_ARM_TARGET1:
       break;
     case R_ARM_REL32:
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
       break;
     case R_ARM_THM_CALL: {
       if (sym.is_remaining_undef_weak()) {
         // On ARM, calling an weak undefined symbol jumps to the
         // next instruction.
-        *(ul32 *)loc = 0x8000'f3af; // NOP.W
+        *(U32<E> *)loc = 0x8000'f3af; // NOP.W
         break;
       }
 
@@ -337,61 +339,61 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       i64 val2 = align_to(S + A - P, 4);
 
       if (T && is_int(val1, 25)) {
-        *(ul16 *)(loc + 2) |= 0x1000;  // BL
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
         write_thm_b25(loc, val1);
       } else if (!T && is_int(val2, 25)) {
-        *(ul16 *)(loc + 2) &= ~0x1000; // BLX
+        *(U16<E> *)(loc + 2) &= ~0x1000; // BLX
         write_thm_b25(loc, val2);
       } else {
-        *(ul16 *)(loc + 2) |= 0x1000;  // BL
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
         write_thm_b25(loc, get_thumb_thunk_addr() + A - P);
       }
       break;
     }
     case R_ARM_BASE_PREL:
-      *(ul32 *)loc = GOT + A - P;
+      *(U32<E> *)loc = GOT + A - P;
       break;
     case R_ARM_GOTOFF32:
-      *(ul32 *)loc = ((S + A) | T) - GOT;
+      *(U32<E> *)loc = ((S + A) | T) - GOT;
       break;
     case R_ARM_GOT_PREL:
     case R_ARM_TARGET2:
-      *(ul32 *)loc = GOT + G + A - P;
+      *(U32<E> *)loc = GOT + G + A - P;
       break;
     case R_ARM_GOT_BREL:
-      *(ul32 *)loc = G + A;
+      *(U32<E> *)loc = G + A;
       break;
     case R_ARM_CALL: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
         break;
       }
 
       // Just like THM_CALL, ARM_CALL relocation refers to either BL or
       // BLX instruction. We may need to rewrite BL → BLX or BLX → BL.
-      bool is_bl = ((*(ul32 *)loc & 0xff00'0000) == 0xeb00'0000);
-      bool is_blx = ((*(ul32 *)loc & 0xfe00'0000) == 0xfa00'0000);
+      bool is_bl = ((*(U32<E> *)loc & 0xff00'0000) == 0xeb00'0000);
+      bool is_blx = ((*(U32<E> *)loc & 0xfe00'0000) == 0xfa00'0000);
       if (!is_bl && !is_blx)
         Fatal(ctx) << *this << ": R_ARM_CALL refers to neither BL nor BLX";
 
       i64 val = S + A - P;
       if (is_int(val, 26)) {
         if (T) {
-          *(ul32 *)loc = 0xfa00'0000; // BLX
-          *(ul32 *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
+          *(U32<E> *)loc = 0xfa00'0000; // BLX
+          *(U32<E> *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
         } else {
-          *(ul32 *)loc = 0xeb00'0000; // BL
-          *(ul32 *)loc |= bits(val, 25, 2);
+          *(U32<E> *)loc = 0xeb00'0000; // BL
+          *(U32<E> *)loc |= bits(val, 25, 2);
         }
       } else {
-        *(ul32 *)loc = 0xeb00'0000; // BL
-        *(ul32 *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // BL
+        *(U32<E> *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
       }
       break;
     }
     case R_ARM_JUMP24: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
         break;
       }
 
@@ -404,26 +406,26 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       i64 val = S + A - P;
       if (T || !is_int(val, 26))
         val = get_arm_thunk_addr() + A - P;
-      *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+      *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
       break;
     }
     case R_ARM_PLT32:
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
       } else {
         u64 val = (T ? get_arm_thunk_addr() : S) + A - P;
-        *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+        *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
       }
       break;
     case R_ARM_THM_JUMP8:
       check(S + A - P, -(1 << 8), 1 << 8);
-      *(ul16 *)loc &= 0xff00;
-      *(ul16 *)loc |= bits(S + A - P, 8, 1);
+      *(U16<E> *)loc &= 0xff00;
+      *(U16<E> *)loc |= bits(S + A - P, 8, 1);
       break;
     case R_ARM_THM_JUMP11:
       check(S + A - P, -(1 << 11), 1 << 11);
-      *(ul16 *)loc &= 0xf800;
-      *(ul16 *)loc |= bits(S + A - P, 11, 1);
+      *(U16<E> *)loc &= 0xf800;
+      *(U16<E> *)loc |= bits(S + A - P, 11, 1);
       break;
     case R_ARM_THM_JUMP19:
       check(S + A - P, -(1 << 20), 1 << 20);
@@ -431,7 +433,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_ARM_THM_JUMP24: {
       if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0x8000'f3af; // NOP
+        *(U32<E> *)loc = 0x8000'f3af; // NOP
         break;
       }
 
@@ -454,8 +456,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_ARM_PREL31:
       check(S + A - P, -(1LL << 30), 1LL << 30);
-      *(ul32 *)loc &= 0x8000'0000;
-      *(ul32 *)loc |= (S + A - P) & 0x7fff'ffff;
+      *(U32<E> *)loc &= 0x8000'0000;
+      *(U32<E> *)loc |= (S + A - P) & 0x7fff'ffff;
       break;
     case R_ARM_THM_MOVW_ABS_NC:
       write_thm_mov(loc, (S + A) | T);
@@ -473,19 +475,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_thm_mov(loc, (S + A) >> 16);
       break;
     case R_ARM_TLS_GD32:
-      *(ul32 *)loc = sym.get_tlsgd_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_tlsgd_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LDM32:
-      *(ul32 *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
+      *(U32<E> *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LDO32:
-      *(ul32 *)loc = S + A - ctx.dtp_addr;
+      *(U32<E> *)loc = S + A - ctx.dtp_addr;
       break;
     case R_ARM_TLS_IE32:
-      *(ul32 *)loc = sym.get_gottp_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_gottp_addr(ctx) + A - P;
       break;
     case R_ARM_TLS_LE32:
-      *(ul32 *)loc = S + A - ctx.tp_addr;
+      *(U32<E> *)loc = S + A - ctx.tp_addr;
       break;
     case R_ARM_TLS_GOTDESC:
       // ARM32 TLSDESC uses the following code sequence to materialize
@@ -514,35 +516,35 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       //  .L2: .word   foo(gottpoff) + . - .L1
       if (sym.has_tlsdesc(ctx)) {
         // A is odd if the corresponding TLS_CALL is Thumb.
-        *(ul32 *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
+        *(U32<E> *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
       } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
+        *(U32<E> *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
       } else {
-        *(ul32 *)loc = S - ctx.tp_addr;
+        *(U32<E> *)loc = S - ctx.tp_addr;
       }
       break;
     case R_ARM_TLS_CALL:
       if (sym.has_tlsdesc(ctx)) {
-        *(ul32 *)loc = 0xeb00'0000; // bl 0
-        *(ul32 *)loc |= bits(get_tlsdesc_trampoline_addr() - P - 8, 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // bl 0
+        *(U32<E> *)loc |= bits(get_tlsdesc_trampoline_addr() - P - 8, 25, 2);
       } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
+        *(U32<E> *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
       } else {
-        *(ul32 *)loc = 0xe320'f000; // nop
+        *(U32<E> *)loc = 0xe320'f000; // nop
       }
       break;
     case R_ARM_THM_TLS_CALL:
       if (sym.has_tlsdesc(ctx)) {
         u64 val = align_to(get_tlsdesc_trampoline_addr() - P - 4, 4);
         write_thm_b25(loc, val);
-        *(ul16 *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
+        *(U16<E> *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
       } else if (sym.has_gottp(ctx)) {
         // Since `ldr r0, [pc, r0]` is not representable in Thumb,
         // we use two instructions instead.
-        *(ul16 *)loc = 0x4478;         // add r0, pc
-        *(ul16 *)(loc + 2) = 0x6800;   // ldr r0, [r0]
+        *(U16<E> *)loc = 0x4478;         // add r0, pc
+        *(U16<E> *)(loc + 2) = 0x6800;   // ldr r0, [r0]
       } else {
-        *(ul32 *)loc = 0x8000'f3af;    // nop.w
+        *(U32<E> *)loc = 0x8000'f3af;    // nop.w
       }
       break;
     default:
@@ -553,12 +555,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -573,22 +572,22 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     switch (rel.r_type) {
     case R_ARM_ABS32:
       if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
       else
-        *(ul32 *)loc = S + A;
+        *(U32<E> *)loc = S + A;
       break;
     case R_ARM_TLS_LDO32:
       if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
       else
-        *(ul32 *)loc = S + A - ctx.dtp_addr;
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
       break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -695,27 +694,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_hdr_size == sizeof(hdr));
   static_assert(E::thunk_size == sizeof(entry));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  memcpy(buf, hdr, sizeof(hdr));
-  buf += sizeof(hdr);
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  memcpy(base, hdr, sizeof(hdr));
 
-  u64 P = output_section.shdr.sh_addr + offset + sizeof(hdr);
-
-  for (Symbol<E> *sym : symbols) {
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
     memcpy(buf, entry, sizeof(entry));
-    *(ul32 *)(buf + 12) = sym->get_addr(ctx) - P - 16;
-
-    buf += sizeof(entry);
-    P += sizeof(entry);
+    *(U32<E> *)(buf + 12) = S - P - 16;
   }
 }
 
 template <>
 u64 get_eflags(Context<E> &ctx) {
-  return EF_ARM_EABI_VER5;
+  if constexpr (E::is_le)
+    return EF_ARM_EABI_VER5;
+  else
+    return EF_ARM_EABI_VER5 | EF_ARM_BE8;
 }
 
-void create_arm_exidx_section(Context<E> &ctx) {
+template <>
+void create_arm_exidx_section<E>(Context<E> &ctx) {
   for (i64 i = 0; i < ctx.chunks.size(); i++) {
     OutputSection<E> *osec = ctx.chunks[i]->to_osec();
 
@@ -732,28 +732,20 @@ void create_arm_exidx_section(Context<E> &ctx) {
   }
 }
 
-void Arm32ExidxSection::compute_section_size(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::compute_section_size(Context<E> &ctx) {
   output_section.compute_section_size(ctx);
   this->shdr.sh_size = output_section.shdr.sh_size + 8; // +8 for sentinel
 }
 
-void Arm32ExidxSection::update_shdr(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::update_shdr(Context<E> &ctx) {
   // .ARM.exidx's sh_link should be set to the .text section index.
   // Runtime doesn't care about it, but the binutils's strip command does.
   if (Chunk<E> *chunk = find_chunk(ctx, ".text"))
     this->shdr.sh_link = chunk->shndx;
 }
 
-void Arm32ExidxSection::remove_duplicate_entries(Context<E> &ctx) {
-  this->shdr.sh_size = get_contents(ctx).size();
-}
-
-void Arm32ExidxSection::copy_buf(Context<E> &ctx) {
-  std::vector<u8> contents = get_contents(ctx);
-  assert(this->shdr.sh_size == contents.size());
-  write_vector(ctx.buf + this->shdr.sh_offset, contents);
-}
-
 // Returns the end of the text segment
 static u64 get_text_end(Context<E> &ctx) {
   u64 ret = 0;
@@ -772,7 +764,8 @@ static u64 get_text_end(Context<E> &ctx) {
 // likely that it's due to some historical reason.
 //
 // This function returns contents of .ARM.exidx.
-std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
+template <>
+std::vector<u8> Arm32ExidxSection<E>::get_contents(Context<E> &ctx) {
   // .ARM.exidx records consists of a signed 31-bit relative address
   // and a 32-bit value. The relative address indicates the start
   // address of a function that the record covers. The value is one of
@@ -788,8 +781,8 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
   const u32 CANTUNWIND = 1;
 
   struct Entry {
-    ul32 addr;
-    ul32 val;
+    U32<E> addr;
+    U32<E> val;
   };
 
   // We reserve one extra slot for the sentinel
@@ -820,19 +813,13 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
       ent[i].val = 0x7fff'ffff & (ent[i].val + offset);
   });
 
-  std::sort(ent, ent + num_entries, [](const Entry &a, const Entry &b) {
-    return a.addr < b.addr;
-  });
+  ranges::sort(ent, ent + num_entries, {}, &Entry::addr);
 
   // Remove duplicate adjacent entries. That is, if two adjacent functions
   // have the same compact unwind info or are both CANTUNWIND, we can
   // merge them into a single address range.
-  auto it = std::unique(ent, ent + num_entries,
-                        [](const Entry &a, const Entry &b) {
-    return a.val == b.val;
-  });
-
-  num_entries = it - ent;
+  auto tail = ranges::unique(ent, ent + num_entries, {}, &Entry::val);
+  num_entries -= tail.size();
   buf.resize(num_entries * sizeof(Entry));
 
   // Make addresses relative to themselves.
@@ -846,4 +833,84 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
   return buf;
 }
 
+template <>
+void Arm32ExidxSection<E>::remove_duplicate_entries(Context<E> &ctx) {
+  this->shdr.sh_size = get_contents(ctx).size();
+}
+
+template <>
+void Arm32ExidxSection<E>::copy_buf(Context<E> &ctx) {
+  std::vector<u8> contents = get_contents(ctx);
+  assert(this->shdr.sh_size == contents.size());
+  write_vector(ctx.buf + this->shdr.sh_offset, contents);
+}
+
+// Even though using ARM32 in big-endian mode is very rare, the processor
+// technically supports both little- and big-endian modes. There are two
+// variants of big-endian mode: BE32 and BE8. In BE32, instructions and
+// data are encoded in big-endian. In BE8, instructions are encoded in
+// little-endian, and only data is in big-endian. BE8 is the de facto
+// standard for ARMv6 or later. We support only BE8.
+//
+// A tricky thing is that instructions in an object file are always
+// big-endian if the file is compiled for big-endian mode. In other words,
+// the compiler always emit code in BE32 if -mbig-endian is specified. It
+// is the linker's responsibility to rewrite instructions from big-endian
+// to little-endian for an BE8 output. This function does that.
+//
+// The text section may contain a mix of 32-bit ARM instructions, 16-bit
+// Thumb instructions, and data. We need to distinguish them to swap 4
+// bytes, 2 bytes, or not swap bytes, respectively. The beginning of ARM
+// code, Thumb code, and data is labeled with a mapping symbol of $a, $t,
+// and $d, respectively. We use mapping symbols to determine what to do
+// with the text section.
+//
+// This function is called after we copy the input section contents to the
+// output file. We rewrite instructions in the output buffer in place.
+#if MOLD_ARM32BE
+void arm32be_swap_bytes(Context<E> &ctx) {
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    // Collect mapping symbols
+    std::vector<Symbol<E> *> syms;
+
+    for (Symbol<E> *sym : file->get_local_syms())
+      if (InputSection<E> *isec = sym->get_input_section())
+        if (isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
+          if (std::string_view x = sym->name();
+              x == "$a" || x.starts_with("$a.") ||
+              x == "$t" || x.starts_with("$t.") ||
+              x == "$d" || x.starts_with("$d."))
+            syms.push_back(sym);
+
+    // Group mapping symbols by input section and sort by address
+    ranges::stable_sort(syms, {}, [](const Symbol<E> *sym) {
+      return std::tuple{(uintptr_t)sym->get_input_section(), sym->value};
+    });
+
+    // Swap bytes
+    for (i64 i = 0; i < syms.size(); i++) {
+      Symbol<E> &sym = *syms[i];
+      if (sym.name().starts_with("$d"))
+        continue;
+
+      InputSection<E> &isec = *sym.get_input_section();
+      u8 *base = ctx.buf + isec.output_section->shdr.sh_offset + isec.offset;
+      u8 *start = base + sym.value;
+      u8 *end;
+
+      if (i + 1 < syms.size() && syms[i + 1]->get_input_section() == &isec)
+        end = base + syms[i + 1]->value;
+      else
+        end = base + isec.sh_size;
+
+      i64 sz = sym.name().starts_with("$a") ? 4 : 2;
+      for (u8 *p = start; p < end; p += sz)
+        std::reverse(p, p + sz);
+    }
+  });
+}
+#endif
+
 } // namespace mold
+
+#endif

src/arch-arm64.cc

@@ -145,14 +145,17 @@ static bool is_add(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -165,6 +168,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_AARCH64_ABS64:
       break;
@@ -208,6 +216,30 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_AARCH64_MOVW_UABS_G3:
       *(ul32 *)loc |= bits(S + A, 63, 48) << 5;
       break;
+    case R_AARCH64_MOVW_PREL_G0:
+      check(S + A - P, -(1 << 15), 1 << 15);
+      write_movn_movz(loc, S + A - P);
+      break;
+    case R_AARCH64_MOVW_PREL_G0_NC:
+      *(ul32 *)loc |= bits(S + A - P, 15, 0) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G1:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
+      write_movn_movz(loc, (i64)(S + A - P) >> 16);
+      break;
+    case R_AARCH64_MOVW_PREL_G1_NC:
+      *(ul32 *)loc |= bits(S + A - P, 31, 16) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G2:
+      check(S + A - P, -(1LL << 47), 1LL << 47);
+      write_movn_movz(loc, (i64)(S + A - P) >> 32);
+      break;
+    case R_AARCH64_MOVW_PREL_G2_NC:
+      *(ul32 *)loc |= bits(S + A - P, 47, 32) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G3:
+      *(ul32 *)loc |= bits(S + A - P, 63, 48) << 5;
+      break;
     case R_AARCH64_ADR_GOT_PAGE:
       if (sym.has_got(ctx)) {
         i64 val = page(G + GOT + A) - page(P);
@@ -222,6 +254,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         u32 reg = bits(*(ul32 *)loc, 4, 0);
         *(ul32 *)(loc + 4) = 0x9100'0000 | (reg << 5) | reg; // ADD
         *(ul32 *)(loc + 4) |= bits(S + A, 11, 0) << 10;
+
+        rewrite(i, R_AARCH64_ADR_PREL_PG_HI21);
+        rewrite(i + 1, R_AARCH64_ADD_ABS_LO12_NC);
         i++;
       }
       break;
@@ -248,6 +283,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             *(ul32 *)loc = 0xd503'201f;              // nop
             *(ul32 *)(loc + 4) = 0x1000'0000 | reg1; // adr
             write_adr(loc + 4, val);
+            rewrite(i, R_NONE);
+            rewrite(i + 1, R_AARCH64_ADR_PREL_LO21);
             i++;
             break;
           }
@@ -283,6 +320,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check(S + A - P, -(1LL << 31), 1LL << 31);
       *(U32<E> *)loc = S + A - P;
       break;
+    case R_AARCH64_GOTPCREL32:
+      check(G + GOT + A - P, -(1LL << 31), 1LL << 31);
+      *(U32<E> *)loc = G + GOT + A - P;
+      break;
     case R_AARCH64_CONDBR19:
     case R_AARCH64_LD_PREL_LO19:
       check(S + A - P, -(1LL << 20), 1LL << 20);
@@ -397,13 +438,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         write_adrp(loc, val);
       } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
       }
       break;
     case R_AARCH64_TLSDESC_LD64_LO12:
-      if (sym.has_tlsdesc(ctx))
+      if (sym.has_tlsdesc(ctx)) {
         *(ul32 *)loc |= bits(sym.get_tlsdesc_addr(ctx) + A, 11, 3) << 10;
-      else
+      } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
+      }
       break;
     case R_AARCH64_TLSDESC_ADD_LO12:
       if (sym.has_tlsdesc(ctx)) {
@@ -411,9 +455,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x9000'0000; // adrp x0, 0
         write_adrp(loc, page(sym.get_gottp_addr(ctx) + A) - page(P));
+        rewrite(i, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21);
       } else {
         *(ul32 *)loc = 0xd2a0'0000; // movz x0, 0, lsl #16
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 32, 16) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G1);
       }
       break;
     case R_AARCH64_TLSDESC_CALL:
@@ -422,9 +468,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0xf940'0000; // ldr x0, [x0, 0]
         *(ul32 *)loc |= bits(sym.get_gottp_addr(ctx) + A, 11, 3) << 10;
+        rewrite(i, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC);
       } else {
         *(ul32 *)loc = 0xf280'0000; // movk x0, 0
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 15, 0) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G0_NC);
       }
       break;
     default:
@@ -435,12 +483,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -467,12 +512,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       check(S + A, 0, 1LL << 32);
       *(U32<E> *)loc = S + A;
       break;
+    case R_AARCH64_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -487,7 +538,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -525,6 +576,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       break;
     case R_AARCH64_LD64_GOT_LO12_NC:
     case R_AARCH64_LD64_GOTPAGE_LO15:
+    case R_AARCH64_GOTPCREL32:
       sym.flags |= NEEDS_GOT;
       break;
     case R_AARCH64_CALL26:
@@ -568,6 +620,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_AARCH64_MOVW_UABS_G1_NC:
     case R_AARCH64_MOVW_UABS_G2:
     case R_AARCH64_MOVW_UABS_G2_NC:
+    case R_AARCH64_MOVW_PREL_G0:
+    case R_AARCH64_MOVW_PREL_G0_NC:
+    case R_AARCH64_MOVW_PREL_G1:
+    case R_AARCH64_MOVW_PREL_G1_NC:
+    case R_AARCH64_MOVW_PREL_G2:
+    case R_AARCH64_MOVW_PREL_G2_NC:
+    case R_AARCH64_MOVW_PREL_G3:
     case R_AARCH64_PREL16:
     case R_AARCH64_PREL32:
     case R_AARCH64_PREL64:
@@ -587,31 +646,81 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
   }
 }
 
+// The size of a thunk entry varies on ARM64 depending on the distance to
+// the branch target. This function computes the size of each thunk entry.
+template <>
+void Thunk<E>::shrink_size(Context<E> &ctx) {
+  offsets.clear();
+  offsets.push_back(0);
+  i64 off = 0;
+
+  // The distance between S and P is only reduced by shrink_size(), but
+  // page(S) – page(P) may still increase by one page due to address
+  // changes, so we add a safety margin.
+  //
+  // For example, page(0x1200) – page(0x1000) is 0, whereas
+  // page(0x1100) – page(0xfff) is 0x1000, even though the latter
+  // distance is shorter than the former.
+  auto is_small = [](i64 prel) {
+    return is_int(prel + 0x1000, 33) && is_int(prel - 0x1000, 33);
+  };
+
+  for (Symbol<E> *sym : symbols) {
+    u64 S = sym->get_addr(ctx);
+    u64 P = get_addr() + off;
+    i64 prel = page(S) - page(P);
+    off += is_small(prel) ? 16 : 32;
+    offsets.push_back(off);
+  }
+}
+
 template <>
 void Thunk<E>::copy_buf(Context<E> &ctx) {
-  constexpr ul32 insn[] = {
-    0x9000'0010, // adrp x16, 0   # R_AARCH64_ADR_PREL_PG_HI21
-    0x9100'0210, // add  x16, x16 # R_AARCH64_ADD_ABS_LO12_NC
+  // Short thunk with a 33 bit displacement
+  constexpr ul32 insn1[] = {
+    0x9000'0010, // adrp x16, 0
+    0x9100'0210, // add  x16, x16
     0xd61f'0200, // br   x16
     0xd420'7d00, // brk
   };
 
-  static_assert(E::thunk_size == sizeof(insn));
+  // Long thunk with a 64 bit displacement
+  constexpr ul32 insn2[] = {
+    0x1000'0010, // adr  x16, 0
+    0xd2a0'0011, // movz x17, 0, lsl #16
+    0xf2c0'0011, // movk x17, 0, lsl #32
+    0xf2e0'0011, // movk x17, 0, lsl #48
+    0x8b11'0210, // add  x16, x16, x17
+    0xd61f'0200, // br   x16
+    0xd420'7d00, // brk
+    0xd420'7d00, // brk
+  };
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
 
-  for (Symbol<E> *sym : symbols) {
-    u64 S = sym->get_addr(ctx);
-    memcpy(buf, insn, E::thunk_size);
-    write_adrp(buf, page(S) - page(P));
-    *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
 
-    buf += E::thunk_size;
-    P += E::thunk_size;
+    if (offsets[i + 1] - offsets[i] == 16) {
+      i64 prel = page(S) - page(P);
+      assert(is_int(prel, 33));
+      memcpy(buf, insn1, sizeof(insn1));
+      write_adrp(buf, prel);
+      *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
+    } else {
+      memcpy(buf, insn2, sizeof(insn2));
+      write_adr(buf, bits(S - P, 15, 0));
+      *(ul32 *)(buf + 4) |= bits(S - P, 31, 16) << 5;
+      *(ul32 *)(buf + 8) |= bits(S - P, 47, 32) << 5;
+      *(ul32 *)(buf + 12) |= bits(S - P, 63, 48) << 5;
+    }
   }
 }
 
+template class Thunk<E>;
+
 } // namespace mold
 
 #endif

src/arch-i386.cc

@@ -33,6 +33,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/i386.pdf
 
+#if MOLD_I386
+
 #include "mold.h"
 
 namespace mold {
@@ -44,10 +46,10 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   switch (rel.r_type) {
   case R_386_8:
   case R_386_PC8:
-    return *loc;
+    return *(i8 *)loc;
   case R_386_16:
   case R_386_PC16:
-    return *(ul16 *)loc;
+    return *(il16 *)loc;
   case R_386_32:
   case R_386_PC32:
   case R_386_GOT32:
@@ -63,7 +65,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   case R_386_TLS_LDO_32:
   case R_386_SIZE32:
   case R_386_TLS_GOTDESC:
-    return *(ul32 *)loc;
+    return *(il32 *)loc;
   default:
     return 0;
   }
@@ -278,14 +280,17 @@ static u32 relax_tlsdesc_to_le(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -333,6 +338,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         loc[-2] = insn >> 8;
         loc[-1] = insn;
         *(ul32 *)loc = S + A - GOT;
+        if (ctx.arg.emit_relocs)
+          rel.r_type = R_386_GOTOFF;
       }
       break;
     case R_386_GOTOFF:
@@ -434,12 +441,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -500,7 +504,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       unreachable();
     }
-  }
+  });
 }
 
 template <>
@@ -515,7 +519,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -596,3 +600,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-loongarch.cc

@@ -66,7 +66,7 @@ static u64 higher(u64 val, u64 pc) {
   // LU52I.D simply set bits to [51:31] and to [63:53], respectively.
   //
   // Compensating all the sign-extensions is a bit complicated. The
-  // psABI gave the following formula.
+  // psABI gives the following formula.
   val = val + 0x8000'0000 + ((val & 0x800) ? (0x1000 - 0x1'0000'0000) : 0);
   return page(val) - page(pc - 8);
 }
@@ -133,7 +133,7 @@ static void set_rj(u8 *loc, u32 rj) {
 static bool is_relaxable_got_load(Context<E> &ctx, InputSection<E> &isec, i64 i) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   Symbol<E> &sym = *isec.file.symbols[rels[i].r_sym];
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   if (ctx.arg.relax &&
       sym.is_pcrel_linktime_const(ctx) &&
@@ -269,13 +269,13 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
 
     if (rel.r_type == R_NONE || rel.r_type == R_LARCH_RELAX ||
         rel.r_type == R_LARCH_MARK_LA || rel.r_type == R_LARCH_MARK_PCREL ||
@@ -295,6 +295,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -332,6 +335,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
                    << " for relocation " << rel;
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_LARCH_32:
       assert(E::is_64);
@@ -375,10 +383,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (removed_bytes == 0) {
         write_j20(loc, hi20(S + A, P));
       } else {
-        // Rewrite pcalau12i + addi.d with pcaddi
+        // Rewrite pcalau12i + addi.d with pcaddi. The high part vanishes and
+        // the low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -417,10 +428,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         }
         write_j20(loc, hi20(GOT + G + A, P));
       } else {
-        // Rewrite pcalau12i + ld.d with pcaddi
+        // Rewrite pcalau12i + ld.d with pcaddi. The high part vanishes and the
+        // low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -518,6 +532,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul64 *)loc -= S + A;
       break;
     case R_LARCH_32_PCREL:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
       *(ul32 *)loc = S + A - P;
       break;
     case R_LARCH_64_PCREL:
@@ -537,6 +552,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x5400'0000; // BL
         write_d10k16(loc, (S + A - P) >> 2);
+        rewrite(i, R_LARCH_B26);
       }
       break;
     case R_LARCH_ADD_ULEB128:
@@ -590,30 +606,49 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       //   jirl      $ra, $ra, 0
       if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
         write_j20(loc, hi20(sym.get_tlsdesc_addr(ctx) + A, P));
+
+      // pcalau12i + addi.d => pcaddi when TLSDESC is kept; both deleted when it
+      // is relaxed to IE/LE. Either way the high part loses its relocation.
+      // The folded pcaddi's relocation is emitted from the LO12 slot below.
+      if (!sym.has_tlsdesc(ctx) || removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_DESC_PC_LO12:
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0) {
-        i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
-        if (is_int(dist, 22)) {
-          *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
-          write_j20(loc, dist >> 2);
-        } else {
-          write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0) {
+          i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
+          if (is_int(dist, 22)) {
+            *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
+            write_j20(loc, dist >> 2);
+            rewrite(i, R_LARCH_TLS_DESC_PCREL20_S2);
+          } else {
+            write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+          }
         }
+      } else {
+        rewrite(i, R_NONE);
       }
       break;
     case R_LARCH_TLS_DESC_LD:
-      if (sym.has_tlsdesc(ctx) || removed_bytes == 4) {
-        // Do nothing
+      // TLSDESC is relaxed to IE or LE. The ld.d slot holds the first
+      // instruction of the resulting sequence.
+      if (sym.has_tlsdesc(ctx)) {
+        // Do nothing (TLSDESC kept)
+      } else if (removed_bytes == 4) {
+        // Small TP offset: the instruction was deleted.
+        rewrite(i, R_NONE);
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x1a00'0004; // pcalau12i $a0, 0
         write_j20(loc, hi20(sym.get_gottp_addr(ctx) + A, P));
+        rewrite(i, R_LARCH_TLS_IE_PC_HI20);
       } else {
         *(ul32 *)loc = 0x1400'0004; // lu12i.w   $a0, 0
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+        rewrite(i, R_LARCH_TLS_LE_HI20);
       }
       break;
     case R_LARCH_TLS_DESC_CALL:
+      // The jirl slot holds the second instruction of the IE or LE sequence.
       if (sym.has_tlsdesc(ctx)) {
         // Do nothing
       } else if (sym.has_gottp(ctx)) {
@@ -622,6 +657,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x2880'0084; // ld.w $a0, $a0, 0
         write_k12(loc, sym.get_gottp_addr(ctx) + A);
+        rewrite(i, R_LARCH_TLS_IE_PC_LO12);
       } else {
         i64 val = S + A - ctx.tp_addr;
         if (0 <= val && val < 0x1000)
@@ -629,11 +665,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x0280'0084; // addi.w $a0, $a0, 0
         write_k12(loc, val);
+        rewrite(i, R_LARCH_TLS_LE_LO12);
       }
       break;
     case R_LARCH_TLS_LE_HI20_R:
+      // lu12i.w + add.d + addi.d => addi.d when the variable is within 2 KiB
+      // of TP, in which case the lu12i.w loses its relocation.
       if (removed_bytes == 0)
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_LE_LO12_R: {
       i64 val = S + A - ctx.tp_addr;
@@ -645,8 +686,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         set_rj(loc, 2);
       break;
     }
-    case R_LARCH_64:
     case R_LARCH_TLS_LE_ADD_R:
+      // add.d that materializes TP + offset; removed together with the
+      // lu12i.w when the variable is within 2 KiB of TP.
+      if (removed_bytes)
+        rewrite(i, R_NONE);
+      break;
+    case R_LARCH_64:
       break;
     default:
       unreachable();
@@ -656,19 +702,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE)
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
-    if (!sym.file) {
+    if (!sym.file && &sym != discarded_comdat_sym<E>) {
       record_undef_error(ctx, rel);
-      continue;
+      return;
     }
 
     SectionFragment<E> *frag;
@@ -741,7 +784,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -856,7 +899,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &r = rels[i];
@@ -980,7 +1023,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
       //   pcaddi    $t0, <offset>
       if (is_relaxable_got_load(ctx, isec, i))
         if (i64 dist = compute_distance(ctx, sym, isec, r);
-            is_int(dist, 22))
+            is_int(dist, 22) && (dist & 3) == 0)
           remove(4);
       break;
     case R_LARCH_TLS_DESC_PC_HI20:

src/arch-m68k.cc

@@ -14,6 +14,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/m68k.pdf
 
+#if MOLD_M68K
+
 #include "mold.h"
 
 namespace mold {
@@ -84,6 +86,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -208,12 +213,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -232,11 +234,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(ub32 *)loc = S + A;
       break;
+    case R_68K_TLS_LDO32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -310,3 +318,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc32.cc

@@ -40,6 +40,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc32.pdf
 
+#if MOLD_PPC32
+
 #include "mold.h"
 
 namespace mold {
@@ -156,6 +158,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -282,12 +287,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -306,11 +308,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(ub32 *)loc = S + A;
       break;
+    case R_PPC_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -416,27 +424,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(plt_entry));
   static_assert(E::thunk_size == sizeof(local_thunk));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
       i64 val = got - P - 8;
       memcpy(buf, plt_entry, sizeof(plt_entry));
       *(ub32 *)(buf + 16) |= higha(val);
       *(ub32 *)(buf + 20) |= lo(val);
     } else {
-      i64 val = sym->get_addr(ctx) - P - 8;
+      i64 val = sym.get_addr(ctx) - P - 8;
       memcpy(buf, local_thunk, sizeof(local_thunk));
       *(ub32 *)(buf + 16) |= higha(val);
       *(ub32 *)(buf + 20) |= lo(val);
     }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
   }
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc64v1.cc

@@ -45,6 +45,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v1.pdf
 
+#if MOLD_PPC64V1
+
 #include "mold.h"
 
 #include <algorithm>
@@ -160,6 +162,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -217,6 +222,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_REL16_LO:
       *(ub16 *)loc = lo(S + A - P);
       break;
+    case R_PPC64_GOT16:
+      *(ub16 *)loc = G - TOC;
+      break;
     case R_PPC64_PLT16_HA:
       *(ub16 *)loc = ha(G + GOT - TOC);
       break;
@@ -280,12 +288,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -319,7 +324,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -351,6 +356,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       if (sym.is_imported)
         sym.flags |= NEEDS_PLT;
       break;
+    case R_PPC64_GOT16:
     case R_PPC64_PLT16_HA:
       sym.flags |= NEEDS_GOT;
       break;
@@ -452,49 +458,46 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(plt_thunk));
   static_assert(E::thunk_size == sizeof(local_thunk));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  u64 TOC = ctx.extra.TOC->value;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_got(ctx)) {
-      i64 val = sym->get_got_addr(ctx) - ctx.extra.TOC->value;
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_got(ctx)) {
       memcpy(buf, pltgot_thunk, sizeof(pltgot_thunk));
+      i64 val = sym.get_got_addr(ctx) - TOC;
       *(ub32 *)(buf + 4) |= higha(val);
       *(ub32 *)(buf + 8) |= lo(val);
-    } else if(sym->has_plt(ctx)) {
-      i64 val = sym->get_gotplt_addr(ctx) - ctx.extra.TOC->value;
+    } else if(sym.has_plt(ctx)) {
       memcpy(buf, plt_thunk, sizeof(plt_thunk));
+      i64 val = sym.get_gotplt_addr(ctx) - TOC;
       *(ub32 *)(buf + 4) |= higha(val);
       *(ub32 *)(buf + 8) |= lo(val);
     } else {
-      i64 val = sym->get_addr(ctx, NO_OPD) - ctx.extra.TOC->value;
       memcpy(buf, local_thunk, sizeof(local_thunk));
+      i64 val = sym.get_addr(ctx, NO_OPD) - TOC;
       *(ub32 *)buf |= higha(val);
       *(ub32 *)(buf + 4) |= lo(val);
     }
-
-    buf += E::thunk_size;
   }
 }
 
 static InputSection<E> *get_opd_section(ObjectFile<E> &file) {
-  for (std::unique_ptr<InputSection<E>> &isec : file.sections)
+  for (InputSection<E> *isec : file.sections)
     if (isec && isec->name() == ".opd")
-      return isec.get();
+      return isec;
   return nullptr;
 }
 
 static ElfRel<E> *
-get_relocation_at(Context<E> &ctx, InputSection<E> &isec, i64 offset) {
+get_relocation_at(Context<E> &ctx, InputSection<E> &isec, u64 offset) {
   std::span<ElfRel<E>> rels = isec.get_rels(ctx);
-
-  auto it = std::lower_bound(rels.begin(), rels.end(), offset,
-                             [](const ElfRel<E> &r, i64 offset) {
-    return r.r_offset < offset;
+  auto it = ranges::lower_bound(rels, offset, {}, [](const ElfRel<E> &r) {
+    return (u64)r.r_offset;
   });
-
-  if (it == rels.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  if (it == rels.end() || it->r_offset != offset)
     return nullptr;
   return &*it;
 }
@@ -509,9 +512,7 @@ struct OpdSymbol {
 static Symbol<E> *
 get_opd_sym_at(std::span<OpdSymbol> syms, u64 offset) {
   auto it = ranges::lower_bound(syms, offset, {}, &OpdSymbol::r_offset);
-  if (it == syms.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  if (it == syms.end() || it->r_offset != offset)
     return nullptr;
   return it->sym;
 }
@@ -593,8 +594,8 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
     ranges::stable_sort(opd_syms, {}, &OpdSymbol::r_offset);
 
     // Rewrite relocations so that they directly refer to .opd.
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (!isec || !isec->is_alive || isec.get() == opd)
+    for (InputSection<E> *isec : file->sections) {
+      if (!isec || !isec->is_alive || isec == opd)
         continue;
 
       for (ElfRel<E> &r : isec->get_rels(ctx)) {
@@ -618,11 +619,10 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
 // refers to the function's .opd entry. This function marks such symbols
 // with NEEDS_PPC_OPD.
 void ppc64v1_scan_symbols(Context<E> &ctx) {
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (Symbol<E> *sym : file->symbols)
-      if (sym->file == file && sym->is_exported)
-        if (u32 ty = sym->get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
-          sym->flags |= NEEDS_PPC_OPD;
+  ctx.symbol_map.parallel_for_each([](Symbol<E> &sym) {
+    if (sym.file && !sym.file->is_dso && sym.is_exported)
+      if (u32 ty = sym.get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
+        sym.flags |= NEEDS_PPC_OPD;
   });
 
   // Functions referenced by the ELF header also have to have .opd entries.
@@ -637,36 +637,55 @@ void ppc64v1_scan_symbols(Context<E> &ctx) {
 }
 
 void PPC64OpdSection::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_opd_idx(ctx, symbols.size());
+  sym->aux->opd_idx = symbols.size();
   symbols.push_back(sym);
   this->shdr.sh_size += ENTRY_SIZE;
 }
 
-i64 PPC64OpdSection::get_reldyn_size(Context<E> &ctx) const {
-  if (ctx.arg.pic)
-    return symbols.size() * 2;
-  return 0;
+i64 PPC64OpdSection::get_num_dynrels(Context<E> &ctx) const {
+  return ctx.arg.pic ? symbols.size() * 2 : 0;
+}
+
+std::vector<u64> PPC64OpdSection::get_relr_offsets(Context<E> &ctx) {
+  if (!ctx.arg.pic)
+    return {};
+
+  std::vector<u64> offsets;
+  offsets.reserve(symbols.size() * 2);
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx) - this->shdr.sh_addr;
+    offsets.push_back(loc);
+    offsets.push_back(loc + 8);
+  }
+  return offsets;
+}
+
+void PPC64OpdSection::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  if (!ctx.arg.pic)
+    return;
+
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx);
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        loc % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc, E::R_RELATIVE, 0, sym->get_addr(ctx, NO_PLT | NO_OPD));
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        (loc + 8) % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
+  }
 }
 
 void PPC64OpdSection::copy_buf(Context<E> &ctx) {
   ub64 *buf = (ub64 *)(ctx.buf + this->shdr.sh_offset);
 
-  ElfRel<E> *rel = nullptr;
-  if (ctx.arg.pic)
-    rel = (ElfRel<E> *)(ctx.buf + ctx.reldyn->shdr.sh_offset + reldyn_offset);
-
   for (Symbol<E> *sym : symbols) {
     u64 addr = sym->get_addr(ctx, NO_PLT | NO_OPD);
     *buf++ = addr;
     *buf++ = ctx.extra.TOC->value;
     *buf++ = 0;
-
-    if (ctx.arg.pic) {
-      u64 loc = sym->get_opd_addr(ctx);
-      *rel++ = ElfRel<E>(loc, E::R_RELATIVE, 0, addr);
-      *rel++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
-    }
   }
 }
 
 } // namespace mold
+
+#endif

src/arch-ppc64v2.cc

@@ -80,6 +80,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v2.pdf
 
+#if MOLD_PPC64V2
+
 #include "mold.h"
 
 namespace mold {
@@ -192,6 +194,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -243,6 +248,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         *(ul32 *)loc |= bits(val, 25, 2) << 2;
       }
       break;
+    case R_PPC64_REL14:
+      *(ul32 *)loc |= bits(S + A - P, 15, 2) << 2;
+      break;
     case R_PPC64_REL32:
       *(ul32 *)loc = S + A - P;
       break;
@@ -255,6 +263,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_REL16_LO:
       *(ul16 *)loc = lo(S + A - P);
       break;
+    case R_PPC64_GOT16:
+      *(ul16 *)loc = G - TOC;
+      break;
+    case R_PPC64_GOT16_LO:
+      *(ul16 *)loc = lo(G - TOC);
+      break;
+    case R_PPC64_GOT16_HI:
+      *(ul16 *)loc = hi(G - TOC);
+      break;
+    case R_PPC64_GOT16_HA:
+      *(ul16 *)loc = ha(G - TOC);
+      break;
     case R_PPC64_PLT16_HA:
       *(ul16 *)loc = ha(G + GOT - TOC);
       break;
@@ -344,12 +364,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -383,7 +400,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -416,6 +433,10 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_PLT;
       ctx.extra.is_power10 = true;
       break;
+    case R_PPC64_GOT16:
+    case R_PPC64_GOT16_LO:
+    case R_PPC64_GOT16_HI:
+    case R_PPC64_GOT16_HA:
     case R_PPC64_PLT16_HA:
     case R_PPC64_PLT_PCREL34:
     case R_PPC64_PLT_PCREL34_NOTOC:
@@ -431,12 +452,11 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       ctx.needs_tlsld = true;
       break;
     case R_PPC64_TPREL16_HA:
-    case R_PPC64_TPREL16_LO:
-    case R_PPC64_TPREL16_LO_DS:
     case R_PPC64_TPREL34:
       check_tlsle(ctx, sym, rel);
       break;
     case R_PPC64_ADDR64:
+    case R_PPC64_REL14:
     case R_PPC64_REL32:
     case R_PPC64_REL64:
     case R_PPC64_TOC16_HA:
@@ -456,6 +476,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_PPC64_GOT_TPREL16_LO_DS:
     case R_PPC64_GOT_TLSGD16_LO:
     case R_PPC64_GOT_TLSLD16_LO:
+    case R_PPC64_TPREL16_LO:
+    case R_PPC64_TPREL16_LO_DS:
     case R_PPC64_TLS:
     case R_PPC64_TLSGD:
     case R_PPC64_TLSLD:
@@ -518,15 +540,16 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(local_thunk));
   static_assert(E::thunk_size == sizeof(local_thunk_power10));
 
-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
   u64 TOC = ctx.extra.TOC->value;
 
-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
 
+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
       if (ctx.extra.is_power10) {
         memcpy(buf, plt_thunk_power10, E::thunk_size);
         write34(buf + 8, got - P - 8);
@@ -536,7 +559,7 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
         *(ul32 *)(buf + 12) |= lo(got - TOC);
       }
     } else {
-      u64 S = sym->get_addr(ctx);
+      u64 S = sym.get_addr(ctx);
       if (ctx.extra.is_power10) {
         memcpy(buf, local_thunk_power10, E::thunk_size);
         write34(buf + 8, S - P - 8);
@@ -546,9 +569,6 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
         *(ul32 *)(buf + 12) |= lo(S - TOC);
       }
     }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
   }
 }
 
@@ -658,3 +678,5 @@ u64 get_eflags(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-riscv.cc

@@ -252,7 +252,7 @@ find_paired_reloc(Context<E> &ctx, InputSection<E> &isec,
 //   ld    t0, 0(t0)  # R_RISCV_PCREL_LO12_I(.L0), R_RISCV_RELAX
 static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
                              std::span<const ElfRel<E>> rels, i64 i) {
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
   return i + 3 < rels.size() &&
          rels[i].r_type == R_RISCV_GOT_HI20 &&
          rels[i + 1].r_type == R_RISCV_RELAX &&
@@ -265,13 +265,13 @@ static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE || rel.r_type == R_RISCV_RELAX)
       continue;
 
@@ -288,6 +288,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -301,10 +304,20 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto utype = [&](i64 val) {
+      check(val, -(1LL << 31) - 0x800, (1LL << 31) - 0x800);
+      write_utype(loc, val);
+    };
+
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_RISCV_32:
-      assert(E::is_64);
-      *(U32<E> *)loc = S + A;
+      if (E::is_64)
+        *(U32<E> *)loc = S + A;
       break;
     case R_RISCV_64:
       break;
@@ -321,29 +334,25 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       i64 val = S + A - P;
       i64 rd = get_rd(buf + rel.r_offset + 4);
 
-      // Calling an undefined weak symbol does not make sense.
-      // We make such call into an infinite loop. This should
-      // help debugging of a faulty program.
-      if (sym.esym().is_undef_weak())
-        val = 0;
-
       if (removed_bytes == 4) {
         // auipc + jalr -> jal
         *(ul32 *)loc = (rd << 7) | 0b1101111;
         write_jtype(loc, val);
+        rewrite(i, R_RISCV_JAL);
       } else if (removed_bytes == 6 && rd == 0) {
         // auipc + jalr -> c.j
         *(ul16 *)loc = 0b101'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else if (removed_bytes == 6 && rd == 1) {
         // auipc + jalr -> c.jal
         assert(!E::is_64);
         *(ul16 *)loc = 0b001'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else {
         assert(removed_bytes == 0);
-        check(val, -(1LL << 31), 1LL << 31);
-        write_utype(loc, val);
+        utype(val);
         write_itype(loc + 4, val);
       }
       break;
@@ -359,11 +368,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         // c.li <rd>, val
         *(ul16 *)loc = 0b010'0'00000'00000'01 | (rd << 7);
         write_citype(loc, sym.get_addr(ctx));
+        // The value is materialized directly, so neither this nor the paired
+        // PCREL_LO12 (the load) needs a relocation anymore.
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else if (removed_bytes == 4) {
         // addi <rd>, zero, val
         *(ul32 *)loc = 0b0010011 | (rd << 7);
         write_itype(loc, sym.get_addr(ctx));
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else {
         assert(removed_bytes == 0);
@@ -372,26 +387,26 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         if (ctx.arg.relax && sym.is_pcrel_linktime_const(ctx) &&
             is_got_load_pair(ctx, *this, rels, i) && is_int(val, 32)) {
           // auipc <rd>, %hi20(val)
-          write_utype(loc, val);
+          utype(val);
 
           // addi <rd>, <rd>, %lo12(val)
           *(ul32 *)(loc + 4) = 0b0010011 | (rd << 15) | (rd << 7);
           write_itype(loc + 4, val);
           i += 3;
         } else {
-          write_utype(loc, G + GOT + A - P);
+          utype(G + GOT + A - P);
         }
       }
       break;
     }
     case R_RISCV_TLS_GOT_HI20:
-      write_utype(loc, sym.get_gottp_addr(ctx) + A - P);
+      utype(sym.get_gottp_addr(ctx) + A - P);
       break;
     case R_RISCV_TLS_GD_HI20:
-      write_utype(loc, sym.get_tlsgd_addr(ctx) + A - P);
+      utype(sym.get_tlsgd_addr(ctx) + A - P);
       break;
     case R_RISCV_PCREL_HI20:
-      write_utype(loc, S + A - P);
+      utype(S + A - P);
       break;
     case R_RISCV_PCREL_LO12_I:
     case R_RISCV_PCREL_LO12_S: {
@@ -423,15 +438,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     }
     case R_RISCV_HI20:
+      // lui (+ addi) => an instruction holding a link-time constant. The lui
+      // may be compressed to c.lui (removed 2 bytes) or deleted outright
+      // (removed 4 bytes); either way it no longer needs a relocation.
       if (removed_bytes == 2) {
         // Rewrite LUI with C.LUI
         i64 rd = get_rd(buf + rel.r_offset);
         *(ul16 *)loc = 0b011'0'00000'00000'01 | (rd << 7);
         write_citype(loc, (S + A + 0x800) >> 12);
       } else if (removed_bytes == 0) {
-        check(S + A, -(1LL << 31), 1LL << 31);
-        write_utype(loc, S + A);
+        utype(S + A);
       }
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_LO12_I:
     case R_RISCV_LO12_S:
@@ -448,14 +467,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_RISCV_TPREL_HI20:
       assert(removed_bytes == 0 || removed_bytes == 4);
+      // lui + add => deleted; the variable is accessed relative to tp directly.
       if (removed_bytes == 0)
-        write_utype(loc, S + A - ctx.tp_addr);
+        utype(S + A - ctx.tp_addr);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_ADD:
       // This relocation just annotates an ADD instruction that can be
       // removed when a TPREL is relaxed. No value is needed to be
       // written.
       assert(removed_bytes == 0 || removed_bytes == 4);
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_LO12_I:
     case R_RISCV_TPREL_LO12_S: {
@@ -473,7 +497,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
     case R_RISCV_TLSDESC_HI20:
       // RISC-V TLSDESC uses the following code sequence to materialize
-      // a TP-relative address in x0.
+      // a TP-relative address in a0.
       //
       //   .L0:
       //   auipc  tX, 0
@@ -510,18 +534,25 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       // If the code-shrinking relaxation is disabled, we may leave
       // original useless instructions instead of deleting them, but we
       // accept that because relaxations are enabled by default.
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
-        write_utype(loc, sym.get_tlsdesc_addr(ctx) + A - P);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0)
+          utype(sym.get_tlsdesc_addr(ctx) + A - P);
+      } else {
+        rewrite(i, R_NONE);
+      }
       break;
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12:
     case R_RISCV_TLSDESC_CALL: {
-      if (removed_bytes == 4)
-        break;
-
       const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
       Symbol<E> &sym2 = *file.symbols[rel2.r_sym];
 
+      if (!sym2.has_tlsdesc(ctx))
+        rewrite(i, R_NONE);
+
+      if (removed_bytes == 4)
+        break;
+
       u64 S = sym2.get_addr(ctx);
       u64 A = rel2.r_addend;
       u64 P = get_addr() + rel2.r_offset - get_r_delta(*this, rel2.r_offset);
@@ -538,10 +569,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           write_itype(loc, sym2.get_tlsdesc_addr(ctx) + A - P);
         } else if (sym2.has_gottp(ctx)) {
           *(ul32 *)loc = 0x517; // auipc a0,<hi20>
-          write_utype(loc, sym2.get_gottp_addr(ctx) + A - P);
+          utype(sym2.get_gottp_addr(ctx) + A - P);
         } else {
           *(ul32 *)loc = 0x537; // lui a0,<hi20>
-          write_utype(loc, S + A - ctx.tp_addr);
+          utype(S + A - ctx.tp_addr);
         }
         break;
       case R_RISCV_TLSDESC_CALL:
@@ -632,6 +663,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_RISCV_32_PCREL:
       *(U32<E> *)loc = S + A - P;
       break;
+    case R_RISCV_GOT32_PCREL:
+      *(U32<E> *)loc = G + GOT + A - P;
+      break;
     case R_RISCV_SET_ULEB128:
       overwrite_uleb(loc, S + A);
       break;
@@ -646,12 +680,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -673,6 +704,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U64<E> *)loc = S + A;
       break;
+    case R_RISCV_TLS_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
+    case R_RISCV_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     case R_RISCV_ADD8:
       *loc += S + A;
       break;
@@ -723,7 +766,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -757,6 +800,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_PLT;
       break;
     case R_RISCV_GOT_HI20:
+    case R_RISCV_GOT32_PCREL:
       sym.flags |= NEEDS_GOT;
       break;
     case R_RISCV_TLS_GOT_HI20:
@@ -773,9 +817,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       scan_pcrel(ctx, sym, rel);
       break;
     case R_RISCV_TPREL_HI20:
-    case R_RISCV_TPREL_LO12_I:
-    case R_RISCV_TPREL_LO12_S:
-    case R_RISCV_TPREL_ADD:
       check_tlsle(ctx, sym, rel);
       break;
     case R_RISCV_64:
@@ -785,6 +826,9 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_RISCV_PCREL_LO12_S:
     case R_RISCV_LO12_I:
     case R_RISCV_LO12_S:
+    case R_RISCV_TPREL_LO12_I:
+    case R_RISCV_TPREL_LO12_S:
+    case R_RISCV_TPREL_ADD:
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12:
     case R_RISCV_TLSDESC_CALL:
@@ -845,7 +889,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   // True if we can use 2-byte instructions. This is usually true on
   // Unix because RV64GC is generally considered the baseline hardware.

src/arch-s390x.cc

@@ -35,6 +35,8 @@
 //
 // https://github.com/IBM/s390x-abi/releases/download/v1.6.1/lzsabi_s390x.pdf
 
+#if MOLD_S390X
+
 #include "mold.h"
 
 namespace mold {
@@ -48,18 +50,24 @@ static void write_mid20(u8 *loc, u64 val) {
 template <>
 void write_plt_header(Context<E> &ctx, u8 *buf) {
   static u8 insn[] = {
-    // Compute PLT_INDEX
+    // Compute the offset into .rela.plt. This is equivalent to
+    // (%r0 - %r1 - 48 - 14) * 3/2 where %r0 is the PLT entry address
+    // plus 14, %r1 is the start address of .plt, and 48 is the size
+    // of this PLT header. We multiply by 3/2 because each PLT entry
+    // is 16 bytes, whereas each .rela.plt entry is 24 bytes.
     0xb9, 0x09, 0x00, 0x01,             // sgr   %r0, %r1
     0xa7, 0x0b, 0xff, 0xc2,             // aghi  %r0, -62
     0xeb, 0x10, 0x00, 0x01, 0x00, 0x0c, // srlg  %r1, %r0, 1
     0xb9, 0x08, 0x00, 0x01,             // agr   %r0, %r1
+    // Store the computed value to 56(%r15) and .got.plt[1] to 48(%15)
+    // where %r15 is the stack pointer.
     0xe3, 0x00, 0xf0, 0x38, 0x00, 0x24, // stg   %r0, 56(%r15)
-    // Branch to _dl_runtime_resolve
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_OFFSET
     0xd2, 0x07, 0xf0, 0x30, 0x10, 0x08, // mvc   48(8, %r15), 8(%r1)
+    // Branch to _dl_runtime_resolve
     0xe3, 0x10, 0x10, 0x10, 0x00, 0x04, // lg    %r1, 16(%r1)
     0x07, 0xf1,                         // br    %r1
-    0x07, 0x00, 0x07, 0x00,             // nopr; nopr
+    0x00, 0x00, 0x00, 0x00,             // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -73,7 +81,7 @@ void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_ENTRY_OFFSET
     0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
     0x0d, 0x01,                         // basr  %r0, %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -86,7 +94,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOT_ENTRY_OFFSET
     0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
     0x07, 0xf1,                         // br    %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -114,14 +122,17 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -254,6 +265,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             is_int(val, 33)) {
           *(ub16 *)(loc - 2) = 0xc000 | (op & 0x00f0);
           *(ub32 *)loc = val >> 1;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_390_PC32DBL;
           break;
         }
       }
@@ -334,12 +347,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -375,7 +385,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -483,3 +493,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/arch-sh4.cc

@@ -47,9 +47,10 @@
 //    with a piece of code to store the address of .got to %r12.
 //    We can use the register in our PLT for position-independent output.
 //
-//  - Even though it uses the RELA-type relocations, relocation addends
-//    are stored not to the r_addend field but to the relocated section
-//    contents for some reason. Therefore, it's effectively REL.
+//  - Even though it uses RELA-type relocations, object files store
+//    addends not in the r_addend field but in the relocated section
+//    contents. Dynamic relocations, however, follow the usual RELA
+//    convention.
 //
 //  - It looks like the ecosystem has bit-rotted. Some tests, especially
 //    one using C++ exceptions, don't pass even with GNU ld.
@@ -66,8 +67,8 @@ namespace mold {
 
 using E = MOLD_TARGET;
 
-// Even though SH-4 uses RELA-type relocations, addends are stored to
-// relocated places for some reason.
+// Even though SH-4 uses RELA-type relocations, addends are stored in
+// the relocated places for some reason.
 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   switch (rel.r_type) {
@@ -137,7 +138,7 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
       0x5221, //    mov.l   @(4, r2), r2
       0x402b, //    jmp     @r0
       0xe000, //    mov     #0, r0
-      0x0009, //    nop
+      0xfffd, //    (illegal)
       0, 0,   // 1: .long GOTPLT
     };
 
@@ -151,32 +152,36 @@ template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   if (ctx.arg.pic) {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x00ce, //    mov.l   @(r0, r12), r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   } else {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x6002, //    mov.l   @r0, r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx);
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx);
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   }
 }
 
@@ -238,6 +243,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -285,12 +293,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -309,11 +314,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U32<E> *)loc = S + A;
       break;
+    case R_SH_TLS_LDO_32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-sparc64.cc

@@ -53,6 +53,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/sparc.pdf
 
+#if MOLD_SPARC64
+
 #include "mold.h"
 
 namespace mold {
@@ -76,26 +78,77 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
   memset(buf, 0, E::plt_hdr_size);
 }
 
+// Returns the byte offset within .plt of the data pointer for a large SPARC
+// PLT entry. See write_plt_entry below for the block layout this assumes.
+i64 sparc_plt_ptr_offset(Context<E> &ctx, i64 pltidx) {
+  i64 i = pltidx - sparc_num_small_plt;
+  i64 block = i / 160;
+  i64 nlarge = (i64)ctx.plt->symbols.size() - sparc_num_small_plt;
+  i64 nstub = std::min<i64>(160, nlarge - block * 160);
+  return 0x100000 + block * 5120 + nstub * 24 + (i % 160) * 8;
+}
+
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  static ub32 insn[] = {
-    0x0300'0000, // sethi (. - .PLT0), %g1
-    0x3068'0000, // ba,a  %xcc, .PLT1
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-  };
+  // SPARC uses two PLT entry formats. A "small" entry branches directly to
+  // the resolver stub (.PLT1) with a BPcc instruction whose reach is only
+  // ±1 MiB. Once the PLT grows past that (0x100000 bytes), we switch to a
+  // "large" format: rather than branching to the resolver, a large entry
+  // loads a 64-bit value from a nearby data pointer and jumps to (that value
+  // + its own address).
+  //
+  // Large entries are grouped into blocks of 160: each block is 160 code
+  // stubs followed by 160 8-byte data pointers, one per stub. The stubs must
+  // sit on a fixed grid, 24 bytes apart, because that is how the loader finds
+  // them: on a lazy bind the stub jumps to the resolver (.PLT0) with only its
+  // own address, and the loader derives which symbol to resolve from that
+  // address. Nothing may sit between two stubs, so each stub's pointer lives
+  // in the block's pointer region, which it reaches with a signed 13-bit ldx
+  // offset (see to_plt_offset). This layout is dictated by the loader; we
+  // cannot rearrange or simplify it.
+  i64 idx = sym.get_plt_idx(ctx);
 
-  u64 plt0 = ctx.plt->shdr.sh_addr;
-  u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
-  u64 entry = sym.get_plt_addr(ctx);
+  if (idx < sparc_num_small_plt) {
+    static ub32 insn[] = {
+      0x0300'0000, // sethi (. - .PLT0), %g1
+      0x3068'0000, // ba,a  %xcc, .PLT1
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+    };
 
-  memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)buf |= bits(entry - plt0, 21, 0);
-  *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+    u64 plt0 = ctx.plt->shdr.sh_addr;
+    u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
+    u64 entry = sym.get_plt_addr(ctx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)buf |= bits(entry - plt0, 21, 0);
+    *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+  } else {
+    static ub32 insn[] = {
+      0x8a10'000f, // mov  %o7, %g5
+      0x4000'0002, // call . + 8
+      0x0100'0000, // nop
+      0xc25b'e000, // ldx  [ %o7 + .ptr ], %g1
+      0x83c3'c001, // jmpl %o7 + %g1, %g1
+      0x9e10'0005, // mov  %g5, %o7
+    };
+
+    u64 plt = ctx.plt->shdr.sh_addr;
+    u64 call = sym.get_plt_addr(ctx) + 4;
+    i64 ptroff = sparc_plt_ptr_offset(ctx, idx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)(buf + 12) |= bits(plt + ptroff - call, 12, 0);
+
+    // The data pointer initially holds (.PLT0 - call) so that the first call
+    // jumps to .PLT0, where the loader's lazy resolver lives. The resolver
+    // later overwrites it with (target - call).
+    *(ub64 *)(ctx.buf + ctx.plt->shdr.sh_offset + ptroff) = plt - call;
+  }
 }
 
 template <>
@@ -139,12 +192,17 @@ template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
 
-  for (i64 i = 0; i < rels.size(); i++) {
+  // We iterate over relocations in reverse order so that it is easy
+  // to swap instructions for R_SPARC_TLS_GD_CALL.
+  for (i64 i = (i64)rels.size() - 1; i >= 0; i--) {
     const ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -157,6 +215,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto rs1 = [&] { return *(ub32 *)loc & (0b11111 << 14); };
+    auto rs2 = [&] { return *(ub32 *)loc & 0b11111; };
+    auto rd  = [&] { return *(ub32 *)loc & (0b11111 << 25); };
+
     switch (rel.r_type) {
     case R_SPARC_5:
       check(S + A, 0, 1 << 5);
@@ -172,7 +234,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_SPARC_8:
       check(S + A, 0, 1 << 8);
-      *(u8 *)loc = S + A;
+      *loc = S + A;
       break;
     case R_SPARC_10:
       check(S + A, 0, 1 << 10);
@@ -206,13 +268,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ub32 *)loc = S + A;
       break;
     case R_SPARC_PLT64:
-    case R_SPARC_UA64:
     case R_SPARC_REGISTER:
       *(ub64 *)loc = S + A;
       break;
     case R_SPARC_DISP8:
       check(S + A - P, -(1 << 7), 1 << 7);
-      *(u8 *)loc = S + A - P;
+      *loc = S + A - P;
       break;
     case R_SPARC_DISP16:
       check(S + A - P, -(1 << 15), 1 << 15);
@@ -297,12 +358,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_SPARC_GOTDATA_OP:
       if (sym.is_absolute()) {
-        // ldx [ %g2 + %g1 ], %g1  →  nop
-        *(ub32 *)loc = 0x0100'0000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  mov %rs2, %rd
+        *(ub32 *)loc = 0x8010'0000 | rs2() | rd();
       } else if (sym.is_pcrel_linktime_const(ctx)) {
-        // ldx [ %g2 + %g1 ], %g1  →  add %g2, %g1, %g1
-        *(ub32 *)loc &= 0b00'11111'000000'11111'1'11111111'11111;
-        *(ub32 *)loc |= 0b10'00000'000000'00000'0'00000000'00000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  add %rs1, %rs2, %rd
+        *(ub32 *)loc = 0x8000'0000 | rs1() | rs2() | rd();
       }
       break;
     case R_SPARC_PC10:
@@ -357,12 +417,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (sym.has_tlsgd(ctx)) {
         *(ub32 *)loc |= bits(sym.get_tlsgd_addr(ctx) + A - GOT, 9, 0);
       } else if (sym.has_gottp(ctx)) {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8010'2000 | (rd << 25) | (rd << 14); // or  %reg, $0, %reg
+        // add %rs1, %rs2, %rd → or  %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8010'2000 | rs1() | rd();
         *(ub32 *)loc |= bits(sym.get_gottp_addr(ctx) + A - GOT, 9, 0);
       } else {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8018'2000 | (rd << 25) | (rd << 14); // xor %reg, $0, %reg
+        // add %rs1, %rs2, %rd → xor %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8018'2000 | rs1() | rd();
         *(ub32 *)loc |= bits(S + A - ctx.tp_addr, 9, 0) | 0b1'1100'0000'0000;
       }
       break;
@@ -370,11 +430,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (sym.has_tlsgd(ctx)) {
         // do nothing
       } else if (sym.has_gottp(ctx)) {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0xd05d'c000 | rs2; // ldx [ %l7 + %reg ], %o0
+        // add %rs1, %rs2, %rd → ldx [ %rs1 + %rs2 ], %rd
+        *(ub32 *)loc = 0xc058'0000 | rs1() | rs2() | rd();
       } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9001'c000 | rs2; // add %g7, %reg, %o0
+        // add %rs1, %rs2, %rd → add %g7, %rs2, %rd
+        *(ub32 *)loc = 0x8001'c000 | rs2() | rd();
       }
       break;
     case R_SPARC_TLS_GD_CALL:
@@ -382,9 +442,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         u64 addr = ctx.extra.tls_get_addr->get_addr(ctx);
         *(ub32 *)loc |= bits(addr + A - P, 31, 2);
       } else if (sym.has_gottp(ctx)) {
-        *(ub32 *)loc = 0x9001'c008; // add %g7, %o0, %o0
+        // When we rewrite a branch instruction with a non-branch one,
+        // we need to swap the instruction and the following one so that
+        // the original execution order, which is inverted due to the
+        // branch delay slot, is preserved.
+        //
+        // Since we apply relocations from the end to the beginning,
+        // the instruction at loc + 4 is already complete.
+        memcpy(loc, loc + 4, 4);
+        *(ub32 *)(loc + 4) = 0x9001'c008; // add %g7, %o0, %o0
       } else {
-        *(ub32 *)loc = 0x0100'0000; // nop
+        *(ub32 *)loc = 0x0100'0000; // call → nop
       }
       break;
     case R_SPARC_TLS_LDM_HI22:
@@ -400,12 +468,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         *(ub32 *)loc |= bits(ctx.tp_addr - ctx.tls_begin, 9, 0);
       break;
     case R_SPARC_TLS_LDM_ADD:
-      if (ctx.got->has_tlsld(ctx)) {
-        // do nothing
-      } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9021'c000 | rs2; // sub %g7, %reg, %o0
-      }
+      if (!ctx.got->has_tlsld(ctx))
+        *(ub32 *)loc = 0x8021'c000 | rs2() | rd(); // sub %g7, %rs2, %rd
       break;
     case R_SPARC_TLS_LDM_CALL:
       if (ctx.got->has_tlsld(ctx)) {
@@ -437,6 +501,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ub32 *)loc = sym.esym().st_size + A;
       break;
     case R_SPARC_64:
+    case R_SPARC_UA64:
     case R_SPARC_TLS_LDO_ADD:
     case R_SPARC_TLS_IE_LD:
     case R_SPARC_TLS_IE_LDX:
@@ -450,12 +515,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -493,13 +555,14 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>
 void InputSection<E>::scan_relocations(Context<E> &ctx) {
   assert(shdr().sh_flags & SHF_ALLOC);
   std::span<const ElfRel<E>> rels = get_rels(ctx);
+  bool needs_tlsgd = false;
 
   // Scan relocations
   for (i64 i = 0; i < rels.size(); i++) {
@@ -526,7 +589,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_SPARC_REGISTER:
     case R_SPARC_UA16:
     case R_SPARC_UA32:
-    case R_SPARC_UA64:
     case R_SPARC_PC_HM10:
     case R_SPARC_OLO10:
     case R_SPARC_LOX10:
@@ -584,6 +646,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
         sym.flags |= NEEDS_GOTTP;
       } else {
         sym.flags |= NEEDS_TLSGD;
+        needs_tlsgd = true;
       }
       break;
     case R_SPARC_TLS_LDM_HI22:
@@ -597,23 +660,21 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_SPARC_TLS_IE_HI22:
       sym.flags |= NEEDS_GOTTP;
       break;
-    case R_SPARC_TLS_GD_CALL:
-    case R_SPARC_TLS_LDM_CALL:
-      if (Symbol<E> *sym = ctx.extra.tls_get_addr; sym->is_imported)
-        sym->flags |= NEEDS_PLT;
-      break;
     case R_SPARC_TLS_LE_HIX22:
     case R_SPARC_TLS_LE_LOX10:
       check_tlsle(ctx, sym, rel);
       break;
     case R_SPARC_64:
+    case R_SPARC_UA64:
     case R_SPARC_GOTDATA_OP_LOX10:
     case R_SPARC_GOTDATA_OP:
     case R_SPARC_GOTDATA_LOX10:
     case R_SPARC_TLS_GD_LO10:
     case R_SPARC_TLS_GD_ADD:
+    case R_SPARC_TLS_GD_CALL:
     case R_SPARC_TLS_LDM_LO10:
     case R_SPARC_TLS_LDM_ADD:
+    case R_SPARC_TLS_LDM_CALL:
     case R_SPARC_TLS_LDO_HIX22:
     case R_SPARC_TLS_LDO_LOX10:
     case R_SPARC_TLS_LDO_ADD:
@@ -627,6 +688,14 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       Error(ctx) << *this << ": unknown relocation: " << rel;
     }
   }
+
+  // TLS_GD_CALL and TLS_LDM_CALL relocations implicitly refer to
+  // __tls_get_addr, which may be dynamically linked from libc.so.
+  Symbol<E> &sym = *ctx.extra.tls_get_addr;
+  if (sym.is_imported && (needs_tlsgd || ctx.needs_tlsld))
+    sym.flags |= NEEDS_PLT;
 }
 
 } // namespace mold
+
+#endif

src/arch-x86-64.cc

@@ -25,8 +25,9 @@
 //
 // https://gitlab.com/x86-psABIs/x86-64-ABI
 
-#include "mold.h"
+#if MOLD_X86_64
 
+#include "mold.h"
 #include <tbb/parallel_for_each.h>
 
 namespace mold {
@@ -432,14 +433,17 @@ static void relax_ld_to_le(u8 *loc, const ElfRel<E> &rel, i64 tls_size) {
 // scan_relocations().
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -527,6 +531,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           loc[-2] = insn >> 8;
           loc[-1] = insn;
           *(ul32 *)loc = S + A - P;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_X86_64_PC32;
           break;
         }
       }
@@ -656,12 +662,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 // scan_relocations.
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -740,7 +743,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 // Linker has to create data structures in an output file to apply
@@ -760,7 +763,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -906,10 +909,14 @@ void rewrite_endbr(Context<E> &ctx) {
     // If isec has an endbr64 at a given offset, copy that instruction to
     // the output buffer, possibly overwriting a nop written in the above
     // loop.
+    i64 size = 0;
+    if (isec)
+      size = isec->get_contents().size();
+
     if (isec && isec->output_section &&
         (isec->shdr().sh_flags & SHF_EXECINSTR) &&
-        0 <= offset && offset <= isec->contents.size() - 4 &&
-        memcmp(isec->contents.data() + offset, endbr64, 4) == 0)
+        0 <= offset && offset <= size - 4 &&
+        memcmp(isec->contents + offset, endbr64, 4) == 0)
       memcpy(ctx.buf + isec->output_section->shdr.sh_offset + isec->offset + offset,
              endbr64, 4);
   };
@@ -917,7 +924,7 @@ void rewrite_endbr(Context<E> &ctx) {
   // Write back endbr64 instructions if they are referred to by address-taking
   // relocations.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC)) {
         for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
           if (!is_func_call_rel(rel)) {
@@ -950,3 +957,5 @@ void rewrite_endbr(Context<E> &ctx) {
 }
 
 } // namespace mold
+
+#endif

src/archive-file.cc

@@ -23,12 +23,11 @@
 // see the contents of libc.a by running `ar t
 // /usr/lib/x86_64-linux-gnu/libc.a`.
 
-#pragma once
-
-#include "common.h"
+#include "mold.h"
 
 namespace mold {
 
+namespace {
 struct ArHdr {
   char ar_name[16];
   char ar_date[12];
@@ -74,10 +73,11 @@ struct ArHdr {
     return {ar_name, sizeof(ar_name)};
   }
 };
+}
 
-template <typename Context>
+template <typename E>
 std::vector<MappedFile *>
-read_thin_archive_members(Context &ctx, MappedFile *mf) {
+read_thin_archive_members(Context<E> &ctx, MappedFile *mf) {
   u8 *begin = mf->data;
   u8 *data = begin + 8;
   std::vector<MappedFile *> vec;
@@ -123,8 +123,9 @@ read_thin_archive_members(Context &ctx, MappedFile *mf) {
   return vec;
 }
 
-template <typename Context>
-std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &ctx, MappedFile *mf) {
   u8 *begin = mf->data;
   u8 *data = begin + 8;
   std::vector<MappedFile *> vec;
@@ -161,13 +162,27 @@ std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf)
   return vec;
 }
 
-template <typename Context>
-std::vector<MappedFile *> read_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_archive_members(Context<E> &ctx, MappedFile *mf) {
   std::string_view str = mf->get_contents();
+
   if (str.starts_with("!<arch>\n"))
     return read_fat_archive_members(ctx, mf);
+
   assert(str.starts_with("!<thin>\n"));
   return read_thin_archive_members(ctx, mf);
 }
 
+using E = MOLD_TARGET;
+
+template std::vector<MappedFile *>
+read_thin_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_archive_members(Context<E> &, MappedFile *);
+
 } // namespace mold

src/config.cc

@@ -1,12 +0,0 @@
-#include "mold.h"
-
-namespace mold {
-
-std::string get_mold_version() {
-  if (mold_git_hash.empty())
-    return "mold "s + MOLD_VERSION + " (compatible with GNU ld)";
-  return "mold "s + MOLD_VERSION + " (" + mold_git_hash +
-         "; compatible with GNU ld)";
-}
-
-} // namespace mold

src/elf.cc

@@ -1,11 +1,12 @@
 #include "elf.h"
+#include <sstream>
 
 namespace mold {
 
 static std::string unknown_type(u32 r_type) {
-  char buf[50];
-  snprintf(buf, sizeof(buf), "unknown (0x%x)", r_type);
-  return buf;
+  std::stringstream ss;
+  ss << "unknown (0x" << std::hex << r_type << ")";
+  return ss.str();
 }
 
 #define CASE(x) case x: return #x
@@ -160,6 +161,7 @@ std::string rel_to_string<ARM64LE>(u32 r_type) {
   CASE(R_AARCH64_LD64_GOT_LO12_NC);
   CASE(R_AARCH64_LD64_GOTPAGE_LO15);
   CASE(R_AARCH64_PLT32);
+  CASE(R_AARCH64_GOTPCREL32);
   CASE(R_AARCH64_TLSGD_ADR_PREL21);
   CASE(R_AARCH64_TLSGD_ADR_PAGE21);
   CASE(R_AARCH64_TLSGD_ADD_LO12_NC);
@@ -232,7 +234,7 @@ std::string rel_to_string<ARM64BE>(u32 r_type) {
 }
 
 template <>
-std::string rel_to_string<ARM32>(u32 r_type) {
+std::string rel_to_string<ARM32LE>(u32 r_type) {
   switch (r_type) {
   CASE(R_ARM_NONE);
   CASE(R_ARM_PC24);
@@ -373,6 +375,11 @@ std::string rel_to_string<ARM32>(u32 r_type) {
   return unknown_type(r_type);
 }
 
+template <>
+std::string rel_to_string<ARM32BE>(u32 r_type) {
+  return rel_to_string<ARM32LE>(r_type);
+}
+
 template <>
 std::string rel_to_string<RV64LE>(u32 r_type) {
   switch (r_type) {

src/elf.h

@@ -13,7 +13,8 @@ struct X86_64;
 struct I386;
 struct ARM64LE;
 struct ARM64BE;
-struct ARM32;
+struct ARM32LE;
+struct ARM32BE;
 struct RV64LE;
 struct RV64BE;
 struct RV32LE;
@@ -80,7 +81,11 @@ enum : u32 {
   SHT_RELR = 19,
   SHT_CREL = 0x40000014,
   SHT_LOOS = 0x60000000,
+  SHT_ANDROID_REL = 0x60000001,
+  SHT_ANDROID_RELA = 0x60000002,
   SHT_LLVM_ADDRSIG = 0x6fff4c03,
+  SHT_ANDROID_RELR = 0x6fffff00,
+  SHT_GNU_SFRAME = 0x6ffffff4,
   SHT_GNU_HASH = 0x6ffffff6,
   SHT_GNU_VERDEF = 0x6ffffffd,
   SHT_GNU_VERNEED = 0x6ffffffe,
@@ -165,6 +170,7 @@ enum : u32 {
   VER_NDX_GLOBAL = 1,
   VER_NDX_LAST_RESERVED = 1,
   VER_NDX_UNSPECIFIED = 0xffff,
+  VERSYM_HIDDEN = 0x8000,
 };
 
 enum : u32 {
@@ -173,10 +179,6 @@ enum : u32 {
   VER_FLG_INFO = 4,
 };
 
-enum : u32 {
-  VERSYM_HIDDEN = 0x8000,
-};
-
 enum : u32 {
   PT_NULL = 0,
   PT_LOAD = 1,
@@ -190,6 +192,7 @@ enum : u32 {
   PT_GNU_STACK = 0x6474e551,
   PT_GNU_RELRO = 0x6474e552,
   PT_GNU_PROPERTY = 0x6474e553,
+  PT_GNU_SFRAME = 0x6474e554,
   PT_OPENBSD_RANDOMIZE = 0x65a3dbe6,
   PT_ARM_EXIDX = 0x70000001,
   PT_RISCV_ATTRIBUTES = 0x70000003,
@@ -284,6 +287,13 @@ enum : u32 {
   DT_RELRSZ = 35,
   DT_RELR = 36,
   DT_RELRENT = 37,
+  DT_ANDROID_REL = 0x6000000f,
+  DT_ANDROID_RELSZ = 0x60000010,
+  DT_ANDROID_RELA = 0x60000011,
+  DT_ANDROID_RELASZ = 0x60000012,
+  DT_ANDROID_RELR = 0x6fffe000,
+  DT_ANDROID_RELRSZ = 0x6fffe001,
+  DT_ANDROID_RELRENT = 0x6fffe003,
   DT_GNU_HASH = 0x6ffffef5,
   DT_DEPAUDIT = 0x6ffffefb,
   DT_AUDIT = 0x6ffffefc,
@@ -352,14 +362,20 @@ enum : u32 {
 };
 
 enum : u32 {
+  ELFCOMPRESS_NONE = 0,
   ELFCOMPRESS_ZLIB = 1,
   ELFCOMPRESS_ZSTD = 2,
 };
 
 enum : u32 {
-  EF_ARM_ABI_FLOAT_SOFT = 0x00000200,
-  EF_ARM_ABI_FLOAT_HARD = 0x00000400,
-  EF_ARM_EABI_VER5 = 0x05000000,
+  EF_ARM_ABI_FLOAT_SOFT = 0x200,
+  EF_ARM_ABI_FLOAT_HARD = 0x400,
+  EF_ARM_BE8 = 0x800000,
+  EF_ARM_EABI_VER5 = 0x5000000,
+};
+
+enum : u32 {
+  EF_PPC64_ABI = 3,
 };
 
 enum : u32 {
@@ -554,6 +570,7 @@ enum : u32 {
   R_AARCH64_LD64_GOT_LO12_NC = 0x138,
   R_AARCH64_LD64_GOTPAGE_LO15 = 0x139,
   R_AARCH64_PLT32 = 0x13a,
+  R_AARCH64_GOTPCREL32 = 0x13b,
   R_AARCH64_TLSGD_ADR_PREL21 = 0x200,
   R_AARCH64_TLSGD_ADR_PAGE21 = 0x201,
   R_AARCH64_TLSGD_ADD_LO12_NC = 0x202,
@@ -804,6 +821,7 @@ enum : u32 {
   R_RISCV_SET8 = 54,
   R_RISCV_SET16 = 55,
   R_RISCV_SET32 = 56,
+  R_RISCV_GOT32_PCREL = 41,
   R_RISCV_32_PCREL = 57,
   R_RISCV_IRELATIVE = 58,
   R_RISCV_PLT32 = 59,
@@ -1488,6 +1506,52 @@ template <typename E> using U64 = std::conditional_t<E::is_le, ul64, ub64>;
 template <typename E> using Word = std::conditional_t<E::is_64, U64<E>, U32<E>>;
 template <typename E> using SWord = std::conditional_t<E::is_64, I64<E>, I32<E>>;
 
+//
+// SFrame is a simple unwind information format used as a lightweight
+// alternative to .eh_frame. A .sframe section consists of a header, an
+// array of Function Descriptor Entries (FDEs) sorted by PC, and a blob
+// of Frame Row Entries (FREs). mold understands SFrame Version 3.
+//
+// https://sourceware.org/binutils/docs/sframe-spec.html
+//
+
+static constexpr u16 SFRAME_MAGIC = 0xdee2;
+
+static constexpr u8 SFRAME_F_FDE_SORTED = 0x1;
+static constexpr u8 SFRAME_F_FRAME_POINTER = 0x2;
+static constexpr u8 SFRAME_F_FDE_FUNC_START_PCREL = 0x4;
+
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_BIG = 1;
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_LITTLE = 2;
+static constexpr u8 SFRAME_ABI_AMD64_ENDIAN_LITTLE = 3;
+static constexpr u8 SFRAME_ABI_S390X_ENDIAN_BIG = 4;
+
+template <typename E>
+struct SFrameHeader {
+  U16<E> magic;
+  u8 version;
+  u8 flags;
+  u8 abi_arch;
+  i8 cfa_fixed_fp_offset;
+  i8 cfa_fixed_ra_offset;
+  u8 auxhdr_len;
+  U32<E> num_fdes;
+  U32<E> num_fres;
+  U32<E> fre_len;
+  U32<E> fdeoff;
+  U32<E> freoff;
+};
+
+// The index part of an SFrame Version 3 FDE. The func_start_offset field
+// is PC-relative (relative to its own address) when the section flag
+// SFRAME_F_FDE_FUNC_START_PCREL is set, which is how mold always emits it.
+template <typename E>
+struct SFrameFdeIdx {
+  I64<E> func_start_offset;
+  U32<E> func_size;
+  U32<E> func_start_fre_off;
+};
+
 template <typename E> requires E::is_64
 struct ElfSym<E> {
   bool is_undef() const { return st_shndx == SHN_UNDEF; }
@@ -1770,47 +1834,21 @@ struct ElfRel<SPARC64> {
   ib64 r_addend;
 };
 
-template <>
-struct ElfRel<SH4LE> {
-  ElfRel() = default;
-
-  // Addend is ignored except for base relocations because even though
-  // SH4 is RELA, r_addend is ignored in most cases and works as if it
-  // were REL.
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_type(type), r_sym(sym), r_addend(sym ? 0 : addend) {}
-
-  ul32 r_offset;
-  u8 r_type;
-  ul24 r_sym;
-  il32 r_addend;
-};
-
-template <>
-struct ElfRel<SH4BE> {
-  ElfRel() = default;
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_sym(sym), r_type(type), r_addend(sym ? 0 : addend) {}
-
-  ub32 r_offset;
-  ub24 r_sym;
-  u8 r_type;
-  ib32 r_addend;
-};
-
 //
 // Machine descriptions
 //
 
 template <typename E> concept supports_ifunc = requires { E::R_IRELATIVE; };
 template <typename E> concept supports_tlsdesc = requires { E::R_TLSDESC; };
+template <typename E> concept supports_sframe = requires { E::sframe_abi; };
 template <typename E> concept needs_thunk = requires { E::thunk_size; };
 
 template <typename E> concept is_x86_64 = std::same_as<E, X86_64>;
 template <typename E> concept is_i386 = std::same_as<E, I386>;
 template <typename E> concept is_arm64le = std::same_as<E, ARM64LE>;
 template <typename E> concept is_arm64be = std::same_as<E, ARM64BE>;
-template <typename E> concept is_arm32 = std::same_as<E, ARM32>;
+template <typename E> concept is_arm32le = std::same_as<E, ARM32LE>;
+template <typename E> concept is_arm32be = std::same_as<E, ARM32BE>;
 template <typename E> concept is_rv64le = std::same_as<E, RV64LE>;
 template <typename E> concept is_rv64be = std::same_as<E, RV64BE>;
 template <typename E> concept is_rv32le = std::same_as<E, RV32LE>;
@@ -1827,6 +1865,7 @@ template <typename E> concept is_loongarch64 = std::same_as<E, LOONGARCH64>;
 template <typename E> concept is_loongarch32 = std::same_as<E, LOONGARCH32>;
 
 template <typename E> concept is_x86 = is_x86_64<E> || is_i386<E>;
+template <typename E> concept is_arm32 = is_arm32le<E> || is_arm32be<E>;
 template <typename E> concept is_arm64 = is_arm64le<E> || is_arm64be<E>;
 template <typename E> concept is_arm = is_arm64<E> || is_arm32<E>;
 template <typename E> concept is_rv64 = is_rv64le<E> || is_rv64be<E>;
@@ -1845,10 +1884,11 @@ struct X86_64 {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_X86_64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AMD64_ENDIAN_LITTLE;
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u8 trap[] = { 0xcc }; // int3
 
   static constexpr u32 R_COPY = R_X86_64_COPY;
   static constexpr u32 R_GLOB_DAT = R_X86_64_GLOB_DAT;
@@ -1860,6 +1900,7 @@ struct X86_64 {
   static constexpr u32 R_TPOFF = R_X86_64_TPOFF64;
   static constexpr u32 R_DTPMOD = R_X86_64_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_X86_64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_X86_64_PC64;
   static constexpr u32 R_FUNCALL[] = { R_X86_64_PLT32, R_X86_64_PLTOFF64 };
 };
 
@@ -1873,7 +1914,7 @@ struct I386 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u8 trap[] = { 0xcc }; // int3
 
   static constexpr u32 R_COPY = R_386_COPY;
   static constexpr u32 R_GLOB_DAT = R_386_GLOB_DAT;
@@ -1895,12 +1936,13 @@ struct ARM64LE {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 65536;
   static constexpr u32 e_machine = EM_AARCH64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_LITTLE;
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk
+  static constexpr u32 thunk_size = 32;
+  static constexpr u8 trap[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk
 
   static constexpr u32 R_COPY = R_AARCH64_COPY;
   static constexpr u32 R_GLOB_DAT = R_AARCH64_GLOB_DAT;
@@ -1912,15 +1954,17 @@ struct ARM64LE {
   static constexpr u32 R_TPOFF = R_AARCH64_TLS_TPREL64;
   static constexpr u32 R_DTPMOD = R_AARCH64_TLS_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_AARCH64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_AARCH64_PREL64;
   static constexpr u32 R_FUNCALL[] = { R_AARCH64_JUMP26, R_AARCH64_CALL26 };
 };
 
 struct ARM64BE : ARM64LE {
   static constexpr std::string_view name = "arm64be";
   static constexpr bool is_le = false;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_BIG;
 };
 
-struct ARM32 {
+struct ARM32LE {
   static constexpr std::string_view name = "arm32";
   static constexpr bool is_64 = false;
   static constexpr bool is_le = true;
@@ -1932,7 +1976,7 @@ struct ARM32 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 16;
   static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0xff, 0xde }; // udf
+  static constexpr u8 trap[] = { 0xff, 0xde }; // udf
 
   static constexpr u32 R_COPY = R_ARM_COPY;
   static constexpr u32 R_GLOB_DAT = R_ARM_GLOB_DAT;
@@ -1950,6 +1994,11 @@ struct ARM32 {
   };
 };
 
+struct ARM32BE : ARM32LE {
+  static constexpr std::string_view name = "arm32be";
+  static constexpr bool is_le = false;
+};
+
 struct RV64LE {
   static constexpr std::string_view name = "riscv64";
   static constexpr bool is_64 = true;
@@ -1960,7 +2009,7 @@ struct RV64LE {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_64;
@@ -1990,7 +2039,7 @@ struct RV32LE {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_32;
@@ -2022,7 +2071,7 @@ struct PPC32 {
   static constexpr u32 pltgot_size = 36;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 36;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
 
   static constexpr u32 R_COPY = R_PPC_COPY;
   static constexpr u32 R_GLOB_DAT = R_PPC_GLOB_DAT;
@@ -2064,7 +2113,7 @@ struct PPC64V1 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 28;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
 };
 
 struct PPC64V2 : PPC64 {
@@ -2075,7 +2124,7 @@ struct PPC64V2 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 24;
-  static constexpr u8 filler[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
+  static constexpr u8 trap[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
 };
 
 struct S390X {
@@ -2085,10 +2134,11 @@ struct S390X {
   static constexpr bool is_rela = true;
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_S390X;
+  static constexpr u8 sframe_abi = SFRAME_ABI_S390X_ENDIAN_BIG;
   static constexpr u32 plt_hdr_size = 48;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x07, 0x00 }; // nop
+  static constexpr u8 trap[] = { 0x00, 0x00 }; // invalid
 
   static constexpr u32 R_COPY = R_390_COPY;
   static constexpr u32 R_GLOB_DAT = R_390_GLOB_DAT;
@@ -2099,6 +2149,7 @@ struct S390X {
   static constexpr u32 R_DTPOFF = R_390_TLS_DTPOFF;
   static constexpr u32 R_TPOFF = R_390_TLS_TPOFF;
   static constexpr u32 R_DTPMOD = R_390_TLS_DTPMOD;
+  static constexpr u32 R_SFRAME = R_390_PC64;
   static constexpr u32 R_FUNCALL[] = { R_390_PLT32DBL };
 };
 
@@ -2112,7 +2163,7 @@ struct SPARC64 {
   static constexpr u32 plt_hdr_size = 128;
   static constexpr u32 plt_size = 32;
   static constexpr u32 pltgot_size = 32;
-  static constexpr u8 filler[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5
+  static constexpr u8 trap[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5
 
   static constexpr u32 R_COPY = R_SPARC_COPY;
   static constexpr u32 R_GLOB_DAT = R_SPARC_GLOB_DAT;
@@ -2136,7 +2187,7 @@ struct M68K {
   static constexpr u32 plt_hdr_size = 18;
   static constexpr u32 plt_size = 14;
   static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0x4a, 0xfc }; // illegal
+  static constexpr u8 trap[] = { 0x4a, 0xfc }; // illegal
 
   static constexpr u32 R_COPY = R_68K_COPY;
   static constexpr u32 R_GLOB_DAT = R_68K_GLOB_DAT;
@@ -2157,9 +2208,9 @@ struct SH4LE {
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_SH;
   static constexpr u32 plt_hdr_size = 16;
-  static constexpr u32 plt_size = 16;
+  static constexpr u32 plt_size = 20;
   static constexpr u32 pltgot_size = 12;
-  static constexpr u8 filler[] = { 0x00, 0x90 }; // nop
+  static constexpr u8 trap[] = { 0xfd, 0xff }; // illegal
 
   static constexpr u32 R_COPY = R_SH_COPY;
   static constexpr u32 R_GLOB_DAT = R_SH_GLOB_DAT;
@@ -2175,7 +2226,7 @@ struct SH4LE {
 struct SH4BE : SH4LE {
   static constexpr std::string_view name = "sh4be";
   static constexpr bool is_le = false;
-  static constexpr u8 filler[] = { 0x90, 0x00 }; // nop
+  static constexpr u8 trap[] = { 0xff, 0xfd }; // illegal
 };
 
 struct LOONGARCH64 {
@@ -2188,7 +2239,7 @@ struct LOONGARCH64 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_64;
@@ -2213,7 +2264,7 @@ struct LOONGARCH32 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_32;

src/entry.cc

@@ -0,0 +1,43 @@
+#include "config.h"
+#include "mold.h"
+#include "mold-git-hash.h"
+
+#if MOLD_USE_MIMALLOC
+# include <mimalloc.h>
+#endif
+
+#if MOLD_USE_SYSTEM_MIMALLOC
+// Including mimalloc-new-delete.h overrides the new/delete operators.
+// We need it only when using mimalloc as a dynamic library.
+// This header should be included in only one source file, so we do
+// it in this file.
+# include <mimalloc-new-delete.h>
+#endif
+
+#ifdef __linux__
+# include <sys/prctl.h>
+#endif
+
+std::string mold::mold_version =
+#ifdef MOLD_GIT_HASH
+  "mold " MOLD_VERSION " (" MOLD_GIT_HASH "; compatible with GNU ld)";
+#else
+  "mold " MOLD_VERSION " (compatible with GNU ld)";
+#endif
+
+int main(int argc, char **argv) {
+#if MOLD_USE_MIMALLOC
+  // Silence mimalloc warnings that users can ignore
+  mi_option_disable(mi_option_verbose);
+  mi_option_disable(mi_option_show_errors);
+#endif
+
+#if defined(__linux__) && defined(PR_SET_THP_DISABLE)
+  // A parent process may have disabled transparent huge pages, and the
+  // flag is inherited. Huge pages make mold considerably faster on
+  // large links, so re-enable them for this process.
+  prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
+#endif
+
+  return mold::mold_main<mold::MOLD_FIRST_TARGET>(argc, argv);
+}

src/error.cc

@@ -0,0 +1,55 @@
+#include "mold.h"
+
+namespace mold {
+
+static std::string_view fatal_mono = "mold: fatal: ";
+static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
+static std::string_view error_mono = "mold: error: ";
+static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
+static std::string_view warning_mono = "mold: warning: ";
+static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
+
+template <typename E>
+Fatal<E>::Fatal(Context<E> &ctx) {
+  out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
+}
+
+template <typename E>
+[[noreturn]] Fatal<E>::~Fatal() {
+  out.emit();
+  cleanup();
+  _exit(1);
+}
+
+template <typename E>
+Error<E>::Error(Context<E> &ctx) {
+  if (ctx.arg.noinhibit_exec) {
+    out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  } else {
+    out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  }
+}
+
+template <typename E>
+Warn<E>::Warn(Context<E> &ctx) {
+  if (ctx.arg.suppress_warnings)
+    return;
+
+  out.emplace(std::cerr);
+
+  if (ctx.arg.fatal_warnings) {
+    *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  } else {
+    *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  }
+}
+
+using E = MOLD_TARGET;
+
+template class Fatal<E>;
+template class Error<E>;
+template class Warn<E>;
+
+} // namespace mold

src/filetype.cc

@@ -1,5 +1,4 @@
 #include "mold.h"
-#include "../lib/archive-file.h"
 
 namespace mold {
 
@@ -14,7 +13,7 @@ static bool is_text_file(MappedFile *mf) {
 }
 
 template <typename E>
-static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
+static bool is_gcc_lto_obj(MappedFile *mf, bool has_gcc_plugin) {
   const char *data = mf->get_contents().data();
   ElfEhdr<E> &ehdr = *(ElfEhdr<E> *)data;
   ElfShdr<E> *sh_begin = (ElfShdr<E> *)(data + ehdr.e_shoff);
@@ -31,7 +30,7 @@ static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
     // the LTO linker plugin is available and falls back as regular
     // objects otherwise. GCC FAT LTO object can be identified by the
     // presence of `.gcc.lto_.symtab` section.
-    if (has_plugin) {
+    if (has_gcc_plugin) {
       std::string_view name = data + shdrs[shstrtab_idx].sh_offset + sec.sh_name;
       if (name.starts_with(".gnu.lto_.symtab."))
         return true;
@@ -69,11 +68,15 @@ static bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
 template <typename E>
 FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
   std::string_view data = mf->get_contents();
-  bool has_plugin = !ctx.arg.plugin.empty();
-
   if (data.empty())
     return FileType::EMPTY;
 
+  // GCC FAT LTO objects can be linked as regular ELF objects. If the active
+  // plugin is LLVM's, treat them as regular objects so that we can fall back
+  // to native code instead of routing them through GCC LTO handling.
+  bool has_gcc_plugin = !ctx.arg.plugin.empty() &&
+                        ctx.arg.plugin.find("LLVMgold.") == std::string_view::npos;
+
   if (data.starts_with("\177ELF")) {
     u8 byte_order = ((ElfEhdr<I386> *)data.data())->e_ident[EI_DATA];
 
@@ -82,10 +85,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
 
       if (ehdr.e_type == ET_REL) {
         if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<I386>(mf, has_plugin))
+          if (is_gcc_lto_obj<I386>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         } else {
-          if (is_gcc_lto_obj<X86_64>(mf, has_plugin))
+          if (is_gcc_lto_obj<X86_64>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         }
         return FileType::ELF_OBJ;
@@ -98,10 +101,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
 
       if (ehdr.e_type == ET_REL) {
         if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<M68K>(mf, has_plugin))
+          if (is_gcc_lto_obj<M68K>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         } else {
-          if (is_gcc_lto_obj<SPARC64>(mf, has_plugin))
+          if (is_gcc_lto_obj<SPARC64>(mf, has_gcc_plugin))
             return FileType::GCC_LTO_OBJ;
         }
         return FileType::ELF_OBJ;
@@ -130,16 +133,16 @@ static std::string_view get_elf_type(u8 *buf) {
   bool is_le = (buf[EI_DATA] == ELFDATA2LSB);
   bool is_64 = (buf[EI_CLASS] == ELFCLASS64);
 
-  auto *ehdr_le = (ElfEhdr<I386> *)buf;
-  auto *ehdr_be = (ElfEhdr<M68K> *)buf;
+  auto *ehdr64_le = (ElfEhdr<ARM64LE> *)buf;
+  auto *ehdr64_be = (ElfEhdr<ARM64BE> *)buf;
 
-  switch (is_le ? ehdr_le->e_machine : ehdr_be->e_machine) {
+  switch (is_le ? ehdr64_le->e_machine : ehdr64_be->e_machine) {
   case EM_386:
     return I386::name;
   case EM_X86_64:
     return X86_64::name;
   case EM_ARM:
-    return ARM32::name;
+    return is_le ? ARM32LE::name : ARM32BE::name;
   case EM_AARCH64:
     return is_le ? ARM64LE::name : ARM64BE::name;
   case EM_RISCV:
@@ -149,7 +152,18 @@ static std::string_view get_elf_type(u8 *buf) {
   case EM_PPC:
     return PPC32::name;
   case EM_PPC64:
-    return is_le ? PPC64V2::name : PPC64V1::name;
+    // ELFv1 is big-endian and ELFv2 is little-endian by convention, but
+    // the correspondence is not a rule; musl for example uses ELFv2 on
+    // big-endian too. We support only the usual combinations, so treat
+    // the others as unrecognizable rather than silently linking them
+    // against the wrong ABI.
+    if (!is_le)
+      if (u32 abi = (ehdr64_be->e_flags & EF_PPC64_ABI); abi == 0 || abi == 1)
+        return PPC64V1::name;
+    if (is_le)
+      if (u32 abi = (ehdr64_le->e_flags & EF_PPC64_ABI); abi == 0 || abi == 2)
+        return PPC64V2::name;
+    return "";
   case EM_S390X:
     return S390X::name;
   case EM_SPARC64:

src/gc-sections.cc

@@ -4,6 +4,8 @@
 
 #include "mold.h"
 
+#include <fstream>
+#include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/parallel_for_each.h>
 
@@ -27,8 +29,30 @@ static bool should_keep(const InputSection<E> &isec) {
          name.starts_with(".ctors") ||
          name.starts_with(".dtors") ||
          name.starts_with(".init") ||
-         name.starts_with(".fini") ||
-         is_c_identifier(name);
+         name.starts_with(".fini");
+}
+
+// Sections whose names are valid C identifiers can be referenced via
+// __start_<name>/__stop_<name> symbols, which the linker synthesizes.
+// Such sections must be kept alive only if such a marker symbol is
+// referenced from a live section. This map lets us find all sections
+// of a given name when we encounter such a reference during marking.
+template <typename E>
+using StartStopMap =
+  tbb::concurrent_unordered_map<std::string_view,
+                                tbb::concurrent_vector<InputSection<E> *>>;
+
+template <typename E>
+static StartStopMap<E> build_start_stop_map(Context<E> &ctx) {
+  StartStopMap<E> map;
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    for (InputSection<E> *isec : file->sections)
+      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC))
+        if (std::string_view name = isec->name();
+            is_c_identifier(name))
+          map[name].push_back(isec);
+  });
+  return map;
 }
 
 template <typename E>
@@ -58,7 +82,7 @@ collect_root_set(Context<E> &ctx) {
 
   // Add sections that are not subject to garbage collection.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (!isec || !isec->is_alive)
         continue;
 
@@ -66,12 +90,13 @@ collect_root_set(Context<E> &ctx) {
       // reduce the amount of non-memory-mapped segments, you should
       // use `strip` command, compile without debug info or use
       // --strip-all linker option.
-      u32 flags = isec->shdr().sh_flags;
-      if (!(flags & SHF_ALLOC))
+      if (!(isec->shdr().sh_flags & SHF_ALLOC)) {
         isec->is_visited = true;
+        continue;
+      }
 
       if (should_keep(*isec))
-        enqueue_section(isec.get());
+        enqueue_section(isec);
     }
   });
 
@@ -94,9 +119,18 @@ collect_root_set(Context<E> &ctx) {
   return rootset;
 }
 
+static std::string_view start_stop_name(std::string_view sym) {
+  if (sym.starts_with("__start_"))
+    return sym.substr(8);
+  if (sym.starts_with("__stop_"))
+    return sym.substr(7);
+  return "";
+}
+
 template <typename E>
-static void visit(Context<E> &ctx, InputSection<E> *isec,
-                  tbb::feeder<InputSection<E> *> &feeder, i64 depth) {
+static void visit_section(Context<E> &ctx, InputSection<E> *isec,
+                          tbb::feeder<InputSection<E> *> &feeder, i64 depth,
+                          const StartStopMap<E> &start_stop_map) {
   assert(isec->is_visited);
 
   // Mark a section alive. For better performacne, we don't call
@@ -104,7 +138,7 @@ static void visit(Context<E> &ctx, InputSection<E> *isec,
   auto mark = [&](InputSection<E> *sec) {
     if (mark_section(sec)) {
       if (depth < 3)
-        visit(ctx, sec, feeder, depth + 1);
+        visit_section(ctx, sec, feeder, depth + 1, start_stop_map);
       else
         feeder.add(sec);
     }
@@ -121,10 +155,38 @@ static void visit(Context<E> &ctx, InputSection<E> *isec,
   for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
     // Symbol can refer to either a section fragment or an input section.
     Symbol<E> &sym = *isec->file.symbols[rel.r_sym];
-    if (SectionFragment<E> *frag = sym.get_frag())
+
+    if (sym.file && sym.file->is_dso) {
+      sym.file->is_reachable.test_and_set();
+      continue;
+    }
+
+    if (SectionFragment<E> *frag = sym.get_frag()) {
       frag->is_alive = true;
-    else
-      mark(sym.get_input_section());
+      continue;
+    }
+
+    mark(sym.get_input_section());
+
+    // A reference to __start_<name> or __stop_<name> keeps every
+    // section named <name> alive, mirroring how those symbols are
+    // defined. A single such reference can keep an enormous number of
+    // sections alive, so we spread the fanout over threads instead
+    // of marking the sections one by one.
+    if (std::string_view name = start_stop_name(sym.name());
+        !name.empty()) {
+      if (auto it = start_stop_map.find(name);
+          it != start_stop_map.end()) {
+        // Mark and visit the sections with a nested parallel loop.
+        // As in mark() below, a section added to a feeder has
+        // already been marked, so a feeder's loop body must visit
+        // it unconditionally.
+        tbb::parallel_for_each(it->second, [&](InputSection<E> *isec) {
+          if (mark_section(isec))
+            visit_section(ctx, isec, feeder, 0, start_stop_map);
+        });
+      }
+    }
   }
 
   if constexpr (is_arm32<E>)
@@ -134,12 +196,13 @@ static void visit(Context<E> &ctx, InputSection<E> *isec,
 // Mark all reachable sections
 template <typename E>
 static void mark(Context<E> &ctx,
-                 tbb::concurrent_vector<InputSection<E> *> &rootset) {
+                 tbb::concurrent_vector<InputSection<E> *> &rootset,
+                 const StartStopMap<E> &start_stop_map) {
   Timer t(ctx, "mark");
 
   tbb::parallel_for_each(rootset, [&](InputSection<E> *isec,
                                       tbb::feeder<InputSection<E> *> &feeder) {
-    visit(ctx, isec, feeder, 0);
+    visit_section(ctx, isec, feeder, 0, start_stop_map);
   });
 }
 
@@ -147,26 +210,65 @@ static void mark(Context<E> &ctx,
 template <typename E>
 static void sweep(Context<E> &ctx) {
   Timer t(ctx, "sweep");
-  static Counter counter("garbage_sections");
 
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+  std::vector<std::vector<InputSection<E> *>> sections(ctx.objs.size());
+
+  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
+    ObjectFile<E> &file = *ctx.objs[i];
+
+    for (InputSection<E> *isec : file.sections) {
       if (isec && isec->is_alive && !isec->is_visited) {
-        if (ctx.arg.print_gc_sections)
-          Out(ctx) << "removing unused section " << *isec;
         isec->kill();
-        counter++;
+        sections[i].push_back(isec);
       }
     }
   });
+
+  std::string &path = ctx.arg.print_gc_sections;
+
+  if (!path.empty()) {
+    std::ostream *out = &std::cout;
+    std::ofstream file;
+
+    if (path != "-") {
+      file.open(path);
+      if (file.fail())
+        Fatal(ctx) << "--print-gc-sections: cannot open " << path << ": "
+                   << errno_string();
+      out = &file;
+    }
+
+    i64 saved_bytes = 0;
+    for (std::span<InputSection<E> *> vec : sections)
+      for (InputSection<E> *isec : vec)
+        *out << "removing unused section " << *isec << '\n';
+
+    *out << "GC saved " << saved_bytes << " bytes\n";
+  }
 }
 
 template <typename E>
 void gc_sections(Context<E> &ctx) {
   Timer t(ctx, "gc");
+
+  for (SharedFile<E> *file : ctx.dsos)
+    if (file->as_needed)
+      file->is_reachable = false;
+
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
   tbb::concurrent_vector<InputSection<E> *> rootset = collect_root_set(ctx);
-  mark(ctx, rootset);
+  StartStopMap<E> start_stop_map = build_start_stop_map(ctx);
+  mark(ctx, rootset, start_stop_map);
   sweep(ctx);
+
+  std::erase_if(ctx.dsos, [](SharedFile<E> *file) { return !file->is_reachable; });
 }
 
 using E = MOLD_TARGET;

src/gdb-index.cc

@@ -58,6 +58,7 @@
 
 #include "mold.h"
 #include <tbb/parallel_for_each.h>
+#include <tbb/parallel_scan.h>
 
 namespace mold {
 
@@ -125,13 +126,6 @@ struct SectionHeader {
   ul32 const_pool_offset = 0;
 };
 
-struct NameType {
-  auto operator<=>(const NameType &) const = default;
-  u64 hash;
-  u8 type;
-  std::string_view name;
-};
-
 struct MapValue {
   u32 gdb_hash = 0;
   Atomic<u32> count;
@@ -139,24 +133,146 @@ struct MapValue {
   u32 type_offset = 0;
 };
 
+struct NameType {
+  NameType(u64 hash, u8 type, const char *name)
+    : hash_and_type((hash << 8) | type), name(name) {}
+
+  u64 get_hash() const {
+    return hash_and_type >> 8;
+  }
+
+  u8 get_type() const {
+    return hash_and_type & 0xff;
+  }
+
+  bool operator==(const NameType &other) const {
+    return hash_and_type == other.hash_and_type && !strcmp(name, other.name);
+  }
+
+  // The low byte stores the type. Hash collisions, including collisions in
+  // the discarded high byte, are resolved by comparing the complete name.
+  u64 hash_and_type;
+  const char *name;
+};
+
+struct IndexedName {
+  MapValue *entry;
+  u32 type_idx;
+  u8 type;
+};
+
+// A name record is read as a NameType and replaced with an IndexedName after
+// the name has been interned. The union avoids retaining both forms for the
+// millions of public names in a large debug link.
+union NameRecord {
+  NameRecord(u64 hash, u8 type, const char *name)
+    : nametype(hash, type, name) {}
+
+  NameType nametype;
+  IndexedName indexed;
+};
+
 struct Compunit {
   DwarfKind kind;
   i64 offset;
   i64 size;
+  i32 file_idx;
   std::vector<std::pair<u64, u64>> ranges;
-  std::vector<NameType> nametypes;
-  std::vector<MapValue *> entries;
+  std::vector<NameRecord> names;
 };
 
-// The hash function for .gdb_index.
-static u32 gdb_hash(std::string_view name) {
+using GdbNameMap = ConcurrentMap<MapValue>;
+
+struct PoolSize {
+  i64 type_bytes = 0;
+  i64 name_bytes = 0;
+};
+
+struct GdbIndexData {
+  std::vector<Compunit> cus;
+  std::unique_ptr<GdbNameMap> map;
+  std::vector<GdbNameMap::Entry *> entries;
+  std::unique_ptr<u8[]> tables;
+  PoolSize pool_size;
+  i64 ht_size = 0;
+};
+
+// GDB index type lists contain fixed-width 32-bit records. Four byte-wise
+// passes are faster than comparison sorting once a list is sufficiently large.
+static void radix_sort(std::span<ul32> values, std::vector<ul32> &scratch) {
+  scratch.resize(values.size());
+  ul32 *src = values.data();
+  ul32 *dst = scratch.data();
+
+  for (i64 shift = 0; shift < 32; shift += 8) {
+    u32 counts[256] = {};
+    for (i64 i = 0; i < values.size(); i++)
+      counts[((u32)src[i] >> shift) & 255]++;
+
+    u32 offsets[256];
+    offsets[0] = 0;
+    for (i64 i = 1; i < 256; i++)
+      offsets[i] = offsets[i - 1] + counts[i - 1];
+
+    for (i64 i = 0; i < values.size(); i++) {
+      u32 val = src[i];
+      dst[offsets[(val >> shift) & 255]++] = val;
+    }
+    std::swap(src, dst);
+  }
+
+  assert(src == values.data());
+}
+
+// GCC can emit the same public name once for each COMDAT group. Remove these
+// duplicates with a local hash table instead of sorting the strings.
+static void dedup_names(Compunit &cu) {
+  if (cu.names.size() < 2)
+    return;
+
+  u64 capacity = bit_ceil(cu.names.size() * 2);
+  std::vector<u32> buckets(capacity, UINT32_MAX);
+  u64 mask = capacity - 1;
+  i64 out = 0;
+
+  for (NameRecord &record : cu.names) {
+    NameType &nt = record.nametype;
+    u64 idx =
+      (nt.get_hash() ^ ((u64)nt.get_type() * 0x9e37'79b9)) & mask;
+    bool duplicate = false;
+
+    while (buckets[idx] != UINT32_MAX) {
+      if (cu.names[buckets[idx]].nametype == nt) {
+        duplicate = true;
+        break;
+      }
+      idx = (idx + 1) & mask;
+    }
+
+    if (duplicate)
+      continue;
+
+    if (&record != &cu.names[out])
+      cu.names[out] = record;
+    buckets[idx] = out++;
+  }
+
+  cu.names.erase(cu.names.begin() + out, cu.names.end());
+}
+
+// Compute the .gdb_index hash and length together when a name is first
+// inserted.
+static u32 initialize_gdb_name(MapValue &value, const char *name) {
   u32 h = 0;
-  for (u8 c : name) {
+  u32 size = 0;
+
+  for (u8 c = *name; c; c = name[++size]) {
     if ('A' <= c && c <= 'Z')
       c = 'a' + c - 'A';
     h = h * 67 + c - 113;
   }
-  return h;
+  value.gdb_hash = h;
+  return size;
 }
 
 template <typename E>
@@ -456,11 +572,8 @@ read_address_ranges(Context<E> &ctx, const Compunit &cu) {
         Fatal(ctx) << "--gdb-index: missing DW_AT_rnglists_base";
 
       u8 *base = buf + rnglists_base;
-      i64 num_offsets = *(U32<E> *)(base - 4);
       Offset *offsets = (Offset *)base;
-
-      for (i64 i = 0; i < num_offsets; i++)
-        read_rnglist_range<E>(vec, base + offsets[i], addrx, low_pc.value);
+      read_rnglist_range<E>(vec, base + offsets[ranges.value], addrx, low_pc.value);
     }
     return vec;
   }
@@ -512,14 +625,15 @@ static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
                             std::vector<Compunit> &cus, ObjectFile<E> &file) {
   using Offset = decltype(hdr.size);
 
+  // Compunits are sorted by offset, so we can use binary search.
   auto get_cu = [&](i64 offset) {
-    for (i64 i = 0; i < cus.size(); i++)
-      if (cus[i].offset == offset)
-        return &cus[i];
-    Fatal(ctx) << file << ": corrupted debug_info_offset";
+    auto it = ranges::lower_bound(cus, offset, {}, &Compunit::offset);
+    if (it == cus.end() || it->offset != offset)
+      Fatal(ctx) << file << ": corrupted debug_info_offset";
+    return &*it;
   };
 
-  Compunit *cu = get_cu(file.debug_info->offset + hdr.debug_info_offset);
+  Compunit *cu = get_cu(hdr.debug_info_offset);
   i64 size = hdr.size + offsetof(PubnamesHdr, size) + sizeof(hdr.size);
   u8 *p = (u8 *)&hdr + sizeof(hdr);
   u8 *end = (u8 *)&hdr + size;
@@ -530,9 +644,11 @@ static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
     p += sizeof(Offset);
 
     u8 type = *p++;
-    std::string_view name = (char *)p;
-    p += name.size() + 1;
-    cu->nametypes.push_back(NameType{hash_string(name), type, name});
+    const char *name = (char *)p;
+    i64 len = strlen(name);
+    p += len + 1;
+    cu->names.emplace_back(
+      hash_string(std::string_view(name, len)), type, name);
   }
 
   return size;
@@ -554,11 +670,12 @@ static void read_pubnames(Context<E> &ctx, std::vector<Compunit> &cus,
       continue;
 
     isec->uncompress(ctx);
-    if (isec->contents.empty())
+    std::string_view contents = isec->get_contents();
+    if (contents.empty())
       continue;
 
-    u8 *p = (u8*)&isec->contents[0];
-    u8 *end = p + isec->contents.size();
+    u8 *p = (u8 *)contents.data();
+    u8 *end = p + contents.size();
 
     while (p < end) {
       if (*(U32<E> *)p == 0xffff'ffff)
@@ -566,30 +683,55 @@ static void read_pubnames(Context<E> &ctx, std::vector<Compunit> &cus,
       else
         p += read_pubnames_cu(ctx, *(PubnamesHdr32<E> *)p, cus, file);
     }
-  };
+  }
 }
 
 template <typename E>
 static std::vector<Compunit> read_compunits(Context<E> &ctx) {
+  std::vector<std::vector<Compunit>> file_cus(ctx.objs.size());
+
+  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 file_idx) {
+    ObjectFile<E> &file = *ctx.objs[file_idx];
+    if (!file.debug_info)
+      return;
+
+    file.debug_info->uncompress(ctx);
+    std::string_view contents = file.debug_info->get_contents();
+    std::vector<Compunit> &cus = file_cus[file_idx];
+    u8 *begin = (u8 *)contents.data();
+    u8 *end = begin + contents.size();
+
+    for (u8 *p = begin; p < end;) {
+      DwarfKind kind = get_dwarf_kind(ctx, p);
+      i64 size;
+      if (kind == DWARF2_32 || kind == DWARF5_32)
+        size = ((CuHdrDwarf2_32<E> *)p)->size + 4;
+      else
+        size = ((CuHdrDwarf2_64<E> *)p)->size + 12;
+
+      cus.push_back(Compunit{kind, p - begin, size, (i32)file_idx});
+      p += size;
+    }
+
+    read_pubnames(ctx, cus, file);
+    for (Compunit &cu : cus)
+      dedup_names(cu);
+  });
+
+  i64 size = 0;
+  for (std::vector<Compunit> &cus : file_cus)
+    size += cus.size();
+
   std::vector<Compunit> cus;
+  cus.reserve(size);
+  for (std::vector<Compunit> &vec : file_cus)
+    for (Compunit &cu : vec)
+      cus.push_back(std::move(cu));
+  return cus;
+}
 
-  // Read compunits from the output .debug_info section.
-  u8 *begin = &ctx.debug_info[0];
-  u8 *end = begin + ctx.debug_info.size();
-
-  for (u8 *p = begin; p < end;) {
-    DwarfKind kind = get_dwarf_kind(ctx, p);
-    i64 size;
-    if (kind == DWARF2_32 || kind == DWARF5_32)
-      size = ((CuHdrDwarf2_32<E> *)p)->size + 4;
-    else
-      size = ((CuHdrDwarf2_64<E> *)p)->size + 12;
-
-    cus.push_back(Compunit{kind, p - begin, size});
-    p += size;
-  }
-
-  // Read address ranges for each compunit.
+template <typename E>
+static void read_address_ranges(Context<E> &ctx, std::vector<Compunit> &cus) {
   tbb::parallel_for_each(cus, [&](Compunit &cu) {
     switch (cu.kind) {
     case DWARF2_32:
@@ -606,38 +748,191 @@ static std::vector<Compunit> read_compunits(Context<E> &ctx) {
       break;
     }
 
-    // Remove empty ranges
-    std::erase_if(cu.ranges, [](std::pair<u64, u64> p) {
-      return p.first == 0 || p.first == p.second;
+    std::erase_if(cu.ranges, [](std::pair<u64, u64> range) {
+      return range.first == 0 || range.first == range.second;
     });
   });
-
-  // Read symbols from .debug_gnu_pubnames and .debug_gnu_pubtypes.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    read_pubnames(ctx, cus, *file);
-  });
-
-  // Uniquify elements because GCC 11 seems to emit one record for each
-  // comdat group which results in having a lot of duplicate records.
-  tbb::parallel_for_each(cus, [](Compunit &cu) {
-    ranges::stable_sort(cu.nametypes);
-    remove_duplicates(cu.nametypes);
-  });
-
-  return cus;
 }
 
 template <typename E>
-std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
-  if (chunk->is_compressed)
-    return chunk->uncompressed_data;
+static std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
+  if (chunk->is_compressed) {
+    CompressedSection<E> &sec = *(CompressedSection<E> *)chunk;
+    return {sec.uncompressed_data.get(), (size_t)sec.chdr.ch_size};
+  }
   return {ctx.buf + chunk->shdr.sh_offset, (size_t)chunk->shdr.sh_size};
 }
 
+// Read compilation units and their public names, deduplicate and intern the
+// names, and determine the constant-pool layout. This stage needs only input
+// sections, so it can run before output-section offsets are assigned.
+template <typename E>
+void read_gdb_index_inputs(Context<E> &ctx) {
+  Timer t(ctx, "read_gdb_index_inputs");
+
+  ctx.gdb_index_data = std::make_shared<GdbIndexData>();
+
+  GdbIndexData &data = *ctx.gdb_index_data;
+  data.cus = read_compunits(ctx);
+
+  HyperLogLog estimator;
+  tbb::parallel_for_each(data.cus, [&](Compunit &cu) {
+    HyperLogLog::Sketch &sketch = estimator.local();
+
+    // NameType keeps 56 hash bits. Spread them across a 64-bit word because
+    // HyperLogLog uses the number of leading zero bits.
+    for (NameRecord &record : cu.names)
+      sketch.insert(record.nametype.get_hash() * 0x9e37'79b9'7f4a'7c15);
+  });
+
+  data.map = std::make_unique<GdbNameMap>(estimator.get_cardinality() * 3 / 2);
+
+  tbb::parallel_for_each(data.cus, [&](Compunit &cu) {
+    for (NameRecord &record : cu.names) {
+      NameType &nt = record.nametype;
+      const char *name = nt.name;
+      u64 hash = nt.get_hash();
+      u8 type = nt.get_type();
+      MapValue *ent =
+        data.map->insert_cstr(name, hash, {}, initialize_gdb_name).first;
+      record.indexed = {ent, ent->count++ + 1, type};
+    }
+  });
+
+  data.entries = data.map->get_sorted_entries_all();
+  data.ht_size = bit_ceil(data.entries.size() * 5 / 4 + 1);
+
+  // The map may contain millions of names. Assign their type and string
+  // ranges with a parallel prefix sum.
+  auto scan = [&](const tbb::blocked_range<i64> &range, PoolSize size,
+                  bool is_final) {
+    for (i64 i = range.begin(); i < range.end(); i++) {
+      GdbNameMap::Entry *ent = data.entries[i];
+      if (is_final) {
+        ent->value.type_offset = size.type_bytes;
+        ent->value.name_offset = size.name_bytes;
+      }
+      size.type_bytes += ent->value.count * 4 + 4;
+      size.name_bytes += ent->keylen + 1;
+    }
+    return size;
+  };
+
+  data.pool_size = tbb::parallel_scan(
+    tbb::blocked_range<i64>(0, data.entries.size()), PoolSize{}, scan,
+    [](PoolSize a, PoolSize b) -> PoolSize {
+      return {a.type_bytes + b.type_bytes, a.name_bytes + b.name_bytes};
+    });
+
+  tbb::parallel_for_each(data.entries, [&](GdbNameMap::Entry *ent) {
+    ent->value.name_offset += data.pool_size.type_bytes;
+  });
+}
+
+// Build the name lookup table and the constant pool for .gdb_index. They
+// depend on compilation-unit order but not on relocated address ranges, so
+// they can be built in the background once .debug_info offsets are fixed.
+template <typename E>
+void build_gdb_index_tables(Context<E> &ctx) {
+  Timer t(ctx, "build_gdb_index_tables");
+
+  GdbIndexData &data = *ctx.gdb_index_data;
+  if (data.cus.empty() || data.tables)
+    return;
+
+  std::vector<Compunit> &cus = data.cus;
+  std::vector<GdbNameMap::Entry *> &entries = data.entries;
+  using Entry = GdbNameMap::Entry;
+
+  // CU offsets are relative to their input .debug_info sections. Convert them
+  // to output-section offsets and sort the CUs in output order. A CU's position
+  // in this vector is the CU number stored in type vectors and address ranges.
+  for (Compunit &cu : cus)
+    cu.offset += ctx.objs[cu.file_idx]->debug_info->offset;
+  ranges::sort(cus, {}, &Compunit::offset);
+
+  // `tables` contains the name hash table followed by the constant pool. The
+  // constant pool contains all type vectors followed by all name strings.
+  i64 symtab_size = data.ht_size * 8;
+  i64 pool_size = data.pool_size.type_bytes + data.pool_size.name_bytes;
+  data.tables = std::unique_ptr<u8[]>(new u8[symtab_size + pool_size]);
+
+  // Each occupied hash-table slot contains the constant-pool offsets of a name
+  // and its type vector.
+  ul32 *ht = (ul32 *)data.tables.get();
+  memset(ht, 0, symtab_size);
+
+  u32 mask = data.ht_size - 1;
+  for (Entry *ent : entries) {
+    u32 hash = ent->value.gdb_hash;
+
+    // This probing sequence is part of the .gdb_index format. The table size
+    // is a power of two, so an odd step visits every slot.
+    u32 step = ((hash * 17) & mask) | 1;
+    u32 i = hash & mask;
+
+    while (ht[i * 2] || ht[i * 2 + 1])
+      i = (i + step) & mask;
+
+    ht[i * 2] = ent->value.name_offset;
+    ht[i * 2 + 1] = ent->value.type_offset;
+  }
+
+  u8 *base = data.tables.get() + symtab_size;
+
+  // Each occurrence of a name contributes one value to its type vector. Each
+  // occurrence was assigned a distinct slot while the names were interned, so
+  // the vectors can be filled in parallel. The high byte is the name's type
+  // and the low 24 bits are the CU number.
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    i64 i = &cu - cus.data();
+    for (NameRecord &record : cu.names) {
+      IndexedName &name = record.indexed;
+      MapValue *ent = name.entry;
+      ul32 *p = (ul32 *)(base + ent->type_offset);
+      p[name.type_idx] = (name.type << 24) | i;
+    }
+  });
+
+  // Prefix each type vector with its length and sort it for deterministic
+  // output. Store the NUL-terminated name at its assigned string-pool offset.
+  tbb::enumerable_thread_specific<std::vector<ul32>> scratch;
+  tbb::parallel_for_each(entries, [&](Entry *ent) {
+    ul32 *p = (ul32 *)(base + ent->value.type_offset);
+    p[0] = ent->value.count;
+    std::span<ul32> values(p + 1, ent->value.count);
+
+    if (values.size() < 256)
+      ranges::sort(values);
+    else
+      radix_sort(values, scratch.local());
+
+    u8 *dst = base + ent->value.name_offset;
+    memcpy(dst, ent->key, ent->keylen);
+    dst[ent->keylen] = '\0';
+  });
+
+  // The serialized tables contain everything needed from names and the map.
+  // Release their storage here so reclamation remains part of this background
+  // phase rather than delaying the final output path.
+  tbb::parallel_for_each(cus, [](Compunit &cu) {
+    std::vector<NameRecord>().swap(cu.names);
+  });
+  std::vector<GdbNameMap::Entry *>().swap(data.entries);
+  data.map.reset();
+}
+
+// Read relocated address ranges and serialize the index prepared above.
 template <typename E>
 void write_gdb_index(Context<E> &ctx) {
   Timer t(ctx, "write_gdb_index");
 
+  std::shared_ptr<GdbIndexData> owner = std::move(ctx.gdb_index_data);
+  GdbIndexData &data = *owner;
+
+  if (data.cus.empty())
+    return;
+
   // Find debug info sections
   for (Chunk<E> *chunk : ctx.chunks) {
     std::string_view name = chunk->name;
@@ -653,41 +948,11 @@ void write_gdb_index(Context<E> &ctx) {
       ctx.debug_rnglists = get_buffer(ctx, chunk);
   }
 
-  if (ctx.debug_info.empty())
-    return;
+  std::vector<Compunit> &cus = data.cus;
 
-  // Read debug info
-  std::vector<Compunit> cus = read_compunits(ctx);
+  read_address_ranges(ctx, cus);
 
-  // Uniquify symbols
-  HyperLogLog estimator;
-
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    HyperLogLog e;
-    for (NameType &nt : cu.nametypes)
-      e.insert(nt.hash);
-    estimator.merge(e);
-  });
-
-  ConcurrentMap<MapValue> map(estimator.get_cardinality() * 3 / 2);
-
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    cu.entries.reserve(cu.nametypes.size());
-    for (NameType &nt : cu.nametypes) {
-      MapValue *ent;
-      bool inserted;
-      std::tie(ent, inserted) = map.insert(nt.name, nt.hash,
-                                           MapValue{gdb_hash(nt.name)});
-      ent->count++;
-      cu.entries.push_back(ent);
-    }
-  });
-
-  // Sort symbols for build reproducibility
-  using Entry = typename decltype(map)::Entry;
-  std::vector<Entry *> entries = map.get_sorted_entries_all();
-
-  // Compute sizes of each components
+  // Compute sizes of each component
   SectionHeader hdr;
   hdr.cu_list_offset = sizeof(hdr);
   hdr.cu_types_offset = hdr.cu_list_offset + cus.size() * 16;
@@ -697,25 +962,13 @@ void write_gdb_index(Context<E> &ctx) {
   for (Compunit &cu : cus)
     hdr.symtab_offset += cu.ranges.size() * 20;
 
-  i64 ht_size = bit_ceil(entries.size() * 5 / 4 + 1);
+  i64 ht_size = data.ht_size;
   hdr.const_pool_offset = hdr.symtab_offset + ht_size * 8;
 
-  i64 offset = 0;
-  for (Entry *ent : entries) {
-    ent->value.type_offset = offset;
-    offset += ent->value.count * 4 + 4;
-  }
+  i64 bufsize = hdr.const_pool_offset + data.pool_size.type_bytes +
+                data.pool_size.name_bytes;
 
-  for (Entry *ent : entries) {
-    ent->value.name_offset = offset;
-    offset += ent->keylen + 1;
-  }
-
-  i64 bufsize = hdr.const_pool_offset + offset;
-
-  // Allocate an output buffer
-  ctx.output_file->buf2.resize(bufsize);
-  u8 *buf = ctx.output_file->buf2.data();
+  u8 *buf = ctx.output_file->extend(ctx, bufsize);
 
   // Write a section header
   memcpy(buf, &hdr, sizeof(hdr));
@@ -730,49 +983,25 @@ void write_gdb_index(Context<E> &ctx) {
   }
 
   // Write address areas
-  for (i64 i = 0; i < cus.size(); i++) {
-    for (std::pair<u64, u64> range : cus[i].ranges) {
+  std::vector<i64> range_offsets(cus.size());
+  for (i64 i = 1; i < cus.size(); i++)
+    range_offsets[i] = range_offsets[i - 1] + cus[i - 1].ranges.size() * 20;
+
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    i64 i = &cu - cus.data();
+    u8 *p = buf + hdr.ranges_offset + range_offsets[i];
+    for (std::pair<u64, u64> range : cu.ranges) {
       *(ul64 *)p = range.first;
       *(ul64 *)(p + 8) = range.second;
       *(ul32 *)(p + 16) = i;
       p += 20;
     }
-  }
-
-  // Write a symbol table
-  u32 mask = ht_size - 1;
-  ul32 *ht = (ul32 *)(buf + hdr.symtab_offset);
-
-  for (Entry *ent : entries) {
-    u32 hash = ent->value.gdb_hash;
-    u32 step = ((hash * 17) & mask) | 1;
-    u32 j = hash & mask;
-
-    while (ht[j * 2] || ht[j * 2 + 1])
-      j = (j + step) & mask;
-
-    ht[j * 2] = ent->value.name_offset;
-    ht[j * 2 + 1] = ent->value.type_offset;
-  }
-
-  // Write types
-  for (i64 i = 0; i < cus.size(); i++) {
-    Compunit &cu = cus[i];
-    u8 *base = buf + hdr.const_pool_offset;
-
-    for (i64 j = 0; j < cu.nametypes.size(); j++) {
-      ul32 *p = (ul32 *)(base + cu.entries[j]->type_offset);
-      i64 idx = ++p[0];
-      p[idx] = (cu.nametypes[j].type << 24) | i;
-    }
-  }
-
-  // Write names
-  tbb::parallel_for_each(entries, [&](Entry *ent) {
-    memcpy(buf + hdr.const_pool_offset + ent->value.name_offset,
-           ent->key, ent->keylen);
   });
 
+  i64 tables_size = ht_size * 8 + data.pool_size.type_bytes +
+                    data.pool_size.name_bytes;
+  parallel_memcpy(buf + hdr.symtab_offset, data.tables.get(), tables_size);
+
   // Update the section size and rewrite the section header
   if (ctx.shdr) {
     ctx.gdb_index->shdr.sh_size = bufsize;
@@ -782,6 +1011,8 @@ void write_gdb_index(Context<E> &ctx) {
 
 using E = MOLD_TARGET;
 
+template void read_gdb_index_inputs(Context<E> &);
+template void build_gdb_index_tables(Context<E> &);
 template void write_gdb_index(Context<E> &);
 
 } // namespace mold

src/icf.cc

@@ -67,8 +67,7 @@
 #include "mold.h"
 #include "../lib/siphash.h"
 
-#include <array>
-#include <cstdio>
+#include <fstream>
 #include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
@@ -76,24 +75,14 @@
 #include <tbb/parallel_for_each.h>
 #include <tbb/parallel_sort.h>
 
-static constexpr int64_t HASH_SIZE = 16;
-
-using Digest = std::array<uint8_t, HASH_SIZE>;
-
-namespace std {
-template <> struct hash<Digest> {
-  size_t operator()(const Digest &k) const {
-    static_assert(sizeof(size_t) <= HASH_SIZE);
-    size_t val;
-    memcpy(&val, k.data(), sizeof(size_t));
-    return val;
-  }
-};
-}
-
 namespace mold {
 
-static u8 hmac_key[16];
+struct Digest {
+  u64 hi;
+  u64 lo;
+};
+
+static u8 siphash_key[16];
 
 template <typename E>
 static void uniquify_cies(Context<E> &ctx) {
@@ -132,109 +121,22 @@ static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
     return (ctx.arg.icf_all || !isec.address_taken) &&
            name != ".init" && name != ".fini";
 
+  // .gcc_except_table contains a compiler-generated table. Pointer
+  // equality for the section is not significant because only the C++
+  // exception handling code will use the table at runtime.
+  if (name == ".gcc_except_table" ||
+      name.starts_with(".gcc_except_table."))
+    return true;
+
   bool is_readonly = !(shdr.sh_flags & SHF_WRITE);
-  bool is_relro = isec.name().starts_with(".data.rel.ro");
+  bool is_relro = name.starts_with(".data.rel.ro");
   return (ctx.arg.ignore_data_address_equality || !isec.address_taken) &&
          (is_readonly || is_relro);
 }
 
-template <typename E>
-static bool is_leaf(Context<E> &ctx, InputSection<E> &isec) {
-  if (!isec.get_rels(ctx).empty())
-    return false;
-
-  for (FdeRecord<E> &fde : isec.get_fdes())
-    if (fde.get_rels(isec.file).size() > 1)
-      return false;
-
-  return true;
-}
-
-template <typename E>
-struct LeafHasher {
-  size_t operator()(InputSection<E> *isec) const {
-    u64 h = hash_string(isec->contents);
-    for (FdeRecord<E> &fde : isec->get_fdes()) {
-      u64 h2 = hash_string(fde.get_contents(isec->file).substr(8));
-      h = combine_hash(h, h2);
-    }
-    return h;
-  }
-};
-
-template <typename E>
-struct LeafEq {
-  bool operator()(InputSection<E> *a, InputSection<E> *b) const {
-    if (a->contents != b->contents)
-      return false;
-
-    std::span<FdeRecord<E>> x = a->get_fdes();
-    std::span<FdeRecord<E>> y = b->get_fdes();
-
-    if (x.size() != y.size())
-      return false;
-
-    for (i64 i = 0; i < x.size(); i++)
-      if (x[i].get_contents(a->file).substr(8) !=
-          y[i].get_contents(b->file).substr(8))
-        return false;
-    return true;
-  }
-};
-
-// Early merge of leaf nodes, which can be processed without constructing the
-// entire graph. This reduces the vertex count and improves memory efficiency.
-template <typename E>
-static void merge_leaf_nodes(Context<E> &ctx) {
-  Timer t(ctx, "merge_leaf_nodes");
-
-  static Counter eligible("icf_eligibles");
-  static Counter non_eligible("icf_non_eligibles");
-  static Counter leaf("icf_leaf_nodes");
-
-  tbb::concurrent_unordered_map<InputSection<E> *, Atomic<InputSection<E> *>,
-                                LeafHasher<E>, LeafEq<E>> map;
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (!isec || !isec->is_alive)
-        continue;
-
-      if (!is_eligible(ctx, *isec)) {
-        non_eligible++;
-        continue;
-      }
-
-      if (is_leaf(ctx, *isec)) {
-        leaf++;
-        isec->icf_leaf = true;
-        auto [it, inserted] = map.insert({isec.get(), isec.get()});
-        if (!inserted) {
-          InputSection<E> *isec2 = it->second.load();
-          while (isec->get_priority() < isec2->get_priority() &&
-                 !it->second.compare_exchange_strong(isec2, isec.get()));
-        }
-      } else {
-        eligible++;
-        isec->icf_eligible = true;
-      }
-    }
-  });
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (isec && isec->is_alive && isec->icf_leaf) {
-        auto it = map.find(isec.get());
-        assert(it != map.end());
-        isec->leader = it->second;
-      }
-    }
-  });
-}
-
 template <typename E>
 static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
-  SipHash13_128 hasher(hmac_key);
+  SipHash13_128 hasher(siphash_key);
 
   auto hash = [&](auto val) {
     hasher.update((u8 *)&val, sizeof(val));
@@ -248,7 +150,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
   auto hash_symbol = [&](Symbol<E> &sym) {
     InputSection<E> *isec = sym.get_input_section();
 
-    if (!sym.file) {
+    if (!sym.file || sym.is_imported) {
       hash('1');
       hash((u64)&sym);
     } else if (SectionFragment<E> *frag = sym.get_frag()) {
@@ -256,19 +158,16 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
       hash((u64)frag);
     } else if (!isec) {
       hash('3');
-    } else if (isec->leader) {
-      hash('4');
-      hash((u64)isec->leader);
     } else if (isec->icf_eligible) {
-      hash('5');
+      hash('4');
     } else {
-      hash('6');
+      hash('5');
       hash((u64)isec);
     }
     hash(sym.value);
   };
 
-  hash_string(isec.contents);
+  hash_string(isec.get_contents());
   hash(isec.shdr().sh_flags);
   hash(isec.get_fdes().size());
   hash(isec.get_rels(ctx).size());
@@ -290,8 +189,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
     }
   }
 
-  for (i64 i = 0; i < isec.get_rels(ctx).size(); i++) {
-    const ElfRel<E> &rel = isec.get_rels(ctx)[i];
+  for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
     hash(rel.r_offset);
     hash(rel.r_type);
     hash(get_addend(isec, rel));
@@ -299,40 +197,163 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
   }
 
   Digest digest;
-  hasher.finish(digest.data());
+  hasher.finish(&digest);
   return digest;
 }
 
+// A concurrent hash map from digest to section. We use it to count the
+// number of distinct digests and to elect the leader section of each
+// digest's equivalence class.
+//
+// The number of distinct digests cannot exceed the number of digests,
+// so we can allocate a large enough table upfront and never need to
+// grow it.
+//
+// The map is reused across propagation rounds. Instead of clearing the
+// table at the start of each round, we stamp each slot with the round
+// number in which it was written; a slot stamped with an earlier round
+// is treated as vacant.
+//
+// Each slot consists of two 64-bit words and a section pointer. The
+// first word packs the round number, a busy bit, and 48 bits of the
+// digest; the second word holds another 64 bits. An inserter claims a
+// vacant slot by installing the first word with compare-and-swap with
+// the busy bit set, writes the second word and the pointer, and then
+// rewrites the first word with the busy bit cleared to publish the
+// slot. Since the slot index is derived from digest bits that the
+// first word doesn't contain, a successful match effectively compares
+// an entire 128-bit digest, so the map is exact under the same
+// hash-collision assumption the surrounding algorithm is built on.
+//
+// Of all sections inserted with the same digest in the same round, the
+// slot ends up pointing to the one with the lowest priority, which ICF
+// uses as the leader of the digest's equivalence class.
+template <typename E>
+class DigestMap {
+public:
+  DigestMap(i64 n) :
+    mask(bit_ceil(n * 2) - 1),
+    slots(mask + 1) {}
+
+  void next_round() {
+    if (++round == 1 << 15) {
+      // The round number wrapped around, making stale slot stamps
+      // ambiguous, so reset the table. In practice, ICF converges long
+      // before this point.
+      for (Slot &slot : slots)
+        slot.hi = 0;
+      round = 1;
+    }
+  }
+
+  // Returns true if the digest was not in the table.
+  bool insert(const Digest &digest, InputSection<E> *isec) {
+    constexpr u64 busy_bit = 1LL << 48;
+    u64 tag = digest.hi >> 16;
+    u64 value = (round << 49) | tag;
+
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask) {
+      Slot &slot = slots[i];
+      u64 x = slot.hi.load(std::memory_order_acquire);
+
+      // If the slot was last written in an earlier round, it's vacant;
+      // try to claim it.
+      while ((x >> 49) != round) {
+        if (slot.hi.compare_exchange_weak(x, value | busy_bit,
+                                          std::memory_order_acquire)) {
+          slot.lo = digest.lo;
+          slot.leader = isec;
+          slot.hi.store(value, std::memory_order_release);
+          return true;
+        }
+      }
+
+      // The slot is occupied. If it holds a different digest, try the
+      // next slot.
+      if ((x & 0xffff'ffff'ffff) != tag)
+        continue;
+
+      // The tags match; compare the digest bits in the second word,
+      // waiting for the writer to publish them if the slot is still
+      // being claimed.
+      while (x & busy_bit)
+        x = slot.hi.load(std::memory_order_acquire);
+      if (slot.lo != digest.lo)
+        continue;
+
+      // The digest is already in the table; keep the slot pointing to
+      // the lowest-priority section.
+      InputSection<E> *cur = slot.leader;
+      while (isec->get_priority() < cur->get_priority() &&
+             !slot.leader.compare_exchange_strong(cur, isec));
+      return false;
+    }
+  }
+
+  // Returns the section associated with a digest. The digest must have
+  // been inserted in the current round, and no insertion may be running
+  // concurrently.
+  InputSection<E> *find(const Digest &digest) {
+    u64 value = (round << 49) | (digest.hi >> 16);
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask)
+      if (slots[i].hi == value && slots[i].lo == digest.lo)
+        return slots[i].leader;
+  }
+
+private:
+  struct Slot {
+    Atomic<u64> hi;
+    Atomic<u64> lo;
+    Atomic<InputSection<E> *> leader;
+  };
+
+  // Time begins in round 1 so that all-zero slots, the initial state
+  // of the table, read as vacant.
+  u64 round = 1;
+  u64 mask;
+  std::vector<Slot> slots;
+};
+
 template <typename E>
 static std::vector<InputSection<E> *> gather_sections(Context<E> &ctx) {
   Timer t(ctx, "gather_sections");
 
-  // Count the number of input sections for each input file.
-  std::vector<i64> num_sections(ctx.objs.size());
+  static Counter eligible("icf_eligibles");
+  static Counter non_eligible("icf_non_eligibles");
+
+  // Count the number of eligible input sections for each input file
+  // and turn the counts into starting indices with a prefix sum.
+  std::vector<i64> indices(ctx.objs.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        num_sections[i]++;
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (!isec || !isec->is_alive)
+        continue;
+
+      if (is_eligible(ctx, *isec)) {
+        eligible++;
+        isec->icf_eligible = true;
+        indices[i + 1]++;
+      } else {
+        non_eligible++;
+      }
+    }
   });
 
-  std::vector<i64> section_indices(ctx.objs.size());
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
-    section_indices[i + 1] = section_indices[i] + num_sections[i];
+  for (i64 i = 1; i < indices.size(); i++)
+    indices[i] += indices[i - 1];
 
-  std::vector<InputSection<E> *> sections(
-    section_indices.back() + num_sections.back());
+  std::vector<InputSection<E> *> sections(indices.back());
 
   // Fill `sections` contents.
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    i64 idx = section_indices[i];
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        sections[idx++] = isec.get();
-  });
-
-  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-    sections[i]->icf_idx = i;
+    i64 idx = indices[i];
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (isec && isec->is_alive && isec->icf_eligible) {
+        isec->icf_idx = idx;
+        sections[idx++] = isec;
+      }
+    }
   });
 
   return sections;
@@ -353,6 +374,13 @@ compute_digests(Context<E> &ctx, std::span<InputSection<E> *> sections) {
 // Build a graph, treating every function as a vertex and every function call
 // as an edge. See the description at the top for a more detailed formulation.
 // We use u32 indices here to improve cache locality.
+//
+// Relocations in a section's FDEs are edges too, because compute_digest
+// hashes eligible relocation targets without identity, and every such
+// target must be represented as an edge to remain distinguishable. In
+// particular, an FDE's reference to an LSDA is an edge; without it, two
+// identical functions whose exception tables catch different types would
+// be folded into one.
 template <typename E>
 static void gather_edges(Context<E> &ctx,
                          std::span<InputSection<E> *> sections,
@@ -360,94 +388,87 @@ static void gather_edges(Context<E> &ctx,
                          std::vector<u32> &edge_indices) {
   Timer t(ctx, "gather_edges");
 
-  if (sections.empty())
-    return;
-
-  std::vector<i64> num_edges(sections.size());
-  edge_indices.resize(sections.size());
+  // Count the number of outgoing edges for each vertex and turn the
+  // counts into starting indices with a prefix sum. The extra entry at
+  // the end makes edge_indices[i + 1] valid for every vertex, so that
+  // vertex i's edges are edge_indices[i] to edge_indices[i + 1].
+  edge_indices.resize(sections.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
     assert(isec.icf_eligible);
 
-    for (i64 j = 0; j < isec.get_rels(ctx).size(); j++) {
-      const ElfRel<E> &rel = isec.get_rels(ctx)[j];
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (!sym.get_frag())
-        if (InputSection<E> *isec = sym.get_input_section())
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
           if (isec->icf_eligible)
-            num_edges[i]++;
-    }
+            edge_indices[i + 1]++;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
+        if (isec->icf_eligible)
+          edge_indices[i + 1]++;
   });
 
-  for (i64 i = 0; i < num_edges.size() - 1; i++)
-    edge_indices[i + 1] = edge_indices[i] + num_edges[i];
+  for (i64 i = 1; i < edge_indices.size(); i++)
+    edge_indices[i] += edge_indices[i - 1];
 
-  edges.resize(edge_indices.back() + num_edges.back());
+  edges.resize(edge_indices.back());
 
-  tbb::parallel_for((i64)0, (i64)num_edges.size(), [&](i64 i) {
+  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
     i64 idx = edge_indices[i];
 
-    for (ElfRel<E> &rel : isec.get_rels(ctx)) {
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (InputSection<E> *isec = sym.get_input_section())
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
+          if (isec->icf_eligible)
+            edges[idx++] = isec->icf_idx;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
         if (isec->icf_eligible)
           edges[idx++] = isec->icf_idx;
-  }
   });
 }
 
-template <typename E>
-static i64 propagate(std::span<std::vector<Digest>> digests,
-                     std::span<u32> edges, std::span<u32> edge_indices,
-                     bool &slot, std::span<u8> converged,
-                     tbb::affinity_partitioner &ap) {
-  static Counter round("icf_round");
-  round++;
-
-  i64 num_digests = digests[0].size();
-  tbb::enumerable_thread_specific<i64> changed;
-
-  tbb::parallel_for((i64)0, num_digests, [&](i64 i) {
-    if (converged[i])
-      return;
-
-    SipHash13_128 hasher(hmac_key);
-    hasher.update(digests[2][i].data(), HASH_SIZE);
+// Compute the next-round digest of each vertex by hashing its current
+// digest and the current digests of the vertices it refers to. A
+// vertex's digest after the nth round is therefore a hash of its
+// unfolding into a tree of depth n.
+static void propagate(std::span<Digest> cur, std::span<Digest> next,
+                      std::span<u32> edges, std::span<u32> edge_indices) {
+  tbb::parallel_for((i64)0, (i64)cur.size(), [&](i64 i) {
+    SipHash13_128 hasher(siphash_key);
+    hasher.update(&cur[i], sizeof(Digest));
 
     i64 begin = edge_indices[i];
-    i64 end = (i + 1 == num_digests) ? edges.size() : edge_indices[i + 1];
-
+    i64 end = edge_indices[i + 1];
     for (i64 j : edges.subspan(begin, end - begin))
-      hasher.update(digests[slot][j].data(), HASH_SIZE);
+      hasher.update(&cur[j], sizeof(Digest));
 
-    hasher.finish(digests[!slot][i].data());
+    hasher.finish(&next[i]);
+  });
 
-    if (digests[slot][i] == digests[!slot][i]) {
-      // This node has converged. Skip further iterations as it will
-      // yield the same hash.
-      converged[i] = true;
-    } else {
-      changed.local()++;
-    }
-  }, ap);
-
-  slot = !slot;
-  return changed.combine(std::plus());
+  static Counter counter("icf_round");
+  counter++;
 }
 
 template <typename E>
 static i64 count_num_classes(std::span<Digest> digests,
-                             tbb::affinity_partitioner &ap) {
-  std::vector<Digest> vec(digests.begin(), digests.end());
-  tbb::parallel_sort(vec);
+                             std::span<InputSection<E> *> sections,
+                             DigestMap<E> &map) {
+  map.next_round();
 
   tbb::enumerable_thread_specific<i64> num_classes;
-  tbb::parallel_for((i64)0, (i64)vec.size() - 1, [&](i64 i) {
-    if (vec[i] != vec[i + 1])
+  tbb::parallel_for((i64)0, (i64)digests.size(), [&](i64 i) {
+    if (map.insert(digests[i], sections[i]))
       num_classes.local()++;
-  }, ap);
+  });
   return num_classes.combine(std::plus());
 }
 
@@ -457,12 +478,12 @@ static void print_icf_sections(Context<E> &ctx) {
   tbb::concurrent_unordered_multimap<InputSection<E> *, InputSection<E> *> map;
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (isec && isec->is_alive && isec->leader) {
-        if (isec.get() == isec->leader)
-          leaders.push_back(isec.get());
+        if (isec == isec->leader)
+          leaders.push_back(isec);
         else
-          map.insert({isec->leader, isec.get()});
+          map.insert({isec->leader, isec});
       }
     }
   });
@@ -472,24 +493,32 @@ static void print_icf_sections(Context<E> &ctx) {
                        return a->get_priority() < b->get_priority();
                      });
 
+  std::ostream *out = &std::cout;
+  std::ofstream file;
+
+  std::string &path = ctx.arg.print_icf_sections;
+  if (path != "-") {
+    file.open(path);
+    if (file.fail())
+      Fatal(ctx) << "--print-icf-sections: cannot open " << path << ": "
+                 << errno_string();
+    out = &file;
+  }
+
   i64 saved_bytes = 0;
 
   for (InputSection<E> *leader : leaders) {
     auto [begin, end] = map.equal_range(leader);
-    if (begin == end)
-      continue;
-
-    Out(ctx) << "selected section " << *leader;
-
-    i64 n = 0;
-    for (auto it = begin; it != end; it++) {
-      Out(ctx) << "  removing identical section " << *it->second;
-      n++;
+    if (begin != end) {
+      *out << "selected section " << *leader << '\n';
+      for (auto it = begin; it != end; it++) {
+        *out << "  removing identical section " << *it->second << '\n';
+        saved_bytes += leader->get_contents().size();
+      }
     }
-    saved_bytes += leader->contents.size() * n;
   }
 
-  Out(ctx) << "ICF saved " << saved_bytes << " bytes";
+  *out << "ICF saved " << saved_bytes << " bytes\n";
 }
 
 template <typename E>
@@ -498,107 +527,77 @@ void icf_sections(Context<E> &ctx) {
   if (ctx.objs.empty())
     return;
 
-  get_random_bytes(hmac_key, sizeof(hmac_key));
+  get_random_bytes(siphash_key, sizeof(siphash_key));
 
   uniquify_cies(ctx);
-  merge_leaf_nodes(ctx);
 
   // Prepare for the propagation rounds.
   std::vector<InputSection<E> *> sections = gather_sections(ctx);
 
-  // We allocate 3 arrays to store hashes for each vertex.
-  //
-  // Index 0 and 1 are used for tree hashes from the previous
-  // iteration and the current iteration. They switch roles every
-  // iteration. See `slot` below.
-  //
-  // Index 2 stores the initial, single-vertex hash. This is combined
-  // with hashes from the connected vertices to form the tree hash
-  // described above.
-  std::vector<std::vector<Digest>> digests(3);
-  digests[0] = compute_digests<E>(ctx, sections);
-  digests[1].resize(digests[0].size());
-  digests[2] = digests[0];
+  // `digests` holds the current digest of each vertex; `scratch` is
+  // where a propagation round writes the next digests before the two
+  // vectors swap roles.
+  std::vector<Digest> digests = compute_digests<E>(ctx, sections);
+  std::vector<Digest> scratch(digests.size());
 
   std::vector<u32> edges;
   std::vector<u32> edge_indices;
   gather_edges<E>(ctx, sections, edges, edge_indices);
 
-  std::vector<u8> converged(digests[0].size());
-  bool slot = 0;
+  // The digest map is used to count the number of distinct digests in
+  // the loop below. As a side effect, it records the lowest-priority
+  // section for each digest, which the grouping step after the loop
+  // uses as the leader of each equivalence class.
+  DigestMap<E> map(digests.size());
 
   // Execute the propagation rounds until convergence is obtained.
+  //
+  // The number of distinct digests can only monotonically increase as
+  // the rounds hash ever-deeper trees, so once two consecutive rounds
+  // yield the same count, the partition of sections into equivalence
+  // classes has stopped changing and will remain unchanged for further
+  // iterations (proof omitted for brevity). Note that individual
+  // digests may well still be changing at that point; sections that
+  // have a cycle in downstream (i.e. recursive functions and functions
+  // that call them) never settle on a digest. That doesn't matter
+  // because sections in the same class change their digests in
+  // lockstep, keeping the partition intact.
   {
     Timer t(ctx, "propagate");
-    tbb::affinity_partitioner ap;
-
-    // A cheap test that the graph hasn't converged yet.
-    // The loop after this one uses a strict condition, but it's expensive
-    // as it requires sorting the entire hash collection.
-    //
-    // For nodes that have a cycle in downstream (i.e. recursive
-    // functions and functions that calls recursive functions) will always
-    // change with the iterations. Nodes that doesn't (i.e. non-recursive
-    // functions) will stop changing as soon as the propagation depth reaches
-    // the call tree depth.
-    // Here, we test whether we have reached sufficient depth for the latter,
-    // which is a necessary (but not sufficient) condition for convergence.
-    i64 num_changed = -1;
-    for (;;) {
-      i64 n = propagate<E>(digests, edges, edge_indices, slot, converged, ap);
-      if (n == num_changed)
-        break;
-      num_changed = n;
-    }
-
-    // Run the pass until the unique number of hashes stop increasing, at which
-    // point we have achieved convergence (proof omitted for brevity).
     i64 num_classes = -1;
-    for (;;) {
-      // count_num_classes requires sorting which is O(n log n), so do a little
-      // more work beforehand to amortize that log factor.
-      for (i64 i = 0; i < 10; i++)
-        propagate<E>(digests, edges, edge_indices, slot, converged, ap);
 
-      i64 n = count_num_classes<E>(digests[slot], ap);
-      if (n == num_classes)
+    for (;;) {
+      propagate(digests, scratch, edges, edge_indices);
+      std::swap(digests, scratch);
+      i64 m = count_num_classes<E>(digests, sections, map);
+      if (m == num_classes)
         break;
-      num_classes = n;
+      num_classes = m;
     }
   }
 
-  // Group sections by hash values.
+  // Group sections by digest. The final counting round has already
+  // elected a leader for each digest; look it up.
   {
     Timer t(ctx, "group");
-
-    auto *map =
-      new tbb::concurrent_unordered_map<Digest, Atomic<InputSection<E> *>>;
-
-    std::span<Digest> digest = digests[slot];
-
     tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      InputSection<E> *isec = sections[i];
-      auto [it, inserted] = map->insert({digest[i], isec});
-      if (!inserted) {
-        InputSection<E> *isec2 = it->second.load();
-        while (isec->get_priority() < isec2->get_priority() &&
-               !it->second.compare_exchange_strong(isec2, isec));
-      }
+      sections[i]->leader = map.find(digests[i]);
     });
-
-    tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      auto it = map->find(digest[i]);
-      assert(it != map->end());
-      sections[i]->leader = it->second;
-    });
-
-    // Since free'ing the map is slow, postpone it.
-    ctx.on_exit.push_back([=] { delete map; });
   }
 
-  if (ctx.arg.print_icf_sections)
+  if (!ctx.arg.print_icf_sections.empty())
     print_icf_sections(ctx);
 
+  // Update alignment of leaders.
+  {
+    Timer t(ctx, "update_alignment");
+    tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
+      for (InputSection<E> *isec : file->sections)
+        if (isec && isec->is_alive && isec->icf_removed())
+          update_maximum(isec->leader->p2align, isec->p2align);
+    });
+  }
+
   // Eliminate duplicate sections.
   // Symbols pointing to eliminated sections will be redirected on the fly when
   // exporting to the symtab.
@@ -606,7 +605,7 @@ void icf_sections(Context<E> &ctx) {
     Timer t(ctx, "sweep");
     static Counter eliminated("icf_eliminated");
     tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-      for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+      for (InputSection<E> *isec : file->sections) {
         if (isec && isec->is_alive && isec->icf_removed()) {
           isec->kill();
           eliminated++;

src/input-sections.cc

@@ -31,13 +31,20 @@ bool cie_equals(const CieRecord<E> &a, const CieRecord<E> &b) {
 }
 
 template <typename E>
-InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
+InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx,
+                              std::string_view section_name)
   : file(file), shndx(shndx) {
-  if (shndx < file.elf_sections.size())
-    contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};
+  if (shndx < file.elf_sections.size()) {
+    std::string_view name = section_name;
+    if (name.empty())
+      name = file.shstrtab.data() + file.elf_sections[shndx].sh_name;
+    namelen = std::min<i64>(name.size(), UINT16_MAX);
+
+    contents = file.mf->data + shdr().sh_offset;
+  }
 
   if (shdr().sh_flags & SHF_COMPRESSED) {
-    ElfChdr<E> &chdr = *(ElfChdr<E> *)&contents[0];
+    ElfChdr<E> &chdr = *(ElfChdr<E> *)contents;
     sh_size = chdr.ch_size;
     p2align = to_p2align(chdr.ch_addralign);
   } else {
@@ -63,44 +70,69 @@ void InputSection<E>::uncompress(Context<E> &ctx) {
     return;
 
   u8 *buf = new u8[sh_size];
-  copy_contents(ctx, buf);
-  contents = std::string_view((char *)buf, sh_size);
+  copy_contents_to(ctx, buf, sh_size);
+  contents = buf;
   ctx.string_pool.emplace_back(buf);
   uncompressed = true;
 }
 
 template <typename E>
-void InputSection<E>::copy_contents(Context<E> &ctx, u8 *buf) {
+void InputSection<E>::copy_contents_to(Context<E> &ctx, u8 *buf, i64 sz) {
   if (!(shdr().sh_flags & SHF_COMPRESSED) || uncompressed) {
-    memcpy(buf, contents.data(), contents.size());
+    memcpy(buf, contents, sz);
     return;
   }
 
-  if (contents.size() < sizeof(ElfChdr<E>))
+  std::string_view view = get_contents();
+  if (view.size() < sizeof(ElfChdr<E>))
     Fatal(ctx) << *this << ": corrupted compressed section";
 
-  ElfChdr<E> &hdr = *(ElfChdr<E> *)&contents[0];
-  std::string_view data = contents.substr(sizeof(ElfChdr<E>));
+  ElfChdr<E> &hdr = *(ElfChdr<E> *)contents;
+  std::string_view data = view.substr(sizeof(ElfChdr<E>));
 
   switch (hdr.ch_type) {
   case ELFCOMPRESS_ZLIB: {
-    unsigned long size = sh_size;
-    if (::uncompress(buf, &size, (u8 *)data.data(), data.size()) != Z_OK)
-      Fatal(ctx) << *this << ": uncompress failed";
-    assert(size == sh_size);
+    z_stream s = {};
+    inflateInit(&s);
+    s.next_in = (u8 *)data.data();
+    s.avail_in = data.size();
+    s.next_out = buf;
+    s.avail_out = sz;
+
+    int r;
+    while (s.total_out < sz && (r = inflate(&s, Z_NO_FLUSH)) == Z_OK);
+    if (s.total_out < sz && r != Z_STREAM_END)
+      Fatal(ctx) << *this << ": uncompress failed: " << s.msg;
+    inflateEnd(&s);
+    msan_unpoison(buf, sz);
     break;
   }
-  case ELFCOMPRESS_ZSTD:
-    if (ZSTD_decompress(buf, sh_size, (u8 *)data.data(), data.size()) != sh_size)
-      Fatal(ctx) << *this << ": ZSTD_decompress failed";
+  case ELFCOMPRESS_ZSTD: {
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+    ZSTD_inBuffer in = { data.data(), data.size() };
+    ZSTD_outBuffer out = { buf, (size_t)sz };
+
+    while (out.pos < out.size) {
+      size_t r = ZSTD_decompressStream(dctx, &out, &in);
+      if (ZSTD_isError(r))
+        Fatal(ctx) << *this << ": uncompress failed: " << ZSTD_getErrorName(r);
+      if (r == 0 && out.pos < out.size)
+        Fatal(ctx) << *this << ": uncompress failed: premature end of input";
+    }
+    ZSTD_freeDCtx(dctx);
+    msan_unpoison(buf, sz);
     break;
+  }
   default:
     Fatal(ctx) << *this << ": unsupported compression type: 0x"
                << std::hex << hdr.ch_type;
   }
 }
 
-typedef enum : u8 { NONE, ERROR, COPYREL, PLT, CPLT } Action;
+// wingdi.h defines ERROR as a macro, so undefine it before use
+#undef ERROR
+
+typedef enum : u8 { NONE, ERROR, CANONICAL, PLT } Action;
 
 template <typename E>
 static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
@@ -113,23 +145,13 @@ static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
                << std::hex << rel.r_offset << " against symbol `"
                << sym << "' can not be used; recompile with -fPIC";
     break;
-  case COPYREL:
-    // Create a copy relocation
-    if (!sym.file->is_dso) {
-      assert(sym.esym().is_undef_weak());
-      Error(ctx) << isec << ": cannot create a copy relocation for "
-                 << sym <<"; recompile with -fPIE or -fPIC";
-    }
-    sym.flags |= NEEDS_COPYREL;
+  case CANONICAL:
+    sym.flags |= NEEDS_CANONICAL;
     break;
   case PLT:
     // Create a PLT entry
     sym.flags |= NEEDS_PLT;
     break;
-  case CPLT:
-    // Create a canonical PLT entry
-    sym.flags |= NEEDS_CPLT;
-    break;
   }
 }
 
@@ -161,9 +183,9 @@ void InputSection<E>::scan_pcrel(Context<E> &ctx, Symbol<E> &sym,
   // linker generally does not support PC-relative relocations.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  ERROR,    NONE,    ERROR,         PLT    },  // Shared object
-    {  ERROR,    NONE,    COPYREL,       CPLT   },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT   },  // Position-dependent exec
+    {  ERROR,    NONE,    ERROR,         PLT       }, // Shared object
+    {  ERROR,    NONE,    CANONICAL,     CANONICAL }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -180,9 +202,9 @@ void InputSection<E>::scan_absrel(Context<E> &ctx, Symbol<E> &sym,
   // resolved at link-time.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Shared object
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT  },  // Position-dependent exec
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Shared object
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -234,23 +256,26 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 
     if (deltas.empty()) {
       // If a section is not relaxed, we can copy it as a one big chunk.
-      copy_contents(ctx, buf);
+      copy_contents_to(ctx, buf, sh_size);
     } else {
       // A relaxed section is copied piece-wise.
-      memcpy(buf, contents.data(), deltas[0].offset);
+      memcpy(buf, contents, deltas[0].offset);
 
+      i64 input_size = get_contents().size();
       for (i64 i = 0; i < deltas.size(); i++) {
         i64 offset = deltas[i].offset;
         i64 delta = deltas[i].delta;
-        i64 end = (i + 1 == deltas.size()) ? contents.size() : deltas[i + 1].offset;
+        i64 end = input_size;
+        if (i + 1 < deltas.size())
+          end = deltas[i + 1].offset;
         i64 removed_bytes = get_removed_bytes(deltas, i);
         memcpy(buf + offset + removed_bytes - delta,
-               contents.data() + offset + removed_bytes,
+               contents + offset + removed_bytes,
                end - offset - removed_bytes);
       }
     }
   } else {
-    copy_contents(ctx, buf);
+    copy_contents_to(ctx, buf, sh_size);
   }
 
   // Apply relocations
@@ -266,17 +291,12 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 template <typename E>
 std::string_view
 InputSection<E>::get_func_name(Context<E> &ctx, i64 offset) const {
-  for (Symbol<E> *sym : file.symbols) {
-    if (sym->file == &file) {
-      const ElfSym<E> &esym = sym->esym();
-      if (esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
-          esym.st_value <= offset && offset < esym.st_value + esym.st_size) {
-        if (ctx.arg.demangle)
-          return demangle(*sym);
-        return sym->name();
-      }
-    }
-  }
+  for (Symbol<E> *sym : file.symbols)
+    if (sym->file == &file)
+      if (const ElfSym<E> &esym = sym->esym();
+          esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
+          esym.st_value <= offset && offset < esym.st_value + esym.st_size)
+        return ctx.arg.demangle ? demangle(*sym) : sym->name();
   return "";
 }
 
@@ -292,10 +312,10 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
   Symbol<E> &sym = *file.symbols[rel.r_sym];
   const ElfSym<E> &esym = file.elf_syms[rel.r_sym];
 
-  // If a symbol is defined in a comdat group, and the comdat group is
-  // discarded, the symbol may not have an owner. It is technically an
-  // violation of the One Definition Rule, so it is a programmer's fault.
-  if (!sym.file) {
+  // A global symbol in a discarded COMDAT group should resolve to the
+  // corresponding symbol in the prevailing group. If it does not, the
+  // object files violate the One Definition Rule.
+  if (!sym.file && &sym != discarded_comdat_sym<E>) {
     Error(ctx) << *this << ": " << sym << " refers to a discarded COMDAT section"
                << " probably due to an ODR violation";
     return true;
@@ -342,9 +362,9 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
 
 template <typename E>
 MergeableSection<E>::MergeableSection(Context<E> &ctx, MergedSection<E> &parent,
-                                      std::unique_ptr<InputSection<E>> &isec)
-  : parent(parent), p2align(isec->p2align), section(std::move(isec)) {
-  section->uncompress(ctx);
+                                      InputSection<E> *isec)
+  : parent(parent), p2align(isec->p2align), input_section(isec) {
+  input_section->uncompress(ctx);
 
   std::scoped_lock lock(parent.mu);
   parent.members.push_back(this);
@@ -380,9 +400,9 @@ static size_t find_null(std::string_view data, i64 pos, i64 entsize) {
 // We do not support mergeable sections that have relocations.
 template <typename E>
 void MergeableSection<E>::split_contents(Context<E> &ctx) {
-  std::string_view data = section->contents;
+  std::string_view data = input_section->get_contents();
   if (data.size() > UINT32_MAX)
-    Fatal(ctx) << *section << ": mergeable section too large";
+    Fatal(ctx) << *input_section << ": mergeable section too large";
 
   i64 entsize = parent.shdr.sh_entsize;
 
@@ -392,12 +412,13 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
       frag_offsets.push_back(pos);
       size_t end = find_null(data, pos, entsize);
       if (end == data.npos)
-        Fatal(ctx) << *section << ": string is not null terminated";
+        Fatal(ctx) << *input_section << ": string is not null terminated";
       pos = end + entsize;
     }
   } else {
     if (data.size() % entsize)
-      Fatal(ctx) << *section << ": section size is not multiple of sh_entsize";
+      Fatal(ctx) << *input_section
+                 << ": section size is not multiple of sh_entsize";
     frag_offsets.reserve(data.size() / entsize);
 
     for (i64 pos = 0; pos < data.size(); pos += entsize)
@@ -405,17 +426,15 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
   }
 
   // Compute hashes for section pieces
-  HyperLogLog estimator;
+  HyperLogLog::Sketch &sketch = parent.estimator.local();
   hashes.reserve(frag_offsets.size());
 
   for (i64 i = 0; i < frag_offsets.size(); i++) {
     u64 hash = hash_string(get_contents(i));
     hashes.push_back(hash);
-    estimator.insert(hash);
+    sketch.insert(hash);
   }
 
-  parent.estimator.merge(estimator);
-
   static Counter counter("string_fragments");
   counter += frag_offsets.size();
 }
@@ -423,8 +442,17 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
 template <typename E>
 void MergeableSection<E>::resolve_contents(Context<E> &ctx) {
   fragments.reserve(frag_offsets.size());
-  for (i64 i = 0; i < frag_offsets.size(); i++)
+
+  // The hash table is typically much larger than the cache, so each
+  // insertion stalls on a cache miss for its first probe. We know all
+  // hashes upfront, so prefetch the bucket a few insertions ahead.
+  constexpr i64 lookahead = 8;
+
+  for (i64 i = 0; i < frag_offsets.size(); i++) {
+    if (i + lookahead < frag_offsets.size())
+      parent.map.prefetch(hashes[i + lookahead]);
     fragments.push_back(parent.insert(ctx, get_contents(i), hashes[i], p2align));
+  }
 
   // Reclaim memory as we'll never use this vector again
   hashes.clear();

src/jobs-unix.cc

@@ -9,7 +9,7 @@
 // mold processes to just 1 for each user. It is intended to be used as
 // `MOLD_JOBS=1 ninja` or `MOLD_JOBS=1 make -j$(nproc)`.
 
-#include "common.h"
+#include "mold.h"
 
 #include <cstdlib>
 #include <fcntl.h>

src/linker-script.cc

@@ -124,7 +124,11 @@ Script<E>::read_output_format(std::span<std::string_view> tok) {
 
 template <typename E>
 static bool is_in_sysroot(Context<E> &ctx, std::string path) {
-  std::string rel = std::filesystem::relative(path, ctx.arg.sysroot).string();
+  std::string sysroot = ctx.arg.sysroot;
+  if (sysroot.starts_with('/') && !ctx.arg.chroot.empty())
+    sysroot = ctx.arg.chroot + "/" + path_clean(sysroot);
+
+  std::string rel = std::filesystem::relative(path, sysroot).string();
   return rel != "." && !rel.starts_with("../");
 }
 
@@ -196,7 +200,8 @@ Script<E>::read_group(std::span<std::string_view> tok) {
     }
 
     MappedFile *mf = resolve_path(tok[0], true);
-    read_file(ctx, rctx, mf);
+    ReaderContext rctx2 = rctx.next_child();
+    read_file(ctx, rctx2, mf);
     tok = tok.subspan(1);
   }
 
@@ -245,9 +250,12 @@ std::string_view Script<E>::get_script_output_type() {
   }
 
   if (tok.size() >= 3 && (tok[0] == "INPUT" || tok[0] == "GROUP") &&
-      tok[1] == "(")
+      tok[1] == "(") {
+    if (tok.size() >= 5 && tok[2] == "AS_NEEDED" && tok[3] == "(")
+      tok = tok.subspan(2);
     if (MappedFile *mf = resolve_path(tok[2], false))
       return get_machine_type(ctx, rctx, mf);
+  }
   return "";
 }
 

src/lto-unix.cc

@@ -29,8 +29,8 @@
 //
 //  4. Continue the rest of the linking process as usual.
 //
-// When gcc or clang inovkes ld, they pass `-plugin linker-plugin.so` to
-// the linker. The given .so file provides a way to call the compiler
+// When gcc or clang inovkes ld, they pass `-plugin /path/to/linker-plugin.so`
+// to the linker. The given .so file provides a way to call the compiler
 // backend.
 //
 // The linker plugin API is documented at
@@ -179,13 +179,17 @@ static PluginStatus add_input_file(const char *path) {
   MappedFile *mf = must_open_file(ctx, path);
   mf->is_dependency = false;
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, "");
+  ObjectFile<E> *file = ctx.arena.template make<ObjectFile<E>>(ctx, mf, "");
   ctx.obj_pool.emplace_back(file);
   lto_objects<E>.push_back(file);
 
   file->priority = file_priority++;
   file->is_reachable = true;
-  file->parse(ctx);
+  file->parse_symbols(ctx);
+
+  // parse_symbols() only registers global symbols. Create their shared
+  // Symbol objects and fill in the file's pointers before resolving.
+  gather_symbols(ctx);
   file->resolve_symbols(ctx);
   return LDPS_OK;
 }
@@ -307,7 +311,7 @@ get_symbols(const void *handle, int nsyms, PluginSymbol *psyms, bool is_v2) {
     if (sym.file->is_dso)
       return LDPR_RESOLVED_DYN;
 
-    if (((ObjectFile<E> *)sym.file)->is_lto_obj && !sym.is_wrapped)
+    if (sym.file->to_obj()->is_lto_obj && !sym.is_wrapped)
       return esym.is_undef() ? LDPR_RESOLVED_IR : LDPR_PREEMPTED_IR;
     return esym.is_undef() ? LDPR_RESOLVED_EXEC : LDPR_PREEMPTED_REG;
   };
@@ -342,7 +346,7 @@ static void restart_process(Context<E> &ctx) {
   for (std::string_view arg : ctx.cmdline_args)
     args.push_back(strdup(std::string(arg).c_str()));
 
-  for (std::unique_ptr<ObjectFile<E>> &file : ctx.obj_pool)
+  for (ArenaObjectPtr<ObjectFile<E>> &file : ctx.obj_pool)
     if (file->is_lto_obj && !file->is_reachable)
       args.push_back(strdup(("--:ignore-ir-file=" +
                              file->mf->get_identifier()).c_str()));
@@ -425,7 +429,7 @@ get_api_version(const char *plugin_identifier,
   if (LAPI_V1 < minimal_api_supported)
     Fatal(*gctx<E>) << "LTO plugin does not support V0 or V1 API";
 
-  std::string version = get_mold_version() + "\0"s;
+  std::string version = mold_version + "\0"s;
 
   *linker_identifier = "mold";
   *linker_version = version.data();
@@ -536,8 +540,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
   }
 
   switch (psym.symbol_type) {
-  case LDST_UNKNOWN:
-    break;
   case LDST_FUNCTION:
     esym.st_type = STT_FUNC;
     break;
@@ -547,8 +549,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
   };
 
   switch (psym.visibility) {
-  case LDPV_DEFAULT:
-    break;
   case LDPV_PROTECTED:
     esym.st_visibility = STV_PROTECTED;
     break;
@@ -600,25 +600,30 @@ create_plugin_input_file(Context<E> &ctx, MappedFile *mf) {
 template <typename E>
 ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
   if (ctx.arg.plugin.empty())
-    Fatal(ctx) << mf->name << ": don't know how to handle this LTO object file "
-               << "because no -plugin option was given. Please make sure you "
-               << "added -flto not only for creating object files but also for "
-               << "creating the final executable.";
+    Fatal(ctx) << mf->name << ": unable to handle this LTO object file because "
+               << "the -plugin option was not provided. Please make sure you "
+               << "added -flto not only when creating object files but also "
+               << "when linking the final executable.";
 
   load_lto_plugin(ctx);
 
-  // V0 API's claim_file is not thread-safe.
+  // We read input files in parallel, but the plugin interface is not
+  // ready for concurrent claims: claim_file_hook() returns a file's
+  // symbol table through the add_symbols() callback into a global
+  // buffer, and members of the same archive share their parent's file
+  // descriptor, which we close after each claim. Serialize the whole
+  // claim sequence.
   static std::mutex mu;
-  std::unique_lock lock(mu, std::defer_lock);
-  if (!is_gcc_linker_api_v1)
-    lock.lock();
+  std::scoped_lock lock(mu);
 
   // Create mold's object instance
-  ObjectFile<E> *obj = new ObjectFile<E>;
+  ObjectFile<E> *obj =
+    ctx.arena.template make<ObjectFile<E>>(ctx);
   ctx.obj_pool.emplace_back(obj);
 
   obj->filename = mf->name;
-  obj->symbols.push_back(new Symbol<E>);
+  Symbol<E> *dummy = ctx.arena.template make<Symbol<E>>();
+  obj->symbols.emplace_back(dummy);
   obj->first_global = 1;
   obj->is_lto_obj = true;
   obj->mf = mf;
@@ -633,17 +638,18 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
   // claim_file_hook() calls add_symbols() which initializes `plugin_symbols`
   int claimed = false;
   claim_file_hook(&file, &claimed);
-  if (!claimed)
-    Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
-               << " please make sure you are using the same compiler of the"
-               << " same version for all object files";
 
-  // It looks like GCC doesn't need fd after claim_file_hook() while
-  // LLVM needs it and takes the ownership of fd. To prevent "too many
-  // open files" issue, we close fd only for GCC. This is ugly, though.
-  if (!is_llvm(ctx)) {
-    MappedFile *mf2 = mf->parent ? mf->parent : mf;
-    mf2->close_fd();
+  if (mf->parent)
+    mf->parent->close_fd();
+  else
+    mf->close_fd();
+
+  if (!claimed) {
+    if (!mf->parent && !mf->thin_parent)
+      Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
+                 << " please make sure you are using the same compiler of the"
+                 << " same version for all object files";
+    return nullptr;
   }
 
   // Create a symbol strtab
@@ -654,7 +660,8 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
 
   // Initialize esyms
   obj->lto_elf_syms.resize(plugin_symbols.size() + 1);
-  obj->lto_comdat_groups.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_signatures.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_discarded.resize(plugin_symbols.size() + 1);
   i64 strtab_offset = 1;
 
   for (i64 i = 0; i < plugin_symbols.size(); i++) {
@@ -670,13 +677,16 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
     // section. We handle such symbols differently than comdat symbols in
     // a regular file because, unlike regular object files, IR files don't
     // have input sections.
-    if (psym.comdat_key)
-      obj->lto_comdat_groups[i + 1] = insert_comdat_group(ctx, psym.comdat_key);
+    if (psym.comdat_key) {
+      std::string_view key = save_string(ctx, psym.comdat_key);
+      obj->lto_comdat_signatures[i + 1] = get_symbol(ctx, key);
+    }
   }
 
   obj->symbol_strtab = save_string(ctx, strtab);
   obj->elf_syms = obj->lto_elf_syms;
-  obj->initialize_symbols(ctx);
+  obj->populate_symbol_name_lengths();
+  obj->register_global_symbols(ctx);
   plugin_symbols.clear();
   return obj;
 }
@@ -698,7 +708,7 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
     if (!file->is_lto_obj) {
       for (Symbol<E> *sym : file->get_global_syms()) {
         if (sym->file && !sym->file->is_dso &&
-            ((ObjectFile<E> *)sym->file)->is_lto_obj) {
+            sym->file->to_obj()->is_lto_obj) {
           std::scoped_lock lock(sym->mu);
           sym->referenced_by_regular_obj = true;
         }

src/main.cc

@@ -1,122 +1,122 @@
 #include "mold.h"
-#include "../lib/archive-file.h"
+#include "config.h"
 
 #include <cstring>
+#include <filesystem>
 #include <functional>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <tbb/global_control.h>
+#include <tbb/parallel_for_each.h>
 #include <unordered_set>
 
-#ifdef _WIN32
-# include <direct.h>
-# define chdir _chdir
-#else
-# include <unistd.h>
-#endif
-
-#ifdef MOLD_X86_64
-int main(int argc, char **argv) {
-  mold::set_mimalloc_options();
-  return mold::mold_main<mold::X86_64>(argc, argv);
-}
-#endif
-
 namespace mold {
 
 template <typename E>
 static void
 check_file_compatibility(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   std::string_view target = get_machine_type(ctx, rctx, mf);
+  if (target.empty())
+    Fatal(ctx) << mf->name << ": unknown machine type";
   if (target != ctx.arg.emulation)
     Fatal(ctx) << mf->name << ": incompatible file type: "
                << ctx.arg.emulation << " is expected but got " << target;
 }
 
 template <typename E>
-static ObjectFile<E> *new_object_file(Context<E> &ctx, ReaderContext &rctx,
-                                      MappedFile *mf, std::string archive_name) {
+static void new_object_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_objs");
   count++;
 
   check_file_compatibility(ctx, rctx, mf);
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, archive_name);
+  ObjectFile<E> *file =
+    ctx.arena.template make<ObjectFile<E>>(ctx, mf, archive_name);
   ctx.obj_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
-  file->is_reachable =
-    !rctx.in_lib && (archive_name.empty() || rctx.whole_archive);
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse_symbols(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
 template <typename E>
-static ObjectFile<E> *new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
-                                  MappedFile *mf, std::string archive_name) {
+static void new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
+                        MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_lto_objs");
   count++;
 
   if (ctx.arg.ignore_ir_file.count(mf->get_identifier()))
-    return nullptr;
+    return;
 
   ObjectFile<E> *file = read_lto_object(ctx, mf);
-  file->priority = ctx.file_priority++;
-  file->archive_name = archive_name;
-  file->is_reachable =
-    !rctx.in_lib && (archive_name.empty() || rctx.whole_archive);
+  if (!file)
+    return;
 
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->archive_name = archive_name;
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
+
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
 template <typename E>
-static SharedFile<E> *
-new_shared_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
+static void new_shared_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf) {
+  if (rctx.static_)
+    Fatal(ctx) << mf->name << ": attempted static link of a dynamic object";
+
   check_file_compatibility(ctx, rctx, mf);
 
-  SharedFile<E> *file = new SharedFile<E>(ctx, mf);
+  SharedFile<E> *file = ctx.arena.template make<SharedFile<E>>(ctx, mf);
   ctx.dso_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
-  file->is_reachable = !rctx.as_needed;
+  file->as_needed = rctx.as_needed;
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
+// Reads a file inside an archive.
+template <typename E>
+static void read_archive_member(Context<E> &ctx, ReaderContext &rctx,
+                                MappedFile *mf, std::string archive_name) {
+  switch (get_file_type(ctx, mf)) {
+  case FileType::ELF_OBJ:
+    new_object_file(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::GCC_LTO_OBJ:
+  case FileType::LLVM_BITCODE:
+    new_lto_obj(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::ELF_DSO:
+    Warn(ctx) << archive_name << "(" << mf->name
+              << "): shared object file in an archive is ignored";
+    break;
+  default:
+    break;
+  }
+}
+
+// Reads the given file, which is located at rctx.pos in the command
+// line. If the file is a container, i.e. an archive file or a linker
+// script, the files in it are read in place, recursively.
+//
+// read_input_files() reads top-level files with this function too but
+// overrides the container cases to read archive members in parallel.
 template <typename E>
 void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   switch (get_file_type(ctx, mf)) {
   case FileType::ELF_OBJ:
-    ctx.objs.push_back(new_object_file(ctx, rctx, mf, ""));
+    new_object_file(ctx, rctx, mf, "");
     return;
   case FileType::ELF_DSO:
-    ctx.dsos.push_back(new_shared_file(ctx, rctx, mf));
+    new_shared_file(ctx, rctx, mf);
     return;
   case FileType::AR:
   case FileType::THIN_AR:
     for (MappedFile *child : read_archive_members(ctx, mf)) {
-      switch (get_file_type(ctx, child)) {
-      case FileType::ELF_OBJ:
-        ctx.objs.push_back(new_object_file(ctx, rctx, child, mf->name));
-        break;
-      case FileType::GCC_LTO_OBJ:
-      case FileType::LLVM_BITCODE:
-        if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, child, mf->name))
-          ctx.objs.push_back(file);
-        break;
-      case FileType::ELF_DSO:
-        Warn(ctx) << mf->name << "(" << child->name
-                  << "): shared object file in an archive is ignored";
-        break;
-      default:
-        break;
-      }
+      ReaderContext rctx2 = rctx.next_child();
+      read_archive_member(ctx, rctx2, child, mf->name);
     }
     return;
   case FileType::TEXT:
@@ -124,8 +124,7 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
     return;
   case FileType::GCC_LTO_OBJ:
   case FileType::LLVM_BITCODE:
-    if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, mf, ""))
-      ctx.objs.push_back(file);
+    new_lto_obj(ctx, rctx, mf, "");
     return;
   default:
     Fatal(ctx) << mf->name << ": unknown file type";
@@ -134,35 +133,23 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
 
 template <typename E>
 static std::string_view
-detect_machine_type(Context<E> &ctx, std::vector<std::string> args) {
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+detect_machine_type(Context<E> &ctx, std::span<ReaderJob> jobs) {
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) != FileType::TEXT)
-          if (std::string_view target = get_machine_type(ctx, rctx, mf);
+          if (std::string_view target = get_machine_type(ctx, job.rctx, mf);
               !target.empty())
             return target;
-    }
-  }
 
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) == FileType::TEXT)
           if (std::string_view target =
-              Script(ctx, rctx, mf).get_script_output_type();
+              Script(ctx, job.rctx, mf).get_script_output_type();
               !target.empty())
             return target;
-    }
-  }
 
   Fatal(ctx) << "-m option is missing";
 }
@@ -204,62 +191,113 @@ MappedFile *find_library(Context<E> &ctx, ReaderContext &rctx, std::string name)
   Fatal(ctx) << "library not found: " << name;
 }
 
+// Reads all input files.
+//
+// Reading input files is I/O- and CPU-intensive, and a large program
+// can easily consist of tens of thousands of them, so we want to read
+// files in parallel. The command line, on the other hand, is
+// inherently sequential: options such as --as-needed or
+// --whole-archive apply to the files after them, and a file's
+// priority for symbol resolution is its position in the command line.
+//
+// We reconcile the two as follows: the command line parser has
+// already recorded the reader state and the position for each file
+// argument in its ReaderJob. We open and read files in parallel and
+// then sort the files we've found back into the command line order to
+// assign priorities.
 template <typename E>
-static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
+static void read_input_files(Context<E> &ctx, std::vector<ReaderJob> &jobs) {
   Timer t(ctx, "read_input_files");
 
-  ReaderContext rctx;
-  std::vector<ReaderContext> stack;
-  std::unordered_set<std::string_view> visited;
+  // Open and read files in parallel. Archive files are expanded into
+  // one job per member so that members are read in parallel too.
+  //
+  // Linker scripts are the exception to the parallelism: they can
+  // modify the context, e.g. by defining symbol versions, so we only
+  // collect them here and parse them after this loop, one at a time
+  // and in the command line order, to keep their effects
+  // deterministic. Scripts given as input files are rare and small,
+  // such as the GROUP file that glibc installs as libc.so, so the
+  // lost parallelism doesn't matter.
+  //
+  // Parsing scripts late assumes that no script directive affects how
+  // command line arguments after the script are read. That holds for
+  // the directives we currently support: a script can only add input
+  // files, whose positions order them correctly, and define symbol
+  // versions or symbols, which are not used until after this
+  // function. If we add a directive that doesn't satisfy this, such
+  // as SEARCH_DIR, which affects how -l arguments after it are
+  // resolved, this scheme needs to be revisited.
+  tbb::concurrent_vector<ReaderJob> scripts;
 
-  tbb::task_group tg;
-  rctx.tg = &tg;
+  tbb::parallel_for_each(jobs, [&](ReaderJob &job,
+                                   tbb::feeder<ReaderJob> &feeder) {
+    ReaderContext &rctx = job.rctx;
 
-  while (!args.empty()) {
-    std::string_view arg = args[0];
-    args = args.subspan(1);
-
-    if (arg == "--as-needed") {
-      rctx.as_needed = true;
-    } else if (arg == "--no-as-needed") {
-      rctx.as_needed = false;
-    } else if (arg == "--whole-archive") {
-      rctx.whole_archive = true;
-    } else if (arg == "--no-whole-archive") {
-      rctx.whole_archive = false;
-    } else if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (arg == "--start-lib") {
-      rctx.in_lib = true;
-    } else if (arg == "--end-lib") {
-      rctx.in_lib = false;
-    } else if (arg == "--push-state") {
-      stack.push_back(rctx);
-    } else if (arg == "--pop-state") {
-      if (stack.empty())
-        Fatal(ctx) << "no state pushed before popping";
-      rctx = stack.back();
-      stack.pop_back();
-    } else if (arg.starts_with("-l")) {
-      arg = arg.substr(2);
-      if (visited.contains(arg))
-        continue;
-      visited.insert(arg);
-
-      MappedFile *mf = find_library(ctx, rctx, std::string(arg));
-      mf->given_fullpath = false;
-      read_file(ctx, rctx, mf);
-    } else {
-      read_file(ctx, rctx, must_open_file(ctx, std::string(arg)));
+    // An archive member is enqueued in an already-opened form by the
+    // job that read its archive file.
+    if (!job.archive_name.empty()) {
+      read_archive_member(ctx, rctx, job.mf, job.archive_name);
+      return;
     }
+
+    // Everything else is a command line argument that we need to open.
+    MappedFile *mf;
+    if (job.is_lib) {
+      mf = find_library(ctx, rctx, job.name);
+      mf->given_fullpath = false;
+    } else {
+      mf = must_open_file(ctx, job.name);
+    }
+
+    switch (get_file_type(ctx, mf)) {
+    case FileType::AR:
+    case FileType::THIN_AR:
+      for (MappedFile *child : read_archive_members(ctx, mf)) {
+        ReaderJob job2;
+        job2.rctx = rctx.next_child();
+        job2.mf = child;
+        job2.archive_name = mf->name;
+        feeder.add(std::move(job2));
+      }
+      break;
+    case FileType::TEXT:
+      job.mf = mf;
+      scripts.push_back(std::move(job));
+      break;
+    default:
+      read_file(ctx, rctx, mf);
+    }
+  });
+
+  // Parse linker scripts and read the files they name.
+  ranges::sort(scripts, {}, [](const ReaderJob &job) {
+    return job.rctx.pos;
+  });
+
+  for (ReaderJob &job : scripts)
+    Script(ctx, job.rctx, job.mf).parse_linker_script();
+
+  // Sort the files into the command line order and assign priorities.
+  ranges::sort(ctx.unsorted_input_files, {},
+               &std::pair<std::vector<u32>, InputFile<E> *>::first);
+
+  for (auto &pair : ctx.unsorted_input_files) {
+    InputFile<E> *file = pair.second;
+    file->priority = ctx.file_priority++;
+    if (ctx.arg.trace)
+      Out(ctx) << "trace: " << *file;
+
+    if (file->is_dso)
+      ctx.dsos.push_back(file->to_dso());
+    else
+      ctx.objs.push_back(file->to_obj());
   }
 
-  if (ctx.objs.empty())
-    Fatal(ctx) << "no input files";
+  ctx.unsorted_input_files.clear();
 
-  tg.wait();
+  if (ctx.objs.empty() && ctx.dsos.empty())
+    Fatal(ctx) << "no input files";
 }
 
 template <typename E>
@@ -270,6 +308,17 @@ static bool has_lto_obj(Context<E> &ctx) {
   return false;
 }
 
+template <typename E>
+static i64 get_thread_count(Context<E> &ctx) {
+  if (ctx.arg.thread_count.has_value())
+    return *ctx.arg.thread_count;
+
+  // mold doesn't scale well with too many threads, so limit it to 32.
+  int n = tbb::global_control::active_value(
+    tbb::global_control::max_allowed_parallelism);
+  return std::min(n, 32);
+}
+
 template <typename E>
 int mold_main(int argc, char **argv) {
   Context<E> ctx;
@@ -278,36 +327,39 @@ int mold_main(int argc, char **argv) {
   if (argc >= 2 && (argv[1] == "-run"sv || argv[1] == "--run"sv))
     process_run_subcommand(ctx, argc, argv);
 
+  // parse_nonpositional_args() may chdir(2) for -C. If we end up
+  // restarting in redo_main(), we need to re-enter from the original
+  // directory so relative paths (e.g. response files) still resolve.
+  std::error_code ec;
+  std::filesystem::path orig_cwd = std::filesystem::current_path(ec);
+
   // Parse non-positional command line options
   ctx.cmdline_args = expand_response_files(ctx, argv);
-  std::vector<std::string> file_args = parse_nonpositional_args(ctx);
+  std::vector<ReaderJob> jobs = parse_nonpositional_args(ctx);
 
   // If no -m option is given, deduce it from input files.
   if (ctx.arg.emulation.empty())
-    ctx.arg.emulation = detect_machine_type(ctx, file_args);
+    ctx.arg.emulation = detect_machine_type(ctx, jobs);
 
-  // Redo if -m is not x86-64.
-  if constexpr (is_x86_64<E>)
-    if (ctx.arg.emulation != X86_64::name)
-      return redo_main(ctx, argc, argv);
+  // Redo if -m does not match with our speculation.
+  if (ctx.arg.emulation != E::name) {
+    if (!ec)
+      std::filesystem::current_path(orig_cwd, ec);
+    return redo_main<E>(ctx.arg.emulation, argc, argv);
+  }
 
   Timer t_all(ctx, "all");
 
   install_signal_handler();
 
-  if (!ctx.arg.directory.empty())
-    if (chdir(ctx.arg.directory.c_str()) == -1)
-      Fatal(ctx) << "chdir failed: " << ctx.arg.directory
-                 << ": " << errno_string();
-
   // Fork a subprocess unless --no-fork is given.
   if (ctx.arg.fork)
-    fork_child();
+    fork_child<E>();
 
   acquire_global_lock();
 
-  tbb::global_control tbb_cont(tbb::global_control::max_allowed_parallelism,
-                               ctx.arg.thread_count);
+  ctx.global_limit.emplace(tbb::global_control::max_allowed_parallelism,
+                           get_thread_count(ctx));
 
   // Handle --wrap options if any.
   for (std::string_view name : ctx.arg.wrap)
@@ -322,7 +374,7 @@ int mold_main(int argc, char **argv) {
     get_symbol(ctx, arg)->is_traced = true;
 
   // Parse input files
-  read_input_files(ctx, file_args);
+  read_input_files(ctx, jobs);
 
   // Uniquify shared object files by soname
   {
@@ -360,9 +412,25 @@ int mold_main(int argc, char **argv) {
   std::erase_if(ctx.objs, [](InputFile<E> *file) { return !file->is_reachable; });
   std::erase_if(ctx.dsos, [](InputFile<E> *file) { return !file->is_reachable; });
 
+  // Building .gdb_index is split into three stages because the required data
+  // becomes available at different points in the link. Compilation units and
+  // public names depend only on input sections, so read them now in a
+  // low-priority arena while foreground passes continue.
+  bool create_gdb_index = ctx.arg.gdb_index && !ctx.arg.relocatable;
+  tbb::task_arena gdb_input_arena(tbb::task_arena::automatic, 1,
+                                  tbb::task_arena::priority::low);
+  tbb::task_group gdb_task;
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] {
+      gdb_task.run([&] { read_gdb_index_inputs(ctx); });
+    });
+
   // Parse .eh_frame section contents.
   parse_eh_frame_sections(ctx);
 
+  // Parse .sframe section contents.
+  parse_sframe_sections(ctx);
+
   // Split mergeable section contents into section pieces.
   create_merged_sections(ctx);
 
@@ -390,6 +458,11 @@ int mold_main(int argc, char **argv) {
   if (!ctx.arg.allow_multiple_definition)
     check_duplicate_symbols(ctx);
 
+  // Handle --zero-to-bss, which converts data sections containing only
+  // zeros into BSS.
+  if (ctx.arg.zero_to_bss)
+    convert_zero_to_bss(ctx);
+
   // Set "address-taken" bits for input sections.
   if (ctx.arg.icf)
     compute_address_significance(ctx);
@@ -489,6 +562,10 @@ int mold_main(int argc, char **argv) {
   // .got.plt, .dynsym, .dynstr, etc.
   scan_relocations(ctx);
 
+  // Now that we know all exported symbols, make sure that no versioned
+  // name is defined twice.
+  check_symbol_version_conflicts(ctx);
+
   // Compute the is_weak bit for each imported symbol.
   compute_imported_symbol_weakness(ctx);
 
@@ -496,24 +573,45 @@ int mold_main(int argc, char **argv) {
   // create as few segments as possible.
   sort_output_sections(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  // Handle --separate-debug-file.
+  if (ctx.gnu_debuglink)
     separate_debug_sections(ctx);
 
   // Compute sizes of output sections while assigning offsets
   // within an output section to input sections.
   compute_section_sizes(ctx);
 
-  // If --packed_dyn_relocs=relr was given, base relocations are stored
-  // to a .relr.dyn section in a compressed form. Construct a compressed
-  // relocations now so that we can fix section sizes and file layout.
+  // RELR is encoded independently for each output chunk using offsets
+  // relative to that chunk.
   if (ctx.arg.pack_dyn_relocs_relr)
-    construct_relr(ctx);
+    ctx.reldyn->construct_relr(ctx);
 
   // Reserve a space for dynamic symbol strings in .dynstr and sort
   // .dynsym contents if necessary. Beyond this point, no symbol will
   // be added to .dynsym.
   sort_dynsyms(ctx);
 
+  // sort_debug_info_sections may uncompress the same .debug_info sections.
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] { gdb_task.wait(); });
+
+  // Sort .debug_info contents so that DWARF32 debug info precedes that of
+  // DWARF64. This is to mitigate the possibility of a relocation overflow.
+  sort_debug_info_sections(ctx);
+
+  // Type vectors identify compilation units by their order in the output
+  // .debug_info section. That order is now fixed, so build the table while the
+  // remaining layout passes continue. This stage stops scaling after about 12
+  // workers, so limit its arena to avoid competing with foreground work.
+  i64 gdb_table_workers =
+    std::clamp<i64>(get_thread_count(ctx) * 3 / 8, 1, 12);
+  tbb::task_arena gdb_table_arena(gdb_table_workers, 1,
+                                  tbb::task_arena::priority::low);
+  if (ctx.gdb_index && !ctx.gnu_debuglink)
+    gdb_table_arena.execute([&] {
+      gdb_task.run([&] { build_gdb_index_tables(ctx); });
+    });
+
   // Print reports about undefined symbols, if needed.
   if (ctx.arg.unresolved_symbols == UNRESOLVED_ERROR)
     report_undef_errors(ctx);
@@ -531,18 +629,16 @@ int mold_main(int argc, char **argv) {
   // Here, we construct output .eh_frame contents.
   ctx.eh_frame->construct(ctx);
 
+  // .sframe is likewise parsed and reconstructed by the linker. Build
+  // the merged, PC-sorted output .sframe.
+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
   // If --emit-relocs is given, we'll copy relocation sections from input
   // files to an output file.
   if (ctx.arg.emit_relocs)
     create_reloc_sections(ctx);
 
-  // We've created range extension thunks with a pessimistive assumption
-  // that all out-of-section references are out of range. Now that we are
-  // able to assign addresses to all SHF_ALLOC output sections, we can
-  // eliminate excessive thunks.
-  if constexpr (needs_thunk<E>)
-    remove_redundant_thunks(ctx);
-
   // Compute .symtab and .strtab sizes for each file.
   if (!ctx.arg.strip_all)
     create_output_symtab(ctx);
@@ -562,6 +658,14 @@ int mold_main(int argc, char **argv) {
     filesize = set_osec_offsets(ctx);
   }
 
+  // We've created range extension thunks with a pessimistive assumption
+  // that all out-of-section references are out of range. Now that we know
+  // the addresses of all sections,, we can eliminate excessive thunks.
+  if constexpr (needs_thunk<E>) {
+    remove_redundant_thunks(ctx);
+    filesize = set_osec_offsets(ctx);
+  }
+
   if constexpr (is_arm32<E>) {
     if (ctx.extra.exidx) {
       ctx.extra.exidx->remove_duplicate_entries(ctx);
@@ -578,18 +682,29 @@ int mold_main(int argc, char **argv) {
   // GOT or PLT addresses have a correct final value.
 
   // If --compress-debug-sections is given, compress .debug_* sections
-  // using zlib.
-  if (ctx.arg.compress_debug_sections != COMPRESS_NONE) {
+  // using zlib or zstd.
+  if (ctx.arg.compress_debug_sections != ELFCOMPRESS_NONE) {
     compress_debug_sections(ctx);
     filesize = set_osec_offsets(ctx);
   }
 
-  // At this point, both memory and file layouts are fixed.
-
   // Gather thunk symbols and attach them to themselves.
   if constexpr (needs_thunk<E>)
     gather_thunk_addresses(ctx);
 
+  // Re-finalize layout. fix_synthetic_symbols above may have changed
+  // addends for dynamic relocations referencing synthetic symbols, which
+  // can shift the encoded size of .rela.dyn under --pack-dyn-relocs=android
+  // because Android's packed format encodes addends in variable-length
+  // SLEB128. Other modes, including ordinary RELR, encode nothing whose
+  // size depends on addends, so they do not need this pass.
+  if (ctx.arg.pack_dyn_relocs_android)
+    filesize = set_osec_offsets(ctx);
+
+  // At this point, both memory and file layouts are fixed.
+
+  ctx.reldyn->update_shdr(ctx);
+
   t_before_copy.stop();
 
   // Create an output file
@@ -601,6 +716,9 @@ int mold_main(int argc, char **argv) {
   // Copy input sections to the output file and apply relocations.
   copy_chunks(ctx);
 
+  if constexpr (is_arm32be<E>)
+    arm32be_swap_bytes(ctx);
+
   if constexpr (is_x86_64<E>)
     if (ctx.arg.z_rewrite_endbr)
       rewrite_endbr(ctx);
@@ -609,11 +727,12 @@ int mold_main(int argc, char **argv) {
   // so we sort them.
   sort_reldyn(ctx);
 
-  // .gdb_index's contents cannot be constructed before applying
-  // relocations to other debug sections. We have relocated debug
-  // sections now, so write the .gdb_index section.
-  if (ctx.gdb_index && ctx.arg.separate_debug_file.empty())
+  // The final stage reads address ranges, which requires relocated debug
+  // sections. We have applied the relocations now, so finish the index.
+  if (ctx.gdb_index && !ctx.gnu_debuglink) {
+    gdb_table_arena.execute([&] { gdb_task.wait(); });
     write_gdb_index(ctx);
+  }
 
   // .note.gnu.build-id section contains a cryptographic hash of the
   // entire output file. Now that we wrote everything except build-id,
@@ -621,7 +740,7 @@ int mold_main(int argc, char **argv) {
   if (ctx.buildid)
     write_build_id(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
     write_gnu_debuglink(ctx);
 
   t_copy.stop();
@@ -642,7 +761,7 @@ int mold_main(int argc, char **argv) {
   if (ctx.arg.print_map)
     print_map(ctx);
 
-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
     write_separate_debug_file(ctx);
 
   // Show stats numbers
@@ -655,14 +774,22 @@ int mold_main(int argc, char **argv) {
   std::cout << std::flush;
   std::cerr << std::flush;
 
-  notify_parent();
+  notify_parent<E>();
   release_global_lock();
 
+#if HAVE_MADVISE
+  // Dropping page table entries here in parallel makes process exit
+  // faster, as the kernel otherwise reclaims them in a single thread
+  // on exit. File contents stay in the page cache.
+  tbb::parallel_for_each(ctx.mf_pool, [](std::unique_ptr<MappedFile> &mf) {
+    if (!mf->parent && mf->data && mf->size)
+      madvise(mf->data, mf->size, MADV_DONTNEED);
+  });
+#endif
+
   if (ctx.arg.quick_exit)
     _exit(0);
 
-  for (std::function<void()> &fn : ctx.on_exit)
-    fn();
   ctx.checkpoint();
   return 0;
 }
@@ -670,5 +797,6 @@ int mold_main(int argc, char **argv) {
 using E = MOLD_TARGET;
 
 template int mold_main<E>(int, char **);
+template void read_file(Context<E> &, ReaderContext &, MappedFile *);
 
 } // namespace mold

src/mapfile.cc

@@ -19,14 +19,12 @@ static Map<E> get_map(Context<E> &ctx) {
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     for (Symbol<E> *sym : file->symbols) {
-      if (sym->file != file || sym->get_type() == STT_SECTION)
-        continue;
-
-      if (InputSection<E> *isec = sym->get_input_section()) {
-        assert(file == &isec->file);
-        typename Map<E>::accessor acc;
-        map.insert(acc, {isec, {}});
-        acc->second.push_back(sym);
+      if (sym->file == file && sym->get_type() != STT_SECTION) {
+        if (InputSection<E> *isec = sym->get_input_section()) {
+          typename Map<E>::accessor acc;
+          map.insert(acc, {isec, {}});
+          acc->second.push_back(sym);
+        }
       }
     }
   });
@@ -48,10 +46,11 @@ void print_map(Context<E> &ctx) {
   std::ostream *out = &std::cout;
   std::ofstream file;
 
-  if (!ctx.arg.Map.empty()) {
+  if (!ctx.arg.Map.empty() && ctx.arg.Map != "-") {
     file.open(ctx.arg.Map);
-    if (!file.is_open())
-      Fatal(ctx) << "cannot open " << ctx.arg.Map << ": " << errno_string();
+    if (file.fail())
+      Fatal(ctx) << "--print-map: cannot open " << ctx.arg.Map << ": "
+                 << errno_string();
     out = &file;
   }
 

src/mapped-file-unix.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 namespace mold {
 

src/mapped-file-win32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 namespace mold {
 

src/output-file-unix.cc

@@ -1,3 +1,4 @@
+#include "config.h"
 #include "mold.h"
 
 #include <fcntl.h>
@@ -6,6 +7,11 @@
 #include <sys/mman.h>
 #include <system_error>
 
+#ifdef __linux__
+# include <sys/vfs.h>
+# include <linux/magic.h>
+#endif
+
 namespace mold {
 
 static u32 get_umask() {
@@ -53,14 +59,34 @@ public:
 
     output_tmpfile = (char *)save_string(ctx, tmpfile).data();
 
-#ifdef __linux__
-    fallocate(this->fd, 0, 0, filesize);
+#if HAVE_FALLOCATE
+    // Calling fallocate speeds up later linking passes on ext4 by
+    // taking disk block allocation out of the page-fault handler.
+    // On tmpfs, it instead makes things much slower: the kernel
+    // allocates and zeroes every page of the range inside the
+    // syscall, on one thread, which takes ~1.8 s for a 5 GiB file.
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, 0, filesize);
 #endif
 
-    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+    // We map the file with twice as much address space as its size, so
+    // that extend() can grow the file into the mapping in place.
+    // Touching the mapping beyond the end of the file is not allowed,
+    // but growing the file with ftruncate makes the tail of the
+    // mapping accessible without any further mmap call.
+    vasize = filesize * 2;
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                            MAP_SHARED, this->fd, 0);
-    if (this->buf == MAP_FAILED)
-      Fatal(ctx) << path << ": mmap failed: " << errno_string();
+
+    if (this->buf == MAP_FAILED) {
+      // If the address space is too tight, map just the file.
+      vasize = filesize;
+      this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << path << ": mmap failed: " << errno_string();
+    }
 
     mold::output_buffer_start = this->buf;
     mold::output_buffer_end = this->buf + filesize;
@@ -71,20 +97,51 @@ public:
       ::close(fd2);
   }
 
+  // Extend the file so that the caller can fill the appended data
+  // through the tail of the mapping. This is called at most once per
+  // output file.
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    i64 mapsize = this->filesize;
+
+    if (ftruncate(this->fd, mapsize + size) == -1)
+      Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+#if HAVE_FALLOCATE
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, mapsize, size);
+#endif
+
+    if (mapsize + size > vasize) {
+      // The appended data does not fit in the mapping. Map the grown
+      // file again, moving the buffer.
+      munmap(this->buf, vasize);
+      vasize = mapsize + size;
+
+      this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+      ctx.buf = this->buf;
+      mold::output_buffer_start = this->buf;
+    }
+
+    this->filesize += size;
+    mold::output_buffer_end = this->buf + mapsize + size;
+
+#ifdef MADV_HUGEPAGE
+    madvise(this->buf, mapsize + size, MADV_HUGEPAGE);
+#endif
+    return this->buf + mapsize;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     if (!this->is_unmapped)
-      munmap(this->buf, this->filesize);
-
-    if (this->buf2.empty()) {
-      ::close(this->fd);
-    } else {
-      FILE *out = fdopen(this->fd, "w");
-      fseek(out, 0, SEEK_END);
-      fwrite(&this->buf2[0], this->buf2.size(), 1, out);
-      fclose(out);
-    }
+      munmap(this->buf, vasize);
+    ::close(this->fd);
 
     // If an output file already exists, open a file and then remove it.
     // This is the fastest way to unlink a file, as it does not make the
@@ -100,6 +157,9 @@ public:
 
 private:
   int fd2 = -1;
+
+  // Size of the file mapping, which may extend past the end of the file.
+  i64 vasize = 0;
 };
 
 template <typename E>
@@ -122,14 +182,7 @@ OutputFile<E>::open(Context<E> &ctx, std::string path, i64 filesize, int perm) {
 
 #ifdef MADV_HUGEPAGE
   // Enable transparent huge page for an output memory-mapped file.
-  // On Linux, it has an effect only on tmpfs mounted with `huge=advise`,
-  // but it can make the linker ~10% faster. You can try it by creating
-  // a tmpfs with the following commands
-  //
-  //  $ mkdir tmp
-  //  $ sudo mount -t tmpfs -o size=2G,huge=advise none tmp
-  //
-  // and then specifying a path under the directory as an output file.
+  // Linking a Chromium debug build is ~20% faster with this madvise call.
   madvise(file->buf, filesize, MADV_HUGEPAGE);
 #endif
 
@@ -161,10 +214,20 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
   if (ftruncate(this->fd, filesize) == -1)
     Fatal(ctx) << "ftruncate failed: " << errno_string();
 
-  this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+  // As in MemoryMappedOutputFile, we map the file with twice as much
+  // address space as its size so that extend() can grow the file into
+  // the mapping in place.
+  vasize = filesize * 2;
+  this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                          MAP_SHARED, this->fd, 0);
-  if (this->buf == MAP_FAILED)
-    Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+  if (this->buf == MAP_FAILED) {
+    vasize = filesize;
+    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+  }
 
   this->filesize = filesize;
   mold::output_buffer_start = this->buf;
@@ -172,17 +235,36 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
 }
 
 template <typename E>
-void LockingOutputFile<E>::close(Context<E> &ctx) {
-  if (!this->is_unmapped)
-    munmap(this->buf, this->filesize);
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  i64 mapsize = this->filesize;
 
-  if (!this->buf2.empty()) {
-    FILE *out = fdopen(this->fd, "w");
-    fseek(out, 0, SEEK_END);
-    fwrite(&this->buf2[0], this->buf2.size(), 1, out);
-    fclose(out);
+  if (ftruncate(this->fd, mapsize + size) == -1)
+    Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+  if (mapsize + size > vasize) {
+    // The appended data does not fit in the mapping. Map the grown
+    // file again, moving the buffer.
+    munmap(this->buf, vasize);
+    vasize = mapsize + size;
+
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
   }
 
+  this->filesize += size;
+  mold::output_buffer_end = this->buf + mapsize + size;
+  return this->buf + mapsize;
+}
+
+template <typename E>
+void LockingOutputFile<E>::close(Context<E> &ctx) {
+  if (!this->is_unmapped)
+    munmap(this->buf, vasize);
   ::close(this->fd);
 }
 

src/output-file-win32.cc

@@ -39,22 +39,36 @@ public:
       CloseHandle(handle);
   }
 
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    UnmapViewOfFile(this->buf);
+
+    HANDLE map = CreateFileMapping(handle, nullptr, PAGE_READWRITE, 0,
+                                   this->filesize + size, nullptr);
+    if (!map)
+      Fatal(ctx) << this->path << ": CreateFileMapping failed: "
+                 << GetLastError();
+
+    this->buf = (u8 *)MapViewOfFile(map, FILE_MAP_WRITE, 0, 0,
+                                    this->filesize + size);
+    if (!this->buf)
+      Fatal(ctx) << this->path << ": MapViewOfFile failed: " << GetLastError();
+
+    CloseHandle(map);
+
+    u8 *space = this->buf + this->filesize;
+    memset(space, 0, size);
+
+    this->filesize += size;
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
+    mold::output_buffer_end = this->buf + this->filesize;
+    return space;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     UnmapViewOfFile(this->buf);
-
-    if (!this->buf2.empty()) {
-      if (SetFilePointer(handle, 0, nullptr, FILE_END) == INVALID_SET_FILE_POINTER)
-        Fatal(ctx) << this->path << ": SetFilePointer failed: "
-                   << GetLastError();
-
-      DWORD written;
-      if (!WriteFile(handle, this->buf2.data(), this->buf2.size(), &written,
-                     nullptr))
-        Fatal(ctx) << this->path << ": WriteFile failed: " << GetLastError();
-    }
-
     CloseHandle(handle);
     handle = INVALID_HANDLE_VALUE;
   }
@@ -106,6 +120,11 @@ LockingOutputFile<E>::LockingOutputFile(Context<E> &ctx, std::string path,
 template <typename E>
 void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {}
 
+template <typename E>
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  return nullptr;
+}
+
 template <typename E>
 void LockingOutputFile<E>::close(Context<E> &ctx) {}
 

src/relocatable.cc

@@ -50,6 +50,8 @@ static void r_create_synthetic_sections(Context<E> &ctx) {
   ctx.shdr = push(new OutputShdr<E>);
   ctx.eh_frame = push(new EhFrameSection<E>);
   ctx.eh_frame_reloc = push(new EhFrameRelocSection<E>);
+  ctx.sframe = push(new SFrameSection<E>);
+  ctx.sframe_reloc = push(new SFrameRelocSection<E>);
   ctx.strtab = push(new StrtabSection<E>);
   ctx.symtab = push(new SymtabSection<E>);
   ctx.shstrtab = push(new ShstrtabSection<E>);
@@ -74,14 +76,14 @@ static void create_comdat_group_sections(Context<E> &ctx) {
     ObjectFile<E> &file = *ctx.objs[i];
 
     for (ComdatGroupRef<E> &ref : file.comdat_groups) {
-      if (ref.group->owner != file.priority)
+      if (!ref.is_owner)
         continue;
 
       Symbol<E> *sym = file.symbols[file.elf_sections[ref.sect_idx].sh_info];
       assert(sym);
 
       std::vector<Chunk<E> *> members;
-      for (u32 j : ref.members) {
+      for (u32 j : ref.members(file)) {
         const ElfShdr<E> &shdr = file.elf_sections[j];
         if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL)) {
           InputSection<E> &isec = *file.sections[shdr.sh_info];
@@ -125,7 +127,7 @@ static void r_claim_unresolved_symbols(Context<E> &ctx) {
         continue;
 
       sym.file = file;
-      sym.origin = 0;
+      sym.origin = nullptr;
       sym.value = 0;
       sym.sym_idx = i;
     }
@@ -161,6 +163,9 @@ void combine_objects(Context<E> &ctx) {
 
   ctx.eh_frame->construct(ctx);
 
+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
   create_reloc_sections(ctx);
 
   create_comdat_group_sections(ctx);

src/shrink-sections.cc

@@ -89,7 +89,7 @@ void shrink_sections(Context<E> &ctx) {
   // only ~0.04% larger than that of GNU ld), so we don't bother to handle
   // them. We scan relocations only once here.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
+    for (InputSection<E> *isec : file->sections)
       if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
         shrink_section(ctx, *isec);
   });
@@ -122,10 +122,6 @@ i64 compute_distance(Context<E> &ctx, Symbol<E> &sym,
   if (sym.is_absolute())
     return INT64_MAX;
 
-  // Likewise, relocations against weak undefined symbols won't be relaxed.
-  if (sym.esym().is_undef_weak())
-    return INT64_MAX;
-
   // Compute a distance between the relocated place and the symbol.
   i64 S = sym.get_addr(ctx);
   i64 A = rel.r_addend;

src/signal-unix.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 #include <csignal>
 #include <tbb/version.h>
@@ -28,8 +28,7 @@ void cleanup() {
 // for the first time.
 //
 // If a disk becomes full as a result of a write to an mmap'ed memory
-// region, the failure of the write is reported as a SIGBUS or structured
-// exeption with code EXCEPTION_IN_PAGE_ERROR on Windows. This
+// region, the failure of the write is reported as a SIGBUS. This
 // signal handler catches that signal and prints out a user-friendly
 // error message. Without this, it is very hard to realize that the
 // disk might be full.
@@ -74,7 +73,7 @@ void install_signal_handler() {
   sigaction(SIGBUS, &action, NULL);
 
   // OneTBB 2021.9.0 has the interface version 12090.
-  if (TBB_runtime_interface_version() < 12090) {
+  if (TBB_runtime_interface_version() <= 12090) {
     sigabrt_msg = "mold: aborted\n"
       "mold: mold with libtbb version 2021.9.0 or older is known to be unstable "
       "under heavy load. Your libtbb version is " +

src/signal-win32.cc

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"
 
 #include <windows.h>
 

src/subprocess-unix.cc

@@ -1,3 +1,4 @@
+#include "config.h"
 #include "mold.h"
 
 #include <csignal>
@@ -10,12 +11,12 @@
 
 namespace mold {
 
-#ifdef MOLD_X86_64
 static int pipe_write_fd = -1;
 
 // Exiting from a program with large memory usage is slow --
 // it may take a few hundred milliseconds. To hide the latency,
 // we fork a child and let it do the actual linking work.
+template <typename E>
 void fork_child() {
   int pipefd[2];
   if (pipe(pipefd) == -1) {
@@ -52,6 +53,7 @@ void fork_child() {
   pipe_write_fd = pipefd[1];
 }
 
+template <typename E>
 void notify_parent() {
   if (pipe_write_fd == -1)
     return;
@@ -61,7 +63,6 @@ void notify_parent() {
   assert(n == 1);
   pipe_write_fd = -1;
 }
-#endif
 
 template <typename E>
 static std::string find_dso(Context<E> &ctx, std::filesystem::path self) {
@@ -104,7 +105,7 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
   setenv("LD_PRELOAD", dso_path.c_str(), 1);
   setenv("MOLD_PATH", self.c_str(), 1);
 
-  // If ld, ld.lld or ld.gold is specified, run mold itself
+  // If ld, ld.lld or ld.gold is specified, run mold instead
   if (std::string cmd = path_filename(argv[2]);
       cmd == "ld" || cmd == "ld.lld" || cmd == "ld.gold") {
     std::vector<char *> args;
@@ -123,6 +124,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
 
 using E = MOLD_TARGET;
 
+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);
 
 } // namespace mold

src/subprocess-win32.cc

@@ -2,10 +2,11 @@
 
 namespace mold {
 
-#ifdef MOLD_X86_64
+template <typename E>
 void fork_child() {}
+
+template <typename E>
 void notify_parent() {}
-#endif
 
 template <typename E>
 [[noreturn]]
@@ -15,6 +16,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
 
 using E = MOLD_TARGET;
 
+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);
 
 } // namespace mold

src/thunks.cc

@@ -20,8 +20,8 @@
 // we don't need to try too hard to reduce thunk size to the absolute
 // minimum.
 
-#if MOLD_ARM32 || MOLD_ARM64LE || MOLD_ARM64BE || MOLD_PPC32 || \
-    MOLD_PPC64V1 || MOLD_PPC64V2
+#if MOLD_ARM32LE || MOLD_ARM32BE || MOLD_ARM64LE || MOLD_ARM64BE || \
+    MOLD_PPC32 || MOLD_PPC64V1 || MOLD_PPC64V2
 
 #include "mold.h"
 
@@ -32,12 +32,14 @@ namespace mold {
 
 using E = MOLD_TARGET;
 
-// We create thunks for each 25.6/3.2/6.4 MiB code block for
+// We create thunks for each 32/8/16 MiB code block for
 // ARM64/ARM32/PPC, respectively.
-static constexpr i64 batch_size = branch_distance<E> / 5;
+static constexpr i64 batch_size =
+  (is_arm64<E> ? 32 : is_arm32<E> ? 8 : 16) * 1024 * 1024;
 
-// We assume that a single thunk group is smaller than 1 MiB.
-static constexpr i64 max_thunk_size = 1024 * 1024;
+// We assume that a single thunk group is smaller than 16/4/8 MiB
+// for ARM64/ARM32/PPC, respectively.
+static constexpr i64 max_thunk_size = batch_size / 2;
 
 // We align thunks to 16 byte boundaries because many processor vendors
 // recommend we align branch targets to 16 byte boundaries for performance
@@ -45,84 +47,91 @@ static constexpr i64 max_thunk_size = 1024 * 1024;
 static constexpr i64 thunk_align = 16;
 
 template <typename E>
-static bool is_reachable(Context<E> &ctx, bool first_pass, InputSection<E> &isec,
-                         Symbol<E> &sym, const ElfRel<E> &rel) {
-  // If the target section is in the same output section but
-  // hasn't got any address yet, that's unreacahble.
-  InputSection<E> *isec2 = sym.get_input_section();
-  if (isec2 && isec.output_section == isec2->output_section &&
-      isec2->offset == -1)
+static bool
+requires_thunk(Context<E> &ctx, InputSection<E> &isec, const ElfRel<E> &rel,
+               bool first_pass) {
+  if (!is_func_call_rel(rel))
     return false;
+  Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
 
-  // We don't know about the final file layout on the first pass, so
-  // we assume pessimistically that all out-of-section relocations are
-  // out-of-range. Excessive thunks will be removed on the second pass.
   if (first_pass) {
+    // On the first pass, we pessimistically assume that all out-of-section
+    // relocations are out of range.
+    InputSection<E> *isec2 = sym.get_input_section();
     if (!isec2 || isec.output_section != isec2->output_section)
-      return false;
+      return true;
+
+    // If the target section is in the same output section but
+    // hasn't got any address yet, that's unreacahble.
+    if (isec2->offset == -1)
+      return true;
 
     // Even if the target is the same section, we branch to its PLT
     // if it has one. So a symbol with a PLT is also considered an
     // out-of-section reference.
     if (sym.has_plt(ctx))
-      return false;
+      return true;
   }
 
+  // Thumb and ARM B instructions cannot be converted to BX, so we
+  // always have to make them jump to a thunk to switch processor mode
+  // even if their destinations are reachable.
+  if constexpr (is_arm32<E>)
+    if (bool is_thumb = sym.get_addr(ctx) & 1;
+        (rel.r_type == R_ARM_JUMP24 && is_thumb) ||
+        (rel.r_type == R_ARM_PLT32 && is_thumb) ||
+        (rel.r_type == R_ARM_THM_JUMP24 && !is_thumb))
+      return true;
+
+  // On PowerPC, all PLT calls go through range extension thunks.
+  if constexpr (is_ppc<E>)
+    if (sym.has_plt(ctx))
+      return true;
+
+  // PowerPC before Power9 lacks PC-relative load/store instructions.
+  // Functions compiled for Power9 or earlier assume that r2 points to
+  // GOT+0x8000, while those for Power10 uses r2 as a scratch register.
+  // We need a thunk to recompute r2 for interworking.
+  if constexpr (is_ppc64v2<E>)
+    if ((rel.r_type == R_PPC64_REL24 && !sym.esym().ppc64_preserves_r2()) ||
+        (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().ppc64_uses_toc()))
+      return true;
+
   // Compute a distance between the relocated place and the symbol
   // and check if they are within reach.
   i64 S = sym.get_addr(ctx, NO_OPD);
   i64 A = get_addend(isec, rel);
   i64 P = isec.get_addr() + rel.r_offset;
   i64 val = S + A - P;
-  return -branch_distance<E> <= val && val < branch_distance<E>;
-}
-
-template <typename E>
-static bool needs_shim(Context<E> &ctx, Symbol<E> &sym, const ElfRel<E> &rel) {
-  // Thumb and ARM B instructions cannot be converted to BX, so we
-  // always have to make them jump to a thunk to switch processor mode
-  // even if their destinations are within their ranges.
-  if constexpr (is_arm32<E>) {
-    bool is_thumb = sym.get_addr(ctx) & 1;
-    return (rel.r_type == R_ARM_THM_JUMP24 && !is_thumb) ||
-           (rel.r_type == R_ARM_JUMP24 && is_thumb) ||
-           (rel.r_type == R_ARM_PLT32 && is_thumb);
-  }
-
-  // On PowerPC, all PLT calls go through range extension thunks.
-  if constexpr (is_ppc32<E> || is_ppc64v1<E>)
-    return sym.has_plt(ctx);
-
-  // PowerPC before Power9 lacks PC-relative load/store instructions.
-  // Functions compiled for Power9 or earlier assume that r2 points to
-  // GOT+0x8000, while those for Power10 uses r2 as a scratch register.
-  // We need to a thunk to recompute r2 for interworking.
-  if constexpr (is_ppc64v2<E>)
-    return sym.has_plt(ctx) ||
-           (rel.r_type == R_PPC64_REL24 && !sym.esym().ppc64_preserves_r2()) ||
-           (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().ppc64_uses_toc());
-  return false;
+  return val < -branch_distance<E> || branch_distance<E> <= val;
 }
 
 template <>
-void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
-                                                     bool first_pass) {
+void Thunk<E>::compute_size() {
+  offsets.clear();
+  for (i64 i = 0; i <= symbols.size(); i++)
+    offsets.push_back(E::thunk_hdr_size + i * E::thunk_size);
+}
+
+template <>
+void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   std::span<InputSection<E> *> m = members;
   if (m.empty())
     return;
 
   // Initialize input sections with a dummy offset so that we can
-  // distinguish sections that have got an address with the one who
-  // haven't.
-  for (InputSection<E> *isec : m)
+  // distinguish sections whose addresses have been assigned from those
+  // whose addresses have not.
+  tbb::parallel_for_each(m, [](InputSection<E> *isec) {
     isec->offset = -1;
+  });
   thunks.clear();
 
   // We create thunks from the beginning of the section to the end.
   // We manage progress using four offsets which increase monotonically.
   // The locations they point to are always A <= B <= C <= D.
   //
-  // Input sections between B and C are in the current batch.
+  // Input sections between B and C are the current batch.
   //
   // A is the input section with the smallest address than can reach
   // from the current batch.
@@ -161,17 +170,21 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
       d++;
     }
 
-    // Move C forward so that C is apart from B by BATCH_SIZE. We want
-    // to make sure that there's at least one section between B and C
-    // to ensure progress.
-    c = b + 1;
-    while (c < d && m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
-      c++;
+    // Find the end of the current batch. Section end addresses are sorted,
+    // so use binary search. Starting from B + 1 guarantees progress.
+    std::span<InputSection<E> *> range = m.subspan(b + 1, d - b - 1);
+    c = ranges::lower_bound(range, m[b]->offset + batch_size, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset + isec->sh_size;
+                            }) - m.begin();
 
-    // Move A forward so that A is reachable from C.
+    // Find the first section that is within branch range of C.
     i64 c_offset = (c == d) ? offset : m[c]->offset;
-    while (a < b && m[a]->offset + branch_distance<E> < c_offset)
-      a++;
+    range = m.subspan(a, b - a);
+    a = ranges::lower_bound(range, c_offset - branch_distance<E>, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset;
+                            }) - m.begin();
 
     // Erase references to out-of-range thunks.
     for (; t < thunks.size() && thunks[t]->offset < m[a]->offset; t++)
@@ -181,44 +194,28 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
     // Create a new thunk and place it at D.
     offset = align_to(offset, thunk_align);
     thunks.emplace_back(std::make_unique<Thunk<E>>(*this, offset));
+
     Thunk<E> &thunk = *thunks.back();
+    tbb::enumerable_thread_specific<std::vector<Symbol<E> *>> symbols;
 
     // Scan relocations between B and C to collect symbols that need
     // entries in the new thunk.
-    std::mutex mu;
-
     tbb::parallel_for(b, c, [&](i64 i) {
       InputSection<E> &isec = *m[i];
-
-      for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
-        if (!is_func_call_rel(rel))
-          continue;
-
-        // Skip if the symbol is undefined. apply_reloc() will report an error.
-        Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-        if (!sym.file)
-          continue;
-
-        // Skip if we can directly branch to the destination.
-        if (is_reachable(ctx, first_pass, isec, sym, rel) &&
-            !needs_shim(ctx, sym, rel))
-          continue;
-
-        // Add the symbol to the current thunk if it's not added already.
-        if (!sym.flags.test_and_set()) {
-          std::scoped_lock lock(mu);
-          thunk.symbols.push_back(&sym);
-        }
-      }
+      for (const ElfRel<E> &rel : isec.get_rels(ctx))
+        if (requires_thunk(ctx, isec, rel, true))
+          if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+              !sym.flags.test_and_set())
+            symbols.local().push_back(&sym);
     });
 
-    // Sort symbols added to the thunk to make the output deterministic.
-    ranges::sort(thunk.symbols, {}, [](Symbol<E> *x) {
-      return std::tuple{x->file->priority, x->sym_idx};
-    });
+    // Add symbols to the thunk
+    for (std::vector<Symbol<E> *> &vec : symbols)
+      append(thunk.symbols, vec);
 
     // Now that we know the number of symbols in the thunk, we can compute
     // the thunk's size.
+    thunk.compute_size();
     assert(thunk.size() < max_thunk_size);
     offset += thunk.size();
 
@@ -226,22 +223,89 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx,
     b = c;
   }
 
+  // Clear marks for thunks that are still reachable from the last batch.
   for (; t < thunks.size(); t++)
     for (Symbol<E> *sym : thunks[t]->symbols)
       sym->flags = 0;
 
+  // Sort symbols for deterministic output.
+  tbb::parallel_for_each(thunks, [](std::unique_ptr<Thunk<E>> &thunk) {
+    ranges::sort(thunk->symbols, {}, [](Symbol<E> *sym) {
+      return std::tuple{sym->file->priority, sym->sym_idx};
+    });
+  });
+
   this->shdr.sh_size = offset;
 }
 
+// create_range_extension_thunks() creates thunks with a pessimistic
+// assumption that all out-of-section references are out of range.
+// After computing output section addresses, we revisit all thunks to
+// remove unneeded entries from them.
+//
+// We create more thunks than necessary and then eliminate some of
+// them later, instead of just creating thunks at this stage. This is
+// because we can safely shrink sections after assigning addresses to
+// them without worrying about making existing references to thunks go
+// out of range. On the other hand, if we insert thunks after
+// assigning addresses to sections, references to thunks could become
+// out of range due to the new extra gaps for thunks. Thus, the
+// creation of thunks is a two-pass process.
 template <>
 void remove_redundant_thunks(Context<E> &ctx) {
   Timer t(ctx, "remove_redundant_thunks");
-  set_osec_offsets(ctx);
 
+  // Gather output executable sections
+  std::vector<OutputSection<E> *> sections;
   for (Chunk<E> *chunk : ctx.chunks)
     if (OutputSection<E> *osec = chunk->to_osec())
       if (osec->shdr.sh_flags & SHF_EXECINSTR)
-        osec->create_range_extension_thunks(ctx, false);
+        sections.push_back(osec);
+
+  // Mark all symbols that actually need range extension thunks
+  for (OutputSection<E> *osec : sections) {
+    tbb::parallel_for_each(osec->members, [&](InputSection<E> *isec) {
+      for (const ElfRel<E> &rel : isec->get_rels(ctx))
+        if (is_func_call_rel(rel))
+          if (Symbol<E> *sym = isec->file.symbols[rel.r_sym];
+              !sym->flags &&
+              requires_thunk(ctx, *isec, rel, false))
+            sym->flags.test_and_set();
+    });
+  }
+
+  // Remove symbols from thunks if they don't actually need range
+  // extension thunks
+  for (OutputSection<E> *osec : sections) {
+    tbb::parallel_for_each(osec->thunks,
+                           [&](std::unique_ptr<Thunk<E>> &thunk) {
+      std::erase_if(thunk->symbols,
+                    [&](Symbol<E> *sym) { return !sym->flags; });
+      thunk->shrink_size(ctx);
+    });
+  }
+
+  // Recompute section sizes
+  tbb::parallel_for_each(sections, [&](OutputSection<E> *osec) {
+    std::span<InputSection<E> *> m = osec->members;
+    std::span<std::unique_ptr<Thunk<E>>> t = osec->thunks;
+    i64 offset = 0;
+
+    while (!m.empty() || !t.empty()) {
+      if (!m.empty() && (t.empty() || m[0]->offset < t[0]->offset)) {
+        offset = align_to(offset, 1 << m[0]->p2align);
+        m[0]->offset = offset;
+        offset += m[0]->sh_size;
+        m = m.subspan(1);
+      } else {
+        offset = align_to(offset, thunk_align);
+        t[0]->offset = offset;
+        offset += t[0]->size();
+        t = t.subspan(1);
+      }
+    }
+    osec->shdr.sh_size = offset;
+  });
 }
 
 // When applying relocations, we want to know the address in a reachable
@@ -259,7 +323,8 @@ void gather_thunk_addresses(Context<E> &ctx) {
   std::vector<OutputSection<E> *> sections;
   for (Chunk<E> *chunk : ctx.chunks)
     if (OutputSection<E> *osec = chunk->to_osec())
-      sections.push_back(osec);
+      if (osec->shdr.sh_flags & SHF_EXECINSTR)
+        sections.push_back(osec);
 
   ranges::stable_sort(sections, {}, [](OutputSection<E> *x) {
     return x->shdr.sh_addr;
@@ -269,8 +334,8 @@ void gather_thunk_addresses(Context<E> &ctx) {
     for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks) {
       for (i64 i = 0; i < thunk->symbols.size(); i++) {
         Symbol<E> &sym = *thunk->symbols[i];
-        sym.add_aux(ctx);
-        ctx.symbol_aux[sym.aux_idx].thunk_addrs.push_back(thunk->get_addr(i));
+        u64 addr = thunk->get_addr() + thunk->offsets[i];
+        sym.add_thunk_addr(ctx, addr);
       }
     }
   }

test/CMakeLists.txt

@@ -30,8 +30,8 @@ endif()
 
 if(MOLD_ENABLE_QEMU_TESTS)
   list(APPEND QEMU_ARCHS
-    x86_64 i386 arm aarch64 aarch64_be ppc ppc64 ppc64le sparc64 sh4 sh4eb
-    s390x riscv64 riscv32 m68k loongarch64)
+    x86_64 i386 arm armeb aarch64 aarch64_be ppc ppc64 ppc64le sparc64
+    sh4 sh4eb s390x riscv64 riscv32 m68k loongarch64)
 
   LIST(APPEND TRIPLES
     x86_64-linux-gnu
@@ -39,6 +39,7 @@ if(MOLD_ENABLE_QEMU_TESTS)
     aarch64-linux-gnu
     aarch64_be-linux-gnu
     arm-linux-gnueabihf
+    armeb-linux-gnueabihf
     riscv64-linux-gnu
     powerpc-linux-gnu
     powerpc64-linux-gnu
@@ -85,7 +86,7 @@ function(add_target ARCH TRIPLE)
     endif()
 
     add_test(NAME ${TESTNAME}
-      COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
+      COMMAND ${CMAKE_CURRENT_LIST_DIR}/${TEST}
       WORKING_DIRECTORY ${mold_BINARY_DIR})
 
     set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT
@@ -121,6 +122,10 @@ if(${MACHINE} STREQUAL "arm" OR (HAS_qemu-arm AND HAS_arm-linux-gnueabihf-gcc))
   add_target(arm arm-linux-gnueabihf)
 endif()
 
+if(${MACHINE} STREQUAL "armeb" OR (HAS_qemu-armeb AND HAS_armeb-linux-gnueabihf-gcc))
+  add_target(armeb armeb-linux-gnueabihf)
+endif()
+
 if(${MACHINE} STREQUAL "riscv64" OR (HAS_qemu-riscv64 AND HAS_riscv64-linux-gnu-gcc))
   add_target(riscv64 riscv64-linux-gnu)
 endif()

test/abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 [ $MACHINE = aarch64 ] && skip

test/absolute-symbols.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # This test crashes only on qemu-sparc64 running on GitHub Actions,

test/allow-multiple-definition.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc -

test/ar-alignment.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-aarch64-emit-relocs-relax.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# When mold relaxes an instruction sequence, the relocations attached to the
+# original instructions no longer match the relaxed code. With --emit-relocs,
+# mold must rewrite the emitted relocations so that their types describe the
+# relaxed instructions.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xassembler -
+.text
+.globl _start
+_start:
+  # GOT-load relaxation: ADRP+LDR => ADRP+ADD
+  #   R_AARCH64_ADR_GOT_PAGE     => R_AARCH64_ADR_PREL_PG_HI21
+  #   R_AARCH64_LD64_GOT_LO12_NC => R_AARCH64_ADD_ABS_LO12_NC
+  adrp x0, :got:gvar
+  ldr  x0, [x0, #:got_lo12:gvar]
+
+  # ADRP+ADD => NOP+ADR (gvar is within +-1 MiB)
+  #   R_AARCH64_ADR_PREL_PG_HI21 => R_AARCH64_NONE
+  #   R_AARCH64_ADD_ABS_LO12_NC  => R_AARCH64_ADR_PREL_LO21
+  adrp x1, gvar
+  add  x1, x1, #:lo12:gvar
+
+  # Both sequences must materialize the same address.
+  cmp  x0, x1
+  bne  1f
+
+  # ... and it must point at gvar, whose value is 42.
+  ldr  w2, [x0]
+  cmp  w2, #42
+  bne  1f
+
+  mov  x0, #0
+  b    2f
+1:
+  mov  x0, #1
+2:
+  mov  x8, #93           // __NR_exit
+  svc  #0
+
+.data
+.hidden gvar
+gvar:
+  .word 42
+EOF
+
+$CC -B. -o $t/exe $t/a.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax
+$QEMU $t/exe
+
+$OBJDUMP -dr $t/exe > $t/exe.objdump
+
+# Relaxation took place.
+grep -Fw nop $t/exe.objdump
+grep -Fw adr $t/exe.objdump
+grep -Fw add $t/exe.objdump
+
+# The relocations were rewritten to match the relaxed instructions.
+grep -Fw R_AARCH64_ADR_PREL_PG_HI21 $t/exe.objdump
+grep -Fw R_AARCH64_ADD_ABS_LO12_NC $t/exe.objdump
+grep -Fw R_AARCH64_ADR_PREL_LO21 $t/exe.objdump
+
+# The stale pre-relaxation relocations are gone.
+not grep -Fw R_AARCH64_ADR_GOT_PAGE $t/exe.objdump
+not grep -Fw R_AARCH64_LD64_GOT_LO12_NC $t/exe.objdump
+
+# Each rewritten relocation must sit on the relaxed instruction it describes.
+awk '
+  $3 == "adrp" || $3 == "add" || $3 == "adr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_ADR_PREL_PG_HI21" { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADD_ABS_LO12_NC"  { if (insn[$1] != "add")  { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADR_PREL_LO21"    { if (insn[$1] != "adr")  { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe.objdump

test/arch-aarch64-emit-relocs-tlsdesc-relax.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec
+# (LE) or initial-exec (IE) sequence. The four-instruction sequence is rewritten
+# in place: the ADRP and LDR become NOPs, and the ADD and BLR become the two
+# instructions that materialize the address. With --emit-relocs, the emitted
+# relocations must be rewritten to match whichever relaxation was applied.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC
+_Thread_local char foo[4] = "foo";
+_Thread_local char bar[4] = "bar";
+EOF
+
+cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC -mtls-dialect=desc -O2
+extern _Thread_local char foo[4];
+extern _Thread_local char bar[4];
+char *get_foo() { return foo; }
+char *get_bar() { return bar; }
+EOF
+
+cat <<EOF | $CC -o $t/c.o -c -xc - -mtls-dialect=desc
+#include <stdio.h>
+char *get_foo();
+char *get_bar();
+int main() { printf("%s %s\n", get_foo(), get_bar()); }
+EOF
+
+#
+# Local TLS in an executable: TLSDESC => LE
+#
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--emit-relocs,--relax
+$QEMU $t/exe1 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe1 > $t/exe1.objdump
+
+# The ADD/BLR became MOVZ/MOVK that build the TP-relative offset.
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/exe1.objdump
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G0_NC $t/exe1.objdump
+
+# No TLSDESC relocation survives.
+not grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe1.objdump
+
+# The G1 relocation sits on a movz and the G0_NC on a movk, not on the leading
+# adrp/ldr (which are now nops).
+awk '
+  $3 == "movz" || $3 == "movk" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G1"    { if (insn[$1] != "movz") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G0_NC" { if (insn[$1] != "movk") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe1.objdump
+
+#
+# TLS defined in a shared library: TLSDESC => IE
+#
+$CC -B. -shared -o $t/d.so $t/a.o -Wl,--relax
+$CC -B. -o $t/exe2 $t/b.o $t/c.o $t/d.so -Wl,--emit-relocs,--relax -Wl,-rpath=$t
+$QEMU $t/exe2 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe2 > $t/exe2.objdump
+
+# The ADD/BLR became ADRP/LDR that load the address from the GOT.
+grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/exe2.objdump
+grep -Fw R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe2.objdump
+
+awk '
+  $3 == "adrp" || $3 == "ldr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21"  { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC" { if (insn[$1] != "ldr") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe2.objdump
+
+#
+# Relocatable output: no relaxation happens and TLSDESC is not downgraded, so
+# every relocation must be passed through unchanged rather than rewritten as if
+# it had been relaxed to LE/IE.
+#
+$CC -B. -o $t/reloc.o $t/b.o -r
+$OBJDUMP -r $t/reloc.o > $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_CALL $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/reloc.objdump

test/arch-aarch64-long-thunk.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+cat <<EOF > $t/a.c
+#include <stdio.h>
+
+void fn3();
+void fn4();
+
+__attribute__((section(".low"))) void fn1() { printf(" fn1"); fn3(); }
+__attribute__((section(".low"))) void fn2() { printf(" fn2"); fn4(); }
+
+int main() {
+  printf(" main");
+  fn1();
+  printf("\n");
+}
+EOF
+
+cat <<EOF > $t/b.c
+#include <stdio.h>
+
+void fn1();
+void fn2();
+
+__attribute__((section(".high"))) void fn3() { printf(" fn3"); fn2(); }
+__attribute__((section(".high"))) void fn4() { printf(" fn4"); }
+EOF
+
+$CC -c -o $t/c.o $t/a.c
+$CC -c -o $t/d.o $t/b.c
+
+$CC -B. -o $t/exe1 $t/c.o $t/d.o \
+  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x100000000
+$QEMU $t/exe1 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -B. -o $t/exe2 $t/c.o $t/d.o \
+  -Wl,--section-start=.high=0x10000000,--section-start=.low=0x100000000
+$QEMU $t/exe2 | grep 'main fn1 fn3 fn2 fn4'
+
+
+$CC -c -o $t/e.o $t/a.c -fno-PIC -mcmodel=large
+$CC -c -o $t/f.o $t/b.c -fno-PIC -mcmodel=large
+
+$CC -B. -o $t/exe3 $t/e.o $t/f.o -pie \
+  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x400000000
+$QEMU $t/exe3 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -B. -o $t/exe4 $t/e.o $t/f.o -pie \
+  -Wl,--section-start=.high=0x10000000,--section-start=.low=0x400000000
+$QEMU $t/exe4 | grep 'main fn1 fn3 fn2 fn4'

test/arch-aarch64-pack-dyn-relocs.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = aarch64 ] || skip
+command -v timeout >/dev/null || skip
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip
+.section .custom_rodata_a, "a", %progbits
+.hidden A
+.global A
+A:
+  .space 8
+
+.section .custom_rodata_b_huge, "a", %progbits
+.balign 16384
+.hidden B
+.global B
+B:
+  .space 8
+
+.section .data, "aw", %progbits
+.rept 7439
+  .quad A
+.endr
+
+.global P_1
+P_1:
+  .quad A
+
+.global P_2
+P_2:
+  .quad B
+EOF
+
+timeout 5s ./mold -m aarch64elf -shared -o $t/b.so $t/a.o \
+  --pack-dyn-relocs=android -z max-page-size=16384

test/arch-aarch64-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-aarch64-variant-pcs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip

test/arch-arm-abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -fPIC -c -o $t/a.o -xassembler - -mthumb

test/arch-arm-exidx-sentinel.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CXX -o $t/a.o -c -xc++ - -fno-PIC

test/arch-arm-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-arm-range-extension-thunk.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 test_cflags -mthumb || skip

test/arch-arm-target1.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -xassembler -

test/arch-arm-thm-jump19.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-arm-thm-jump8.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -