Remove orphaned comments and tidy ScriptExpr/ScriptCmd enums

Co-Authored-By: Claude Opus 4.8 <[email protected]>
Make the reader state optional for the Script class
2026-11-09 04:49:33 -10:00 · 2026-06-13 14:40:42 +09:00 · 2026-06-12 12:49:56 +09:00 · 2026-06-12 12:21:30 +09:00 · 2026-06-11 14:03:04 +09:00 · 2026-06-11 13:28:28 +09:00
1277 changed files with 50764 additions and 23666 deletions
@@ -1,10 +0,0 @@
---
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
---
-
-Delete this message and describe the bug here. Note that we only fix bugs in the most recent release of mold, so please make sure that you are using the most recent version.
@@ -3,17 +3,39 @@ name: Build all tarballs
 on:
  schedule:
    - cron: '0 0 * * *'
-  release:
-    types: [ published ]
  workflow_dispatch:

 jobs:
-  build-tarballs:
-    runs-on: ubuntu-latest
+  check-changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      has_changes: ${{ github.event_name == 'workflow_dispatch' || steps.check.outputs.has_changes }}
+    steps:
+    - name: Check for new commits since last successful run
+      id: check
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        last=$(gh run list -R ${{ github.repository }} -w build-all.yml -b main -s success -L1 --json createdAt -q '.[0].createdAt')
+        if [ -z "$last" ] || [ "$last" = "null" ] || [ "$(gh api repos/${{ github.repository }}/commits?sha=main\&since=$last\&per_page=1 | jq length)" -gt 0 ]; then
+          echo "has_changes=true" >> "$GITHUB_OUTPUT"
+        fi

+  build-linux:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
    strategy:
      matrix:
-        target: [ x86_64, aarch64, arm, riscv64, ppc64le, s390x ]
+        include:
+          - { target: x86_64,      os: ubuntu-24.04 }
+          - { target: aarch64,     os: ubuntu-24.04-arm }
+          - { target: arm,         os: ubuntu-24.04-arm }
+          - { target: riscv64,     os: ubuntu-24.04 }
+          - { target: ppc64le,     os: ubuntu-24.04 }
+          - { target: s390x,       os: ubuntu-24.04 }
+          - { target: loongarch64, os: ubuntu-24.04 }
+
+    runs-on: ${{ matrix.os }}

    permissions:
      contents: read
@@ -21,26 +43,51 @@ jobs:

    steps:
    - name: Checkout Repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
+
+    - name: Install Podman
+      run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static

    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
+      uses: redhat-actions/podman-login@v1
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
    - name: Build a tarball
      run: ./dist.sh ${{ matrix.target }}

    - name: Upload artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
      with:
        name: ${{ matrix.target }}
-        path: mold-*.tar.gz
+        path: dist/mold-*.tar.gz
+        compression-level: 0
+
+  build-windows:
+    needs: check-changes
+    if: needs.check-changes.outputs.has_changes == 'true'
+    runs-on: windows-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-windows-x86_64.zip
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*
@@ -0,0 +1,44 @@
+name: Build native tarballs
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build-tarballs:
+    strategy:
+      matrix:
+        include:
+          - { target: x86_64,  os: ubuntu-24.04 }
+          - { target: aarch64, os: ubuntu-24.04-arm }
+          - { target: arm,     os: ubuntu-24.04-arm }
+
+    runs-on: ${{ matrix.os }}
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v6
+
+    - name: Install Podman
+      run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
+
+    - name: Login to GitHub Container Registry
+      uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build a tarball
+      run: ./dist.sh ${{ matrix.target }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v7
+      with:
+        name: ${{ matrix.target }}
+        path: dist/mold-*.tar.gz
+        compression-level: 0
@@ -1,37 +0,0 @@
-name: Build x86 tarball
-
-on:
-  push:
-    branches: [ main ]
-  workflow_dispatch:
-
-jobs:
-  build-tarball:
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-    - name: Checkout Repository
-      uses: actions/checkout@v2
-
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ github.repository_owner }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Build a tarball
-      run: ./dist.sh
-
-    - name: Upload artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: tarball
-        path: mold-*.tar.gz
@@ -5,7 +5,7 @@ on:
 env:
  UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
 jobs:
-  build-sanitizers:
+  build-asan-tsan:
    strategy:
      matrix:
        target:
@@ -13,8 +13,9 @@ jobs:
        - '-DMOLD_USE_ASAN=On'
        - '-DMOLD_USE_TSAN=On'
    runs-on: ubuntu-24.04
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
    - uses: rui314/setup-mold@staging
    - run: sudo ./install-build-deps.sh
    - name: build
@@ -24,9 +25,9 @@ jobs:
        cd build
        cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 ${{ matrix.target }} ..
        cmake --build . -j$(nproc)
-    - run: ctest --test-dir build -j$(nproc)
+    - run: cd build; ctest --output-on-failure -j$(nproc)
    - name: archive test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v7
      if: failure()
      with:
        name: test-results-clang
@@ -34,62 +35,40 @@ jobs:
          build
          !build/CMakeFiles

-  build-multi-archs:
-    runs-on: ubuntu-latest
-    container: gcc:11.1.0
+  build-msan:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
+    - uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - run: .github/workflows/run-msan.sh
+
+  build-multi-archs:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v6
    - name: install-build-deps
-      shell: bash
      run: |
-        # Install cross toolchains
-        ./install-build-deps.sh
-        ./install-cross-tools.sh
-
-        # Install a RV32 toolchain from third party since it's not available
-        # as an Ubuntu package.
-        mkdir /rv32
-        wget -O- -q https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2023.07.07/riscv32-glibc-ubuntu-20.04-gcc-nightly-2023.07.07-nightly.tar.gz | tar -C /rv32 --strip-components=1 -xzf -
-
-        ln -sf /rv32/sysroot /usr/riscv32-linux-gnu
-        echo '/rv32/bin/riscv32-unknown-linux-gnu-gcc -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-gcc
-        echo '/rv32/bin/riscv32-unknown-linux-gnu-g++ -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-g++
-        chmod 755 /usr/bin/riscv32-linux-gnu-{gcc,g++}
-
-        for i in objdump objcopy strip; do
-          ln -sf /rv32/bin/riscv32-unknown-linux-gnu-$i /usr/bin/riscv32-linux-gnu-$i
-        done
-
-        # Install a LoongArch toolchain
-        mkdir /larch
-        wget -O- -q https://github.com/loongson/build-tools/releases/download/2024.08.08/x86_64-cross-tools-loongarch64-binutils_2.43-gcc_14.2.0-glibc_2.40.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
-
-        cp -r /larch/loongarch64-unknown-linux-gnu/lib/* /larch/target/lib64
-        ln -sf /larch/target /usr/loongarch64-linux-gnu
-
-        for i in gcc g++ objdump objcopy strip; do
-          ln -sf /larch/bin/loongarch64-unknown-linux-gnu-$i /usr/bin/loongarch64-linux-gnu-$i
-        done
-
-        wget -O /usr/local/bin/qemu-loongarch64 -q https://github.com/loongson/build-tools/releases/download/2023.08.08/qemu-loongarch64
-        chmod 755 /usr/local/bin/qemu-loongarch64
-
-        # Install Intel SDE CPU emulator for CET-related tests
-        mkdir /sde
-        wget -O- -q https://downloadmirror.intel.com/813591/sde-external-9.33.0-2024-01-07-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
-        ln -s /sde/sde /usr/bin
+        sudo ./install-build-deps.sh
+        sudo ./install-cross-tools.sh
+        sudo .github/workflows/install-extras.sh
    - name: build
      run: |
        mkdir build
        cd build
        cmake ..
        cmake --build . -j$(nproc)
-    - run: ctest --test-dir build -j$(nproc)
+    - run: cd build; ctest --output-on-failure -j$(nproc)
    - name: archive test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v7
      if: failure()
      with:
-        name: test-results-gcc
+        name: test-results-multi-archs
        path: |
          build
          !build/CMakeFiles
@@ -100,14 +79,17 @@ jobs:
        distro:
        - alpine
        - archlinux
+        - debian:11 # GCC 10 and CMake 3.18 - the minimum supported by mold
        - fedora
        - gentoo/stage3
        - opensuse/tumbleweed
        - ubuntu:22.04
+        - ubuntu:25.04
    runs-on: ubuntu-latest
+    timeout-minutes: 60
    container: ${{ matrix.distro }}
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
    - run: ./install-build-deps.sh
    - name: build
      run: |
@@ -115,12 +97,25 @@ jobs:
        cd build
        cmake ..
        cmake --build . -j$(nproc)
-    - run: ctest --test-dir build -j$(nproc)
+    - run: cd build; ctest --output-on-failure -j$(nproc)
+    - name: sanitize distro name
+      if: failure()
+      id: sanitize
+      run: echo "name=$(echo '${{ matrix.distro }}' | tr '/:' '--')" >> "$GITHUB_OUTPUT"
+    - name: archive test results
+      uses: actions/upload-artifact@v7
+      if: failure()
+      with:
+        name: test-results-${{ steps.sanitize.outputs.name }}
+        path: |
+          build
+          !build/CMakeFiles

  build-macos:
-    runs-on: macos-12
+    runs-on: macos-latest
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
    - name: build
      run: |
        mkdir build
@@ -130,8 +125,9 @@ jobs:

  build-windows:
    runs-on: windows-latest
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
    - name: build
      run: |
        mkdir build
@@ -141,8 +137,9 @@ jobs:

  build-msys:
    runs-on: windows-latest
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
    - name: Setup MSYS2
      uses: msys2/setup-msys2@v2
      with:
@@ -159,8 +156,9 @@ jobs:

  build-freebsd:
    runs-on: ubuntu-latest
+    timeout-minutes: 60
    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
    - name: Build and test
      uses: vmactions/freebsd-vm@v1
      with:
@@ -171,4 +169,4 @@ jobs:
          cd build
          cmake ..
          cmake --build . -j$(nproc)
-          ctest -j$(nproc)
+          ctest --output-on-failure -j$(nproc)
@@ -0,0 +1,75 @@
+#!/bin/bash -x
+
+apt-get update
+apt-get install -y wget xz-utils
+
+# Install a 32-bit RISC-V toolchain
+mkdir /rv32
+wget -O- --progress=dot:mega https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.06.13/riscv32-glibc-ubuntu-24.04-gcc-nightly-2025.06.13-nightly.tar.xz | tar -C /rv32 --strip-components=1 --xz -xf -
+
+ln -sf /rv32/sysroot /usr/riscv32-linux-gnu
+echo '/rv32/bin/riscv32-unknown-linux-gnu-gcc -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-gcc
+echo '/rv32/bin/riscv32-unknown-linux-gnu-g++ -L/usr/riscv32-linux-gnu "$@"' > /usr/bin/riscv32-linux-gnu-g++
+chmod 755 /usr/bin/riscv32-linux-gnu-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /rv32/bin/riscv32-unknown-linux-gnu-$i /usr/bin/riscv32-linux-gnu-$i
+done
+
+# Install a LoongArch toolchain
+mkdir /larch
+wget -O- --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/x86_64-cross-tools-loongarch64-binutils_2.44-gcc_15.1.0-glibc_2.41.tar.xz | tar -C /larch --strip-components=1 --xz -xf -
+
+cp -r /larch/loongarch64-unknown-linux-gnu/lib/* /larch/target/lib64
+ln -sf /larch/target /usr/loongarch64-linux-gnu
+
+for i in gcc g++ objdump objcopy strip; do
+  ln -sf /larch/bin/loongarch64-unknown-linux-gnu-$i /usr/bin/loongarch64-linux-gnu-$i
+done
+
+wget -O /usr/local/bin/qemu-loongarch64 --progress=dot:mega https://github.com/loongson/build-tools/releases/download/2025.06.06/qemu-loongarch64
+chmod 755 /usr/local/bin/qemu-loongarch64
+
+# Install ARM32 big-endian toolchain
+mkdir /armeb
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/armebv7-eabihf/tarballs/armebv7-eabihf--glibc--stable-2024.05-1.tar.xz | tar -C /armeb --strip-components=1 --xz -xf -
+
+ln -sf /armeb/armeb-buildroot-linux-gnueabihf/sysroot /usr/armeb-linux-gnueabihf
+echo '/armeb/bin/armeb-linux-gcc -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-gcc
+echo '/armeb/bin/armeb-linux-g++ -L/usr/armeb-linux-gnu "$@"' > /usr/bin/armeb-linux-gnueabihf-g++
+chmod 755 /usr/bin/armeb-linux-gnueabihf-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /armeb/bin/armeb-linux-$i /usr/bin/armeb-linux-gnueabihf-$i
+done
+
+# Install ARM64 big-endian toolchain
+mkdir /aarch64be
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64be/tarballs/aarch64be--glibc--stable-2024.05-1.tar.xz | tar -C /aarch64be --strip-components=1 --xz -xf -
+
+ln -sf /aarch64be/aarch64_be-buildroot-linux-gnu/sysroot /usr/aarch64_be-linux-gnu
+echo '/aarch64be/bin/aarch64_be-linux-gcc -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-gcc
+echo '/aarch64be/bin/aarch64_be-linux-g++ -L/usr/aarch64_be-linux-gnu "$@"' > /usr/bin/aarch64_be-linux-gnu-g++
+chmod 755 /usr/bin/aarch64_be-linux-gnu-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /aarch64be/bin/aarch64_be-linux-$i /usr/bin/aarch64_be-linux-gnu-$i
+done
+
+# Install SH4 big-endian toolchain
+mkdir /sh4aeb
+wget -O- --progress=dot:mega https://toolchains.bootlin.com/downloads/releases/toolchains/sh-sh4aeb/tarballs/sh-sh4aeb--glibc--stable-2024.05-1.tar.xz | tar -C /sh4aeb --strip-components=1 --xz -xf -
+
+ln -sf /sh4aeb/sh4aeb-buildroot-linux-gnu/sysroot /usr/sh4aeb-linux-gnu
+echo '/sh4aeb/bin/sh4aeb-linux-gcc -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-gcc
+echo '/sh4aeb/bin/sh4aeb-linux-g++ -L/usr/sh4aeb-linux-gnu "$@"' > /usr/bin/sh4aeb-linux-gnu-g++
+chmod 755 /usr/bin/sh4aeb-linux-gnu-{gcc,g++}
+
+for i in objdump objcopy strip; do
+  ln -sf /sh4aeb/bin/sh4aeb-linux-$i /usr/bin/sh4aeb-linux-gnu-$i
+done
+
+# Install Intel SDE CPU emulator for CET-related tests
+mkdir /sde
+wget -O- --progress=dot:mega https://downloadmirror.intel.com/850782/sde-external-9.53.0-2025-03-16-lin.tar.xz | tar -C /sde --strip-components=1 --xz -xf -
+ln -s /sde/sde64 /usr/bin
@@ -0,0 +1,104 @@
+name: Build & attach tarballs on tag push
+
+on:
+  push:
+    tags: ['v[0-9]*']
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+env:
+  TAG: ${{ github.ref_name }}
+
+jobs:
+  build-linux:
+    strategy:
+      matrix:
+        include:
+        - { target: x86_64,      os: ubuntu-24.04 }
+        - { target: aarch64,     os: ubuntu-24.04-arm }
+        - { target: arm,         os: ubuntu-24.04-arm }
+        - { target: riscv64,     os: ubuntu-24.04 }
+        - { target: ppc64le,     os: ubuntu-24.04 }
+        - { target: s390x,       os: ubuntu-24.04 }
+        - { target: loongarch64, os: ubuntu-24.04 }
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install Podman
+      run: sudo apt-get update && sudo apt-get install -y podman qemu-user-static
+
+    - name: Login to GitHub Container Registry
+      uses: redhat-actions/podman-login@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - run: ./dist.sh ${{ matrix.target }}
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: ${{ matrix.target }}
+        path: dist/mold-*.tar.gz
+
+  build-windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Build and Archive
+      shell: pwsh
+      run: |
+        mkdir build
+        cd build
+        cmake -T clangcl ..
+        cmake --build . --config Release -j $Env:NUMBER_OF_PROCESSORS
+        cmake --install . --config Release --prefix ../mold-install
+        cd ..
+        New-Item -ItemType Directory -Force dist | Out-Null
+        $version = $Env:TAG -replace '^v', ''
+        Compress-Archive -Path mold-install\* -DestinationPath dist\mold-$version-x86_64-windows.zip
+
+    - uses: actions/upload-artifact@v7
+      with:
+        name: win-x86_64
+        path: dist/mold-*.*
+
+  publish:
+    needs: [build-linux, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - uses: actions/download-artifact@v8
+      with:
+        path: dist
+        pattern: '*'
+        merge-multiple: true
+
+    - name: Ensure release exists (create if missing)
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -e
+        if gh release view "$TAG" >& /dev/null; then
+          echo "Existing release found for $TAG"
+        else
+          echo "Creating draft release for $TAG"
+          gh release create "$TAG" --draft \
+            --title "$TAG" \
+            --notes "Automated draft – fill the changelog later."
+        fi
+
+    - name: Upload tarballs
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        gh release upload "$TAG" dist/* --clobber
+        gh release view "$TAG" --json assets --jq '.assets[].name'
@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e -x
+cd "$(dirname $0)"/../..
+
+if [ "$GITHUB_REPOSITORY" = '' ]; then
+  image=mold-msan
+  image_build="podman build -t $image -"
+else
+  # If this script is running on GitHub Actions, we want to cache
+  # the created container image in GitHub's container repostiory.
+  image=ghcr.io/$GITHUB_REPOSITORY/mold-msan
+  image_build="podman build -t $image --output=type=registry --layers --cache-to $image --cache-from $image -"
+fi
+
+cat <<EOF | $image_build
+FROM mirror.gcr.io/library/ubuntu:24.04
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc g++ wget ca-certificates make cmake ninja-build python3 clang-20 llvm-20-dev libclang-20-dev libclang-rt-20-dev libtbb-dev mold && \
+  rm -rf /var/lib/apt/lists
+RUN mkdir /llvm && \
+  cd /llvm && \
+  wget -O- --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-21.1.1.tar.gz | tar xzf - --strip-components=1 && \
+  cmake -S ./runtimes -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi;libunwind' -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DLLVM_USE_SANITIZER=MemoryWithOrigins && \
+  cmake --build build -j\$(nproc) && \
+  cmake --install build --prefix /msan && \
+  rm -rf /llvm
+EOF
+
+# Build mold and run its tests
+podman run -it --rm --pull=never -v "$(pwd):/mold:ro" --security-opt seccomp=unconfined $image bash -c "
+set -e
+mkdir /build
+cd /build
+cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 -DMOLD_USE_MSAN=ON -DMOLD_STDLIB_PREFIX=/msan -DMOLD_USE_MIMALLOC=0 -DMOLD_USE_SYSTEM_TBB=1 -DMOLD_USE_MOLD=1 /mold
+cmake --build . -j\$(nproc)
+setarch -R ctest --output-on-failure -j\$(nproc)
+"
@@ -14,7 +14,7 @@ jobs:

    steps:
    - name: Check out repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6

    - name: Install ronn
      run: sudo apt-get update && sudo apt-get install -y ronn
@@ -8,3 +8,4 @@ gentoo
 /mold-*.tar.gz
 /build*
 /mold
+/dist
@@ -44,18 +44,12 @@
 # features and behave exactly the same.

 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.34.0)
+project(mold VERSION 2.41.0)

 include(CMakeDependentOption)
 include(CheckSymbolExists)
 include(GNUInstallDirs)

-# Build mold itself using mold if -DMOLD_USE_MOLD=ON
-option(MOLD_USE_MOLD "Use mold to build mold" OFF)
-if(MOLD_USE_MOLD)
-  add_link_options(-fuse-ld=mold -Wl,--gdb-index)
-endif()
-
 add_executable(mold)
 target_compile_features(mold PRIVATE cxx_std_20)

@@ -65,11 +59,27 @@ else()
  target_link_libraries(mold PRIVATE ${CMAKE_DL_LIBS})
 endif()

+# Build mold itself using mold if -DMOLD_USE_MOLD=ON
+option(MOLD_USE_MOLD "Use mold to build mold" OFF)
+if(MOLD_USE_MOLD)
+  target_link_options(mold PRIVATE -fuse-ld=mold)
+
+  if(CMAKE_BUILD_TYPE MATCHES "Deb")
+    target_link_options(mold PRIVATE -Wl,--gdb-index)
+  endif()
+
+  if(CMAKE_BUILD_TYPE MATCHES "^Rel")
+    target_link_options(mold PRIVATE -Wl,--gc-sections,--icf=safe)
+  endif()
+endif()
+
 if(NOT "${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC")
  target_compile_options(mold PRIVATE
    -fno-exceptions
    -fno-unwind-tables
    -fno-asynchronous-unwind-tables
+    -ffunction-sections
+    -fdata-sections
    -Wall
    -Wextra
    -Wno-sign-compare
@@ -107,6 +117,22 @@ if(MOLD_USE_TSAN)
  target_link_options(mold PRIVATE -fsanitize=thread)
 endif()

+# Enable MemorySanitizer if -DMOLD_USE_MSAN=ON
+option(MOLD_USE_MSAN "Build mold with MemorySanitizer" OFF)
+if(MOLD_USE_MSAN)
+  set(MOLD_STDLIB_PREFIX /usr CACHE FILEPATH "prefix of instrumented stdlib")
+  target_compile_options(mold PRIVATE
+    -g -Og -fsanitize=memory -fsanitize-memory-track-origins
+    -nostdinc++ -isystem ${MOLD_STDLIB_PREFIX}/include/c++/v1
+    -DENABLE_MSAN_UNPOISON
+  )
+  target_link_options(mold PRIVATE
+    -fsanitize=memory
+    -nostdlib++ -L ${MOLD_STDLIB_PREFIX}/lib -lc++
+    -Wl,-rpath,${MOLD_STDLIB_PREFIX}/lib
+  )
+endif()
+
 # Statically-link libstdc++ if -DMOLD_MOSTLY_STATIC=ON.
 #
 # This option is intended to be used by `./dist.sh` script to create a
@@ -154,13 +180,13 @@ check_include_file(zstd.h HAVE_ZSTD_H)
 if(HAVE_ZSTD_H AND NOT MOLD_MOSTLY_STATIC)
  target_link_libraries(mold PRIVATE zstd)
 else()
+  set(ZSTD_BUILD_PROGRAMS OFF)
+  set(ZSTD_BUILD_CONTRIB OFF)
+  set(ZSTD_BUILD_TESTS OFF)
+  set(ZSTD_MULTITHREAD_SUPPORT OFF)
+  set(ZSTD_BUILD_SHARED OFF)
+  set(ZSTD_BUILD_STATIC ON)
  add_subdirectory(third-party/zstd/build/cmake EXCLUDE_FROM_ALL)
-  target_compile_definitions(libzstd_static PRIVATE
-    ZSTD_BUILD_STATIC=1
-    ZSTD_BUILD_SHARED=0
-    ZSTD_BUILD_PROGRAMS=0
-    ZSTD_MULTITHREAD_SUPPORT=0
-    ZSTD_BUILD_TESTS=0)
  target_include_directories(mold PUBLIC third-party/zstd/lib)
  target_link_libraries(mold PRIVATE libzstd_static)
 endif()
@@ -174,7 +200,7 @@ endif()
 # be stable on 32-bit targets.
 cmake_dependent_option(
  MOLD_USE_MIMALLOC "Use mimalloc" ON
-  "CMAKE_SIZEOF_VOID_P EQUAL 8; NOT APPLE; NOT ANDROID; NOT OPENBSD" OFF)
+  "CMAKE_SIZEOF_VOID_P EQUAL 8; NOT APPLE; NOT ANDROID; NOT OPENBSD; NOT MOLD_USE_ASAN; NOT MOLD_USE_TSAN" OFF)

 cmake_dependent_option(
  MOLD_USE_SYSTEM_MIMALLOC "Use system or vendored mimalloc" OFF
@@ -187,11 +213,11 @@ if(MOLD_USE_MIMALLOC)
  if(MOLD_USE_SYSTEM_MIMALLOC)
    find_package(mimalloc REQUIRED)
    target_link_libraries(mold PRIVATE mimalloc)
-    target_compile_definitions(mold PRIVATE USE_SYSTEM_MIMALLOC)
  else()
    function(mold_add_mimalloc)
      set(MI_BUILD_STATIC ON CACHE INTERNAL "")
      set(MI_BUILD_TESTS OFF CACHE INTERNAL "")
+      set(MI_NO_OPT_ARCH ON CACHE INTERNAL "")
      add_subdirectory(third-party/mimalloc EXCLUDE_FROM_ALL)
      target_compile_definitions(mimalloc-static PRIVATE MI_USE_ENVIRON=0)
      target_link_libraries(mold PRIVATE mimalloc-static)
@@ -208,7 +234,7 @@ endif()
 # to mold. If you want to link to the system's libtbb2.so, pass
 # -DMOLD_USE_SYSTEM_TBB=ON.
 option(MOLD_USE_SYSTEM_TBB "Use system or vendored TBB" OFF)
-if(MOLD_USE_SYSTEM_TBB)
+if(MOLD_USE_SYSTEM_TBB OR BLAKE3_USE_TBB)
  find_package(TBB REQUIRED)
  target_link_libraries(mold PRIVATE TBB::tbb)
 else()
@@ -278,23 +304,20 @@ if(NOT APPLE AND NOT MSVC)
 endif()

 check_symbol_exists(madvise sys/mman.h HAVE_MADVISE)
+check_symbol_exists(uname sys/utsname.h HAVE_UNAME)

 # Create a .cc file containing the current git hash for `mold --version`.
 add_custom_target(git_hash
  COMMAND ${CMAKE_COMMAND}
    -DSOURCE_DIR=${CMAKE_SOURCE_DIR}
-    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/git-hash.cc
+    -DOUTPUT_FILE=${CMAKE_BINARY_DIR}/mold-git-hash.h
    -P ${CMAKE_SOURCE_DIR}/lib/update-git-hash.cmake
  DEPENDS lib/update-git-hash.cmake
-  BYPRODUCTS git-hash.cc
+  BYPRODUCTS mold-git-hash.h
  VERBATIM)

 add_dependencies(mold git_hash)

-# Create config.h file
-configure_file(lib/config.h.in config.h)
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
 # Almost all functions are template in mold which take a target type
 # (e.g. X86_64) as its type parameter. Since we suport more than 10
 # targets, compiling a single source file for all the targets is very
@@ -303,14 +326,39 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # As a workaround, we create a .cc file for each target and spawn many
 # compiler instances. This is hacky but greatly reduces compile time
 # on a multicore machine.
-list(APPEND MOLD_ELF_TARGETS
-  X86_64 I386 ARM64 ARM32 RV32LE RV32BE RV64LE RV64BE PPC32 PPC64V1 PPC64V2
-  S390X SPARC64 M68K SH4 LOONGARCH32 LOONGARCH64)
+#
+# You can build mold for a specific set of targets by passing, e.g.,
+# -DMOLD_TARGETS='X86_64;ARM64LE', though this is strongly discouraged
+# for build reproducibility. Use this option only if you build mold
+# frequently for your personal use; otherwise, always build mold with all
+# targets enabled. We provide this flag in the trust that you will not
+# abuse it, so please don’t betray that trust.
+set(MOLD_TARGETS
+  X86_64 I386 ARM64LE ARM64BE ARM32LE ARM32BE RV32LE RV32BE RV64LE
+  RV64BE PPC32 PPC64V1 PPC64V2 S390X SPARC64 M68K SH4LE SH4BE
+  LOONGARCH32 LOONGARCH64
+  CACHE STRING "List of supported targets")

-list(APPEND MOLD_ELF_TEMPLATE_FILES
+list(GET MOLD_TARGETS 0 MOLD_FIRST_TARGET)
+
+list(APPEND MOLD_TEMPLATE_FILES
+  src/arch-arm32.cc
+  src/arch-arm64.cc
+  src/arch-i386.cc
  src/arch-loongarch.cc
+  src/arch-m68k.cc
+  src/arch-ppc32.cc
+  src/arch-ppc64v1.cc
+  src/arch-ppc64v2.cc
  src/arch-riscv.cc
+  src/arch-s390x.cc
+  src/arch-sh4.cc
+  src/arch-sparc64.cc
+  src/arch-x86-64.cc
+  src/archive-file.cc
  src/cmdline.cc
+  src/error.cc
+  src/filetype.cc
  src/gc-sections.cc
  src/gdb-index.cc
  src/icf.cc
@@ -328,18 +376,18 @@ list(APPEND MOLD_ELF_TEMPLATE_FILES
  )

 if(WIN32 AND NOT MINGW)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-win32.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-win32.cc)
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES src/lto-unix.cc)
+  list(APPEND MOLD_TEMPLATE_FILES src/lto-unix.cc)
 endif()

 if(WIN32)
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
    src/output-file-win32.cc
    src/subprocess-win32.cc
    )
 else()
-  list(APPEND MOLD_ELF_TEMPLATE_FILES
+  list(APPEND MOLD_TEMPLATE_FILES
    src/output-file-unix.cc
    src/subprocess-unix.cc
  )
@@ -356,56 +404,49 @@ function(mold_instantiate_templates SOURCE TARGET)
  target_sources(mold PRIVATE ${PATH})
 endfunction()

-foreach (SOURCE IN LISTS MOLD_ELF_TEMPLATE_FILES)
-  foreach(TARGET IN LISTS MOLD_ELF_TARGETS)
+foreach (SOURCE IN LISTS MOLD_TEMPLATE_FILES)
+  foreach(TARGET IN LISTS MOLD_TARGETS)
    mold_instantiate_templates(${SOURCE} ${TARGET})
+    set(HAVE_TARGET_${TARGET} 1)
  endforeach()
 endforeach()

 # Add other non-template source files.
 target_sources(mold PRIVATE
-  git-hash.cc
+  lib/aho-corasick.cc
  lib/compress.cc
  lib/crc32.cc
  lib/demangle.cc
  lib/filepath.cc
  lib/glob.cc
  lib/hyperloglog.cc
-  lib/malloc.cc
-  lib/multi-glob.cc
  lib/perf.cc
  lib/random.cc
  lib/tar.cc
-  src/arch-arm32.cc
-  src/arch-arm64.cc
-  src/arch-i386.cc
-  src/arch-m68k.cc
-  src/arch-ppc32.cc
-  src/arch-ppc64v1.cc
-  src/arch-ppc64v2.cc
-  src/arch-s390x.cc
-  src/arch-sh4.cc
-  src/arch-sparc64.cc
-  src/arch-x86-64.cc
-  src/config.cc
  src/elf.cc
+  src/entry.cc
  third-party/rust-demangle/rust-demangle.c
  )

 if(WIN32)
  target_sources(mold PRIVATE
-    lib/jobs-win32.cc
-    lib/mapped-file-win32.cc
-    lib/signal-win32.cc
+    src/jobs-win32.cc
+    src/mapped-file-win32.cc
+    src/signal-win32.cc
    )
 else()
  target_sources(mold PRIVATE
-    lib/jobs-unix.cc
-    lib/mapped-file-unix.cc
-    lib/signal-unix.cc
+    src/jobs-unix.cc
+    src/mapped-file-unix.cc
+    src/signal-unix.cc
    )
 endif()

+# Create config.h file
+configure_file(lib/config.h.in config.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+# Test configs
 include(CTest)

 if(BUILD_TESTING)
@@ -428,7 +469,6 @@ if(NOT CMAKE_SKIP_INSTALL_RULES)
  install(TARGETS mold RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
  install(FILES docs/mold.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1/)
  install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR})
-  install(FILES "LICENSE.third-party" DESTINATION "${CMAKE_INSTALL_DOCDIR}")

  function(mold_install_relative_symlink OLD NEW)
    install(CODE "
@@ -442,10 +482,12 @@ if(NOT CMAKE_SKIP_INSTALL_RULES)
      file(CREATE_LINK \${OLD_REL} \$ENV{DESTDIR}\${NEW_ABS} SYMBOLIC)")
  endfunction()

-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
-    ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
-  mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
-    ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  if(NOT WIN32)
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_LIBEXECDIR}/mold/ld${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_BINDIR}/mold${CMAKE_EXECUTABLE_SUFFIX}
+      ${CMAKE_INSTALL_BINDIR}/ld.mold${CMAKE_EXECUTABLE_SUFFIX})
+    mold_install_relative_symlink(${CMAKE_INSTALL_MANDIR}/man1/mold.1
+      ${CMAKE_INSTALL_MANDIR}/man1/ld.mold.1)
+  endif()
 endif()
@@ -48,7 +48,7 @@ Binary packages for the following systems are currently available:

 mold is written in C++20, so if you build mold yourself, you will need a
 recent version of a C++ compiler and a C++ standard library. We recommend GCC
-10.2 or Clang 12.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).
+10.2 or Clang 16.0.0 (or later) and libstdc++ 10 or libc++ 7 (or later).

 ### Install Dependencies

@@ -76,12 +76,12 @@ installation location by passing `-DCMAKE_INSTALL_PREFIX=<directory>`.
 For other cmake options, see the comments in `CMakeLists.txt`.

 If you are not using a recent enough Linux distribution, or if `cmake` does
-not work for you for any reason, you can use Docker to build mold in a Docker
-environment. To do so, run `./dist.sh` in this directory instead of using
-`cmake`. The shell script will pull a Docker image, build mold and auxiliary
+not work for you for any reason, you can use Podman to build mold in a
+container. To do so, run `./dist.sh` in this directory instead of using
+`cmake`. The shell script will pull a container image, build mold and auxiliary
 files inside it, and package them into a single tar file named
-`mold-$version-$arch-linux.tar.gz`. You can extract the tar file anywhere and
-use the mold executable within it.
+`dist/mold-$version-$arch-linux.tar.gz`. You can extract the tar file anywhere
+and use the mold executable in it.

 ## How to use

@@ -120,7 +120,7 @@ accept an absolute path as an argument for `-fuse-ld`.
 Create `.cargo/config.toml` in your project directory with the following:

 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 linker = "clang"
 rustflags = ["-C", "link-arg=-fuse-ld=/path/to/mold"]
 ```
@@ -131,7 +131,7 @@ example above, we use `clang` as a linker driver since it always accepts the
 may be able to remove the `linker = "clang"` line.

 ```toml
-[target.x86_64-unknown-linux-gnu]
+[target.'cfg(target_os = "linux")']
 rustflags = ["-C", "link-arg=-fuse-ld=mold"]
 ```

@@ -158,6 +158,14 @@ If you want to use mold for all projects, add the above snippet to

 </details>

+<details><summary>If you are using Conan package manager</summary>
+
+You can configure [Conan](https://github.com/conan-io) to download the latest
+version of `mold` and use it as the linker when building your dependencies and
+projects from source. Please see the instructions [here](https://conan.io/center/recipes/mold).
+
+</details>
+
 <details><summary>mold -run</summary>

 It is sometimes very hard to pass an appropriate command line option to `cc`
@@ -264,3 +272,4 @@ the following people and organizations who have sponsored $128/month or more:
 - [Bryant Biggs](https://github.com/bryantbiggs)
 - [kraptor23](https://github.com/kraptor23)
 - [Jinkyu Yi](https://github.com/jincreator)
+- [Pedro Navarro](https://github.com/pedronavf)
@@ -1,42 +1,51 @@
 #!/bin/bash
 #
-# This script creates a mold binary distribution. The output is written in
-# this directory as `mold-$version-$arch-linux.tar.gz` (e.g.,
-# `mold-1.0.3-x86_64-linux.tar.gz`).
+# This script creates a mold binary distribution. The output is written to
+# the `dist` directory as `mold-$version-$arch-linux.tar.gz` (e.g.
+# `mold-2.40.0-x86_64-linux.tar.gz`).
 #
-# The mold executable created by this script is statically linked to
-# libstdc++ but dynamically linked to libc, libm, libz, and librt, as
-# these libraries almost always exist on any Linux system. We can't
-# statically link libc because doing so would disable dlopen(), which is
-# necessary to open the LTO linker plugin.
-#
-# This script aims to produce reproducible outputs. That means if you run
-# the script twice on the same git commit, it should produce bit-by-bit
-# identical binary files. This property is crucial as a countermeasure
-# against supply chain attacks. With this, you can verify that the binary
-# files distributed on the GitHub release pages are created from the
-# commit with release tags by rebuilding the binaries yourself.
+# This script aims to produce reproducible outputs. That means each time
+# it's run on the same git commit, it generates a bit-for-bit identical
+# binary file regardless of when or where it's executed. This property
+# serves as a strong safeguard against supply chain attacks. With a
+# reproducible build, anyone can independently verify that the binary
+# files published on our GitHub release page were built from the git
+# commit tagged for release by rebuilding the binaries themselves.
 #
 # Debian provides snapshot.debian.org to host all historical binary
-# packages. We use it to construct Docker images pinned to a
-# particular timestamp.
+# packages. We use it to construct a container image pinned to a
+# particular timestamp. snapshot.debian.org is known to be very slow,
+# but that shouldn't be a big problem for us because we only need that
+# site the first time.
 #
-# We aim to use a reasonably old Debian version because we'll dynamically
-# link glibc to mold, and a binary linked against a newer version of glibc
-# won't work on a system with an older version of glibc.
+# The mold executable created by this script is statically linked to
+# libstdc++, but dynamically linked to glibc, libm and a few other
+# libraries, as these libraries are almost always available on any Linux
+# system. We can't statically link glibc because doing so would disable
+# dlopen(), which is required to load the LTO linker plugin.
 #
-# We need GCC 10 or newer to build mold. If GCC 10 is not available on an
-# old Debian version, we'll build it ourselves.
+# We use a reasonably old Debian version for the build environment because
+# a binary dynamically linked against a newer version of glibc won't work
+# on a system with an older version of glibc.
 #
-# You may need to run the following command to use Docker with Qemu:
+# We prefer to build mold with Clang rather than GCC because mold's
+# Identical Code Folding works best with the LLVM address significance
+# table (.llvm_addrsig). Building a release binary with GCC produces a
+# slightly larger binary than with Clang.
 #
-#  $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# We need a recent version of Clang to build mold. If it's not available
+# via apt-get, we'll build it ourselves.
+#
+# This script can be used to create non-native binaries (e.g., building
+# aarch64 binary on x86-64) because Podman automatically runs everything
+# under QEMU if the container image is not native. To use this script for
+# non-native builds, you may need to install the qemu-user-static package.

 set -e -x
 cd "$(dirname $0)"

 usage() {
-  echo "Usage: $0 [ x86_64 | aarch64 | arm | riscv64 | ppc64le | s390x ]"
+  echo "Usage: $0 [ x86_64 | aarch64 | arm | riscv64 | ppc64le | s390x | loongarch64 ]"
  exit 1
 }

@@ -56,119 +65,183 @@ case $# in
  usage
 esac

-echo "$arch" | grep -Eq '^(x86_64|aarch64|arm|riscv64|ppc64le|s390x)$' || usage
-
-version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' CMakeLists.txt)
-dest=mold-$version-$arch-linux
-
+# Create a Podman image.
 if [ "$GITHUB_REPOSITORY" = '' ]; then
  image=mold-builder-$arch
-  docker_build="docker build --platform linux/$arch -t $image -"
+  image_build="podman build --arch $arch -t $image -"
 else
  # If this script is running on GitHub Actions, we want to cache
-  # the created Docker image in GitHub's Docker repostiory.
+  # the created container image in GitHub's container repostiory.
  image=ghcr.io/$GITHUB_REPOSITORY/mold-builder-$arch
-  docker_build="docker buildx build --platform linux/$arch -t $image --push --cache-to type=inline --cache-from type=registry,ref=ghcr.io/$GITHUB_REPOSITORY/mold-builder-$arch -"
+  image_build="podman build --arch $arch -t $image --output=type=registry --layers --cache-to $image --cache-from $image -"
 fi

-# Create a Docker image.
 case $arch in
 x86_64)
-  # Debian 8 (Jessie) released in April 2015
-  cat <<EOF | $docker_build
-FROM debian:jessie-20210326@sha256:32ad5050caffb2c7e969dac873bce2c370015c2256ff984b70c1c08b3a2816a0
+  # Debian 9 (Stretch) released in June 2017.
+  #
+  # We use a Google-provided mirror (gcr.io) instead of the official Docker
+  # Hub (docker.io) because docker.io has a strict rate limit policy.
+  #
+  # The toolchain in Debian 9 is too old to build mold, so we rebuild it
+  # from source. We download source archives from official sites and build
+  # them locally, rather than downloading pre-built binaries from somewhere
+  # else, to avoid relying on unverifiable third-party binary blobs. Podman
+  # caches the result of each RUN command, so rebuilding is done only once
+  # per host.
+  cat <<EOF | $image_build
+FROM mirror.gcr.io/library/debian:stretch@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
  apt-get update && \
-  apt-get install -y --no-install-recommends wget bzip2 file make autoconf gcc g++ libssl-dev && \
+  apt-get install -y --no-install-recommends wget file make gcc g++ zlib1g-dev libssl-dev ca-certificates && \
  rm -rf /var/lib/apt/lists

 # Build CMake 3.27
 RUN mkdir /build && \
  cd /build && \
-  wget -O- --no-check-certificate https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
  ./bootstrap --parallel=\$(nproc) && \
  make -j\$(nproc) && \
  make install && \
  rm -rf /build

-# Build GCC 10
+# Build GCC 14
 RUN mkdir /build && \
  cd /build && \
-  wget -O- --no-check-certificate https://ftpmirror.gnu.org/gnu/gcc/gcc-10.5.0/gcc-10.5.0.tar.gz | tar xzf - --strip-components=1 && \
-  mkdir isl gmp mpc mpfr && \
-  wget -O- --no-check-certificate https://gcc.gnu.org/pub/gcc/infrastructure/isl-0.18.tar.bz2 | tar xjf - --strip-components=1 -C isl && \
-  wget -O- --no-check-certificate https://ftpmirror.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2 | tar xjf - --strip-components=1 -C gmp && \
-  wget -O- --no-check-certificate https://ftpmirror.gnu.org/gnu/mpc/mpc-1.2.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
-  wget -O- --no-check-certificate https://ftpmirror.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
+  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
+  mkdir gmp mpc mpfr && \
+  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
+  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
+  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
  ./configure --prefix=/usr --enable-languages=c,c++ --disable-bootstrap --disable-multilib && \
  make -j\$(nproc) && \
  make install && \
  ln -sf /usr/lib64/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6 && \
  rm -rf /build
+
+# Build GNU binutils 2.43
+RUN mkdir /build && \
+  cd /build && \
+  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
+  ./configure --prefix=/usr && \
+  make -j\$(nproc) && \
+  make install && \
+  rm -fr /build
+
+# Build Python 3.12.7
+RUN mkdir /build && \
+  cd /build && \
+  wget -O- --progress=dot:mega https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz | tar xzf - --strip-components=1 && \
+  ./configure && \
+  make -j\$(nproc) && \
+  make install && \
+  rm -rf /build
+
+# Build LLVM 20
+RUN mkdir /build && \
+  cd /build && \
+  wget -O- --progress=dot:mega https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.3.tar.gz | tar xzf - --strip-components=1 && \
+  mkdir b && \
+  cd b && \
+  cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang ../llvm && \
+  cmake --build . -j\$(nproc) && \
+  cmake --install . --strip && \
+  rm -rf /build
 EOF
  ;;
 aarch64 | arm | ppc64le | s390x)
-  # Debian 10 (Bullseye) released in July 2019
+  # Debian 11 (Bullseye) released in August 2021
  #
-  # We don't want to build GCC for these targets with Qemu becuase
-  # that'd take extremely long time. Also I believe old build machines
-  # are usually x86-64.
-  [ $arch = aarch64 ] && digest=d5ed76c5265576982e6599b6f12392290d9b52b315b19b28b640aaba6e8af002
-  [ $arch = arm ]     && digest=bede2623dae269454c5b6dd4af15a10810a5f4ef75963d4eb6531628f98bd633
-  [ $arch = ppc64le ] && digest=255f385e735469493b3465befad59a16f9d46f41d0b50e4fa6d5928c5ee7702a
-  [ $arch = s390x ]   && digest=96fb9ce5d3ce7f3dab7c34c18edfee093904cbc7fc19162dbcca22b2cc273b9d
-
-  cat <<EOF | $docker_build
-FROM debian:bullseye-20231030@sha256:$digest
+  # We don't want to build Clang for these targets on QEMU becuase it
+  # would take an extremely long time. Also, I believe old Linux boxes
+  # are typically x86-64.
+  cat <<EOF | $image_build
+FROM mirror.gcr.io/library/debian:bullseye-20240904@sha256:8ccc486c29a3ad02ad5af7f1156e2152dff3ba5634eec9be375269ef123457d8
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e '/^deb/d' -e 's/^# deb /deb [trusted=yes] /g' /etc/apt/sources.list && \
+RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
  apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 cmake && \
-  ln -sf /usr/bin/gcc-10 /usr/bin/cc && \
-  ln -sf /usr/bin/g++-10 /usr/bin/c++ && \
+  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake && \
+  ln -sf /usr/bin/clang-16 /usr/bin/clang && \
+  ln -sf /usr/bin/clang++-16 /usr/bin/clang++ && \
  rm -rf /var/lib/apt/lists
 EOF
  ;;
 riscv64)
-  cat <<EOF | $docker_build
-FROM riscv64/debian:sid-20240311@sha256:8c02dbe4faa999b588e873cc1759dd9b340f39daf4d7aabdb2c1a87cdc586459
+  cat <<EOF | $image_build
+FROM mirror.gcr.io/riscv64/debian:unstable-20240926@sha256:25654919c2926f38952cdd14b3300d83d13f2d820715f78c9f4b7a1d9399bf48
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^URIs/d' -e 's/^# http/URIs: http/' /etc/apt/sources.list.d/debian.sources && \
  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
  apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-12 g++-12 cmake && \
-  ln -sf /usr/bin/gcc-12 /usr/bin/cc && \
-  ln -sf /usr/bin/g++-12 /usr/bin/c++ && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake && \
+  ln -sf /usr/bin/clang-18 /usr/bin/clang && \
+  ln -sf /usr/bin/clang++-18 /usr/bin/clang++ && \
  rm -rf /var/lib/apt/lists
 EOF
  ;;
+loongarch64)
+  cat <<EOF | $image_build
+FROM mirror.gcr.io/loongarch64/debian:sid@sha256:0356df4e494bbb86bb469377a00789a5b42bbf67d5ff649a3f9721b745cbef77
+ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
+RUN sed -i -e 's!http[^ ]*!http://snapshot.debian.org/archive/debian-ports/20250620T014755Z!g' /etc/apt/sources.list && \
+  echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake && \
+  ln -sf /usr/bin/clang-19 /usr/bin/clang && \
+  ln -sf /usr/bin/clang++-19 /usr/bin/clang++ && \
+  rm -rf /var/lib/apt/lists
+EOF
+  ;;
+*)
+  usage
+  ;;
 esac

-# Source tarballs available on GitHub don't contain .git history.
+version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' CMakeLists.txt)
+dest=mold-$version-$arch-linux
+
+# Source tarballs available on GitHub don't contain .git directory.
 # Clone the repo if missing.
 [ -d .git ] || git clone --branch v$version --depth 1 --bare https://github.com/rui314/mold .git

 # We use the timestamp of the last Git commit as the file timestamp
 # for build artifacts.
-timestamp="$(git log -1 --format=%ci)"
+timestamp=$(git log -1 --format=%ct)
+
+# `uname -m` in an ARM32 container running on an ARM64 host reports it
+# not as ARM32 but as ARM64. That confuses BLAKE3's cmake script and
+# erroneously enables NEON SIMD instructions. `setarch` can be used to
+# change the output of `uname -m`.
+setarch=
+[ $arch = arm ] && setarch='setarch linux32'
+
+mkdir -p dist

 # Build mold in a container.
-docker run --platform linux/$arch -i --rm -v "$(pwd):/mold" $image bash -c "
+#
+# SOURCE_DATE_EPOCH is a standardized environment variable that allows
+# build artifacts to appear as if they were built at a specific time.
+# We use it to control how the compiler expands the C/C++ __DATE__ and
+# __TIME__ macros.
+podman run --arch $arch -it --rm --userns=host --pids-limit=-1 --network=none \
+  --pull=never -v "$(pwd):/mold:ro" -v "$(pwd)/dist:/dist" $image \
+   $setarch bash -c "
 set -e
+export SOURCE_DATE_EPOCH=$timestamp
 mkdir /build
 cd /build
-cmake -DCMAKE_BUILD_TYPE=Release -DMOLD_MOSTLY_STATIC=On /mold
+cmake -DCMAKE_BUILD_TYPE=Release -DMOLD_MOSTLY_STATIC=1 -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ /mold
 cmake --build . -j\$(nproc)
-mv mold mold2
-./mold2 -run cmake --build . -j\$(nproc)
-ctest -j\$(nproc)
+cmake --install .
+cmake -DMOLD_USE_MOLD=1 .
+cmake --build . -j\$(nproc)
+ctest --output-on-failure -j\$(nproc)
 cmake --install . --prefix $dest --strip
-find $dest -print | xargs touch --no-dereference --date='$timestamp'
-find $dest -print | sort | tar -cf - --no-recursion --files-from=- | gzip -9nc > /mold/$dest.tar.gz
-cp mold /mold
-chown $(id -u):$(id -g) /mold/$dest.tar.gz /mold/mold
-sha256sum /mold/$dest.tar.gz
+find $dest -print | xargs touch --no-dereference --date=@$timestamp
+find $dest -print | sort | tar -cf - --no-recursion --files-from=- | gzip -9nc > /dist/$dest.tar.gz
+cp mold /dist
+sha256sum /dist/$dest.tar.gz
 "
@@ -0,0 +1,179 @@
+# Instrumenting mold with MemorySanitizer
+
+## Introduction
+
+Per <https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo>:
+
+> If you want MemorySanitizer to work properly and not produce any false
+> positives, you must ensure that all the code in your program and in
+> libraries it uses is instrumented (i.e. built with `-fsanitize=memory`).
+> In particular, you would need to link against an MSan-instrumented C++
+> standard library. We recommend to use [libc++](https://libcxx.llvm.org/)
+> for that purpose.
+
+## Building instrumented libc++
+
+Build an MSan-instrumented libc++ from source:
+
+```sh
+cd ~
+git clone https://github.com/llvm/llvm-project
+cd llvm-project
+cmake -S ./runtimes -B build-libcxx -G Ninja  \
+    -DCMAKE_BUILD_TYPE=Release                \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+    -DCMAKE_C_COMPILER=clang                  \
+    -DCMAKE_CXX_COMPILER=clang++              \
+    -DLLVM_USE_SANITIZER=MemoryWithOrigins
+cmake --build build-libcxx -- cxx cxxabi
+export LIBCXX="$HOME/llvm-project/build-libcxx" # for subsequent build steps
+```
+
+Upon success, `./build-libcxx/{include,lib}` will contain the resulting
+headers and shared libraries.
+
+## Linking mold against instrumented libc++
+
+Use the `MOLD_USE_MSAN` and `MOLD_STDLIB_PREFIX` cmake variables to link
+mold against the instrumented build of libc++:
+
+```sh
+cd ~
+git clone https://github.com/rui314/mold.git
+cd mold
+cmake -B build -G Ninja            \
+    -DCMAKE_BUILD_TYPE=Debug       \
+    -DCMAKE_C_COMPILER=clang       \
+    -DCMAKE_CXX_COMPILER=clang++   \
+    -DMOLD_USE_MSAN=ON             \
+    -DMOLD_STDLIB_PREFIX="$LIBCXX" \
+    -DMOLD_USE_MIMALLOC=OFF        \
+    -DMOLD_USE_SYSTEM_TBB=ON
+cmake --build build
+```
+
+Most of mold's tests (except those for `-flto`) should work at this
+point. Run them like normal:
+
+```sh
+ctest --test-dir build
+```
+
+Any resulting MemorySanitizer errors should be visible in
+`./build/Temporary/Testing/LastTest.log`.
+
+## Building instrumented LTO plugin (experimental)
+
+Exercising `-flto` with MemorySanitizer (and without false positives)
+requires instrumenting the transitive dependencies of `lto-unix.cc`:
+
+- `libiberty.a`
+- `liblto_plugin.so`
+- `LLVMgold.so`
+
+> [!NOTE]
+> This is more involved and time-consuming than building only libc++ from
+> source. The cost-benefit ratio of this additional instrumentation and
+> test coverage may be unfavorable in many cases.
+
+> [!IMPORTANT]
+> The following steps are experimental and unlikely to work exactly as-is
+> under CI runners and individual developer environments. Consider the
+> following a starting point rather than a complete HOWTO.
+
+Build an MSan-instrumented GNU libiberty:
+
+```sh
+cd ~
+git clone [email protected]:gcc-mirror/gcc.git
+cd gcc/libiberty
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `./libiberty.a` into the build toolchain being
+tested with mold. If the system image is ephemeral or disposable (e.g. a
+short-lived VM or container), a quick-and-dirty install could look like:
+
+```sh
+sudo cp /usr/lib/libiberty.a{,.bak}
+sudo cp ./libiberty.a /usr/lib/
+```
+
+Build an MSan-instrumented GCC LTO plugin library:
+
+```sh
+cd ~/gcc/lto-plugin # use already-cloned repo from earlier step
+export CC=clang
+export CFLAGS="-g -Og -fsanitize=memory -fsanitize-memory-track-origins"
+export LDFLAGS="-fsanitize=memory"
+./configure --with-libiberty=/usr/lib
+make -j$(nproc)
+unset CC CFLAGS LDFLAGS
+```
+
+Install the resulting `liblto_plugin.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+export GCC_VERSION="$(gcc -dumpversion)"
+sudo cp /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/liblto_plugin.so{,.bak}
+sudo cp .libs/liblto_plugin.so /usr/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/
+```
+
+Build an MSan-instrumented LLVM LTO plugin library:
+
+```sh
+cd ~/llvm-project # use already-cloned repo from earlier step
+
+CF="-nostdinc++ -isystem $LIBCXX/include/c++/v1"
+INCDIR="$(find /usr -name plugin-api.h -type f | head -1 | xargs dirname)"
+
+function configure
+{
+    cmake -S ./llvm -B build-plugin -G Ninja   \
+        -DLLVM_ENABLE_PROJECTS=clang           \
+        -DLLVM_TARGETS_TO_BUILD=X86            \
+        -DCMAKE_BUILD_TYPE=Release             \
+        -DCMAKE_C_COMPILER=clang               \
+        -DCMAKE_CXX_COMPILER=clang++           \
+        -DCMAKE_C_FLAGS="$CF"                  \
+        -DCMAKE_CXX_FLAGS="$CF"                \
+        -DCMAKE_EXE_LINKER_FLAGS="$1"          \
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON   \
+        -DLLVM_BINUTILS_INCDIR="$INCDIR"       \
+        -DLLVM_USE_SANITIZER=MemoryWithOrigins
+}
+
+# workaround for linker issues: configure twice with different LDFLAGS
+configure "-nostdlib++ -L $LIBCXX/lib -Wl,--rpath=$LIBCXX/lib"
+configure "-nostdlib++ -L $LIBCXX/lib -lc++ -Wl,--rpath=$LIBCXX/lib"
+
+cmake --build build-plugin -- LLVMgold.so
+```
+
+Install the resulting `LLVMgold.so`. With the same caveats discussed
+above, a simple install into an ephemeral environment could look like:
+
+```sh
+sudo cp /usr/lib/LLVMgold.so{,.bak}
+sudo cp ./build-plugin/lib/LLVMgold.so /usr/lib/
+```
+
+mold's tests for `-flto` should now (mostly) work, though there do seem
+to be issues around `gcc` inferring `-flto` when not explicitly specified.
+One workaround is to use `clang` as `TEST_CC`:
+
+```sh
+cd ~/mold # use already-cloned and built repo from earlier step
+TEST_CC=clang TEST_CXX=clang++ ctest --test-dir build
+```
+
+## References
+
+- <https://github.com/google/sanitizers/wiki/MemorySanitizerBootstrappingClang>
+- <https://llvm.org/docs/GoldPlugin.html#lto-how-to-build>
@@ -1,6 +1,6 @@
 .\" generated with Ronn-NG/v0.9.1
 .\" http://github.com/apjanke/ronn-ng/tree/0.9.1
-.TH "MOLD" "1" "August 2024" ""
+.TH "MOLD" "1" "May 2026" ""
 .SH "NAME"
 \fBmold\fR \- a modern linker
 .SH "SYNOPSIS"
@@ -10,9 +10,9 @@
 .SS "How to use"
 See \fIhttps://github\.com/rui314/mold#how\-to\-use\fR\.
 .SS "Compatibility"
-\fBMold\fR is designed to be a drop\-in replacement for the GNU linkers for linking user\-land programs\. If your user\-land program cannot be built due to missing command\-line options, please file a bug at \fIhttps://github\.com/rui314/mold/issues\fR\.
+\fBmold\fR is designed to be a drop\-in replacement for the GNU linkers for linking user\-land programs\. If your user\-land program cannot be built due to missing command\-line options, please file a bug at \fIhttps://github\.com/rui314/mold/issues\fR\.
 .P
-Mold supports a very limited set of linker script features, which is just sufficient to read \fB/usr/lib/x86_64\-linux\-gnu/libc\.so\fR on Linux systems (on Linux, that file is contrary to its name not a shared library but an ASCII linker script that loads a real \fBlibc\.so\fR file\.)
+\fBmold\fR supports a very limited set of linker script features, which is just sufficient to read \fB/usr/lib/x86_64\-linux\-gnu/libc\.so\fR on Linux systems (on Linux, that file is contrary to its name not a shared library but an ASCII linker script that loads a real \fBlibc\.so\fR file\.)
 .P
 Beyond that, we have no plan to support any additional linker script features\. The linker script is an ad\-hoc, over\-designed, complex language which we believe needs to be replaced by a simpler mechanism\. We have a plan to add a replacement for the linker script to \fBmold\fR instead\.
 .SS "Archive symbol resolution"
@@ -40,13 +40,11 @@ Typically, an ELF module that exports a symbol also imports the same symbol\. Su
 .P
 Let's take \fBmalloc\fR as an example\. Assume that you define your version of \fBmalloc\fR in your main executable file\. Then, all \fBmalloc\fR calls from any module are resolved to your function instead of the one in libc, because the executable is always at the beginning of the dynamic symbol search list\. Note that even \fBmalloc\fR calls within libc are resolved to your definition since libc exports and imports \fBmalloc\fR\. Therefore, by defining \fBmalloc\fR yourself, you can overwrite a library function, and the \fBmalloc\fR in libc becomes dead code\.
 .P
-These Unix semantics are tricky and sometimes considered harmful\. For example, assume that you accidentally define \fBatoi\fR as a global function in your executable that behaves completely differently from the one in the C standard\. Then, all \fBatoi\fR function calls from any modules (even function calls within libc) are redirected to your function instead of the one in libc, which will very likely cause a problem\.
-.P
-That is a somewhat surprising consequence for an accidental name conflict\. On the other hand, this semantic is sometimes useful because it allows users to override library functions without rebuilding modules containing them\.
+These Unix semantics are tricky and sometimes considered harmful\. For example, assume that you accidentally define \fBatoi\fR as a global function in your executable that behaves completely differently from the one in the C standard\. Then, all \fBatoi\fR function calls from any modules (even function calls within libc) are redirected to your function instead of the one in libc, which will very likely cause a problem\. That is a somewhat surprising consequence for an accidental name conflict\. On the other hand, this semantic is sometimes useful because it allows users to override library functions without rebuilding modules containing them\.
 .P
 Whether good or bad, you should keep these semantics in mind to understand Unix linkers' behaviors\.
 .SS "Build reproducibility"
-\fBmold\fR's output is deterministic\. That is, if you pass the same object files and the same command\-line options to the same version of \fBmold\fR, it is guaranteed that \fBmold\fR produces the bit\-by\-bit identical output\. The linker's internal randomness, such as the timing of thread scheduling or iteration orders of hash tables, doesn't affect the output\.
+\fBmold\fR's output is deterministic\. That is, if you pass the same object files and the same command\-line options to the same version of \fBmold\fR, it is guaranteed that \fBmold\fR produces the bit\-for\-bit identical output\. The linker's internal randomness, such as the timing of thread scheduling or iteration orders of hash tables, doesn't affect the output\.
 .P
 \fBmold\fR does not have any host\-specific default settings\. This is contrary to the GNU linkers, for which some configurable values, such as system\-dependent library search paths, are hard\-coded\. \fBmold\fR depends only on its command\-line arguments\.
 .SH "OPTION NOTATIONS"
@@ -67,11 +65,13 @@ Synonym for \fB\-\-color\-diagnostics=auto\fR\.
 \fB\-\-no\-color\-diagnostics\fR
 Synonym for \fB\-\-color\-diagnostics=never\fR\.
 .TP
-\fB\-\-detach\fR, `\-\-no\-detach
+\fB\-\-detach\fR, \fB\-\-no\-detach\fR
 Permit or do not permit mold to create a debug info file in the background\.
 .TP
 \fB\-\-fork\fR, \fB\-\-no\-fork\fR
 Spawn a child process and let it do the actual linking\. When linking a large program, the OS kernel can take a few hundred milliseconds to terminate a \fBmold\fR process\. \fB\-\-fork\fR hides that latency\. By default, it does fork\.
+.IP
+Note that \fB\-\-fork\fR also hides the resource usage statistics reported by time(2), since it doesn't call waitpid(2) on the child process\. If you need those statistics, pass \fB\-\-no\-fork\fR\.
 .TP
 \fB\-\-perf\fR
 Print performance statistics\.
@@ -132,6 +132,17 @@ Use multiple threads\. By default, \fBmold\fR uses as many threads as the number
 \fB\-\-quick\-exit\fR, \fB\-\-no\-quick\-exit\fR
 Use or do not use \fBquick_exit\fR to exit\.
 .TP
+\fB\-\-zero\-to\-bss\fR
+Convert all\-zero data sections into BSS\.
+.IP
+When this option is enabled, \fBmold\fR scans input data sections that are not of type \fBSHT_NOBITS\fR and checks whether their contents consist solely of zero bytes\. Such sections are then converted into BSS (\fBSHT_NOBITS\fR) sections\. This reduces the size of the output file, since BSS sections occupy no space in the file image\.
+.IP
+This behavior is especially useful for user\-defined sections created with \fB__attribute__((section("\.sectname")))\fR that contain uninitialized global variables\. GCC and Clang do not automatically mark such sections as BSS even if their contents are entirely zero, and instead emit them as regular data sections\.
+.IP
+For example, consider \fB__attribute__((section("\.sectname"))) int vec[256];\fR\.
+.IP
+By default, this results in a \fB\.sectname\fR section of type \fBSHT_PROGBITS\fR filled with zeros\. With \fB\-\-zero\-to\-bss\fR, the linker will recognize it as empty data and convert it to a \fBSHT_NOBITS\fR section, reducing the output file size without changing runtime semantics\.
+.TP
 \fB\-z rewrite\-endbr\fR, \fB\-z norewrite\-endbr\fR
 As a security measure, some CPU instruction sets have recently gained a feature to protect control flow integrity by disallowing indirect branches by default\. If the feature is enabled, the instruction that is executed immediately after an indirect branch must be an branch target marker instruction, or a CPU\-level fault will raise\. The marker instruction is also known as "landing pad" instruction, to which indirect branches can land\. This feature makes ROP attacks harder to conduct\.
 .IP
@@ -271,8 +282,8 @@ Synonym for \fB\-\-build\-id=sha256\fR\.
 \fB\-\-no\-build\-id\fR
 Synonym for \fB\-\-build\-id=none\fR\.
 .TP
-\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzstd\fR | \fBnone\fR ]
-Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\.
+\fB\-\-compress\-debug\-sections\fR=[ \fBzlib\fR | \fBzlib\-gabi\fR | \fBzlib:\fR\fIN\fR | \fBzstd\fR | \fBzstd:\fR\fIN\fR | \fBnone\fR ]
+Compress DWARF debug info (\fB\.debug_*\fR sections) using the zlib or zstd compression algorithm\. \fBzlib\-gabi\fR is an alias for \fBzlib\fR\. \fBzlib:\fR\fIN\fR selects the zlib compression level, where \fIN\fR is between 0 and 9 (0 means no compression), and \fBzstd:\fR\fIN\fR selects the zstd compression level, where \fIN\fR is between 1 and 22\. Higher levels achieve better compression ratios but are slower\. \fBzlib\fR is equivalent to \fBzlib:1\fR and \fBzstd\fR is equivalent to \fBzstd:3\fR\.
 .TP
 \fB\-\-defsym\fR=\fIsymbol\fR=\fIvalue\fR
 Define \fIsymbol\fR as an alias for \fIvalue\fR\.
@@ -355,10 +366,21 @@ Report undefined symbols (even with \fB\-\-shared\fR)\.
 \fB\-\-noinhibit\-exec\fR
 Create an output file even if errors occur\.
 .TP
-\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBnone\fR ]
+\fB\-\-package\-metadata\fR=\fIpercent\-encoded\-string\fR
+Embed a specified string into the \fB\.note\.package\fR section\. This option is designed for build scripts that generate binary packages, such as \fB\.rpm\fR or \fB\.deb\fR, to include package metadata in each executable\. It simplifies the process of identifying the corresponding package for a given executable or core file\.
+.IP
+An argument to this option is treated as percent\-encoded and decoded before being inserted into the section, allowing you to avoid the use of the comma (\fB,\fR) character in the argument\. This is useful because the compiler replaces all occurrences of commas in \fB\-Wl,\fR with spaces before forwarding them to the linker\. Note that \fBmold\fR always interprets the argument as percent\-encoded, so you also need to escape all occurrences of \fB%\fR as \fB%25\fR\.
+.TP
+\fB\-\-pack\-dyn\-relocs\fR=[ \fBrelr\fR | \fBandroid\fR | \fBandroid+relr\fR | \fBnone\fR ]
+Compress dynamic relocations in the output file\.
+.IP
 If \fBrelr\fR is specified, all \fBR_*_RELATIVE\fR relocations are put into \fB\.relr\.dyn\fR section instead of \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section\. Since \fB\.relr\.dyn\fR section uses a space\-efficient encoding scheme, specifying this flag can reduce the size of the output\. This is typically most effective for position\-independent executable\.
 .IP
-Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2022, only ChromeOS, Android and Fuchsia support it\.
+If \fBandroid\fR is specified, all dynamic relocations are encoded in Android's packed relocation format (APS2) and stored in the \fB\.rel\.dyn\fR or \fB\.rela\.dyn\fR section, whose section type is changed to \fBSHT_ANDROID_REL\fR or \fBSHT_ANDROID_RELA\fR\. The dynamic table is updated to use \fBDT_ANDROID_REL\fR / \fBDT_ANDROID_RELA\fR and their \fBSZ\fR variants in place of the standard tags\.
+.IP
+\fBandroid+relr\fR combines both: \fBR_*_RELATIVE\fR relocations go to \fB\.relr\.dyn\fR and the rest are APS2\-encoded\. This is the configuration used by most Android system binaries\.
+.IP
+Note that a runtime loader has to support \fB\.relr\.dyn\fR to run executables or shared libraries linked with \fB\-\-pack\-dyn\-relocs=relr\fR\. As of 2026, glibc (since 2\.36), musl (since 1\.2\.4), Android, ChromeOS, and Fuchsia all support it\. The APS2 format, on the other hand, is only handled by Android's bionic loader; the encoder itself is target\-agnostic, but the produced binary can only be loaded on platforms where bionic (or a compatible loader) handles \fBDT_ANDROID_REL(A)\fR\.
 .TP
 \fB\-\-pie\fR, \fB\-\-pic\-executable\fR, \fB\-\-no\-pie\fR, \fB\-\-no\-pic\-executable\fR
 Create a position\-independent executable\.
@@ -424,6 +446,9 @@ How to handle undefined symbols\.
 \fB\-\-version\-script\fR=\fIfile\fR
 Read version script from \fIfile\fR\. If \fIfile\fR does not exist in the current directory, it is searched from library search paths for the sake of compatibility with GNU ld\.
 .TP
+\fB\-w\fR, \fB\-\-no\-warnings\fR
+Suppress warnings and cancel \fB\-\-fatal\-warnings\fR\.
+.TP
 \fB\-\-warn\-common\fR, \fB\-\-no\-warn\-common\fR
 Warn about common symbols\.
 .TP
@@ -480,9 +505,9 @@ By default, \fBmold\fR generates a relro segment\. \fB\-z norelro\fR disables th
 \fB\-z separate\-loadable\-segments\fR, \fB\-z separate\-code\fR, \fB\-z noseparate\-code\fR
 If one memory page contains multiple segments, the page protection bits are set in such a way that the needed attributes (writable or executable) are satisfied for all segments\. This usually happens at a boundary of two segments with two different attributes\.
 .IP
-\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\. This is the default\.
+\fBseparate\-loadable\-segments\fR adds paddings between segments with different attributes so that they do not share the same page\.
 .IP
-\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\.
+\fBseparate\-code\fR adds paddings only between executable and non\-executable segments\. This is the default\.
 .IP
 \fBnoseparate\-code\fR does not add any paddings between segments\.
 .TP
@@ -13,12 +13,12 @@ See <https://github.com/rui314/mold#how-to-use>.

 ### Compatibility

-**Mold** is designed to be a drop-in replacement for the GNU linkers for
+`mold` is designed to be a drop-in replacement for the GNU linkers for
 linking user-land programs. If your user-land program cannot be built due to
 missing command-line options, please file a bug at
 <https://github.com/rui314/mold/issues>.

-Mold supports a very limited set of linker script features, which is just
+`mold` supports a very limited set of linker script features, which is just
 sufficient to read `/usr/lib/x86_64-linux-gnu/libc.so` on Linux systems (on
 Linux, that file is contrary to its name not a shared library but an ASCII
 linker script that loads a real `libc.so` file.)
@@ -104,11 +104,10 @@ assume that you accidentally define `atoi` as a global function in your
 executable that behaves completely differently from the one in the C standard.
 Then, all `atoi` function calls from any modules (even function calls within
 libc) are redirected to your function instead of the one in libc, which will
-very likely cause a problem.
-
-That is a somewhat surprising consequence for an accidental name conflict. On
-the other hand, this semantic is sometimes useful because it allows users to
-override library functions without rebuilding modules containing them.
+very likely cause a problem. That is a somewhat surprising consequence for an
+accidental name conflict. On the other hand, this semantic is sometimes useful
+because it allows users to override library functions without rebuilding
+modules containing them.

 Whether good or bad, you should keep these semantics in mind to understand
 Unix linkers' behaviors.
@@ -117,7 +116,7 @@ Unix linkers' behaviors.

 `mold`'s output is deterministic. That is, if you pass the same object files
 and the same command-line options to the same version of `mold`, it is
-guaranteed that `mold` produces the bit-by-bit identical output. The linker's
+guaranteed that `mold` produces the bit-for-bit identical output. The linker's
 internal randomness, such as the timing of thread scheduling or iteration
 orders of hash tables, doesn't affect the output.

@@ -152,7 +151,7 @@ but as `-o magic`.
 * `--no-color-diagnostics`:
  Synonym for `--color-diagnostics=never`.

-* `--detach`, `--no-detach:
+* `--detach`, `--no-detach`:
  Permit or do not permit mold to create a debug info file in the background.

 * `--fork`, `--no-fork`:
@@ -160,6 +159,10 @@ but as `-o magic`.
  program, the OS kernel can take a few hundred milliseconds to terminate a
  `mold` process. `--fork` hides that latency. By default, it does fork.

+  Note that `--fork` also hides the resource usage statistics reported by
+  time(2), since it doesn't call waitpid(2) on the child process. If you
+  need those statistics, pass `--no-fork`.
+
 * `--perf`:
  Print performance statistics.

@@ -268,6 +271,28 @@ but as `-o magic`.
 * `--quick-exit`, `--no-quick-exit`:
  Use or do not use `quick_exit` to exit.

+* `--zero-to-bss`:
+  Convert all-zero data sections into BSS.
+
+  When this option is enabled, `mold` scans input data sections that are not
+  of type `SHT_NOBITS` and checks whether their contents consist solely of
+  zero bytes. Such sections are then converted into BSS (`SHT_NOBITS`)
+  sections. This reduces the size of the output file, since BSS sections
+  occupy no space in the file image.
+
+  This behavior is especially useful for user-defined sections created with
+  `__attribute__((section(".sectname")))` that contain uninitialized global
+  variables. GCC and Clang do not automatically mark such sections as BSS
+  even if their contents are entirely zero, and instead emit them as regular
+  data sections.
+
+  For example, consider `__attribute__((section(".sectname"))) int vec[256];`.
+
+  By default, this results in a `.sectname` section of type `SHT_PROGBITS`
+  filled with zeros. With `--zero-to-bss`, the linker will recognize it as
+  empty data and convert it to a `SHT_NOBITS` section, reducing the output
+  file size without changing runtime semantics.
+
 * `-z rewrite-endbr`, `-z norewrite-endbr`:
  As a security measure, some CPU instruction sets have recently gained a
  feature to protect control flow integrity by disallowing indirect branches
@@ -481,9 +506,14 @@ but as `-o magic`.
 * `--no-build-id`:
  Synonym for `--build-id=none`.

-* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zstd` | `none` ]:
+* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zlib:`_N_ | `zstd` | `zstd:`_N_ | `none` ]:
  Compress DWARF debug info (`.debug_*` sections) using the zlib or zstd
-  compression algorithm. `zlib-gabi` is an alias for `zlib`.
+  compression algorithm. `zlib-gabi` is an alias for `zlib`. `zlib:`_N_
+  selects the zlib compression level, where _N_ is between 0 and 9 (0
+  means no compression), and `zstd:`_N_ selects the zstd compression
+  level, where _N_ is between 1 and 22. Higher levels achieve better
+  compression ratios but are slower. `zlib` is equivalent to `zlib:1`
+  and `zstd` is equivalent to `zstd:3`.

 * `--defsym`=_symbol_=_value_:
  Define _symbol_ as an alias for _value_.
@@ -618,16 +648,47 @@ but as `-o magic`.
 * `--noinhibit-exec`:
  Create an output file even if errors occur.

-* `--pack-dyn-relocs`=[ `relr` | `none` ]:
+* `--package-metadata`=_percent-encoded-string_:
+  Embed a specified string into the `.note.package` section. This option
+  is designed for build scripts that generate binary packages, such as
+  `.rpm` or `.deb`, to include package metadata in each executable. It
+  simplifies the process of identifying the corresponding package for a
+  given executable or core file.
+
+  An argument to this option is treated as percent-encoded and decoded
+  before being inserted into the section, allowing you to avoid the use of
+  the comma (`,`) character in the argument. This is useful because the
+  compiler replaces all occurrences of commas in `-Wl,` with spaces before
+  forwarding them to the linker. Note that `mold` always interprets the
+  argument as percent-encoded, so you also need to escape all occurrences
+  of `%` as `%25`.
+
+* `--pack-dyn-relocs`=[ `relr` | `android` | `android+relr` | `none` ]:
+  Compress dynamic relocations in the output file.
+
  If `relr` is specified, all `R_*_RELATIVE` relocations are put into
  `.relr.dyn` section instead of `.rel.dyn` or `.rela.dyn` section. Since
  `.relr.dyn` section uses a space-efficient encoding scheme, specifying this
  flag can reduce the size of the output. This is typically most effective for
  position-independent executable.

+  If `android` is specified, all dynamic relocations are encoded in Android's
+  packed relocation format (APS2) and stored in the `.rel.dyn` or `.rela.dyn`
+  section, whose section type is changed to `SHT_ANDROID_REL` or
+  `SHT_ANDROID_RELA`. The dynamic table is updated to use `DT_ANDROID_REL`
+  / `DT_ANDROID_RELA` and their `SZ` variants in place of the standard tags.
+
+  `android+relr` combines both: `R_*_RELATIVE` relocations go to `.relr.dyn`
+  and the rest are APS2-encoded. This is the configuration used by most
+  Android system binaries.
+
  Note that a runtime loader has to support `.relr.dyn` to run executables or
-  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2022, only
-  ChromeOS, Android and Fuchsia support it.
+  shared libraries linked with `--pack-dyn-relocs=relr`. As of 2026, glibc
+  (since 2.36), musl (since 1.2.4), Android, ChromeOS, and Fuchsia all
+  support it. The APS2 format, on the other hand, is only handled by
+  Android's bionic loader; the encoder itself is target-agnostic, but the
+  produced binary can only be loaded on platforms where bionic (or a
+  compatible loader) handles `DT_ANDROID_REL(A)`.

 * `--pie`, `--pic-executable`, `--no-pie`, `--no-pic-executable`:
  Create a position-independent executable.
@@ -717,6 +778,9 @@ but as `-o magic`.
  directory, it is searched from library search paths for the sake of
  compatibility with GNU ld.

+* `-w`, `--no-warnings`:
+  Suppress warnings and cancel `--fatal-warnings`.
+
 * `--warn-common`, `--no-warn-common`:
  Warn about common symbols.

@@ -814,10 +878,10 @@ but as `-o magic`.
  segments with two different attributes.

  `separate-loadable-segments` adds paddings between segments with different
-  attributes so that they do not share the same page. This is the default.
+  attributes so that they do not share the same page.

  `separate-code` adds paddings only between executable and non-executable
-  segments.
+  segments. This is the default.

  `noseparate-code` does not add any paddings between segments.

@@ -7,43 +7,46 @@ set -e

 set -x

-case "$ID-$VERSION_ID" in
-ubuntu-20.* | pop-20.*)
+case "$ID" in
+ubuntu | pop | linuxmint | debian | raspbian | neon | zorin)
  apt-get update
-  apt-get install -y cmake gcc g++ g++-10
+  apt-get install -y cmake gcc g++ clang gdb
  ;;
-ubuntu-* | pop-* | linuxmint-* | debian-* | raspbian-*)
-  apt-get update
-  apt-get install -y cmake gcc g++
+fedora | fedora-* | amzn | rhel | centos)
+  dnf install -y gcc-c++ cmake glibc-static libstdc++-static diffutils util-linux tar
  ;;
-fedora-* | amzn-* | rhel-*)
-  dnf install -y gcc-g++ cmake glibc-static libstdc++-static diffutils util-linux
+rocky | ol)
+  dnf install -y gcc-c++ cmake diffutils util-linux
  ;;
 opensuse-*)
-  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux
+  zypper install -y make cmake gcc-c++ glibc-devel-static tar diffutils util-linux gawk
  ;;
-gentoo-*)
+gentoo)
  emerge-webrsync
  FEATURES='getbinpkg binpkg-request-signature' emerge dev-build/cmake
  ;;
-arch-* | archarm-* | artix-* | endeavouros-*)
+arch | archarm | artix | endeavouros | manjaro | cachyos)
  pacman -Sy --needed --noconfirm base-devel cmake util-linux
  ;;
-void-*)
+void)
  xbps-install -Sy xbps bash make cmake gcc tar diffutils util-linux
  ;;
-alpine-*)
+alpine)
  apk update
  apk add bash make linux-headers cmake gcc g++
  ;;
-clear-linux-*)
+clear-linux-os)
  swupd update
  swupd bundle-add c-basic diffutils
  ;;
-almalinux-*)
+almalinux)
  dnf install -y gcc-toolset-13-gcc-c++ gcc-toolset-13-libstdc++-devel cmake diffutils
  ;;
-freebsd-*)
+altlinux)
+  apt-get update
+  apt-get install -y gcc-c++ make cmake ctest diffutils util-linux
+  ;;
+freebsd)
  pkg update
  pkg install -y cmake bash binutils gcc
  ;;
@@ -9,8 +9,8 @@ set -x
 #
 # Feel free to send me a PR if your OS is not on this list.

-case "$ID-$VERSION_ID" in
-ubuntu-* | pop-* | linuxmint-* | debian-* | raspbian-*)
+case "$ID" in
+ubuntu | pop | linuxmint | debian | raspbian)
  apt-get install -y qemu-user {gcc,g++}-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4}-linux-gnu {gcc,g++}-arm-linux-gnueabihf
  ;;
 *)
@@ -0,0 +1,132 @@
+// This file implements the Aho-Corasick algorithm to search multiple
+// strings within an input string simultaneously. It is essentially a
+// trie with additional links. For details, see
+// https://en.wikipedia.org/wiki/Aho-Corasick_algorithm.
+//
+// We use it for simple glob patterns in version scripts or dynamic
+// list files. Here are some examples of glob patterns:
+//
+//    qt_private_api_tag*
+//    *16QAccessibleCache*
+//    *32QAbstractFileIconProviderPrivate*
+//    *17QPixmapIconEngine*
+//
+// Aho-Corasick can do only substring search, so it cannot handle
+// complex glob patterns such as `*foo*bar*`. We handle such patterns
+// with the Glob class.
+
+#include "lib.h"
+
+#include <queue>
+
+namespace mold {
+
+bool AhoCorasick::can_handle(std::string_view pat) {
+  if (pat.starts_with('*'))
+    pat.remove_prefix(1);
+  if (pat.ends_with('*'))
+    pat.remove_suffix(1);
+  return pat.find_first_of("*?[") == pat.npos;
+}
+
+i64 AhoCorasick::find(std::string_view str) {
+  if (nodes.empty())
+    return -1;
+
+  i64 idx = 0;
+  i64 val = -1;
+
+  auto walk = [&](u8 c) {
+    for (i64 j = idx; j != -1; j = nodes[j].suffix_link) {
+      i64 child = nodes[j].children[c];
+      if (child != -1) {
+        idx = child;
+        val = std::max(val, nodes[child].value);
+        return;
+      }
+    }
+    idx = 0;
+  };
+
+  walk('\0');
+  for (u8 c : str)
+    walk(c);
+  walk('\0');
+  return val;
+}
+
+bool AhoCorasick::add(std::string_view pat, i64 val) {
+  assert(can_handle(pat));
+  if (nodes.empty())
+    nodes.resize(1);
+  i64 idx = 0;
+
+  auto walk = [&](u8 c) {
+    if (nodes[idx].children[c] == -1) {
+      nodes[idx].children[c] = nodes.size();
+      nodes.resize(nodes.size() + 1);
+    }
+    idx = nodes[idx].children[c];
+  };
+
+  // We handle "foo" as if "\0foo\0", "*foo" as if "foo\0", "foo*" as
+  // if "\0foo", and "*foo*" as if "foo". Aho-Corasick can do only
+  // substring matching, so we use \0 as a beginning/end-of-string
+  // markers.
+  if (!pat.starts_with('*'))
+    walk('\0');
+  for (u8 c : pat)
+    if (c != '*')
+      walk(c);
+  if (!pat.ends_with('*'))
+    walk('\0');
+
+  nodes[idx].value = std::max(nodes[idx].value, val);
+  return true;
+}
+
+void AhoCorasick::compile() {
+  if (nodes.empty())
+    return;
+  fix_suffix_links(0);
+  fix_values();
+}
+
+void AhoCorasick::fix_suffix_links(i64 idx) {
+  for (i64 i = 0; i < 256; i++) {
+    i64 child = nodes[idx].children[i];
+    if (child == -1)
+      continue;
+
+    i64 j = nodes[idx].suffix_link;
+    for (; j != -1; j = nodes[j].suffix_link) {
+      if (nodes[j].children[i] != -1) {
+        nodes[child].suffix_link = j;
+        break;
+      }
+    }
+    if (j == -1)
+      nodes[child].suffix_link = 0;
+    fix_suffix_links(child);
+  }
+}
+
+void AhoCorasick::fix_values() {
+  std::queue<i64> queue;
+  queue.push(0);
+
+  do {
+    i64 idx = queue.front();
+    queue.pop();
+
+    for (i64 child : nodes[idx].children) {
+      if (child != -1) {
+        i64 suffix = nodes[child].suffix_link;
+        nodes[child].value = std::max(nodes[child].value, nodes[suffix].value);
+        queue.push(child);
+      }
+    }
+  } while (!queue.empty());
+}
+
+} // namespace mold
@@ -0,0 +1,50 @@
+// This is the same as std::atomic except that the default memory
+// order is relaxed instead of sequential consistency.
+
+#pragma once
+
+#include <atomic>
+
+namespace mold {
+
+template <typename T>
+struct Atomic : std::atomic<T> {
+  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
+
+  using std::atomic<T>::atomic;
+
+  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
+
+  Atomic<T> &operator=(const Atomic<T> &other) {
+    store(other.load());
+    return *this;
+  }
+
+  void operator=(T val) { store(val); }
+  operator T() const { return load(); }
+
+  void store(T val, std::memory_order order = relaxed) {
+    std::atomic<T>::store(val, order);
+  }
+
+  T load(std::memory_order order = relaxed) const {
+    return std::atomic<T>::load(order);
+  }
+
+  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
+  T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
+  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
+  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
+  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
+  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
+
+  bool test_and_set() {
+    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
+    // while an atomic RMW can easily take hundreds on x86. We note that it's
+    // common that another thread beat us in marking, so doing an optimistic
+    // early test tends to improve performance in the ~20% ballpark.
+    return load() || exchange(true);
+  }
+};
+
+} // namespace mold
@@ -0,0 +1,78 @@
+#pragma once
+
+#include "integers.h"
+
+#include <cassert>
+#include <vector>
+
+namespace mold {
+
+class BitvectorProxy {
+public:
+  BitvectorProxy(u64 &word, size_t pos)
+    : word(word), mask(1ULL << pos) {}
+
+  BitvectorProxy &operator=(bool val) {
+    if (val)
+      word |= mask;
+    else
+      word &= ~mask;
+    return *this;
+  }
+
+  BitvectorProxy &operator=(const BitvectorProxy &other) {
+    return *this = (bool)other;
+  }
+
+  operator bool() const {
+    return word & mask;
+  }
+
+private:
+  u64 &word;
+  u64 mask;
+};
+
+class Bitvector {
+public:
+  Bitvector() = default;
+  Bitvector(i64 n) : size(n), words((n + 63) / 64) {}
+
+  void resize(i64 n) {
+    words.clear();
+    words.resize((n + 63) / 64);
+    size = n;
+  }
+
+  Bitvector &operator|=(const Bitvector &x) {
+    assert(size == x.size);
+    for (i64 i = 0; i < words.size(); i++)
+      words[i] |= x.words[i];
+    return *this;
+  }
+
+  Bitvector &operator&=(const Bitvector &x) {
+    assert(size == x.size);
+    for (i64 i = 0; i < words.size(); i++)
+      words[i] &= x.words[i];
+    return *this;
+  }
+
+  Bitvector &operator<<=(size_t n) {
+    assert(n == 1);
+    for (i64 i = words.size() - 1; i > 0; i--)
+      words[i] = (words[i] << 1) | (words[i - 1] >> 63);
+    words[0] <<= 1;
+    return *this;
+  }
+
+  BitvectorProxy operator[](size_t pos) {
+    assert(pos < size);
+    return {words[pos / 64], pos % 64};
+  }
+
+  i64 size = 0;
+  std::vector<u64> words;
+};
+
+} // namespace mold
@@ -14,7 +14,7 @@
 // a little bit. However, if a shard size is large enough, that loss
 // is negligible in practice.

-#include "common.h"
+#include "lib.h"

 #include <tbb/parallel_for_each.h>
 #include <zlib.h>
@@ -30,42 +30,41 @@ namespace mold {

 static constexpr i64 SHARD_SIZE = 1024 * 1024;

-static std::vector<std::string_view> split(std::string_view input) {
-  std::vector<std::string_view> shards;
-
-  while (input.size() >= SHARD_SIZE) {
-    shards.push_back(input.substr(0, SHARD_SIZE));
-    input = input.substr(SHARD_SIZE);
-  }
-  if (!input.empty())
-    shards.push_back(input);
-  return shards;
+Compressor::~Compressor() {
+  for (std::span<u8> shard : shards)
+    delete[] shard.data();
 }

-static std::vector<u8> zlib_compress(std::string_view input) {
-  // Initialize zlib stream. Since debug info is generally compressed
-  // pretty well with lower compression levels, we chose compression
-  // level 1.
-  z_stream strm;
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
+static std::vector<std::span<u8>> split(std::span<u8> input) {
+  std::vector<std::span<u8>> vec;
+  while (!input.empty()) {
+    i64 sz = std::min<i64>(SHARD_SIZE, input.size());
+    vec.push_back(input.subspan(0, sz));
+    input = input.subspan(sz);
+  }
+  return vec;
+}

-  CHECK(deflateInit2(&strm, 1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));
+static std::span<u8> zlib_compress(std::span<u8> input, int level) {
+  // Initialize zlib stream. Since debug info is generally compressed
+  // pretty well with lower compression levels, the default level is 1.
+  z_stream strm = {};
+  CHECK(deflateInit2(&strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY));

  // Set an input buffer
  strm.avail_in = input.size();
-  strm.next_in = (u8 *)input.data();
+  strm.next_in = input.data();

  // Set an output buffer. deflateBound() returns an upper bound
  // on the compression size. +16 for Z_SYNC_FLUSH.
-  std::vector<u8> buf(deflateBound(&strm, strm.avail_in) + 16);
+  i64 bufsize = deflateBound(&strm, strm.avail_in) + 16;
+  u8 *buf = new u8[bufsize];

  // Compress data. It writes all compressed bytes except the last
  // partial byte, so up to 7 bits can be held to be written to the
  // buffer.
-  strm.avail_out = buf.size();
-  strm.next_out = buf.data();
+  strm.avail_out = bufsize;
+  strm.next_out = buf;
  CHECK(deflate(&strm, Z_BLOCK));

  // This is a workaround for libbacktrace before 2022-04-06.
@@ -84,29 +83,26 @@ static std::vector<u8> zlib_compress(std::string_view input) {
  // that the bit position is 7 in the next byte.
  //
  // https://github.com/ianlancetaylor/libbacktrace/pull/87
-  int nbits;
+  int nbits = 0;
  deflatePending(&strm, Z_NULL, &nbits);
  if (nbits == 5)
    CHECK(deflatePrime(&strm, 10, 2));
  CHECK(deflate(&strm, Z_SYNC_FLUSH));

-  assert(strm.avail_out > 0);
-  buf.resize(buf.size() - strm.avail_out);
-  buf.shrink_to_fit();
  deflateEnd(&strm);
-  return buf;
+  return {buf, (size_t)(bufsize - strm.avail_out)};
 }

-ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
-  std::vector<u64> adlers(inputs.size());
+ZlibCompressor::ZlibCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
+  std::vector<u32> adlers(inputs.size());
  shards.resize(inputs.size());

  // Compress each shard
  tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    adlers[i] = adler32(1, (u8 *)inputs[i].data(), inputs[i].size());
-    shards[i] = zlib_compress(inputs[i]);
+    std::span<u8> in = inputs[i];
+    adlers[i] = adler32(1, in.data(), in.size());
+    shards[i] = zlib_compress(in, level);
  });

  // Combine checksums
@@ -116,7 +112,7 @@ ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {

  // Comput the total size
  compressed_size = 8; // the header and the trailer
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
    compressed_size += shard.size();
 }

@@ -127,7 +123,7 @@ void ZlibCompressor::write_to(u8 *buf) {

  // Copy compressed data
  std::vector<i64> offsets(shards.size());
-  offsets[0] = 2; // +2 for header
+  offsets[0] = 2; // +2 for the header
  for (i64 i = 1; i < shards.size(); i++)
    offsets[i] = offsets[i - 1] + shards[i - 1].size();

@@ -139,35 +135,28 @@ void ZlibCompressor::write_to(u8 *buf) {
  u8 *end = buf + compressed_size;
  end[-6] = 3;
  end[-5] = 0;
-
-  // Write a checksum
  *(ub32 *)(end - 4) = checksum;
 }

-static std::vector<u8> zstd_compress(std::string_view input) {
-  std::vector<u8> buf(ZSTD_COMPRESSBOUND(input.size()));
-  constexpr int LEVEL = 3; // compression level; must be between 1 to 22
-
-  size_t sz = ZSTD_compress(buf.data(), buf.size(), input.data(), input.size(),
-                            LEVEL);
+static std::span<u8> zstd_compress(std::span<u8> input, int level) {
+  i64 bufsize = ZSTD_COMPRESSBOUND(input.size());
+  u8 *buf = new u8[bufsize];
+  size_t sz = ZSTD_compress(buf, bufsize, input.data(), input.size(), level);
  assert(!ZSTD_isError(sz));
-  buf.resize(sz);
-  buf.shrink_to_fit();
-  return buf;
+  return {buf, sz};
 }

-ZstdCompressor::ZstdCompressor(u8 *buf, i64 size) {
-  std::string_view input{(char *)buf, (size_t)size};
-  std::vector<std::string_view> inputs = split(input);
+ZstdCompressor::ZstdCompressor(u8 *buf, i64 size, i64 level) {
+  std::vector<std::span<u8>> inputs = split(std::span(buf, size));
  shards.resize(inputs.size());

  // Compress each shard
  tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    shards[i] = zstd_compress(inputs[i]);
+    shards[i] = zstd_compress(inputs[i], level);
  });

  compressed_size = 0;
-  for (std::vector<u8> &shard : shards)
+  for (std::span<u8> &shard : shards)
    compressed_size += shard.size();
 }

@@ -1,4 +1,29 @@
 #define MOLD_VERSION "@mold_VERSION@"
 #define MOLD_LIBDIR "@CMAKE_INSTALL_FULL_LIBDIR@"
+#define MOLD_FIRST_TARGET @MOLD_FIRST_TARGET@

-#cmakedefine HAVE_MADVISE 1
+#cmakedefine01 HAVE_MADVISE
+#cmakedefine01 HAVE_UNAME
+#cmakedefine01 MOLD_USE_MIMALLOC
+#cmakedefine01 MOLD_USE_SYSTEM_MIMALLOC
+
+#cmakedefine01 HAVE_TARGET_X86_64
+#cmakedefine01 HAVE_TARGET_I386
+#cmakedefine01 HAVE_TARGET_ARM64LE
+#cmakedefine01 HAVE_TARGET_ARM64BE
+#cmakedefine01 HAVE_TARGET_ARM32LE
+#cmakedefine01 HAVE_TARGET_ARM32BE
+#cmakedefine01 HAVE_TARGET_RV32LE
+#cmakedefine01 HAVE_TARGET_RV32BE
+#cmakedefine01 HAVE_TARGET_RV64LE
+#cmakedefine01 HAVE_TARGET_RV64BE
+#cmakedefine01 HAVE_TARGET_PPC32
+#cmakedefine01 HAVE_TARGET_PPC64V1
+#cmakedefine01 HAVE_TARGET_PPC64V2
+#cmakedefine01 HAVE_TARGET_S390X
+#cmakedefine01 HAVE_TARGET_SPARC64
+#cmakedefine01 HAVE_TARGET_M68K
+#cmakedefine01 HAVE_TARGET_SH4LE
+#cmakedefine01 HAVE_TARGET_SH4BE
+#cmakedefine01 HAVE_TARGET_LOONGARCH64
+#cmakedefine01 HAVE_TARGET_LOONGARCH32
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"

 #include <tbb/parallel_for_each.h>
 #include <zlib.h>
@@ -48,7 +48,7 @@ u32 compute_crc32(u32 crc, u8 *buf, i64 len) {
    len -= sz;
  }

-  tbb::parallel_for_each(shards.begin(), shards.end(), [](Shard &shard) {
+  tbb::parallel_for_each(shards, [](Shard &shard) {
    shard.crc = crc32(0, shard.buf, shard.len);
  });

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"

 #include <cstdlib>

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"

 #include <filesystem>
 #include <sys/stat.h>
@@ -13,27 +13,6 @@

 namespace mold {

-std::string get_realpath(std::string_view path) {
-  std::error_code ec;
-  std::filesystem::path link = std::filesystem::read_symlink(path, ec);
-  if (ec)
-    return std::string(path);
-  return (filepath(path) / ".." / link).lexically_normal().string();
-}
-
-// Removes redundant '/..' or '/.' from a given path.
-// The transformation is done purely by lexical processing.
-// This function does not access file system.
-std::string path_clean(std::string_view path) {
-  return filepath(path).lexically_normal().string();
-}
-
-std::filesystem::path to_abs_path(std::filesystem::path path) {
-  if (path.is_absolute())
-    return path.lexically_normal();
-  return (std::filesystem::current_path() / path).lexically_normal();
-}
-
 // Returns the path of the mold executable itself
 std::string get_self_path() {
 #if __APPLE__ || _WIN32
@@ -1,14 +1,14 @@
 #!/bin/bash
 #
 # This test script takes a Gentoo package name and tries to build it
-# using mold in a Docker environment. We chose Gentoo Linux as a test
+# using mold in a Podman environment. We chose Gentoo Linux as a test
 # target, because its source-based package allows us to build programs
 # locally and run their test suites without any hassle.
 #
 # You can get a complete list of Gentoo packages availalbe for testing
 # with the following command:
 #
-# docker run --rm mold-gentoo emerge --color n -s '' | \
+# podman run --rm mold-gentoo emerge --color n -s '' | \
 #   perl -ne 'next unless m!^\*\s+(\S+/\S+)!; print "$1\n"'

 package="$1"
@@ -20,48 +20,48 @@ fi

 set -x

-# Create a Docker image
-if ! docker image ls mold-gentoo | grep -q mold-gentoo; then
+# Create a Podman image
+if ! podman image ls mold-gentoo | grep -q mold-gentoo; then
  set -e
-  cat <<EOF | docker build -t mold-gentoo -
-FROM gentoo/stage3
+  cat <<EOF | podman build -t mold-gentoo -
+FROM docker.io/gentoo/stage3
 RUN emerge-webrsync
 RUN echo 'USE="X ssl elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary -perl"' >> /etc/portage/make.conf && \
    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
    echo 'FEATURES="\${FEATURE} noclean nostrip ccache -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
-    emerge gdb lld clang vim emacs strace ccache xeyes dev-build/cmake dev-vcs/git && \
+    FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache xeyes dev-build/cmake dev-vcs/git && \
    rm -rf /var/tmp/portage
 EOF
  set +e
 fi

-git_hash=$(./mold --version | perl -ne '/\((\w+)/; print $1;')
+git_hash=$(./dist/mold --version | perl -ne '/\((\w+)/; print $1;')

 if [ "$package" = dev-libs/concurrencykit ]; then
  echo "Skipping known broken package: $package"
  exit 0
 fi

-# Build a given package in Docker
-cmd1='(cd /usr/bin; ln -sf /mold/mold $(realpath ld))'
+# Build a given package in Podman
+cmd1='(cd /usr/bin; ln -sf /mold/dist/mold $(realpath ld))'
 cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
 cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
 filename=`echo "$package" | sed 's!/!_!g'`
-docker="docker run --rm --cap-add=SYS_PTRACE -v `pwd`:/mold -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 3h"
+podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 3h"
 dir=gentoo/$git_hash

 mkdir -p "$dir"/success "$dir"/failure

-$docker nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
+$podman chrt --idle 0 nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
 if [ $? = 0 ]; then
  mv "$dir"/"$filename".mold "$dir"/success
 else
  mv "$dir"/"$filename".mold "$dir"/failure
 fi

-$docker nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
+$podman chrt --idle 0 nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
 if [ $? = 0 ]; then
  mv "$dir"/"$filename".ld "$dir"/success
 else
@@ -1,35 +1,113 @@
-#include "common.h"
+// This file implements a glob matcher that can run multiple glob patterns
+// against an input string. mold uses the glob matcher for symbol name
+// patterns in a version script or a dynamic list file. Since we may need to
+// match hundreds of glob patterns against millions of symbol names, the
+// speed of the matcher is very important.
+//
+// The pattern match implemented in this file is NFA-based, although the
+// cost of the function is O(n*m), where n is the number of NFA states and m
+// is the length of the input string. We do not use recursion or
+// backtracking, unlike a generic NFA-based regular expression matcher. This
+// is doable because glob patterns are very limited subsets of regexes.
+//
+// Here is the explanation of the algorithm. Observe that the only "tricky"
+// meta-character in a glob pattern is "*", which matches zero or more
+// characters. Other characters and meta-characters always match a single
+// input character. So the key of the algorithm is to handle "*" efficiently.
+//
+// We can represent a glob pattern "a*b*" with three NFA states: q_start, q1
+// and q_accept, with the following transition functions:
+//
+//   δ(q_start, "a") = q1
+//   δ(q1, <any character>) = q1
+//   δ(q1, "b") = q_accept
+//   δ(q_accept, <any character>) = q_accept
+//
+// We can construct such an NFA in a straightforward manner. We maintain NFA
+// states as a list, with the initial contents being the start state. Each
+// character except for "*" creates a new NFA state, adds a transition from
+// the last state in the list to the new one, and appends the new state at
+// the end of the list. "*" sets the "is_star" flag on the last NFA state.
+// The flag indicates that the state machine can remain in the state for any
+// input character.
+//
+// An NFA constructed this way doesn't have any complicated loops,
+// ε-transitions, or anything like that. Each state has only one incoming
+// edge. The only loops in the state transition are the self-loops on states
+// followed by a "*". Aside from that, the state machine progresses linearly
+// from the start state to the accept state.
+//
+// Each state of an NFA can be represented by a single bit. If a bit is 1,
+// the non-deterministic state machine is in that state. Otherwise, it's
+// not. Observe that a state with the "is_star" flag will continued to be 1
+// once it becomes 1, since the state machine can loop over the state on any
+// input character.
+//
+// With that observation, we can represent an NFA with a bit vector of N
+// bits, where N is the number of NFA states. For each input character, bit
+// M becomes 1 if
+//
+//   - bit M-1 is 1 and there's a transition from state_{M-1} to state_M
+//     with the given character, or
+//   - bit M is 1 and state_M's "is_star" flag is 1.
+//
+// Initially, the 0th bit is 1 for the start state. At each step, the bits
+// propagate from least significant to most significant positions, at most
+// one bit at a time. If the most significant bit is 1 after the entire
+// input has been processed, the string matches.
+//
+// This propagation can be implemented with bitwise OR, bitwise AND, and a
+// one-bit bit shift on the bit vector. All these operations are very cheap.
+//
+// We can combine multiple glob matchers into a single matcher by simply
+// concatenating the bit vectors of their state machines.
+
+#include "lib.h"

 #include <cstring>

 namespace mold {

-std::optional<Glob> Glob::compile(std::string_view pat) {
-  std::vector<Element> vec;
+static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
+  std::vector<MultiGlob::State> vec(1);

  while (!pat.empty()) {
    u8 c = pat[0];
    pat = pat.substr(1);
+    std::bitset<256> chars;

    switch (c) {
+    case '*':
+      vec.back().is_star = true;
+      continue;
+    case '?':
+      chars.set();
+      break;
+    case '\\':
+      if (pat.empty())
+        return {};
+      chars[pat[0]] = true;
+      pat = pat.substr(1);
+      break;
    case '[': {
      // Here are a few bracket pattern examples:
      //
      // [abc]: a, b or c
      // [$\]!]: $, ] or !
      // [a-czg-i]: a, b, c, z, g, h, or i
-      // [^a-z]: Any character except lowercase letters
-      vec.push_back({BRACKET});
-      std::bitset<256> &bitset = vec.back().bitset;
-
+      // [!a-z]: Any character except lowercase letters
+      //
+      // Both `!` and `^` are accepted as negation markers. `!` is the
+      // POSIX/shell convention used by other linkers. `^` was mold's
+      // original syntax and is kept for backward compatibility.
      bool negate = false;
-      if (!pat.empty() && pat[0] == '^') {
+      bool closed = false;
+
+      if (!pat.empty() && (pat[0] == '!' || pat[0] == '^')) {
        negate = true;
        pat = pat.substr(1);
      }

-      bool closed = false;
-
      while (!pat.empty()) {
        if (pat[0] == ']') {
          pat = pat.substr(1);
@@ -59,9 +137,9 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
            return {};

          for (i64 i = start; i <= end; i++)
-            bitset[i] = true;
+            chars[i] = true;
        } else {
-          bitset[(u8)pat[0]] = true;
+          chars[pat[0]] = true;
          pat = pat.substr(1);
        }
      }
@@ -70,87 +148,109 @@ std::optional<Glob> Glob::compile(std::string_view pat) {
        return {};

      if (negate)
-        bitset.flip();
+        chars.flip();
      break;
    }
-    case '?':
-      vec.push_back({QUESTION});
-      break;
-    case '*':
-      vec.push_back({STAR});
-      break;
-    case '\\':
-      if (pat.empty())
-        return {};
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.push_back({STRING});
-      vec.back().str += pat[0];
-      pat = pat.substr(1);
-      break;
    default:
-      if (vec.empty() || vec.back().kind != STRING)
-        vec.push_back({STRING});
-      vec.back().str += c;
+      chars[c] = true;
      break;
    }
-  }

-  return {Glob{std::move(vec)}};
+    vec.push_back({chars, false});
+  }
+  return vec;
 }

-bool Glob::match(std::string_view str) {
-  return do_match(str, elements);
+// Instead of returning just a match/no match boolean value, our glob
+// matcher returns an integer value associated with each given pattern.
+// If multiple patterns match at the same time, the largest associated
+// value will be returned by find().
+bool MultiGlob::add(std::string_view pat, i64 val) {
+  std::vector<State> vec = parse_glob(pat);
+  if (vec.empty())
+    return false;
+  patterns.push_back({std::move(vec), val});
+  return true;
 }

-bool Glob::do_match(std::string_view str, std::span<Element> elements) {
-  while (!elements.empty()) {
-    Element &e = elements[0];
-    elements = elements.subspan(1);
+void MultiGlob::compile() {
+  if (patterns.empty())
+    return;

-    switch (e.kind) {
-    case STRING:
-      if (!str.starts_with(e.str))
-        return false;
-      str = str.substr(e.str.size());
-      break;
-    case STAR:
-      if (elements.empty())
-        return true;
+  ranges::stable_sort(patterns, ranges::greater(), &GlobPattern::value);

-      // Patterns like "*foo*bar*" should be much more common than more
-      // complex ones like "*foo*[abc]*" or "*foo**?bar*", so we optimize
-      // the former case here.
-      if (elements[0].kind == STRING) {
-        for (;;) {
-          size_t pos = str.find(elements[0].str);
-          if (pos == str.npos)
-            return false;
-          if (do_match(str.substr(pos + elements[0].str.size()),
-                       elements.subspan(1)))
-            return true;
-          str = str.substr(pos + 1);
-        }
-      }
+  std::vector<State> states;
+  for (GlobPattern &p : patterns)
+    append(states, p.states);
+  i64 sz = states.size();

-      // Other cases are handled here.
-      for (i64 j = 0; j < str.size(); j++)
-        if (do_match(str.substr(j), elements))
-          return true;
-      return false;
-    case QUESTION:
-      if (str.empty())
-        return false;
-      str = str.substr(1);
-      break;
-    case BRACKET:
-      if (str.empty() || !e.bitset[str[0]])
-        return false;
-      str = str.substr(1);
-      break;
-    }
+  start_states.resize(sz);
+  for (i64 pos = 0; GlobPattern &p : patterns) {
+    start_states[pos] = true;
+    pos += p.states.size();
  }

-  return str.empty();
+  star_mask.resize(sz);
+  for (i64 i = 0; i < sz; i++)
+    if (states[i].is_star)
+      star_mask[i] = true;
+
+  for (i64 i = 0; i < 256; i++) {
+    char_mask[i].resize(sz);
+    for (i64 j = 0; j < sz; j++)
+      if (states[j].incoming_edge[i])
+        char_mask[i][j] = true;
+  }
+}
+
+i64 MultiGlob::find(std::string_view str) {
+  if (patterns.empty())
+    return -1;
+
+  Bitvector bits = start_states;
+  Bitvector tmp;
+
+  for (u8 c : str) {
+    // This is equivalent to
+    //
+    //   bits = (bits & star_mask) | ((bits << 1) & char_mask[c])
+    //
+    // but we update the existing objects in place to avoid allocating
+    // temporary objects.
+    tmp = bits;
+    tmp &= star_mask;
+    bits <<= 1;
+    bits &= char_mask[c];
+    bits |= tmp;
+  }
+
+  for (i64 pos = 0; GlobPattern &p : patterns) {
+    pos += p.states.size();
+    if (bits[pos - 1])
+      return p.value;
+  }
+  return -1;
+}
+
+bool Glob::add(std::string_view pat, i64 val) {
+  assert(val >= 0);
+  assert(!is_compiled);
+
+  // If the pattern requires only a single substring search, the
+  // Aho-Corasick algorithm is even faster than our glob matcher.
+  if (aho_corasick.can_handle(pat))
+    return aho_corasick.add(pat, val);
+  return multi_glob.add(pat, val);
+}
+
+i64 Glob::find(std::string_view str) {
+  std::call_once(once, [&] {
+    multi_glob.compile();
+    aho_corasick.compile();
+    is_compiled = true;
+  });
+
+  return std::max(multi_glob.find(str), aho_corasick.find(str));
 }

 } // namespace mold
@@ -4,7 +4,7 @@
 // For more info, read
 // https://engineering.fb.com/2018/12/13/data-infrastructure/hyperloglog

-#include "common.h"
+#include "lib.h"

 #include <cmath>

@@ -5,24 +5,30 @@
 //
 // 1. mold is always a cross linker and should not depend on what host it
 //    is running on. For example, users should be able to run mold on a
-//    big-endian SPARC machine to create a little-endian RV64 binary.
+//    little-endian x86 machine to create a big-endian s390x binary.
 //
 // 2. Even though data members in all ELF data strucutres are naturally
 //    aligned, they are not guaranteed to be aligned on memory because of
-//    archive files. Archive files (.a files) align each member only to a
+//    archive files. Archive files (.a files) align each file only to a
 //    2 byte boundary, so anything larger than 2 bytes may be misaligned
 //    in an mmap'ed memory. Misaligned access is an undefined behavior in
 //    C/C++, so we shouldn't cast an arbitrary pointer to a uint32_t, for
 //    example, to read a 32 bit value.
 //
-// The data types defined in this file don't depend on host byte order and
-// don't do unaligned access.
+// The data types defined in this file are independent of the host byte
+// order and are designed to avoid unaligned access.
+//
+// Note that in C/C++, memcpy is a portable and efficient way to access
+// unaligned data, as it is typically treated as an intrinsic. Compilers
+// can easily optimize memcpy calls in this file into a single load or
+// store instruction.

 #pragma once

 #include <bit>
 #include <cstdint>
 #include <cstring>
+#include <type_traits>

 #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
 # if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
@@ -36,109 +42,86 @@

 namespace mold {

-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-
-typedef int8_t i8;
-typedef int16_t i16;
-typedef int32_t i32;
-typedef int64_t i64;
-
-template <typename T, std::endian endian, int size = sizeof(T)>
+template <typename T, bool is_le, int size = sizeof(T)>
 class Integer {
 public:
  constexpr Integer() = default;

-  constexpr Integer(T x) requires (endian == std::endian::little && size == 2)
-    : buf{(u8)x, (u8)(x >> 8)} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::little && size == 3)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16)} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::little && size == 4)
-    : buf{(u8)x, (u8)(x >> 8), (u8)(x >> 16), (u8)(x >> 24)} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::little && size == 8)
-    : buf{(u8)x,         (u8)(x >> 8),  (u8)(x >> 16), (u8)(x >> 24),
-          (u8)(x >> 32), (u8)(x >> 40), (u8)(x >> 48), (u8)(x >> 56)} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::big && size == 2)
-    : buf{(u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::big && size == 3)
-    : buf{(u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::big && size == 4)
-    : buf{(u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8), (u8)x} {}
-
-  constexpr Integer(T x) requires (endian == std::endian::big && size == 8)
-    : buf{(u8)(x >> 56), (u8)(x >> 48), (u8)(x >> 40), (u8)(x >> 32),
-          (u8)(x >> 24), (u8)(x >> 16), (u8)(x >> 8),  (u8)x} {}
-
-  Integer &operator=(T x) {
-    new (this) Integer(x);
-    return *this;
-  }
-
-  operator T() const {
-    if constexpr (endian == std::endian::little) {
-      if constexpr (size == 2)
-        return buf[1] << 8 | buf[0];
-      else if constexpr (size == 3)
-        return buf[2] << 16 | buf[1] << 8 | buf[0];
-      else if constexpr (size == 4)
-        return buf[3] << 24 | buf[2] << 16 | buf[1] << 8 | buf[0];
-      else
-        return (u64)buf[7] << 56 | (u64)buf[6] << 48 |
-               (u64)buf[5] << 40 | (u64)buf[4] << 32 |
-               (u64)buf[3] << 24 | (u64)buf[2] << 16 |
-               (u64)buf[1] << 8  | (u64)buf[0];
+  constexpr Integer(T v) {
+    if (std::is_constant_evaluated() || size == 3) {
+      for (int i = 0; i < size; i++)
+        buf[is_le ? i : (size - i - 1)] = v >> (i * 8);
    } else {
-      if constexpr (size == 2)
-        return buf[0] << 8 | buf[1];
-      else if constexpr (size == 3)
-        return buf[0] << 16 | buf[1] << 8 | buf[2];
-      else if constexpr (size == 4)
-        return buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3];
-      else
-        return (u64)buf[0] << 56 | (u64)buf[1] << 48 |
-               (u64)buf[2] << 40 | (u64)buf[3] << 32 |
-               (u64)buf[4] << 24 | (u64)buf[5] << 16 |
-               (u64)buf[6] << 8  | (u64)buf[7];
+      if (!is_native)
+        v = bswap(v);
+      memcpy(buf, &v, size);
    }
  }

+  operator T() const {
+    if (size == 3) {
+      if (is_le)
+        return buf[2] << 16 | buf[1] << 8 | buf[0];
+      return buf[0] << 16 | buf[1] << 8 | buf[2];
+    }
+
+    T v;
+    memcpy(&v, buf, size);
+    return is_native ? v : bswap(v);
+  }
+
+  Integer &operator=(T v)  { new (this) Integer(v); return *this; }
  Integer &operator++()    { return *this = *this + 1; }
-  Integer operator++(int)  { return ++*this - 1; }
+  Integer operator++(int)  { auto x = *this; ++*this; return x; }
  Integer &operator--()    { return *this = *this - 1; }
-  Integer operator--(int)  { return --*this + 1; }
-  Integer &operator+=(T x) { return *this = *this + x; }
-  Integer &operator-=(T x) { return *this = *this - x; }
-  Integer &operator&=(T x) { return *this = *this & x; }
-  Integer &operator|=(T x) { return *this = *this | x; }
+  Integer operator--(int)  { auto x = *this; --*this; return x; }
+  Integer &operator+=(T v) { return *this = *this + v; }
+  Integer &operator-=(T v) { return *this = *this - v; }
+  Integer &operator&=(T v) { return *this = *this & v; }
+  Integer &operator|=(T v) { return *this = *this | v; }

 private:
-  u8 buf[size];
-};;
+  static constexpr bool is_native =
+    (std::endian::native == (is_le ? std::endian::little : std::endian::big));

-using il16 = Integer<i16, std::endian::little>;
-using il32 = Integer<i32, std::endian::little>;
-using il64 = Integer<i64, std::endian::little>;
+  static T bswap(T v) {
+    switch (size) {
+    case 2: return __builtin_bswap16(v);
+    case 4: return __builtin_bswap32(v);
+    case 8: return __builtin_bswap64(v);
+    default: __builtin_unreachable();
+    }
+  }

-using ul16 = Integer<u16, std::endian::little>;
-using ul24 = Integer<u32, std::endian::little, 3>;
-using ul32 = Integer<u32, std::endian::little>;
-using ul64 = Integer<u64, std::endian::little>;
+  uint8_t buf[size];
+};

-using ib16 = Integer<i16, std::endian::big>;
-using ib32 = Integer<i32, std::endian::big>;
-using ib64 = Integer<i64, std::endian::big>;
+using i8 = int8_t;
+using i16 = int16_t;
+using i32 = int32_t;
+using i64 = int64_t;

-using ub16 = Integer<u16, std::endian::big>;
-using ub24 = Integer<u32, std::endian::big, 3>;
-using ub32 = Integer<u32, std::endian::big>;
-using ub64 = Integer<u64, std::endian::big>;
+using u8 = uint8_t;
+using u16 = uint16_t;
+using u32 = uint32_t;
+using u64 = uint64_t;
+
+using il16 = Integer<i16, true>;
+using il32 = Integer<i32, true>;
+using il64 = Integer<i64, true>;
+
+using ul16 = Integer<u16, true>;
+using ul24 = Integer<u32, true, 3>;
+using ul32 = Integer<u32, true>;
+using ul64 = Integer<u64, true>;
+
+using ib16 = Integer<i16, false>;
+using ib32 = Integer<i32, false>;
+using ib64 = Integer<i64, false>;
+
+using ub16 = Integer<u16, false>;
+using ub24 = Integer<u32, false, 3>;
+using ub32 = Integer<u32, false>;
+using ub64 = Integer<u64, false>;

 } // namespace mold
@@ -1,6 +1,8 @@
 #pragma once

+#include "atomics.h"
 #include "integers.h"
+#include "bitvector.h"

 #include <array>
 #include <atomic>
@@ -59,197 +61,14 @@ public:

 namespace mold {

+namespace ranges = std::ranges;
 using namespace std::literals::string_literals;
 using namespace std::literals::string_view_literals;

-inline char *output_tmpfile;
-
-inline u8 *output_buffer_start = nullptr;
-inline u8 *output_buffer_end = nullptr;
-
-extern std::string mold_git_hash;
-
-std::string errno_string();
-std::string get_self_path();
-void cleanup();
-void install_signal_handler();
-
 inline u64 combine_hash(u64 a, u64 b) {
  return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
 }

-//
-// Error output
-//
-
-// Some C++ stdlibs don't support std::osyncstream even though
-// it's is in the C++20 standard. So we implement it ourselves.
-class SyncStream {
-public:
-  SyncStream(std::ostream &out) : out(out) {}
-
-  ~SyncStream() {
-    emit();
-  }
-
-  template <typename T> SyncStream &operator<<(T &&val) {
-    ss << std::forward<T>(val);
-    return *this;
-  }
-
-  void emit() {
-    if (emitted)
-      return;
-
-    std::scoped_lock lock(mu);
-    out << ss.str() << '\n';
-    emitted = true;
-  }
-
-private:
-  std::ostream &out;
-  std::stringstream ss;
-  bool emitted = false;
-  static inline std::mutex mu;
-};
-
-template <typename Context>
-class Out {
-public:
-  Out(Context &ctx) {}
-
-  template <typename T> Out &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cout};
-};
-
-static std::string_view fatal_mono = "mold: fatal: ";
-static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
-static std::string_view error_mono = "mold: error: ";
-static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
-static std::string_view warning_mono = "mold: warning: ";
-static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
-
-template <typename Context>
-class Fatal {
-public:
-  Fatal(Context &ctx) {
-    out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
-  }
-
-  [[noreturn]] ~Fatal() {
-    out.emit();
-    cleanup();
-    _exit(1);
-  }
-
-  template <typename T> Fatal &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Error {
-public:
-  Error(Context &ctx) {
-    if (ctx.arg.noinhibit_exec) {
-      out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    } else {
-      out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    }
-  }
-
-  template <typename T> Error &operator<<(T &&val) {
-    out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  SyncStream out{std::cerr};
-};
-
-template <typename Context>
-class Warn {
-public:
-  Warn(Context &ctx) {
-    if (ctx.arg.suppress_warnings)
-      return;
-
-    out.emplace(std::cerr);
-
-    if (ctx.arg.fatal_warnings) {
-      *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
-      ctx.has_error = true;
-    } else {
-      *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
-    }
-  }
-
-  template <typename T> Warn &operator<<(T &&val) {
-    if (out)
-      *out << std::forward<T>(val);
-    return *this;
-  }
-
-private:
-  std::optional<SyncStream> out;
-};
-
-//
-// Atomics
-//
-
-// This is the same as std::atomic except that the default memory
-// order is relaxed instead of sequential consistency.
-template <typename T>
-struct Atomic : std::atomic<T> {
-  static constexpr std::memory_order relaxed = std::memory_order_relaxed;
-
-  using std::atomic<T>::atomic;
-
-  Atomic(const Atomic<T> &other) : std::atomic<T>(other.load()) {}
-
-  Atomic<T> &operator=(const Atomic<T> &other) {
-    store(other.load());
-    return *this;
-  }
-
-  void operator=(T val) { store(val); }
-  operator T() const { return load(); }
-
-  void store(T val, std::memory_order order = relaxed) {
-    std::atomic<T>::store(val, order);
-  }
-
-  T load(std::memory_order order = relaxed) const {
-    return std::atomic<T>::load(order);
-  }
-
-  T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
-  T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
-  T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
-  T operator--() { return std::atomic<T>::fetch_sub(1, relaxed) - 1; }
-  T operator++(int) { return std::atomic<T>::fetch_add(1, relaxed); }
-  T operator--(int) { return std::atomic<T>::fetch_sub(1, relaxed); }
-
-  bool test_and_set() {
-    // A relaxed load + branch (assuming miss) takes only around 20 cycles,
-    // while an atomic RMW can easily take hundreds on x86. We note that it's
-    // common that another thread beat us in marking, so doing an optimistic
-    // early test tends to improve performance in the ~20% ballpark.
-    return load() || exchange(true);
-  }
-};
-
 //
 // perf.cc
 //
@@ -312,7 +131,7 @@ class Timer {
 public:
  Timer(Context &ctx, std::string name, Timer *parent = nullptr) {
    record = new TimerRecord(name, parent ? parent->record : nullptr);
-    ctx.timer_records.push_back(std::unique_ptr<TimerRecord>(record));
+    ctx.timer_records.emplace_back(record);
  }

  Timer(const Timer &) = delete;
@@ -329,23 +148,6 @@ private:
  TimerRecord *record;
 };

-//
-// Bit vector
-//
-
-class BitVector {
-public:
-  BitVector() = default;
-  BitVector(u32 size) : vec((size + 7) / 8) {}
-
-  void resize(u32 size) { vec.resize((size + 7) / 8); }
-  bool get(u32 idx) const { return vec[idx / 8] & (1 << (idx % 8)); }
-  void set(u32 idx) { vec[idx / 8] |= 1 << (idx % 8); }
-
-private:
-  std::vector<u8> vec;
-};
-
 //
 // Utility functions
 //
@@ -383,8 +185,14 @@ inline u64 bits(u64 val, u64 hi, u64 lo) {
  return (val >> lo) & ((1LL << (hi - lo + 1)) - 1);
 }

-inline i64 sign_extend(u64 val, i64 size) {
-  return (i64)(val << (63 - size)) >> (63 - size);
+// Cast val to a signed N bit integer.
+// For example, sign_extend(x, 32) == (i32)x for any integer x.
+inline i64 sign_extend(u64 val, i64 n) {
+  return (i64)(val << (64 - n)) >> (64 - n);
+}
+
+inline bool is_int(u64 val, i64 n) {
+  return sign_extend(val, n) == val;
 }

 template <typename T, typename Compare = std::less<T>>
@@ -421,14 +229,6 @@ inline std::vector<T> flatten(std::vector<std::vector<T>> &vec) {
  return ret;
 }

-inline void sort(auto &vec) {
-  std::stable_sort(vec.begin(), vec.end());
-}
-
-inline void sort(auto &vec, auto less) {
-  std::stable_sort(vec.begin(), vec.end(), less);
-}
-
 template <typename T>
 inline void remove_duplicates(std::vector<T> &vec) {
  vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
@@ -495,6 +295,23 @@ inline u64 read_uleb(u8 *buf) {
  return read_uleb(&tmp);
 }

+inline i64 read_sleb(u8 **buf) {
+  u64 val = 0;
+  u8 shift = 0;
+  u8 byte;
+  do {
+    byte = *(*buf)++;
+    val |= (byte & 0x7f) << shift;
+    shift += 7;
+  } while (byte & 0x80);
+  return sign_extend(val, shift);
+}
+
+inline i64 read_sleb(u8 *buf) {
+  u8 *tmp = buf;
+  return read_sleb(&tmp);
+}
+
 inline u64 read_uleb(std::string_view *str) {
  u8 *start = (u8 *)str->data();
  u8 *ptr = start;
@@ -578,25 +395,29 @@ public:
    this->nbuckets = std::max<i64>(MIN_NBUCKETS, bit_ceil(nbuckets));
    i64 bufsize = sizeof(Entry) * this->nbuckets;

-    // Allocate a zero-initialized buffer. We use mmap() if available
-    // because it's faster than malloc() and memset().
+    // Allocate a zero-initialized buffer. mmap() is faster than
+    // malloc()+memset(), and MAP_POPULATE pre-faults pages here to
+    // avoid CoW faults and TLB shootdowns under concurrent inserts.
 #ifdef _WIN32
    entries = (Entry *)_aligned_malloc(bufsize, alignof(Entry));
    memset((void *)entries, 0, bufsize);
+#elif MAP_POPULATE
+    entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
+                            MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
 #else
    entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
                            MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 #endif
  }

-  std::pair<T *, bool> insert(std::string_view key, u32 hash, const T &val) {
+  std::pair<T *, bool> insert(std::string_view key, u64 hash, const T &val) {
    assert(has_single_bit(nbuckets));

-    i64 begin = hash & (nbuckets - 1);
+    u64 begin = hash & (nbuckets - 1);
    u64 mask = nbuckets / NUM_SHARDS - 1;

    for (i64 i = 0; i < MAX_RETRY; i++) {
-      i64 idx = (begin & ~mask) | ((begin + i) & mask);
+      u64 idx = (begin & ~mask) | ((begin + i) & mask);
      Entry &ent = entries[idx];

      // It seems avoiding compare-and-swap is faster overall at least
@@ -634,8 +455,8 @@ public:
        return {&ent.value, false};
    }

-    assert(false && "ConcurrentMap is full");
-    return {nullptr, false};
+    std::cerr << "ConcurrentMap is full\n";
+    abort();
  }

  i64 get_idx(T *value) const {
@@ -662,7 +483,7 @@ public:

    // Since the shard is circular, we need to handle the last entries
    // as if they were next to the first entries.
-    while (entries[end - 1].key)
+    while (begin < end && entries[end - 1].key)
      vec.push_back(entries + --end);

    // Find entries contiguous in the buckets and sort them.
@@ -693,12 +514,12 @@ public:
    return flatten(vec);
  }

-  static constexpr i64 MIN_NBUCKETS = 2048;
+  static constexpr i64 MIN_NBUCKETS = 4096;
  static constexpr i64 NUM_SHARDS = 16;
-  static constexpr i64 MAX_RETRY = 128;
+  static constexpr i64 MAX_RETRY = 256;

  Entry *entries = nullptr;
-  i64 nbuckets = 0;
+  u64 nbuckets = 0;
 };

 //
@@ -732,71 +553,91 @@ private:
 };

 //
-// glob.cc
+// aho-corasick.cc
 //

-class Glob {
-  typedef enum { STRING, STAR, QUESTION, BRACKET } Kind;
-
-  struct Element {
-    Element(Kind k) : kind(k) {}
-    Kind kind;
-    std::string str;
-    std::bitset<256> bitset;
-  };
-
+class AhoCorasick {
 public:
-  static std::optional<Glob> compile(std::string_view pat);
-  bool match(std::string_view str);
+  bool add(std::string_view pat, i64 val);
+  bool empty() const { return nodes.empty(); }
+  void compile();
+  i64 find(std::string_view str);
+
+  static bool can_handle(std::string_view str);

 private:
-  Glob(std::vector<Element> &&vec) : elements(vec) {}
-  static bool do_match(std::string_view str, std::span<Element> elements);
+  struct TrieNode {
+    TrieNode() { children.fill(-1); }
+    i64 value = -1;
+    i32 suffix_link = -1;
+    std::array<i32, 256> children;
+  };

-  std::vector<Element> elements;
+  void fix_suffix_links(i64 idx);
+  void fix_values();
+
+  std::vector<TrieNode> nodes;
 };

 //
-// multi-glob.cc
+// glob.cc
 //

 class MultiGlob {
 public:
  bool add(std::string_view pat, i64 val);
-  bool empty() const { return strings.empty(); }
-  std::optional<i64> find(std::string_view str);
+  bool empty() const { return patterns.empty(); }
+  void compile();
+  i64 find(std::string_view str);

-private:
-  struct TrieNode {
-    i64 value = -1;
-    TrieNode *suffix_link = nullptr;
-    std::unique_ptr<TrieNode> children[256];
+  struct State {
+    std::bitset<256> incoming_edge;
+    bool is_star = false;
  };

-  void compile();
-  void fix_suffix_links(TrieNode &node);
-  void fix_values();
-  i64 find_aho_corasick(std::string_view str);
+  struct GlobPattern {
+    std::vector<State> states;
+    i64 value = -1;
+  };

-  std::vector<std::string> strings;
-  std::unique_ptr<TrieNode> root;
-  std::vector<std::pair<Glob, i64>> globs;
+private:
+  std::vector<GlobPattern> patterns;
+  Bitvector start_states;
+  Bitvector star_mask;
+  Bitvector char_mask[256];
+};
+
+class Glob {
+public:
+  bool add(std::string_view pat, i64 val);
+  bool empty() const { return multi_glob.empty() && aho_corasick.empty(); }
+  i64 find(std::string_view str);
+
+private:
  std::once_flag once;
  bool is_compiled = false;
-  bool prefix_match = false;
+
+  MultiGlob multi_glob;
+  AhoCorasick aho_corasick;
 };

 //
 // filepath.cc
 //

-std::filesystem::path filepath(const auto &path) {
-  return {path, std::filesystem::path::format::generic_format};
+inline std::filesystem::path path_dirname(std::string_view path) {
+  return std::filesystem::path(path).parent_path();
 }

-std::string get_realpath(std::string_view path);
-std::string path_clean(std::string_view path);
-std::filesystem::path to_abs_path(std::filesystem::path path);
+inline std::string path_filename(std::string_view path) {
+  return std::filesystem::path(path).filename().string();
+}
+
+inline std::string path_clean(std::string_view path) {
+  return std::filesystem::path(path).lexically_normal().string();
+}
+
+std::string get_self_path();

 //
 // demangle.cc
@@ -805,13 +646,6 @@ std::filesystem::path to_abs_path(std::filesystem::path path);
 std::optional<std::string_view> demangle_cpp(std::string_view name);
 std::optional<std::string_view> demangle_rust(std::string_view name);

-//
-// jbos.cc
-//
-
-void acquire_global_lock();
-void release_global_lock();
-
 //
 // crc32.cc
 //
@@ -826,27 +660,26 @@ std::vector<u8> crc32_solve(u32 current, u32 desired);
 class Compressor {
 public:
  virtual void write_to(u8 *buf) = 0;
-  virtual ~Compressor() {}
+  virtual ~Compressor();
  i64 compressed_size = 0;
+
+protected:
+  std::vector<std::span<u8>> shards;
 };

 class ZlibCompressor : public Compressor {
 public:
-  ZlibCompressor(u8 *buf, i64 size);
+  ZlibCompressor(u8 *buf, i64 size, i64 level);
  void write_to(u8 *buf) override;

 private:
-  std::vector<std::vector<u8>> shards;
-  u64 checksum = 0;
+  u32 checksum = 0;
 };

 class ZstdCompressor : public Compressor {
 public:
-  ZstdCompressor(u8 *buf, i64 size);
+  ZstdCompressor(u8 *buf, i64 size, i64 level);
  void write_to(u8 *buf) override;
-
-private:
-  std::vector<std::vector<u8>> shards;
 };

 //
@@ -873,96 +706,4 @@ private:
  std::string basedir;
 };

-//
-// Memory-mapped file
-//
-
-// MappedFile represents an mmap'ed input file.
-// mold uses mmap-IO only.
-class MappedFile {
-public:
-  ~MappedFile() { unmap(); }
-  void unmap();
-  void close_fd();
-  void reopen_fd(const std::string &path);
-
-  template <typename Context>
-  MappedFile *slice(Context &ctx, std::string name, u64 start, u64 size) {
-    MappedFile *mf = new MappedFile;
-    mf->name = name;
-    mf->data = data + start;
-    mf->size = size;
-    mf->parent = this;
-
-    ctx.mf_pool.push_back(std::unique_ptr<MappedFile>(mf));
-    return mf;
-  }
-
-  std::string_view get_contents() {
-    return std::string_view((char *)data, size);
-  }
-
-  i64 get_offset() const {
-    return parent ? (data - parent->data + parent->get_offset()) : 0;
-  }
-
-  // Returns a string that uniquely identify a file that is possibly
-  // in an archive.
-  std::string get_identifier() const {
-    if (parent) {
-      // We use the file offset within an archive as an identifier
-      // because archive members may have the same name.
-      return parent->name + ":" + std::to_string(get_offset());
-    }
-
-    if (thin_parent) {
-      // If this is a thin archive member, the filename part is
-      // guaranteed to be unique.
-      return thin_parent->name + ":" + name;
-    }
-    return name;
-  }
-
-  std::string name;
-  u8 *data = nullptr;
-  i64 size = 0;
-  bool given_fullpath = true;
-  MappedFile *parent = nullptr;
-  MappedFile *thin_parent = nullptr;
-
-  // For --dependency-file
-  bool is_dependency = true;
-
-#ifdef _WIN32
-  HANDLE fd = INVALID_HANDLE_VALUE;
-#else
-  int fd = -1;
-#endif
-};
-
-MappedFile *open_file_impl(const std::string &path, std::string &error);
-
-template <typename Context>
-MappedFile *open_file(Context &ctx, std::string path) {
-  if (path.starts_with('/') && !ctx.arg.chroot.empty())
-    path = ctx.arg.chroot + "/" + path_clean(path);
-
-  std::string error;
-  MappedFile *mf = open_file_impl(path, error);
-  if (!error.empty())
-    Fatal(ctx) << error;
-
-  if (mf)
-    ctx.mf_pool.push_back(std::unique_ptr<MappedFile>(mf));
-  return mf;
-}
-
-template <typename Context>
-MappedFile *must_open_file(Context &ctx, std::string path) {
-  MappedFile *mf = open_file(ctx, path);
-  if (!mf)
-    Fatal(ctx) << "cannot open " << path << ": " << errno_string();
-  return mf;
-}
-
 } // namespace mold
@@ -1,5 +0,0 @@
-#include "config.h"
-
-#ifdef USE_SYSTEM_MIMALLOC
-# include <mimalloc-new-delete.h>
-#endif
@@ -1,186 +0,0 @@
-// This file implements the Aho-Corasick algorithm to match multiple
-// glob patterns to symbol strings as quickly as possible.
-//
-// Here are some examples of glob patterns:
-//
-//    qt_private_api_tag*
-//    *16QAccessibleCache*
-//    *32QAbstractFileIconProviderPrivate*
-//    *17QPixmapIconEngine*
-//
-// `*` is a wildcard that matches any substring. We sometimes have
-// hundreds of glob patterns and have to match them against millions
-// of symbol strings.
-//
-// Aho-Corasick cannot handle complex patterns such as `*foo*bar*`.
-// We handle such patterns with the Glob class. Glob is relatively
-// slow, but complex patterns are rare in practice, so it should be
-// OK.
-
-#include "common.h"
-
-#include <queue>
-#include <regex>
-
-namespace mold {
-
-std::optional<i64> MultiGlob::find(std::string_view str) {
-  std::call_once(once, [&] { compile(); });
-  i64 val = -1;
-
-  // Match against simple glob patterns
-  if (root)
-    val = find_aho_corasick(str);
-
-  // Match against complex glob patterns
-  for (std::pair<Glob, i64> &glob : globs)
-    if (glob.first.match(str))
-      val = std::max(val, glob.second);
-
-  if (val == -1)
-    return {};
-  return val;
-}
-
-i64 MultiGlob::find_aho_corasick(std::string_view str) {
-  TrieNode *node = root.get();
-  i64 val = -1;
-
-  auto walk = [&](u8 c) {
-    for (;;) {
-      if (node->children[c]) {
-        node = node->children[c].get();
-        val = std::max(val, node->value);
-        return;
-      }
-
-      if (!node->suffix_link)
-        return;
-      node = node->suffix_link;
-    }
-  };
-
-  walk('\0');
-
-  for (u8 c : str) {
-    if (prefix_match && node == root.get())
-      return val;
-    walk(c);
-  }
-
-  walk('\0');
-  return val;
-}
-
-static bool is_simple_pattern(std::string_view pat) {
-  static std::regex re(R"(\*?[^*[?]+\*?)", std::regex_constants::optimize);
-  return std::regex_match(pat.begin(), pat.end(), re);
-}
-
-static std::string handle_stars(std::string_view pat) {
-  std::string str(pat);
-
-  // Convert "foo" -> "\0foo\0", "*foo" -> "foo\0", "foo*" -> "\0foo"
-  // and "*foo*" -> "foo". Aho-Corasick can do only substring matching,
-  // so we use \0 as beginning/end-of-string markers.
-  if (str.starts_with('*') && str.ends_with('*'))
-    return str.substr(1, str.size() - 2);
-  if (str.starts_with('*'))
-    return str.substr(1) + "\0"s;
-  if (str.ends_with('*'))
-    return "\0"s + str.substr(0, str.size() - 1);
-  return "\0"s + str + "\0"s;
-}
-
-bool MultiGlob::add(std::string_view pat, i64 val) {
-  assert(!is_compiled);
-  assert(!pat.empty());
-
-  strings.push_back(std::string(pat));
-
-  // Complex glob pattern
-  if (!is_simple_pattern(pat)) {
-    if (std::optional<Glob> glob = Glob::compile(pat)) {
-      globs.push_back({std::move(*glob), val});
-      return true;
-    }
-    return false;
-  }
-
-  // Simple glob pattern
-  if (!root)
-    root.reset(new TrieNode);
-  TrieNode *node = root.get();
-
-  for (u8 c : handle_stars(pat)) {
-    if (!node->children[c])
-      node->children[c].reset(new TrieNode);
-    node = node->children[c].get();
-  }
-
-  node->value = std::max(node->value, val);
-  return true;
-}
-
-void MultiGlob::compile() {
-  is_compiled = true;
-  if (root) {
-    fix_suffix_links(*root);
-    fix_values();
-
-    // If no pattern starts with '*', set prefix_match to true.
-    // We'll use this flag for optimization.
-    prefix_match = true;
-    for (i64 i = 1; i < 256; i++) {
-      if (root->children[i]) {
-        prefix_match = false;
-        break;
-      }
-    }
-  }
-}
-
-void MultiGlob::fix_suffix_links(TrieNode &node) {
-  for (i64 i = 0; i < 256; i++) {
-    if (!node.children[i])
-      continue;
-
-    TrieNode &child = *node.children[i];
-
-    TrieNode *cur = node.suffix_link;
-    for (;;) {
-      if (!cur) {
-        child.suffix_link = root.get();
-        break;
-      }
-
-      if (cur->children[i]) {
-        child.suffix_link = cur->children[i].get();
-        break;
-      }
-
-      cur = cur->suffix_link;
-    }
-
-    fix_suffix_links(child);
-  }
-}
-
-void MultiGlob::fix_values() {
-  std::queue<TrieNode *> queue;
-  queue.push(root.get());
-
-  do {
-    TrieNode *node = queue.front();
-    queue.pop();
-
-    for (std::unique_ptr<TrieNode> &child : node->children) {
-      if (!child)
-        continue;
-      child->value = std::max(child->value, child->suffix_link->value);
-      queue.push(child.get());
-    }
-  } while (!queue.empty());
-}
-
-} // namespace mold
@@ -1,10 +1,13 @@
-#include "common.h"
+#include "lib.h"

 #include <functional>
 #include <iomanip>
 #include <ios>
+#include <tbb/concurrent_vector.h>

-#ifndef _WIN32
+#ifdef _WIN32
+#include <windows.h>
+#else
 #include <sys/resource.h>
 #include <sys/time.h>
 #endif
@@ -16,9 +19,7 @@ i64 Counter::get_value() {
 }

 void Counter::print() {
-  sort(instances, [](Counter *a, Counter *b) {
-    return a->get_value() > b->get_value();
-  });
+  ranges::stable_sort(instances, {}, [](Counter *x) { return x->get_value(); });

  for (Counter *c : instances)
    std::cout << std::setw(20) << std::right << c->name
@@ -79,9 +80,7 @@ static void print_rec(TimerRecord &rec, i64 indent) {
         std::string(indent * 2, ' ').c_str(),
         rec.name.c_str());

-  sort(rec.children, [](TimerRecord *a, TimerRecord *b) {
-    return a->start < b->start;
-  });
+  ranges::stable_sort(rec.children, {}, &TimerRecord::start);

  for (TimerRecord *child : rec.children)
    print_rec(*child, indent + 1);
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "lib.h"

 #include <random>

@@ -1,6 +1,6 @@
 // This file contains functions to create a tar file.

-#include "common.h"
+#include "lib.h"

 #ifdef _WIN32
 # define ftruncate _chsize_s
@@ -86,6 +86,9 @@ void TarWriter::append(std::string path, std::string_view data) {
  UstarHeader pax = {};

  std::string attr = encode_path(basedir, path);
+
+  if (attr.size() >= 0100'000'000'000)
+    unreachable();
  snprintf(pax.size, sizeof(pax.size), "%011zo", attr.size());
  pax.name[0] = '/';
  pax.typeflag[0] = 'x';
@@ -99,6 +102,8 @@ void TarWriter::append(std::string path, std::string_view data) {
  // Write Ustar header
  UstarHeader ustar = {};
  memcpy(ustar.mode, "0000664", 8);
+  if (data.size() >= 0100'000'000'000)
+    unreachable();
  snprintf(ustar.size, sizeof(ustar.size), "%011zo", data.size());
  finalize(ustar);
  fwrite(&ustar, sizeof(ustar), 1, out);
@@ -17,11 +17,11 @@ if(EXISTS "${SOURCE_DIR}/.git/HEAD")
 endif()

 # Create new file contents and update a given file if necessary.
-set(NEW_CONTENTS "#include <string>
-namespace mold {
-std::string mold_git_hash = \"${HASH}\";
-}
-")
+if("${HASH}" STREQUAL "")
+  set(NEW_CONTENTS "")
+else()
+  set(NEW_CONTENTS "#define MOLD_GIT_HASH \"${HASH}\"\n")
+endif()

 if(EXISTS "${OUTPUT_FILE}")
  file(READ "${OUTPUT_FILE}" OLD_CONTENTS)
@@ -31,22 +31,25 @@
 //
 // https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst

+#if MOLD_ARM32LE || MOLD_ARM32BE
+
 #include "mold.h"

 #include <tbb/parallel_for.h>
 #include <tbb/parallel_for_each.h>
-#include <tbb/parallel_sort.h>

 namespace mold {

-using E = ARM32;
+using E = MOLD_TARGET;

 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
+  U32<E> *arm = (U32<E> *)loc;
+  U16<E> *thm = (U16<E> *)loc;
+
  switch (rel.r_type) {
  case R_ARM_ABS32:
  case R_ARM_REL32:
-  case R_ARM_TARGET1:
  case R_ARM_BASE_PREL:
  case R_ARM_GOTOFF32:
  case R_ARM_GOT_PREL:
@@ -57,83 +60,110 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
  case R_ARM_TLS_IE32:
  case R_ARM_TLS_LE32:
  case R_ARM_TLS_GOTDESC:
+  case R_ARM_TARGET1:
  case R_ARM_TARGET2:
-    return *(il32 *)loc;
+    return (I32<E>)*arm;
+  case R_ARM_THM_JUMP8:
+    return sign_extend(thm[0], 8) << 1;
  case R_ARM_THM_JUMP11:
-    return sign_extend(*(ul16 *)loc, 10) << 1;
+    return sign_extend(thm[0], 11) << 1;
+  case R_ARM_THM_JUMP19: {
+    // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/B--Branch-
+    u32 S = bit(thm[0], 10);
+    u32 J2 = bit(thm[1], 11);
+    u32 J1 = bit(thm[1], 13);
+    u32 imm6 = bits(thm[0], 5, 0);
+    u32 imm11 = bits(thm[1], 10, 0);
+    u32 val = (S << 20) | (J2 << 19) | (J1 << 18) | (imm6 << 12) | (imm11 << 1);
+    return sign_extend(val, 21);
+  }
  case R_ARM_THM_CALL:
  case R_ARM_THM_JUMP24:
  case R_ARM_THM_TLS_CALL: {
-    u32 S = bit(*(ul16 *)loc, 10);
-    u32 J1 = bit(*(ul16 *)(loc + 2), 13);
-    u32 J2 = bit(*(ul16 *)(loc + 2), 11);
+    // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/BL--BLX--immediate---Branch-with-Link-and-optional-Exchange--immediate--
+    u32 S = bit(thm[0], 10);
+    u32 J1 = bit(thm[1], 13);
+    u32 J2 = bit(thm[1], 11);
    u32 I1 = !(J1 ^ S);
    u32 I2 = !(J2 ^ S);
-    u32 imm10 = bits(*(ul16 *)loc, 9, 0);
-    u32 imm11 = bits(*(ul16 *)(loc + 2), 10, 0);
+    u32 imm10 = bits(thm[0], 9, 0);
+    u32 imm11 = bits(thm[1], 10, 0);
    u32 val = (S << 24) | (I1 << 23) | (I2 << 22) | (imm10 << 12) | (imm11 << 1);
-    return sign_extend(val, 24);
+    return sign_extend(val, 25);
  }
  case R_ARM_CALL:
  case R_ARM_JUMP24:
  case R_ARM_PLT32:
  case R_ARM_TLS_CALL:
-    return sign_extend(*(ul32 *)loc, 23) << 2;
+    return sign_extend(*arm, 24) << 2;
  case R_ARM_MOVW_PREL_NC:
  case R_ARM_MOVW_ABS_NC:
  case R_ARM_MOVT_PREL:
  case R_ARM_MOVT_ABS: {
-    u32 imm12 = bits(*(ul32 *)loc, 11, 0);
-    u32 imm4 = bits(*(ul32 *)loc, 19, 16);
-    return sign_extend((imm4 << 12) | imm12, 15);
+    // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/MOV--MOVS--immediate---Move--immediate--
+    u32 imm4 = bits(*arm, 19, 16);
+    u32 imm12 = bits(*arm, 11, 0);
+    u32 val = (imm4 << 12) | imm12;
+    return sign_extend(val, 16);
  }
  case R_ARM_PREL31:
-    return sign_extend(*(ul32 *)loc, 30);
+    return sign_extend(*arm, 31);
  case R_ARM_THM_MOVW_PREL_NC:
  case R_ARM_THM_MOVW_ABS_NC:
  case R_ARM_THM_MOVT_PREL:
  case R_ARM_THM_MOVT_ABS: {
-    u32 imm4 = bits(*(ul16 *)loc, 3, 0);
-    u32 i = bit(*(ul16 *)loc, 10);
-    u32 imm3 = bits(*(ul16 *)(loc + 2), 14, 12);
-    u32 imm8 = bits(*(ul16 *)(loc + 2), 7, 0);
+    // https://developer.arm.com/documentation/ddi0597/2024-12/Base-Instructions/MOVT--Move-Top-
+    u32 imm4 = bits(thm[0], 3, 0);
+    u32 i = bit(thm[0], 10);
+    u32 imm3 = bits(thm[1], 14, 12);
+    u32 imm8 = bits(thm[1], 7, 0);
    u32 val = (imm4 << 12) | (i << 11) | (imm3 << 8) | imm8;
-    return sign_extend(val, 15);
+    return sign_extend(val, 16);
  }
  default:
    return 0;
  }
 }

-static void write_mov_imm(u8 *loc, u32 val) {
+static void write_arm_mov(u8 *loc, u32 val) {
  u32 imm12 = bits(val, 11, 0);
  u32 imm4 = bits(val, 15, 12);
-  *(ul32 *)loc = (*(ul32 *)loc & 0xfff0f000) | (imm4 << 16) | imm12;
+  *(U32<E> *)loc = (*(U32<E> *)loc & 0xfff0'f000) | (imm4 << 16) | imm12;
 }

-static void write_thm_b_imm(u8 *loc, u32 val) {
-  // https://developer.arm.com/documentation/ddi0406/cb/Application-Level-Architecture/Instruction-Details/Alphabetical-list-of-instructions/BL--BLX--immediate-
-  u32 sign = bit(val, 24);
-  u32 I1 = bit(val, 23);
-  u32 I2 = bit(val, 22);
-  u32 J1 = !I1 ^ sign;
-  u32 J2 = !I2 ^ sign;
-  u32 imm10 = bits(val, 21, 12);
+static void write_thm_b21(u8 *loc, u32 val) {
+  u32 S = bit(val, 20);
+  u32 J2 = bit(val, 19);
+  u32 J1 = bit(val, 18);
+  u32 imm6 = bits(val, 17, 12);
  u32 imm11 = bits(val, 11, 1);

-  ul16 *buf = (ul16 *)loc;
-  buf[0] = (buf[0] & 0b1111'1000'0000'0000) | (sign << 10) | imm10;
+  U16<E> *buf = (U16<E> *)loc;
+  buf[0] = (buf[0] & 0b1111'1011'1100'0000) | (S << 10) | imm6;
  buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
 }

-static void write_thm_mov_imm(u8 *loc, u32 val) {
-  // https://developer.arm.com/documentation/ddi0406/cb/Application-Level-Architecture/Instruction-Details/Alphabetical-list-of-instructions/MOVT
+static void write_thm_b25(u8 *loc, u32 val) {
+  u32 S = bit(val, 24);
+  u32 I1 = bit(val, 23);
+  u32 I2 = bit(val, 22);
+  u32 J1 = !I1 ^ S;
+  u32 J2 = !I2 ^ S;
+  u32 imm10 = bits(val, 21, 12);
+  u32 imm11 = bits(val, 11, 1);
+
+  U16<E> *buf = (U16<E> *)loc;
+  buf[0] = (buf[0] & 0b1111'1000'0000'0000) | (S << 10) | imm10;
+  buf[1] = (buf[1] & 0b1101'0000'0000'0000) | (J1 << 13) | (J2 << 11) | imm11;
+}
+
+static void write_thm_mov(u8 *loc, u32 val) {
  u32 imm4 = bits(val, 15, 12);
  u32 i = bit(val, 11);
  u32 imm3 = bits(val, 10, 8);
  u32 imm8 = bits(val, 7, 0);

-  ul16 *buf = (ul16 *)loc;
+  U16<E> *buf = (U16<E> *)loc;
  buf[0] = (buf[0] & 0b1111'1011'1111'0000) | (i << 10) | imm4;
  buf[1] = (buf[1] & 0b1000'1111'0000'0000) | (imm3 << 12) | imm8;
 }
@@ -145,7 +175,6 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
    break;
  case R_ARM_ABS32:
  case R_ARM_REL32:
-  case R_ARM_TARGET1:
  case R_ARM_BASE_PREL:
  case R_ARM_GOTOFF32:
  case R_ARM_GOT_PREL:
@@ -156,36 +185,40 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
  case R_ARM_TLS_IE32:
  case R_ARM_TLS_LE32:
  case R_ARM_TLS_GOTDESC:
+  case R_ARM_TARGET1:
  case R_ARM_TARGET2:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
+    break;
+  case R_ARM_THM_JUMP8:
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xff00) | bits(val, 8, 1);
    break;
  case R_ARM_THM_JUMP11:
-    *(ul16 *)loc = (*(ul16 *)loc & 0xf800) | bits(val, 11, 1);
+    *(U16<E> *)loc = (*(U16<E> *)loc & 0xf800) | bits(val, 11, 1);
    break;
  case R_ARM_THM_CALL:
  case R_ARM_THM_JUMP24:
  case R_ARM_THM_TLS_CALL:
-    write_thm_b_imm(loc, val);
+    write_thm_b25(loc, val);
    break;
  case R_ARM_CALL:
  case R_ARM_JUMP24:
  case R_ARM_PLT32:
-    *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
    break;
  case R_ARM_MOVW_PREL_NC:
  case R_ARM_MOVW_ABS_NC:
  case R_ARM_MOVT_PREL:
  case R_ARM_MOVT_ABS:
-    write_mov_imm(loc, val);
+    write_arm_mov(loc, val);
    break;
  case R_ARM_PREL31:
-    *(ul32 *)loc = (*(ul32 *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
+    *(U32<E> *)loc = (*(U32<E> *)loc & 0x8000'0000) | (val & 0x7fff'ffff);
    break;
  case R_ARM_THM_MOVW_PREL_NC:
  case R_ARM_THM_MOVW_ABS_NC:
  case R_ARM_THM_MOVT_PREL:
  case R_ARM_THM_MOVT_ABS:
-    write_thm_mov_imm(loc, val);
+    write_thm_mov(loc, val);
    break;
  default:
    unreachable();
@@ -206,10 +239,10 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
  };

  memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
+  *(U32<E> *)(buf + 16) = ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr - 16;
 }

-constexpr ul32 plt_entry[] = {
+static constexpr ul32 plt_entry[] = {
  0xe59f'c004, // 1: ldr ip, 2f
  0xe08c'c00f, //    add ip, ip, pc
  0xe59c'f000, //    ldr pc, [ip]
@@ -219,13 +252,13 @@ constexpr ul32 plt_entry[] = {
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }

 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(U32<E> *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }

 template <>
@@ -237,35 +270,20 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
  case R_NONE:
    break;
  case R_ARM_ABS32:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
    break;
  case R_ARM_REL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
    break;
  default:
    Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
  }
 }

-// ARM and Thumb branch instructions can jump within ±16 MiB.
-static bool is_jump_reachable(i64 val) {
-  return sign_extend(val, 24) == val;
-}
-
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
  std::span<const ElfRel<E>> rels = get_rels(ctx);

-  auto get_tls_trampoline_addr = [&, i = 0](u64 addr) mutable {
-    for (; i < output_section->thunks.size(); i++) {
-      i64 disp = output_section->shdr.sh_addr + output_section->thunks[i]->offset -
-                 addr;
-      if (is_jump_reachable(disp))
-        return disp;
-    }
-    unreachable();
-  };
-
  for (i64 i = 0; i < rels.size(); i++) {
    const ElfRel<E> &rel = rels[i];
    if (rel.r_type == R_NONE || rel.r_type == R_ARM_V4BX)
@@ -274,13 +292,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = get_addend(*this, rel);
    u64 P = get_addr() + rel.r_offset;
@@ -288,86 +299,98 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

-    auto get_thumb_thunk_addr = [&] { return get_thunk_addr(i); };
-    auto get_arm_thunk_addr   = [&] { return get_thunk_addr(i) + 4; };
+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
+    auto get_thumb_thunk_addr = [&] { return sym.get_thunk_addr(ctx, P); };
+    auto get_arm_thunk_addr   = [&] { return sym.get_thunk_addr(ctx, P) + 4; };
+
+    auto get_tlsdesc_trampoline_addr = [&] {
+      auto it = ranges::upper_bound(output_section->thunks, P, {},
+                                    [](std::unique_ptr<Thunk<E>> &thunk) {
+        return thunk->get_addr();
+      });
+      return (*it)->get_addr();
+    };

    switch (rel.r_type) {
    case R_ARM_ABS32:
    case R_ARM_TARGET1:
      break;
    case R_ARM_REL32:
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
      break;
    case R_ARM_THM_CALL: {
      if (sym.is_remaining_undef_weak()) {
        // On ARM, calling an weak undefined symbol jumps to the
        // next instruction.
-        *(ul32 *)loc = 0x8000'f3af; // NOP.W
+        *(U32<E> *)loc = 0x8000'f3af; // NOP.W
        break;
      }

-      // THM_CALL relocation refers either BL or BLX instruction.
+      // THM_CALL relocation refers to either BL or BLX instruction.
      // They are different in only one bit. We need to use BL if
      // the jump target is Thumb. Otherwise, use BLX.
-      i64 val = S + A - P;
-      if (is_jump_reachable(val)) {
-        if (T) {
-          write_thm_b_imm(loc, val);
-          *(ul16 *)(loc + 2) |= 0x1000;  // rewrite to BL
-        } else {
-          write_thm_b_imm(loc, align_to(val, 4));
-          *(ul16 *)(loc + 2) &= ~0x1000; // rewrite to BLX
-        }
+      i64 val1 = S + A - P;
+      i64 val2 = align_to(S + A - P, 4);
+
+      if (T && is_int(val1, 25)) {
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
+        write_thm_b25(loc, val1);
+      } else if (!T && is_int(val2, 25)) {
+        *(U16<E> *)(loc + 2) &= ~0x1000; // BLX
+        write_thm_b25(loc, val2);
      } else {
-        write_thm_b_imm(loc, align_to(get_arm_thunk_addr() + A - P, 4));
-        *(ul16 *)(loc + 2) &= ~0x1000;  // rewrite to BLX
+        *(U16<E> *)(loc + 2) |= 0x1000;  // BL
+        write_thm_b25(loc, get_thumb_thunk_addr() + A - P);
      }
      break;
    }
    case R_ARM_BASE_PREL:
-      *(ul32 *)loc = GOT + A - P;
+      *(U32<E> *)loc = GOT + A - P;
      break;
    case R_ARM_GOTOFF32:
-      *(ul32 *)loc = ((S + A) | T) - GOT;
+      *(U32<E> *)loc = ((S + A) | T) - GOT;
      break;
    case R_ARM_GOT_PREL:
    case R_ARM_TARGET2:
-      *(ul32 *)loc = GOT + G + A - P;
+      *(U32<E> *)loc = GOT + G + A - P;
      break;
    case R_ARM_GOT_BREL:
-      *(ul32 *)loc = G + A;
+      *(U32<E> *)loc = G + A;
      break;
    case R_ARM_CALL: {
      if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
        break;
      }

      // Just like THM_CALL, ARM_CALL relocation refers to either BL or
      // BLX instruction. We may need to rewrite BL → BLX or BLX → BL.
-      bool is_bl = ((*(ul32 *)loc & 0xff00'0000) == 0xeb00'0000);
-      bool is_blx = ((*(ul32 *)loc & 0xfe00'0000) == 0xfa00'0000);
+      bool is_bl = ((*(U32<E> *)loc & 0xff00'0000) == 0xeb00'0000);
+      bool is_blx = ((*(U32<E> *)loc & 0xfe00'0000) == 0xfa00'0000);
      if (!is_bl && !is_blx)
        Fatal(ctx) << *this << ": R_ARM_CALL refers to neither BL nor BLX";

-      u64 val = S + A - P;
-      if (is_jump_reachable(val)) {
+      i64 val = S + A - P;
+      if (is_int(val, 26)) {
        if (T) {
-          *(ul32 *)loc = 0xfa00'0000; // BLX
-          *(ul32 *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
+          *(U32<E> *)loc = 0xfa00'0000; // BLX
+          *(U32<E> *)loc |= (bit(val, 1) << 24) | bits(val, 25, 2);
        } else {
-          *(ul32 *)loc = 0xeb00'0000; // BL
-          *(ul32 *)loc |= bits(val, 25, 2);
+          *(U32<E> *)loc = 0xeb00'0000; // BL
+          *(U32<E> *)loc |= bits(val, 25, 2);
        }
      } else {
-        *(ul32 *)loc = 0xeb00'0000; // BL
-        *(ul32 *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // BL
+        *(U32<E> *)loc |= bits(get_arm_thunk_addr() + A - P, 25, 2);
      }
      break;
    }
    case R_ARM_JUMP24: {
      if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
        break;
      }

@@ -377,101 +400,91 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      // immediate; it takes only a register. So if mode switch is
      // required, we jump to a linker-synthesized thunk which does the
      // job with a longer code sequence.
-      u64 val = S + A - P;
-      if (!is_jump_reachable(val) || T)
+      i64 val = S + A - P;
+      if (T || !is_int(val, 26))
        val = get_arm_thunk_addr() + A - P;
-      *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+      *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
      break;
    }
    case R_ARM_PLT32:
      if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0xe320'f000; // NOP
+        *(U32<E> *)loc = 0xe320'f000; // NOP
      } else {
        u64 val = (T ? get_arm_thunk_addr() : S) + A - P;
-        *(ul32 *)loc = (*(ul32 *)loc & 0xff00'0000) | bits(val, 25, 2);
+        *(U32<E> *)loc = (*(U32<E> *)loc & 0xff00'0000) | bits(val, 25, 2);
      }
      break;
+    case R_ARM_THM_JUMP8:
+      check(S + A - P, -(1 << 8), 1 << 8);
+      *(U16<E> *)loc &= 0xff00;
+      *(U16<E> *)loc |= bits(S + A - P, 8, 1);
+      break;
    case R_ARM_THM_JUMP11:
-      assert(T);
      check(S + A - P, -(1 << 11), 1 << 11);
-      *(ul16 *)loc &= 0xf800;
-      *(ul16 *)loc |= bits(S + A - P, 11, 1);
+      *(U16<E> *)loc &= 0xf800;
+      *(U16<E> *)loc |= bits(S + A - P, 11, 1);
      break;
-    case R_ARM_THM_JUMP19: {
-      i64 val = S + A - P;
-      check(val, -(1 << 19), 1 << 19);
-
-      // sign:J2:J1:imm6:imm11:'0'
-      u32 sign = bit(val, 20);
-      u32 J2 = bit(val, 19);
-      u32 J1 = bit(val, 18);
-      u32 imm6 = bits(val, 17, 12);
-      u32 imm11 = bits(val, 11, 1);
-
-      *(ul16 *)loc &= 0b1111'1011'1100'0000;
-      *(ul16 *)loc |= (sign << 10) | imm6;
-
-      *(ul16 *)(loc + 2) &= 0b1101'0000'0000'0000;
-      *(ul16 *)(loc + 2) |= (J2 << 13) | (J1 << 11) | imm11;
+    case R_ARM_THM_JUMP19:
+      check(S + A - P, -(1 << 20), 1 << 20);
+      write_thm_b21(loc, S + A - P);
      break;
-    }
    case R_ARM_THM_JUMP24: {
      if (sym.is_remaining_undef_weak()) {
-        *(ul32 *)loc = 0x8000'f3af; // NOP
+        *(U32<E> *)loc = 0x8000'f3af; // NOP
        break;
      }

      // Just like R_ARM_JUMP24, we need to jump to a thunk if we need to
      // switch processor mode.
-      u64 val = S + A - P;
-      if (!is_jump_reachable(val) || !T)
+      i64 val = S + A - P;
+      if (!T || !is_int(val, 25))
        val = get_thumb_thunk_addr() + A - P;
-      write_thm_b_imm(loc, val);
+      write_thm_b25(loc, val);
      break;
    }
    case R_ARM_MOVW_PREL_NC:
-      write_mov_imm(loc, ((S + A) | T) - P);
+      write_arm_mov(loc, ((S + A) | T) - P);
      break;
    case R_ARM_MOVW_ABS_NC:
-      write_mov_imm(loc, (S + A) | T);
+      write_arm_mov(loc, (S + A) | T);
      break;
    case R_ARM_THM_MOVW_PREL_NC:
-      write_thm_mov_imm(loc, ((S + A) | T) - P);
+      write_thm_mov(loc, ((S + A) | T) - P);
      break;
    case R_ARM_PREL31:
      check(S + A - P, -(1LL << 30), 1LL << 30);
-      *(ul32 *)loc &= 0x8000'0000;
-      *(ul32 *)loc |= (S + A - P) & 0x7fff'ffff;
+      *(U32<E> *)loc &= 0x8000'0000;
+      *(U32<E> *)loc |= (S + A - P) & 0x7fff'ffff;
      break;
    case R_ARM_THM_MOVW_ABS_NC:
-      write_thm_mov_imm(loc, (S + A) | T);
+      write_thm_mov(loc, (S + A) | T);
      break;
    case R_ARM_MOVT_PREL:
-      write_mov_imm(loc, (S + A - P) >> 16);
+      write_arm_mov(loc, (S + A - P) >> 16);
      break;
    case R_ARM_THM_MOVT_PREL:
-      write_thm_mov_imm(loc, (S + A - P) >> 16);
+      write_thm_mov(loc, (S + A - P) >> 16);
      break;
    case R_ARM_MOVT_ABS:
-      write_mov_imm(loc, (S + A) >> 16);
+      write_arm_mov(loc, (S + A) >> 16);
      break;
    case R_ARM_THM_MOVT_ABS:
-      write_thm_mov_imm(loc, (S + A) >> 16);
+      write_thm_mov(loc, (S + A) >> 16);
      break;
    case R_ARM_TLS_GD32:
-      *(ul32 *)loc = sym.get_tlsgd_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_tlsgd_addr(ctx) + A - P;
      break;
    case R_ARM_TLS_LDM32:
-      *(ul32 *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
+      *(U32<E> *)loc = ctx.got->get_tlsld_addr(ctx) + A - P;
      break;
    case R_ARM_TLS_LDO32:
-      *(ul32 *)loc = S + A - ctx.dtp_addr;
+      *(U32<E> *)loc = S + A - ctx.dtp_addr;
      break;
    case R_ARM_TLS_IE32:
-      *(ul32 *)loc = sym.get_gottp_addr(ctx) + A - P;
+      *(U32<E> *)loc = sym.get_gottp_addr(ctx) + A - P;
      break;
    case R_ARM_TLS_LE32:
-      *(ul32 *)loc = S + A - ctx.tp_addr;
+      *(U32<E> *)loc = S + A - ctx.tp_addr;
      break;
    case R_ARM_TLS_GOTDESC:
      // ARM32 TLSDESC uses the following code sequence to materialize
@@ -500,35 +513,35 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      //  .L2: .word   foo(gottpoff) + . - .L1
      if (sym.has_tlsdesc(ctx)) {
        // A is odd if the corresponding TLS_CALL is Thumb.
-        *(ul32 *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
+        *(U32<E> *)loc = sym.get_tlsdesc_addr(ctx) - P + A - ((A & 1) ? 6 : 4);
      } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
+        *(U32<E> *)loc = sym.get_gottp_addr(ctx) - P + A - ((A & 1) ? 5 : 8);
      } else {
-        *(ul32 *)loc = S - ctx.tp_addr;
+        *(U32<E> *)loc = S - ctx.tp_addr;
      }
      break;
    case R_ARM_TLS_CALL:
      if (sym.has_tlsdesc(ctx)) {
-        // BL <tls_trampoline>
-        *(ul32 *)loc = 0xeb00'0000 | bits(get_tls_trampoline_addr(P + 8), 25, 2);
+        *(U32<E> *)loc = 0xeb00'0000; // bl 0
+        *(U32<E> *)loc |= bits(get_tlsdesc_trampoline_addr() - P - 8, 25, 2);
      } else if (sym.has_gottp(ctx)) {
-        *(ul32 *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
+        *(U32<E> *)loc = 0xe79f'0000; // ldr r0, [pc, r0]
      } else {
-        *(ul32 *)loc = 0xe320'f000; // nop
+        *(U32<E> *)loc = 0xe320'f000; // nop
      }
      break;
    case R_ARM_THM_TLS_CALL:
      if (sym.has_tlsdesc(ctx)) {
-        u64 val = align_to(get_tls_trampoline_addr(P + 4), 4);
-        write_thm_b_imm(loc, val);
-        *(ul16 *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
+        u64 val = align_to(get_tlsdesc_trampoline_addr() - P - 4, 4);
+        write_thm_b25(loc, val);
+        *(U16<E> *)(loc + 2) &= ~0x1000; // rewrite BL with BLX
      } else if (sym.has_gottp(ctx)) {
        // Since `ldr r0, [pc, r0]` is not representable in Thumb,
        // we use two instructions instead.
-        *(ul16 *)loc = 0x4478;         // add r0, pc
-        *(ul16 *)(loc + 2) = 0x6800;   // ldr r0, [r0]
+        *(U16<E> *)loc = 0x4478;         // add r0, pc
+        *(U16<E> *)(loc + 2) = 0x6800;   // ldr r0, [r0]
      } else {
-        *(ul32 *)loc = 0x8000'f3af;    // nop.w
+        *(U32<E> *)loc = 0x8000'f3af;    // nop.w
      }
      break;
    default:
@@ -559,15 +572,15 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    switch (rel.r_type) {
    case R_ARM_ABS32:
      if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
      else
-        *(ul32 *)loc = S + A;
+        *(U32<E> *)loc = S + A;
      break;
    case R_ARM_TLS_LDO32:
      if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
      else
-        *(ul32 *)loc = S + A - ctx.dtp_addr;
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
      break;
    default:
      Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
@@ -633,12 +646,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      check_tlsle(ctx, sym, rel);
      break;
    case R_ARM_ABS32:
+    case R_ARM_TARGET1:
    case R_ARM_MOVT_ABS:
    case R_ARM_THM_MOVT_ABS:
-    case R_ARM_TARGET1:
    case R_ARM_REL32:
    case R_ARM_BASE_PREL:
    case R_ARM_GOTOFF32:
+    case R_ARM_THM_JUMP8:
    case R_ARM_THM_JUMP11:
    case R_ARM_THM_JUMP19:
    case R_ARM_MOVW_PREL_NC:
@@ -667,7 +681,7 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {

  // This is a range extension and mode switch thunk.
  // It has two entry points: +0 for Thumb and +4 for ARM.
-  static const u8 entry[] = {
+  constexpr u8 entry[] = {
    // .thumb
    0x78, 0x47,             //    bx   pc  # jumps to 1f
    0xc0, 0x46,             //    nop
@@ -680,27 +694,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
  static_assert(E::thunk_hdr_size == sizeof(hdr));
  static_assert(E::thunk_size == sizeof(entry));

-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  memcpy(buf, hdr, sizeof(hdr));
-  buf += sizeof(hdr);
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  memcpy(base, hdr, sizeof(hdr));

-  u64 P = output_section.shdr.sh_addr + offset + sizeof(hdr);
-
-  for (Symbol<E> *sym : symbols) {
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
    memcpy(buf, entry, sizeof(entry));
-    *(ul32 *)(buf + 12) = sym->get_addr(ctx) - P - 16;
-
-    buf += sizeof(entry);
-    P += sizeof(entry);
+    *(U32<E> *)(buf + 12) = S - P - 16;
  }
 }

 template <>
 u64 get_eflags(Context<E> &ctx) {
-  return EF_ARM_EABI_VER5;
+  if constexpr (E::is_le)
+    return EF_ARM_EABI_VER5;
+  else
+    return EF_ARM_EABI_VER5 | EF_ARM_BE8;
 }

-void create_arm_exidx_section(Context<E> &ctx) {
+template <>
+void create_arm_exidx_section<E>(Context<E> &ctx) {
  for (i64 i = 0; i < ctx.chunks.size(); i++) {
    OutputSection<E> *osec = ctx.chunks[i]->to_osec();

@@ -717,26 +732,27 @@ void create_arm_exidx_section(Context<E> &ctx) {
  }
 }

-void Arm32ExidxSection::compute_section_size(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::compute_section_size(Context<E> &ctx) {
  output_section.compute_section_size(ctx);
-  this->shdr.sh_size = output_section.shdr.sh_size;
+  this->shdr.sh_size = output_section.shdr.sh_size + 8; // +8 for sentinel
 }

-void Arm32ExidxSection::update_shdr(Context<E> &ctx) {
+template <>
+void Arm32ExidxSection<E>::update_shdr(Context<E> &ctx) {
  // .ARM.exidx's sh_link should be set to the .text section index.
  // Runtime doesn't care about it, but the binutils's strip command does.
  if (Chunk<E> *chunk = find_chunk(ctx, ".text"))
    this->shdr.sh_link = chunk->shndx;
 }

-void Arm32ExidxSection::remove_duplicate_entries(Context<E> &ctx) {
-  this->shdr.sh_size = get_contents(ctx).size();
-}
-
-void Arm32ExidxSection::copy_buf(Context<E> &ctx) {
-  std::vector<u8> contents = get_contents(ctx);
-  assert(this->shdr.sh_size = contents.size());
-  write_vector(ctx.buf + this->shdr.sh_offset, contents);
+// Returns the end of the text segment
+static u64 get_text_end(Context<E> &ctx) {
+  u64 ret = 0;
+  for (Chunk<E> *chunk : ctx.chunks)
+    if (chunk->shdr.sh_flags & SHF_EXECINSTR)
+      ret = std::max<u64>(ret, chunk->shdr.sh_addr + chunk->shdr.sh_size);
+  return ret;
 }

 // ARM executables use an .ARM.exidx section to look up an exception
@@ -748,12 +764,8 @@ void Arm32ExidxSection::copy_buf(Context<E> &ctx) {
 // likely that it's due to some historical reason.
 //
 // This function returns contents of .ARM.exidx.
-std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
-  std::vector<u8> buf(output_section.shdr.sh_size);
-
-  output_section.shdr.sh_addr = this->shdr.sh_addr;
-  output_section.write_to(ctx, buf.data(), nullptr);
-
+template <>
+std::vector<u8> Arm32ExidxSection<E>::get_contents(Context<E> &ctx) {
  // .ARM.exidx records consists of a signed 31-bit relative address
  // and a 32-bit value. The relative address indicates the start
  // address of a function that the record covers. The value is one of
@@ -769,15 +781,23 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
  const u32 CANTUNWIND = 1;

  struct Entry {
-    ul32 addr;
-    ul32 val;
+    U32<E> addr;
+    U32<E> val;
  };

-  if (buf.size() % sizeof(Entry))
-    Fatal(ctx) << "invalid .ARM.exidx section size";
-
+  // We reserve one extra slot for the sentinel
+  i64 num_entries = output_section.shdr.sh_size / sizeof(Entry) + 1;
+  std::vector<u8> buf(num_entries * sizeof(Entry));
  Entry *ent = (Entry *)buf.data();
-  i64 num_entries = buf.size() / sizeof(Entry);
+
+  // Write section contents to the buffer
+  output_section.shdr.sh_addr = this->shdr.sh_addr;
+  output_section.write_to(ctx, buf.data());
+
+  // Fill in sentinel fields
+  u64 sentinel_addr = this->shdr.sh_addr + sizeof(Entry) * (num_entries - 1);
+  ent[num_entries - 1].addr = get_text_end(ctx) - sentinel_addr;
+  ent[num_entries - 1].val = CANTUNWIND;

  // Entry's addresses are relative to themselves. In order to sort
  // records by address, we first translate them so that the addresses
@@ -788,24 +808,18 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {

  tbb::parallel_for((i64)0, num_entries, [&](i64 i) {
    i64 offset = sizeof(Entry) * i;
-    ent[i].addr = sign_extend(ent[i].addr, 30) + offset;
+    ent[i].addr = sign_extend(ent[i].addr, 31) + offset;
    if (is_relative(ent[i].val))
      ent[i].val = 0x7fff'ffff & (ent[i].val + offset);
  });

-  std::sort(ent, ent + num_entries, [](const Entry &a, const Entry &b) {
-    return a.addr < b.addr;
-  });
+  ranges::sort(ent, ent + num_entries, {}, &Entry::addr);

  // Remove duplicate adjacent entries. That is, if two adjacent functions
  // have the same compact unwind info or are both CANTUNWIND, we can
  // merge them into a single address range.
-  auto it = std::unique(ent, ent + num_entries,
-                        [](const Entry &a, const Entry &b) {
-    return a.val == b.val;
-  });
-
-  num_entries = it - ent;
+  auto tail = ranges::unique(ent, ent + num_entries, {}, &Entry::val);
+  num_entries -= tail.size();
  buf.resize(num_entries * sizeof(Entry));

  // Make addresses relative to themselves.
@@ -819,4 +833,84 @@ std::vector<u8> Arm32ExidxSection::get_contents(Context<E> &ctx) {
  return buf;
 }

+template <>
+void Arm32ExidxSection<E>::remove_duplicate_entries(Context<E> &ctx) {
+  this->shdr.sh_size = get_contents(ctx).size();
+}
+
+template <>
+void Arm32ExidxSection<E>::copy_buf(Context<E> &ctx) {
+  std::vector<u8> contents = get_contents(ctx);
+  assert(this->shdr.sh_size == contents.size());
+  write_vector(ctx.buf + this->shdr.sh_offset, contents);
+}
+
+// Even though using ARM32 in big-endian mode is very rare, the processor
+// technically supports both little- and big-endian modes. There are two
+// variants of big-endian mode: BE32 and BE8. In BE32, instructions and
+// data are encoded in big-endian. In BE8, instructions are encoded in
+// little-endian, and only data is in big-endian. BE8 is the de facto
+// standard for ARMv6 or later. We support only BE8.
+//
+// A tricky thing is that instructions in an object file are always
+// big-endian if the file is compiled for big-endian mode. In other words,
+// the compiler always emit code in BE32 if -mbig-endian is specified. It
+// is the linker's responsibility to rewrite instructions from big-endian
+// to little-endian for an BE8 output. This function does that.
+//
+// The text section may contain a mix of 32-bit ARM instructions, 16-bit
+// Thumb instructions, and data. We need to distinguish them to swap 4
+// bytes, 2 bytes, or not swap bytes, respectively. The beginning of ARM
+// code, Thumb code, and data is labeled with a mapping symbol of $a, $t,
+// and $d, respectively. We use mapping symbols to determine what to do
+// with the text section.
+//
+// This function is called after we copy the input section contents to the
+// output file. We rewrite instructions in the output buffer in place.
+#if MOLD_ARM32BE
+void arm32be_swap_bytes(Context<E> &ctx) {
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    // Collect mapping symbols
+    std::vector<Symbol<E> *> syms;
+
+    for (Symbol<E> *sym : file->get_local_syms())
+      if (InputSection<E> *isec = sym->get_input_section())
+        if (isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
+          if (std::string_view x = sym->name();
+              x == "$a" || x.starts_with("$a.") ||
+              x == "$t" || x.starts_with("$t.") ||
+              x == "$d" || x.starts_with("$d."))
+            syms.push_back(sym);
+
+    // Group mapping symbols by input section and sort by address
+    ranges::stable_sort(syms, {}, [](const Symbol<E> *sym) {
+      return std::tuple{(uintptr_t)sym->get_input_section(), sym->value};
+    });
+
+    // Swap bytes
+    for (i64 i = 0; i < syms.size(); i++) {
+      Symbol<E> &sym = *syms[i];
+      if (sym.name().starts_with("$d"))
+        continue;
+
+      InputSection<E> &isec = *sym.get_input_section();
+      u8 *base = ctx.buf + isec.output_section->shdr.sh_offset + isec.offset;
+      u8 *start = base + sym.value;
+      u8 *end;
+
+      if (i + 1 < syms.size() && syms[i + 1]->get_input_section() == &isec)
+        end = base + syms[i + 1]->value;
+      else
+        end = base + isec.sh_size;
+
+      i64 sz = sym.name().starts_with("$a") ? 4 : 2;
+      for (u8 *p = start; p < end; p += sz)
+        std::reverse(p, p + sz);
+    }
+  });
+}
+#endif
+
 } // namespace mold
+
+#endif
@@ -17,11 +17,13 @@
 //
 // https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst

+#if MOLD_ARM64LE || MOLD_ARM64BE
+
 #include "mold.h"

 namespace mold {

-using E = ARM64;
+using E = MOLD_TARGET;

 static void write_adrp(u8 *buf, u64 val) {
  *(ul32 *)buf |= (bits(val, 13, 12) << 29) | (bits(val, 32, 14) << 5);
@@ -110,13 +112,13 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
  case R_NONE:
    break;
  case R_AARCH64_ABS64:
-    *(ul64 *)loc = val;
+    *(U64<E> *)loc = val;
    break;
  case R_AARCH64_PREL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
    break;
  case R_AARCH64_PREL64:
-    *(ul64 *)loc = val - this->shdr.sh_addr - offset;
+    *(U64<E> *)loc = val - this->shdr.sh_addr - offset;
    break;
  default:
    Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -153,19 +155,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = rel.r_addend;
    u64 P = get_addr() + rel.r_offset;
    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_AARCH64_ABS64:
      break;
@@ -209,6 +208,30 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_AARCH64_MOVW_UABS_G3:
      *(ul32 *)loc |= bits(S + A, 63, 48) << 5;
      break;
+    case R_AARCH64_MOVW_PREL_G0:
+      check(S + A - P, -(1 << 15), 1 << 15);
+      write_movn_movz(loc, S + A - P);
+      break;
+    case R_AARCH64_MOVW_PREL_G0_NC:
+      *(ul32 *)loc |= bits(S + A - P, 15, 0) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G1:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
+      write_movn_movz(loc, (i64)(S + A - P) >> 16);
+      break;
+    case R_AARCH64_MOVW_PREL_G1_NC:
+      *(ul32 *)loc |= bits(S + A - P, 31, 16) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G2:
+      check(S + A - P, -(1LL << 47), 1LL << 47);
+      write_movn_movz(loc, (i64)(S + A - P) >> 32);
+      break;
+    case R_AARCH64_MOVW_PREL_G2_NC:
+      *(ul32 *)loc |= bits(S + A - P, 47, 32) << 5;
+      break;
+    case R_AARCH64_MOVW_PREL_G3:
+      *(ul32 *)loc |= bits(S + A - P, 63, 48) << 5;
+      break;
    case R_AARCH64_ADR_GOT_PAGE:
      if (sym.has_got(ctx)) {
        i64 val = page(G + GOT + A) - page(P);
@@ -236,7 +259,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
          i + 1 < rels.size()) {
        i64 val = S + A - P - 4;
        const ElfRel<E> &rel2 = rels[i + 1];
-        if (sign_extend(val, 20) == val &&
+        if (is_int(val, 21) &&
            rel2.r_type == R_AARCH64_ADD_ABS_LO12_NC &&
            rel2.r_sym == rel.r_sym &&
            rel2.r_offset == rel.r_offset + 4 &&
@@ -275,14 +298,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      }

      i64 val = S + A - P;
-      if (val < -(1 << 27) || (1 << 27) <= val)
-        val = get_thunk_addr(i) + A - P;
+      if (!is_int(val, 28))
+        val = sym.get_thunk_addr(ctx, P) + A - P;
      *(ul32 *)loc |= bits(val, 27, 2);
      break;
    }
    case R_AARCH64_PLT32:
      check(S + A - P, -(1LL << 31), 1LL << 31);
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
+      break;
+    case R_AARCH64_GOTPCREL32:
+      check(G + GOT + A - P, -(1LL << 31), 1LL << 31);
+      *(U32<E> *)loc = G + GOT + A - P;
      break;
    case R_AARCH64_CONDBR19:
    case R_AARCH64_LD_PREL_LO19:
@@ -290,15 +317,15 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul32 *)loc |= bits(S + A - P, 20, 2) << 5;
      break;
    case R_AARCH64_PREL16:
-      check(S + A - P, -(1LL << 15), 1LL << 15);
-      *(ul16 *)loc = S + A - P;
+      check(S + A - P, -(1LL << 15), 1LL << 16);
+      *(U16<E> *)loc = S + A - P;
      break;
    case R_AARCH64_PREL32:
      check(S + A - P, -(1LL << 31), 1LL << 32);
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
      break;
    case R_AARCH64_PREL64:
-      *(ul64 *)loc = S + A - P;
+      *(U64<E> *)loc = S + A - P;
      break;
    case R_AARCH64_LD64_GOT_LO12_NC:
      *(ul32 *)loc |= bits(G + GOT + A, 11, 3) << 10;
@@ -446,13 +473,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -460,19 +480,21 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
    u64 A = frag ? frag_addend : (i64)rel.r_addend;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, val, i, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_AARCH64_ABS64:
      if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul64 *)loc = *val;
+        *(U64<E> *)loc = *val;
      else
-        *(ul64 *)loc = S + A;
+        *(U64<E> *)loc = S + A;
      break;
-    case R_AARCH64_ABS32: {
-      i64 val = S + A;
-      check(val, 0, 1LL << 32);
-      *(ul32 *)loc = val;
+    case R_AARCH64_ABS32:
+      check(S + A, 0, 1LL << 32);
+      *(U32<E> *)loc = S + A;
      break;
-    }
    default:
      Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                 << rel;
@@ -531,6 +553,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      break;
    case R_AARCH64_LD64_GOT_LO12_NC:
    case R_AARCH64_LD64_GOTPAGE_LO15:
+    case R_AARCH64_GOTPCREL32:
      sym.flags |= NEEDS_GOT;
      break;
    case R_AARCH64_CALL26:
@@ -574,6 +597,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_AARCH64_MOVW_UABS_G1_NC:
    case R_AARCH64_MOVW_UABS_G2:
    case R_AARCH64_MOVW_UABS_G2_NC:
+    case R_AARCH64_MOVW_PREL_G0:
+    case R_AARCH64_MOVW_PREL_G0_NC:
+    case R_AARCH64_MOVW_PREL_G1:
+    case R_AARCH64_MOVW_PREL_G1_NC:
+    case R_AARCH64_MOVW_PREL_G2:
+    case R_AARCH64_MOVW_PREL_G2_NC:
+    case R_AARCH64_MOVW_PREL_G3:
    case R_AARCH64_PREL16:
    case R_AARCH64_PREL32:
    case R_AARCH64_PREL64:
@@ -593,29 +623,81 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
  }
 }

+// The size of a thunk entry varies on ARM64 depending on the distance to
+// the branch target. This function computes the size of each thunk entry.
+template <>
+void Thunk<E>::shrink_size(Context<E> &ctx) {
+  offsets.clear();
+  offsets.push_back(0);
+  i64 off = 0;
+
+  // The distance between S and P is only reduced by shrink_size(), but
+  // page(S) – page(P) may still increase by one page due to address
+  // changes, so we add a safety margin.
+  //
+  // For example, page(0x1200) – page(0x1000) is 0, whereas
+  // page(0x1100) – page(0xfff) is 0x1000, even though the latter
+  // distance is shorter than the former.
+  auto is_small = [](i64 prel) {
+    return is_int(prel + 0x1000, 33) && is_int(prel - 0x1000, 33);
+  };
+
+  for (Symbol<E> *sym : symbols) {
+    u64 S = sym->get_addr(ctx);
+    u64 P = get_addr() + off;
+    i64 prel = page(S) - page(P);
+    off += is_small(prel) ? 16 : 32;
+    offsets.push_back(off);
+  }
+}
+
 template <>
 void Thunk<E>::copy_buf(Context<E> &ctx) {
-  constexpr ul32 insn[] = {
-    0x9000'0010, // adrp x16, 0   # R_AARCH64_ADR_PREL_PG_HI21
-    0x9100'0210, // add  x16, x16 # R_AARCH64_ADD_ABS_LO12_NC
+  // Short thunk with a 33 bit displacement
+  constexpr ul32 insn1[] = {
+    0x9000'0010, // adrp x16, 0
+    0x9100'0210, // add  x16, x16
    0xd61f'0200, // br   x16
    0xd420'7d00, // brk
  };

-  static_assert(E::thunk_size == sizeof(insn));
+  // Long thunk with a 64 bit displacement
+  constexpr ul32 insn2[] = {
+    0x1000'0010, // adr  x16, 0
+    0xd2a0'0011, // movz x17, 0, lsl #16
+    0xf2c0'0011, // movk x17, 0, lsl #32
+    0xf2e0'0011, // movk x17, 0, lsl #48
+    0x8b11'0210, // add  x16, x16, x17
+    0xd61f'0200, // br   x16
+    0xd420'7d00, // brk
+    0xd420'7d00, // brk
+  };

-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;

-  for (Symbol<E> *sym : symbols) {
-    u64 S = sym->get_addr(ctx);
-    memcpy(buf, insn, E::thunk_size);
-    write_adrp(buf, page(S) - page(P));
-    *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
+  for (i64 i = 0; i < symbols.size(); i++) {
+    u64 S = symbols[i]->get_addr(ctx);
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];

-    buf += E::thunk_size;
-    P += E::thunk_size;
+    if (offsets[i + 1] - offsets[i] == 16) {
+      i64 prel = page(S) - page(P);
+      assert(is_int(prel, 33));
+      memcpy(buf, insn1, sizeof(insn1));
+      write_adrp(buf, prel);
+      *(ul32 *)(buf + 4) |= bits(S, 11, 0) << 10;
+    } else {
+      memcpy(buf, insn2, sizeof(insn2));
+      write_adr(buf, bits(S - P, 15, 0));
+      *(ul32 *)(buf + 4) |= bits(S - P, 31, 16) << 5;
+      *(ul32 *)(buf + 8) |= bits(S - P, 47, 32) << 5;
+      *(ul32 *)(buf + 12) |= bits(S - P, 63, 48) << 5;
+    }
  }
 }

+template class Thunk<E>;
+
 } // namespace mold
+
+#endif
@@ -33,6 +33,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/i386.pdf

+#if MOLD_I386
+
 #include "mold.h"

 namespace mold {
@@ -44,10 +46,10 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
  switch (rel.r_type) {
  case R_386_8:
  case R_386_PC8:
-    return *loc;
+    return *(i8 *)loc;
  case R_386_16:
  case R_386_PC16:
-    return *(ul16 *)loc;
+    return *(il16 *)loc;
  case R_386_32:
  case R_386_PC32:
  case R_386_GOT32:
@@ -63,7 +65,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
  case R_386_TLS_LDO_32:
  case R_386_SIZE32:
  case R_386_TLS_GOTDESC:
-    return *(ul32 *)loc;
+    return *(il32 *)loc;
  default:
    return 0;
  }
@@ -108,25 +110,24 @@ template <>
 void write_plt_header(Context<E> &ctx, u8 *buf) {
  if (ctx.arg.pic) {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb, // endbr32
      0x51,                   // push   %ecx
      0x8d, 0x8b, 0, 0, 0, 0, // lea    GOTPLT+4(%ebx), %ecx
      0xff, 0x31,             // push   (%ecx)
      0xff, 0x61, 0x04,       // jmp    *0x4(%ecx)
+      0xcc, 0xcc, 0xcc, 0xcc, // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 7) = ctx.gotplt->shdr.sh_addr - ctx.got->shdr.sh_addr + 4;
+    *(ul32 *)(buf + 3) = ctx.gotplt->shdr.sh_addr - ctx.got->shdr.sh_addr + 4;
  } else {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb, // endbr32
-      0x51,                   // push   %ecx
-      0xb9, 0, 0, 0, 0,       // mov    GOTPLT+4, %ecx
-      0xff, 0x31,             // push   (%ecx)
-      0xff, 0x61, 0x04,       // jmp    *0x4(%ecx)
-      0xcc,                   // (padding)
+      0x51,                         // push   %ecx
+      0xb9, 0, 0, 0, 0,             // mov    GOTPLT+4, %ecx
+      0xff, 0x31,                   // push   (%ecx)
+      0xff, 0x61, 0x04,             // jmp    *0x4(%ecx)
+      0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = ctx.gotplt->shdr.sh_addr + 4;
+    *(ul32 *)(buf + 2) = ctx.gotplt->shdr.sh_addr + 4;
  }
 }

@@ -134,24 +135,22 @@ template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  if (ctx.arg.pic) {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb, // endbr32
-      0xb9, 0, 0, 0, 0,       // mov $reloc_offset, %ecx
-      0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx)
-      0xcc,                   // (padding)
+      0xb9, 0, 0, 0, 0,             // mov $reloc_offset, %ecx
+      0xff, 0xa3, 0, 0, 0, 0,       // jmp *foo@GOT(%ebx)
+      0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 5) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
-    *(ul32 *)(buf + 11) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 1) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(ul32 *)(buf + 7) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
  } else {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb, // endbr32
-      0xb9, 0, 0, 0, 0,       // mov $reloc_offset, %ecx
-      0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT
-      0xcc,                   // (padding)
+      0xb9, 0, 0, 0, 0,             // mov $reloc_offset, %ecx
+      0xff, 0x25, 0, 0, 0, 0,       // jmp *foo@GOT
+      0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 5) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
-    *(ul32 *)(buf + 11) = sym.get_gotplt_addr(ctx);
+    *(ul32 *)(buf + 1) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(ul32 *)(buf + 7) = sym.get_gotplt_addr(ctx);
  }
 }

@@ -159,20 +158,18 @@ template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  if (ctx.arg.pic) {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb,             // endbr32
-      0xff, 0xa3, 0, 0, 0, 0,             // jmp *foo@GOT(%ebx)
-      0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
+      0xff, 0xa3, 0, 0, 0, 0, // jmp *foo@GOT(%ebx)
+      0xcc, 0xcc,             // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 2) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
  } else {
    static const u8 insn[] = {
-      0xf3, 0x0f, 0x1e, 0xfb,             // endbr32
-      0xff, 0x25, 0, 0, 0, 0,             // jmp *foo@GOT
-      0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
+      0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT
+      0xcc, 0xcc,             // (padding)
    };
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx);
+    *(ul32 *)(buf + 2) = sym.get_got_pltgot_addr(ctx);
  }
 }

@@ -242,11 +239,10 @@ static void relax_ld_to_le(u8 *loc, ElfRel<E> rel, u64 tls_size) {
  case R_386_GOT32X: {
    static const u8 insn[] = {
      0x65, 0xa1, 0, 0, 0, 0, // mov %gs:0, %eax
-      0x2d, 0, 0, 0, 0,       // sub $tls_size, %eax
-      0x90,                   // nop
+      0x81, 0xe8, 0, 0, 0, 0, // sub $tls_size, %eax
    };
    memcpy(loc - 2, insn, sizeof(insn));
-    *(ul32 *)(loc + 5) = tls_size;
+    *(ul32 *)(loc + 6) = tls_size;
    break;
  }
  default:
@@ -294,19 +290,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = get_addend(*this, rel);
    u64 P = get_addr() + rel.r_offset;
    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_386_8:
      check(S + A, 0, 1 << 8);
@@ -406,6 +399,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      //   mov    %reg, %eax
      //   call   *(%eax)
      //       R_386_TLS_DESC_CALL foo
+      //
+      // Note that the compiler always uses the local-exec TLS model
+      // for -fno-pic, so TLSDESC code is always PIC (i.e. uses %ebx to
+      // store the address of GOT.)
      if (sym.has_tlsdesc(ctx)) {
        *(ul32 *)loc = sym.get_tlsdesc_addr(ctx) + A - GOT;
      } else if (sym.has_gottp(ctx)) {
@@ -449,13 +446,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -464,6 +454,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 A = frag ? frag_addend : get_addend(*this, rel);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, val, i, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_386_8:
      check(S + A, 0, 1 << 8);
@@ -604,3 +598,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -51,7 +51,7 @@ static u64 hi20(u64 val, u64 pc) {
  return bits(page(val + 0x800) - page(pc), 31, 12);
 }

-static u64 higher20(u64 val, u64 pc) {
+static u64 higher(u64 val, u64 pc) {
  // A PC-relative 64-bit address is materialized with the following
  // instructions for the large code model:
  //
@@ -66,14 +66,17 @@ static u64 higher20(u64 val, u64 pc) {
  // LU52I.D simply set bits to [51:31] and to [63:53], respectively.
  //
  // Compensating all the sign-extensions is a bit complicated. The
-  // psABI gave the following formula.
+  // psABI gives the following formula.
  val = val + 0x8000'0000 + ((val & 0x800) ? (0x1000 - 0x1'0000'0000) : 0);
-  return bits(page(val) - page(pc - 8), 51, 32);
+  return page(val) - page(pc - 8);
+}
+
+static u64 higher20(u64 val, u64 pc) {
+  return bits(higher(val, pc), 51, 32);
 }

 static u64 highest12(u64 val, u64 pc) {
-  val = val + 0x8000'0000 + ((val & 0x800) ? (0x1000 - 0x1'0000'0000) : 0);
-  return bits(page(val) - page(pc - 12), 63, 52);
+  return bits(higher(val, pc), 63, 52);
 }

 static void write_k12(u8 *loc, u32 val) {
@@ -125,20 +128,22 @@ static void set_rj(u8 *loc, u32 rj) {
 // Returns true if isec's i'th relocation refers to the following
 // relaxable instructioon pair.
 //
-//   pcalau12i $t0, 0         # R_LARCH_GOT_PC_HI20
-//   ld.d      $t0, $t0, 0    # R_LARCH_GOT_PC_LO12
+//   pcalau12i $t0, 0         # R_LARCH_GOT_PC_HI20, R_LARCH_RELAX
+//   ld.d      $t0, $t0, 0    # R_LARCH_GOT_PC_LO12, R_LARCH_RELAX
 static bool is_relaxable_got_load(Context<E> &ctx, InputSection<E> &isec, i64 i) {
  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
  Symbol<E> &sym = *isec.file.symbols[rels[i].r_sym];
+  u8 *buf = (u8 *)isec.contents.data();

  if (ctx.arg.relax &&
      sym.is_pcrel_linktime_const(ctx) &&
      i + 3 < rels.size() &&
+      rels[i + 1].r_type == R_LARCH_RELAX &&
      rels[i + 2].r_type == R_LARCH_GOT_PC_LO12 &&
      rels[i + 2].r_offset == rels[i].r_offset + 4 &&
      rels[i + 3].r_type == R_LARCH_RELAX) {
-    u32 insn1 = *(ul32 *)(isec.contents.data() + rels[i].r_offset);
-    u32 insn2 = *(ul32 *)(isec.contents.data() + rels[i].r_offset + 4);
+    u32 insn1 = *(ul32 *)(buf + rels[i].r_offset);
+    u32 insn2 = *(ul32 *)(buf + rels[i].r_offset + 4);
    bool is_ld_d = (insn2 & 0xffc0'0000) == 0x28c0'0000;
    return get_rd(insn1) == get_rd(insn2) && get_rd(insn2) == get_rj(insn2) &&
           is_ld_d;
@@ -179,14 +184,14 @@ void write_plt_header<E>(Context<E> &ctx, u8 *buf) {
  write_k12(buf + 16, gotplt);
 }

-constexpr ul32 plt_entry_64[] = {
+static constexpr ul32 plt_entry_64[] = {
  0x1a00'000f, // pcalau12i $t3, %pc_hi20([email protected])
  0x28c0'01ef, // ld.d      $t3, $t3, %lo12([email protected])
  0x4c00'01ed, // jirl      $t1, $t3, 0
  0x002a'0000, // break
 };

-constexpr ul32 plt_entry_32[] = {
+static constexpr ul32 plt_entry_32[] = {
  0x1a00'000f, // pcalau12i $t3, %pc_hi20([email protected])
  0x2880'01ef, // ld.w      $t3, $t3, %lo12([email protected])
  0x4c00'01ed, // jirl      $t1, $t3, 0
@@ -265,10 +270,9 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  auto get_r_delta = [&](i64 idx) {
-    return extra.r_deltas.empty() ? 0 : extra.r_deltas[idx];
-  };
+  std::span<RelocDelta> deltas = extra.r_deltas;
+  i64 k = 0;
+  u8 *buf = (u8 *)contents.data();

  for (i64 i = 0; i < rels.size(); i++) {
    const ElfRel<E> &rel = rels[i];
@@ -278,25 +282,22 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        rel.r_type == R_LARCH_ALIGN)
      continue;

+    i64 removed_bytes = 0;
+    i64 r_delta = 0;
+
+    if (!deltas.empty()) {
+      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
+        k++;
+      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
+        removed_bytes = get_removed_bytes(deltas, k);
+      if (k > 0)
+        r_delta = deltas[k - 1].delta;
+    }
+
    Symbol<E> &sym = *file.symbols[rel.r_sym];
-    i64 r_offset = rel.r_offset - get_r_delta(i);
-    i64 removed_bytes = get_r_delta(i + 1) - get_r_delta(i);
+    i64 r_offset = rel.r_offset - r_delta;
    u8 *loc = base + r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
-    auto check_branch = [&](i64 val, i64 lo, i64 hi) {
-      if (val & 0b11)
-        Error(ctx) << *this << ": misaligned symbol " << sym
-                   << " for relocation " << rel;
-      check(val, lo, hi);
-    };
-
    // Unlike other psABIs, the LoongArch ABI uses the same relocation
    // types to refer to GOT entries for thread-local symbols and regular
    // ones. Therefore, G may refer to a TLSGD or a regular GOT slot
@@ -320,10 +321,21 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    u64 G = got_idx * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
+    auto check_branch = [&](i64 val, i64 lo, i64 hi) {
+      check(val, lo, hi);
+      if (val & 0b11)
+        Error(ctx) << *this << ": misaligned symbol " << sym
+                   << " for relocation " << rel;
+    };
+
    switch (rel.r_type) {
    case R_LARCH_32:
-      if constexpr (E::is_64)
-        *(ul32 *)loc = S + A;
+      assert(E::is_64);
+      *(ul32 *)loc = S + A;
      break;
    case R_LARCH_B16:
      check_branch(S + A - P, -(1 << 17), 1 << 17);
@@ -355,7 +367,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      // It is contrary to the psABI document, but GNU ld has special
      // code to handle it, so we accept it too.
      if ((*(ul32 *)loc & 0xfc00'0000) == 0x4c00'0000)
-        write_k16(loc, sign_extend(S + A, 11) >> 2);
+        write_k16(loc, sign_extend(S + A, 12) >> 2);
      else
        write_k12(loc, S + A);
      break;
@@ -393,7 +405,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        //   addi.d    $t0, $t0, 0
        if (is_relaxable_got_load(ctx, *this, i)) {
          i64 dist = compute_distance(ctx, sym, *this, rel);
-          if (-(1LL << 31) <= dist && dist < (1LL << 31)) {
+          if (is_int(dist, 32)) {
            u32 rd = get_rd(*(ul32 *)loc);
            *(ul32 *)(loc + 4) = 0x02c0'0000 | (rd << 5) | rd; // addi.d

@@ -506,6 +518,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul64 *)loc -= S + A;
      break;
    case R_LARCH_32_PCREL:
+      check(S + A - P, -(1LL << 31), 1LL << 31);
      *(ul32 *)loc = S + A - P;
      break;
    case R_LARCH_64_PCREL:
@@ -513,12 +526,14 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      break;
    case R_LARCH_CALL36:
      if (removed_bytes == 0) {
-        write_j20(loc, (S + A - P + 0x20000) >> 18);
-        write_k16(loc + 4, (S + A - P) >> 2);
+        i64 val = S + A - P;
+        check_branch(val, -(1LL << 37) - 0x20000, (1LL << 37) - 0x20000);
+        write_j20(loc, (val + 0x20000) >> 18);
+        write_k16(loc + 4, val >> 2);
      } else {
        // Rewrite PCADDU18I + JIRL to B or BL
        assert(removed_bytes == 4);
-        if (get_rd(*(ul32 *)(contents.data() + rel.r_offset + 4)) == 0)
+        if (get_rd(*(ul32 *)(buf + rel.r_offset + 4)) == 0)
          *(ul32 *)loc = 0x5000'0000; // B
        else
          *(ul32 *)loc = 0x5400'0000; // BL
@@ -547,63 +562,44 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      // We may relax the instructions to the following if its TP-relative
      // address is known at link-time
      //
-      //   <nop>
-      //   <nop>
+      //   <deleted>
+      //   <deleted>
      //   lu12i.w   $a0, foo@TPOFF
      //   addi.w    $a0, $a0, foo@TPOFF
      //
      // or to the following if the TP offset is small enough.
      //
-      //   <nop>
-      //   <nop>
-      //   <nop>
+      //   <deleted>
+      //   <deleted>
+      //   <deleted>
      //   ori       $a0, $zero, foo@TPOFF
      //
      // If the TP-relative address is known at process startup time, we
      // may relax the instructions to the following.
      //
-      //   <nop>
-      //   <nop>
+      //   <deleted>
+      //   <deleted>
      //   pcalau12i $a0, foo@GOTTP
      //   ld.[dw]   $a0, $a0, foo@GOTTP
      //
      // If we don't know anything about the symbol, we can still relax
      // the first two instructions to a single pcaddi as shown below.
      //
-      //   <nop>
+      //   <deleted>
      //   pcaddi    $a0, foo@GOTDESC
      //   ld.d      $ra, $a0, 0
      //   jirl      $ra, $ra, 0
-      //
-      // Note that if section-shrinking relaxation is enabled, nop may be
-      // completely deleted.
-      if (removed_bytes == 0) {
-        if (sym.has_tlsdesc(ctx)) {
-          i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
-          if (ctx.arg.relax && -(1 << 21) <= dist && dist < (1 << 21)) {
-            *(ul32 *)loc = 0x0340'0000; // nop
-          } else {
-            write_j20(loc, hi20(sym.get_tlsdesc_addr(ctx) + A, P));
-          }
-        } else {
-          *(ul32 *)loc = 0x0340'0000; // nop
-        }
-      }
+      if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
+        write_j20(loc, hi20(sym.get_tlsdesc_addr(ctx) + A, P));
      break;
    case R_LARCH_TLS_DESC_PC_LO12:
-      if (removed_bytes == 0) {
-        if (sym.has_tlsdesc(ctx)) {
-          i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
-          if (ctx.arg.relax && -(1 << 21) <= dist && dist < (1 << 21)) {
-            // If we can directly materialize the PC-relative address
-            // with pcaddi, do that.
-            *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
-            write_j20(loc, dist >> 2);
-          } else {
-            write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
-          }
+      if (sym.has_tlsdesc(ctx) && removed_bytes == 0) {
+        i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
+        if (is_int(dist, 22)) {
+          *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
+          write_j20(loc, dist >> 2);
        } else {
-          *(ul32 *)loc = 0x0340'0000; // nop
+          write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
        }
      }
      break;
@@ -629,7 +625,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        write_k12(loc, sym.get_gottp_addr(ctx) + A);
      } else {
        i64 val = S + A - ctx.tp_addr;
-        if (val < 0x1000)
+        if (0 <= val && val < 0x1000)
          *(ul32 *)loc = 0x0380'0004; // ori    $a0, $zero, 0
        else
          *(ul32 *)loc = 0x0280'0084; // addi.w $a0, $a0, 0
@@ -646,7 +642,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {

      // Rewrite `addi.d $t0, $t0, <offset>` with `addi.d $t0, $tp, <offset>`
      // if the offset is directly accessible using tp. tp is r2.
-      if (sign_extend(val, 11) == val)
+      if (is_int(val, 12))
        set_rj(loc, 2);
      break;
    }
@@ -857,15 +853,20 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }

 template <>
-void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
+void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  isec.extra.r_deltas.resize(rels.size() + 1);
-  i64 delta = 0;
+  std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
+  i64 r_delta = 0;
+  u8 *buf = (u8 *)isec.contents.data();

  for (i64 i = 0; i < rels.size(); i++) {
    const ElfRel<E> &r = rels[i];
    Symbol<E> &sym = *isec.file.symbols[r.r_sym];
-    isec.extra.r_deltas[i] = delta;
+
+    auto remove = [&](i64 d) {
+      r_delta += d;
+      deltas.push_back(RelocDelta{r.r_offset, r_delta});
+    };

    // A R_LARCH_ALIGN relocation refers to the beginning of a nop
    // sequence. We need to remove some or all of them so that the
@@ -890,9 +891,11 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
        alignment = r.r_addend + 4;
      }

-      u64 loc = isec.get_addr() + r.r_offset - delta;
-      u64 next_loc = loc + alignment - 4;
-      delta += next_loc - align_to(loc, alignment);
+      u64 P = isec.get_addr() + r.r_offset - r_delta;
+      u64 desired = align_to(P, alignment);
+      u64 actual = P + alignment - 4;
+      if (desired != actual)
+        remove(actual - desired);
      continue;
    }

@@ -921,8 +924,8 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
      //
      //  addi.d  $t0, $tp, <tp-offset>
      if (i64 val = sym.get_addr(ctx) + r.r_addend - ctx.tp_addr;
-          sign_extend(val, 11) == val)
-        delta += 4;
+          is_int(val, 12))
+        remove(4);
      break;
    case R_LARCH_PCALA_HI20:
      // The following two instructions are used to materialize a
@@ -940,14 +943,14 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
          rels[i + 2].r_offset == rels[i].r_offset + 4 &&
          rels[i + 3].r_type == R_LARCH_RELAX) {
        i64 dist = compute_distance(ctx, sym, isec, r);
-        u32 insn1 = *(ul32 *)(isec.contents.data() + rels[i].r_offset);
-        u32 insn2 = *(ul32 *)(isec.contents.data() + rels[i].r_offset + 4);
+        u32 insn1 = *(ul32 *)(buf + rels[i].r_offset);
+        u32 insn2 = *(ul32 *)(buf + rels[i].r_offset + 4);
        bool is_addi_d = (insn2 & 0xffc0'0000) == 0x02c0'0000;

-        if (dist % 4 == 0 && -(1 << 21) <= dist && dist < (1 << 21) &&
+        if ((dist & 0b11) == 0 && is_int(dist, 22) &&
            is_addi_d && get_rd(insn1) == get_rd(insn2) &&
            get_rd(insn2) == get_rj(insn2))
-          delta += 4;
+          remove(4);
      }
      break;
    case R_LARCH_CALL36:
@@ -960,10 +963,10 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
      // If the displacement is PC ± 128 MiB, we can use B or BL instead.
      // Note that $zero is $r0 and $ra is $r1.
      if (i64 dist = compute_distance(ctx, sym, isec, r);
-          -(1 << 27) <= dist && dist < (1 << 27))
-        if (u32 jirl = *(ul32 *)(isec.contents.data() + rels[i].r_offset + 4);
+          is_int(dist, 28))
+        if (u32 jirl = *(ul32 *)(buf + rels[i].r_offset + 4);
            get_rd(jirl) == 0 || get_rd(jirl) == 1)
-          delta += 4;
+          remove(4);
      break;
    case R_LARCH_GOT_PC_HI20:
      // The following two instructions are used to load a symbol address
@@ -976,36 +979,116 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
      // relax them to the following instruction.
      //
      //   pcaddi    $t0, <offset>
-      if (is_relaxable_got_load(ctx, isec, i)) {
-        i64 dist = compute_distance(ctx, sym, isec, r);
-        if (dist % 4 == 0 && -(1 << 21) <= dist && dist < (1 << 21))
-          delta += 4;
-      }
+      if (is_relaxable_got_load(ctx, isec, i))
+        if (i64 dist = compute_distance(ctx, sym, isec, r);
+            is_int(dist, 22) && (dist & 3) == 0)
+          remove(4);
      break;
    case R_LARCH_TLS_DESC_PC_HI20:
      if (sym.has_tlsdesc(ctx)) {
        u64 P = isec.get_addr() + r.r_offset;
        i64 dist = sym.get_tlsdesc_addr(ctx) + r.r_addend - P;
-        if (-(1 << 21) <= dist && dist < (1 << 21))
-          delta += 4;
+        if (is_int(dist, 22))
+          remove(4);
      } else {
-        delta += 4;
+        remove(4);
      }
      break;
    case R_LARCH_TLS_DESC_PC_LO12:
      if (!sym.has_tlsdesc(ctx))
-        delta += 4;
+        remove(4);
      break;
    case R_LARCH_TLS_DESC_LD:
-      if (!sym.has_tlsdesc(ctx) && !sym.has_gottp(ctx) &&
-          sym.get_addr(ctx) + r.r_addend - ctx.tp_addr < 0x1000)
-        delta += 4;
+      if (!sym.has_tlsdesc(ctx) && !sym.has_gottp(ctx))
+        if (i64 val = sym.get_addr(ctx) + r.r_addend - ctx.tp_addr;
+            0 <= val && val < 0x1000)
+          remove(4);
      break;
    }
  }

-  isec.extra.r_deltas[rels.size()] = delta;
-  isec.sh_size -= delta;
+  isec.sh_size -= r_delta;
+}
+
+// With --emit-relocs, rewrite the emitted relocations to match the
+// instructions that relaxation produced. The logic parallels apply_reloc_alloc.
+template <>
+void rewrite_reloc_types(Context<E> &ctx, InputSection<E> &isec, ElfRel<E> *buf) {
+  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
+  std::span<RelocDelta> deltas = isec.extra.r_deltas;
+  i64 k = 0;
+
+  for (i64 i = 0; i < rels.size(); i++) {
+    const ElfRel<E> &rel = rels[i];
+    Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+
+    i64 removed_bytes = 0;
+    if (!deltas.empty()) {
+      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
+        k++;
+      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
+        removed_bytes = get_removed_bytes(deltas, k);
+    }
+
+    switch (rel.r_type) {
+    case R_LARCH_TLS_LE_HI20_R:
+    case R_LARCH_TLS_LE_ADD_R:
+      // lu12i.w + add.d + addi.d => addi.d
+      if (removed_bytes)
+        buf[i].r_type = R_LARCH_NONE;
+      break;
+    case R_LARCH_PCALA_HI20:
+    case R_LARCH_GOT_PC_HI20:
+      // pcalau12i + addi.d/ld.d => pcaddi. The high part vanishes and the low
+      // part becomes the pcaddi's PC-relative relocation.
+      if (removed_bytes) {
+        buf[i].r_type = R_LARCH_NONE;
+        buf[i + 2].r_type = R_LARCH_PCREL20_S2;
+        i += 3;
+      }
+      break;
+    case R_LARCH_CALL36:
+      // pcaddu18i + jirl => b or bl
+      if (removed_bytes)
+        buf[i].r_type = R_LARCH_B26;
+      break;
+    case R_LARCH_TLS_DESC_PC_HI20:
+      // pcalau12i + addi.d => pcaddi when TLSDESC is kept; both deleted when it
+      // is relaxed to IE/LE.
+      if (!sym.has_tlsdesc(ctx)) {
+        buf[i].r_type = R_LARCH_NONE;
+      } else if (removed_bytes) {
+        buf[i].r_type = R_LARCH_NONE;
+        buf[i + 2].r_type = R_LARCH_TLS_DESC_PCREL20_S2;
+      }
+      break;
+    case R_LARCH_TLS_DESC_PC_LO12:
+      if (!sym.has_tlsdesc(ctx))
+        buf[i].r_type = R_LARCH_NONE;
+      break;
+    case R_LARCH_TLS_DESC_LD:
+      // TLSDESC is relaxed to IE or LE. The ld.d slot holds the first
+      // instruction of the resulting sequence.
+      if (sym.has_tlsdesc(ctx))
+        break;
+      if (sym.has_gottp(ctx))
+        buf[i].r_type = R_LARCH_TLS_IE_PC_HI20;   // pcalau12i
+      else if (removed_bytes)
+        buf[i].r_type = R_LARCH_NONE;             // small TP offset (deleted)
+      else
+        buf[i].r_type = R_LARCH_TLS_LE_HI20;      // large TP offset (lu12i.w)
+      break;
+    case R_LARCH_TLS_DESC_CALL:
+      // The jirl slot holds the second instruction of the IE or LE sequence.
+      if (sym.has_tlsdesc(ctx))
+        break;
+      if (sym.has_gottp(ctx))
+        buf[i].r_type = R_LARCH_TLS_IE_PC_LO12;   // ld.d
+      else
+        buf[i].r_type = R_LARCH_TLS_LE_LO12;      // ori or addi.w
+      break;
+    }
+  }
 }

 } // namespace mold
@@ -1,10 +1,10 @@
 // This file contains code for the Motorola 68000 series microprocessors,
 // which is often abbreviated as m68k. Running a Unix-like system on a
-// m68k-based machine today is probably a retro-computing hobby activity,
-// but the processor was a popular choice to build Unix computers during
-// '80s. Early Sun workstations for example used m68k. Macintosh until
-// 1994 were based on m68k as well until they switched to PowerPC (and
-// then to x86 and to ARM.)
+// m68k-based machine today is a retro-computing hobby activity, but the
+// processor was a popular choice to build Unix computers during '80s.
+// Early Sun workstations for example used m68k. Macintosh until 1994 were
+// based on m68k as well until they switched to PowerPC (and then to x86
+// and to ARM.)
 //
 // From the linker's point of view, it is not hard to support m68k. It's
 // just a 32-bit big-endian CISC ISA. Compared to comtemporary i386,
@@ -14,6 +14,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/m68k.pdf

+#if MOLD_M68K
+
 #include "mold.h"

 namespace mold {
@@ -86,11 +88,14 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

+    u64 S = sym.get_addr(ctx);
+    u64 A = rel.r_addend;
+    u64 P = get_addr() + rel.r_offset;
+    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
+    u64 GOT = ctx.got->shdr.sh_addr;
+
    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
+      check_range(ctx, i, val, lo, hi);
    };

    auto write16 = [&](u64 val) {
@@ -113,12 +118,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *loc = val;
    };

-    u64 S = sym.get_addr(ctx);
-    u64 A = rel.r_addend;
-    u64 P = get_addr() + rel.r_offset;
-    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
-    u64 GOT = ctx.got->shdr.sh_addr;
-
    switch (rel.r_type) {
    case R_68K_32:
      break;
@@ -313,3 +312,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -40,6 +40,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc32.pdf

+#if MOLD_PPC32
+
 #include "mold.h"

 namespace mold {
@@ -88,7 +90,7 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
  loc[5] |= lo(ctx.gotplt->shdr.sh_addr - ctx.plt->shdr.sh_addr + 4);
 }

-constexpr ub32 plt_entry[] = {
+static constexpr ub32 plt_entry[] = {
  // Get the address of this PLT entry
  0x7c08'02a6, // mflr    r0
  0x429f'0005, // bcl     20, 31, 4
@@ -213,15 +215,15 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_PPC_REL24:
    case R_PPC_LOCAL24PC: {
      i64 val = S + A - P;
-      if (sign_extend(val, 25) != val)
-        val = get_thunk_addr(i) - P;
+      if (!is_int(val, 26))
+        val = sym.get_thunk_addr(ctx, P) - P;
      *(ub32 *)loc |= bits(val, 25, 2) << 2;
      break;
    }
    case R_PPC_PLTREL24: {
      i64 val = S - P;
-      if (sym.has_plt(ctx) || sign_extend(val, 25) != val)
-        val = get_thunk_addr(i) - P;
+      if (sym.has_plt(ctx) || !is_int(val, 26))
+        val = sym.get_thunk_addr(ctx, P) - P;
      *(ub32 *)loc |= bits(val, 25, 2) << 2;
      break;
    }
@@ -306,6 +308,12 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
      else
        *(ub32 *)loc = S + A;
      break;
+    case R_PPC_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
    default:
      Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                 << rel;
@@ -416,27 +424,28 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
  static_assert(E::thunk_size == sizeof(plt_entry));
  static_assert(E::thunk_size == sizeof(local_thunk));

-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;

-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
      i64 val = got - P - 8;
      memcpy(buf, plt_entry, sizeof(plt_entry));
      *(ub32 *)(buf + 16) |= higha(val);
      *(ub32 *)(buf + 20) |= lo(val);
    } else {
-      i64 val = sym->get_addr(ctx) - P - 8;
+      i64 val = sym.get_addr(ctx) - P - 8;
      memcpy(buf, local_thunk, sizeof(local_thunk));
      *(ub32 *)(buf + 16) |= higha(val);
      *(ub32 *)(buf + 20) |= lo(val);
    }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
  }
 }

 } // namespace mold
+
+#endif
@@ -45,6 +45,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v1.pdf

+#if MOLD_PPC64V1
+
 #include "mold.h"

 #include <algorithm>
@@ -162,13 +164,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = rel.r_addend;
    u64 P = get_addr() + rel.r_offset;
@@ -176,6 +171,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    u64 GOT = ctx.got->shdr.sh_addr;
    u64 TOC = ctx.extra.TOC->value;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_PPC64_TOC:
      break;
@@ -194,8 +193,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      break;
    case R_PPC64_REL24: {
      i64 val = sym.get_addr(ctx, NO_OPD) + A - P;
-      if (sym.has_plt(ctx) || sign_extend(val, 25) != val)
-        val = get_thunk_addr(i) + A - P;
+      if (sym.has_plt(ctx) || !is_int(val, 26))
+        val = sym.get_thunk_addr(ctx, P) + A - P;

      check(val, -(1 << 25), 1 << 25);
      *(ub32 *)loc |= bits(val, 25, 2) << 2;
@@ -220,6 +219,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_PPC64_REL16_LO:
      *(ub16 *)loc = lo(S + A - P);
      break;
+    case R_PPC64_GOT16:
+      *(ub16 *)loc = G - TOC;
+      break;
    case R_PPC64_PLT16_HA:
      *(ub16 *)loc = ha(G + GOT - TOC);
      break;
@@ -293,13 +295,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -307,6 +302,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
    u64 A = frag ? frag_addend : (i64)rel.r_addend;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, val, i, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_PPC64_ADDR64:
      if (std::optional<u64> val = get_tombstone(sym, frag))
@@ -314,12 +313,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
      else
        *(ub64 *)loc = S + A;
      break;
-    case R_PPC64_ADDR32: {
-      i64 val = S + A;
-      check(val, 0, 1LL << 32);
-      *(ub32 *)loc = val;
+    case R_PPC64_ADDR32:
+      check(S + A, 0, 1LL << 32);
+      *(ub32 *)loc = S + A;
      break;
-    }
    case R_PPC64_DTPREL64:
      *(ub64 *)loc = S + A - ctx.dtp_addr;
      break;
@@ -359,6 +356,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      if (sym.is_imported)
        sym.flags |= NEEDS_PLT;
      break;
+    case R_PPC64_GOT16:
    case R_PPC64_PLT16_HA:
      sym.flags |= NEEDS_GOT;
      break;
@@ -460,66 +458,61 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
  static_assert(E::thunk_size == sizeof(plt_thunk));
  static_assert(E::thunk_size == sizeof(local_thunk));

-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
+  u64 TOC = ctx.extra.TOC->value;

-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_got(ctx)) {
-      i64 val = sym->get_got_addr(ctx) - ctx.extra.TOC->value;
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u8 *buf = base + offsets[i];
+
+    if (sym.has_got(ctx)) {
      memcpy(buf, pltgot_thunk, sizeof(pltgot_thunk));
+      i64 val = sym.get_got_addr(ctx) - TOC;
      *(ub32 *)(buf + 4) |= higha(val);
      *(ub32 *)(buf + 8) |= lo(val);
-    } else if(sym->has_plt(ctx)) {
-      i64 val = sym->get_gotplt_addr(ctx) - ctx.extra.TOC->value;
+    } else if(sym.has_plt(ctx)) {
      memcpy(buf, plt_thunk, sizeof(plt_thunk));
+      i64 val = sym.get_gotplt_addr(ctx) - TOC;
      *(ub32 *)(buf + 4) |= higha(val);
      *(ub32 *)(buf + 8) |= lo(val);
    } else {
-      i64 val = sym->get_addr(ctx, NO_OPD) - ctx.extra.TOC->value;
      memcpy(buf, local_thunk, sizeof(local_thunk));
+      i64 val = sym.get_addr(ctx, NO_OPD) - TOC;
      *(ub32 *)buf |= higha(val);
      *(ub32 *)(buf + 4) |= lo(val);
    }
-
-    buf += E::thunk_size;
  }
 }

 static InputSection<E> *get_opd_section(ObjectFile<E> &file) {
  for (std::unique_ptr<InputSection<E>> &isec : file.sections)
-    if (isec && isec->name() == ".opd")
+    if (isec && isec->name == ".opd")
      return isec.get();
  return nullptr;
 }

 static ElfRel<E> *
-get_relocation_at(Context<E> &ctx, InputSection<E> &isec, i64 offset) {
+get_relocation_at(Context<E> &ctx, InputSection<E> &isec, u64 offset) {
  std::span<ElfRel<E>> rels = isec.get_rels(ctx);
-
-  auto it = std::lower_bound(rels.begin(), rels.end(), offset,
-                             [](const ElfRel<E> &r, i64 offset) {
-    return r.r_offset < offset;
+  auto it = ranges::lower_bound(rels, offset, {}, [](const ElfRel<E> &r) {
+    return (u64)r.r_offset;
  });
-
-  if (it == rels.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  if (it == rels.end() || it->r_offset != offset)
    return nullptr;
  return &*it;
 }

+namespace {
 struct OpdSymbol {
-  bool operator<(const OpdSymbol &x) const { return r_offset < x.r_offset; }
-
  u64 r_offset = 0;
  Symbol<E> *sym = nullptr;
 };
+}

 static Symbol<E> *
 get_opd_sym_at(std::span<OpdSymbol> syms, u64 offset) {
-  auto it = std::lower_bound(syms.begin(), syms.end(), OpdSymbol{offset});
-  if (it == syms.end())
-    return nullptr;
-  if (it->r_offset != offset)
+  auto it = ranges::lower_bound(syms, offset, {}, &OpdSymbol::r_offset);
+  if (it == syms.end() || it->r_offset != offset)
    return nullptr;
  return it->sym;
 }
@@ -589,7 +582,7 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {

      Symbol<E> *sym2 = file->symbols[rel->r_sym];
      if (sym2->get_type() != STT_SECTION)
-        Fatal(ctx) << *file << ": bad relocation in .opd referring " << *sym2;
+        Fatal(ctx) << *file << ": bad relocation in .opd referring to " << *sym2;

      opd_syms.push_back({sym->value, sym});

@@ -598,7 +591,7 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
    }

    // Sort symbols so that get_opd_sym_at() can do binary search.
-    sort(opd_syms);
+    ranges::stable_sort(opd_syms, {}, &OpdSymbol::r_offset);

    // Rewrite relocations so that they directly refer to .opd.
    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
@@ -650,31 +643,31 @@ void PPC64OpdSection::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
  this->shdr.sh_size += ENTRY_SIZE;
 }

-i64 PPC64OpdSection::get_reldyn_size(Context<E> &ctx) const {
-  if (ctx.arg.pic)
-    return symbols.size() * 2;
-  return 0;
+std::vector<ElfRel<E>> PPC64OpdSection::collect_dynrels(Context<E> &ctx) const {
+  if (!ctx.arg.pic)
+    return {};
+
+  std::vector<ElfRel<E>> rels;
+
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx);
+    rels.emplace_back(loc, E::R_RELATIVE, 0, sym->get_addr(ctx, NO_PLT | NO_OPD));
+    rels.emplace_back(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
+  }
+  return rels;
 }

 void PPC64OpdSection::copy_buf(Context<E> &ctx) {
  ub64 *buf = (ub64 *)(ctx.buf + this->shdr.sh_offset);

-  ElfRel<E> *rel = nullptr;
-  if (ctx.arg.pic)
-    rel = (ElfRel<E> *)(ctx.buf + ctx.reldyn->shdr.sh_offset + reldyn_offset);
-
  for (Symbol<E> *sym : symbols) {
    u64 addr = sym->get_addr(ctx, NO_PLT | NO_OPD);
    *buf++ = addr;
    *buf++ = ctx.extra.TOC->value;
    *buf++ = 0;
-
-    if (ctx.arg.pic) {
-      u64 loc = sym->get_opd_addr(ctx);
-      *rel++ = ElfRel<E>(loc, E::R_RELATIVE, 0, addr);
-      *rel++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
-    }
  }
 }

 } // namespace mold
+
+#endif
@@ -80,6 +80,8 @@
 //
 // https://github.com/rui314/psabi/blob/main/ppc64v2.pdf

+#if MOLD_PPC64V2
+
 #include "mold.h"

 namespace mold {
@@ -172,7 +174,7 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 }

 static u64 get_local_entry_offset(Context<E> &ctx, Symbol<E> &sym) {
-  i64 val = sym.esym().ppc_local_entry;
+  i64 val = sym.esym().ppc64_local_entry;
  assert(val <= 7);
  if (val == 7)
    Fatal(ctx) << sym << ": local entry offset 7 is reserved";
@@ -201,8 +203,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    u64 GOT = ctx.got->shdr.sh_addr;
    u64 TOC = ctx.extra.TOC->value;

-    auto r2save_thunk_addr = [&] { return get_thunk_addr(i); };
-    auto no_r2save_thunk_addr = [&] { return get_thunk_addr(i) + 8; };
+    auto r2save_thunk_addr = [&] { return sym.get_thunk_addr(ctx, P); };
+    auto no_r2save_thunk_addr = [&] { return sym.get_thunk_addr(ctx, P) + 8; };

    switch (rel.r_type) {
    case R_PPC64_TOC16_HA:
@@ -216,7 +218,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul16 *)loc |= (S + A - TOC) & 0xfffc;
      break;
    case R_PPC64_REL24:
-      if (sym.has_plt(ctx) || !sym.esym().preserves_r2()) {
+      if (sym.has_plt(ctx) || !sym.esym().ppc64_preserves_r2()) {
        i64 val = r2save_thunk_addr() + A - P;
        *(ul32 *)loc |= bits(val, 25, 2) << 2;

@@ -227,22 +229,25 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
          *(ul32 *)(loc + 4) = 0xe841'0018; // ld r2, 24(r1)
      } else {
        i64 val = S + get_local_entry_offset(ctx, sym) + A - P;
-        if (sign_extend(val, 25) != val)
+        if (!is_int(val, 26))
          val = no_r2save_thunk_addr() + A - P;
        *(ul32 *)loc |= bits(val, 25, 2) << 2;
      }
      break;
    case R_PPC64_REL24_NOTOC:
-      if (sym.has_plt(ctx) || sym.esym().uses_toc()) {
+      if (sym.has_plt(ctx) || sym.esym().ppc64_uses_toc()) {
        i64 val = no_r2save_thunk_addr() + A - P;
        *(ul32 *)loc |= bits(val, 25, 2) << 2;
      } else {
        i64 val = S + A - P;
-        if (sign_extend(val, 25) != val)
+        if (!is_int(val, 26))
          val = no_r2save_thunk_addr() + A - P;
        *(ul32 *)loc |= bits(val, 25, 2) << 2;
      }
      break;
+    case R_PPC64_REL14:
+      *(ul32 *)loc |= bits(S + A - P, 15, 2) << 2;
+      break;
    case R_PPC64_REL32:
      *(ul32 *)loc = S + A - P;
      break;
@@ -255,6 +260,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_PPC64_REL16_LO:
      *(ul16 *)loc = lo(S + A - P);
      break;
+    case R_PPC64_GOT16:
+      *(ul16 *)loc = G - TOC;
+      break;
+    case R_PPC64_GOT16_LO:
+      *(ul16 *)loc = lo(G - TOC);
+      break;
+    case R_PPC64_GOT16_HI:
+      *(ul16 *)loc = hi(G - TOC);
+      break;
+    case R_PPC64_GOT16_HA:
+      *(ul16 *)loc = ha(G - TOC);
+      break;
    case R_PPC64_PLT16_HA:
      *(ul16 *)loc = ha(G + GOT - TOC);
      break;
@@ -334,6 +351,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_PPC64_TLS:
    case R_PPC64_TLSGD:
    case R_PPC64_TLSLD:
+    case R_PPC64_ENTRY:
      break;
    default:
      unreachable();
@@ -353,13 +371,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -367,6 +378,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
    u64 A = frag ? frag_addend : (i64)rel.r_addend;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_PPC64_ADDR64:
      if (std::optional<u64> val = get_tombstone(sym, frag))
@@ -374,12 +389,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
      else
        *(ul64 *)loc = S + A;
      break;
-    case R_PPC64_ADDR32: {
-      i64 val = S + A;
-      check(val, 0, 1LL << 32);
-      *(ul32 *)loc = val;
+    case R_PPC64_ADDR32:
+      check(S + A, 0, 1LL << 32);
+      *(ul32 *)loc = S + A;
      break;
-    }
    case R_PPC64_DTPREL64:
      *(ul64 *)loc = S + A - ctx.dtp_addr;
      break;
@@ -420,6 +433,10 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
        sym.flags |= NEEDS_PLT;
      ctx.extra.is_power10 = true;
      break;
+    case R_PPC64_GOT16:
+    case R_PPC64_GOT16_LO:
+    case R_PPC64_GOT16_HI:
+    case R_PPC64_GOT16_HA:
    case R_PPC64_PLT16_HA:
    case R_PPC64_PLT_PCREL34:
    case R_PPC64_PLT_PCREL34_NOTOC:
@@ -435,12 +452,11 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      ctx.needs_tlsld = true;
      break;
    case R_PPC64_TPREL16_HA:
-    case R_PPC64_TPREL16_LO:
-    case R_PPC64_TPREL16_LO_DS:
    case R_PPC64_TPREL34:
      check_tlsle(ctx, sym, rel);
      break;
    case R_PPC64_ADDR64:
+    case R_PPC64_REL14:
    case R_PPC64_REL32:
    case R_PPC64_REL64:
    case R_PPC64_TOC16_HA:
@@ -460,6 +476,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_PPC64_GOT_TPREL16_LO_DS:
    case R_PPC64_GOT_TLSGD16_LO:
    case R_PPC64_GOT_TLSLD16_LO:
+    case R_PPC64_TPREL16_LO:
+    case R_PPC64_TPREL16_LO_DS:
    case R_PPC64_TLS:
    case R_PPC64_TLSGD:
    case R_PPC64_TLSLD:
@@ -467,6 +485,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_PPC64_DTPREL16_LO:
    case R_PPC64_DTPREL16_LO_DS:
    case R_PPC64_DTPREL34:
+    case R_PPC64_ENTRY:
      break;
    default:
      Error(ctx) << *this << ": unknown relocation: " << rel;
@@ -521,15 +540,16 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
  static_assert(E::thunk_size == sizeof(local_thunk));
  static_assert(E::thunk_size == sizeof(local_thunk_power10));

-  u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
+  u8 *base = ctx.buf + output_section.shdr.sh_offset + offset;
  u64 TOC = ctx.extra.TOC->value;

-  for (Symbol<E> *sym : symbols) {
-    if (sym->has_plt(ctx)) {
-      u64 got =
-        sym->has_got(ctx) ? sym->get_got_addr(ctx) : sym->get_gotplt_addr(ctx);
+  for (i64 i = 0; i < symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    u64 P = get_addr() + offsets[i];
+    u8 *buf = base + offsets[i];

+    if (sym.has_plt(ctx)) {
+      u64 got = sym.has_got(ctx) ? sym.get_got_addr(ctx) : sym.get_gotplt_addr(ctx);
      if (ctx.extra.is_power10) {
        memcpy(buf, plt_thunk_power10, E::thunk_size);
        write34(buf + 8, got - P - 8);
@@ -539,7 +559,7 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
        *(ul32 *)(buf + 12) |= lo(got - TOC);
      }
    } else {
-      u64 S = sym->get_addr(ctx);
+      u64 S = sym.get_addr(ctx);
      if (ctx.extra.is_power10) {
        memcpy(buf, local_thunk_power10, E::thunk_size);
        write34(buf + 8, S - P - 8);
@@ -549,9 +569,6 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
        *(ul32 *)(buf + 12) |= lo(S - TOC);
      }
    }
-
-    buf += E::thunk_size;
-    P += E::thunk_size;
  }
 }

@@ -661,3 +678,5 @@ u64 get_eflags(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -22,8 +22,6 @@
 #include "mold.h"

 #include <regex>
-#include <tbb/parallel_for.h>
-#include <tbb/parallel_for_each.h>

 namespace mold {

@@ -88,6 +86,10 @@ static void set_rs1(u8 *loc, u32 rs1) {
  *(ul32 *)loc |= rs1 << 15;
 }

+static u32 get_rd(u8 *loc) {
+  return bits(*(u32 *)loc, 11, 7);
+};
+
 template <>
 void write_plt_header<E>(Context<E> &ctx, u8 *buf) {
  constexpr ul32 insn_64[] = {
@@ -121,14 +123,14 @@ void write_plt_header<E>(Context<E> &ctx, u8 *buf) {
  write_itype(buf + 16, gotplt - plt);
 }

-constexpr ul32 plt_entry_64[] = {
+static constexpr ul32 plt_entry_64[] = {
  0x0000'0e17, // auipc   t3, %pcrel_hi([email protected])
  0x000e'3e03, // ld      t3, %pcrel_lo(1b)(t3)
  0x000e'0367, // jalr    t1, t3
  0x0010'0073, // ebreak
 };

-constexpr ul32 plt_entry_32[] = {
+static constexpr ul32 plt_entry_32[] = {
  0x0000'0e17, // auipc   t3, %pcrel_hi([email protected])
  0x000e'2e03, // lw      t3, %pcrel_lo(1b)(t3)
  0x000e'0367, // jalr    t1, t3
@@ -198,67 +200,115 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
  }
 }

-static inline bool is_hi20(const ElfRel<E> &rel) {
-  u32 ty = rel.r_type;
-  return ty == R_RISCV_GOT_HI20 || ty == R_RISCV_TLS_GOT_HI20 ||
-         ty == R_RISCV_TLS_GD_HI20 || ty == R_RISCV_PCREL_HI20 ||
-         ty == R_RISCV_TLSDESC_HI20;
+// RISC-V generally uses the AUIPC + ADDI/LW/SW/etc instruction pair
+// to access the AUIPC's address ± 2 GiB. AUIPC materializes the most
+// significant 52 bits in a PC-relative manner, and the following
+// instruction specifies the remaining least significant 12 bits.
+// There are several HI20 and LO12 relocation types for them.
+//
+// LO12 relocations need to materialize an address relative to AUIPC's
+// address, not relative to the instruction that the relocation
+// directly refers to.
+//
+// The problem here is that the instruction pair may not always be
+// adjacent. We need a mechanism to find a paired AUIPC for a given
+// LO12 relocation. For this purpose, the compiler creates a local
+// symbol for each location to which HI20 refers, and the LO12
+// relocation refers to that symbol.
+//
+// This function returns a paired HI20 relocation for a given LO12.
+// Since the instructions are typically adjacent, we do a linear
+// search.
+static const ElfRel<E> &
+find_paired_reloc(Context<E> &ctx, InputSection<E> &isec,
+                  std::span<const ElfRel<E>> rels,
+                  Symbol<E> &sym, i64 i) {
+  auto is_hi20 = [](u32 ty) {
+    return ty == R_RISCV_GOT_HI20 || ty == R_RISCV_TLS_GOT_HI20 ||
+           ty == R_RISCV_TLS_GD_HI20 || ty == R_RISCV_PCREL_HI20 ||
+           ty == R_RISCV_TLSDESC_HI20;
+  };
+
+  u64 value = sym.esym().st_value;
+
+  if (value <= rels[i].r_offset) {
+    for (i64 j = i - 1; j >= 0; j--)
+      if (is_hi20(rels[j].r_type) && value == rels[j].r_offset)
+        return rels[j];
+  } else {
+    for (i64 j = i + 1; j < rels.size(); j++)
+      if (is_hi20(rels[j].r_type) && value == rels[j].r_offset)
+        return rels[j];
+  }
+  Fatal(ctx) << isec << ": paired relocation is missing: " << i;
+}
+
+// Returns true if isec's i'th relocation refers to the following
+// GOT-load instructioon pair, which is an expeanded form of
+// `la t0, foo` pseudo assembly instruction.
+//
+// .L0
+//   auipc t0, 0      # R_RISCV_GOT_HI20(foo),     R_RISCV_RELAX
+//   ld    t0, 0(t0)  # R_RISCV_PCREL_LO12_I(.L0), R_RISCV_RELAX
+static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
+                             std::span<const ElfRel<E>> rels, i64 i) {
+  u8 *buf = (u8 *)isec.contents.data();
+  return i + 3 < rels.size() &&
+         rels[i].r_type == R_RISCV_GOT_HI20 &&
+         rels[i + 1].r_type == R_RISCV_RELAX &&
+         rels[i + 2].r_type == R_RISCV_PCREL_LO12_I &&
+         rels[i + 3].r_type == R_RISCV_RELAX &&
+         rels[i].r_offset == rels[i + 2].r_offset - 4 &&
+         rels[i].r_offset == isec.file.symbols[rels[i + 2].r_sym]->value &&
+         get_rd(buf + rels[i].r_offset) == get_rd(buf + rels[i + 2].r_offset);
 }

 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
  std::span<const ElfRel<E>> rels = get_rels(ctx);
-  u64 GP = ctx.__global_pointer ? ctx.__global_pointer->get_addr(ctx) : 0;
-
-  auto get_r_delta = [&](i64 idx) {
-    return extra.r_deltas.empty() ? 0 : extra.r_deltas[idx];
-  };
+  std::span<RelocDelta> deltas = extra.r_deltas;
+  i64 k = 0;
+  u8 *buf = (u8 *)contents.data();

  for (i64 i = 0; i < rels.size(); i++) {
    const ElfRel<E> &rel = rels[i];
    if (rel.r_type == R_NONE || rel.r_type == R_RISCV_RELAX)
      continue;

+    i64 removed_bytes = 0;
+    i64 r_delta = 0;
+
+    if (!deltas.empty()) {
+      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
+        k++;
+      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
+        removed_bytes = get_removed_bytes(deltas, k);
+      if (k > 0)
+        r_delta = deltas[k - 1].delta;
+    }
+
    Symbol<E> &sym = *file.symbols[rel.r_sym];
-    i64 r_offset = rel.r_offset - get_r_delta(i);
-    i64 removed_bytes = get_r_delta(i + 1) - get_r_delta(i);
+    i64 r_offset = rel.r_offset - r_delta;
    u8 *loc = base + r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
-    auto find_paired_reloc = [&] {
-      if (sym.value <= rels[i].r_offset - get_r_delta(i)) {
-        for (i64 j = i - 1; j >= 0; j--)
-          if (is_hi20(rels[j]) && sym.value == rels[j].r_offset - get_r_delta(j))
-            return j;
-      } else {
-        for (i64 j = i + 1; j < rels.size(); j++)
-          if (is_hi20(rels[j]) && sym.value == rels[j].r_offset - get_r_delta(j))
-            return j;
-      }
-
-      Fatal(ctx) << *this << ": paired relocation is missing: " << i;
-    };
-
-    auto get_rd = [&](i64 offset) {
-      // Returns the rd register of an R/I/U/J-type instruction.
-      return bits(*(ul32 *)(contents.data() + offset), 11, 7);
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = rel.r_addend;
    u64 P = get_addr() + r_offset;
    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
+    auto utype = [&](i64 val) {
+      check(val, -(1LL << 31) - 0x800, (1LL << 31) - 0x800);
+      write_utype(loc, val);
+    };
+
    switch (rel.r_type) {
    case R_RISCV_32:
-      if constexpr (E::is_64)
+      if (E::is_64)
        *(U32<E> *)loc = S + A;
      break;
    case R_RISCV_64:
@@ -274,13 +324,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_RISCV_CALL:
    case R_RISCV_CALL_PLT: {
      i64 val = S + A - P;
-      i64 rd = get_rd(rel.r_offset + 4);
-
-      // Calling an undefined weak symbol does not make sense.
-      // We make such call into an infinite loop. This should
-      // help debugging of a faulty program.
-      if (sym.esym().is_undef_weak())
-        val = 0;
+      i64 rd = get_rd(buf + rel.r_offset + 4);

      if (removed_bytes == 4) {
        // auipc + jalr -> jal
@@ -297,8 +341,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        write_cjtype(loc, val);
      } else {
        assert(removed_bytes == 0);
-        check(val, -(1LL << 31), 1LL << 31);
-        write_utype(loc, val);
+        utype(val);
        write_itype(loc + 4, val);
      }
      break;
@@ -308,109 +351,83 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      // pair to load a symbol value from the GOT. If the symbol value
      // is actually a link-time constant, we can materialize the value
      // directly into a register to eliminate a memory load.
-      i64 rd = get_rd(rel.r_offset);
+      i64 rd = get_rd(buf + rel.r_offset);

-      switch (removed_bytes) {
-      case 6:
+      if (removed_bytes == 6) {
        // c.li <rd>, val
        *(ul16 *)loc = 0b010'0'00000'00000'01 | (rd << 7);
        write_citype(loc, sym.get_addr(ctx));
        i += 3;
-        break;
-      case 4:
+      } else if (removed_bytes == 4) {
        // addi <rd>, zero, val
        *(ul32 *)loc = 0b0010011 | (rd << 7);
        write_itype(loc, sym.get_addr(ctx));
        i += 3;
-        break;
-      case 0:
-        if (ctx.arg.relax &&
-            sym.is_pcrel_linktime_const(ctx) &&
-            i + 3 < rels.size() &&
-            rels[i + 1].r_type == R_RISCV_RELAX &&
-            rels[i + 2].r_type == R_RISCV_PCREL_LO12_I &&
-            rels[i + 2].r_offset == rels[i].r_offset + 4 &&
-            file.symbols[rels[i + 2].r_sym]->value == r_offset &&
-            rels[i + 3].r_type == R_RISCV_RELAX) {
-          i64 val = S + A - P;
-          if (rd == get_rd(rel.r_offset + 4) && (i32)val == val) {
-            // auipc <rd>, %hi20(val)
-            write_utype(loc, val);
+      } else {
+        assert(removed_bytes == 0);

-            // addi <rd>, <rd>, %lo12(val)
-            *(ul32 *)(loc + 4) = 0b0010011 | (rd << 15) | (rd << 7);
-            write_itype(loc + 4, val);
-            i += 3;
-            break;
-          }
+        i64 val = S + A - P;
+        if (ctx.arg.relax && sym.is_pcrel_linktime_const(ctx) &&
+            is_got_load_pair(ctx, *this, rels, i) && is_int(val, 32)) {
+          // auipc <rd>, %hi20(val)
+          utype(val);
+
+          // addi <rd>, <rd>, %lo12(val)
+          *(ul32 *)(loc + 4) = 0b0010011 | (rd << 15) | (rd << 7);
+          write_itype(loc + 4, val);
+          i += 3;
+        } else {
+          utype(G + GOT + A - P);
        }
-
-        write_utype(loc, G + GOT + A - P);
-        break;
-      default:
-        unreachable();
      }
      break;
    }
    case R_RISCV_TLS_GOT_HI20:
-      write_utype(loc, sym.get_gottp_addr(ctx) + A - P);
+      utype(sym.get_gottp_addr(ctx) + A - P);
      break;
    case R_RISCV_TLS_GD_HI20:
-      write_utype(loc, sym.get_tlsgd_addr(ctx) + A - P);
+      utype(sym.get_tlsgd_addr(ctx) + A - P);
      break;
    case R_RISCV_PCREL_HI20:
-      write_utype(loc, S + A - P);
+      utype(S + A - P);
      break;
    case R_RISCV_PCREL_LO12_I:
-    case R_RISCV_PCREL_LO12_S:
-    case R_RISCV_GPREL_LO12_I:
-    case R_RISCV_GPREL_LO12_S: {
-      i64 idx2 = find_paired_reloc();
-      const ElfRel<E> &rel2 = rels[idx2];
+    case R_RISCV_PCREL_LO12_S: {
+      const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
      Symbol<E> &sym2 = *file.symbols[rel2.r_sym];

+      auto write =
+        (rel.r_type == R_RISCV_PCREL_LO12_I) ? write_itype : write_stype;
+
      u64 S = sym2.get_addr(ctx);
      u64 A = rel2.r_addend;
-      u64 P = get_addr() + rel2.r_offset - get_r_delta(idx2);
+      u64 P = get_addr() + rel2.r_offset - get_r_delta(*this, rel2.r_offset);
      u64 G = sym2.get_got_idx(ctx) * sizeof(Word<E>);
-      u64 val;

      switch (rel2.r_type) {
      case R_RISCV_GOT_HI20:
-        val = G + GOT + A - P;
+        write(loc, G + GOT + A - P);
        break;
      case R_RISCV_TLS_GOT_HI20:
-        val = sym2.get_gottp_addr(ctx) + A - P;
+        write(loc, sym2.get_gottp_addr(ctx) + A - P);
        break;
      case R_RISCV_TLS_GD_HI20:
-        val = sym2.get_tlsgd_addr(ctx) + A - P;
+        write(loc, sym2.get_tlsgd_addr(ctx) + A - P);
        break;
      case R_RISCV_PCREL_HI20:
-        val = S + A - P;
+        write(loc, S + A - P);
        break;
-      case R_RISCV_GPREL_HI20:
-        val = S + A - GP;
-        break;
-      default:
-        unreachable();
      }
-
-      if (rel.r_type == R_RISCV_PCREL_LO12_I ||
-          rel.r_type == R_RISCV_GPREL_LO12_I)
-        write_itype(loc, val);
-      else
-        write_stype(loc, val);
      break;
    }
    case R_RISCV_HI20:
      if (removed_bytes == 2) {
        // Rewrite LUI with C.LUI
-        i64 rd = get_rd(rel.r_offset);
+        i64 rd = get_rd(buf + rel.r_offset);
        *(ul16 *)loc = 0b011'0'00000'00000'01 | (rd << 7);
        write_citype(loc, (S + A + 0x800) >> 12);
      } else if (removed_bytes == 0) {
-        check(S + A, -(1LL << 31), 1LL << 31);
-        write_utype(loc, S + A);
+        utype(S + A);
      }
      break;
    case R_RISCV_LO12_I:
@@ -423,13 +440,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      // Rewrite `lw t1, 0(t0)` with `lw t1, 0(x0)` if the address is
      // accessible relative to the zero register because if that's the
      // case, corresponding LUI might have been removed by relaxation.
-      if (sign_extend(S + A, 11) == S + A)
+      if (is_int(S + A, 12))
        set_rs1(loc, 0);
      break;
    case R_RISCV_TPREL_HI20:
      assert(removed_bytes == 0 || removed_bytes == 4);
      if (removed_bytes == 0)
-        write_utype(loc, S + A - ctx.tp_addr);
+        utype(S + A - ctx.tp_addr);
      break;
    case R_RISCV_TPREL_ADD:
      // This relocation just annotates an ADD instruction that can be
@@ -447,13 +464,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {

      // Rewrite `lw t1, 0(t0)` with `lw t1, 0(tp)` if the address is
      // directly accessible using tp. tp is x4.
-      if (sign_extend(val, 11) == val)
+      if (is_int(val, 12))
        set_rs1(loc, 4);
      break;
    }
    case R_RISCV_TLSDESC_HI20:
      // RISC-V TLSDESC uses the following code sequence to materialize
-      // a TP-relative address in x0.
+      // a TP-relative address in a0.
      //
      //   .L0:
      //   auipc  tX, 0
@@ -486,8 +503,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      //   <deleted>
      //   lui    a0, %tpoff_hi(a0)
      //   addi   a0, a0, %tpoff_lo(a0)
-      if (removed_bytes == 0)
-        write_utype(loc, sym.get_tlsdesc_addr(ctx) + A - P);
+      //
+      // If the code-shrinking relaxation is disabled, we may leave
+      // original useless instructions instead of deleting them, but we
+      // accept that because relaxations are enabled by default.
+      if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
+        utype(sym.get_tlsdesc_addr(ctx) + A - P);
      break;
    case R_RISCV_TLSDESC_LOAD_LO12:
    case R_RISCV_TLSDESC_ADD_LO12:
@@ -495,27 +516,29 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      if (removed_bytes == 4)
        break;

-      i64 idx2 = find_paired_reloc();
-      const ElfRel<E> &rel2 = rels[idx2];
+      const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
      Symbol<E> &sym2 = *file.symbols[rel2.r_sym];

      u64 S = sym2.get_addr(ctx);
      u64 A = rel2.r_addend;
-      u64 P = get_addr() + rel2.r_offset - get_r_delta(idx2);
+      u64 P = get_addr() + rel2.r_offset - get_r_delta(*this, rel2.r_offset);

      switch (rel.r_type) {
      case R_RISCV_TLSDESC_LOAD_LO12:
-        write_itype(loc, sym2.get_tlsdesc_addr(ctx) + A - P);
+        if (sym2.has_tlsdesc(ctx))
+          write_itype(loc, sym2.get_tlsdesc_addr(ctx) + A - P);
+        else
+          *(ul32 *)loc = 0x13; // nop
        break;
      case R_RISCV_TLSDESC_ADD_LO12:
        if (sym2.has_tlsdesc(ctx)) {
          write_itype(loc, sym2.get_tlsdesc_addr(ctx) + A - P);
        } else if (sym2.has_gottp(ctx)) {
          *(ul32 *)loc = 0x517; // auipc a0,<hi20>
-          write_utype(loc, sym2.get_gottp_addr(ctx) + A - P);
+          utype(sym2.get_gottp_addr(ctx) + A - P);
        } else {
          *(ul32 *)loc = 0x537; // lui a0,<hi20>
-          write_utype(loc, S + A - ctx.tp_addr);
+          utype(S + A - ctx.tp_addr);
        }
        break;
      case R_RISCV_TLSDESC_CALL:
@@ -527,7 +550,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
          write_itype(loc, sym2.get_gottp_addr(ctx) + A - P);
        } else {
          i64 val = S + A - ctx.tp_addr;
-          if (sign_extend(val, 11) == val)
+          if (is_int(val, 12))
            *(ul32 *)loc = 0x513;   // addi a0,zero,<lo12>
          else
            *(ul32 *)loc = 0x50513; // addi a0,a0,<lo12>
@@ -587,10 +610,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      check(S + A - P, -(1 << 11), 1 << 11);
      write_cjtype(loc, S + A - P);
      break;
-    case R_RISCV_GPREL_HI20:
-      check(S + A - GP, -(1LL << 31), 1LL << 31);
-      write_utype(loc, S + A - GP);
-      break;
    case R_RISCV_SUB6:
      *loc = (*loc & 0b1100'0000) | ((*loc - S - A) & 0b0011'1111);
      break;
@@ -610,6 +629,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    case R_RISCV_32_PCREL:
      *(U32<E> *)loc = S + A - P;
      break;
+    case R_RISCV_GOT32_PCREL:
+      *(U32<E> *)loc = G + GOT + A - P;
+      break;
    case R_RISCV_SET_ULEB128:
      overwrite_uleb(loc, S + A);
      break;
@@ -735,6 +757,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
        sym.flags |= NEEDS_PLT;
      break;
    case R_RISCV_GOT_HI20:
+    case R_RISCV_GOT32_PCREL:
      sym.flags |= NEEDS_GOT;
      break;
    case R_RISCV_TLS_GOT_HI20:
@@ -751,15 +774,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      scan_pcrel(ctx, sym, rel);
      break;
    case R_RISCV_TPREL_HI20:
-    case R_RISCV_TPREL_LO12_I:
-    case R_RISCV_TPREL_LO12_S:
-    case R_RISCV_TPREL_ADD:
      check_tlsle(ctx, sym, rel);
      break;
-    case R_RISCV_GPREL_HI20:
-      if (ctx.arg.shared)
-        Error(ctx) << *this << ": R_RISCV_GPREL_HI20 may not be used with -shared";
-      break;
    case R_RISCV_64:
    case R_RISCV_BRANCH:
    case R_RISCV_JAL:
@@ -767,6 +783,9 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_RISCV_PCREL_LO12_S:
    case R_RISCV_LO12_I:
    case R_RISCV_LO12_S:
+    case R_RISCV_TPREL_LO12_I:
+    case R_RISCV_TPREL_LO12_S:
+    case R_RISCV_TPREL_ADD:
    case R_RISCV_TLSDESC_LOAD_LO12:
    case R_RISCV_TLSDESC_ADD_LO12:
    case R_RISCV_TLSDESC_CALL:
@@ -781,8 +800,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_RISCV_ALIGN:
    case R_RISCV_RVC_BRANCH:
    case R_RISCV_RVC_JUMP:
-    case R_RISCV_GPREL_LO12_I:
-    case R_RISCV_GPREL_LO12_S:
    case R_RISCV_RELAX:
    case R_RISCV_SUB6:
    case R_RISCV_SET6:
@@ -806,9 +823,9 @@ u64 get_eflags(Context<E> &ctx) {
  if (objs.empty())
    return 0;

-  u32 ret = objs[0]->get_ehdr().e_flags;
+  u32 ret = objs[0]->get_eflags();
  for (i64 i = 1; i < objs.size(); i++) {
-    u32 flags = objs[i]->get_ehdr().e_flags;
+    u32 flags = objs[i]->get_eflags();
    if (flags & EF_RISCV_RVC)
      ret |= EF_RISCV_RVC;

@@ -823,22 +840,26 @@ u64 get_eflags(Context<E> &ctx) {
  return ret;
 }

-// Scan relocations to a given shrink section.
+// Scan relocations to shrink a given section.
 template <>
-void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
+void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  isec.extra.r_deltas.resize(rels.size() + 1);
+  std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
+  i64 r_delta = 0;
+  u8 *buf = (u8 *)isec.contents.data();

-  auto get_rd = [&](i64 offset) {
-    return bits(*(ul32 *)(isec.contents.data() + offset), 11, 7);
-  };
-
-  i64 delta = 0;
+  // True if we can use 2-byte instructions. This is usually true on
+  // Unix because RV64GC is generally considered the baseline hardware.
+  bool use_rvc = isec.file.get_eflags() & EF_RISCV_RVC;

  for (i64 i = 0; i < rels.size(); i++) {
    const ElfRel<E> &r = rels[i];
    Symbol<E> &sym = *isec.file.symbols[r.r_sym];
-    isec.extra.r_deltas[i] = delta;
+
+    auto remove = [&](i64 d) {
+      r_delta += d;
+      deltas.push_back(RelocDelta{r.r_offset, r_delta});
+    };

    // Handling R_RISCV_ALIGN is mandatory.
    //
@@ -848,11 +869,11 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
    if (r.r_type == R_RISCV_ALIGN) {
      // The total bytes of NOPs is stored to r_addend, so the next
      // instruction is r_addend away.
-      u64 loc = isec.get_addr() + r.r_offset - delta;
-      u64 next_loc = loc + r.r_addend;
-      u64 alignment = bit_ceil(r.r_addend + 1);
-      assert(alignment <= (1 << isec.p2align));
-      delta += next_loc - align_to(loc, alignment);
+      u64 P = isec.get_addr() + r.r_offset - r_delta;
+      u64 desired = align_to(P, bit_ceil(r.r_addend));
+      u64 actual = P + r.r_addend;
+      if (desired != actual)
+        remove(actual - desired);
      continue;
    }

@@ -869,20 +890,6 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
    if (sym.file == ctx.internal_obj)
      continue;

-    auto find_paired_reloc = [&] {
-      if (sym.value <= rels[i].r_offset) {
-        for (i64 j = i - 1; j >= 0; j--)
-          if (is_hi20(rels[j]) && sym.value == rels[j].r_offset)
-            return j;
-      } else {
-        for (i64 j = i + 1; j < rels.size(); j++)
-          if (is_hi20(rels[j]) && sym.value == rels[j].r_offset)
-            return j;
-      }
-
-      Fatal(ctx) << isec << ": paired relocation is missing: " << i;
-    };
-
    switch (r.r_type) {
    case R_RISCV_CALL:
    case R_RISCV_CALL_PLT: {
@@ -893,63 +900,51 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
      if (dist & 1)
        break;

-      i64 rd = get_rd(r.r_offset + 4);
+      i64 rd = get_rd(buf + r.r_offset + 4);

-      if (use_rvc && rd == 0 && sign_extend(dist, 11) == dist) {
+      if (use_rvc && rd == 0 && is_int(dist, 12)) {
        // If rd is x0 and the jump target is within ±2 KiB, we can use
        // C.J, saving 6 bytes.
-        delta += 6;
-      } else if (use_rvc && !E::is_64 && rd == 1 && sign_extend(dist, 11) == dist) {
+        remove(6);
+      } else if (use_rvc && !E::is_64 && rd == 1 && is_int(dist, 12)) {
        // If rd is x1 and the jump target is within ±2 KiB, we can use
        // C.JAL. This is RV32 only because C.JAL is RV32-only instruction.
-        delta += 6;
-      } else if (sign_extend(dist, 20) == dist) {
+        remove(6);
+      } else if (is_int(dist, 21)) {
        // If the jump target is within ±1 MiB, we can use JAL.
-        delta += 4;
+        remove(4);
      }
      break;
    }
-    case R_RISCV_GOT_HI20: {
+    case R_RISCV_GOT_HI20:
      // A GOT_HI20 followed by a PCREL_LO12_I is used to load a value from
      // GOT. If the loaded value is a link-time constant, we can rewrite
      // the instructions to directly materialize the value, eliminating a
      // memory load.
-      if (sym.is_absolute() &&
-          i + 3 < rels.size() &&
-          rels[i + 1].r_type == R_RISCV_RELAX &&
-          rels[i + 2].r_type == R_RISCV_PCREL_LO12_I &&
-          rels[i + 2].r_offset == rels[i].r_offset + 4 &&
-          isec.file.symbols[rels[i + 2].r_sym]->value == rels[i].r_offset &&
-          rels[i + 3].r_type == R_RISCV_RELAX) {
-        i64 rd = get_rd(r.r_offset);
-
-        if (rd == get_rd(r.r_offset + 4)) {
-          u64 val = sym.get_addr(ctx) + r.r_addend;
-
-          if (use_rvc && rd != 0 && sign_extend(val, 5) == val) {
-            // Replace AUIPC + LD with C.LI.
-            delta += 6;
-          } else if (sign_extend(val, 11) == val) {
-            // Replace AUIPC + LD with ADDI.
-            delta += 4;
-          }
+      if (sym.is_absolute() && is_got_load_pair(ctx, isec, rels, i)) {
+        u64 val = sym.get_addr(ctx) + r.r_addend;
+        if (use_rvc && is_int(val, 6) && get_rd(buf + r.r_offset) != 0) {
+          // Replace AUIPC + LD with C.LI.
+          remove(6);
+        } else if (is_int(val, 12)) {
+          // Replace AUIPC + LD with ADDI.
+          remove(4);
        }
      }
      break;
-    }
    case R_RISCV_HI20: {
      u64 val = sym.get_addr(ctx) + r.r_addend;
-      i64 rd = get_rd(r.r_offset);
+      i64 rd = get_rd(buf + r.r_offset);

-      if (sign_extend(val, 11) == val) {
+      if (is_int(val, 12)) {
        // We can replace `lui t0, %hi(foo)` and `add t0, t0, %lo(foo)`
        // instruction pair with `add t0, x0, %lo(foo)` if foo's bits
        // [32:11] are all one or all zero.
-        delta += 4;
-      } else if (use_rvc && rd != 0 && rd != 2 && sign_extend(val, 17) == val) {
+        remove(4);
+      } else if (use_rvc && rd != 0 && rd != 2 && is_int(val + 0x800, 18)) {
        // If the upper 20 bits can actually be represented in 6 bits,
        // we can use C.LUI instead of LUI.
-        delta += 2;
+        remove(2);
      }
      break;
    }
@@ -975,35 +970,100 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec, bool use_rvc) {
      //
      // Here, we remove `lui` and `add` if the offset is within ±2 KiB.
      if (i64 val = sym.get_addr(ctx) + r.r_addend - ctx.tp_addr;
-          sign_extend(val, 11) == val)
-        delta += 4;
+          is_int(val, 12))
+        remove(4);
      break;
    case R_RISCV_TLSDESC_HI20:
      if (!sym.has_tlsdesc(ctx))
-        delta += 4;
+        remove(4);
      break;
    case R_RISCV_TLSDESC_LOAD_LO12:
    case R_RISCV_TLSDESC_ADD_LO12: {
-      const ElfRel<E> &rel2 = rels[find_paired_reloc()];
+      const ElfRel<E> &rel2 = find_paired_reloc(ctx, isec, rels, sym, i);
      Symbol<E> &sym2 = *isec.file.symbols[rel2.r_sym];

      if (r.r_type == R_RISCV_TLSDESC_LOAD_LO12) {
        if (!sym2.has_tlsdesc(ctx))
-          delta += 4;
+          remove(4);
      } else {
        assert(r.r_type == R_RISCV_TLSDESC_ADD_LO12);
        if (!sym2.has_tlsdesc(ctx) && !sym2.has_gottp(ctx))
          if (i64 val = sym2.get_addr(ctx) + rel2.r_addend - ctx.tp_addr;
-              sign_extend(val, 11) == val)
-            delta += 4;
+              is_int(val, 12))
+            remove(4);
      }
      break;
    }
    }
  }

-  isec.extra.r_deltas[rels.size()] = delta;
-  isec.sh_size -= delta;
+  isec.sh_size -= r_delta;
+}
+
+// With --emit-relocs, rewrite the emitted relocations to match the
+// instructions that relaxation produced. The logic parallels apply_reloc_alloc.
+template <>
+void rewrite_reloc_types(Context<E> &ctx, InputSection<E> &isec, ElfRel<E> *buf) {
+  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
+  std::span<RelocDelta> deltas = isec.extra.r_deltas;
+  i64 k = 0;
+
+  for (i64 i = 0; i < rels.size(); i++) {
+    const ElfRel<E> &rel = rels[i];
+    Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+
+    i64 removed_bytes = 0;
+    if (!deltas.empty()) {
+      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
+        k++;
+      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
+        removed_bytes = get_removed_bytes(deltas, k);
+    }
+
+    switch (rel.r_type) {
+    case R_RISCV_CALL:
+    case R_RISCV_CALL_PLT:
+      // auipc + jalr => jal, or c.j/c.jal
+      if (removed_bytes == 4)
+        buf[i].r_type = R_RISCV_JAL;
+      else if (removed_bytes == 6)
+        buf[i].r_type = R_RISCV_RVC_JUMP;
+      break;
+    case R_RISCV_GOT_HI20:
+      // auipc + l[dw] GOT load => the value is materialized directly, so both
+      // this and the paired PCREL_LO12 (the load) no longer need a relocation.
+      if (removed_bytes) {
+        buf[i].r_type = R_RISCV_NONE;
+        buf[i + 2].r_type = R_RISCV_NONE;
+        i += 3;
+      }
+      break;
+    case R_RISCV_HI20:
+      // lui (+ addi) => an instruction holding a link-time constant.
+      if (removed_bytes)
+        buf[i].r_type = R_RISCV_NONE;
+      break;
+    case R_RISCV_TPREL_HI20:
+    case R_RISCV_TPREL_ADD:
+      // lui + add => deleted; the variable is accessed relative to tp directly.
+      if (removed_bytes)
+        buf[i].r_type = R_RISCV_NONE;
+      break;
+    case R_RISCV_TLSDESC_HI20:
+      if (!sym.has_tlsdesc(ctx))
+        buf[i].r_type = R_RISCV_NONE;
+      break;
+    case R_RISCV_TLSDESC_LOAD_LO12:
+    case R_RISCV_TLSDESC_ADD_LO12:
+    case R_RISCV_TLSDESC_CALL: {
+      Symbol<E> &sym2 =
+        *isec.file.symbols[find_paired_reloc(ctx, isec, rels, sym, i).r_sym];
+      if (!sym2.has_tlsdesc(ctx))
+        buf[i].r_type = R_RISCV_NONE;
+      break;
+    }
+    }
+  }
 }

 // ISA name handlers
@@ -1122,15 +1182,12 @@ static std::vector<Extn> merge_extensions(std::span<Extn> x, std::span<Extn> y)
  return vec;
 }

-static std::string to_string(const Extn &e) {
-  return e.name + std::to_string(e.major) + "p" + std::to_string(e.minor);
-}
-
 static std::string to_string(std::span<Extn> v) {
-  std::string str = to_string(v[0]);
-  for (i64 i = 1; i < v.size(); i++)
-    str += "_" + to_string(v[i]);
-  return str;
+  std::ostringstream os;
+  os << v[0].name << v[0].major << 'p' << v[0].minor;
+  for (Extn &e : v.subspan(1))
+    os << '_' << e.name << e.major << 'p' << e.minor;
+  return os.str();
 }

 //
@@ -33,7 +33,9 @@
 // we need to add TP to a return value before use. I don't know why it is
 // different, but that is the way it is.
 //
-// https://github.com/rui314/psabi/blob/main/s390x.pdf
+// https://github.com/IBM/s390x-abi/releases/download/v1.6.1/lzsabi_s390x.pdf
+
+#if MOLD_S390X

 #include "mold.h"

@@ -48,18 +50,24 @@ static void write_mid20(u8 *loc, u64 val) {
 template <>
 void write_plt_header(Context<E> &ctx, u8 *buf) {
  static u8 insn[] = {
-    // Compute PLT_INDEX
+    // Compute the offset into .rela.plt. This is equivalent to
+    // (%r0 - %r1 - 48 - 14) * 3/2 where %r0 is the PLT entry address
+    // plus 14, %r1 is the start address of .plt, and 48 is the size
+    // of this PLT header. We multiply by 3/2 because each PLT entry
+    // is 16 bytes, whereas each .rela.plt entry is 24 bytes.
    0xb9, 0x09, 0x00, 0x01,             // sgr   %r0, %r1
    0xa7, 0x0b, 0xff, 0xc2,             // aghi  %r0, -62
    0xeb, 0x10, 0x00, 0x01, 0x00, 0x0c, // srlg  %r1, %r0, 1
    0xb9, 0x08, 0x00, 0x01,             // agr   %r0, %r1
+    // Store the computed value to 56(%r15) and .got.plt[1] to 48(%15)
+    // where %r15 is the stack pointer.
    0xe3, 0x00, 0xf0, 0x38, 0x00, 0x24, // stg   %r0, 56(%r15)
-    // Branch to _dl_runtime_resolve
    0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_OFFSET
    0xd2, 0x07, 0xf0, 0x30, 0x10, 0x08, // mvc   48(8, %r15), 8(%r1)
+    // Branch to _dl_runtime_resolve
    0xe3, 0x10, 0x10, 0x10, 0x00, 0x04, // lg    %r1, 16(%r1)
    0x07, 0xf1,                         // br    %r1
-    0x07, 0x00, 0x07, 0x00,             // nopr; nopr
+    0x00, 0x00, 0x00, 0x00,             // (filler)
  };

  memcpy(buf, insn, sizeof(insn));
@@ -73,7 +81,7 @@ void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
    0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOTPLT_ENTRY_OFFSET
    0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
    0x0d, 0x01,                         // basr  %r0, %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
  };

  memcpy(buf, insn, sizeof(insn));
@@ -86,7 +94,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
    0xc0, 0x10, 0, 0, 0, 0,             // larl  %r1, GOT_ENTRY_OFFSET
    0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg    %r1, (%r1)
    0x07, 0xf1,                         // br    %r1
-    0x07, 0x00,                         // nopr
+    0x00, 0x00,                         // (filler)
  };

  memcpy(buf, insn, sizeof(insn));
@@ -124,28 +132,24 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
-    auto check_dbl = [&](i64 val, i64 lo, i64 hi) {
-      check(val, lo, hi);
-
-      // R_390_*DBL relocs should never refer a symbol at an odd address
-      if (val & 1)
-        Error(ctx) << *this << ": misaligned symbol " << sym
-                   << " for relocation " << rel;
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = rel.r_addend;
    u64 P = get_addr() + rel.r_offset;
    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
+    auto check_dbl = [&](i64 val, i64 lo, i64 hi) {
+      // R_390_*DBL relocs should never refer to a symbol at an odd address
+      check(val, lo, hi);
+      if (val & 1)
+        Error(ctx) << *this << ": misaligned symbol " << sym
+                   << " for relocation " << rel;
+    };
+
    switch (rel.r_type) {
    case R_390_64:
      break;
@@ -170,9 +174,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      check(S + A, 0, 1LL << 32);
      *(ub32 *)loc = S + A;
      break;
-    case R_390_PLT64:
-      *(ub64 *)loc = S + A;
-      break;
    case R_390_PC12DBL:
    case R_390_PLT12DBL:
      check_dbl(S + A - P, -(1 << 12), 1 << 12);
@@ -187,6 +188,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ub32 *)loc = S + A - P;
      break;
    case R_390_PC64:
+    case R_390_PLT64:
      *(ub64 *)loc = S + A - P;
      break;
    case R_390_PC16DBL:
@@ -250,6 +252,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ub32 *)loc = (GOT + A - P) >> 1;
      break;
    case R_390_GOTENT:
+      // If we can relax a GOT-loading LGRL to an address-materializing
+      // LARL, do that. The format of LGRL is 0xc 0x4 <reg> 0x8 followed
+      // by a 32-bit offset. LARL is 0xc 0x0 <reg> 0x0.
+      if (ctx.arg.relax && sym.is_pcrel_linktime_const(ctx)) {
+        u64 op = *(ub16 *)(loc - 2);
+        u64 val = S + A - P;
+        if ((op & 0xff0f) == 0xc408 && A == 2 && (val & 1) == 0 &&
+            is_int(val, 33)) {
+          *(ub16 *)(loc - 2) = 0xc000 | (op & 0x00f0);
+          *(ub32 *)loc = val >> 1;
+          break;
+        }
+      }
      check_dbl(GOT + G + A - P, -(1LL << 32), 1LL << 32);
      *(ub32 *)loc = (GOT + G + A - P) >> 1;
      break;
@@ -337,13 +352,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -351,6 +359,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
    u64 A = frag ? frag_addend : (i64)rel.r_addend;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, val, i, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_390_32:
      check(S + A, 0, 1LL << 32);
@@ -398,8 +410,10 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_390_32:
      scan_absrel(ctx, sym, rel);
      break;
+    case R_390_PC12DBL:
    case R_390_PC16:
    case R_390_PC16DBL:
+    case R_390_PC24DBL:
    case R_390_PC32:
    case R_390_PC32DBL:
    case R_390_PC64:
@@ -477,3 +491,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -58,11 +58,13 @@
 //    output from the linker contains lots of text relocations. That's not
 //    a problem with embedded programming, I guess.

+#if MOLD_SH4LE || MOLD_SH4BE
+
 #include "mold.h"

 namespace mold {

-using E = SH4;
+using E = MOLD_TARGET;

 // Even though SH-4 uses RELA-type relocations, addends are stored to
 // relocated places for some reason.
@@ -84,7 +86,7 @@ i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
  case R_SH_GOTOFF:
  case R_SH_GOTPC:
  case R_SH_GOTPLT32:
-    return *(ul32 *)loc;
+    return *(U32<E> *)loc;
  default:
    return 0;
  }
@@ -108,102 +110,102 @@ void write_addend(u8 *loc, i64 val, const ElfRel<E> &rel) {
  case R_SH_GOTOFF:
  case R_SH_GOTPC:
  case R_SH_GOTPLT32:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
  }
 }

 template <>
 void write_plt_header(Context<E> &ctx, u8 *buf) {
  if (ctx.arg.pic) {
-    static const u8 insn[] = {
-      0x02, 0xd2, //    mov.l   1f, r2
-      0xcc, 0x32, //    add     r12, r2
-      0x22, 0x50, //    mov.l   @(8, r2), r0
-      0x21, 0x52, //    mov.l   @(4, r2), r2
-      0x2b, 0x40, //    jmp     @r0
-      0x00, 0xe0, //    mov     #0, r0
-      0, 0, 0, 0, // 1: .long GOTPLT
+    constexpr U16<E> insn[] = {
+      0xd202, //    mov.l   1f, r2
+      0x32cc, //    add     r12, r2
+      0x5022, //    mov.l   @(8, r2), r0
+      0x5221, //    mov.l   @(4, r2), r2
+      0x402b, //    jmp     @r0
+      0xe000, //    mov     #0, r0
+      0, 0,   // 1: .long GOTPLT
    };

    static_assert(sizeof(insn) == E::plt_hdr_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 12) = ctx.gotplt->shdr.sh_addr - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 12) = ctx.gotplt->shdr.sh_addr - ctx.got->shdr.sh_addr;
  } else {
-    static const u8 insn[] = {
-      0x02, 0xd2, //    mov.l   1f, r2
-      0x22, 0x50, //    mov.l   @(8, r2), r0
-      0x21, 0x52, //    mov.l   @(4, r2), r2
-      0x2b, 0x40, //    jmp     @r0
-      0x00, 0xe0, //    mov     #0, r0
-      0x09, 0x00, //    nop
-      0, 0, 0, 0, // 1: .long GOTPLT
+    constexpr U16<E> insn[] = {
+      0xd202, //    mov.l   1f, r2
+      0x5022, //    mov.l   @(8, r2), r0
+      0x5221, //    mov.l   @(4, r2), r2
+      0x402b, //    jmp     @r0
+      0xe000, //    mov     #0, r0
+      0xfffd, //    (illegal)
+      0, 0,   // 1: .long GOTPLT
    };

    static_assert(sizeof(insn) == E::plt_hdr_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 12) = ctx.gotplt->shdr.sh_addr;
+    *(U32<E> *)(buf + 12) = ctx.gotplt->shdr.sh_addr;
  }
 }

 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  if (ctx.arg.pic) {
-    static const u8 insn[] = {
-      0x01, 0xd0, //    mov.l   1f, r0
-      0xce, 0x00, //    mov.l   @(r0, r12), r0
-      0x2b, 0x40, //    jmp     @r0
-      0x01, 0xd1, //    mov.l   2f, r1
-      0, 0, 0, 0, // 1: .long GOTPLT_ENTRY
-      0, 0, 0, 0, // 2: .long INDEX_IN_RELPLT
+    constexpr U16<E> insn[] = {
+      0xd001, //    mov.l   1f, r0
+      0x00ce, //    mov.l   @(r0, r12), r0
+      0x402b, //    jmp     @r0
+      0xd101, //    mov.l   2f, r1
+      0, 0,   // 1: .long GOTPLT_ENTRY
+      0, 0,   // 2: .long INDEX_IN_RELPLT
    };

    static_assert(sizeof(insn) == E::plt_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
-    *(ul32 *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
  } else {
-    static const u8 insn[] = {
-      0x01, 0xd0, //    mov.l   1f, r0
-      0x02, 0x60, //    mov.l   @r0, r0
-      0x2b, 0x40, //    jmp     @r0
-      0x01, 0xd1, //    mov.l   2f, r1
-      0, 0, 0, 0, // 1: .long GOTPLT_ENTRY
-      0, 0, 0, 0, // 2: .long INDEX_IN_RELPLT
+    constexpr U16<E> insn[] = {
+      0xd001, //    mov.l   1f, r0
+      0x6002, //    mov.l   @r0, r0
+      0x402b, //    jmp     @r0
+      0xd101, //    mov.l   2f, r1
+      0, 0,   // 1: .long GOTPLT_ENTRY
+      0, 0,   // 2: .long INDEX_IN_RELPLT
    };

    static_assert(sizeof(insn) == E::plt_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_gotplt_addr(ctx);
-    *(ul32 *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx);
+    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
  }
 }

 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
  if (ctx.arg.pic) {
-    static const u8 insn[] = {
-      0x01, 0xd0, //    mov.l   1f, r0
-      0xce, 0x00, //    mov.l   @(r0, r12), r0
-      0x2b, 0x40, //    jmp     @r0
-      0x09, 0x00, //    nop
-      0, 0, 0, 0, // 1: .long GOT_ENTRY
+    constexpr U16<E> insn[] = {
+      0xd001, //    mov.l   1f, r0
+      0x00ce, //    mov.l   @(r0, r12), r0
+      0x402b, //    jmp     @r0
+      0x0009, //    nop
+      0, 0,   // 1: .long GOT_ENTRY
    };

    static_assert(sizeof(insn) == E::pltgot_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 8) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
  } else {
-    static const u8 insn[] = {
-      0x01, 0xd0, //    mov.l   1f, r0
-      0x02, 0x60, //    mov.l   @r0, r0
-      0x2b, 0x40, //    jmp     @r0
-      0x09, 0x00, //    nop
-      0, 0, 0, 0, // 1: .long GOT_ENTRY
+    constexpr U16<E> insn[] = {
+      0xd001, //    mov.l   1f, r0
+      0x6002, //    mov.l   @r0, r0
+      0x402b, //    jmp     @r0
+      0x0009, //    nop
+      0, 0,   // 1: .long GOT_ENTRY
    };

    static_assert(sizeof(insn) == E::pltgot_size);
    memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx);
+    *(U32<E> *)(buf + 8) = sym.get_got_pltgot_addr(ctx);
  }
 }

@@ -216,10 +218,10 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
  case R_NONE:
    break;
  case R_SH_DIR32:
-    *(ul32 *)loc = val;
+    *(U32<E> *)loc = val;
    break;
  case R_SH_REL32:
-    *(ul32 *)loc = val - this->shdr.sh_addr - offset;
+    *(U32<E> *)loc = val - this->shdr.sh_addr - offset;
    break;
  default:
    Fatal(ctx) << "unsupported relocation in .eh_frame: " << rel;
@@ -249,31 +251,31 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      break;
    case R_SH_REL32:
    case R_SH_PLT32:
-      *(ul32 *)loc = S + A - P;
+      *(U32<E> *)loc = S + A - P;
      break;
    case R_SH_GOT32:
-      *(ul32 *)loc = G;
+      *(U32<E> *)loc = G;
      break;
    case R_SH_GOTPC:
-      *(ul32 *)loc = GOT + A - P;
+      *(U32<E> *)loc = GOT + A - P;
      break;
    case R_SH_GOTOFF:
-      *(ul32 *)loc = S + A - GOT;
+      *(U32<E> *)loc = S + A - GOT;
      break;
    case R_SH_TLS_GD_32:
-      *(ul32 *)loc = sym.get_tlsgd_addr(ctx) + A - GOT;
+      *(U32<E> *)loc = sym.get_tlsgd_addr(ctx) + A - GOT;
      break;
    case R_SH_TLS_LD_32:
-      *(ul32 *)loc = ctx.got->get_tlsld_addr(ctx) + A - GOT;
+      *(U32<E> *)loc = ctx.got->get_tlsld_addr(ctx) + A - GOT;
      break;
    case R_SH_TLS_LDO_32:
-      *(ul32 *)loc = S + A - ctx.dtp_addr;
+      *(U32<E> *)loc = S + A - ctx.dtp_addr;
      break;
    case R_SH_TLS_IE_32:
-      *(ul32 *)loc = sym.get_gottp_addr(ctx) + A - GOT;
+      *(U32<E> *)loc = sym.get_gottp_addr(ctx) + A - GOT;
      break;
    case R_SH_TLS_LE_32:
-      *(ul32 *)loc = S + A - ctx.tp_addr;
+      *(U32<E> *)loc = S + A - ctx.tp_addr;
      break;
    default:
      unreachable();
@@ -303,9 +305,9 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    switch (rel.r_type) {
    case R_SH_DIR32:
      if (std::optional<u64> val = get_tombstone(sym, frag))
-        *(ul32 *)loc = *val;
+        *(U32<E> *)loc = *val;
      else
-        *(ul32 *)loc = S + A;
+        *(U32<E> *)loc = S + A;
      break;
    default:
      Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
@@ -364,3 +366,5 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -51,11 +51,10 @@
 // address. That also means each function needs to execute the above piece
 // of code to become position-independent.
 //
-// This scheme is very similar to i386. That may not be a coincidence
-// because the i386 ELF psABI is created by Sun Microsystems too.
-//
 // https://github.com/rui314/psabi/blob/main/sparc.pdf

+#if MOLD_SPARC64
+
 #include "mold.h"

 namespace mold {
@@ -79,26 +78,77 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
  memset(buf, 0, E::plt_hdr_size);
 }

+// Returns the byte offset within .plt of the data pointer for a large SPARC
+// PLT entry. See write_plt_entry below for the block layout this assumes.
+i64 sparc_plt_ptr_offset(Context<E> &ctx, i64 pltidx) {
+  i64 i = pltidx - sparc_num_small_plt;
+  i64 block = i / 160;
+  i64 nlarge = (i64)ctx.plt->symbols.size() - sparc_num_small_plt;
+  i64 nstub = std::min<i64>(160, nlarge - block * 160);
+  return 0x100000 + block * 5120 + nstub * 24 + (i % 160) * 8;
+}
+
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  static ub32 insn[] = {
-    0x0300'0000, // sethi (. - .PLT0), %g1
-    0x3068'0000, // ba,a  %xcc, .PLT1
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-    0x0100'0000, // nop
-  };
+  // SPARC uses two PLT entry formats. A "small" entry branches directly to
+  // the resolver stub (.PLT1) with a BPcc instruction whose reach is only
+  // ±1 MiB. Once the PLT grows past that (0x100000 bytes), we switch to a
+  // "large" format: rather than branching to the resolver, a large entry
+  // loads a 64-bit value from a nearby data pointer and jumps to (that value
+  // + its own address).
+  //
+  // Large entries are grouped into blocks of 160: each block is 160 code
+  // stubs followed by 160 8-byte data pointers, one per stub. The stubs must
+  // sit on a fixed grid, 24 bytes apart, because that is how the loader finds
+  // them: on a lazy bind the stub jumps to the resolver (.PLT0) with only its
+  // own address, and the loader derives which symbol to resolve from that
+  // address. Nothing may sit between two stubs, so each stub's pointer lives
+  // in the block's pointer region, which it reaches with a signed 13-bit ldx
+  // offset (see to_plt_offset). This layout is dictated by the loader; we
+  // cannot rearrange or simplify it.
+  i64 idx = sym.get_plt_idx(ctx);

-  u64 plt0 = ctx.plt->shdr.sh_addr;
-  u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
-  u64 entry = sym.get_plt_addr(ctx);
+  if (idx < sparc_num_small_plt) {
+    static ub32 insn[] = {
+      0x0300'0000, // sethi (. - .PLT0), %g1
+      0x3068'0000, // ba,a  %xcc, .PLT1
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+      0x0100'0000, // nop
+    };

-  memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)buf |= bits(entry - plt0, 21, 0);
-  *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+    u64 plt0 = ctx.plt->shdr.sh_addr;
+    u64 plt1 = ctx.plt->shdr.sh_addr + E::plt_size;
+    u64 entry = sym.get_plt_addr(ctx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)buf |= bits(entry - plt0, 21, 0);
+    *(ub32 *)(buf + 4) |= bits(plt1 - entry - 4, 20, 2);
+  } else {
+    static ub32 insn[] = {
+      0x8a10'000f, // mov  %o7, %g5
+      0x4000'0002, // call . + 8
+      0x0100'0000, // nop
+      0xc25b'e000, // ldx  [ %o7 + .ptr ], %g1
+      0x83c3'c001, // jmpl %o7 + %g1, %g1
+      0x9e10'0005, // mov  %g5, %o7
+    };
+
+    u64 plt = ctx.plt->shdr.sh_addr;
+    u64 call = sym.get_plt_addr(ctx) + 4;
+    i64 ptroff = sparc_plt_ptr_offset(ctx, idx);
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ub32 *)(buf + 12) |= bits(plt + ptroff - call, 12, 0);
+
+    // The data pointer initially holds (.PLT0 - call) so that the first call
+    // jumps to .PLT0, where the loader's lazy resolver lives. The resolver
+    // later overwrites it with (target - call).
+    *(ub64 *)(ctx.buf + ctx.plt->shdr.sh_offset + ptroff) = plt - call;
+  }
 }

 template <>
@@ -142,7 +192,9 @@ template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
  std::span<const ElfRel<E>> rels = get_rels(ctx);

-  for (i64 i = 0; i < rels.size(); i++) {
+  // We iterate over relocations in reverse order so that it is easy
+  // to swap instructions for R_SPARC_TLS_GD_CALL.
+  for (i64 i = (i64)rels.size() - 1; i >= 0; i--) {
    const ElfRel<E> &rel = rels[i];
    if (rel.r_type == R_NONE)
      continue;
@@ -150,19 +202,20 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    u64 S = sym.get_addr(ctx);
    u64 A = rel.r_addend;
-    u64 P = (get_addr() + rel.r_offset);
-    u64 G = (sym.get_got_idx(ctx) * sizeof(Word<E>));
+    u64 P = get_addr() + rel.r_offset;
+    u64 G = sym.get_got_idx(ctx) * sizeof(Word<E>);
    u64 GOT = ctx.got->shdr.sh_addr;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, i, val, lo, hi);
+    };
+
+    auto rs1 = [&] { return *(ub32 *)loc & (0b11111 << 14); };
+    auto rs2 = [&] { return *(ub32 *)loc & 0b11111; };
+    auto rd  = [&] { return *(ub32 *)loc & (0b11111 << 25); };
+
    switch (rel.r_type) {
    case R_SPARC_5:
      check(S + A, 0, 1 << 5);
@@ -178,7 +231,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      break;
    case R_SPARC_8:
      check(S + A, 0, 1 << 8);
-      *(u8 *)loc = S + A;
+      *loc = S + A;
      break;
    case R_SPARC_10:
      check(S + A, 0, 1 << 10);
@@ -212,13 +265,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ub32 *)loc = S + A;
      break;
    case R_SPARC_PLT64:
-    case R_SPARC_UA64:
    case R_SPARC_REGISTER:
      *(ub64 *)loc = S + A;
      break;
    case R_SPARC_DISP8:
      check(S + A - P, -(1 << 7), 1 << 7);
-      *(u8 *)loc = S + A - P;
+      *loc = S + A - P;
      break;
    case R_SPARC_DISP16:
      check(S + A - P, -(1 << 15), 1 << 15);
@@ -303,12 +355,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      break;
    case R_SPARC_GOTDATA_OP:
      if (sym.is_absolute()) {
-        // ldx [ %g2 + %g1 ], %g1  →  nop
-        *(ub32 *)loc = 0x0100'0000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  mov %rs2, %rd
+        *(ub32 *)loc = 0x8010'0000 | rs2() | rd();
      } else if (sym.is_pcrel_linktime_const(ctx)) {
-        // ldx [ %g2 + %g1 ], %g1  →  add %g2, %g1, %g1
-        *(ub32 *)loc &= 0b00'11111'000000'11111'1'11111111'11111;
-        *(ub32 *)loc |= 0b10'00000'000000'00000'0'00000000'00000;
+        // ldx [ %rs1 + %rs2 ], %rd  →  add %rs1, %rs2, %rd
+        *(ub32 *)loc = 0x8000'0000 | rs1() | rs2() | rd();
      }
      break;
    case R_SPARC_PC10:
@@ -363,12 +414,12 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      if (sym.has_tlsgd(ctx)) {
        *(ub32 *)loc |= bits(sym.get_tlsgd_addr(ctx) + A - GOT, 9, 0);
      } else if (sym.has_gottp(ctx)) {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8010'2000 | (rd << 25) | (rd << 14); // or  %reg, $0, %reg
+        // add %rs1, %rs2, %rd → or  %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8010'2000 | rs1() | rd();
        *(ub32 *)loc |= bits(sym.get_gottp_addr(ctx) + A - GOT, 9, 0);
      } else {
-        u32 rd = bits(*(ub32 *)loc, 29, 25);
-        *(ub32 *)loc = 0x8018'2000 | (rd << 25) | (rd << 14); // xor %reg, $0, %reg
+        // add %rs1, %rs2, %rd → xor %rs1, $imm, %rd
+        *(ub32 *)loc = 0x8018'2000 | rs1() | rd();
        *(ub32 *)loc |= bits(S + A - ctx.tp_addr, 9, 0) | 0b1'1100'0000'0000;
      }
      break;
@@ -376,11 +427,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      if (sym.has_tlsgd(ctx)) {
        // do nothing
      } else if (sym.has_gottp(ctx)) {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0xd05d'c000 | rs2; // ldx [ %l7 + %reg ], %o0
+        // add %rs1, %rs2, %rd → ldx [ %rs1 + %rs2 ], %rd
+        *(ub32 *)loc = 0xc058'0000 | rs1() | rs2() | rd();
      } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9001'c000 | rs2; // add %g7, %reg, %o0
+        // add %rs1, %rs2, %rd → add %g7, %rs2, %rd
+        *(ub32 *)loc = 0x8001'c000 | rs2() | rd();
      }
      break;
    case R_SPARC_TLS_GD_CALL:
@@ -388,9 +439,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        u64 addr = ctx.extra.tls_get_addr->get_addr(ctx);
        *(ub32 *)loc |= bits(addr + A - P, 31, 2);
      } else if (sym.has_gottp(ctx)) {
-        *(ub32 *)loc = 0x9001'c008; // add %g7, %o0, %o0
+        // When we rewrite a branch instruction with a non-branch one,
+        // we need to swap the instruction and the following one so that
+        // the original execution order, which is inverted due to the
+        // branch delay slot, is preserved.
+        //
+        // Since we apply relocations from the end to the beginning,
+        // the instruction at loc + 4 is already complete.
+        memcpy(loc, loc + 4, 4);
+        *(ub32 *)(loc + 4) = 0x9001'c008; // add %g7, %o0, %o0
      } else {
-        *(ub32 *)loc = 0x0100'0000; // nop
+        *(ub32 *)loc = 0x0100'0000; // call → nop
      }
      break;
    case R_SPARC_TLS_LDM_HI22:
@@ -406,12 +465,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
        *(ub32 *)loc |= bits(ctx.tp_addr - ctx.tls_begin, 9, 0);
      break;
    case R_SPARC_TLS_LDM_ADD:
-      if (ctx.got->has_tlsld(ctx)) {
-        // do nothing
-      } else {
-        u32 rs2 = bits(*(ub32 *)loc, 4, 0);
-        *(ub32 *)loc = 0x9021'c000 | rs2; // sub %g7, %reg, %o0
-      }
+      if (!ctx.got->has_tlsld(ctx))
+        *(ub32 *)loc = 0x8021'c000 | rs2() | rd(); // sub %g7, %rs2, %rd
      break;
    case R_SPARC_TLS_LDM_CALL:
      if (ctx.got->has_tlsld(ctx)) {
@@ -443,6 +498,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ub32 *)loc = sym.esym().st_size + A;
      break;
    case R_SPARC_64:
+    case R_SPARC_UA64:
    case R_SPARC_TLS_LDO_ADD:
    case R_SPARC_TLS_IE_LD:
    case R_SPARC_TLS_IE_LDX:
@@ -466,13 +522,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

-    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
-    };
-
    SectionFragment<E> *frag;
    i64 frag_addend;
    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
@@ -480,6 +529,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
    u64 A = frag ? frag_addend : (i64)rel.r_addend;

+    auto check = [&](i64 val, i64 lo, i64 hi) {
+      check_range(ctx, val, i, lo, hi);
+    };
+
    switch (rel.r_type) {
    case R_SPARC_64:
    case R_SPARC_UA64:
@@ -489,12 +542,10 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
        *(ub64 *)loc = S + A;
      break;
    case R_SPARC_32:
-    case R_SPARC_UA32: {
-      i64 val = S + A;
-      check(val, 0, 1LL << 32);
-      *(ub32 *)loc = val;
+    case R_SPARC_UA32:
+      check(S + A, 0, 1LL << 32);
+      *(ub32 *)loc = S + A;
      break;
-    }
    case R_SPARC_TLS_DTPOFF32:
      *(ub32 *)loc = S + A - ctx.dtp_addr;
      break;
@@ -511,6 +562,7 @@ template <>
 void InputSection<E>::scan_relocations(Context<E> &ctx) {
  assert(shdr().sh_flags & SHF_ALLOC);
  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  bool needs_tlsgd = false;

  // Scan relocations
  for (i64 i = 0; i < rels.size(); i++) {
@@ -537,7 +589,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_SPARC_REGISTER:
    case R_SPARC_UA16:
    case R_SPARC_UA32:
-    case R_SPARC_UA64:
    case R_SPARC_PC_HM10:
    case R_SPARC_OLO10:
    case R_SPARC_LOX10:
@@ -595,6 +646,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
        sym.flags |= NEEDS_GOTTP;
      } else {
        sym.flags |= NEEDS_TLSGD;
+        needs_tlsgd = true;
      }
      break;
    case R_SPARC_TLS_LDM_HI22:
@@ -608,23 +660,21 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_SPARC_TLS_IE_HI22:
      sym.flags |= NEEDS_GOTTP;
      break;
-    case R_SPARC_TLS_GD_CALL:
-    case R_SPARC_TLS_LDM_CALL:
-      if (Symbol<E> *sym = ctx.extra.tls_get_addr; sym->is_imported)
-        sym->flags |= NEEDS_PLT;
-      break;
    case R_SPARC_TLS_LE_HIX22:
    case R_SPARC_TLS_LE_LOX10:
      check_tlsle(ctx, sym, rel);
      break;
    case R_SPARC_64:
+    case R_SPARC_UA64:
    case R_SPARC_GOTDATA_OP_LOX10:
    case R_SPARC_GOTDATA_OP:
    case R_SPARC_GOTDATA_LOX10:
    case R_SPARC_TLS_GD_LO10:
    case R_SPARC_TLS_GD_ADD:
+    case R_SPARC_TLS_GD_CALL:
    case R_SPARC_TLS_LDM_LO10:
    case R_SPARC_TLS_LDM_ADD:
+    case R_SPARC_TLS_LDM_CALL:
    case R_SPARC_TLS_LDO_HIX22:
    case R_SPARC_TLS_LDO_LOX10:
    case R_SPARC_TLS_LDO_ADD:
@@ -638,6 +688,14 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
      Error(ctx) << *this << ": unknown relocation: " << rel;
    }
  }
+
+  // TLS_GD_CALL and TLS_LDM_CALL relocations implicitly refer to
+  // __tls_get_addr, which may be dynamically linked from libc.so.
+  Symbol<E> &sym = *ctx.extra.tls_get_addr;
+  if (sym.is_imported && (needs_tlsgd || ctx.needs_tlsld))
+    sym.flags |= NEEDS_PLT;
 }

 } // namespace mold
+
+#endif
@@ -23,11 +23,11 @@
 // TLS block as TP (with some addend). As a result, offsets from TP to
 // thread-local variables (TLVs) in the main executable are all negative.
 //
-// https://github.com/rui314/psabi/blob/main/x86-64.pdf
-// https://github.com/rui314/psabi/blob/main/i386.pdf
+// https://gitlab.com/x86-psABIs/x86-64-ABI
+
+#if MOLD_X86_64

 #include "mold.h"
-
 #include <tbb/parallel_for_each.h>

 namespace mold {
@@ -128,98 +128,165 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
  }
 }

-static u32 relax_gotpcrelx(u8 *loc) {
-  switch ((loc[0] << 8) | loc[1]) {
-  case 0xff15: return 0x90e8; // call *0(%rip) -> call 0
-  case 0xff25: return 0x90e9; // jmp  *0(%rip) -> jmp  0
+static u32 relax_gotpcrelx(u8 *loc, const ElfRel<E> &rel) {
+  if (rel.r_type == R_X86_64_GOTPCRELX) {
+    switch ((loc[-2] << 8) | loc[-1]) {
+    case 0xff15: return 0x40e8; // call *0(%rip) -> call 0
+    case 0xff25: return 0x40e9; // jmp  *0(%rip) -> jmp  0
+    }
+  } else {
+    assert(rel.r_type == R_X86_64_REX_GOTPCRELX ||
+           rel.r_type == R_X86_64_CODE_4_GOTPCRELX);
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488b05: return 0x8d05; // mov 0(%rip), %rax -> lea 0(%rip), %rax
+    case 0x488b0d: return 0x8d0d; // mov 0(%rip), %rcx -> lea 0(%rip), %rcx
+    case 0x488b15: return 0x8d15; // mov 0(%rip), %rdx -> lea 0(%rip), %rdx
+    case 0x488b1d: return 0x8d1d; // mov 0(%rip), %rbx -> lea 0(%rip), %rbx
+    case 0x488b25: return 0x8d25; // mov 0(%rip), %rsp -> lea 0(%rip), %rsp
+    case 0x488b2d: return 0x8d2d; // mov 0(%rip), %rbp -> lea 0(%rip), %rbp
+    case 0x488b35: return 0x8d35; // mov 0(%rip), %rsi -> lea 0(%rip), %rsi
+    case 0x488b3d: return 0x8d3d; // mov 0(%rip), %rdi -> lea 0(%rip), %rdi
+    case 0x4c8b05: return 0x8d05; // mov 0(%rip), %r8  -> lea 0(%rip), %r8
+    case 0x4c8b0d: return 0x8d0d; // mov 0(%rip), %r9  -> lea 0(%rip), %r9
+    case 0x4c8b15: return 0x8d15; // mov 0(%rip), %r10 -> lea 0(%rip), %r10
+    case 0x4c8b1d: return 0x8d1d; // mov 0(%rip), %r11 -> lea 0(%rip), %r11
+    case 0x4c8b25: return 0x8d25; // mov 0(%rip), %r12 -> lea 0(%rip), %r12
+    case 0x4c8b2d: return 0x8d2d; // mov 0(%rip), %r13 -> lea 0(%rip), %r13
+    case 0x4c8b35: return 0x8d35; // mov 0(%rip), %r14 -> lea 0(%rip), %r14
+    case 0x4c8b3d: return 0x8d3d; // mov 0(%rip), %r15 -> lea 0(%rip), %r15
+    }
  }
  return 0;
 }

-static u32 relax_rex_gotpcrelx(u8 *loc) {
-  switch ((loc[0] << 16) | (loc[1] << 8) | loc[2]) {
-  case 0x488b05: return 0x488d05; // mov 0(%rip), %rax -> lea 0(%rip), %rax
-  case 0x488b0d: return 0x488d0d; // mov 0(%rip), %rcx -> lea 0(%rip), %rcx
-  case 0x488b15: return 0x488d15; // mov 0(%rip), %rdx -> lea 0(%rip), %rdx
-  case 0x488b1d: return 0x488d1d; // mov 0(%rip), %rbx -> lea 0(%rip), %rbx
-  case 0x488b25: return 0x488d25; // mov 0(%rip), %rsp -> lea 0(%rip), %rsp
-  case 0x488b2d: return 0x488d2d; // mov 0(%rip), %rbp -> lea 0(%rip), %rbp
-  case 0x488b35: return 0x488d35; // mov 0(%rip), %rsi -> lea 0(%rip), %rsi
-  case 0x488b3d: return 0x488d3d; // mov 0(%rip), %rdi -> lea 0(%rip), %rdi
-  case 0x4c8b05: return 0x4c8d05; // mov 0(%rip), %r8  -> lea 0(%rip), %r8
-  case 0x4c8b0d: return 0x4c8d0d; // mov 0(%rip), %r9  -> lea 0(%rip), %r9
-  case 0x4c8b15: return 0x4c8d15; // mov 0(%rip), %r10 -> lea 0(%rip), %r10
-  case 0x4c8b1d: return 0x4c8d1d; // mov 0(%rip), %r11 -> lea 0(%rip), %r11
-  case 0x4c8b25: return 0x4c8d25; // mov 0(%rip), %r12 -> lea 0(%rip), %r12
-  case 0x4c8b2d: return 0x4c8d2d; // mov 0(%rip), %r13 -> lea 0(%rip), %r13
-  case 0x4c8b35: return 0x4c8d35; // mov 0(%rip), %r14 -> lea 0(%rip), %r14
-  case 0x4c8b3d: return 0x4c8d3d; // mov 0(%rip), %r15 -> lea 0(%rip), %r15
+static u32 relax_gottpoff(u8 *loc, const ElfRel<E> &rel) {
+  if (rel.r_type == R_X86_64_GOTTPOFF) {
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488b05: return 0x48c7c0; // mov 0(%rip), %rax -> mov $0, %rax
+    case 0x488b0d: return 0x48c7c1; // mov 0(%rip), %rcx -> mov $0, %rcx
+    case 0x488b15: return 0x48c7c2; // mov 0(%rip), %rdx -> mov $0, %rdx
+    case 0x488b1d: return 0x48c7c3; // mov 0(%rip), %rbx -> mov $0, %rbx
+    case 0x488b25: return 0x48c7c4; // mov 0(%rip), %rsp -> mov $0, %rsp
+    case 0x488b2d: return 0x48c7c5; // mov 0(%rip), %rbp -> mov $0, %rbp
+    case 0x488b35: return 0x48c7c6; // mov 0(%rip), %rsi -> mov $0, %rsi
+    case 0x488b3d: return 0x48c7c7; // mov 0(%rip), %rdi -> mov $0, %rdi
+    case 0x4c8b05: return 0x49c7c0; // mov 0(%rip), %r8  -> mov $0, %r8
+    case 0x4c8b0d: return 0x49c7c1; // mov 0(%rip), %r9  -> mov $0, %r9
+    case 0x4c8b15: return 0x49c7c2; // mov 0(%rip), %r10 -> mov $0, %r10
+    case 0x4c8b1d: return 0x49c7c3; // mov 0(%rip), %r11 -> mov $0, %r11
+    case 0x4c8b25: return 0x49c7c4; // mov 0(%rip), %r12 -> mov $0, %r12
+    case 0x4c8b2d: return 0x49c7c5; // mov 0(%rip), %r13 -> mov $0, %r13
+    case 0x4c8b35: return 0x49c7c6; // mov 0(%rip), %r14 -> mov $0, %r14
+    case 0x4c8b3d: return 0x49c7c7; // mov 0(%rip), %r15 -> mov $0, %r15
+    }
+  } else {
+    assert(rel.r_type == R_X86_64_CODE_4_GOTTPOFF);
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488b05: return 0x18c7c0; // mov 0(%rip), %r16 -> mov $0, %r16
+    case 0x488b0d: return 0x18c7c1; // mov 0(%rip), %r17 -> mov $0, %r17
+    case 0x488b15: return 0x18c7c2; // mov 0(%rip), %r18 -> mov $0, %r18
+    case 0x488b1d: return 0x18c7c3; // mov 0(%rip), %r19 -> mov $0, %r19
+    case 0x488b25: return 0x18c7c4; // mov 0(%rip), %r20 -> mov $0, %r20
+    case 0x488b2d: return 0x18c7c5; // mov 0(%rip), %r21 -> mov $0, %r21
+    case 0x488b35: return 0x18c7c6; // mov 0(%rip), %r22 -> mov $0, %r22
+    case 0x488b3d: return 0x18c7c7; // mov 0(%rip), %r23 -> mov $0, %r23
+    case 0x4c8b05: return 0x19c7c0; // mov 0(%rip), %r24 -> mov $0, %r24
+    case 0x4c8b0d: return 0x19c7c1; // mov 0(%rip), %r25 -> mov $0, %r25
+    case 0x4c8b15: return 0x19c7c2; // mov 0(%rip), %r26 -> mov $0, %r26
+    case 0x4c8b1d: return 0x19c7c3; // mov 0(%rip), %r27 -> mov $0, %r27
+    case 0x4c8b25: return 0x19c7c4; // mov 0(%rip), %r28 -> mov $0, %r28
+    case 0x4c8b2d: return 0x19c7c5; // mov 0(%rip), %r29 -> mov $0, %r29
+    case 0x4c8b35: return 0x19c7c6; // mov 0(%rip), %r30 -> mov $0, %r30
+    case 0x4c8b3d: return 0x19c7c7; // mov 0(%rip), %r31 -> mov $0, %r31
+    }
  }
  return 0;
 }

-static u32 relax_gottpoff(u8 *loc) {
-  switch ((loc[0] << 16) | (loc[1] << 8) | loc[2]) {
-  case 0x488b05: return 0x48c7c0; // mov 0(%rip), %rax -> mov $0, %rax
-  case 0x488b0d: return 0x48c7c1; // mov 0(%rip), %rcx -> mov $0, %rcx
-  case 0x488b15: return 0x48c7c2; // mov 0(%rip), %rdx -> mov $0, %rdx
-  case 0x488b1d: return 0x48c7c3; // mov 0(%rip), %rbx -> mov $0, %rbx
-  case 0x488b25: return 0x48c7c4; // mov 0(%rip), %rsp -> mov $0, %rsp
-  case 0x488b2d: return 0x48c7c5; // mov 0(%rip), %rbp -> mov $0, %rbp
-  case 0x488b35: return 0x48c7c6; // mov 0(%rip), %rsi -> mov $0, %rsi
-  case 0x488b3d: return 0x48c7c7; // mov 0(%rip), %rdi -> mov $0, %rdi
-  case 0x4c8b05: return 0x49c7c0; // mov 0(%rip), %r8  -> mov $0, %r8
-  case 0x4c8b0d: return 0x49c7c1; // mov 0(%rip), %r9  -> mov $0, %r9
-  case 0x4c8b15: return 0x49c7c2; // mov 0(%rip), %r10 -> mov $0, %r10
-  case 0x4c8b1d: return 0x49c7c3; // mov 0(%rip), %r11 -> mov $0, %r11
-  case 0x4c8b25: return 0x49c7c4; // mov 0(%rip), %r12 -> mov $0, %r12
-  case 0x4c8b2d: return 0x49c7c5; // mov 0(%rip), %r13 -> mov $0, %r13
-  case 0x4c8b35: return 0x49c7c6; // mov 0(%rip), %r14 -> mov $0, %r14
-  case 0x4c8b3d: return 0x49c7c7; // mov 0(%rip), %r15 -> mov $0, %r15
+static u32 relax_tlsdesc_to_ie(u8 *loc, const ElfRel<E> &rel) {
+  if (rel.r_type == R_X86_64_GOTPC32_TLSDESC) {
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488d05: return 0x488b05; // lea 0(%rip), %rax -> mov 0(%rip), %rax
+    case 0x488d0d: return 0x488b0d; // lea 0(%rip), %rcx -> mov 0(%rip), %rcx
+    case 0x488d15: return 0x488b15; // lea 0(%rip), %rdx -> mov 0(%rip), %rdx
+    case 0x488d1d: return 0x488b1d; // lea 0(%rip), %rbx -> mov 0(%rip), %rbx
+    case 0x488d25: return 0x488b25; // lea 0(%rip), %rsp -> mov 0(%rip), %rsp
+    case 0x488d2d: return 0x488b2d; // lea 0(%rip), %rbp -> mov 0(%rip), %rbp
+    case 0x488d35: return 0x488b35; // lea 0(%rip), %rsi -> mov 0(%rip), %rsi
+    case 0x488d3d: return 0x488b3d; // lea 0(%rip), %rdi -> mov 0(%rip), %rdi
+    case 0x4c8d05: return 0x4c8b05; // lea 0(%rip), %r8  -> mov 0(%rip), %r8
+    case 0x4c8d0d: return 0x4c8b0d; // lea 0(%rip), %r9  -> mov 0(%rip), %r9
+    case 0x4c8d15: return 0x4c8b15; // lea 0(%rip), %r10 -> mov 0(%rip), %r10
+    case 0x4c8d1d: return 0x4c8b1d; // lea 0(%rip), %r11 -> mov 0(%rip), %r11
+    case 0x4c8d25: return 0x4c8b25; // lea 0(%rip), %r12 -> mov 0(%rip), %r12
+    case 0x4c8d2d: return 0x4c8b2d; // lea 0(%rip), %r13 -> mov 0(%rip), %r13
+    case 0x4c8d35: return 0x4c8b35; // lea 0(%rip), %r14 -> mov 0(%rip), %r14
+    case 0x4c8d3d: return 0x4c8b3d; // lea 0(%rip), %r15 -> mov 0(%rip), %r15
+    }
+  } else {
+    assert(rel.r_type == R_X86_64_CODE_4_GOTPC32_TLSDESC);
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488d05: return 0x488b05; // lea 0(%rip), %r16 -> mov 0(%rip), %r16
+    case 0x488d0d: return 0x488b0d; // lea 0(%rip), %r17 -> mov 0(%rip), %r17
+    case 0x488d15: return 0x488b15; // lea 0(%rip), %r18 -> mov 0(%rip), %r18
+    case 0x488d1d: return 0x488b1d; // lea 0(%rip), %r19 -> mov 0(%rip), %r19
+    case 0x488d25: return 0x488b25; // lea 0(%rip), %r20 -> mov 0(%rip), %r20
+    case 0x488d2d: return 0x488b2d; // lea 0(%rip), %r21 -> mov 0(%rip), %r21
+    case 0x488d35: return 0x488b35; // lea 0(%rip), %r22 -> mov 0(%rip), %r22
+    case 0x488d3d: return 0x488b3d; // lea 0(%rip), %r23 -> mov 0(%rip), %r23
+    case 0x4c8d05: return 0x4c8b05; // lea 0(%rip), %r24 -> mov 0(%rip), %r24
+    case 0x4c8d0d: return 0x4c8b0d; // lea 0(%rip), %r25 -> mov 0(%rip), %r25
+    case 0x4c8d15: return 0x4c8b15; // lea 0(%rip), %r26 -> mov 0(%rip), %r26
+    case 0x4c8d1d: return 0x4c8b1d; // lea 0(%rip), %r27 -> mov 0(%rip), %r27
+    case 0x4c8d25: return 0x4c8b25; // lea 0(%rip), %r28 -> mov 0(%rip), %r28
+    case 0x4c8d2d: return 0x4c8b2d; // lea 0(%rip), %r29 -> mov 0(%rip), %r29
+    case 0x4c8d35: return 0x4c8b35; // lea 0(%rip), %r30 -> mov 0(%rip), %r30
+    case 0x4c8d3d: return 0x4c8b3d; // lea 0(%rip), %r31 -> mov 0(%rip), %r31
+    }
  }
  return 0;
 }

-static u32 relax_tlsdesc_to_ie(u8 *loc) {
-  switch ((loc[0] << 16) | (loc[1] << 8) | loc[2]) {
-  case 0x488d05: return 0x488b05; // lea 0(%rip), %rax -> mov 0(%rip), %rax
-  case 0x488d0d: return 0x488b0d; // lea 0(%rip), %rcx -> mov 0(%rip), %rcx
-  case 0x488d15: return 0x488b15; // lea 0(%rip), %rdx -> mov 0(%rip), %rdx
-  case 0x488d1d: return 0x488b1d; // lea 0(%rip), %rbx -> mov 0(%rip), %rbx
-  case 0x488d25: return 0x488b25; // lea 0(%rip), %rsp -> mov 0(%rip), %rsp
-  case 0x488d2d: return 0x488b2d; // lea 0(%rip), %rbp -> mov 0(%rip), %rbp
-  case 0x488d35: return 0x488b35; // lea 0(%rip), %rsi -> mov 0(%rip), %rsi
-  case 0x488d3d: return 0x488b3d; // lea 0(%rip), %rdi -> mov 0(%rip), %rdi
-  case 0x4c8d05: return 0x4c8b05; // lea 0(%rip), %r8  -> mov 0(%rip), %r8
-  case 0x4c8d0d: return 0x4c8b0d; // lea 0(%rip), %r9  -> mov 0(%rip), %r9
-  case 0x4c8d15: return 0x4c8b15; // lea 0(%rip), %r10 -> mov 0(%rip), %r10
-  case 0x4c8d1d: return 0x4c8b1d; // lea 0(%rip), %r11 -> mov 0(%rip), %r11
-  case 0x4c8d25: return 0x4c8b25; // lea 0(%rip), %r12 -> mov 0(%rip), %r12
-  case 0x4c8d2d: return 0x4c8b2d; // lea 0(%rip), %r13 -> mov 0(%rip), %r13
-  case 0x4c8d35: return 0x4c8b35; // lea 0(%rip), %r14 -> mov 0(%rip), %r14
-  case 0x4c8d3d: return 0x4c8b3d; // lea 0(%rip), %r15 -> mov 0(%rip), %r15
-  }
-  return 0;
-}
-
-static u32 relax_tlsdesc_to_le(u8 *loc) {
-  switch ((loc[0] << 16) | (loc[1] << 8) | loc[2]) {
-  case 0x488d05: return 0x48c7c0; // lea 0(%rip), %rax -> mov $0, %rax
-  case 0x488d0d: return 0x48c7c1; // lea 0(%rip), %rcx -> mov $0, %rcx
-  case 0x488d15: return 0x48c7c2; // lea 0(%rip), %rdx -> mov $0, %rdx
-  case 0x488d1d: return 0x48c7c3; // lea 0(%rip), %rbx -> mov $0, %rbx
-  case 0x488d25: return 0x48c7c4; // lea 0(%rip), %rsp -> mov $0, %rsp
-  case 0x488d2d: return 0x48c7c5; // lea 0(%rip), %rbp -> mov $0, %rbp
-  case 0x488d35: return 0x48c7c6; // lea 0(%rip), %rsi -> mov $0, %rsi
-  case 0x488d3d: return 0x48c7c7; // lea 0(%rip), %rdi -> mov $0, %rdi
-  case 0x4c8d05: return 0x49c7c0; // lea 0(%rip), %r8  -> mov $0, %r8
-  case 0x4c8d0d: return 0x49c7c1; // lea 0(%rip), %r9  -> mov $0, %r9
-  case 0x4c8d15: return 0x49c7c2; // lea 0(%rip), %r10 -> mov $0, %r10
-  case 0x4c8d1d: return 0x49c7c3; // lea 0(%rip), %r11 -> mov $0, %r11
-  case 0x4c8d25: return 0x49c7c4; // lea 0(%rip), %r12 -> mov $0, %r12
-  case 0x4c8d2d: return 0x49c7c5; // lea 0(%rip), %r13 -> mov $0, %r13
-  case 0x4c8d35: return 0x49c7c6; // lea 0(%rip), %r14 -> mov $0, %r14
-  case 0x4c8d3d: return 0x49c7c7; // lea 0(%rip), %r15 -> mov $0, %r15
+static u32 relax_tlsdesc_to_le(u8 *loc, const ElfRel<E> &rel) {
+  if (rel.r_type == R_X86_64_GOTPC32_TLSDESC) {
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488d05: return 0x48c7c0; // lea 0(%rip), %rax -> mov $0, %rax
+    case 0x488d0d: return 0x48c7c1; // lea 0(%rip), %rcx -> mov $0, %rcx
+    case 0x488d15: return 0x48c7c2; // lea 0(%rip), %rdx -> mov $0, %rdx
+    case 0x488d1d: return 0x48c7c3; // lea 0(%rip), %rbx -> mov $0, %rbx
+    case 0x488d25: return 0x48c7c4; // lea 0(%rip), %rsp -> mov $0, %rsp
+    case 0x488d2d: return 0x48c7c5; // lea 0(%rip), %rbp -> mov $0, %rbp
+    case 0x488d35: return 0x48c7c6; // lea 0(%rip), %rsi -> mov $0, %rsi
+    case 0x488d3d: return 0x48c7c7; // lea 0(%rip), %rdi -> mov $0, %rdi
+    case 0x4c8d05: return 0x49c7c0; // lea 0(%rip), %r8  -> mov $0, %r8
+    case 0x4c8d0d: return 0x49c7c1; // lea 0(%rip), %r9  -> mov $0, %r9
+    case 0x4c8d15: return 0x49c7c2; // lea 0(%rip), %r10 -> mov $0, %r10
+    case 0x4c8d1d: return 0x49c7c3; // lea 0(%rip), %r11 -> mov $0, %r11
+    case 0x4c8d25: return 0x49c7c4; // lea 0(%rip), %r12 -> mov $0, %r12
+    case 0x4c8d2d: return 0x49c7c5; // lea 0(%rip), %r13 -> mov $0, %r13
+    case 0x4c8d35: return 0x49c7c6; // lea 0(%rip), %r14 -> mov $0, %r14
+    case 0x4c8d3d: return 0x49c7c7; // lea 0(%rip), %r15 -> mov $0, %r15
+    }
+  } else {
+    assert(rel.r_type == R_X86_64_CODE_4_GOTPC32_TLSDESC);
+    switch ((loc[-3] << 16) | (loc[-2] << 8) | loc[-1]) {
+    case 0x488d05: return 0x18c7c0; // lea 0(%rip), %r16 -> mov $0, %r16
+    case 0x488d0d: return 0x18c7c1; // lea 0(%rip), %r17 -> mov $0, %r17
+    case 0x488d15: return 0x18c7c2; // lea 0(%rip), %r18 -> mov $0, %r18
+    case 0x488d1d: return 0x18c7c3; // lea 0(%rip), %r19 -> mov $0, %r19
+    case 0x488d25: return 0x18c7c4; // lea 0(%rip), %r20 -> mov $0, %r20
+    case 0x488d2d: return 0x18c7c5; // lea 0(%rip), %r21 -> mov $0, %r21
+    case 0x488d35: return 0x18c7c6; // lea 0(%rip), %r22 -> mov $0, %r22
+    case 0x488d3d: return 0x18c7c7; // lea 0(%rip), %r23 -> mov $0, %r23
+    case 0x4c8d05: return 0x19c7c0; // lea 0(%rip), %r24 -> mov $0, %r24
+    case 0x4c8d0d: return 0x19c7c1; // lea 0(%rip), %r25 -> mov $0, %r25
+    case 0x4c8d15: return 0x19c7c2; // lea 0(%rip), %r26 -> mov $0, %r26
+    case 0x4c8d1d: return 0x19c7c3; // lea 0(%rip), %r27 -> mov $0, %r27
+    case 0x4c8d25: return 0x19c7c4; // lea 0(%rip), %r28 -> mov $0, %r28
+    case 0x4c8d2d: return 0x19c7c5; // lea 0(%rip), %r29 -> mov $0, %r29
+    case 0x4c8d35: return 0x19c7c6; // lea 0(%rip), %r30 -> mov $0, %r30
+    case 0x4c8d3d: return 0x19c7c7; // lea 0(%rip), %r31 -> mov $0, %r31
+    }
  }
  return 0;
 }
@@ -227,7 +294,7 @@ static u32 relax_tlsdesc_to_le(u8 *loc) {
 // Rewrite a function call to __tls_get_addr to a cheaper instruction
 // sequence. We can do this when we know the thread-local variable's TP-
 // relative address at link-time.
-static void relax_gd_to_le(u8 *loc, ElfRel<E> rel, u64 val) {
+static void relax_gd_to_le(u8 *loc, const ElfRel<E> &rel, u64 val) {
  switch (rel.r_type) {
  case R_X86_64_PLT32:
  case R_X86_64_PC32:
@@ -271,7 +338,7 @@ static void relax_gd_to_le(u8 *loc, ElfRel<E> rel, u64 val) {
  }
 }

-static void relax_gd_to_ie(u8 *loc, ElfRel<E> rel, u64 val) {
+static void relax_gd_to_ie(u8 *loc, const ElfRel<E> &rel, u64 val) {
  switch (rel.r_type) {
  case R_X86_64_PLT32:
  case R_X86_64_PC32:
@@ -304,7 +371,7 @@ static void relax_gd_to_ie(u8 *loc, ElfRel<E> rel, u64 val) {
 // sequence. The difference from relax_gd_to_le is that we are materializing
 // the address of the beginning of TLS block instead of an address of a
 // particular thread-local variable.
-static void relax_ld_to_le(u8 *loc, ElfRel<E> rel, i64 tls_size) {
+static void relax_ld_to_le(u8 *loc, const ElfRel<E> &rel, i64 tls_size) {
  switch (rel.r_type) {
  case R_X86_64_PLT32:
  case R_X86_64_PC32: {
@@ -376,11 +443,14 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

+    u64 S = sym.get_addr(ctx);
+    u64 A = rel.r_addend;
+    u64 P = get_addr() + rel.r_offset;
+    u64 G = sym.get_got_addr(ctx) - ctx.gotplt->shdr.sh_addr;
+    u64 GOT = ctx.gotplt->shdr.sh_addr;
+
    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
+      check_range(ctx, i, val, lo, hi);
    };

    auto write32 = [&](u64 val) {
@@ -393,12 +463,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul32 *)loc = val;
    };

-    u64 S = sym.get_addr(ctx);
-    u64 A = rel.r_addend;
-    u64 P = get_addr() + rel.r_offset;
-    u64 G = sym.get_got_addr(ctx) - ctx.gotplt->shdr.sh_addr;
-    u64 GOT = ctx.gotplt->shdr.sh_addr;
-
    switch (rel.r_type) {
    case R_X86_64_8:
      check(S + A, 0, 1 << 8);
@@ -454,30 +518,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul64 *)loc = G + GOT + A - P;
      break;
    case R_X86_64_GOTPCRELX:
+    case R_X86_64_REX_GOTPCRELX:
+    case R_X86_64_CODE_4_GOTPCRELX:
      // We always want to relax GOTPCRELX relocs even if --no-relax
      // was given because some static PIE runtime code depends on these
      // relaxations.
-      if (sym.is_pcrel_linktime_const(ctx)) {
-        u32 insn = relax_gotpcrelx(loc - 2);
-        i64 val = S + A - P;
-        if (insn && (i32)val == val) {
+      if (sym.is_pcrel_linktime_const(ctx) && is_int(S + A - P, 32)) {
+        if (u32 insn = relax_gotpcrelx(loc, rel)) {
          loc[-2] = insn >> 8;
          loc[-1] = insn;
-          *(ul32 *)loc = val;
-          break;
-        }
-      }
-      write32s(G + GOT + A - P);
-      break;
-    case R_X86_64_REX_GOTPCRELX:
-      if (sym.is_pcrel_linktime_const(ctx)) {
-        u32 insn = relax_rex_gotpcrelx(loc - 3);
-        i64 val = S + A - P;
-        if (insn && (i32)val == val) {
-          loc[-3] = insn >> 16;
-          loc[-2] = insn >> 8;
-          loc[-1] = insn;
-          *(ul32 *)loc = val;
+          *(ul32 *)loc = S + A - P;
          break;
        }
      }
@@ -510,18 +560,22 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      *(ul64 *)loc = S + A - ctx.tp_addr;
      break;
    case R_X86_64_GOTTPOFF:
+    case R_X86_64_CODE_4_GOTTPOFF:
      if (sym.has_gottp(ctx)) {
        write32s(sym.get_gottp_addr(ctx) + A - P);
      } else {
-        u32 insn = relax_gottpoff(loc - 3);
+        u32 insn = relax_gottpoff(loc, rel);
        loc[-3] = insn >> 16;
        loc[-2] = insn >> 8;
        loc[-1] = insn;
        write32s(S - ctx.tp_addr);
-        assert(A == -4);
      }
      break;
+    case R_X86_64_CODE_6_GOTTPOFF:
+      write32s(sym.get_gottp_addr(ctx) + A - P);
+      break;
    case R_X86_64_GOTPC32_TLSDESC:
+    case R_X86_64_CODE_4_GOTPC32_TLSDESC:
      // x86-64 TLSDESC uses the following code sequence to materialize
      // a TP-relative address in %rax.
      //
@@ -553,17 +607,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      if (sym.has_tlsdesc(ctx)) {
        write32s(sym.get_tlsdesc_addr(ctx) + A - P);
      } else if (sym.has_gottp(ctx)) {
-        u32 insn = relax_tlsdesc_to_ie(loc - 3);
+        u32 insn = relax_tlsdesc_to_ie(loc, rel);
        if (!insn)
-          Fatal(ctx) << *this << ": illegal instruction sequence for TLSDESC";
+          Fatal(ctx) << *this << ": illegal instruction sequence for " << rel;
        loc[-3] = insn >> 16;
        loc[-2] = insn >> 8;
        loc[-1] = insn;
        write32s(sym.get_gottp_addr(ctx) + A - P);
      } else {
-        u32 insn = relax_tlsdesc_to_le(loc - 3);
+        u32 insn = relax_tlsdesc_to_le(loc, rel);
        if (!insn)
-          Fatal(ctx) << *this << ": illegal instruction sequence for TLSDESC";
+          Fatal(ctx) << *this << ": illegal instruction sequence for " << rel;
        loc[-3] = insn >> 16;
        loc[-2] = insn >> 8;
        loc[-1] = insn;
@@ -571,9 +625,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
      }
      break;
    case R_X86_64_TLSDESC_CALL:
-      if (sym.has_tlsdesc(ctx)) {
-        // Do nothing
-      } else {
+      if (!sym.has_tlsdesc(ctx)) {
        // call *(%rax) -> nop
        loc[0] = 0x66;
        loc[1] = 0x90;
@@ -615,11 +667,15 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
    Symbol<E> &sym = *file.symbols[rel.r_sym];
    u8 *loc = base + rel.r_offset;

+    SectionFragment<E> *frag;
+    i64 frag_addend;
+    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
+
+    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
+    u64 A = frag ? frag_addend : (i64)rel.r_addend;
+
    auto check = [&](i64 val, i64 lo, i64 hi) {
-      if (val < lo || hi <= val)
-        Error(ctx) << *this << ": relocation " << rel << " against "
-                   << sym << " out of range: " << val << " is not in ["
-                   << lo << ", " << hi << ")";
+      check_range(ctx, i, val, lo, hi);
    };

    auto write32 = [&](u64 val) {
@@ -632,13 +688,6 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
      *(ul32 *)loc = val;
    };

-    SectionFragment<E> *frag;
-    i64 frag_addend;
-    std::tie(frag, frag_addend) = get_fragment(ctx, rel);
-
-    u64 S = frag ? frag->get_addr(ctx) : sym.get_addr(ctx);
-    u64 A = frag ? frag_addend : (i64)rel.r_addend;
-
    switch (rel.r_type) {
    case R_X86_64_8:
      check(S + A, 0, 1 << 8);
@@ -751,6 +800,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_X86_64_GOTPCREL64:
    case R_X86_64_GOTPCRELX:
    case R_X86_64_REX_GOTPCRELX:
+    case R_X86_64_CODE_4_GOTPCRELX:
      sym.flags |= NEEDS_GOT;
      break;
    case R_X86_64_PLT32:
@@ -779,12 +829,13 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
        ctx.needs_tlsld = true;
      break;
    case R_X86_64_GOTTPOFF:
-      if (ctx.arg.relax && relax_gottpoff(loc - 3) &&
-          sym.is_tprel_linktime_const(ctx)) {
-        // do nothing
-      } else {
+    case R_X86_64_CODE_4_GOTTPOFF:
+      if (!ctx.arg.relax || !sym.is_tprel_linktime_const(ctx) ||
+          !relax_gottpoff(loc, rel))
        sym.flags |= NEEDS_GOTTP;
-      }
+      break;
+    case R_X86_64_CODE_6_GOTTPOFF:
+      sym.flags |= NEEDS_GOTTP;
      break;
    case R_X86_64_TLSDESC_CALL:
      scan_tlsdesc(ctx, sym);
@@ -800,6 +851,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
    case R_X86_64_SIZE32:
    case R_X86_64_SIZE64:
    case R_X86_64_GOTPC32_TLSDESC:
+    case R_X86_64_CODE_4_GOTPC32_TLSDESC:
      break;
    default:
      Error(ctx) << *this << ": unknown relocation: " << rel;
@@ -899,3 +951,5 @@ void rewrite_endbr(Context<E> &ctx) {
 }

 } // namespace mold
+
+#endif
@@ -23,12 +23,11 @@
 // see the contents of libc.a by running `ar t
 // /usr/lib/x86_64-linux-gnu/libc.a`.

-#pragma once
-
-#include "common.h"
+#include "mold.h"

 namespace mold {

+namespace {
 struct ArHdr {
  char ar_name[16];
  char ar_date[12];
@@ -74,10 +73,11 @@ struct ArHdr {
    return {ar_name, sizeof(ar_name)};
  }
 };
+}

-template <typename Context>
+template <typename E>
 std::vector<MappedFile *>
-read_thin_archive_members(Context &ctx, MappedFile *mf) {
+read_thin_archive_members(Context<E> &ctx, MappedFile *mf) {
  u8 *begin = mf->data;
  u8 *data = begin + 8;
  std::vector<MappedFile *> vec;
@@ -115,7 +115,7 @@ read_thin_archive_members(Context &ctx, MappedFile *mf) {
      continue;

    std::string path = name.starts_with('/') ?
-      name : (filepath(mf->name).parent_path() / name).string();
+      name : (path_dirname(mf->name) / name).string();
    vec.push_back(must_open_file(ctx, path));
    vec.back()->thin_parent = mf;
    data = body;
@@ -123,8 +123,9 @@ read_thin_archive_members(Context &ctx, MappedFile *mf) {
  return vec;
 }

-template <typename Context>
-std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &ctx, MappedFile *mf) {
  u8 *begin = mf->data;
  u8 *data = begin + 8;
  std::vector<MappedFile *> vec;
@@ -161,13 +162,27 @@ std::vector<MappedFile *> read_fat_archive_members(Context &ctx, MappedFile *mf)
  return vec;
 }

-template <typename Context>
-std::vector<MappedFile *> read_archive_members(Context &ctx, MappedFile *mf) {
+template <typename E>
+std::vector<MappedFile *>
+read_archive_members(Context<E> &ctx, MappedFile *mf) {
  std::string_view str = mf->get_contents();
+
  if (str.starts_with("!<arch>\n"))
    return read_fat_archive_members(ctx, mf);
+
  assert(str.starts_with("!<thin>\n"));
  return read_thin_archive_members(ctx, mf);
 }

+using E = MOLD_TARGET;
+
+template std::vector<MappedFile *>
+read_thin_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_fat_archive_members(Context<E> &, MappedFile *);
+
+template std::vector<MappedFile *>
+read_archive_members(Context<E> &, MappedFile *);
+
 } // namespace mold
@@ -1,13 +0,0 @@
-#include "mold.h"
-#include "config.h"
-
-namespace mold {
-
-std::string get_mold_version() {
-  if (mold_git_hash.empty())
-    return "mold "s + MOLD_VERSION + " (compatible with GNU ld)";
-  return "mold "s + MOLD_VERSION + " (" + mold_git_hash +
-         "; compatible with GNU ld)";
-}
-
-} // namespace mold
@@ -1,11 +1,12 @@
 #include "elf.h"
+#include <sstream>

 namespace mold {

 static std::string unknown_type(u32 r_type) {
-  char buf[50];
-  snprintf(buf, sizeof(buf), "unknown (0x%x)", r_type);
-  return buf;
+  std::stringstream ss;
+  ss << "unknown (0x" << std::hex << r_type << ")";
+  return ss.str();
 }

 #define CASE(x) case x: return #x
@@ -53,6 +54,15 @@ std::string rel_to_string<X86_64>(u32 r_type) {
  CASE(R_X86_64_IRELATIVE);
  CASE(R_X86_64_GOTPCRELX);
  CASE(R_X86_64_REX_GOTPCRELX);
+  CASE(R_X86_64_CODE_4_GOTPCRELX);
+  CASE(R_X86_64_CODE_4_GOTTPOFF);
+  CASE(R_X86_64_CODE_4_GOTPC32_TLSDESC);
+  CASE(R_X86_64_CODE_5_GOTPCRELX);
+  CASE(R_X86_64_CODE_5_GOTTPOFF);
+  CASE(R_X86_64_CODE_5_GOTPC32_TLSDESC);
+  CASE(R_X86_64_CODE_6_GOTPCRELX);
+  CASE(R_X86_64_CODE_6_GOTTPOFF);
+  CASE(R_X86_64_CODE_6_GOTPC32_TLSDESC);
  }
  return unknown_type(r_type);
 }
@@ -107,7 +117,7 @@ std::string rel_to_string<I386>(u32 r_type) {
 }

 template <>
-std::string rel_to_string<ARM64>(u32 r_type) {
+std::string rel_to_string<ARM64LE>(u32 r_type) {
  switch (r_type) {
  CASE(R_AARCH64_NONE);
  CASE(R_AARCH64_ABS64);
@@ -151,6 +161,7 @@ std::string rel_to_string<ARM64>(u32 r_type) {
  CASE(R_AARCH64_LD64_GOT_LO12_NC);
  CASE(R_AARCH64_LD64_GOTPAGE_LO15);
  CASE(R_AARCH64_PLT32);
+  CASE(R_AARCH64_GOTPCREL32);
  CASE(R_AARCH64_TLSGD_ADR_PREL21);
  CASE(R_AARCH64_TLSGD_ADR_PAGE21);
  CASE(R_AARCH64_TLSGD_ADD_LO12_NC);
@@ -218,7 +229,12 @@ std::string rel_to_string<ARM64>(u32 r_type) {
 }

 template <>
-std::string rel_to_string<ARM32>(u32 r_type) {
+std::string rel_to_string<ARM64BE>(u32 r_type) {
+  return rel_to_string<ARM64LE>(r_type);
+}
+
+template <>
+std::string rel_to_string<ARM32LE>(u32 r_type) {
  switch (r_type) {
  CASE(R_ARM_NONE);
  CASE(R_ARM_PC24);
@@ -359,6 +375,11 @@ std::string rel_to_string<ARM32>(u32 r_type) {
  return unknown_type(r_type);
 }

+template <>
+std::string rel_to_string<ARM32BE>(u32 r_type) {
+  return rel_to_string<ARM32LE>(r_type);
+}
+
 template <>
 std::string rel_to_string<RV64LE>(u32 r_type) {
  switch (r_type) {
@@ -402,7 +423,6 @@ std::string rel_to_string<RV64LE>(u32 r_type) {
  CASE(R_RISCV_ALIGN);
  CASE(R_RISCV_RVC_BRANCH);
  CASE(R_RISCV_RVC_JUMP);
-  CASE(R_RISCV_RVC_LUI);
  CASE(R_RISCV_RELAX);
  CASE(R_RISCV_SUB6);
  CASE(R_RISCV_SET6);
@@ -615,6 +635,7 @@ std::string rel_to_string<PPC64V1>(u32 r_type) {
  CASE(R_PPC64_DTPREL16_HIGH);
  CASE(R_PPC64_DTPREL16_HIGHA);
  CASE(R_PPC64_REL24_NOTOC);
+  CASE(R_PPC64_ENTRY);
  CASE(R_PPC64_PLTSEQ);
  CASE(R_PPC64_PLTCALL);
  CASE(R_PPC64_PLTSEQ_NOTOC);
@@ -857,7 +878,7 @@ std::string rel_to_string<M68K>(u32 r_type) {
 }

 template <>
-std::string rel_to_string<SH4>(u32 r_type) {
+std::string rel_to_string<SH4LE>(u32 r_type) {
  switch (r_type) {
  CASE(R_SH_NONE);
  CASE(R_SH_DIR32);
@@ -890,6 +911,11 @@ std::string rel_to_string<SH4>(u32 r_type) {
  return unknown_type(r_type);
 }

+template <>
+std::string rel_to_string<SH4BE>(u32 r_type) {
+  return rel_to_string<SH4LE>(r_type);
+}
+
 template <>
 std::string rel_to_string<LOONGARCH64>(u32 r_type) {
  switch (r_type) {
@@ -11,8 +11,10 @@ namespace mold {

 struct X86_64;
 struct I386;
-struct ARM64;
-struct ARM32;
+struct ARM64LE;
+struct ARM64BE;
+struct ARM32LE;
+struct ARM32BE;
 struct RV64LE;
 struct RV64BE;
 struct RV32LE;
@@ -23,7 +25,8 @@ struct PPC64V2;
 struct S390X;
 struct SPARC64;
 struct M68K;
-struct SH4;
+struct SH4LE;
+struct SH4BE;
 struct LOONGARCH64;
 struct LOONGARCH32;

@@ -49,10 +52,6 @@ std::ostream &operator<<(std::ostream &out, const ElfRel<E> &rel) {
  return out;
 }

-enum : u32 {
-  R_NONE = 0,
-};
-
 enum : u32 {
  SHN_UNDEF = 0,
  SHN_LORESERVE = 0xff00,
@@ -80,8 +79,13 @@ enum : u32 {
  SHT_GROUP = 17,
  SHT_SYMTAB_SHNDX = 18,
  SHT_RELR = 19,
+  SHT_CREL = 0x40000014,
  SHT_LOOS = 0x60000000,
+  SHT_ANDROID_REL = 0x60000001,
+  SHT_ANDROID_RELA = 0x60000002,
  SHT_LLVM_ADDRSIG = 0x6fff4c03,
+  SHT_ANDROID_RELR = 0x6fffff00,
+  SHT_GNU_SFRAME = 0x6ffffff4,
  SHT_GNU_HASH = 0x6ffffff6,
  SHT_GNU_VERDEF = 0x6ffffffd,
  SHT_GNU_VERNEED = 0x6ffffffe,
@@ -191,6 +195,7 @@ enum : u32 {
  PT_GNU_STACK = 0x6474e551,
  PT_GNU_RELRO = 0x6474e552,
  PT_GNU_PROPERTY = 0x6474e553,
+  PT_GNU_SFRAME = 0x6474e554,
  PT_OPENBSD_RANDOMIZE = 0x65a3dbe6,
  PT_ARM_EXIDX = 0x70000001,
  PT_RISCV_ATTRIBUTES = 0x70000003,
@@ -285,7 +290,16 @@ enum : u32 {
  DT_RELRSZ = 35,
  DT_RELR = 36,
  DT_RELRENT = 37,
+  DT_ANDROID_REL = 0x6000000f,
+  DT_ANDROID_RELSZ = 0x60000010,
+  DT_ANDROID_RELA = 0x60000011,
+  DT_ANDROID_RELASZ = 0x60000012,
+  DT_ANDROID_RELR = 0x6fffe000,
+  DT_ANDROID_RELRSZ = 0x6fffe001,
+  DT_ANDROID_RELRENT = 0x6fffe003,
  DT_GNU_HASH = 0x6ffffef5,
+  DT_DEPAUDIT = 0x6ffffefb,
+  DT_AUDIT = 0x6ffffefc,
  DT_VERSYM = 0x6ffffff0,
  DT_RELACOUNT = 0x6ffffff9,
  DT_RELCOUNT = 0x6ffffffa,
@@ -296,6 +310,7 @@ enum : u32 {
  DT_VERNEEDNUM = 0x6fffffff,
  DT_PPC_GOT = 0x70000000,
  DT_PPC64_GLINK = 0x70000000,
+  DT_RISCV_VARIANT_CC = 0x70000001,
  DT_AARCH64_VARIANT_PCS = 0x70000005,
  DT_AUXILIARY = 0x7ffffffd,
  DT_FILTER = 0x7fffffff,
@@ -350,14 +365,16 @@ enum : u32 {
 };

 enum : u32 {
+  ELFCOMPRESS_NONE = 0,
  ELFCOMPRESS_ZLIB = 1,
  ELFCOMPRESS_ZSTD = 2,
 };

 enum : u32 {
-  EF_ARM_ABI_FLOAT_SOFT = 0x00000200,
-  EF_ARM_ABI_FLOAT_HARD = 0x00000400,
-  EF_ARM_EABI_VER5 = 0x05000000,
+  EF_ARM_ABI_FLOAT_SOFT = 0x200,
+  EF_ARM_ABI_FLOAT_HARD = 0x400,
+  EF_ARM_BE8 = 0x800000,
+  EF_ARM_EABI_VER5 = 0x5000000,
 };

 enum : u32 {
@@ -408,6 +425,10 @@ enum : u32 {
 // Relocation types
 //

+enum : u32 {
+  R_NONE = 0,
+};
+
 enum : u32 {
  R_X86_64_NONE = 0,
  R_X86_64_64 = 1,
@@ -449,6 +470,15 @@ enum : u32 {
  R_X86_64_IRELATIVE = 37,
  R_X86_64_GOTPCRELX = 41,
  R_X86_64_REX_GOTPCRELX = 42,
+  R_X86_64_CODE_4_GOTPCRELX = 43,
+  R_X86_64_CODE_4_GOTTPOFF = 44,
+  R_X86_64_CODE_4_GOTPC32_TLSDESC = 45,
+  R_X86_64_CODE_5_GOTPCRELX = 46,
+  R_X86_64_CODE_5_GOTTPOFF = 47,
+  R_X86_64_CODE_5_GOTPC32_TLSDESC = 48,
+  R_X86_64_CODE_6_GOTPCRELX = 49,
+  R_X86_64_CODE_6_GOTTPOFF = 50,
+  R_X86_64_CODE_6_GOTPC32_TLSDESC = 51,
 };

 enum : u32 {
@@ -539,6 +569,7 @@ enum : u32 {
  R_AARCH64_LD64_GOT_LO12_NC = 0x138,
  R_AARCH64_LD64_GOTPAGE_LO15 = 0x139,
  R_AARCH64_PLT32 = 0x13a,
+  R_AARCH64_GOTPCREL32 = 0x13b,
  R_AARCH64_TLSGD_ADR_PREL21 = 0x200,
  R_AARCH64_TLSGD_ADR_PAGE21 = 0x201,
  R_AARCH64_TLSGD_ADD_LO12_NC = 0x202,
@@ -783,16 +814,13 @@ enum : u32 {
  R_RISCV_ALIGN = 43,
  R_RISCV_RVC_BRANCH = 44,
  R_RISCV_RVC_JUMP = 45,
-  R_RISCV_RVC_LUI = 46,
-  R_RISCV_GPREL_LO12_I = 47,
-  R_RISCV_GPREL_LO12_S = 48,
-  R_RISCV_GPREL_HI20 = 49,
  R_RISCV_RELAX = 51,
  R_RISCV_SUB6 = 52,
  R_RISCV_SET6 = 53,
  R_RISCV_SET8 = 54,
  R_RISCV_SET16 = 55,
  R_RISCV_SET32 = 56,
+  R_RISCV_GOT32_PCREL = 41,
  R_RISCV_32_PCREL = 57,
  R_RISCV_IRELATIVE = 58,
  R_RISCV_PLT32 = 59,
@@ -976,6 +1004,7 @@ enum : u32 {
  R_PPC64_DTPREL16_HIGH = 114,
  R_PPC64_DTPREL16_HIGHA = 115,
  R_PPC64_REL24_NOTOC = 116,
+  R_PPC64_ENTRY = 118,
  R_PPC64_PLTSEQ = 119,
  R_PPC64_PLTCALL = 120,
  R_PPC64_PLTSEQ_NOTOC = 121,
@@ -1350,6 +1379,16 @@ enum : u32 {
  R_LARCH_TLS_DESC_PCREL20_S2 = 126,
 };

+// Returns true if a given relocation is of type used for direct
+// function call.
+template <typename E>
+inline bool is_func_call_rel(const ElfRel<E> &r) {
+  for (u32 r_type : E::R_FUNCALL)
+    if (r.r_type == r_type)
+      return true;
+  return false;
+}
+
 //
 // DWARF data types
 //
@@ -1464,7 +1503,53 @@ template <typename E> using U32 = std::conditional_t<E::is_le, ul32, ub32>;
 template <typename E> using U64 = std::conditional_t<E::is_le, ul64, ub64>;

 template <typename E> using Word = std::conditional_t<E::is_64, U64<E>, U32<E>>;
-template <typename E> using IWord = std::conditional_t<E::is_64, I64<E>, I32<E>>;
+template <typename E> using SWord = std::conditional_t<E::is_64, I64<E>, I32<E>>;
+
+//
+// SFrame is a simple unwind information format used as a lightweight
+// alternative to .eh_frame. A .sframe section consists of a header, an
+// array of Function Descriptor Entries (FDEs) sorted by PC, and a blob
+// of Frame Row Entries (FREs). mold understands SFrame Version 3.
+//
+// https://sourceware.org/binutils/docs/sframe-spec.html
+//
+
+static constexpr u16 SFRAME_MAGIC = 0xdee2;
+
+static constexpr u8 SFRAME_F_FDE_SORTED = 0x1;
+static constexpr u8 SFRAME_F_FRAME_POINTER = 0x2;
+static constexpr u8 SFRAME_F_FDE_FUNC_START_PCREL = 0x4;
+
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_BIG = 1;
+static constexpr u8 SFRAME_ABI_AARCH64_ENDIAN_LITTLE = 2;
+static constexpr u8 SFRAME_ABI_AMD64_ENDIAN_LITTLE = 3;
+static constexpr u8 SFRAME_ABI_S390X_ENDIAN_BIG = 4;
+
+template <typename E>
+struct SFrameHeader {
+  U16<E> magic;
+  u8 version;
+  u8 flags;
+  u8 abi_arch;
+  i8 cfa_fixed_fp_offset;
+  i8 cfa_fixed_ra_offset;
+  u8 auxhdr_len;
+  U32<E> num_fdes;
+  U32<E> num_fres;
+  U32<E> fre_len;
+  U32<E> fdeoff;
+  U32<E> freoff;
+};
+
+// The index part of an SFrame Version 3 FDE. The func_start_offset field
+// is PC-relative (relative to its own address) when the section flag
+// SFRAME_F_FDE_FUNC_START_PCREL is set, which is how mold always emits it.
+template <typename E>
+struct SFrameFdeIdx {
+  I64<E> func_start_offset;
+  U32<E> func_size;
+  U32<E> func_start_fre_off;
+};

 template <typename E> requires E::is_64
 struct ElfSym<E> {
@@ -1474,18 +1559,41 @@ struct ElfSym<E> {
  bool is_weak() const { return st_bind == STB_WEAK; }
  bool is_undef_weak() const { return is_undef() && is_weak(); }

+  bool ppc64_preserves_r2() const { return ppc64_local_entry != 1; }
+  bool ppc64_uses_toc() const { return ppc64_local_entry > 1; }
+
  U32<E> st_name;

 #ifdef __LITTLE_ENDIAN__
  u8 st_type : 4;
  u8 st_bind : 4;
-  u8 st_visibility : 2;
-  u8 : 6;
+  union {
+    u8 st_visibility : 2;
+    struct {
+      u8 : 7;
+      u8 arm64_variant_pcs : 1;
+    };
+    struct {
+      u8 : 7;
+      u8 riscv_variant_cc : 1;
+    };
+    struct {
+      u8 : 5;
+      u8 ppc64_local_entry : 3;
+    };
+  };
 #else
  u8 st_bind : 4;
  u8 st_type : 4;
-  u8 : 6;
-  u8 st_visibility : 2;
+  union {
+    struct {
+      u8 : 6;
+      u8 st_visibility : 2;
+    };
+    u8 arm64_variant_pcs : 1;
+    u8 riscv_variant_cc : 1;
+    u8 ppc64_local_entry : 3;
+  };
 #endif

  U16<E> st_shndx;
@@ -1508,13 +1616,23 @@ struct ElfSym<E> {
 #ifdef __LITTLE_ENDIAN__
  u8 st_type : 4;
  u8 st_bind : 4;
-  u8 st_visibility : 2;
-  u8 : 6;
+  union {
+    u8 st_visibility : 2;
+    struct {
+      u8 : 7;
+      u8 riscv_variant_cc : 1;
+    };
+  };
 #else
  u8 st_bind : 4;
  u8 st_type : 4;
-  u8 : 6;
-  u8 st_visibility : 2;
+  union {
+    struct {
+      u8 : 6;
+      u8 st_visibility : 2;
+    };
+    u8 riscv_variant_cc : 1;
+  };
 #endif

  U16<E> st_shndx;
@@ -1588,7 +1706,7 @@ struct ElfPhdr<E> {
 // We don't want to have too many `if (REL)`s and `if (RELA)`s in our
 // codebase, so ElfRel always takes r_addend as a constructor argument.
 // If it's REL, the argument will simply be ignored.
-template <typename E> requires E::is_le && E::is_rela
+template <typename E> requires (E::is_le && E::is_rela)
 struct ElfRel<E> {
  ElfRel() = default;
  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
@@ -1597,10 +1715,10 @@ struct ElfRel<E> {
  Word<E> r_offset;
  std::conditional_t<E::is_64, U32<E>, u8> r_type;
  std::conditional_t<E::is_64, U32<E>, U24<E>> r_sym;
-  IWord<E> r_addend;
+  SWord<E> r_addend;
 };

-template <typename E> requires (!E::is_le) && E::is_rela
+template <typename E> requires (!E::is_le && E::is_rela)
 struct ElfRel<E> {
  ElfRel() = default;
  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
@@ -1609,10 +1727,10 @@ struct ElfRel<E> {
  Word<E> r_offset;
  std::conditional_t<E::is_64, U32<E>, U24<E>> r_sym;
  std::conditional_t<E::is_64, U32<E>, u8> r_type;
-  IWord<E> r_addend;
+  SWord<E> r_addend;
 };

-template <typename E> requires E::is_le && (!E::is_rela)
+template <typename E> requires (E::is_le && !E::is_rela)
 struct ElfRel<E> {
  ElfRel() = default;
  ElfRel(u64 offset, u32 type, u32 sym, i64 addend = 0)
@@ -1623,7 +1741,7 @@ struct ElfRel<E> {
  std::conditional_t<E::is_64, U32<E>, U24<E>> r_sym;
 };

-template <typename E> requires (!E::is_le) && (!E::is_rela)
+template <typename E> requires (!E::is_le && !E::is_rela)
 struct ElfRel<E> {
  ElfRel() = default;
  ElfRel(u64 offset, u32 type, u32 sym, i64 addend = 0)
@@ -1634,16 +1752,6 @@ struct ElfRel<E> {
  std::conditional_t<E::is_64, U32<E>, u8> r_type;
 };

-// Returns true if a given relocation is of type used for direct
-// function call.
-template <typename E>
-inline bool is_func_call_rel(const ElfRel<E> &r) {
-  for (u32 r_type : E::R_FUNCALL)
-    if (r.r_type == r_type)
-      return true;
-  return false;
-}
-
 template <typename E>
 struct ElfDyn {
  Word<E> d_tag;
@@ -1711,67 +1819,6 @@ struct ElfNhdr {
 // Target-specific ELF data types
 //

-template <>
-struct ElfSym<ARM64> {
-  bool is_undef() const { return st_shndx == SHN_UNDEF; }
-  bool is_abs() const { return st_shndx == SHN_ABS; }
-  bool is_common() const { return st_shndx == SHN_COMMON; }
-  bool is_weak() const { return st_bind == STB_WEAK; }
-  bool is_undef_weak() const { return is_undef() && is_weak(); }
-
-  ul32 st_name;
-
-#ifdef __LITTLE_ENDIAN__
-  u8 st_type : 4;
-  u8 st_bind : 4;
-  u8 st_visibility : 2;
-  u8 : 5;
-  u8 arm64_variant_pcs : 1; // ARM64-specific
-#else
-  u8 st_bind : 4;
-  u8 st_type : 4;
-  u8 arm64_variant_pcs : 1;
-  u8 : 5;
-  u8 st_visibility : 2;
-#endif
-
-  ul16 st_shndx;
-  ul64 st_value;
-  ul64 st_size;
-};
-
-template <>
-struct ElfSym<PPC64V2> {
-  bool is_undef() const { return st_shndx == SHN_UNDEF; }
-  bool is_abs() const { return st_shndx == SHN_ABS; }
-  bool is_common() const { return st_shndx == SHN_COMMON; }
-  bool is_weak() const { return st_bind == STB_WEAK; }
-  bool is_undef_weak() const { return is_undef() && is_weak(); }
-
-  bool preserves_r2() const { return ppc_local_entry != 1; }
-  bool uses_toc() const { return ppc_local_entry > 1; }
-
-  ul32 st_name;
-
-#ifdef __LITTLE_ENDIAN__
-  u8 st_type : 4;
-  u8 st_bind : 4;
-  u8 st_visibility : 2;
-  u8 : 3;
-  u8 ppc_local_entry : 3; // This is PPC64V2-specific
-#else
-  u8 st_bind : 4;
-  u8 st_type : 4;
-  u8 ppc_local_entry : 3;
-  u8 : 3;
-  u8 st_visibility : 2;
-#endif
-
-  ul16 st_shndx;
-  ul64 st_value;
-  ul64 st_size;
-};
-
 template <>
 struct ElfRel<SPARC64> {
  ElfRel() = default;
@@ -1787,7 +1834,7 @@ struct ElfRel<SPARC64> {
 };

 template <>
-struct ElfRel<SH4> {
+struct ElfRel<SH4LE> {
  ElfRel() = default;

  // Addend is ignored except for base relocations because even though
@@ -1802,18 +1849,33 @@ struct ElfRel<SH4> {
  il32 r_addend;
 };

+template <>
+struct ElfRel<SH4BE> {
+  ElfRel() = default;
+  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
+    : r_offset(offset), r_sym(sym), r_type(type), r_addend(sym ? 0 : addend) {}
+
+  ub32 r_offset;
+  ub24 r_sym;
+  u8 r_type;
+  ib32 r_addend;
+};
+
 //
 // Machine descriptions
 //

 template <typename E> concept supports_ifunc = requires { E::R_IRELATIVE; };
 template <typename E> concept supports_tlsdesc = requires { E::R_TLSDESC; };
+template <typename E> concept supports_sframe = requires { E::sframe_abi; };
 template <typename E> concept needs_thunk = requires { E::thunk_size; };

 template <typename E> concept is_x86_64 = std::same_as<E, X86_64>;
 template <typename E> concept is_i386 = std::same_as<E, I386>;
-template <typename E> concept is_arm64 = std::same_as<E, ARM64>;
-template <typename E> concept is_arm32 = std::same_as<E, ARM32>;
+template <typename E> concept is_arm64le = std::same_as<E, ARM64LE>;
+template <typename E> concept is_arm64be = std::same_as<E, ARM64BE>;
+template <typename E> concept is_arm32le = std::same_as<E, ARM32LE>;
+template <typename E> concept is_arm32be = std::same_as<E, ARM32BE>;
 template <typename E> concept is_rv64le = std::same_as<E, RV64LE>;
 template <typename E> concept is_rv64be = std::same_as<E, RV64BE>;
 template <typename E> concept is_rv32le = std::same_as<E, RV32LE>;
@@ -1824,11 +1886,14 @@ template <typename E> concept is_ppc64v2 = std::same_as<E, PPC64V2>;
 template <typename E> concept is_s390x = std::same_as<E, S390X>;
 template <typename E> concept is_sparc64 = std::same_as<E, SPARC64>;
 template <typename E> concept is_m68k = std::same_as<E, M68K>;
-template <typename E> concept is_sh4 = std::same_as<E, SH4>;
+template <typename E> concept is_sh4le = std::same_as<E, SH4LE>;
+template <typename E> concept is_sh4be = std::same_as<E, SH4BE>;
 template <typename E> concept is_loongarch64 = std::same_as<E, LOONGARCH64>;
 template <typename E> concept is_loongarch32 = std::same_as<E, LOONGARCH32>;

 template <typename E> concept is_x86 = is_x86_64<E> || is_i386<E>;
+template <typename E> concept is_arm32 = is_arm32le<E> || is_arm32be<E>;
+template <typename E> concept is_arm64 = is_arm64le<E> || is_arm64be<E>;
 template <typename E> concept is_arm = is_arm64<E> || is_arm32<E>;
 template <typename E> concept is_rv64 = is_rv64le<E> || is_rv64be<E>;
 template <typename E> concept is_rv32 = is_rv32le<E> || is_rv32be<E>;
@@ -1836,19 +1901,21 @@ template <typename E> concept is_riscv = is_rv64<E> || is_rv32<E>;
 template <typename E> concept is_ppc64 = is_ppc64v1<E> || is_ppc64v2<E>;
 template <typename E> concept is_ppc = is_ppc64<E> || is_ppc32<E>;
 template <typename E> concept is_sparc = is_sparc64<E>;
+template <typename E> concept is_sh4 = is_sh4le<E> || is_sh4be<E>;
 template <typename E> concept is_loongarch = is_loongarch64<E> || is_loongarch32<E>;

 struct X86_64 {
-  static constexpr std::string_view target_name = "x86_64";
+  static constexpr std::string_view name = "x86_64";
  static constexpr bool is_64 = true;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
  static constexpr u32 page_size = 4096;
  static constexpr u32 e_machine = EM_X86_64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AMD64_ENDIAN_LITTLE;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u8 trap[] = { 0xcc }; // int3

  static constexpr u32 R_COPY = R_X86_64_COPY;
  static constexpr u32 R_GLOB_DAT = R_X86_64_GLOB_DAT;
@@ -1860,11 +1927,12 @@ struct X86_64 {
  static constexpr u32 R_TPOFF = R_X86_64_TPOFF64;
  static constexpr u32 R_DTPMOD = R_X86_64_DTPMOD64;
  static constexpr u32 R_TLSDESC = R_X86_64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_X86_64_PC64;
  static constexpr u32 R_FUNCALL[] = { R_X86_64_PLT32, R_X86_64_PLTOFF64 };
 };

 struct I386 {
-  static constexpr std::string_view target_name = "i386";
+  static constexpr std::string_view name = "i386";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = false;
@@ -1872,8 +1940,8 @@ struct I386 {
  static constexpr u32 e_machine = EM_386;
  static constexpr u32 plt_hdr_size = 16;
  static constexpr u32 plt_size = 16;
-  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0xcc }; // int3
+  static constexpr u32 pltgot_size = 8;
+  static constexpr u8 trap[] = { 0xcc }; // int3

  static constexpr u32 R_COPY = R_386_COPY;
  static constexpr u32 R_GLOB_DAT = R_386_GLOB_DAT;
@@ -1888,19 +1956,20 @@ struct I386 {
  static constexpr u32 R_FUNCALL[] = { R_386_PLT32 };
 };

-struct ARM64 {
-  static constexpr std::string_view target_name = "arm64";
+struct ARM64LE {
+  static constexpr std::string_view name = "arm64";
  static constexpr bool is_64 = true;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
  static constexpr u32 page_size = 65536;
  static constexpr u32 e_machine = EM_AARCH64;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_LITTLE;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
  static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk
+  static constexpr u32 thunk_size = 32;
+  static constexpr u8 trap[] = { 0x00, 0x7d, 0x20, 0xd4 }; // brk

  static constexpr u32 R_COPY = R_AARCH64_COPY;
  static constexpr u32 R_GLOB_DAT = R_AARCH64_GLOB_DAT;
@@ -1912,22 +1981,29 @@ struct ARM64 {
  static constexpr u32 R_TPOFF = R_AARCH64_TLS_TPREL64;
  static constexpr u32 R_DTPMOD = R_AARCH64_TLS_DTPMOD64;
  static constexpr u32 R_TLSDESC = R_AARCH64_TLSDESC;
+  static constexpr u32 R_SFRAME = R_AARCH64_PREL64;
  static constexpr u32 R_FUNCALL[] = { R_AARCH64_JUMP26, R_AARCH64_CALL26 };
 };

-struct ARM32 {
-  static constexpr std::string_view target_name = "arm32";
+struct ARM64BE : ARM64LE {
+  static constexpr std::string_view name = "arm64be";
+  static constexpr bool is_le = false;
+  static constexpr u8 sframe_abi = SFRAME_ABI_AARCH64_ENDIAN_BIG;
+};
+
+struct ARM32LE {
+  static constexpr std::string_view name = "arm32";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = false;
-  static constexpr u32 page_size = 4096;
+  static constexpr u32 page_size = 65536;
  static constexpr u32 e_machine = EM_ARM;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
  static constexpr u32 thunk_hdr_size = 16;
  static constexpr u32 thunk_size = 16;
-  static constexpr u8 filler[] = { 0xff, 0xde }; // udf
+  static constexpr u8 trap[] = { 0xff, 0xde }; // udf

  static constexpr u32 R_COPY = R_ARM_COPY;
  static constexpr u32 R_GLOB_DAT = R_ARM_GLOB_DAT;
@@ -1945,15 +2021,22 @@ struct ARM32 {
  };
 };

-struct RV64 {
+struct ARM32BE : ARM32LE {
+  static constexpr std::string_view name = "arm32be";
+  static constexpr bool is_le = false;
+};
+
+struct RV64LE {
+  static constexpr std::string_view name = "riscv64";
  static constexpr bool is_64 = true;
+  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
  static constexpr u32 page_size = 4096;
  static constexpr u32 e_machine = EM_RISCV;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak

  static constexpr u32 R_COPY = R_RISCV_COPY;
  static constexpr u32 R_GLOB_DAT = R_RISCV_64;
@@ -1968,25 +2051,22 @@ struct RV64 {
  static constexpr u32 R_FUNCALL[] = { R_RISCV_CALL, R_RISCV_CALL_PLT };
 };

-struct RV64LE : RV64 {
-  static constexpr std::string_view target_name = "riscv64";
-  static constexpr bool is_le = true;
-};
-
-struct RV64BE : RV64 {
-  static constexpr std::string_view target_name = "riscv64be";
+struct RV64BE : RV64LE {
+  static constexpr std::string_view name = "riscv64be";
  static constexpr bool is_le = false;
 };

-struct RV32 {
+struct RV32LE {
+  static constexpr std::string_view name = "riscv32";
  static constexpr bool is_64 = false;
+  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
  static constexpr u32 page_size = 4096;
  static constexpr u32 e_machine = EM_RISCV;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x02, 0x90 }; // c.ebreak
+  static constexpr u8 trap[] = { 0x02, 0x90 }; // c.ebreak

  static constexpr u32 R_COPY = R_RISCV_COPY;
  static constexpr u32 R_GLOB_DAT = R_RISCV_32;
@@ -2001,18 +2081,13 @@ struct RV32 {
  static constexpr u32 R_FUNCALL[] = { R_RISCV_CALL, R_RISCV_CALL_PLT };
 };

-struct RV32LE : RV32 {
-  static constexpr std::string_view target_name = "riscv32";
-  static constexpr bool is_le = true;
-};
-
-struct RV32BE : RV32 {
-  static constexpr std::string_view target_name = "riscv32be";
+struct RV32BE : RV32LE {
+  static constexpr std::string_view name = "riscv32be";
  static constexpr bool is_le = false;
 };

 struct PPC32 {
-  static constexpr std::string_view target_name = "ppc32";
+  static constexpr std::string_view name = "ppc32";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = false;
  static constexpr bool is_rela = true;
@@ -2023,7 +2098,7 @@ struct PPC32 {
  static constexpr u32 pltgot_size = 36;
  static constexpr u32 thunk_hdr_size = 0;
  static constexpr u32 thunk_size = 36;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap

  static constexpr u32 R_COPY = R_PPC_COPY;
  static constexpr u32 R_GLOB_DAT = R_PPC_GLOB_DAT;
@@ -2059,37 +2134,38 @@ struct PPC64 {
 };

 struct PPC64V1 : PPC64 {
-  static constexpr std::string_view target_name = "ppc64v1";
+  static constexpr std::string_view name = "ppc64v1";
  static constexpr bool is_le = false;
  static constexpr u32 plt_hdr_size = 44;
  static constexpr u32 pltgot_size = 0;
  static constexpr u32 thunk_hdr_size = 0;
  static constexpr u32 thunk_size = 28;
-  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
+  static constexpr u8 trap[] = { 0x7f, 0xe0, 0x00, 0x08 }; // trap
 };

 struct PPC64V2 : PPC64 {
-  static constexpr std::string_view target_name = "ppc64v2";
+  static constexpr std::string_view name = "ppc64v2";
  static constexpr bool is_le = true;
  static constexpr u32 plt_hdr_size = 52;
  static constexpr u32 plt_size = 4;
  static constexpr u32 pltgot_size = 0;
  static constexpr u32 thunk_hdr_size = 0;
  static constexpr u32 thunk_size = 24;
-  static constexpr u8 filler[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
+  static constexpr u8 trap[] = { 0x08, 0x00, 0xe0, 0x7f }; // trap
 };

 struct S390X {
-  static constexpr std::string_view target_name = "s390x";
+  static constexpr std::string_view name = "s390x";
  static constexpr bool is_64 = true;
  static constexpr bool is_le = false;
  static constexpr bool is_rela = true;
  static constexpr u32 page_size = 4096;
  static constexpr u32 e_machine = EM_S390X;
+  static constexpr u8 sframe_abi = SFRAME_ABI_S390X_ENDIAN_BIG;
  static constexpr u32 plt_hdr_size = 48;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x07, 0x00 }; // nop
+  static constexpr u8 trap[] = { 0x00, 0x00 }; // invalid

  static constexpr u32 R_COPY = R_390_COPY;
  static constexpr u32 R_GLOB_DAT = R_390_GLOB_DAT;
@@ -2100,11 +2176,12 @@ struct S390X {
  static constexpr u32 R_DTPOFF = R_390_TLS_DTPOFF;
  static constexpr u32 R_TPOFF = R_390_TLS_TPOFF;
  static constexpr u32 R_DTPMOD = R_390_TLS_DTPMOD;
+  static constexpr u32 R_SFRAME = R_390_PC64;
  static constexpr u32 R_FUNCALL[] = { R_390_PLT32DBL };
 };

 struct SPARC64 {
-  static constexpr std::string_view target_name = "sparc64";
+  static constexpr std::string_view name = "sparc64";
  static constexpr bool is_64 = true;
  static constexpr bool is_le = false;
  static constexpr bool is_rela = true;
@@ -2113,7 +2190,7 @@ struct SPARC64 {
  static constexpr u32 plt_hdr_size = 128;
  static constexpr u32 plt_size = 32;
  static constexpr u32 pltgot_size = 32;
-  static constexpr u8 filler[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5
+  static constexpr u8 trap[] = { 0x91, 0xd0, 0x20, 0x05 }; // ta 5

  static constexpr u32 R_COPY = R_SPARC_COPY;
  static constexpr u32 R_GLOB_DAT = R_SPARC_GLOB_DAT;
@@ -2128,7 +2205,7 @@ struct SPARC64 {
 };

 struct M68K {
-  static constexpr std::string_view target_name = "m68k";
+  static constexpr std::string_view name = "m68k";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = false;
  static constexpr bool is_rela = true;
@@ -2137,7 +2214,7 @@ struct M68K {
  static constexpr u32 plt_hdr_size = 18;
  static constexpr u32 plt_size = 14;
  static constexpr u32 pltgot_size = 8;
-  static constexpr u8 filler[] = { 0x4a, 0xfc }; // illegal
+  static constexpr u8 trap[] = { 0x4a, 0xfc }; // illegal

  static constexpr u32 R_COPY = R_68K_COPY;
  static constexpr u32 R_GLOB_DAT = R_68K_GLOB_DAT;
@@ -2150,8 +2227,8 @@ struct M68K {
  static constexpr u32 R_FUNCALL[] = { R_68K_PLT32 };
 };

-struct SH4 {
-  static constexpr std::string_view target_name = "sh4";
+struct SH4LE {
+  static constexpr std::string_view name = "sh4";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
@@ -2160,7 +2237,7 @@ struct SH4 {
  static constexpr u32 plt_hdr_size = 16;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 12;
-  static constexpr u8 filler[] = { 0x00, 0x90 }; // nop
+  static constexpr u8 trap[] = { 0xfd, 0xff }; // illegal

  static constexpr u32 R_COPY = R_SH_COPY;
  static constexpr u32 R_GLOB_DAT = R_SH_GLOB_DAT;
@@ -2173,17 +2250,23 @@ struct SH4 {
  static constexpr u32 R_FUNCALL[] = { R_SH_PLT32 };
 };

+struct SH4BE : SH4LE {
+  static constexpr std::string_view name = "sh4be";
+  static constexpr bool is_le = false;
+  static constexpr u8 trap[] = { 0xff, 0xfd }; // illegal
+};
+
 struct LOONGARCH64 {
-  static constexpr std::string_view target_name = "loongarch64";
+  static constexpr std::string_view name = "loongarch64";
  static constexpr bool is_64 = true;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
-  static constexpr u32 page_size = 16384;
+  static constexpr u32 page_size = 65536;
  static constexpr u32 e_machine = EM_LOONGARCH;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0

  static constexpr u32 R_COPY = R_LARCH_COPY;
  static constexpr u32 R_GLOB_DAT = R_LARCH_64;
@@ -2199,16 +2282,16 @@ struct LOONGARCH64 {
 };

 struct LOONGARCH32 {
-  static constexpr std::string_view target_name = "loongarch32";
+  static constexpr std::string_view name = "loongarch32";
  static constexpr bool is_64 = false;
  static constexpr bool is_le = true;
  static constexpr bool is_rela = true;
-  static constexpr u32 page_size = 16384;
+  static constexpr u32 page_size = 65536;
  static constexpr u32 e_machine = EM_LOONGARCH;
  static constexpr u32 plt_hdr_size = 32;
  static constexpr u32 plt_size = 16;
  static constexpr u32 pltgot_size = 16;
-  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0
+  static constexpr u8 trap[] = { 0x00, 0x00, 0x2a, 0x00 }; // break 0

  static constexpr u32 R_COPY = R_LARCH_COPY;
  static constexpr u32 R_GLOB_DAT = R_LARCH_32;
@@ -0,0 +1,32 @@
+#include "config.h"
+#include "mold.h"
+#include "mold-git-hash.h"
+
+#if MOLD_USE_MIMALLOC
+# include <mimalloc.h>
+#endif
+
+#if MOLD_USE_SYSTEM_MIMALLOC
+// Including mimalloc-new-delete.h overrides the new/delete operators.
+// We need it only when using mimalloc as a dynamic library.
+// This header should be included in only one source file, so we do
+// it in this file.
+# include <mimalloc-new-delete.h>
+#endif
+
+std::string mold::mold_version =
+#ifdef MOLD_GIT_HASH
+  "mold " MOLD_VERSION " (" MOLD_GIT_HASH "; compatible with GNU ld)";
+#else
+  "mold " MOLD_VERSION " (compatible with GNU ld)";
+#endif
+
+int main(int argc, char **argv) {
+#if MOLD_USE_MIMALLOC
+  // Silence mimalloc warnings that users can ignore
+  mi_option_disable(mi_option_verbose);
+  mi_option_disable(mi_option_show_errors);
+#endif
+
+  return mold::mold_main<mold::MOLD_FIRST_TARGET>(argc, argv);
+}
@@ -0,0 +1,55 @@
+#include "mold.h"
+
+namespace mold {
+
+static std::string_view fatal_mono = "mold: fatal: ";
+static std::string_view fatal_color = "mold: \033[0;1;31mfatal:\033[0m ";
+static std::string_view error_mono = "mold: error: ";
+static std::string_view error_color = "mold: \033[0;1;31merror:\033[0m ";
+static std::string_view warning_mono = "mold: warning: ";
+static std::string_view warning_color = "mold: \033[0;1;35mwarning:\033[0m ";
+
+template <typename E>
+Fatal<E>::Fatal(Context<E> &ctx) {
+  out << (ctx.arg.color_diagnostics ? fatal_color : fatal_mono);
+}
+
+template <typename E>
+[[noreturn]] Fatal<E>::~Fatal() {
+  out.emit();
+  cleanup();
+  _exit(1);
+}
+
+template <typename E>
+Error<E>::Error(Context<E> &ctx) {
+  if (ctx.arg.noinhibit_exec) {
+    out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  } else {
+    out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  }
+}
+
+template <typename E>
+Warn<E>::Warn(Context<E> &ctx) {
+  if (ctx.arg.suppress_warnings)
+    return;
+
+  out.emplace(std::cerr);
+
+  if (ctx.arg.fatal_warnings) {
+    *out << (ctx.arg.color_diagnostics ? error_color : error_mono);
+    ctx.has_error = true;
+  } else {
+    *out << (ctx.arg.color_diagnostics ? warning_color : warning_mono);
+  }
+}
+
+using E = MOLD_TARGET;
+
+template class Fatal<E>;
+template class Error<E>;
+template class Warn<E>;
+
+} // namespace mold
@@ -1,23 +1,8 @@
-#pragma once
-
-#include "../lib/common.h"
-#include "elf.h"
+#include "mold.h"

 namespace mold {

-enum class FileType {
-  UNKNOWN,
-  EMPTY,
-  ELF_OBJ,
-  ELF_DSO,
-  AR,
-  THIN_AR,
-  TEXT,
-  GCC_LTO_OBJ,
-  LLVM_BITCODE,
-};
-
-inline bool is_text_file(MappedFile *mf) {
+static bool is_text_file(MappedFile *mf) {
  auto istext = [](char c) {
    return isprint(c) || c == '\n' || c == '\t';
  };
@@ -28,7 +13,7 @@ inline bool is_text_file(MappedFile *mf) {
 }

 template <typename E>
-inline bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
+static bool is_gcc_lto_obj(MappedFile *mf, bool has_gcc_plugin) {
  const char *data = mf->get_contents().data();
  ElfEhdr<E> &ehdr = *(ElfEhdr<E> *)data;
  ElfShdr<E> *sh_begin = (ElfShdr<E> *)(data + ehdr.e_shoff);
@@ -45,7 +30,7 @@ inline bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
    // the LTO linker plugin is available and falls back as regular
    // objects otherwise. GCC FAT LTO object can be identified by the
    // presence of `.gcc.lto_.symtab` section.
-    if (has_plugin) {
+    if (has_gcc_plugin) {
      std::string_view name = data + shdrs[shstrtab_idx].sh_offset + sec.sh_name;
      if (name.starts_with(".gnu.lto_.symtab."))
        return true;
@@ -83,11 +68,15 @@ inline bool is_gcc_lto_obj(MappedFile *mf, bool has_plugin) {
 template <typename E>
 FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
  std::string_view data = mf->get_contents();
-  bool has_plugin = !ctx.arg.plugin.empty();
-
  if (data.empty())
    return FileType::EMPTY;

+  // GCC FAT LTO objects can be linked as regular ELF objects. If the active
+  // plugin is LLVM's, treat them as regular objects so that we can fall back
+  // to native code instead of routing them through GCC LTO handling.
+  bool has_gcc_plugin = !ctx.arg.plugin.empty() &&
+                        ctx.arg.plugin.find("LLVMgold.") == std::string_view::npos;
+
  if (data.starts_with("\177ELF")) {
    u8 byte_order = ((ElfEhdr<I386> *)data.data())->e_ident[EI_DATA];

@@ -96,10 +85,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {

      if (ehdr.e_type == ET_REL) {
        if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<I386>(mf, has_plugin))
+          if (is_gcc_lto_obj<I386>(mf, has_gcc_plugin))
            return FileType::GCC_LTO_OBJ;
        } else {
-          if (is_gcc_lto_obj<X86_64>(mf, has_plugin))
+          if (is_gcc_lto_obj<X86_64>(mf, has_gcc_plugin))
            return FileType::GCC_LTO_OBJ;
        }
        return FileType::ELF_OBJ;
@@ -112,10 +101,10 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {

      if (ehdr.e_type == ET_REL) {
        if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
-          if (is_gcc_lto_obj<M68K>(mf, has_plugin))
+          if (is_gcc_lto_obj<M68K>(mf, has_gcc_plugin))
            return FileType::GCC_LTO_OBJ;
        } else {
-          if (is_gcc_lto_obj<SPARC64>(mf, has_plugin))
+          if (is_gcc_lto_obj<SPARC64>(mf, has_gcc_plugin))
            return FileType::GCC_LTO_OBJ;
        }
        return FileType::ELF_OBJ;
@@ -140,24 +129,79 @@ FileType get_file_type(Context<E> &ctx, MappedFile *mf) {
  return FileType::UNKNOWN;
 }

-inline std::ostream &operator<<(std::ostream &out, FileType type) {
-  auto to_string = [&] {
-    switch (type) {
-    case FileType::UNKNOWN: return "UNKNOWN";
-    case FileType::EMPTY: return "EMPTY";
-    case FileType::ELF_OBJ: return "ELF_OBJ";
-    case FileType::ELF_DSO: return "ELF_DSO";
-    case FileType::AR: return "AR";
-    case FileType::THIN_AR: return "THIN_AR";
-    case FileType::TEXT: return "TEXT";
-    case FileType::GCC_LTO_OBJ: return "GCC_LTO_OBJ";
-    case FileType::LLVM_BITCODE: return "LLVM_BITCODE";
-    default: return "UNKNOWN";
-    }
-  };
+static std::string_view get_elf_type(u8 *buf) {
+  bool is_le = (buf[EI_DATA] == ELFDATA2LSB);
+  bool is_64 = (buf[EI_CLASS] == ELFCLASS64);

-  out << to_string();
-  return out;
+  auto *ehdr_le = (ElfEhdr<I386> *)buf;
+  auto *ehdr_be = (ElfEhdr<M68K> *)buf;
+
+  switch (is_le ? ehdr_le->e_machine : ehdr_be->e_machine) {
+  case EM_386:
+    return I386::name;
+  case EM_X86_64:
+    return X86_64::name;
+  case EM_ARM:
+    return is_le ? ARM32LE::name : ARM32BE::name;
+  case EM_AARCH64:
+    return is_le ? ARM64LE::name : ARM64BE::name;
+  case EM_RISCV:
+    if (is_le)
+      return is_64 ? RV64LE::name : RV32LE::name;
+    return is_64 ? RV64BE::name : RV32BE::name;
+  case EM_PPC:
+    return PPC32::name;
+  case EM_PPC64:
+    return is_le ? PPC64V2::name : PPC64V1::name;
+  case EM_S390X:
+    return S390X::name;
+  case EM_SPARC64:
+    return SPARC64::name;
+  case EM_68K:
+    return M68K::name;
+  case EM_SH:
+    return is_le ? SH4LE::name : SH4BE::name;
+  case EM_LOONGARCH:
+    return is_64 ? LOONGARCH64::name : LOONGARCH32::name;
+  default:
+    return "";
+  }
 }

+// Read the beginning of a given file and returns its machine type
+// (e.g. EM_X86_64 or EM_386).
+template <typename E>
+std::string_view
+get_machine_type(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
+  switch (get_file_type(ctx, mf)) {
+  case FileType::ELF_OBJ:
+  case FileType::ELF_DSO:
+  case FileType::GCC_LTO_OBJ:
+    return get_elf_type(mf->data);
+  case FileType::AR:
+    for (MappedFile *child : read_fat_archive_members(ctx, mf))
+      if (FileType ty = get_file_type(ctx, child);
+          ty == FileType::ELF_OBJ || ty == FileType::GCC_LTO_OBJ)
+        return get_elf_type(child->data);
+    return "";
+  case FileType::THIN_AR:
+    for (MappedFile *child : read_thin_archive_members(ctx, mf))
+      if (FileType ty = get_file_type(ctx, child);
+          ty == FileType::ELF_OBJ || ty == FileType::GCC_LTO_OBJ)
+        return get_elf_type(child->data);
+    return "";
+  case FileType::TEXT:
+    return Script(ctx, mf, &rctx).get_script_output_type();
+  default:
+    return "";
+  }
+}
+
+using E = MOLD_TARGET;
+
+template FileType get_file_type(Context<E> &, MappedFile *);
+
+template std::string_view
+get_machine_type(Context<E> &, ReaderContext &, MappedFile *);
+
 } // namespace mold
@@ -4,32 +4,58 @@

 #include "mold.h"

+#include <fstream>
+#include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/parallel_for_each.h>

 namespace mold {

 template <typename E>
-static bool should_keep(const InputSection<E> &isec) {
+static bool should_keep(Context<E> &ctx, const InputSection<E> &isec) {
  u32 type = isec.shdr().sh_type;
  u32 flags = isec.shdr().sh_flags;
-  std::string_view name = isec.name();
-
  if constexpr (is_ppc32<E>)
-    if (name == ".got2")
+    if (isec.name == ".got2")
      return true;

+  // KEEP'ed by a linker script
+  if (!isec.file.script_match.empty()) {
+    i32 rank = isec.file.script_match[isec.shndx];
+    if (rank != -1 && ctx.script_ranks[rank].keep)
+      return true;
+  }
+
  return (flags & SHF_GNU_RETAIN) ||
         type == SHT_NOTE ||
         type == SHT_INIT_ARRAY ||
         type == SHT_FINI_ARRAY ||
         type == SHT_PREINIT_ARRAY ||
-         (is_arm32<E> && type == SHT_ARM_EXIDX) ||
-         name.starts_with(".ctors") ||
-         name.starts_with(".dtors") ||
-         name.starts_with(".init") ||
-         name.starts_with(".fini") ||
-         is_c_identifier(name);
+         isec.name.starts_with(".ctors") ||
+         isec.name.starts_with(".dtors") ||
+         isec.name.starts_with(".init") ||
+         isec.name.starts_with(".fini");
+}
+
+// Sections whose names are valid C identifiers can be referenced via
+// __start_<name>/__stop_<name> symbols, which the linker synthesizes.
+// Such sections must be kept alive only if such a marker symbol is
+// referenced from a live section. This map lets us find all sections
+// of a given name when we encounter such a reference during marking.
+template <typename E>
+using StartStopMap =
+  tbb::concurrent_unordered_multimap<std::string_view, InputSection<E> *>;
+
+template <typename E>
+static StartStopMap<E> build_start_stop_map(Context<E> &ctx) {
+  StartStopMap<E> map;
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
+      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC) &&
+          is_c_identifier(isec->name))
+        map.insert({isec->name, isec.get()});
+  });
+  return map;
 }

 template <typename E>
@@ -67,11 +93,12 @@ collect_root_set(Context<E> &ctx) {
      // reduce the amount of non-memory-mapped segments, you should
      // use `strip` command, compile without debug info or use
      // --strip-all linker option.
-      u32 flags = isec->shdr().sh_flags;
-      if (!(flags & SHF_ALLOC))
+      if (!(isec->shdr().sh_flags & SHF_ALLOC)) {
        isec->is_visited = true;
+        continue;
+      }

-      if (should_keep(*isec))
+      if (should_keep(ctx, *isec))
        enqueue_section(isec.get());
    }
  });
@@ -95,50 +122,73 @@ collect_root_set(Context<E> &ctx) {
  return rootset;
 }

+static std::string_view start_stop_name(std::string_view sym) {
+  if (sym.starts_with("__start_"))
+    return sym.substr(8);
+  if (sym.starts_with("__stop_"))
+    return sym.substr(7);
+  return "";
+}
+
 template <typename E>
-static void visit(Context<E> &ctx, InputSection<E> *isec,
-                  tbb::feeder<InputSection<E> *> &feeder, i64 depth) {
+static void visit_section(Context<E> &ctx, InputSection<E> *isec,
+                          tbb::feeder<InputSection<E> *> &feeder, i64 depth,
+                          const StartStopMap<E> &start_stop_map) {
  assert(isec->is_visited);

+  // Mark a section alive. For better performacne, we don't call
+  // `feeder.add` too often.
+  auto mark = [&](InputSection<E> *sec) {
+    if (mark_section(sec)) {
+      if (depth < 3)
+        visit_section(ctx, sec, feeder, depth + 1, start_stop_map);
+      else
+        feeder.add(sec);
+    }
+  };
+
  // If this is a text section, .eh_frame may contain records
  // describing how to handle exceptions for that function.
  // We want to keep associated .eh_frame records.
  for (FdeRecord<E> &fde : isec->get_fdes())
    for (const ElfRel<E> &rel : fde.get_rels(isec->file).subspan(1))
      if (Symbol<E> *sym = isec->file.symbols[rel.r_sym])
-        if (mark_section(sym->get_input_section()))
-          feeder.add(sym->get_input_section());
+        mark(sym->get_input_section());

  for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
-    Symbol<E> &sym = *isec->file.symbols[rel.r_sym];
-
    // Symbol can refer to either a section fragment or an input section.
-    // Mark a fragment as alive.
+    Symbol<E> &sym = *isec->file.symbols[rel.r_sym];
    if (SectionFragment<E> *frag = sym.get_frag()) {
      frag->is_alive = true;
      continue;
    }

-    // Mark a section alive. For better performacne, we don't call
-    // `feeder.add` too often.
-    if (mark_section(sym.get_input_section())) {
-      if (depth < 3)
-        visit(ctx, sym.get_input_section(), feeder, depth + 1);
-      else
-        feeder.add(sym.get_input_section());
+    mark(sym.get_input_section());
+
+    // A reference to __start_<name> or __stop_<name> keeps every section
+    // named <name> alive, mirroring how those symbols are defined.
+    if (std::string_view sec = start_stop_name(sym.name());
+        !sec.empty()) {
+      auto [i, end] = start_stop_map.equal_range(sec);
+      for (; i != end; ++i)
+        mark(i->second);
    }
  }
+
+  if constexpr (is_arm32<E>)
+    mark(isec->extra.exidx);
 }

 // Mark all reachable sections
 template <typename E>
 static void mark(Context<E> &ctx,
-                 tbb::concurrent_vector<InputSection<E> *> &rootset) {
+                 tbb::concurrent_vector<InputSection<E> *> &rootset,
+                 const StartStopMap<E> &start_stop_map) {
  Timer t(ctx, "mark");

  tbb::parallel_for_each(rootset, [&](InputSection<E> *isec,
                                      tbb::feeder<InputSection<E> *> &feeder) {
-    visit(ctx, isec, feeder, 0);
+    visit_section(ctx, isec, feeder, 0, start_stop_map);
  });
 }

@@ -146,25 +196,49 @@ static void mark(Context<E> &ctx,
 template <typename E>
 static void sweep(Context<E> &ctx) {
  Timer t(ctx, "sweep");
-  static Counter counter("garbage_sections");

-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+  std::vector<std::vector<InputSection<E> *>> sections(ctx.objs.size());
+
+  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
+    ObjectFile<E> &file = *ctx.objs[i];
+
+    for (std::unique_ptr<InputSection<E>> &isec : file.sections) {
      if (isec && isec->is_alive && !isec->is_visited) {
-        if (ctx.arg.print_gc_sections)
-          Out(ctx) << "removing unused section " << *isec;
        isec->kill();
-        counter++;
+        sections[i].push_back(isec.get());
      }
    }
  });
+
+  std::string &path = ctx.arg.print_gc_sections;
+
+  if (!path.empty()) {
+    std::ostream *out = &std::cout;
+    std::ofstream file;
+
+    if (path != "-") {
+      file.open(path);
+      if (file.fail())
+        Fatal(ctx) << "--print-gc-sections: cannot open " << path << ": "
+                   << errno_string();
+      out = &file;
+    }
+
+    i64 saved_bytes = 0;
+    for (std::span<InputSection<E> *> vec : sections)
+      for (InputSection<E> *isec : vec)
+        *out << "removing unused section " << *isec << '\n';
+
+    *out << "GC saved " << saved_bytes << " bytes\n";
+  }
 }

 template <typename E>
 void gc_sections(Context<E> &ctx) {
  Timer t(ctx, "gc");
  tbb::concurrent_vector<InputSection<E> *> rootset = collect_root_set(ctx);
-  mark(ctx, rootset);
+  StartStopMap<E> start_stop_map = build_start_stop_map(ctx);
+  mark(ctx, rootset, start_stop_map);
  sweep(ctx);
 }

@@ -58,7 +58,6 @@

 #include "mold.h"
 #include <tbb/parallel_for_each.h>
-#include <tbb/parallel_sort.h>

 namespace mold {

@@ -127,16 +126,10 @@ struct SectionHeader {
 };

 struct NameType {
-  bool operator==(const NameType &) const = default;
-
-  bool operator<(const NameType &other) const {
-    return std::tuple(hash, type, name) <
-           std::tuple(other.hash, other.type, other.name);
-  }
-
-  std::string_view name;
+  auto operator<=>(const NameType &) const = default;
  u64 hash;
  u8 type;
+  std::string_view name;
 };

 struct MapValue {
@@ -463,11 +456,8 @@ read_address_ranges(Context<E> &ctx, const Compunit &cu) {
        Fatal(ctx) << "--gdb-index: missing DW_AT_rnglists_base";

      u8 *base = buf + rnglists_base;
-      i64 num_offsets = *(U32<E> *)(base - 4);
      Offset *offsets = (Offset *)base;
-
-      for (i64 i = 0; i < num_offsets; i++)
-        read_rnglist_range<E>(vec, base + offsets[i], addrx, low_pc.value);
+      read_rnglist_range<E>(vec, base + offsets[ranges.value], addrx, low_pc.value);
    }
    return vec;
  }
@@ -539,7 +529,7 @@ static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
    u8 type = *p++;
    std::string_view name = (char *)p;
    p += name.size() + 1;
-    cu->nametypes.push_back({name, hash_string(name), type});
+    cu->nametypes.push_back(NameType{hash_string(name), type, name});
  }

  return size;
@@ -626,8 +616,8 @@ static std::vector<Compunit> read_compunits(Context<E> &ctx) {

  // Uniquify elements because GCC 11 seems to emit one record for each
  // comdat group which results in having a lot of duplicate records.
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    sort(cu.nametypes);
+  tbb::parallel_for_each(cus, [](Compunit &cu) {
+    ranges::stable_sort(cu.nametypes);
    remove_duplicates(cu.nametypes);
  });

@@ -636,8 +626,10 @@ static std::vector<Compunit> read_compunits(Context<E> &ctx) {

 template <typename E>
 std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
-  if (chunk->is_compressed)
-    return chunk->uncompressed_data;
+  if (chunk->is_compressed) {
+    CompressedSection<E> &sec = *(CompressedSection<E> *)chunk;
+    return {sec.uncompressed_data.get(), (size_t)sec.chdr.ch_size};
+  }
  return {ctx.buf + chunk->shdr.sh_offset, (size_t)chunk->shdr.sh_size};
 }

@@ -720,9 +712,11 @@ void write_gdb_index(Context<E> &ctx) {

  i64 bufsize = hdr.const_pool_offset + offset;

-  // Allocate an output buffer
-  ctx.output_file->buf2.resize(bufsize);
-  u8 *buf = ctx.output_file->buf2.data();
+  // Allocate an output buffer. We use malloc instead of vector to
+  // avoid zero-initializing the entire buffer.
+  ctx.output_file->buf2 = (u8 *)malloc(bufsize);
+  ctx.output_file->buf2_size = bufsize;
+  u8 *buf = ctx.output_file->buf2;

  // Write a section header
  memcpy(buf, &hdr, sizeof(hdr));
@@ -737,18 +731,25 @@ void write_gdb_index(Context<E> &ctx) {
  }

  // Write address areas
-  for (i64 i = 0; i < cus.size(); i++) {
-    for (std::pair<u64, u64> range : cus[i].ranges) {
+  std::vector<i64> range_offsets(cus.size());
+  for (i64 i = 1; i < cus.size(); i++)
+    range_offsets[i] = range_offsets[i - 1] + cus[i - 1].ranges.size() * 20;
+
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    i64 i = &cu - cus.data();
+    u8 *p = buf + hdr.ranges_offset + range_offsets[i];
+    for (std::pair<u64, u64> range : cu.ranges) {
      *(ul64 *)p = range.first;
      *(ul64 *)(p + 8) = range.second;
      *(ul32 *)(p + 16) = i;
      p += 20;
    }
-  }
+  });

  // Write a symbol table
  u32 mask = ht_size - 1;
  ul32 *ht = (ul32 *)(buf + hdr.symtab_offset);
+  memset(ht, 0, ht_size * 8);

  for (Entry *ent : entries) {
    u32 hash = ent->value.gdb_hash;
@@ -762,22 +763,35 @@ void write_gdb_index(Context<E> &ctx) {
    ht[j * 2 + 1] = ent->value.type_offset;
  }

-  // Write types
-  for (i64 i = 0; i < cus.size(); i++) {
-    Compunit &cu = cus[i];
-    u8 *base = buf + hdr.const_pool_offset;
+  // Write types. Use MapValue::count as an atomic slot counter.
+  u8 *base = buf + hdr.const_pool_offset;

+  for (Entry *ent : entries)
+    ent->value.count = 0;
+
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    i64 i = &cu - cus.data();
    for (i64 j = 0; j < cu.nametypes.size(); j++) {
-      ul32 *p = (ul32 *)(base + cu.entries[j]->type_offset);
-      i64 idx = ++p[0];
+      MapValue *ent = cu.entries[j];
+      ul32 *p = (ul32 *)(base + ent->type_offset);
+      i64 idx = ++ent->count;
      p[idx] = (cu.nametypes[j].type << 24) | i;
    }
-  }
+  });
+
+  // Write the final counts into the buffer.
+  tbb::parallel_for_each(entries, [&](Entry *ent) {
+    ul32 *p = (ul32 *)(base + ent->value.type_offset);
+    p[0] = ent->value.count;
+    // Sort entries for deterministic output
+    std::sort(p + 1, p + 1 + ent->value.count);
+  });

  // Write names
  tbb::parallel_for_each(entries, [&](Entry *ent) {
-    memcpy(buf + hdr.const_pool_offset + ent->value.name_offset,
-           ent->key, ent->keylen);
+    u8 *dst = buf + hdr.const_pool_offset + ent->value.name_offset;
+    memcpy(dst, ent->key, ent->keylen);
+    dst[ent->keylen] = '\0';
  });

  // Update the section size and rewrite the section header
@@ -69,6 +69,7 @@

 #include <array>
 #include <cstdio>
+#include <fstream>
 #include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
@@ -122,18 +123,24 @@ static void uniquify_cies(Context<E> &ctx) {
 template <typename E>
 static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
  const ElfShdr<E> &shdr = isec.shdr();
-  std::string_view name = isec.name();

  if (shdr.sh_size == 0 || !(shdr.sh_flags & SHF_ALLOC) ||
-      shdr.sh_type == SHT_NOBITS || is_c_identifier(name))
+      shdr.sh_type == SHT_NOBITS || is_c_identifier(isec.name))
    return false;

  if (shdr.sh_flags & SHF_EXECINSTR)
    return (ctx.arg.icf_all || !isec.address_taken) &&
-           name != ".init" && name != ".fini";
+           isec.name != ".init" && isec.name != ".fini";
+
+  // .gcc_except_table contains a compiler-generated table. Pointer
+  // equality for the section is not significant because only the C++
+  // exception handling code will use the table at runtime.
+  if (isec.name == ".gcc_except_table" ||
+      isec.name.starts_with(".gcc_except_table."))
+    return true;

  bool is_readonly = !(shdr.sh_flags & SHF_WRITE);
-  bool is_relro = isec.output_section && isec.output_section->is_relro;
+  bool is_relro = isec.name.starts_with(".data.rel.ro");
  return (ctx.arg.ignore_data_address_equality || !isec.address_taken) &&
         (is_readonly || is_relro);
 }
@@ -192,7 +199,7 @@ static void merge_leaf_nodes(Context<E> &ctx) {
  static Counter non_eligible("icf_non_eligibles");
  static Counter leaf("icf_leaf_nodes");

-  tbb::concurrent_unordered_map<InputSection<E> *, InputSection<E> *,
+  tbb::concurrent_unordered_map<InputSection<E> *, Atomic<InputSection<E> *>,
                                LeafHasher<E>, LeafEq<E>> map;

  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
@@ -209,8 +216,11 @@ static void merge_leaf_nodes(Context<E> &ctx) {
        leaf++;
        isec->icf_leaf = true;
        auto [it, inserted] = map.insert({isec.get(), isec.get()});
-        if (!inserted && isec->get_priority() < it->second->get_priority())
-          it->second = isec.get();
+        if (!inserted) {
+          InputSection<E> *isec2 = it->second.load();
+          while (isec->get_priority() < isec2->get_priority() &&
+                 !it->second.compare_exchange_strong(isec2, isec.get()));
+        }
      } else {
        eligible++;
        isec->icf_eligible = true;
@@ -245,7 +255,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
  auto hash_symbol = [&](Symbol<E> &sym) {
    InputSection<E> *isec = sym.get_input_section();

-    if (!sym.file) {
+    if (!sym.file || sym.is_imported) {
      hash('1');
      hash((u64)&sym);
    } else if (SectionFragment<E> *frag = sym.get_frag()) {
@@ -465,28 +475,36 @@ static void print_icf_sections(Context<E> &ctx) {
  });

  tbb::parallel_sort(leaders.begin(), leaders.end(),
-                     [&](InputSection<E> *a, InputSection<E> *b) {
+                     [](InputSection<E> *a, InputSection<E> *b) {
                       return a->get_priority() < b->get_priority();
                     });

+  std::ostream *out = &std::cout;
+  std::ofstream file;
+
+  std::string &path = ctx.arg.print_icf_sections;
+  if (path != "-") {
+    file.open(path);
+    if (file.fail())
+      Fatal(ctx) << "--print-icf-sections: cannot open " << path << ": "
+                 << errno_string();
+    out = &file;
+  }
+
  i64 saved_bytes = 0;

  for (InputSection<E> *leader : leaders) {
    auto [begin, end] = map.equal_range(leader);
-    if (begin == end)
-      continue;
-
-    Out(ctx) << "selected section " << *leader;
-
-    i64 n = 0;
-    for (auto it = begin; it != end; it++) {
-      Out(ctx) << "  removing identical section " << *it->second;
-      n++;
+    if (begin != end) {
+      *out << "selected section " << *leader << '\n';
+      for (auto it = begin; it != end; it++) {
+        *out << "  removing identical section " << *it->second << '\n';
+        saved_bytes += leader->contents.size();
+      }
    }
-    saved_bytes += leader->contents.size() * n;
  }

-  Out(ctx) << "ICF saved " << saved_bytes << " bytes";
+  *out << "ICF saved " << saved_bytes << " bytes\n";
 }

 template <typename E>
@@ -568,14 +586,19 @@ void icf_sections(Context<E> &ctx) {
  {
    Timer t(ctx, "group");

-    auto *map = new tbb::concurrent_unordered_map<Digest, InputSection<E> *>;
+    auto *map =
+      new tbb::concurrent_unordered_map<Digest, Atomic<InputSection<E> *>>;
+
    std::span<Digest> digest = digests[slot];

    tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
      InputSection<E> *isec = sections[i];
      auto [it, inserted] = map->insert({digest[i], isec});
-      if (!inserted && isec->get_priority() < it->second->get_priority())
-        it->second = isec;
+      if (!inserted) {
+        InputSection<E> *isec2 = it->second.load();
+        while (isec->get_priority() < isec2->get_priority() &&
+               !it->second.compare_exchange_strong(isec2, isec));
+      }
    });

    tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
@@ -588,9 +611,21 @@ void icf_sections(Context<E> &ctx) {
    ctx.on_exit.push_back([=] { delete map; });
  }

-  if (ctx.arg.print_icf_sections)
+  if (!ctx.arg.print_icf_sections.empty())
    print_icf_sections(ctx);

+  // Update alignment of leaders.
+  {
+    Timer t(ctx, "update_alignment");
+    tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
+      for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+        if (isec && isec->is_alive && isec->leader && isec->leader != isec.get()) {
+          update_maximum(isec->leader->p2align, isec->p2align);
+        }
+      }
+    });
+  }
+
  // Eliminate duplicate sections.
  // Symbols pointing to eliminated sections will be redirected on the fly when
  // exporting to the symtab.
@@ -94,9 +94,12 @@ InputFile<E>::InputFile(Context<E> &ctx, MappedFile *mf)
 }

 template <typename E>
-std::span<ElfPhdr<E>> InputFile<E>::get_phdrs() {
-  ElfEhdr<E> &ehdr = get_ehdr();
-  return {(ElfPhdr<E> *)(mf->data + ehdr.e_phoff), ehdr.e_phnum};
+void InputFile<E>::populate_symbol_names() {
+  symbol_names.reserve(elf_syms.size());
+  for (const ElfSym<E> &esym : elf_syms) {
+    const char *p = symbol_strtab.data() + esym.st_name;
+    symbol_names.emplace_back(p, strlen(p));
+  }
 }

 template <typename E>
@@ -116,25 +119,9 @@ std::string_view InputFile<E>::get_source_name() const {
  return "";
 }

-template <typename E>
-ObjectFile<E>::ObjectFile(Context<E> &ctx, MappedFile *mf,
-                          std::string archive_name, bool is_in_lib)
-  : InputFile<E>(ctx, mf), archive_name(archive_name), is_in_lib(is_in_lib) {
-  this->is_alive = !is_in_lib;
-}
-
-template <typename E>
-ObjectFile<E> *
-ObjectFile<E>::create(Context<E> &ctx, MappedFile *mf,
-                      std::string archive_name, bool is_in_lib) {
-  ObjectFile<E> *obj = new ObjectFile<E>(ctx, mf, archive_name, is_in_lib);
-  ctx.obj_pool.emplace_back(obj);
-  return obj;
-}
-
 template <typename E>
 static bool is_debug_section(const ElfShdr<E> &shdr, std::string_view name) {
-  return !(shdr.sh_flags & SHF_ALLOC) && name.starts_with(".debug");
+  return !(shdr.sh_flags & SHF_ALLOC) && name.starts_with(".debug_");
 }

 template <typename E>
@@ -177,7 +164,7 @@ ObjectFile<E>::parse_note_gnu_property(Context<E> &ctx, const ElfShdr<E> &shdr)

 // <format-version>
 // [ <section-length> "vendor-name" <file-tag> <size> <attribute>*]+ ]*
-template <typename E>
+template <is_riscv E>
 static void read_riscv_attributes(Context<E> &ctx, ObjectFile<E> &file,
                                  std::string_view data) {
  if (data.empty())
@@ -252,6 +239,65 @@ static bool is_known_section_type(const ElfShdr<E> &shdr) {
  return false;
 }

+// SHT_CREL is an experimental alternative relocation table format
+// designed to reduce the size of the table. Only LLVM supports it
+// at the moment.
+//
+// This function converts a CREL relocation table to a regular one.
+template <typename E>
+std::vector<ElfRel<E>> decode_crel(Context<E> &ctx, ObjectFile<E> &file,
+                                   const ElfShdr<E> &shdr) {
+  u8 *p = (u8 *)file.get_string(ctx, shdr).data();
+  u64 hdr = read_uleb(&p);
+  i64 nrels = hdr >> 3;
+  bool is_rela = hdr & 0b100;
+  i64 scale = hdr & 0b11;
+
+  if (is_rela && !E::is_rela)
+    Fatal(ctx) << file << ": CREL with addends is not supported for " << E::name;
+
+  u64 offset = 0;
+  i64 type = 0;
+  i64 symidx = 0;
+  i64 addend = 0;
+
+  std::vector<ElfRel<E>> vec;
+  vec.reserve(nrels);
+
+  while (vec.size() < nrels) {
+    u8 flags = *p++;
+    i64 nflags = is_rela ? 3 : 2;
+
+    // The first ULEB-128 encoded value is a concatenation of bit flags and
+    // an offset delta. The delta may be very large to decrease the
+    // current offset value by wrapping around. Combined, the encoded value
+    // can be up to 67 bit long. Thus we can't simply use read_uleb() which
+    // returns a u64.
+    u64 delta;
+    if (flags & 0x80)
+      delta = (read_uleb(&p) << (7 - nflags)) | ((flags & 0x7f) >> nflags);
+    else
+      delta = flags >> nflags;
+    offset += delta << scale;
+
+    if (flags & 1)
+      symidx += read_sleb(&p);
+    if (flags & 2)
+      type += read_sleb(&p);
+    if (is_rela && (flags & 4))
+      addend += read_sleb(&p);
+    vec.emplace_back(offset, type, symidx, addend);
+  }
+  return vec;
+}
+
+template <typename E>
+ComdatGroup *insert_comdat_group(Context<E> &ctx, std::string_view name) {
+  typename decltype(ctx.comdat_groups)::const_accessor acc;
+  ctx.comdat_groups.insert(acc, {name, ComdatGroup()});
+  return const_cast<ComdatGroup *>(&acc->second);
+}
+
 template <typename E>
 void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
  // Read sections
@@ -292,7 +338,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
        signature = this->shstrtab.data() +
                    this->elf_sections[get_shndx(esym)].sh_name;
      } else {
-        signature = this->symbol_strtab.data() + esym.st_name;
+        signature = this->symbol_names[shdr.sh_info];
      }

      // Ignore a broken comdat group GCC emits for .debug_macros.
@@ -310,12 +356,14 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
      if (entries[0] != GRP_COMDAT)
        Fatal(ctx) << *this << ": unsupported SHT_GROUP format";

-      typename decltype(ctx.comdat_groups)::const_accessor acc;
-      ctx.comdat_groups.insert(acc, {signature, ComdatGroup()});
-      ComdatGroup *group = const_cast<ComdatGroup *>(&acc->second);
+      ComdatGroup *group = insert_comdat_group(ctx, signature);
      comdat_groups.push_back({group, (i32)i, entries.subspan(1)});
      break;
    }
+    case SHT_CREL:
+      decoded_crel.resize(i + 1);
+      decoded_crel[i] = decode_crel(ctx, *this, shdr);
+      break;
    case SHT_REL:
    case SHT_RELA:
    case SHT_SYMTAB:
@@ -369,6 +417,11 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
          is_debug_section(shdr, name))
        continue;

+      // Ignore section is specified by --discard-section.
+      if (!ctx.arg.discard_section.empty() &&
+          ctx.arg.discard_section.contains(name))
+        continue;
+
      if (name == ".comment" &&
          this->get_string(ctx, shdr).starts_with("rustc "))
        this->is_rust_obj = true;
@@ -381,6 +434,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
        continue;

      this->sections[i] = std::make_unique<InputSection<E>>(ctx, *this, i);
+      InputSection<E> *isec = this->sections[i].get();

      // Save .llvm_addrsig for --icf=safe.
      if (shdr.sh_type == SHT_LLVM_ADDRSIG && !ctx.arg.relocatable) {
@@ -402,19 +456,21 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
        this->has_ctors = true;

      if (name == ".eh_frame")
-        eh_frame_sections.push_back(this->sections[i].get());
+        eh_frame_sections.push_back(isec);
+
+      if constexpr (supports_sframe<E>)
+        if (name == ".sframe")
+          sframe_sections.push_back(isec);
+
+      if (name == ".debug_info" && !(shdr.sh_flags & SHF_ALLOC))
+        debug_info = isec;

      if constexpr (is_ppc32<E>)
        if (name == ".got2")
-          extra.got2 = this->sections[i].get();
+          extra.got2 = isec;

      // Save debug sections for --gdb-index.
      if (ctx.arg.gdb_index) {
-        InputSection<E> *isec = this->sections[i].get();
-
-        if (name == ".debug_info")
-          debug_info = isec;
-
        // If --gdb-index is given, contents of .debug_gnu_pubnames and
        // .debug_gnu_pubtypes are copied to .gdb_index, so keeping them
        // in an output file is just a waste of space.
@@ -448,18 +504,21 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
  // Attach relocation sections to their target sections.
  for (i64 i = 0; i < this->elf_sections.size(); i++) {
    const ElfShdr<E> &shdr = this->elf_sections[i];
-    if (shdr.sh_type != (E::is_rela ? SHT_RELA : SHT_REL))
-      continue;
-
-    if (shdr.sh_info >= sections.size())
-      Fatal(ctx) << *this << ": invalid relocated section index: "
-                 << (u32)shdr.sh_info;
-
-    if (std::unique_ptr<InputSection<E>> &target = sections[shdr.sh_info]) {
-      assert(target->relsec_idx == -1);
-      target->relsec_idx = i;
+    if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL) ||
+        shdr.sh_type == SHT_CREL) {
+      if (std::unique_ptr<InputSection<E>> &target = sections[shdr.sh_info]) {
+        assert(target->relsec_idx == -1);
+        target->relsec_idx = i;
+      }
    }
  }
+
+  // Attach .arm.exidx sections to their corresponding sections
+  if constexpr (is_arm32<E>)
+    for (std::unique_ptr<InputSection<E>> &isec : this->sections)
+      if (isec && isec->shdr().sh_type == SHT_ARM_EXIDX)
+        if (InputSection<E> *target = sections[isec->shdr().sh_link].get())
+          target->extra.exidx = isec.get();
 }

 // .eh_frame contains data records explaining how to handle exceptions.
@@ -548,6 +607,8 @@ void ObjectFile<E>::parse_ehframe(Context<E> &ctx) {
      i64 cie_offset = *(I32<E> *)(contents.data() + fdes[i].input_offset + 4);
      fdes[i].cie_idx = find_cie(fdes[i].input_offset + 4 - cie_offset);
    }
+
+    isec->is_alive = false;
  }

  auto get_isec = [&](const FdeRecord<E> &fde) {
@@ -556,8 +617,8 @@ void ObjectFile<E>::parse_ehframe(Context<E> &ctx) {

  // We assume that FDEs for the same input sections are contiguous
  // in `fdes` vector.
-  sort(fdes, [&](const FdeRecord<E> &a, const FdeRecord<E> &b) {
-    return get_isec(a)->get_priority() < get_isec(b)->get_priority();
+  ranges::stable_sort(fdes, {}, [&](const FdeRecord<E> &x) {
+    return get_isec(x)->get_priority();
  });

  // Associate FDEs to input sections.
@@ -576,6 +637,96 @@ void ObjectFile<E>::parse_ehframe(Context<E> &ctx) {
  }
 }

+// Returns the byte length of the SFrame FRE block at offset `offset`:
+// a 5-byte attribute header followed by a series of frame row
+// entries, each of which is a start address (whose width is given by
+// the attribute header), a one-byte info field and a number of
+// variable-width data words encoded in that info field.
+template <typename E>
+static i64 sframe_fre_block_size(std::string_view data, i64 offset) {
+  const u8 *attr = (const u8 *)data.data() + offset;
+  i64 num_fres = *(U16<E> *)attr;
+  i64 addr_size = 1 << bits(attr[2], 3, 0);
+
+  i64 p = offset + 5;
+  for (i64 i = 0; i < num_fres; i++) {
+    u8 info = data[p + addr_size];
+    i64 num_words = bits(info, 4, 1);
+    i64 word_size = 1 << bits(info, 6, 5);
+    p += addr_size + 1 + num_words * word_size;
+  }
+  return p - offset;
+}
+
+// .sframe is a compact stack-unwinding format. Just like .eh_frame, the
+// linker has to understand its contents: an output .sframe section is a
+// single header followed by a PC-sorted index of function descriptor
+// entries (FDEs) and a blob of frame row entries (FREs). This function
+// parses an input .sframe section into individual FDEs so that we can
+// later drop the dead ones, sort the survivors and rewrite the header.
+//
+// Unlike .eh_frame, the variable-length FRE data carries no relocations,
+// so we copy it to the output verbatim. The only relocated field is each
+// FDE's func_start, which points to the function the FDE describes; we
+// use it both to find the function (for garbage collection and to obtain
+// its output address) and to re-emit a PC-relative offset.
+template <typename E>
+void ObjectFile<E>::parse_sframe(Context<E> &ctx) requires supports_sframe<E> {
+  for (InputSection<E> *isec : sframe_sections) {
+    // The input section is consumed by the linker; it's not copied to the
+    // output as-is but reconstructed into ctx.sframe.
+    isec->is_alive = false;
+
+    std::string_view data = this->get_string(ctx, isec->shdr());
+    const SFrameHeader<E> &hdr = *(const SFrameHeader<E> *)data.data();
+
+    if (hdr.magic != SFRAME_MAGIC)
+      Fatal(ctx) << *isec << ": corrupted .sframe section";
+    if (hdr.abi_arch != E::sframe_abi)
+      Fatal(ctx) << *isec << ": .sframe is incompatible with " << E::name;
+
+    // We support only SFrame Version 3. A section written in an older
+    // version (e.g. by an old assembler) isn't an error; we just ignore
+    // it, so the functions it covers won't have SFrame info in the output.
+    if (hdr.version != 3)
+      continue;
+
+    i64 hdr_len = sizeof(SFrameHeader<E>) + hdr.auxhdr_len;
+    i64 num_fdes = hdr.num_fdes;
+    i64 fde_off = hdr_len + hdr.fdeoff;
+    i64 fre_off = hdr_len + hdr.freoff;
+
+    std::span<ElfRel<E>> rels = isec->get_rels(ctx);
+    i64 rel_idx = 0;
+
+    for (i64 i = 0; i < num_fdes; i++) {
+      i64 idx_off = fde_off + i * sizeof(SFrameFdeIdx<E>);
+      const SFrameFdeIdx<E> &ent =
+        *(const SFrameFdeIdx<E> *)(data.data() + idx_off);
+
+      // Find the relocation for this FDE's func_start field. An FDE without
+      // one isn't tied to any function (`ld -r` can emit such dead FDEs),
+      // so we drop it.
+      while (rel_idx < rels.size() && rels[rel_idx].r_offset < idx_off)
+        rel_idx++;
+      if (rel_idx == rels.size() || rels[rel_idx].r_offset != idx_off)
+        continue;
+
+      const ElfRel<E> &rel = rels[rel_idx];
+      i64 off = fre_off + ent.func_start_fre_off;
+
+      SFrameFde<E> fde;
+      fde.isec = get_section(this->elf_syms[rel.r_sym]);
+      fde.sym = this->symbols[rel.r_sym];
+      fde.addend = rel.r_addend;
+      fde.fre = data.substr(off, sframe_fre_block_size<E>(data, off));
+      fde.func_size = ent.func_size;
+      fde.num_fres = *(U16<E> *)(data.data() + off);
+      sframe_fdes.push_back(fde);
+    }
+  }
+}
+
 template <typename E>
 void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
  if (this->elf_syms.empty())
@@ -598,7 +749,7 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
    if (esym.st_type == STT_SECTION)
      name = this->shstrtab.data() + this->elf_sections[get_shndx(esym)].sh_name;
    else
-      name = this->symbol_strtab.data() + esym.st_name;
+      name = this->symbol_names[i];

    Symbol<E> &sym = this->local_syms[i];
    sym.set_name(name);
@@ -626,7 +777,7 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
      has_common_symbol = true;

    // Get a symbol name
-    std::string_view key = this->symbol_strtab.data() + esym.st_name;
+    std::string_view key = this->symbol_names[i];
    std::string_view name = key;

    // Parse symbol version after atsign
@@ -637,7 +788,7 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
      if (ver != "@" && ver != "@@") {
        if (ver.starts_with("@@"))
          key = name;
-        has_symver.set(i - this->first_global);
+        has_symver[i - this->first_global] = true;
      }
    }

@@ -665,18 +816,14 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
 template <typename E>
 void ObjectFile<E>::sort_relocations(Context<E> &ctx) {
  if constexpr (is_riscv<E> || is_loongarch<E>) {
-    auto less = [&](const ElfRel<E> &a, const ElfRel<E> &b) {
-      return a.r_offset < b.r_offset;
-    };
-
    for (i64 i = 1; i < sections.size(); i++) {
      std::unique_ptr<InputSection<E>> &isec = sections[i];
      if (!isec || !isec->is_alive || !(isec->shdr().sh_flags & SHF_ALLOC))
        continue;

      std::span<ElfRel<E>> rels = isec->get_rels(ctx);
-      if (!std::is_sorted(rels.begin(), rels.end(), less))
-        sort(rels, less);
+      if (!ranges::is_sorted(rels, {}, &ElfRel<E>::r_offset))
+        ranges::stable_sort(rels, {}, &ElfRel<E>::r_offset);
    }
  }
 }
@@ -690,11 +837,21 @@ void ObjectFile<E>::convert_mergeable_sections(Context<E> &ctx) {
      continue;

    const ElfShdr<E> &shdr = isec->shdr();
-    if (!(shdr.sh_flags & SHF_MERGE))
+    if (!(shdr.sh_flags & SHF_MERGE) || !isec->is_alive)
      continue;

+    // Under a SECTIONS-driven layout, sections matched by different
+    // output section descriptions merge separately
+    i64 cmd = -1;
+    i32 rank = INT32_MAX;
+    if (!this->script_match.empty())
+      if (i32 r = this->script_match[i]; r != -1) {
+        cmd = ctx.script_ranks[r].cmd;
+        rank = r;
+      }
+
    MergedSection<E> *parent =
-      MergedSection<E>::get_instance(ctx, isec->name(), shdr);
+      MergedSection<E>::get_instance(ctx, isec->name, shdr, cmd, rank);

    if (parent) {
      this->mergeable_sections[i] =
@@ -845,6 +1002,7 @@ void ObjectFile<E>::parse(Context<E> &ctx) {
    this->first_global = symtab_sec->sh_info;
    this->elf_syms = this->template get_data<ElfSym<E>>(ctx, *symtab_sec);
    this->symbol_strtab = this->get_string(ctx, symtab_sec->sh_link);
+    this->populate_symbol_names();

    if (ElfShdr<E> *shdr = this->find_section(SHT_SYMTAB_SHNDX))
      symtab_shndx_sec = this->template get_data<U32<E>>(ctx, *shdr);
@@ -867,6 +1025,10 @@ void ObjectFile<E>::parse(Context<E> &ctx) {
 //  7. Unclaimed (nonexistent) symbol
 //
 // Ties are broken by file priority.
+//
+// Note that the above priorities are based on heuristics and not on exact
+// science. We tried several different orders and settled on the current
+// one just because it avoids link errors in all programs we've tested.
 template <typename E>
 static u64 get_rank(InputFile<E> *file, const ElfSym<E> &esym, bool is_in_archive) {
  auto get_sym_rank = [&] {
@@ -890,7 +1052,7 @@ template <typename E>
 static u64 get_rank(const Symbol<E> &sym) {
  if (!sym.file)
    return 7 << 24;
-  return get_rank(sym.file, sym.esym(), !sym.file->is_alive);
+  return get_rank(sym.file, sym.esym(), !sym.file->is_reachable);
 }

 // Symbol's visibility is set to the most restrictive one. For example,
@@ -950,13 +1112,15 @@ void ObjectFile<E>::resolve_symbols(Context<E> &ctx) {
    }

    std::scoped_lock lock(sym.mu);
-    if (get_rank(this, esym, !this->is_alive) < get_rank(sym)) {
+
+    if (get_rank(this, esym, !this->is_reachable) < get_rank(sym)) {
      sym.file = this;
      sym.set_input_section(isec);
      sym.value = esym.st_value;
      sym.sym_idx = i;
      sym.ver_idx = ctx.default_version;
      sym.is_weak = esym.is_weak();
+      sym.is_versioned_default = false;
    }
  }
 }
@@ -965,7 +1129,7 @@ template <typename E>
 void
 ObjectFile<E>::mark_live_objects(Context<E> &ctx,
                                 std::function<void(InputFile<E> *)> feeder) {
-  assert(this->is_alive);
+  assert(this->is_reachable);

  for (i64 i = this->first_global; i < this->elf_syms.size(); i++) {
    const ElfSym<E> &esym = this->elf_syms[i];
@@ -983,7 +1147,7 @@ ObjectFile<E>::mark_live_objects(Context<E> &ctx,
      bool undef_ref = esym.is_undef() && (!esym.is_weak() || sym.file->is_dso);
      bool common_ref = esym.is_common() && !sym.esym().is_common();

-      if ((undef_ref || common_ref) && !sym.file->is_alive.test_and_set()) {
+      if ((undef_ref || common_ref) && !sym.file->is_reachable.test_and_set()) {
        feeder(sym.file);
        if (sym.is_traced)
          Out(ctx) << "trace-symbol: " << *this << " keeps " << *sym.file
@@ -1089,7 +1253,7 @@ static bool should_write_to_local_symtab(Context<E> &ctx, Symbol<E> &sym) {
  // merged, so their origins shouldn't matter, but I don't really
  // know the rationale. Anyway, this is the behavior of the
  // traditional linkers.
-  if (sym.name().starts_with(".L")) {
+  if (sym.name().starts_with(".L") || sym.name() == "L0\001") {
    if (ctx.arg.discard_locals)
      return false;

@@ -1105,7 +1269,7 @@ template <typename E>
 void ObjectFile<E>::compute_symtab_size(Context<E> &ctx) {
  this->output_sym_indices.resize(this->elf_syms.size(), -1);

-  auto is_alive = [&](Symbol<E> &sym) -> bool {
+  auto is_alive = [](Symbol<E> &sym) -> bool {
    if (SectionFragment<E> *frag = sym.get_frag())
      return frag->is_alive;
    if (InputSection<E> *isec = sym.get_input_section())
@@ -1156,25 +1320,21 @@ void ObjectFile<E>::populate_symtab(Context<E> &ctx) {
    if (ctx.symtab_shndx)
      xindex = (U32<E> *)(ctx.buf + ctx.symtab_shndx->shdr.sh_offset) + idx;

-    symtab_base[idx] = to_output_esym(ctx, sym, strtab_off, xindex);
+    symtab_base[idx] = *to_output_esym(ctx, sym, strtab_off, xindex);
    strtab_off += write_string(strtab_base + strtab_off, sym.name());
  };

-  i64 local_symtab_idx = this->local_symtab_idx;
-  i64 global_symtab_idx = this->global_symtab_idx;
+  i64 local_idx = this->local_symtab_idx;
+  i64 global_idx = this->global_symtab_idx;

  for (i64 i = 1; i < this->first_global; i++)
    if (Symbol<E> &sym = *this->symbols[i]; sym.write_to_symtab)
-      write_sym(sym, local_symtab_idx++);
+      write_sym(sym, local_idx++);

  for (i64 i = this->first_global; i < this->elf_syms.size(); i++) {
    Symbol<E> &sym = *this->symbols[i];
-    if (sym.file == this && sym.write_to_symtab) {
-      if (sym.is_local(ctx))
-        write_sym(sym, local_symtab_idx++);
-      else
-        write_sym(sym, global_symtab_idx++);
-    }
+    if (sym.file == this && sym.write_to_symtab)
+      write_sym(sym, sym.is_local(ctx) ? local_idx++ : global_idx++);
  }
 }

@@ -1193,24 +1353,17 @@ std::ostream &operator<<(std::ostream &out, const InputFile<E> &file) {
  return out;
 }

-template <typename E>
-SharedFile<E> *SharedFile<E>::create(Context<E> &ctx, MappedFile *mf) {
-  SharedFile<E> *obj = new SharedFile(ctx, mf);
-  ctx.dso_pool.emplace_back(obj);
-  return obj;
-}
-
 template <typename E>
 std::string SharedFile<E>::get_soname(Context<E> &ctx) {
  if (ElfShdr<E> *sec = this->find_section(SHT_DYNAMIC))
    for (ElfDyn<E> &dyn : this->template get_data<ElfDyn<E>>(ctx, *sec))
      if (dyn.d_tag == DT_SONAME)
-        return this->symbol_strtab.data() + dyn.d_val;
+        return this->get_string(ctx, sec->sh_link).data() + dyn.d_val;

  if (this->mf->given_fullpath)
    return this->filename;

-  return filepath(this->filename).filename().string();
+  return path_filename(this->filename);
 }

 template <typename E>
@@ -1221,7 +1374,7 @@ void SharedFile<E>::parse(Context<E> &ctx) {

  this->symbol_strtab = this->get_string(ctx, symtab_sec->sh_link);
  soname = get_soname(ctx);
-  version_strings = read_verdef(ctx);
+  version_strings = read_version_strings(ctx);

  // Read a symbol table.
  std::span<ElfSym<E>> esyms = this->template get_data<ElfSym<E>>(ctx, *symtab_sec);
@@ -1231,27 +1384,61 @@ void SharedFile<E>::parse(Context<E> &ctx) {
    vers = this->template get_data<U16<E>>(ctx, *sec);

  for (i64 i = symtab_sec->sh_info; i < esyms.size(); i++) {
-    u16 ver;
-    if (vers.empty() || esyms[i].is_undef())
-      ver = VER_NDX_GLOBAL;
-    else
-      ver = (vers[i] & ~VERSYM_HIDDEN);
+    u16 ver = vers.empty() ? VER_NDX_GLOBAL : (vers[i] & ~VERSYM_HIDDEN);

-    if (ver == VER_NDX_LOCAL)
+    // A defined symbol with VER_NDX_LOCAL is bound locally and isn't
+    // really exposed for dynamic linking; skip it. We never skip
+    // undefined references — the dynamic loader doesn't distinguish
+    // local vs global on the undef side, and dropping them would hide
+    // the DSO's reference from us.
+    if (ver == VER_NDX_LOCAL && !esyms[i].is_undef())
      continue;

-    std::string_view name = this->symbol_strtab.data() + esyms[i].st_name;
-    bool is_default = vers.empty() || !(vers[i] & VERSYM_HIDDEN);
-
    this->elf_syms2.push_back(esyms[i]);
-    this->versyms.push_back(ver);

-    if (is_default) {
-      this->symbols.push_back(get_symbol(ctx, name));
-    } else {
-      std::string_view mangled_name = save_string(
+    // resolve_symbols only consults versyms[] for defined symbols
+    // (see SharedFile::resolve_symbols), so VER_NDX_GLOBAL is fine
+    // for undefined entries.
+    this->versyms.push_back(esyms[i].is_undef() ? (u16)VER_NDX_GLOBAL : ver);
+
+    std::string_view name = this->symbol_strtab.data() + esyms[i].st_name;
+
+    bool has_version = ver != VER_NDX_GLOBAL && ver < version_strings.size() &&
+                       !version_strings[ver].empty();
+
+    auto get_versioned_sym = [&] {
+      std::string_view key = save_string(
        ctx, std::string(name) + "@" + std::string(version_strings[ver]));
-      this->symbols.push_back(get_symbol(ctx, mangled_name, name));
+      return get_symbol(ctx, key, name);
+    };
+
+    // Symbol resolution involving symbol versioning is tricky because one
+    // symbol can be resolved with two different identifiers. Among
+    // symbols with the same name but different versions, one of them is
+    // always marked as the "default" one. This symbol is often denoted
+    // with two atsigns as `foo@@VERSION` and can be referred to either
+    // as `foo` or `foo@VERSION`. No other symbols have two names like that.
+    //
+    // On contrary, a versioned non-default symbol can be referred only
+    // with an explicit version suffix, e.g., `foo@VERSION`.
+    //
+    // Here is how we resolve versioned default symbols. We resolve `foo`
+    // and `foo@VERSION` as usual, but with information to forward
+    // references to `foo@VERSION` to `foo`. After name resolution, we
+    // visit all symbol references to redirect `foo@VERSION` to `foo`.
+    if (!has_version) {
+      // Unversioned symbol
+      this->symbols.push_back(get_symbol(ctx, name));
+      this->symbols2.push_back(nullptr);
+    } else if (esyms[i].is_undef() || (vers[i] & VERSYM_HIDDEN)) {
+      // Versioned non-default symbol, or undefined reference whose
+      // version comes from .gnu.version_r.
+      this->symbols.push_back(get_versioned_sym());
+      this->symbols2.push_back(nullptr);
+    } else {
+      // Versioned default symbol
+      this->symbols.push_back(get_symbol(ctx, name));
+      this->symbols2.push_back(get_versioned_sym());
    }
  }

@@ -1264,30 +1451,23 @@ void SharedFile<E>::parse(Context<E> &ctx) {

 template <typename E>
 std::vector<std::string_view> SharedFile<E>::get_dt_needed(Context<E> &ctx) {
-  // Get the contents of the dynamic segment
-  std::span<Word<E>> dynamic;
-  for (ElfPhdr<E> &phdr : this->get_phdrs())
-    if (phdr.p_type == PT_DYNAMIC)
-      dynamic = {(Word<E> *)(this->mf->data + phdr.p_offset),
-                 (size_t)phdr.p_memsz / sizeof(Word<E>)};
-
-  // Find a string table
-  char *strtab = nullptr;
-  for (i64 i = 0; i < dynamic.size(); i += 2)
-    if (dynamic[i] == DT_STRTAB)
-      strtab = (char *)this->mf->data + dynamic[i + 1];
-
-  if (!strtab)
-    return {};
-
-  // Find all DT_NEEDED entries
  std::vector<std::string_view> vec;
-  for (i64 i = 0; i < dynamic.size(); i += 2)
-    if (dynamic[i] == DT_NEEDED)
-      vec.push_back(strtab + dynamic[i + 1]);
+  if (ElfShdr<E> *sec = this->find_section(SHT_DYNAMIC))
+    for (ElfDyn<E> &dyn : this->template get_data<ElfDyn<E>>(ctx, *sec))
+      if (dyn.d_tag == DT_NEEDED)
+        vec.push_back(this->get_string(ctx, sec->sh_link).data() + dyn.d_val);
  return vec;
 }

+template <typename E>
+std::string_view SharedFile<E>::get_dt_audit(Context<E> &ctx) {
+  if (ElfShdr<E> *sec = this->find_section(SHT_DYNAMIC))
+    for (ElfDyn<E> &dyn : this->template get_data<ElfDyn<E>>(ctx, *sec))
+      if (dyn.d_tag == DT_AUDIT)
+        return this->get_string(ctx, sec->sh_link).data() + dyn.d_val;
+  return "";
+}
+
 // Symbol versioning is a GNU extension to the ELF file format. I don't
 // particularly like the feature as it complicates the semantics of
 // dynamic linking, but we need to support it anyway because it is
@@ -1318,32 +1498,61 @@ std::vector<std::string_view> SharedFile<E>::get_dt_needed(Context<E> &ctx) {
 // If an undefiend symbol `foo` is resolved to a symbol defined by the
 // shared object, it's marked so that it'll be resolved to (`foo`, the
 // default version of the library) at load-time.
+//
+// Reads .gnu.version_d and .gnu.version_r and returns a vector of
+// version names indexed by versym value. The versym index space is
+// shared between the two sections (vd_ndx for defined symbols and
+// vna_other for undefined ones), so a single vector covers both.
 template <typename E>
-std::vector<std::string_view> SharedFile<E>::read_verdef(Context<E> &ctx) {
-  ElfShdr<E> *verdef_sec = this->find_section(SHT_GNU_VERDEF);
-  if (!verdef_sec)
-    return {};
-
-  std::string_view verdef = this->get_string(ctx, *verdef_sec);
-  std::string_view strtab = this->get_string(ctx, verdef_sec->sh_link);
-
+std::vector<std::string_view>
+SharedFile<E>::read_version_strings(Context<E> &ctx) {
  std::vector<std::string_view> vec;
-  u8 *ptr = (u8 *)verdef.data();

-  for (;;) {
-    ElfVerdef<E> *ver = (ElfVerdef<E> *)ptr;
-    if (ver->vd_ndx == VER_NDX_UNSPECIFIED)
-      Fatal(ctx) << *this << ": symbol version too large";
+  if (ElfShdr<E> *sec = this->find_section(SHT_GNU_VERDEF)) {
+    std::string_view verdef = this->get_string(ctx, *sec);
+    std::string_view strtab = this->get_string(ctx, sec->sh_link);
+    u8 *ptr = (u8 *)verdef.data();

-    if (vec.size() <= ver->vd_ndx)
-      vec.resize(ver->vd_ndx + 1);
+    for (;;) {
+      ElfVerdef<E> *ver = (ElfVerdef<E> *)ptr;
+      if (ver->vd_ndx == VER_NDX_UNSPECIFIED)
+        Fatal(ctx) << *this << ": symbol version too large";

-    ElfVerdaux<E> *aux = (ElfVerdaux<E> *)(ptr + ver->vd_aux);
-    vec[ver->vd_ndx] = strtab.data() + aux->vda_name;
-    if (!ver->vd_next)
-      break;
-    ptr += ver->vd_next;
+      if (vec.size() <= ver->vd_ndx)
+        vec.resize(ver->vd_ndx + 1);
+
+      ElfVerdaux<E> *aux = (ElfVerdaux<E> *)(ptr + ver->vd_aux);
+      vec[ver->vd_ndx] = strtab.data() + aux->vda_name;
+      if (!ver->vd_next)
+        break;
+      ptr += ver->vd_next;
+    }
  }
+
+  if (ElfShdr<E> *sec = this->find_section(SHT_GNU_VERNEED)) {
+    std::string_view verneed = this->get_string(ctx, *sec);
+    std::string_view strtab = this->get_string(ctx, sec->sh_link);
+    u8 *ptr = (u8 *)verneed.data();
+
+    for (;;) {
+      ElfVerneed<E> *vn = (ElfVerneed<E> *)ptr;
+      u8 *aux_ptr = ptr + vn->vn_aux;
+      for (i64 i = 0; i < vn->vn_cnt; i++) {
+        ElfVernaux<E> *aux = (ElfVernaux<E> *)aux_ptr;
+        u16 idx = aux->vna_other & ~VERSYM_HIDDEN;
+        if (vec.size() <= idx)
+          vec.resize(idx + 1);
+        vec[idx] = strtab.data() + aux->vna_name;
+        if (!aux->vna_next)
+          break;
+        aux_ptr += aux->vna_next;
+      }
+      if (!vn->vn_next)
+        break;
+      ptr += vn->vn_next;
+    }
+  }
+
  return vec;
 }

@@ -1365,6 +1574,22 @@ void SharedFile<E>::resolve_symbols(Context<E> &ctx) {
      sym.sym_idx = i;
      sym.ver_idx = versyms[i];
      sym.is_weak = true;
+      sym.is_versioned_default = false;
+    }
+
+    // A symbol with the default version is a special case because, unlike
+    // other symbols, the symbol can be referred by two names, `foo` and
+    // `foo@VERSION`. Here, we resolve `foo@VERSOIN` as a proxy of `foo`.
+    Symbol<E> *sym2 = this->symbols2[i];
+    if (sym2 && sym2 != &sym) {
+      std::scoped_lock lock2(sym2->mu);
+
+      if (get_rank(this, esym, false) < get_rank(*sym2)) {
+        sym2->file = this;
+        sym2->origin = (uintptr_t)&sym;
+        sym2->sym_idx = i;
+        sym2->is_versioned_default = true;
+      }
    }
  }
 }
@@ -1380,8 +1605,11 @@ SharedFile<E>::mark_live_objects(Context<E> &ctx,
    if (sym.is_traced)
      print_trace_symbol(ctx, *this, esym, sym);

+    // We follow undefined symbols in a DSO only to handle
+    // --no-allow-shlib-undefined.
    if (esym.is_undef() && !esym.is_weak() && sym.file &&
-        !sym.file->is_alive.test_and_set()) {
+        (!sym.file->is_dso || !ctx.arg.allow_shlib_undefined) &&
+        !sym.file->is_reachable.test_and_set()) {
      feeder(sym.file);

      if (sym.is_traced)
@@ -1408,11 +1636,10 @@ std::span<Symbol<E> *> SharedFile<E>::get_symbols_at(Symbol<E> *sym) {
    });
  });

-  auto [begin, end] = std::equal_range(sorted_syms.begin(), sorted_syms.end(),
-                                       sym, [&](Symbol<E> *a, Symbol<E> *b) {
-    return a->esym().st_value < b->esym().st_value;
+  auto [begin, end] = ranges::equal_range(sorted_syms, sym->esym().st_value, {},
+                                          [](Symbol<E> *x) {
+    return x->esym().st_value;
  });
-
  return {&*begin, (size_t)(end - begin)};
 }

@@ -1434,9 +1661,12 @@ i64 SharedFile<E>::get_alignment(Symbol<E> *sym) {

 template <typename E>
 bool SharedFile<E>::is_readonly(Symbol<E> *sym) {
+  ElfEhdr<E> &ehdr = *(ElfEhdr<E> *)this->mf->data;
+  std::span<ElfPhdr<E>> phdrs((ElfPhdr<E> *)(this->mf->data + ehdr.e_phoff),
+                              ehdr.e_phnum);
  u64 val = sym->esym().st_value;

-  for (ElfPhdr<E> &phdr : this->get_phdrs())
+  for (ElfPhdr<E> &phdr : phdrs)
    if ((phdr.p_type == PT_LOAD || phdr.p_type == PT_GNU_RELRO) &&
        !(phdr.p_flags & PF_W) &&
        phdr.p_vaddr <= val && val < phdr.p_vaddr + phdr.p_memsz)
@@ -1478,7 +1708,7 @@ void SharedFile<E>::populate_symtab(Context<E> &ctx) {
      xindex = (U32<E> *)(ctx.buf + ctx.symtab_shndx->shdr.sh_offset) +
               this->global_symtab_idx + i;

-    *symtab++ = to_output_esym(ctx, *sym, strtab_off, xindex);
+    *symtab++ = *to_output_esym(ctx, *sym, strtab_off, xindex);
    strtab_off += write_string(strtab + strtab_off, sym->name());
    i++;
  }
@@ -1492,6 +1722,7 @@ template class SharedFile<E>;
 template Symbol<E> *get_symbol(Context<E> &, std::string_view, std::string_view);
 template Symbol<E> *get_symbol(Context<E> &, std::string_view);
 template std::string_view demangle(const Symbol<E> &);
+template ComdatGroup *insert_comdat_group(Context<E> &, std::string_view);
 template std::ostream &operator<<(std::ostream &, const Symbol<E> &);
 template std::ostream &operator<<(std::ostream &, const InputFile<E> &);

@@ -1,6 +1,5 @@
 #include "mold.h"

-#include <limits>
 #include <zlib.h>
 #include <zstd.h>

@@ -34,8 +33,12 @@ bool cie_equals(const CieRecord<E> &a, const CieRecord<E> &b) {
 template <typename E>
 InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
  : file(file), shndx(shndx) {
-  if (shndx < file.elf_sections.size())
+  if (shndx < file.elf_sections.size()) {
+    name = file.shstrtab.data() + file.elf_sections[shndx].sh_name;
    contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};
+  } else {
+    name = (shdr().sh_flags & SHF_TLS) ? ".tls_common" : ".common";
+  }

  if (shdr().sh_flags & SHF_COMPRESSED) {
    ElfChdr<E> &chdr = *(ElfChdr<E> *)&contents[0];
@@ -64,16 +67,16 @@ void InputSection<E>::uncompress(Context<E> &ctx) {
    return;

  u8 *buf = new u8[sh_size];
-  copy_contents(ctx, buf);
+  copy_contents_to(ctx, buf, sh_size);
  contents = std::string_view((char *)buf, sh_size);
  ctx.string_pool.emplace_back(buf);
  uncompressed = true;
 }

 template <typename E>
-void InputSection<E>::copy_contents(Context<E> &ctx, u8 *buf) {
+void InputSection<E>::copy_contents_to(Context<E> &ctx, u8 *buf, i64 sz) {
  if (!(shdr().sh_flags & SHF_COMPRESSED) || uncompressed) {
-    memcpy(buf, contents.data(), contents.size());
+    memcpy(buf, contents.data(), sz);
    return;
  }

@@ -85,22 +88,46 @@ void InputSection<E>::copy_contents(Context<E> &ctx, u8 *buf) {

  switch (hdr.ch_type) {
  case ELFCOMPRESS_ZLIB: {
-    unsigned long size = sh_size;
-    if (::uncompress(buf, &size, (u8 *)data.data(), data.size()) != Z_OK)
-      Fatal(ctx) << *this << ": uncompress failed";
-    assert(size == sh_size);
+    z_stream s = {};
+    inflateInit(&s);
+    s.next_in = (u8 *)data.data();
+    s.avail_in = data.size();
+    s.next_out = buf;
+    s.avail_out = sz;
+
+    int r;
+    while (s.total_out < sz && (r = inflate(&s, Z_NO_FLUSH)) == Z_OK);
+    if (s.total_out < sz && r != Z_STREAM_END)
+      Fatal(ctx) << *this << ": uncompress failed: " << s.msg;
+    inflateEnd(&s);
+    msan_unpoison(buf, sz);
    break;
  }
-  case ELFCOMPRESS_ZSTD:
-    if (ZSTD_decompress(buf, sh_size, (u8 *)data.data(), data.size()) != sh_size)
-      Fatal(ctx) << *this << ": ZSTD_decompress failed";
+  case ELFCOMPRESS_ZSTD: {
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+    ZSTD_inBuffer in = { data.data(), data.size() };
+    ZSTD_outBuffer out = { buf, (size_t)sz };
+
+    while (out.pos < out.size) {
+      size_t r = ZSTD_decompressStream(dctx, &out, &in);
+      if (ZSTD_isError(r))
+        Fatal(ctx) << *this << ": uncompress failed: " << ZSTD_getErrorName(r);
+      if (r == 0 && out.pos < out.size)
+        Fatal(ctx) << *this << ": uncompress failed: premature end of input";
+    }
+    ZSTD_freeDCtx(dctx);
+    msan_unpoison(buf, sz);
    break;
+  }
  default:
    Fatal(ctx) << *this << ": unsupported compression type: 0x"
               << std::hex << hdr.ch_type;
  }
 }

+// wingdi.h defines ERROR as a macro, so undefine it before use
+#undef ERROR
+
 typedef enum : u8 { NONE, ERROR, COPYREL, PLT, CPLT } Action;

 template <typename E>
@@ -115,7 +142,6 @@ static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
               << sym << "' can not be used; recompile with -fPIC";
    break;
  case COPYREL:
-    // Create a copy relocation
    sym.flags |= NEEDS_COPYREL;
    break;
  case PLT:
@@ -226,30 +252,27 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
  // relocations are allowed to remove bytes from the middle of a
  // section and shrink the overall size of it.
  if constexpr (is_riscv<E> || is_loongarch<E>) {
-    if (extra.r_deltas.empty()) {
+    std::span<RelocDelta> deltas = extra.r_deltas;
+
+    if (deltas.empty()) {
      // If a section is not relaxed, we can copy it as a one big chunk.
-      copy_contents(ctx, buf);
+      copy_contents_to(ctx, buf, sh_size);
    } else {
      // A relaxed section is copied piece-wise.
-      std::span<const ElfRel<E>> rels = get_rels(ctx);
-      u8 *buf2 = buf;
-      i64 pos = 0;
+      memcpy(buf, contents.data(), deltas[0].offset);

-      for (i64 i = 0; i < rels.size(); i++) {
-        i64 delta = extra.r_deltas[i + 1] - extra.r_deltas[i];
-        if (delta == 0)
-          continue;
-        assert(delta > 0);
-
-        const ElfRel<E> &r = rels[i];
-        memcpy(buf2, contents.data() + pos, r.r_offset - pos);
-        buf2 += r.r_offset - pos;
-        pos = r.r_offset + delta;
+      for (i64 i = 0; i < deltas.size(); i++) {
+        i64 offset = deltas[i].offset;
+        i64 delta = deltas[i].delta;
+        i64 end = (i + 1 == deltas.size()) ? contents.size() : deltas[i + 1].offset;
+        i64 removed_bytes = get_removed_bytes(deltas, i);
+        memcpy(buf + offset + removed_bytes - delta,
+               contents.data() + offset + removed_bytes,
+               end - offset - removed_bytes);
      }
-      memcpy(buf2, contents.data() + pos, contents.size() - pos);
    }
  } else {
-    copy_contents(ctx, buf);
+    copy_contents_to(ctx, buf, sh_size);
  }

  // Apply relocations
@@ -265,17 +288,12 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 template <typename E>
 std::string_view
 InputSection<E>::get_func_name(Context<E> &ctx, i64 offset) const {
-  for (Symbol<E> *sym : file.symbols) {
-    if (sym->file == &file) {
-      const ElfSym<E> &esym = sym->esym();
-      if (esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
-          esym.st_value <= offset && offset < esym.st_value + esym.st_size) {
-        if (ctx.arg.demangle)
-          return demangle(*sym);
-        return sym->name();
-      }
-    }
-  }
+  for (Symbol<E> *sym : file.symbols)
+    if (sym->file == &file)
+      if (const ElfSym<E> &esym = sym->esym();
+          esym.st_shndx == shndx && esym.st_type == STT_FUNC &&
+          esym.st_value <= offset && offset < esym.st_value + esym.st_size)
+        return ctx.arg.demangle ? demangle(*sym) : sym->name();
  return "";
 }

@@ -342,8 +360,8 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
 template <typename E>
 MergeableSection<E>::MergeableSection(Context<E> &ctx, MergedSection<E> &parent,
                                      std::unique_ptr<InputSection<E>> &isec)
-  : parent(parent), section(std::move(isec)), p2align(section->p2align) {
-  section->uncompress(ctx);
+  : parent(parent), p2align(isec->p2align), input_section(std::move(isec)) {
+  input_section->uncompress(ctx);

  std::scoped_lock lock(parent.mu);
  parent.members.push_back(this);
@@ -379,10 +397,9 @@ static size_t find_null(std::string_view data, i64 pos, i64 entsize) {
 // We do not support mergeable sections that have relocations.
 template <typename E>
 void MergeableSection<E>::split_contents(Context<E> &ctx) {
-  std::string_view data = section->contents;
+  std::string_view data = input_section->contents;
  if (data.size() > UINT32_MAX)
-    Fatal(ctx) << *section
-               << ": mergeable section too large";
+    Fatal(ctx) << *input_section << ": mergeable section too large";

  i64 entsize = parent.shdr.sh_entsize;

@@ -392,12 +409,13 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
      frag_offsets.push_back(pos);
      size_t end = find_null(data, pos, entsize);
      if (end == data.npos)
-        Fatal(ctx) << *section << ": string is not null terminated";
+        Fatal(ctx) << *input_section << ": string is not null terminated";
      pos = end + entsize;
    }
  } else {
    if (data.size() % entsize)
-      Fatal(ctx) << *section << ": section size is not multiple of sh_entsize";
+      Fatal(ctx) << *input_section
+                 << ": section size is not multiple of sh_entsize";
    frag_offsets.reserve(data.size() / entsize);

    for (i64 pos = 0; pos < data.size(); pos += entsize)
@@ -9,11 +9,11 @@
 // mold processes to just 1 for each user. It is intended to be used as
 // `MOLD_JOBS=1 ninja` or `MOLD_JOBS=1 make -j$(nproc)`.

-#include "common.h"
+#include "mold.h"

+#include <cstdlib>
 #include <fcntl.h>
 #include <pwd.h>
-#include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -29,8 +29,8 @@
 //
 //  4. Continue the rest of the linking process as usual.
 //
-// When gcc or clang inovkes ld, they pass `-plugin linker-plugin.so` to
-// the linker. The given .so file provides a way to call the compiler
+// When gcc or clang inovkes ld, they pass `-plugin /path/to/linker-plugin.so`
+// to the linker. The given .so file provides a way to call the compiler
 // backend.
 //
 // The linker plugin API is documented at
@@ -177,13 +177,14 @@ static PluginStatus add_input_file(const char *path) {
  static i64 file_priority = 100;

  MappedFile *mf = must_open_file(ctx, path);
+  mf->is_dependency = false;

-  ObjectFile<E> *file = ObjectFile<E>::create(ctx, mf, "", false);
+  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, "");
  ctx.obj_pool.emplace_back(file);
  lto_objects<E>.push_back(file);

  file->priority = file_priority++;
-  file->is_alive = true;
+  file->is_reachable = true;
  file->parse(ctx);
  file->resolve_symbols(ctx);
  return LDPS_OK;
@@ -284,7 +285,7 @@ get_symbols(const void *handle, int nsyms, PluginSymbol *psyms, bool is_v2) {
  // If file is an archive member which was not chose to be included in
  // to the final result, we need to make the plugin to ignore all
  // symbols.
-  if (!file.is_alive) {
+  if (!file.is_reachable) {
    assert(!is_v2);
    for (int i = 0; i < nsyms; i++)
      psyms[i].resolution = LDPR_PREEMPTED_REG;
@@ -342,7 +343,7 @@ static void restart_process(Context<E> &ctx) {
    args.push_back(strdup(std::string(arg).c_str()));

  for (std::unique_ptr<ObjectFile<E>> &file : ctx.obj_pool)
-    if (file->is_lto_obj && !file->is_alive)
+    if (file->is_lto_obj && !file->is_reachable)
      args.push_back(strdup(("--:ignore-ir-file=" +
                             file->mf->get_identifier()).c_str()));

@@ -424,7 +425,7 @@ get_api_version(const char *plugin_identifier,
  if (LAPI_V1 < minimal_api_supported)
    Fatal(*gctx<E>) << "LTO plugin does not support V0 or V1 API";

-  std::string version = get_mold_version() + "\0"s;
+  std::string version = mold_version + "\0"s;

  *linker_identifier = "mold";
  *linker_version = version.data();
@@ -535,8 +536,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
  }

  switch (psym.symbol_type) {
-  case LDST_UNKNOWN:
-    break;
  case LDST_FUNCTION:
    esym.st_type = STT_FUNC;
    break;
@@ -546,8 +545,6 @@ static ElfSym<E> to_elf_sym(PluginSymbol &psym) {
  };

  switch (psym.visibility) {
-  case LDPV_DEFAULT:
-    break;
  case LDPV_PROTECTED:
    esym.st_visibility = STV_PROTECTED;
    break;
@@ -598,6 +595,12 @@ create_plugin_input_file(Context<E> &ctx, MappedFile *mf) {

 template <typename E>
 ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
+  if (ctx.arg.plugin.empty())
+    Fatal(ctx) << mf->name << ": unable to handle this LTO object file because "
+               << "the -plugin option was not provided. Please make sure you "
+               << "added -flto not only when creating object files but also "
+               << "when linking the final executable.";
+
  load_lto_plugin(ctx);

  // V0 API's claim_file is not thread-safe.
@@ -606,12 +609,6 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
  if (!is_gcc_linker_api_v1)
    lock.lock();

-  if (ctx.arg.plugin.empty())
-    Fatal(ctx) << mf->name << ": don't know how to handle this LTO object file "
-               << "because no -plugin option was given. Please make sure you "
-               << "added -flto not only for creating object files but also for "
-               << "creating the final executable.";
-
  // Create mold's object instance
  ObjectFile<E> *obj = new ObjectFile<E>;
  ctx.obj_pool.emplace_back(obj);
@@ -632,17 +629,18 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
  // claim_file_hook() calls add_symbols() which initializes `plugin_symbols`
  int claimed = false;
  claim_file_hook(&file, &claimed);
-  if (!claimed)
-    Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
-               << " please make sure you are using the same compiler of the"
-               << " same version for all object files";

-  // It looks like GCC doesn't need fd after claim_file_hook() while
-  // LLVM needs it and takes the ownership of fd. To prevent "too many
-  // open files" issue, we close fd only for GCC. This is ugly, though.
-  if (!is_llvm(ctx)) {
-    MappedFile *mf2 = mf->parent ? mf->parent : mf;
-    mf2->close_fd();
+  if (mf->parent)
+    mf->parent->close_fd();
+  else
+    mf->close_fd();
+
+  if (!claimed) {
+    if (!mf->parent && !mf->thin_parent)
+      Fatal(ctx) << mf->name << ": not claimed by the LTO plugin;"
+                 << " please make sure you are using the same compiler of the"
+                 << " same version for all object files";
+    return nullptr;
  }

  // Create a symbol strtab
@@ -653,6 +651,7 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {

  // Initialize esyms
  obj->lto_elf_syms.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_groups.resize(plugin_symbols.size() + 1);
  i64 strtab_offset = 1;

  for (i64 i = 0; i < plugin_symbols.size(); i++) {
@@ -663,10 +662,20 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
    i64 len = strlen(psym.name);
    memcpy(strtab.data() + strtab_offset, psym.name, len);
    strtab_offset += len + 1;
+
+    // comdat_key is non-null if the symbol is defined in a comdat member
+    // section. We handle such symbols differently than comdat symbols in
+    // a regular file because, unlike regular object files, IR files don't
+    // have input sections.
+    if (psym.comdat_key) {
+      std::string_view key = save_string(ctx, psym.comdat_key);
+      obj->lto_comdat_groups[i + 1] = insert_comdat_group(ctx, key);
+    }
  }

  obj->symbol_strtab = save_string(ctx, strtab);
  obj->elf_syms = obj->lto_elf_syms;
+  obj->populate_symbol_names();
  obj->initialize_symbols(ctx);
  plugin_symbols.clear();
  return obj;
@@ -685,7 +694,7 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
  phase = 2;

  // Set `referenced_by_regular_obj` bit.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+  tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
    if (!file->is_lto_obj) {
      for (Symbol<E> *sym : file->get_global_syms()) {
        if (sym->file && !sym->file->is_dso &&
@@ -717,7 +726,7 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
  // given to the LTO backend. Such sections contains code and data for
  // peripherails (typically GPUs).
  for (ObjectFile<E> *file : ctx.objs) {
-    if (file->is_alive && !file->is_lto_obj && file->is_gcc_offload_obj) {
+    if (file->is_reachable && !file->is_lto_obj && file->is_gcc_offload_obj) {
      PluginInputFile pfile = create_plugin_input_file(ctx, file->mf);
      int claimed = false;
      claim_file_hook(&pfile, &claimed);
@@ -1,110 +1,15 @@
 #include "mold.h"
-#include "filetype.h"
-#include "../lib/archive-file.h"
+#include "config.h"

 #include <cstring>
+#include <filesystem>
 #include <functional>
-#include <iomanip>
-#include <map>
-#include <regex>
-#include <signal.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <tbb/global_control.h>
-#include <tbb/parallel_for_each.h>
 #include <unordered_set>

-#ifdef _WIN32
-# include <direct.h>
-# define chdir _chdir
-#else
-# include <unistd.h>
-#endif
-
-#ifdef MOLD_X86_64
-int main(int argc, char **argv) {
-  return mold::mold_main<mold::X86_64>(argc, argv);
-}
-#endif
-
 namespace mold {

-// Read the beginning of a given file and returns its machine type
-// (e.g. EM_X86_64 or EM_386).
-template <typename E>
-std::string_view
-get_machine_type(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
-  auto get_elf_type = [&](u8 *buf) -> std::string_view {
-    bool is_le = (((ElfEhdr<I386> *)buf)->e_ident[EI_DATA] == ELFDATA2LSB);
-    bool is_64;
-    u32 e_machine;
-
-    if (is_le) {
-      auto &ehdr = *(ElfEhdr<I386> *)buf;
-      is_64 = (ehdr.e_ident[EI_CLASS] == ELFCLASS64);
-      e_machine = ehdr.e_machine;
-    } else {
-      auto &ehdr = *(ElfEhdr<M68K> *)buf;
-      is_64 = (ehdr.e_ident[EI_CLASS] == ELFCLASS64);
-      e_machine = ehdr.e_machine;
-    }
-
-    switch (e_machine) {
-    case EM_386:
-      return I386::target_name;
-    case EM_X86_64:
-      return X86_64::target_name;
-    case EM_ARM:
-      return ARM32::target_name;
-    case EM_AARCH64:
-      return ARM64::target_name;
-    case EM_RISCV:
-      if (is_le)
-        return is_64 ? RV64LE::target_name : RV32LE::target_name;
-      return is_64 ? RV64BE::target_name : RV32BE::target_name;
-    case EM_PPC:
-      return PPC32::target_name;
-    case EM_PPC64:
-      return is_le ? PPC64V2::target_name : PPC64V1::target_name;
-    case EM_S390X:
-      return S390X::target_name;
-    case EM_SPARC64:
-      return SPARC64::target_name;
-    case EM_68K:
-      return M68K::target_name;
-    case EM_SH:
-      return SH4::target_name;
-    case EM_LOONGARCH:
-      return is_64 ? LOONGARCH64::target_name : LOONGARCH32::target_name;
-    default:
-      return "";
-    }
-  };
-
-  switch (get_file_type(ctx, mf)) {
-  case FileType::ELF_OBJ:
-  case FileType::ELF_DSO:
-  case FileType::GCC_LTO_OBJ:
-    return get_elf_type(mf->data);
-  case FileType::AR:
-    for (MappedFile *child : read_fat_archive_members(ctx, mf))
-      if (FileType ty = get_file_type(ctx, child);
-          ty == FileType::ELF_OBJ || ty == FileType::GCC_LTO_OBJ)
-        return get_elf_type(child->data);
-    return "";
-  case FileType::THIN_AR:
-    for (MappedFile *child : read_thin_archive_members(ctx, mf))
-      if (FileType ty = get_file_type(ctx, child);
-          ty == FileType::ELF_OBJ || ty == FileType::GCC_LTO_OBJ)
-        return get_elf_type(child->data);
-    return "";
-  case FileType::TEXT:
-    return Script(ctx, rctx, mf).get_script_output_type();
-  default:
-    return "";
-  }
-}
-
 template <typename E>
 static void
 check_file_compatibility(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
@@ -122,9 +27,13 @@ static ObjectFile<E> *new_object_file(Context<E> &ctx, ReaderContext &rctx,

  check_file_compatibility(ctx, rctx, mf);

-  bool in_lib = rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
-  ObjectFile<E> *file = ObjectFile<E>::create(ctx, mf, archive_name, in_lib);
+  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, archive_name);
+  ctx.obj_pool.emplace_back(file);
  file->priority = ctx.file_priority++;
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
+
+  assert(rctx.tg);
  rctx.tg->run([file, &ctx] { file->parse(ctx); });
  if (ctx.arg.trace)
    Out(ctx) << "trace: " << *file;
@@ -141,10 +50,14 @@ static ObjectFile<E> *new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
    return nullptr;

  ObjectFile<E> *file = read_lto_object(ctx, mf);
+  if (!file)
+    return nullptr;
+
  file->priority = ctx.file_priority++;
  file->archive_name = archive_name;
-  file->is_in_lib = rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
-  file->is_alive = !file->is_in_lib;
+  file->as_needed =
+    rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
+
  if (ctx.arg.trace)
    Out(ctx) << "trace: " << *file;
  return file;
@@ -155,9 +68,12 @@ static SharedFile<E> *
 new_shared_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
  check_file_compatibility(ctx, rctx, mf);

-  SharedFile<E> *file = SharedFile<E>::create(ctx, mf);
+  SharedFile<E> *file = new SharedFile<E>(ctx, mf);
+  ctx.dso_pool.emplace_back(file);
  file->priority = ctx.file_priority++;
-  file->is_alive = !rctx.as_needed;
+  file->as_needed = rctx.as_needed;
+
+  assert(rctx.tg);
  rctx.tg->run([file, &ctx] { file->parse(ctx); });
  if (ctx.arg.trace)
    Out(ctx) << "trace: " << *file;
@@ -195,7 +111,7 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
    }
    return;
  case FileType::TEXT:
-    Script(ctx, rctx, mf).parse_linker_script();
+    Script(ctx, mf, &rctx).parse_linker_script();
    return;
  case FileType::GCC_LTO_OBJ:
  case FileType::LLVM_BITCODE:
@@ -233,7 +149,7 @@ detect_machine_type(Context<E> &ctx, std::vector<std::string> args) {
      if (MappedFile *mf = open_file(ctx, arg))
        if (get_file_type(ctx, mf) == FileType::TEXT)
          if (std::string_view target =
-              Script(ctx, rctx, mf).get_script_output_type();
+              Script(ctx, mf, &rctx).get_script_output_type();
              !target.empty())
            return target;
    }
@@ -249,7 +165,7 @@ MappedFile *open_library(Context<E> &ctx, ReaderContext &rctx, std::string path)
    return nullptr;

  std::string_view target = get_machine_type(ctx, rctx, mf);
-  if (!target.empty() && target != E::target_name) {
+  if (!target.empty() && target != E::name) {
    Warn(ctx) << path << ": skipping incompatible file: " << target
              << " (e_machine " << (int)E::e_machine << ")";
    return nullptr;
@@ -331,20 +247,36 @@ static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
    }
  }

-  if (ctx.objs.empty())
+  if (ctx.objs.empty() && ctx.dsos.empty())
    Fatal(ctx) << "no input files";

+  if (rctx.static_ || ctx.arg.relocatable) {
+    ctx.arg.static_ = true;
+    ctx.arg.dynamic_linker = "";
+  }
+
  tg.wait();
 }

 template <typename E>
 static bool has_lto_obj(Context<E> &ctx) {
  for (ObjectFile<E> *file : ctx.objs)
-    if (file->is_alive && (file->is_lto_obj || file->is_gcc_offload_obj))
+    if (file->is_reachable && (file->is_lto_obj || file->is_gcc_offload_obj))
      return true;
  return false;
 }

+template <typename E>
+static i64 get_thread_count(Context<E> &ctx) {
+  if (ctx.arg.thread_count.has_value())
+    return *ctx.arg.thread_count;
+
+  // mold doesn't scale well with too many threads, so limit it to 32.
+  int n = tbb::global_control::active_value(
+    tbb::global_control::max_allowed_parallelism);
+  return std::min(n, 32);
+}
+
 template <typename E>
 int mold_main(int argc, char **argv) {
  Context<E> ctx;
@@ -353,6 +285,12 @@ int mold_main(int argc, char **argv) {
  if (argc >= 2 && (argv[1] == "-run"sv || argv[1] == "--run"sv))
    process_run_subcommand(ctx, argc, argv);

+  // parse_nonpositional_args() may chdir(2) for -C. If we end up
+  // restarting in redo_main(), we need to re-enter from the original
+  // directory so relative paths (e.g. response files) still resolve.
+  std::error_code ec;
+  std::filesystem::path orig_cwd = std::filesystem::current_path(ec);
+
  // Parse non-positional command line options
  ctx.cmdline_args = expand_response_files(ctx, argv);
  std::vector<std::string> file_args = parse_nonpositional_args(ctx);
@@ -361,28 +299,25 @@ int mold_main(int argc, char **argv) {
  if (ctx.arg.emulation.empty())
    ctx.arg.emulation = detect_machine_type(ctx, file_args);

-  // Redo if -m is not x86-64.
-  if constexpr (is_x86_64<E>)
-    if (ctx.arg.emulation != X86_64::target_name)
-      return redo_main(ctx, argc, argv);
+  // Redo if -m does not match with our speculation.
+  if (ctx.arg.emulation != E::name) {
+    if (!ec)
+      std::filesystem::current_path(orig_cwd, ec);
+    return redo_main<E>(ctx.arg.emulation, argc, argv);
+  }

  Timer t_all(ctx, "all");

  install_signal_handler();

-  if (!ctx.arg.directory.empty())
-    if (chdir(ctx.arg.directory.c_str()) == -1)
-      Fatal(ctx) << "chdir failed: " << ctx.arg.directory
-                 << ": " << errno_string();
-
  // Fork a subprocess unless --no-fork is given.
  if (ctx.arg.fork)
-    fork_child();
+    fork_child<E>();

  acquire_global_lock();

-  tbb::global_control tbb_cont(tbb::global_control::max_allowed_parallelism,
-                               ctx.arg.thread_count);
+  ctx.global_limit.emplace(tbb::global_control::max_allowed_parallelism,
+                           get_thread_count(ctx));

  // Handle --wrap options if any.
  for (std::string_view name : ctx.arg.wrap)
@@ -399,6 +334,13 @@ int mold_main(int argc, char **argv) {
  // Parse input files
  read_input_files(ctx, file_args);

+  // The entry point is given by -e, by an ENTRY command in a linker
+  // script, or defaults to _start
+  if (!ctx.arg.entry)
+    ctx.arg.entry = get_symbol(ctx, "_start");
+  ctx.arg.entry->gc_root = true;
+  ctx.arg.undefined.push_back(ctx.arg.entry);
+
  // Uniquify shared object files by soname
  {
    std::unordered_set<std::string_view> seen;
@@ -407,6 +349,10 @@ int mold_main(int argc, char **argv) {
    });
  }

+  // Handle -repro
+  if (ctx.arg.repro)
+    write_repro_file(ctx);
+
  Timer t_before_copy(ctx, "before_copy");

  // Apply -exclude-libs
@@ -428,12 +374,22 @@ int mold_main(int argc, char **argv) {

  // Now that we know which object files are to be included to the
  // final output, we can remove unnecessary files.
-  std::erase_if(ctx.objs, [](InputFile<E> *file) { return !file->is_alive; });
-  std::erase_if(ctx.dsos, [](InputFile<E> *file) { return !file->is_alive; });
+  std::erase_if(ctx.objs, [](InputFile<E> *file) { return !file->is_reachable; });
+  std::erase_if(ctx.dsos, [](InputFile<E> *file) { return !file->is_reachable; });
+
+  // Remove input sections that match a /DISCARD/ pattern and
+  // compute, for each input section, the output section description
+  // it matches. This runs before mergeable sections are split
+  // because the patterns match the original input sections.
+  apply_script_discards(ctx);
+  match_script_sections(ctx);

  // Parse .eh_frame section contents.
  parse_eh_frame_sections(ctx);

+  // Parse .sframe section contents.
+  parse_sframe_sections(ctx);
+
  // Split mergeable section contents into section pieces.
  create_merged_sections(ctx);

@@ -448,6 +404,9 @@ int mold_main(int argc, char **argv) {
  // Create .bss sections for common symbols.
  convert_common_symbols(ctx);

+  // Define symbols PROVIDE'd by linker scripts.
+  add_provided_symbols(ctx);
+
  // Apply version scripts.
  apply_version_script(ctx);

@@ -457,10 +416,23 @@ int mold_main(int argc, char **argv) {
  // Set is_imported and is_exported bits for each symbol.
  compute_import_export(ctx);

+  // Make sure that there's no duplicate symbol
+  if (!ctx.arg.allow_multiple_definition)
+    check_duplicate_symbols(ctx);
+
+  // Handle --zero-to-bss, which converts data sections containing only
+  // zeros into BSS.
+  if (ctx.arg.zero_to_bss)
+    convert_zero_to_bss(ctx);
+
  // Set "address-taken" bits for input sections.
  if (ctx.arg.icf)
    compute_address_significance(ctx);

+  // Handle PPC64-specific .opd sections.
+  if constexpr (is_ppc64v1<E>)
+    ppc64v1_rewrite_opd(ctx);
+
  // Garbage-collect unreachable sections.
  if (ctx.arg.gc_sections)
    gc_sections(ctx);
@@ -472,19 +444,13 @@ int mold_main(int argc, char **argv) {
  // Create linker-synthesized sections such as .got or .plt.
  create_synthetic_sections(ctx);

-  // Make sure that there's no duplicate symbol
-  if (!ctx.arg.allow_multiple_definition)
-    check_duplicate_symbols(ctx);
-
+  // Handle --no-allow-shlib-undefined
  if (!ctx.arg.allow_shlib_undefined)
    check_shlib_undefined(ctx);

  // Warn if symbols with different types are defined under the same name.
  check_symbol_types(ctx);

-  if constexpr (is_ppc64v1<E>)
-    ppc64v1_rewrite_opd(ctx);
-
  // Bin input sections into output sections.
  create_output_sections(ctx);

@@ -527,10 +493,6 @@ int mold_main(int argc, char **argv) {
  if (ctx.arg.print_dependencies)
    print_dependencies(ctx);

-  // Handle -repro
-  if (ctx.arg.repro)
-    write_repro_file(ctx);
-
  // Handle --require-defined
  for (Symbol<E> *sym : ctx.arg.require_defined)
    if (!sym->file)
@@ -553,16 +515,7 @@ int mold_main(int argc, char **argv) {
    shuffle_sections(ctx);

  // Copy string referred by .dynamic to .dynstr.
-  for (SharedFile<E> *file : ctx.dsos)
-    ctx.dynstr->add_string(file->soname);
-  for (std::string_view str : ctx.arg.auxiliary)
-    ctx.dynstr->add_string(str);
-  for (std::string_view str : ctx.arg.filter)
-    ctx.dynstr->add_string(str);
-  if (!ctx.arg.rpaths.empty())
-    ctx.dynstr->add_string(ctx.arg.rpaths);
-  if (!ctx.arg.soname.empty())
-    ctx.dynstr->add_string(ctx.arg.soname);
+  add_dynamic_strings(ctx);

  if constexpr (is_ppc64v1<E>)
    ppc64v1_scan_symbols(ctx);
@@ -578,24 +531,23 @@ int mold_main(int argc, char **argv) {
  // create as few segments as possible.
  sort_output_sections(ctx);

-  if (!ctx.arg.separate_debug_file.empty())
+  // Handle --separate-debug-file.
+  if (ctx.gnu_debuglink)
    separate_debug_sections(ctx);

  // Compute sizes of output sections while assigning offsets
  // within an output section to input sections.
  compute_section_sizes(ctx);

-  // If --packed_dyn_relocs=relr was given, base relocations are stored
-  // to a .relr.dyn section in a compressed form. Construct a compressed
-  // relocations now so that we can fix section sizes and file layout.
-  if (ctx.arg.pack_dyn_relocs_relr)
-    construct_relr(ctx);
-
  // Reserve a space for dynamic symbol strings in .dynstr and sort
  // .dynsym contents if necessary. Beyond this point, no symbol will
  // be added to .dynsym.
  sort_dynsyms(ctx);

+  // Sort .debug_info contents so that DWARF32 debug info precedes that of
+  // DWARF64. This is to mitigate the possibility of a relocation overflow.
+  sort_debug_info_sections(ctx);
+
  // Print reports about undefined symbols, if needed.
  if (ctx.arg.unresolved_symbols == UNRESOLVED_ERROR)
    report_undef_errors(ctx);
@@ -607,21 +559,26 @@ int mold_main(int argc, char **argv) {
  // Fill .gnu.version_r section contents.
  ctx.verneed->construct(ctx);

-  // Compute .symtab and .strtab sizes for each file.
-  if (!ctx.arg.strip_all)
-    create_output_symtab(ctx);
-
  // .eh_frame is a special section from the linker's point of view,
  // as its contents are parsed and reconstructed by the linker,
  // unlike other sections that are regarded as opaque bytes.
  // Here, we construct output .eh_frame contents.
  ctx.eh_frame->construct(ctx);

+  // .sframe is likewise parsed and reconstructed by the linker. Build
+  // the merged, PC-sorted output .sframe.
+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
  // If --emit-relocs is given, we'll copy relocation sections from input
  // files to an output file.
  if (ctx.arg.emit_relocs)
    create_reloc_sections(ctx);

+  // Compute .symtab and .strtab sizes for each file.
+  if (!ctx.arg.strip_all)
+    create_output_symtab(ctx);
+
  // Compute the section header values for all sections.
  compute_section_headers(ctx);

@@ -637,6 +594,14 @@ int mold_main(int argc, char **argv) {
    filesize = set_osec_offsets(ctx);
  }

+  // We've created range extension thunks with a pessimistive assumption
+  // that all out-of-section references are out of range. Now that we know
+  // the addresses of all sections,, we can eliminate excessive thunks.
+  if constexpr (needs_thunk<E>) {
+    remove_redundant_thunks(ctx);
+    filesize = set_osec_offsets(ctx);
+  }
+
  if constexpr (is_arm32<E>) {
    if (ctx.extra.exidx) {
      ctx.extra.exidx->remove_duplicate_entries(ctx);
@@ -653,14 +618,26 @@ int mold_main(int argc, char **argv) {
  // GOT or PLT addresses have a correct final value.

  // If --compress-debug-sections is given, compress .debug_* sections
-  // using zlib.
-  if (ctx.arg.compress_debug_sections != COMPRESS_NONE) {
+  // using zlib or zstd.
+  if (ctx.arg.compress_debug_sections != ELFCOMPRESS_NONE) {
    compress_debug_sections(ctx);
    filesize = set_osec_offsets(ctx);
  }

+  // Gather thunk symbols and attach them to themselves.
+  if constexpr (needs_thunk<E>)
+    gather_thunk_addresses(ctx);
+
+  // Re-finalize layout. fix_synthetic_symbols above may have changed
+  // addends for dynamic relocations referencing synthetic symbols, which
+  // can shift the encoded size of .rela.dyn under --pack-dyn-relocs=android.
+  // For other modes this is a cheap no-op convergence.
+  filesize = set_osec_offsets(ctx);
+
  // At this point, both memory and file layouts are fixed.

+  ctx.reldyn->update_shdr(ctx);
+
  t_before_copy.stop();

  // Create an output file
@@ -672,13 +649,22 @@ int mold_main(int argc, char **argv) {
  // Copy input sections to the output file and apply relocations.
  copy_chunks(ctx);

+  if constexpr (is_arm32be<E>)
+    arm32be_swap_bytes(ctx);
+
  if constexpr (is_x86_64<E>)
    if (ctx.arg.z_rewrite_endbr)
      rewrite_endbr(ctx);

  // Dynamic linker works better with sorted .rela.dyn section,
  // so we sort them.
-  ctx.reldyn->sort(ctx);
+  sort_reldyn(ctx);
+
+  // .gdb_index's contents cannot be constructed before applying
+  // relocations to other debug sections. We have relocated debug
+  // sections now, so write the .gdb_index section.
+  if (ctx.gdb_index && !ctx.gnu_debuglink)
+    write_gdb_index(ctx);

  // .note.gnu.build-id section contains a cryptographic hash of the
  // entire output file. Now that we wrote everything except build-id,
@@ -686,13 +672,7 @@ int mold_main(int argc, char **argv) {
  if (ctx.buildid)
    write_build_id(ctx);

-  // .gdb_index's contents cannot be constructed before applying
-  // relocations to other debug sections. We have relocated debug
-  // sections now, so write the .gdb_index section.
-  if (ctx.gdb_index && ctx.arg.separate_debug_file.empty())
-    write_gdb_index(ctx);
-
-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
    write_gnu_debuglink(ctx);

  t_copy.stop();
@@ -713,7 +693,7 @@ int mold_main(int argc, char **argv) {
  if (ctx.arg.print_map)
    print_map(ctx);

-  if (!ctx.arg.separate_debug_file.empty())
+  if (ctx.gnu_debuglink)
    write_separate_debug_file(ctx);

  // Show stats numbers
@@ -726,7 +706,7 @@ int mold_main(int argc, char **argv) {
  std::cout << std::flush;
  std::cerr << std::flush;

-  notify_parent();
+  notify_parent<E>();
  release_global_lock();

  if (ctx.arg.quick_exit)
@@ -4,8 +4,8 @@
 #include <iomanip>
 #include <ios>
 #include <sstream>
+#include <tbb/concurrent_hash_map.h>
 #include <tbb/parallel_for_each.h>
-#include <unordered_map>

 namespace mold {

@@ -13,29 +13,18 @@ template <typename E>
 using Map =
  tbb::concurrent_hash_map<InputSection<E> *, std::vector<Symbol<E> *>>;

-template <typename E>
-static std::unique_ptr<std::ofstream> open_output_file(Context<E> &ctx) {
-  std::unique_ptr<std::ofstream> file(new std::ofstream);
-  file->open(ctx.arg.Map.c_str());
-  if (!file->is_open())
-    Fatal(ctx) << "cannot open " << ctx.arg.Map << ": " << errno_string();
-  return file;
-}
-
 template <typename E>
 static Map<E> get_map(Context<E> &ctx) {
  Map<E> map;

  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    for (Symbol<E> *sym : file->symbols) {
-      if (sym->file != file || sym->get_type() == STT_SECTION)
-        continue;
-
-      if (InputSection<E> *isec = sym->get_input_section()) {
-        assert(file == &isec->file);
-        typename Map<E>::accessor acc;
-        map.insert(acc, {isec, {}});
-        acc->second.push_back(sym);
+      if (sym->file == file && sym->get_type() != STT_SECTION) {
+        if (InputSection<E> *isec = sym->get_input_section()) {
+          typename Map<E>::accessor acc;
+          map.insert(acc, {isec, {}});
+          acc->second.push_back(sym);
+        }
      }
    }
  });
@@ -44,10 +33,8 @@ static Map<E> get_map(Context<E> &ctx) {
    return map;

  tbb::parallel_for(map.range(), [](const typename Map<E>::range_type &range) {
-    for (auto it = range.begin(); it != range.end(); it++) {
-      std::vector<Symbol<E> *> &vec = it->second;
-      sort(vec, [](Symbol<E> *a, Symbol<E> *b) { return a->value < b->value; });
-    }
+    for (auto &[k, v] : range)
+      ranges::stable_sort(v, {}, &Symbol<E>::value);
  });
  return map;
 }
@@ -57,11 +44,14 @@ void print_map(Context<E> &ctx) {
  Timer t(ctx, "print_map");

  std::ostream *out = &std::cout;
-  std::unique_ptr<std::ofstream> file;
+  std::ofstream file;

-  if (!ctx.arg.Map.empty()) {
-    file = open_output_file(ctx);
-    out = file.get();
+  if (!ctx.arg.Map.empty() && ctx.arg.Map != "-") {
+    file.open(ctx.arg.Map);
+    if (file.fail())
+      Fatal(ctx) << "--print-map: cannot open " << ctx.arg.Map << ": "
+                 << errno_string();
+    out = &file;
  }

  // Construct a section-to-symbol map.
@@ -70,28 +60,32 @@ void print_map(Context<E> &ctx) {
  // Print a mapfile.
  *out << "               VMA       Size Align Out     In      Symbol\n";

-  for (Chunk<E> *osec : ctx.chunks) {
+  for (Chunk<E> *chunk : ctx.chunks) {
    *out << std::showbase
-         << std::setw(18) << std::hex << (u64)osec->shdr.sh_addr << std::dec
-         << std::setw(11) << (u64)osec->shdr.sh_size
-         << std::setw(6) << (u64)osec->shdr.sh_addralign
-         << " " << osec->name << "\n";
+         << std::setw(18) << std::hex << (u64)chunk->shdr.sh_addr << std::dec
+         << std::setw(11) << (u64)chunk->shdr.sh_size
+         << std::setw(6) << (u64)chunk->shdr.sh_addralign
+         << " " << chunk->name << "\n";

-    if (!osec->to_osec())
+    OutputSection<E> *osec = chunk->to_osec();
+    if (!osec)
      continue;

-    std::span<InputSection<E> *> members = ((OutputSection<E> *)osec)->members;
+    std::span<InputSection<E> *> members = osec->members;
    std::vector<std::string> bufs(members.size());

    tbb::parallel_for((i64)0, (i64)members.size(), [&](i64 i) {
      InputSection<E> *mem = members[i];
      std::ostringstream ss;
-      u64 addr = osec->shdr.sh_addr + mem->offset;
+
+      u64 addr = 0;
+      if (osec->shdr.sh_flags & SHF_ALLOC)
+        addr = osec->shdr.sh_addr + mem->offset;

      ss << std::showbase
         << std::setw(18) << std::hex << addr << std::dec
         << std::setw(11) << (u64)mem->sh_size
-         << std::setw(6) << (1 << (u64)mem->p2align)
+         << std::setw(6) << (1 << mem->p2align)
         << "         " << *mem << "\n";

      typename Map<E>::const_accessor acc;
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"

 namespace mold {

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"

 namespace mold {

@@ -4,8 +4,12 @@
 #include <filesystem>
 #include <sys/file.h>
 #include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/types.h>
+#include <system_error>
+
+#ifdef __linux__
+# include <sys/vfs.h>
+# include <linux/magic.h>
+#endif

 namespace mold {

@@ -40,9 +44,9 @@ class MemoryMappedOutputFile : public OutputFile<E> {
 public:
  MemoryMappedOutputFile(Context<E> &ctx, std::string path, i64 filesize, int perm)
    : OutputFile<E>(path, filesize, true) {
-    std::filesystem::path dir = filepath(path).parent_path();
-    std::string filename = filepath(path).filename().string();
-    std::string tmpfile = dir / ("." + filename + "." + std::to_string(getpid()));
+    std::string pid = std::to_string(getpid());
+    std::string tmpfile =
+      path_dirname(path) / ("." + path_filename(path) + "." + pid);

    this->fd = open_or_create_file(ctx, path, tmpfile, perm);

@@ -55,7 +59,11 @@ public:
    output_tmpfile = (char *)save_string(ctx, tmpfile).data();

 #ifdef __linux__
-    fallocate(this->fd, 0, 0, filesize);
+    // Calling falllocate speeds up later linking passes on ext4,
+    // while it just takes time with not benefits on tmpfs.
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, 0, filesize);
 #endif

    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
@@ -78,12 +86,12 @@ public:
    if (!this->is_unmapped)
      munmap(this->buf, this->filesize);

-    if (this->buf2.empty()) {
+    if (!this->buf2) {
      ::close(this->fd);
    } else {
      FILE *out = fdopen(this->fd, "w");
      fseek(out, 0, SEEK_END);
-      fwrite(&this->buf2[0], this->buf2.size(), 1, out);
+      fwrite(this->buf2, this->buf2_size, 1, out);
      fclose(out);
    }

@@ -111,14 +119,9 @@ OutputFile<E>::open(Context<E> &ctx, std::string path, i64 filesize, int perm) {
  if (path.starts_with('/') && !ctx.arg.chroot.empty())
    path = ctx.arg.chroot + "/" + path_clean(path);

-  bool is_special = false;
-  if (path == "-") {
-    is_special = true;
-  } else {
-    struct stat st;
-    if (stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) != S_IFREG)
-      is_special = true;
-  }
+  std::error_code error;
+  bool is_special = path == "-" ||
+                    (!std::filesystem::is_regular_file(path, error) && !error);

  OutputFile<E> *file;
  if (is_special)
@@ -182,10 +185,10 @@ void LockingOutputFile<E>::close(Context<E> &ctx) {
  if (!this->is_unmapped)
    munmap(this->buf, this->filesize);

-  if (!this->buf2.empty()) {
+  if (this->buf2) {
    FILE *out = fdopen(this->fd, "w");
    fseek(out, 0, SEEK_END);
-    fwrite(&this->buf2[0], this->buf2.size(), 1, out);
+    fwrite(this->buf2, this->buf2_size, 1, out);
    fclose(out);
  }

@@ -1,7 +1,6 @@
 #include "mold.h"

 #include <fcntl.h>
-#include <filesystem>
 #include <windows.h>

 namespace mold {
@@ -45,13 +44,13 @@ public:

    UnmapViewOfFile(this->buf);

-    if (!this->buf2.empty()) {
+    if (this->buf2) {
      if (SetFilePointer(handle, 0, nullptr, FILE_END) == INVALID_SET_FILE_POINTER)
        Fatal(ctx) << this->path << ": SetFilePointer failed: "
                   << GetLastError();

      DWORD written;
-      if (!WriteFile(handle, this->buf2.data(), this->buf2.size(), &written,
+      if (!WriteFile(handle, this->buf2, this->buf2_size, &written,
                     nullptr))
        Fatal(ctx) << this->path << ": WriteFile failed: " << GetLastError();
    }
@@ -50,10 +50,14 @@ static void r_create_synthetic_sections(Context<E> &ctx) {
  ctx.shdr = push(new OutputShdr<E>);
  ctx.eh_frame = push(new EhFrameSection<E>);
  ctx.eh_frame_reloc = push(new EhFrameRelocSection<E>);
+  ctx.sframe = push(new SFrameSection<E>);
+  ctx.sframe_reloc = push(new SFrameRelocSection<E>);
  ctx.strtab = push(new StrtabSection<E>);
  ctx.symtab = push(new SymtabSection<E>);
  ctx.shstrtab = push(new ShstrtabSection<E>);
-  ctx.note_property = push(new NotePropertySection<E>);
+
+  if constexpr (is_x86<E>)
+    ctx.extra.note_property = push(new NotePropertySection<E>);

  if constexpr (is_riscv<E>)
    ctx.extra.riscv_attributes = push(new RiscvAttributesSection<E>);
@@ -109,10 +113,7 @@ template <typename E>
 static void r_claim_unresolved_symbols(Context<E> &ctx) {
  Timer t(ctx, "r_claim_unresolved_symbols");

-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    if (!file->is_alive)
-      return;
-
+  tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
    for (i64 i = file->first_global; i < file->elf_syms.size(); i++) {
      const ElfSym<E> &esym = file->elf_syms[i];
      Symbol<E> &sym = *file->symbols[i];
@@ -162,6 +163,9 @@ void combine_objects(Context<E> &ctx) {

  ctx.eh_frame->construct(ctx);

+  if constexpr (supports_sframe<E>)
+    ctx.sframe->construct(ctx);
+
  create_reloc_sections(ctx);

  create_comdat_group_sections(ctx);
@@ -66,21 +66,18 @@ namespace mold {

 using E = MOLD_TARGET;

-static bool is_resizable(InputSection<E> *isec) {
-  return isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC) &&
-         (isec->shdr().sh_flags & SHF_EXECINSTR);
+template <>
+i64 get_r_delta(InputSection<E> &isec, u64 offset) {
+  std::span<RelocDelta> deltas = isec.extra.r_deltas;
+  auto it = ranges::upper_bound(deltas, offset, std::less_equal<>{},
+                                &RelocDelta::offset);
+  return (it == deltas.begin()) ? 0 : (it - 1)->delta;
 }

 template <>
-void shrink_sections<E>(Context<E> &ctx) {
+void shrink_sections(Context<E> &ctx) {
  Timer t(ctx, "shrink_sections");

-  // True if we can use the 2-byte instructions. This is usually true on
-  // Unix because RV64GC is generally considered the baseline hardware.
-  bool use_rvc = false;
-  if constexpr (is_riscv<E>)
-    use_rvc = get_eflags(ctx) & EF_RISCV_RVC;
-
  // Find all relaxable relocations and record how many bytes we can save
  // into r_deltas.
  //
@@ -93,28 +90,18 @@ void shrink_sections<E>(Context<E> &ctx) {
  // them. We scan relocations only once here.
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
-      if (is_resizable(isec.get()))
-        shrink_section(ctx, *isec, use_rvc);
+      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
+        shrink_section(ctx, *isec);
  });

  // Fix symbol values.
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (Symbol<E> *sym : file->symbols) {
-      if (sym->file != file)
-        continue;
-
-      InputSection<E> *isec = sym->get_input_section();
-      if (!isec || isec->extra.r_deltas.empty())
-        continue;
-
-      std::span<const ElfRel<E>> rels = isec->get_rels(ctx);
-      auto it = std::lower_bound(rels.begin(), rels.end(), sym->value,
-                                 [&](const ElfRel<E> &r, u64 val) {
-        return r.r_offset < val;
-      });
-
-      sym->value -= isec->extra.r_deltas[it - rels.begin()];
-    }
+    for (Symbol<E> *sym : file->symbols)
+      if (sym->file == file)
+        if (InputSection<E> *isec = sym->get_input_section())
+          if (isec->shdr().sh_flags & SHF_EXECINSTR)
+            if (i64 delta = get_r_delta(*isec, sym->value))
+              sym->value -= delta;
  });

  // Recompute sizes of executable sections
@@ -126,8 +113,8 @@ void shrink_sections<E>(Context<E> &ctx) {

 // Returns the distance between a relocated place and a symbol.
 template <>
-i64 compute_distance<E>(Context<E> &ctx, Symbol<E> &sym,
-                        InputSection<E> &isec, const ElfRel<E> &rel) {
+i64 compute_distance(Context<E> &ctx, Symbol<E> &sym,
+                     InputSection<E> &isec, const ElfRel<E> &rel) {
  // We handle absolute symbols as if they were infinitely far away
  // because `shrink_section` may increase a distance between a branch
  // instruction and an absolute symbol. Branching to an absolute
@@ -135,10 +122,6 @@ i64 compute_distance<E>(Context<E> &ctx, Symbol<E> &sym,
  if (sym.is_absolute())
    return INT64_MAX;

-  // Likewise, relocations against weak undefined symbols won't be relaxed.
-  if (sym.esym().is_undef_weak())
-    return INT64_MAX;
-
  // Compute a distance between the relocated place and the symbol.
  i64 S = sym.get_addr(ctx);
  i64 A = rel.r_addend;
@@ -1,6 +1,6 @@
-#include "common.h"
+#include "mold.h"

-#include <signal.h>
+#include <csignal>
 #include <tbb/version.h>

 #ifdef __FreeBSD__
@@ -28,8 +28,7 @@ void cleanup() {
 // for the first time.
 //
 // If a disk becomes full as a result of a write to an mmap'ed memory
-// region, the failure of the write is reported as a SIGBUS or structured
-// exeption with code EXCEPTION_IN_PAGE_ERROR on Windows. This
+// region, the failure of the write is reported as a SIGBUS. This
 // signal handler catches that signal and prints out a user-friendly
 // error message. Without this, it is very hard to realize that the
 // disk might be full.
@@ -74,7 +73,7 @@ void install_signal_handler() {
  sigaction(SIGBUS, &action, NULL);

  // OneTBB 2021.9.0 has the interface version 12090.
-  if (TBB_runtime_interface_version() < 12090) {
+  if (TBB_runtime_interface_version() <= 12090) {
    sigabrt_msg = "mold: aborted\n"
      "mold: mold with libtbb version 2021.9.0 or older is known to be unstable "
      "under heavy load. Your libtbb version is " +
@@ -1,4 +1,4 @@
-#include "common.h"
+#include "mold.h"

 #include <windows.h>

@@ -1,8 +1,8 @@
-#include "mold.h"
 #include "config.h"
+#include "mold.h"

+#include <csignal>
 #include <filesystem>
-#include <signal.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
@@ -11,12 +11,12 @@

 namespace mold {

-#ifdef MOLD_X86_64
 static int pipe_write_fd = -1;

 // Exiting from a program with large memory usage is slow --
 // it may take a few hundred milliseconds. To hide the latency,
 // we fork a child and let it do the actual linking work.
+template <typename E>
 void fork_child() {
  int pipefd[2];
  if (pipe(pipefd) == -1) {
@@ -53,6 +53,7 @@ void fork_child() {
  pipe_write_fd = pipefd[1];
 }

+template <typename E>
 void notify_parent() {
  if (pipe_write_fd == -1)
    return;
@@ -62,7 +63,6 @@ void notify_parent() {
  assert(n == 1);
  pipe_write_fd = -1;
 }
-#endif

 template <typename E>
 static std::string find_dso(Context<E> &ctx, std::filesystem::path self) {
@@ -102,11 +102,11 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
  std::string dso_path = find_dso(ctx, self);

  // Set environment variables
-  putenv(strdup(("LD_PRELOAD=" + dso_path).c_str()));
-  putenv(strdup(("MOLD_PATH=" + self).c_str()));
+  setenv("LD_PRELOAD", dso_path.c_str(), 1);
+  setenv("MOLD_PATH", self.c_str(), 1);

-  // If ld, ld.lld or ld.gold is specified, run mold itself
-  if (std::string cmd = filepath(argv[2]).filename();
+  // If ld, ld.lld or ld.gold is specified, run mold instead
+  if (std::string cmd = path_filename(argv[2]);
      cmd == "ld" || cmd == "ld.lld" || cmd == "ld.gold") {
    std::vector<char *> args;
    args.push_back(argv[0]);
@@ -124,6 +124,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {

 using E = MOLD_TARGET;

+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);

 } // namespace mold
@@ -2,10 +2,11 @@

 namespace mold {

-#ifdef MOLD_X86_64
+template <typename E>
 void fork_child() {}
+
+template <typename E>
 void notify_parent() {}
-#endif

 template <typename E>
 [[noreturn]]
@@ -15,6 +16,8 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {

 using E = MOLD_TARGET;

+template void fork_child<E>();
+template void notify_parent<E>();
 template void process_run_subcommand(Context<E> &, int, char **);

 } // namespace mold
@@ -20,7 +20,8 @@
 // we don't need to try too hard to reduce thunk size to the absolute
 // minimum.

-#if MOLD_ARM32 || MOLD_ARM64 || MOLD_PPC32 || MOLD_PPC64V1 || MOLD_PPC64V2
+#if MOLD_ARM32LE || MOLD_ARM32BE || MOLD_ARM64LE || MOLD_ARM64BE || \
+    MOLD_PPC32 || MOLD_PPC64V1 || MOLD_PPC64V2

 #include "mold.h"

@@ -31,81 +32,70 @@ namespace mold {

 using E = MOLD_TARGET;

-// Returns a branch reach in bytes for a given target.
-static consteval i64 max_distance() {
-  // ARM64's branch has 26 bits immediate. The immediate is padded with
-  // implicit two-bit zeros because all instructions are 4 bytes aligned
-  // and therefore the least two bits are always zero. So the branch
-  // operand is effectively 28 bits long. That means the branch range is
-  // [-2^27, 2^27) or PC ± 128 MiB.
-  if (is_arm64<E>)
-    return 1 << 27;
-
-  // ARM32's Thumb branch has 24 bits immediate, and the instructions are
-  // aligned to 2, so it's effectively 25 bits. It's [-2^24, 2^24) or PC ±
-  // 16 MiB.
-  //
-  // ARM32's non-Thumb branches have twice longer range than its Thumb
-  // counterparts, but we conservatively use the Thumb's limitation.
-  if (is_arm32<E>)
-    return 1 << 24;
-
-  // PPC's branch has 24 bits immediate, and the instructions are aligned
-  // to 4, therefore the reach is [-2^25, 2^25) or PC ± 32 MiB.
-  assert(is_ppc<E>);
-  return 1 << 25;
-}
-
-// We create thunks for each 12.8/1.6/3.2 MiB code block for
+// We create thunks for each 32/8/16 MiB code block for
 // ARM64/ARM32/PPC, respectively.
-static constexpr i64 batch_size = max_distance() / 10;
+static constexpr i64 batch_size =
+  (is_arm64<E> ? 32 : is_arm32<E> ? 8 : 16) * 1024 * 1024;

-// We assume that a single thunk group is smaller than 900 KiB.
-static constexpr i64 max_thunk_size = 900 * 1024;
+// We assume that a single thunk group is smaller than 16/4/8 MiB
+// for ARM64/ARM32/PPC, respectively.
+static constexpr i64 max_thunk_size = batch_size / 2;

-static_assert(max_thunk_size / E::thunk_size < ThunkRef::MAX_SYM_IDX);
+// We align thunks to 16 byte boundaries because many processor vendors
+// recommend we align branch targets to 16 byte boundaries for performance
+// reasons.
+static constexpr i64 thunk_align = 16;

 template <typename E>
-static bool is_reachable(Context<E> &ctx, InputSection<E> &isec,
-                         Symbol<E> &sym, const ElfRel<E> &rel) {
-  // We create thunks with a pessimistic assumption that all
-  // out-of-section relocations would be out-of-range.
-  InputSection<E> *isec2 = sym.get_input_section();
-  if (!isec2 || isec.output_section != isec2->output_section)
+static bool
+requires_thunk(Context<E> &ctx, InputSection<E> &isec, const ElfRel<E> &rel,
+               bool first_pass) {
+  if (!is_func_call_rel(rel))
    return false;
+  Symbol<E> &sym = *isec.file.symbols[rel.r_sym];

-  // Even if the target is the same section, we branch to its PLT
-  // if it has one. So a symbol with a PLT is also considered an
-  // out-of-section reference.
-  if (sym.has_plt(ctx))
-    return false;
+  if (first_pass) {
+    // On the first pass, we pessimistically assume that all out-of-section
+    // relocations are out of range.
+    InputSection<E> *isec2 = sym.get_input_section();
+    if (!isec2 || isec.output_section != isec2->output_section)
+      return true;

-  // If the target section is in the same output section but
-  // hasn't got any address yet, that's unreacahble.
-  if (isec2->offset == -1)
-    return false;
+    // If the target section is in the same output section but
+    // hasn't got any address yet, that's unreacahble.
+    if (isec2->offset == -1)
+      return true;
+
+    // Even if the target is the same section, we branch to its PLT
+    // if it has one. So a symbol with a PLT is also considered an
+    // out-of-section reference.
+    if (sym.has_plt(ctx))
+      return true;
+  }

  // Thumb and ARM B instructions cannot be converted to BX, so we
  // always have to make them jump to a thunk to switch processor mode
-  // even if their destinations are within their ranges.
-  if constexpr (is_arm32<E>) {
-    bool is_thumb = sym.get_addr(ctx) & 1;
-    if ((rel.r_type == R_ARM_THM_JUMP24 && !is_thumb) ||
+  // even if their destinations are reachable.
+  if constexpr (is_arm32<E>)
+    if (bool is_thumb = sym.get_addr(ctx) & 1;
        (rel.r_type == R_ARM_JUMP24 && is_thumb) ||
-        (rel.r_type == R_ARM_PLT32 && is_thumb))
-      return false;
-  }
+        (rel.r_type == R_ARM_PLT32 && is_thumb) ||
+        (rel.r_type == R_ARM_THM_JUMP24 && !is_thumb))
+      return true;
+
+  // On PowerPC, all PLT calls go through range extension thunks.
+  if constexpr (is_ppc<E>)
+    if (sym.has_plt(ctx))
+      return true;

  // PowerPC before Power9 lacks PC-relative load/store instructions.
  // Functions compiled for Power9 or earlier assume that r2 points to
  // GOT+0x8000, while those for Power10 uses r2 as a scratch register.
-  // We need to a thunk to recompute r2 for interworking.
-  if constexpr (is_ppc64v2<E>) {
-    if (rel.r_type == R_PPC64_REL24 && !sym.esym().preserves_r2())
-      return false;
-    if (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().uses_toc())
-      return false;
-  }
+  // We need a thunk to recompute r2 for interworking.
+  if constexpr (is_ppc64v2<E>)
+    if ((rel.r_type == R_PPC64_REL24 && !sym.esym().ppc64_preserves_r2()) ||
+        (rel.r_type == R_PPC64_REL24_NOTOC && sym.esym().ppc64_uses_toc()))
+      return true;

  // Compute a distance between the relocated place and the symbol
  // and check if they are within reach.
@@ -113,56 +103,14 @@ static bool is_reachable(Context<E> &ctx, InputSection<E> &isec,
  i64 A = get_addend(isec, rel);
  i64 P = isec.get_addr() + rel.r_offset;
  i64 val = S + A - P;
-  return -max_distance() <= val && val < max_distance();
+  return val < -branch_distance<E> || branch_distance<E> <= val;
 }

-static void reset_thunk(Thunk<E> &thunk) {
-  for (Symbol<E> *sym : thunk.symbols) {
-    sym->extra.thunk_idx = -1;
-    sym->extra.thunk_sym_idx = -1;
-    sym->flags = 0;
-  }
-}
-
-// Scan relocations to collect symbols that need thunks.
-static void scan_rels(Context<E> &ctx, InputSection<E> &isec,
-                      Thunk<E> &thunk, i64 thunk_idx) {
-  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  std::vector<ThunkRef> &thunk_refs = isec.extra.thunk_refs;
-  thunk_refs.resize(rels.size());
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
-    if (!is_func_call_rel(rel))
-      continue;
-
-    // Skip if the symbol is undefined. apply_reloc() will report an error.
-    Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-    if (!sym.file)
-      continue;
-
-    // Skip if the destination is within reach.
-    if (is_reachable(ctx, isec, sym, rel))
-      continue;
-
-    // This relocation needs a thunk. If the symbol is already in a
-    // previous thunk, reuse it.
-    if (sym.extra.thunk_idx != -1) {
-      thunk_refs[i].thunk_idx = sym.extra.thunk_idx;
-      thunk_refs[i].sym_idx = sym.extra.thunk_sym_idx;
-      continue;
-    }
-
-    // Otherwise, add the symbol to the current thunk if it's not
-    // added already.
-    thunk_refs[i].thunk_idx = thunk_idx;
-    thunk_refs[i].sym_idx = -1;
-
-    if (sym.flags.exchange(-1) == 0) {
-      std::scoped_lock lock(thunk.mu);
-      thunk.symbols.push_back(&sym);
-    }
-  }
+template <>
+void Thunk<E>::compute_size() {
+  offsets.clear();
+  for (i64 i = 0; i <= symbols.size(); i++)
+    offsets.push_back(E::thunk_hdr_size + i * E::thunk_size);
 }

 template <>
@@ -182,20 +130,20 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
  // We manage progress using four offsets which increase monotonically.
  // The locations they point to are always A <= B <= C <= D.
  //
-  // Input sections between B and C are in the current batch.
+  // Input sections between B and C are the current batch.
  //
  // A is the input section with the smallest address than can reach
  // from the current batch.
  //
  // D is the input section with the largest address such that the thunk
-  // is reachable from the current batch if it's inserted right before D.
+  // is reachable from the current batch if it's inserted at D.
  //
  //  ................................ <input sections> ............
  //     A    B    C    D
  //                    ^ We insert a thunk for the current batch just before D
  //          <--->       The current batch, which is smaller than BATCH_SIZE
-  //     <-------->       Smaller than MAX_DISTANCE
-  //          <-------->  Smaller than MAX_DISTANCE
+  //     <-------->       Smaller than BRANCH_DISTANCE
+  //          <-------->  Smaller than BRANCH_DISTANCE
  //     <------------->  Reachable from the current batch
  i64 a = 0;
  i64 b = 0;
@@ -210,11 +158,11 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
    // Move D foward as far as we can jump from B to a thunk at D.
    auto d_thunk_end = [&] {
      u64 d_end = align_to(offset, 1 << m[d]->p2align) + m[d]->sh_size;
-      return align_to(d_end, Thunk<E>::alignment) + max_thunk_size;
+      return align_to(d_end, thunk_align) + max_thunk_size;
    };

    while (d < m.size() &&
-           (b == d || d_thunk_end() <= m[b]->offset + max_distance())) {
+           (b == d || d_thunk_end() <= m[b]->offset + branch_distance<E>)) {
      offset = align_to(offset, 1 << m[d]->p2align);
      m[d]->offset = offset;
      offset += m[d]->sh_size;
@@ -229,65 +177,162 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
      c++;

    // Move A forward so that A is reachable from C.
-    i64 c_offset = (c == m.size()) ? offset : m[c]->offset;
-    while (a < b && m[a]->offset + max_distance() < c_offset)
+    i64 c_offset = (c == d) ? offset : m[c]->offset;
+    while (a < b && m[a]->offset + branch_distance<E> < c_offset)
      a++;

    // Erase references to out-of-range thunks.
-    while (t < thunks.size() && thunks[t]->offset < m[a]->offset)
-      reset_thunk(*thunks[t++]);
+    for (; t < thunks.size() && thunks[t]->offset < m[a]->offset; t++)
+      for (Symbol<E> *sym : thunks[t]->symbols)
+        sym->flags = 0;

    // Create a new thunk and place it at D.
-    offset = align_to(offset, Thunk<E>::alignment);
-    i64 thunk_idx = thunks.size();
-    Thunk<E> *thunk = new Thunk<E>(*this, offset);
-    thunks.emplace_back(thunk);
+    offset = align_to(offset, thunk_align);
+    thunks.emplace_back(std::make_unique<Thunk<E>>(*this, offset));
+
+    Thunk<E> &thunk = *thunks.back();
+    std::mutex mu;

    // Scan relocations between B and C to collect symbols that need
    // entries in the new thunk.
-    for (i64 i = b; i < c; i++)
-      scan_rels(ctx, *m[i], *thunk, thunk_idx);
+    tbb::parallel_for(b, c, [&](i64 i) {
+      InputSection<E> &isec = *m[i];
+      for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
+        if (requires_thunk(ctx, isec, rel, true)) {
+          if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+              !sym.flags.test_and_set()) {
+            std::scoped_lock lock(mu);
+            thunk.symbols.push_back(&sym);
+          }
+        }
+      }
+    });
+
+    // Sort symbols added to the thunk to make the output deterministic.
+    ranges::sort(thunk.symbols, {}, [](Symbol<E> *x) {
+      return std::tuple{x->file->priority, x->sym_idx};
+    });

    // Now that we know the number of symbols in the thunk, we can compute
    // the thunk's size.
-    assert(thunk->size() < max_thunk_size);
-    offset += thunk->size();
-
-    // Sort symbols added to the thunk to make the output deterministic.
-    sort(thunk->symbols, [](Symbol<E> *a, Symbol<E> *b) {
-      return std::tuple{a->file->priority, a->sym_idx} <
-             std::tuple{b->file->priority, b->sym_idx};
-    });
-
-    // Assign offsets within the thunk to the symbols.
-    for (i64 i = 0; Symbol<E> *sym : thunk->symbols) {
-      sym->extra.thunk_idx = thunk_idx;
-      sym->extra.thunk_sym_idx = i++;
-    }
-
-    // Scan relocations again to fix symbol offsets in the last thunk.
-    for (i64 i = b; i < c; i++) {
-      std::span<Symbol<E> *> syms = m[i]->file.symbols;
-      std::span<const ElfRel<E>> rels = m[i]->get_rels(ctx);
-      std::span<ThunkRef> thunk_refs = m[i]->extra.thunk_refs;
-
-      for (i64 j = 0; j < rels.size(); j++)
-        if (thunk_refs[j].thunk_idx == thunk_idx)
-          thunk_refs[j].sym_idx = syms[rels[j].r_sym]->extra.thunk_sym_idx;
-    }
+    thunk.compute_size();
+    assert(thunk.size() < max_thunk_size);
+    offset += thunk.size();

    // Move B forward to point to the begining of the next batch.
    b = c;
  }

-  while (t < thunks.size())
-    reset_thunk(*thunks[t++]);
+  // Reset flags for future use
+  for (; t < thunks.size(); t++)
+    for (Symbol<E> *sym : thunks[t]->symbols)
+      sym->flags = 0;

  this->shdr.sh_size = offset;
+}

-  for (InputSection<E> *isec : members)
-    this->shdr.sh_addralign =
-      std::max<u32>(this->shdr.sh_addralign, 1 << isec->p2align);
+// create_range_extension_thunks() creates thunks with a pessimistic
+// assumption that all out-of-section references are out of range.
+// After computing output section addresses, we revisit all thunks to
+// remove unneeded entries from them.
+//
+// We create more thunks than necessary and then eliminate some of
+// them later, instead of just creating thunks at this stage. This is
+// because we can safely shrink sections after assigning addresses to
+// them without worrying about making existing references to thunks go
+// out of range. On the other hand, if we insert thunks after
+// assigning addresses to sections, references to thunks could become
+// out of range due to the new extra gaps for thunks. Thus, the
+// creation of thunks is a two-pass process.
+template <>
+void remove_redundant_thunks(Context<E> &ctx) {
+  Timer t(ctx, "remove_redundant_thunks");
+
+  // Gather output executable sections
+  std::vector<OutputSection<E> *> sections;
+  for (Chunk<E> *chunk : ctx.chunks)
+    if (OutputSection<E> *osec = chunk->to_osec())
+      if (osec->shdr.sh_flags & SHF_EXECINSTR)
+        sections.push_back(osec);
+
+  // Mark all symbols that actually need range extension thunks
+  for (OutputSection<E> *osec : sections) {
+    tbb::parallel_for_each(osec->members, [&](InputSection<E> *isec) {
+      for (const ElfRel<E> &rel : isec->get_rels(ctx))
+        if (requires_thunk(ctx, *isec, rel, false))
+          isec->file.symbols[rel.r_sym]->flags.test_and_set();
+    });
+  }
+
+  // Remove symbols from thunks if they don't actually need range
+  // extension thunks
+  for (OutputSection<E> *osec : sections) {
+    for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks) {
+      std::erase_if(thunk->symbols, [&](Symbol<E> *sym) { return !sym->flags; });
+      thunk->shrink_size(ctx);
+    }
+  }
+
+  // Recompute section sizes
+  tbb::parallel_for_each(sections, [&](OutputSection<E> *osec) {
+    std::span<InputSection<E> *> m = osec->members;
+    std::span<std::unique_ptr<Thunk<E>>> t = osec->thunks;
+    i64 offset = 0;
+
+    while (!m.empty() || !t.empty()) {
+      if (!m.empty() && (t.empty() || m[0]->offset < t[0]->offset)) {
+        offset = align_to(offset, 1 << m[0]->p2align);
+        m[0]->offset = offset;
+        offset += m[0]->sh_size;
+        m = m.subspan(1);
+      } else {
+        offset = align_to(offset, thunk_align);
+        t[0]->offset = offset;
+        offset += t[0]->size();
+        t = t.subspan(1);
+      }
+    }
+    osec->shdr.sh_size = offset;
+  });
+
+  // Reset flags for future use
+  for (OutputSection<E> *osec : sections)
+    for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks)
+      for (Symbol<E> *sym : thunk->symbols)
+        sym->flags = 0;
+}
+
+// When applying relocations, we want to know the address in a reachable
+// range extension thunk for a given symbol. Doing it by scanning all
+// reachable range extension thunks is too expensive.
+//
+// In this function, we create a list of all addresses in range extension
+// thunks for each symbol, so that it is easy to find one.
+//
+// Note that thunk_addrs must be sorted for binary search.
+template <>
+void gather_thunk_addresses(Context<E> &ctx) {
+  Timer t(ctx, "gather_thunk_addresses");
+
+  std::vector<OutputSection<E> *> sections;
+  for (Chunk<E> *chunk : ctx.chunks)
+    if (OutputSection<E> *osec = chunk->to_osec())
+      if (osec->shdr.sh_flags & SHF_EXECINSTR)
+        sections.push_back(osec);
+
+  ranges::stable_sort(sections, {}, [](OutputSection<E> *x) {
+    return x->shdr.sh_addr;
+  });
+
+  for (OutputSection<E> *osec : sections) {
+    for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks) {
+      for (i64 i = 0; i < thunk->symbols.size(); i++) {
+        Symbol<E> &sym = *thunk->symbols[i];
+        u64 addr = thunk->get_addr() + thunk->offsets[i];
+        sym.add_thunk_addr(ctx, addr);
+      }
+    }
+  }
 }

 } // namespace mold
@@ -72,21 +72,46 @@
 //     TLV is sometimes compiled to a function call! The module ID and the
 //     offset are usually stored to GOT as two consecutive words.
 //
-// The last access method is the most generic, so the compiler emits such
-// code by default. But that's the most expensive one, so the linker
-// rewrites instructions if possible so that 3) is relaxed to 2) or even
-// to 1).
-//
 // 1) is called the Local Exec access model. 2) is Initial Exec, and 3) is
 // General Dynamic.
 //
 // There's another little trick that the compiler can use if it knows two
 // TLVs are in the same ELF file (usually in the same file as the code is).
-// In this case, we can call __tls_get_addr() only once with a module ID and
-// the offset 0 to obtain the base address of the ELF file's TLS block. The
-// base address obtained this way is sometimes called Dynamic Thread Pointer
-// or DTP. We can then compute TLVs' addresses by adding their DTP-relative
-// addresses to DTP. This access model is called the Local Dynamic.
+// In this case, we can call __tls_get_addr() only once with the module ID
+// and the offset 0 to obtain the base address of the ELF file's TLS block.
+// The base address obtained this way is sometimes called Dynamic Thread
+// Pointer or DTP. We can then compute TLVs' addresses by adding their
+// DTP-relative addresses to DTP. This access model is called the Local
+// Dynamic.
+//
+// The compiler tries to emit the most efficient code for a TLV access based
+// on compiler command line options and TLV properties as follows:
+//
+//  1. If -fno-PIC or -fPIE is given, and a TLV is defined as a file-local
+//     variable, the compiler knows that it is compiling code for an
+//     executable (i.e. not for a shared library) and the TLV is defined
+//     within the executable. In this case, Local Exec is used to access the
+//     variable.
+//
+//  2. If -fno-PIC or -fPIE is given (i.e. the compiler is compiling code for
+//     a main executable), but a TLV is defined in another translation unit,
+//     then the variable may be defined not in the main executable but in a
+//     shared library. In this case, Initial Exec is used to access the
+//     variable.
+//
+//  3. If -fPIC is given, it may be compiling code for dlopen'able shared
+//     library. In this case, Local Dynamic or General Dynamic is used to
+//     access TLVs.
+//
+// You can also manually control how the compiler emits TLV access code
+// globally with `-ftls-model=<model-name>` or per-variable basis with
+// `__attribute__((tls_model(<model-name>)))`. For example, if you are
+// building a shared library that you have no plan to use with dlopen(), you
+// may want to compile the source files with `-ftls-model=initial-exec` to
+// avoid the cost associated with the General Dynamic access model.
+//
+// The linker may rewrite instructions with a code sequence for a cheaper
+// access model at link-time.
 //
 // === TLS Descriptor access model ===
 //
@@ -18,7 +18,7 @@ endif()

 if(${MACHINE} MATCHES "amd64")
  set(MACHINE x86_64)
-elseif(${MACHINE} MATCHES "i386")
+elseif(${MACHINE} MATCHES "i.86")
  set(MACHINE i686)
 elseif(${MACHINE} MATCHES "arm.*")
  set(MACHINE arm)
@@ -30,14 +30,16 @@ endif()

 if(MOLD_ENABLE_QEMU_TESTS)
  list(APPEND QEMU_ARCHS
-    x86_64 i386 arm aarch64 ppc ppc64 ppc64le sparc64 sh4 s390x
-    riscv64 riscv32 m68k loongarch64)
+    x86_64 i386 arm armeb aarch64 aarch64_be ppc ppc64 ppc64le sparc64
+    sh4 sh4eb s390x riscv64 riscv32 m68k loongarch64)

  LIST(APPEND TRIPLES
    x86_64-linux-gnu
    i686-linux-gnu
    aarch64-linux-gnu
+    aarch64_be-linux-gnu
    arm-linux-gnueabihf
+    armeb-linux-gnueabihf
    riscv64-linux-gnu
    powerpc-linux-gnu
    powerpc64-linux-gnu
@@ -45,6 +47,7 @@ if(MOLD_ENABLE_QEMU_TESTS)
    sparc64-linux-gnu
    s390x-linux-gnu
    sh4-linux-gnu
+    sh4aeb-linux-gnu
    riscv32-linux-gnu
    m68k-linux-gnu
    loongarch64-linux-gnu)
@@ -111,10 +114,18 @@ if(${MACHINE} STREQUAL "aarch64" OR (HAS_qemu-aarch64 AND HAS_aarch64-linux-gnu-
  add_target(aarch64 aarch64-linux-gnu)
 endif()

+if(${MACHINE} STREQUAL "aarch64_be" OR (HAS_qemu-aarch64_be AND HAS_aarch64_be-linux-gnu-gcc))
+  add_target(aarch64_be aarch64_be-linux-gnu)
+endif()
+
 if(${MACHINE} STREQUAL "arm" OR (HAS_qemu-arm AND HAS_arm-linux-gnueabihf-gcc))
  add_target(arm arm-linux-gnueabihf)
 endif()

+if(${MACHINE} STREQUAL "armeb" OR (HAS_qemu-armeb AND HAS_armeb-linux-gnueabihf-gcc))
+  add_target(armeb armeb-linux-gnueabihf)
+endif()
+
 if(${MACHINE} STREQUAL "riscv64" OR (HAS_qemu-riscv64 AND HAS_riscv64-linux-gnu-gcc))
  add_target(riscv64 riscv64-linux-gnu)
 endif()
@@ -167,6 +178,10 @@ if(${MACHINE} STREQUAL "sh4" OR (HAS_qemu-sh4 AND HAS_sh4-linux-gnu-gcc))
  add_target(sh4 sh4-linux-gnu)
 endif()

+if(${MACHINE} STREQUAL "sh4aeb" OR (HAS_qemu-sh4eb AND HAS_sh4aeb-linux-gnu-gcc))
+  add_target(sh4aeb sh4aeb-linux-gnu)
+endif()
+
 if(${MACHINE} STREQUAL "m68k" OR (HAS_qemu-m68k AND HAS_m68k-linux-gnu-gcc))
  add_target(m68k m68k-linux-gnu)
 endif()
@@ -18,5 +18,5 @@ extern char foo;
 int main() { printf("foo=%p\n", &foo); }
 EOF

-! $CC -B. -o $t/exe -pie $t/a.o $t/b.o -Wl,-z,text >& $t/log
-grep -q 'recompile with -fPIC' $t/log
+not $CC -B. -o $t/exe -pie $t/a.o $t/b.o -Wl,-z,text |&
+  grep 'recompile with -fPIC'
@@ -36,4 +36,4 @@ int main() {
 EOF

 $CC -B. -o $t/exe -no-pie $t/a.o $t/b.o
-$QEMU $t/exe | grep -q '^ip=0xa0000.$'
+$QEMU $t/exe | grep '^ip=0xa0000.$'
@@ -4,6 +4,6 @@
 echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc -
 echo 'int main() { return 1; }' | $CC -c -o $t/b.o -xc -

-! $CC -B. -o $t/exe $t/a.o $t/b.o 2> /dev/null || false
+not $CC -B. -o $t/exe $t/a.o $t/b.o 2> /dev/null
 $CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-allow-multiple-definition
 $CC -B. -o $t/exe $t/a.o $t/b.o -Wl,-z,muldefs
@@ -0,0 +1,51 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF > $t/a.c
+#include <stdio.h>
+
+void fn3();
+void fn4();
+
+__attribute__((section(".low"))) void fn1() { printf(" fn1"); fn3(); }
+__attribute__((section(".low"))) void fn2() { printf(" fn2"); fn4(); }
+
+int main() {
+  printf(" main");
+  fn1();
+  printf("\n");
+}
+EOF
+
+cat <<EOF > $t/b.c
+#include <stdio.h>
+
+void fn1();
+void fn2();
+
+__attribute__((section(".high"))) void fn3() { printf(" fn3"); fn2(); }
+__attribute__((section(".high"))) void fn4() { printf(" fn4"); }
+EOF
+
+$CC -c -o $t/c.o $t/a.c
+$CC -c -o $t/d.o $t/b.c
+
+$CC -B. -o $t/exe1 $t/c.o $t/d.o \
+  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x100000000
+$QEMU $t/exe1 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -B. -o $t/exe2 $t/c.o $t/d.o \
+  -Wl,--section-start=.high=0x10000000,--section-start=.low=0x100000000
+$QEMU $t/exe2 | grep 'main fn1 fn3 fn2 fn4'
+
+
+$CC -c -o $t/e.o $t/a.c -fno-PIC -mcmodel=large
+$CC -c -o $t/f.o $t/b.c -fno-PIC -mcmodel=large
+
+$CC -B. -o $t/exe3 $t/e.o $t/f.o -pie \
+  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x400000000
+$QEMU $t/exe3 | grep 'main fn1 fn3 fn2 fn4'
+
+$CC -B. -o $t/exe4 $t/e.o $t/f.o -pie \
+  -Wl,--section-start=.high=0x10000000,--section-start=.low=0x400000000
+$QEMU $t/exe4 | grep 'main fn1 fn3 fn2 fn4'
@@ -0,0 +1,36 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = aarch64 ] || skip
+command -v timeout >/dev/null || skip
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip
+.section .custom_rodata_a, "a", %progbits
+.hidden A
+.global A
+A:
+  .space 8
+
+.section .custom_rodata_b_huge, "a", %progbits
+.balign 16384
+.hidden B
+.global B
+B:
+  .space 8
+
+.section .data, "aw", %progbits
+.rept 7439
+  .quad A
+.endr
+
+.global P_1
+P_1:
+  .quad A
+
+.global P_2
+P_2:
+  .quad B
+EOF
+
+timeout 5s ./mold -m aarch64elf -shared -o $t/b.so $t/a.o \
+  --pack-dyn-relocs=android -z max-page-size=16384
@@ -18,4 +18,4 @@ EOF
 $CC -B. -o $t/exe $t/a.o \
  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000

-$OBJDUMP -dr $t/exe | grep -Fq '<fn1$thunk>:'
+$OBJDUMP -dr $t/exe | grep -E '<fn1\$thunk[0-9]+>:'
@@ -10,7 +10,7 @@ foo:
 EOF

 $CC -B. -shared -o $t/b.so $t/a.o
-readelf -W --dyn-syms $t/b.so | grep foo | grep -q '[VARIANT_PCS]'
+readelf -W --dyn-syms $t/b.so | grep foo | grep -F '[VARIANT_PCS]'

 cat <<EOF | $CC -c -o $t/c.o -xc -
 void foo();
@@ -18,4 +18,4 @@ int main() { foo(); }
 EOF

 $CC -B. -o $t/exe $t/c.o $t/b.so
-readelf -W --dynamic $t/exe | grep -q AARCH64_VARIANT_PCS
+readelf -W --dynamic $t/exe | grep AARCH64_VARIANT_PCS
@@ -14,5 +14,5 @@ EOF

 $CC -o $t/exe -pie $t/a.o $t/b.o >& /dev/null && skip

-! $CC -B. -o $t/exe -pie $t/a.o $t/b.o >& $t/log
-grep -q 'recompile with -fPIC' $t/log
+not $CC -B. -o $t/exe -pie $t/a.o $t/b.o |&
+  grep 'recompile with -fPIC'
@@ -0,0 +1,13 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CXX -o $t/a.o -c -xc++ - -fno-PIC
+#include <stdio.h>
+extern char _etext[];
+int main() {
+  printf("%p\n", _etext);
+}
+EOF
+
+$CXX -B. -o $t/exe $t/a.o -no-pie
+readelf --unwind $t/exe | grep "$($QEMU $t/exe) .*cantunwind"
@@ -18,7 +18,7 @@ EOF
 $CC -B. -o $t/exe $t/a.o \
  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000

-$OBJDUMP -dr $t/exe | grep -F -A7 '<fn1$thunk>:' > $t/log
+$OBJDUMP -dr $t/exe | grep -E -A7 '<fn1\$thunk[0-9]+>:' > $t/log

-grep -Eq 'bx\s+pc' $t/log
-grep -Eq 'add\s+pc, ip, pc' $t/log
+grep -E 'bx\s+pc' $t/log
+grep -E 'add\s+pc, ip, pc' $t/log
@@ -1,8 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc

-echo 'int main() {}' | $CC -c -o /dev/null -xc - -O0 -mthumb >& /dev/null \
-  || skip
+test_cflags -mthumb || skip

 cat <<EOF > $t/a.c
 #include <stdio.h>
@@ -35,11 +34,11 @@ $CC -c -o $t/d.o $t/b.c -O0 -marm

 $CC -B. -o $t/exe $t/c.o $t/d.o \
  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000
-$QEMU $t/exe | grep -q 'main fn1 fn3 fn2 fn4'
+$QEMU $t/exe | grep 'main fn1 fn3 fn2 fn4'

 $CC -c -o $t/e.o $t/a.c -O2 -mthumb
 $CC -c -o $t/f.o $t/b.c -O2 -marm

 $CC -B. -o $t/exe $t/e.o $t/f.o \
  -Wl,--section-start=.low=0x10000000,--section-start=.high=0x20000000
-$QEMU $t/exe | grep -q 'main fn1 fn3 fn2 fn4'
+$QEMU $t/exe | grep 'main fn1 fn3 fn2 fn4'
@@ -0,0 +1,19 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -c -o $t/a.o -xassembler -
+.globl foo
+.data
+foo:
+.word bar(TARGET1)
+EOF
+
+cat <<EOF | $CC -fPIC -c -o $t/b.o -xc -
+#include <stdio.h>
+extern char *foo;
+char bar[] = "Hello world";
+int main() { printf("%s\n", foo); }
+EOF
+
+$CC -B. -o $t/exe -pie $t/a.o $t/b.o
+$QEMU $t/exe | grep 'Hello world'
@@ -0,0 +1,29 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xassembler -
+.syntax unified
+.globl bar
+.thumb
+bar:
+ beq foo-2
+EOF
+
+$OBJDUMP -d $t/a.o | grep -E 'beq\.w' || skip
+
+cat <<EOF | $CC -o $t/b.o -c -xassembler -
+.globl foo, baz
+.thumb
+baz:
+ nop
+foo:
+ nop
+EOF
+
+cat <<EOF | $CC -o $t/c.o -c -xc -
+void bar();
+int main() { bar(); }
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
+$OBJDUMP -d $t/exe | grep -E 'beq\.w.*<baz>'
@@ -0,0 +1,29 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xassembler -
+.syntax unified
+.globl bar
+.thumb
+bar:
+ beq foo-2
+EOF
+
+$OBJDUMP -d $t/a.o | grep -E 'beq\.n' || skip
+
+cat <<EOF | $CC -o $t/b.o -c -xassembler -
+.globl foo, baz
+.thumb
+baz:
+ nop
+foo:
+ nop
+EOF
+
+cat <<EOF | $CC -o $t/c.o -c -xc -
+void bar();
+int main() { bar(); }
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
+$OBJDUMP -d $t/exe | grep -E 'beq\.n.*<baz>'
@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc

-echo 'int foo() { return 0; }' | $CC -o /dev/null -c -xc - -mthumb 2> /dev/null || skip
+test_cflags -mthumb || skip

 cat <<EOF | $CC -o $t/a.o -c -xc - -mthumb
 #include <stdio.h>
@@ -27,4 +27,4 @@ int main() {
 EOF

 $CC -B. -o $t/exe $t/a.o $t/b.o
-$QEMU $t/exe | grep -q 'main foo bar'
+$QEMU $t/exe | grep 'main foo bar'