Bump mold version to 2.3.3

Debian 8 was released in April 2015, which is old enough for
binary distribution. Previously, we used to use Debian 10 (Bullseye),
which was released in July 2019.

.github/workflows/ci.yml

@@ -98,7 +98,7 @@ jobs:
         echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH
         mkdir build
         cd build
-        cmake -DMOLD_ENABLE_QEMU_TESTS=On -DMOLD_ENABLE_QEMU_TESTS_RV32=On -DMOLD_ENABLE_QEMU_TESTS_LOONGARCH=On ..
+        cmake ..
         cmake --build . -j$(nproc)
         ctest . -j$(nproc)
     - name: archive test results

.gitignore

@@ -3,15 +3,7 @@
 *~
 **/.DS_Store
 .cache
-compile_commands.json
-/mold
-/test/tmp
-options.inc
 core
-ld
-ld64
-ld64.mold
 gentoo
-out
 /mold-*.tar.gz
 /build*

CMakeLists.txt

@@ -44,7 +44,7 @@
 # features and behave exactly the same.
 
 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.3.0)
+project(mold VERSION 2.3.3)
 
 include(CMakeDependentOption)
 include(GNUInstallDirs)
@@ -160,12 +160,10 @@ endif()
 #
 # We enable mimalloc by default for 64-bit targets. It doesn't seem to
 # be stable on 32-bit targets.
-include(CheckCXXSourceCompiles)
-check_cxx_source_compiles(
-  "static_assert(sizeof(void *) == 4); int main() {}" P32)
+cmake_dependent_option(
+  MOLD_USE_MIMALLOC "Use mimalloc" ON
+  "CMAKE_SIZEOF_VOID_P EQUAL 8; NOT APPLE; NOT ANDROID; NOT OPENBSD" OFF)
 
-cmake_dependent_option(MOLD_USE_MIMALLOC "Use mimalloc" ON
-  "NOT APPLE; NOT ANDROID; NOT OPENBSD; NOT P32" OFF)
 cmake_dependent_option(
   MOLD_USE_SYSTEM_MIMALLOC "Use system or vendored mimalloc" OFF
   MOLD_USE_MIMALLOC OFF)

README.md

@@ -71,7 +71,7 @@ necessary packages. You may need to run it as root.
 git clone https://github.com/rui314/mold.git
 mkdir mold/build
 cd mold/build
-git checkout v2.3.0
+git checkout v2.3.3
 ../install-build-deps.sh
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ ..
 cmake --build . -j $(nproc)

common/Dockerfile

@@ -1,17 +0,0 @@
-# This dockerfile creates a reproducible build environment for mold.
-#
-# $ docker buildx create --use
-# $ docker buildx build --platform x86_64,aarch64,arm,ppc64le,s390x -t rui314/mold-builder:latest --push .
-
-FROM ubuntu:18.04
-ENV TZ=Europe/London
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common && \
-  add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
-  apt-get update && \
-  apt-get install -y --no-install-recommends build-essential wget libstdc++-11-dev zlib1g-dev gcc-10 g++-10 python3 libssl-dev && \
-  \
-  mkdir /cmake && cd /cmake && \
-  wget -O- -q https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz | tar --strip-components=1 -xzf - && \
-  ./bootstrap --parallel=$(nproc) && make -j$(nproc) && make -j$(nproc) install && \
-  rm -rf /var/lib/apt/lists/* /cmake

common/common.h

@@ -23,6 +23,7 @@
 #include <sys/types.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
+#include <tbb/parallel_for.h>
 #include <vector>
 
 #ifdef _WIN32
@@ -639,15 +640,61 @@ public:
     return {nullptr, false};
   }
 
-  const char *get_key(i64 idx) {
-    return entries[idx].key;
-  }
-
   i64 get_idx(T *value) const {
     uintptr_t addr = (uintptr_t)value - (uintptr_t)value % sizeof(Entry);
     return (Entry *)addr - entries;
   }
 
+  // Return a list of map entries sorted in a deterministic order.
+  std::vector<Entry *> get_sorted_entries(i64 shard_idx) {
+    if (nbuckets == 0)
+      return {};
+
+    i64 shard_size = nbuckets / NUM_SHARDS;
+    i64 begin = shard_idx * shard_size;
+    i64 end = begin + shard_size;
+
+    i64 sz = 0;
+    for (i64 i = begin; i < end; i++)
+      if (entries[i].key)
+        sz++;
+
+    std::vector<Entry *> vec;
+    vec.reserve(sz);
+
+    // Since the shard is circular, we need to handle the last entries
+    // as if they were next to the first entries.
+    while (entries[end - 1].key)
+      vec.push_back(entries + --end);
+
+    // Find entries contiguous in the buckets and sort them.
+    i64 last = 0;
+    for (i64 i = begin; i < end;) {
+      while (i < end && entries[i].key)
+        vec.push_back(entries + i++);
+
+      std::sort(vec.begin() + last, vec.end(), [](Entry *a, Entry *b) {
+        if (a->keylen != b->keylen)
+          return a->keylen < b->keylen;
+        return memcmp(a->key, b->key, a->keylen) < 0;
+      });
+
+      last = vec.size();
+
+      while (i < end && !entries[i].key)
+        i++;
+    }
+    return vec;
+  }
+
+  std::vector<Entry *> get_sorted_entries_all() {
+    std::vector<std::vector<Entry *>> vec(NUM_SHARDS);
+    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
+      vec[i] = get_sorted_entries(i);
+    });
+    return flatten(vec);
+  }
+
   static constexpr i64 MIN_NBUCKETS = 2048;
   static constexpr i64 NUM_SHARDS = 16;
   static constexpr i64 MAX_RETRY = 128;

dist.sh

@@ -1,11 +1,43 @@
 #!/bin/bash
-# This script creates a mold binary distribution. The output is
-# written in this directory as `mold-$version-$arch-linux.tar.gz`
-# (e.g. `mold-1.0.3-x86_64-linux.tar.gz`).
 #
-# The mold executable created by this script is statically-linked to
-# libstdc++ and libcrypto but dynamically-linked to libc, libm, libz
-# and librt, as they almost always exist on any Linux systems.
+# This script creates a mold binary distribution. The output is written in
+# this directory as `mold-$version-$arch-linux.tar.gz` (e.g.,
+# `mold-1.0.3-x86_64-linux.tar.gz`).
+#
+# The mold executable created by this script is statically linked to
+# libstdc++ but dynamically linked to libc, libm, libz, and librt, as
+# these libraries almost always exist on any Linux system. We can't
+# statically link libc because doing so would disable dlopen(), which is
+# necessary to open the LTO linker plugin.
+#
+# This script aims to produce reproducible outputs. That means if you run
+# the script twice on the same git commit, it should produce bit-by-bit
+# identical binary files. This property is crucial as a countermeasure
+# against supply chain attacks. With this, you can verify that the binary
+# files distributed on the GitHub release pages are created from the
+# commit with release tags by rebuilding the binaries yourself.
+#
+# Debian provides snapshot.debian.org to host all historical binary
+# packages. We use it to construct Docker images pinned to a
+# particular timestamp.
+#
+# We aim to use a reasonably old Debian version because we'll dynamically
+# link glibc to mold, and a binary linked against a newer version of glibc
+# won't work on a system with an older version of glibc.
+#
+# We need GCC 10 or newer to build mold. If GCC 10 is not available on an
+# old Debian version, we'll build it ourselves.
+#
+# You may need to run the following command to use Docker with Qemu:
+#
+#  $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+set -e -x
+
+usage() {
+  echo "Usage: $0 [ x86_64 | aarch64 | arm | riscv64 | ppc64le | s390x ]"
+  exit 1
+}
 
 case $# in
 0)
@@ -16,25 +48,93 @@ case $# in
   arch="$1"
   ;;
 *)
-  echo "Usage: $0 [ x86_64 | aarch64 | arm | ppc64le | s390x ]"
-  exit 1
+  usage
 esac
 
-echo "$arch" | grep -Eq '^(x86_64|aarch64|arm|ppc64le|s390x)$' || \
-  { echo "Error: no docker image for $arch"; exit 1; }
+echo "$arch" | grep -Eq '^(x86_64|aarch64|arm|riscv64|ppc64le|s390x)$' || usage
 
+image=mold-builder-$arch
 version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' $(dirname $0)/CMakeLists.txt)
 dest=mold-$version-$arch-linux
-set -e -x
 
-docker run --platform linux/$arch -i --rm -v "$(pwd):/mold" \
-  -e "OWNER=$(id -u):$(id -g)" rui314/mold-builder:latest \
-  bash -c "mkdir /tmp/build &&
-cd /tmp/build &&
-cmake -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 -DMOLD_MOSTLY_STATIC=On -DCMAKE_BUILD_TYPE=Release /mold &&
-cmake --build . -j\$(nproc) &&
-[ $arch = arm ] || ctest -j\$(nproc) &&
+# Create a Docker image.
+case $arch in
+x86_64)
+  # Debian 8 (Jessie) released in April 2015
+  cat <<EOF | docker build --platform $arch -t $image -
+FROM debian:jessie-20210326
+ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
+RUN sed -i -e '/^deb/d' -e 's/^# //g' /etc/apt/sources.list && \
+  echo 'Acquire { Retries "10"; http::timeout "10"; Check-Valid-Until "false"; };' > /etc/apt/apt.conf.d/80-retries && \
+  apt-get update && \
+  apt-get install -y --force-yes --no-install-recommends wget bzip2 file make autoconf gcc g++ libssl-dev && \
+  rm -rf /var/lib/apt/lists
+
+# Build CMake 3.27
+RUN mkdir -p /build/cmake && \
+  cd /build/cmake && \
+  wget -O- --no-check-certificate https://cmake.org/files/v3.27/cmake-3.27.7.tar.gz | tar xzf - --strip-components=1 && \
+  ./bootstrap --parallel=$(nproc) && \
+  make -j$(nproc) && \
+  make install && \
+  rm -rf /build
+
+# Build GCC 10
+RUN mkdir -p /build/gcc && \
+  cd /build/gcc && \
+  wget -O- http://ftp.gnu.org/gnu/gcc/gcc-10.5.0/gcc-10.5.0.tar.gz | tar xzf - --strip-components=1 && \
+  mkdir isl gmp mpc mpfr && \
+  wget -O- http://gcc.gnu.org/pub/gcc/infrastructure/isl-0.18.tar.bz2 | tar xjf - --strip-components=1 -C isl && \
+  wget -O- http://ftp.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2 | tar xjf - --strip-components=1 -C gmp && \
+  wget -O- http://ftp.gnu.org/gnu/mpc/mpc-1.2.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
+  wget -O- http://ftp.gnu.org/gnu/mpfr/mpfr-4.1.0.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
+  ./configure --prefix=/usr --enable-languages=c,c++ --disable-bootstrap --disable-multilib && \
+  make -j$(nproc) && \
+  make install && \
+  ln -sf /usr/lib64/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6 && \
+  rm -rf /build
+EOF
+  ;;
+aarch64 | arm | ppc64le | s390x)
+  # Debian 10 (Bullseye) released in July 2019
+  cat <<EOF | docker build --platform $arch -t $image -
+FROM debian:bullseye-20231030
+ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
+RUN sed -i -e '/^deb/d' -e 's/^# //g' /etc/apt/sources.list && \
+  echo 'Acquire { Retries "10"; http::timeout "10"; Check-Valid-Until "false"; };' > /etc/apt/apt.conf.d/80-retries && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 cmake && \
+  ln -sf /usr/bin/gcc-10 /usr/bin/cc && \
+  ln -sf /usr/bin/g++-10 /usr/bin/c++ && \
+  rm -rf /var/lib/apt/lists
+EOF
+  ;;
+riscv64)
+  # snapshot.debian.org is not available for RISC-V binaries
+  cat <<EOF | docker build --platform $arch -t $image -
+FROM riscv64/debian:unstable-20231030
+ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends build-essential gcc-12 g++-12 cmake && \
+  ln -sf /usr/bin/gcc-12 /usr/bin/cc && \
+  ln -sf /usr/bin/g++-12 /usr/bin/c++ && \
+  rm -rf /var/lib/apt/lists
+EOF
+  ;;
+esac
+
+# We use the timestamp of the last Git commit as the file timestamp
+# for build artifacts.
+timestamp="$(git log -1 --format=%ci)"
+
+# Build mold in a container.
+docker run --platform linux/$arch -i --rm -v "$(realpath $(dirname $0)):/mold" $image \
+  bash -c "mkdir -p /build/mold &&
+cd /build/mold &&
+cmake -DCMAKE_BUILD_TYPE=Release -DMOLD_MOSTLY_STATIC=On /mold &&
+cmake --build . -j$(nproc) &&
+ctest -j$(nproc) &&
 cmake --install . --prefix $dest --strip &&
-tar czf /mold/$dest.tar.gz $dest &&
-cp mold mold-wrapper.so /mold &&
-chown \$OWNER /mold/mold /mold/mold-wrapper.so /mold/$dest.tar.gz"
+find $dest -print | xargs touch --no-dereference --date='$timestamp' &&
+find $dest -print | sort | tar -cf - --no-recursion --files-from=- | gzip -9nc > /mold/$dest.tar.gz &&
+chown $(id -u):$(id -g) /mold/$dest.tar.gz"

docs/mold.md

@@ -764,22 +764,24 @@ arguments.
 ## ENVIRONMENT VARIABLES
 
 * `MOLD_JOBS`:
-  If this variable is set to `1`, only one process of `mold` runs actively. A
-  mold process invoked while another active mold process is running will wait
-  before doing anything until the active process exits.
+  If this variable is set to `1`, only one `mold` process will run at a time.
+  If a new mold process is initiated while another is already active, the new
+  process will wait until the active one completes before starting.
 
-  The purpose of this environment variable is to reduce peak memory usage.
-  Since mold is highly parallelized, there's no point in running it
-  simultaneously. If you run N instances of mold in parallel, it would take N
-  times more time and N times more memory. If you run them serially, it would
-  still take N times more to finish, but their peak memory usage is reduced to
-  normal.
+  The primary reason for this environment variable is to minimize peak memory
+  usage. Since mold is designed to operate with high parallelism, running
+  multiple mold instances simultaneously may not be beneficial. If you execute
+  N instances of mold concurrently, it could require N times the time and N
+  times the memory. On the other hand, running them one after the other might
+  still take N times longer, but the peak memory usage would be the same as
+  running just a single instance.
 
-  If your build system tends to invoke multiple linker processes
-  simultaneously, you may want to try to set this environment variable to
-  `1` to see if it could improve overall performance.
+  If your build system invokes multiple linker processes simultaneously and
+  some of them often get killed due to out-of-memory errors, you might
+  consider setting this environment variable to `1` to see if it addresses the
+  OOM issue.
 
-  Currently, any value other than 1 is silently ignored.
+  Currently, any value other than `1` is silently ignored.
 
 * `MOLD_DEBUG`:
   If this variable is set to a non-empty string, `mold` embeds its

elf/arch-arm32.cc

@@ -200,9 +200,9 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
     0xe08f'e00e, // 1: add lr, pc, lr
     0xe5be'f008, //    ldr pc, [lr, #8]!
     0x0000'0000, // 2: .word .got.plt - 1b - 8
-    0xe320'f000, //    nop
-    0xe320'f000, //    nop
-    0xe320'f000, //    nop
+    0x0000'0000, //    (padding)
+    0x0000'0000, //    (padding)
+    0x0000'0000, //    (padding)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -225,7 +225,7 @@ void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(ul32 *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 // ARM does not use .eh_frame for exception handling. Instead, it uses

elf/arch-arm64.cc

@@ -93,7 +93,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xd503'201f, // nop
   };
 
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, insn, sizeof(insn));

elf/arch-i386.cc

@@ -164,7 +164,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
       0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
     };
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
   } else {
     static const u8 insn[] = {
       0xf3, 0x0f, 0x1e, 0xfb,             // endbr32
@@ -172,7 +172,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
       0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
     };
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_addr(ctx);
+    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx);
   }
 }
 

elf/arch-loongarch.cc

@@ -65,14 +65,12 @@ static u64 hi64(u64 val, u64 pc) {
   // LU52I.D simply set bits to [51:31] and to [63:53], respectively.
   //
   // Compensating all the sign-extensions is a bit complicated.
-  bool x = val & 0x800;
-  bool y = (page(val + 0x800) - page(pc)) & 0x8000'0000;
-
-  if (x && !y)
-    return val - 0x1'0000'0000;
-  if (!x && y)
-    return val + 0x1'0000'0000;
-  return val;
+  u64 x = page(val) - page(pc);
+  if (val & 0x800)
+    x += 0x1000 - 0x1'0000'0000;
+  if (x & 0x8000'0000)
+    x += 0x1'0000'0000;
+  return x;
 }
 
 static u64 higher20(u64 val, u64 pc) {
@@ -174,7 +172,7 @@ void write_plt_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
 template <>
 void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, E::is_64 ? plt_entry_64 : plt_entry_32, E::plt_size);
@@ -325,10 +323,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, (S + A) >> 52);
       break;
     case R_LARCH_PCALA_LO12:
-      write_k12(loc, S + A);
+      // It looks like R_LARCH_PCALA_LO12 is sometimes used for JIRL even
+      // though the instruction takes a 16 bit immediate rather than 12 bits.
+      // It is contrary to the psABI document, but GNU ld has special
+      // code to handle it, so we accept it too.
+      if ((*(ul32 *)loc & 0xfc00'0000) == 0x4c00'0000)
+        write_k16(loc, sign_extend(S + A, 11) >> 2);
+      else
+        write_k12(loc, S + A);
       break;
     case R_LARCH_PCALA_HI20:
-      check(S + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(S + A, P));
       break;
     case R_LARCH_PCALA64_LO20:
@@ -341,7 +345,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, GOT + G + A);
       break;
     case R_LARCH_GOT_PC_HI20:
-      check(GOT + G + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(GOT + G + A, P));
       break;
     case R_LARCH_GOT64_PC_LO20:
@@ -378,7 +381,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, sym.get_gottp_addr(ctx) + A);
       break;
     case R_LARCH_TLS_IE_PC_HI20:
-      check(sym.get_gottp_addr(ctx) + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(sym.get_gottp_addr(ctx) + A, P));
       break;
     case R_LARCH_TLS_IE64_PC_LO20:
@@ -567,11 +569,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
 
-    if (!sym.file) {
-      record_undef_error(ctx, rel);
-      continue;
-    }
-
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
 
@@ -587,6 +584,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       scan_dyn_absrel(ctx, sym, rel);
       break;
     case R_LARCH_B26:
+    case R_LARCH_PCALA_HI20:
       if (sym.is_imported)
         sym.flags |= NEEDS_PLT;
       break;
@@ -620,7 +618,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_LARCH_ABS_LO12:
     case R_LARCH_ABS64_LO20:
     case R_LARCH_ABS64_HI12:
-    case R_LARCH_PCALA_HI20:
     case R_LARCH_PCALA_LO12:
     case R_LARCH_PCALA64_LO20:
     case R_LARCH_PCALA64_HI12:
@@ -658,10 +655,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 template <>
 void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
   static const ul32 insn[] = {
-    0x1a00'000c, // pcalau12i $t0, 0
-    0x02c0'018c, // addi.d    $t0, $t0, 0
+    0x1e00'000c, // pcaddu18i $t0, 0
     0x4c00'0180, // jirl      $zero, $t0, 0
-    0x0340'0000, // nop
   };
 
   static_assert(E::thunk_size == sizeof(insn));
@@ -673,8 +668,8 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
     u64 S = sym->get_addr(ctx);
 
     memcpy(buf, insn, sizeof(insn));
-    write_j20(buf, hi20(S, P));
-    write_k12(buf + 4, S);
+    write_j20(buf, (S - P + 0x20000) >> 18);
+    write_k16(buf + 4, (S - P) >> 2);
 
     buf += sizeof(insn);
     P += sizeof(insn);

elf/arch-m68k.cc

@@ -52,7 +52,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)(buf + 4) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 2;
+  *(ub32 *)(buf + 4) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 2;
 }
 
 template <>

elf/arch-ppc32.cc

@@ -120,7 +120,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
 
   ub32 *loc = (ub32 *)buf;
-  i64 offset = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 8;
+  i64 offset = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 8;
   loc[4] |= higha(offset);
   loc[5] |= lo(offset);
 }

elf/arch-riscv.cc

@@ -198,7 +198,7 @@ void write_plt_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
 template <>
 void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, E::is_64 ? plt_entry_64 : plt_entry_32, E::plt_size);

elf/arch-s390x.cc

@@ -90,7 +90,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)(buf + 2) = (sym.get_got_addr(ctx) - sym.get_plt_addr(ctx)) >> 1;
+  *(ub32 *)(buf + 2) = (sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx)) >> 1;
 }
 
 template <>

elf/arch-sh4.cc

@@ -169,7 +169,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
     static_assert(sizeof(insn) == E::pltgot_size);
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
   } else {
     static const u8 insn[] = {
       0x01, 0xd0, //    mov.l   1f, r0
@@ -181,7 +181,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
     static_assert(sizeof(insn) == E::pltgot_size);
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_addr(ctx);
+    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx);
   }
 }
 

elf/arch-sparc64.cc

@@ -115,7 +115,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, entry, sizeof(entry));
-  *(ub64 *)(buf + 24) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 4;
+  *(ub64 *)(buf + 24) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 4;
 }
 
 template <>

elf/arch-x86-64.cc

@@ -98,7 +98,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 2) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 6;
+  *(ul32 *)(buf + 2) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 6;
 }
 
 template <>

elf/cmdline.cc

@@ -660,9 +660,9 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       else
         Fatal(ctx) << "unknown --unresolved-symbols argument: " << arg;
     } else if (read_arg("undefined") || read_arg("u")) {
-      ctx.arg.undefined.push_back(arg);
+      ctx.arg.undefined.push_back(get_symbol(ctx, arg));
     } else if (read_arg("require-defined")) {
-      ctx.arg.require_defined.push_back(arg);
+      ctx.arg.require_defined.push_back(get_symbol(ctx, arg));
     } else if (read_arg("init")) {
       ctx.arg.init = arg;
     } else if (read_arg("fini")) {
@@ -1104,21 +1104,21 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     } else if (read_flag("no-keep-memory")) {
     } else if (read_arg("max-cache-size")) {
     } else if (read_arg("version-script")) {
-      // --version-script, --dynamic-list and --export-dynamic-symbol[-list]
-      // are treated as positional arguments even though they are actually not
-      // positional. This is because linker scripts (a positional argument)
-      // can also specify a version script, and it's better to consolidate
-      // parsing in read_input_files. In particular, version scripts can
-      // modify ctx.default_version which we initialize *after* parsing
-      // non-positional args, so the parsing cannot be done right here.
+      // --version-script is treated as positional arguments even though
+      // they are actually not positional. This is because linker scripts
+      // (a positional argument) can also specify a version script, and
+      // it's better to consolidate parsing in read_input_files. In
+      // particular, version scripts can modify ctx.default_version which
+      // we initialize *after* parsing non-positional args, so the parsing
+      // cannot be done right here.
       remaining.push_back("--version-script=" + std::string(arg));
     } else if (read_arg("dynamic-list")) {
       ctx.arg.Bsymbolic = true;
-      remaining.push_back("--dynamic-list=" + std::string(arg));
+      append(ctx.dynamic_list_patterns, parse_dynamic_list(ctx, arg));
     } else if (read_arg("export-dynamic-symbol")) {
-      remaining.push_back("--export-dynamic-symbol=" + std::string(arg));
+      ctx.dynamic_list_patterns.push_back({arg, "<command line>"});
     } else if (read_arg("export-dynamic-symbol-list")) {
-      remaining.push_back("--export-dynamic-symbol-list=" + std::string(arg));
+      append(ctx.dynamic_list_patterns, parse_dynamic_list(ctx, arg));
     } else if (read_flag("as-needed")) {
       remaining.push_back("--as-needed");
     } else if (read_flag("no-as-needed")) {
@@ -1228,11 +1228,6 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   if (char *env = getenv("MOLD_REPRO"); env && env[0])
     ctx.arg.repro = true;
 
-  if (ctx.arg.shared || ctx.arg.export_dynamic)
-    ctx.default_version = VER_NDX_GLOBAL;
-  else
-    ctx.default_version = VER_NDX_LOCAL;
-
   if (ctx.arg.default_symver) {
     std::string ver = ctx.arg.soname.empty() ?
       filepath(ctx.arg.output).filename().string() : std::string(ctx.arg.soname);
@@ -1243,7 +1238,13 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   if (ctx.arg.shared && warn_shared_textrel)
     ctx.arg.warn_textrel = true;
 
-  ctx.arg.undefined.push_back(ctx.arg.entry);
+  ctx.arg.undefined.push_back(get_symbol(ctx, ctx.arg.entry));
+
+  for (i64 i = 0; i < ctx.arg.defsyms.size(); i++) {
+    std::variant<Symbol<E> *, u64> &val = ctx.arg.defsyms[i].second;
+    if (Symbol<E> **sym = std::get_if<Symbol<E> *>(&val))
+      ctx.arg.undefined.push_back(*sym);
+  }
 
   // --oformat=binary implies --strip-all because without a section
   // header, there's no way to identify the locations of a symbol
@@ -1264,6 +1265,14 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   if (ctx.arg.shared)
     ctx.overwrite_output_file = false;
 
+  if (!ctx.arg.chroot.empty()) {
+    if (!ctx.arg.Map.empty())
+      ctx.arg.Map = ctx.arg.chroot + "/" + ctx.arg.Map;
+
+    if (!ctx.arg.dependency_file.empty())
+      ctx.arg.dependency_file = ctx.arg.chroot + "/" + ctx.arg.dependency_file;
+  }
+
   if (version_shown && remaining.empty())
     exit(0);
   return remaining;

elf/elf.h

@@ -161,6 +161,7 @@ enum : u32 {
   VER_NDX_LOCAL = 0,
   VER_NDX_GLOBAL = 1,
   VER_NDX_LAST_RESERVED = 1,
+  VER_NDX_UNSPECIFIED = 0xffff,
 };
 
 enum : u32 {
@@ -1868,6 +1869,7 @@ struct X86_64 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 8;
+  static constexpr u8 filler[] = { 0xcc };
 
   static constexpr u32 R_COPY = R_X86_64_COPY;
   static constexpr u32 R_GLOB_DAT = R_X86_64_GLOB_DAT;
@@ -1892,6 +1894,7 @@ struct I386 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0xcc };
 
   static constexpr u32 R_COPY = R_386_COPY;
   static constexpr u32 R_GLOB_DAT = R_386_GLOB_DAT;
@@ -1918,6 +1921,7 @@ struct ARM64 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 16;
+  static constexpr u8 filler[] = { 0x00, 0x7d, 0x20, 0xd4 };
 
   static constexpr u32 R_COPY = R_AARCH64_COPY;
   static constexpr u32 R_GLOB_DAT = R_AARCH64_GLOB_DAT;
@@ -1944,6 +1948,7 @@ struct ARM32 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 16;
   static constexpr u32 thunk_size = 16;
+  static constexpr u8 filler[] = { 0xff, 0xde };
 
   static constexpr u32 R_COPY = R_ARM_COPY;
   static constexpr u32 R_GLOB_DAT = R_ARM_GLOB_DAT;
@@ -1969,6 +1974,7 @@ struct RV64 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x02, 0x90 };
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_64;
@@ -2001,6 +2007,7 @@ struct RV32 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x02, 0x90 };
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_32;
@@ -2037,7 +2044,7 @@ struct PPC32 {
   static constexpr u32 pltgot_size = 36;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 36;
-
+  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 };
 
   static constexpr u32 R_COPY = R_PPC_COPY;
   static constexpr u32 R_GLOB_DAT = R_PPC_GLOB_DAT;
@@ -2079,6 +2086,7 @@ struct PPC64V1 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 28;
+  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 };
 };
 
 struct PPC64V2 : PPC64 {
@@ -2089,6 +2097,7 @@ struct PPC64V2 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 24;
+  static constexpr u8 filler[] = { 0x08, 0x00, 0xe0, 0x7f };
 };
 
 struct S390X {
@@ -2101,6 +2110,7 @@ struct S390X {
   static constexpr u32 plt_hdr_size = 48;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x07, 0x00 };
 
   static constexpr u32 R_COPY = R_390_COPY;
   static constexpr u32 R_GLOB_DAT = R_390_GLOB_DAT;
@@ -2124,6 +2134,7 @@ struct SPARC64 {
   static constexpr u32 plt_hdr_size = 128;
   static constexpr u32 plt_size = 32;
   static constexpr u32 pltgot_size = 32;
+  static constexpr u8 filler[] = { 0x91, 0xd0, 0x20, 0x05 };
 
   static constexpr u32 R_COPY = R_SPARC_COPY;
   static constexpr u32 R_GLOB_DAT = R_SPARC_GLOB_DAT;
@@ -2147,6 +2158,7 @@ struct M68K {
   static constexpr u32 plt_hdr_size = 18;
   static constexpr u32 plt_size = 14;
   static constexpr u32 pltgot_size = 8;
+  static constexpr u8 filler[] = { 0x4a, 0xfc };
 
   static constexpr u32 R_COPY = R_68K_COPY;
   static constexpr u32 R_GLOB_DAT = R_68K_GLOB_DAT;
@@ -2169,6 +2181,7 @@ struct SH4 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 12;
+  static constexpr u8 filler[] = { 0x00, 0x00 };
 
   static constexpr u32 R_COPY = R_SH_COPY;
   static constexpr u32 R_GLOB_DAT = R_SH_GLOB_DAT;
@@ -2191,6 +2204,7 @@ struct ALPHA {
   static constexpr u32 plt_hdr_size = 0;
   static constexpr u32 plt_size = 0;
   static constexpr u32 pltgot_size = 0;
+  static constexpr u8 filler[] = { 0x81, 0x00, 0x00, 0x00 };
 
   static constexpr u32 R_COPY = R_ALPHA_COPY;
   static constexpr u32 R_GLOB_DAT = R_ALPHA_GLOB_DAT;
@@ -2214,7 +2228,8 @@ struct LOONGARCH64 {
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
+  static constexpr u32 thunk_size = 8;
+  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 };
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_64;
@@ -2239,7 +2254,8 @@ struct LOONGARCH32 {
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
+  static constexpr u32 thunk_size = 8;
+  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 };
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_32;

elf/gc-sections.cc

@@ -116,11 +116,11 @@ static void collect_root_set(Context<E> &ctx,
   // Add sections referenced by root symbols.
   enqueue_symbol(get_symbol(ctx, ctx.arg.entry));
 
-  for (std::string_view name : ctx.arg.undefined)
-    enqueue_symbol(get_symbol(ctx, name));
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    enqueue_symbol(sym);
 
-  for (std::string_view name : ctx.arg.require_defined)
-    enqueue_symbol(get_symbol(ctx, name));
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    enqueue_symbol(sym);
 
   // .eh_frame consists of variable-length records called CIE and FDE
   // records, and they are a unit of inclusion or exclusion.

elf/gdb-index.cc

@@ -140,7 +140,6 @@ struct NameType {
 };
 
 struct MapValue {
-  std::string_view name;
   u32 hash = 0;
   Atomic<u32> count = 0;
   u32 name_offset = 0;
@@ -639,7 +638,7 @@ template <typename E>
 std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
   if (chunk->is_compressed)
     return chunk->uncompressed_data;
-  return {ctx.buf + chunk->shdr.sh_offset, chunk->shdr.sh_size};
+  return {ctx.buf + chunk->shdr.sh_offset, (size_t)chunk->shdr.sh_size};
 }
 
 template <typename E>
@@ -678,36 +677,21 @@ void write_gdb_index(Context<E> &ctx) {
   });
 
   ConcurrentMap<MapValue> map(estimator.get_cardinality() * 3 / 2);
-  Atomic<i64> num_entries;
 
   tbb::parallel_for_each(cus, [&](Compunit &cu) {
     cu.entries.reserve(cu.nametypes.size());
-    i64 count = 0;
-
     for (NameType &nt : cu.nametypes) {
       MapValue *ent;
       bool inserted;
-      MapValue value = {nt.name, nt.hash};
-      std::tie(ent, inserted) = map.insert(nt.name, nt.hash, value);
+      std::tie(ent, inserted) = map.insert(nt.name, nt.hash, MapValue{nt.hash});
       ent->count++;
-      if (inserted)
-        count++;
       cu.entries.push_back(ent);
     }
-    num_entries += count;
   });
 
   // Sort symbols for build reproducibility
-  std::vector<MapValue *> entries;
-  entries.reserve(num_entries);
-
-  for (i64 i = 0; i < map.nbuckets; i++)
-    if (map.entries[i].key)
-      entries.push_back(&map.entries[i].value);
-
-  tbb::parallel_sort(entries, [](MapValue *a, MapValue *b) {
-    return std::tuple(a->hash, a->name) < std::tuple(b->hash, b->name);
-  });
+  using Entry = typename decltype(map)::Entry;
+  std::vector<Entry *> entries = map.get_sorted_entries_all();
 
   // Compute sizes of each components
   SectionHeader hdr;
@@ -719,18 +703,18 @@ void write_gdb_index(Context<E> &ctx) {
   for (Compunit &cu : cus)
     hdr.symtab_offset += cu.ranges.size() * 20;
 
-  i64 ht_size = bit_ceil(num_entries * 5 / 4 + 1);
+  i64 ht_size = bit_ceil(entries.size() * 5 / 4 + 1);
   hdr.const_pool_offset = hdr.symtab_offset + ht_size * 8;
 
   i64 offset = 0;
-  for (MapValue *ent : entries) {
-    ent->type_offset = offset;
-    offset += ent->count * 4 + 4;
+  for (Entry *ent : entries) {
+    ent->value.type_offset = offset;
+    offset += ent->value.count * 4 + 4;
   }
 
-  for (MapValue *ent : entries) {
-    ent->name_offset = offset;
-    offset += ent->name.size() + 1;
+  for (Entry *ent : entries) {
+    ent->value.name_offset = offset;
+    offset += ent->keylen + 1;
   }
 
   i64 bufsize = hdr.const_pool_offset + offset;
@@ -765,16 +749,16 @@ void write_gdb_index(Context<E> &ctx) {
   u32 mask = ht_size - 1;
   ul32 *ht = (ul32 *)(buf + hdr.symtab_offset);
 
-  for (MapValue *ent : entries) {
-    u32 hash = ent->hash;
+  for (Entry *ent : entries) {
+    u32 hash = ent->value.hash;
     u32 step = ((hash * 17) & mask) | 1;
     u32 j = hash & mask;
 
     while (ht[j * 2] || ht[j * 2 + 1])
       j = (j + step) & mask;
 
-    ht[j * 2] = ent->name_offset;
-    ht[j * 2 + 1] = ent->type_offset;
+    ht[j * 2] = ent->value.name_offset;
+    ht[j * 2 + 1] = ent->value.type_offset;
   }
 
   // Write types
@@ -790,8 +774,9 @@ void write_gdb_index(Context<E> &ctx) {
   }
 
   // Write names
-  tbb::parallel_for_each(entries, [&](MapValue *ent) {
-    write_string(buf + hdr.const_pool_offset + ent->name_offset, ent->name);
+  tbb::parallel_for_each(entries, [&](Entry *ent) {
+    memcpy(buf + hdr.const_pool_offset + ent->value.name_offset,
+           ent->key, ent->keylen);
   });
 
   // Update the section size and rewrite the section header

elf/input-files.cc

@@ -63,7 +63,7 @@ void InputFile<E>::clear_symbols() {
       sym->origin = 0;
       sym->value = -1;
       sym->sym_idx = -1;
-      sym->ver_idx = 0;
+      sym->ver_idx = VER_NDX_UNSPECIFIED;
       sym->is_weak = false;
       sym->is_imported = false;
       sym->is_exported = false;
@@ -249,15 +249,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
       break;
     }
     case SHT_REL:
-      if constexpr (E::is_rela)
-        Fatal(ctx) << *this << ": REL-type relocation table is not supported"
-                   << " for this target";
-      break;
     case SHT_RELA:
-      if constexpr (!E::is_rela)
-        Fatal(ctx) << *this <<": RELA-type relocation table is not supported"
-                   << " for this target";
-      break;
     case SHT_SYMTAB:
     case SHT_SYMTAB_SHNDX:
     case SHT_STRTAB:
@@ -322,6 +314,15 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
         continue;
       }
 
+      if (shdr.sh_type == SHT_INIT_ARRAY ||
+          shdr.sh_type == SHT_FINI_ARRAY ||
+          shdr.sh_type == SHT_PREINIT_ARRAY)
+        ctx.has_init_array = true;
+
+      if (name == ".ctors" || name.starts_with(".ctors.") ||
+          name == ".dtors" || name.starts_with(".dtors."))
+        ctx.has_ctors = true;
+
       if (name == ".eh_frame")
         eh_frame_section = this->sections[i].get();
 
@@ -645,9 +646,20 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
   if (!(shdr.sh_flags & SHF_MERGE))
     return nullptr;
 
+  i64 entsize = shdr.sh_entsize;
+  if (entsize == 0)
+    entsize = (shdr.sh_flags & SHF_STRINGS) ? 1 : (int)shdr.sh_addralign;
+
+  if (entsize == 0)
+    return nullptr;
+
+  i64 addralign = shdr.sh_addralign;
+  if (addralign == 0)
+    addralign = 1;
+
   std::unique_ptr<MergeableSection<E>> rec(new MergeableSection<E>);
   rec->parent = MergedSection<E>::get_instance(ctx, sec.name(), shdr.sh_type,
-                                               shdr.sh_flags);
+                                               shdr.sh_flags, entsize, addralign);
   rec->p2align = sec.p2align;
 
   if (sec.sh_size == 0)
@@ -658,18 +670,10 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
 
   std::string_view data = sec.contents;
   const char *begin = data.data();
-  u64 entsize = shdr.sh_entsize;
   HyperLogLog estimator;
 
   // Split sections
   if (shdr.sh_flags & SHF_STRINGS) {
-    if (entsize == 0) {
-      // GHC (Glasgow Haskell Compiler) sometimes creates a mergeable
-      // string section with entsize of 0 instead of 1, though such
-      // entsize is technically wrong. This is a workaround for the issue.
-      entsize = 1;
-    }
-
     while (!data.empty()) {
       size_t end = find_null(data, entsize);
       if (end == data.npos)
@@ -686,11 +690,6 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
       estimator.insert(hash);
     }
   } else {
-    // OCaml compiler seems to create a mergeable non-string section with
-    // entisze of 0. Such section is malformed. We do not split such section.
-    if (entsize == 0)
-      return nullptr;
-
     if (data.size() % entsize)
       Fatal(ctx) << sec << ": section size is not multiple of sh_entsize";
 
@@ -1291,17 +1290,17 @@ void SharedFile<E>::parse(Context<E> &ctx) {
       continue;
 
     std::string_view name = this->symbol_strtab.data() + esyms[i].st_name;
-    bool is_hidden = (!vers.empty() && (vers[i] & VERSYM_HIDDEN));
+    bool is_default = vers.empty() || !(vers[i] & VERSYM_HIDDEN);
 
     this->elf_syms2.push_back(esyms[i]);
     this->versyms.push_back(ver);
 
-    if (is_hidden) {
+    if (is_default) {
+      this->symbols.push_back(get_symbol(ctx, name));
+    } else {
       std::string_view mangled_name = save_string(
         ctx, std::string(name) + "@" + std::string(version_strings[ver]));
       this->symbols.push_back(get_symbol(ctx, mangled_name, name));
-    } else {
-      this->symbols.push_back(get_symbol(ctx, name));
     }
   }
 
@@ -1356,6 +1355,9 @@ std::vector<std::string_view> SharedFile<E>::read_verdef(Context<E> &ctx) {
   ElfVerdef<E> *ver = (ElfVerdef<E> *)verdef.data();
 
   for (;;) {
+    if (ver->vd_ndx == VER_NDX_UNSPECIFIED)
+      Fatal(ctx) << *this << ": symbol version too large";
+
     if (ret.size() <= ver->vd_ndx)
       ret.resize(ver->vd_ndx + 1);
 

elf/input-sections.cc

@@ -7,7 +7,8 @@
 namespace mold::elf {
 
 typedef enum {
-  NONE, ERROR, COPYREL, DYN_COPYREL, PLT, CPLT, DYN_CPLT, DYNREL, BASEREL, IFUNC,
+  NONE, ERROR, COPYREL, DYN_COPYREL, PLT, CPLT, DYN_CPLT, DYNREL,
+  BASEREL, IFUNC_DYNREL,
 } Action;
 
 template <typename E>
@@ -203,7 +204,7 @@ static void scan_rel(Context<E> &ctx, InputSection<E> &isec, Symbol<E> &sym,
     if (!isec.is_relr_reloc(ctx, rel))
       isec.file.num_dynrel++;
     break;
-  case IFUNC:
+  case IFUNC_DYNREL:
     // Create an IRELATIVE relocation for a GNU ifunc symbol.
     //
     // We usually create an IRELATIVE relocation in .got for each ifunc.
@@ -273,7 +274,7 @@ static Action get_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
 template <typename E>
 static Action get_dyn_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
   if (sym.is_ifunc())
-    return IFUNC;
+    return sym.is_pde_ifunc(ctx) ? NONE : IFUNC_DYNREL;
 
   // This is a decision table for absolute relocations for the pointer
   // size data (e.g. R_X86_64_64). Unlike the absrel_table, we can emit
@@ -291,7 +292,7 @@ static Action get_dyn_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
 template <typename E>
 static Action get_ppc64_toc_action(Context<E> &ctx, Symbol<E> &sym) {
   if (sym.is_ifunc())
-    return IFUNC;
+    return IFUNC_DYNREL;
 
   // As a special case, we do not create copy relocations nor canonical
   // PLTs for .toc sections. PPC64's .toc is a compiler-generated
@@ -406,7 +407,7 @@ static void apply_absrel(Context<E> &ctx, InputSection<E> &isec,
   case DYNREL:
     emit_abs_dynrel();
     break;
-  case IFUNC:
+  case IFUNC_DYNREL:
     if constexpr (supports_ifunc<E>) {
       u64 addr = sym.get_addr(ctx, NO_PLT) + A;
       *dynrel++ = ElfRel<E>(P, E::R_IRELATIVE, 0, addr);
@@ -458,14 +459,14 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
       apply_reloc_nonalloc(ctx, buf);
 
     if constexpr (is_x86_64<E>) {
-      if (ctx.arg.z_rewrite_endbr) {
-        // Rewrite the leading endbr instruction with a nop if the section
-        // is not address-taken.
-        u8 endbr[] = {0xf3, 0x0f, 0x1e, 0xfa};
-        u8 nop[] = {0x0f, 0x1f, 0x40, 0x00};
-        if (!address_taken && (shdr().sh_flags & SHF_EXECINSTR) &&
-            sh_size >= 4 && memcmp(buf, endbr, 4) == 0)
-          memcpy(buf, nop, 4);
+      u8 endbr[] = {0xf3, 0x0f, 0x1e, 0xfa};
+      u8 nop[] = {0x0f, 0x1f, 0x40, 0x00};
+
+      // Rewrite the leading endbr instruction with a nop if the section
+      // is not address-taken.
+      if (ctx.arg.z_rewrite_endbr && (shdr().sh_flags & SHF_EXECINSTR) &&
+          !address_taken && sh_size >= 4 && memcmp(buf, endbr, 4) == 0) {
+        memcpy(buf, nop, 4);
       }
     }
   }

elf/jobs.cc

@@ -1,3 +1,14 @@
+// Many build systems attempt to invoke as many linker processes as there
+// are cores, based on the assumption that the linker is single-threaded.
+// However, since mold is multi-threaded, such build systems' behavior is
+// not beneficial and just increases the overall peak memory usage.
+// On machines with limited memory, this could lead to an out-of-memory
+// error.
+//
+// This file implements a feature that limits the number of concurrent
+// mold processes to just 1 for each user. It is intended to be used as
+// `MOLD_JOBS=1 ninja` or `MOLD_JOBS=1 make -j$(nproc)`.
+
 #include "mold.h"
 
 #ifndef _WIN32
@@ -14,14 +25,15 @@ template <typename E>
 void acquire_global_lock(Context<E> &ctx) {
 #ifndef _WIN32
   char *jobs = getenv("MOLD_JOBS");
-  if (!jobs || std::string(jobs) != "1")
+  if (!jobs || jobs != "1"s)
     return;
 
-  char *home = getenv("HOME");
-  if (!home)
-    home = getpwuid(getuid())->pw_dir;
+  std::string path;
+  if (char *dir = getenv("XDG_RUNTIME_DIR"))
+    path = dir + "/mold-lock"s;
+  else
+    path = "/tmp/mold-lock-"s + getpwuid(getuid())->pw_name;
 
-  std::string path = std::string(home) + "/.mold-lock";
   int fd = open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600);
   if (fd == -1)
     return;

elf/linker-script.cc

@@ -312,7 +312,6 @@ read_version_script_commands(Context<E> &ctx, std::span<std::string_view> &tok,
 
     if (tok[0] == "*") {
       ctx.default_version = (is_global ? ver_idx : (u32)VER_NDX_LOCAL);
-      ctx.default_version_from_version_script = true;
     } else if (is_global) {
       ctx.version_patterns.push_back({unquote(tok[0]), current_file<E>->name,
                                       ver_str, ver_idx, is_cpp});
@@ -367,7 +366,9 @@ void parse_version_script(Context<E> &ctx, MappedFile<Context<E>> *mf) {
 }
 
 template <typename E>
-void read_dynamic_list_commands(Context<E> &ctx, std::span<std::string_view> &tok,
+void read_dynamic_list_commands(Context<E> &ctx,
+                                std::vector<DynamicPattern> &result,
+                                std::span<std::string_view> &tok,
                                 bool is_cpp) {
   while (!tok.empty() && tok[0] != "}") {
     if (tok[0] == "extern") {
@@ -376,11 +377,11 @@ void read_dynamic_list_commands(Context<E> &ctx, std::span<std::string_view> &to
       if (!tok.empty() && tok[0] == "\"C\"") {
         tok = tok.subspan(1);
         tok = skip(ctx, tok, "{");
-        read_dynamic_list_commands(ctx, tok, false);
+        read_dynamic_list_commands(ctx, result, tok, false);
       } else {
         tok = skip(ctx, tok, "\"C++\"");
         tok = skip(ctx, tok, "{");
-        read_dynamic_list_commands(ctx, tok, true);
+        read_dynamic_list_commands(ctx, result, tok, true);
       }
 
       tok = skip(ctx, tok, "}");
@@ -388,29 +389,32 @@ void read_dynamic_list_commands(Context<E> &ctx, std::span<std::string_view> &to
       continue;
     }
 
-    if (tok[0] == "*")
-      ctx.default_version = VER_NDX_GLOBAL;
-    else
-      ctx.version_patterns.push_back({unquote(tok[0]), current_file<E>->name,
-                                      "global", VER_NDX_GLOBAL, is_cpp});
-
+    result.push_back({unquote(tok[0]), "", is_cpp});
     tok = skip(ctx, tok.subspan(1), ";");
   }
 }
 
 template <typename E>
-void parse_dynamic_list(Context<E> &ctx, MappedFile<Context<E>> *mf) {
-  current_file<E> = mf;
-  std::vector<std::string_view> vec = tokenize(ctx, mf->get_contents());
+std::vector<DynamicPattern>
+parse_dynamic_list(Context<E> &ctx, std::string_view path) {
+  std::string_view contents =
+    MappedFile<Context<E>>::must_open(ctx, std::string(path))->get_contents();
+  std::vector<std::string_view> vec = tokenize(ctx, contents);
   std::span<std::string_view> tok = vec;
+  std::vector<DynamicPattern> result;
 
   tok = skip(ctx, tok, "{");
-  read_dynamic_list_commands(ctx, tok, false);
+  read_dynamic_list_commands(ctx, result, tok, false);
   tok = skip(ctx, tok, "}");
   tok = skip(ctx, tok, ";");
 
   if (!tok.empty())
     SyntaxError(ctx, tok[0]) << "trailing garbage token";
+
+  for (DynamicPattern &p : result)
+    p.source = path;
+
+  return result;
 }
 
 using E = MOLD_TARGET;
@@ -418,6 +422,7 @@ using E = MOLD_TARGET;
 template void parse_linker_script(Context<E> &, MappedFile<Context<E>> *);
 template std::string_view get_script_output_type(Context<E> &, MappedFile<Context<E>> *);
 template void parse_version_script(Context<E> &, MappedFile<Context<E>> *);
-template void parse_dynamic_list(Context<E> &, MappedFile<Context<E>> *);
+template std::vector<DynamicPattern> parse_dynamic_list(Context<E> &, std::string_view);
+
 
 } // namespace mold::elf

elf/lto-unix.cc

@@ -709,8 +709,9 @@ std::vector<ObjectFile<E> *> do_lto(Context<E> &ctx) {
     get_symbol(ctx, y)->referenced_by_regular_obj = true;
   }
 
-  // Handle --entry
-  get_symbol(ctx, ctx.arg.entry)->referenced_by_regular_obj = true;
+  // Keep some symbols
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    sym->referenced_by_regular_obj = true;
 
   // all_symbols_read_hook() calls add_input_file() and add_input_library()
   LOG << "all symbols read\n";

elf/main.cc

@@ -299,22 +299,6 @@ static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
       if (!mf)
         Fatal(ctx) << "--version-script: file not found: " << arg;
       parse_version_script(ctx, mf);
-    } else if (remove_prefix(arg, "--dynamic-list=")) {
-      MappedFile<Context<E>> *mf = find_from_search_paths(ctx, std::string(arg));
-      if (!mf)
-        Fatal(ctx) << "--dynamic-list: file not found: " << arg;
-      parse_dynamic_list(ctx, mf);
-    } else if (remove_prefix(arg, "--export-dynamic-symbol=")) {
-      if (arg == "*")
-        ctx.default_version = VER_NDX_GLOBAL;
-      else
-        ctx.version_patterns.push_back({arg, "--export-dynamic-symbol",
-                                        "global", VER_NDX_GLOBAL, false});
-    } else if (remove_prefix(arg, "--export-dynamic-symbol-list=")) {
-      MappedFile<Context<E>> *mf = find_from_search_paths(ctx, std::string(arg));
-      if (!mf)
-        Fatal(ctx) << "--export-dynamic-symbol-list: file not found: " << arg;
-      parse_dynamic_list(ctx, mf);
     } else if (arg == "--push-state") {
       state.push_back({ctx.as_needed, ctx.whole_archive, ctx.is_static,
                        ctx.in_lib});
@@ -523,9 +507,9 @@ int elf_main(int argc, char **argv) {
     write_repro_file(ctx);
 
   // Handle --require-defined
-  for (std::string_view name : ctx.arg.require_defined)
-    if (!get_symbol(ctx, name)->file)
-      Error(ctx) << "--require-defined: undefined symbol: " << name;
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    if (!sym->file)
+      Error(ctx) << "--require-defined: undefined symbol: " << *sym;
 
   // .init_array and .fini_array contents have to be sorted by
   // a special rule. Sort them.
@@ -535,6 +519,11 @@ int elf_main(int argc, char **argv) {
   // because they are superceded by .init_array/.fini_array, though.
   sort_ctor_dtor(ctx);
 
+  // If .ctors/.dtors are to be placed to .init_array/.fini_array,
+  // we need to reverse their contents.
+  if (ctx.has_init_array && ctx.has_ctors)
+    fixup_ctors_in_init_array(ctx);
+
   // Handle --shuffle-sections
   if (ctx.arg.shuffle_sections != SHUFFLE_SECTIONS_NONE)
     shuffle_sections(ctx);

elf/mold.h

@@ -622,6 +622,12 @@ public:
   }
 
   void update_shdr(Context<E> &ctx) override;
+  void copy_buf(Context<E> &ctx) override;
+
+  // Offsets in .strtab for ARM32 mapping symbols
+  static constexpr i64 ARM = 1;
+  static constexpr i64 THUMB = 4;
+  static constexpr i64 DATA = 7;
 };
 
 template <typename E>
@@ -761,7 +767,8 @@ template <typename E>
 class MergedSection : public Chunk<E> {
 public:
   static MergedSection<E> *
-  get_instance(Context<E> &ctx, std::string_view name, u64 type, u64 flags);
+  get_instance(Context<E> &ctx, std::string_view name, i64 type, i64 flags,
+               i64 entsize, i64 addralign);
 
   SectionFragment<E> *insert(Context<E> &ctx, std::string_view data,
                              u64 hash, i64 p2align);
@@ -774,7 +781,7 @@ public:
   HyperLogLog estimator;
 
 private:
-  MergedSection(std::string_view name, u64 flags, u32 type);
+  MergedSection(std::string_view name, i64 flags, i64 type, i64 entsize);
 
   ConcurrentMap<SectionFragment<E>> map;
   std::vector<i64> shard_offsets;
@@ -1274,8 +1281,15 @@ get_script_output_type(Context<E> &ctx, MappedFile<Context<E>> *mf);
 template <typename E>
 void parse_version_script(Context<E> &ctx, MappedFile<Context<E>> *mf);
 
+struct DynamicPattern {
+  std::string_view pattern;
+  std::string_view source;
+  bool is_cpp = false;
+};
+
 template <typename E>
-void parse_dynamic_list(Context<E> &ctx, MappedFile<Context<E>> *mf);
+std::vector<DynamicPattern>
+parse_dynamic_list(Context<E> &ctx, std::string_view path);
 
 //
 // lto.cc
@@ -1365,6 +1379,7 @@ template <typename E> void check_duplicate_symbols(Context<E> &);
 template <typename E> void check_symbol_types(Context<E> &);
 template <typename E> void sort_init_fini(Context<E> &);
 template <typename E> void sort_ctor_dtor(Context<E> &);
+template <typename E> void fixup_ctors_in_init_array(Context<E> &);
 template <typename E> void shuffle_sections(Context<E> &);
 template <typename E> void compute_section_sizes(Context<E> &);
 template <typename E> void sort_output_sections(Context<E> &);
@@ -1711,6 +1726,8 @@ struct Context {
     std::unordered_set<std::string_view> ignore_ir_file;
     std::unordered_set<std::string_view> wrap;
     std::vector<SectionOrder> section_order;
+    std::vector<Symbol<E> *> require_defined;
+    std::vector<Symbol<E> *> undefined;
     std::vector<std::pair<Symbol<E> *, std::variant<Symbol<E> *, u64>>> defsyms;
     std::vector<std::string> library_paths;
     std::vector<std::string> plugin_opt;
@@ -1718,20 +1735,16 @@ struct Context {
     std::vector<std::string_view> auxiliary;
     std::vector<std::string_view> exclude_libs;
     std::vector<std::string_view> filter;
-    std::vector<std::string_view> require_defined;
     std::vector<std::string_view> trace_symbol;
-    std::vector<std::string_view> undefined;
     u64 image_base = 0x200000;
   } arg;
 
   std::vector<VersionPattern> version_patterns;
-  u16 default_version = VER_NDX_GLOBAL;
+  std::vector<DynamicPattern> dynamic_list_patterns;
+  i64 default_version = VER_NDX_UNSPECIFIED;
   i64 page_size = E::page_size;
   std::optional<int> global_lock_fd;
 
-  // true if default_version is set by a wildcard in version script.
-  bool default_version_from_version_script = false;
-
   // Reader context
   bool as_needed = false;
   bool whole_archive = false;
@@ -1743,6 +1756,8 @@ struct Context {
 
   bool has_error = false;
   bool has_lto_object = false;
+  Atomic<bool> has_init_array = false;
+  Atomic<bool> has_ctors = false;
 
   // Symbol table
   tbb::concurrent_hash_map<std::string_view, Symbol<E>, HashCmp> symbol_map;
@@ -1934,6 +1949,7 @@ public:
   u64 get_tlsdesc_addr(Context<E> &ctx) const;
   u64 get_plt_addr(Context<E> &ctx) const;
   u64 get_opd_addr(Context<E> &ctx) const;
+  u64 get_got_pltgot_addr(Context<E> &ctx) const;
 
   void set_got_idx(Context<E> &ctx, i32 idx);
   void set_gottp_idx(Context<E> &ctx, i32 idx);
@@ -1967,6 +1983,7 @@ public:
   bool is_relative() const { return !is_absolute(); }
   bool is_local(Context<E> &ctx) const;
   bool is_ifunc() const { return get_type() == STT_GNU_IFUNC; }
+  bool is_pde_ifunc(Context<E> &ctx) const;
   bool is_remaining_undef_weak() const;
 
   bool is_pcrel_linktime_const(Context<E> &ctx) const;
@@ -2022,7 +2039,7 @@ public:
   i32 sym_idx = -1;
 
   i32 aux_idx = -1;
-  u16 ver_idx = 0;
+  u16 ver_idx = VER_NDX_UNSPECIFIED;
 
   // `flags` has NEEDS_ flags.
   Atomic<u8> flags = 0;
@@ -2568,6 +2585,25 @@ inline u64 Symbol<E>::get_opd_addr(Context<E> &ctx) const {
          get_opd_idx(ctx) * PPC64OpdSection::ENTRY_SIZE;
 }
 
+template <typename E>
+inline u64 Symbol<E>::get_got_pltgot_addr(Context<E> &ctx) const {
+  // An ifunc symbol occupies two consecutive GOT slots in a
+  // position-dependent executable (PDE). The first slot contains the
+  // symbol's PLT address, and the second slot holds the resolved
+  // address. A PDE uses the ifunc symbol's PLT entry as the address
+  // for the symbol, akin to a canonical PLT.
+  //
+  // This function returns the address that the PLT entry should use
+  // to jump to the resolved address.
+  //
+  // Note that we don't use this function for PPC64. In PPC64, symbols
+  // are always accessed through the TOC table regardless of the
+  // -fno-PIE setting. We don't need canonical PLTs on the psABIs too.
+  if (is_pde_ifunc(ctx))
+    return get_got_addr(ctx) + sizeof(Word<E>);
+  return get_got_addr(ctx);
+}
+
 template <typename E>
 inline void Symbol<E>::set_got_idx(Context<E> &ctx, i32 idx) {
   assert(aux_idx != -1);
@@ -2696,6 +2732,12 @@ inline bool Symbol<E>::is_local(Context<E> &ctx) const {
   return !is_imported && !is_exported;
 }
 
+template <typename E>
+inline bool Symbol<E>::is_pde_ifunc(Context<E> &ctx) const {
+  // Returns true if this is an ifunc tha uses two GOT slots
+  return is_ifunc() && !ctx.arg.pic && !is_ppc64<E>;
+}
+
 // A remaining weak undefined symbol is promoted to a dynamic symbol
 // in DSO and resolved to 0 in an executable. This function returns
 // true if it's latter.

elf/output-chunks.cc

@@ -462,6 +462,14 @@ template <typename E>
 void StrtabSection<E>::update_shdr(Context<E> &ctx) {
   i64 offset = 1;
 
+  // ARM32 uses $a, $t and $t mapping symbols to mark the beginning of
+  // ARM, Thumb and data in text, respectively. These symbols don't
+  // affect correctness of the program but helps disassembler to
+  // disassemble machine code appropriately.
+  if constexpr (is_arm32<E>)
+    if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file)
+      offset += sizeof("$a\0$t\0$d");
+
   for (Chunk<E> *chunk : ctx.chunks) {
     chunk->strtab_offset = offset;
     offset += chunk->strtab_size;
@@ -480,6 +488,16 @@ void StrtabSection<E>::update_shdr(Context<E> &ctx) {
   this->shdr.sh_size = (offset == 1) ? 0 : offset;
 }
 
+template <typename E>
+void StrtabSection<E>::copy_buf(Context<E> &ctx) {
+  u8 *buf = ctx.buf + this->shdr.sh_offset;
+  buf[0] = '\0';
+
+  if constexpr (is_arm32<E>)
+    if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file)
+      memcpy(buf + 1, "$a\0$t\0$d", 9);
+}
+
 template <typename E>
 void ShstrtabSection<E>::update_shdr(Context<E> &ctx) {
   std::unordered_map<std::string_view, i64> map;
@@ -591,9 +609,6 @@ void SymtabSection<E>::copy_buf(Context<E> &ctx) {
   ElfSym<E> *buf = (ElfSym<E> *)(ctx.buf + this->shdr.sh_offset);
   memset(buf, 0, sizeof(ElfSym<E>));
 
-  // Write the initial NUL byte to .strtab.
-  ctx.buf[ctx.strtab->shdr.sh_offset] = '\0';
-
   if (ctx.symtab_shndx) {
     ElfShdr<E> &shdr = ctx.symtab_shndx->shdr;
     memset(ctx.buf + shdr.sh_offset, 0, shdr.sh_size);
@@ -850,7 +865,7 @@ OutputSection<E>::OutputSection(Context<E> &ctx, std::string_view name,
                                 u32 type, u64 flags) {
   this->name = name;
   this->shdr.sh_type = type;
-  this->shdr.sh_flags = flags;
+  this->shdr.sh_flags = flags & ~SHF_MERGE & ~SHF_STRINGS;
 
   if (auto it = ctx.arg.section_align.find(name);
       it != ctx.arg.section_align.end())
@@ -883,31 +898,30 @@ void OutputSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void OutputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
-  auto clear = [&](u8 *loc, i64 size) {
-    // As a special case, .init and .fini are filled with NOPs for s390x
-    // because the runtime executes the sections as if they were a single
-    // function. .init and .fini are superceded by .init_array and
-    // .fini_array but being actively used only on s390x.
-    if constexpr (is_s390x<E>) {
-      if (this->name == ".init" || this->name == ".fini") {
-        for (i64 i = 0; i < size; i += 2)
-          *(ub16 *)(loc + i) = 0x0700; // nop
-        return;
-      }
-    }
-    memset(loc, 0, size);
-  };
-
   tbb::parallel_for((i64)0, (i64)members.size(), [&](i64 i) {
-    // Copy section contents to an output file
+    // Copy section contents to an output file.
     InputSection<E> &isec = *members[i];
     isec.write_to(ctx, buf + isec.offset);
 
-    // Clear trailing padding
+    // Clear trailing padding. We write trap or nop instructions for
+    // an executable segment so that a disassembler wouldn't try to
+    // disassemble garbage as instructions.
     u64 this_end = isec.offset + isec.sh_size;
-    u64 next_start = (i == members.size() - 1) ?
-      (u64)this->shdr.sh_size : members[i + 1]->offset;
-    clear(buf + this_end, next_start - this_end);
+    u64 next_start;
+    if (i + 1 < members.size())
+      next_start = members[i + 1]->offset;
+    else
+      next_start = this->shdr.sh_size;
+
+    u8 *loc = buf + this_end;
+    i64 size = next_start - this_end;
+
+    if (this->shdr.sh_flags & SHF_EXECINSTR) {
+      for (i64 i = 0; i + sizeof(E::filler) <= size; i += sizeof(E::filler))
+        memcpy(loc + i, E::filler, sizeof(E::filler));
+    } else {
+      memset(loc, 0, size);
+    }
   });
 
   if constexpr (needs_thunk<E>) {
@@ -1001,16 +1015,10 @@ void OutputSection<E>::construct_relr(Context<E> &ctx) {
 // Compute spaces needed for thunk symbols
 template <typename E>
 void OutputSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file || ctx.arg.relocatable)
-    return;
-
   if constexpr (needs_thunk<E>) {
     this->strtab_size = 0;
     this->num_local_symtab = 0;
 
-    if constexpr (is_arm32<E>)
-      this->strtab_size = 9; // for "$t", "$a" and "$d" symbols
-
     for (std::unique_ptr<RangeExtensionThunk<E>> &thunk : thunks) {
       // For ARM32, we emit additional symbol "$t", "$a" and "$d" for
       // each thunk to mark the beginning of Thumb code, ARM code and
@@ -1031,7 +1039,7 @@ void OutputSection<E>::compute_symtab_size(Context<E> &ctx) {
 // disassembling and/or debugging our output.
 template <typename E>
 void OutputSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   if constexpr (needs_thunk<E>) {
@@ -1041,37 +1049,30 @@ void OutputSection<E>::populate_symtab(Context<E> &ctx) {
     u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
     u8 *strtab = strtab_base + this->strtab_offset;
 
-    if constexpr (is_arm32<E>) {
-      // ARM uses these symbols to mark the begining of Thumb code, ARM
-      // code and data, respectively. Our thunk contains all of them.
-      strtab += write_string(strtab, "$t");
-      strtab += write_string(strtab, "$a");
-      strtab += write_string(strtab, "$d");
-    }
+    auto write_esym = [&](u64 addr, i64 st_name) {
+      memset(esym, 0, sizeof(*esym));
+      esym->st_name = st_name;
+      esym->st_type = STT_FUNC;
+      esym->st_shndx = this->shndx;
+      esym->st_value = addr;
+      esym++;
+    };
 
     for (std::unique_ptr<RangeExtensionThunk<E>> &thunk : thunks) {
       for (i64 i = 0; i < thunk->symbols.size(); i++) {
         Symbol<E> &sym = *thunk->symbols[i];
+        u64 addr = thunk->get_addr(i);
 
-        auto write_esym = [&](i64 st_name, i64 off) {
-          memset(esym, 0, sizeof(*esym));
-          esym->st_name = st_name;
-          esym->st_type = STT_FUNC;
-          esym->st_shndx = this->shndx;
-          esym->st_value = thunk->get_addr(i) + off;
-          esym++;
-        };
-
-        write_esym(strtab - strtab_base, 0);
+        write_esym(addr, strtab - strtab_base);
 
         strtab += write_string(strtab, sym.name()) - 1;
         strtab += write_string(strtab, "$thunk");
 
         // Emit "$t", "$a" and "$d" if ARM32.
         if constexpr (is_arm32<E>) {
-          write_esym(this->strtab_offset, 0);
-          write_esym(this->strtab_offset + 3, 4);
-          write_esym(this->strtab_offset + 6, 12);
+          write_esym(addr, ctx.strtab->THUMB);
+          write_esym(addr + 4, ctx.strtab->ARM);
+          write_esym(addr + 12, ctx.strtab->DATA);
         }
       }
     }
@@ -1081,7 +1082,14 @@ void OutputSection<E>::populate_symtab(Context<E> &ctx) {
 template <typename E>
 void GotSection<E>::add_got_symbol(Context<E> &ctx, Symbol<E> *sym) {
   sym->set_got_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
-  this->shdr.sh_size += sizeof(Word<E>);
+
+  // An IFUNC symbol uses two GOT slots in a position-dependent
+  // executable.
+  if (sym->is_pde_ifunc(ctx))
+    this->shdr.sh_size += sizeof(Word<E>) * 2;
+  else
+    this->shdr.sh_size += sizeof(Word<E>);
+
   got_syms.push_back(sym);
 }
 
@@ -1105,10 +1113,10 @@ void GotSection<E>::add_tlsdesc_symbol(Context<E> &ctx, Symbol<E> *sym) {
   // always emit a dynamic relocation for each TLSDESC entry.
   //
   // If dynamic relocation is not available (i.e. if we are creating a
-  // non-PIC executable), we always relax TLSDESC relocations so that
-  // no TLSDESC relocation exist at runtime.
+  // statically-linked executable), we always relax TLSDESC relocations
+  // so that no TLSDESC relocation exist at runtime.
   assert(supports_tlsdesc<E>);
-  assert(ctx.arg.pic);
+  assert(!ctx.arg.is_static);
 
   sym->set_tlsdesc_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
   this->shdr.sh_size += sizeof(Word<E>) * 2;
@@ -1175,7 +1183,12 @@ static std::vector<GotEntry<E>> get_got_entries(Context<E> &ctx) {
     // IFUNC always needs to be fixed up by the dynamic linker.
     if constexpr (supports_ifunc<E>) {
       if (sym->is_ifunc()) {
-        add({idx, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        if (sym->is_pde_ifunc(ctx)) {
+          add({idx, sym->get_plt_addr(ctx)});
+          add({idx + 1, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        } else {
+          add({idx, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        }
         continue;
       }
     }
@@ -1330,9 +1343,6 @@ void GotSection<E>::construct_relr(Context<E> &ctx) {
 
 template <typename E>
 void GotSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->strtab_size = 0;
   this->num_local_symtab = 0;
 
@@ -1364,7 +1374,7 @@ void GotSection<E>::compute_symtab_size(Context<E> &ctx) {
 
 template <typename E>
 void GotSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1455,19 +1465,19 @@ void PltSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void PltSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->num_local_symtab = symbols.size();
   this->strtab_size = 0;
 
   for (Symbol<E> *sym : symbols)
     this->strtab_size += sym->name().size() + sizeof("$plt");
+
+  if constexpr (is_arm32<E>)
+    this->num_local_symtab += symbols.size() * 2 + 2;
 }
 
 template <typename E>
 void PltSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1476,16 +1486,30 @@ void PltSection<E>::populate_symtab(Context<E> &ctx) {
   u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
   u8 *strtab = strtab_base + this->strtab_offset;
 
-  for (Symbol<E> *sym : symbols) {
+  auto write_esym = [&](u64 addr, i64 st_name) {
     memset(esym, 0, sizeof(*esym));
-    esym->st_name = strtab - strtab_base;
+    esym->st_name = st_name;
     esym->st_type = STT_FUNC;
     esym->st_shndx = this->shndx;
-    esym->st_value = sym->get_plt_addr(ctx);
+    esym->st_value = addr;
     esym++;
+  };
 
+  if constexpr (is_arm32<E>) {
+    write_esym(this->shdr.sh_addr, ctx.strtab->ARM);
+    write_esym(this->shdr.sh_addr + 16, ctx.strtab->DATA);
+  }
+
+  for (Symbol<E> *sym : symbols) {
+    u64 addr = sym->get_plt_addr(ctx);
+    write_esym(addr, strtab - strtab_base);
     strtab += write_string(strtab, sym->name()) - 1;
     strtab += write_string(strtab, "$plt");
+
+    if constexpr (is_arm32<E>) {
+      write_esym(addr, ctx.strtab->ARM);
+      write_esym(addr + 12, ctx.strtab->DATA);
+    }
   }
 }
 
@@ -1508,19 +1532,19 @@ void PltGotSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void PltGotSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->num_local_symtab = symbols.size();
   this->strtab_size = 0;
 
   for (Symbol<E> *sym : symbols)
     this->strtab_size += sym->name().size() + sizeof("$pltgot");
+
+  if constexpr (is_arm32<E>)
+    this->num_local_symtab += symbols.size() * 2;
 }
 
 template <typename E>
 void PltGotSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1529,16 +1553,25 @@ void PltGotSection<E>::populate_symtab(Context<E> &ctx) {
   u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
   u8 *strtab = strtab_base + this->strtab_offset;
 
-  for (Symbol<E> *sym : symbols) {
+  auto write_esym = [&](u64 addr, i64 st_name) {
     memset(esym, 0, sizeof(*esym));
-    esym->st_name = strtab - strtab_base;
+    esym->st_name = st_name;
     esym->st_type = STT_FUNC;
     esym->st_shndx = this->shndx;
-    esym->st_value = sym->get_plt_addr(ctx);
+    esym->st_value = addr;
     esym++;
+  };
 
+  for (Symbol<E> *sym : symbols) {
+    u64 addr = sym->get_plt_addr(ctx);
+    write_esym(addr, strtab - strtab_base);
     strtab += write_string(strtab, sym->name()) - 1;
     strtab += write_string(strtab, "$pltgot");
+
+    if constexpr (is_arm32<E>) {
+      write_esym(addr, ctx.strtab->ARM);
+      write_esym(addr + 12, ctx.strtab->DATA);
+    }
   }
 }
 
@@ -1635,8 +1668,15 @@ ElfSym<E> to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name,
     esym.st_shndx = SHN_ABS;
     esym.st_value = sym.get_addr(ctx);
   } else if (sym.get_type() == STT_TLS) {
+    // TLS symbol
     shndx = get_st_shndx(sym);
     esym.st_value = sym.get_addr(ctx) - ctx.tls_begin;
+  } else if (sym.is_pde_ifunc(ctx)) {
+    // IFUNC symbol in PDE that uses two GOT slots
+    shndx = get_st_shndx(sym);
+    esym.st_type = STT_FUNC;
+    esym.st_visibility = sym.visibility;
+    esym.st_value = sym.get_addr(ctx);
   } else {
     shndx = get_st_shndx(sym);
     esym.st_visibility = sym.visibility;
@@ -1872,34 +1912,51 @@ void GnuHashSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 std::string_view
-get_merged_output_name(Context<E> &ctx, std::string_view name, u64 flags) {
+get_merged_output_name(Context<E> &ctx, std::string_view name, u64 flags,
+                       i64 entsize, i64 addralign) {
   if (ctx.arg.relocatable && !ctx.arg.relocatable_merge_sections)
     return name;
   if (ctx.arg.unique && ctx.arg.unique->match(name))
     return name;
-  if (name == ".rodata" || name.starts_with(".rodata."))
-    return (flags & SHF_STRINGS) ? ".rodata.str" : ".rodata.cst";
+
+  // GCC seems to create sections named ".rodata.strN.<mangled-symbol-name>.M".
+  // We want to eliminate the symbol name part from the section name.
+  if ((flags & SHF_STRINGS) && name.starts_with(".rodata.")) {
+    if (entsize == 1 && addralign == 1)
+      return ".rodata.str1.1";
+    if (entsize == 2 && addralign == 2)
+      return ".rodata.str2.2";
+    if (entsize == 4 && addralign == 4)
+      return ".rodata.str4.4";
+    return save_string(ctx,".rodata.str"s + std::to_string(entsize) + "." +
+                       std::to_string(addralign));
+  }
+
   return name;
 }
 
 template <typename E>
-MergedSection<E>::MergedSection(std::string_view name, u64 flags, u32 type) {
+MergedSection<E>::MergedSection(std::string_view name, i64 flags, i64 type,
+                                i64 entsize) {
   this->name = name;
   this->shdr.sh_flags = flags;
   this->shdr.sh_type = type;
+  this->shdr.sh_entsize = entsize;
 }
 
 template <typename E>
 MergedSection<E> *
 MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
-                               u64 type, u64 flags) {
-  name = get_merged_output_name(ctx, name, flags);
+                               i64 type, i64 flags,
+                               i64 entsize, i64 addralign) {
+  name = get_merged_output_name(ctx, name, flags, entsize, addralign);
   flags = flags & ~(u64)SHF_GROUP & ~(u64)SHF_COMPRESSED;
 
   auto find = [&]() -> MergedSection * {
     for (std::unique_ptr<MergedSection<E>> &osec : ctx.merged_sections)
-      if (std::tuple(name, flags, type) ==
-          std::tuple(osec->name, osec->shdr.sh_flags, osec->shdr.sh_type))
+      if (std::tuple(name, flags, type, entsize) ==
+          std::tuple(osec->name, osec->shdr.sh_flags, osec->shdr.sh_type,
+                     osec->shdr.sh_entsize))
         return osec.get();
     return nullptr;
   };
@@ -1917,7 +1974,7 @@ MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
   if (MergedSection *osec = find())
     return osec;
 
-  MergedSection *osec = new MergedSection(name, flags, type);
+  MergedSection *osec = new MergedSection(name, flags, type, entsize);
   ctx.merged_sections.emplace_back(osec);
   return osec;
 }
@@ -1947,67 +2004,49 @@ MergedSection<E>::insert(Context<E> &ctx, std::string_view data, u64 hash,
 template <typename E>
 void MergedSection<E>::assign_offsets(Context<E> &ctx) {
   std::vector<i64> sizes(map.NUM_SHARDS);
-  std::vector<i64> max_p2aligns(map.NUM_SHARDS);
-  shard_offsets.resize(map.NUM_SHARDS + 1);
-
-  i64 shard_size = map.nbuckets / map.NUM_SHARDS;
+  Atomic<i64> alignment = 1;
 
   tbb::parallel_for((i64)0, map.NUM_SHARDS, [&](i64 i) {
-    struct KeyVal {
-      std::string_view key;
-      SectionFragment<E> *val;
-    };
+    using Entry = typename decltype(map)::Entry;
+    std::vector<Entry *> entries = map.get_sorted_entries(i);
 
-    std::vector<KeyVal> fragments;
-    fragments.reserve(shard_size);
-
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (const char *key = map.get_key(j))
-        if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
-          fragments.push_back({{key, map.entries[j].keylen}, &frag});
-
-    // Sort fragments to make output deterministic.
-    tbb::parallel_sort(fragments.begin(), fragments.end(),
-                       [](const KeyVal &a, const KeyVal &b) {
-      return std::tuple{(u32)a.val->p2align, a.key.size(), a.key} <
-             std::tuple{(u32)b.val->p2align, b.key.size(), b.key};
-    });
-
-    // Assign offsets.
     i64 offset = 0;
     i64 p2align = 0;
 
-    for (KeyVal &kv : fragments) {
-      SectionFragment<E> &frag = *kv.val;
-      offset = align_to(offset, 1 << frag.p2align);
-      frag.offset = offset;
-      offset += kv.key.size();
-      p2align = std::max<i64>(p2align, frag.p2align);
+    for (Entry *ent : entries) {
+      SectionFragment<E> &frag = ent->value;
+      if (frag.is_alive) {
+        offset = align_to(offset, 1 << frag.p2align);
+        frag.offset = offset;
+        offset += ent->keylen;
+        p2align = std::max<i64>(p2align, frag.p2align);
+      }
     }
 
     sizes[i] = offset;
-    max_p2aligns[i] = p2align;
+    update_maximum(alignment, 1 << p2align);
 
     static Counter merged_strings("merged_strings");
-    merged_strings += fragments.size();
+    merged_strings += entries.size();
   });
 
-  i64 p2align = 0;
-  for (i64 x : max_p2aligns)
-    p2align = std::max(p2align, x);
+  i64 shard_size = map.nbuckets / map.NUM_SHARDS;
+  shard_offsets.resize(map.NUM_SHARDS + 1);
 
   for (i64 i = 1; i < map.NUM_SHARDS + 1; i++)
     shard_offsets[i] =
-      align_to(shard_offsets[i - 1] + sizes[i - 1], 1 << p2align);
+      align_to(shard_offsets[i - 1] + sizes[i - 1], alignment);
 
   tbb::parallel_for((i64)1, map.NUM_SHARDS, [&](i64 i) {
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
+    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
+      SectionFragment<E> &frag = map.entries[j].value;
+      if (frag.is_alive)
         frag.offset += shard_offsets[i];
+    }
   });
 
   this->shdr.sh_size = shard_offsets[map.NUM_SHARDS];
-  this->shdr.sh_addralign = 1 << p2align;
+  this->shdr.sh_addralign = alignment;
 }
 
 template <typename E>
@@ -2023,9 +2062,11 @@ void MergedSection<E>::write_to(Context<E> &ctx, u8 *buf) {
     memset(buf + shard_offsets[i], 0, shard_offsets[i + 1] - shard_offsets[i]);
 
     for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (const char *key = map.get_key(j))
-        if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
+      if (const char *key = map.entries[j].key) {
+        SectionFragment<E> &frag = map.entries[j].value;
+        if (frag.is_alive)
           memcpy(buf + frag.offset, key, map.entries[j].keylen);
+      }
   });
 }
 
@@ -2033,7 +2074,7 @@ template <typename E>
 void MergedSection<E>::print_stats(Context<E> &ctx) {
   i64 used = 0;
   for (i64 i = 0; i < map.nbuckets; i++)
-    if (map.get_key(i))
+    if (map.entries[i].key)
       used++;
 
   SyncOut(ctx) << this->name
@@ -2315,8 +2356,7 @@ void CopyrelSection<E>::update_shdr(Context<E> &ctx) {
   // a segment, so a .copyrel.rel.ro usually requires one extra
   // segment for it. We turn a .copyrel.rel.ro into a regular section
   // if it is very small to avoid the cost of the extra segment.
-  constexpr i64 threshold = 4096;
-  if (this->is_relro && ctx.arg.z_relro && this->shdr.sh_size < threshold)
+  if (this->is_relro && ctx.arg.z_relro && this->shdr.sh_size < E::page_size)
     this->shdr.sh_type = SHT_PROGBITS;
 }
 
@@ -2510,8 +2550,12 @@ void VerdefSection<E>::construct(Context<E> &ctx) {
   for (std::string_view verstr : ctx.arg.version_definitions)
     write(verstr, idx++, 0);
 
-  for (Symbol<E> *sym : std::span<Symbol<E> *>(ctx.dynsym->symbols).subspan(1))
-    ctx.versym->contents[sym->get_dynsym_idx(ctx)] = sym->ver_idx;
+  for (Symbol<E> *sym : std::span<Symbol<E> *>(ctx.dynsym->symbols).subspan(1)) {
+    i64 ver = sym->ver_idx;
+    if (ver == VER_NDX_UNSPECIFIED)
+      ver = VER_NDX_GLOBAL;
+    ctx.versym->contents[sym->get_dynsym_idx(ctx)] = ver;
+  }
 }
 
 template <typename E>

elf/passes.cc

@@ -189,15 +189,13 @@ void create_synthetic_sections(Context<E> &ctx) {
 
 template <typename E>
 static void mark_live_objects(Context<E> &ctx) {
-  auto mark_symbol = [&](std::string_view name) {
-    if (InputFile<E> *file = get_symbol(ctx, name)->file)
-      file->is_alive = true;
-  };
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    if (sym->file)
+      sym->file->is_alive = true;
 
-  for (std::string_view name : ctx.arg.undefined)
-    mark_symbol(name);
-  for (std::string_view name : ctx.arg.require_defined)
-    mark_symbol(name);
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    if (sym->file)
+      sym->file->is_alive = true;
 
   std::vector<InputFile<E> *> roots;
 
@@ -397,7 +395,7 @@ template <typename E>
 void add_comment_string(Context<E> &ctx, std::string str) {
   MergedSection<E> *sec =
     MergedSection<E>::get_instance(ctx, ".comment", SHT_PROGBITS,
-                                   SHF_MERGE | SHF_STRINGS);
+                                   SHF_MERGE | SHF_STRINGS, 1, 1);
 
   std::string_view buf = save_string(ctx, str);
   std::string_view data(buf.data(), buf.size() + 1);
@@ -510,9 +508,31 @@ get_output_name(Context<E> &ctx, std::string_view name, u64 flags) {
   return name;
 }
 
+// Returns true if a given input section is a .ctors/.dtors that
+// should be put into .init_array/.fini_array.
+//
+// CRT object files contain .ctors/.dtors sections without any
+// relocations. They contain sentinel values, 0 and -1, to mark the
+// beginning and the end of the initializer/finalizer pointer arrays. We
+// do not place them into .init_array/.fini_array because such invalid
+// pointer values would simply make the program to crash.
+template <typename E>
+static bool is_ctors_in_init_array(Context<E> &ctx, InputSection<E> &isec) {
+  std::string_view name = isec.name();
+  return ctx.has_init_array && !isec.get_rels(ctx).empty() &&
+         (name == ".ctors" || name.starts_with(".ctors.") ||
+          name == ".dtors" || name.starts_with(".dtors."));
+}
+
 template <typename E>
 static OutputSectionKey
 get_output_section_key(Context<E> &ctx, InputSection<E> &isec) {
+  if (is_ctors_in_init_array(ctx, isec)) {
+    if (isec.name().starts_with(".ctors"))
+      return {".init_array", SHT_INIT_ARRAY, SHF_ALLOC | SHF_WRITE};
+    return {".fini_array", SHT_FINI_ARRAY, SHF_ALLOC | SHF_WRITE};
+  }
+
   const ElfShdr<E> &shdr = isec.shdr();
   std::string_view name = get_output_name(ctx, isec.name(), shdr.sh_flags);
   u64 type = canonicalize_type<E>(name, shdr.sh_type);
@@ -523,6 +543,7 @@ get_output_section_key(Context<E> &ctx, InputSection<E> &isec) {
   // So is .fini_array.
   if (type == SHT_INIT_ARRAY || type == SHT_FINI_ARRAY)
     flags |= SHF_WRITE;
+
   return {name, type, flags};
 }
 
@@ -661,34 +682,9 @@ void create_internal_file(Context<E> &ctx) {
     ctx.internal_esyms.push_back(esym);
   };
 
-  auto add_undef = [&](Symbol<E> *sym) {
-    obj->symbols.push_back(sym);
-    sym->value = 0xdeadbeef;
-
-    ElfSym<E> esym;
-    memset(&esym, 0, sizeof(esym));
-    esym.st_type = STT_NOTYPE;
-    esym.st_shndx = SHN_UNDEF;
-    esym.st_bind = STB_GLOBAL;
-    esym.st_visibility = STV_DEFAULT;
-    ctx.internal_esyms.push_back(esym);
-  };
-
-  // Add --defsym symbols
-  for (i64 i = 0; i < ctx.arg.defsyms.size(); i++) {
-    Symbol<E> *sym = ctx.arg.defsyms[i].first;
-    std::variant<Symbol<E> *, u64> val = ctx.arg.defsyms[i].second;
-    add(sym);
-
-    if (Symbol<E> **ref = std::get_if<Symbol<E> *>(&val)) {
-      // Add an undefined symbol to keep a reference to the defsym target.
-      // This prevents elimination by e.g. LTO or gc-sections.
-      // The undefined symbol will never make to the final object file; we
-      // double-check that the defsym target is not undefined in
-      // fix_synthetic_symbols.
-      add_undef(*ref);
-    }
-  }
+  // Add --defsym'd symbols
+  for (i64 i = 0; i < ctx.arg.defsyms.size(); i++)
+    add(ctx.arg.defsyms[i].first);
 
   // Add --section-order symbols
   for (SectionOrder &ord : ctx.arg.section_order)
@@ -821,26 +817,27 @@ void add_synthetic_symbols(Context<E> &ctx) {
 
   // Handle --defsym symbols.
   for (i64 i = 0; i < ctx.arg.defsyms.size(); i++) {
-    Symbol<E> *sym = ctx.arg.defsyms[i].first;
+    Symbol<E> *sym1 = ctx.arg.defsyms[i].first;
     std::variant<Symbol<E> *, u64> val = ctx.arg.defsyms[i].second;
 
-    Symbol<E> *target = nullptr;
-    if (Symbol<E> **ref = std::get_if<Symbol<E> *>(&val))
-      target = *ref;
+    if (Symbol<E> **ref = std::get_if<Symbol<E> *>(&val)) {
+      Symbol<E> *sym2 = *ref;
+      if (!sym2->file) {
+        Error(ctx) << "--defsym: undefined symbol: " << *sym2;
+        continue;
+      }
 
-    // If the alias refers another symobl, copy ELF symbol attributes.
-    if (target) {
       ElfSym<E> &esym = obj.elf_syms[i + 1];
-      esym.st_type = target->esym().st_type;
+      esym.st_type = sym2->esym().st_type;
       if constexpr (is_ppc64v2<E>)
-        esym.ppc_local_entry = target->esym().ppc_local_entry;
+        esym.ppc_local_entry = sym2->esym().ppc_local_entry;
+
+      if (sym2->is_absolute())
+        sym1->origin = 0;
+    } else {
+      sym1->origin = 0;
     }
-
-    // Make the target absolute if necessary.
-    if (!target || target->is_absolute())
-      sym->origin = 0;
   }
-
 }
 
 template <typename E>
@@ -1047,19 +1044,42 @@ void check_symbol_types(Context<E> &ctx) {
   });
 }
 
+template <typename E>
+static i64 get_init_fini_priority(InputSection<E> *isec) {
+  static std::regex re(R"(\.(\d+)$)", std::regex_constants::optimize);
+  std::string_view name = isec->name();
+  std::cmatch m;
+  if (std::regex_search(name.data(), name.data() + name.size(), m, re))
+    return std::stoi(m[1]);
+  return 65536;
+}
+
+template <typename E>
+static i64 get_ctor_dtor_priority(InputSection<E> *isec) {
+  auto opts = std::regex_constants::optimize | std::regex_constants::ECMAScript;
+  static std::regex re1(R"((?:clang_rt\.)?crtbegin)", opts);
+  static std::regex re2(R"((?:clang_rt\.)?crtend)", opts);
+  static std::regex re3(R"(\.(\d+)$)", opts);
+
+  // crtbegin.o and crtend.o contain marker symbols such as
+  // __CTOR_LIST__ or __DTOR_LIST__. So they have to be at the
+  // beginning or end of the section.
+  std::smatch m;
+  if (std::regex_search(isec->file.filename, m, re1))
+    return -2;
+  if (std::regex_search(isec->file.filename, m, re2))
+    return 65536;
+
+  std::string name(isec->name());
+  if (std::regex_search(name, m, re3))
+    return std::stoi(m[1]);
+  return -1;
+}
+
 template <typename E>
 void sort_init_fini(Context<E> &ctx) {
   Timer t(ctx, "sort_init_fini");
 
-  auto get_priority = [](InputSection<E> *isec) {
-    static std::regex re(R"(\.(\d+)$)", std::regex_constants::optimize);
-    std::string_view name = isec->name();
-    std::cmatch m;
-    if (std::regex_search(name.data(), name.data() + name.size(), m, re))
-      return std::stoi(m[1]);
-    return 65536;
-  };
-
   for (Chunk<E> *chunk : ctx.chunks) {
     if (OutputSection<E> *osec = chunk->to_osec()) {
       if (osec->name == ".init_array" || osec->name == ".preinit_array" ||
@@ -1068,8 +1088,14 @@ void sort_init_fini(Context<E> &ctx) {
           std::reverse(osec->members.begin(), osec->members.end());
 
         std::unordered_map<InputSection<E> *, i64> map;
-        for (InputSection<E> *isec : osec->members)
-          map.insert({isec, get_priority(isec)});
+
+        for (InputSection<E> *isec : osec->members) {
+          std::string_view name = isec->name();
+          if (name.starts_with(".ctors") || name.starts_with(".dtors"))
+            map.insert({isec, 65535 - get_ctor_dtor_priority(isec)});
+          else
+            map.insert({isec, get_init_fini_priority(isec)});
+        }
 
         sort(osec->members, [&](InputSection<E> *a, InputSection<E> *b) {
           return map[a] < map[b];
@@ -1083,27 +1109,6 @@ template <typename E>
 void sort_ctor_dtor(Context<E> &ctx) {
   Timer t(ctx, "sort_ctor_dtor");
 
-  auto get_priority = [](InputSection<E> *isec) {
-    auto opts = std::regex_constants::optimize | std::regex_constants::ECMAScript;
-    static std::regex re1(R"((?:clang_rt\.)?crtbegin)", opts);
-    static std::regex re2(R"((?:clang_rt\.)?crtend)", opts);
-    static std::regex re3(R"(\.(\d+)$)", opts);
-
-    // crtbegin.o and crtend.o contain marker symbols such as
-    // __CTOR_LIST__ or __DTOR_LIST__. So they have to be at the
-    // beginning or end of the section.
-    std::smatch m;
-    if (std::regex_search(isec->file.filename, m, re1))
-      return -2;
-    if (std::regex_search(isec->file.filename, m, re2))
-      return 65536;
-
-    std::string name(isec->name());
-    if (std::regex_search(name, m, re3))
-      return std::stoi(m[1]);
-    return -1;
-  };
-
   for (Chunk<E> *chunk : ctx.chunks) {
     if (OutputSection<E> *osec = chunk->to_osec()) {
       if (osec->name == ".ctors" || osec->name == ".dtors") {
@@ -1112,7 +1117,7 @@ void sort_ctor_dtor(Context<E> &ctx) {
 
         std::unordered_map<InputSection<E> *, i64> map;
         for (InputSection<E> *isec : osec->members)
-          map.insert({isec, get_priority(isec)});
+          map.insert({isec, get_ctor_dtor_priority(isec)});
 
         sort(osec->members, [&](InputSection<E> *a, InputSection<E> *b) {
           return map[a] < map[b];
@@ -1122,6 +1127,45 @@ void sort_ctor_dtor(Context<E> &ctx) {
   }
 }
 
+// .ctors/.dtors serves the same purpose as .init_array/.fini_array,
+// albeit with very subtly differences. Both contain pointers to
+// initializer/finalizer functions. The runtime executes them one by one
+// but in the exact opposite order to one another. Therefore, if we are to
+// place the contents of .ctors/.dtors into .init_array/.fini_array, we
+// need to reverse them.
+//
+// It's unfortunate that we have both .ctors/.dtors and
+// .init_array/.fini_array in ELF for historical reasons, but that's
+// the reality we need to deal with.
+template <typename E>
+void fixup_ctors_in_init_array(Context<E> &ctx) {
+  Timer t(ctx, "fixup_ctors_in_init_array");
+
+  for (Chunk<E> *chunk : ctx.chunks) {
+    if (OutputSection<E> *osec = chunk->to_osec()) {
+      if (osec->name == ".init_array" || osec->name == ".fini_array") {
+        for (InputSection<E> *isec : osec->members) {
+          if (isec->name().starts_with(".ctors") ||
+              isec->name().starts_with(".dtors")) {
+            if (isec->sh_size % sizeof(Word<E>)) {
+              Error(ctx) << *isec << ": section corrupted";
+              continue;
+            }
+
+            u8 *buf = (u8 *)isec->contents.data();
+            std::reverse((Word<E> *)buf, (Word<E> *)(buf + isec->sh_size));
+
+            std::span<ElfRel<E>> rels = isec->get_rels(ctx);
+            for (ElfRel<E> &r : rels)
+              r.r_offset = isec->sh_size - r.r_offset - sizeof(Word<E>);
+            std::reverse(rels.begin(), rels.end());
+          }
+        }
+      }
+    }
+  }
+}
+
 template <typename T>
 static void shuffle(std::vector<T> &vec, u64 seed) {
   if (vec.empty())
@@ -1250,12 +1294,15 @@ void compute_section_sizes(Context<E> &ctx) {
   // inserting thunks. This pass cannot be parallelized. That is,
   // create_range_extension_thunks is parallelized internally, but the
   // function itself is not thread-safe.
-  if constexpr (needs_thunk<E>)
+  if constexpr (needs_thunk<E>) {
+    Timer t2(ctx, "create_range_extension_thunks");
+
     if (!ctx.arg.relocatable)
       for (Chunk<E> *chunk : ctx.chunks)
         if (OutputSection<E> *osec = chunk->to_osec())
           if (osec->shdr.sh_flags & SHF_EXECINSTR)
             osec->create_range_extension_thunks(ctx);
+  }
 }
 
 // Find all unresolved symbols and attach them to the most appropriate files.
@@ -1532,9 +1579,11 @@ template <typename E>
 void create_output_symtab(Context<E> &ctx) {
   Timer t(ctx, "compute_symtab_size");
 
-  tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
-    chunk->compute_symtab_size(ctx);
-  });
+  if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file) {
+    tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
+      chunk->compute_symtab_size(ctx);
+    });
+  }
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     file->compute_symtab_size(ctx);
@@ -1549,9 +1598,6 @@ template <typename E>
 void apply_version_script(Context<E> &ctx) {
   Timer t(ctx, "apply_version_script");
 
-  // If all patterns are simple (i.e. not containing any meta-
-  // characters and is not a C++ name), we can simply look up
-  // symbols.
   auto is_simple = [&] {
     for (VersionPattern &v : ctx.version_patterns)
       if (v.is_cpp || v.pattern.find_first_of("*?[") != v.pattern.npos)
@@ -1559,6 +1605,9 @@ void apply_version_script(Context<E> &ctx) {
     return true;
   };
 
+  // If all patterns are simple (i.e. not containing any meta-
+  // characters and is not a C++ name), we can simply look up
+  // symbols.
   if (is_simple()) {
     for (VersionPattern &v : ctx.version_patterns) {
       Symbol<E> *sym = get_symbol(ctx, v.pattern);
@@ -1684,44 +1733,124 @@ void compute_import_export(Context<E> &ctx) {
   if (!ctx.arg.shared) {
     tbb::parallel_for_each(ctx.dsos, [&](SharedFile<E> *file) {
       for (Symbol<E> *sym : file->symbols) {
-        if (sym->file && !sym->file->is_dso && sym->visibility != STV_HIDDEN) {
-          if (sym->ver_idx != VER_NDX_LOCAL ||
-              !ctx.default_version_from_version_script) {
-            std::scoped_lock lock(sym->mu);
-            sym->is_exported = true;
-          }
+        if (sym->file && !sym->file->is_dso && sym->visibility != STV_HIDDEN &&
+            sym->ver_idx != VER_NDX_LOCAL) {
+          std::scoped_lock lock(sym->mu);
+          sym->is_exported = true;
         }
       }
     });
   }
 
+  auto should_export = [&](Symbol<E> &sym) {
+    if (sym.visibility == STV_HIDDEN)
+      return false;
+
+    switch (sym.ver_idx) {
+    case VER_NDX_UNSPECIFIED:
+      if (ctx.arg.shared)
+        return !((ObjectFile<E> *)sym.file)->exclude_libs;
+      return ctx.arg.export_dynamic;
+    case VER_NDX_LOCAL:
+      return false;
+    default:
+      return true;
+    }
+  };
+
   // Export symbols that are not hidden or marked as local.
   // We also want to mark imported symbols as such.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     for (Symbol<E> *sym : file->get_global_syms()) {
-      if (!sym->file || sym->visibility == STV_HIDDEN ||
-          sym->ver_idx == VER_NDX_LOCAL)
-        continue;
-
-      // If we are using a symbol in a DSO, we need to import it at runtime.
-      if (sym->file != file && sym->file->is_dso && !sym->is_absolute()) {
-        std::scoped_lock lock(sym->mu);
-        sym->is_imported = true;
+      // If we are using a symbol in a DSO, we need to import it.
+      if (sym->file && sym->file->is_dso) {
+        if (!sym->is_absolute()) {
+          std::scoped_lock lock(sym->mu);
+          sym->is_imported = true;
+        }
         continue;
       }
 
-      // If we are creating a DSO, all global symbols are exported by default.
-      if (sym->file == file) {
-        std::scoped_lock lock(sym->mu);
+      // If we have a definition of a symbol, we may want to export it.
+      if (sym->file == file && should_export(*sym)) {
         sym->is_exported = true;
 
-        if (ctx.arg.shared && sym->visibility != STV_PROTECTED &&
+        // Exported symbols are marked as imported as well by default
+        // for DSOs.
+        if (ctx.arg.shared &&
+            sym->visibility != STV_PROTECTED &&
             !ctx.arg.Bsymbolic &&
             !(ctx.arg.Bsymbolic_functions && sym->get_type() == STT_FUNC))
           sym->is_imported = true;
       }
     }
   });
+
+
+  // Apply --dynamic-list, --export-dynamic-symbol and
+  // --export-dynamic-symbol-list options.
+  //
+  // The semantics of these options vary depending on whether we are
+  // creating an executalbe or a shared object.
+  //
+  // For executable, matched symbols are exported.
+  //
+  // For shared objects, matched symbols are imported if it is already
+  // exported so that they are interposable. In other words, symbols
+  // that did not match will be bound locally within the output file,
+  // effectively turning them into protected symbols.
+  MultiGlob matcher;
+  MultiGlob cpp_matcher;
+
+  auto handle_match = [&](Symbol<E> *sym) {
+    if (ctx.arg.shared) {
+      if (sym->is_exported)
+        sym->is_imported = true;
+    } else {
+      if (sym->file && !sym->file->is_dso && sym->visibility != STV_HIDDEN)
+        sym->is_exported = true;
+    }
+  };
+
+  for (DynamicPattern &p : ctx.dynamic_list_patterns) {
+    if (p.is_cpp) {
+      if (!cpp_matcher.add(p.pattern, 1))
+        Fatal(ctx) << p.source << ": invalid dynamic list entry: "
+                   << p.pattern;
+      continue;
+    }
+
+    if (p.pattern.find_first_of("*?[") != p.pattern.npos) {
+      if (!matcher.add(p.pattern, 1))
+        Fatal(ctx) << p.source << ": invalid dynamic list entry: "
+                   << p.pattern;
+      continue;
+    }
+
+    handle_match(get_symbol(ctx, p.pattern));
+  }
+
+  if (!matcher.empty() || !cpp_matcher.empty()) {
+    tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+      for (Symbol<E> *sym : file->get_global_syms()) {
+        if (sym->file != file)
+          continue;
+        if (ctx.arg.shared && !sym->is_exported)
+          continue;
+
+        std::string_view name = sym->name();
+
+        if (matcher.find(name)) {
+          handle_match(sym);
+        } else if (!cpp_matcher.empty()) {
+          if (std::optional<std::string_view> s = cpp_demangle(name))
+            name = *s;
+          if (cpp_matcher.find(name))
+            handle_match(sym);
+        }
+      }
+    });
+  }
 }
 
 // Compute the "address-taken" bit for each input section.
@@ -1932,23 +2061,31 @@ void sort_output_sections_regular(Context<E> &ctx) {
 
   // Ties are broken by additional rules
   auto get_rank2 = [&](Chunk<E> *chunk) -> i64 {
-    if (chunk->shdr.sh_type == SHT_NOTE)
-      return -chunk->shdr.sh_addralign;
+    ElfShdr<E> &shdr = chunk->shdr;
+    if (shdr.sh_type == SHT_NOTE)
+      return -shdr.sh_addralign;
 
     if (chunk == ctx.got)
-      return 1;
-    if (chunk->name == ".toc")
       return 2;
-    if (chunk->name == ".alpha_got")
+    if (chunk->name == ".toc")
       return 3;
+    if (chunk->name == ".alpha_got")
+      return 4;
+
+    if (shdr.sh_flags & SHF_MERGE) {
+      if (shdr.sh_flags & SHF_STRINGS)
+        return (5LL << 32) | shdr.sh_entsize;
+      return (6LL << 32) | shdr.sh_entsize;
+    }
+
     if (chunk == ctx.relro_padding)
-      return INT_MAX;
+      return INT64_MAX;
     return 0;
   };
 
   sort(ctx.chunks, [&](Chunk<E> *a, Chunk<E> *b) {
-    return std::tuple{get_rank1(a), get_rank2(a)} <
-           std::tuple{get_rank1(b), get_rank2(b)};
+    return std::tuple{get_rank1(a), get_rank2(a), a->name} <
+           std::tuple{get_rank1(b), get_rank2(b), b->name};
   });
 }
 
@@ -2579,18 +2716,12 @@ void fix_synthetic_symbols(Context<E> &ctx) {
     if (u64 *addr = std::get_if<u64>(&val)) {
       sym->origin = 0;
       sym->value = *addr;
-      continue;
+    } else {
+      Symbol<E> *sym2 = std::get<Symbol<E> *>(val);
+      sym->value = sym2->value;
+      sym->origin = sym2->origin;
+      sym->visibility = sym2->visibility.load();
     }
-
-    Symbol<E> *sym2 = std::get<Symbol<E> *>(val);
-    if (!sym2->file) {
-      Error(ctx) << "--defsym: undefined symbol: " << *sym2;
-      continue;
-    }
-
-    sym->value = sym2->value;
-    sym->origin = sym2->origin;
-    sym->visibility = sym2->visibility.load();
   }
 
 
@@ -2743,6 +2874,7 @@ template void check_duplicate_symbols(Context<E> &);
 template void check_symbol_types(Context<E> &);
 template void sort_init_fini(Context<E> &);
 template void sort_ctor_dtor(Context<E> &);
+template void fixup_ctors_in_init_array(Context<E> &);
 template void shuffle_sections(Context<E> &);
 template void compute_section_sizes(Context<E> &);
 template void sort_output_sections(Context<E> &);

elf/thunks.cc

@@ -170,17 +170,17 @@ static void scan_rels(Context<E> &ctx, InputSection<E> &isec,
 
 template <>
 void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
+  using Thunk = RangeExtensionThunk<E>;
+
   std::span<InputSection<E> *> m = members;
   if (m.empty())
     return;
 
-  m[0]->offset = 0;
-
   // Initialize input sections with a dummy offset so that we can
   // distinguish sections that have got an address with the one who
   // haven't.
-  tbb::parallel_for((i64)1, (i64)m.size(), [&](i64 i) {
-    m[i]->offset = -1;
+  tbb::parallel_for_each(m, [](InputSection<E> *isec) {
+    isec->offset = -1;
   });
 
   // We create thunks from the beginning of the section to the end.
@@ -190,7 +190,7 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   // Input sections between B and C are in the current batch.
   //
   // A is the input section with the smallest address than can reach
-  // anywhere from the current batch.
+  // from the current batch.
   //
   // D is the input section with the largest address such that the thunk
   // is reachable from the current batch if it's inserted right before D.
@@ -198,9 +198,9 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   //  ................................ <input sections> ............
   //     A    B    C    D
   //                    ^ We insert a thunk for the current batch just before D
-  //          <--->       The current batch, which is smaller than batch_size
-  //     <-------->       Smaller than max_distance
-  //          <-------->  Smaller than max_distance
+  //          <--->       The current batch, which is smaller than BATCH_SIZE
+  //     <-------->       Smaller than MAX_DISTANCE
+  //          <-------->  Smaller than MAX_DISTANCE
   //     <------------->  Reachable from the current batch
   i64 a = 0;
   i64 b = 0;
@@ -212,41 +212,40 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   i64 t = 0;
 
   while (b < m.size()) {
+    // Move D foward as far as we can jump from B to a thunk at D.
+    auto d_thunk_end = [&] {
+      u64 d_end = align_to(offset, 1 << m[d]->p2align) + m[d]->sh_size;
+      return align_to(d_end, Thunk::alignment) + max_thunk_size;
+    };
+
+    while (d < m.size() &&
+           (b == d || d_thunk_end() <= m[b]->offset + max_distance())) {
+      offset = align_to(offset, 1 << m[d]->p2align);
+      m[d]->offset = offset;
+      offset += m[d]->sh_size;
+      d++;
+    }
+
     // Move C forward so that C is apart from B by BATCH_SIZE. We want
     // to make sure that there's at least one section between B and C
     // to ensure progress.
     c = b + 1;
-    while (c < m.size() &&
-           m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
+    while (c < d && m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
       c++;
 
-    // Move D foward as far as we can jump from B to anywhere in a thunk at D.
-    d = c;
-    while (d < m.size() &&
-           align_to(offset, 1 << m[d]->p2align) + m[d]->sh_size + max_thunk_size <
-           m[b]->offset + max_distance())
-      d++;
-
     // Move A forward so that A is reachable from C.
     i64 c_offset = (c == m.size()) ? offset : m[c]->offset;
-    while (a < b && a < m.size() && m[a]->offset + max_distance() < c_offset)
+    while (a < b && m[a]->offset + max_distance() < c_offset)
       a++;
 
-    // Assign offsets to all sections before D.
-    for (i64 i = b; i < d; i++) {
-      offset = align_to(offset, 1 << m[i]->p2align);
-      m[i]->offset = offset;
-      offset += m[i]->sh_size;
-    }
-
     // Erase references to out-of-range thunks.
     while (t < thunks.size() && thunks[t]->offset < m[a]->offset)
       reset_thunk(*thunks[t++]);
 
     // Create a new thunk and place it at D.
-    offset = align_to(offset, RangeExtensionThunk<E>::alignment);
+    offset = align_to(offset, Thunk::alignment);
     i64 thunk_idx = thunks.size();
-    RangeExtensionThunk<E> *thunk = new RangeExtensionThunk<E>(*this, offset);
+    Thunk *thunk = new Thunk(*this, offset);
     thunks.emplace_back(thunk);
 
     // Scan relocations between B and C to collect symbols that need

install-build-deps.sh

@@ -36,7 +36,7 @@ gentoo-*)
   emerge-webrsync
   emerge dev-util/cmake
   ;;
-arch-* | archarm-*)
+arch-* | archarm-* | artix-*)
   pacman -Sy
   pacman -S --needed --noconfirm base-devel cmake util-linux
   ;;

test/elf/CMakeLists.txt

@@ -1,84 +1,181 @@
-option(MOLD_ENABLE_QEMU_TESTS "Enable tests on non-native targets" OFF)
+option(MOLD_ENABLE_QEMU_TESTS "Enable tests on non-native targets" ON)
+set(HOST ${CMAKE_HOST_SYSTEM_PROCESSOR})
 
-function(add_target TRIPLE)
-  set(HOST ${CMAKE_HOST_SYSTEM_PROCESSOR})
+if(EXISTS "/proc/cpuinfo")
+  file(READ "/proc/cpuinfo" CPUINFO)
+endif()
 
-  if(${HOST} MATCHES "amd64")
-    set(HOST x86_64)
-  elseif(${HOST} MATCHES "arm.*")
-    set(HOST arm)
-  elseif(${HOST} STREQUAL "ppc64")
-    set(HOST powerpc64)
-  elseif(${HOST} STREQUAL "ppc64le")
-    set(HOST powerpc64le)
-  endif()
+# CMAKE_HOST_SYSTEM_PROCESSOR returns "aarch64" on ARM32 userland
+# with ARM64 kernel. Here, we run `cc -dumpmachine` to determine the
+# compiler's default target.
+execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpmachine
+  RESULT_VARIABLE EXIT_CODE
+  OUTPUT_VARIABLE ARCH
+  ERROR_QUIET)
+if(NOT EXIT_CODE AND ARCH MATCHES "([^-]+).*")
+  set(HOST ${CMAKE_MATCH_1})
+endif()
 
-  if(${TRIPLE} MATCHES "${HOST}-.*")
+if(${HOST} MATCHES "amd64")
+  set(HOST x86_64)
+elseif(${HOST} MATCHES "i386")
+  set(HOST i686)
+elseif(${HOST} MATCHES "arm.*")
+  set(HOST arm)
+elseif(${HOST} STREQUAL "powerpc64")
+  set(HOST ppc64)
+elseif(${HOST} STREQUAL "powerpc64le")
+  set(HOST ppc64le)
+endif()
+
+if(MOLD_ENABLE_QEMU_TESTS)
+  list(APPEND QEMU_ARCHS
+    x86_64 i386 arm aarch64 ppc ppc64 ppc64le sparc64 sh4 s390x
+    alpha riscv64 riscv32 m68k loongarch64)
+
+  LIST(APPEND TRIPLES
+    x86_64-linux-gnu
+    i686-linux-gnu
+    aarch64-linux-gnu
+    arm-linux-gnueabihf
+    riscv64-linux-gnu
+    powerpc-linux-gnu
+    powerpc64-linux-gnu
+    powerpc64le-linux-gnu
+    sparc64-linux-gnu
+    s390x-linux-gnu
+    sh4-linux-gnu
+    alpha-linux-gnu
+    riscv32-linux-gnu
+    m68k-linux-gnu
+    loongarch64-linux-gnu)
+
+  foreach(ARCH IN LISTS QEMU_ARCHS)
+    find_program(HAS_qemu-${ARCH} qemu-${ARCH})
+  endforeach()
+
+  foreach(TRIPLE IN LISTS TRIPLES)
+    find_program(HAS_${TRIPLE}-gcc ${TRIPLE}-gcc)
+  endforeach()
+endif()
+
+function(add_target ARCH TRIPLE)
+  set(CPU ${ARGV2})
+
+  if(${ARCH} STREQUAL ${HOST})
     set(IS_NATIVE 1)
   endif()
 
-  if(${TRIPLE} MATCHES "([^-]+)-.")
-    set(MACHINE ${CMAKE_MATCH_1})
-  endif()
+  file(GLOB ALL_TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
+    "*.sh")
 
-  if(IS_NATIVE OR MOLD_ENABLE_QEMU_TESTS)
-    file(GLOB ALL_TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
-      "*.sh")
+  list(FILTER ALL_TESTS EXCLUDE REGEX "_")
 
-    list(FILTER ALL_TESTS EXCLUDE REGEX "_")
+  file(GLOB TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
+    "${ARCH}_*.sh")
 
-    file(GLOB TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
-      "${MACHINE}_*.sh")
+  list(APPEND TESTS ${ALL_TESTS})
 
-    list(APPEND TESTS ${ALL_TESTS})
+  foreach(TEST IN LISTS TESTS)
+    if(CPU)
+      string(REGEX REPLACE "\\.sh$" "" TESTNAME "${ARCH}_${CPU}-${TEST}")
+    else()
+      string(REGEX REPLACE "\\.sh$" "" TESTNAME "${ARCH}-${TEST}")
+    endif()
 
-    foreach(TEST IN LISTS TESTS)
-      string(REGEX REPLACE "\\.sh$" "" TESTNAME "${MACHINE}-${TEST}")
+    add_test(NAME ${TESTNAME}
+      COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
+      WORKING_DIRECTORY ${mold_BINARY_DIR})
 
-      add_test(NAME ${TESTNAME}
-        COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
-        WORKING_DIRECTORY ${mold_BINARY_DIR})
+    set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT
+      "HOST=${HOST};CPU=${CPU}")
 
-      if(IS_NATIVE)
-        set_tests_properties(${TESTNAME} PROPERTIES
-          SKIP_REGULAR_EXPRESSION "skipped")
-      else()
-        set_tests_properties(${TESTNAME} PROPERTIES
-          ENVIRONMENT "TRIPLE=${TRIPLE}")
-      endif()
-    endforeach()
-  endif()
+    if(IS_NATIVE)
+      set_property(TEST ${TESTNAME} APPEND PROPERTY SKIP_REGULAR_EXPRESSION
+        "skipped")
+    else()
+      set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT
+        "TRIPLE=${TRIPLE}")
+    endif()
+  endforeach()
 endfunction()
 
-add_target(x86_64-linux-gnu)
-add_target(i686-linux-gnu)
-add_target(aarch64-linux-gnu)
-add_target(arm-linux-gnueabihf)
-add_target(riscv64-linux-gnu)
-add_target(powerpc-linux-gnu)
-add_target(powerpc64-linux-gnu)
-add_target(powerpc64le-linux-gnu)
-add_target(sparc64-linux-gnu)
-add_target(s390x-linux-gnu)
-add_target(sh4-linux-gnu)
-add_target(alpha-linux-gnu)
-
-option(MOLD_ENABLE_QEMU_TESTS_RV32 "Enable tests for RV32" OFF)
-if(MOLD_ENABLE_QEMU_TESTS_RV32)
-  add_target(riscv32-linux-gnu)
+if(${HOST} STREQUAL "x86_64" OR (HAS_qemu-x86_64 AND HAS_x86_64-linux-gnu-gcc))
+  add_target(x86_64 x86_64-linux-gnu)
 endif()
 
-option(MOLD_ENABLE_QEMU_TESTS_POWER10 "Enable tests for Power10" OFF)
-if(MOLD_ENABLE_QEMU_TESTS_POWER10)
-  add_target(powerpc64le_power10-linux-gnu)
+if(${HOST} STREQUAL "i686" OR (HAS_qemu-i386 AND HAS_i686-linux-gnu-gcc))
+  add_target(i686 i686-linux-gnu)
 endif()
 
-option(MOLD_ENABLE_QEMU_TESTS_M68K "Enable tests for m68k" OFF)
-if(MOLD_ENABLE_QEMU_TESTS_M68K)
-  add_target(m68k-linux-gnu)
+if(${HOST} STREQUAL "aarch64" OR (HAS_qemu-aarch64 AND HAS_aarch64-linux-gnu-gcc))
+  add_target(aarch64 aarch64-linux-gnu)
 endif()
 
-option(MOLD_ENABLE_QEMU_TESTS_LOONGARCH "Enable tests for LoongArch" OFF)
-if(MOLD_ENABLE_QEMU_TESTS_LOONGARCH)
-  add_target(loongarch64-linux-gnu)
+if(${HOST} STREQUAL "arm" OR (HAS_qemu-arm AND HAS_arm-linux-gnueabihf-gcc))
+  add_target(arm arm-linux-gnueabihf)
+endif()
+
+if(${HOST} STREQUAL "riscv64" OR (HAS_qemu-riscv64 AND HAS_riscv64-linux-gnu-gcc))
+  add_target(riscv64 riscv64-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "riscv32" OR (HAS_qemu-riscv32 AND HAS_riscv32-linux-gnu-gcc))
+  add_target(riscv32 riscv32-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "ppc" OR (HAS_qemu-ppc AND HAS_powerpc-linux-gnu-gcc))
+  add_target(ppc powerpc-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "ppc64" OR (HAS_qemu-ppc64 AND HAS_powerpc64-linux-gnu-gcc))
+  add_target(ppc64 powerpc64-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "ppc64le" OR (HAS_qemu-ppc64le AND HAS_powerpc64le-linux-gnu-gcc))
+  add_target(ppc64le powerpc64le-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "ppc64le" AND "${CPUINFO}" MATCHES "POWER10")
+  add_target(ppc64le powerpc64le-linux-gnu power10)
+elseif(HAS_qemu-ppc64le AND HAS_powerpc64le-linux-gnu-gcc)
+  file(WRITE "${CMAKE_BINARY_DIR}/empty.c" "")
+
+  execute_process(
+    COMMAND powerpc64le-linux-gnu-gcc -mcpu=power10 -E
+      "${CMAKE_BINARY_DIR}/empty.c"
+    RESULT_VARIABLE GCC_EXIT_CODE
+    OUTPUT_QUIET ERROR_QUIET)
+
+  execute_process(
+    COMMAND qemu-ppc64le -cpu help
+    OUTPUT_VARIABLE QEMU_CPUS)
+
+  if("${GCC_EXIT_CODE}" EQUAL "0" AND "${QEMU_CPUS}" MATCHES power10_v2.0)
+    add_target(ppc64le powerpc64le-linux-gnu power10)
+  endif()
+endif()
+
+if(${HOST} STREQUAL "sparc64" OR (HAS_qemu-sparc64 AND HAS_sparc64-linux-gnu-gcc))
+  add_target(sparc64 sparc64-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "s390x" OR (HAS_qemu-s390x AND HAS_s390x-linux-gnu-gcc))
+  add_target(s390x s390x-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "sh4" OR (HAS_qemu-sh4 AND HAS_sh4-linux-gnu-gcc))
+  add_target(sh4 sh4-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "alpha" OR (HAS_qemu-alpha AND HAS_alpha-linux-gnu-gcc))
+  add_target(alpha alpha-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "m68k" OR (HAS_qemu-m68k AND HAS_m68k-linux-gnu-gcc))
+  add_target(m68k m68k-linux-gnu)
+endif()
+
+if(${HOST} STREQUAL "loongarch64" OR (HAS_qemu-loongarch64 AND HAS_loongarch64-linux-gnu-gcc))
+  add_target(loongarch64 loongarch64-linux-gnu)
 endif()

test/elf/aarch64_variant-pcs.sh

@@ -12,7 +12,7 @@ foo:
 EOF
 
 $CC -B. -shared -o $t/b.so $t/a.o
-readelf -W --dyn-syms $t/b.so | grep -Eq '\[VARIANT_PCS\].* foo$'
+readelf -W --dyn-syms $t/b.so | grep foo | grep -q '[VARIANT_PCS]'
 
 cat <<EOF | $CC -c -o $t/c.o -xc -
 void foo();

test/elf/arm_tlsdesc.sh

@@ -2,6 +2,7 @@
 . $(dirname $0)/common.inc
 
 [ $MACHINE = arm ] || skip
+is_musl && skip
 
 echo 'int main() {}' | $GCC -c -o /dev/null -xc - -O0 -mthumb >& /dev/null \
   || skip

test/elf/build-id.sh

@@ -7,8 +7,7 @@ $CC -B. -o $t/exe $t/a.c -Wl,-build-id
 readelf -n $t/exe | grep -qv 'GNU.*0x00000010.*NT_GNU_BUILD_ID'
 
 $CC -B. -o $t/exe $t/a.c -Wl,-build-id=uuid
-readelf -nW $t/exe |
-  grep -Eq 'GNU.*0x00000010.*NT_GNU_BUILD_ID.*Build ID: ............4...[89abcdef]'
+readelf -nW $t/exe | grep -Eq 'Build ID: ............4...[89abcdef]'
 
 $CC -B. -o $t/exe $t/a.c -Wl,-build-id=md5
 readelf -n $t/exe | grep -q 'GNU.*0x00000010.*NT_GNU_BUILD_ID'

test/elf/common.inc

@@ -5,7 +5,7 @@ export LC_ALL=C
 
 canonical_name() {
   case $1 in
-  i?86) echo i386 ;;
+  i?86) echo i686 ;;
   arm*) echo arm ;;
   powerpc) echo ppc ;;
   powerpc64) echo ppc64 ;;
@@ -14,10 +14,12 @@ canonical_name() {
   esac
 }
 
-HOST=$(canonical_name $(uname -m))
+if [ -z "$HOST" ]; then
+  HOST=$(canonical_name $(uname -m))
+fi
 
 # Set tool names
-if [ "$TRIPLE" = "" ]; then
+if [ -z "$TRIPLE" ]; then
   MACHINE=$HOST
   TESTDIR=out/test/elf/$HOST
   CC="${TEST_CC:-cc}"
@@ -28,8 +30,7 @@ if [ "$TRIPLE" = "" ]; then
   OBJCOPY=objcopy
   STRIP=strip
   QEMU=
-elif [ "$TRIPLE" = powerpc64le_power10-linux-gnu ]; then
-  TRIPLE=powerpc64le-linux-gnu
+elif [ "$TRIPLE" = powerpc64le-linux-gnu -a "$CPU" = power10 ]; then
   MACHINE=ppc64le
   TESTDIR=out/test/elf/ppc64le-power10
   CC="${TEST_CC:-$TRIPLE-gcc} -mcpu=power10"
@@ -50,22 +51,20 @@ else
   OBJDUMP="$TRIPLE-objdump"
   OBJCOPY="$TRIPLE-objcopy"
   STRIP="$TRIPLE-strip"
-  QEMU="qemu-$MACHINE -L /usr/$TRIPLE"
+
+  if [ $MACHINE = i686 ]; then
+    QEMU="qemu-i386 -L /usr/$TRIPLE"
+  else
+    QEMU="qemu-$MACHINE -L /usr/$TRIPLE"
+  fi
 fi
 
-if [ $MACHINE = x86_64 -o $MACHINE = i386 -o $MACHINE = arm ]; then
+if [ $MACHINE = x86_64 -o $MACHINE = i686 -o $MACHINE = arm ]; then
   tlsdesc_opt=-mtls-dialect=gnu2
 elif [ $MACHINE = aarch64 ]; then
   tlsdesc_opt=-mtls-dialect=desc
 fi
 
-if [ $MACHINE = ppc64le -a -e /proc/cpuinfo ] && grep -q POWER10 /proc/cpuinfo; then
-  CC="$CC -mcpu=power10"
-  CXX="$CXX -mcpu=power10"
-  GCC="$GCC -mcpu=power10"
-  GXX="$GXX -mcpu=power10"
-fi
-
 # Common functions
 test_cflags() {
   echo 'int main() {}' | $CC "$@" -o /dev/null -xc - >& /dev/null

test/elf/copyrel-relro2.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-
 cat <<EOF | $CC -o $t/a.o -c -xc -fno-PIE -
 #include <setjmp.h>
 #include <signal.h>
@@ -32,7 +31,7 @@ int main() {
 }
 EOF
 
-cat <<EOF | $CC -fPIC -shared -o $t/b.so -xc -
+cat <<EOF | $CC -B. -fPIC -shared -o $t/b.so -xc -
 __attribute__((section (".data.rel.ro"))) char readonly[100] = "abc";
 char readwrite[100] = "abc";
 EOF

test/elf/ctors-in-init-array.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc -
+#include <stdio.h>
+
+static void ctor1() { printf("ctor1 "); }
+static void ctor2() { printf("ctor2 "); }
+static void ctor3() { printf("ctor3 "); }
+static void ctor4() { printf("ctor4 "); }
+
+static void dtor1() { printf("dtor1 "); }
+static void dtor2() { printf("dtor2 "); }
+static void dtor3() { printf("dtor3 "); }
+static void dtor4() { printf("dtor4 "); }
+
+__attribute__((aligned(sizeof(void *)), section(".ctors.65435")))
+void (*ctors65435[])()  = { ctor1 };
+
+__attribute__((aligned(sizeof(void *)), section(".ctors.65433")))
+void (*ctors65433[])()  = { ctor2 };
+
+__attribute__((aligned(sizeof(void *)), section(".ctors")))
+void (*ctors[])()  = { ctor4, ctor3 };
+
+__attribute__((aligned(sizeof(void *)), section(".dtors")))
+void (*dtors[])()  = { dtor1, dtor2 };
+
+__attribute__((aligned(sizeof(void *)), section(".dtors.65433")))
+void (*dtors65433[])()  = { dtor3 };
+
+__attribute__((aligned(sizeof(void *)), section(".dtors.65435")))
+void (*dtors65435[])()  = { dtor4 };
+EOF
+
+cat <<EOF | $CC -o $t/b.o -c -xc -
+#include <stdio.h>
+
+__attribute__((constructor(101)))
+static void init1() { printf("init1 "); }
+
+__attribute__((constructor))
+static void init2() { printf("init2 "); }
+
+__attribute__((destructor(101)))
+static void fini1() { printf("fini1 "); }
+
+__attribute__((destructor))
+static void fini2() { printf("fini2 "); }
+
+int main() {}
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o
+$QEMU $t/exe | grep -q 'ctor1 init1 ctor2 ctor3 ctor4 init2 fini2 dtor1 dtor2 dtor3 fini1 dtor4'

test/elf/defsym-missing-symbol.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc -
+int main() {}
+EOF
+
+! $CC -B. -o $t/exe $t/a.o -Wl,-defsym=foo=bar 2> $t/log
+grep -q 'undefined symbol: bar' $t/log

test/elf/dynamic-list4.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -fPIC
+#include <stdio.h>
+
+void foo() { printf("foo1 "); }
+void bar() { printf("bar1 "); }
+void baz() { printf("baz1 "); }
+
+void print() {
+  foo();
+  bar();
+  baz();
+  printf("\n");
+}
+EOF
+
+cat <<EOF > $t/dyn
+{ foo; bar; };
+EOF
+
+$CC -B. -shared -o $t/b.so $t/a.o -Wl,--dynamic-list=$t/dyn
+
+cat <<EOF | $CC -o $t/c.o -c -xc - -fPIC
+#include <stdio.h>
+void foo() { printf("foo2 "); }
+void bar() { printf("bar2 "); }
+void baz() { printf("baz2 "); }
+EOF
+
+$CC -B. -shared -o $t/d.so $t/c.o
+
+cat <<EOF | $CC -o $t/e.o -c -xc -
+#include <stdio.h>
+void print();
+int main() { print(); }
+EOF
+
+$CC -B. -o $t/exe1 $t/e.o -Wl,-push-state,-no-as-needed $t/b.so -Wl,-pop-state
+$QEMU $t/exe1 | grep -q 'foo1 bar1 baz1'
+
+$CC -B. -o $t/exe2 $t/e.o -Wl,-push-state,-no-as-needed $t/d.so $t/b.so -Wl,-pop-state
+$QEMU $t/exe2 | grep -q 'foo2 bar2 baz1'

test/elf/emit-relocs-dead-sections.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+[ $MACHINE = m68k ] && skip
 [ $MACHINE = sh4 ] && skip
 
 cat <<EOF | $CXX -o $t/a.o -c -fPIC -xc++ -g -ffunction-sections -

test/elf/exception.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+[ $MACHINE = m68k ] && skip
 [ $MACHINE = sh4 ] && skip
 
 static=

test/elf/global-offset-table.sh

@@ -20,7 +20,7 @@ GOT_ADDR=$($QEMU $t/exe)
 
 # _GLOBAL_OFFSET_TABLE_ refers the end of .got only on x86.
 # We assume .got is followed by .gotplt.
-if [ $MACHINE = x86_64 -o $MACHINE = i386 ]; then
+if [ $MACHINE = x86_64 -o $MACHINE = i686 ]; then
   readelf -WS $t/exe | grep -q "\.got\.plt .*$GOT_ADDR "
 else
   readelf -WS $t/exe | grep -q "\.got .*$GOT_ADDR "

test/elf/gnu-unique.sh

@@ -18,5 +18,5 @@ int main() {
 }
 EOF
 
-$CC -B. -o $t/exe $t/a.o $t/b.o
+$CC -B. -o $t/exe $t/a.o $t/b.o -no-pie
 $QEMU $t/exe | grep -q 'foo=5'

test/elf/gnu-warning.sh

@@ -16,5 +16,5 @@ int main() {
 EOF
 
 # Make sure that we do not copy .gnu.warning.* sections.
-$CC -B. -o $t/exe $t/a.o
+$CC -B. -o $t/exe $t/a.o -no-pie
 $QEMU $t/exe | grep -q 'Hello world'

test/elf/i386_tls-module-base.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-[ $MACHINE = i386 ] || skip
+[ $MACHINE = i686 ] || skip
 
 cat <<'EOF' | $CC -fPIC -o $t/a.o -c -xassembler -
 .globl get_foo

test/elf/icf-safe.sh

@@ -1,7 +1,14 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-cat <<EOF | $GCC -c -o $t/a.o -ffunction-sections -fdata-sections -xc -
+# GCC 11 or older uses R_390_PLT32DBL to take an address of a function
+# instead of R_390_PC32DBL
+if [ $MACHINE = s390x ]; then
+  echo 'void *foo() { return foo; }' | $CC -c -o $t/a.o -xc -
+  readelf -r $t/a.o | grep -q R_390_PLT32DBL && skip
+fi
+
+cat <<EOF | $CC -c -o $t/a.o -ffunction-sections -fdata-sections -xc -
 int bar() {
   return 5;
 }
@@ -34,7 +41,7 @@ EOF
 $CC -B. -o $t/exe1 -Wl,-icf=safe $t/a.o $t/b.o
 $QEMU $t/exe1 | grep -q '^0 0$'
 
-cat <<EOF | $GCC -c -o $t/c.o -ffunction-sections -fdata-sections -xc -
+cat <<EOF | $CC -c -o $t/c.o -ffunction-sections -fdata-sections -xc -
 int foo1();
 int foo2();
 int foo3();

test/elf/ifunc-address-equality-exported.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+supports_ifunc || skip
+
+cat <<EOF | $CC -c -fPIC -o $t/a.o -xc -
+typedef void Func();
+void foo();
+Func *get_foo() { return foo; }
+EOF
+
+$CC -B. -shared -o $t/b.so $t/a.o
+
+cat <<EOF | $CC -c -fno-PIE -o $t/c.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+__attribute__((ifunc("resolve_foo"))) void foo(void);
+void real_foo(void) { printf("foo "); }
+Func *resolve_foo() { return real_foo; }
+
+Func *get_foo();
+
+int main() {
+  printf("%p %p\n", foo, get_foo());
+  foo();
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe $t/c.o $t/b.so -no-pie
+$QEMU $t/exe | grep -Eq '^(\S+) \1'

test/elf/ifunc-address-equality.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+supports_ifunc || skip
+
+cat <<EOF | $CC -c -fno-PIE -o $t/a.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+__attribute__((ifunc("resolve_foo"))) void foo(void);
+void real_foo(void) { printf("foo "); }
+Func *resolve_foo() { return real_foo; }
+
+__attribute__((ifunc("resolve_bar"))) void bar(void);
+void real_bar(void) { printf("bar "); }
+Func *resolve_bar() { return real_bar; }
+EOF
+
+cat <<EOF | $CC -c -fPIC -o $t/b.o -xc -
+typedef void Func();
+
+void foo();
+void bar();
+
+Func *get_foo() { return foo; }
+Func *get_bar() { return bar; }
+EOF
+
+cat <<EOF | $CC -c -fno-PIE -o $t/c.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+void foo();
+void bar();
+Func *get_foo();
+Func *get_bar();
+
+int main() {
+  printf("%p %p %p %p\n", foo, get_foo(), bar, get_bar());
+  foo();
+  bar();
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -no-pie
+$QEMU $t/exe1 | grep -Eq '^(\S+) \1 (\S+) \2'
+
+readelf --dynamic $t/exe1 > $t/log1
+! grep -q TEXTREL $t/log1 || false

test/elf/ifunc-alias.sh

@@ -16,5 +16,8 @@ int main() {
 }
 EOF
 
-$CC -B. -o $t/exe $t/a.o
-$QEMU $t/exe | grep -Eq '^(\S+) \1$'
+$CC -B. -o $t/exe1 $t/a.o -pie
+$QEMU $t/exe1 | grep -Eq '^(\S+) \1$'
+
+$CC -B. -o $t/exe2 $t/a.o -no-pie
+$QEMU $t/exe2 | grep -Eq '^(\S+) \1$'

test/elf/ifunc-funcptr.sh

@@ -33,5 +33,8 @@ int main() {
 }
 EOF
 
-$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
-$QEMU $t/exe | grep -q '^3$'
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -pie
+$QEMU $t/exe1 | grep -q '^3$'
+
+$CC -B. -o $t/exe2 $t/a.o $t/b.o $t/c.o -no-pie
+$QEMU $t/exe2 | grep -q '^3$'

test/elf/ifunc-noplt.sh

@@ -24,5 +24,8 @@ int main() {
 }
 EOF
 
-$CC -B. -o $t/exe $t/a.o
-$QEMU $t/exe | grep -q 'Hello world'
+$CC -B. -o $t/exe1 $t/a.o -pie
+$QEMU $t/exe1 | grep -q 'Hello world'
+
+$CC -B. -o $t/exe2 $t/a.o -no-pie
+$QEMU $t/exe2 | grep -q 'Hello world'

test/elf/issue646.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+[ $MACHINE = m68k ] && skip
 [ $MACHINE = sh4 ] && skip
 
 cat <<EOF | $CXX -o $t/a.o -c -xc++ -

test/elf/large-alignment-dso.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-[ $MACHINE = i386 ] && skip
+[ $MACHINE = i686 ] && skip
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections -fPIC
 #include <stdio.h>

test/elf/large-alignment.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-[ $MACHINE = i386 ] && skip
+[ $MACHINE = i686 ] && skip
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections
 #include <stdio.h>

test/elf/loongarch64_mcmodel-extreme.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -mcmodel=extreme
+#include <stdio.h>
+char msg[] = "Hello world\n";
+int main() { printf(msg); }
+EOF
+
+$CC -B. -o $t/exe1 $t/a.o
+$QEMU $t/exe1 | grep -q 'Hello world'

test/elf/nocopyreloc.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-[ $MACHINE = i386 ] && skip
+[ $MACHINE = i686 ] && skip
 [ $MACHINE = m68k ] && skip
 [ $MACHINE = arm ] && skip
 [ $MACHINE = ppc64 ] && skip

test/elf/range-extension-thunk.sh

@@ -2,12 +2,16 @@
 . $(dirname $0)/common.inc
 
 # Skip if 32 bits as we use very large addresses in this test.
-[ $MACHINE = i386 ] && skip
+[ $MACHINE = i686 ] && skip
 [ $MACHINE = riscv32 ] && skip
 
 # It looks like SPARC's runtime can't handle PLT if it's too far from GOT.
 [ $MACHINE = sparc64 ] && skip
 
+# qemu aborts with the "Unknown exception 0x5" error, although this
+# test passes on a real POWER10 machine.
+[ -n "$QEMU" -a "$CPU" = power10 ] && skip
+
 cat <<EOF > $t/a.c
 #include <stdio.h>
 

test/elf/range-extension-thunk2.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections
+#include <stdio.h>
+
+void f0();
+void f1();
+void f2();
+void f3();
+void f4();
+void f5();
+void f6();
+void f7();
+void f8();
+void f9();
+void f10();
+void f11();
+void f12();
+void f13();
+void f14();
+void f15();
+void f16();
+void f17();
+void f18();
+void f19();
+
+void f0(int x) { printf("0 "); if (!x) f9(); }
+void space0() { __asm__(".space 1024*1024"); }
+
+void f1(int x) { printf("1 "); f8(x); }
+void space1() { __asm__(".space 1024*1024"); }
+
+void f2(int x) { printf("2 "); f7(x); }
+void space2() { __asm__(".space 1024*1024"); }
+
+void f3(int x) { printf("3 "); f6(x); }
+void space3() { __asm__(".space 1024*1024"); }
+
+void f4(int x) { printf("4 "); f5(x); }
+void space4() { __asm__(".space 1024*1024"); }
+
+void f5(int x) { printf("5 "); f10(x); }
+void space5() { __asm__(".space 1024*1024"); }
+
+void f6(int x) { printf("6 "); f4(x); }
+void space6() { __asm__(".space 1024*1024"); }
+
+void f7(int x) { printf("7 "); f3(x); }
+void space7() { __asm__(".space 1024*1024"); }
+
+void f8(int x) { printf("8 "); f2(x); }
+void space8() { __asm__(".space 1024*1024"); }
+
+void f9(int x) { printf("9 "); f1(x); }
+void space9() { __asm__(".space 1024*1024"); }
+
+void f10(int x) { printf("10 "); f19(x); }
+void space10() { __asm__(".space 8*1024*1024"); }
+
+void f11(int x) { printf("11 "); f18(x); }
+void space11() { __asm__(".space 8*1024*1024"); }
+
+void f12(int x) { printf("12 "); f17(x); }
+void space12() { __asm__(".space 8*1024*1024"); }
+
+void f13(int x) { printf("13 "); f16(x); }
+void space13() { __asm__(".space 8*1024*1024"); }
+
+void f14(int x) { printf("14 "); f15(x); }
+void space14() { __asm__(".space 8*1024*1024"); }
+
+void f15(int x) { printf("15 "); f0(x + 1); }
+void space15() { __asm__(".space 8*1024*1024"); }
+
+void f16(int x) { printf("16 "); f14(x); }
+void space16() { __asm__(".space 8*1024*1024"); }
+
+void f17(int x) { printf("17 "); f13(x); }
+void space17() { __asm__(".space 8*1024*1024"); }
+
+void f18(int x) { printf("18 "); f12(x); }
+void space18() { __asm__(".space 8*1024*1024"); }
+
+void f19(int x) { printf("19 "); f11(x); }
+void space19() { __asm__(".space 8*1024*1024"); }
+
+int main() {
+  f0(0);
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe $t/a.o
+$QEMU $t/exe | grep -Eq '^0 9 1 8 2 7 3 6 4 5 10 19 11 18 12 17 13 16 14 15 0 $'

test/elf/relocatable-c++.sh

@@ -4,7 +4,8 @@
 # OneTBB isn't tsan-clean
 nm mold | grep -q '__tsan_init' && skip
 
-# Ubuntu 22.04 SH4 GCC is broken
+# Ubuntu 22.04 GCC is broken
+[ $MACHINE = m68k ] && skip
 [ $MACHINE = sh4 ] && skip
 
 cat <<EOF | $CXX -c -o $t/a.o -xc++ -

test/elf/relocatable-exception.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+[ $MACHINE = m68k ] && skip
 [ $MACHINE = sh4 ] && skip
 
 # OneTBB isn't tsan-clean

test/elf/relocatable-mergeable-sections.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+# OneTBB isn't tsan-clean
+nm mold | grep -q '__tsan_init' && skip
+
+cat <<EOF | $CC -c -o $t/a.o -xassembler -
+.section .rodata.str1.1,"aMS",@progbits,1
+val1:
+.ascii "Hello \0"
+
+.section .rodata.str1.1,"aMS",@progbits,1
+val5:
+.ascii "World \0"
+EOF
+
+./mold --relocatable -o $t/b.o $t/a.o
+
+readelf -W -p .rodata.str1.1 $t/b.o | grep -Eq '\b0\b.*Hello'
+readelf -W -p .rodata.str1.1 $t/b.o | grep -Eq '\b7\b.*World'

test/elf/rodata-name.sh

@@ -12,23 +12,23 @@ cat <<'EOF' | $CC -c -o $t/a.o -x assembler -
 val1:
 .ascii "Hello \0"
 
-.section .rodata.str4.4,"aMS",@progbits,4
-.align 4
+.section .rodata.foo,"aMS",@progbits,4
+.p2align 2
 val2:
 .ascii "world   \0\0\0\0"
 
-.section .rodata.foo,"aMS",@progbits,1
+.section .rodata.x,"aMS",@progbits,1
 val3:
 .ascii "foobar\0"
 
 .section .rodata.cst8,"aM",@progbits,8
-.align 8
+.p2align 3
 val4:
 .ascii "abcdefgh"
 
-.section .rodatabar,"aMS",@progbits,1
+.section .rodatabaz,"aMS",@progbits,1
 val5:
-.ascii "bar\0"
+.ascii "baz\0"
 EOF
 
 cat <<'EOF' | $CC -c -o $t/b.o -xc -
@@ -43,8 +43,8 @@ EOF
 
 $CC -B. -o $t/exe $t/a.o $t/b.o
 
-readelf -p .rodata.str $t/exe | grep -q Hello
-readelf -p .rodata.str $t/exe | grep -q world
-readelf -p .rodata.str $t/exe | grep -q foobar
-readelf -p .rodata.cst $t/exe | grep -q abcdefgh
-readelf -p .rodatabar $t/exe | grep -q bar
+readelf -p .rodata.str1.1 $t/exe | grep -q Hello
+readelf -p .rodata.str4.4 $t/exe | grep -q world
+readelf -p .rodata.str1.1 $t/exe | grep -q foobar
+readelf -p .rodata.cst8 $t/exe | grep -q abcdefgh
+readelf -p .rodatabaz $t/exe | grep -q baz

test/elf/run.sh

@@ -4,7 +4,7 @@
 [ "$CC" = cc ] || skip
 
 # ASAN doesn't work with LD_PRELOAD
-nm mold-wrapper.so | grep -q '__[at]san_init' && skip
+nm mold | grep -q '__[at]san_init' && skip
 
 cat <<'EOF' | $CC -xc -c -o $t/a.o -
 #include <stdio.h>

test/elf/tls-ie.sh

@@ -19,7 +19,7 @@ EOF
 
 $CC -B. -shared -o $t/b.so $t/a.o
 
-cat <<EOF | $GCC -c -o $t/c.o -xc -
+cat <<EOF | $GCC -c -o $t/c.o -xc - -fPIC
 #include <stdio.h>
 
 _Thread_local int baz;

test/elf/tls-nopic.sh

@@ -23,5 +23,5 @@ cat <<EOF | $CC -xc -c -o $t/b.o -
 __attribute__((tls_model("global-dynamic"))) _Thread_local int foo;
 EOF
 
-$CC -B. -o $t/exe $t/a.o $t/b.o
+$CC -B. -o $t/exe $t/a.o $t/b.o -no-pie
 $QEMU $t/exe | grep -q '3 5 3 5'

test/elf/version-script6.sh

@@ -9,10 +9,10 @@ EOF
 cat <<EOF | $CXX -fPIC -c -o $t/b.o -xc -
 int foo = 5;
 int bar = 6;
+int quux = 100;
 EOF
 
-$CC -B. -shared -Wl,--version-script=$t/a.ver \
-  -o $t/c.so $t/b.o
+$CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 cat <<'EOF' > $t/d.ver
 VER_Y1 { local; *; };

test/elf/x86_64_empty-mergeable-section.sh

@@ -3,6 +3,7 @@
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -
 .section .rodata.str1.1, "aMS", @progbits, 1
+.section .rodata.foo, "aMS", @progbits, 1
 EOF
 
 cat <<EOF | $CC -o $t/b.o -c -xassembler -
@@ -19,4 +20,5 @@ $CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
 $t/exe
 
 readelf -SW $t/exe > $t/log
-! grep -Fq .rodata.str1.1 $t/log || false
+grep -q '.rodata.str1.1 .* AMS ' $t/log
+! grep -q '.rodata.str1.1 .* AM ' $t/log || false

test/elf/x86_64_endbr.sh

@@ -4,12 +4,12 @@
 [ $MACHINE = x86_64 ] || skip
 test_cflags -fcf-protection || skip
 
-cat <<EOF | $GCC -o $t/a.o -c -xc - -ffunction-sections -O -fcf-protection
+cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections -O -fcf-protection
 int foo() { return 3; }
 int bar() { return foo(); }
 EOF
 
-cat <<EOF | $GCC -o $t/b.o -c -xc - -ffunction-sections -O -fcf-protection
+cat <<EOF | $CC -o $t/b.o -c -xc - -ffunction-sections -O -fcf-protection
 int main() {}
 EOF
 

test/elf/x86_64_gnu-linkonce.sh

@@ -22,5 +22,6 @@ int main() {}
 EOF
 
 $CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
+$OBJDUMP -d $t/exe >& /dev/null || skip
 $OBJDUMP -d $t/exe | grep -A1 '<__x86.get_pc_thunk.bx>:' | \
   grep -Fq 'puts$plt'

test/elf/x86_64_note-property.sh

@@ -4,6 +4,8 @@
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip
 
+echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip
+
 $CC -fcf-protection=branch -c /dev/null -o /dev/null -xc 2> /dev/null || skip
 
 cat <<EOF | $CC -fcf-protection=branch -c -o $t/a.o -xc -

test/elf/x86_64_note-property2.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+# OneTBB isn't tsan-clean
+nm mold | grep -q '__tsan_init' && skip
+
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip
 

test/elf/x86_64_relax.sh

@@ -3,6 +3,7 @@
 
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip
+echo ret | cc -c -o /dev/null -xassembler -Wa,-mrelax-relocations=yes - 2> /dev/null || skip
 
 cat <<EOF | $CC -o $t/a.o -c -x assembler -Wa,-mrelax-relocations=yes -
 .globl bar

test/elf/x86_64_z-ibt.sh

@@ -2,6 +2,7 @@
 . $(dirname $0)/common.inc
 
 [ $MACHINE = x86_64 ] || skip
+echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip
 
 cat <<EOF | $CC -o $t/a.o -c -x assembler -
 .globl main

test/elf/x86_64_z-shstk.sh

@@ -2,6 +2,7 @@
 . $(dirname $0)/common.inc
 
 [ $MACHINE = x86_64 ] || skip
+echo endbr64 | $CC -o /dev/null -c -xassembler - 2> /dev/null || skip
 
 cat <<EOF | $CC -o $t/a.o -c -x assembler -
 .globl main

third-party/blake3/c/CMakeLists.txt

@@ -6,6 +6,7 @@ project(libblake3
   LANGUAGES C ASM
 )
 
+include(CheckCCompilerFlag)
 include(FeatureSummary)
 include(GNUInstallDirs)
 
@@ -131,7 +132,12 @@ elseif(CMAKE_SYSTEM_PROCESSOR IN_LIST BLAKE3_ARMv8_NAMES
   target_sources(blake3 PRIVATE
     blake3_neon.c
   )
-  set_source_files_properties(blake3_dispatch.c PROPERTIES COMPILE_DEFINITIONS BLAKE3_USE_NEON=1)
+  target_compile_options(blake3 PRIVATE -DBLAKE3_USE_NEON=1)
+
+  check_c_compiler_flag(-mfpu=neon BLAKE3_MFPU_NEON_SUPPORTED)
+  if (BLAKE3_MFPU_NEON_SUPPORTED)
+    target_compile_options(blake3 PRIVATE -mfpu=neon)
+  endif()
 
   if (DEFINED BLAKE3_CFLAGS_NEON)
     set_source_files_properties(blake3_neon.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_NEON}")

third-party/rust-demangle/README.md

@@ -18,21 +18,22 @@ As this C port was originally (see the [History](#history) section) *only* for t
 it may lag behind `rustc-demangle` in functionality, for now.
 
 The current port status by category is:
-* **(UNPORTED)** `legacy` (pre-RFC2603 Rust symbols) demangling
+* **ported** `legacy` (pre-RFC2603 Rust symbols) demangling
 * `v0` ([RFC2603](https://rust-lang.github.io/rfcs/2603-rust-symbol-name-mangling-v0.html) Rust symbols) demangling
   * **ported** PRs:
     * [[#23] Support demangling the new Rust mangling scheme (v0).](https://github.com/rust-lang/rustc-demangle/pull/23)
     * [[#26] v0: allow identifiers to start with a digit.](https://github.com/rust-lang/rustc-demangle/pull/26)
     * [[#53] v0: replace `skip_*` methods with `print_*` methods in a "skip printing" mode.](https://github.com/rust-lang/rustc-demangle/pull/53)
       * arguably backported to Rust, as the C port always took this approach
-  * **(UNPORTED)** symbol prefix flexibility (`__R` and `R`, instead of `_R`)
-  * **(UNPORTED)** `min_const_generics` constants (`bool`, `char`, negative signed integers)
-    * this arguably also includes `p` as an *untyped* placeholder constant
-  * **(UNPORTED)** [`str` and structural constants](https://github.com/rust-lang/rfcs/pull/3161)
-    (only usable in `const` generics on unstable Rust)
+    * symbol prefix flexibility (`__R` and `R`, instead of `_R`)
+    * [[#39] Add support for `min_const_generics` constants](https://github.com/rust-lang/rustc-demangle/pull/39)
+      * [[#40] Elide the type when the const value is a placeholder `p`](https://github.com/rust-lang/rustc-demangle/pull/40)
+    * [[#55] v0: demangle structural constants and &str.](https://github.com/rust-lang/rustc-demangle/pull/55)
+      (only usable in `const` generics on unstable Rust)
   * **(UNPORTED)** recursion limits
 * miscellaneous
-  * **(UNPORTED)** extraneous symbol suffix (e.g. `.llvm.*`) removal
+  * **ported** PRs:
+    * [[#30] v0: also support preserving extra suffixes found after mangled symbol.](https://github.com/rust-lang/rustc-demangle/pull/30)
   * **(UNPORTED)** output size limits
 
 Notable differences (intentionally) introduced by porting:

third-party/rust-demangle/rust-demangle.c

@@ -70,6 +70,54 @@ static char next(struct rust_demangler *rdm) {
     return c;
 }
 
+struct hex_nibbles {
+    const char *nibbles;
+    size_t nibbles_len;
+};
+
+static struct hex_nibbles parse_hex_nibbles(struct rust_demangler *rdm) {
+    struct hex_nibbles hex;
+
+    hex.nibbles = NULL;
+    hex.nibbles_len = 0;
+
+    size_t start = rdm->next, hex_len = 0;
+    while (!eat(rdm, '_')) {
+        char c = next(rdm);
+        CHECK_OR(IS_DIGIT(c) || (c >= 'a' && c <= 'f'), return hex);
+        hex_len++;
+    }
+
+    hex.nibbles = rdm->sym + start;
+    hex.nibbles_len = hex_len;
+    return hex;
+}
+
+static struct hex_nibbles
+parse_hex_nibbles_for_const_uint(struct rust_demangler *rdm) {
+    struct hex_nibbles hex = parse_hex_nibbles(rdm);
+    CHECK_OR(!rdm->errored, return hex);
+
+    // Trim leading `0`s.
+    while (hex.nibbles_len > 0 && *hex.nibbles == '0') {
+        hex.nibbles++;
+        hex.nibbles_len--;
+    }
+
+    return hex;
+}
+
+static struct hex_nibbles
+parse_hex_nibbles_for_const_bytes(struct rust_demangler *rdm) {
+    struct hex_nibbles hex = parse_hex_nibbles(rdm);
+    CHECK_OR(!rdm->errored && (hex.nibbles_len % 2 == 0), return hex);
+    return hex;
+}
+
+static uint8_t decode_hex_nibble(char nibble) {
+    return nibble >= 'a' ? 10 + (nibble - 'a') : nibble - '0';
+}
+
 static uint64_t parse_integer_62(struct rust_demangler *rdm) {
     if (eat(rdm, '_'))
         return 0;
@@ -118,7 +166,10 @@ static struct rust_mangled_ident parse_ident(struct rust_demangler *rdm) {
     ident.punycode = NULL;
     ident.punycode_len = 0;
 
-    bool is_punycode = eat(rdm, 'u');
+    bool is_punycode = false;
+    if (rdm->version != -1) {
+        is_punycode = eat(rdm, 'u');
+    }
 
     char c = next(rdm);
     CHECK_OR(IS_DIGIT(c), return ident);
@@ -128,8 +179,10 @@ static struct rust_mangled_ident parse_ident(struct rust_demangler *rdm) {
         while (IS_DIGIT(peek(rdm)))
             len = len * 10 + (next(rdm) - '0');
 
-    // Skip past the optional `_` separator.
-    eat(rdm, '_');
+    if (rdm->version != -1) {
+        // Skip past the optional `_` separator.
+        eat(rdm, '_');
+    }
 
     size_t start = rdm->next;
     rdm->next += len;
@@ -182,6 +235,63 @@ static void print_uint64_hex(struct rust_demangler *rdm, uint64_t x) {
     PRINT(s);
 }
 
+static void
+print_quoted_escaped_char(struct rust_demangler *rdm, char quote, uint32_t c) {
+    CHECK_OR(c < 0xd800 || (c > 0xdfff && c < 0x10ffff), return);
+
+    switch (c) {
+    case '\0':
+        PRINT("\\0");
+        break;
+
+    case '\t':
+        PRINT("\\t");
+        break;
+
+    case '\r':
+        PRINT("\\r");
+        break;
+
+    case '\n':
+        PRINT("\\n");
+        break;
+
+    case '\\':
+        PRINT("\\\\");
+        break;
+
+    case '"':
+        if (quote == '"') {
+            PRINT("\\\"");
+        } else {
+            PRINT("\"");
+        }
+        break;
+
+    case '\'':
+        if (quote == '\'') {
+            PRINT("\\'");
+        } else {
+            PRINT("'");
+        }
+        break;
+
+    default:
+        if (c >= 0x20 && c <= 0x7e) {
+            // Printable ASCII
+            char v = (char)c;
+            print_str(rdm, &v, 1);
+        } else {
+            // FIXME show printable unicode characters without hex encoding
+            PRINT("\\u{");
+            char s[9] = {0};
+            sprintf(s, "%" PRIx32, c);
+            PRINT(s);
+            PRINT("}");
+        }
+    }
+}
+
 static void
 print_ident(struct rust_demangler *rdm, struct rust_mangled_ident ident) {
     if (rdm->errored || rdm->skipping_printing)
@@ -197,12 +307,12 @@ print_ident(struct rust_demangler *rdm, struct rust_mangled_ident ident) {
     while (cap < ident.ascii_len) {
         cap *= 2;
         // Check for overflows.
-        CHECK_OR((cap * 4) / 4 == cap, return );
+        CHECK_OR((cap * 4) / 4 == cap, return);
     }
 
     // Store the output codepoints as groups of 4 UTF-8 bytes.
     uint8_t *out = (uint8_t *)malloc(cap * 4);
-    CHECK_OR(out, return );
+    CHECK_OR(out, return);
 
     // Populate initial output from ASCII fragment.
     for (len = 0; len < ident.ascii_len; len++) {
@@ -342,14 +452,15 @@ static void demangle_generic_arg(struct rust_demangler *rdm);
 static void demangle_type(struct rust_demangler *rdm);
 static bool demangle_path_maybe_open_generics(struct rust_demangler *rdm);
 static void demangle_dyn_trait(struct rust_demangler *rdm);
-static void demangle_const(struct rust_demangler *rdm);
-static void demangle_const_uint(struct rust_demangler *rdm);
+static void demangle_const(struct rust_demangler *rdm, bool in_value);
+static void demangle_const_uint(struct rust_demangler *rdm, char ty_tag);
+static void demangle_const_str_literal(struct rust_demangler *rdm);
 
 /// Optionally enter a binder ('G') for late-bound lifetimes,
 /// printing e.g. `for<'a, 'b> `, and make those lifetimes visible
 /// to the caller (via depth level, which the caller should reset).
 static void demangle_binder(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     uint64_t bound_lifetimes = parse_opt_integer_62(rdm, 'G');
     if (bound_lifetimes > 0) {
@@ -365,7 +476,7 @@ static void demangle_binder(struct rust_demangler *rdm) {
 }
 
 static void demangle_path(struct rust_demangler *rdm, bool in_value) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     char tag = next(rdm);
     switch (tag) {
@@ -383,7 +494,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
     }
     case 'N': {
         char ns = next(rdm);
-        CHECK_OR(IS_LOWER(ns) || IS_UPPER(ns), return );
+        CHECK_OR(IS_LOWER(ns) || IS_UPPER(ns), return);
 
         demangle_path(rdm, in_value);
 
@@ -428,6 +539,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
         rdm->skipping_printing = true;
         demangle_path(rdm, in_value);
         rdm->skipping_printing = was_skipping_printing;
+        __attribute__((fallthrough));
     case 'Y':
         PRINT("<");
         demangle_type(rdm);
@@ -460,7 +572,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
         break;
     }
     default:
-        ERROR_AND(return );
+        ERROR_AND(return);
     }
 }
 
@@ -469,7 +581,7 @@ static void demangle_generic_arg(struct rust_demangler *rdm) {
         uint64_t lt = parse_integer_62(rdm);
         print_lifetime_from_index(rdm, lt);
     } else if (eat(rdm, 'K'))
-        demangle_const(rdm);
+        demangle_const(rdm, false);
     else
         demangle_type(rdm);
 }
@@ -525,7 +637,7 @@ static const char *basic_type(char tag) {
 }
 
 static void demangle_type(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     char tag = next(rdm);
 
@@ -565,11 +677,11 @@ static void demangle_type(struct rust_demangler *rdm) {
         demangle_type(rdm);
         if (tag == 'A') {
             PRINT("; ");
-            demangle_const(rdm);
+            demangle_const(rdm, true);
         }
         PRINT("]");
         break;
-    case 'T':
+    case 'T': {
         PRINT("(");
         size_t i;
         for (i = 0; !rdm->errored && !eat(rdm, 'E'); i++) {
@@ -581,6 +693,7 @@ static void demangle_type(struct rust_demangler *rdm) {
             PRINT(",");
         PRINT(")");
         break;
+    }
     case 'F': {
         uint64_t old_bound_lifetime_depth = rdm->bound_lifetime_depth;
         demangle_binder(rdm);
@@ -652,7 +765,7 @@ static void demangle_type(struct rust_demangler *rdm) {
         // Restore `bound_lifetime_depth` to outside the binder.
         rdm->bound_lifetime_depth = old_bound_lifetime_depth;
 
-        CHECK_OR(eat(rdm, 'L'), return );
+        CHECK_OR(eat(rdm, 'L'), return);
         uint64_t lt = parse_integer_62(rdm);
         if (lt) {
             PRINT(" + ");
@@ -709,7 +822,7 @@ static bool demangle_path_maybe_open_generics(struct rust_demangler *rdm) {
 }
 
 static void demangle_dyn_trait(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     bool open = demangle_path_maybe_open_generics(rdm);
 
@@ -730,22 +843,17 @@ static void demangle_dyn_trait(struct rust_demangler *rdm) {
         PRINT(">");
 }
 
-static void demangle_const(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+static void demangle_const(struct rust_demangler *rdm, bool in_value) {
+    CHECK_OR(!rdm->errored, return);
 
-    if (eat(rdm, 'B')) {
-        size_t backref = parse_integer_62(rdm);
-        if (!rdm->skipping_printing) {
-            size_t old_next = rdm->next;
-            rdm->next = backref;
-            demangle_const(rdm);
-            rdm->next = old_next;
-        }
-        return;
-    }
+    bool opened_brace = false;
 
     char ty_tag = next(rdm);
     switch (ty_tag) {
+    case 'p':
+        PRINT("_");
+        break;
+
     // Unsigned integer types.
     case 'h':
     case 't':
@@ -753,66 +861,464 @@ static void demangle_const(struct rust_demangler *rdm) {
     case 'y':
     case 'o':
     case 'j':
+        demangle_const_uint(rdm, ty_tag);
         break;
 
-    default:
-        ERROR_AND(return );
+    case 'a':
+    case 's':
+    case 'l':
+    case 'x':
+    case 'n':
+    case 'i':
+        if (eat(rdm, 'n')) {
+            PRINT("-");
+        }
+        demangle_const_uint(rdm, ty_tag);
+        break;
+
+    case 'b': {
+        struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+        CHECK_OR(!rdm->errored && hex.nibbles_len <= 1, return);
+        uint8_t v = hex.nibbles_len > 0 ? decode_hex_nibble(hex.nibbles[0]) : 0;
+        CHECK_OR(v <= 1, return);
+        PRINT(v == 1 ? "true" : "false");
+        break;
     }
 
-    if (eat(rdm, 'p'))
-        PRINT("_");
-    else {
-        demangle_const_uint(rdm);
-        if (rdm->verbose)
-            PRINT(basic_type(ty_tag));
+    case 'c': {
+        struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+        CHECK_OR(!rdm->errored && hex.nibbles_len <= 6, return);
+
+        uint32_t c = 0;
+        for (size_t i = 0; i < hex.nibbles_len; i++)
+            c = (c << 4) | decode_hex_nibble(hex.nibbles[i]);
+
+        PRINT("'");
+        print_quoted_escaped_char(rdm, '\'', c);
+        PRINT("'");
+
+        break;
+    }
+
+    case 'e':
+        // NOTE(eddyb) a string literal `"..."` has type `&str`, so
+        // to get back the type `str`, `*"..."` syntax is needed
+        // (even if that may not be valid in Rust itself).
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+        PRINT("*");
+
+        demangle_const_str_literal(rdm);
+        break;
+
+    case 'R':
+    case 'Q':
+        if (ty_tag == 'R' && eat(rdm, 'e')) {
+            // NOTE(eddyb) this prints `"..."` instead of `&*"..."`, which
+            // is what `Re..._` would imply (see comment for `str` above).
+            demangle_const_str_literal(rdm);
+            break;
+        }
+
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("&");
+        if (ty_tag != 'R') {
+            PRINT("mut ");
+        }
+
+        demangle_const(rdm, true);
+        break;
+
+    case 'A': {
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("[");
+
+        size_t i = 0;
+        while (!eat(rdm, 'E')) {
+            CHECK_OR(!rdm->errored, return);
+
+            if (i > 0)
+                PRINT(", ");
+
+            demangle_const(rdm, true);
+
+            i += 1;
+        }
+
+        PRINT("]");
+        break;
+    }
+
+    case 'T': {
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("(");
+
+        size_t i = 0;
+        while (!eat(rdm, 'E')) {
+            CHECK_OR(!rdm->errored, return);
+
+            if (i > 0)
+                PRINT(", ");
+
+            demangle_const(rdm, true);
+
+            i += 1;
+        }
+
+        if (i == 1)
+            PRINT(",");
+
+        PRINT(")");
+        break;
+    }
+
+    case 'V':
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        demangle_path(rdm, true);
+
+        switch (next(rdm)) {
+        case 'U':
+            break;
+
+        case 'T': {
+            PRINT("(");
+
+            size_t i = 0;
+            while (!eat(rdm, 'E')) {
+                CHECK_OR(!rdm->errored, return);
+
+                if (i > 0)
+                    PRINT(", ");
+
+                demangle_const(rdm, true);
+
+                i += 1;
+            }
+
+            PRINT(")");
+            break;
+        }
+
+        case 'S': {
+            PRINT(" { ");
+
+            size_t i = 0;
+            while (!eat(rdm, 'E')) {
+                CHECK_OR(!rdm->errored, return);
+
+                if (i > 0)
+                    PRINT(", ");
+
+                parse_disambiguator(rdm);
+
+                struct rust_mangled_ident name = parse_ident(rdm);
+                print_ident(rdm, name);
+
+                PRINT(": ");
+
+                demangle_const(rdm, true);
+
+                i += 1;
+            }
+
+            PRINT(" }");
+            break;
+        }
+
+        default:
+            ERROR_AND(return);
+        }
+
+        break;
+
+    case 'B': {
+        size_t backref = parse_integer_62(rdm);
+        if (!rdm->skipping_printing) {
+            size_t old_next = rdm->next;
+            rdm->next = backref;
+            demangle_const(rdm, in_value);
+            rdm->next = old_next;
+        }
+        break;
+    }
+
+    default:
+        ERROR_AND(return);
+    }
+
+    if (opened_brace) {
+        PRINT("}");
     }
 }
 
-static void demangle_const_uint(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+static void demangle_const_uint(struct rust_demangler *rdm, char ty_tag) {
+    CHECK_OR(!rdm->errored, return);
 
-    uint64_t value = 0;
-    size_t hex_len = 0;
-    while (!eat(rdm, '_')) {
-        value <<= 4;
-
-        char c = next(rdm);
-        if (IS_DIGIT(c))
-            value |= c - '0';
-        else if (c >= 'a' && c <= 'f')
-            value |= 10 + (c - 'a');
-        else
-            ERROR_AND(return );
-        hex_len++;
-    }
+    struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+    CHECK_OR(!rdm->errored, return);
 
     // Print anything that doesn't fit in `uint64_t` verbatim.
-    if (hex_len > 16) {
+    if (hex.nibbles_len > 16) {
         PRINT("0x");
-        print_str(rdm, rdm->sym + (rdm->next - hex_len), hex_len);
-        return;
+        print_str(rdm, hex.nibbles, hex.nibbles_len);
+    } else {
+        uint64_t v = 0;
+        for (size_t i = 0; i < hex.nibbles_len; i++)
+            v = (v << 4) | decode_hex_nibble(hex.nibbles[i]);
+        print_uint64(rdm, v);
     }
 
-    print_uint64(rdm, value);
+    if (rdm->verbose)
+        PRINT(basic_type(ty_tag));
+}
+
+// UTF-8 uses an unary encoding for its "length" field (`1`s followed by a `0`).
+struct utf8_byte {
+    // Decoded "length" field of an UTF-8 byte, including the special cases:
+    // - `0` indicates this is a lone ASCII byte
+    // - `1` indicates a continuation byte (cannot start an UTF-8 sequence)
+    size_t seq_len;
+
+    // Remaining (`payload_width`) bits in the UTF-8 byte, contributing to
+    // the Unicode scalar value being encoded in the UTF-8 sequence.
+    uint8_t payload;
+    size_t payload_width;
+};
+static struct utf8_byte utf8_decode(uint8_t byte) {
+    struct utf8_byte utf8;
+
+    utf8.seq_len = 0;
+    utf8.payload = byte;
+    utf8.payload_width = 8;
+
+    // FIXME(eddyb) figure out if using "count leading ones/zeros" is an option.
+    while (utf8.seq_len <= 6) {
+        uint8_t msb = 0x80 >> utf8.seq_len;
+        utf8.payload &= ~msb;
+        utf8.payload_width--;
+        if ((byte & msb) == 0)
+            break;
+        utf8.seq_len++;
+    }
+
+    return utf8;
+}
+
+static void demangle_const_str_literal(struct rust_demangler *rdm) {
+    CHECK_OR(!rdm->errored, return);
+
+    struct hex_nibbles hex = parse_hex_nibbles_for_const_bytes(rdm);
+    CHECK_OR(!rdm->errored, return);
+
+    PRINT("\"");
+    for (size_t i = 0; i < hex.nibbles_len; i += 2) {
+        struct utf8_byte utf8 = utf8_decode(
+            (decode_hex_nibble(hex.nibbles[i]) << 4) |
+            decode_hex_nibble(hex.nibbles[i + 1])
+        );
+        uint32_t c = utf8.payload;
+        if (utf8.seq_len > 0) {
+            CHECK_OR(utf8.seq_len >= 2 && utf8.seq_len <= 4, return);
+            for (size_t extra = utf8.seq_len - 1; extra > 0; extra--) {
+                i += 2;
+                utf8 = utf8_decode(
+                    (decode_hex_nibble(hex.nibbles[i]) << 4) |
+                    decode_hex_nibble(hex.nibbles[i + 1])
+                );
+                CHECK_OR(utf8.seq_len == 1, return);
+                c = (c << utf8.payload_width) | utf8.payload;
+            }
+        }
+        print_quoted_escaped_char(rdm, '"', c);
+    }
+    PRINT("\"");
+}
+
+static bool is_rust_hash(struct rust_mangled_ident name) {
+    if (name.ascii[0] != 'h') {
+        return false;
+    }
+    for (size_t i = 1; i < name.ascii_len; i++) {
+        if (!IS_DIGIT(name.ascii[i]) &&
+            !(name.ascii[i] >= 'a' && name.ascii[i] <= 'f')) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static void print_legacy_ident(
+    struct rust_demangler *rdm, struct rust_mangled_ident ident
+) {
+    if (rdm->errored || rdm->skipping_printing)
+        return;
+
+    CHECK_OR(!ident.punycode, return);
+
+    if (ident.ascii[0] == '_' && ident.ascii[1] == '$') {
+        ident.ascii += 1;
+        ident.ascii_len -= 1;
+    }
+
+    while (1) {
+        if (ident.ascii_len == 0) {
+            break;
+        } else if (ident.ascii[0] == '.') {
+            if (ident.ascii_len >= 2 && ident.ascii[1] == '.') {
+                PRINT("::");
+                ident.ascii += 2;
+                ident.ascii_len -= 2;
+            } else {
+                PRINT(".");
+                ident.ascii += 1;
+                ident.ascii_len -= 1;
+            }
+        } else if (ident.ascii[0] == '$') {
+            const char *end_ptr =
+                (const char *)memchr(&ident.ascii[1], '$', ident.ascii_len - 1);
+            if (!end_ptr)
+                break;
+            const char *escape = &ident.ascii[1];
+            size_t escape_len = end_ptr - escape;
+
+            if (strncmp(escape, "SP", 2) == 0) {
+                PRINT("@");
+            } else if (strncmp(escape, "BP", 2) == 0) {
+                PRINT("*");
+            } else if (strncmp(escape, "RF", 2) == 0) {
+                PRINT("&");
+            } else if (strncmp(escape, "LT", 2) == 0) {
+                PRINT("<");
+            } else if (strncmp(escape, "GT", 2) == 0) {
+                PRINT(">");
+            } else if (strncmp(escape, "LP", 2) == 0) {
+                PRINT("(");
+            } else if (strncmp(escape, "RP", 2) == 0) {
+                PRINT(")");
+            } else if (strncmp(escape, "C", 1) == 0) {
+                PRINT(",");
+            } else {
+                if (escape[0] != 'u') {
+                    break;
+                }
+
+                const char *digits = &escape[1];
+                size_t digits_len = escape_len - 1;
+
+                bool invalid = false;
+                for (size_t i = 1; i < digits_len; i++) {
+                    if (!IS_DIGIT(digits[i]) &&
+                        !(digits[i] >= 'a' && digits[i] <= 'f')) {
+                        invalid = true;
+                        break;
+                    }
+                }
+                if (invalid)
+                    break;
+
+                struct hex_nibbles hex;
+
+                hex.nibbles = digits;
+                hex.nibbles_len = digits_len;
+
+                uint32_t c = 0;
+                for (size_t i = 0; i < hex.nibbles_len; i++)
+                    c = (c << 4) | decode_hex_nibble(hex.nibbles[i]);
+
+                if (!(c < 0xd800 || (c > 0xdfff && c < 0x10ffff))) {
+                    break; // Not a valid unicode scalar
+                }
+
+                if (c >= 0x20 && c <= 0x7e) {
+                    // Printable ASCII
+                    char v = (char)c;
+                    print_str(rdm, &v, 1);
+                } else {
+                    // FIXME show printable unicode characters without hex
+                    // encoding
+                    PRINT("\\u{");
+                    char s[9] = {0};
+                    sprintf(s, "%" PRIx32, c);
+                    PRINT(s);
+                    PRINT("}");
+                }
+            }
+
+            ident.ascii += escape_len + 2;
+            ident.ascii_len -= escape_len + 2;
+        } else {
+            bool found = false;
+            for (size_t i = 0; i < ident.ascii_len; i++) {
+                if (ident.ascii[i] == '$' || ident.ascii[i] == '.') {
+                    print_str(rdm, ident.ascii, i);
+                    ident.ascii += i;
+                    ident.ascii_len -= i;
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                break;
+            }
+        }
+    }
+
+    print_str(rdm, ident.ascii, ident.ascii_len);
+}
+
+static void demangle_legacy_path(struct rust_demangler *rdm) {
+    bool first = true;
+
+    while (1) {
+        if (eat(rdm, 'E')) {
+            // FIXME Maybe check if at end of symbol?
+            return;
+        }
+
+        struct rust_mangled_ident name = parse_ident(rdm);
+
+        if (!rdm->verbose && peek(rdm) == 'E' && is_rust_hash(name)) {
+            // Skip printing the hash if verbose mode is disabled.
+            eat(rdm, 'E');
+            break;
+        }
+
+        if (!first) {
+            PRINT("::");
+        }
+        first = false;
+
+        print_legacy_ident(rdm, name);
+
+        CHECK_OR(!rdm->errored, return);
+    }
 }
 
 bool rust_demangle_with_callback(
-    const char *mangled, int flags,
+    const char *whole_mangled_symbol, int flags,
     void (*callback)(const char *data, size_t len, void *opaque), void *opaque
 ) {
-    // Rust symbols always start with _R.
-    if (mangled[0] == '_' && mangled[1] == 'R')
-        mangled += 2;
-    else
-        return false;
-
-    // Paths always start with uppercase characters.
-    if (!IS_UPPER(mangled[0]))
-        return false;
-
     struct rust_demangler rdm;
 
-    rdm.sym = mangled;
+    rdm.sym = whole_mangled_symbol;
     rdm.sym_len = 0;
 
     rdm.callback_opaque = opaque;
@@ -822,26 +1328,84 @@ bool rust_demangle_with_callback(
     rdm.errored = false;
     rdm.skipping_printing = false;
     rdm.verbose = (flags & RUST_DEMANGLE_FLAG_VERBOSE) != 0;
-    rdm.version = 0;
+    rdm.version = -2; // Invalid version
     rdm.bound_lifetime_depth = 0;
 
-    // Rust symbols use only [_0-9a-zA-Z] characters.
-    for (const char *p = mangled; *p; p++) {
-        if (!(*p == '_' || IS_DIGIT(*p) || IS_LOWER(*p) || IS_UPPER(*p)))
+    // Rust symbols always start with R, _R or __R for the v0 scheme or ZN, _ZN
+    // or __ZN for the legacy scheme.
+    if (strncmp(rdm.sym, "_R", 2) == 0) {
+        rdm.sym += 2;
+        rdm.version = 0; // v0
+    } else if (rdm.sym[0] == 'R') {
+        // On Windows, dbghelp strips leading underscores, so we accept "R..."
+        // form too.
+        rdm.sym += 1;
+        rdm.version = 0; // v0
+    } else if (strncmp(rdm.sym, "__R", 3) == 0) {
+        // On OSX, symbols are prefixed with an extra _
+        rdm.sym += 3;
+        rdm.version = 0; // v0
+    } else if (strncmp(rdm.sym, "_ZN", 3) == 0) {
+        rdm.sym += 3;
+        rdm.version = -1; // legacy
+    } else if (strncmp(rdm.sym, "ZN", 2) == 0) {
+        // On Windows, dbghelp strips leading underscores, so we accept "R..."
+        // form too.
+        rdm.sym += 2;
+        rdm.version = -1; // legacy
+    } else if (strncmp(rdm.sym, "__ZN", 4) == 0) {
+        // On OSX, symbols are prefixed with an extra _
+        rdm.sym += 4;
+        rdm.version = -1; // legacy
+    } else {
+        return false;
+    }
+
+    if (rdm.version != -1) {
+        // Paths always start with uppercase characters.
+        if (!IS_UPPER(rdm.sym[0]))
             return false;
+    }
+
+    // Rust symbols only use ASCII characters.
+    for (const char *p = rdm.sym; *p; p++) {
+        if ((*p & 0x80) != 0)
+            return false;
+
+        if (*p == '.' && strncmp(p, ".llvm.", 6) == 0) {
+            // Ignore .llvm.<hash> suffixes
+            break;
+        }
+
         rdm.sym_len++;
     }
 
-    demangle_path(&rdm, true);
+    if (rdm.version == -1) {
+        demangle_legacy_path(&rdm);
+    } else {
+        demangle_path(&rdm, true);
 
-    // Skip instantiating crate.
-    if (!rdm.errored && rdm.next < rdm.sym_len) {
-        rdm.skipping_printing = true;
-        demangle_path(&rdm, false);
+        // Skip instantiating crate.
+        if (!rdm.errored && rdm.next < rdm.sym_len && peek(&rdm) >= 'A' &&
+            peek(&rdm) <= 'Z') {
+            rdm.skipping_printing = true;
+            demangle_path(&rdm, false);
+        }
     }
 
-    // It's an error to not reach the end.
-    rdm.errored |= rdm.next != rdm.sym_len;
+    if (!rdm.errored && (rdm.sym_len - rdm.next > 0)) {
+        for (const char *p = rdm.sym + rdm.next; *p; p++) {
+            // FIXME match is_symbol_like from rustc-demangle
+            if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
+                  (*p >= '0' && *p <= '9') || *p == '.')) {
+                // Suffix is not a symbol like string
+                return false;
+            }
+        }
+
+        // Print LLVM produced suffix
+        print_str(&rdm, rdm.sym + rdm.next, rdm.sym_len - rdm.next);
+    }
 
     return !rdm.errored;
 }

third-party/rust-demangle/test-harness/src/lib.rs

@@ -54,8 +54,7 @@ pub fn try_demangle(s: &str) -> Result<Demangle<'_>, rustc_demangle::TryDemangle
             rustc_demangle: d,
         }),
         Err(e) => {
-            // HACK(eddyb) ensure the C port also errors.
-            assert_eq!(demangle_via_c(s, false), "");
+            assert!(demangle_via_c(s, false).is_err());
 
             Err(e)
         }
@@ -68,6 +67,36 @@ impl fmt::Display for Demangle<'_> {
     }
 }
 
+// HACK(eddyb) the C port doesn't have the "is printable Unicode" heuristic,
+// to avoid having to include the non-trivial amount of data that requires,
+// so instead we allow mismatches when `b` has more `\u{...}` escapes than `a`.
+fn equal_modulo_unicode_escapes(a: &str, b: &str) -> bool {
+    let mut a_chars = a.chars();
+    let mut a_active_escape: Option<std::char::EscapeUnicode> = None;
+    let mut b_chars = b.chars();
+    loop {
+        let a_ch = a_active_escape
+            .as_mut()
+            .and_then(|escape| escape.next())
+            .or_else(|| {
+                a_active_escape = None;
+                a_chars.next()
+            });
+        let b_ch = b_chars.next();
+        match (a_ch, b_ch) {
+            (Some(a_ch), Some(b_ch)) if a_ch == b_ch => {}
+            // Compare with the `\u{...}` escape instead, if possible.
+            (Some(a_ch), Some('\\')) if a_active_escape.is_none() => {
+                let mut escape = a_ch.escape_unicode();
+                assert_eq!(escape.next(), Some('\\'));
+                a_active_escape = Some(escape);
+            }
+            (None, None) => return true,
+            _ => return false,
+        }
+    }
+}
+
 impl Demangle<'_> {
     pub fn to_string_maybe_verbose(&self, verbose: bool) -> String {
         let rust = if verbose {
@@ -75,8 +104,8 @@ impl Demangle<'_> {
         } else {
             format!("{:#}", self.rustc_demangle)
         };
-        let c = demangle_via_c(self.original, verbose);
-        if rust != c {
+        let c = demangle_via_c(self.original, verbose).unwrap_or_else(|_| self.original.to_owned());
+        if rust != c && !equal_modulo_unicode_escapes(&rust, &c) {
             panic!(
                 "Rust vs C demangling difference:\
             \n mangled: {mangled:?}\
@@ -91,7 +120,7 @@ impl Demangle<'_> {
     }
 }
 
-fn demangle_via_c(mangled: &str, verbose: bool) -> String {
+fn demangle_via_c(mangled: &str, verbose: bool) -> Result<String, ()> {
     use std::ffi::{CStr, CString};
     use std::os::raw::c_char;
 
@@ -101,15 +130,18 @@ fn demangle_via_c(mangled: &str, verbose: bool) -> String {
     }
 
     let flags = if verbose { 1 } else { 0 };
-    let mangled = CString::new(mangled).unwrap();
+    let Ok(mangled) = CString::new(mangled) else {
+        // C can't handle strings containing nul bytes
+        return Err(());
+    };
     let out = unsafe { rust_demangle(mangled.as_ptr(), flags) };
     if out.is_null() {
-        String::new()
+        Err(())
     } else {
         unsafe {
             let s = CStr::from_ptr(out).to_string_lossy().into_owned();
             free(out);
-            s
+            Ok(s)
         }
     }
 }

third-party/rust-demangle/test-harness/tests/legacy.rs

@@ -53,8 +53,6 @@ fn ok_err(sym: &str) -> bool {
     }
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle() {
     t_err!("test");
@@ -63,8 +61,6 @@ fn demangle() {
     t!("_ZN4test1a2bcE", "test::a::bc");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_dollars() {
     t!("_ZN4$RP$E", ")");
@@ -74,16 +70,12 @@ fn demangle_dollars() {
     t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_many_dollars() {
     t!("_ZN13test$u20$test4foobE", "test test::foob");
     t!("_ZN12test$BP$test4foobE", "test*test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_osx() {
     t!(
@@ -94,8 +86,6 @@ fn demangle_osx() {
     t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::<impl core::iter::traits::IntoIterator for &'a [T]>::into_iter::h450e234d27262170");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_windows() {
     t!("ZN4testE", "test");
@@ -103,8 +93,6 @@ fn demangle_windows() {
     t!("ZN12test$RF$test4foobE", "test&test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_elements_beginning_with_underscore() {
     t!("_ZN13_$LT$test$GT$E", "<test>");
@@ -112,8 +100,6 @@ fn demangle_elements_beginning_with_underscore() {
     t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_trait_impls() {
     t!(
@@ -122,8 +108,6 @@ fn demangle_trait_impls() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash() {
     let s = "_ZN3foo17h05af221e174051e9E";
@@ -131,8 +115,6 @@ fn demangle_without_hash() {
     t_nohash!(s, "foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash_edgecases() {
     // One element, no hash.
@@ -151,8 +133,6 @@ fn demangle_without_hash_edgecases() {
     t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     // One element, no hash.
@@ -164,23 +144,17 @@ fn demangle_thinlto() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_llvm_ir_branch_labels() {
     t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i");
     t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut.exit.i.i");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() {
     t_err!("_ZN3fooE.llvm moocow");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn dont_panic() {
     rust_demangle_c_test_harness::demangle("_ZN2222222222222222222222EE").to_string();
@@ -196,22 +170,16 @@ fn dont_panic() {
     .to_string();
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn invalid_no_chop() {
     t_err!("_ZNfooE");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_assoc_types() {
     t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", "<alloc::boxed::Box<alloc::boxed::FnBox<A, Output=R> + 'a> as core::ops::function::FnOnce<A>>::call_once::h69e8f44b3723e1ca");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_bang() {
     t!(
@@ -220,8 +188,6 @@ fn handle_bang() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_utf8_idents() {
     t_nohash!(
@@ -230,8 +196,6 @@ fn demangle_utf8_idents() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_issue_60925() {
     t_nohash!(

third-party/rust-demangle/test-harness/tests/top_level.rs

@@ -56,8 +56,6 @@ fn ok_err(sym: &str) -> bool {
     }
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle() {
     t_err!("test");
@@ -66,8 +64,6 @@ fn demangle() {
     t!("_ZN4test1a2bcE", "test::a::bc");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_dollars() {
     t!("_ZN4$RP$E", ")");
@@ -77,16 +73,12 @@ fn demangle_dollars() {
     t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_many_dollars() {
     t!("_ZN13test$u20$test4foobE", "test test::foob");
     t!("_ZN12test$BP$test4foobE", "test*test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_osx() {
     t!(
@@ -97,8 +89,6 @@ fn demangle_osx() {
     t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::<impl core::iter::traits::IntoIterator for &'a [T]>::into_iter::h450e234d27262170");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_windows() {
     t!("ZN4testE", "test");
@@ -106,8 +96,6 @@ fn demangle_windows() {
     t!("ZN12test$RF$test4foobE", "test&test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_elements_beginning_with_underscore() {
     t!("_ZN13_$LT$test$GT$E", "<test>");
@@ -115,8 +103,6 @@ fn demangle_elements_beginning_with_underscore() {
     t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_trait_impls() {
     t!(
@@ -125,8 +111,6 @@ fn demangle_trait_impls() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash() {
     let s = "_ZN3foo17h05af221e174051e9E";
@@ -134,8 +118,6 @@ fn demangle_without_hash() {
     t_nohash!(s, "foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash_edgecases() {
     // One element, no hash.
@@ -154,8 +136,6 @@ fn demangle_without_hash_edgecases() {
     t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     // One element, no hash.
@@ -167,23 +147,17 @@ fn demangle_thinlto() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_llvm_ir_branch_labels() {
     t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i");
     t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut.exit.i.i");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() {
     t_err!("_ZN3fooE.llvm moocow");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn dont_panic() {
     rust_demangle_c_test_harness::demangle("_ZN2222222222222222222222EE").to_string();
@@ -199,22 +173,16 @@ fn dont_panic() {
     .to_string();
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn invalid_no_chop() {
     t_err!("_ZNfooE");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_assoc_types() {
     t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", "<alloc::boxed::Box<alloc::boxed::FnBox<A, Output=R> + 'a> as core::ops::function::FnOnce<A>>::call_once::h69e8f44b3723e1ca");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_bang() {
     t!(
@@ -238,14 +206,20 @@ fn limit_recursion() {
 }
 
 // FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
+#[ignore = "would slowly use up all RAM before being OOM-killed"]
 #[test]
-fn limit_output() {
+fn limit_output_oom_hazard() {
     assert_ends_with!(
         rust_demangle_c_test_harness::demangle("RYFG_FGyyEvRYFF_EvRYFFEvERLB_B_B_ERLRjB_B_B_")
             .to_string(),
         "{size limit reached}"
     );
+}
+
+// FIXME(eddyb) port the relevant functionality to C.
+#[should_panic]
+#[test]
+fn limit_output() {
     // NOTE(eddyb) somewhat reduced version of the above, effectively
     // `<for<...> fn()>` with a larger number of lifetimes in `...`.
     assert_ends_with!(

third-party/rust-demangle/test-harness/tests/v0.rs

@@ -92,8 +92,6 @@ fn demangle_const_generics_preview() {
     t_const_suffixed!("j7b_", "123", "usize");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_min_const_generics() {
     t_const!("p", "_");
@@ -109,8 +107,6 @@ fn demangle_min_const_generics() {
     t_const!("c2202_", "'∂'");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_str() {
     t_const!("e616263_", "{*\"abc\"}");
@@ -129,8 +125,6 @@ fn demangle_const_str() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 // NOTE(eddyb) this uses the same strings as `demangle_const_str` and should
 // be kept in sync with it - while a macro could be used to generate both
 // `str` and `&str` tests, from a single list of strings, this seems clearer.
@@ -152,8 +146,6 @@ fn demangle_const_ref_str() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_ref() {
     t_const!("Rp", "{&_}");
@@ -165,8 +157,6 @@ fn demangle_const_ref() {
     t_const!("QAE", "{&mut []}");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_array() {
     t_const!("AE", "{[]}");
@@ -176,8 +166,6 @@ fn demangle_const_array() {
     t_const!("AAh1_h2_EAh3_h4_EE", "{[[1, 2], [3, 4]]}");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_tuple() {
     t_const!("TE", "{()}");
@@ -189,8 +177,6 @@ fn demangle_const_tuple() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_adt() {
     t_const!(
@@ -223,8 +209,6 @@ fn demangle_exponential_explosion() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     t_nohash!("_RC3foo.llvm.9D1C9369", "foo");
@@ -232,8 +216,6 @@ fn demangle_thinlto() {
     t_nohash!("_RNvC9backtrace3foo.llvm.A5310EB9", "backtrace::foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_extra_suffix() {
     // From alexcrichton/rustc-demangle#27: