Update mold.1 (automated commit)

A weak TLS variable is usually guarded by a separate weak non-TLS
symbol indicating whether the TLS definition is available. A typical
use looks like this:

  extern char foo_used __attribute__((weak));
  extern __thread int foo __attribute__((weak));

  if (&foo_used)
    foo = 1;

If foo remains undefined, its access is unreachable and there is no
meaningful value to apply to its TLS relocations. Leave those
relocations unapplied.

This prevents AArch64 from rejecting an otherwise unreachable TLS
access when an R_AARCH64_TLSLE_ADD_TPREL_HI12 relocation is out of range.

https://github.com/rui314/mold/issues/1626

CMakeLists.txt

@@ -44,7 +44,7 @@
 # features and behave exactly the same.
 
 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.41.0)
+project(mold VERSION 2.42.0)
 
 include(CMakeDependentOption)
 include(CheckSymbolExists)
@@ -149,10 +149,9 @@ find_package(ZLIB QUIET)
 if(ZLIB_FOUND AND NOT MOLD_MOSTLY_STATIC)
   target_link_libraries(mold PRIVATE ZLIB::ZLIB)
 else()
-  set(ZLIB_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_SHARED OFF CACHE INTERNAL "")
+  set(ZLIB_BUILD_TESTING OFF CACHE INTERNAL "")
   add_subdirectory(third-party/zlib EXCLUDE_FROM_ALL)
-  target_include_directories(zlibstatic INTERFACE third-party/zlib
-    $<TARGET_PROPERTY:zlibstatic,BINARY_DIR>)
   target_link_libraries(mold PRIVATE zlibstatic)
 endif()
 
@@ -209,9 +208,13 @@ cmake_dependent_option(
 # By default, we build a bundled mimalloc and statically-link it to
 # mold. If you want to dynamically link to the system's
 # libmimalloc.so, pass -DMOLD_USE_SYSTEM_MIMALLOC=ON.
+#
+# We require mimalloc v3 or newer. mimalloc v2 doesn't use transparent
+# huge pages for the allocation path that serves mold's largest data
+# structures, which makes linking large programs significantly slower.
 if(MOLD_USE_MIMALLOC)
   if(MOLD_USE_SYSTEM_MIMALLOC)
-    find_package(mimalloc REQUIRED)
+    find_package(mimalloc 3 REQUIRED)
     target_link_libraries(mold PRIVATE mimalloc)
   else()
     function(mold_add_mimalloc)
@@ -306,6 +309,13 @@ endif()
 check_symbol_exists(madvise sys/mman.h HAVE_MADVISE)
 check_symbol_exists(uname sys/utsname.h HAVE_UNAME)
 
+# glibc declares fallocate only under _GNU_SOURCE. C++ compilers
+# predefine it on Linux, but this check runs the C compiler, which
+# does not.
+set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+check_symbol_exists(fallocate fcntl.h HAVE_FALLOCATE)
+unset(CMAKE_REQUIRED_DEFINITIONS)
+
 # Create a .cc file containing the current git hash for `mold --version`.
 add_custom_target(git_hash
   COMMAND ${CMAKE_COMMAND}
@@ -414,6 +424,7 @@ endforeach()
 # Add other non-template source files.
 target_sources(mold PRIVATE
   lib/aho-corasick.cc
+  lib/arena.cc
   lib/compress.cc
   lib/crc32.cc
   lib/demangle.cc

README.md

@@ -54,14 +54,14 @@ recent version of a C++ compiler and a C++ standard library. We recommend GCC
 
 To install build dependencies, run `./install-build-deps.sh` in this
 directory. It will detect your Linux distribution and attempt to install the
-necessary packages. You may need to run it as root.
+necessary packages.
 
 ### Compile mold
 
 ```shell
 git clone --branch stable https://github.com/rui314/mold.git
 cd mold
-./install-build-deps.sh
+sudo ./install-build-deps.sh
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ -B build
 cmake --build build -j$(nproc)
 sudo cmake --build build --target install

dist.sh

@@ -95,7 +95,7 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends wget file make gcc g++ zlib1g-dev libssl-dev ca-certificates && \
+  apt-get install -y --no-install-recommends wget file make gcc g++ git zlib1g-dev libssl-dev ca-certificates && \
   rm -rf /var/lib/apt/lists
 
 # Build CMake 3.27
@@ -110,11 +110,11 @@ RUN mkdir /build && \
 # Build GCC 14
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/gcc/gcc-14.2.0/gcc-14.2.0.tar.gz | tar xzf - --strip-components=1 && \
   mkdir gmp mpc mpfr && \
-  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
-  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
-  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz | tar xzf - --strip-components=1 -C gmp && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz | tar xzf - --strip-components=1 -C mpc && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/mpfr/mpfr-4.2.1.tar.gz | tar xzf - --strip-components=1 -C mpfr && \
   ./configure --prefix=/usr --enable-languages=c,c++ --disable-bootstrap --disable-multilib && \
   make -j\$(nproc) && \
   make install && \
@@ -124,7 +124,7 @@ RUN mkdir /build && \
 # Build GNU binutils 2.43
 RUN mkdir /build && \
   cd /build && \
-  wget -O- --progress=dot:mega https://ftpmirror.gnu.org/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
+  wget -O- --progress=dot:mega https://ftp.gnu.org/gnu/binutils/binutils-2.43.tar.gz | tar xzf - --strip-components=1 && \
   ./configure --prefix=/usr && \
   make -j\$(nproc) && \
   make install && \
@@ -163,7 +163,7 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^deb/d' -e 's/^# deb /deb /g' /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 clang-16 cmake git && \
   ln -sf /usr/bin/clang-16 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-16 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -176,7 +176,7 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
 RUN sed -i -e '/^URIs/d' -e 's/^# http/URIs: http/' /etc/apt/sources.list.d/debian.sources && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-18 cmake git && \
   ln -sf /usr/bin/clang-18 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-18 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists
@@ -186,10 +186,10 @@ loongarch64)
   cat <<EOF | $image_build
 FROM mirror.gcr.io/loongarch64/debian:sid@sha256:0356df4e494bbb86bb469377a00789a5b42bbf67d5ff649a3f9721b745cbef77
 ENV DEBIAN_FRONTEND=noninteractive TZ=UTC
-RUN sed -i -e 's!http[^ ]*!http://snapshot.debian.org/archive/debian-ports/20250620T014755Z!g' /etc/apt/sources.list && \
+RUN echo 'deb http://deb.debian.org/debian sid main' > /etc/apt/sources.list && \
   echo 'Acquire::Retries "10"; Acquire::http::timeout "10"; Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/80-retries && \
   apt-get update && \
-  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake && \
+  apt-get install -y --no-install-recommends build-essential gcc-14 g++-14 clang-19 cmake git && \
   ln -sf /usr/bin/clang-19 /usr/bin/clang && \
   ln -sf /usr/bin/clang++-19 /usr/bin/clang++ && \
   rm -rf /var/lib/apt/lists

docs/mold.1

@@ -1,6 +1,6 @@
 .\" generated with Ronn-NG/v0.9.1
 .\" http://github.com/apjanke/ronn-ng/tree/0.9.1
-.TH "MOLD" "1" "May 2026" ""
+.TH "MOLD" "1" "August 2026" ""
 .SH "NAME"
 \fBmold\fR \- a modern linker
 .SH "SYNOPSIS"
@@ -193,7 +193,10 @@ Omit \fB\.debug_*\fR sections from the output file\.
 Read linker script from \fIfile\fR\.
 .TP
 \fB\-X\fR, \fB\-\-discard\-locals\fR
-Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\.
+Discard temporary local symbols to reduce the sizes of the symbol table and the string table\. Temporary local symbols are local symbols starting with \fB\.L\fR\. Compilers usually generate such symbols for unnamed program elements such as string literals or floating\-point literals\. This is the default; use \fB\-\-discard\-none\fR to keep temporary local symbols\.
+.TP
+\fB\-\-discard\-none\fR
+Keep all local symbols in the symbol table, including temporary ones starting with \fB\.L\fR, except ones in mergeable sections\. This restores the default behavior of GNU ld\. \fBmold\fR and \fBld\.lld\fR discard temporary local symbols by default\.
 .TP
 \fB\-e\fR \fIsymbol\fR, \fB\-\-entry\fR=\fIsymbol\fR:
 
@@ -258,6 +261,9 @@ Alias for \fB\-\-section\-start=\.data=\fR\fIaddress\fR\.
 \fB\-\-Ttext\fR=\fIaddress\fR
 Alias for \fB\-\-section\-start=\.text=\fR\fIaddress\fR\.
 .TP
+\fB\-\-Ttext\-segment\fR=\fIaddress\fR
+Set the address of the first byte of the text segment\.
+.TP
 \fB\-\-allow\-multiple\-definition\fR
 Normally, the linker reports an error if there are more than one definition of a symbol\. This option changes the default behavior so that it doesn't report an error for duplicate definitions and instead use the first definition\.
 .TP

docs/mold.md

@@ -375,7 +375,14 @@ but as `-o magic`.
   Discard temporary local symbols to reduce the sizes of the symbol table and
   the string table. Temporary local symbols are local symbols starting with
   `.L`. Compilers usually generate such symbols for unnamed program elements
-  such as string literals or floating-point literals.
+  such as string literals or floating-point literals. This is the default;
+  use `--discard-none` to keep temporary local symbols.
+
+* `--discard-none`:
+  Keep all local symbols in the symbol table, including temporary ones
+  starting with `.L`, except ones in mergeable sections. This restores the
+  default behavior of GNU ld. `mold` and `ld.lld` discard temporary local
+  symbols by default.
 
 * `-e` _symbol_, `--entry`=_symbol_:
 
@@ -458,6 +465,9 @@ but as `-o magic`.
 * `--Ttext`=_address_:
   Alias for `--section-start=.text=`_address_.
 
+* `--Ttext-segment`=_address_:
+  Set the address of the first byte of the text segment.
+
 * `--allow-multiple-definition`:
   Normally, the linker reports an error if there are more than one definition
   of a symbol. This option changes the default behavior so that it doesn't

lib/aho-corasick.cc

@@ -26,20 +26,45 @@ bool AhoCorasick::can_handle(std::string_view pat) {
     pat.remove_prefix(1);
   if (pat.ends_with('*'))
     pat.remove_suffix(1);
-  return pat.find_first_of("*?[") == pat.npos;
+  return !pat.empty() && pat.find_first_of("*?[\\") == pat.npos;
+}
+
+i32 AhoCorasick::find_child(i32 node, u8 ch) const {
+  if (node == 0)
+    return root_children[ch];
+
+  for (i32 i = nodes[node].first_child; i != -1; i = nodes[i].next_sibling)
+    if (nodes[i].ch == ch)
+      return i;
+  return -1;
+}
+
+i32 AhoCorasick::add_child(i32 node, u8 ch) {
+  if (i32 child = find_child(node, ch); child != -1)
+    return child;
+
+  i32 child = nodes.size();
+  i32 sibling = nodes[node].first_child;
+  nodes.emplace_back();
+  nodes[child].next_sibling = sibling;
+  nodes[child].ch = ch;
+  nodes[node].first_child = child;
+
+  if (node == 0)
+    root_children[ch] = child;
+  return child;
 }
 
 i64 AhoCorasick::find(std::string_view str) {
   if (nodes.empty())
     return -1;
 
-  i64 idx = 0;
+  i32 idx = 0;
   i64 val = -1;
 
   auto walk = [&](u8 c) {
-    for (i64 j = idx; j != -1; j = nodes[j].suffix_link) {
-      i64 child = nodes[j].children[c];
-      if (child != -1) {
+    for (i32 j = idx; j != -1; j = nodes[j].suffix_link) {
+      if (i32 child = find_child(j, c); child != -1) {
         idx = child;
         val = std::max(val, nodes[child].value);
         return;
@@ -57,17 +82,13 @@ i64 AhoCorasick::find(std::string_view str) {
 
 bool AhoCorasick::add(std::string_view pat, i64 val) {
   assert(can_handle(pat));
-  if (nodes.empty())
-    nodes.resize(1);
-  i64 idx = 0;
+  if (nodes.empty()) {
+    root_children.fill(-1);
+    nodes.emplace_back();
+  }
+  i32 idx = 0;
 
-  auto walk = [&](u8 c) {
-    if (nodes[idx].children[c] == -1) {
-      nodes[idx].children[c] = nodes.size();
-      nodes.resize(nodes.size() + 1);
-    }
-    idx = nodes[idx].children[c];
-  };
+  auto walk = [&](u8 c) { idx = add_child(idx, c); };
 
   // We handle "foo" as if "\0foo\0", "*foo" as if "foo\0", "foo*" as
   // if "\0foo", and "*foo*" as if "foo". Aho-Corasick can do only
@@ -88,45 +109,34 @@ bool AhoCorasick::add(std::string_view pat, i64 val) {
 void AhoCorasick::compile() {
   if (nodes.empty())
     return;
-  fix_suffix_links(0);
-  fix_values();
-}
 
-void AhoCorasick::fix_suffix_links(i64 idx) {
-  for (i64 i = 0; i < 256; i++) {
-    i64 child = nodes[idx].children[i];
-    if (child == -1)
-      continue;
+  // A failure link may refer to any node at the previous depth, so failure
+  // links must be constructed breadth-first.
+  std::queue<i32> queue;
+  for (i32 child = nodes[0].first_child; child != -1;
+       child = nodes[child].next_sibling) {
+    nodes[child].suffix_link = 0;
+    queue.push(child);
+  }
 
-    i64 j = nodes[idx].suffix_link;
-    for (; j != -1; j = nodes[j].suffix_link) {
-      if (nodes[j].children[i] != -1) {
-        nodes[child].suffix_link = j;
-        break;
-      }
+  while (!queue.empty()) {
+    i32 idx = queue.front();
+    queue.pop();
+
+    for (i32 child = nodes[idx].first_child; child != -1;
+         child = nodes[child].next_sibling) {
+      i32 suffix = nodes[idx].suffix_link;
+      while (suffix != 0 && find_child(suffix, nodes[child].ch) == -1)
+        suffix = nodes[suffix].suffix_link;
+
+      if (i32 next = find_child(suffix, nodes[child].ch); next != -1)
+        suffix = next;
+
+      nodes[child].suffix_link = suffix;
+      nodes[child].value = std::max(nodes[child].value, nodes[suffix].value);
+      queue.push(child);
     }
-    if (j == -1)
-      nodes[child].suffix_link = 0;
-    fix_suffix_links(child);
   }
 }
 
-void AhoCorasick::fix_values() {
-  std::queue<i64> queue;
-  queue.push(0);
-
-  do {
-    i64 idx = queue.front();
-    queue.pop();
-
-    for (i64 child : nodes[idx].children) {
-      if (child != -1) {
-        i64 suffix = nodes[child].suffix_link;
-        nodes[child].value = std::max(nodes[child].value, nodes[suffix].value);
-        queue.push(child);
-      }
-    }
-  } while (!queue.empty());
-}
-
 } // namespace mold

lib/arena.cc

@@ -0,0 +1,77 @@
+#include "lib.h"
+
+#ifdef _WIN32
+# include <windows.h>
+#endif
+
+namespace mold {
+
+[[noreturn]]
+static void memory_error(std::string_view action, u64 size) {
+  std::cerr << "mold: cannot " << action << ' ' << size
+            << " bytes of virtual memory\n";
+  std::exit(1);
+}
+
+ArenaResource::ArenaResource() {
+#ifdef _WIN32
+  data = (u8 *)VirtualAlloc(nullptr, (SIZE_T)SIZE, MEM_RESERVE, PAGE_NOACCESS);
+  if (!data)
+    memory_error("reserve", SIZE);
+#else
+  int flags = MAP_ANONYMOUS | MAP_PRIVATE;
+# ifdef MAP_NORESERVE
+  // The arena is much larger than most links need. Do not reserve swap for
+  // pages that may never be touched.
+  flags |= MAP_NORESERVE;
+# endif
+  data = (u8 *)mmap(nullptr, SIZE, PROT_READ | PROT_WRITE, flags, -1, 0);
+  if (data == MAP_FAILED)
+    memory_error("reserve", SIZE);
+#endif
+
+#ifdef MADV_HUGEPAGE
+  // Large links fill the beginning of the arena densely. Transparent huge
+  // pages reduce address-translation overhead without populating unused pages.
+  madvise(data, SIZE, MADV_HUGEPAGE);
+#endif
+}
+
+ArenaResource::~ArenaResource() {
+#ifdef _WIN32
+  VirtualFree(data, 0, MEM_RELEASE);
+#else
+  munmap(data, SIZE);
+#endif
+}
+
+void *ArenaResource::allocate(u64 size, u64 alignment) {
+  assert(std::has_single_bit(alignment));
+
+  // ArenaPtr and base-relative indices both encode offsets in four-byte units.
+  alignment = std::max<u64>(alignment, 4);
+  u64 old = offset.load(std::memory_order_relaxed);
+
+  for (;;) {
+    u64 padding = -old & (alignment - 1);
+    if (old > SIZE || padding > SIZE - old || size > SIZE - old - padding) {
+      std::cerr << "mold: cannot allocate more than " << SIZE
+                << " bytes for linker data structures on this host\n";
+      std::exit(1);
+    }
+
+    u64 begin = old + padding;
+    u64 end = begin + size;
+    if (offset.compare_exchange_weak(old, end, std::memory_order_relaxed)) {
+#ifdef _WIN32
+      // VirtualAlloc reserves and commits address space separately.
+      if (size &&
+          !VirtualAlloc(data + begin, (SIZE_T)size, MEM_COMMIT, PAGE_READWRITE))
+        memory_error("commit", size);
+#endif
+      return data + begin;
+    }
+  }
+}
+
+} // namespace mold

lib/bitvector.h

@@ -1,78 +0,0 @@
-#pragma once
-
-#include "integers.h"
-
-#include <cassert>
-#include <vector>
-
-namespace mold {
-
-class BitvectorProxy {
-public:
-  BitvectorProxy(u64 &word, size_t pos)
-    : word(word), mask(1ULL << pos) {}
-
-  BitvectorProxy &operator=(bool val) {
-    if (val)
-      word |= mask;
-    else
-      word &= ~mask;
-    return *this;
-  }
-
-  BitvectorProxy &operator=(const BitvectorProxy &other) {
-    return *this = (bool)other;
-  }
-
-  operator bool() const {
-    return word & mask;
-  }
-
-private:
-  u64 &word;
-  u64 mask;
-};
-
-class Bitvector {
-public:
-  Bitvector() = default;
-  Bitvector(i64 n) : size(n), words((n + 63) / 64) {}
-
-  void resize(i64 n) {
-    words.clear();
-    words.resize((n + 63) / 64);
-    size = n;
-  }
-
-  Bitvector &operator|=(const Bitvector &x) {
-    assert(size == x.size);
-    for (i64 i = 0; i < words.size(); i++)
-      words[i] |= x.words[i];
-    return *this;
-  }
-
-  Bitvector &operator&=(const Bitvector &x) {
-    assert(size == x.size);
-    for (i64 i = 0; i < words.size(); i++)
-      words[i] &= x.words[i];
-    return *this;
-  }
-
-  Bitvector &operator<<=(size_t n) {
-    assert(n == 1);
-    for (i64 i = words.size() - 1; i > 0; i--)
-      words[i] = (words[i] << 1) | (words[i - 1] >> 63);
-    words[0] <<= 1;
-    return *this;
-  }
-
-  BitvectorProxy operator[](size_t pos) {
-    assert(pos < size);
-    return {words[pos / 64], pos % 64};
-  }
-
-  i64 size = 0;
-  std::vector<u64> words;
-};
-
-} // namespace mold

lib/config.h.in

@@ -2,6 +2,7 @@
 #define MOLD_LIBDIR "@CMAKE_INSTALL_FULL_LIBDIR@"
 #define MOLD_FIRST_TARGET @MOLD_FIRST_TARGET@
 
+#cmakedefine01 HAVE_FALLOCATE
 #cmakedefine01 HAVE_MADVISE
 #cmakedefine01 HAVE_UNAME
 #cmakedefine01 MOLD_USE_MIMALLOC

lib/gentoo-gpkg-workaround.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Work around a deadlock in portage's binary package extraction.
+#
+# tar_stream_reader.close() joins its stdin-feeder thread and then waits
+# for the external decompressor to exit without draining the
+# decompressor's remaining stdout. tarfile stops reading at the tar
+# end-of-archive marker, so trailing archive padding is left in the
+# pipe. If that residue exceeds the pipe buffer, the decompressor
+# blocks writing forever and close() never returns, freezing the whole
+# emerge. Small pipe buffers make this reachable in practice: the
+# kernel shrinks new pipes to one page once a user exceeds
+# fs.pipe-user-pages-soft, which our 80 parallel containers do.
+#
+# The fix is to drain the decompressor's stdout to EOF before joining
+# and waiting. This script patches the installed gpkg.py at container
+# image build time and fails loudly if the code no longer matches, so
+# a portage upgrade that changes (or fixes) this code is noticed.
+
+import sys
+import portage.gpkg
+
+path = portage.gpkg.__file__
+src = open(path).read()
+
+old = """        if self.proc is not None:
+            self.thread.join()
+"""
+
+new = """        if self.proc is not None:
+            # Drain remaining decompressor output before joining and
+            # waiting; otherwise the decompressor can block forever on
+            # a full stdout pipe. Injected by mold's lib/gentoo-test.sh.
+            try:
+                while self.read_io.read(65536):
+                    pass
+            except Exception:
+                pass
+            self.thread.join()
+"""
+
+if src.count(old) != 1:
+    sys.exit(f"gpkg workaround: expected pattern not found in {path}; "
+             "portage changed, update lib/gentoo-gpkg-workaround.py")
+
+with open(path, "w") as f:
+    f.write(src.replace(old, new))
+
+import py_compile
+py_compile.compile(path, doraise=True)
+print(f"gpkg deadlock workaround applied to {path}")

lib/gentoo-test.sh

@@ -2,10 +2,25 @@
 #
 # This test script takes a Gentoo package name and tries to build it
 # using mold in a Podman environment. We chose Gentoo Linux as a test
-# target, because its source-based package allows us to build programs
+# target, because its source-based packages allow us to build programs
 # locally and run their test suites without any hassle.
 #
-# You can get a complete list of Gentoo packages availalbe for testing
+# We first build a package with mold acting as /usr/bin/ld. Only if
+# that fails, we build the same package again with GNU ld to tell
+# whether the failure is mold's fault. The package's own test suite
+# runs in both cases; FEATURES=test is scoped to just the package
+# under test via package.env, because applying it to the entire
+# dependency graph makes a large fraction of the tree unresolvable.
+# Results are classified into three directories:
+#
+#   success: the package built fine with mold (or, for a control
+#            build, with GNU ld)
+#   failure: the package built with GNU ld but not with mold, which
+#            means it hit a mold bug
+#   broken:  the package didn't build even with GNU ld, so the
+#            failure is not mold's fault
+#
+# You can get a complete list of Gentoo packages available for testing
 # with the following command:
 #
 # podman run --rm mold-gentoo emerge --color n -s '' | \
@@ -20,50 +35,152 @@ fi
 
 set -x
 
-# Create a Podman image
-if ! podman image ls mold-gentoo | grep -q mold-gentoo; then
-  set -e
-  cat <<EOF | podman build -t mold-gentoo -
-FROM docker.io/gentoo/stage3
-RUN emerge-webrsync
-RUN echo 'USE="X ssl elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary -perl"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
-    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
-    echo 'FEATURES="\${FEATURE} noclean nostrip ccache -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
-    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
-    FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache xeyes dev-build/cmake dev-vcs/git && \
-    rm -rf /var/tmp/portage
-EOF
-  set +e
+# Test results are keyed by the commit hash embedded in the mold
+# binary, so refuse to run with a binary that doesn't have one.
+git_hash=$(./dist/mold --version | perl -ne '/\(([0-9a-f]{40})\b/ and print $1;')
+
+if [ "$git_hash" = "" ]; then
+  echo "Cannot get a git hash from ./dist/mold --version. Run this script at the mold source root, with dist/mold built from a git checkout."
+  exit 1
 fi
 
-git_hash=$(./dist/mold --version | perl -ne '/\((\w+)/; print $1;')
+cache_dir=/var/cache/mold-gentoo
+
+# Create a Podman image. The lock prevents parallel invocations of
+# this script from racing to build the same image.
+#
+# A few notes on the container configuration:
+#
+# - We accept ~amd64 (testing) packages to test the newest version of
+#   everything. The stage3 image is built for the stable profile, so
+#   we update the whole system to ~amd64 once here; without that,
+#   half-upgraded packages (e.g. perl and its virtuals) cause
+#   unsolvable dependency conflicts in every container.
+#
+# - EMERGE_DEFAULT_OPTS makes emerge automatically apply the USE flag
+#   adjustments it would otherwise just print and give up on. An
+#   ephemeral container has no /etc/portage/package.use accumulated by
+#   an administrator, so without this, thousands of packages are
+#   unbuildable out of the box.
+#
+# - freetype and harfbuzz mutually depend on each other, so we
+#   pre-install them with the documented two-step bootstrap. During
+#   that step, pillow is built without truetype support to break
+#   another dependency cycle (docutils -> pillow -> harfbuzz -> glib
+#   -> docutils). The override is removed afterwards; once the cycle
+#   participants are installed, they satisfy buildtime dependencies
+#   and the cycle can no longer form.
+#
+# - qtbase's test suite requires icu (REQUIRED_USE), which autounmask
+#   doesn't resolve on its own, hence the package.use entry.
+#
+# - gcc doesn't auto-select a newly installed major version, so we
+#   switch to the newest slot explicitly after the world update.
+#
+# - lib/gentoo-gpkg-workaround.py patches a deadlock in portage's
+#   binary package extraction. See that file for details.
+#
+# - static-libs is not in USE because it contradicts REQUIRED_USE
+#   constraints in some packages (e.g. lvm2 forbids static-libs with
+#   udev), and building static archives doesn't exercise the linker
+#   anyway. Where a dependency genuinely needs it, emerge re-enables
+#   it per package via autounmask.
+#
+# - /etc/portage/env/test.conf allows marking a single package for
+#   FEATURES=test through package.env, which is how we scope test
+#   suites to just the package under test.
+#
+# - The image build compiles a couple hundred packages from source,
+#   because the Gentoo binhost only carries stable versions built with
+#   default USE flags, which rarely match our ~amd64/USE combination.
+#   To pay that cost only once, the build publishes what it compiles
+#   to the GNU ld binary package pool and reuses it, so a rebuilt
+#   image only compiles packages whose version has changed since the
+#   last build. The toolchain step still uses the binhost, where our
+#   configuration does match (e.g. clang). Since it verifies binhost
+#   package signatures and our own pool is unsigned, it runs with a
+#   private PKGDIR so the two never mix.
+(
+  flock 9
+  if ! podman image exists mold-gentoo; then
+    cat <<EOF | podman build -v $cache_dir/distfiles:/var/cache/distfiles -v $cache_dir/ccache:/ccache -v $cache_dir/binpkgs-ld:/var/cache/binpkgs -v `pwd`/lib:/mold-lib:ro -t mold-gentoo - || exit 1
+FROM docker.io/gentoo/stage3
+RUN emerge-webrsync
+RUN echo 'USE="X ssl elogind -systemd truetype jpeg jpeg2k tiff zstd -perl udev lcms alsa"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_KEYWORDS="~amd64"' >> /etc/portage/make.conf && \
+    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
+    echo 'FEATURES="nostrip ccache binpkg-multi-instance -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
+    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf && \
+    echo 'MAKEOPTS="-j$(nproc) --load-average=100"' >> /etc/portage/make.conf && \
+    echo 'EMERGE_DEFAULT_OPTS="--autounmask-continue=y --autounmask-license=n --autounmask-backtrack=y --backtrack=100 --jobs=16 --load-average=100"' >> /etc/portage/make.conf && \
+    mkdir -p /etc/portage/env /etc/portage/package.use && \
+    echo 'FEATURES="test"' > /etc/portage/env/test.conf && \
+    printf 'media-libs/freetype harfbuzz\ndev-qt/qtbase icu\n' > /etc/portage/package.use/mold && \
+    echo 'dev-python/pillow -truetype' > /etc/portage/package.use/cycle-break && \
+    FEATURES=buildpkg emerge --usepkg --update --deep --newuse @world && \
+    gcc-config \$(gcc-config -l | tail -n 1 | awk '{print \$2}') && env-update && \
+    USE=-harfbuzz FEATURES=buildpkg emerge --oneshot media-libs/freetype && \
+    FEATURES=buildpkg emerge --usepkg --oneshot media-libs/freetype media-libs/harfbuzz && \
+    rm /etc/portage/package.use/cycle-break && \
+    PKGDIR=/var/tmp/binpkgs FEATURES='getbinpkg binpkg-request-signature' emerge gdb lld llvm-core/clang vim emacs strace ccache dev-build/cmake dev-vcs/git && \
+    rm -rf /var/tmp/portage /var/tmp/binpkgs
+RUN python3 /mold-lib/gentoo-gpkg-workaround.py
+EOF
+  fi
+) 9> /tmp/mold-gentoo-image.lock || exit 1
 
 if [ "$package" = dev-libs/concurrencykit ]; then
   echo "Skipping known broken package: $package"
   exit 0
 fi
 
-# Build a given package in Podman
+# Build a given package in Podman.
+#
+# Build artifacts are shared between containers through host
+# directories bind-mounted into every container:
+#
+#   /var/cache/mold-gentoo/ccache: compiler cache
+#   /var/cache/mold-gentoo/distfiles: source tarballs, so that each
+#     distfile is downloaded only once across all containers and runs
+#   /var/cache/mold-gentoo/binpkgs-{mold,ld}: binary packages, so that each
+#     dependency is built only once instead of once per container that
+#     needs it
+#
+# Every emerge runs with FEATURES=buildpkg to publish what it built to
+# the binary package pool, and dependencies are installed with
+# --usepkg from the pool when an exact version and USE match exists.
+# The package under test itself is always built from source. In the
+# mold run, dependencies are still linked with mold and their binaries
+# are still exercised by the test suites of everything built on top of
+# them; the pool just memoizes that work across containers. The mold
+# and GNU ld runs use separate pools so that a control build can never
+# install mold-linked binaries.
 cmd1='(cd /usr/bin; ln -sf /mold/dist/mold $(realpath ld))'
-cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
-cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
+cmd2="echo '$package test.conf' > /etc/portage/package.env"
+cmd3="FEATURES=buildpkg emerge --usepkg --onlydeps $package"
+cmd4="FEATURES=buildpkg emerge $package"
 filename=`echo "$package" | sed 's!/!_!g'`
-podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 3h"
+podman="podman run --rm --pids-limit=-1 --cap-add=SYS_PTRACE -v `pwd`:/mold:ro -v $cache_dir/ccache:/ccache -v $cache_dir/distfiles:/var/cache/distfiles"
+run="mold-gentoo timeout -v -k 15s 3h chrt --idle 0 nice -n 19 bash -c"
 dir=gentoo/$git_hash
 
-mkdir -p "$dir"/success "$dir"/failure
+mkdir -p "$dir"/success "$dir"/failure "$dir"/broken
 
-$podman chrt --idle 0 nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
+# Build the package with mold. If this succeeds, we don't need a
+# control build.
+$podman -v $cache_dir/binpkgs-mold:/var/cache/binpkgs $run "$cmd1 && $cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".mold
 if [ $? = 0 ]; then
   mv "$dir"/"$filename".mold "$dir"/success
-else
-  mv "$dir"/"$filename".mold "$dir"/failure
+  exit 0
 fi
 
-$podman chrt --idle 0 nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
+# Build the package again with GNU ld to tell whether the failure is
+# mold's fault.
+$podman -v $cache_dir/binpkgs-ld:/var/cache/binpkgs $run "$cmd2 && $cmd3 && $cmd4" >& "$dir"/"$filename".ld
 if [ $? = 0 ]; then
+  mv "$dir"/"$filename".mold "$dir"/failure
   mv "$dir"/"$filename".ld "$dir"/success
 else
-  mv "$dir"/"$filename".ld "$dir"/failure
+  mv "$dir"/"$filename".mold "$dir"/broken
+  mv "$dir"/"$filename".ld "$dir"/broken
 fi

lib/glob.cc

@@ -1,94 +1,23 @@
-// This file implements a glob matcher that can run multiple glob patterns
-// against an input string. mold uses the glob matcher for symbol name
-// patterns in a version script or a dynamic list file. Since we may need to
-// match hundreds of glob patterns against millions of symbol names, the
-// speed of the matcher is very important.
-//
-// The pattern match implemented in this file is NFA-based, although the
-// cost of the function is O(n*m), where n is the number of NFA states and m
-// is the length of the input string. We do not use recursion or
-// backtracking, unlike a generic NFA-based regular expression matcher. This
-// is doable because glob patterns are very limited subsets of regexes.
-//
-// Here is the explanation of the algorithm. Observe that the only "tricky"
-// meta-character in a glob pattern is "*", which matches zero or more
-// characters. Other characters and meta-characters always match a single
-// input character. So the key of the algorithm is to handle "*" efficiently.
-//
-// We can represent a glob pattern "a*b*" with three NFA states: q_start, q1
-// and q_accept, with the following transition functions:
-//
-//   δ(q_start, "a") = q1
-//   δ(q1, <any character>) = q1
-//   δ(q1, "b") = q_accept
-//   δ(q_accept, <any character>) = q_accept
-//
-// We can construct such an NFA in a straightforward manner. We maintain NFA
-// states as a list, with the initial contents being the start state. Each
-// character except for "*" creates a new NFA state, adds a transition from
-// the last state in the list to the new one, and appends the new state at
-// the end of the list. "*" sets the "is_star" flag on the last NFA state.
-// The flag indicates that the state machine can remain in the state for any
-// input character.
-//
-// An NFA constructed this way doesn't have any complicated loops,
-// ε-transitions, or anything like that. Each state has only one incoming
-// edge. The only loops in the state transition are the self-loops on states
-// followed by a "*". Aside from that, the state machine progresses linearly
-// from the start state to the accept state.
-//
-// Each state of an NFA can be represented by a single bit. If a bit is 1,
-// the non-deterministic state machine is in that state. Otherwise, it's
-// not. Observe that a state with the "is_star" flag will continued to be 1
-// once it becomes 1, since the state machine can loop over the state on any
-// input character.
-//
-// With that observation, we can represent an NFA with a bit vector of N
-// bits, where N is the number of NFA states. For each input character, bit
-// M becomes 1 if
-//
-//   - bit M-1 is 1 and there's a transition from state_{M-1} to state_M
-//     with the given character, or
-//   - bit M is 1 and state_M's "is_star" flag is 1.
-//
-// Initially, the 0th bit is 1 for the start state. At each step, the bits
-// propagate from least significant to most significant positions, at most
-// one bit at a time. If the most significant bit is 1 after the entire
-// input has been processed, the string matches.
-//
-// This propagation can be implemented with bitwise OR, bitwise AND, and a
-// one-bit bit shift on the bit vector. All these operations are very cheap.
-//
-// We can combine multiple glob matchers into a single matcher by simply
-// concatenating the bit vectors of their state machines.
+// This file implements the glob matcher used for symbol name patterns.
+// Exact, prefix and suffix patterns are matched directly, and simple
+// substring patterns are combined into an Aho-Corasick matcher. The
+// remaining patterns are matched with the non-recursive algorithm described
+// at https://research.swtch.com/glob. If there are many such patterns, a
+// bit-parallel NFA matches them together.
 
 #include "lib.h"
 
-#include <cstring>
-
 namespace mold {
 
-static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
-  std::vector<MultiGlob::State> vec(1);
+std::optional<Glob::Pattern> Glob::Pattern::compile(std::string_view pat,
+                                                    i64 value) {
+  std::vector<Token> tokens;
 
   while (!pat.empty()) {
     u8 c = pat[0];
-    pat = pat.substr(1);
-    std::bitset<256> chars;
+    pat.remove_prefix(1);
 
     switch (c) {
-    case '*':
-      vec.back().is_star = true;
-      continue;
-    case '?':
-      chars.set();
-      break;
-    case '\\':
-      if (pat.empty())
-        return {};
-      chars[pat[0]] = true;
-      pat = pat.substr(1);
-      break;
     case '[': {
       // Here are a few bracket pattern examples:
       //
@@ -100,23 +29,25 @@ static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
       // Both `!` and `^` are accepted as negation markers. `!` is the
       // POSIX/shell convention used by other linkers. `^` was mold's
       // original syntax and is kept for backward compatibility.
+      tokens.emplace_back(BRACKET);
+      std::bitset<256> &chars = tokens.back().chars;
       bool negate = false;
       bool closed = false;
 
       if (!pat.empty() && (pat[0] == '!' || pat[0] == '^')) {
         negate = true;
-        pat = pat.substr(1);
+        pat.remove_prefix(1);
       }
 
       while (!pat.empty()) {
         if (pat[0] == ']') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           closed = true;
           break;
         }
 
         if (pat[0] == '\\') {
-          pat = pat.substr(1);
+          pat.remove_prefix(1);
           if (pat.empty())
             return {};
         }
@@ -124,13 +55,13 @@ static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
         if (pat.size() >= 3 && pat[1] == '-') {
           u8 start = pat[0];
           u8 end = pat[2];
-          pat = pat.substr(3);
+          pat.remove_prefix(3);
 
           if (end == '\\') {
             if (pat.empty())
               return {};
             end = pat[0];
-            pat = pat.substr(1);
+            pat.remove_prefix(1);
           }
 
           if (end < start)
@@ -139,8 +70,8 @@ static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
           for (i64 i = start; i <= end; i++)
             chars[i] = true;
         } else {
-          chars[pat[0]] = true;
-          pat = pat.substr(1);
+          chars[(u8)pat[0]] = true;
+          pat.remove_prefix(1);
         }
       }
 
@@ -151,106 +82,276 @@ static std::vector<MultiGlob::State> parse_glob(std::string_view pat) {
         chars.flip();
       break;
     }
+    case '?':
+      tokens.emplace_back(QUESTION);
+      break;
+    case '*':
+      if (tokens.empty() || tokens.back().kind != STAR)
+        tokens.emplace_back(STAR);
+      break;
+    case '\\':
+      if (pat.empty())
+        return {};
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += pat[0];
+      pat.remove_prefix(1);
+      break;
     default:
-      chars[c] = true;
+      if (tokens.empty() || tokens.back().kind != STRING)
+        tokens.emplace_back(STRING);
+      tokens.back().str += c;
       break;
     }
-
-    vec.push_back({chars, false});
   }
-  return vec;
+  return Pattern(std::move(tokens), value);
 }
 
-// Instead of returning just a match/no match boolean value, our glob
-// matcher returns an integer value associated with each given pattern.
-// If multiple patterns match at the same time, the largest associated
-// value will be returned by find().
-bool MultiGlob::add(std::string_view pat, i64 val) {
-  std::vector<State> vec = parse_glob(pat);
-  if (vec.empty())
+bool Glob::Pattern::match(std::string_view str) const {
+  if (!tokens.empty() && tokens.back().kind == STRING &&
+      !str.ends_with(tokens.back().str))
     return false;
-  patterns.push_back({std::move(vec), val});
+
+  i64 x = 0;
+  i64 y = 0;
+  i64 next_x = -1;
+  i64 next_y = -1;
+
+  while (x < str.size() || y < tokens.size()) {
+    if (y < tokens.size()) {
+      const Token &tok = tokens[y];
+
+      switch (tok.kind) {
+      case STRING:
+        if (str.substr(x).starts_with(tok.str)) {
+          x += tok.str.size();
+          y++;
+          continue;
+        }
+        break;
+      case STAR:
+        next_y = y++;
+        next_x = x + 1;
+
+        if (y < tokens.size() && tokens[y].kind == STRING) {
+          const std::string &s = tokens[y].str;
+          i64 pos = str.find(s, x);
+          if (pos == str.npos)
+            return false;
+          next_x = pos + 1;
+          x = pos + s.size();
+          y++;
+        }
+        continue;
+      case QUESTION:
+        if (x < str.size()) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
+      case BRACKET:
+        if (x < str.size() && tok.chars[(u8)str[x]]) {
+          x++;
+          y++;
+          continue;
+        }
+        break;
+      }
+    }
+
+    // Retry the last star after assigning one more input byte to it.
+    if (next_x != -1 && next_x <= str.size()) {
+      x = next_x;
+      y = next_y;
+      continue;
+    }
+    return false;
+  }
   return true;
 }
 
-void MultiGlob::compile() {
-  if (patterns.empty())
-    return;
+void Glob::Nfa::compile(std::span<const Pattern> patterns) {
+  i64 num_states = 0;
 
-  ranges::stable_sort(patterns, ranges::greater(), &GlobPattern::value);
+  for (const Pattern &pattern : patterns) {
+    num_states++;
 
-  std::vector<State> states;
-  for (GlobPattern &p : patterns)
-    append(states, p.states);
-  i64 sz = states.size();
-
-  start_states.resize(sz);
-  for (i64 pos = 0; GlobPattern &p : patterns) {
-    start_states[pos] = true;
-    pos += p.states.size();
+    for (const Pattern::Token &tok : pattern.tokens) {
+      if (tok.kind == Pattern::STRING)
+        num_states += tok.str.size();
+      else if (tok.kind != Pattern::STAR)
+        num_states++;
+    }
   }
 
-  star_mask.resize(sz);
-  for (i64 i = 0; i < sz; i++)
-    if (states[i].is_star)
-      star_mask[i] = true;
+  i64 num_words = (num_states + 63) / 64;
+  initial_states.resize(num_words);
+  star_states.resize(num_words);
+  accept_states.resize(num_words);
+  char_masks.resize(256 * num_words);
+  values.resize(num_states, -1);
 
-  for (i64 i = 0; i < 256; i++) {
-    char_mask[i].resize(sz);
-    for (i64 j = 0; j < sz; j++)
-      if (states[j].incoming_edge[i])
-        char_mask[i][j] = true;
+  auto set_bit = [](std::vector<u64> &vec, i64 pos) {
+    vec[pos / 64] |= 1ULL << (pos % 64);
+  };
+
+  i64 state = 0;
+
+  for (const Pattern &pattern : patterns) {
+    set_bit(initial_states, state);
+
+    for (const Pattern::Token &tok : pattern.tokens) {
+      switch (tok.kind) {
+      case Pattern::STRING:
+        for (u8 c : tok.str) {
+          state++;
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        }
+        break;
+      case Pattern::STAR:
+        set_bit(star_states, state);
+        break;
+      case Pattern::QUESTION:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      case Pattern::BRACKET:
+        state++;
+        for (i64 c = 0; c < 256; c++)
+          if (tok.chars[c])
+            char_masks[c * num_words + state / 64] |= 1ULL << (state % 64);
+        break;
+      }
+    }
+
+    set_bit(accept_states, state);
+    values[state] = pattern.value;
+    state++;
   }
 }
 
-i64 MultiGlob::find(std::string_view str) {
-  if (patterns.empty())
-    return -1;
+i64 Glob::Nfa::match(std::string_view str) const {
+  static thread_local std::vector<u64> states;
+  states.assign(initial_states.begin(), initial_states.end());
 
-  Bitvector bits = start_states;
-  Bitvector tmp;
+  i64 num_words = states.size();
 
   for (u8 c : str) {
-    // This is equivalent to
-    //
-    //   bits = (bits & star_mask) | ((bits << 1) & char_mask[c])
-    //
-    // but we update the existing objects in place to avoid allocating
-    // temporary objects.
-    tmp = bits;
-    tmp &= star_mask;
-    bits <<= 1;
-    bits &= char_mask[c];
-    bits |= tmp;
+    const u64 *mask = char_masks.data() + c * num_words;
+    u64 carry = 0;
+
+    for (i64 i = 0; i < num_words; i++) {
+      u64 old = states[i];
+      u64 next = (old << 1) | carry;
+      states[i] = (old & star_states[i]) | (next & mask[i]);
+      carry = old >> 63;
+    }
   }
 
-  for (i64 pos = 0; GlobPattern &p : patterns) {
-    pos += p.states.size();
-    if (bits[pos - 1])
-      return p.value;
+  i64 value = -1;
+
+  for (i64 i = 0; i < num_words; i++) {
+    u64 word = states[i] & accept_states[i];
+
+    while (word) {
+      i64 bit = std::countr_zero(word);
+      value = std::max(value, values[i * 64 + bit]);
+      word &= word - 1;
+    }
   }
-  return -1;
+  return value;
+}
+
+static bool is_literal(std::string_view pat) {
+  return pat.find_first_of("*?[\\") == pat.npos;
 }
 
 bool Glob::add(std::string_view pat, i64 val) {
   assert(val >= 0);
   assert(!is_compiled);
+  is_empty = false;
+
+  // Match-all, exact, prefix and suffix patterns are handled with
+  // plain string comparisons instead of the matchers below, which
+  // have to scan the entire input string on every query.
+  if (pat == "*") {
+    match_all = std::max(match_all, val);
+    return true;
+  }
+
+  if (is_literal(pat)) {
+    exacts.push_back({std::string(pat), val});
+    return true;
+  }
+
+  if (pat.ends_with('*') && is_literal(pat.substr(0, pat.size() - 1))) {
+    prefixes.push_back({std::string(pat.substr(0, pat.size() - 1)), val});
+    return true;
+  }
+
+  if (pat.starts_with('*') && is_literal(pat.substr(1))) {
+    suffixes.push_back({std::string(pat.substr(1)), val});
+    return true;
+  }
 
   // If the pattern requires only a single substring search, the
   // Aho-Corasick algorithm is even faster than our glob matcher.
   if (aho_corasick.can_handle(pat))
     return aho_corasick.add(pat, val);
-  return multi_glob.add(pat, val);
+
+  if (std::optional<Pattern> pattern = Pattern::compile(pat, val)) {
+    patterns.push_back(std::move(*pattern));
+    return true;
+  }
+  return false;
 }
 
 i64 Glob::find(std::string_view str) {
   std::call_once(once, [&] {
-    multi_glob.compile();
+    // If the same name was added more than once, keep only the entry
+    // with the largest value, as find() returns the largest match.
+    // Sorting by (name, negated value) places that entry first in
+    // each run of duplicates, which is the one unique() keeps.
+    ranges::sort(exacts, {}, [](const LiteralPattern &p) {
+      return std::pair<std::string_view, i64>(p.pat, -p.value);
+    });
+
+    auto dup = ranges::unique(exacts, {}, &LiteralPattern::pat);
+    exacts.erase(dup.begin(), dup.end());
+
+    if (patterns.size() >= 64) {
+      nfa.compile(patterns);
+      patterns.clear();
+    }
+
     aho_corasick.compile();
     is_compiled = true;
   });
 
-  return std::max(multi_glob.find(str), aho_corasick.find(str));
+  i64 val = match_all;
+
+  auto it = ranges::lower_bound(exacts, str, {}, &LiteralPattern::pat);
+  if (it != exacts.end() && it->pat == str)
+    val = std::max(val, it->value);
+
+  for (const LiteralPattern &p : prefixes)
+    if (val < p.value && str.starts_with(p.pat))
+      val = p.value;
+
+  for (const LiteralPattern &p : suffixes)
+    if (val < p.value && str.ends_with(p.pat))
+      val = p.value;
+
+  if (!nfa.empty())
+    val = std::max(val, nfa.match(str));
+
+  for (const Pattern &p : patterns)
+    if (val < p.value && p.match(str))
+      val = p.value;
+
+  return std::max(val, aho_corasick.find(str));
 }
 
 } // namespace mold

lib/hyperloglog.cc

@@ -11,6 +11,11 @@
 namespace mold {
 
 i64 HyperLogLog::get_cardinality() const {
+  u8 buckets[NBUCKETS] = {};
+  for (const Sketch &sketch : sketches)
+    for (i64 i = 0; i < NBUCKETS; i++)
+      buckets[i] = std::max(buckets[i], sketch.buckets[i]);
+
   double z = 0;
   for (i64 val : buckets)
     z += std::ldexp(1.0, -val);

lib/integers.h

@@ -59,15 +59,15 @@ public:
   }
 
   operator T() const {
-    if (size == 3) {
-      if (is_le)
+    if constexpr (size == 3) {
+      if constexpr (is_le)
         return buf[2] << 16 | buf[1] << 8 | buf[0];
       return buf[0] << 16 | buf[1] << 8 | buf[2];
+    } else {
+      T v;
+      memcpy(&v, buf, size);
+      return is_native ? v : bswap(v);
     }
-
-    T v;
-    memcpy(&v, buf, size);
-    return is_native ? v : bswap(v);
   }
 
   Integer &operator=(T v)  { new (this) Integer(v); return *this; }

lib/lib.h

@@ -2,7 +2,6 @@
 
 #include "atomics.h"
 #include "integers.h"
-#include "bitvector.h"
 
 #include <array>
 #include <atomic>
@@ -15,6 +14,7 @@
 #include <fcntl.h>
 #include <filesystem>
 #include <iostream>
+#include <memory>
 #include <mutex>
 #include <optional>
 #include <span>
@@ -23,6 +23,7 @@
 #include <string_view>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <tbb/blocked_range.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
 #include <tbb/parallel_for.h>
@@ -69,6 +70,57 @@ inline u64 combine_hash(u64 a, u64 b) {
   return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
 }
 
+// TaggedPtr stores one of several pointer types and uses the low pointer bits
+// to record which type it contains.
+template <typename... Ts>
+class TaggedPtr {
+public:
+  static_assert(sizeof...(Ts) > 0);
+
+  TaggedPtr() = default;
+
+  template <typename T> requires (std::is_same_v<T, Ts> || ...)
+  TaggedPtr(T *ptr) {
+    *this = ptr;
+  }
+
+  template <typename T> requires (std::is_same_v<T, Ts> || ...)
+  TaggedPtr &operator=(T *ptr) {
+    static_assert(alignof(T) >= 1ULL << TAG_BITS);
+    value = (uintptr_t)ptr;
+    if (ptr)
+      value |= get_tag<T>();
+    return *this;
+  }
+
+  TaggedPtr &operator=(std::nullptr_t) {
+    value = 0;
+    return *this;
+  }
+
+  template <typename T> requires (std::is_same_v<T, Ts> || ...)
+  T *get() const {
+    if (!value || (value & TAG_MASK) != get_tag<T>())
+      return nullptr;
+    return (T *)(value & ~TAG_MASK);
+  }
+
+private:
+  template <typename T>
+  static consteval uintptr_t get_tag() {
+    bool matches[] = {std::is_same_v<T, Ts>...};
+    for (i64 i = 0; i < sizeof...(Ts); i++)
+      if (matches[i])
+        return i;
+    return 0;
+  }
+
+  static constexpr i64 TAG_BITS = std::bit_width((u64)sizeof...(Ts) - 1);
+  static constexpr uintptr_t TAG_MASK = (1ULL << TAG_BITS) - 1;
+
+  uintptr_t value = 0;
+};
+
 //
 // perf.cc
 //
@@ -164,6 +216,41 @@ inline u64 bit_ceil(u64 val) {
   return 1LL << (64 - std::countl_zero(val));
 }
 
+// ExactArray owns a fixed-size array without value-initializing trivial
+// elements that the caller is about to overwrite.
+template <typename T>
+class ExactArray {
+public:
+  ExactArray() = default;
+
+  explicit ExactArray(i64 sz)
+    : ptr(new T[sz]), size_(sz) {}
+
+  T &operator[](i64 i) {
+    return ptr[i];
+  }
+
+  const T &operator[](i64 i) const {
+    return ptr[i];
+  }
+
+  T *data() {
+    return ptr.get();
+  }
+
+  const T *data() const {
+    return ptr.get();
+  }
+
+  i64 size() const {
+    return size_;
+  }
+
+private:
+  std::unique_ptr<T[]> ptr;
+  i64 size_ = 0;
+};
+
 inline u64 align_to(u64 val, u64 align) {
   if (align == 0)
     return val;
@@ -246,6 +333,16 @@ inline void write_vector(void *buf, const std::vector<T> &vec) {
     memcpy(buf, vec.data(), vec.size() * sizeof(T));
 }
 
+inline void parallel_memcpy(void *dst, const void *src, i64 size) {
+  constexpr i64 BLOCK_SIZE = 2 * 1024 * 1024;
+
+  tbb::parallel_for(tbb::blocked_range<i64>(0, size, BLOCK_SIZE),
+                    [&](const tbb::blocked_range<i64> &range) {
+                      memcpy((u8 *)dst + range.begin(),
+                             (const u8 *)src + range.begin(), range.size());
+                    });
+}
+
 inline void encode_uleb(std::vector<u8> &vec, u64 val) {
   do {
     u8 byte = val & 0x7f;
@@ -395,68 +492,37 @@ public:
     this->nbuckets = std::max<i64>(MIN_NBUCKETS, bit_ceil(nbuckets));
     i64 bufsize = sizeof(Entry) * this->nbuckets;
 
-    // Allocate a zero-initialized buffer. mmap() is faster than
-    // malloc()+memset(), and MAP_POPULATE pre-faults pages here to
-    // avoid CoW faults and TLB shootdowns under concurrent inserts.
+    // Allocate a zero-initialized buffer. mmap is faster than
+    // malloc + memset.
 #ifdef _WIN32
     entries = (Entry *)_aligned_malloc(bufsize, alignof(Entry));
     memset((void *)entries, 0, bufsize);
-#elif MAP_POPULATE
-    entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
-                            MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
 #else
     entries = (Entry *)mmap(nullptr, bufsize, PROT_READ | PROT_WRITE,
                             MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 #endif
+
+#ifdef MADV_HUGEPAGE
+    madvise((void *)entries, bufsize, MADV_HUGEPAGE);
+#endif
   }
 
   std::pair<T *, bool> insert(std::string_view key, u64 hash, const T &val) {
-    assert(has_single_bit(nbuckets));
+    return insert_entry(key.data(), hash, val,
+                        [&](const char *ptr, u32 len) {
+                          return key == std::string_view(ptr, len);
+                        },
+                        [&](T &) { return (u32)key.size(); });
+  }
 
-    u64 begin = hash & (nbuckets - 1);
-    u64 mask = nbuckets / NUM_SHARDS - 1;
-
-    for (i64 i = 0; i < MAX_RETRY; i++) {
-      u64 idx = (begin & ~mask) | ((begin + i) & mask);
-      Entry &ent = entries[idx];
-
-      // It seems avoiding compare-and-swap is faster overall at least
-      // on my Zen4 machine, so do it.
-      if (const char *ptr = ent.key.load(std::memory_order_acquire);
-          ptr != nullptr && ptr != (char *)-1) {
-        if (key == std::string_view(ptr, ent.keylen))
-          return {&ent.value, false};
-        continue;
-      }
-
-      // Otherwise, use CAS to atomically claim the ownership of the slot.
-      const char *ptr = nullptr;
-      bool claimed = ent.key.compare_exchange_strong(ptr, (char *)-1,
-                                                     std::memory_order_acquire);
-
-      // If we successfully claimed the ownership of the slot,
-      // copy values to it.
-      if (claimed) {
-        new (&ent.value) T(val);
-        ent.keylen = key.size();
-        ent.key.store(key.data(), std::memory_order_release);
-        return {&ent.value, true};
-      }
-
-      // If someone is copying values to the slot, do busy wait.
-      while (ptr == (char *)-1) {
-        pause();
-        ptr = ent.key.load(std::memory_order_acquire);
-      }
-
-      // If the same key is already present, this is the slot we are
-      // looking for.
-      if (key == std::string_view(ptr, ent.keylen))
-        return {&ent.value, false};
-    }
-
-    std::cerr << "ConcurrentMap is full\n";
-    abort();
+  std::pair<T *, bool> insert_cstr(const char *key, u64 hash, const T &val,
+                                   auto initialize) {
+    // This variant avoids storing a length alongside every caller-side key.
+    // Only a newly inserted key needs its length computed by `initialize`.
+    return insert_entry(
+      key, hash, val,
+      [&](const char *ptr, u32) { return !strcmp(key, ptr); },
+      [&](T &value) { return initialize(value, key); });
   }
 
   i64 get_idx(T *value) const {
@@ -464,6 +530,16 @@ public:
     return (Entry *)addr - entries;
   }
 
+  // Prefetch the bucket where a key with the given hash would be
+  // probed first. Useful when a caller knows the hashes of upcoming
+  // insertions, as probes into a large table miss the cache almost
+  // every time.
+  void prefetch(u64 hash) const {
+#ifdef __GNUC__
+    __builtin_prefetch(entries + (hash & (nbuckets - 1)));
+#endif
+  }
+
   // Return a list of map entries sorted in a deterministic order.
   std::vector<Entry *> get_sorted_entries(i64 shard_idx) {
     if (nbuckets == 0)
@@ -514,12 +590,382 @@ public:
     return flatten(vec);
   }
 
-  static constexpr i64 MIN_NBUCKETS = 4096;
-  static constexpr i64 NUM_SHARDS = 16;
+  // MIN_NBUCKETS is chosen so that even the smallest map has
+  // MAX_RETRY buckets per shard; probing is confined to a shard, and
+  // a probe that visits MAX_RETRY distinct occupied slots aborts.
+  static constexpr i64 MIN_NBUCKETS = 16384;
+  static constexpr i64 NUM_SHARDS = 64;
   static constexpr i64 MAX_RETRY = 256;
 
   Entry *entries = nullptr;
   u64 nbuckets = 0;
+
+private:
+  std::pair<T *, bool> insert_entry(const char *key, u64 hash, const T &val,
+                                    auto equals, auto initialize) {
+    assert(has_single_bit(nbuckets));
+
+    u64 begin = hash & (nbuckets - 1);
+    u64 mask = nbuckets / NUM_SHARDS - 1;
+
+    for (i64 i = 0; i < MAX_RETRY; i++) {
+      u64 idx = (begin & ~mask) | ((begin + i) & mask);
+      Entry &ent = entries[idx];
+
+      // Avoid an atomic update when the slot is already occupied.
+      if (const char *ptr = ent.key.load(std::memory_order_acquire);
+          ptr != nullptr && ptr != (char *)-1) {
+        if (equals(ptr, ent.keylen))
+          return {&ent.value, false};
+        continue;
+      }
+
+      const char *ptr = nullptr;
+      bool claimed = ent.key.compare_exchange_strong(ptr, (char *)-1,
+                                                     std::memory_order_acquire);
+
+      // -1 marks the slot as claimed until its value has been initialized.
+      if (claimed) {
+        new (&ent.value) T(val);
+        ent.keylen = initialize(ent.value);
+        ent.key.store(key, std::memory_order_release);
+        return {&ent.value, true};
+      }
+
+      // Wait for the thread that claimed the slot to publish its key.
+      while (ptr == (char *)-1) {
+        pause();
+        ptr = ent.key.load(std::memory_order_acquire);
+      }
+
+      if (equals(ptr, ent.keylen))
+        return {&ent.value, false};
+    }
+
+    std::cerr << "ConcurrentMap is full\n";
+    abort();
+  }
+};
+
+static_assert(sizeof(void *) == 4 || sizeof(void *) == 8);
+
+// ArenaResource owns a sparsely-backed address range for symbols, input files,
+// and related linker data structures. Allocation is thread-safe and monotonic;
+// individual allocations are not freed. Keeping related objects in this range
+// lets ArenaPtr represent references between them as 32-bit self-relative
+// offsets.
+class ArenaResource {
+public:
+  // An 8 GiB arena ensures that the distance between any two allocations fits
+  // in ArenaPtr's signed 32-bit offset. A smaller reservation is used on
+  // 32-bit hosts, where address space is more limited.
+  static constexpr u64 SIZE = sizeof(void *) == 8 ? 1ULL << 33 : 1ULL << 28;
+
+  ArenaResource();
+  ~ArenaResource();
+  ArenaResource(const ArenaResource &) = delete;
+  ArenaResource &operator=(const ArenaResource &) = delete;
+
+  void *allocate(u64 size, u64 alignment);
+
+  template <typename T>
+  T *allocate(i64 count) {
+    if (count < 0 || (u64)count > SIZE / sizeof(T)) {
+      std::cerr << "mold: cannot allocate more than " << SIZE
+                << " bytes for linker data structures on this host\n";
+      std::exit(1);
+    }
+    return (T *)allocate(sizeof(T) * (u64)count, alignof(T));
+  }
+
+  template <typename T, typename... Args>
+  T *make(Args &&...args) {
+    return std::construct_at(allocate<T>(1), std::forward<Args>(args)...);
+  }
+
+  // ArenaPtr works only if the pointer field itself is within 8 GiB of its
+  // target. Records stored outside the arena instead use an index from the
+  // beginning of the arena. Indices count four-byte slots, and zero represents
+  // a null pointer.
+  u32 get_index(const void *ptr) const {
+    uintptr_t addr = (uintptr_t)ptr;
+    uintptr_t begin = (uintptr_t)data;
+    assert(begin < addr && addr < begin + SIZE);
+    assert((addr - begin) % 4 == 0);
+    return (addr - begin) / 4;
+  }
+
+  template <typename T>
+  T *get_pointer(u32 idx) const {
+    assert(0 < idx && (u64)idx * 4 < SIZE);
+    return (T *)(data + (u64)idx * 4);
+  }
+
+private:
+  u8 *data;
+
+  // Leave the first slots unused so that a base-relative index is never zero.
+  std::atomic<u64> offset = 8;
+};
+
+// ArenaAllocator adapts ArenaResource to the standard allocator interface so
+// containers can place their backing storage in the resource's address range.
+// It does not own the resource, and deallocation is deferred until the
+// resource itself is destroyed.
+template <typename T>
+class ArenaAllocator {
+public:
+  using value_type = T;
+
+  ArenaAllocator(ArenaResource &arena) : arena(arena) {}
+
+  template <typename U>
+  ArenaAllocator(const ArenaAllocator<U> &other) : arena(other.arena) {}
+
+  T *allocate(i64 count) {
+    return arena.template allocate<T>(count);
+  }
+
+  void deallocate(T *, i64) {}
+
+  template <typename U>
+  bool operator==(const ArenaAllocator<U> &other) const {
+    return &arena == &other.arena;
+  }
+
+private:
+  template <typename U> friend class ArenaAllocator;
+
+  ArenaResource &arena;
+};
+
+// ArenaPtr stores a pointer as a signed 32-bit offset from itself, in units of
+// four bytes. It is used for references between objects in an ArenaResource;
+// the ArenaPtr and its target must be four-byte aligned and less than 8 GiB
+// apart. An offset of zero represents a null pointer.
+template <typename T>
+class alignas(4) ArenaPtr {
+public:
+  ArenaPtr() = default;
+  ArenaPtr(std::nullptr_t) {}
+  ArenaPtr(T *ptr) { *this = ptr; }
+
+  // The offset is relative to this ArenaPtr, so copying it verbatim would
+  // make it point somewhere else.
+  ArenaPtr(const ArenaPtr &other) { *this = (T *)other; }
+  ArenaPtr(ArenaPtr &&other) { *this = (T *)other; }
+
+  ArenaPtr &operator=(T *ptr) {
+    if (!ptr) {
+      offset = 0;
+      return *this;
+    }
+
+    i64 byte_offset = (intptr_t)ptr - (intptr_t)this;
+    assert(byte_offset % 4 == 0);
+    assert(is_int(byte_offset / 4, 32));
+    offset = byte_offset / 4;
+    assert(offset != 0);
+    return *this;
+  }
+
+  ArenaPtr &operator=(const ArenaPtr &other) {
+    return *this = (T *)other;
+  }
+
+  ArenaPtr &operator=(ArenaPtr &&other) {
+    return *this = (T *)other;
+  }
+
+  operator T *() const {
+    return offset ? (T *)((intptr_t)this + (i64)offset * 4) : nullptr;
+  }
+
+  T *operator->() const {
+    return (T *)*this;
+  }
+
+  T &operator*() const {
+    return *(T *)*this;
+  }
+
+private:
+  i32 offset = 0;
+};
+
+static_assert(sizeof(ArenaPtr<u64>) == 4);
+
+// ArenaObjectDeleter runs an arena object's destructor without freeing its
+// storage. ArenaObjectPtr uses it to retain normal unique_ptr ownership
+// semantics for objects whose storage belongs to ArenaResource.
+template <typename T>
+struct ArenaObjectDeleter {
+  void operator()(T *ptr) const {
+    std::destroy_at(ptr);
+  }
+};
+
+template <typename T>
+using ArenaObjectPtr = std::unique_ptr<T, ArenaObjectDeleter<T>>;
+
+template <typename T>
+struct ShardedMapEntry : T {
+  ShardedMapEntry(std::string_view key) : key(key) {}
+
+  // Keeping the key outside T means that values not stored in a map do not
+  // pay for it. A mapped T is the base subobject of this entry.
+  std::string_view key;
+};
+
+template <typename T>
+std::string_view get_sharded_map_key(const T &object) {
+  return static_cast<const ShardedMapEntry<T> &>(object).key;
+}
+
+// ShardedMap is a map from strings to values of type T, built in two phases.
+// In the first phase, which may run in parallel, add() records a key and an
+// ArenaPtr<T> slot that needs the key's value. gather() then deduplicates the
+// keys, finds or creates one value for each key, and writes its address to
+// every recorded slot. The slots must remain at stable addresses until
+// gather().
+//
+// Keys whose hashes fall into different shards never interact, and
+// each shard is processed by exactly one thread during gather(), so
+// unlike with a concurrent hash table, no synchronization is needed,
+// and each key is hashed only once, in add().
+//
+// If a value needs to be initialized from its key, pass an
+// `on_create(key, value)` callback to gather() or insert(). It is called
+// exactly once when a value is created.
+//
+// insert() handles keys that arrive outside the two-phase pattern under a
+// shard mutex. All values live in stable arena blocks.
+template <typename T>
+class ShardedMap {
+public:
+  explicit ShardedMap(ArenaResource &arena) {
+    for (Shard &shard : shards)
+      shard.arena = &arena;
+  }
+
+  void add(std::string_view key, ArenaPtr<T> &slot) {
+    u64 hash = hash_string(key);
+    bins.local()[hash % NUM_SHARDS].push_back({key, hash, &slot});
+  }
+
+  T *insert(std::string_view key, auto on_create) {
+    u64 hash = hash_string(key);
+    Shard &shard = shards[hash % NUM_SHARDS];
+    std::scoped_lock lock(shard.mu);
+    return shard.insert(key, hash, on_create);
+  }
+
+  void gather(auto on_create) {
+    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
+      Shard &shard = shards[i];
+
+      i64 count = 0;
+      for (Bin &bin : bins)
+        count += bin[i].size();
+      if (count == 0)
+        return;
+
+      shard.reserve(count);
+
+      for (Bin &bin : bins)
+        for (Pending &p : bin[i])
+          *p.slot = shard.insert(p.key, p.hash, on_create);
+    });
+
+    bins.clear();
+  }
+
+  void parallel_for_each(auto fn) {
+    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
+      Shard &shard = shards[i];
+      for (Block &block : shard.blocks)
+        for (i64 j = 0; j < block.size; j++)
+          fn(block.data[j]);
+    });
+  }
+
+private:
+  static constexpr i64 NUM_SHARDS = 64;
+  using Entry = ShardedMapEntry<T>;
+  static_assert(std::is_trivially_destructible_v<Entry>);
+
+  struct Block {
+    Entry *data;
+    i64 size;
+    i64 capacity;
+  };
+
+  struct Pending {
+    std::string_view key;
+    u64 hash;
+    ArenaPtr<T> *slot;
+  };
+
+  // add() already computed the string hash. Store it in the map key so that
+  // unordered_map does not scan the string again.
+  struct PassThroughHash {
+    size_t operator()(const std::pair<u64, std::string_view> &key) const {
+      return key.first;
+    }
+  };
+
+  using Map = std::unordered_map<std::pair<u64, std::string_view>, Entry *,
+                                 PassThroughHash>;
+
+  struct Shard {
+    Shard() = default;
+    Shard(const Shard &) = delete;
+    Shard &operator=(const Shard &) = delete;
+
+    T *insert(std::string_view key, u64 hash, auto on_create) {
+      auto [it, inserted] = map.try_emplace({hash, key}, nullptr);
+      if (!inserted)
+        return it->second;
+
+      if (blocks.empty() || blocks.back().size == blocks.back().capacity)
+        add_block(INSERT_BLOCK_SIZE);
+
+      Entry *entry = emplace(blocks.back(), key);
+      it->second = entry;
+      on_create(key, *entry);
+      return entry;
+    }
+
+    void reserve(i64 count) {
+      map.reserve(map.size() + count);
+      if (blocks.empty() ||
+          blocks.back().capacity - blocks.back().size < count)
+        add_block(std::max<i64>(INSERT_BLOCK_SIZE, count));
+    }
+
+    static constexpr i64 INSERT_BLOCK_SIZE = 256;
+
+    void add_block(i64 capacity) {
+      Entry *data = arena->allocate<Entry>(capacity);
+      blocks.push_back({data, 0, capacity});
+    }
+
+    Entry *emplace(Block &block, std::string_view key) {
+      assert(block.size < block.capacity);
+      Entry *entry = block.data + block.size++;
+      return std::construct_at(entry, key);
+    }
+
+    std::mutex mu;
+    ArenaResource *arena = nullptr;
+    std::vector<Block> blocks;
+    Map map;
+  };
+
+  using Bin = std::array<std::vector<Pending>, NUM_SHARDS>;
+
+  Shard shards[NUM_SHARDS];
+  tbb::enumerable_thread_specific<Bin> bins;
 };
 
 //
@@ -533,23 +979,31 @@ void get_random_bytes(u8 *buf, i64 size);
 //
 
 class HyperLogLog {
-public:
-  void insert(u64 hash) {
-    update_maximum(buckets[hash & (NBUCKETS - 1)], std::countl_zero(hash) + 1);
-  }
-
-  i64 get_cardinality() const;
-
-  void merge(const HyperLogLog &other) {
-    for (i64 i = 0; i < NBUCKETS; i++)
-      update_maximum(buckets[i], other.buckets[i]);
-  }
-
 private:
   static constexpr i64 NBUCKETS = 2048;
   static constexpr double ALPHA = 0.79402;
 
-  Atomic<u8> buckets[NBUCKETS];
+public:
+  class Sketch {
+  public:
+    void insert(u64 hash) {
+      u8 &val = buckets[hash & (NBUCKETS - 1)];
+      val = std::max<u8>(val, std::countl_zero(hash) + 1);
+    }
+
+  private:
+    friend class HyperLogLog;
+    u8 buckets[NBUCKETS] = {};
+  };
+
+  Sketch &local() {
+    return sketches.local();
+  }
+
+  i64 get_cardinality() const;
+
+private:
+  tbb::enumerable_thread_specific<Sketch> sketches;
 };
 
 //
@@ -567,57 +1021,86 @@ public:
 
 private:
   struct TrieNode {
-    TrieNode() { children.fill(-1); }
     i64 value = -1;
     i32 suffix_link = -1;
-    std::array<i32, 256> children;
+    i32 first_child = -1;
+    i32 next_sibling = -1;
+    u8 ch = 0;
   };
 
-  void fix_suffix_links(i64 idx);
-  void fix_values();
+  i32 find_child(i32 node, u8 ch) const;
+  i32 add_child(i32 node, u8 ch);
 
+  // Most trie nodes have only one child. The root uses a dense table because
+  // it is visited for almost every input byte; other edges are stored sparsely.
+  std::array<i32, 256> root_children;
   std::vector<TrieNode> nodes;
 };
 
-//
-// glob.cc
-//
-
-class MultiGlob {
-public:
-  bool add(std::string_view pat, i64 val);
-  bool empty() const { return patterns.empty(); }
-  void compile();
-  i64 find(std::string_view str);
-
-  struct State {
-    std::bitset<256> incoming_edge;
-    bool is_star = false;
-  };
-
-  struct GlobPattern {
-    std::vector<State> states;
-    i64 value = -1;
-  };
-
-private:
-  std::vector<GlobPattern> patterns;
-  Bitvector start_states;
-  Bitvector star_mask;
-  Bitvector char_mask[256];
-};
-
 class Glob {
 public:
   bool add(std::string_view pat, i64 val);
-  bool empty() const { return multi_glob.empty() && aho_corasick.empty(); }
+  bool empty() const { return is_empty; }
   i64 find(std::string_view str);
 
 private:
   std::once_flag once;
+  bool is_empty = true;
   bool is_compiled = false;
 
-  MultiGlob multi_glob;
+  // Patterns that need only a literal string comparison are kept out
+  // of the automaton-based matchers below, which scan the entire input
+  // string per query. Real version scripts consist almost entirely of
+  // such patterns (e.g. `local: *;` or `v8dbg_*;`), and we match them
+  // against every defined symbol name.
+  struct LiteralPattern {
+    std::string pat;
+    i64 value;
+  };
+
+  struct Pattern {
+    enum Kind { STRING, STAR, QUESTION, BRACKET };
+
+    struct Token {
+      Token(Kind kind) : kind(kind) {}
+
+      Kind kind;
+      std::string str;
+      std::bitset<256> chars;
+    };
+
+    Pattern(std::vector<Token> &&tokens, i64 value)
+      : tokens(std::move(tokens)), value(value) {}
+
+    static std::optional<Pattern> compile(std::string_view pat, i64 value);
+    bool match(std::string_view str) const;
+
+    std::vector<Token> tokens;
+    i64 value;
+  };
+
+  // Nfa matches many glob patterns in parallel by representing each state
+  // with one bit. It is used only for large pattern sets; matching individual
+  // patterns is faster when there are only a few of them.
+  struct Nfa {
+    void compile(std::span<const Pattern> patterns);
+    i64 match(std::string_view str) const;
+    bool empty() const { return initial_states.empty(); }
+
+    std::vector<u64> initial_states;
+    std::vector<u64> star_states;
+    std::vector<u64> accept_states;
+    std::vector<u64> char_masks;
+    std::vector<i64> values;
+  };
+
+  i64 match_all = -1;                   // "*"
+  std::vector<LiteralPattern> exacts;   // "foo"
+  std::vector<LiteralPattern> prefixes; // "foo*"
+  std::vector<LiteralPattern> suffixes; // "*foo"
+  std::vector<Pattern> patterns;        // "foo*bar"
+
+  Nfa nfa;
   AhoCorasick aho_corasick;
 };
 

lib/update-git-hash.cmake

@@ -1,18 +1,17 @@
-# Get a git hash value. We do not want to use git command here
-# because we don't want to make git a build-time dependency.
-if(EXISTS "${SOURCE_DIR}/.git/HEAD")
-  file(READ "${SOURCE_DIR}/.git/HEAD" HASH)
-  string(STRIP "${HASH}" HASH)
+# Embed the current git commit hash into the mold executable. We ask git
+# instead of parsing files under .git because the repository format varies
+# (e.g. packed refs, reftable). When building from a source tarball, there's
+# no .git, and the hash is simply omitted.
+if(EXISTS "${SOURCE_DIR}/.git")
+  execute_process(
+    COMMAND git -C "${SOURCE_DIR}" rev-parse HEAD
+    OUTPUT_VARIABLE HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+    RESULT_VARIABLE EXIT_CODE)
 
-  if(HASH MATCHES "^ref: (.*)")
-    set(HEAD "${CMAKE_MATCH_1}")
-    if(EXISTS "${SOURCE_DIR}/.git/${HEAD}")
-      file(READ "${SOURCE_DIR}/.git/${HEAD}" HASH)
-      string(STRIP "${HASH}" HASH)
-    else()
-      file(READ "${SOURCE_DIR}/.git/packed-refs" PACKED_REFS)
-      string(REGEX REPLACE ".*\n([0-9a-f]+) ${HEAD}\n.*" "\\1" HASH "\n${PACKED_REFS}")
-    endif()
+  if(NOT EXIT_CODE EQUAL 0)
+    set(HASH "")
   endif()
 endif()
 

src/arch-arm32.cc

@@ -290,6 +290,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -552,12 +555,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -587,7 +587,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>

src/arch-arm64.cc

@@ -145,14 +145,17 @@ static bool is_add(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -165,6 +168,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check_range(ctx, i, val, lo, hi);
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_AARCH64_ABS64:
       break;
@@ -246,6 +254,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         u32 reg = bits(*(ul32 *)loc, 4, 0);
         *(ul32 *)(loc + 4) = 0x9100'0000 | (reg << 5) | reg; // ADD
         *(ul32 *)(loc + 4) |= bits(S + A, 11, 0) << 10;
+
+        rewrite(i, R_AARCH64_ADR_PREL_PG_HI21);
+        rewrite(i + 1, R_AARCH64_ADD_ABS_LO12_NC);
         i++;
       }
       break;
@@ -272,6 +283,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             *(ul32 *)loc = 0xd503'201f;              // nop
             *(ul32 *)(loc + 4) = 0x1000'0000 | reg1; // adr
             write_adr(loc + 4, val);
+            rewrite(i, R_NONE);
+            rewrite(i + 1, R_AARCH64_ADR_PREL_LO21);
             i++;
             break;
           }
@@ -425,13 +438,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         write_adrp(loc, val);
       } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
       }
       break;
     case R_AARCH64_TLSDESC_LD64_LO12:
-      if (sym.has_tlsdesc(ctx))
+      if (sym.has_tlsdesc(ctx)) {
         *(ul32 *)loc |= bits(sym.get_tlsdesc_addr(ctx) + A, 11, 3) << 10;
-      else
+      } else {
         *(ul32 *)loc = 0xd503'201f; // nop
+        rewrite(i, R_NONE);
+      }
       break;
     case R_AARCH64_TLSDESC_ADD_LO12:
       if (sym.has_tlsdesc(ctx)) {
@@ -439,9 +455,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x9000'0000; // adrp x0, 0
         write_adrp(loc, page(sym.get_gottp_addr(ctx) + A) - page(P));
+        rewrite(i, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21);
       } else {
         *(ul32 *)loc = 0xd2a0'0000; // movz x0, 0, lsl #16
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 32, 16) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G1);
       }
       break;
     case R_AARCH64_TLSDESC_CALL:
@@ -450,9 +468,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0xf940'0000; // ldr x0, [x0, 0]
         *(ul32 *)loc |= bits(sym.get_gottp_addr(ctx) + A, 11, 3) << 10;
+        rewrite(i, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC);
       } else {
         *(ul32 *)loc = 0xf280'0000; // movk x0, 0
         *(ul32 *)loc |= bits(S + A - ctx.tp_addr, 15, 0) << 5;
+        rewrite(i, R_AARCH64_TLSLE_MOVW_TPREL_G0_NC);
       }
       break;
     default:
@@ -463,12 +483,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -495,12 +512,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       check(S + A, 0, 1LL << 32);
       *(U32<E> *)loc = S + A;
       break;
+    case R_AARCH64_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -515,7 +538,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;

src/arch-i386.cc

@@ -280,14 +280,17 @@ static u32 relax_tlsdesc_to_le(u8 *loc) {
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -335,6 +338,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         loc[-2] = insn >> 8;
         loc[-1] = insn;
         *(ul32 *)loc = S + A - GOT;
+        if (ctx.arg.emit_relocs)
+          rel.r_type = R_386_GOTOFF;
       }
       break;
     case R_386_GOTOFF:
@@ -436,12 +441,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -502,7 +504,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       unreachable();
     }
-  }
+  });
 }
 
 template <>
@@ -517,7 +519,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;

src/arch-loongarch.cc

@@ -133,7 +133,7 @@ static void set_rj(u8 *loc, u32 rj) {
 static bool is_relaxable_got_load(Context<E> &ctx, InputSection<E> &isec, i64 i) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   Symbol<E> &sym = *isec.file.symbols[rels[i].r_sym];
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   if (ctx.arg.relax &&
       sym.is_pcrel_linktime_const(ctx) &&
@@ -269,13 +269,13 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
 
     if (rel.r_type == R_NONE || rel.r_type == R_LARCH_RELAX ||
         rel.r_type == R_LARCH_MARK_LA || rel.r_type == R_LARCH_MARK_PCREL ||
@@ -295,6 +295,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -332,6 +335,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
                    << " for relocation " << rel;
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_LARCH_32:
       assert(E::is_64);
@@ -375,10 +383,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       if (removed_bytes == 0) {
         write_j20(loc, hi20(S + A, P));
       } else {
-        // Rewrite pcalau12i + addi.d with pcaddi
+        // Rewrite pcalau12i + addi.d with pcaddi. The high part vanishes and
+        // the low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -417,10 +428,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         }
         write_j20(loc, hi20(GOT + G + A, P));
       } else {
-        // Rewrite pcalau12i + ld.d with pcaddi
+        // Rewrite pcalau12i + ld.d with pcaddi. The high part vanishes and the
+        // low part becomes the pcaddi's PC-relative relocation.
         assert(removed_bytes == 4);
         *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
         write_j20(loc, (S + A - P) >> 2);
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_LARCH_PCREL20_S2);
         i += 3;
       }
       break;
@@ -538,6 +552,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x5400'0000; // BL
         write_d10k16(loc, (S + A - P) >> 2);
+        rewrite(i, R_LARCH_B26);
       }
       break;
     case R_LARCH_ADD_ULEB128:
@@ -591,30 +606,49 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       //   jirl      $ra, $ra, 0
       if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
         write_j20(loc, hi20(sym.get_tlsdesc_addr(ctx) + A, P));
+
+      // pcalau12i + addi.d => pcaddi when TLSDESC is kept; both deleted when it
+      // is relaxed to IE/LE. Either way the high part loses its relocation.
+      // The folded pcaddi's relocation is emitted from the LO12 slot below.
+      if (!sym.has_tlsdesc(ctx) || removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_DESC_PC_LO12:
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0) {
-        i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
-        if (is_int(dist, 22)) {
-          *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
-          write_j20(loc, dist >> 2);
-        } else {
-          write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0) {
+          i64 dist = sym.get_tlsdesc_addr(ctx) + A - P;
+          if (is_int(dist, 22)) {
+            *(ul32 *)loc = 0x1800'0000 | get_rd(*(ul32 *)loc); // pcaddi
+            write_j20(loc, dist >> 2);
+            rewrite(i, R_LARCH_TLS_DESC_PCREL20_S2);
+          } else {
+            write_k12(loc, sym.get_tlsdesc_addr(ctx) + A);
+          }
         }
+      } else {
+        rewrite(i, R_NONE);
       }
       break;
     case R_LARCH_TLS_DESC_LD:
-      if (sym.has_tlsdesc(ctx) || removed_bytes == 4) {
-        // Do nothing
+      // TLSDESC is relaxed to IE or LE. The ld.d slot holds the first
+      // instruction of the resulting sequence.
+      if (sym.has_tlsdesc(ctx)) {
+        // Do nothing (TLSDESC kept)
+      } else if (removed_bytes == 4) {
+        // Small TP offset: the instruction was deleted.
+        rewrite(i, R_NONE);
       } else if (sym.has_gottp(ctx)) {
         *(ul32 *)loc = 0x1a00'0004; // pcalau12i $a0, 0
         write_j20(loc, hi20(sym.get_gottp_addr(ctx) + A, P));
+        rewrite(i, R_LARCH_TLS_IE_PC_HI20);
       } else {
         *(ul32 *)loc = 0x1400'0004; // lu12i.w   $a0, 0
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+        rewrite(i, R_LARCH_TLS_LE_HI20);
       }
       break;
     case R_LARCH_TLS_DESC_CALL:
+      // The jirl slot holds the second instruction of the IE or LE sequence.
       if (sym.has_tlsdesc(ctx)) {
         // Do nothing
       } else if (sym.has_gottp(ctx)) {
@@ -623,6 +657,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x2880'0084; // ld.w $a0, $a0, 0
         write_k12(loc, sym.get_gottp_addr(ctx) + A);
+        rewrite(i, R_LARCH_TLS_IE_PC_LO12);
       } else {
         i64 val = S + A - ctx.tp_addr;
         if (0 <= val && val < 0x1000)
@@ -630,11 +665,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         else
           *(ul32 *)loc = 0x0280'0084; // addi.w $a0, $a0, 0
         write_k12(loc, val);
+        rewrite(i, R_LARCH_TLS_LE_LO12);
       }
       break;
     case R_LARCH_TLS_LE_HI20_R:
+      // lu12i.w + add.d + addi.d => addi.d when the variable is within 2 KiB
+      // of TP, in which case the lu12i.w loses its relocation.
       if (removed_bytes == 0)
         write_j20(loc, (S + A + 0x800 - ctx.tp_addr) >> 12);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_LARCH_TLS_LE_LO12_R: {
       i64 val = S + A - ctx.tp_addr;
@@ -646,8 +686,13 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         set_rj(loc, 2);
       break;
     }
-    case R_LARCH_64:
     case R_LARCH_TLS_LE_ADD_R:
+      // add.d that materializes TP + offset; removed together with the
+      // lu12i.w when the variable is within 2 KiB of TP.
+      if (removed_bytes)
+        rewrite(i, R_NONE);
+      break;
+    case R_LARCH_64:
       break;
     default:
       unreachable();
@@ -657,19 +702,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE)
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
 
-    if (!sym.file) {
+    if (!sym.file && &sym != discarded_comdat_sym<E>) {
       record_undef_error(ctx, rel);
-      continue;
+      return;
     }
 
     SectionFragment<E> *frag;
@@ -742,7 +784,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -857,7 +899,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &r = rels[i];
@@ -1010,87 +1052,6 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   isec.sh_size -= r_delta;
 }
 
-// With --emit-relocs, rewrite the emitted relocations to match the
-// instructions that relaxation produced. The logic parallels apply_reloc_alloc.
-template <>
-void rewrite_reloc_types(Context<E> &ctx, InputSection<E> &isec, ElfRel<E> *buf) {
-  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  std::span<RelocDelta> deltas = isec.extra.r_deltas;
-  i64 k = 0;
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
-    Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-
-    i64 removed_bytes = 0;
-    if (!deltas.empty()) {
-      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
-        k++;
-      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
-        removed_bytes = get_removed_bytes(deltas, k);
-    }
-
-    switch (rel.r_type) {
-    case R_LARCH_TLS_LE_HI20_R:
-    case R_LARCH_TLS_LE_ADD_R:
-      // lu12i.w + add.d + addi.d => addi.d
-      if (removed_bytes)
-        buf[i].r_type = R_LARCH_NONE;
-      break;
-    case R_LARCH_PCALA_HI20:
-    case R_LARCH_GOT_PC_HI20:
-      // pcalau12i + addi.d/ld.d => pcaddi. The high part vanishes and the low
-      // part becomes the pcaddi's PC-relative relocation.
-      if (removed_bytes) {
-        buf[i].r_type = R_LARCH_NONE;
-        buf[i + 2].r_type = R_LARCH_PCREL20_S2;
-        i += 3;
-      }
-      break;
-    case R_LARCH_CALL36:
-      // pcaddu18i + jirl => b or bl
-      if (removed_bytes)
-        buf[i].r_type = R_LARCH_B26;
-      break;
-    case R_LARCH_TLS_DESC_PC_HI20:
-      // pcalau12i + addi.d => pcaddi when TLSDESC is kept; both deleted when it
-      // is relaxed to IE/LE.
-      if (!sym.has_tlsdesc(ctx)) {
-        buf[i].r_type = R_LARCH_NONE;
-      } else if (removed_bytes) {
-        buf[i].r_type = R_LARCH_NONE;
-        buf[i + 2].r_type = R_LARCH_TLS_DESC_PCREL20_S2;
-      }
-      break;
-    case R_LARCH_TLS_DESC_PC_LO12:
-      if (!sym.has_tlsdesc(ctx))
-        buf[i].r_type = R_LARCH_NONE;
-      break;
-    case R_LARCH_TLS_DESC_LD:
-      // TLSDESC is relaxed to IE or LE. The ld.d slot holds the first
-      // instruction of the resulting sequence.
-      if (sym.has_tlsdesc(ctx))
-        break;
-      if (sym.has_gottp(ctx))
-        buf[i].r_type = R_LARCH_TLS_IE_PC_HI20;   // pcalau12i
-      else if (removed_bytes)
-        buf[i].r_type = R_LARCH_NONE;             // small TP offset (deleted)
-      else
-        buf[i].r_type = R_LARCH_TLS_LE_HI20;      // large TP offset (lu12i.w)
-      break;
-    case R_LARCH_TLS_DESC_CALL:
-      // The jirl slot holds the second instruction of the IE or LE sequence.
-      if (sym.has_tlsdesc(ctx))
-        break;
-      if (sym.has_gottp(ctx))
-        buf[i].r_type = R_LARCH_TLS_IE_PC_LO12;   // ld.d
-      else
-        buf[i].r_type = R_LARCH_TLS_LE_LO12;      // ori or addi.w
-      break;
-    }
-  }
-}
-
 } // namespace mold
 
 #endif

src/arch-m68k.cc

@@ -86,6 +86,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -210,12 +213,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -234,11 +234,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(ub32 *)loc = S + A;
       break;
+    case R_68K_TLS_LDO32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(ub32 *)loc = *val;
+      else
+        *(ub32 *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-ppc32.cc

@@ -158,6 +158,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -284,12 +287,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -318,7 +318,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-ppc64v1.cc

@@ -162,6 +162,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -285,12 +288,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -324,7 +324,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>
@@ -485,9 +485,9 @@ void Thunk<E>::copy_buf(Context<E> &ctx) {
 }
 
 static InputSection<E> *get_opd_section(ObjectFile<E> &file) {
-  for (std::unique_ptr<InputSection<E>> &isec : file.sections)
-    if (isec && isec->name == ".opd")
-      return isec.get();
+  for (InputSection<E> *isec : file.sections)
+    if (isec && isec->name() == ".opd")
+      return isec;
   return nullptr;
 }
 
@@ -594,8 +594,8 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
     ranges::stable_sort(opd_syms, {}, &OpdSymbol::r_offset);
 
     // Rewrite relocations so that they directly refer to .opd.
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (!isec || !isec->is_alive || isec.get() == opd)
+    for (InputSection<E> *isec : file->sections) {
+      if (!isec || !isec->is_alive || isec == opd)
         continue;
 
       for (ElfRel<E> &r : isec->get_rels(ctx)) {
@@ -619,11 +619,10 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
 // refers to the function's .opd entry. This function marks such symbols
 // with NEEDS_PPC_OPD.
 void ppc64v1_scan_symbols(Context<E> &ctx) {
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (Symbol<E> *sym : file->symbols)
-      if (sym->file == file && sym->is_exported)
-        if (u32 ty = sym->get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
-          sym->flags |= NEEDS_PPC_OPD;
+  ctx.symbol_map.parallel_for_each([](Symbol<E> &sym) {
+    if (sym.file && !sym.file->is_dso && sym.is_exported)
+      if (u32 ty = sym.get_type(); ty == STT_FUNC || ty == STT_GNU_IFUNC)
+        sym.flags |= NEEDS_PPC_OPD;
   });
 
   // Functions referenced by the ELF header also have to have .opd entries.
@@ -638,23 +637,42 @@ void ppc64v1_scan_symbols(Context<E> &ctx) {
 }
 
 void PPC64OpdSection::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_opd_idx(ctx, symbols.size());
+  sym->aux->opd_idx = symbols.size();
   symbols.push_back(sym);
   this->shdr.sh_size += ENTRY_SIZE;
 }
 
-std::vector<ElfRel<E>> PPC64OpdSection::collect_dynrels(Context<E> &ctx) const {
+i64 PPC64OpdSection::get_num_dynrels(Context<E> &ctx) const {
+  return ctx.arg.pic ? symbols.size() * 2 : 0;
+}
+
+std::vector<u64> PPC64OpdSection::get_relr_offsets(Context<E> &ctx) {
   if (!ctx.arg.pic)
     return {};
 
-  std::vector<ElfRel<E>> rels;
+  std::vector<u64> offsets;
+  offsets.reserve(symbols.size() * 2);
+  for (Symbol<E> *sym : symbols) {
+    u64 loc = sym->get_opd_addr(ctx) - this->shdr.sh_addr;
+    offsets.push_back(loc);
+    offsets.push_back(loc + 8);
+  }
+  return offsets;
+}
+
+void PPC64OpdSection::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  if (!ctx.arg.pic)
+    return;
 
   for (Symbol<E> *sym : symbols) {
     u64 loc = sym->get_opd_addr(ctx);
-    rels.emplace_back(loc, E::R_RELATIVE, 0, sym->get_addr(ctx, NO_PLT | NO_OPD));
-    rels.emplace_back(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        loc % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc, E::R_RELATIVE, 0, sym->get_addr(ctx, NO_PLT | NO_OPD));
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs ||
+        (loc + 8) % sizeof(Word<E>) != 0)
+      *buf++ = ElfRel<E>(loc + 8, E::R_RELATIVE, 0, ctx.extra.TOC->value);
   }
-  return rels;
 }
 
 void PPC64OpdSection::copy_buf(Context<E> &ctx) {

src/arch-ppc64v2.cc

@@ -194,6 +194,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -361,12 +364,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -400,7 +400,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-riscv.cc

@@ -252,7 +252,7 @@ find_paired_reloc(Context<E> &ctx, InputSection<E> &isec,
 //   ld    t0, 0(t0)  # R_RISCV_PCREL_LO12_I(.L0), R_RISCV_RELAX
 static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
                              std::span<const ElfRel<E>> rels, i64 i) {
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
   return i + 3 < rels.size() &&
          rels[i].r_type == R_RISCV_GOT_HI20 &&
          rels[i + 1].r_type == R_RISCV_RELAX &&
@@ -265,13 +265,13 @@ static bool is_got_load_pair(Context<E> &ctx, InputSection<E> &isec,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
   std::span<RelocDelta> deltas = extra.r_deltas;
   i64 k = 0;
-  u8 *buf = (u8 *)contents.data();
+  u8 *buf = contents;
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE || rel.r_type == R_RISCV_RELAX)
       continue;
 
@@ -288,6 +288,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     }
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     i64 r_offset = rel.r_offset - r_delta;
     u8 *loc = base + r_offset;
 
@@ -306,6 +309,11 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_utype(loc, val);
     };
 
+    auto rewrite = [&](i64 idx, u32 ty) {
+      if (ctx.arg.emit_relocs)
+        rels[idx].r_type = ty;
+    };
+
     switch (rel.r_type) {
     case R_RISCV_32:
       if (E::is_64)
@@ -330,15 +338,18 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         // auipc + jalr -> jal
         *(ul32 *)loc = (rd << 7) | 0b1101111;
         write_jtype(loc, val);
+        rewrite(i, R_RISCV_JAL);
       } else if (removed_bytes == 6 && rd == 0) {
         // auipc + jalr -> c.j
         *(ul16 *)loc = 0b101'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else if (removed_bytes == 6 && rd == 1) {
         // auipc + jalr -> c.jal
         assert(!E::is_64);
         *(ul16 *)loc = 0b001'00000000000'01;
         write_cjtype(loc, val);
+        rewrite(i, R_RISCV_RVC_JUMP);
       } else {
         assert(removed_bytes == 0);
         utype(val);
@@ -357,11 +368,17 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
         // c.li <rd>, val
         *(ul16 *)loc = 0b010'0'00000'00000'01 | (rd << 7);
         write_citype(loc, sym.get_addr(ctx));
+        // The value is materialized directly, so neither this nor the paired
+        // PCREL_LO12 (the load) needs a relocation anymore.
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else if (removed_bytes == 4) {
         // addi <rd>, zero, val
         *(ul32 *)loc = 0b0010011 | (rd << 7);
         write_itype(loc, sym.get_addr(ctx));
+        rewrite(i, R_NONE);
+        rewrite(i + 2, R_NONE);
         i += 3;
       } else {
         assert(removed_bytes == 0);
@@ -421,6 +438,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     }
     case R_RISCV_HI20:
+      // lui (+ addi) => an instruction holding a link-time constant. The lui
+      // may be compressed to c.lui (removed 2 bytes) or deleted outright
+      // (removed 4 bytes); either way it no longer needs a relocation.
       if (removed_bytes == 2) {
         // Rewrite LUI with C.LUI
         i64 rd = get_rd(buf + rel.r_offset);
@@ -429,6 +449,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       } else if (removed_bytes == 0) {
         utype(S + A);
       }
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_LO12_I:
     case R_RISCV_LO12_S:
@@ -445,14 +467,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_RISCV_TPREL_HI20:
       assert(removed_bytes == 0 || removed_bytes == 4);
+      // lui + add => deleted; the variable is accessed relative to tp directly.
       if (removed_bytes == 0)
         utype(S + A - ctx.tp_addr);
+      else
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_ADD:
       // This relocation just annotates an ADD instruction that can be
       // removed when a TPREL is relaxed. No value is needed to be
       // written.
       assert(removed_bytes == 0 || removed_bytes == 4);
+      if (removed_bytes)
+        rewrite(i, R_NONE);
       break;
     case R_RISCV_TPREL_LO12_I:
     case R_RISCV_TPREL_LO12_S: {
@@ -507,18 +534,25 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       // If the code-shrinking relaxation is disabled, we may leave
       // original useless instructions instead of deleting them, but we
       // accept that because relaxations are enabled by default.
-      if (sym.has_tlsdesc(ctx) && removed_bytes == 0)
-        utype(sym.get_tlsdesc_addr(ctx) + A - P);
+      if (sym.has_tlsdesc(ctx)) {
+        if (removed_bytes == 0)
+          utype(sym.get_tlsdesc_addr(ctx) + A - P);
+      } else {
+        rewrite(i, R_NONE);
+      }
       break;
     case R_RISCV_TLSDESC_LOAD_LO12:
     case R_RISCV_TLSDESC_ADD_LO12:
     case R_RISCV_TLSDESC_CALL: {
-      if (removed_bytes == 4)
-        break;
-
       const ElfRel<E> &rel2 = find_paired_reloc(ctx, *this, rels, sym, i);
       Symbol<E> &sym2 = *file.symbols[rel2.r_sym];
 
+      if (!sym2.has_tlsdesc(ctx))
+        rewrite(i, R_NONE);
+
+      if (removed_bytes == 4)
+        break;
+
       u64 S = sym2.get_addr(ctx);
       u64 A = rel2.r_addend;
       u64 P = get_addr() + rel2.r_offset - get_r_delta(*this, rel2.r_offset);
@@ -646,12 +680,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -673,6 +704,18 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U64<E> *)loc = S + A;
       break;
+    case R_RISCV_TLS_DTPREL32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
+    case R_RISCV_TLS_DTPREL64:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U64<E> *)loc = *val;
+      else
+        *(U64<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     case R_RISCV_ADD8:
       *loc += S + A;
       break;
@@ -723,7 +766,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 template <>
@@ -846,7 +889,7 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
   std::vector<RelocDelta> &deltas = isec.extra.r_deltas;
   i64 r_delta = 0;
-  u8 *buf = (u8 *)isec.contents.data();
+  u8 *buf = isec.contents;
 
   // True if we can use 2-byte instructions. This is usually true on
   // Unix because RV64GC is generally considered the baseline hardware.
@@ -1000,72 +1043,6 @@ void shrink_section(Context<E> &ctx, InputSection<E> &isec) {
   isec.sh_size -= r_delta;
 }
 
-// With --emit-relocs, rewrite the emitted relocations to match the
-// instructions that relaxation produced. The logic parallels apply_reloc_alloc.
-template <>
-void rewrite_reloc_types(Context<E> &ctx, InputSection<E> &isec, ElfRel<E> *buf) {
-  std::span<const ElfRel<E>> rels = isec.get_rels(ctx);
-  std::span<RelocDelta> deltas = isec.extra.r_deltas;
-  i64 k = 0;
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
-    Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-
-    i64 removed_bytes = 0;
-    if (!deltas.empty()) {
-      while (k < deltas.size() && deltas[k].offset < rel.r_offset)
-        k++;
-      if (k < deltas.size() && deltas[k].offset == rel.r_offset)
-        removed_bytes = get_removed_bytes(deltas, k);
-    }
-
-    switch (rel.r_type) {
-    case R_RISCV_CALL:
-    case R_RISCV_CALL_PLT:
-      // auipc + jalr => jal, or c.j/c.jal
-      if (removed_bytes == 4)
-        buf[i].r_type = R_RISCV_JAL;
-      else if (removed_bytes == 6)
-        buf[i].r_type = R_RISCV_RVC_JUMP;
-      break;
-    case R_RISCV_GOT_HI20:
-      // auipc + l[dw] GOT load => the value is materialized directly, so both
-      // this and the paired PCREL_LO12 (the load) no longer need a relocation.
-      if (removed_bytes) {
-        buf[i].r_type = R_RISCV_NONE;
-        buf[i + 2].r_type = R_RISCV_NONE;
-        i += 3;
-      }
-      break;
-    case R_RISCV_HI20:
-      // lui (+ addi) => an instruction holding a link-time constant.
-      if (removed_bytes)
-        buf[i].r_type = R_RISCV_NONE;
-      break;
-    case R_RISCV_TPREL_HI20:
-    case R_RISCV_TPREL_ADD:
-      // lui + add => deleted; the variable is accessed relative to tp directly.
-      if (removed_bytes)
-        buf[i].r_type = R_RISCV_NONE;
-      break;
-    case R_RISCV_TLSDESC_HI20:
-      if (!sym.has_tlsdesc(ctx))
-        buf[i].r_type = R_RISCV_NONE;
-      break;
-    case R_RISCV_TLSDESC_LOAD_LO12:
-    case R_RISCV_TLSDESC_ADD_LO12:
-    case R_RISCV_TLSDESC_CALL: {
-      Symbol<E> &sym2 =
-        *isec.file.symbols[find_paired_reloc(ctx, isec, rels, sym, i).r_sym];
-      if (!sym2.has_tlsdesc(ctx))
-        buf[i].r_type = R_RISCV_NONE;
-      break;
-    }
-    }
-  }
-}
-
 // ISA name handlers
 //
 // An example of ISA name is "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0".

src/arch-s390x.cc

@@ -122,14 +122,17 @@ void EhFrameSection<E>::apply_eh_reloc(Context<E> &ctx, const ElfRel<E> &rel,
 
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -262,6 +265,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
             is_int(val, 33)) {
           *(ub16 *)(loc - 2) = 0xc000 | (op & 0x00f0);
           *(ub32 *)loc = val >> 1;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_390_PC32DBL;
           break;
         }
       }
@@ -342,12 +347,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -383,7 +385,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-sh4.cc

@@ -47,9 +47,10 @@
 //    with a piece of code to store the address of .got to %r12.
 //    We can use the register in our PLT for position-independent output.
 //
-//  - Even though it uses the RELA-type relocations, relocation addends
-//    are stored not to the r_addend field but to the relocated section
-//    contents for some reason. Therefore, it's effectively REL.
+//  - Even though it uses RELA-type relocations, object files store
+//    addends not in the r_addend field but in the relocated section
+//    contents. Dynamic relocations, however, follow the usual RELA
+//    convention.
 //
 //  - It looks like the ecosystem has bit-rotted. Some tests, especially
 //    one using C++ exceptions, don't pass even with GNU ld.
@@ -66,8 +67,8 @@ namespace mold {
 
 using E = MOLD_TARGET;
 
-// Even though SH-4 uses RELA-type relocations, addends are stored to
-// relocated places for some reason.
+// Even though SH-4 uses RELA-type relocations, addends are stored in
+// the relocated places for some reason.
 template <>
 i64 get_addend(u8 *loc, const ElfRel<E> &rel) {
   switch (rel.r_type) {
@@ -151,32 +152,36 @@ template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   if (ctx.arg.pic) {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x00ce, //    mov.l   @(r0, r12), r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   } else {
     constexpr U16<E> insn[] = {
-      0xd001, //    mov.l   1f, r0
+      0xd002, //    mov.l   1f, r0
       0x6002, //    mov.l   @r0, r0
+      0xd102, //    mov.l   2f, r1
       0x402b, //    jmp     @r0
-      0xd101, //    mov.l   2f, r1
+      0x0009, //    nop
+      0x0009, //    nop
       0, 0,   // 1: .long GOTPLT_ENTRY
       0, 0,   // 2: .long INDEX_IN_RELPLT
     };
 
     static_assert(sizeof(insn) == E::plt_size);
     memcpy(buf, insn, sizeof(insn));
-    *(U32<E> *)(buf + 8) = sym.get_gotplt_addr(ctx);
-    *(U32<E> *)(buf + 12) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
+    *(U32<E> *)(buf + 12) = sym.get_gotplt_addr(ctx);
+    *(U32<E> *)(buf + 16) = sym.get_plt_idx(ctx) * sizeof(ElfRel<E>);
   }
 }
 
@@ -238,6 +243,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -285,12 +293,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -309,11 +314,17 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
       else
         *(U32<E> *)loc = S + A;
       break;
+    case R_SH_TLS_LDO_32:
+      if (std::optional<u64> val = get_tombstone(sym, frag))
+        *(U32<E> *)loc = *val;
+      else
+        *(U32<E> *)loc = S + A - ctx.dtp_addr;
+      break;
     default:
       Fatal(ctx) << *this << ": invalid relocation for non-allocated sections: "
                  << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-sparc64.cc

@@ -200,6 +200,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -512,12 +515,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -555,7 +555,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
     default:
       Fatal(ctx) << *this << ": apply_reloc_nonalloc: " << rel;
     }
-  }
+  });
 }
 
 template <>

src/arch-x86-64.cc

@@ -433,14 +433,17 @@ static void relax_ld_to_le(u8 *loc, const ElfRel<E> &rel, i64 tls_size) {
 // scan_relocations().
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
+  std::span<ElfRel<E>> rels = get_rels(ctx);
 
   for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+    ElfRel<E> &rel = rels[i];
     if (rel.r_type == R_NONE)
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
+    if (sym.get_type() == STT_TLS && sym.is_remaining_undef_weak())
+      continue;
+
     u8 *loc = base + rel.r_offset;
 
     u64 S = sym.get_addr(ctx);
@@ -528,6 +531,8 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           loc[-2] = insn >> 8;
           loc[-1] = insn;
           *(ul32 *)loc = S + A - P;
+          if (ctx.arg.emit_relocs)
+            rel.r_type = R_X86_64_PC32;
           break;
         }
       }
@@ -657,12 +662,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
 // scan_relocations.
 template <>
 void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
-  std::span<const ElfRel<E>> rels = get_rels(ctx);
-
-  for (i64 i = 0; i < rels.size(); i++) {
-    const ElfRel<E> &rel = rels[i];
+  for_each_reloc(ctx, [&](const ElfRel<E> &rel, i64 i) ALWAYS_INLINE {
     if (rel.r_type == R_NONE || record_undef_error(ctx, rel))
-      continue;
+      return;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
     u8 *loc = base + rel.r_offset;
@@ -741,7 +743,7 @@ void InputSection<E>::apply_reloc_nonalloc(Context<E> &ctx, u8 *base) {
                  << rel;
       break;
     }
-  }
+  });
 }
 
 // Linker has to create data structures in an output file to apply
@@ -761,7 +763,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       continue;
 
     Symbol<E> &sym = *file.symbols[rel.r_sym];
-    u8 *loc = (u8 *)(contents.data() + rel.r_offset);
+    u8 *loc = contents + rel.r_offset;
 
     if (sym.is_ifunc())
       sym.flags |= NEEDS_GOT | NEEDS_PLT;
@@ -907,10 +909,14 @@ void rewrite_endbr(Context<E> &ctx) {
     // If isec has an endbr64 at a given offset, copy that instruction to
     // the output buffer, possibly overwriting a nop written in the above
     // loop.
+    i64 size = 0;
+    if (isec)
+      size = isec->get_contents().size();
+
     if (isec && isec->output_section &&
         (isec->shdr().sh_flags & SHF_EXECINSTR) &&
-        0 <= offset && offset <= isec->contents.size() - 4 &&
-        memcmp(isec->contents.data() + offset, endbr64, 4) == 0)
+        0 <= offset && offset <= size - 4 &&
+        memcmp(isec->contents + offset, endbr64, 4) == 0)
       memcpy(ctx.buf + isec->output_section->shdr.sh_offset + isec->offset + offset,
              endbr64, 4);
   };
@@ -918,7 +924,7 @@ void rewrite_endbr(Context<E> &ctx) {
   // Write back endbr64 instructions if they are referred to by address-taking
   // relocations.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC)) {
         for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
           if (!is_func_call_rel(rel)) {

src/cmdline.cc

@@ -47,7 +47,7 @@ Options:
                               Set DT_DEPAUDIT to the specified value
   -S, --strip-debug           Strip .debug_* sections
   -T FILE, --script FILE      Read linker script
-  -X, --discard-locals        Discard temporary local symbols
+  -X, --discard-locals        Discard temporary local symbols (default)
   -e SYMBOL, --entry SYMBOL   Set program entry point
   -f SHLIB, --auxiliary SHLIB Set DT_AUXILIARY to the specified value
   -h LIBNAME, --soname LIBNAME
@@ -76,6 +76,7 @@ Options:
   --Tbss=ADDR                 Set address to .bss
   --Tdata=ADDR                Set address to .data
   --Ttext=ADDR                Set address to .text
+  --Ttext-segment=ADDR        Set address of text segment
   --allow-multiple-definition Allow multiple definitions
   --apply-dynamic-relocs      Apply link-time values for dynamic relocations (default)
     --no-apply-dynamic-relocs
@@ -98,6 +99,7 @@ Options:
     --no-demangle
   --detach                    Create separate debug info file in the background (default)
     --no-detach
+  --discard-none              Keep all local symbols in the symbol table
   --enable-new-dtags          Emit DT_RUNPATH for --rpath (default)
     --disable-new-dtags       Emit DT_RPATH for --rpath
   --execute-only              Make executable segments unreadable
@@ -371,19 +373,40 @@ static std::string_view string_trim(std::string_view str) {
   return str.substr(0, pos + 1);
 }
 
-static std::vector<std::string> add_dashes(std::string name) {
-  // Single-letter option
-  if (name.size() == 1)
-    return {"-" + name};
+// This function matches a command line argument against an option
+// name and, on success, returns the remainder of the argument. For
+// example, matching "--foo=bar" against "foo" yields "=bar", and
+// matching "--foo" against "foo" yields an empty string. On
+// mismatch, it returns std::nullopt.
+//
+// Multi-letter option names can be preceded by either a single dash
+// or double dashes except ones starting with "o", which must be
+// preceded by double dashes. For example, "-omagic" is interpreted
+// as "-o magic". If you really want to specify the "omagic" option,
+// you have to pass "--omagic". Single-letter option names take a
+// single dash.
+static std::optional<std::string_view>
+match_option(std::string_view arg, std::string_view name) {
+  assert(arg.starts_with('-'));
+  arg.remove_prefix(1);
 
-  // Multi-letter linker options can be preceded by either a single
-  // dash or double dashes except ones starting with "o", which must
-  // be preceded by double dashes. For example, "-omagic" is
-  // interpreted as "-o magic". If you really want to specify the
-  // "omagic" option, you have to pass "--omagic".
-  if (name[0] == 'o')
-    return {"--" + name};
-  return {"-" + name, "--" + name};
+  // Single-letter option
+  if (name.size() == 1) {
+    if (arg.starts_with(name))
+      return arg.substr(1);
+    return {};
+  }
+
+  // Options beginning with "o" require double dashes
+  if (name[0] == 'o' && !arg.starts_with('-'))
+    return {};
+
+  if (arg.starts_with('-'))
+    arg.remove_prefix(1);
+
+  if (arg.starts_with(name))
+    return arg.substr(name.size());
+  return {};
 }
 
 template <typename E>
@@ -608,11 +631,18 @@ static bool returns_etxtbsy() {
 }
 
 template <typename E>
-std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
+std::vector<ReaderJob> parse_nonpositional_args(Context<E> &ctx) {
   std::span<std::string_view> args = ctx.cmdline_args;
   args = args.subspan(1);
 
-  std::vector<std::string> remaining;
+  // Input file arguments are turned into ReaderJobs for
+  // read_input_files(). rctx tracks the reader state options, such as
+  // --as-needed, that apply to the files after them; each job gets a
+  // snapshot of the state at its position.
+  std::vector<ReaderJob> jobs;
+  ReaderContext rctx;
+  std::vector<ReaderContext> rctx_stack;
+  std::unordered_set<std::string_view> visited_libs;
   std::string_view arg;
 
   ctx.arg.color_diagnostics = isatty(STDERR_FILENO);
@@ -638,11 +668,6 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     }
   };
 
-  // RISC-V and LoongArch object files contains lots of local symbols,
-  // so by default we discard them. This is compatible with GNU ld.
-  if constexpr (is_riscv<E> || is_loongarch<E>)
-    ctx.arg.discard_locals = true;
-
   // We generally don't need to write addends to relocated places if the
   // relocation type is RELA because RELA records contain addends.
   // However, there are too much code that wrongly assumes that addends
@@ -657,76 +682,95 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   //   we write addends to relocated places.
   ctx.arg.apply_dynamic_relocs = !is_sparc<E> && !is_riscv<E>;
 
-  auto read_arg = [&](std::string name) {
-    for (const std::string &opt : add_dashes(name)) {
-      if (args[0] == opt) {
-        if (args.size() == 1)
-          Fatal(ctx) << "option -" << name << ": argument missing";
-        arg = args[1];
-        args = args.subspan(2);
-        return true;
-      }
+  auto read_arg = [&](std::string_view name) {
+    std::optional<std::string_view> rest = match_option(args[0], name);
+    if (!rest)
+      return false;
 
-      std::string prefix = (name.size() == 1) ? opt : opt + "=";
-      if (args[0].starts_with(prefix)) {
-        arg = args[0].substr(prefix.size());
-        args = args.subspan(1);
-        return true;
-      }
+    // An option and its argument are either separate command line
+    // arguments or a single one, as in "-o foo" vs. "-ofoo" or
+    // "--output foo" vs. "--output=foo".
+    if (rest->empty()) {
+      if (args.size() == 1)
+        Fatal(ctx) << "option -" << name << ": argument missing";
+      arg = args[1];
+      args = args.subspan(2);
+      return true;
+    }
+
+    if (name.size() == 1) {
+      arg = *rest;
+      args = args.subspan(1);
+      return true;
+    }
+
+    if (rest->starts_with('=')) {
+      arg = rest->substr(1);
+      args = args.subspan(1);
+      return true;
     }
     return false;
   };
 
-  auto read_eq = [&](std::string name) {
-    for (const std::string &opt : add_dashes(name)) {
-      if (args[0].starts_with(opt + "=")) {
-        arg = args[0].substr(opt.size() + 1);
-        args = args.subspan(1);
-        return true;
-      }
+  auto read_eq = [&](std::string_view name) {
+    std::optional<std::string_view> rest = match_option(args[0], name);
+    if (rest && rest->starts_with('=')) {
+      arg = rest->substr(1);
+      args = args.subspan(1);
+      return true;
     }
     return false;
   };
 
-  auto read_flag = [&](std::string name) {
-    for (const std::string &opt : add_dashes(name)) {
-      if (args[0] == opt) {
-        args = args.subspan(1);
-        return true;
-      }
+  auto read_flag = [&](std::string_view name) {
+    if (match_option(args[0], name) == "") {
+      args = args.subspan(1);
+      return true;
     }
     return false;
   };
 
-  auto read_z_flag = [&](std::string name) {
+  auto read_z_flag = [&](std::string_view name) {
     if (args.size() >= 2 && args[0] == "-z" && args[1] == name) {
       args = args.subspan(2);
       return true;
     }
 
-    if (!args.empty() && args[0] == "-z" + name) {
+    if (args[0].starts_with("-z") && args[0].substr(2) == name) {
       args = args.subspan(1);
       return true;
     }
     return false;
   };
 
-  auto read_z_arg = [&](std::string name) {
-    if (args.size() >= 2 && args[0] == "-z" && args[1].starts_with(name + "=")) {
+  auto read_z_arg = [&](std::string_view name) {
+    if (args.size() >= 2 && args[0] == "-z" && args[1].starts_with(name) &&
+        args[1].substr(name.size()).starts_with('=')) {
       arg = args[1].substr(name.size() + 1);
       args = args.subspan(2);
       return true;
     }
 
-    if (!args.empty() && args[0].starts_with("-z" + name + "=")) {
-      arg = args[0].substr(name.size() + 3);
-      args = args.subspan(1);
-      return true;
+    if (args[0].starts_with("-z")) {
+      std::string_view s = args[0].substr(2);
+      if (s.starts_with(name) && s.substr(name.size()).starts_with('=')) {
+        arg = s.substr(name.size() + 1);
+        args = args.subspan(1);
+        return true;
+      }
     }
     return false;
   };
 
   while (!args.empty()) {
+    if (!args[0].starts_with('-')) {
+      ReaderJob job = {rctx, std::string(args[0])};
+      job.rctx.pos = {(u32)jobs.size()};
+      jobs.push_back(std::move(job));
+      args = args.subspan(1);
+      continue;
+    }
+
     if (read_flag("help")) {
       Out(ctx) << "Usage: " << ctx.cmdline_args[0]
                << " [options] file...\n" << helpmsg;
@@ -828,7 +872,7 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
         Fatal(ctx) << "unknown -m argument: " << arg;
       }
     } else if (read_flag("end-lib")) {
-      remaining.emplace_back("--end-lib");
+      rctx.in_lib = false;
     } else if (read_flag("export-dynamic") || read_flag("E")) {
       ctx.arg.export_dynamic = true;
     } else if (read_flag("no-export-dynamic")) {
@@ -859,9 +903,9 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     } else if (read_flag("print-map") || read_flag("M")) {
       ctx.arg.print_map = true;
     } else if (read_flag("Bstatic") || read_flag("dn") || read_flag("static")) {
-      remaining.emplace_back("--Bstatic");
+      rctx.static_ = true;
     } else if (read_flag("Bdynamic") || read_flag("dy")) {
-      remaining.emplace_back("--Bdynamic");
+      rctx.static_ = false;
     } else if (read_flag("shared") || read_flag("Bshareable")) {
       ctx.arg.shared = true;
     } else if (read_arg("spare-dynamic-tags")) {
@@ -870,7 +914,7 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       ctx.arg.spare_program_headers
         = parse_number(ctx, "spare-program-headers", arg);
     } else if (read_flag("start-lib")) {
-      remaining.emplace_back("--start-lib");
+      rctx.in_lib = true;
     } else if (read_flag("start-stop")) {
       ctx.arg.start_stop = true;
     } else if (read_arg("dependency-file")) {
@@ -1080,7 +1124,7 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       ctx.arg.wrap.insert(arg);
     } else if (read_flag("omagic") || read_flag("N")) {
       ctx.arg.omagic = true;
-      remaining.emplace_back("--Bstatic");
+      rctx.static_ = true;
     } else if (read_flag("no-omagic")) {
       ctx.arg.omagic = false;
     } else if (read_arg("oformat")) {
@@ -1111,6 +1155,8 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       ctx.arg.section_start[".data"] = parse_hex(ctx, "Tdata", arg);
     } else if (read_arg("Ttext")) {
       ctx.arg.section_start[".text"] = parse_hex(ctx, "Ttext", arg);
+    } else if (read_arg("Ttext-segment")) {
+      ctx.arg.ttext_segment = parse_number(ctx, "Ttext-segment", arg);
     } else if (read_flag("repro")) {
       ctx.arg.repro = true;
     } else if (read_z_flag("now")) {
@@ -1344,6 +1390,9 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       ctx.arg.discard_all = true;
     } else if (read_flag("discard-locals") || read_flag("X")) {
       ctx.arg.discard_locals = true;
+    } else if (read_flag("discard-none")) {
+      ctx.arg.discard_all = false;
+      ctx.arg.discard_locals = false;
     } else if (read_flag("strip-all") || read_flag("s")) {
       ctx.arg.strip_all = true;
     } else if (read_flag("strip-debug") || read_flag("S")) {
@@ -1462,21 +1511,31 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     } else if (read_arg("export-dynamic-symbol-list")) {
       append(ctx.dynamic_list_patterns, parse_dynamic_list(ctx, arg));
     } else if (read_flag("as-needed")) {
-      remaining.emplace_back("--as-needed");
+      rctx.as_needed = true;
     } else if (read_flag("no-as-needed")) {
-      remaining.emplace_back("--no-as-needed");
+      rctx.as_needed = false;
     } else if (read_flag("whole-archive")) {
-      remaining.emplace_back("--whole-archive");
+      rctx.whole_archive = true;
     } else if (read_flag("no-whole-archive")) {
-      remaining.emplace_back("--no-whole-archive");
+      rctx.whole_archive = false;
     } else if (read_arg("l") || read_arg("library")) {
-      remaining.push_back("-l" + std::string(arg));
+      if (visited_libs.insert(arg).second) {
+        ReaderJob job = {rctx, std::string(arg)};
+        job.is_lib = true;
+        job.rctx.pos = {(u32)jobs.size()};
+        jobs.push_back(std::move(job));
+      }
     } else if (read_arg("script") || read_arg("T")) {
-      remaining.emplace_back(arg);
+      ReaderJob job = {rctx, std::string(arg)};
+      job.rctx.pos = {(u32)jobs.size()};
+      jobs.push_back(std::move(job));
     } else if (read_flag("push-state")) {
-      remaining.emplace_back("--push-state");
+      rctx_stack.push_back(rctx);
     } else if (read_flag("pop-state")) {
-      remaining.emplace_back("--pop-state");
+      if (rctx_stack.empty())
+        Fatal(ctx) << "no state pushed before popping";
+      rctx = rctx_stack.back();
+      rctx_stack.pop_back();
     } else if (args[0].starts_with("-z") && args[0].size() > 2) {
       Warn(ctx) << "unknown command line option: " << args[0];
       args = args.subspan(1);
@@ -1487,10 +1546,7 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       Fatal(ctx) << "unknown command line option: -dynamic; -dynamic is a "
                  << "macOS linker's option. mold does not support macOS.";
     } else {
-      if (args[0].starts_with('-'))
-        Fatal(ctx) << "unknown command line option: " << args[0];
-      remaining.emplace_back(args[0]);
-      args = args.subspan(1);
+      Fatal(ctx) << "unknown command line option: " << args[0];
     }
   }
 
@@ -1527,8 +1583,14 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   if (ctx.arg.shared)
     ctx.arg.pic = true;
 
-  if (ctx.arg.pic)
+  if (ctx.arg.ttext_segment) {
+    u64 val = *ctx.arg.ttext_segment;
+    if (val % ctx.page_size)
+      Warn(ctx) << "-Ttext-segment is not a multiple of page size: " << val;
+    ctx.arg.image_base = align_down(val, ctx.page_size);
+  } else if (ctx.arg.pic) {
     ctx.arg.image_base = 0;
+  }
 
   // A shared library is loaded by the dynamic linker, so it doesn't need
   // PT_INTERP. (-pie makes the output an executable even with -shared.)
@@ -1685,14 +1747,19 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     sym->gc_root = true;
   ctx.arg.entry->gc_root = true;
 
-  if (version_shown && remaining.empty())
+  if (version_shown && jobs.empty())
     exit(0);
-  return remaining;
+
+  if (rctx.static_ || ctx.arg.relocatable) {
+    ctx.arg.static_ = true;
+    ctx.arg.dynamic_linker = "";
+  }
+  return jobs;
 }
 
 using E = MOLD_TARGET;
 
 template std::vector<std::string_view> expand_response_files(Context<E> &, char **);
-template std::vector<std::string> parse_nonpositional_args(Context<E> &ctx);
+template std::vector<ReaderJob> parse_nonpositional_args(Context<E> &ctx);
 
 } // namespace mold

src/elf.h

@@ -170,6 +170,7 @@ enum : u32 {
   VER_NDX_GLOBAL = 1,
   VER_NDX_LAST_RESERVED = 1,
   VER_NDX_UNSPECIFIED = 0xffff,
+  VERSYM_HIDDEN = 0x8000,
 };
 
 enum : u32 {
@@ -178,10 +179,6 @@ enum : u32 {
   VER_FLG_INFO = 4,
 };
 
-enum : u32 {
-  VERSYM_HIDDEN = 0x8000,
-};
-
 enum : u32 {
   PT_NULL = 0,
   PT_LOAD = 1,
@@ -377,6 +374,10 @@ enum : u32 {
   EF_ARM_EABI_VER5 = 0x5000000,
 };
 
+enum : u32 {
+  EF_PPC64_ABI = 3,
+};
+
 enum : u32 {
   EF_RISCV_RVC = 1,
   EF_RISCV_FLOAT_ABI = 6,
@@ -1833,34 +1834,6 @@ struct ElfRel<SPARC64> {
   ib64 r_addend;
 };
 
-template <>
-struct ElfRel<SH4LE> {
-  ElfRel() = default;
-
-  // Addend is ignored except for base relocations because even though
-  // SH4 is RELA, r_addend is ignored in most cases and works as if it
-  // were REL.
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_type(type), r_sym(sym), r_addend(sym ? 0 : addend) {}
-
-  ul32 r_offset;
-  u8 r_type;
-  ul24 r_sym;
-  il32 r_addend;
-};
-
-template <>
-struct ElfRel<SH4BE> {
-  ElfRel() = default;
-  ElfRel(u64 offset, u32 type, u32 sym, i64 addend)
-    : r_offset(offset), r_sym(sym), r_type(type), r_addend(sym ? 0 : addend) {}
-
-  ub32 r_offset;
-  ub24 r_sym;
-  u8 r_type;
-  ib32 r_addend;
-};
-
 //
 // Machine descriptions
 //
@@ -2235,7 +2208,7 @@ struct SH4LE {
   static constexpr u32 page_size = 4096;
   static constexpr u32 e_machine = EM_SH;
   static constexpr u32 plt_hdr_size = 16;
-  static constexpr u32 plt_size = 16;
+  static constexpr u32 plt_size = 20;
   static constexpr u32 pltgot_size = 12;
   static constexpr u8 trap[] = { 0xfd, 0xff }; // illegal
 

src/entry.cc

@@ -14,6 +14,10 @@
 # include <mimalloc-new-delete.h>
 #endif
 
+#ifdef __linux__
+# include <sys/prctl.h>
+#endif
+
 std::string mold::mold_version =
 #ifdef MOLD_GIT_HASH
   "mold " MOLD_VERSION " (" MOLD_GIT_HASH "; compatible with GNU ld)";
@@ -28,5 +32,12 @@ int main(int argc, char **argv) {
   mi_option_disable(mi_option_show_errors);
 #endif
 
+#if defined(__linux__) && defined(PR_SET_THP_DISABLE)
+  // A parent process may have disabled transparent huge pages, and the
+  // flag is inherited. Huge pages make mold considerably faster on
+  // large links, so re-enable them for this process.
+  prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
+#endif
+
   return mold::mold_main<mold::MOLD_FIRST_TARGET>(argc, argv);
 }

src/filetype.cc

@@ -133,10 +133,10 @@ static std::string_view get_elf_type(u8 *buf) {
   bool is_le = (buf[EI_DATA] == ELFDATA2LSB);
   bool is_64 = (buf[EI_CLASS] == ELFCLASS64);
 
-  auto *ehdr_le = (ElfEhdr<I386> *)buf;
-  auto *ehdr_be = (ElfEhdr<M68K> *)buf;
+  auto *ehdr64_le = (ElfEhdr<ARM64LE> *)buf;
+  auto *ehdr64_be = (ElfEhdr<ARM64BE> *)buf;
 
-  switch (is_le ? ehdr_le->e_machine : ehdr_be->e_machine) {
+  switch (is_le ? ehdr64_le->e_machine : ehdr64_be->e_machine) {
   case EM_386:
     return I386::name;
   case EM_X86_64:
@@ -152,7 +152,18 @@ static std::string_view get_elf_type(u8 *buf) {
   case EM_PPC:
     return PPC32::name;
   case EM_PPC64:
-    return is_le ? PPC64V2::name : PPC64V1::name;
+    // ELFv1 is big-endian and ELFv2 is little-endian by convention, but
+    // the correspondence is not a rule; musl for example uses ELFv2 on
+    // big-endian too. We support only the usual combinations, so treat
+    // the others as unrecognizable rather than silently linking them
+    // against the wrong ABI.
+    if (!is_le)
+      if (u32 abi = (ehdr64_be->e_flags & EF_PPC64_ABI); abi == 0 || abi == 1)
+        return PPC64V1::name;
+    if (is_le)
+      if (u32 abi = (ehdr64_le->e_flags & EF_PPC64_ABI); abi == 0 || abi == 2)
+        return PPC64V2::name;
+    return "";
   case EM_S390X:
     return S390X::name;
   case EM_SPARC64:

src/gc-sections.cc

@@ -15,8 +15,10 @@ template <typename E>
 static bool should_keep(const InputSection<E> &isec) {
   u32 type = isec.shdr().sh_type;
   u32 flags = isec.shdr().sh_flags;
+  std::string_view name = isec.name();
+
   if constexpr (is_ppc32<E>)
-    if (isec.name == ".got2")
+    if (name == ".got2")
       return true;
 
   return (flags & SHF_GNU_RETAIN) ||
@@ -24,10 +26,10 @@ static bool should_keep(const InputSection<E> &isec) {
          type == SHT_INIT_ARRAY ||
          type == SHT_FINI_ARRAY ||
          type == SHT_PREINIT_ARRAY ||
-         isec.name.starts_with(".ctors") ||
-         isec.name.starts_with(".dtors") ||
-         isec.name.starts_with(".init") ||
-         isec.name.starts_with(".fini");
+         name.starts_with(".ctors") ||
+         name.starts_with(".dtors") ||
+         name.starts_with(".init") ||
+         name.starts_with(".fini");
 }
 
 // Sections whose names are valid C identifiers can be referenced via
@@ -37,16 +39,18 @@ static bool should_keep(const InputSection<E> &isec) {
 // of a given name when we encounter such a reference during marking.
 template <typename E>
 using StartStopMap =
-  tbb::concurrent_unordered_multimap<std::string_view, InputSection<E> *>;
+  tbb::concurrent_unordered_map<std::string_view,
+                                tbb::concurrent_vector<InputSection<E> *>>;
 
 template <typename E>
 static StartStopMap<E> build_start_stop_map(Context<E> &ctx) {
   StartStopMap<E> map;
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
-      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC) &&
-          is_c_identifier(isec->name))
-        map.insert({isec->name, isec.get()});
+    for (InputSection<E> *isec : file->sections)
+      if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC))
+        if (std::string_view name = isec->name();
+            is_c_identifier(name))
+          map[name].push_back(isec);
   });
   return map;
 }
@@ -78,7 +82,7 @@ collect_root_set(Context<E> &ctx) {
 
   // Add sections that are not subject to garbage collection.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (!isec || !isec->is_alive)
         continue;
 
@@ -92,7 +96,7 @@ collect_root_set(Context<E> &ctx) {
       }
 
       if (should_keep(*isec))
-        enqueue_section(isec.get());
+        enqueue_section(isec);
     }
   });
 
@@ -151,6 +155,12 @@ static void visit_section(Context<E> &ctx, InputSection<E> *isec,
   for (const ElfRel<E> &rel : isec->get_rels(ctx)) {
     // Symbol can refer to either a section fragment or an input section.
     Symbol<E> &sym = *isec->file.symbols[rel.r_sym];
+
+    if (sym.file && sym.file->is_dso) {
+      sym.file->is_reachable.test_and_set();
+      continue;
+    }
+
     if (SectionFragment<E> *frag = sym.get_frag()) {
       frag->is_alive = true;
       continue;
@@ -158,13 +168,24 @@ static void visit_section(Context<E> &ctx, InputSection<E> *isec,
 
     mark(sym.get_input_section());
 
-    // A reference to __start_<name> or __stop_<name> keeps every section
-    // named <name> alive, mirroring how those symbols are defined.
-    if (std::string_view sec = start_stop_name(sym.name());
-        !sec.empty()) {
-      auto [i, end] = start_stop_map.equal_range(sec);
-      for (; i != end; ++i)
-        mark(i->second);
+    // A reference to __start_<name> or __stop_<name> keeps every
+    // section named <name> alive, mirroring how those symbols are
+    // defined. A single such reference can keep an enormous number of
+    // sections alive, so we spread the fanout over threads instead
+    // of marking the sections one by one.
+    if (std::string_view name = start_stop_name(sym.name());
+        !name.empty()) {
+      if (auto it = start_stop_map.find(name);
+          it != start_stop_map.end()) {
+        // Mark and visit the sections with a nested parallel loop.
+        // As in mark() below, a section added to a feeder has
+        // already been marked, so a feeder's loop body must visit
+        // it unconditionally.
+        tbb::parallel_for_each(it->second, [&](InputSection<E> *isec) {
+          if (mark_section(isec))
+            visit_section(ctx, isec, feeder, 0, start_stop_map);
+        });
+      }
     }
   }
 
@@ -195,10 +216,10 @@ static void sweep(Context<E> &ctx) {
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
     ObjectFile<E> &file = *ctx.objs[i];
 
-    for (std::unique_ptr<InputSection<E>> &isec : file.sections) {
+    for (InputSection<E> *isec : file.sections) {
       if (isec && isec->is_alive && !isec->is_visited) {
         isec->kill();
-        sections[i].push_back(isec.get());
+        sections[i].push_back(isec);
       }
     }
   });
@@ -229,10 +250,25 @@ static void sweep(Context<E> &ctx) {
 template <typename E>
 void gc_sections(Context<E> &ctx) {
   Timer t(ctx, "gc");
+
+  for (SharedFile<E> *file : ctx.dsos)
+    if (file->as_needed)
+      file->is_reachable = false;
+
+  for (Symbol<E> *sym : ctx.arg.undefined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
+  for (Symbol<E> *sym : ctx.arg.require_defined)
+    if (sym->file && sym->file->is_dso)
+      sym->file->is_reachable = true;
+
   tbb::concurrent_vector<InputSection<E> *> rootset = collect_root_set(ctx);
   StartStopMap<E> start_stop_map = build_start_stop_map(ctx);
   mark(ctx, rootset, start_stop_map);
   sweep(ctx);
+
+  std::erase_if(ctx.dsos, [](SharedFile<E> *file) { return !file->is_reachable; });
 }
 
 using E = MOLD_TARGET;

src/gdb-index.cc

@@ -58,6 +58,7 @@
 
 #include "mold.h"
 #include <tbb/parallel_for_each.h>
+#include <tbb/parallel_scan.h>
 
 namespace mold {
 
@@ -125,13 +126,6 @@ struct SectionHeader {
   ul32 const_pool_offset = 0;
 };
 
-struct NameType {
-  auto operator<=>(const NameType &) const = default;
-  u64 hash;
-  u8 type;
-  std::string_view name;
-};
-
 struct MapValue {
   u32 gdb_hash = 0;
   Atomic<u32> count;
@@ -139,24 +133,146 @@ struct MapValue {
   u32 type_offset = 0;
 };
 
+struct NameType {
+  NameType(u64 hash, u8 type, const char *name)
+    : hash_and_type((hash << 8) | type), name(name) {}
+
+  u64 get_hash() const {
+    return hash_and_type >> 8;
+  }
+
+  u8 get_type() const {
+    return hash_and_type & 0xff;
+  }
+
+  bool operator==(const NameType &other) const {
+    return hash_and_type == other.hash_and_type && !strcmp(name, other.name);
+  }
+
+  // The low byte stores the type. Hash collisions, including collisions in
+  // the discarded high byte, are resolved by comparing the complete name.
+  u64 hash_and_type;
+  const char *name;
+};
+
+struct IndexedName {
+  MapValue *entry;
+  u32 type_idx;
+  u8 type;
+};
+
+// A name record is read as a NameType and replaced with an IndexedName after
+// the name has been interned. The union avoids retaining both forms for the
+// millions of public names in a large debug link.
+union NameRecord {
+  NameRecord(u64 hash, u8 type, const char *name)
+    : nametype(hash, type, name) {}
+
+  NameType nametype;
+  IndexedName indexed;
+};
+
 struct Compunit {
   DwarfKind kind;
   i64 offset;
   i64 size;
+  i32 file_idx;
   std::vector<std::pair<u64, u64>> ranges;
-  std::vector<NameType> nametypes;
-  std::vector<MapValue *> entries;
+  std::vector<NameRecord> names;
 };
 
-// The hash function for .gdb_index.
-static u32 gdb_hash(std::string_view name) {
+using GdbNameMap = ConcurrentMap<MapValue>;
+
+struct PoolSize {
+  i64 type_bytes = 0;
+  i64 name_bytes = 0;
+};
+
+struct GdbIndexData {
+  std::vector<Compunit> cus;
+  std::unique_ptr<GdbNameMap> map;
+  std::vector<GdbNameMap::Entry *> entries;
+  std::unique_ptr<u8[]> tables;
+  PoolSize pool_size;
+  i64 ht_size = 0;
+};
+
+// GDB index type lists contain fixed-width 32-bit records. Four byte-wise
+// passes are faster than comparison sorting once a list is sufficiently large.
+static void radix_sort(std::span<ul32> values, std::vector<ul32> &scratch) {
+  scratch.resize(values.size());
+  ul32 *src = values.data();
+  ul32 *dst = scratch.data();
+
+  for (i64 shift = 0; shift < 32; shift += 8) {
+    u32 counts[256] = {};
+    for (i64 i = 0; i < values.size(); i++)
+      counts[((u32)src[i] >> shift) & 255]++;
+
+    u32 offsets[256];
+    offsets[0] = 0;
+    for (i64 i = 1; i < 256; i++)
+      offsets[i] = offsets[i - 1] + counts[i - 1];
+
+    for (i64 i = 0; i < values.size(); i++) {
+      u32 val = src[i];
+      dst[offsets[(val >> shift) & 255]++] = val;
+    }
+    std::swap(src, dst);
+  }
+
+  assert(src == values.data());
+}
+
+// GCC can emit the same public name once for each COMDAT group. Remove these
+// duplicates with a local hash table instead of sorting the strings.
+static void dedup_names(Compunit &cu) {
+  if (cu.names.size() < 2)
+    return;
+
+  u64 capacity = bit_ceil(cu.names.size() * 2);
+  std::vector<u32> buckets(capacity, UINT32_MAX);
+  u64 mask = capacity - 1;
+  i64 out = 0;
+
+  for (NameRecord &record : cu.names) {
+    NameType &nt = record.nametype;
+    u64 idx =
+      (nt.get_hash() ^ ((u64)nt.get_type() * 0x9e37'79b9)) & mask;
+    bool duplicate = false;
+
+    while (buckets[idx] != UINT32_MAX) {
+      if (cu.names[buckets[idx]].nametype == nt) {
+        duplicate = true;
+        break;
+      }
+      idx = (idx + 1) & mask;
+    }
+
+    if (duplicate)
+      continue;
+
+    if (&record != &cu.names[out])
+      cu.names[out] = record;
+    buckets[idx] = out++;
+  }
+
+  cu.names.erase(cu.names.begin() + out, cu.names.end());
+}
+
+// Compute the .gdb_index hash and length together when a name is first
+// inserted.
+static u32 initialize_gdb_name(MapValue &value, const char *name) {
   u32 h = 0;
-  for (u8 c : name) {
+  u32 size = 0;
+
+  for (u8 c = *name; c; c = name[++size]) {
     if ('A' <= c && c <= 'Z')
       c = 'a' + c - 'A';
     h = h * 67 + c - 113;
   }
-  return h;
+  value.gdb_hash = h;
+  return size;
 }
 
 template <typename E>
@@ -509,14 +625,15 @@ static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
                             std::vector<Compunit> &cus, ObjectFile<E> &file) {
   using Offset = decltype(hdr.size);
 
+  // Compunits are sorted by offset, so we can use binary search.
   auto get_cu = [&](i64 offset) {
-    for (i64 i = 0; i < cus.size(); i++)
-      if (cus[i].offset == offset)
-        return &cus[i];
-    Fatal(ctx) << file << ": corrupted debug_info_offset";
+    auto it = ranges::lower_bound(cus, offset, {}, &Compunit::offset);
+    if (it == cus.end() || it->offset != offset)
+      Fatal(ctx) << file << ": corrupted debug_info_offset";
+    return &*it;
   };
 
-  Compunit *cu = get_cu(file.debug_info->offset + hdr.debug_info_offset);
+  Compunit *cu = get_cu(hdr.debug_info_offset);
   i64 size = hdr.size + offsetof(PubnamesHdr, size) + sizeof(hdr.size);
   u8 *p = (u8 *)&hdr + sizeof(hdr);
   u8 *end = (u8 *)&hdr + size;
@@ -527,9 +644,11 @@ static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
     p += sizeof(Offset);
 
     u8 type = *p++;
-    std::string_view name = (char *)p;
-    p += name.size() + 1;
-    cu->nametypes.push_back(NameType{hash_string(name), type, name});
+    const char *name = (char *)p;
+    i64 len = strlen(name);
+    p += len + 1;
+    cu->names.emplace_back(
+      hash_string(std::string_view(name, len)), type, name);
   }
 
   return size;
@@ -551,11 +670,12 @@ static void read_pubnames(Context<E> &ctx, std::vector<Compunit> &cus,
       continue;
 
     isec->uncompress(ctx);
-    if (isec->contents.empty())
+    std::string_view contents = isec->get_contents();
+    if (contents.empty())
       continue;
 
-    u8 *p = (u8*)&isec->contents[0];
-    u8 *end = p + isec->contents.size();
+    u8 *p = (u8 *)contents.data();
+    u8 *end = p + contents.size();
 
     while (p < end) {
       if (*(U32<E> *)p == 0xffff'ffff)
@@ -563,30 +683,55 @@ static void read_pubnames(Context<E> &ctx, std::vector<Compunit> &cus,
       else
         p += read_pubnames_cu(ctx, *(PubnamesHdr32<E> *)p, cus, file);
     }
-  };
+  }
 }
 
 template <typename E>
 static std::vector<Compunit> read_compunits(Context<E> &ctx) {
+  std::vector<std::vector<Compunit>> file_cus(ctx.objs.size());
+
+  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 file_idx) {
+    ObjectFile<E> &file = *ctx.objs[file_idx];
+    if (!file.debug_info)
+      return;
+
+    file.debug_info->uncompress(ctx);
+    std::string_view contents = file.debug_info->get_contents();
+    std::vector<Compunit> &cus = file_cus[file_idx];
+    u8 *begin = (u8 *)contents.data();
+    u8 *end = begin + contents.size();
+
+    for (u8 *p = begin; p < end;) {
+      DwarfKind kind = get_dwarf_kind(ctx, p);
+      i64 size;
+      if (kind == DWARF2_32 || kind == DWARF5_32)
+        size = ((CuHdrDwarf2_32<E> *)p)->size + 4;
+      else
+        size = ((CuHdrDwarf2_64<E> *)p)->size + 12;
+
+      cus.push_back(Compunit{kind, p - begin, size, (i32)file_idx});
+      p += size;
+    }
+
+    read_pubnames(ctx, cus, file);
+    for (Compunit &cu : cus)
+      dedup_names(cu);
+  });
+
+  i64 size = 0;
+  for (std::vector<Compunit> &cus : file_cus)
+    size += cus.size();
+
   std::vector<Compunit> cus;
+  cus.reserve(size);
+  for (std::vector<Compunit> &vec : file_cus)
+    for (Compunit &cu : vec)
+      cus.push_back(std::move(cu));
+  return cus;
+}
 
-  // Read compunits from the output .debug_info section.
-  u8 *begin = &ctx.debug_info[0];
-  u8 *end = begin + ctx.debug_info.size();
-
-  for (u8 *p = begin; p < end;) {
-    DwarfKind kind = get_dwarf_kind(ctx, p);
-    i64 size;
-    if (kind == DWARF2_32 || kind == DWARF5_32)
-      size = ((CuHdrDwarf2_32<E> *)p)->size + 4;
-    else
-      size = ((CuHdrDwarf2_64<E> *)p)->size + 12;
-
-    cus.push_back(Compunit{kind, p - begin, size});
-    p += size;
-  }
-
-  // Read address ranges for each compunit.
+template <typename E>
+static void read_address_ranges(Context<E> &ctx, std::vector<Compunit> &cus) {
   tbb::parallel_for_each(cus, [&](Compunit &cu) {
     switch (cu.kind) {
     case DWARF2_32:
@@ -603,29 +748,14 @@ static std::vector<Compunit> read_compunits(Context<E> &ctx) {
       break;
     }
 
-    // Remove empty ranges
-    std::erase_if(cu.ranges, [](std::pair<u64, u64> p) {
-      return p.first == 0 || p.first == p.second;
+    std::erase_if(cu.ranges, [](std::pair<u64, u64> range) {
+      return range.first == 0 || range.first == range.second;
     });
   });
-
-  // Read symbols from .debug_gnu_pubnames and .debug_gnu_pubtypes.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    read_pubnames(ctx, cus, *file);
-  });
-
-  // Uniquify elements because GCC 11 seems to emit one record for each
-  // comdat group which results in having a lot of duplicate records.
-  tbb::parallel_for_each(cus, [](Compunit &cu) {
-    ranges::stable_sort(cu.nametypes);
-    remove_duplicates(cu.nametypes);
-  });
-
-  return cus;
 }
 
 template <typename E>
-std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
+static std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
   if (chunk->is_compressed) {
     CompressedSection<E> &sec = *(CompressedSection<E> *)chunk;
     return {sec.uncompressed_data.get(), (size_t)sec.chdr.ch_size};
@@ -633,10 +763,176 @@ std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
   return {ctx.buf + chunk->shdr.sh_offset, (size_t)chunk->shdr.sh_size};
 }
 
+// Read compilation units and their public names, deduplicate and intern the
+// names, and determine the constant-pool layout. This stage needs only input
+// sections, so it can run before output-section offsets are assigned.
+template <typename E>
+void read_gdb_index_inputs(Context<E> &ctx) {
+  Timer t(ctx, "read_gdb_index_inputs");
+
+  ctx.gdb_index_data = std::make_shared<GdbIndexData>();
+
+  GdbIndexData &data = *ctx.gdb_index_data;
+  data.cus = read_compunits(ctx);
+
+  HyperLogLog estimator;
+  tbb::parallel_for_each(data.cus, [&](Compunit &cu) {
+    HyperLogLog::Sketch &sketch = estimator.local();
+
+    // NameType keeps 56 hash bits. Spread them across a 64-bit word because
+    // HyperLogLog uses the number of leading zero bits.
+    for (NameRecord &record : cu.names)
+      sketch.insert(record.nametype.get_hash() * 0x9e37'79b9'7f4a'7c15);
+  });
+
+  data.map = std::make_unique<GdbNameMap>(estimator.get_cardinality() * 3 / 2);
+
+  tbb::parallel_for_each(data.cus, [&](Compunit &cu) {
+    for (NameRecord &record : cu.names) {
+      NameType &nt = record.nametype;
+      const char *name = nt.name;
+      u64 hash = nt.get_hash();
+      u8 type = nt.get_type();
+      MapValue *ent =
+        data.map->insert_cstr(name, hash, {}, initialize_gdb_name).first;
+      record.indexed = {ent, ent->count++ + 1, type};
+    }
+  });
+
+  data.entries = data.map->get_sorted_entries_all();
+  data.ht_size = bit_ceil(data.entries.size() * 5 / 4 + 1);
+
+  // The map may contain millions of names. Assign their type and string
+  // ranges with a parallel prefix sum.
+  auto scan = [&](const tbb::blocked_range<i64> &range, PoolSize size,
+                  bool is_final) {
+    for (i64 i = range.begin(); i < range.end(); i++) {
+      GdbNameMap::Entry *ent = data.entries[i];
+      if (is_final) {
+        ent->value.type_offset = size.type_bytes;
+        ent->value.name_offset = size.name_bytes;
+      }
+      size.type_bytes += ent->value.count * 4 + 4;
+      size.name_bytes += ent->keylen + 1;
+    }
+    return size;
+  };
+
+  data.pool_size = tbb::parallel_scan(
+    tbb::blocked_range<i64>(0, data.entries.size()), PoolSize{}, scan,
+    [](PoolSize a, PoolSize b) -> PoolSize {
+      return {a.type_bytes + b.type_bytes, a.name_bytes + b.name_bytes};
+    });
+
+  tbb::parallel_for_each(data.entries, [&](GdbNameMap::Entry *ent) {
+    ent->value.name_offset += data.pool_size.type_bytes;
+  });
+}
+
+// Build the name lookup table and the constant pool for .gdb_index. They
+// depend on compilation-unit order but not on relocated address ranges, so
+// they can be built in the background once .debug_info offsets are fixed.
+template <typename E>
+void build_gdb_index_tables(Context<E> &ctx) {
+  Timer t(ctx, "build_gdb_index_tables");
+
+  GdbIndexData &data = *ctx.gdb_index_data;
+  if (data.cus.empty() || data.tables)
+    return;
+
+  std::vector<Compunit> &cus = data.cus;
+  std::vector<GdbNameMap::Entry *> &entries = data.entries;
+  using Entry = GdbNameMap::Entry;
+
+  // CU offsets are relative to their input .debug_info sections. Convert them
+  // to output-section offsets and sort the CUs in output order. A CU's position
+  // in this vector is the CU number stored in type vectors and address ranges.
+  for (Compunit &cu : cus)
+    cu.offset += ctx.objs[cu.file_idx]->debug_info->offset;
+  ranges::sort(cus, {}, &Compunit::offset);
+
+  // `tables` contains the name hash table followed by the constant pool. The
+  // constant pool contains all type vectors followed by all name strings.
+  i64 symtab_size = data.ht_size * 8;
+  i64 pool_size = data.pool_size.type_bytes + data.pool_size.name_bytes;
+  data.tables = std::unique_ptr<u8[]>(new u8[symtab_size + pool_size]);
+
+  // Each occupied hash-table slot contains the constant-pool offsets of a name
+  // and its type vector.
+  ul32 *ht = (ul32 *)data.tables.get();
+  memset(ht, 0, symtab_size);
+
+  u32 mask = data.ht_size - 1;
+  for (Entry *ent : entries) {
+    u32 hash = ent->value.gdb_hash;
+
+    // This probing sequence is part of the .gdb_index format. The table size
+    // is a power of two, so an odd step visits every slot.
+    u32 step = ((hash * 17) & mask) | 1;
+    u32 i = hash & mask;
+
+    while (ht[i * 2] || ht[i * 2 + 1])
+      i = (i + step) & mask;
+
+    ht[i * 2] = ent->value.name_offset;
+    ht[i * 2 + 1] = ent->value.type_offset;
+  }
+
+  u8 *base = data.tables.get() + symtab_size;
+
+  // Each occurrence of a name contributes one value to its type vector. Each
+  // occurrence was assigned a distinct slot while the names were interned, so
+  // the vectors can be filled in parallel. The high byte is the name's type
+  // and the low 24 bits are the CU number.
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    i64 i = &cu - cus.data();
+    for (NameRecord &record : cu.names) {
+      IndexedName &name = record.indexed;
+      MapValue *ent = name.entry;
+      ul32 *p = (ul32 *)(base + ent->type_offset);
+      p[name.type_idx] = (name.type << 24) | i;
+    }
+  });
+
+  // Prefix each type vector with its length and sort it for deterministic
+  // output. Store the NUL-terminated name at its assigned string-pool offset.
+  tbb::enumerable_thread_specific<std::vector<ul32>> scratch;
+  tbb::parallel_for_each(entries, [&](Entry *ent) {
+    ul32 *p = (ul32 *)(base + ent->value.type_offset);
+    p[0] = ent->value.count;
+    std::span<ul32> values(p + 1, ent->value.count);
+
+    if (values.size() < 256)
+      ranges::sort(values);
+    else
+      radix_sort(values, scratch.local());
+
+    u8 *dst = base + ent->value.name_offset;
+    memcpy(dst, ent->key, ent->keylen);
+    dst[ent->keylen] = '\0';
+  });
+
+  // The serialized tables contain everything needed from names and the map.
+  // Release their storage here so reclamation remains part of this background
+  // phase rather than delaying the final output path.
+  tbb::parallel_for_each(cus, [](Compunit &cu) {
+    std::vector<NameRecord>().swap(cu.names);
+  });
+  std::vector<GdbNameMap::Entry *>().swap(data.entries);
+  data.map.reset();
+}
+
+// Read relocated address ranges and serialize the index prepared above.
 template <typename E>
 void write_gdb_index(Context<E> &ctx) {
   Timer t(ctx, "write_gdb_index");
 
+  std::shared_ptr<GdbIndexData> owner = std::move(ctx.gdb_index_data);
+  GdbIndexData &data = *owner;
+
+  if (data.cus.empty())
+    return;
+
   // Find debug info sections
   for (Chunk<E> *chunk : ctx.chunks) {
     std::string_view name = chunk->name;
@@ -652,41 +948,11 @@ void write_gdb_index(Context<E> &ctx) {
       ctx.debug_rnglists = get_buffer(ctx, chunk);
   }
 
-  if (ctx.debug_info.empty())
-    return;
+  std::vector<Compunit> &cus = data.cus;
 
-  // Read debug info
-  std::vector<Compunit> cus = read_compunits(ctx);
+  read_address_ranges(ctx, cus);
 
-  // Uniquify symbols
-  HyperLogLog estimator;
-
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    HyperLogLog e;
-    for (NameType &nt : cu.nametypes)
-      e.insert(nt.hash);
-    estimator.merge(e);
-  });
-
-  ConcurrentMap<MapValue> map(estimator.get_cardinality() * 3 / 2);
-
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    cu.entries.reserve(cu.nametypes.size());
-    for (NameType &nt : cu.nametypes) {
-      MapValue *ent;
-      bool inserted;
-      std::tie(ent, inserted) = map.insert(nt.name, nt.hash,
-                                           MapValue{gdb_hash(nt.name)});
-      ent->count++;
-      cu.entries.push_back(ent);
-    }
-  });
-
-  // Sort symbols for build reproducibility
-  using Entry = typename decltype(map)::Entry;
-  std::vector<Entry *> entries = map.get_sorted_entries_all();
-
-  // Compute sizes of each components
+  // Compute sizes of each component
   SectionHeader hdr;
   hdr.cu_list_offset = sizeof(hdr);
   hdr.cu_types_offset = hdr.cu_list_offset + cus.size() * 16;
@@ -696,27 +962,13 @@ void write_gdb_index(Context<E> &ctx) {
   for (Compunit &cu : cus)
     hdr.symtab_offset += cu.ranges.size() * 20;
 
-  i64 ht_size = bit_ceil(entries.size() * 5 / 4 + 1);
+  i64 ht_size = data.ht_size;
   hdr.const_pool_offset = hdr.symtab_offset + ht_size * 8;
 
-  i64 offset = 0;
-  for (Entry *ent : entries) {
-    ent->value.type_offset = offset;
-    offset += ent->value.count * 4 + 4;
-  }
+  i64 bufsize = hdr.const_pool_offset + data.pool_size.type_bytes +
+                data.pool_size.name_bytes;
 
-  for (Entry *ent : entries) {
-    ent->value.name_offset = offset;
-    offset += ent->keylen + 1;
-  }
-
-  i64 bufsize = hdr.const_pool_offset + offset;
-
-  // Allocate an output buffer. We use malloc instead of vector to
-  // avoid zero-initializing the entire buffer.
-  ctx.output_file->buf2 = (u8 *)malloc(bufsize);
-  ctx.output_file->buf2_size = bufsize;
-  u8 *buf = ctx.output_file->buf2;
+  u8 *buf = ctx.output_file->extend(ctx, bufsize);
 
   // Write a section header
   memcpy(buf, &hdr, sizeof(hdr));
@@ -746,53 +998,9 @@ void write_gdb_index(Context<E> &ctx) {
     }
   });
 
-  // Write a symbol table
-  u32 mask = ht_size - 1;
-  ul32 *ht = (ul32 *)(buf + hdr.symtab_offset);
-  memset(ht, 0, ht_size * 8);
-
-  for (Entry *ent : entries) {
-    u32 hash = ent->value.gdb_hash;
-    u32 step = ((hash * 17) & mask) | 1;
-    u32 j = hash & mask;
-
-    while (ht[j * 2] || ht[j * 2 + 1])
-      j = (j + step) & mask;
-
-    ht[j * 2] = ent->value.name_offset;
-    ht[j * 2 + 1] = ent->value.type_offset;
-  }
-
-  // Write types. Use MapValue::count as an atomic slot counter.
-  u8 *base = buf + hdr.const_pool_offset;
-
-  for (Entry *ent : entries)
-    ent->value.count = 0;
-
-  tbb::parallel_for_each(cus, [&](Compunit &cu) {
-    i64 i = &cu - cus.data();
-    for (i64 j = 0; j < cu.nametypes.size(); j++) {
-      MapValue *ent = cu.entries[j];
-      ul32 *p = (ul32 *)(base + ent->type_offset);
-      i64 idx = ++ent->count;
-      p[idx] = (cu.nametypes[j].type << 24) | i;
-    }
-  });
-
-  // Write the final counts into the buffer.
-  tbb::parallel_for_each(entries, [&](Entry *ent) {
-    ul32 *p = (ul32 *)(base + ent->value.type_offset);
-    p[0] = ent->value.count;
-    // Sort entries for deterministic output
-    std::sort(p + 1, p + 1 + ent->value.count);
-  });
-
-  // Write names
-  tbb::parallel_for_each(entries, [&](Entry *ent) {
-    u8 *dst = buf + hdr.const_pool_offset + ent->value.name_offset;
-    memcpy(dst, ent->key, ent->keylen);
-    dst[ent->keylen] = '\0';
-  });
+  i64 tables_size = ht_size * 8 + data.pool_size.type_bytes +
+                    data.pool_size.name_bytes;
+  parallel_memcpy(buf + hdr.symtab_offset, data.tables.get(), tables_size);
 
   // Update the section size and rewrite the section header
   if (ctx.shdr) {
@@ -803,6 +1011,8 @@ void write_gdb_index(Context<E> &ctx) {
 
 using E = MOLD_TARGET;
 
+template void read_gdb_index_inputs(Context<E> &);
+template void build_gdb_index_tables(Context<E> &);
 template void write_gdb_index(Context<E> &);
 
 } // namespace mold

src/icf.cc

@@ -67,8 +67,6 @@
 #include "mold.h"
 #include "../lib/siphash.h"
 
-#include <array>
-#include <cstdio>
 #include <fstream>
 #include <tbb/concurrent_unordered_map.h>
 #include <tbb/concurrent_vector.h>
@@ -77,24 +75,14 @@
 #include <tbb/parallel_for_each.h>
 #include <tbb/parallel_sort.h>
 
-static constexpr int64_t HASH_SIZE = 16;
-
-using Digest = std::array<uint8_t, HASH_SIZE>;
-
-namespace std {
-template <> struct hash<Digest> {
-  size_t operator()(const Digest &k) const {
-    static_assert(sizeof(size_t) <= HASH_SIZE);
-    size_t val;
-    memcpy(&val, k.data(), sizeof(size_t));
-    return val;
-  }
-};
-}
-
 namespace mold {
 
-static u8 hmac_key[16];
+struct Digest {
+  u64 hi;
+  u64 lo;
+};
+
+static u8 siphash_key[16];
 
 template <typename E>
 static void uniquify_cies(Context<E> &ctx) {
@@ -123,125 +111,32 @@ static void uniquify_cies(Context<E> &ctx) {
 template <typename E>
 static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
   const ElfShdr<E> &shdr = isec.shdr();
+  std::string_view name = isec.name();
 
   if (shdr.sh_size == 0 || !(shdr.sh_flags & SHF_ALLOC) ||
-      shdr.sh_type == SHT_NOBITS || is_c_identifier(isec.name))
+      shdr.sh_type == SHT_NOBITS || is_c_identifier(name))
     return false;
 
   if (shdr.sh_flags & SHF_EXECINSTR)
     return (ctx.arg.icf_all || !isec.address_taken) &&
-           isec.name != ".init" && isec.name != ".fini";
+           name != ".init" && name != ".fini";
 
   // .gcc_except_table contains a compiler-generated table. Pointer
   // equality for the section is not significant because only the C++
   // exception handling code will use the table at runtime.
-  if (isec.name == ".gcc_except_table" ||
-      isec.name.starts_with(".gcc_except_table."))
+  if (name == ".gcc_except_table" ||
+      name.starts_with(".gcc_except_table."))
     return true;
 
   bool is_readonly = !(shdr.sh_flags & SHF_WRITE);
-  bool is_relro = isec.name.starts_with(".data.rel.ro");
+  bool is_relro = name.starts_with(".data.rel.ro");
   return (ctx.arg.ignore_data_address_equality || !isec.address_taken) &&
          (is_readonly || is_relro);
 }
 
-template <typename E>
-static bool is_leaf(Context<E> &ctx, InputSection<E> &isec) {
-  if (!isec.get_rels(ctx).empty())
-    return false;
-
-  for (FdeRecord<E> &fde : isec.get_fdes())
-    if (fde.get_rels(isec.file).size() > 1)
-      return false;
-
-  return true;
-}
-
-template <typename E>
-struct LeafHasher {
-  size_t operator()(InputSection<E> *isec) const {
-    u64 h = hash_string(isec->contents);
-    for (FdeRecord<E> &fde : isec->get_fdes()) {
-      u64 h2 = hash_string(fde.get_contents(isec->file).substr(8));
-      h = combine_hash(h, h2);
-    }
-    return h;
-  }
-};
-
-template <typename E>
-struct LeafEq {
-  bool operator()(InputSection<E> *a, InputSection<E> *b) const {
-    if (a->contents != b->contents)
-      return false;
-
-    std::span<FdeRecord<E>> x = a->get_fdes();
-    std::span<FdeRecord<E>> y = b->get_fdes();
-
-    if (x.size() != y.size())
-      return false;
-
-    for (i64 i = 0; i < x.size(); i++)
-      if (x[i].get_contents(a->file).substr(8) !=
-          y[i].get_contents(b->file).substr(8))
-        return false;
-    return true;
-  }
-};
-
-// Early merge of leaf nodes, which can be processed without constructing the
-// entire graph. This reduces the vertex count and improves memory efficiency.
-template <typename E>
-static void merge_leaf_nodes(Context<E> &ctx) {
-  Timer t(ctx, "merge_leaf_nodes");
-
-  static Counter eligible("icf_eligibles");
-  static Counter non_eligible("icf_non_eligibles");
-  static Counter leaf("icf_leaf_nodes");
-
-  tbb::concurrent_unordered_map<InputSection<E> *, Atomic<InputSection<E> *>,
-                                LeafHasher<E>, LeafEq<E>> map;
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (!isec || !isec->is_alive)
-        continue;
-
-      if (!is_eligible(ctx, *isec)) {
-        non_eligible++;
-        continue;
-      }
-
-      if (is_leaf(ctx, *isec)) {
-        leaf++;
-        isec->icf_leaf = true;
-        auto [it, inserted] = map.insert({isec.get(), isec.get()});
-        if (!inserted) {
-          InputSection<E> *isec2 = it->second.load();
-          while (isec->get_priority() < isec2->get_priority() &&
-                 !it->second.compare_exchange_strong(isec2, isec.get()));
-        }
-      } else {
-        eligible++;
-        isec->icf_eligible = true;
-      }
-    }
-  });
-
-  tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections) {
-      if (isec && isec->is_alive && isec->icf_leaf) {
-        auto it = map.find(isec.get());
-        assert(it != map.end());
-        isec->leader = it->second;
-      }
-    }
-  });
-}
-
 template <typename E>
 static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
-  SipHash13_128 hasher(hmac_key);
+  SipHash13_128 hasher(siphash_key);
 
   auto hash = [&](auto val) {
     hasher.update((u8 *)&val, sizeof(val));
@@ -263,19 +158,16 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
       hash((u64)frag);
     } else if (!isec) {
       hash('3');
-    } else if (isec->leader) {
-      hash('4');
-      hash((u64)isec->leader);
     } else if (isec->icf_eligible) {
-      hash('5');
+      hash('4');
     } else {
-      hash('6');
+      hash('5');
       hash((u64)isec);
     }
     hash(sym.value);
   };
 
-  hash_string(isec.contents);
+  hash_string(isec.get_contents());
   hash(isec.shdr().sh_flags);
   hash(isec.get_fdes().size());
   hash(isec.get_rels(ctx).size());
@@ -297,8 +189,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
     }
   }
 
-  for (i64 i = 0; i < isec.get_rels(ctx).size(); i++) {
-    const ElfRel<E> &rel = isec.get_rels(ctx)[i];
+  for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
     hash(rel.r_offset);
     hash(rel.r_type);
     hash(get_addend(isec, rel));
@@ -306,40 +197,163 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
   }
 
   Digest digest;
-  hasher.finish(digest.data());
+  hasher.finish(&digest);
   return digest;
 }
 
+// A concurrent hash map from digest to section. We use it to count the
+// number of distinct digests and to elect the leader section of each
+// digest's equivalence class.
+//
+// The number of distinct digests cannot exceed the number of digests,
+// so we can allocate a large enough table upfront and never need to
+// grow it.
+//
+// The map is reused across propagation rounds. Instead of clearing the
+// table at the start of each round, we stamp each slot with the round
+// number in which it was written; a slot stamped with an earlier round
+// is treated as vacant.
+//
+// Each slot consists of two 64-bit words and a section pointer. The
+// first word packs the round number, a busy bit, and 48 bits of the
+// digest; the second word holds another 64 bits. An inserter claims a
+// vacant slot by installing the first word with compare-and-swap with
+// the busy bit set, writes the second word and the pointer, and then
+// rewrites the first word with the busy bit cleared to publish the
+// slot. Since the slot index is derived from digest bits that the
+// first word doesn't contain, a successful match effectively compares
+// an entire 128-bit digest, so the map is exact under the same
+// hash-collision assumption the surrounding algorithm is built on.
+//
+// Of all sections inserted with the same digest in the same round, the
+// slot ends up pointing to the one with the lowest priority, which ICF
+// uses as the leader of the digest's equivalence class.
+template <typename E>
+class DigestMap {
+public:
+  DigestMap(i64 n) :
+    mask(bit_ceil(n * 2) - 1),
+    slots(mask + 1) {}
+
+  void next_round() {
+    if (++round == 1 << 15) {
+      // The round number wrapped around, making stale slot stamps
+      // ambiguous, so reset the table. In practice, ICF converges long
+      // before this point.
+      for (Slot &slot : slots)
+        slot.hi = 0;
+      round = 1;
+    }
+  }
+
+  // Returns true if the digest was not in the table.
+  bool insert(const Digest &digest, InputSection<E> *isec) {
+    constexpr u64 busy_bit = 1LL << 48;
+    u64 tag = digest.hi >> 16;
+    u64 value = (round << 49) | tag;
+
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask) {
+      Slot &slot = slots[i];
+      u64 x = slot.hi.load(std::memory_order_acquire);
+
+      // If the slot was last written in an earlier round, it's vacant;
+      // try to claim it.
+      while ((x >> 49) != round) {
+        if (slot.hi.compare_exchange_weak(x, value | busy_bit,
+                                          std::memory_order_acquire)) {
+          slot.lo = digest.lo;
+          slot.leader = isec;
+          slot.hi.store(value, std::memory_order_release);
+          return true;
+        }
+      }
+
+      // The slot is occupied. If it holds a different digest, try the
+      // next slot.
+      if ((x & 0xffff'ffff'ffff) != tag)
+        continue;
+
+      // The tags match; compare the digest bits in the second word,
+      // waiting for the writer to publish them if the slot is still
+      // being claimed.
+      while (x & busy_bit)
+        x = slot.hi.load(std::memory_order_acquire);
+      if (slot.lo != digest.lo)
+        continue;
+
+      // The digest is already in the table; keep the slot pointing to
+      // the lowest-priority section.
+      InputSection<E> *cur = slot.leader;
+      while (isec->get_priority() < cur->get_priority() &&
+             !slot.leader.compare_exchange_strong(cur, isec));
+      return false;
+    }
+  }
+
+  // Returns the section associated with a digest. The digest must have
+  // been inserted in the current round, and no insertion may be running
+  // concurrently.
+  InputSection<E> *find(const Digest &digest) {
+    u64 value = (round << 49) | (digest.hi >> 16);
+    for (i64 i = digest.hi & mask;; i = (i + 1) & mask)
+      if (slots[i].hi == value && slots[i].lo == digest.lo)
+        return slots[i].leader;
+  }
+
+private:
+  struct Slot {
+    Atomic<u64> hi;
+    Atomic<u64> lo;
+    Atomic<InputSection<E> *> leader;
+  };
+
+  // Time begins in round 1 so that all-zero slots, the initial state
+  // of the table, read as vacant.
+  u64 round = 1;
+  u64 mask;
+  std::vector<Slot> slots;
+};
+
 template <typename E>
 static std::vector<InputSection<E> *> gather_sections(Context<E> &ctx) {
   Timer t(ctx, "gather_sections");
 
-  // Count the number of input sections for each input file.
-  std::vector<i64> num_sections(ctx.objs.size());
+  static Counter eligible("icf_eligibles");
+  static Counter non_eligible("icf_non_eligibles");
+
+  // Count the number of eligible input sections for each input file
+  // and turn the counts into starting indices with a prefix sum.
+  std::vector<i64> indices(ctx.objs.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        num_sections[i]++;
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (!isec || !isec->is_alive)
+        continue;
+
+      if (is_eligible(ctx, *isec)) {
+        eligible++;
+        isec->icf_eligible = true;
+        indices[i + 1]++;
+      } else {
+        non_eligible++;
+      }
+    }
   });
 
-  std::vector<i64> section_indices(ctx.objs.size());
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
-    section_indices[i + 1] = section_indices[i] + num_sections[i];
+  for (i64 i = 1; i < indices.size(); i++)
+    indices[i] += indices[i - 1];
 
-  std::vector<InputSection<E> *> sections(
-    section_indices.back() + num_sections.back());
+  std::vector<InputSection<E> *> sections(indices.back());
 
   // Fill `sections` contents.
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    i64 idx = section_indices[i];
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
-      if (isec && isec->is_alive && isec->icf_eligible)
-        sections[idx++] = isec.get();
-  });
-
-  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-    sections[i]->icf_idx = i;
+    i64 idx = indices[i];
+    for (InputSection<E> *isec : ctx.objs[i]->sections) {
+      if (isec && isec->is_alive && isec->icf_eligible) {
+        isec->icf_idx = idx;
+        sections[idx++] = isec;
+      }
+    }
   });
 
   return sections;
@@ -360,6 +374,13 @@ compute_digests(Context<E> &ctx, std::span<InputSection<E> *> sections) {
 // Build a graph, treating every function as a vertex and every function call
 // as an edge. See the description at the top for a more detailed formulation.
 // We use u32 indices here to improve cache locality.
+//
+// Relocations in a section's FDEs are edges too, because compute_digest
+// hashes eligible relocation targets without identity, and every such
+// target must be represented as an edge to remain distinguishable. In
+// particular, an FDE's reference to an LSDA is an edge; without it, two
+// identical functions whose exception tables catch different types would
+// be folded into one.
 template <typename E>
 static void gather_edges(Context<E> &ctx,
                          std::span<InputSection<E> *> sections,
@@ -367,94 +388,87 @@ static void gather_edges(Context<E> &ctx,
                          std::vector<u32> &edge_indices) {
   Timer t(ctx, "gather_edges");
 
-  if (sections.empty())
-    return;
-
-  std::vector<i64> num_edges(sections.size());
-  edge_indices.resize(sections.size());
+  // Count the number of outgoing edges for each vertex and turn the
+  // counts into starting indices with a prefix sum. The extra entry at
+  // the end makes edge_indices[i + 1] valid for every vertex, so that
+  // vertex i's edges are edge_indices[i] to edge_indices[i + 1].
+  edge_indices.resize(sections.size() + 1);
 
   tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
     assert(isec.icf_eligible);
 
-    for (i64 j = 0; j < isec.get_rels(ctx).size(); j++) {
-      const ElfRel<E> &rel = isec.get_rels(ctx)[j];
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (!sym.get_frag())
-        if (InputSection<E> *isec = sym.get_input_section())
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
           if (isec->icf_eligible)
-            num_edges[i]++;
-    }
+            edge_indices[i + 1]++;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
+        if (isec->icf_eligible)
+          edge_indices[i + 1]++;
   });
 
-  for (i64 i = 0; i < num_edges.size() - 1; i++)
-    edge_indices[i + 1] = edge_indices[i] + num_edges[i];
+  for (i64 i = 1; i < edge_indices.size(); i++)
+    edge_indices[i] += edge_indices[i - 1];
 
-  edges.resize(edge_indices.back() + num_edges.back());
+  edges.resize(edge_indices.back());
 
-  tbb::parallel_for((i64)0, (i64)num_edges.size(), [&](i64 i) {
+  tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
     InputSection<E> &isec = *sections[i];
     i64 idx = edge_indices[i];
 
-    for (ElfRel<E> &rel : isec.get_rels(ctx)) {
-      Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-      if (InputSection<E> *isec = sym.get_input_section())
+    for (FdeRecord<E> &fde : isec.get_fdes())
+      for (const ElfRel<E> &rel : fde.get_rels(isec.file).subspan(1))
+        if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+            InputSection<E> *isec = sym.get_input_section())
+          if (isec->icf_eligible)
+            edges[idx++] = isec->icf_idx;
+
+    for (const ElfRel<E> &rel : isec.get_rels(ctx))
+      if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
+          InputSection<E> *isec = sym.get_input_section())
         if (isec->icf_eligible)
           edges[idx++] = isec->icf_idx;
-  }
   });
 }
 
-template <typename E>
-static i64 propagate(std::span<std::vector<Digest>> digests,
-                     std::span<u32> edges, std::span<u32> edge_indices,
-                     bool &slot, std::span<u8> converged,
-                     tbb::affinity_partitioner &ap) {
-  static Counter round("icf_round");
-  round++;
-
-  i64 num_digests = digests[0].size();
-  tbb::enumerable_thread_specific<i64> changed;
-
-  tbb::parallel_for((i64)0, num_digests, [&](i64 i) {
-    if (converged[i])
-      return;
-
-    SipHash13_128 hasher(hmac_key);
-    hasher.update(digests[2][i].data(), HASH_SIZE);
+// Compute the next-round digest of each vertex by hashing its current
+// digest and the current digests of the vertices it refers to. A
+// vertex's digest after the nth round is therefore a hash of its
+// unfolding into a tree of depth n.
+static void propagate(std::span<Digest> cur, std::span<Digest> next,
+                      std::span<u32> edges, std::span<u32> edge_indices) {
+  tbb::parallel_for((i64)0, (i64)cur.size(), [&](i64 i) {
+    SipHash13_128 hasher(siphash_key);
+    hasher.update(&cur[i], sizeof(Digest));
 
     i64 begin = edge_indices[i];
-    i64 end = (i + 1 == num_digests) ? edges.size() : edge_indices[i + 1];
-
+    i64 end = edge_indices[i + 1];
     for (i64 j : edges.subspan(begin, end - begin))
-      hasher.update(digests[slot][j].data(), HASH_SIZE);
+      hasher.update(&cur[j], sizeof(Digest));
 
-    hasher.finish(digests[!slot][i].data());
+    hasher.finish(&next[i]);
+  });
 
-    if (digests[slot][i] == digests[!slot][i]) {
-      // This node has converged. Skip further iterations as it will
-      // yield the same hash.
-      converged[i] = true;
-    } else {
-      changed.local()++;
-    }
-  }, ap);
-
-  slot = !slot;
-  return changed.combine(std::plus());
+  static Counter counter("icf_round");
+  counter++;
 }
 
 template <typename E>
 static i64 count_num_classes(std::span<Digest> digests,
-                             tbb::affinity_partitioner &ap) {
-  std::vector<Digest> vec(digests.begin(), digests.end());
-  tbb::parallel_sort(vec);
+                             std::span<InputSection<E> *> sections,
+                             DigestMap<E> &map) {
+  map.next_round();
 
   tbb::enumerable_thread_specific<i64> num_classes;
-  tbb::parallel_for((i64)0, (i64)vec.size() - 1, [&](i64 i) {
-    if (vec[i] != vec[i + 1])
+  tbb::parallel_for((i64)0, (i64)digests.size(), [&](i64 i) {
+    if (map.insert(digests[i], sections[i]))
       num_classes.local()++;
-  }, ap);
+  });
   return num_classes.combine(std::plus());
 }
 
@@ -464,12 +478,12 @@ static void print_icf_sections(Context<E> &ctx) {
   tbb::concurrent_unordered_multimap<InputSection<E> *, InputSection<E> *> map;
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (isec && isec->is_alive && isec->leader) {
-        if (isec.get() == isec->leader)
-          leaders.push_back(isec.get());
+        if (isec == isec->leader)
+          leaders.push_back(isec);
         else
-          map.insert({isec->leader, isec.get()});
+          map.insert({isec->leader, isec});
       }
     }
   });
@@ -499,7 +513,7 @@ static void print_icf_sections(Context<E> &ctx) {
       *out << "selected section " << *leader << '\n';
       for (auto it = begin; it != end; it++) {
         *out << "  removing identical section " << *it->second << '\n';
-        saved_bytes += leader->contents.size();
+        saved_bytes += leader->get_contents().size();
       }
     }
   }
@@ -513,102 +527,62 @@ void icf_sections(Context<E> &ctx) {
   if (ctx.objs.empty())
     return;
 
-  get_random_bytes(hmac_key, sizeof(hmac_key));
+  get_random_bytes(siphash_key, sizeof(siphash_key));
 
   uniquify_cies(ctx);
-  merge_leaf_nodes(ctx);
 
   // Prepare for the propagation rounds.
   std::vector<InputSection<E> *> sections = gather_sections(ctx);
 
-  // We allocate 3 arrays to store hashes for each vertex.
-  //
-  // Index 0 and 1 are used for tree hashes from the previous
-  // iteration and the current iteration. They switch roles every
-  // iteration. See `slot` below.
-  //
-  // Index 2 stores the initial, single-vertex hash. This is combined
-  // with hashes from the connected vertices to form the tree hash
-  // described above.
-  std::vector<std::vector<Digest>> digests(3);
-  digests[0] = compute_digests<E>(ctx, sections);
-  digests[1].resize(digests[0].size());
-  digests[2] = digests[0];
+  // `digests` holds the current digest of each vertex; `scratch` is
+  // where a propagation round writes the next digests before the two
+  // vectors swap roles.
+  std::vector<Digest> digests = compute_digests<E>(ctx, sections);
+  std::vector<Digest> scratch(digests.size());
 
   std::vector<u32> edges;
   std::vector<u32> edge_indices;
   gather_edges<E>(ctx, sections, edges, edge_indices);
 
-  std::vector<u8> converged(digests[0].size());
-  bool slot = 0;
+  // The digest map is used to count the number of distinct digests in
+  // the loop below. As a side effect, it records the lowest-priority
+  // section for each digest, which the grouping step after the loop
+  // uses as the leader of each equivalence class.
+  DigestMap<E> map(digests.size());
 
   // Execute the propagation rounds until convergence is obtained.
+  //
+  // The number of distinct digests can only monotonically increase as
+  // the rounds hash ever-deeper trees, so once two consecutive rounds
+  // yield the same count, the partition of sections into equivalence
+  // classes has stopped changing and will remain unchanged for further
+  // iterations (proof omitted for brevity). Note that individual
+  // digests may well still be changing at that point; sections that
+  // have a cycle in downstream (i.e. recursive functions and functions
+  // that call them) never settle on a digest. That doesn't matter
+  // because sections in the same class change their digests in
+  // lockstep, keeping the partition intact.
   {
     Timer t(ctx, "propagate");
-    tbb::affinity_partitioner ap;
-
-    // A cheap test that the graph hasn't converged yet.
-    // The loop after this one uses a strict condition, but it's expensive
-    // as it requires sorting the entire hash collection.
-    //
-    // For nodes that have a cycle in downstream (i.e. recursive
-    // functions and functions that calls recursive functions) will always
-    // change with the iterations. Nodes that doesn't (i.e. non-recursive
-    // functions) will stop changing as soon as the propagation depth reaches
-    // the call tree depth.
-    // Here, we test whether we have reached sufficient depth for the latter,
-    // which is a necessary (but not sufficient) condition for convergence.
-    i64 num_changed = -1;
-    for (;;) {
-      i64 n = propagate<E>(digests, edges, edge_indices, slot, converged, ap);
-      if (n == num_changed)
-        break;
-      num_changed = n;
-    }
-
-    // Run the pass until the unique number of hashes stop increasing, at which
-    // point we have achieved convergence (proof omitted for brevity).
     i64 num_classes = -1;
-    for (;;) {
-      // count_num_classes requires sorting which is O(n log n), so do a little
-      // more work beforehand to amortize that log factor.
-      for (i64 i = 0; i < 10; i++)
-        propagate<E>(digests, edges, edge_indices, slot, converged, ap);
 
-      i64 n = count_num_classes<E>(digests[slot], ap);
-      if (n == num_classes)
+    for (;;) {
+      propagate(digests, scratch, edges, edge_indices);
+      std::swap(digests, scratch);
+      i64 m = count_num_classes<E>(digests, sections, map);
+      if (m == num_classes)
         break;
-      num_classes = n;
+      num_classes = m;
     }
   }
 
-  // Group sections by hash values.
+  // Group sections by digest. The final counting round has already
+  // elected a leader for each digest; look it up.
   {
     Timer t(ctx, "group");
-
-    auto *map =
-      new tbb::concurrent_unordered_map<Digest, Atomic<InputSection<E> *>>;
-
-    std::span<Digest> digest = digests[slot];
-
     tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      InputSection<E> *isec = sections[i];
-      auto [it, inserted] = map->insert({digest[i], isec});
-      if (!inserted) {
-        InputSection<E> *isec2 = it->second.load();
-        while (isec->get_priority() < isec2->get_priority() &&
-               !it->second.compare_exchange_strong(isec2, isec));
-      }
+      sections[i]->leader = map.find(digests[i]);
     });
-
-    tbb::parallel_for((i64)0, (i64)sections.size(), [&](i64 i) {
-      auto it = map->find(digest[i]);
-      assert(it != map->end());
-      sections[i]->leader = it->second;
-    });
-
-    // Since free'ing the map is slow, postpone it.
-    ctx.on_exit.push_back([=] { delete map; });
   }
 
   if (!ctx.arg.print_icf_sections.empty())
@@ -618,11 +592,9 @@ void icf_sections(Context<E> &ctx) {
   {
     Timer t(ctx, "update_alignment");
     tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-      for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-        if (isec && isec->is_alive && isec->leader && isec->leader != isec.get()) {
+      for (InputSection<E> *isec : file->sections)
+        if (isec && isec->is_alive && isec->icf_removed())
           update_maximum(isec->leader->p2align, isec->p2align);
-        }
-      }
     });
   }
 
@@ -633,7 +605,7 @@ void icf_sections(Context<E> &ctx) {
     Timer t(ctx, "sweep");
     static Counter eliminated("icf_eliminated");
     tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-      for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+      for (InputSection<E> *isec : file->sections) {
         if (isec && isec->is_alive && isec->icf_removed()) {
           isec->kill();
           eliminated++;

src/input-files.cc

@@ -10,15 +10,28 @@
 
 namespace mold {
 
+template <typename E>
+static std::span<Symbol<E>>
+allocate_symbols(Context<E> &ctx, i64 size) {
+  if (size == 0)
+    return {};
+
+  Symbol<E> *data = ctx.arena.template allocate<Symbol<E>>(size);
+  std::uninitialized_value_construct_n(data, size);
+  return std::span<Symbol<E>>(data, size);
+}
+
 // If we haven't seen the same `key` before, create a new instance
 // of Symbol and returns it. Otherwise, returns the previously-
 // instantiated object. `key` is usually the same as `name`.
 template <typename E>
 Symbol<E> *get_symbol(Context<E> &ctx, std::string_view key,
                       std::string_view name) {
-  typename decltype(ctx.symbol_map)::const_accessor acc;
-  ctx.symbol_map.insert(acc, {key, Symbol<E>(name, ctx.arg.demangle)});
-  return const_cast<Symbol<E> *>(&acc->second);
+  return ctx.symbol_map.insert(key, [&](std::string_view, Symbol<E> &sym) {
+    sym.has_map_name = true;
+    sym.set_name(name);
+    sym.demangle = ctx.arg.demangle;
+  });
 }
 
 template <typename E>
@@ -33,7 +46,7 @@ static bool is_rust_symbol(const Symbol<E> &sym) {
   // We don't want to accidentally demangle C++ symbols as Rust ones.
   // So, the legacy mangling scheme will be demangled only when we
   // know the object file was created by rustc.
-  if (sym.file && !sym.file->is_dso && ((ObjectFile<E> *)sym.file)->is_rust_obj)
+  if (sym.file && !sym.file->is_dso && sym.file->to_obj()->is_rust_obj)
     return true;
 
   // "_R" is the prefix of the new Rust mangling scheme.
@@ -63,7 +76,8 @@ std::ostream &operator<<(std::ostream &out, const Symbol<E> &sym) {
 
 template <typename E>
 InputFile<E>::InputFile(Context<E> &ctx, MappedFile *mf)
-  : mf(mf), filename(mf->name) {
+  : mf(mf), symbols(ArenaAllocator<ArenaPtr<Symbol<E>>>(ctx.arena)),
+    filename(mf->name) {
   if (mf->size < sizeof(ElfEhdr<E>))
     Fatal(ctx) << *this << ": file too small";
   if (memcmp(mf->data, "\177ELF", 4))
@@ -94,11 +108,16 @@ InputFile<E>::InputFile(Context<E> &ctx, MappedFile *mf)
 }
 
 template <typename E>
-void InputFile<E>::populate_symbol_names() {
-  symbol_names.reserve(elf_syms.size());
+InputFile<E>::InputFile(Context<E> &ctx)
+  : symbols(ArenaAllocator<ArenaPtr<Symbol<E>>>(ctx.arena)),
+    filename("<internal>") {}
+
+template <typename E>
+void InputFile<E>::populate_symbol_name_lengths() {
+  symname_lens.reserve(elf_syms.size());
   for (const ElfSym<E> &esym : elf_syms) {
     const char *p = symbol_strtab.data() + esym.st_name;
-    symbol_names.emplace_back(p, strlen(p));
+    symname_lens.emplace_back(strlen(p));
   }
 }
 
@@ -245,57 +264,67 @@ static bool is_known_section_type(const ElfShdr<E> &shdr) {
 //
 // This function converts a CREL relocation table to a regular one.
 template <typename E>
-std::vector<ElfRel<E>> decode_crel(Context<E> &ctx, ObjectFile<E> &file,
-                                   const ElfShdr<E> &shdr) {
-  u8 *p = (u8 *)file.get_string(ctx, shdr).data();
-  u64 hdr = read_uleb(&p);
-  i64 nrels = hdr >> 3;
-  bool is_rela = hdr & 0b100;
-  i64 scale = hdr & 0b11;
-
-  if (is_rela && !E::is_rela)
-    Fatal(ctx) << file << ": CREL with addends is not supported for " << E::name;
-
-  u64 offset = 0;
-  i64 type = 0;
-  i64 symidx = 0;
-  i64 addend = 0;
-
-  std::vector<ElfRel<E>> vec;
-  vec.reserve(nrels);
-
-  while (vec.size() < nrels) {
-    u8 flags = *p++;
-    i64 nflags = is_rela ? 3 : 2;
-
-    // The first ULEB-128 encoded value is a concatenation of bit flags and
-    // an offset delta. The delta may be very large to decrease the
-    // current offset value by wrapping around. Combined, the encoded value
-    // can be up to 67 bit long. Thus we can't simply use read_uleb() which
-    // returns a u64.
-    u64 delta;
-    if (flags & 0x80)
-      delta = (read_uleb(&p) << (7 - nflags)) | ((flags & 0x7f) >> nflags);
-    else
-      delta = flags >> nflags;
-    offset += delta << scale;
-
-    if (flags & 1)
-      symidx += read_sleb(&p);
-    if (flags & 2)
-      type += read_sleb(&p);
-    if (is_rela && (flags & 4))
-      addend += read_sleb(&p);
-    vec.emplace_back(offset, type, symidx, addend);
-  }
-  return vec;
+ExactArray<ElfRel<E>> decode_crel(Context<E> &ctx, ObjectFile<E> &file,
+                                  const ElfShdr<E> &shdr) {
+  CrelReader<E> reader(ctx, file, shdr);
+  ExactArray<ElfRel<E>> rels(reader.size());
+  reader.for_each([&](const ElfRel<E> &rel, i64 i) { rels[i] = rel; });
+  return rels;
 }
 
+// Read COMDAT groups and detect GCC offload objects. Both affect which input
+// sections can be discarded before LTO.
 template <typename E>
-ComdatGroup *insert_comdat_group(Context<E> &ctx, std::string_view name) {
-  typename decltype(ctx.comdat_groups)::const_accessor acc;
-  ctx.comdat_groups.insert(acc, {name, ComdatGroup()});
-  return const_cast<ComdatGroup *>(&acc->second);
+void ObjectFile<E>::read_section_metadata(Context<E> &ctx) {
+  assert(!sections_parsed);
+
+  for (i64 i = 0; i < this->elf_sections.size(); i++) {
+    const ElfShdr<E> &shdr = this->elf_sections[i];
+
+    if (shdr.sh_flags & SHF_EXCLUDE) {
+      std::string_view name = this->shstrtab.data() + shdr.sh_name;
+      if (name.starts_with(".gnu.offload_lto_.symtab."))
+        this->is_gcc_offload_obj = true;
+    }
+
+    if (shdr.sh_type != SHT_GROUP)
+      continue;
+
+    if (shdr.sh_info >= this->elf_syms.size())
+      Fatal(ctx) << *this << ": invalid symbol index";
+
+    const ElfSym<E> &esym = this->elf_syms[shdr.sh_info];
+    std::string_view name;
+    if (esym.st_type == STT_SECTION)
+      name = this->shstrtab.data() +
+             this->elf_sections[get_shndx(esym)].sh_name;
+    else
+      name = this->get_symbol_name(shdr.sh_info);
+
+    // Ignore a broken comdat group GCC emits for .debug_macros.
+    // https://github.com/rui314/mold/issues/438
+    if (name.starts_with("wm4."))
+      continue;
+
+    std::span<U32<E>> entries = this->template get_data<U32<E>>(ctx, shdr);
+    if (entries.empty())
+      Fatal(ctx) << *this << ": empty SHT_GROUP";
+    if (entries[0] == 0)
+      continue;
+    if (entries[0] != GRP_COMDAT)
+      Fatal(ctx) << *this << ": unsupported SHT_GROUP format";
+
+    // Ordinary global signatures already have a Symbol. Local, section and
+    // versioned signatures use the same symbol table through their full name.
+    Symbol<E> *signature;
+    if (esym.st_type != STT_SECTION && esym.st_bind != STB_LOCAL &&
+        !esym.is_undef() && name.find('@') == name.npos)
+      signature = this->symbols[shdr.sh_info];
+    else
+      signature = get_symbol(ctx, name);
+
+    comdat_groups.emplace_back(ctx, signature, i);
+  }
 }
 
 template <typename E>
@@ -303,13 +332,14 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
   // Read sections
   for (i64 i = 0; i < this->elf_sections.size(); i++) {
     const ElfShdr<E> &shdr = this->elf_sections[i];
-    std::string_view name = this->shstrtab.data() + shdr.sh_name;
 
-    if ((shdr.sh_flags & SHF_EXCLUDE) &&
-        name.starts_with(".gnu.offload_lto_.symtab.")) {
-      this->is_gcc_offload_obj = true;
+    if (!comdat_discarded.empty() && comdat_discarded[i])
       continue;
-    }
+
+    std::string_view name;
+
+    if (shdr.sh_flags & SHF_EXCLUDE)
+      name = this->shstrtab.data() + shdr.sh_name;
 
     if ((shdr.sh_flags & SHF_EXCLUDE) && !(shdr.sh_flags & SHF_ALLOC) &&
         shdr.sh_type != SHT_LLVM_ADDRSIG && !ctx.arg.relocatable)
@@ -327,42 +357,13 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
     }
 
     switch (shdr.sh_type) {
-    case SHT_GROUP: {
-      // Get the signature of this section group.
-      if (shdr.sh_info >= this->elf_syms.size())
-        Fatal(ctx) << *this << ": invalid symbol index";
-      const ElfSym<E> &esym = this->elf_syms[shdr.sh_info];
-
-      std::string_view signature;
-      if (esym.st_type == STT_SECTION) {
-        signature = this->shstrtab.data() +
-                    this->elf_sections[get_shndx(esym)].sh_name;
-      } else {
-        signature = this->symbol_names[shdr.sh_info];
-      }
-
-      // Ignore a broken comdat group GCC emits for .debug_macros.
-      // https://github.com/rui314/mold/issues/438
-      if (signature.starts_with("wm4."))
-        continue;
-
-      // Get comdat group members.
-      std::span<U32<E>> entries = this->template get_data<U32<E>>(ctx, shdr);
-
-      if (entries.empty())
-        Fatal(ctx) << *this << ": empty SHT_GROUP";
-      if (entries[0] == 0)
-        continue;
-      if (entries[0] != GRP_COMDAT)
-        Fatal(ctx) << *this << ": unsupported SHT_GROUP format";
-
-      ComdatGroup *group = insert_comdat_group(ctx, signature);
-      comdat_groups.push_back({group, (i32)i, entries.subspan(1)});
+    case SHT_GROUP:
       break;
-    }
     case SHT_CREL:
       decoded_crel.resize(i + 1);
-      decoded_crel[i] = decode_crel(ctx, *this, shdr);
+      if ((this->elf_sections[shdr.sh_info].sh_flags & SHF_ALLOC) ||
+          ctx.arg.relocatable || ctx.arg.emit_relocs)
+        decoded_crel[i] = decode_crel(ctx, *this, shdr);
       break;
     case SHT_REL:
     case SHT_RELA:
@@ -372,6 +373,9 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
     case SHT_NULL:
       break;
     default:
+      if (!name.data())
+        name = this->shstrtab.data() + shdr.sh_name;
+
       if (!is_known_section_type(shdr))
         Fatal(ctx) << *this << ": " << name << ": unsupported section type: 0x"
                    << std::hex << (u32)shdr.sh_type;
@@ -433,16 +437,17 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
       if (ctx.arg.oformat_binary && !(shdr.sh_flags & SHF_ALLOC))
         continue;
 
-      this->sections[i] = std::make_unique<InputSection<E>>(ctx, *this, i);
-      InputSection<E> *isec = this->sections[i].get();
+      InputSection<E> *isec = this->sections.emplace(ctx, *this, i, name);
 
       // Save .llvm_addrsig for --icf=safe.
       if (shdr.sh_type == SHT_LLVM_ADDRSIG && !ctx.arg.relocatable) {
         // sh_link should be the index of the symbol table section.
         // Tools that mutates the symbol table, such as objcopy or `ld -r`
         // tend to not preserve sh_link, so we ignore such section.
-        if (shdr.sh_link != 0)
-          llvm_addrsig = std::move(this->sections[i]);
+        if (shdr.sh_link != 0) {
+          llvm_addrsig = isec;
+          this->sections.erase(i);
+        }
         continue;
       }
 
@@ -506,7 +511,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
     const ElfShdr<E> &shdr = this->elf_sections[i];
     if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL) ||
         shdr.sh_type == SHT_CREL) {
-      if (std::unique_ptr<InputSection<E>> &target = sections[shdr.sh_info]) {
+      if (InputSection<E> *target = sections[shdr.sh_info]) {
         assert(target->relsec_idx == -1);
         target->relsec_idx = i;
       }
@@ -515,10 +520,10 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
 
   // Attach .arm.exidx sections to their corresponding sections
   if constexpr (is_arm32<E>)
-    for (std::unique_ptr<InputSection<E>> &isec : this->sections)
+    for (InputSection<E> *isec : this->sections)
       if (isec && isec->shdr().sh_type == SHT_ARM_EXIDX)
-        if (InputSection<E> *target = sections[isec->shdr().sh_link].get())
-          target->extra.exidx = isec.get();
+        if (InputSection<E> *target = sections[isec->shdr().sh_link])
+          target->extra.exidx = isec;
 }
 
 // .eh_frame contains data records explaining how to handle exceptions.
@@ -591,6 +596,10 @@ void ObjectFile<E>::parse_ehframe(Context<E> &ctx) {
         if (rels[rel_begin].r_offset - begin_offset != 8)
           Fatal(ctx) << *isec << ": FDE's first relocation should have offset 8";
 
+        // The function may belong to a discarded COMDAT group.
+        if (!get_section(this->elf_syms[rels[rel_begin].r_sym]))
+          continue;
+
         fdes.emplace_back(begin_offset, rel_begin);
       }
     }
@@ -715,8 +724,12 @@ void ObjectFile<E>::parse_sframe(Context<E> &ctx) requires supports_sframe<E> {
       const ElfRel<E> &rel = rels[rel_idx];
       i64 off = fre_off + ent.func_start_fre_off;
 
+      InputSection<E> *func = get_section(this->elf_syms[rel.r_sym]);
+      if (!func)
+        continue;
+
       SFrameFde<E> fde;
-      fde.isec = get_section(this->elf_syms[rel.r_sym]);
+      fde.isec = func;
       fde.sym = this->symbols[rel.r_sym];
       fde.addend = rel.r_addend;
       fde.fre = data.substr(off, sframe_fre_block_size<E>(data, off));
@@ -728,48 +741,19 @@ void ObjectFile<E>::parse_sframe(Context<E> &ctx) requires supports_sframe<E> {
 }
 
 template <typename E>
-void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
+void ObjectFile<E>::register_global_symbols(Context<E> &ctx) {
   if (this->elf_syms.empty())
     return;
 
   static Counter counter("all_syms");
   counter += this->elf_syms.size();
 
-  // Initialize local symbols
-  this->local_syms.resize(this->first_global);
-  this->local_syms[0].file = this;
-  this->local_syms[0].sym_idx = 0;
-
-  for (i64 i = 1; i < this->first_global; i++) {
-    const ElfSym<E> &esym = this->elf_syms[i];
-    if (esym.is_common())
-      Fatal(ctx) << *this << ": common local symbol?";
-
-    std::string_view name;
-    if (esym.st_type == STT_SECTION)
-      name = this->shstrtab.data() + this->elf_sections[get_shndx(esym)].sh_name;
-    else
-      name = this->symbol_names[i];
-
-    Symbol<E> &sym = this->local_syms[i];
-    sym.set_name(name);
-    sym.file = this;
-    sym.value = esym.st_value;
-    sym.sym_idx = i;
-
-    if (!esym.is_abs())
-      sym.set_input_section(sections[get_shndx(esym)].get());
-  }
-
   this->symbols.resize(this->elf_syms.size());
 
   i64 num_globals = this->elf_syms.size() - this->first_global;
   has_symver.resize(num_globals);
 
-  for (i64 i = 0; i < this->first_global; i++)
-    this->symbols[i] = &this->local_syms[i];
-
-  // Initialize global symbols
+  // Register global symbols
   for (i64 i = this->first_global; i < this->elf_syms.size(); i++) {
     const ElfSym<E> &esym = this->elf_syms[i];
 
@@ -777,7 +761,7 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
       has_common_symbol = true;
 
     // Get a symbol name
-    std::string_view key = this->symbol_names[i];
+    std::string_view key = this->get_symbol_name(i);
     std::string_view name = key;
 
     // Parse symbol version after atsign
@@ -785,7 +769,7 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
       std::string_view ver = name.substr(pos);
       name = name.substr(0, pos);
 
-      if (ver != "@" && ver != "@@") {
+      if (ver != "@") {
         if (ver.starts_with("@@"))
           key = name;
         has_symver[i - this->first_global] = true;
@@ -793,23 +777,72 @@ void ObjectFile<E>::initialize_symbols(Context<E> &ctx) {
     }
 
     // Handle --wrap option
-    Symbol<E> *sym;
     if (esym.is_undef() && name.starts_with("__real_") &&
         ctx.arg.wrap.contains(name.substr(7))) {
-      sym = get_symbol(ctx, key.substr(7), name.substr(7));
-    } else {
-      sym = get_symbol(ctx, key, name);
-      if (esym.is_undef() && sym->is_wrapped) {
-        key = save_string(ctx, "__wrap_" + std::string(key));
-        name = save_string(ctx, "__wrap_" + std::string(name));
-        sym = get_symbol(ctx, key, name);
-      }
+      key = key.substr(7);
+    } else if (esym.is_undef() && ctx.arg.wrap.contains(key)) {
+      key = save_string(ctx, "__wrap_" + std::string(key));
     }
 
-    this->symbols[i] = sym;
+    // Only record the symbol reference here; gather_symbols() creates
+    // the Symbols and fills in the `symbols` slots once all files
+    // have been read.
+    ctx.symbol_map.add(key, this->symbols[i]);
   }
 }
 
+template <typename E>
+void ObjectFile<E>::initialize_local_symbols(Context<E> &ctx) {
+  if (this->elf_syms.empty())
+    return;
+
+  i64 num_local_syms = 1;
+  for (i64 i = 1; i < this->first_global; i++)
+    if (!is_discarded_comdat(this->elf_syms[i]))
+      num_local_syms++;
+
+  this->local_syms = allocate_symbols<E>(ctx, num_local_syms);
+  this->local_syms[0].file = this;
+  this->local_syms[0].sym_idx = 0;
+  this->symbols[0] = &this->local_syms[0];
+
+  i64 local_idx = 1;
+  for (i64 i = 1; i < this->first_global; i++) {
+    const ElfSym<E> &esym = this->elf_syms[i];
+    if (esym.is_common())
+      Fatal(ctx) << *this << ": common local symbol?";
+
+    if (is_discarded_comdat(esym)) {
+      this->symbols[i] = discarded_comdat_sym<E>;
+      continue;
+    }
+    this->symbols[i] = &this->local_syms[local_idx++];
+
+    std::string_view name;
+    if (esym.st_type == STT_SECTION) {
+      i64 shndx = get_shndx(esym);
+      if (InputSection<E> *isec = sections[shndx]) {
+        name = isec->name();
+      } else {
+        name = this->shstrtab.data() + this->elf_sections[shndx].sh_name;
+      }
+    } else {
+      name = this->get_symbol_name(i);
+    }
+
+    Symbol<E> &sym = *this->symbols[i];
+    sym.set_name(name);
+    sym.file = this;
+    sym.value = esym.st_value;
+    sym.sym_idx = i;
+
+    if (!esym.is_abs())
+      sym.set_input_section(sections[get_shndx(esym)]);
+  }
+
+  assert(local_idx == this->local_syms.size());
+}
+
 // Relocations are usually sorted by r_offset in relocation tables,
 // but for some reason only RISC-V does not follow that convention.
 // We expect them to be sorted, so sort them if necessary.
@@ -817,7 +850,7 @@ template <typename E>
 void ObjectFile<E>::sort_relocations(Context<E> &ctx) {
   if constexpr (is_riscv<E> || is_loongarch<E>) {
     for (i64 i = 1; i < sections.size(); i++) {
-      std::unique_ptr<InputSection<E>> &isec = sections[i];
+      InputSection<E> *isec = sections[i];
       if (!isec || !isec->is_alive || !(isec->shdr().sh_flags & SHF_ALLOC))
         continue;
 
@@ -832,7 +865,7 @@ template <typename E>
 void ObjectFile<E>::convert_mergeable_sections(Context<E> &ctx) {
   // Convert InputSections to MergeableSections
   for (i64 i = 0; i < this->sections.size(); i++) {
-    InputSection<E> *isec = this->sections[i].get();
+    InputSection<E> *isec = this->sections[i];
     if (!isec || isec->sh_size == 0 || isec->relsec_idx != -1)
       continue;
 
@@ -841,12 +874,12 @@ void ObjectFile<E>::convert_mergeable_sections(Context<E> &ctx) {
       continue;
 
     MergedSection<E> *parent =
-      MergedSection<E>::get_instance(ctx, isec->name, shdr);
+      MergedSection<E>::get_instance(ctx, isec->name(), shdr);
 
     if (parent) {
-      this->mergeable_sections[i] =
-        std::make_unique<MergeableSection<E>>(ctx, *parent, this->sections[i]);
-      this->sections[i] = nullptr;
+      std::unique_ptr<MergeableSection<E>> m =
+        std::make_unique<MergeableSection<E>>(ctx, *parent, isec);
+      this->sections.set_mergeable(i, std::move(m));
     }
   }
 }
@@ -907,7 +940,7 @@ void ObjectFile<E>::reattach_section_pieces(Context<E> &ctx) {
       continue;
 
     i64 shndx = get_shndx(esym);
-    std::unique_ptr<MergeableSection<E>> &m = mergeable_sections[shndx];
+    MergeableSection<E> *m = sections.get_mergeable(shndx);
     if (!m || !m->parent.resolved)
       continue;
 
@@ -924,21 +957,23 @@ void ObjectFile<E>::reattach_section_pieces(Context<E> &ctx) {
 
   // Compute the size of frag_syms.
   i64 nfrag_syms = 0;
-  for (std::unique_ptr<InputSection<E>> &isec : sections)
+  for (InputSection<E> *isec : sections)
     if (isec && (isec->shdr().sh_flags & SHF_ALLOC))
       for (ElfRel<E> &r : isec->get_rels(ctx))
         if (const ElfSym<E> &esym = this->elf_syms[r.r_sym];
             esym.st_type == STT_SECTION)
-          if (mergeable_sections[get_shndx(esym)])
+          if (sections.get_mergeable(get_shndx(esym)))
             nfrag_syms++;
 
-  this->frag_syms.resize(nfrag_syms);
+  // Arena allocations cannot be reclaimed, so grow this vector only once.
+  this->symbols.reserve(this->symbols.size() + nfrag_syms);
+  this->frag_syms = allocate_symbols<E>(ctx, nfrag_syms);
 
   // For each relocation referring to a mergeable section symbol, we
   // create a new dummy non-section symbol and redirect the relocation
   // to the newly created symbol.
   i64 idx = 0;
-  for (std::unique_ptr<InputSection<E>> &isec : sections) {
+  for (InputSection<E> *isec : sections) {
     if (isec && (isec->shdr().sh_flags & SHF_ALLOC)) {
       for (ElfRel<E> &r : isec->get_rels(ctx)) {
         const ElfSym<E> &esym = this->elf_syms[r.r_sym];
@@ -946,7 +981,7 @@ void ObjectFile<E>::reattach_section_pieces(Context<E> &ctx) {
           continue;
 
         i64 shndx = get_shndx(esym);
-        std::unique_ptr<MergeableSection<E>> &m = mergeable_sections[shndx];
+        MergeableSection<E> *m = sections.get_mergeable(shndx);
         if (!m)
           continue;
 
@@ -962,7 +997,7 @@ void ObjectFile<E>::reattach_section_pieces(Context<E> &ctx) {
 
         Symbol<E> &sym = this->frag_syms[idx];
         sym.file = this;
-        sym.set_name("<fragment>");
+        sym.is_fragment_dummy = true;
         sym.sym_idx = r.r_sym;
         sym.visibility = STV_HIDDEN;
         sym.set_frag(frag);
@@ -976,14 +1011,13 @@ void ObjectFile<E>::reattach_section_pieces(Context<E> &ctx) {
   assert(idx == this->frag_syms.size());
 
   for (Symbol<E> &sym : this->frag_syms)
-    this->symbols.push_back(&sym);
+    this->symbols.emplace_back(&sym);
 }
 
+// Read global symbols before archive extraction so they can participate in
+// symbol resolution without constructing input sections.
 template <typename E>
-void ObjectFile<E>::parse(Context<E> &ctx) {
-  sections.resize(this->elf_sections.size());
-  mergeable_sections.resize(sections.size());
-
+void ObjectFile<E>::parse_symbols(Context<E> &ctx) {
   symtab_sec = this->find_section(SHT_SYMTAB);
 
   if (symtab_sec) {
@@ -992,15 +1026,36 @@ void ObjectFile<E>::parse(Context<E> &ctx) {
     this->first_global = symtab_sec->sh_info;
     this->elf_syms = this->template get_data<ElfSym<E>>(ctx, *symtab_sec);
     this->symbol_strtab = this->get_string(ctx, symtab_sec->sh_link);
-    this->populate_symbol_names();
+    this->populate_symbol_name_lengths();
 
     if (ElfShdr<E> *shdr = this->find_section(SHT_SYMTAB_SHNDX))
       symtab_shndx_sec = this->template get_data<U32<E>>(ctx, *shdr);
   }
 
+  register_global_symbols(ctx);
+}
+
+// Construct sections after COMDAT ownership is known. Members of losing groups
+// are normally skipped. If another selection will run after LTO, construct
+// them too so a different copy can become live.
+template <typename E>
+void ObjectFile<E>::parse_sections(Context<E> &ctx,
+                                   bool keep_discarded_comdat) {
+  assert(!sections_parsed);
+  sections.resize(this->elf_sections.size());
+
+  if (!keep_discarded_comdat && !comdat_groups.empty()) {
+    comdat_discarded.resize(sections.size());
+    for (ComdatGroupRef<E> &ref : comdat_groups)
+      if (!ref.is_owner)
+        for (u32 i : ref.members(*this))
+          comdat_discarded[i] = true;
+  }
+
   initialize_sections(ctx);
-  initialize_symbols(ctx);
+  initialize_local_symbols(ctx);
   sort_relocations(ctx);
+  sections_parsed = true;
 }
 
 // Symbols with higher priorities overwrites symbols with lower priorities.
@@ -1042,7 +1097,8 @@ template <typename E>
 static u64 get_rank(const Symbol<E> &sym) {
   if (!sym.file)
     return 7 << 24;
-  return get_rank(sym.file, sym.esym(), !sym.file->is_reachable);
+  InputFile<E> *file = sym.file;
+  return get_rank(file, sym.esym(), !file->is_reachable);
 }
 
 // Symbol's visibility is set to the most restrictive one. For example,
@@ -1094,8 +1150,10 @@ void ObjectFile<E>::resolve_symbols(Context<E> &ctx) {
     if (esym.is_undef())
       continue;
 
+    // Before sections are parsed, pre-liveness resolution treats all
+    // definitions as live. The final round uses the actual section state.
     InputSection<E> *isec = nullptr;
-    if (!esym.is_abs() && !esym.is_common()) {
+    if (!esym.is_abs() && !esym.is_common() && sections_parsed) {
       isec = get_section(esym);
       if (!isec || !isec->is_alive)
         continue;
@@ -1150,7 +1208,7 @@ ObjectFile<E>::mark_live_objects(Context<E> &ctx,
 template <typename E>
 void ObjectFile<E>::scan_relocations(Context<E> &ctx) {
   // Scan relocations against seciton contents
-  for (std::unique_ptr<InputSection<E>> &isec : sections)
+  for (InputSection<E> *isec : sections)
     if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC))
       isec->scan_relocations(ctx);
 
@@ -1221,14 +1279,13 @@ void ObjectFile<E>::convert_common_symbols(Context<E> &ctx) {
     elf_sections2.push_back(shdr);
 
     i64 idx = this->elf_sections.size() + elf_sections2.size() - 1;
-    auto isec = std::make_unique<InputSection<E>>(ctx, *this, idx);
+    InputSection<E> *isec = sections.emplace(ctx, *this, idx, "");
 
-    sym.set_input_section(isec.get());
+    sym.set_input_section(isec);
     sym.value = 0;
     sym.sym_idx = i;
     sym.ver_idx = ctx.default_version;
     sym.is_weak = false;
-    sections.push_back(std::move(isec));
   }
 }
 
@@ -1237,12 +1294,18 @@ static bool should_write_to_local_symtab(Context<E> &ctx, Symbol<E> &sym) {
   if (sym.get_type() == STT_SECTION)
     return false;
 
-  // Local symbols are discarded if --discard-local is given or they
-  // are in a mergeable section. I *believe* we exclude symbols in
-  // mergeable sections because (1) there are too many and (2) they are
-  // merged, so their origins shouldn't matter, but I don't really
-  // know the rationale. Anyway, this is the behavior of the
-  // traditional linkers.
+  // Temporary local symbols such as .L.str.42 are compiler-internal
+  // and numerous; a Chromium debug build contains more than two
+  // million of them, adding ~180 MiB of .symtab and .strtab. They are
+  // not referenced by DWARF, so we discard them by default, as lld
+  // does. GNU ld keeps them unless -X is given; --discard-none
+  // restores that behavior.
+  //
+  // Even with --discard-none, we discard temporary symbols in
+  // mergeable sections. I *believe* they are excluded because (1)
+  // there are too many and (2) they are merged, so their origins
+  // shouldn't matter, but I don't really know the rationale. Anyway,
+  // this is the behavior of the traditional linkers.
   if (sym.name().starts_with(".L") || sym.name() == "L0\001") {
     if (ctx.arg.discard_locals)
       return false;
@@ -1270,6 +1333,9 @@ void ObjectFile<E>::compute_symtab_size(Context<E> &ctx) {
   // Compute the size of local symbols
   if (!ctx.arg.discard_all && !ctx.arg.strip_all && !ctx.arg.retain_symbols_file) {
     for (i64 i = 1; i < this->first_global; i++) {
+      if (is_discarded_comdat(this->elf_syms[i]))
+        continue;
+
       Symbol<E> &sym = *this->symbols[i];
 
       if (is_alive(sym) && should_write_to_local_symtab(ctx, sym)) {
@@ -1368,6 +1434,14 @@ void SharedFile<E>::parse(Context<E> &ctx) {
 
   // Read a symbol table.
   std::span<ElfSym<E>> esyms = this->template get_data<ElfSym<E>>(ctx, *symtab_sec);
+  if (esyms.size() < symtab_sec->sh_info)
+    Fatal(ctx) << *this << ": invalid symbol table";
+
+  i64 num_syms = esyms.size() - symtab_sec->sh_info;
+  this->elf_syms2.reserve(num_syms);
+  this->versyms.reserve(num_syms);
+  this->symbols.reserve(num_syms);
+  this->symbols2.reserve(num_syms);
 
   std::span<U16<E>> vers;
   if (ElfShdr<E> *sec = this->find_section(SHT_GNU_VERSYM))
@@ -1376,12 +1450,30 @@ void SharedFile<E>::parse(Context<E> &ctx) {
   for (i64 i = symtab_sec->sh_info; i < esyms.size(); i++) {
     u16 ver = vers.empty() ? VER_NDX_GLOBAL : (vers[i] & ~VERSYM_HIDDEN);
 
-    // A defined symbol with VER_NDX_LOCAL is bound locally and isn't
-    // really exposed for dynamic linking; skip it. We never skip
-    // undefined references — the dynamic loader doesn't distinguish
-    // local vs global on the undef side, and dropping them would hide
-    // the DSO's reference from us.
-    if (ver == VER_NDX_LOCAL && !esyms[i].is_undef())
+    // A version index of 0 (VER_NDX_LOCAL) is valid only for unversioned
+    // undefined symbols. A symbol that's actually local to a DSO doesn't
+    // appear in .dynsym in the first place, so index 0 on a defined
+    // symbol is ill-formed. GNU ld briefly emitted it
+    // (https://sourceware.org/bugzilla/show_bug.cgi?id=33577). We used
+    // to silently ignore such symbols, which turned into confusing
+    // "undefined symbol" errors down the line. Reject the file instead,
+    // like lld.
+    if (ver == VER_NDX_LOCAL) {
+      if (!esyms[i].is_undef())
+        Fatal(ctx) << *this << ": invalid version index 0 for defined symbol "
+                   << (this->symbol_strtab.data() + esyms[i].st_name);
+      ver = VER_NDX_GLOBAL;
+    }
+
+    // A definition whose versym is exactly VERSYM_HIDDEN | VER_NDX_GLOBAL
+    // is a compatibility alias that exists only for binaries linked before
+    // the library adopted symbol versioning. GNU ld creates one for
+    // `.symver foo_impl, foo@`. New references must not bind to it; skip
+    // it so that an unversioned reference binds to the default-versioned
+    // `foo@@VERSION` definition instead. Versyms of undefined symbols
+    // encode required versions, so they are exempt.
+    if (!vers.empty() && vers[i] == (VERSYM_HIDDEN | VER_NDX_GLOBAL) &&
+        !esyms[i].is_undef())
       continue;
 
     this->elf_syms2.push_back(esyms[i]);
@@ -1418,16 +1510,16 @@ void SharedFile<E>::parse(Context<E> &ctx) {
     // visit all symbol references to redirect `foo@VERSION` to `foo`.
     if (!has_version) {
       // Unversioned symbol
-      this->symbols.push_back(get_symbol(ctx, name));
+      this->symbols.emplace_back(get_symbol(ctx, name));
       this->symbols2.push_back(nullptr);
     } else if (esyms[i].is_undef() || (vers[i] & VERSYM_HIDDEN)) {
       // Versioned non-default symbol, or undefined reference whose
       // version comes from .gnu.version_r.
-      this->symbols.push_back(get_versioned_sym());
+      this->symbols.emplace_back(get_versioned_sym());
       this->symbols2.push_back(nullptr);
     } else {
       // Versioned default symbol
-      this->symbols.push_back(get_symbol(ctx, name));
+      this->symbols.emplace_back(get_symbol(ctx, name));
       this->symbols2.push_back(get_versioned_sym());
     }
   }
@@ -1559,7 +1651,7 @@ void SharedFile<E>::resolve_symbols(Context<E> &ctx) {
 
     if (get_rank(this, esym, false) < get_rank(sym)) {
       sym.file = this;
-      sym.origin = 0;
+      sym.origin = nullptr;
       sym.value = esym.st_value;
       sym.sym_idx = i;
       sym.ver_idx = versyms[i];
@@ -1576,7 +1668,7 @@ void SharedFile<E>::resolve_symbols(Context<E> &ctx) {
 
       if (get_rank(this, esym, false) < get_rank(*sym2)) {
         sym2->file = this;
-        sym2->origin = (uintptr_t)&sym;
+        sym2->set_symbol_origin(&sym);
         sym2->sym_idx = i;
         sym2->is_versioned_default = true;
       }
@@ -1712,8 +1804,10 @@ template class SharedFile<E>;
 template Symbol<E> *get_symbol(Context<E> &, std::string_view, std::string_view);
 template Symbol<E> *get_symbol(Context<E> &, std::string_view);
 template std::string_view demangle(const Symbol<E> &);
-template ComdatGroup *insert_comdat_group(Context<E> &, std::string_view);
 template std::ostream &operator<<(std::ostream &, const Symbol<E> &);
 template std::ostream &operator<<(std::ostream &, const InputFile<E> &);
 
+template ExactArray<ElfRel<E>>
+decode_crel(Context<E> &, ObjectFile<E> &, const ElfShdr<E> &);
+
 } // namespace mold

src/input-sections.cc

@@ -31,17 +31,20 @@ bool cie_equals(const CieRecord<E> &a, const CieRecord<E> &b) {
 }
 
 template <typename E>
-InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
+InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx,
+                              std::string_view section_name)
   : file(file), shndx(shndx) {
   if (shndx < file.elf_sections.size()) {
-    name = file.shstrtab.data() + file.elf_sections[shndx].sh_name;
-    contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};
-  } else {
-    name = (shdr().sh_flags & SHF_TLS) ? ".tls_common" : ".common";
+    std::string_view name = section_name;
+    if (name.empty())
+      name = file.shstrtab.data() + file.elf_sections[shndx].sh_name;
+    namelen = std::min<i64>(name.size(), UINT16_MAX);
+
+    contents = file.mf->data + shdr().sh_offset;
   }
 
   if (shdr().sh_flags & SHF_COMPRESSED) {
-    ElfChdr<E> &chdr = *(ElfChdr<E> *)&contents[0];
+    ElfChdr<E> &chdr = *(ElfChdr<E> *)contents;
     sh_size = chdr.ch_size;
     p2align = to_p2align(chdr.ch_addralign);
   } else {
@@ -68,7 +71,7 @@ void InputSection<E>::uncompress(Context<E> &ctx) {
 
   u8 *buf = new u8[sh_size];
   copy_contents_to(ctx, buf, sh_size);
-  contents = std::string_view((char *)buf, sh_size);
+  contents = buf;
   ctx.string_pool.emplace_back(buf);
   uncompressed = true;
 }
@@ -76,15 +79,16 @@ void InputSection<E>::uncompress(Context<E> &ctx) {
 template <typename E>
 void InputSection<E>::copy_contents_to(Context<E> &ctx, u8 *buf, i64 sz) {
   if (!(shdr().sh_flags & SHF_COMPRESSED) || uncompressed) {
-    memcpy(buf, contents.data(), sz);
+    memcpy(buf, contents, sz);
     return;
   }
 
-  if (contents.size() < sizeof(ElfChdr<E>))
+  std::string_view view = get_contents();
+  if (view.size() < sizeof(ElfChdr<E>))
     Fatal(ctx) << *this << ": corrupted compressed section";
 
-  ElfChdr<E> &hdr = *(ElfChdr<E> *)&contents[0];
-  std::string_view data = contents.substr(sizeof(ElfChdr<E>));
+  ElfChdr<E> &hdr = *(ElfChdr<E> *)contents;
+  std::string_view data = view.substr(sizeof(ElfChdr<E>));
 
   switch (hdr.ch_type) {
   case ELFCOMPRESS_ZLIB: {
@@ -128,7 +132,7 @@ void InputSection<E>::copy_contents_to(Context<E> &ctx, u8 *buf, i64 sz) {
 // wingdi.h defines ERROR as a macro, so undefine it before use
 #undef ERROR
 
-typedef enum : u8 { NONE, ERROR, COPYREL, PLT, CPLT } Action;
+typedef enum : u8 { NONE, ERROR, CANONICAL, PLT } Action;
 
 template <typename E>
 static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
@@ -141,17 +145,13 @@ static void do_action(Context<E> &ctx, Action action, InputSection<E> &isec,
                << std::hex << rel.r_offset << " against symbol `"
                << sym << "' can not be used; recompile with -fPIC";
     break;
-  case COPYREL:
-    sym.flags |= NEEDS_COPYREL;
+  case CANONICAL:
+    sym.flags |= NEEDS_CANONICAL;
     break;
   case PLT:
     // Create a PLT entry
     sym.flags |= NEEDS_PLT;
     break;
-  case CPLT:
-    // Create a canonical PLT entry
-    sym.flags |= NEEDS_CPLT;
-    break;
   }
 }
 
@@ -183,9 +183,9 @@ void InputSection<E>::scan_pcrel(Context<E> &ctx, Symbol<E> &sym,
   // linker generally does not support PC-relative relocations.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  ERROR,    NONE,    ERROR,         PLT    },  // Shared object
-    {  ERROR,    NONE,    COPYREL,       CPLT   },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT   },  // Position-dependent exec
+    {  ERROR,    NONE,    ERROR,         PLT       }, // Shared object
+    {  ERROR,    NONE,    CANONICAL,     CANONICAL }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -202,9 +202,9 @@ void InputSection<E>::scan_absrel(Context<E> &ctx, Symbol<E> &sym,
   // resolved at link-time.
   static Action table[][4] = {
     // Absolute  Local    Imported data  Imported code
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Shared object
-    {  NONE,     ERROR,   ERROR,         ERROR },  // Position-independent exec
-    {  NONE,     NONE,    COPYREL,       CPLT  },  // Position-dependent exec
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Shared object
+    {  NONE,     ERROR,   ERROR,         ERROR     }, // Position-independent exec
+    {  NONE,     NONE,    CANONICAL,     CANONICAL }, // Position-dependent exec
   };
 
   Action action = table[get_output_type(ctx)][get_sym_type(sym)];
@@ -259,15 +259,18 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
       copy_contents_to(ctx, buf, sh_size);
     } else {
       // A relaxed section is copied piece-wise.
-      memcpy(buf, contents.data(), deltas[0].offset);
+      memcpy(buf, contents, deltas[0].offset);
 
+      i64 input_size = get_contents().size();
       for (i64 i = 0; i < deltas.size(); i++) {
         i64 offset = deltas[i].offset;
         i64 delta = deltas[i].delta;
-        i64 end = (i + 1 == deltas.size()) ? contents.size() : deltas[i + 1].offset;
+        i64 end = input_size;
+        if (i + 1 < deltas.size())
+          end = deltas[i + 1].offset;
         i64 removed_bytes = get_removed_bytes(deltas, i);
         memcpy(buf + offset + removed_bytes - delta,
-               contents.data() + offset + removed_bytes,
+               contents + offset + removed_bytes,
                end - offset - removed_bytes);
       }
     }
@@ -309,10 +312,10 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
   Symbol<E> &sym = *file.symbols[rel.r_sym];
   const ElfSym<E> &esym = file.elf_syms[rel.r_sym];
 
-  // If a symbol is defined in a comdat group, and the comdat group is
-  // discarded, the symbol may not have an owner. It is technically an
-  // violation of the One Definition Rule, so it is a programmer's fault.
-  if (!sym.file) {
+  // A global symbol in a discarded COMDAT group should resolve to the
+  // corresponding symbol in the prevailing group. If it does not, the
+  // object files violate the One Definition Rule.
+  if (!sym.file && &sym != discarded_comdat_sym<E>) {
     Error(ctx) << *this << ": " << sym << " refers to a discarded COMDAT section"
                << " probably due to an ODR violation";
     return true;
@@ -359,8 +362,8 @@ bool InputSection<E>::record_undef_error(Context<E> &ctx, const ElfRel<E> &rel)
 
 template <typename E>
 MergeableSection<E>::MergeableSection(Context<E> &ctx, MergedSection<E> &parent,
-                                      std::unique_ptr<InputSection<E>> &isec)
-  : parent(parent), p2align(isec->p2align), input_section(std::move(isec)) {
+                                      InputSection<E> *isec)
+  : parent(parent), p2align(isec->p2align), input_section(isec) {
   input_section->uncompress(ctx);
 
   std::scoped_lock lock(parent.mu);
@@ -397,7 +400,7 @@ static size_t find_null(std::string_view data, i64 pos, i64 entsize) {
 // We do not support mergeable sections that have relocations.
 template <typename E>
 void MergeableSection<E>::split_contents(Context<E> &ctx) {
-  std::string_view data = input_section->contents;
+  std::string_view data = input_section->get_contents();
   if (data.size() > UINT32_MAX)
     Fatal(ctx) << *input_section << ": mergeable section too large";
 
@@ -423,17 +426,15 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
   }
 
   // Compute hashes for section pieces
-  HyperLogLog estimator;
+  HyperLogLog::Sketch &sketch = parent.estimator.local();
   hashes.reserve(frag_offsets.size());
 
   for (i64 i = 0; i < frag_offsets.size(); i++) {
     u64 hash = hash_string(get_contents(i));
     hashes.push_back(hash);
-    estimator.insert(hash);
+    sketch.insert(hash);
   }
 
-  parent.estimator.merge(estimator);
-
   static Counter counter("string_fragments");
   counter += frag_offsets.size();
 }
@@ -441,8 +442,17 @@ void MergeableSection<E>::split_contents(Context<E> &ctx) {
 template <typename E>
 void MergeableSection<E>::resolve_contents(Context<E> &ctx) {
   fragments.reserve(frag_offsets.size());
-  for (i64 i = 0; i < frag_offsets.size(); i++)
+
+  // The hash table is typically much larger than the cache, so each
+  // insertion stalls on a cache miss for its first probe. We know all
+  // hashes upfront, so prefetch the bucket a few insertions ahead.
+  constexpr i64 lookahead = 8;
+
+  for (i64 i = 0; i < frag_offsets.size(); i++) {
+    if (i + lookahead < frag_offsets.size())
+      parent.map.prefetch(hashes[i + lookahead]);
     fragments.push_back(parent.insert(ctx, get_contents(i), hashes[i], p2align));
+  }
 
   // Reclaim memory as we'll never use this vector again
   hashes.clear();

src/linker-script.cc

@@ -200,7 +200,8 @@ Script<E>::read_group(std::span<std::string_view> tok) {
     }
 
     MappedFile *mf = resolve_path(tok[0], true);
-    read_file(ctx, rctx, mf);
+    ReaderContext rctx2 = rctx.next_child();
+    read_file(ctx, rctx2, mf);
     tok = tok.subspan(1);
   }
 

src/lto-unix.cc

@@ -179,13 +179,17 @@ static PluginStatus add_input_file(const char *path) {
   MappedFile *mf = must_open_file(ctx, path);
   mf->is_dependency = false;
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, "");
+  ObjectFile<E> *file = ctx.arena.template make<ObjectFile<E>>(ctx, mf, "");
   ctx.obj_pool.emplace_back(file);
   lto_objects<E>.push_back(file);
 
   file->priority = file_priority++;
   file->is_reachable = true;
-  file->parse(ctx);
+  file->parse_symbols(ctx);
+
+  // parse_symbols() only registers global symbols. Create their shared
+  // Symbol objects and fill in the file's pointers before resolving.
+  gather_symbols(ctx);
   file->resolve_symbols(ctx);
   return LDPS_OK;
 }
@@ -307,7 +311,7 @@ get_symbols(const void *handle, int nsyms, PluginSymbol *psyms, bool is_v2) {
     if (sym.file->is_dso)
       return LDPR_RESOLVED_DYN;
 
-    if (((ObjectFile<E> *)sym.file)->is_lto_obj && !sym.is_wrapped)
+    if (sym.file->to_obj()->is_lto_obj && !sym.is_wrapped)
       return esym.is_undef() ? LDPR_RESOLVED_IR : LDPR_PREEMPTED_IR;
     return esym.is_undef() ? LDPR_RESOLVED_EXEC : LDPR_PREEMPTED_REG;
   };
@@ -342,7 +346,7 @@ static void restart_process(Context<E> &ctx) {
   for (std::string_view arg : ctx.cmdline_args)
     args.push_back(strdup(std::string(arg).c_str()));
 
-  for (std::unique_ptr<ObjectFile<E>> &file : ctx.obj_pool)
+  for (ArenaObjectPtr<ObjectFile<E>> &file : ctx.obj_pool)
     if (file->is_lto_obj && !file->is_reachable)
       args.push_back(strdup(("--:ignore-ir-file=" +
                              file->mf->get_identifier()).c_str()));
@@ -603,18 +607,23 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
 
   load_lto_plugin(ctx);
 
-  // V0 API's claim_file is not thread-safe.
+  // We read input files in parallel, but the plugin interface is not
+  // ready for concurrent claims: claim_file_hook() returns a file's
+  // symbol table through the add_symbols() callback into a global
+  // buffer, and members of the same archive share their parent's file
+  // descriptor, which we close after each claim. Serialize the whole
+  // claim sequence.
   static std::mutex mu;
-  std::unique_lock lock(mu, std::defer_lock);
-  if (!is_gcc_linker_api_v1)
-    lock.lock();
+  std::scoped_lock lock(mu);
 
   // Create mold's object instance
-  ObjectFile<E> *obj = new ObjectFile<E>;
+  ObjectFile<E> *obj =
+    ctx.arena.template make<ObjectFile<E>>(ctx);
   ctx.obj_pool.emplace_back(obj);
 
   obj->filename = mf->name;
-  obj->symbols.push_back(new Symbol<E>);
+  Symbol<E> *dummy = ctx.arena.template make<Symbol<E>>();
+  obj->symbols.emplace_back(dummy);
   obj->first_global = 1;
   obj->is_lto_obj = true;
   obj->mf = mf;
@@ -651,7 +660,8 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
 
   // Initialize esyms
   obj->lto_elf_syms.resize(plugin_symbols.size() + 1);
-  obj->lto_comdat_groups.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_signatures.resize(plugin_symbols.size() + 1);
+  obj->lto_comdat_discarded.resize(plugin_symbols.size() + 1);
   i64 strtab_offset = 1;
 
   for (i64 i = 0; i < plugin_symbols.size(); i++) {
@@ -669,14 +679,14 @@ ObjectFile<E> *read_lto_object(Context<E> &ctx, MappedFile *mf) {
     // have input sections.
     if (psym.comdat_key) {
       std::string_view key = save_string(ctx, psym.comdat_key);
-      obj->lto_comdat_groups[i + 1] = insert_comdat_group(ctx, key);
+      obj->lto_comdat_signatures[i + 1] = get_symbol(ctx, key);
     }
   }
 
   obj->symbol_strtab = save_string(ctx, strtab);
   obj->elf_syms = obj->lto_elf_syms;
-  obj->populate_symbol_names();
-  obj->initialize_symbols(ctx);
+  obj->populate_symbol_name_lengths();
+  obj->register_global_symbols(ctx);
   plugin_symbols.clear();
   return obj;
 }
@@ -698,7 +708,7 @@ std::vector<ObjectFile<E> *> run_lto_plugin(Context<E> &ctx) {
     if (!file->is_lto_obj) {
       for (Symbol<E> *sym : file->get_global_syms()) {
         if (sym->file && !sym->file->is_dso &&
-            ((ObjectFile<E> *)sym->file)->is_lto_obj) {
+            sym->file->to_obj()->is_lto_obj) {
           std::scoped_lock lock(sym->mu);
           sym->referenced_by_regular_obj = true;
         }

src/main.cc

@@ -6,6 +6,7 @@
 #include <functional>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <tbb/parallel_for_each.h>
 #include <unordered_set>
 
 namespace mold {
@@ -14,98 +15,108 @@ template <typename E>
 static void
 check_file_compatibility(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   std::string_view target = get_machine_type(ctx, rctx, mf);
+  if (target.empty())
+    Fatal(ctx) << mf->name << ": unknown machine type";
   if (target != ctx.arg.emulation)
     Fatal(ctx) << mf->name << ": incompatible file type: "
                << ctx.arg.emulation << " is expected but got " << target;
 }
 
 template <typename E>
-static ObjectFile<E> *new_object_file(Context<E> &ctx, ReaderContext &rctx,
-                                      MappedFile *mf, std::string archive_name) {
+static void new_object_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_objs");
   count++;
 
   check_file_compatibility(ctx, rctx, mf);
 
-  ObjectFile<E> *file = new ObjectFile<E>(ctx, mf, archive_name);
+  ObjectFile<E> *file =
+    ctx.arena.template make<ObjectFile<E>>(ctx, mf, archive_name);
   ctx.obj_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
   file->as_needed =
     rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse_symbols(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
 template <typename E>
-static ObjectFile<E> *new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
-                                  MappedFile *mf, std::string archive_name) {
+static void new_lto_obj(Context<E> &ctx, ReaderContext &rctx,
+                        MappedFile *mf, std::string archive_name) {
   static Counter count("parsed_lto_objs");
   count++;
 
   if (ctx.arg.ignore_ir_file.count(mf->get_identifier()))
-    return nullptr;
+    return;
 
   ObjectFile<E> *file = read_lto_object(ctx, mf);
   if (!file)
-    return nullptr;
+    return;
 
-  file->priority = ctx.file_priority++;
   file->archive_name = archive_name;
   file->as_needed =
     rctx.in_lib || (!archive_name.empty() && !rctx.whole_archive);
 
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
 template <typename E>
-static SharedFile<E> *
-new_shared_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
+static void new_shared_file(Context<E> &ctx, ReaderContext &rctx,
+                            MappedFile *mf) {
+  if (rctx.static_)
+    Fatal(ctx) << mf->name << ": attempted static link of a dynamic object";
+
   check_file_compatibility(ctx, rctx, mf);
 
-  SharedFile<E> *file = new SharedFile<E>(ctx, mf);
+  SharedFile<E> *file = ctx.arena.template make<SharedFile<E>>(ctx, mf);
   ctx.dso_pool.emplace_back(file);
-  file->priority = ctx.file_priority++;
   file->as_needed = rctx.as_needed;
 
-  rctx.tg->run([file, &ctx] { file->parse(ctx); });
-  if (ctx.arg.trace)
-    Out(ctx) << "trace: " << *file;
-  return file;
+  file->parse(ctx);
+  ctx.unsorted_input_files.push_back({rctx.pos, file});
 }
 
+// Reads a file inside an archive.
+template <typename E>
+static void read_archive_member(Context<E> &ctx, ReaderContext &rctx,
+                                MappedFile *mf, std::string archive_name) {
+  switch (get_file_type(ctx, mf)) {
+  case FileType::ELF_OBJ:
+    new_object_file(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::GCC_LTO_OBJ:
+  case FileType::LLVM_BITCODE:
+    new_lto_obj(ctx, rctx, mf, archive_name);
+    break;
+  case FileType::ELF_DSO:
+    Warn(ctx) << archive_name << "(" << mf->name
+              << "): shared object file in an archive is ignored";
+    break;
+  default:
+    break;
+  }
+}
+
+// Reads the given file, which is located at rctx.pos in the command
+// line. If the file is a container, i.e. an archive file or a linker
+// script, the files in it are read in place, recursively.
+//
+// read_input_files() reads top-level files with this function too but
+// overrides the container cases to read archive members in parallel.
 template <typename E>
 void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
   switch (get_file_type(ctx, mf)) {
   case FileType::ELF_OBJ:
-    ctx.objs.push_back(new_object_file(ctx, rctx, mf, ""));
+    new_object_file(ctx, rctx, mf, "");
     return;
   case FileType::ELF_DSO:
-    ctx.dsos.push_back(new_shared_file(ctx, rctx, mf));
+    new_shared_file(ctx, rctx, mf);
     return;
   case FileType::AR:
   case FileType::THIN_AR:
     for (MappedFile *child : read_archive_members(ctx, mf)) {
-      switch (get_file_type(ctx, child)) {
-      case FileType::ELF_OBJ:
-        ctx.objs.push_back(new_object_file(ctx, rctx, child, mf->name));
-        break;
-      case FileType::GCC_LTO_OBJ:
-      case FileType::LLVM_BITCODE:
-        if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, child, mf->name))
-          ctx.objs.push_back(file);
-        break;
-      case FileType::ELF_DSO:
-        Warn(ctx) << mf->name << "(" << child->name
-                  << "): shared object file in an archive is ignored";
-        break;
-      default:
-        break;
-      }
+      ReaderContext rctx2 = rctx.next_child();
+      read_archive_member(ctx, rctx2, child, mf->name);
     }
     return;
   case FileType::TEXT:
@@ -113,8 +124,7 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
     return;
   case FileType::GCC_LTO_OBJ:
   case FileType::LLVM_BITCODE:
-    if (ObjectFile<E> *file = new_lto_obj(ctx, rctx, mf, ""))
-      ctx.objs.push_back(file);
+    new_lto_obj(ctx, rctx, mf, "");
     return;
   default:
     Fatal(ctx) << mf->name << ": unknown file type";
@@ -123,35 +133,23 @@ void read_file(Context<E> &ctx, ReaderContext &rctx, MappedFile *mf) {
 
 template <typename E>
 static std::string_view
-detect_machine_type(Context<E> &ctx, std::vector<std::string> args) {
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+detect_machine_type(Context<E> &ctx, std::span<ReaderJob> jobs) {
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) != FileType::TEXT)
-          if (std::string_view target = get_machine_type(ctx, rctx, mf);
+          if (std::string_view target = get_machine_type(ctx, job.rctx, mf);
               !target.empty())
             return target;
-    }
-  }
 
-  for (ReaderContext rctx; const std::string &arg : args) {
-    if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (!arg.starts_with('-')) {
-      if (MappedFile *mf = open_file(ctx, arg))
+  for (ReaderJob &job : jobs)
+    if (!job.is_lib)
+      if (MappedFile *mf = open_file(ctx, job.name))
         if (get_file_type(ctx, mf) == FileType::TEXT)
           if (std::string_view target =
-              Script(ctx, rctx, mf).get_script_output_type();
+              Script(ctx, job.rctx, mf).get_script_output_type();
               !target.empty())
             return target;
-    }
-  }
 
   Fatal(ctx) << "-m option is missing";
 }
@@ -193,67 +191,113 @@ MappedFile *find_library(Context<E> &ctx, ReaderContext &rctx, std::string name)
   Fatal(ctx) << "library not found: " << name;
 }
 
+// Reads all input files.
+//
+// Reading input files is I/O- and CPU-intensive, and a large program
+// can easily consist of tens of thousands of them, so we want to read
+// files in parallel. The command line, on the other hand, is
+// inherently sequential: options such as --as-needed or
+// --whole-archive apply to the files after them, and a file's
+// priority for symbol resolution is its position in the command line.
+//
+// We reconcile the two as follows: the command line parser has
+// already recorded the reader state and the position for each file
+// argument in its ReaderJob. We open and read files in parallel and
+// then sort the files we've found back into the command line order to
+// assign priorities.
 template <typename E>
-static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
+static void read_input_files(Context<E> &ctx, std::vector<ReaderJob> &jobs) {
   Timer t(ctx, "read_input_files");
 
-  ReaderContext rctx;
-  std::vector<ReaderContext> stack;
-  std::unordered_set<std::string_view> visited;
+  // Open and read files in parallel. Archive files are expanded into
+  // one job per member so that members are read in parallel too.
+  //
+  // Linker scripts are the exception to the parallelism: they can
+  // modify the context, e.g. by defining symbol versions, so we only
+  // collect them here and parse them after this loop, one at a time
+  // and in the command line order, to keep their effects
+  // deterministic. Scripts given as input files are rare and small,
+  // such as the GROUP file that glibc installs as libc.so, so the
+  // lost parallelism doesn't matter.
+  //
+  // Parsing scripts late assumes that no script directive affects how
+  // command line arguments after the script are read. That holds for
+  // the directives we currently support: a script can only add input
+  // files, whose positions order them correctly, and define symbol
+  // versions or symbols, which are not used until after this
+  // function. If we add a directive that doesn't satisfy this, such
+  // as SEARCH_DIR, which affects how -l arguments after it are
+  // resolved, this scheme needs to be revisited.
+  tbb::concurrent_vector<ReaderJob> scripts;
 
-  tbb::task_group tg;
-  rctx.tg = &tg;
+  tbb::parallel_for_each(jobs, [&](ReaderJob &job,
+                                   tbb::feeder<ReaderJob> &feeder) {
+    ReaderContext &rctx = job.rctx;
 
-  while (!args.empty()) {
-    std::string_view arg = args[0];
-    args = args.subspan(1);
-
-    if (arg == "--as-needed") {
-      rctx.as_needed = true;
-    } else if (arg == "--no-as-needed") {
-      rctx.as_needed = false;
-    } else if (arg == "--whole-archive") {
-      rctx.whole_archive = true;
-    } else if (arg == "--no-whole-archive") {
-      rctx.whole_archive = false;
-    } else if (arg == "--Bstatic") {
-      rctx.static_ = true;
-    } else if (arg == "--Bdynamic") {
-      rctx.static_ = false;
-    } else if (arg == "--start-lib") {
-      rctx.in_lib = true;
-    } else if (arg == "--end-lib") {
-      rctx.in_lib = false;
-    } else if (arg == "--push-state") {
-      stack.push_back(rctx);
-    } else if (arg == "--pop-state") {
-      if (stack.empty())
-        Fatal(ctx) << "no state pushed before popping";
-      rctx = stack.back();
-      stack.pop_back();
-    } else if (arg.starts_with("-l")) {
-      arg = arg.substr(2);
-      if (visited.contains(arg))
-        continue;
-      visited.insert(arg);
-
-      MappedFile *mf = find_library(ctx, rctx, std::string(arg));
-      mf->given_fullpath = false;
-      read_file(ctx, rctx, mf);
-    } else {
-      read_file(ctx, rctx, must_open_file(ctx, std::string(arg)));
+    // An archive member is enqueued in an already-opened form by the
+    // job that read its archive file.
+    if (!job.archive_name.empty()) {
+      read_archive_member(ctx, rctx, job.mf, job.archive_name);
+      return;
     }
+
+    // Everything else is a command line argument that we need to open.
+    MappedFile *mf;
+    if (job.is_lib) {
+      mf = find_library(ctx, rctx, job.name);
+      mf->given_fullpath = false;
+    } else {
+      mf = must_open_file(ctx, job.name);
+    }
+
+    switch (get_file_type(ctx, mf)) {
+    case FileType::AR:
+    case FileType::THIN_AR:
+      for (MappedFile *child : read_archive_members(ctx, mf)) {
+        ReaderJob job2;
+        job2.rctx = rctx.next_child();
+        job2.mf = child;
+        job2.archive_name = mf->name;
+        feeder.add(std::move(job2));
+      }
+      break;
+    case FileType::TEXT:
+      job.mf = mf;
+      scripts.push_back(std::move(job));
+      break;
+    default:
+      read_file(ctx, rctx, mf);
+    }
+  });
+
+  // Parse linker scripts and read the files they name.
+  ranges::sort(scripts, {}, [](const ReaderJob &job) {
+    return job.rctx.pos;
+  });
+
+  for (ReaderJob &job : scripts)
+    Script(ctx, job.rctx, job.mf).parse_linker_script();
+
+  // Sort the files into the command line order and assign priorities.
+  ranges::sort(ctx.unsorted_input_files, {},
+               &std::pair<std::vector<u32>, InputFile<E> *>::first);
+
+  for (auto &pair : ctx.unsorted_input_files) {
+    InputFile<E> *file = pair.second;
+    file->priority = ctx.file_priority++;
+    if (ctx.arg.trace)
+      Out(ctx) << "trace: " << *file;
+
+    if (file->is_dso)
+      ctx.dsos.push_back(file->to_dso());
+    else
+      ctx.objs.push_back(file->to_obj());
   }
 
+  ctx.unsorted_input_files.clear();
+
   if (ctx.objs.empty() && ctx.dsos.empty())
     Fatal(ctx) << "no input files";
-
-  if (rctx.static_ || ctx.arg.relocatable) {
-    ctx.arg.static_ = true;
-    ctx.arg.dynamic_linker = "";
-  }
-
-  tg.wait();
 }
 
 template <typename E>
@@ -291,11 +335,11 @@ int mold_main(int argc, char **argv) {
 
   // Parse non-positional command line options
   ctx.cmdline_args = expand_response_files(ctx, argv);
-  std::vector<std::string> file_args = parse_nonpositional_args(ctx);
+  std::vector<ReaderJob> jobs = parse_nonpositional_args(ctx);
 
   // If no -m option is given, deduce it from input files.
   if (ctx.arg.emulation.empty())
-    ctx.arg.emulation = detect_machine_type(ctx, file_args);
+    ctx.arg.emulation = detect_machine_type(ctx, jobs);
 
   // Redo if -m does not match with our speculation.
   if (ctx.arg.emulation != E::name) {
@@ -330,7 +374,7 @@ int mold_main(int argc, char **argv) {
     get_symbol(ctx, arg)->is_traced = true;
 
   // Parse input files
-  read_input_files(ctx, file_args);
+  read_input_files(ctx, jobs);
 
   // Uniquify shared object files by soname
   {
@@ -368,6 +412,19 @@ int mold_main(int argc, char **argv) {
   std::erase_if(ctx.objs, [](InputFile<E> *file) { return !file->is_reachable; });
   std::erase_if(ctx.dsos, [](InputFile<E> *file) { return !file->is_reachable; });
 
+  // Building .gdb_index is split into three stages because the required data
+  // becomes available at different points in the link. Compilation units and
+  // public names depend only on input sections, so read them now in a
+  // low-priority arena while foreground passes continue.
+  bool create_gdb_index = ctx.arg.gdb_index && !ctx.arg.relocatable;
+  tbb::task_arena gdb_input_arena(tbb::task_arena::automatic, 1,
+                                  tbb::task_arena::priority::low);
+  tbb::task_group gdb_task;
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] {
+      gdb_task.run([&] { read_gdb_index_inputs(ctx); });
+    });
+
   // Parse .eh_frame section contents.
   parse_eh_frame_sections(ctx);
 
@@ -505,6 +562,10 @@ int mold_main(int argc, char **argv) {
   // .got.plt, .dynsym, .dynstr, etc.
   scan_relocations(ctx);
 
+  // Now that we know all exported symbols, make sure that no versioned
+  // name is defined twice.
+  check_symbol_version_conflicts(ctx);
+
   // Compute the is_weak bit for each imported symbol.
   compute_imported_symbol_weakness(ctx);
 
@@ -520,15 +581,37 @@ int mold_main(int argc, char **argv) {
   // within an output section to input sections.
   compute_section_sizes(ctx);
 
+  // RELR is encoded independently for each output chunk using offsets
+  // relative to that chunk.
+  if (ctx.arg.pack_dyn_relocs_relr)
+    ctx.reldyn->construct_relr(ctx);
+
   // Reserve a space for dynamic symbol strings in .dynstr and sort
   // .dynsym contents if necessary. Beyond this point, no symbol will
   // be added to .dynsym.
   sort_dynsyms(ctx);
 
+  // sort_debug_info_sections may uncompress the same .debug_info sections.
+  if (create_gdb_index)
+    gdb_input_arena.execute([&] { gdb_task.wait(); });
+
   // Sort .debug_info contents so that DWARF32 debug info precedes that of
   // DWARF64. This is to mitigate the possibility of a relocation overflow.
   sort_debug_info_sections(ctx);
 
+  // Type vectors identify compilation units by their order in the output
+  // .debug_info section. That order is now fixed, so build the table while the
+  // remaining layout passes continue. This stage stops scaling after about 12
+  // workers, so limit its arena to avoid competing with foreground work.
+  i64 gdb_table_workers =
+    std::clamp<i64>(get_thread_count(ctx) * 3 / 8, 1, 12);
+  tbb::task_arena gdb_table_arena(gdb_table_workers, 1,
+                                  tbb::task_arena::priority::low);
+  if (ctx.gdb_index && !ctx.gnu_debuglink)
+    gdb_table_arena.execute([&] {
+      gdb_task.run([&] { build_gdb_index_tables(ctx); });
+    });
+
   // Print reports about undefined symbols, if needed.
   if (ctx.arg.unresolved_symbols == UNRESOLVED_ERROR)
     report_undef_errors(ctx);
@@ -611,9 +694,12 @@ int mold_main(int argc, char **argv) {
 
   // Re-finalize layout. fix_synthetic_symbols above may have changed
   // addends for dynamic relocations referencing synthetic symbols, which
-  // can shift the encoded size of .rela.dyn under --pack-dyn-relocs=android.
-  // For other modes this is a cheap no-op convergence.
-  filesize = set_osec_offsets(ctx);
+  // can shift the encoded size of .rela.dyn under --pack-dyn-relocs=android
+  // because Android's packed format encodes addends in variable-length
+  // SLEB128. Other modes, including ordinary RELR, encode nothing whose
+  // size depends on addends, so they do not need this pass.
+  if (ctx.arg.pack_dyn_relocs_android)
+    filesize = set_osec_offsets(ctx);
 
   // At this point, both memory and file layouts are fixed.
 
@@ -641,11 +727,12 @@ int mold_main(int argc, char **argv) {
   // so we sort them.
   sort_reldyn(ctx);
 
-  // .gdb_index's contents cannot be constructed before applying
-  // relocations to other debug sections. We have relocated debug
-  // sections now, so write the .gdb_index section.
-  if (ctx.gdb_index && !ctx.gnu_debuglink)
+  // The final stage reads address ranges, which requires relocated debug
+  // sections. We have applied the relocations now, so finish the index.
+  if (ctx.gdb_index && !ctx.gnu_debuglink) {
+    gdb_table_arena.execute([&] { gdb_task.wait(); });
     write_gdb_index(ctx);
+  }
 
   // .note.gnu.build-id section contains a cryptographic hash of the
   // entire output file. Now that we wrote everything except build-id,
@@ -690,11 +777,19 @@ int mold_main(int argc, char **argv) {
   notify_parent<E>();
   release_global_lock();
 
+#if HAVE_MADVISE
+  // Dropping page table entries here in parallel makes process exit
+  // faster, as the kernel otherwise reclaims them in a single thread
+  // on exit. File contents stay in the page cache.
+  tbb::parallel_for_each(ctx.mf_pool, [](std::unique_ptr<MappedFile> &mf) {
+    if (!mf->parent && mf->data && mf->size)
+      madvise(mf->data, mf->size, MADV_DONTNEED);
+  });
+#endif
+
   if (ctx.arg.quick_exit)
     _exit(0);
 
-  for (std::function<void()> &fn : ctx.on_exit)
-    fn();
   ctx.checkpoint();
   return 0;
 }
@@ -702,5 +797,6 @@ int mold_main(int argc, char **argv) {
 using E = MOLD_TARGET;
 
 template int mold_main<E>(int, char **);
+template void read_file(Context<E> &, ReaderContext &, MappedFile *);
 
 } // namespace mold

src/output-chunks.cc

@@ -11,7 +11,7 @@
 namespace mold {
 
 template <typename E>
-static std::vector<u64> encode_relr(std::span<ElfRel<E>> rels);
+static std::vector<u64> encode_relr(std::span<u64> offsets);
 
 template <typename E>
 static std::vector<u8> encode_android(std::span<ElfRel<E>> rels);
@@ -85,6 +85,8 @@ void OutputEhdr<E>::copy_buf(Context<E> &ctx) {
 
   if (ctx.arg.relocatable)
     hdr.e_type = ET_REL;
+  else if (ctx.arg.pie && ctx.arg.ttext_segment)
+    hdr.e_type = ET_EXEC;
   else if (ctx.arg.pic)
     hdr.e_type = ET_DYN;
   else
@@ -409,22 +411,72 @@ static bool is_relr(const ElfRel<E> &rel) {
          (u64)rel.r_offset % sizeof(Word<E>) == 0;
 }
 
+static constexpr i64 DYNREL_SHARD_SIZE = 65536;
+
+template <typename E>
+void RelDynSection<E>::write_relocs(Context<E> &ctx, ElfRel<E> *buf) {
+  for (Chunk<E> *chunk : ctx.chunks) {
+    i64 size = chunk->num_dynrels - chunk->num_relrs;
+    if (size) {
+      chunk->write_dynrels(ctx, buf);
+      buf += size;
+    }
+  }
+}
+
+template <typename E>
+void RelDynSection<E>::construct_relr(Context<E> &ctx) {
+  assert(ctx.arg.pack_dyn_relocs_relr);
+
+  tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
+    chunk->num_dynrels = chunk->get_num_dynrels(ctx);
+    chunk->num_relrs = 0;
+
+    // Do not use RELR for executable chunks, as they don't usually contain
+    // base relocations.
+    if (chunk->shdr.sh_flags & SHF_EXECINSTR)
+      return;
+
+    // --section-start can override a chunk's alignment. Conservatively use
+    // .rel[a].dyn if the explicitly assigned address is not word-aligned.
+    if (auto it = ctx.arg.section_start.find(chunk->name);
+        it != ctx.arg.section_start.end() &&
+        it->second % sizeof(Word<E>) != 0)
+      return;
+
+    if (chunk->num_dynrels) {
+      std::vector<u64> offsets = chunk->get_relr_offsets(ctx);
+      chunk->num_relrs = offsets.size();
+      assert(chunk->num_relrs <= chunk->num_dynrels);
+      chunk->relr = encode_relr<E>(offsets);
+    }
+  });
+  i64 size = 0;
+  for (Chunk<E> *chunk : ctx.chunks)
+    size += chunk->relr.size() * sizeof(Word<E>);
+  ctx.relrdyn->shdr.sh_size = size;
+}
+
 template <typename E>
 void RelDynSection<E>::update_shdr(Context<E> &ctx) {
-  relocs.clear();
-  android_encoded.clear();
-  for (Chunk<E> *chunk : ctx.chunks)
-    append(relocs, chunk->collect_dynrels(ctx));
+  if (!ctx.arg.pack_dyn_relocs_relr) {
+    tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
+      chunk->num_dynrels = chunk->get_num_dynrels(ctx);
+      chunk->num_relrs = 0;
+    });
+  }
 
-  if (ctx.relrdyn) {
-    auto relrs = ranges::partition(relocs, std::not_fn(is_relr<E>));
-    ranges::sort(relrs, {}, &ElfRel<E>::r_offset);
-    ctx.relrdyn->relocs = encode_relr<E>(relrs);
-    ctx.relrdyn->shdr.sh_size = ctx.relrdyn->relocs.size() * sizeof(Word<E>);
-    relocs.erase(relrs.begin(), relrs.end());
+  i64 num_relocs = 0;
+  i64 num_relrs = 0;
+  for (Chunk<E> *chunk : ctx.chunks) {
+    num_relocs += chunk->num_dynrels;
+    num_relrs += chunk->num_relrs;
   }
 
   if (ctx.arg.pack_dyn_relocs_android) {
+    relocs.resize(num_relocs - num_relrs);
+    write_relocs(ctx, relocs.data());
+
     // APS2 uses SLEB128-encoded deltas, so .rela.dyn size may oscillate
     // as addresses move. If a shrink is followed by a growth, stop
     // shrinking and pad the encoded stream to converge.
@@ -436,7 +488,7 @@ void RelDynSection<E>::update_shdr(Context<E> &ctx) {
       android_encoded.resize(old_size);
     this->shdr.sh_size = android_encoded.size();
   } else {
-    this->shdr.sh_size = relocs.size() * sizeof(ElfRel<E>);
+    this->shdr.sh_size = (num_relocs - num_relrs) * sizeof(ElfRel<E>);
   }
   this->shdr.sh_link = ctx.dynsym->shndx;
 }
@@ -446,14 +498,16 @@ void RelDynSection<E>::copy_buf(Context<E> &ctx) {
   if (ctx.arg.pack_dyn_relocs_android)
     write_vector(ctx.buf + this->shdr.sh_offset, android_encoded);
   else
-    write_vector(ctx.buf + this->shdr.sh_offset, relocs);
+    write_relocs(ctx, (ElfRel<E> *)(ctx.buf + this->shdr.sh_offset));
 }
 
 template <typename E>
 void RelrDynSection<E>::copy_buf(Context<E> &ctx) {
   Word<E> *buf = (Word<E> *)(ctx.buf + this->shdr.sh_offset);
-  for (u64 val : relocs)
-    *buf++ = val;
+
+  for (Chunk<E> *chunk : ctx.chunks)
+    for (u64 val : chunk->relr)
+      *buf++ = (val & 1) ? val : chunk->shdr.sh_addr + val;
 }
 
 template <typename E>
@@ -967,6 +1021,7 @@ void OutputSection<E>::copy_buf(Context<E> &ctx) {
 
     switch (r.kind) {
     case ABS_REL_NONE:
+    case ABS_REL_RELR:
       *(Word<E> *)loc = S + A;
       break;
     case ABS_REL_BASEREL:
@@ -987,38 +1042,102 @@ void OutputSection<E>::copy_buf(Context<E> &ctx) {
 }
 
 template <typename E>
-std::vector<ElfRel<E>>
-OutputSection<E>::collect_dynrels(Context<E> &ctx) const {
-  std::vector<ElfRel<E>> rels;
+i64 OutputSection<E>::get_num_dynrels(Context<E> &) const {
+  return dynrel_offsets.empty() ? 0 : dynrel_offsets.back();
+}
 
-  for (const AbsRel<E> &r : abs_rels) {
-    Symbol<E> &sym = *r.sym;
-    u64 S = sym.get_addr(ctx);
-    u64 A = r.addend;
-    u64 P = this->shdr.sh_addr + r.isec->offset + r.offset;
+template <typename E>
+std::vector<u64> OutputSection<E>::get_relr_offsets(Context<E> &) {
+  i64 nshards = dynrel_offsets.size() - 1;
+  relr_offsets.assign(nshards + 1, 0);
+  std::vector<u64> offsets;
+  offsets.reserve(this->num_dynrels);
 
-    if constexpr (is_riscv<E> || is_loongarch<E>) {
-      i64 delta = get_r_delta(*r.isec, r.offset);
-      P -= delta;
+  auto scan = [&](i64 begin, i64 end, std::vector<u64> &out) {
+    for (AbsRel<E> &r : std::span(abs_rels).subspan(begin, end - begin)) {
+      if (r.kind != ABS_REL_BASEREL)
+        continue;
+
+      if (r.isec->shdr().sh_addralign % sizeof(Word<E>) == 0 &&
+          r.offset % sizeof(Word<E>) == 0) {
+        r.kind = ABS_REL_RELR;
+        out.push_back(r.isec->offset + r.offset);
+      }
     }
+  };
 
-    switch (r.kind) {
-    case ABS_REL_NONE:
-      break;
-    case ABS_REL_BASEREL:
-      rels.emplace_back(P, E::R_RELATIVE, 0, S + A);
-      break;
-    case ABS_REL_IFUNC:
-      if constexpr (supports_ifunc<E>)
-        rels.emplace_back(P, E::R_IRELATIVE, 0, sym.get_addr(ctx, NO_PLT) + A);
-      break;
-    case ABS_REL_DYNREL:
-      rels.emplace_back(P, E::R_ABS, sym.get_dynsym_idx(ctx), A);
-      break;
-    }
+  if (nshards <= 1) {
+    scan(0, abs_rels.size(), offsets);
+    relr_offsets[nshards] = offsets.size();
+    return offsets;
   }
 
-  return rels;
+  std::vector<std::vector<u64>> shards(nshards);
+  tbb::parallel_for((i64)0, nshards, [&](i64 idx) {
+    i64 begin = idx * DYNREL_SHARD_SIZE;
+    i64 end = std::min<i64>(begin + DYNREL_SHARD_SIZE, abs_rels.size());
+    scan(begin, end, shards[idx]);
+  });
+
+  for (i64 i = 0; i < nshards; i++) {
+    append(offsets, shards[i]);
+    relr_offsets[i + 1] = relr_offsets[i] + shards[i].size();
+  }
+  return offsets;
+}
+
+template <typename E>
+void OutputSection<E>::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  // A single output section such as .data.rel.ro can account for
+  // most of an output's dynamic relocations, so we process its
+  // absolute relocations in parallel shards.
+  i64 nshards = dynrel_offsets.size() - 1;
+  std::vector<i64> offsets = dynrel_offsets;
+
+  if (ctx.arg.pack_dyn_relocs_relr && this->num_relrs) {
+    assert(relr_offsets.size() == offsets.size());
+    for (i64 i = 0; i <= nshards; i++)
+      offsets[i] -= relr_offsets[i];
+  }
+
+  assert(offsets.back() == this->num_dynrels - this->num_relrs);
+
+  tbb::parallel_for((i64)0, nshards, [&](i64 idx) {
+    i64 begin = idx * DYNREL_SHARD_SIZE;
+    i64 end = std::min<i64>(begin + DYNREL_SHARD_SIZE, abs_rels.size());
+    ElfRel<E> *loc = buf + offsets[idx];
+
+    for (const AbsRel<E> &r : std::span(abs_rels).subspan(begin, end - begin)) {
+      Symbol<E> &sym = *r.sym;
+      u64 S = sym.get_addr(ctx);
+      u64 A = r.addend;
+      u64 P = this->shdr.sh_addr + r.isec->offset + r.offset;
+
+      if constexpr (is_riscv<E> || is_loongarch<E>) {
+        i64 delta = get_r_delta(*r.isec, r.offset);
+        P -= delta;
+      }
+
+      switch (r.kind) {
+      case ABS_REL_NONE:
+      case ABS_REL_RELR:
+        break;
+      case ABS_REL_BASEREL:
+        *loc++ = ElfRel<E>(P, E::R_RELATIVE, 0, S + A);
+        break;
+      case ABS_REL_IFUNC:
+        if constexpr (supports_ifunc<E>)
+          *loc++ = ElfRel<E>(P, E::R_IRELATIVE, 0,
+                             sym.get_addr(ctx, NO_PLT) + A);
+        break;
+      case ABS_REL_DYNREL:
+        *loc++ = ElfRel<E>(P, E::R_ABS, sym.get_dynsym_idx(ctx), A);
+        break;
+      }
+    }
+
+    assert(loc == buf + offsets[idx + 1]);
+  });
 }
 
 template <typename E>
@@ -1084,21 +1203,25 @@ void OutputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
 // representable in this encoding and such relocation must be stored to
 // the .rel.dyn section). A bitmap has LSB 1.
 template <typename E>
-static std::vector<u64> encode_relr(std::span<ElfRel<E>> rels) {
+static std::vector<u64> encode_relr(std::span<u64> offsets) {
   std::vector<u64> vec;
   i64 num_bits = E::is_64 ? 63 : 31;
   i64 max_delta = sizeof(Word<E>) * num_bits;
 
-  for (i64 i = 0; i < rels.size();) {
-    u64 first = rels[i].r_offset;
+  for (i64 i = 0; i < offsets.size();) {
+    u64 first = offsets[i];
     vec.push_back(first);
     u64 base = first + sizeof(Word<E>);
     i++;
 
     for (;;) {
       u64 bits = 0;
-      for (; i < rels.size() && (u64)rels[i].r_offset - base < max_delta; i++)
-        bits |= (u64)1 << (((u64)rels[i].r_offset - base) / sizeof(Word<E>));
+      for (; i < offsets.size(); i++) {
+        u64 offset = offsets[i];
+        if (offset - base >= max_delta)
+          break;
+        bits |= (u64)1 << ((offset - base) / sizeof(Word<E>));
+      }
 
       if (!bits)
         break;
@@ -1205,7 +1328,7 @@ static AbsRelKind get_abs_rel_kind(Context<E> &ctx, Symbol<E> &sym) {
     return ABS_REL_NONE;
 
   // True if the symbol's address is in the output file.
-  if (!sym.is_imported || (sym.flags & NEEDS_CPLT) || (sym.flags & NEEDS_COPYREL))
+  if (!sym.is_imported || (sym.flags & NEEDS_CANONICAL))
     return ctx.arg.pic ? ABS_REL_BASEREL : ABS_REL_NONE;
 
   return ABS_REL_DYNREL;
@@ -1249,13 +1372,26 @@ void OutputSection<E>::scan_abs_relocations(Context<E> &ctx) {
     for (AbsRel<E> &r : abs_rels)
       if (Symbol<E> &sym = *r.sym;
           sym.is_imported && !sym.is_absolute())
-        sym.flags |= (sym.get_type() == STT_FUNC) ? NEEDS_CPLT : NEEDS_COPYREL;
+        sym.flags |= NEEDS_CANONICAL;
 
-  // Now we can compute whether they need to be promoted to dynamic
-  // relocations or not.
-  for (AbsRel<E> &r : abs_rels)
+  // Classify relocations and retain exact per-shard output counts.
+  i64 nshards = (abs_rels.size() + DYNREL_SHARD_SIZE - 1) / DYNREL_SHARD_SIZE;
+  dynrel_offsets.assign(nshards + 1, 0);
+
+  for (i64 i = 0; i < (i64)abs_rels.size(); i++) {
+    AbsRel<E> &r = abs_rels[i];
     r.kind = get_abs_rel_kind(ctx, *r.sym);
 
+    bool emit = r.kind == ABS_REL_BASEREL || r.kind == ABS_REL_DYNREL;
+    if constexpr (supports_ifunc<E>)
+      emit |= (r.kind == ABS_REL_IFUNC);
+    if (emit)
+      dynrel_offsets[i / DYNREL_SHARD_SIZE + 1]++;
+  }
+
+  for (i64 i = 0; i < nshards; i++)
+    dynrel_offsets[i + 1] += dynrel_offsets[i];
+
   // If we have a relocation against a read-only section, we need to
   // set the DT_TEXTREL flag for the loader.
   for (AbsRel<E> &r : abs_rels) {
@@ -1343,7 +1479,7 @@ void OutputSection<E>::populate_symtab(Context<E> &ctx) {
 
 template <typename E>
 void GotSection<E>::add_got_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_got_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
+  sym->aux->got_idx = this->shdr.sh_size / sizeof(Word<E>);
 
   // An IFUNC symbol uses two GOT slots in a position-dependent
   // executable.
@@ -1357,14 +1493,14 @@ void GotSection<E>::add_got_symbol(Context<E> &ctx, Symbol<E> *sym) {
 
 template <typename E>
 void GotSection<E>::add_gottp_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_gottp_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
+  sym->aux->gottp_idx = this->shdr.sh_size / sizeof(Word<E>);
   this->shdr.sh_size += sizeof(Word<E>);
   gottp_syms.push_back(sym);
 }
 
 template <typename E>
 void GotSection<E>::add_tlsgd_symbol(Context<E> &ctx, Symbol<E> *sym) {
-  sym->set_tlsgd_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
+  sym->aux->tlsgd_idx = this->shdr.sh_size / sizeof(Word<E>);
   this->shdr.sh_size += sizeof(Word<E>) * 2;
   tlsgd_syms.push_back(sym);
 }
@@ -1380,7 +1516,7 @@ void GotSection<E>::add_tlsdesc_symbol(Context<E> &ctx, Symbol<E> *sym) {
   assert(supports_tlsdesc<E>);
   assert(!ctx.arg.static_);
 
-  sym->set_tlsdesc_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
+  sym->aux->tlsdesc_idx = this->shdr.sh_size / sizeof(Word<E>);
   this->shdr.sh_size += sizeof(Word<E>) * 2;
   tlsdesc_syms.push_back(sym);
 }
@@ -1526,18 +1662,82 @@ static std::vector<GotEntry<E>> get_got_entries(Context<E> &ctx) {
   return entries;
 }
 
+// Count the dynamic relocations that get_got_entries will emit, without
+// materializing the entries; computing each entry's value involves a
+// symbol address lookup, which is too expensive for a function that runs
+// on every layout iteration. The cases below must mirror the r_type
+// choices in get_got_entries.
 template <typename E>
-std::vector<ElfRel<E>> GotSection<E>::collect_dynrels(Context<E> &ctx) const {
-  std::vector<GotEntry<E>> entries = get_got_entries(ctx);
-  std::vector<ElfRel<E>> rels;
+i64 GotSection<E>::get_num_dynrels(Context<E> &ctx) const {
+  i64 n = 0;
 
-  for (GotEntry<E> &ent : entries)
-    if (ent.r_type != R_NONE)
-      rels.emplace_back(this->shdr.sh_addr + ent.idx * sizeof(Word<E>),
-                        ent.r_type,
-                        ent.sym ? ent.sym->get_dynsym_idx(ctx) : 0,
-                        ent.val);
-  return rels;
+  for (Symbol<E> *sym : got_syms) {
+    if constexpr (supports_ifunc<E>) {
+      if (sym->is_ifunc()) {
+        n++; // R_IRELATIVE
+        continue;
+      }
+    }
+    if (sym->is_imported)
+      n++; // R_GLOB_DAT
+    else if (ctx.arg.pic && sym->is_relative())
+      n++; // R_RELATIVE
+  }
+
+  for (Symbol<E> *sym : tlsgd_syms) {
+    if (sym->is_imported)
+      n += 2; // R_DTPMOD + R_DTPOFF
+    else if (ctx.arg.shared)
+      n++; // R_DTPMOD
+  }
+
+  if constexpr (supports_tlsdesc<E>)
+    n += tlsdesc_syms.size(); // R_TLSDESC each
+
+  for (Symbol<E> *sym : gottp_syms)
+    if (sym->is_imported || ctx.arg.shared)
+      n++; // R_TPOFF
+
+  if (tlsld_idx != -1 && ctx.arg.shared)
+    n++; // R_DTPMOD
+
+  return n;
+}
+
+template <typename E>
+std::vector<u64> GotSection<E>::get_relr_offsets(Context<E> &ctx) {
+  if (!ctx.arg.pic)
+    return {};
+
+  std::vector<u64> offsets;
+  offsets.reserve(this->num_dynrels);
+
+  for (Symbol<E> *sym : got_syms) {
+    if constexpr (supports_ifunc<E>)
+      if (sym->is_ifunc())
+        continue;
+
+    if (!sym->is_imported && sym->is_relative())
+      offsets.push_back(sym->get_got_idx(ctx) * sizeof(Word<E>));
+  }
+  return offsets;
+}
+
+template <typename E>
+void GotSection<E>::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  std::vector<GotEntry<E>> entries = get_got_entries(ctx);
+
+  for (GotEntry<E> &ent : entries) {
+    if (ent.r_type == R_NONE)
+      continue;
+
+    ElfRel<E> rel(this->shdr.sh_addr + ent.idx * sizeof(Word<E>),
+                  ent.r_type,
+                  ent.sym ? ent.sym->get_dynsym_idx(ctx) : 0,
+                  ent.val);
+    if (!ctx.arg.pack_dyn_relocs_relr || !this->num_relrs || !is_relr(rel))
+      *buf++ = rel;
+  }
 }
 
 // Fill .got.
@@ -1682,7 +1882,7 @@ void GotPltSection<E>::copy_buf(Context<E> &ctx) {
 template <typename E>
 void PltSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
   assert(!sym->has_plt(ctx));
-  sym->set_plt_idx(ctx, symbols.size());
+  sym->aux->plt_idx = symbols.size();
   symbols.push_back(sym);
   ctx.dynsym->add_symbol(ctx, sym);
 }
@@ -1762,7 +1962,7 @@ void PltGotSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
   assert(!sym->has_plt(ctx));
   assert(sym->has_got(ctx));
 
-  sym->set_pltgot_idx(ctx, symbols.size());
+  sym->aux->pltgot_idx = symbols.size();
   symbols.push_back(sym);
   this->shdr.sh_size = symbols.size() * E::pltgot_size;
 }
@@ -1907,7 +2107,7 @@ to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name, U32<E> *shn_xindex)
   auto get_st_shndx = [&](Symbol<E> &sym) -> u32 {
     if (SectionFragment<E> *frag = sym.get_frag())
       if (frag->is_alive)
-        return frag->output_section.shndx;
+        return frag->get_output_section(ctx).shndx;
 
     if constexpr (is_ppc64v1<E>)
       if (sym.has_opd(ctx))
@@ -1942,7 +2142,7 @@ to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name, U32<E> *shn_xindex)
     esym.st_value = sym.get_addr(ctx);
   } else if (SectionFragment<E> *frag = sym.get_frag()) {
     // Section fragment
-    shndx = frag->output_section.shndx;
+    shndx = frag->get_output_section(ctx).shndx;
     esym.st_value = sym.get_addr(ctx);
   } else if (!isec) {
     // Absolute symbol
@@ -1961,9 +2161,9 @@ to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name, U32<E> *shn_xindex)
   } else if ((isec->shdr().sh_flags & SHF_MERGE) &&
              !(isec->shdr().sh_flags & SHF_ALLOC)) {
     // Symbol in a mergeable non-SHF_ALLOC section, such as .debug_str
-    ObjectFile<E> *file = (ObjectFile<E> *)sym.file;
+    ObjectFile<E> *file = sym.file->to_obj();
     MergeableSection<E> &m =
-      *file->mergeable_sections[file->get_shndx(sym.esym())];
+      *file->sections.get_mergeable(file->get_shndx(sym.esym()));
 
     SectionFragment<E> *frag;
     i64 frag_addend;
@@ -2002,7 +2202,7 @@ void DynsymSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
     symbols.resize(1);
 
   if (sym->get_dynsym_idx(ctx) == -1) {
-    sym->set_dynsym_idx(ctx, -2);
+    sym->aux->dynsym_idx = -2;
     symbols.push_back(sym);
   }
 }
@@ -2110,7 +2310,7 @@ void GnuHashSection<E>::copy_buf(Context<E> &ctx) {
   for (i64 i = 0; i < syms.size(); i++) {
     constexpr i64 word_bits = sizeof(Word<E>) * 8;
 
-    u32 h = syms[i]->get_djb_hash(ctx);
+    u32 h = syms[i]->aux->djb_hash;
     indices[i] = h % num_buckets;
 
     i64 idx = (h / word_bits) % num_bloom;
@@ -2130,7 +2330,7 @@ void GnuHashSection<E>::copy_buf(Context<E> &ctx) {
   for (i64 i = 0; i < syms.size(); i++) {
     // The last entry in a chain must be terminated with an entry with
     // least-significant bit 1.
-    u32 h = syms[i]->get_djb_hash(ctx);
+    u32 h = syms[i]->aux->djb_hash;
     if (i == syms.size() - 1 || indices[i] != indices[i + 1])
       table[i] = h | 1;
     else
@@ -2196,7 +2396,7 @@ MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
   name = get_merged_output_name(ctx, name, flags, entsize, addralign);
 
   auto find = [&]() -> MergedSection * {
-    for (std::unique_ptr<MergedSection<E>> &osec : ctx.merged_sections)
+    for (ArenaObjectPtr<MergedSection<E>> &osec : ctx.merged_sections)
       if (name == osec->name && flags == osec->shdr.sh_flags &&
           shdr.sh_type == osec->shdr.sh_type &&
           entsize == osec->shdr.sh_entsize)
@@ -2217,7 +2417,9 @@ MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
   if (MergedSection *osec = find())
     return osec;
 
-  MergedSection *osec = new MergedSection(name, flags, shdr.sh_type, entsize);
+  void *buf = ctx.arena.template allocate<MergedSection>(1);
+  MergedSection *osec =
+    new (buf) MergedSection(name, flags, shdr.sh_type, entsize);
   ctx.merged_sections.emplace_back(osec);
   return osec;
 }
@@ -2232,7 +2434,7 @@ MergedSection<E>::insert(Context<E> &ctx, std::string_view data, u64 hash,
   bool is_alive = !ctx.arg.gc_sections || !(this->shdr.sh_flags & SHF_ALLOC);
 
   SectionFragment<E> *frag =
-    map.insert(data, hash, SectionFragment(this, is_alive)).first;
+    map.insert(data, hash, SectionFragment(ctx, *this, is_alive)).first;
   update_maximum(frag->p2align, p2align);
   return frag;
 }
@@ -2768,7 +2970,7 @@ void CopyrelSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
 
   symbols.push_back(sym);
 
-  SharedFile<E> &file = *(SharedFile<E> *)sym->file;
+  SharedFile<E> &file = *sym->file->to_dso();
   i64 alignment = file.get_alignment(sym);
   u64 offset = align_to(this->shdr.sh_size, alignment);
 
@@ -2782,7 +2984,8 @@ void CopyrelSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
   // aliases. If one of the symbols is copied by a copy relocation, other
   // symbols have to refer to the copied place as well.
   for (Symbol<E> *sym2 : file.get_symbols_at(sym)) {
-    sym2->add_aux(ctx);
+    if (!sym2->aux)
+      sym2->aux = ctx.arena.template make<SymbolAux<E>>();
     sym2->is_imported = true;
     sym2->is_exported = true;
     sym2->has_copyrel = true;
@@ -2793,15 +2996,16 @@ void CopyrelSection<E>::add_symbol(Context<E> &ctx, Symbol<E> *sym) {
 }
 
 template <typename E>
-std::vector<ElfRel<E>>
-CopyrelSection<E>::collect_dynrels(Context<E> &ctx) const {
-  std::vector<ElfRel<E>> rels;
+i64 CopyrelSection<E>::get_num_dynrels(Context<E> &) const {
+  return symbols.size();
+}
 
-  for (Symbol<E> *sym : symbols)
-    rels.emplace_back(sym->get_addr(ctx), E::R_COPY,
-                      sym->get_dynsym_idx(ctx), 0);
-
-  return rels;
+template <typename E>
+void CopyrelSection<E>::write_dynrels(Context<E> &ctx, ElfRel<E> *buf) const {
+  for (i64 i = 0; i < (i64)symbols.size(); i++) {
+    Symbol<E> &sym = *symbols[i];
+    buf[i] = ElfRel<E>(sym.get_addr(ctx), E::R_COPY, sym.get_dynsym_idx(ctx), 0);
+  }
 }
 
 // .gnu.version section contains version indices as a parallel array for
@@ -2858,7 +3062,7 @@ void VerneedSection<E>::construct(Context<E> &ctx) {
     return;
 
   ranges::stable_sort(syms, {}, [](Symbol<E> *x) {
-    return std::tuple{((SharedFile<E> *)x->file)->soname, x->ver_idx};
+    return std::tuple{x->file->to_dso()->soname, x->ver_idx};
   });
 
   // Resize .gnu.version
@@ -2909,7 +3113,7 @@ void VerneedSection<E>::construct(Context<E> &ctx) {
   // Create version entries.
   for (i64 i = 0; i < syms.size(); i++) {
     if (i == 0 || syms[i - 1]->file != syms[i]->file) {
-      start_group(*(SharedFile<E> *)syms[i]->file);
+      start_group(*syms[i]->file->to_dso());
       add_entry(syms[i]->get_version());
     } else if (syms[i - 1]->ver_idx != syms[i]->ver_idx) {
       add_entry(syms[i]->get_version());
@@ -2942,7 +3146,7 @@ void VerdefSection<E>::construct(Context<E> &ctx) {
   // Handle --default-symver
   if (ctx.arg.default_symver)
     for (Symbol<E> *sym : ctx.dynsym->symbols)
-      if (sym && !sym->file->is_dso)
+      if (sym && !sym->file->is_dso && !sym->esym().is_undef())
         if (u16 ver = sym->ver_idx;
             ver == VER_NDX_GLOBAL || ver == VER_NDX_UNSPECIFIED)
           sym->ver_idx = VER_NDX_LAST_RESERVED + 1;
@@ -2951,9 +3155,16 @@ void VerdefSection<E>::construct(Context<E> &ctx) {
   ctx.versym->contents.resize(ctx.dynsym->symbols.size(), VER_NDX_GLOBAL);
   ctx.versym->contents[0] = VER_NDX_LOCAL;
 
-  for (Symbol<E> *sym : ctx.dynsym->symbols)
-    if (sym && !sym->file->is_dso && sym->ver_idx != VER_NDX_UNSPECIFIED)
+  for (Symbol<E> *sym : ctx.dynsym->symbols) {
+    if (!sym || sym->file->is_dso)
+      continue;
+
+    // An unversioned undefined symbol takes version index 0.
+    if (sym->ver_idx != VER_NDX_UNSPECIFIED)
       ctx.versym->contents[sym->get_dynsym_idx(ctx)] = sym->ver_idx;
+    else if (sym->esym().is_undef())
+      ctx.versym->contents[sym->get_dynsym_idx(ctx)] = VER_NDX_LOCAL;
+  }
 
   // Allocate a buffer for .gnu.version_d and write to it
   contents.resize((sizeof(ElfVerdef<E>) + sizeof(ElfVerdaux<E>)) *
@@ -3216,17 +3427,18 @@ get_symidx_addend(Context<E> &ctx, InputSection<E> &isec, const ElfRel<E> &rel)
     i64 frag_addend;
     std::tie(frag, frag_addend) = isec.get_fragment(ctx, rel);
     if (frag)
-      return {frag->output_section.shndx, frag->offset + frag_addend};
+      return {frag->get_output_section(ctx).shndx,
+              frag->offset + frag_addend};
   }
 
   if (sym.esym().st_type == STT_SECTION) {
     if (SectionFragment<E> *frag = sym.get_frag())
-      return {frag->output_section.shndx,
+      return {frag->get_output_section(ctx).shndx,
               frag->offset + sym.value + get_addend(isec, rel)};
 
-    InputSection<E> *isec2 = sym.get_input_section();
-    if (OutputSection<E> *osec = isec2->output_section)
-      return {osec->shndx, get_addend(isec, rel) + isec2->offset};
+    if (InputSection<E> *isec2 = sym.get_input_section())
+      if (OutputSection<E> *osec = isec2->output_section)
+        return {osec->shndx, get_addend(isec, rel) + isec2->offset};
 
     // This is usually a dead debug section referring to a
     // COMDAT-eliminated section.
@@ -3257,19 +3469,18 @@ void RelocSection<E>::copy_buf(Context<E> &ctx) {
       if constexpr (is_riscv<E> || is_loongarch<E>)
         r_offset -= get_r_delta(isec, rel.r_offset);
 
-      buf[j++] = ElfRel<E>(r_offset, rel.r_type, symidx, addend);
+      // SH4 object files store addends in the relocated places rather
+      // than in r_addend, and the relocation records we emit here are
+      // meant to be consumed as if they were in an object file, so we
+      // follow that convention.
+      buf[j++] = ElfRel<E>(r_offset, rel.r_type, symidx,
+                           is_sh4<E> ? 0 : addend);
 
       if (ctx.arg.relocatable) {
         u8 *base = ctx.buf + isec.output_section->shdr.sh_offset + isec.offset;
         write_addend(base + rel.r_offset, addend, rel);
       }
     }
-
-    // Relaxation can also change a relocation's type to match the rewritten
-    // instruction. No relaxation happens in -r mode, so types are left as-is.
-    if constexpr (is_riscv<E> || is_loongarch<E>)
-      if (!ctx.arg.relocatable)
-        rewrite_reloc_types(ctx, isec, buf);
   });
 }
 

src/output-file-unix.cc

@@ -1,3 +1,4 @@
+#include "config.h"
 #include "mold.h"
 
 #include <fcntl.h>
@@ -58,18 +59,34 @@ public:
 
     output_tmpfile = (char *)save_string(ctx, tmpfile).data();
 
-#ifdef __linux__
-    // Calling falllocate speeds up later linking passes on ext4,
-    // while it just takes time with not benefits on tmpfs.
+#if HAVE_FALLOCATE
+    // Calling fallocate speeds up later linking passes on ext4 by
+    // taking disk block allocation out of the page-fault handler.
+    // On tmpfs, it instead makes things much slower: the kernel
+    // allocates and zeroes every page of the range inside the
+    // syscall, on one thread, which takes ~1.8 s for a 5 GiB file.
     if (struct statfs fs;
         fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
       fallocate(this->fd, 0, 0, filesize);
 #endif
 
-    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+    // We map the file with twice as much address space as its size, so
+    // that extend() can grow the file into the mapping in place.
+    // Touching the mapping beyond the end of the file is not allowed,
+    // but growing the file with ftruncate makes the tail of the
+    // mapping accessible without any further mmap call.
+    vasize = filesize * 2;
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                            MAP_SHARED, this->fd, 0);
-    if (this->buf == MAP_FAILED)
-      Fatal(ctx) << path << ": mmap failed: " << errno_string();
+
+    if (this->buf == MAP_FAILED) {
+      // If the address space is too tight, map just the file.
+      vasize = filesize;
+      this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << path << ": mmap failed: " << errno_string();
+    }
 
     mold::output_buffer_start = this->buf;
     mold::output_buffer_end = this->buf + filesize;
@@ -80,20 +97,51 @@ public:
       ::close(fd2);
   }
 
+  // Extend the file so that the caller can fill the appended data
+  // through the tail of the mapping. This is called at most once per
+  // output file.
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    i64 mapsize = this->filesize;
+
+    if (ftruncate(this->fd, mapsize + size) == -1)
+      Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+#if HAVE_FALLOCATE
+    if (struct statfs fs;
+        fstatfs(this->fd, &fs) || fs.f_type != TMPFS_MAGIC)
+      fallocate(this->fd, 0, mapsize, size);
+#endif
+
+    if (mapsize + size > vasize) {
+      // The appended data does not fit in the mapping. Map the grown
+      // file again, moving the buffer.
+      munmap(this->buf, vasize);
+      vasize = mapsize + size;
+
+      this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                             MAP_SHARED, this->fd, 0);
+      if (this->buf == MAP_FAILED)
+        Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+      ctx.buf = this->buf;
+      mold::output_buffer_start = this->buf;
+    }
+
+    this->filesize += size;
+    mold::output_buffer_end = this->buf + mapsize + size;
+
+#ifdef MADV_HUGEPAGE
+    madvise(this->buf, mapsize + size, MADV_HUGEPAGE);
+#endif
+    return this->buf + mapsize;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     if (!this->is_unmapped)
-      munmap(this->buf, this->filesize);
-
-    if (!this->buf2) {
-      ::close(this->fd);
-    } else {
-      FILE *out = fdopen(this->fd, "w");
-      fseek(out, 0, SEEK_END);
-      fwrite(this->buf2, this->buf2_size, 1, out);
-      fclose(out);
-    }
+      munmap(this->buf, vasize);
+    ::close(this->fd);
 
     // If an output file already exists, open a file and then remove it.
     // This is the fastest way to unlink a file, as it does not make the
@@ -109,6 +157,9 @@ public:
 
 private:
   int fd2 = -1;
+
+  // Size of the file mapping, which may extend past the end of the file.
+  i64 vasize = 0;
 };
 
 template <typename E>
@@ -131,14 +182,7 @@ OutputFile<E>::open(Context<E> &ctx, std::string path, i64 filesize, int perm) {
 
 #ifdef MADV_HUGEPAGE
   // Enable transparent huge page for an output memory-mapped file.
-  // On Linux, it has an effect only on tmpfs mounted with `huge=advise`,
-  // but it can make the linker ~10% faster. You can try it by creating
-  // a tmpfs with the following commands
-  //
-  //  $ mkdir tmp
-  //  $ sudo mount -t tmpfs -o size=2G,huge=advise none tmp
-  //
-  // and then specifying a path under the directory as an output file.
+  // Linking a Chromium debug build is ~20% faster with this madvise call.
   madvise(file->buf, filesize, MADV_HUGEPAGE);
 #endif
 
@@ -170,10 +214,20 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
   if (ftruncate(this->fd, filesize) == -1)
     Fatal(ctx) << "ftruncate failed: " << errno_string();
 
-  this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+  // As in MemoryMappedOutputFile, we map the file with twice as much
+  // address space as its size so that extend() can grow the file into
+  // the mapping in place.
+  vasize = filesize * 2;
+  this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
                          MAP_SHARED, this->fd, 0);
-  if (this->buf == MAP_FAILED)
-    Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+  if (this->buf == MAP_FAILED) {
+    vasize = filesize;
+    this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+  }
 
   this->filesize = filesize;
   mold::output_buffer_start = this->buf;
@@ -181,17 +235,36 @@ void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {
 }
 
 template <typename E>
-void LockingOutputFile<E>::close(Context<E> &ctx) {
-  if (!this->is_unmapped)
-    munmap(this->buf, this->filesize);
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  i64 mapsize = this->filesize;
 
-  if (this->buf2) {
-    FILE *out = fdopen(this->fd, "w");
-    fseek(out, 0, SEEK_END);
-    fwrite(this->buf2, this->buf2_size, 1, out);
-    fclose(out);
+  if (ftruncate(this->fd, mapsize + size) == -1)
+    Fatal(ctx) << "ftruncate failed: " << errno_string();
+
+  if (mapsize + size > vasize) {
+    // The appended data does not fit in the mapping. Map the grown
+    // file again, moving the buffer.
+    munmap(this->buf, vasize);
+    vasize = mapsize + size;
+
+    this->buf = (u8 *)mmap(nullptr, vasize, PROT_READ | PROT_WRITE,
+                           MAP_SHARED, this->fd, 0);
+    if (this->buf == MAP_FAILED)
+      Fatal(ctx) << this->path << ": mmap failed: " << errno_string();
+
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
   }
 
+  this->filesize += size;
+  mold::output_buffer_end = this->buf + mapsize + size;
+  return this->buf + mapsize;
+}
+
+template <typename E>
+void LockingOutputFile<E>::close(Context<E> &ctx) {
+  if (!this->is_unmapped)
+    munmap(this->buf, vasize);
   ::close(this->fd);
 }
 

src/output-file-win32.cc

@@ -39,22 +39,36 @@ public:
       CloseHandle(handle);
   }
 
+  u8 *extend(Context<E> &ctx, i64 size) override {
+    UnmapViewOfFile(this->buf);
+
+    HANDLE map = CreateFileMapping(handle, nullptr, PAGE_READWRITE, 0,
+                                   this->filesize + size, nullptr);
+    if (!map)
+      Fatal(ctx) << this->path << ": CreateFileMapping failed: "
+                 << GetLastError();
+
+    this->buf = (u8 *)MapViewOfFile(map, FILE_MAP_WRITE, 0, 0,
+                                    this->filesize + size);
+    if (!this->buf)
+      Fatal(ctx) << this->path << ": MapViewOfFile failed: " << GetLastError();
+
+    CloseHandle(map);
+
+    u8 *space = this->buf + this->filesize;
+    memset(space, 0, size);
+
+    this->filesize += size;
+    ctx.buf = this->buf;
+    mold::output_buffer_start = this->buf;
+    mold::output_buffer_end = this->buf + this->filesize;
+    return space;
+  }
+
   void close(Context<E> &ctx) override {
     Timer t(ctx, "close_file");
 
     UnmapViewOfFile(this->buf);
-
-    if (this->buf2) {
-      if (SetFilePointer(handle, 0, nullptr, FILE_END) == INVALID_SET_FILE_POINTER)
-        Fatal(ctx) << this->path << ": SetFilePointer failed: "
-                   << GetLastError();
-
-      DWORD written;
-      if (!WriteFile(handle, this->buf2, this->buf2_size, &written,
-                     nullptr))
-        Fatal(ctx) << this->path << ": WriteFile failed: " << GetLastError();
-    }
-
     CloseHandle(handle);
     handle = INVALID_HANDLE_VALUE;
   }
@@ -106,6 +120,11 @@ LockingOutputFile<E>::LockingOutputFile(Context<E> &ctx, std::string path,
 template <typename E>
 void LockingOutputFile<E>::resize(Context<E> &ctx, i64 filesize) {}
 
+template <typename E>
+u8 *LockingOutputFile<E>::extend(Context<E> &ctx, i64 size) {
+  return nullptr;
+}
+
 template <typename E>
 void LockingOutputFile<E>::close(Context<E> &ctx) {}
 

src/passes.cc

@@ -276,32 +276,123 @@ static void resolve_default_symver(Context<E> &ctx) {
   append(files, ctx.dsos);
 
   tbb::parallel_for_each(files, [](InputFile<E> *file) {
-    std::span<Symbol<E> *> syms = file->get_global_syms();
+    std::span<ArenaPtr<Symbol<E>>> syms = file->get_global_syms();
     for (i64 i = 0; i < syms.size(); i++)
       if (syms[i]->is_versioned_default)
-        syms[i] = (Symbol<E> *)syms[i]->origin;
+        syms[i] = syms[i]->get_symbol_origin();
   });
 }
 
 template <typename E>
 static void clear_symbols(Context<E> &ctx) {
-  std::vector<InputFile<E> *> files;
-  append(files, ctx.objs);
-  append(files, ctx.dsos);
+  ctx.symbol_map.parallel_for_each([](Symbol<E> &sym) {
+    if (sym.file) {
+      sym.file = nullptr;
+      sym.origin = nullptr;
+      sym.value = -1;
+      sym.sym_idx = -1;
+      sym.ver_idx = VER_NDX_UNSPECIFIED;
+      sym.is_weak = false;
+      sym.is_imported = false;
+      sym.is_exported = false;
+    }
+  });
+}
 
-  tbb::parallel_for_each(files, [](InputFile<E> *file) {
-    for (Symbol<E> *sym : file->get_global_syms()) {
-      if (__atomic_load_n(&sym->file, __ATOMIC_ACQUIRE) == file) {
-        sym->origin = 0;
-        sym->value = -1;
-        sym->sym_idx = -1;
-        sym->ver_idx = VER_NDX_UNSPECIFIED;
-        sym->is_weak = false;
-        sym->is_imported = false;
-        sym->is_exported = false;
-        __atomic_store_n(&sym->file, nullptr, __ATOMIC_RELEASE);
+// Creates a Symbol for each global symbol name recorded during file
+// parsing and fills in the files' symbol pointers. Each hash shard of
+// the symbol table is populated by a single thread, so no
+// synchronization is needed, unlike interning symbols directly into a
+// concurrent hash table as files are parsed.
+template <typename E>
+void gather_symbols(Context<E> &ctx) {
+  Timer t(ctx, "gather_symbols");
+
+  ctx.symbol_map.gather(
+    [&](std::string_view key, Symbol<E> &sym) {
+      sym.has_map_name = true;
+      sym.set_name(key.substr(0, key.find('@')));
+      sym.demangle = ctx.arg.demangle;
+    });
+}
+
+template <typename E>
+static void record_comdat_owner(Symbol<E> &signature, i32 priority) {
+  // Symbol resolution is clear while COMDAT groups are selected, so sym_idx
+  // can temporarily hold the winning file priority.
+  std::atomic_ref<i32> owner(signature.sym_idx);
+  i32 old = owner.load(std::memory_order_relaxed);
+  while ((old == -1 || priority < old) &&
+         !owner.compare_exchange_weak(old, priority,
+                                      std::memory_order_relaxed));
+}
+
+// Select COMDAT groups and construct input sections. If LTO will run,
+// the first invocation also constructs the losing copies of COMDAT
+// members because this function runs again after LTO and may then
+// select a different winner.
+template <typename E>
+static void parse_input_sections(Context<E> &ctx) {
+  Timer t(ctx, "parse_input_sections");
+
+  // Read COMDAT metadata and choose an owner among reachable regular objects.
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    if (!file->is_reachable)
+      return;
+
+    if (file->mf && !file->is_lto_obj && !file->sections_parsed)
+      file->read_section_metadata(ctx);
+
+    for (ComdatGroupRef<E> &ref : file->comdat_groups)
+      record_comdat_owner(*ref.signature(ctx), file->priority);
+  });
+
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    if (file->is_reachable)
+      for (ComdatGroupRef<E> &ref : file->comdat_groups)
+        ref.is_owner = (ref.signature(ctx)->sym_idx == file->priority);
+  });
+
+  // LTO plugin symbol tables may not enumerate all section-level helper
+  // symbols (e.g. some thunks). Therefore, an IR file may claim a signature
+  // only if no reachable regular object has already claimed it.
+  for (ObjectFile<E> *file : ctx.objs) {
+    if (file->is_reachable) {
+      for (i64 i = 0; i < file->lto_comdat_signatures.size(); i++) {
+        if (Symbol<E> *sig = file->lto_comdat_signatures[i]) {
+          if (sig->sym_idx == -1)
+            sig->sym_idx = file->priority;
+          file->lto_comdat_discarded[i] = (sig->sym_idx != file->priority);
+        }
       }
     }
+  }
+
+  // Restore sym_idx before the final symbol-resolution pass.
+  ctx.symbol_map.parallel_for_each([](Symbol<E> &sym) { sym.sym_idx = -1; });
+
+  // LTO can change archive extraction and therefore the winning COMDAT group.
+  // Construct the losing copies too so one can become the winner after LTO.
+  bool keep_discarded_comdat =
+    ranges::any_of(ctx.objs, [](ObjectFile<E> *file) {
+      return file->is_reachable &&
+             (file->is_lto_obj || file->is_gcc_offload_obj);
+    });
+
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    if (file->is_reachable && file->mf && !file->is_lto_obj &&
+        !file->sections_parsed)
+      file->parse_sections(ctx, keep_discarded_comdat);
+  });
+
+  // Apply the selection to all group members. This also updates sections
+  // parsed before LTO if ownership has changed.
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    if (file->is_reachable)
+      for (ComdatGroupRef<E> &ref : file->comdat_groups)
+        for (u32 i : ref.members(*file))
+          if (InputSection<E> *isec = file->sections[i])
+            isec->is_alive = ref.is_owner;
   });
 }
 
@@ -309,6 +400,8 @@ template <typename E>
 void resolve_symbols(Context<E> &ctx) {
   Timer t(ctx, "resolve_symbols");
 
+  gather_symbols(ctx);
+
   std::vector<InputFile<E> *> files;
   append(files, ctx.objs);
   append(files, ctx.dsos);
@@ -329,14 +422,11 @@ void resolve_symbols(Context<E> &ctx) {
     // symbol resolution from scratch with the flag to skip that symbol
     // next time. This should be rare.
     std::atomic_bool redo = false;
-    tbb::parallel_for_each(ctx.dsos, [&](SharedFile<E> *file) {
-      if (file->is_reachable) {
-        for (Symbol<E> *sym : file->symbols) {
-          if (sym->file == file && sym->visibility == STV_HIDDEN) {
-            sym->skip_dso = true;
-            redo = true;
-          }
-        }
+    ctx.symbol_map.parallel_for_each([&](Symbol<E> &sym) {
+      if (sym.file && sym.file->is_dso && sym.file->is_reachable &&
+          sym.visibility == STV_HIDDEN) {
+        sym.skip_dso = true;
+        redo = true;
       }
     });
 
@@ -353,42 +443,10 @@ void resolve_symbols(Context<E> &ctx) {
   // To redo symbol resolution, we want to clear the state first.
   clear_symbols(ctx);
 
-  // COMDAT elimination needs to happen exactly here.
-  //
-  // It needs to be after archive extraction, otherwise we might
-  // assign COMDAT leader to an archive member that is not supposed to
-  // be extracted.
-  //
-  // It needs to happen before the final symbol resolution, otherwise
-  // we could eliminate a symbol that is already resolved to and cause
-  // dangling references.
-  tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-    if (file->is_reachable)
-      for (ComdatGroupRef<E> &ref : file->comdat_groups)
-        update_minimum(ref.group->owner, file->priority);
-  });
-
-  // LTO plugin symbol tables may not enumerate all section-level helper
-  // symbols (e.g. some thunks). If an LTO file wins COMDAT ownership for
-  // a key shared with regular object files, section elimination may discard
-  // needed regular COMDAT members and create dangling relocations.
-  //
-  // Therefore, only let IR files claim ownership for COMDAT keys that have
-  // no reachable regular-object owner.
-  for (ObjectFile<E> *file : ctx.objs)
-    if (file->is_reachable)
-      for (ComdatGroup *g : file->lto_comdat_groups)
-        if (g && g->owner == (u32)-1)
-          g->owner = file->priority;
-
-  tbb::parallel_for_each(ctx.objs, [](ObjectFile<E> *file) {
-    if (file->is_reachable)
-      for (ComdatGroupRef<E> &ref : file->comdat_groups)
-        if (ref.group->owner != file->priority)
-          for (u32 i : ref.members)
-            if (InputSection<E> *isec = file->sections[i].get())
-              isec->is_alive = false;
-  });
+  // Parse input sections after archive extraction, so COMDAT selection
+  // considers only reachable objects. This must happen before final symbol
+  // resolution; otherwise discarding a group could leave a dangling reference.
+  parse_input_sections(ctx);
 
   // Redo symbol resolution
   tbb::parallel_for_each(files, [&](InputFile<E> *file) {
@@ -466,7 +524,7 @@ void create_merged_sections(Context<E> &ctx) {
   });
 
   tbb::parallel_for_each(ctx.merged_sections,
-                         [&](std::unique_ptr<MergedSection<E>> &sec) {
+                         [&](ArenaObjectPtr<MergedSection<E>> &sec) {
     if (sec->shdr.sh_flags & SHF_ALLOC)
       sec->resolve(ctx);
   });
@@ -577,6 +635,8 @@ template <typename E>
 static OutputSectionKey
 get_output_section_key(Context<E> &ctx, InputSection<E> &isec,
                        bool ctors_in_init_array) {
+  std::string_view name = isec.name();
+
   // If .init_array/.fini_array exist, .ctors/.dtors must be merged
   // with them.
   //
@@ -586,14 +646,14 @@ get_output_section_key(Context<E> &ctx, InputSection<E> &isec,
   // We do not place them into .init_array/.fini_array because such
   // invalid pointer values would simply make the program to crash.
   if (ctors_in_init_array && !isec.get_rels(ctx).empty()) {
-    if (isec.name == ".ctors" || isec.name.starts_with(".ctors."))
+    if (name == ".ctors" || name.starts_with(".ctors."))
       return {".init_array", SHT_INIT_ARRAY};
-    if (isec.name == ".dtors" || isec.name.starts_with(".dtors."))
+    if (name == ".dtors" || name.starts_with(".dtors."))
       return {".fini_array", SHT_FINI_ARRAY};
   }
 
   const ElfShdr<E> &shdr = isec.shdr();
-  std::string_view name = get_output_name(ctx, isec.name, shdr.sh_flags);
+  name = get_output_name(ctx, name, shdr.sh_flags);
   u64 type = canonicalize_type<E>(name, shdr.sh_type);
   return {name, type};
 }
@@ -645,7 +705,7 @@ void create_output_sections(Context<E> &ctx) {
     // It makes a noticeable difference if we have millions of input sections.
     MapType &cache = caches.local();
 
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (!isec || !isec->is_alive)
         continue;
 
@@ -654,7 +714,8 @@ void create_output_sections(Context<E> &ctx) {
                      ~SHF_COMPRESSED & ~SHF_GNU_RETAIN;
 
       if (ctx.arg.relocatable && (sh_flags & SHF_GROUP)) {
-        OutputSection<E> *osec = new OutputSection<E>(isec->name, shdr.sh_type);
+        OutputSection<E> *osec =
+          new OutputSection<E>(isec->name(), shdr.sh_type);
         osec->sh_flags = sh_flags;
         isec->output_section = osec;
         ctx.osec_pool.emplace_back(osec);
@@ -701,9 +762,9 @@ void create_output_sections(Context<E> &ctx) {
     osec->members_vec.resize(ctx.objs.size());
 
   tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) {
-    for (std::unique_ptr<InputSection<E>> &isec : ctx.objs[i]->sections)
+    for (InputSection<E> *isec : ctx.objs[i]->sections)
       if (isec && isec->output_section)
-        isec->output_section->members_vec[i].push_back(isec.get());
+        isec->output_section->members_vec[i].push_back(isec);
   });
 
   // Compute section alignment
@@ -730,7 +791,7 @@ void create_output_sections(Context<E> &ctx) {
   std::vector<Chunk<E> *> chunks;
   for (std::unique_ptr<OutputSection<E>> &osec : ctx.osec_pool)
     chunks.push_back(osec.get());
-  for (std::unique_ptr<MergedSection<E>> &osec : ctx.merged_sections)
+  for (ArenaObjectPtr<MergedSection<E>> &osec : ctx.merged_sections)
     chunks.push_back(osec.get());
 
   // Sections are added to the section lists in an arbitrary order
@@ -748,7 +809,8 @@ void create_output_sections(Context<E> &ctx) {
 // symbols.
 template <typename E>
 void create_internal_file(Context<E> &ctx) {
-  ObjectFile<E> *obj = new ObjectFile<E>;
+  ObjectFile<E> *obj =
+    ctx.arena.template make<ObjectFile<E>>(ctx);
   ctx.obj_pool.emplace_back(obj);
   ctx.internal_obj = obj;
   ctx.objs.push_back(obj);
@@ -756,13 +818,14 @@ void create_internal_file(Context<E> &ctx) {
   // Create linker-synthesized symbols.
   ctx.internal_esyms.resize(1);
 
-  obj->symbols.push_back(new Symbol<E>);
+  Symbol<E> *dummy = ctx.arena.template make<Symbol<E>>();
+  obj->symbols.emplace_back(dummy);
   obj->first_global = 1;
   obj->is_reachable = true;
   obj->priority = 1;
 
   auto add = [&](Symbol<E> *sym) {
-    obj->symbols.push_back(sym);
+    obj->symbols.emplace_back(sym);
 
     // An actual value will be set to a linker-synthesized symbol by
     // fix_synthetic_symbols(). Until then, `value` doesn't have a valid
@@ -833,7 +896,7 @@ void add_synthetic_symbols(Context<E> &ctx) {
 
     Symbol<E> *sym = get_symbol(ctx, name);
     sym->value = 0xdeadbeef; // unique dummy value
-    obj.symbols.push_back(sym);
+    obj.symbols.emplace_back(sym);
     return sym;
   };
 
@@ -943,9 +1006,9 @@ void add_synthetic_symbols(Context<E> &ctx) {
         esym.ppc64_local_entry = sym2->esym().ppc64_local_entry;
 
       if (sym2->is_absolute())
-        sym1->origin = 0;
+        sym1->origin = nullptr;
     } else {
-      sym1->origin = 0;
+      sym1->origin = nullptr;
     }
   }
 }
@@ -1022,7 +1085,7 @@ R"(# This is an output of the mold linker's --print-dependencies option.
   };
 
   for (ObjectFile<E> *file : ctx.objs) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (!isec)
         continue;
 
@@ -1130,16 +1193,14 @@ void check_duplicate_symbols(Context<E> &ctx) {
 
       // Skip if the symbol is a deduplicated comdat symbol that is in
       // an IR file.
-      if (file->is_lto_obj)
-        if (ComdatGroup *g = file->lto_comdat_groups[i])
-          if (g->owner != file->priority)
-            continue;
+      if (file->is_lto_obj && file->lto_comdat_discarded[i])
+        continue;
 
       // Skip if one side is an LTO IR object and the other is not.
       // The LTO backend resolves conflicts between IR and regular objects
       // on its own; only IR-vs-IR duplicates need to be caught here.
       if (!sym.file->is_dso &&
-          file->is_lto_obj != ((ObjectFile<E> *)sym.file)->is_lto_obj)
+          file->is_lto_obj != sym.file->to_obj()->is_lto_obj)
         continue;
 
       Error(ctx) << "duplicate symbol: " << *file << ": " << *sym.file
@@ -1150,6 +1211,37 @@ void check_duplicate_symbols(Context<E> &ctx) {
   ctx.checkpoint();
 }
 
+// A default-versioned symbol `foo@@VER` can also be referred to as
+// `foo@VER`, so exporting both `foo@@VER` and `foo@VER` would produce a
+// dynamic symbol table with two definitions of the same versioned name,
+// and which one a versioned reference binds to would be up to the
+// dynamic loader. GNU ld and lld reject this; so do we.
+template <typename E>
+void check_symbol_version_conflicts(Context<E> &ctx) {
+  if (!ctx.dynsym || ctx.arg.allow_multiple_definition)
+    return;
+
+  Timer t(ctx, "check_symbol_version_conflicts");
+
+  for (i64 i = 1; i < ctx.dynsym->symbols.size(); i++) {
+    Symbol<E> *sym = ctx.dynsym->symbols[i];
+    if (sym->file->is_dso || sym->is_weak ||
+        sym->ver_idx == VER_NDX_UNSPECIFIED ||
+        !(sym->ver_idx & VERSYM_HIDDEN))
+      continue;
+
+    Symbol<E> *sym2 = get_symbol(ctx, sym->name());
+    if (sym2 != sym && sym2->file && !sym2->file->is_dso && !sym2->is_weak &&
+        sym2->ver_idx == (sym->ver_idx & ~VERSYM_HIDDEN)) {
+      ObjectFile<E> *file = sym->file->to_obj();
+      Error(ctx) << "duplicate symbol: " << *file << ": " << *sym2->file
+                 << ": " << file->get_symbol_name(sym->sym_idx);
+    }
+  }
+
+  ctx.checkpoint();
+}
+
 // GCC and Clang set the SHT_NOBITS flag for an output section only if the
 // section name is .bss or similar. Sections with nonstandard names, such
 // as those defined with __attribute__((section(".sectname"))), are always
@@ -1166,17 +1258,20 @@ void convert_zero_to_bss(Context<E> &ctx) {
     if (!file->is_reachable)
       return;
 
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
-      if (isec && isec->is_alive &&
-          isec->shdr().sh_type == SHT_PROGBITS &&
+    for (InputSection<E> *isec : file->sections) {
+      if (!isec || !isec->is_alive)
+        continue;
+
+      std::string_view contents = isec->get_contents();
+      if (isec->shdr().sh_type == SHT_PROGBITS &&
           (isec->shdr().sh_flags & SHF_ALLOC) &&
           (isec->shdr().sh_flags & SHF_WRITE) &&
           !(isec->shdr().sh_flags & SHF_EXECINSTR) &&
           isec->get_rels(ctx).empty() &&
-          !isec->contents.empty() &&
-          isec->contents.find_first_not_of('\0') == isec->contents.npos) {
+          !contents.empty() &&
+          contents.find_first_not_of('\0') == contents.npos) {
         isec->shdr().sh_type = SHT_NOBITS;
-        isec->contents = {};
+        isec->contents = nullptr;
       }
     }
   });
@@ -1184,7 +1279,7 @@ void convert_zero_to_bss(Context<E> &ctx) {
 
 template <typename E>
 static bool has_dso_definition(Context<E> &ctx, Symbol<E> &sym) {
-  for (std::unique_ptr<SharedFile<E>> &file : ctx.dso_pool)
+  for (ArenaObjectPtr<SharedFile<E>> &file : ctx.dso_pool)
     for (i64 i = 0; i < file->symbols.size(); i++)
       if (file->symbols[i] == &sym && !file->elf_syms[i].is_undef())
         return true;
@@ -1216,7 +1311,7 @@ void check_shlib_undefined(Context<E> &ctx) {
   // symbol might be defined by that library.
   auto do_test = [&] {
     std::unordered_set<std::string_view> sonames;
-    for (std::unique_ptr<SharedFile<E>> &file : ctx.dso_pool)
+    for (ArenaObjectPtr<SharedFile<E>> &file : ctx.dso_pool)
       sonames.insert(file->soname);
 
     for (SharedFile<E> *file : ctx.dsos)
@@ -1307,7 +1402,7 @@ void check_symbol_types(Context<E> &ctx) {
 template <typename E>
 static i64 get_init_fini_priority(InputSection<E> *isec) {
   static std::regex re(R"(\.(\d+)$)", std::regex_constants::optimize);
-  std::string_view name = isec->name;
+  std::string_view name = isec->name();
   std::cmatch m;
   if (std::regex_search(name.data(), name.data() + name.size(), m, re))
     return std::stoi(m[1]);
@@ -1329,7 +1424,7 @@ static i64 get_ctor_dtor_priority(InputSection<E> *isec) {
   if (std::regex_search(isec->file.filename, re2))
     return 65536;
 
-  std::string_view name = isec->name;
+  std::string_view name = isec->name();
   std::cmatch m;
   if (std::regex_search(name.data(), name.data() + name.size(), m, re3))
     return std::stoi(m[1]);
@@ -1355,8 +1450,8 @@ void sort_init_fini(Context<E> &ctx) {
         std::vector<Entry> vec;
 
         for (InputSection<E> *isec : osec->members) {
-          if (isec->name.starts_with(".ctors") ||
-              isec->name.starts_with(".dtors"))
+          std::string_view name = isec->name();
+          if (name.starts_with(".ctors") || name.starts_with(".dtors"))
             vec.push_back({isec, 65535 - get_ctor_dtor_priority(isec)});
           else
             vec.push_back({isec, get_init_fini_priority(isec)});
@@ -1436,8 +1531,8 @@ static bool is_dwarf32(Context<E> &ctx, InputSection<E> *isec) {
   // we need to uncompress it before accessing `isec->contents`.
   isec->uncompress(ctx);
 
-  u8 *p = (u8 *)isec->contents.data() + *(U64<E> *)(buf + 4) + 12;
-  u8 *end = (u8 *)isec->contents.data() + isec->sh_size;
+  u8 *p = isec->contents + *(U64<E> *)(buf + 4) + 12;
+  u8 *end = isec->contents + isec->sh_size;
 
   while (end - p >= 12) {
     if (*(U32<E> *)p != 0xffff'ffff)
@@ -1490,7 +1585,7 @@ void sort_debug_info_sections(Context<E> &ctx) {
         if (osec->shdr.sh_size >= UINT32_MAX || is_in_test)
           vec1.push_back(osec);
 
-  for (std::unique_ptr<MergedSection<E>> &osec : ctx.merged_sections)
+  for (ArenaObjectPtr<MergedSection<E>> &osec : ctx.merged_sections)
     if (!(osec->shdr.sh_flags & SHF_ALLOC) && osec->name.starts_with(".debug_"))
       if (osec->shdr.sh_size >= UINT32_MAX || is_in_test)
         vec2.push_back(osec.get());
@@ -1551,7 +1646,7 @@ void fixup_ctors_in_init_array(Context<E> &ctx) {
     if (isec.sh_size % sizeof(Word<E>))
       Fatal(ctx) << isec << ": section corrupted";
 
-    u8 *buf = (u8 *)isec.contents.data();
+    u8 *buf = isec.contents;
     std::reverse((Word<E> *)buf, (Word<E> *)(buf + isec.sh_size));
 
     std::span<ElfRel<E>> rels = isec.get_rels(ctx);
@@ -1564,13 +1659,13 @@ void fixup_ctors_in_init_array(Context<E> &ctx) {
   if (Chunk<E> *chunk = find_chunk(ctx, ".init_array"))
     if (OutputSection<E> *osec = chunk->to_osec())
       for (InputSection<E> *isec : osec->members)
-        if (isec->name.starts_with(".ctors"))
+        if (isec->name().starts_with(".ctors"))
           reverse_contents(*isec);
 
   if (Chunk<E> *chunk = find_chunk(ctx, ".fini_array"))
     if (OutputSection<E> *osec = chunk->to_osec())
       for (InputSection<E> *isec : osec->members)
-        if (isec->name.starts_with(".dtors"))
+        if (isec->name().starts_with(".dtors"))
           reverse_contents(*isec);
 }
 
@@ -1725,13 +1820,14 @@ void claim_unresolved_symbols(Context<E> &ctx) {
                    << " symbol " << sym;
 
         sym.file = file;
-        sym.origin = 0;
+        sym.origin = nullptr;
         sym.value = 0;
         sym.sym_idx = i;
         sym.is_weak = false;
         sym.is_imported = is_imported;
         sym.is_exported = false;
-        sym.ver_idx = is_imported ? 0 : ctx.default_version;
+        sym.ver_idx =
+          is_imported ? (u16)VER_NDX_UNSPECIFIED : ctx.default_version;
       };
 
       if (esym.is_undef_weak()) {
@@ -1813,14 +1909,14 @@ void scan_relocations(Context<E> &ctx) {
   });
 
   std::vector<Symbol<E> *> syms = flatten(vec);
-  ctx.symbol_aux.reserve(syms.size());
 
   if (ctx.needs_tlsld)
     ctx.got->add_tlsld(ctx);
 
   // Assign offsets in additional tables for each dynamic symbol.
   for (Symbol<E> *sym : syms) {
-    sym->add_aux(ctx);
+    if (!sym->aux)
+      sym->aux = ctx.arena.template make<SymbolAux<E>>();
 
     if (sym->is_imported || sym->is_exported)
       ctx.dynsym->add_symbol(ctx, sym);
@@ -1828,7 +1924,7 @@ void scan_relocations(Context<E> &ctx) {
     if (sym->flags & NEEDS_GOT)
       ctx.got->add_got_symbol(ctx, sym);
 
-    if (sym->flags & NEEDS_CPLT) {
+    if ((sym->flags & NEEDS_CANONICAL) && sym->get_type() == STT_FUNC) {
       sym->is_canonical = true;
 
       // A canonical PLT needs to be visible from DSOs.
@@ -1854,9 +1950,9 @@ void scan_relocations(Context<E> &ctx) {
     if (sym->flags & NEEDS_TLSDESC)
       ctx.got->add_tlsdesc_symbol(ctx, sym);
 
-    if (sym->flags & NEEDS_COPYREL) {
+    if ((sym->flags & NEEDS_CANONICAL) && sym->get_type() != STT_FUNC) {
       if (ctx.arg.z_relro && sym->file->is_dso &&
-          ((SharedFile<E> *)sym->file)->is_readonly(sym))
+          sym->file->to_dso()->is_readonly(sym))
         ctx.copyrel_relro->add_symbol(ctx, sym);
       else
         ctx.copyrel->add_symbol(ctx, sym);
@@ -1958,23 +2054,28 @@ void copy_chunks(Context<E> &ctx) {
   };
 
   // For --relocatable and --emit-relocs, we want to copy non-relocation
-  // sections first. This is because REL-type relocation sections (as
-  // opposed to RELA-type) stores relocation addends to target sections.
+  // sections first, for two reasons. First, REL-type relocation sections (as
+  // opposed to RELA-type) store relocation addends to target sections, so the
+  // targets must be written first. Second, relaxation may retype an emitted
+  // relocation in place while applying relocations (e.g. AArch64 GOT/TLS
+  // relaxations), and RelocSection has to observe the updated type, so it
+  // must run after the target sections.
   //
-  // We also does that for SH4 because despite being RELA, we always need
+  // We also do that for SH4 because despite being RELA, we always need
   // to write addends to relocated places for SH4.
-  auto is_rel = [](Chunk<E> &chunk) {
+  auto copy_last = [](Chunk<E> &chunk) {
     return chunk.shdr.sh_type == SHT_REL ||
-           (is_sh4<E> && chunk.shdr.sh_type == SHT_RELA);
+           (is_sh4<E> && chunk.shdr.sh_type == SHT_RELA) ||
+           chunk.to_reloc_sec();
   };
 
   tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
-    if (!is_rel(*chunk))
+    if (!copy_last(*chunk))
       copy(*chunk);
   });
 
   tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
-    if (is_rel(*chunk))
+    if (copy_last(*chunk))
       copy(*chunk);
   });
 
@@ -2034,12 +2135,12 @@ void sort_dynsyms(Context<E> &ctx) {
     u32 num_buckets = num_exported / ctx.gnu_hash->LOAD_FACTOR + 1;
 
     tbb::parallel_for_each(exported_syms, [&](Symbol<E> *sym) {
-      sym->set_djb_hash(ctx, djb_hash(sym->name()));
+      sym->aux->djb_hash = djb_hash(sym->name());
     });
 
     tbb::parallel_sort(exported_syms, [&](Symbol<E> *a, Symbol<E> *b) {
-      return std::tuple(a->get_djb_hash(ctx) % num_buckets, a->name()) <
-             std::tuple(b->get_djb_hash(ctx) % num_buckets, b->name());
+      return std::tuple(a->aux->djb_hash % num_buckets, a->name()) <
+             std::tuple(b->aux->djb_hash % num_buckets, b->name());
     });
 
     ctx.gnu_hash->num_buckets = num_buckets;
@@ -2051,7 +2152,7 @@ void sort_dynsyms(Context<E> &ctx) {
 
   tbb::enumerable_thread_specific<i64> size;
   tbb::parallel_for((i64)1, (i64)syms.size(), [&](i64 i) {
-    syms[i]->set_dynsym_idx(ctx, i);
+    syms[i]->aux->dynsym_idx = i;
     size.local() += syms[i]->name().size() + 1;
   });
 
@@ -2124,25 +2225,23 @@ void apply_version_script(Context<E> &ctx) {
   }
 
   if (!matcher.empty() || !cpp_matcher.empty()) {
-    tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-      for (Symbol<E> *sym : file->get_global_syms()) {
-        if (sym->file != file)
-          continue;
+    ctx.symbol_map.parallel_for_each([&](Symbol<E> &sym) {
+      if (!sym.file || sym.file->is_dso)
+        return;
 
-        std::string_view name = sym->name();
-        i64 match = matcher.find(name);
+      std::string_view name = sym.name();
+      i64 match = matcher.find(name);
 
-        // Match non-mangled symbols against the C++ pattern as well.
-        // Weird, but required to match other linkers' behavior.
-        if (!cpp_matcher.empty()) {
-          if (std::optional<std::string_view> s = demangle_cpp(name))
-            name = *s;
-          match = std::max(match, cpp_matcher.find(name));
-        }
-
-        if (match != -1)
-          sym->ver_idx = patterns[match].ver_idx;
+      // Match non-mangled symbols against the C++ pattern as well.
+      // Weird, but required to match other linkers' behavior.
+      if (!cpp_matcher.empty()) {
+        if (std::optional<std::string_view> s = demangle_cpp(name))
+          name = *s;
+        match = std::max(match, cpp_matcher.find(name));
       }
+
+      if (match != -1)
+        sym.ver_idx = patterns[match].ver_idx;
     });
   }
 
@@ -2187,7 +2286,7 @@ void parse_symbol_version(Context<E> &ctx) {
       if (sym->file != file)
         continue;
 
-      std::string_view name = file->symbol_names[i];
+      std::string_view name = file->get_symbol_name(i);
       std::string_view ver = name.substr(name.find('@') + 1);
 
       bool is_default = false;
@@ -2196,6 +2295,13 @@ void parse_symbol_version(Context<E> &ctx) {
         ver = ver.substr(1);
       }
 
+      // Empty version (`foo@@`) is the unversioned default; export it
+      // globally, overriding any `local: *` from apply_version_script().
+      if (ver.empty()) {
+        sym->ver_idx = VER_NDX_GLOBAL;
+        continue;
+      }
+
       auto it = verdefs.find(ver);
       if (it == verdefs.end()) {
         Error(ctx) << *file << ": symbol " << *sym <<  " has undefined version "
@@ -2232,7 +2338,7 @@ static bool should_export(Context<E> &ctx, Symbol<E> &sym) {
       if (u32 ty = sym.get_type(); ty != STT_FUNC && ty != STT_GNU_IFUNC)
         return true;
     if (ctx.arg.shared)
-      return !((ObjectFile<E> *)sym.file)->exclude_libs;
+      return !sym.file->to_obj()->exclude_libs;
     return ctx.arg.export_dynamic;
   case VER_NDX_LOCAL:
     return false;
@@ -2347,23 +2453,21 @@ void compute_import_export(Context<E> &ctx) {
   }
 
   if (!matcher.empty() || !cpp_matcher.empty()) {
-    tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-      for (Symbol<E> *sym : file->get_global_syms()) {
-        if (sym->file != file)
-          continue;
-        if (ctx.arg.shared && !sym->is_exported)
-          continue;
+    ctx.symbol_map.parallel_for_each([&](Symbol<E> &sym) {
+      if (!sym.file || sym.file->is_dso)
+        return;
+      if (ctx.arg.shared && !sym.is_exported)
+        return;
 
-        std::string_view name = sym->name();
+      std::string_view name = sym.name();
 
-        if (matcher.find(name) != -1) {
-          handle_match(sym);
-        } else if (!cpp_matcher.empty()) {
-          if (std::optional<std::string_view> s = demangle_cpp(name))
-            name = *s;
-          if (cpp_matcher.find(name) != -1)
-            handle_match(sym);
-        }
+      if (matcher.find(name) != -1) {
+        handle_match(&sym);
+      } else if (!cpp_matcher.empty()) {
+        if (std::optional<std::string_view> s = demangle_cpp(name))
+          name = *s;
+        if (cpp_matcher.find(name) != -1)
+          handle_match(&sym);
       }
     });
   }
@@ -2407,9 +2511,9 @@ void compute_address_significance(Context<E> &ctx) {
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     // If .llvm_addrsig is available, use it.
-    if (InputSection<E> *sec = file->llvm_addrsig.get()) {
-      u8 *p = (u8 *)sec->contents.data();
-      u8 *end = p + sec->contents.size();
+    if (InputSection<E> *sec = file->llvm_addrsig) {
+      u8 *p = sec->contents;
+      u8 *end = p + sec->get_contents().size();
       while (p != end) {
         Symbol<E> *sym = file->symbols[read_uleb(&p)];
         if (InputSection<E> *isec = sym->get_input_section())
@@ -2419,7 +2523,7 @@ void compute_address_significance(Context<E> &ctx) {
     }
 
     // Otherwise, infer address significance.
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections) {
+    for (InputSection<E> *isec : file->sections) {
       if (!isec || !isec->is_alive || !(isec->shdr().sh_flags & SHF_ALLOC))
         continue;
 
@@ -3003,13 +3107,11 @@ i64 set_osec_offsets(Context<E> &ctx) {
     else
       set_virtual_addresses_by_order(ctx);
 
-    if (ctx.arg.pack_dyn_relocs_relr || ctx.arg.pack_dyn_relocs_android) {
+    if (ctx.arg.pack_dyn_relocs_android) {
       i64 x = ctx.reldyn->shdr.sh_size;
-      i64 y = ctx.relrdyn ? (i64)ctx.relrdyn->shdr.sh_size : 0;
       ctx.reldyn->update_shdr(ctx);
 
-      if (x != (i64)ctx.reldyn->shdr.sh_size ||
-          y != (ctx.relrdyn ? (i64)ctx.relrdyn->shdr.sh_size : 0))
+      if (x != (i64)ctx.reldyn->shdr.sh_size)
         continue;
     }
 
@@ -3103,8 +3205,8 @@ void fix_synthetic_symbols(Context<E> &ctx) {
   } else {
     // If the symbols are not ncessary, we turn them to absolute
     // symbols at address 0.
-    ctx.__rel_iplt_start->origin = 0;
-    ctx.__rel_iplt_end->origin = 0;
+    ctx.__rel_iplt_start->origin = nullptr;
+    ctx.__rel_iplt_end->origin = nullptr;
   }
 
   // __{init,fini}_array_{start,end}
@@ -3235,7 +3337,7 @@ void fix_synthetic_symbols(Context<E> &ctx) {
     std::variant<Symbol<E> *, u64> val = ctx.arg.defsyms[i].second;
 
     if (u64 *addr = std::get_if<u64>(&val)) {
-      sym->origin = 0;
+      sym->origin = nullptr;
       sym->value = *addr;
     } else {
       Symbol<E> *sym2 = std::get<Symbol<E> *>(val);
@@ -3515,23 +3617,19 @@ void write_separate_debug_file(Context<E> &ctx) {
 
   copy_chunks(ctx);
 
-  if (ctx.gdb_index)
+  if (ctx.gdb_index) {
+    build_gdb_index_tables(ctx);
     write_gdb_index(ctx);
+  }
 
   // Reverse-compute a CRC32 value so that the CRC32 checksum embedded to
   // the .gnu_debuglink section in the main executable matches with the
   // debug info file's CRC32 checksum.
   u32 crc = compute_crc32(0, ctx.buf, ctx.output_file->filesize);
 
-  u8 *&buf2 = ctx.output_file->buf2;
-  i64 &buf2_size = ctx.output_file->buf2_size;
-  if (buf2)
-    crc = compute_crc32(crc, buf2, buf2_size);
-
   std::vector<u8> trailer = crc32_solve(crc, ctx.gnu_debuglink->crc32);
-  buf2 = (u8 *)realloc(buf2, buf2_size + trailer.size());
-  memcpy(buf2 + buf2_size, trailer.data(), trailer.size());
-  buf2_size += trailer.size();
+  memcpy(ctx.output_file->extend(ctx, trailer.size()),
+         trailer.data(), trailer.size());
   ctx.output_file->close(ctx);
 }
 
@@ -3577,7 +3675,7 @@ void show_stats(Context<E> &ctx) {
     static Counter undefined("undefined_syms");
     undefined += obj->symbols.size() - obj->first_global;
 
-    for (std::unique_ptr<InputSection<E>> &sec : obj->sections) {
+    for (InputSection<E> *sec : obj->sections) {
       if (!sec || !sec->is_alive)
         continue;
 
@@ -3595,8 +3693,13 @@ void show_stats(Context<E> &ctx) {
 
     static Counter removed_comdats("removed_comdat_mem");
     for (ComdatGroupRef<E> &ref : obj->comdat_groups)
-      if (ref.group->owner != obj->priority)
-        removed_comdats += ref.members.size();
+      if (!ref.is_owner)
+        removed_comdats += ref.members(*obj).size();
+
+    static Counter unique_comdats("unique_comdats");
+    for (ComdatGroupRef<E> &ref : obj->comdat_groups)
+      if (ref.is_owner)
+        unique_comdats++;
 
     static Counter num_cies("num_cies");
     num_cies += obj->cies.size();
@@ -3625,7 +3728,7 @@ void show_stats(Context<E> &ctx) {
   using Entry = typename ConcurrentMap<SectionFragment<E>>::Entry;
 
   static Counter merged_strings("merged_strings");
-  for (std::unique_ptr<MergedSection<E>> &sec : ctx.merged_sections)
+  for (ArenaObjectPtr<MergedSection<E>> &sec : ctx.merged_sections)
     for (Entry &ent : std::span(sec->map.entries, sec->map.nbuckets))
       if (ent.key)
         merged_strings++;
@@ -3649,7 +3752,7 @@ void show_stats(Context<E> &ctx) {
 
   Counter::print();
 
-  for (std::unique_ptr<MergedSection<E>> &sec : ctx.merged_sections)
+  for (ArenaObjectPtr<MergedSection<E>> &sec : ctx.merged_sections)
     sec->print_stats(ctx);
 }
 
@@ -3659,6 +3762,7 @@ template int redo_main<E>(std::string_view, int, char **);
 template void create_internal_file(Context<E> &);
 template void apply_exclude_libs(Context<E> &);
 template void create_synthetic_sections(Context<E> &);
+template void gather_symbols(Context<E> &);
 template void resolve_symbols(Context<E> &);
 template void do_lto(Context<E> &);
 template void parse_eh_frame_sections(Context<E> &);
@@ -3672,6 +3776,7 @@ template void apply_section_align(Context<E> &);
 template void print_dependencies(Context<E> &);
 template void write_repro_file(Context<E> &);
 template void check_duplicate_symbols(Context<E> &);
+template void check_symbol_version_conflicts(Context<E> &);
 template void convert_zero_to_bss(Context<E> &);
 template void check_shlib_undefined(Context<E> &);
 template void check_symbol_types(Context<E> &);

src/relocatable.cc

@@ -76,14 +76,14 @@ static void create_comdat_group_sections(Context<E> &ctx) {
     ObjectFile<E> &file = *ctx.objs[i];
 
     for (ComdatGroupRef<E> &ref : file.comdat_groups) {
-      if (ref.group->owner != file.priority)
+      if (!ref.is_owner)
         continue;
 
       Symbol<E> *sym = file.symbols[file.elf_sections[ref.sect_idx].sh_info];
       assert(sym);
 
       std::vector<Chunk<E> *> members;
-      for (u32 j : ref.members) {
+      for (u32 j : ref.members(file)) {
         const ElfShdr<E> &shdr = file.elf_sections[j];
         if (shdr.sh_type == (E::is_rela ? SHT_RELA : SHT_REL)) {
           InputSection<E> &isec = *file.sections[shdr.sh_info];
@@ -127,7 +127,7 @@ static void r_claim_unresolved_symbols(Context<E> &ctx) {
         continue;
 
       sym.file = file;
-      sym.origin = 0;
+      sym.origin = nullptr;
       sym.value = 0;
       sym.sym_idx = i;
     }

src/shrink-sections.cc

@@ -89,7 +89,7 @@ void shrink_sections(Context<E> &ctx) {
   // only ~0.04% larger than that of GNU ld), so we don't bother to handle
   // them. We scan relocations only once here.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (std::unique_ptr<InputSection<E>> &isec : file->sections)
+    for (InputSection<E> *isec : file->sections)
       if (isec && isec->is_alive && (isec->shdr().sh_flags & SHF_EXECINSTR))
         shrink_section(ctx, *isec);
   });

src/thunks.cc

@@ -120,10 +120,11 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
     return;
 
   // Initialize input sections with a dummy offset so that we can
-  // distinguish sections that have got an address with the one who
-  // haven't.
-  for (InputSection<E> *isec : m)
+  // distinguish sections whose addresses have been assigned from those
+  // whose addresses have not.
+  tbb::parallel_for_each(m, [](InputSection<E> *isec) {
     isec->offset = -1;
+  });
   thunks.clear();
 
   // We create thunks from the beginning of the section to the end.
@@ -169,17 +170,21 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
       d++;
     }
 
-    // Move C forward so that C is apart from B by BATCH_SIZE. We want
-    // to make sure that there's at least one section between B and C
-    // to ensure progress.
-    c = b + 1;
-    while (c < d && m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
-      c++;
+    // Find the end of the current batch. Section end addresses are sorted,
+    // so use binary search. Starting from B + 1 guarantees progress.
+    std::span<InputSection<E> *> range = m.subspan(b + 1, d - b - 1);
+    c = ranges::lower_bound(range, m[b]->offset + batch_size, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset + isec->sh_size;
+                            }) - m.begin();
 
-    // Move A forward so that A is reachable from C.
+    // Find the first section that is within branch range of C.
     i64 c_offset = (c == d) ? offset : m[c]->offset;
-    while (a < b && m[a]->offset + branch_distance<E> < c_offset)
-      a++;
+    range = m.subspan(a, b - a);
+    a = ranges::lower_bound(range, c_offset - branch_distance<E>, {},
+                            [](InputSection<E> *isec) {
+                              return isec->offset;
+                            }) - m.begin();
 
     // Erase references to out-of-range thunks.
     for (; t < thunks.size() && thunks[t]->offset < m[a]->offset; t++)
@@ -191,27 +196,22 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
     thunks.emplace_back(std::make_unique<Thunk<E>>(*this, offset));
 
     Thunk<E> &thunk = *thunks.back();
-    std::mutex mu;
+    tbb::enumerable_thread_specific<std::vector<Symbol<E> *>> symbols;
 
     // Scan relocations between B and C to collect symbols that need
     // entries in the new thunk.
     tbb::parallel_for(b, c, [&](i64 i) {
       InputSection<E> &isec = *m[i];
-      for (const ElfRel<E> &rel : isec.get_rels(ctx)) {
-        if (requires_thunk(ctx, isec, rel, true)) {
+      for (const ElfRel<E> &rel : isec.get_rels(ctx))
+        if (requires_thunk(ctx, isec, rel, true))
           if (Symbol<E> &sym = *isec.file.symbols[rel.r_sym];
-              !sym.flags.test_and_set()) {
-            std::scoped_lock lock(mu);
-            thunk.symbols.push_back(&sym);
-          }
-        }
-      }
+              !sym.flags.test_and_set())
+            symbols.local().push_back(&sym);
     });
 
-    // Sort symbols added to the thunk to make the output deterministic.
-    ranges::sort(thunk.symbols, {}, [](Symbol<E> *x) {
-      return std::tuple{x->file->priority, x->sym_idx};
-    });
+    // Add symbols to the thunk
+    for (std::vector<Symbol<E> *> &vec : symbols)
+      append(thunk.symbols, vec);
 
     // Now that we know the number of symbols in the thunk, we can compute
     // the thunk's size.
@@ -223,11 +223,18 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
     b = c;
   }
 
-  // Reset flags for future use
+  // Clear marks for thunks that are still reachable from the last batch.
   for (; t < thunks.size(); t++)
     for (Symbol<E> *sym : thunks[t]->symbols)
       sym->flags = 0;
 
+  // Sort symbols for deterministic output.
+  tbb::parallel_for_each(thunks, [](std::unique_ptr<Thunk<E>> &thunk) {
+    ranges::sort(thunk->symbols, {}, [](Symbol<E> *sym) {
+      return std::tuple{sym->file->priority, sym->sym_idx};
+    });
+  });
+
   this->shdr.sh_size = offset;
 }
 
@@ -259,18 +266,23 @@ void remove_redundant_thunks(Context<E> &ctx) {
   for (OutputSection<E> *osec : sections) {
     tbb::parallel_for_each(osec->members, [&](InputSection<E> *isec) {
       for (const ElfRel<E> &rel : isec->get_rels(ctx))
-        if (requires_thunk(ctx, *isec, rel, false))
-          isec->file.symbols[rel.r_sym]->flags.test_and_set();
+        if (is_func_call_rel(rel))
+          if (Symbol<E> *sym = isec->file.symbols[rel.r_sym];
+              !sym->flags &&
+              requires_thunk(ctx, *isec, rel, false))
+            sym->flags.test_and_set();
     });
   }
 
   // Remove symbols from thunks if they don't actually need range
   // extension thunks
   for (OutputSection<E> *osec : sections) {
-    for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks) {
-      std::erase_if(thunk->symbols, [&](Symbol<E> *sym) { return !sym->flags; });
+    tbb::parallel_for_each(osec->thunks,
+                           [&](std::unique_ptr<Thunk<E>> &thunk) {
+      std::erase_if(thunk->symbols,
+                    [&](Symbol<E> *sym) { return !sym->flags; });
       thunk->shrink_size(ctx);
-    }
+    });
   }
 
   // Recompute section sizes
@@ -294,12 +306,6 @@ void remove_redundant_thunks(Context<E> &ctx) {
     }
     osec->shdr.sh_size = offset;
   });
-
-  // Reset flags for future use
-  for (OutputSection<E> *osec : sections)
-    for (std::unique_ptr<Thunk<E>> &thunk : osec->thunks)
-      for (Symbol<E> *sym : thunk->symbols)
-        sym->flags = 0;
 }
 
 // When applying relocations, we want to know the address in a reachable

test/CMakeLists.txt

@@ -86,7 +86,7 @@ function(add_target ARCH TRIPLE)
     endif()
 
     add_test(NAME ${TESTNAME}
-      COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
+      COMMAND ${CMAKE_CURRENT_LIST_DIR}/${TEST}
       WORKING_DIRECTORY ${mold_BINARY_DIR})
 
     set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT

test/abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 [ $MACHINE = aarch64 ] && skip

test/absolute-symbols.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # This test crashes only on qemu-sparc64 running on GitHub Actions,

test/allow-multiple-definition.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 echo 'int main() { return 0; }' | $CC -c -o $t/a.o -xc -

test/ar-alignment.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-aarch64-emit-relocs-relax.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# When mold relaxes an instruction sequence, the relocations attached to the
+# original instructions no longer match the relaxed code. With --emit-relocs,
+# mold must rewrite the emitted relocations so that their types describe the
+# relaxed instructions.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xassembler -
+.text
+.globl _start
+_start:
+  # GOT-load relaxation: ADRP+LDR => ADRP+ADD
+  #   R_AARCH64_ADR_GOT_PAGE     => R_AARCH64_ADR_PREL_PG_HI21
+  #   R_AARCH64_LD64_GOT_LO12_NC => R_AARCH64_ADD_ABS_LO12_NC
+  adrp x0, :got:gvar
+  ldr  x0, [x0, #:got_lo12:gvar]
+
+  # ADRP+ADD => NOP+ADR (gvar is within +-1 MiB)
+  #   R_AARCH64_ADR_PREL_PG_HI21 => R_AARCH64_NONE
+  #   R_AARCH64_ADD_ABS_LO12_NC  => R_AARCH64_ADR_PREL_LO21
+  adrp x1, gvar
+  add  x1, x1, #:lo12:gvar
+
+  # Both sequences must materialize the same address.
+  cmp  x0, x1
+  bne  1f
+
+  # ... and it must point at gvar, whose value is 42.
+  ldr  w2, [x0]
+  cmp  w2, #42
+  bne  1f
+
+  mov  x0, #0
+  b    2f
+1:
+  mov  x0, #1
+2:
+  mov  x8, #93           // __NR_exit
+  svc  #0
+
+.data
+.hidden gvar
+gvar:
+  .word 42
+EOF
+
+$CC -B. -o $t/exe $t/a.o --static -nostdlib -Wl,-e,_start,--emit-relocs,--relax
+$QEMU $t/exe
+
+$OBJDUMP -dr $t/exe > $t/exe.objdump
+
+# Relaxation took place.
+grep -Fw nop $t/exe.objdump
+grep -Fw adr $t/exe.objdump
+grep -Fw add $t/exe.objdump
+
+# The relocations were rewritten to match the relaxed instructions.
+grep -Fw R_AARCH64_ADR_PREL_PG_HI21 $t/exe.objdump
+grep -Fw R_AARCH64_ADD_ABS_LO12_NC $t/exe.objdump
+grep -Fw R_AARCH64_ADR_PREL_LO21 $t/exe.objdump
+
+# The stale pre-relaxation relocations are gone.
+not grep -Fw R_AARCH64_ADR_GOT_PAGE $t/exe.objdump
+not grep -Fw R_AARCH64_LD64_GOT_LO12_NC $t/exe.objdump
+
+# Each rewritten relocation must sit on the relaxed instruction it describes.
+awk '
+  $3 == "adrp" || $3 == "add" || $3 == "adr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_ADR_PREL_PG_HI21" { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADD_ABS_LO12_NC"  { if (insn[$1] != "add")  { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_ADR_PREL_LO21"    { if (insn[$1] != "adr")  { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe.objdump

test/arch-aarch64-emit-relocs-tlsdesc-relax.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec
+# (LE) or initial-exec (IE) sequence. The four-instruction sequence is rewritten
+# in place: the ADRP and LDR become NOPs, and the ADD and BLR become the two
+# instructions that materialize the address. With --emit-relocs, the emitted
+# relocations must be rewritten to match whichever relaxation was applied.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC
+_Thread_local char foo[4] = "foo";
+_Thread_local char bar[4] = "bar";
+EOF
+
+cat <<'EOF' | $CC -o $t/b.o -c -xc - -fPIC -mtls-dialect=desc -O2
+extern _Thread_local char foo[4];
+extern _Thread_local char bar[4];
+char *get_foo() { return foo; }
+char *get_bar() { return bar; }
+EOF
+
+cat <<EOF | $CC -o $t/c.o -c -xc - -mtls-dialect=desc
+#include <stdio.h>
+char *get_foo();
+char *get_bar();
+int main() { printf("%s %s\n", get_foo(), get_bar()); }
+EOF
+
+#
+# Local TLS in an executable: TLSDESC => LE
+#
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -Wl,--emit-relocs,--relax
+$QEMU $t/exe1 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe1 > $t/exe1.objdump
+
+# The ADD/BLR became MOVZ/MOVK that build the TP-relative offset.
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/exe1.objdump
+grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G0_NC $t/exe1.objdump
+
+# No TLSDESC relocation survives.
+not grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe1.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe1.objdump
+
+# The G1 relocation sits on a movz and the G0_NC on a movk, not on the leading
+# adrp/ldr (which are now nops).
+awk '
+  $3 == "movz" || $3 == "movk" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G1"    { if (insn[$1] != "movz") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSLE_MOVW_TPREL_G0_NC" { if (insn[$1] != "movk") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe1.objdump
+
+#
+# TLS defined in a shared library: TLSDESC => IE
+#
+$CC -B. -shared -o $t/d.so $t/a.o -Wl,--relax
+$CC -B. -o $t/exe2 $t/b.o $t/c.o $t/d.so -Wl,--emit-relocs,--relax -Wl,-rpath=$t
+$QEMU $t/exe2 | grep -F 'foo bar'
+
+$OBJDUMP -dr $t/exe2 > $t/exe2.objdump
+
+# The ADD/BLR became ADRP/LDR that load the address from the GOT.
+grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/exe2.objdump
+grep -Fw R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/exe2.objdump
+not grep -Fw R_AARCH64_TLSDESC_CALL $t/exe2.objdump
+
+awk '
+  $3 == "adrp" || $3 == "ldr" || $3 == "nop" { insn[$1] = $3 }
+  $2 == "R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21"  { if (insn[$1] != "adrp") { bad=1; print "bad offset: " $0 } }
+  $2 == "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC" { if (insn[$1] != "ldr") { bad=1; print "bad offset: " $0 } }
+  END { exit bad+0 }
+' $t/exe2.objdump
+
+#
+# Relocatable output: no relaxation happens and TLSDESC is not downgraded, so
+# every relocation must be passed through unchanged rather than rewritten as if
+# it had been relaxed to LE/IE.
+#
+$CC -B. -o $t/reloc.o $t/b.o -r
+$OBJDUMP -r $t/reloc.o > $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADR_PAGE21 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_LD64_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_ADD_LO12 $t/reloc.objdump
+grep -Fw R_AARCH64_TLSDESC_CALL $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSLE_MOVW_TPREL_G1 $t/reloc.objdump
+not grep -Fw R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 $t/reloc.objdump

test/arch-aarch64-long-thunk.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF > $t/a.c

test/arch-aarch64-pack-dyn-relocs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 [ $MACHINE = aarch64 ] || skip

test/arch-aarch64-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-aarch64-variant-pcs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip

test/arch-arm-abs-error.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -fPIC -c -o $t/a.o -xassembler - -mthumb

test/arch-arm-exidx-sentinel.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CXX -o $t/a.o -c -xc++ - -fno-PIC

test/arch-arm-range-extension-thunk-disassembly.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -

test/arch-arm-range-extension-thunk.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 test_cflags -mthumb || skip

test/arch-arm-target1.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -xassembler -

test/arch-arm-thm-jump19.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-arm-thm-jump8.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-arm-thumb-interwork.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 test_cflags -mthumb || skip

test/arch-arm-tlsdesc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 is_musl && skip

test/arch-armeb-be32.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-i686-addend-sign-extension.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # In the REL relocation format, an addend is read from the relocated

test/arch-i686-emit-relocs-relax.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# mold relaxes a GOT load of a non-preemptible symbol from
+#   mov foo@GOT(%ebx), %reg   =>   lea foo@GOTOFF(%ebx), %reg
+# With --emit-relocs the R_386_GOT32X relocation must be rewritten to
+# R_386_GOTOFF to match the lea (GNU ld does the same).
+
+cat <<'EOF' | $CC -o $t/a.o -c -xc -
+int foo = 42;
+EOF
+
+cat <<'EOF' | $CC -fPIC -O2 -o $t/b.o -c -xc -
+extern int foo;
+int get(void) { return foo; }
+EOF
+
+cat <<'EOF' | $CC -o $t/c.o -c -xc -
+extern int get(void);
+int main(void) { return get() != 42; }   // exits 0 iff foo resolved to 42
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o -no-pie -Wl,--emit-relocs
+$QEMU $t/exe
+
+$OBJDUMP -dr $t/exe > $t/exe.objdump
+
+# The GOT load of foo was relaxed and its relocation rewritten to GOTOFF. The
+# compiler only emits GOT32X for the extern access, so a GOTOFF naming foo can
+# only come from the relaxation.
+grep -E $'R_386_GOTOFF[ \t]+foo' $t/exe.objdump
+not grep -E $'R_386_GOT32X[ \t]+foo' $t/exe.objdump

test/arch-i686-tls-module-base.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<'EOF' | $CC -fPIC -o $t/a.o -c -xassembler -

test/arch-i686-tlsdesc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 supports_tlsdesc || skip

test/arch-loongarch64-emit-relocs-relax.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # When mold relaxes an instruction sequence, the relocations attached to the

test/arch-loongarch64-emit-relocs-tlsdesc-relax.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # Depending on the symbol, mold relaxes a TLSDESC sequence into a local-exec

test/arch-loongarch64-mcmodel-extreme.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -mcmodel=extreme

test/arch-loongarch64-relax-call36.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<'EOF' | $CC -o $t/a.o -c -xassembler -

test/arch-loongarch64-relax-got-load.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -fPIC

test/arch-loongarch64-relax-got-load2.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # pcaddi takes a 20-bit immediate that is shifted left by 2 to form

test/arch-loongarch64-relax-pcala-addi.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<'EOF' | $CC -o $t/a.o -c -xassembler -

test/arch-loongarch64-relax-tlsdesc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<'EOF' | $CC -o $t/a.o -c -xc - -fPIC

test/arch-ppc64-elfv2-error.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# We support ELFv2 only for little-endian PPC64. A big-endian ELFv2 file
+# must be rejected instead of silently linked as ELFv1.
+# https://github.com/rui314/mold/issues/1498
+
+[ $MACHINE = ppc64 ] || skip
+
+cat <<EOF | $CC -mabi=elfv2 -c -o $t/a.o -xc - 2> /dev/null || skip
+int main() {}
+EOF
+
+readelf -h $t/a.o | grep -q abiv2 || skip
+
+! ./mold -m elf64ppc -o $t/exe $t/a.o 2> $t/log
+grep 'unknown machine type' $t/log

test/arch-ppc64le-save-restore-gprs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-riscv64-attributes.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -march=rv64imafd_xtheadba1p5 -o $t/a.o -c -xc -

test/arch-riscv64-attributes2.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc -

test/arch-riscv64-emit-relocs-relax.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # When mold relaxes instructions, the relocations attached to the relaxed code

test/arch-riscv64-global-pointer-dso.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-riscv64-global-pointer.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -fPIC

test/arch-riscv64-obj-compatible.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 set -x

test/arch-riscv64-relax-align.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-riscv64-relax-got.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-riscv64-relax-hi20.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-riscv64-relax-j.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xc - -O2 -march=rv64g

test/arch-riscv64-reloc-overflow.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -x assembler -

test/arch-riscv64-symbol-size.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -

test/arch-riscv64-variant-cc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip

test/arch-riscv64-weak-undef.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 test_cflags -static || skip

test/arch-s390x-emit-relocs-relax.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+. $(dirname $0)/common.inc
+
+# mold relaxes a GOT-loading LGRL into an address-materializing LARL when the
+# symbol's address is a link-time constant. With --emit-relocs the R_390_GOTENT
+# relocation must be rewritten to R_390_PC32DBL to match the LARL; otherwise it
+# would describe a GOT slot that the relaxation removed.
+
+cat <<'EOF' | $CC -o $t/a.o -c -xc -
+int foo = 42;
+EOF
+
+cat <<'EOF' | $CC -fPIC -O2 -o $t/b.o -c -xc -
+extern int foo;
+int get(void) { return foo; }
+EOF
+
+cat <<'EOF' | $CC -o $t/c.o -c -xc -
+extern int get(void);
+int main(void) { return get() != 42; }   // exits 0 iff foo resolved to 42
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o -static -Wl,--emit-relocs
+$QEMU $t/exe
+
+$OBJDUMP -dr $t/exe > $t/exe.objdump
+
+# The GOT load was relaxed to a direct larl of foo.
+grep -E 'larl.*<foo>' $t/exe.objdump
+
+# Its relocation was rewritten to match the larl, and the stale GOTENT is gone.
+grep -E $'R_390_PC32DBL[ \t]+foo' $t/exe.objdump
+not grep -E $'R_390_GOTENT[ \t]+foo' $t/exe.objdump

test/arch-s390x-got.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 . $(dirname $0)/common.inc
 
 # GOT[0] must be set to the link-time address of .dynamic on s390x.