Bump mold version to 2.3.2

For --relocatable output, we need to pass through these attributes
for correctness. With this change, we'll have more .rodata sections
such as .rodata.str1.1, rodata.str1.4, etc. in the final executable,
but that should be fine because the runtime doesn't care about
section tables at all. The table isn't even loaded to memory at runtime.

.gitignore

@@ -3,15 +3,7 @@
 *~
 **/.DS_Store
 .cache
-compile_commands.json
-/mold
-/test/tmp
-options.inc
 core
-ld
-ld64
-ld64.mold
 gentoo
-out
 /mold-*.tar.gz
 /build*

CMakeLists.txt

@@ -44,7 +44,7 @@
 # features and behave exactly the same.
 
 cmake_minimum_required(VERSION 3.14)
-project(mold VERSION 2.2.0)
+project(mold VERSION 2.3.2)
 
 include(CMakeDependentOption)
 include(GNUInstallDirs)
@@ -64,9 +64,12 @@ if(NOT "${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC")
     -fno-exceptions
     -fno-unwind-tables
     -fno-asynchronous-unwind-tables
+    -Wall
+    -Wextra
     -Wno-sign-compare
     -Wno-unused-function
-    -Wunused-variable
+    -Wno-unused-parameter
+    -Wno-missing-field-initializers
     -ggnu-pubnames)
 endif()
 
@@ -157,12 +160,10 @@ endif()
 #
 # We enable mimalloc by default for 64-bit targets. It doesn't seem to
 # be stable on 32-bit targets.
-include(CheckCXXSourceCompiles)
-check_cxx_source_compiles(
-  "static_assert(sizeof(void *) == 4); int main() {}" P32)
+cmake_dependent_option(
+  MOLD_USE_MIMALLOC "Use mimalloc" ON
+  "CMAKE_SIZEOF_VOID_P EQUAL 8; NOT APPLE; NOT ANDROID; NOT OPENBSD" OFF)
 
-cmake_dependent_option(MOLD_USE_MIMALLOC "Use mimalloc" ON
-  "NOT APPLE; NOT ANDROID; NOT OPENBSD; NOT P32" OFF)
 cmake_dependent_option(
   MOLD_USE_SYSTEM_MIMALLOC "Use system or vendored mimalloc" OFF
   MOLD_USE_MIMALLOC OFF)
@@ -301,8 +302,8 @@ list(APPEND MOLD_ELF_TEMPLATE_FILES
   elf/arch-loongarch.cc
   elf/arch-riscv.cc
   elf/cmdline.cc
-  elf/dwarf.cc
   elf/gc-sections.cc
+  elf/gdb-index.cc
   elf/icf.cc
   elf/input-files.cc
   elf/input-sections.cc

README.md

@@ -71,7 +71,7 @@ necessary packages. You may need to run it as root.
 git clone https://github.com/rui314/mold.git
 mkdir mold/build
 cd mold/build
-git checkout v2.2.0
+git checkout v2.3.2
 ../install-build-deps.sh
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ ..
 cmake --build . -j $(nproc)

common/Dockerfile

@@ -1,17 +0,0 @@
-# This dockerfile creates a reproducible build environment for mold.
-#
-# $ docker buildx create --use
-# $ docker buildx build --platform x86_64,aarch64,arm,ppc64le,s390x -t rui314/mold-builder:latest --push .
-
-FROM ubuntu:18.04
-ENV TZ=Europe/London
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common && \
-  add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
-  apt-get update && \
-  apt-get install -y --no-install-recommends build-essential wget libstdc++-11-dev zlib1g-dev gcc-10 g++-10 python3 && \
-  \
-  mkdir /cmake && cd /cmake && \
-  wget -O- -q https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz | tar --strip-components=1 -xzf - && \
-  ./bootstrap --parallel=$(nproc) && make -j$(nproc) && make -j$(nproc) install && \
-  rm -rf /var/lib/apt/lists/* /cmake

common/cmdline.h

@@ -6,58 +6,69 @@ namespace mold {
 
 template <typename Context>
 std::vector<std::string_view>
-read_response_file(Context &ctx, std::string_view path) {
+read_response_file(Context &ctx, std::string_view path, i64 depth) {
+  if (depth > 10)
+    Fatal(ctx) << path << ": response file nesting too deep";
+
   std::vector<std::string_view> vec;
   MappedFile<Context> *mf = MappedFile<Context>::must_open(ctx, std::string(path));
-  u8 *data = mf->data;
+  std::string_view data((char *)mf->data, mf->size);
 
-  auto read_quoted = [&](i64 i, char quote) {
-    std::string buf;
-    while (i < mf->size && data[i] != quote) {
-      if (data[i] == '\\') {
-        buf.append(1, data[i + 1]);
-        i += 2;
-      } else {
-        buf.append(1, data[i++]);
-      }
-    }
-    if (i >= mf->size)
-      Fatal(ctx) << path << ": premature end of input";
-    vec.push_back(save_string(ctx, buf));
-    return i + 1;
-  };
-
-  auto read_unquoted = [&](i64 i) {
-    std::string buf;
-
-    while (i < mf->size) {
-      if (data[i] == '\\' && i + 1 < mf->size) {
-        buf.append(1, data[i + 1]);
-        i += 2;
-        continue;
-      }
-
-      if (!isspace(data[i])) {
-        buf.append(1, data[i++]);
-        continue;
-      }
-
-      break;
+  while (!data.empty()) {
+    if (isspace(data[0])) {
+      data = data.substr(1);
+      continue;
     }
 
-    vec.push_back(save_string(ctx, buf));
-    return i;
-  };
+    auto read_quoted = [&]() {
+      char quote = data[0];
+      data = data.substr(1);
 
-  for (i64 i = 0; i < mf->size;) {
-    if (isspace(data[i]))
-      i++;
-    else if (data[i] == '\'')
-      i = read_quoted(i + 1, '\'');
-    else if (data[i] == '\"')
-      i = read_quoted(i + 1, '\"');
+      std::string buf;
+      while (!data.empty() && data[0] != quote) {
+        if (data[0] == '\\' && data.size() >= 1) {
+          buf.append(1, data[1]);
+          data = data.substr(2);
+        } else {
+          buf.append(1, data[0]);
+          data = data.substr(1);
+        }
+      }
+      if (data.empty())
+        Fatal(ctx) << path << ": premature end of input";
+      data = data.substr(1);
+      return save_string(ctx, buf);
+    };
+
+    auto read_unquoted = [&] {
+      std::string buf;
+      while (!data.empty()) {
+        if (data[0] == '\\' && data.size() >= 1) {
+          buf.append(1, data[1]);
+          data = data.substr(2);
+          continue;
+        }
+
+        if (!isspace(data[0])) {
+          buf.append(1, data[0]);
+          data = data.substr(1);
+          continue;
+        }
+        break;
+      }
+      return save_string(ctx, buf);
+    };
+
+    std::string_view tok;
+    if (data[0] == '\'' || data[0] == '\"')
+      tok = read_quoted();
     else
-      i = read_unquoted(i);
+      tok = read_unquoted();
+
+    if (tok.starts_with('@'))
+      append(vec, read_response_file(ctx, tok.substr(1), depth + 1));
+    else
+      vec.push_back(tok);
   }
   return vec;
 }
@@ -68,7 +79,7 @@ std::vector<std::string_view> expand_response_files(Context &ctx, char **argv) {
   std::vector<std::string_view> vec;
   for (i64 i = 0; argv[i]; i++) {
     if (argv[i][0] == '@')
-      append(vec, read_response_file(ctx, argv[i] + 1));
+      append(vec, read_response_file(ctx, argv[i] + 1, 1));
     else
       vec.push_back(argv[i]);
   }

common/common.h

@@ -23,6 +23,7 @@
 #include <sys/types.h>
 #include <tbb/concurrent_vector.h>
 #include <tbb/enumerable_thread_specific.h>
+#include <tbb/parallel_for.h>
 #include <vector>
 
 #ifdef _WIN32
@@ -225,8 +226,14 @@ struct Atomic : std::atomic<T> {
   void operator=(T val) { store(val); }
   operator T() const { return load(); }
 
-  void store(T val) { std::atomic<T>::store(val, relaxed); }
-  T load() const { return std::atomic<T>::load(relaxed); }
+  void store(T val, std::memory_order order = relaxed) {
+    std::atomic<T>::store(val, order);
+  }
+
+  T load(std::memory_order order = relaxed) const {
+    return std::atomic<T>::load(order);
+  }
+
   T exchange(T val) { return std::atomic<T>::exchange(val, relaxed); }
   T operator|=(T val) { return std::atomic<T>::fetch_or(val, relaxed); }
   T operator++() { return std::atomic<T>::fetch_add(1, relaxed) + 1; }
@@ -243,6 +250,85 @@ struct Atomic : std::atomic<T> {
   }
 };
 
+//
+// perf.cc
+//
+
+// Counter is used to collect statistics numbers.
+class Counter {
+public:
+  Counter(std::string_view name, i64 value = 0) : name(name), values(value) {
+    static std::mutex mu;
+    std::scoped_lock lock(mu);
+    instances.push_back(this);
+  }
+
+  Counter &operator++(int) {
+    if (enabled) [[unlikely]]
+      values.local()++;
+    return *this;
+  }
+
+  Counter &operator+=(int delta) {
+    if (enabled) [[unlikely]]
+      values.local() += delta;
+    return *this;
+  }
+
+  static void print();
+
+  static inline bool enabled = false;
+
+private:
+  i64 get_value();
+
+  std::string_view name;
+  tbb::enumerable_thread_specific<i64> values;
+
+  static inline std::vector<Counter *> instances;
+};
+
+// Timer and TimeRecord records elapsed time (wall clock time)
+// used by each pass of the linker.
+struct TimerRecord {
+  TimerRecord(std::string name, TimerRecord *parent = nullptr);
+  void stop();
+
+  std::string name;
+  TimerRecord *parent;
+  tbb::concurrent_vector<TimerRecord *> children;
+  i64 start;
+  i64 end;
+  i64 user;
+  i64 sys;
+  bool stopped = false;
+};
+
+void
+print_timer_records(tbb::concurrent_vector<std::unique_ptr<TimerRecord>> &);
+
+template <typename Context>
+class Timer {
+public:
+  Timer(Context &ctx, std::string name, Timer *parent = nullptr) {
+    record = new TimerRecord(name, parent ? parent->record : nullptr);
+    ctx.timer_records.push_back(std::unique_ptr<TimerRecord>(record));
+  }
+
+  Timer(const Timer &) = delete;
+
+  ~Timer() {
+    record->stop();
+  }
+
+  void stop() {
+    record->stop();
+  }
+
+private:
+  TimerRecord *record;
+};
+
 //
 // Bit vector
 //
@@ -486,7 +572,7 @@ public:
   // to make this object to be aligned to a reasonably large
   // power-of-two address.
   struct alignas(32) Entry {
-    std::atomic<const char *> key;
+    Atomic<const char *> key;
     T value;
     u32 keylen;
   };
@@ -554,15 +640,61 @@ public:
     return {nullptr, false};
   }
 
-  const char *get_key(i64 idx) {
-    return entries[idx].key.load(std::memory_order_relaxed);
-  }
-
   i64 get_idx(T *value) const {
     uintptr_t addr = (uintptr_t)value - (uintptr_t)value % sizeof(Entry);
     return (Entry *)addr - entries;
   }
 
+  // Return a list of map entries sorted in a deterministic order.
+  std::vector<Entry *> get_sorted_entries(i64 shard_idx) {
+    if (nbuckets == 0)
+      return {};
+
+    i64 shard_size = nbuckets / NUM_SHARDS;
+    i64 begin = shard_idx * shard_size;
+    i64 end = begin + shard_size;
+
+    i64 sz = 0;
+    for (i64 i = begin; i < end; i++)
+      if (entries[i].key)
+        sz++;
+
+    std::vector<Entry *> vec;
+    vec.reserve(sz);
+
+    // Since the shard is circular, we need to handle the last entries
+    // as if they were next to the first entries.
+    while (entries[end - 1].key)
+      vec.push_back(entries + --end);
+
+    // Find entries contiguous in the buckets and sort them.
+    i64 last = 0;
+    for (i64 i = begin; i < end;) {
+      while (i < end && entries[i].key)
+        vec.push_back(entries + i++);
+
+      std::sort(vec.begin() + last, vec.end(), [](Entry *a, Entry *b) {
+        if (a->keylen != b->keylen)
+          return a->keylen < b->keylen;
+        return memcmp(a->key, b->key, a->keylen) < 0;
+      });
+
+      last = vec.size();
+
+      while (i < end && !entries[i].key)
+        i++;
+    }
+    return vec;
+  }
+
+  std::vector<Entry *> get_sorted_entries_all() {
+    std::vector<std::vector<Entry *>> vec(NUM_SHARDS);
+    tbb::parallel_for((i64)0, NUM_SHARDS, [&](i64 i) {
+      vec[i] = get_sorted_entries(i);
+    });
+    return flatten(vec);
+  }
+
   static constexpr i64 MIN_NBUCKETS = 2048;
   static constexpr i64 NUM_SHARDS = 16;
   static constexpr i64 MAX_RETRY = 128;
@@ -594,9 +726,11 @@ public:
   virtual ~OutputFile() = default;
 
   u8 *buf = nullptr;
+  std::vector<u8> buf2;
   std::string path;
-  i64 filesize;
-  bool is_mmapped;
+  i64 fd = -1;
+  i64 filesize = 0;
+  bool is_mmapped = false;
   bool is_unmapped = false;
 
 protected:
@@ -604,6 +738,43 @@ protected:
     : path(path), filesize(filesize), is_mmapped(is_mmapped) {}
 };
 
+template <typename Context>
+class MallocOutputFile : public OutputFile<Context> {
+public:
+  MallocOutputFile(Context &ctx, std::string path, i64 filesize, i64 perm)
+    : OutputFile<Context>(path, filesize, false), perm(perm) {
+    this->buf = (u8 *)malloc(filesize);
+    if (!this->buf)
+      Fatal(ctx) << "malloc failed";
+  }
+
+  ~MallocOutputFile() {
+    free(this->buf);
+  }
+
+  void close(Context &ctx) override {
+    Timer t(ctx, "close_file");
+    FILE *fp;
+
+    if (this->path == "-") {
+      fp = stdout;
+    } else {
+      i64 fd = ::open(this->path.c_str(), O_RDWR | O_CREAT, perm);
+      if (fd == -1)
+        Fatal(ctx) << "cannot open " << this->path << ": " << errno_string();
+      fp = fdopen(fd, "w");
+    }
+
+    fwrite(this->buf, this->filesize, 1, fp);
+    if (!this->buf2.empty())
+      fwrite(this->buf2.data(), this->buf2.size(), 1, fp);
+    fclose(fp);
+  }
+
+private:
+  i64 perm;
+};
+
 //
 // hyperloglog.cc
 //
@@ -739,85 +910,6 @@ private:
   std::vector<std::vector<u8>> shards;
 };
 
-//
-// perf.cc
-//
-
-// Counter is used to collect statistics numbers.
-class Counter {
-public:
-  Counter(std::string_view name, i64 value = 0) : name(name), values(value) {
-    static std::mutex mu;
-    std::scoped_lock lock(mu);
-    instances.push_back(this);
-  }
-
-  Counter &operator++(int) {
-    if (enabled) [[unlikely]]
-      values.local()++;
-    return *this;
-  }
-
-  Counter &operator+=(int delta) {
-    if (enabled) [[unlikely]]
-      values.local() += delta;
-    return *this;
-  }
-
-  static void print();
-
-  static inline bool enabled = false;
-
-private:
-  i64 get_value();
-
-  std::string_view name;
-  tbb::enumerable_thread_specific<i64> values;
-
-  static inline std::vector<Counter *> instances;
-};
-
-// Timer and TimeRecord records elapsed time (wall clock time)
-// used by each pass of the linker.
-struct TimerRecord {
-  TimerRecord(std::string name, TimerRecord *parent = nullptr);
-  void stop();
-
-  std::string name;
-  TimerRecord *parent;
-  tbb::concurrent_vector<TimerRecord *> children;
-  i64 start;
-  i64 end;
-  i64 user;
-  i64 sys;
-  bool stopped = false;
-};
-
-void
-print_timer_records(tbb::concurrent_vector<std::unique_ptr<TimerRecord>> &);
-
-template <typename Context>
-class Timer {
-public:
-  Timer(Context &ctx, std::string name, Timer *parent = nullptr) {
-    record = new TimerRecord(name, parent ? parent->record : nullptr);
-    ctx.timer_records.push_back(std::unique_ptr<TimerRecord>(record));
-  }
-
-  Timer(const Timer &) = delete;
-
-  ~Timer() {
-    record->stop();
-  }
-
-  void stop() {
-    record->stop();
-  }
-
-private:
-  TimerRecord *record;
-};
-
 //
 // tar.cc
 //
@@ -905,9 +997,9 @@ MappedFile<Context> *MappedFile<Context>::open(Context &ctx, std::string path) {
 
   i64 fd;
 #ifdef _WIN32
-    fd = ::_open(path.c_str(), O_RDONLY);
+  fd = ::_open(path.c_str(), O_RDONLY);
 #else
-    fd = ::open(path.c_str(), O_RDONLY);
+  fd = ::open(path.c_str(), O_RDONLY);
 #endif
 
   if (fd == -1) {
@@ -943,7 +1035,7 @@ MappedFile<Context> *MappedFile<Context>::open(Context &ctx, std::string path) {
     if (mf->data == MAP_FAILED)
       Fatal(ctx) << path << ": mmap failed: " << errno_string();
 #endif
-    }
+  }
 
   close(fd);
   return mf;

common/output-file-unix.h

@@ -52,14 +52,13 @@ class MemoryMappedOutputFile : public OutputFile<Context> {
 public:
   MemoryMappedOutputFile(Context &ctx, std::string path, i64 filesize, i64 perm)
     : OutputFile<Context>(path, filesize, true) {
-    i64 fd;
-    std::tie(fd, output_tmpfile) = open_or_create_file(ctx, path, filesize, perm);
+    std::tie(this->fd, output_tmpfile) =
+      open_or_create_file(ctx, path, filesize, perm);
 
     this->buf = (u8 *)mmap(nullptr, filesize, PROT_READ | PROT_WRITE,
-                           MAP_SHARED, fd, 0);
+                           MAP_SHARED, this->fd, 0);
     if (this->buf == MAP_FAILED)
       Fatal(ctx) << path << ": mmap failed: " << errno_string();
-    ::close(fd);
 
     mold::output_buffer_start = this->buf;
     mold::output_buffer_end = this->buf + filesize;
@@ -76,6 +75,15 @@ public:
     if (!this->is_unmapped)
       munmap(this->buf, this->filesize);
 
+    if (this->buf2.empty()) {
+      ::close(this->fd);
+    } else {
+      FILE *out = fdopen(this->fd, "w");
+      fseek(out, 0, SEEK_END);
+      fwrite(&this->buf2[0], this->buf2.size(), 1, out);
+      fclose(out);
+    }
+
     // If an output file already exists, open a file and then remove it.
     // This is the fastest way to unlink a file, as it does not make the
     // system to immediately release disk blocks occupied by the file.
@@ -92,39 +100,6 @@ private:
   int fd2 = -1;
 };
 
-template <typename Context>
-class MallocOutputFile : public OutputFile<Context> {
-public:
-  MallocOutputFile(Context &ctx, std::string path, i64 filesize, i64 perm)
-    : OutputFile<Context>(path, filesize, false), perm(perm) {
-    this->buf = (u8 *)mmap(NULL, filesize, PROT_READ | PROT_WRITE,
-                           MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-    if (this->buf == MAP_FAILED)
-      Fatal(ctx) << "mmap failed: " << errno_string();
-  }
-
-  void close(Context &ctx) override {
-    Timer t(ctx, "close_file");
-
-    if (this->path == "-") {
-      fwrite(this->buf, this->filesize, 1, stdout);
-      fclose(stdout);
-      return;
-    }
-
-    i64 fd = ::open(this->path.c_str(), O_RDWR | O_CREAT, perm);
-    if (fd == -1)
-      Fatal(ctx) << "cannot open " << this->path << ": " << errno_string();
-
-    FILE *fp = fdopen(fd, "w");
-    fwrite(this->buf, this->filesize, 1, fp);
-    fclose(fp);
-  }
-
-private:
-  i64 perm;
-};
-
 template <typename Context>
 std::unique_ptr<OutputFile<Context>>
 OutputFile<Context>::open(Context &ctx, std::string path, i64 filesize, i64 perm) {

common/output-file-win32.h

@@ -6,39 +6,6 @@
 
 namespace mold {
 
-template <typename Context>
-class MallocOutputFile : public OutputFile<Context> {
-public:
-  MallocOutputFile(Context &ctx, std::string path, i64 filesize, i64 perm)
-    : OutputFile<Context>(path, filesize, false), perm(perm) {
-    this->buf = (u8 *)malloc(filesize);
-    if (!this->buf)
-      Fatal(ctx) << "malloc failed";
-  }
-
-  void close(Context &ctx) override {
-    Timer t(ctx, "close_file");
-
-    if (this->path == "-") {
-      fwrite(this->buf, this->filesize, 1, stdout);
-      fclose(stdout);
-      return;
-    }
-
-    i64 fd = ::open(this->path.c_str(), O_RDWR | O_CREAT, perm);
-    if (fd == -1)
-      Fatal(ctx) << "cannot open " << this->path << ": " << errno_string();
-
-    FILE *fp = fdopen(fd, "w");
-    fwrite(this->buf, this->filesize, 1, fp);
-    fclose(fp);
-    free(this->buf);
-  }
-
-private:
-  i64 perm;
-};
-
 template <typename Context>
 std::unique_ptr<OutputFile<Context>>
 OutputFile<Context>::open(Context &ctx, std::string path, i64 filesize, i64 perm) {

dist.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+#
 # This script creates a mold binary distribution. The output is
 # written in this directory as `mold-$version-$arch-linux.tar.gz`
 # (e.g. `mold-1.0.3-x86_64-linux.tar.gz`).
@@ -7,6 +8,8 @@
 # libstdc++ and libcrypto but dynamically-linked to libc, libm, libz
 # and librt, as they almost always exist on any Linux systems.
 
+set -e -x
+
 case $# in
 0)
   arch=$(uname -m)
@@ -16,25 +19,60 @@ case $# in
   arch="$1"
   ;;
 *)
-  echo "Usage: $0 [ x86_64 | aarch64 | arm | ppc64le | s390x ]"
+  echo "Usage: $0 [ x86_64 | aarch64 | arm | riscv64 | ppc64le | s390x ]"
   exit 1
 esac
 
-echo "$arch" | grep -Eq '^(x86_64|aarch64|arm|ppc64le|s390x)$' || \
+echo "$arch" | grep -Eq '^(x86_64|aarch64|arm|riscv64|ppc64le|s390x)$' || \
   { echo "Error: no docker image for $arch"; exit 1; }
 
+image=mold-builder-$arch:1
 version=$(sed -n 's/^project(mold VERSION \(.*\))/\1/p' $(dirname $0)/CMakeLists.txt)
 dest=mold-$version-$arch-linux
-set -e -x
 
-docker run --platform linux/$arch -i --rm -v "$(pwd):/mold" \
-  -e "OWNER=$(id -u):$(id -g)" rui314/mold-builder:latest \
-  bash -c "mkdir /tmp/build &&
-cd /tmp/build &&
-cmake -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 -DMOLD_MOSTLY_STATIC=On -DCMAKE_BUILD_TYPE=Release /mold &&
+# Create a Docker image if not exists.
+#
+# We want to use a reasonably old Linux distro because we'll dynamically
+# link glibc to mold, and a binary linked against a newer version of
+# glibc won't work on a system with a older version of glibc.
+#
+# We might want to instead statically-link everything, but that'll
+# disable dlopen(), which means we can't use the linker plugin for LTO.
+# So we don't want to do that.
+if ! docker image ls $image | grep -q $image; then
+  if [ $arch = riscv64 ]; then
+    # Ubuntu 18 didn't support RISC-V, so use Ubuntu 20 instead
+    cat <<EOF | docker build --platform $arch -t $image -
+FROM riscv64/ubuntu:20.04
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive TZ=UTC apt-get install -y --no-install-recommends build-essential gcc-10 g++-10 cmake && \
+  rm -rf /var/lib/apt/lists
+EOF
+  else
+    cat <<EOF | docker build --platform $arch -t $image -
+FROM ubuntu:18.04
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends software-properties-common && \
+  add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive TZ=UTC apt-get install -y --no-install-recommends build-essential wget libstdc++-11-dev gcc-10 g++-10 python3 libssl-dev && \
+  \
+  mkdir /cmake && cd /cmake && \
+  wget -O- -q https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz | tar --strip-components=1 -xzf - && \
+  ./bootstrap --parallel=$(nproc) && make -j$(nproc) && make -j$(nproc) install && \
+  rm -rf /var/lib/apt/lists /cmake
+EOF
+  fi
+fi
+
+# Build mold in a container.
+docker run --platform linux/$arch -i --rm -v "$(realpath $(dirname $0)):/mold" \
+  -e "OWNER=$(id -u):$(id -g)" $image \
+  bash -c "mkdir /build &&
+cd /build &&
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 -DMOLD_MOSTLY_STATIC=On /mold &&
 cmake --build . -j\$(nproc) &&
 [ $arch = arm ] || ctest -j\$(nproc) &&
 cmake --install . --prefix $dest --strip &&
 tar czf /mold/$dest.tar.gz $dest &&
-cp mold mold-wrapper.so /mold &&
-chown \$OWNER /mold/mold /mold/mold-wrapper.so /mold/$dest.tar.gz"
+chown \$OWNER /mold/$dest.tar.gz"

docs/mold.md

@@ -764,22 +764,24 @@ arguments.
 ## ENVIRONMENT VARIABLES
 
 * `MOLD_JOBS`:
-  If this variable is set to `1`, only one process of `mold` runs actively. A
-  mold process invoked while another active mold process is running will wait
-  before doing anything until the active process exits.
+  If this variable is set to `1`, only one `mold` process will run at a time.
+  If a new mold process is initiated while another is already active, the new
+  process will wait until the active one completes before starting.
 
-  The purpose of this environment variable is to reduce peak memory usage.
-  Since mold is highly parallelized, there's no point in running it
-  simultaneously. If you run N instances of mold in parallel, it would take N
-  times more time and N times more memory. If you run them serially, it would
-  still take N times more to finish, but their peak memory usage is reduced to
-  normal.
+  The primary reason for this environment variable is to minimize peak memory
+  usage. Since mold is designed to operate with high parallelism, running
+  multiple mold instances simultaneously may not be beneficial. If you execute
+  N instances of mold concurrently, it could require N times the time and N
+  times the memory. On the other hand, running them one after the other might
+  still take N times longer, but the peak memory usage would be the same as
+  running just a single instance.
 
-  If your build system tends to invoke multiple linker processes
-  simultaneously, you may want to try to set this environment variable to
-  `1` to see if it could improve overall performance.
+  If your build system invokes multiple linker processes simultaneously and
+  some of them often get killed due to out-of-memory errors, you might
+  consider setting this environment variable to `1` to see if it addresses the
+  OOM issue.
 
-  Currently, any value other than 1 is silently ignored.
+  Currently, any value other than `1` is silently ignored.
 
 * `MOLD_DEBUG`:
   If this variable is set to a non-empty string, `mold` embeds its

elf/arch-alpha.cc

@@ -318,7 +318,7 @@ void AlphaGotSection::copy_buf(Context<E> &ctx) {
 
     if (e.sym->is_imported) {
       *buf = ctx.arg.apply_dynamic_relocs ? e.addend : 0;
-      *dynrel++ = ElfRel<E>(P, E::R_DYNAMIC, e.sym->get_dynsym_idx(ctx), e.addend);
+      *dynrel++ = ElfRel<E>(P, E::R_ABS, e.sym->get_dynsym_idx(ctx), e.addend);
     } else {
       *buf = e.sym->get_addr(ctx) + e.addend;
       if (ctx.arg.pic && !e.sym->is_absolute())

elf/arch-arm32.cc

@@ -200,9 +200,9 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
     0xe08f'e00e, // 1: add lr, pc, lr
     0xe5be'f008, //    ldr pc, [lr, #8]!
     0x0000'0000, // 2: .word .got.plt - 1b - 8
-    0xe320'f000, //    nop
-    0xe320'f000, //    nop
-    0xe320'f000, //    nop
+    0x0000'0000, //    (padding)
+    0x0000'0000, //    (padding)
+    0x0000'0000, //    (padding)
   };
 
   memcpy(buf, insn, sizeof(insn));
@@ -225,7 +225,7 @@ void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
-  *(ul32 *)(buf + 12) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  *(ul32 *)(buf + 12) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 12;
 }
 
 // ARM does not use .eh_frame for exception handling. Instead, it uses

elf/arch-arm64.cc

@@ -93,7 +93,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
     0xd503'201f, // nop
   };
 
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, insn, sizeof(insn));

elf/arch-i386.cc

@@ -164,7 +164,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
       0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
     };
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
   } else {
     static const u8 insn[] = {
       0xf3, 0x0f, 0x1e, 0xfb,             // endbr32
@@ -172,7 +172,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
       0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // (padding)
     };
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 6) = sym.get_got_addr(ctx);
+    *(ul32 *)(buf + 6) = sym.get_got_pltgot_addr(ctx);
   }
 }
 

elf/arch-loongarch.cc

@@ -65,14 +65,12 @@ static u64 hi64(u64 val, u64 pc) {
   // LU52I.D simply set bits to [51:31] and to [63:53], respectively.
   //
   // Compensating all the sign-extensions is a bit complicated.
-  bool x = val & 0x800;
-  bool y = (page(val + 0x800) - page(pc)) & 0x8000'0000;
-
-  if (x && !y)
-    return val - 0x1'0000'0000;
-  if (!x && y)
-    return val + 0x1'0000'0000;
-  return val;
+  u64 x = page(val) - page(pc);
+  if (val & 0x800)
+    x += 0x1000 - 0x1'0000'0000;
+  if (x & 0x8000'0000)
+    x += 0x1'0000'0000;
+  return x;
 }
 
 static u64 higher20(u64 val, u64 pc) {
@@ -174,7 +172,7 @@ void write_plt_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
 template <>
 void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, E::is_64 ? plt_entry_64 : plt_entry_32, E::plt_size);
@@ -325,10 +323,16 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, (S + A) >> 52);
       break;
     case R_LARCH_PCALA_LO12:
-      write_k12(loc, S + A);
+      // It looks like R_LARCH_PCALA_LO12 is sometimes used for JIRL even
+      // though the instruction takes a 16 bit immediate rather than 12 bits.
+      // It is contrary to the psABI document, but GNU ld has special
+      // code to handle it, so we accept it too.
+      if ((*(ul32 *)loc & 0xfc00'0000) == 0x4c00'0000)
+        write_k16(loc, sign_extend(S + A, 11) >> 2);
+      else
+        write_k12(loc, S + A);
       break;
     case R_LARCH_PCALA_HI20:
-      check(S + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(S + A, P));
       break;
     case R_LARCH_PCALA64_LO20:
@@ -341,7 +345,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, GOT + G + A);
       break;
     case R_LARCH_GOT_PC_HI20:
-      check(GOT + G + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(GOT + G + A, P));
       break;
     case R_LARCH_GOT64_PC_LO20:
@@ -378,7 +381,6 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_k12(loc, sym.get_gottp_addr(ctx) + A);
       break;
     case R_LARCH_TLS_IE_PC_HI20:
-      check(sym.get_gottp_addr(ctx) + A - P, -(1LL << 31), 1LL << 31);
       write_j20(loc, hi20(sym.get_gottp_addr(ctx) + A, P));
       break;
     case R_LARCH_TLS_IE64_PC_LO20:
@@ -587,6 +589,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       scan_dyn_absrel(ctx, sym, rel);
       break;
     case R_LARCH_B26:
+    case R_LARCH_PCALA_HI20:
       if (sym.is_imported)
         sym.flags |= NEEDS_PLT;
       break;
@@ -620,7 +623,6 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_LARCH_ABS_LO12:
     case R_LARCH_ABS64_LO20:
     case R_LARCH_ABS64_HI12:
-    case R_LARCH_PCALA_HI20:
     case R_LARCH_PCALA_LO12:
     case R_LARCH_PCALA64_LO20:
     case R_LARCH_PCALA64_HI12:
@@ -658,10 +660,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
 template <>
 void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
   static const ul32 insn[] = {
-    0x1a00'000c, // pcalau12i $t0, 0
-    0x02c0'018c, // addi.d    $t0, $t0, 0
+    0x1e00'000c, // pcaddu18i $t0, 0
     0x4c00'0180, // jirl      $zero, $t0, 0
-    0x0340'0000, // nop
   };
 
   static_assert(E::thunk_size == sizeof(insn));
@@ -673,8 +673,8 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
     u64 S = sym->get_addr(ctx);
 
     memcpy(buf, insn, sizeof(insn));
-    write_j20(buf, hi20(S, P));
-    write_k12(buf + 4, S);
+    write_j20(buf, (S - P + 0x20000) >> 18);
+    write_k16(buf + 4, (S - P) >> 2);
 
     buf += sizeof(insn);
     P += sizeof(insn);

elf/arch-m68k.cc

@@ -52,7 +52,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)(buf + 4) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 2;
+  *(ub32 *)(buf + 4) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 2;
 }
 
 template <>

elf/arch-ppc32.cc

@@ -120,7 +120,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   memcpy(buf, plt_entry, sizeof(plt_entry));
 
   ub32 *loc = (ub32 *)buf;
-  i64 offset = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 8;
+  i64 offset = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 8;
   loc[4] |= higha(offset);
   loc[5] |= lo(offset);
 }

elf/arch-ppc64v1.cc

@@ -465,7 +465,6 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
   static_assert(E::thunk_size == sizeof(local_thunk));
 
   u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
-  u64 P = output_section.shdr.sh_addr + offset;
 
   for (Symbol<E> *sym : symbols) {
     if (sym->has_got(ctx)) {
@@ -486,7 +485,6 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
     }
 
     buf += E::thunk_size;
-    P += E::thunk_size;
   }
 }
 
@@ -521,7 +519,7 @@ struct OpdSymbol {
 };
 
 static Symbol<E> *
-get_opd_sym_at(Context<E> &ctx, std::span<OpdSymbol> syms, u64 offset) {
+get_opd_sym_at(std::span<OpdSymbol> syms, u64 offset) {
   auto it = std::lower_bound(syms.begin(), syms.end(), OpdSymbol{offset});
   if (it == syms.end())
     return nullptr;
@@ -616,7 +614,7 @@ void ppc64v1_rewrite_opd(Context<E> &ctx) {
         if (sym.get_input_section() != opd)
           continue;
 
-        Symbol<E> *real_sym = get_opd_sym_at(ctx, opd_syms, r.r_addend);
+        Symbol<E> *real_sym = get_opd_sym_at(opd_syms, r.r_addend);
         if (!real_sym)
           Fatal(ctx) << *isec << ": cannot find a symbol in .opd for " << r
                      << " at offset 0x" << std::hex << (u64)r.r_addend;

elf/arch-ppc64v2.cc

@@ -92,8 +92,10 @@ static u64 ha(u64 x)    { return (x + 0x8000) >> 16; }
 static u64 high(u64 x)  { return (x >> 16) & 0xffff; }
 static u64 higha(u64 x) { return ((x + 0x8000) >> 16) & 0xffff; }
 
-static u64 prefix34(u64 x) {
-  return bits(x, 33, 16) | (bits(x, 15, 0) << 32);
+static void write34(u8 *loc, u64 x) {
+  ul32 *buf = (ul32 *)loc;
+  buf[0] = (buf[0] & 0xfffc'0000) | bits(x, 33, 16);
+  buf[1] = (buf[1] & 0xffff'0000) | bits(x, 15, 0);
 }
 
 // .plt is used only for lazy symbol resolution on PPC64. All PLT
@@ -205,7 +207,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     u64 TOC = ctx.extra.TOC->value;
 
     auto r2save_thunk_addr = [&] { return get_thunk_addr(i); };
-    auto no_r2save_thunk_addr = [&] { return get_thunk_addr(i) + 4; };
+    auto no_r2save_thunk_addr = [&] { return get_thunk_addr(i) + 8; };
 
     switch (rel.r_type) {
     case R_PPC64_ADDR64:
@@ -279,10 +281,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_PLT_PCREL34:
     case R_PPC64_PLT_PCREL34_NOTOC:
     case R_PPC64_GOT_PCREL34:
-      *(ul64 *)loc |= prefix34(G + GOT - P);
+      write34(loc, G + GOT - P);
       break;
     case R_PPC64_PCREL34:
-      *(ul64 *)loc |= prefix34(S + A - P);
+      write34(loc, S + A - P);
       break;
     case R_PPC64_GOT_TPREL16_HA:
       *(ul16 *)loc = ha(sym.get_gottp_addr(ctx) - TOC);
@@ -291,7 +293,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc |= (sym.get_gottp_addr(ctx) - TOC) & 0xfffc;
       break;
     case R_PPC64_GOT_TPREL_PCREL34:
-      *(ul64 *)loc |= prefix34(sym.get_gottp_addr(ctx) - P);
+      write34(loc, sym.get_gottp_addr(ctx) - P);
       break;
     case R_PPC64_GOT_TLSGD16_HA:
       *(ul16 *)loc = ha(sym.get_tlsgd_addr(ctx) - TOC);
@@ -300,7 +302,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc = lo(sym.get_tlsgd_addr(ctx) - TOC);
       break;
     case R_PPC64_GOT_TLSGD_PCREL34:
-      *(ul64 *)loc |= prefix34(sym.get_tlsgd_addr(ctx) - P);
+      write34(loc, sym.get_tlsgd_addr(ctx) - P);
       break;
     case R_PPC64_GOT_TLSLD16_HA:
       *(ul16 *)loc = ha(ctx.got->get_tlsld_addr(ctx) - TOC);
@@ -309,7 +311,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc = lo(ctx.got->get_tlsld_addr(ctx) - TOC);
       break;
     case R_PPC64_GOT_TLSLD_PCREL34:
-      *(ul64 *)loc |= prefix34(ctx.got->get_tlsld_addr(ctx) - P);
+      write34(loc, ctx.got->get_tlsld_addr(ctx) - P);
       break;
     case R_PPC64_DTPREL16_HA:
       *(ul16 *)loc = ha(S + A - ctx.dtp_addr);
@@ -318,7 +320,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       *(ul16 *)loc = lo(S + A - ctx.dtp_addr);
       break;
     case R_PPC64_DTPREL34:
-      *(ul64 *)loc |= prefix34(S + A - ctx.dtp_addr);
+      write34(loc, S + A - ctx.dtp_addr);
       break;
     case R_PPC64_TPREL16_HA:
       *(ul16 *)loc = ha(S + A - ctx.tp_addr);
@@ -326,6 +328,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     case R_PPC64_TPREL16_LO:
       *(ul16 *)loc = lo(S + A - ctx.tp_addr);
       break;
+    case R_PPC64_TPREL34:
+      write34(loc, S + A - ctx.tp_addr);
+      break;
     case R_PPC64_PLTSEQ:
     case R_PPC64_PLTSEQ_NOTOC:
     case R_PPC64_PLTCALL:
@@ -443,6 +448,7 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
       break;
     case R_PPC64_TPREL16_HA:
     case R_PPC64_TPREL16_LO:
+    case R_PPC64_TPREL34:
       check_tlsle(ctx, sym, rel);
       break;
     case R_PPC64_REL32:
@@ -483,6 +489,7 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
   // and jump there.
   static const ul32 plt_thunk[] = {
     0xf841'0018, // std   r2, 24(r1)
+    0x6000'0000, // nop
     0x3d82'0000, // addis r12, r2, foo@gotplt@toc@ha
     0xe98c'0000, // ld    r12, foo@gotplt@toc@lo(r12)
     0x7d89'03a6, // mtctr r12
@@ -491,6 +498,7 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
 
   static const ul32 plt_thunk_power10[] = {
     0xf841'0018, // std   r2, 24(r1)
+    0x6000'0000, // nop
     0x0410'0000, // pld   r12, foo@gotplt@pcrel
     0xe580'0000,
     0x7d89'03a6, // mtctr r12
@@ -501,6 +509,7 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
   // to its local entry point.
   static const ul32 local_thunk[] = {
     0xf841'0018, // std   r2, 24(r1)
+    0x6000'0000, // nop
     0x3d82'0000, // addis r12, r2,  foo@toc@ha
     0x398c'0000, // addi  r12, r12, foo@toc@lo
     0x7d89'03a6, // mtctr r12
@@ -509,6 +518,7 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
 
   static const ul32 local_thunk_power10[] = {
     0xf841'0018, // std   r2, 24(r1)
+    0x6000'0000, // nop
     0x0610'0000, // pla   r12, foo@pcrel
     0x3980'0000,
     0x7d89'03a6, // mtctr r12
@@ -522,6 +532,7 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
 
   u8 *buf = ctx.buf + output_section.shdr.sh_offset + offset;
   u64 P = output_section.shdr.sh_addr + offset;
+  u64 TOC = ctx.extra.TOC->value;
 
   for (Symbol<E> *sym : symbols) {
     if (sym->has_plt(ctx)) {
@@ -530,22 +541,21 @@ void RangeExtensionThunk<E>::copy_buf(Context<E> &ctx) {
 
       if (ctx.extra.is_power10) {
         memcpy(buf, plt_thunk_power10, E::thunk_size);
-        *(ul64 *)(buf + 1) |= prefix34(got - P - 4);
+        write34(buf + 8, got - P - 8);
       } else {
-        i64 val = got - ctx.extra.TOC->value;
         memcpy(buf, plt_thunk, E::thunk_size);
-        *(ul32 *)(buf + 4) |= higha(val);
-        *(ul32 *)(buf + 8) |= lo(val);
+        *(ul32 *)(buf + 8) |= higha(got - TOC);
+        *(ul32 *)(buf + 12) |= lo(got - TOC);
       }
     } else {
+      u64 S = sym->get_addr(ctx);
       if (ctx.extra.is_power10) {
         memcpy(buf, local_thunk_power10, E::thunk_size);
-        *(ul64 *)(buf + 1) |= prefix34(sym->get_addr(ctx) - P - 4);
+        write34(buf + 8, S - P - 8);
       } else {
-        i64 val = sym->get_addr(ctx) - ctx.extra.TOC->value;
         memcpy(buf, local_thunk, E::thunk_size);
-        *(ul32 *)(buf + 4) |= higha(val);
-        *(ul32 *)(buf + 8) |= lo(val);
+        *(ul32 *)(buf + 8) |= higha(S - TOC);
+        *(ul32 *)(buf + 12) |= lo(S - TOC);
       }
     }
 

elf/arch-riscv.cc

@@ -198,7 +198,7 @@ void write_plt_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
 template <>
 void write_pltgot_entry<E>(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  u64 got = sym.get_got_addr(ctx);
+  u64 got = sym.get_got_pltgot_addr(ctx);
   u64 plt = sym.get_plt_addr(ctx);
 
   memcpy(buf, E::is_64 ? plt_entry_64 : plt_entry_32, E::plt_size);
@@ -259,6 +259,7 @@ static inline bool is_hi20(const ElfRel<E> &rel) {
 template <>
 void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
   std::span<const ElfRel<E>> rels = get_rels(ctx);
+  u64 GP = ctx.__global_pointer ? ctx.__global_pointer->get_addr(ctx) : 0;
 
   ElfRel<E> *dynrel = nullptr;
   if (ctx.reldyn)
@@ -375,7 +376,9 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       write_utype(loc, S + A - P);
       break;
     case R_RISCV_PCREL_LO12_I:
-    case R_RISCV_PCREL_LO12_S: {
+    case R_RISCV_PCREL_LO12_S:
+    case R_RISCV_GPREL_LO12_I:
+    case R_RISCV_GPREL_LO12_S: {
       i64 idx2 = find_paired_reloc();
       const ElfRel<E> &rel2 = rels[idx2];
       Symbol<E> &sym2 = *file.symbols[rel2.r_sym];
@@ -399,11 +402,15 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       case R_RISCV_PCREL_HI20:
         val = S + A - P;
         break;
+      case R_RISCV_GPREL_HI20:
+        val = S + A - GP;
+        break;
       default:
         unreachable();
       }
 
-      if (rel.r_type == R_RISCV_PCREL_LO12_I)
+      if (rel.r_type == R_RISCV_PCREL_LO12_I ||
+          rel.r_type == R_RISCV_GPREL_LO12_I)
         write_itype(loc, val);
       else
         write_stype(loc, val);
@@ -560,6 +567,10 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       check(S + A - P, -(1 << 11), 1 << 11);
       write_cjtype(loc, S + A - P);
       break;
+    case R_RISCV_GPREL_HI20:
+      check(S + A - GP, -(1LL << 31), 1LL << 31);
+      write_utype(loc, S + A - GP);
+      break;
     case R_RISCV_SUB6:
       *loc = (*loc & 0b1100'0000) | ((*loc - S - A) & 0b0011'1111);
       break;
@@ -761,6 +772,10 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_RISCV_TPREL_ADD:
       check_tlsle(ctx, sym, rel);
       break;
+    case R_RISCV_GPREL_HI20:
+      if (ctx.arg.shared)
+        Fatal(ctx) << *this << ": R_RISCV_GPREL_HI20 may not be used with -shared";
+      break;
     case R_RISCV_BRANCH:
     case R_RISCV_JAL:
     case R_RISCV_PCREL_LO12_I:
@@ -781,6 +796,8 @@ void InputSection<E>::scan_relocations(Context<E> &ctx) {
     case R_RISCV_ALIGN:
     case R_RISCV_RVC_BRANCH:
     case R_RISCV_RVC_JUMP:
+    case R_RISCV_GPREL_LO12_I:
+    case R_RISCV_GPREL_LO12_S:
     case R_RISCV_RELAX:
     case R_RISCV_SUB6:
     case R_RISCV_SET6:
@@ -821,7 +838,7 @@ u64 get_eflags(Context<E> &ctx) {
   return ret;
 }
 
-static bool is_resizable(Context<E> &ctx, InputSection<E> *isec) {
+static bool is_resizable(InputSection<E> *isec) {
   return isec && isec->is_alive && (isec->shdr().sh_flags & SHF_ALLOC) &&
          (isec->shdr().sh_flags & SHF_EXECINSTR);
 }
@@ -1021,7 +1038,7 @@ i64 riscv_resize_sections<E>(Context<E> &ctx) {
   // This step should only shrink sections.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     for (std::unique_ptr<InputSection<E>> &isec : file->sections)
-      if (is_resizable(ctx, isec.get()))
+      if (is_resizable(isec.get()))
         shrink_section(ctx, *isec, use_rvc);
   });
 

elf/arch-s390x.cc

@@ -90,7 +90,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ub32 *)(buf + 2) = (sym.get_got_addr(ctx) - sym.get_plt_addr(ctx)) >> 1;
+  *(ub32 *)(buf + 2) = (sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx)) >> 1;
 }
 
 template <>

elf/arch-sh4.cc

@@ -169,7 +169,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
     static_assert(sizeof(insn) == E::pltgot_size);
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_addr(ctx) - ctx.got->shdr.sh_addr;
+    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx) - ctx.got->shdr.sh_addr;
   } else {
     static const u8 insn[] = {
       0x01, 0xd0, //    mov.l   1f, r0
@@ -181,7 +181,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
 
     static_assert(sizeof(insn) == E::pltgot_size);
     memcpy(buf, insn, sizeof(insn));
-    *(ul32 *)(buf + 8) = sym.get_got_addr(ctx);
+    *(ul32 *)(buf + 8) = sym.get_got_pltgot_addr(ctx);
   }
 }
 

elf/arch-sparc64.cc

@@ -115,7 +115,7 @@ void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   };
 
   memcpy(buf, entry, sizeof(entry));
-  *(ub64 *)(buf + 24) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 4;
+  *(ub64 *)(buf + 24) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 4;
 }
 
 template <>

elf/arch-x86-64.cc

@@ -64,28 +64,41 @@ void write_plt_header(Context<E> &ctx, u8 *buf) {
 
 template <>
 void write_plt_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
-  static const u8 insn[] = {
-    0xf3, 0x0f, 0x1e, 0xfa, // endbr64
-    0x41, 0xbb, 0, 0, 0, 0, // mov $index_in_relplt, %r11d
-    0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOTPLT
-  };
+  // Only a canonical PLT can be address-taken; there's no way to take
+  // an address of a non-canonical PLT. Therefore, a non-canonical PLT
+  // doesn't have to start with an endbr64.
+  if (sym.is_canonical) {
+    static const u8 insn[] = {
+      0xf3, 0x0f, 0x1e, 0xfa, // endbr64
+      0x41, 0xbb, 0, 0, 0, 0, // mov $index_in_relplt, %r11d
+      0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOTPLT
+    };
 
-  memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 6) = sym.get_plt_idx(ctx);
-  *(ul32 *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 16;
+    memcpy(buf, insn, sizeof(insn));
+    *(ul32 *)(buf + 6) = sym.get_plt_idx(ctx);
+    *(ul32 *)(buf + 12) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 16;
+  } else {
+    static const u8 insn[] = {
+      0x41, 0xbb, 0, 0, 0, 0, // mov $index_in_relplt, %r11d
+      0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOTPLT
+      0xcc, 0xcc, 0xcc, 0xcc, // (padding)
+    };
+
+    memcpy(buf, insn, sizeof(insn));
+    *(ul32 *)(buf + 2) = sym.get_plt_idx(ctx);
+    *(ul32 *)(buf + 8) = sym.get_gotplt_addr(ctx) - sym.get_plt_addr(ctx) - 12;
+  }
 }
 
 template <>
 void write_pltgot_entry(Context<E> &ctx, u8 *buf, Symbol<E> &sym) {
   static const u8 insn[] = {
-    0xf3, 0x0f, 0x1e, 0xfa, // endbr64
     0xff, 0x25, 0, 0, 0, 0, // jmp *foo@GOT
-    0xcc, 0xcc, 0xcc, 0xcc, // (padding)
     0xcc, 0xcc,             // (padding)
   };
 
   memcpy(buf, insn, sizeof(insn));
-  *(ul32 *)(buf + 6) = sym.get_got_addr(ctx) - sym.get_plt_addr(ctx) - 10;
+  *(ul32 *)(buf + 2) = sym.get_got_pltgot_addr(ctx) - sym.get_plt_addr(ctx) - 6;
 }
 
 template <>
@@ -345,7 +358,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
     u64 A = rel.r_addend;
     u64 P = get_addr() + rel.r_offset;
     u64 G = sym.get_got_addr(ctx) - ctx.gotplt->shdr.sh_addr;
-    u64 GOTPLT = ctx.gotplt->shdr.sh_addr;
+    u64 GOT = ctx.gotplt->shdr.sh_addr;
 
     switch (rel.r_type) {
     case R_X86_64_8:
@@ -388,19 +401,19 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
       break;
     case R_X86_64_GOTOFF64:
     case R_X86_64_PLTOFF64:
-      *(ul64 *)loc = S + A - GOTPLT;
+      *(ul64 *)loc = S + A - GOT;
       break;
     case R_X86_64_GOTPC32:
-      write32s(GOTPLT + A - P);
+      write32s(GOT + A - P);
       break;
     case R_X86_64_GOTPC64:
-      *(ul64 *)loc = GOTPLT + A - P;
+      *(ul64 *)loc = GOT + A - P;
       break;
     case R_X86_64_GOTPCREL:
-      write32s(G + GOTPLT + A - P);
+      write32s(G + GOT + A - P);
       break;
     case R_X86_64_GOTPCREL64:
-      *(ul64 *)loc = G + GOTPLT + A - P;
+      *(ul64 *)loc = G + GOT + A - P;
       break;
     case R_X86_64_GOTPCRELX:
       // We always want to relax GOTPCRELX relocs even if --no-relax
@@ -416,7 +429,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           break;
         }
       }
-      write32s(G + GOTPLT + A - P);
+      write32s(G + GOT + A - P);
       break;
     case R_X86_64_REX_GOTPCRELX:
       if (sym.is_pcrel_linktime_const(ctx)) {
@@ -430,7 +443,7 @@ void InputSection<E>::apply_reloc_alloc(Context<E> &ctx, u8 *base) {
           break;
         }
       }
-      write32s(G + GOTPLT + A - P);
+      write32s(G + GOT + A - P);
       break;
     case R_X86_64_TLSGD:
       if (sym.has_tlsgd(ctx))

elf/cmdline.cc

@@ -883,6 +883,8 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
       ctx.arg.z_sectionheader = true;
     } else if (read_z_flag("nosectionheader")) {
       ctx.arg.z_sectionheader = false;
+    } else if (read_z_flag("rewrite-endbr")) {
+      ctx.arg.z_rewrite_endbr = true;
     } else if (read_flag("no-undefined")) {
       ctx.arg.z_defs = true;
     } else if (read_flag("fatal-warnings")) {
@@ -1262,6 +1264,14 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
   if (ctx.arg.shared)
     ctx.overwrite_output_file = false;
 
+  if (!ctx.arg.chroot.empty()) {
+    if (!ctx.arg.Map.empty())
+      ctx.arg.Map = ctx.arg.chroot + "/" + ctx.arg.Map;
+
+    if (!ctx.arg.dependency_file.empty())
+      ctx.arg.dependency_file = ctx.arg.chroot + "/" + ctx.arg.dependency_file;
+  }
+
   if (version_shown && remaining.empty())
     exit(0);
   return remaining;

elf/dwarf.cc

@@ -1,554 +0,0 @@
-// This file contains code to read DWARF debug info to create .gdb_index.
-//
-// .gdb_index is an optional section to speed up GNU debugger. It contains
-// two maps: 1) a map from function/variable/type names to compunits, and
-// 2) a map from function address ranges to compunits. gdb uses these
-// maps to quickly find a compunit given a name or an instruction pointer.
-//
-// (Terminology: a compilation unit, which often abbreviated as compunit
-// or cu, is a unit of debug info. An input .debug_info section usually
-// contains one compunit, and thus an output .debug_info contains as
-// many compunits as the number of input files.)
-//
-// .gdb_index is not mandatory. All the information in .gdb_index is
-// also in other debug info sections. You can actually create an
-// executable without .gdb_index and later add it using `gdb-add-index`
-// post-processing tool that comes with gdb.
-//
-// The mapping from names to compunits is 1:n while the mapping from
-// address ranges to compunits is 1:1. That is, two object files may
-// define the same type name (with the same definition), while there
-// should be no two functions that overlap with each other in memory.
-//
-// .gdb_index contains an on-disk hash table for names, so gdb can
-// lookup names without loading all strings into memory and construct an
-// in-memory hash table.
-//
-// Names are in .debug_gnu_pubnames and .debug_gnu_pubtypes input
-// sections. These sections are created if `-ggnu-pubnames` is given.
-// Besides names, these sections contains attributes for each name so
-// that gdb can distinguish type names from function names, for example.
-//
-// A compunit contains one or more function address ranges. If an
-// object file is compiled without -ffunction-sections, it contains
-// only one .text section and therefore contains a single address range.
-// Such range is typically stored directly to the compunit.
-//
-// If an object file is compiled with -fucntion-sections, it contains
-// more than one .text section, and it has as many address ranges as
-// the number of .text sections. Such discontiguous address ranges are
-// stored to .debug_ranges in DWARF 2/3/4/5 and
-// .debug_rnglists/.debug_addr in DWARF 5.
-//
-// .debug_info section contains DWARF debug info. Although we don't need
-// to parse the whole .debug_info section to read address ranges, we
-// have to do a little bit. DWARF is complicated and often handled using
-// a library such as libdwarf. But we don't use any library because we
-// don't want to add an extra run-time dependency just for --gdb-index.
-//
-// This page explains the format of .gdb_index:
-// https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html
-
-#include "mold.h"
-
-namespace mold::elf {
-
-// The hash function for .gdb_index.
-static u32 gdb_hash(std::string_view name) {
-  u32 h = 0;
-  for (u8 c : name) {
-    if ('A' <= c && c <= 'Z')
-      c = 'a' + c - 'A';
-    h = h * 67 + c - 113;
-  }
-  return h;
-}
-
-// Split .debug_info into so-called "compilation units". A .debug_info
-// section usually contains one compunit unless it was created by `ld -r`.
-// This is for --gdb-index.
-template <typename E>
-std::vector<std::string_view>
-read_compunits(Context<E> &ctx, ObjectFile<E> &file) {
-  file.debug_info->uncompress(ctx);
-  std::string_view data = file.debug_info->contents;
-  std::vector<std::string_view> vec;
-
-  while (!data.empty()) {
-    if (data.size() < 4)
-      Fatal(ctx) << *file.debug_info << ": corrupted .debug_info";
-    if (*(U32<E> *)data.data() == 0xffff'ffff)
-      Fatal(ctx) << *file.debug_info << ": --gdb-index: DWARF64 not supported";
-    i64 len = *(U32<E> *)data.data() + 4;
-    vec.push_back(data.substr(0, len));
-    data = data.substr(len);
-  }
-  return vec;
-}
-
-// Parses .debug_gnu_pubnames and .debug_gnu_pubtypes. These sections
-// start with a 14 bytes header followed by (4-byte offset, 1-byte type,
-// null-terminated string) tuples.
-//
-// The 4-byte offset is an offset into .debug_info that contains details
-// about the name. The 1-byte type is a type of the corresponding name
-// (e.g. function, variable or datatype). The string is a name of a
-// function, a variable or a type.
-template <typename E>
-std::vector<GdbIndexName> read_pubnames(Context<E> &ctx, ObjectFile<E> &file) {
-  std::vector<GdbIndexName> vec;
-
-  auto get_cu_idx = [&](InputSection<E> &isec, i64 offset) {
-    i64 off = 0;
-    for (i64 i = 0; i < file.compunits.size(); i++) {
-      if (offset == off)
-        return file.compunits_idx + i;
-      off += file.compunits[i].size();
-    }
-    Fatal(ctx) << isec << ": corrupted debug_info_offset";
-  };
-
-  auto read = [&](InputSection<E> &isec) {
-    isec.uncompress(ctx);
-    std::string_view contents = isec.contents;
-
-    while (!contents.empty()) {
-      if (contents.size() < 14)
-        Fatal(ctx) << isec << ": corrupted header";
-
-      u32 len = *(U32<E> *)contents.data() + 4;
-      u32 debug_info_offset = *(U32<E> *)(contents.data() + 6);
-      u32 cu_idx = get_cu_idx(isec, debug_info_offset);
-
-      std::string_view data = contents.substr(14, len - 14);
-      contents = contents.substr(len);
-
-      while (!data.empty()) {
-        u32 offset = *(U32<E> *)data.data();
-        data = data.substr(4);
-        if (offset == 0)
-          break;
-
-        u8 type = data[0];
-        data = data.substr(1);
-
-        std::string_view name = data.data();
-        data = data.substr(name.size() + 1);
-
-        vec.push_back({name, gdb_hash(name), (type << 24) | cu_idx});
-      }
-    }
-  };
-
-  if (file.debug_pubnames)
-    read(*file.debug_pubnames);
-  if (file.debug_pubtypes)
-    read(*file.debug_pubtypes);
-
-  // Uniquify elements because GCC 11 seems to emit one record for each
-  // comdat group which results in having a lot of duplicate records.
-  auto less = [](const GdbIndexName &a, const GdbIndexName &b) {
-    return std::tuple{a.hash, a.attr, a.name} <
-           std::tuple{b.hash, b.attr, b.name};
-  };
-
-  auto equal = [](const GdbIndexName &a, const GdbIndexName &b) {
-    return std::tuple{a.hash, a.attr, a.name} ==
-           std::tuple{b.hash, b.attr, b.name};
-  };
-
-  std::sort(vec.begin(), vec.end(), less);
-  vec.erase(std::unique(vec.begin(), vec.end(), equal), vec.end());
-  return vec;
-}
-
-template <typename E>
-static u8 *get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
-  if (u8 *buf = chunk->get_uncompressed_data())
-    return buf;
-  return ctx.buf + chunk->shdr.sh_offset;
-}
-
-// Try to find a compilation unit from .debug_info and its
-// corresponding record from .debug_abbrev and returns them.
-template <typename E>
-static std::tuple<u8 *, u8 *, u32>
-find_compunit(Context<E> &ctx, ObjectFile<E> &file, i64 offset) {
-  // Read .debug_info to find the record at a given offset.
-  u8 *cu = get_buffer(ctx, ctx.debug_info) + offset;
-  u32 dwarf_version = *(U16<E> *)(cu + 4);
-  u32 abbrev_offset;
-
-  // Skip a header.
-  switch (dwarf_version) {
-  case 2:
-  case 3:
-  case 4:
-    abbrev_offset = *(U32<E> *)(cu + 6);
-    if (u32 address_size = cu[10]; address_size != sizeof(Word<E>))
-      Fatal(ctx) << file << ": --gdb-index: unsupported address size "
-                 << address_size;
-    cu += 11;
-    break;
-  case 5: {
-    abbrev_offset = *(U32<E> *)(cu + 8);
-    if (u32 address_size = cu[7]; address_size != sizeof(Word<E>))
-      Fatal(ctx) << file << ": --gdb-index: unsupported address size "
-                 << address_size;
-
-    switch (u32 unit_type = cu[6]; unit_type) {
-    case DW_UT_compile:
-    case DW_UT_partial:
-      cu += 12;
-      break;
-    case DW_UT_skeleton:
-    case DW_UT_split_compile:
-      cu += 20;
-      break;
-    default:
-      Fatal(ctx) << file << ": --gdb-index: unknown DW_UT_* value: 0x"
-                 << std::hex << unit_type;
-    }
-    break;
-  }
-  default:
-    Fatal(ctx) << file << ": --gdb-index: unknown DWARF version: "
-               << dwarf_version;
-  }
-
-  u32 abbrev_code = read_uleb(&cu);
-
-  // Find a .debug_abbrev record corresponding to the .debug_info record.
-  // We assume the .debug_info record at a given offset is of
-  // DW_TAG_compile_unit which describes a compunit.
-  u8 *abbrev = get_buffer(ctx, ctx.debug_abbrev) + abbrev_offset;
-
-  for (;;) {
-    u32 code = read_uleb(&abbrev);
-    if (code == 0)
-      Fatal(ctx) << file << ": --gdb-index: .debug_abbrev does not contain"
-                 << " a record for the first .debug_info record";
-
-    if (code == abbrev_code) {
-      // Found a record
-      u64 abbrev_tag = read_uleb(&abbrev);
-      if (abbrev_tag != DW_TAG_compile_unit && abbrev_tag != DW_TAG_skeleton_unit)
-        Fatal(ctx) << file << ": --gdb-index: the first entry's tag is not"
-                   << " DW_TAG_compile_unit/DW_TAG_skeleton_unit but 0x"
-                   << std::hex << abbrev_tag;
-      break;
-    }
-
-    // Skip an uninteresting record
-    read_uleb(&abbrev); // tag
-    abbrev++; // has_children byte
-    for (;;) {
-      u64 name = read_uleb(&abbrev);
-      u64 form = read_uleb(&abbrev);
-      if (name == 0 && form == 0)
-        break;
-      if (form == DW_FORM_implicit_const)
-        read_uleb(&abbrev);
-    }
-  }
-
-  abbrev++; // skip has_children byte
-  return {cu, abbrev, dwarf_version};
-}
-
-// Estimate the number of address ranges contained in a given file.
-// It may over-estimate but never under-estimate.
-template <typename E>
-i64 estimate_address_areas(Context<E> &ctx, ObjectFile<E> &file) {
-  // Each CU contains zero or one address area.
-  i64 ret = file.compunits.size();
-
-  // In DWARF 4, a CU can refer address ranges in .debug_ranges.
-  // .debug_ranges contains a vector of [begin, end) address pairs.
-  // The last entry must be a null terminator, so we do -1.
-  if (file.debug_ranges)
-    ret += file.debug_ranges->sh_size / sizeof(Word<E>) / 2 - 1;
-
-  // In DWARF 5, a CU can refer address ranges in .debug_rnglists, which
-  // contains variable-length entries. The smallest possible range entry
-  // is one byte for the code and two ULEB128 values (each can be as
-  // small as one byte), so 3 bytes.
-  if (file.debug_rnglists)
-    ret += file.debug_rnglists->sh_size / 3;
-  return ret;
-}
-
-// .debug_info contains variable-length fields. This class reads them.
-template <typename E>
-class DebugInfoReader {
-public:
-  DebugInfoReader(Context<E> &ctx, ObjectFile<E> &file, u8 *cu)
-    : ctx(ctx), file(file), cu(cu) {}
-
-  u64 read(u64 form);
-
-  Context<E> &ctx;
-  ObjectFile<E> &file;
-  u8 *cu;
-};
-
-// Read value of the given DW_FORM_* form. If a value is not scalar,
-// returns a dummy value 0.
-template <typename E>
-u64 DebugInfoReader<E>::read(u64 form) {
-  switch (form) {
-  case DW_FORM_flag_present:
-    return 0;
-  case DW_FORM_data1:
-  case DW_FORM_flag:
-  case DW_FORM_strx1:
-  case DW_FORM_addrx1:
-  case DW_FORM_ref1:
-    return *cu++;
-  case DW_FORM_data2:
-  case DW_FORM_strx2:
-  case DW_FORM_addrx2:
-  case DW_FORM_ref2: {
-    u64 val = *(U16<E> *)cu;
-    cu += 2;
-    return val;
-  }
-  case DW_FORM_strx3:
-  case DW_FORM_addrx3: {
-    u64 val = *(U24<E> *)cu;
-    cu += 3;
-    return val;
-  }
-  case DW_FORM_data4:
-  case DW_FORM_strp:
-  case DW_FORM_sec_offset:
-  case DW_FORM_line_strp:
-  case DW_FORM_strx4:
-  case DW_FORM_addrx4:
-  case DW_FORM_ref4: {
-    u64 val = *(U32<E> *)cu;
-    cu += 4;
-    return val;
-  }
-  case DW_FORM_data8:
-  case DW_FORM_ref8: {
-    u64 val = *(U64<E> *)cu;
-    cu += 8;
-    return val;
-  }
-  case DW_FORM_addr:
-  case DW_FORM_ref_addr: {
-    u64 val = *(Word<E> *)cu;
-    cu += sizeof(Word<E>);
-    return val;
-  }
-  case DW_FORM_strx:
-  case DW_FORM_addrx:
-  case DW_FORM_udata:
-  case DW_FORM_ref_udata:
-  case DW_FORM_loclistx:
-  case DW_FORM_rnglistx:
-    return read_uleb(&cu);
-  case DW_FORM_string:
-    cu += strlen((char *)cu) + 1;
-    return 0;
-  default:
-    Fatal(ctx) << file << ": --gdb-index: unhandled debug info form: 0x"
-               << std::hex << form;
-    return 0;
-  }
-}
-
-// Read a range list from .debug_ranges starting at the given offset.
-template <typename E>
-static std::vector<u64>
-read_debug_range(Context<E> &ctx, ObjectFile<E> &file, Word<E> *range) {
-  std::vector<u64> vec;
-  u64 base = 0;
-
-  for (i64 i = 0; range[i] || range[i + 1]; i += 2) {
-    if (range[i] + 1 == 0) {
-      // base address selection entry
-      base = range[i + 1];
-    } else {
-      vec.push_back(range[i] + base);
-      vec.push_back(range[i + 1] + base);
-    }
-  }
-  return vec;
-}
-
-// Read a range list from .debug_rnglists starting at the given offset.
-template <typename E>
-static std::vector<u64>
-read_rnglist_range(Context<E> &ctx, ObjectFile<E> &file, u8 *rnglist,
-                   Word<E> *addrx) {
-  std::vector<u64> vec;
-  u64 base = 0;
-
-  for (;;) {
-    switch (*rnglist++) {
-    case DW_RLE_end_of_list:
-      return vec;
-    case DW_RLE_base_addressx:
-      base = addrx[read_uleb(&rnglist)];
-      break;
-    case DW_RLE_startx_endx:
-      vec.push_back(addrx[read_uleb(&rnglist)]);
-      vec.push_back(addrx[read_uleb(&rnglist)]);
-      break;
-    case DW_RLE_startx_length:
-      vec.push_back(addrx[read_uleb(&rnglist)]);
-      vec.push_back(vec.back() + read_uleb(&rnglist));
-      break;
-    case DW_RLE_offset_pair:
-      vec.push_back(base + read_uleb(&rnglist));
-      vec.push_back(base + read_uleb(&rnglist));
-      break;
-    case DW_RLE_base_address:
-      base = *(Word<E> *)rnglist;
-      rnglist += sizeof(Word<E>);
-      break;
-    case DW_RLE_start_end:
-      vec.push_back(*(Word<E> *)rnglist);
-      rnglist += sizeof(Word<E>);
-      vec.push_back(*(Word<E> *)rnglist);
-      rnglist += sizeof(Word<E>);
-      break;
-    case DW_RLE_start_length:
-      vec.push_back(*(Word<E> *)rnglist);
-      rnglist += sizeof(Word<E>);
-      vec.push_back(vec.back() + read_uleb(&rnglist));
-      break;
-    }
-  }
-}
-
-// Returns a list of address ranges explained by a compunit at the
-// `offset` in an output .debug_info section.
-//
-// .debug_info contains DWARF debug info records, so this function
-// parses DWARF. If a designated compunit contains multiple ranges, the
-// ranges are read from .debug_ranges (or .debug_rnglists for DWARF5).
-// Otherwise, a range is read directly from .debug_info (or possibly
-// from .debug_addr for DWARF5).
-template <typename E>
-std::vector<u64>
-read_address_areas(Context<E> &ctx, ObjectFile<E> &file, i64 offset) {
-  u8 *cu;
-  u8 *abbrev;
-  u32 dwarf_version;
-  std::tie(cu, abbrev, dwarf_version) = find_compunit(ctx, file, offset);
-
-  DebugInfoReader<E> reader{ctx, file, cu};
-
-  struct Record {
-    u64 form = 0;
-    u64 value = 0;
-  };
-
-  Record low_pc;
-  Record high_pc;
-  Record ranges;
-  std::optional<u64> rnglists_base;
-  Word<E> *addrx = nullptr;
-
-  // Read all interesting debug records.
-  for (;;) {
-    u64 name = read_uleb(&abbrev);
-    u64 form = read_uleb(&abbrev);
-    if (name == 0 && form == 0)
-      break;
-
-    u64 val = reader.read(form);
-
-    switch (name) {
-    case DW_AT_low_pc:
-      low_pc = {form, val};
-      break;
-    case DW_AT_high_pc:
-      high_pc = {form, val};
-      break;
-    case DW_AT_rnglists_base:
-      rnglists_base = val;
-      break;
-    case DW_AT_addr_base:
-      addrx = (Word<E> *)(get_buffer(ctx, ctx.debug_addr) + val);
-      break;
-    case DW_AT_ranges:
-      ranges = {form, val};
-      break;
-    }
-  }
-
-  // Handle non-contiguous address ranges.
-  if (ranges.form) {
-    if (dwarf_version <= 4) {
-       Word<E> *range_begin =
-        (Word<E> *)(get_buffer(ctx, ctx.debug_ranges) + ranges.value);
-      return read_debug_range(ctx, file, range_begin);
-    }
-
-    assert(dwarf_version == 5);
-
-    u8 *buf = get_buffer(ctx, ctx.debug_rnglists);
-    if (ranges.form == DW_FORM_sec_offset)
-      return read_rnglist_range(ctx, file, buf + ranges.value, addrx);
-
-    if (!rnglists_base)
-      Fatal(ctx) << file << ": --gdb-index: missing DW_AT_rnglists_base";
-
-    u8 *base = buf + *rnglists_base;
-    return read_rnglist_range(ctx, file, base + *(U32<E> *)base, addrx);
-  }
-
-  // Handle a contiguous address range.
-  if (low_pc.form && high_pc.form) {
-    u64 lo;
-
-    switch (low_pc.form) {
-    case DW_FORM_addr:
-      lo = low_pc.value;
-      break;
-    case DW_FORM_addrx:
-    case DW_FORM_addrx1:
-    case DW_FORM_addrx2:
-    case DW_FORM_addrx4:
-      lo = addrx[low_pc.value];
-      break;
-    default:
-      Fatal(ctx) << file << ": --gdb-index: unhandled form for DW_AT_low_pc: 0x"
-                 << std::hex << high_pc.form;
-    }
-
-    switch (high_pc.form) {
-    case DW_FORM_addr:
-      return {lo, high_pc.value};
-    case DW_FORM_addrx:
-    case DW_FORM_addrx1:
-    case DW_FORM_addrx2:
-    case DW_FORM_addrx4:
-      return {lo, addrx[high_pc.value]};
-    case DW_FORM_udata:
-    case DW_FORM_data1:
-    case DW_FORM_data2:
-    case DW_FORM_data4:
-    case DW_FORM_data8:
-      return {lo, lo + high_pc.value};
-    default:
-      Fatal(ctx) << file << ": --gdb-index: unhandled form for DW_AT_high_pc: 0x"
-                 << std::hex << high_pc.form;
-    }
-  }
-
-  return {};
-}
-
-using E = MOLD_TARGET;
-
-template std::vector<std::string_view> read_compunits(Context<E> &, ObjectFile<E> &);
-template std::vector<GdbIndexName> read_pubnames(Context<E> &, ObjectFile<E> &);
-template i64 estimate_address_areas(Context<E> &, ObjectFile<E> &);
-template std::vector<u64> read_address_areas(Context<E> &, ObjectFile<E> &, i64);
-
-} // namespace mold::elf

elf/elf.h

@@ -291,6 +291,7 @@ enum : u32 {
   DT_VERNEEDNUM = 0x6fffffff,
   DT_PPC_GOT = 0x70000000,
   DT_PPC64_GLINK = 0x70000000,
+  DT_AARCH64_VARIANT_PCS = 0x70000005,
   DT_AUXILIARY = 0x7ffffffd,
   DT_FILTER = 0x7fffffff,
 };
@@ -774,6 +775,9 @@ enum : u32 {
   R_RISCV_RVC_BRANCH = 44,
   R_RISCV_RVC_JUMP = 45,
   R_RISCV_RVC_LUI = 46,
+  R_RISCV_GPREL_LO12_I = 47,
+  R_RISCV_GPREL_LO12_S = 48,
+  R_RISCV_GPREL_HI20 = 49,
   R_RISCV_RELAX = 51,
   R_RISCV_SUB6 = 52,
   R_RISCV_SET6 = 53,
@@ -1638,6 +1642,16 @@ struct ElfRel<E> {
   std::conditional_t<E::is_64, U32<E>, u8> r_type;
 };
 
+// Returns true if a given relocation is of type used for direct
+// function call.
+template <typename E>
+inline bool is_func_call_rel(const ElfRel<E> &r) {
+  for (u32 r_type : E::R_FUNCALL)
+    if (r.r_type == r_type)
+      return true;
+  return false;
+}
+
 template <typename E>
 struct ElfDyn {
   Word<E> d_tag;
@@ -1705,6 +1719,35 @@ struct ElfNhdr {
 // Target-specific ELF data types
 //
 
+template <>
+struct ElfSym<ARM64> {
+  bool is_undef() const { return st_shndx == SHN_UNDEF; }
+  bool is_abs() const { return st_shndx == SHN_ABS; }
+  bool is_common() const { return st_shndx == SHN_COMMON; }
+  bool is_weak() const { return st_bind == STB_WEAK; }
+  bool is_undef_weak() const { return is_undef() && is_weak(); }
+
+  ul32 st_name;
+
+#ifdef __LITTLE_ENDIAN__
+  u8 st_type : 4;
+  u8 st_bind : 4;
+  u8 st_visibility : 2;
+  u8 : 5;
+  u8 arm64_variant_pcs : 1; // ARM64-specific
+#else
+  u8 st_bind : 4;
+  u8 st_type : 4;
+  u8 arm64_variant_pcs : 1;
+  u8 : 6;
+  u8 st_visibility : 2;
+#endif
+
+  ul16 st_shndx;
+  ul64 st_value;
+  ul64 st_size;
+};
+
 template <>
 struct ElfSym<PPC64V2> {
   bool is_undef() const { return st_shndx == SHN_UNDEF; }
@@ -1824,19 +1867,20 @@ struct X86_64 {
   static constexpr u32 e_machine = EM_X86_64;
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
-  static constexpr u32 pltgot_size = 16;
+  static constexpr u32 pltgot_size = 8;
+  static constexpr u8 filler[] = { 0xcc };
 
   static constexpr u32 R_COPY = R_X86_64_COPY;
   static constexpr u32 R_GLOB_DAT = R_X86_64_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_X86_64_JUMP_SLOT;
   static constexpr u32 R_ABS = R_X86_64_64;
-  static constexpr u32 R_DYNAMIC = R_X86_64_64;
   static constexpr u32 R_RELATIVE = R_X86_64_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_X86_64_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_X86_64_DTPOFF64;
   static constexpr u32 R_TPOFF = R_X86_64_TPOFF64;
   static constexpr u32 R_DTPMOD = R_X86_64_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_X86_64_TLSDESC;
+  static constexpr u32 R_FUNCALL[] = { R_X86_64_PLT32, R_X86_64_PLTOFF64 };
 };
 
 struct I386 {
@@ -1849,18 +1893,19 @@ struct I386 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0xcc };
 
   static constexpr u32 R_COPY = R_386_COPY;
   static constexpr u32 R_GLOB_DAT = R_386_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_386_JUMP_SLOT;
   static constexpr u32 R_ABS = R_386_32;
-  static constexpr u32 R_DYNAMIC = R_386_32;
   static constexpr u32 R_RELATIVE = R_386_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_386_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_386_TLS_DTPOFF32;
   static constexpr u32 R_TPOFF = R_386_TLS_TPOFF;
   static constexpr u32 R_DTPMOD = R_386_TLS_DTPMOD32;
   static constexpr u32 R_TLSDESC = R_386_TLS_DESC;
+  static constexpr u32 R_FUNCALL[] = { R_386_PLT32 };
 };
 
 struct ARM64 {
@@ -1875,18 +1920,19 @@ struct ARM64 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 16;
+  static constexpr u8 filler[] = { 0x00, 0x7d, 0x20, 0xd4 };
 
   static constexpr u32 R_COPY = R_AARCH64_COPY;
   static constexpr u32 R_GLOB_DAT = R_AARCH64_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_AARCH64_JUMP_SLOT;
   static constexpr u32 R_ABS = R_AARCH64_ABS64;
-  static constexpr u32 R_DYNAMIC = R_AARCH64_ABS64;
   static constexpr u32 R_RELATIVE = R_AARCH64_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_AARCH64_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_AARCH64_TLS_DTPREL64;
   static constexpr u32 R_TPOFF = R_AARCH64_TLS_TPREL64;
   static constexpr u32 R_DTPMOD = R_AARCH64_TLS_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_AARCH64_TLSDESC;
+  static constexpr u32 R_FUNCALL[] = { R_AARCH64_JUMP26, R_AARCH64_CALL26 };
 };
 
 struct ARM32 {
@@ -1901,18 +1947,22 @@ struct ARM32 {
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 16;
   static constexpr u32 thunk_size = 16;
+  static constexpr u8 filler[] = { 0xff, 0xde };
 
   static constexpr u32 R_COPY = R_ARM_COPY;
   static constexpr u32 R_GLOB_DAT = R_ARM_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_ARM_JUMP_SLOT;
   static constexpr u32 R_ABS = R_ARM_ABS32;
-  static constexpr u32 R_DYNAMIC = R_ARM_ABS32;
   static constexpr u32 R_RELATIVE = R_ARM_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_ARM_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_ARM_TLS_DTPOFF32;
   static constexpr u32 R_TPOFF = R_ARM_TLS_TPOFF32;
   static constexpr u32 R_DTPMOD = R_ARM_TLS_DTPMOD32;
   static constexpr u32 R_TLSDESC = R_ARM_TLS_DESC;
+
+  static constexpr u32 R_FUNCALL[] = {
+    R_ARM_JUMP24, R_ARM_THM_JUMP24, R_ARM_CALL, R_ARM_THM_CALL, R_ARM_PLT32,
+  };
 };
 
 struct RV64 {
@@ -1923,18 +1973,19 @@ struct RV64 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x02, 0x90 };
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_64;
   static constexpr u32 R_JUMP_SLOT = R_RISCV_JUMP_SLOT;
   static constexpr u32 R_ABS = R_RISCV_64;
-  static constexpr u32 R_DYNAMIC = R_RISCV_64;
   static constexpr u32 R_RELATIVE = R_RISCV_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_RISCV_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_RISCV_TLS_DTPREL64;
   static constexpr u32 R_TPOFF = R_RISCV_TLS_TPREL64;
   static constexpr u32 R_DTPMOD = R_RISCV_TLS_DTPMOD64;
   static constexpr u32 R_TLSDESC = R_RISCV_TLSDESC;
+  static constexpr u32 R_FUNCALL[] = { R_RISCV_CALL, R_RISCV_CALL_PLT };
 };
 
 struct RV64LE : RV64 {
@@ -1955,18 +2006,19 @@ struct RV32 {
   static constexpr u32 plt_hdr_size = 32;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x02, 0x90 };
 
   static constexpr u32 R_COPY = R_RISCV_COPY;
   static constexpr u32 R_GLOB_DAT = R_RISCV_32;
   static constexpr u32 R_JUMP_SLOT = R_RISCV_JUMP_SLOT;
   static constexpr u32 R_ABS = R_RISCV_32;
-  static constexpr u32 R_DYNAMIC = R_RISCV_32;
   static constexpr u32 R_RELATIVE = R_RISCV_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_RISCV_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_RISCV_TLS_DTPREL32;
   static constexpr u32 R_TPOFF = R_RISCV_TLS_TPREL32;
   static constexpr u32 R_DTPMOD = R_RISCV_TLS_DTPMOD32;
   static constexpr u32 R_TLSDESC = R_RISCV_TLSDESC;
+  static constexpr u32 R_FUNCALL[] = { R_RISCV_CALL, R_RISCV_CALL_PLT };
 };
 
 struct RV32LE : RV32 {
@@ -1991,17 +2043,21 @@ struct PPC32 {
   static constexpr u32 pltgot_size = 36;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 36;
+  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 };
 
   static constexpr u32 R_COPY = R_PPC_COPY;
   static constexpr u32 R_GLOB_DAT = R_PPC_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_PPC_JMP_SLOT;
   static constexpr u32 R_ABS = R_PPC_ADDR32;
-  static constexpr u32 R_DYNAMIC = R_PPC_ADDR32;
   static constexpr u32 R_RELATIVE = R_PPC_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_PPC_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_PPC_DTPREL32;
   static constexpr u32 R_TPOFF = R_PPC_TPREL32;
   static constexpr u32 R_DTPMOD = R_PPC_DTPMOD32;
+
+  static constexpr u32 R_FUNCALL[] = {
+    R_PPC_REL24, R_PPC_PLTREL24, R_PPC_LOCAL24PC,
+  };
 };
 
 struct PPC64 {
@@ -2014,12 +2070,12 @@ struct PPC64 {
   static constexpr u32 R_GLOB_DAT = R_PPC64_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_PPC64_JMP_SLOT;
   static constexpr u32 R_ABS = R_PPC64_ADDR64;
-  static constexpr u32 R_DYNAMIC = R_PPC64_ADDR64;
   static constexpr u32 R_RELATIVE = R_PPC64_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_PPC64_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_PPC64_DTPREL64;
   static constexpr u32 R_TPOFF = R_PPC64_TPREL64;
   static constexpr u32 R_DTPMOD = R_PPC64_DTPMOD64;
+  static constexpr u32 R_FUNCALL[] = { R_PPC64_REL24, R_PPC64_REL24_NOTOC };
 };
 
 struct PPC64V1 : PPC64 {
@@ -2029,6 +2085,7 @@ struct PPC64V1 : PPC64 {
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
   static constexpr u32 thunk_size = 28;
+  static constexpr u8 filler[] = { 0x7f, 0xe0, 0x00, 0x08 };
 };
 
 struct PPC64V2 : PPC64 {
@@ -2038,7 +2095,8 @@ struct PPC64V2 : PPC64 {
   static constexpr u32 plt_size = 4;
   static constexpr u32 pltgot_size = 0;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 20;
+  static constexpr u32 thunk_size = 24;
+  static constexpr u8 filler[] = { 0x08, 0x00, 0xe0, 0x7f };
 };
 
 struct S390X {
@@ -2051,17 +2109,18 @@ struct S390X {
   static constexpr u32 plt_hdr_size = 48;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
+  static constexpr u8 filler[] = { 0x07, 0x00 };
 
   static constexpr u32 R_COPY = R_390_COPY;
   static constexpr u32 R_GLOB_DAT = R_390_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_390_JMP_SLOT;
   static constexpr u32 R_ABS = R_390_64;
-  static constexpr u32 R_DYNAMIC = R_390_64;
   static constexpr u32 R_RELATIVE = R_390_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_390_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_390_TLS_DTPOFF;
   static constexpr u32 R_TPOFF = R_390_TLS_TPOFF;
   static constexpr u32 R_DTPMOD = R_390_TLS_DTPMOD;
+  static constexpr u32 R_FUNCALL[] = { R_390_PLT32DBL };
 };
 
 struct SPARC64 {
@@ -2074,17 +2133,18 @@ struct SPARC64 {
   static constexpr u32 plt_hdr_size = 128;
   static constexpr u32 plt_size = 32;
   static constexpr u32 pltgot_size = 32;
+  static constexpr u8 filler[] = { 0x91, 0xd0, 0x20, 0x05 };
 
   static constexpr u32 R_COPY = R_SPARC_COPY;
   static constexpr u32 R_GLOB_DAT = R_SPARC_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_SPARC_JMP_SLOT;
   static constexpr u32 R_ABS = R_SPARC_64;
-  static constexpr u32 R_DYNAMIC = R_SPARC_64;
   static constexpr u32 R_RELATIVE = R_SPARC_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_SPARC_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_SPARC_TLS_DTPOFF64;
   static constexpr u32 R_TPOFF = R_SPARC_TLS_TPOFF64;
   static constexpr u32 R_DTPMOD = R_SPARC_TLS_DTPMOD64;
+  static constexpr u32 R_FUNCALL[] = { R_SPARC_WPLT30, R_SPARC_WDISP30 };
 };
 
 struct M68K {
@@ -2097,16 +2157,17 @@ struct M68K {
   static constexpr u32 plt_hdr_size = 18;
   static constexpr u32 plt_size = 14;
   static constexpr u32 pltgot_size = 8;
+  static constexpr u8 filler[] = { 0x4a, 0xfc };
 
   static constexpr u32 R_COPY = R_68K_COPY;
   static constexpr u32 R_GLOB_DAT = R_68K_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_68K_JMP_SLOT;
   static constexpr u32 R_ABS = R_68K_32;
-  static constexpr u32 R_DYNAMIC = R_68K_32;
   static constexpr u32 R_RELATIVE = R_68K_RELATIVE;
   static constexpr u32 R_DTPOFF = R_68K_TLS_DTPREL32;
   static constexpr u32 R_TPOFF = R_68K_TLS_TPREL32;
   static constexpr u32 R_DTPMOD = R_68K_TLS_DTPMOD32;
+  static constexpr u32 R_FUNCALL[] = { R_68K_PLT32 };
 };
 
 struct SH4 {
@@ -2119,16 +2180,17 @@ struct SH4 {
   static constexpr u32 plt_hdr_size = 16;
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 12;
+  static constexpr u8 filler[] = { 0x00, 0x00 };
 
   static constexpr u32 R_COPY = R_SH_COPY;
   static constexpr u32 R_GLOB_DAT = R_SH_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_SH_JMP_SLOT;
   static constexpr u32 R_ABS = R_SH_DIR32;
-  static constexpr u32 R_DYNAMIC = R_SH_DIR32;
   static constexpr u32 R_RELATIVE = R_SH_RELATIVE;
   static constexpr u32 R_DTPOFF = R_SH_TLS_DTPOFF32;
   static constexpr u32 R_TPOFF = R_SH_TLS_TPOFF32;
   static constexpr u32 R_DTPMOD = R_SH_TLS_DTPMOD32;
+  static constexpr u32 R_FUNCALL[] = { R_SH_PLT32 };
 };
 
 struct ALPHA {
@@ -2141,16 +2203,17 @@ struct ALPHA {
   static constexpr u32 plt_hdr_size = 0;
   static constexpr u32 plt_size = 0;
   static constexpr u32 pltgot_size = 0;
+  static constexpr u8 filler[] = { 0x81, 0x00, 0x00, 0x00 };
 
   static constexpr u32 R_COPY = R_ALPHA_COPY;
   static constexpr u32 R_GLOB_DAT = R_ALPHA_GLOB_DAT;
   static constexpr u32 R_JUMP_SLOT = R_ALPHA_JMP_SLOT;
   static constexpr u32 R_ABS = R_ALPHA_REFQUAD;
-  static constexpr u32 R_DYNAMIC = R_ALPHA_REFQUAD;
   static constexpr u32 R_RELATIVE = R_ALPHA_RELATIVE;
   static constexpr u32 R_DTPOFF = R_ALPHA_DTPREL64;
   static constexpr u32 R_TPOFF = R_ALPHA_TPREL64;
   static constexpr u32 R_DTPMOD = R_ALPHA_DTPMOD64;
+  static constexpr u32 R_FUNCALL[] = {};
 };
 
 struct LOONGARCH64 {
@@ -2164,18 +2227,19 @@ struct LOONGARCH64 {
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
+  static constexpr u32 thunk_size = 8;
+  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 };
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_64;
   static constexpr u32 R_JUMP_SLOT = R_LARCH_JUMP_SLOT;
   static constexpr u32 R_ABS = R_LARCH_64;
-  static constexpr u32 R_DYNAMIC = R_LARCH_64;
   static constexpr u32 R_RELATIVE = R_LARCH_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_LARCH_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_LARCH_TLS_DTPREL64;
   static constexpr u32 R_TPOFF = R_LARCH_TLS_TPREL64;
   static constexpr u32 R_DTPMOD = R_LARCH_TLS_DTPMOD64;
+  static constexpr u32 R_FUNCALL[] = { R_LARCH_B26 };
 };
 
 struct LOONGARCH32 {
@@ -2189,18 +2253,19 @@ struct LOONGARCH32 {
   static constexpr u32 plt_size = 16;
   static constexpr u32 pltgot_size = 16;
   static constexpr u32 thunk_hdr_size = 0;
-  static constexpr u32 thunk_size = 16;
+  static constexpr u32 thunk_size = 8;
+  static constexpr u8 filler[] = { 0x00, 0x00, 0x2a, 0x00 };
 
   static constexpr u32 R_COPY = R_LARCH_COPY;
   static constexpr u32 R_GLOB_DAT = R_LARCH_32;
   static constexpr u32 R_JUMP_SLOT = R_LARCH_JUMP_SLOT;
   static constexpr u32 R_ABS = R_LARCH_32;
-  static constexpr u32 R_DYNAMIC = R_LARCH_32;
   static constexpr u32 R_RELATIVE = R_LARCH_RELATIVE;
   static constexpr u32 R_IRELATIVE = R_LARCH_IRELATIVE;
   static constexpr u32 R_DTPOFF = R_LARCH_TLS_DTPREL32;
   static constexpr u32 R_TPOFF = R_LARCH_TLS_TPREL32;
   static constexpr u32 R_DTPMOD = R_LARCH_TLS_DTPMOD32;
+  static constexpr u32 R_FUNCALL[] = { R_LARCH_B26 };
 };
 
 } // namespace mold::elf

elf/gdb-index.cc

@@ -0,0 +1,793 @@
+// This file contains code to read DWARF debug info to create .gdb_index.
+//
+// .gdb_index is an optional section to speed up GNU debugger. It contains
+// two maps: 1) a map from function/variable/type names to compunits, and
+// 2) a map from function address ranges to compunits. gdb uses these
+// maps to quickly find a compunit given a name or an instruction pointer.
+//
+// (Terminology: a compilation unit, often abbreviated as compunit or
+// CU, is a unit of debug info. An input .debug_info section usually
+// contains one compunit, and thus an output .debug_info contains as
+// many compunits as the number of input files.)
+//
+// .gdb_index is not mandatory. All the information in .gdb_index is
+// also in other debug info sections. You can actually create an
+// executable without .gdb_index and later add it using the
+// `gdb-add-index` post-processing tool that comes with gdb.
+//
+// Post-relocated debug section contents are needed to create a
+// .gdb_index. Therefore, we create it after relocating all the other
+// sections. The size of the section is also hard to estimate before
+// applying relocations to debug info sections, so a .gdb_index is
+// placed at the very end of the output file, even after the section
+// header.
+//
+// The mapping from names to compunits is 1:n while the mapping from
+// address ranges to compunits is 1:1. That is, two object files may
+// define the same type name, while there should be no two functions
+// that overlap with each other in memory.
+//
+// .gdb_index contains an on-disk hash table for names, so gdb can
+// lookup names without loading all strings into memory and construct an
+// in-memory hash table.
+//
+// Names are in .debug_gnu_pubnames and .debug_gnu_pubtypes input
+// sections. These sections are created if `-ggnu-pubnames` is given.
+// Besides names, these sections contain attributes for each name so
+// that gdb can distinguish type names from function names, for example.
+//
+// A compunit contains one or more function address ranges. If an
+// object file is compiled without -ffunction-sections, it contains
+// only one .text section and therefore contains a single address range.
+// Such range is typically stored directly to the compunit.
+//
+// If an object file is compiled with -ffunction-sections, it contains
+// more than one .text section, and it has as many address ranges as
+// the number of .text sections. Such discontiguous address ranges are
+// stored to .debug_ranges in DWARF 2/3/4 and .debug_rnglists/.debug_addr
+// in DWARF 5.
+//
+// .debug_info section contains DWARF debug info. Although we don't need
+// to parse the whole .debug_info section to read address ranges, we
+// have to do a little bit. DWARF is complicated and often handled using
+// a library such as libdwarf. But we don't use any library because we
+// don't want to add an extra run-time dependency just for --gdb-index.
+//
+// This page explains the format of .gdb_index:
+// https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html
+
+#include "mold.h"
+#include <tbb/parallel_for_each.h>
+#include <tbb/parallel_sort.h>
+
+namespace mold::elf {
+
+enum DwarfKind { DWARF2_32, DWARF5_32, DWARF2_64, DWARF5_64 };
+
+template <typename E>
+struct CuHdrDwarf2_32 {
+  U32<E> size;
+  U16<E> version;
+  U32<E> abbrev_offset;
+  u8 address_size;
+};
+
+template <typename E>
+struct CuHdrDwarf5_32 {
+  U32<E> size;
+  U16<E> version;
+  u8 unit_type;
+  u8 address_size;
+  U32<E> abbrev_offset;
+};
+
+template <typename E>
+struct CuHdrDwarf2_64 {
+  U32<E> magic;
+  U64<E> size;
+  U16<E> version;
+  U64<E> abbrev_offset;
+  u8 address_size;
+};
+
+template <typename E>
+struct CuHdrDwarf5_64 {
+  U32<E> magic;
+  U64<E> size;
+  U16<E> version;
+  u8 unit_type;
+  u8 address_size;
+  U64<E> abbrev_offset;
+};
+
+template <typename E>
+struct PubnamesHdr32 {
+  U32<E> size;
+  U16<E> version;
+  U32<E> debug_info_offset;
+  U32<E> debug_info_size;
+};
+
+template <typename E>
+struct PubnamesHdr64 {
+  U32<E> magic;
+  U64<E> size;
+  U16<E> version;
+  U64<E> debug_info_offset;
+  U64<E> debug_info_size;
+};
+
+struct SectionHeader {
+  ul32 version = 7;
+  ul32 cu_list_offset = 0;
+  ul32 cu_types_offset = 0;
+  ul32 ranges_offset = 0;
+  ul32 symtab_offset = 0;
+  ul32 const_pool_offset = 0;
+};
+
+struct NameType {
+  bool operator==(const NameType &) const = default;
+
+  bool operator<(const NameType &other) const {
+    return std::tuple(hash, type, name) <
+           std::tuple(other.hash, other.type, other.name);
+  }
+
+  std::string_view name;
+  u32 hash;
+  u8 type;
+};
+
+struct MapValue {
+  u32 hash = 0;
+  Atomic<u32> count = 0;
+  u32 name_offset = 0;
+  u32 type_offset = 0;
+};
+
+struct Compunit {
+  DwarfKind kind;
+  i64 offset;
+  i64 size;
+  std::vector<std::pair<u64, u64>> ranges;
+  std::vector<NameType> nametypes;
+  std::vector<MapValue *> entries;
+};
+
+// The hash function for .gdb_index.
+static u32 gdb_hash(std::string_view name) {
+  u32 h = 0;
+  for (u8 c : name) {
+    if ('A' <= c && c <= 'Z')
+      c = 'a' + c - 'A';
+    h = h * 67 + c - 113;
+  }
+  return h;
+}
+
+template <typename E>
+static DwarfKind get_dwarf_kind(Context<E> &ctx, u8 *p) {
+  if (*(U32<E> *)p == 0xffff'ffff) {
+    CuHdrDwarf2_64<E> &hdr = *(CuHdrDwarf2_64<E> *)p;
+    if (hdr.version > 5)
+      Fatal(ctx) << "--gdb-index: DWARF version " << hdr.version
+                 << " is not supported";
+    return (hdr.version == 5) ? DWARF5_64 : DWARF2_64;
+  }
+
+  CuHdrDwarf2_32<E> &hdr = *(CuHdrDwarf2_32<E> *)p;
+  if (hdr.version > 5)
+    Fatal(ctx) << "--gdb-index: DWARF version " << hdr.version
+               << " is not supported";
+  return (hdr.version == 5) ? DWARF5_32 : DWARF2_32;
+}
+
+template <typename E, typename CuHdr>
+u8 *find_cu_abbrev(Context<E> &ctx, u8 **p, const CuHdr &hdr) {
+  if (hdr.address_size != sizeof(Word<E>))
+    Fatal(ctx) << "--gdb-index: unsupported address size " << hdr.address_size;
+
+  if constexpr (requires { hdr.unit_type; }) {
+    switch (hdr.unit_type) {
+    case DW_UT_compile:
+    case DW_UT_partial:
+      break;
+    case DW_UT_skeleton:
+    case DW_UT_split_compile:
+      *p += 8;
+      break;
+    default:
+      Fatal(ctx) << "--gdb-index: unknown unit type: 0x"
+                 << std::hex << hdr.unit_type;
+    }
+  }
+
+  i64 abbrev_code = read_uleb(p);
+
+  // Find a .debug_abbrev record corresponding to the .debug_info record.
+  // We assume the .debug_info record at a given offset is of
+  // DW_TAG_compile_unit which describes a compunit.
+  u8 *abbrev = &ctx.debug_abbrev[0] + hdr.abbrev_offset;
+
+  for (;;) {
+    u32 code = read_uleb(&abbrev);
+    if (code == 0)
+      Fatal(ctx) << "--gdb-index: .debug_abbrev does not contain"
+                 << " a record for the first .debug_info record";
+
+    if (code == abbrev_code) {
+      // Found a record
+      u64 abbrev_tag = read_uleb(&abbrev);
+      if (abbrev_tag != DW_TAG_compile_unit && abbrev_tag != DW_TAG_skeleton_unit)
+        Fatal(ctx) << "--gdb-index: the first entry's tag is not"
+                   << " DW_TAG_compile_unit/DW_TAG_skeleton_unit but 0x"
+                   << std::hex << abbrev_tag;
+      break;
+    }
+
+    // Skip an uninteresting record
+    read_uleb(&abbrev); // tag
+    abbrev++; // has_children byte
+    for (;;) {
+      u64 name = read_uleb(&abbrev);
+      u64 form = read_uleb(&abbrev);
+      if (name == 0 && form == 0)
+        break;
+      if (form == DW_FORM_implicit_const)
+        read_uleb(&abbrev);
+    }
+  }
+
+  abbrev++; // skip has_children byte
+  return abbrev;
+}
+
+// .debug_info contains variable-length fields.
+// This function reads one scalar value from a given location.
+template <typename E, typename Offset>
+u64 read_scalar(Context<E> &ctx, u8 **p, u64 form) {
+  switch (form) {
+  case DW_FORM_flag_present:
+    return 0;
+  case DW_FORM_data1:
+  case DW_FORM_flag:
+  case DW_FORM_strx1:
+  case DW_FORM_addrx1:
+  case DW_FORM_ref1:
+    return *(*p)++;
+  case DW_FORM_data2:
+  case DW_FORM_strx2:
+  case DW_FORM_addrx2:
+  case DW_FORM_ref2: {
+    u64 val = *(U16<E> *)*p;
+    *p += 2;
+    return val;
+  }
+  case DW_FORM_strx3:
+  case DW_FORM_addrx3: {
+    u64 val = *(U24<E> *)*p;
+    *p += 3;
+    return val;
+  }
+  case DW_FORM_data4:
+  case DW_FORM_strx4:
+  case DW_FORM_addrx4:
+  case DW_FORM_ref4: {
+    u64 val = *(U32<E> *)*p;
+    *p += 4;
+    return val;
+  }
+  case DW_FORM_data8:
+  case DW_FORM_ref8: {
+    u64 val = *(U64<E> *)*p;
+    *p += 8;
+    return val;
+  }
+  case DW_FORM_strp:
+  case DW_FORM_sec_offset:
+  case DW_FORM_line_strp: {
+    u64 val = *(Offset *)*p;
+    *p += sizeof(Offset);
+    return val;
+  }
+  case DW_FORM_addr:
+  case DW_FORM_ref_addr: {
+    u64 val = *(Word<E> *)*p;
+    *p += sizeof(Word<E>);
+    return val;
+  }
+  case DW_FORM_strx:
+  case DW_FORM_addrx:
+  case DW_FORM_udata:
+  case DW_FORM_ref_udata:
+  case DW_FORM_loclistx:
+  case DW_FORM_rnglistx:
+    return read_uleb(p);
+  case DW_FORM_string:
+    *p += strlen((char *)*p) + 1;
+    return 0;
+  default:
+    Fatal(ctx) << "--gdb-index: unhandled debug info form: 0x"
+               << std::hex << form;
+  }
+}
+
+// Read a range list from .debug_ranges starting at the given offset.
+template <typename E>
+static std::vector<std::pair<u64, u64>>
+read_debug_range(Word<E> *range, u64 base) {
+  std::vector<std::pair<u64, u64>> vec;
+
+  for (i64 i = 0; range[i] || range[i + 1]; i += 2) {
+    if (range[i] + 1 == 0)
+      base = range[i + 1];
+    else
+      vec.emplace_back(range[i] + base, range[i + 1] + base);
+  }
+  return vec;
+}
+
+// Read a range list from .debug_rnglists starting at the given offset.
+template <typename E>
+static void
+read_rnglist_range(std::vector<std::pair<u64, u64>> &vec, u8 *p,
+                   Word<E> *addrx, u64 base) {
+  for (;;) {
+    switch (*p++) {
+    case DW_RLE_end_of_list:
+      return;
+    case DW_RLE_base_addressx:
+      base = addrx[read_uleb(&p)];
+      break;
+    case DW_RLE_startx_endx: {
+      u64 val1 = read_uleb(&p);
+      u64 val2 = read_uleb(&p);
+      vec.emplace_back(addrx[val1], addrx[val2]);
+      break;
+    }
+    case DW_RLE_startx_length: {
+      u64 val1 = read_uleb(&p);
+      u64 val2 = read_uleb(&p);
+      vec.emplace_back(addrx[val1], addrx[val1] + val2);
+      break;
+    }
+    case DW_RLE_offset_pair: {
+      u64 val1 = read_uleb(&p);
+      u64 val2 = read_uleb(&p);
+
+      // If the base is 0, this address range is for an eliminated
+      // section. We only emit it if it's alive.
+      if (base)
+        vec.emplace_back(base + val1, base + val2);
+      break;
+    }
+    case DW_RLE_base_address:
+      base = *(Word<E> *)p;
+      p += sizeof(Word<E>);
+      break;
+    case DW_RLE_start_end: {
+      u64 val1 = ((Word<E> *)p)[0];
+      u64 val2 = ((Word<E> *)p)[1];
+      p += sizeof(Word<E>) * 2;
+      vec.emplace_back(val1, val2);
+      break;
+    }
+    case DW_RLE_start_length: {
+      u64 val1 = *(Word<E> *)p;
+      p += sizeof(Word<E>);
+      u64 val2 = read_uleb(&p);
+      vec.emplace_back(val1, val1 + val2);
+      break;
+    }
+    }
+  }
+}
+
+// Returns a list of address ranges explained by a compunit at the
+// `offset` in an output .debug_info section.
+//
+// .debug_info contains DWARF debug info records, so this function
+// parses DWARF. If a designated compunit contains multiple ranges, the
+// ranges are read from .debug_ranges (or .debug_rnglists for DWARF5).
+// Otherwise, a range is read directly from .debug_info (or possibly
+// from .debug_addr for DWARF5).
+template <typename E, typename CuHdr>
+static std::vector<std::pair<u64, u64>>
+read_address_ranges(Context<E> &ctx, const Compunit &cu) {
+  // Read .debug_info to find the record at a given offset.
+  u8 *p = &ctx.debug_info[0] + cu.offset;
+  CuHdr &hdr = *(CuHdr *)p;
+  p += sizeof(hdr);
+
+  u8 *abbrev = find_cu_abbrev(ctx, &p, hdr);
+
+  // Now, read debug info records.
+  struct Record {
+    u64 form = 0;
+    u64 value = 0;
+  };
+
+  using Offset = decltype(hdr.size);
+
+  Record low_pc;
+  Record high_pc;
+  Record ranges;
+  u64 rnglists_base = -1;
+  Word<E> *addrx = nullptr;
+
+  // Read all interesting debug records.
+  for (;;) {
+    u64 name = read_uleb(&abbrev);
+    u64 form = read_uleb(&abbrev);
+    if (name == 0 && form == 0)
+      break;
+
+    u64 val = read_scalar<E, Offset>(ctx, &p, form);
+
+    switch (name) {
+    case DW_AT_low_pc:
+      low_pc = {form, val};
+      break;
+    case DW_AT_high_pc:
+      high_pc = {form, val};
+      break;
+    case DW_AT_rnglists_base:
+      rnglists_base = val;
+      break;
+    case DW_AT_addr_base:
+      addrx = (Word<E> *)(&ctx.debug_addr[0] + val);
+      break;
+    case DW_AT_ranges:
+      ranges = {form, val};
+      break;
+    }
+  }
+
+  // Handle non-contiguous address ranges.
+  if (ranges.form) {
+    if (hdr.version <= 4) {
+      Word<E> *p = (Word<E> *)(&ctx.debug_ranges[0] + ranges.value);
+      return read_debug_range<E>(p, low_pc.value);
+    }
+
+    assert(hdr.version == 5);
+
+    std::vector<std::pair<u64, u64>> vec;
+    u8 *buf = &ctx.debug_rnglists[0];
+
+    if (ranges.form == DW_FORM_sec_offset) {
+      read_rnglist_range<E>(vec, buf + ranges.value, addrx, low_pc.value);
+    } else {
+      if (rnglists_base == -1)
+        Fatal(ctx) << "--gdb-index: missing DW_AT_rnglists_base";
+
+      u8 *base = buf + rnglists_base;
+      i64 num_offsets = *(U32<E> *)(base - 4);
+      Offset *offsets = (Offset *)base;
+
+      for (i64 i = 0; i < num_offsets; i++)
+        read_rnglist_range<E>(vec, base + offsets[i], addrx, low_pc.value);
+    }
+    return vec;
+  }
+
+  // Handle a contiguous address range.
+  if (low_pc.form && high_pc.form) {
+    u64 lo;
+
+    switch (low_pc.form) {
+    case DW_FORM_addr:
+      lo = low_pc.value;
+      break;
+    case DW_FORM_addrx:
+    case DW_FORM_addrx1:
+    case DW_FORM_addrx2:
+    case DW_FORM_addrx4:
+      lo = addrx[low_pc.value];
+      break;
+    default:
+      Fatal(ctx) << "--gdb-index: unhandled form for DW_AT_low_pc: 0x"
+                 << std::hex << high_pc.form;
+    }
+
+    switch (high_pc.form) {
+    case DW_FORM_addr:
+      return {{lo, high_pc.value}};
+    case DW_FORM_addrx:
+    case DW_FORM_addrx1:
+    case DW_FORM_addrx2:
+    case DW_FORM_addrx4:
+      return {{lo, addrx[high_pc.value]}};
+    case DW_FORM_udata:
+    case DW_FORM_data1:
+    case DW_FORM_data2:
+    case DW_FORM_data4:
+    case DW_FORM_data8:
+      return {{lo, lo + high_pc.value}};
+    default:
+      Fatal(ctx) << "--gdb-index: unhandled form for DW_AT_high_pc: 0x"
+                 << std::hex << high_pc.form;
+    }
+  }
+
+  return {};
+}
+
+template <typename E, typename PubnamesHdr>
+static i64 read_pubnames_cu(Context<E> &ctx, const PubnamesHdr &hdr,
+                            std::vector<Compunit> &cus, ObjectFile<E> &file) {
+  using Offset = decltype(hdr.size);
+
+  auto get_cu = [&](i64 offset) {
+    for (i64 i = 0; i < cus.size(); i++)
+      if (cus[i].offset == offset)
+        return &cus[i];
+    Fatal(ctx) << file << ": corrupted debug_info_offset";
+  };
+
+  Compunit *cu = get_cu(file.debug_info->offset + hdr.debug_info_offset);
+  i64 size = hdr.size + offsetof(PubnamesHdr, size) + sizeof(hdr.size);
+  u8 *p = (u8 *)&hdr + sizeof(hdr);
+  u8 *end = (u8 *)&hdr + size;
+
+  while (p < end) {
+    if (*(Offset *)p == 0)
+      break;
+    p += sizeof(Offset);
+
+    u8 type = *p++;
+    std::string_view name = (char *)p;
+    p += name.size() + 1;
+    cu->nametypes.push_back({name, gdb_hash(name), type});
+  }
+
+  return size;
+}
+
+// Parses .debug_gnu_pubnames and .debug_gnu_pubtypes. These sections
+// start with a 14 bytes header followed by (4-byte offset, 1-byte type,
+// null-terminated string) tuples.
+//
+// The 4-byte offset is an offset into .debug_info that contains details
+// about the name. The 1-byte type is a type of the corresponding name
+// (e.g. function, variable or datatype). The string is a name of a
+// function, a variable or a type.
+template <typename E>
+static void read_pubnames(Context<E> &ctx, std::vector<Compunit> &cus,
+                          ObjectFile<E> &file) {
+  for (InputSection<E> *isec : { file.debug_pubnames, file.debug_pubtypes }) {
+    if (!isec)
+      continue;
+
+    isec->uncompress(ctx);
+    if (isec->contents.empty())
+      continue;
+
+    u8 *p = (u8*)&isec->contents[0];
+    u8 *end = p + isec->contents.size();
+
+    while (p < end) {
+      if (*(U32<E> *)p == 0xffff'ffff)
+        p += read_pubnames_cu(ctx, *(PubnamesHdr64<E> *)p, cus, file);
+      else
+        p += read_pubnames_cu(ctx, *(PubnamesHdr32<E> *)p, cus, file);
+    }
+  };
+}
+
+template <typename E>
+static std::vector<Compunit> read_compunits(Context<E> &ctx) {
+  std::vector<Compunit> cus;
+
+  // Read compunits from the output .debug_info section.
+  u8 *begin = &ctx.debug_info[0];
+  u8 *end = begin + ctx.debug_info.size();
+
+  for (u8 *p = begin; p < end;) {
+    DwarfKind kind = get_dwarf_kind(ctx, p);
+    i64 size;
+    if (kind == DWARF2_32 || kind == DWARF5_32)
+      size = ((CuHdrDwarf2_32<E> *)p)->size + 4;
+    else
+      size = ((CuHdrDwarf2_64<E> *)p)->size + 12;
+
+    cus.push_back(Compunit{kind, p - begin, size});
+    p += size;
+  }
+
+  // Read address ranges for each compunit.
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    switch (cu.kind) {
+    case DWARF2_32:
+      cu.ranges = read_address_ranges<E, CuHdrDwarf2_32<E>>(ctx, cu);
+      break;
+    case DWARF5_32:
+      cu.ranges = read_address_ranges<E, CuHdrDwarf5_32<E>>(ctx, cu);
+      break;
+    case DWARF2_64:
+      cu.ranges = read_address_ranges<E, CuHdrDwarf2_64<E>>(ctx, cu);
+      break;
+    case DWARF5_64:
+      cu.ranges = read_address_ranges<E, CuHdrDwarf5_64<E>>(ctx, cu);
+      break;
+    }
+
+    // Remove empty ranges
+    std::erase_if(cu.ranges, [](std::pair<u64, u64> p) {
+      return p.first == 0 || p.first == p.second;
+    });
+  });
+
+  // Read symbols from .debug_gnu_pubnames and .debug_gnu_pubtypes.
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    read_pubnames(ctx, cus, *file);
+  });
+
+  // Uniquify elements because GCC 11 seems to emit one record for each
+  // comdat group which results in having a lot of duplicate records.
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    sort(cu.nametypes);
+    remove_duplicates(cu.nametypes);
+  });
+
+  return cus;
+}
+
+template <typename E>
+std::span<u8> get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
+  if (chunk->is_compressed)
+    return chunk->uncompressed_data;
+  return {ctx.buf + chunk->shdr.sh_offset, (size_t)chunk->shdr.sh_size};
+}
+
+template <typename E>
+void write_gdb_index(Context<E> &ctx) {
+  Timer t(ctx, "write_gdb_index");
+
+  // Find debug info sections
+  for (Chunk<E> *chunk : ctx.chunks) {
+    std::string_view name = chunk->name;
+    if (name == ".debug_info")
+      ctx.debug_info = get_buffer(ctx, chunk);
+    if (name == ".debug_abbrev")
+      ctx.debug_abbrev = get_buffer(ctx, chunk);
+    if (name == ".debug_ranges")
+      ctx.debug_ranges = get_buffer(ctx, chunk);
+    if (name == ".debug_addr")
+      ctx.debug_addr = get_buffer(ctx, chunk);
+    if (name == ".debug_rnglists")
+      ctx.debug_rnglists = get_buffer(ctx, chunk);
+  }
+
+  if (ctx.debug_info.empty())
+    return;
+
+  // Read debug info
+  std::vector<Compunit> cus = read_compunits(ctx);
+
+  // Uniquify symbols
+  HyperLogLog estimator;
+
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    HyperLogLog e;
+    for (NameType &nt : cu.nametypes)
+      e.insert(nt.hash);
+    estimator.merge(e);
+  });
+
+  ConcurrentMap<MapValue> map(estimator.get_cardinality() * 3 / 2);
+
+  tbb::parallel_for_each(cus, [&](Compunit &cu) {
+    cu.entries.reserve(cu.nametypes.size());
+    for (NameType &nt : cu.nametypes) {
+      MapValue *ent;
+      bool inserted;
+      std::tie(ent, inserted) = map.insert(nt.name, nt.hash, MapValue{nt.hash});
+      ent->count++;
+      cu.entries.push_back(ent);
+    }
+  });
+
+  // Sort symbols for build reproducibility
+  using Entry = typename decltype(map)::Entry;
+  std::vector<Entry *> entries = map.get_sorted_entries_all();
+
+  // Compute sizes of each components
+  SectionHeader hdr;
+  hdr.cu_list_offset = sizeof(hdr);
+  hdr.cu_types_offset = hdr.cu_list_offset + cus.size() * 16;
+  hdr.ranges_offset = hdr.cu_types_offset;
+
+  hdr.symtab_offset = hdr.ranges_offset;
+  for (Compunit &cu : cus)
+    hdr.symtab_offset += cu.ranges.size() * 20;
+
+  i64 ht_size = bit_ceil(entries.size() * 5 / 4 + 1);
+  hdr.const_pool_offset = hdr.symtab_offset + ht_size * 8;
+
+  i64 offset = 0;
+  for (Entry *ent : entries) {
+    ent->value.type_offset = offset;
+    offset += ent->value.count * 4 + 4;
+  }
+
+  for (Entry *ent : entries) {
+    ent->value.name_offset = offset;
+    offset += ent->keylen + 1;
+  }
+
+  i64 bufsize = hdr.const_pool_offset + offset;
+
+  // Allocate an output buffer
+  ctx.output_file->buf2.resize(bufsize);
+  u8 *buf = ctx.output_file->buf2.data();
+
+  // Write a section header
+  memcpy(buf, &hdr, sizeof(hdr));
+
+  // Write a CU list
+  u8 *p = buf + sizeof(hdr);
+
+  for (Compunit &cu : cus) {
+    *(ul64 *)p = cu.offset;
+    *(ul64 *)(p + 8) = cu.size;
+    p += 16;
+  }
+
+  // Write address areas
+  for (i64 i = 0; i < cus.size(); i++) {
+    for (std::pair<u64, u64> range : cus[i].ranges) {
+      *(ul64 *)p = range.first;
+      *(ul64 *)(p + 8) = range.second;
+      *(ul32 *)(p + 16) = i;
+      p += 20;
+    }
+  }
+
+  // Write a symbol table
+  u32 mask = ht_size - 1;
+  ul32 *ht = (ul32 *)(buf + hdr.symtab_offset);
+
+  for (Entry *ent : entries) {
+    u32 hash = ent->value.hash;
+    u32 step = ((hash * 17) & mask) | 1;
+    u32 j = hash & mask;
+
+    while (ht[j * 2] || ht[j * 2 + 1])
+      j = (j + step) & mask;
+
+    ht[j * 2] = ent->value.name_offset;
+    ht[j * 2 + 1] = ent->value.type_offset;
+  }
+
+  // Write types
+  for (i64 i = 0; i < cus.size(); i++) {
+    Compunit &cu = cus[i];
+    u8 *base = buf + hdr.const_pool_offset;
+
+    for (i64 j = 0; j < cu.nametypes.size(); j++) {
+      ul32 *p = (ul32 *)(base + cu.entries[j]->type_offset);
+      i64 idx = ++p[0];
+      p[idx] = (cu.nametypes[j].type << 24) | i;
+    }
+  }
+
+  // Write names
+  tbb::parallel_for_each(entries, [&](Entry *ent) {
+    memcpy(buf + hdr.const_pool_offset + ent->value.name_offset,
+           ent->key, ent->keylen);
+  });
+
+  // Update the section size and rewrite the section header
+  if (ctx.shdr) {
+    ctx.gdb_index->shdr.sh_size = bufsize;
+    ctx.shdr->copy_buf(ctx);
+  }
+}
+
+using E = MOLD_TARGET;
+
+template void write_gdb_index(Context<E> &);
+
+} // namespace mold::elf

elf/icf.cc

@@ -115,21 +115,19 @@ static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
   const ElfShdr<E> &shdr = isec.shdr();
   std::string_view name = isec.name();
 
-  bool is_alloc = (shdr.sh_flags & SHF_ALLOC);
-  bool is_exec = (shdr.sh_flags & SHF_EXECINSTR) ||
-                 ctx.arg.ignore_data_address_equality;
-  bool is_relro = (name == ".data.rel.ro" ||
-                   name.starts_with(".data.rel.ro."));
-  bool is_readonly = !(shdr.sh_flags & SHF_WRITE) || is_relro;
-  bool is_bss = (shdr.sh_type == SHT_NOBITS);
-  bool is_empty = (shdr.sh_size == 0);
-  bool is_init = (shdr.sh_type == SHT_INIT_ARRAY || name == ".init");
-  bool is_fini = (shdr.sh_type == SHT_FINI_ARRAY || name == ".fini");
-  bool is_enumerable = is_c_identifier(name);
-  bool is_addr_taken = !ctx.arg.icf_all && isec.address_significant;
+  if (shdr.sh_size == 0 || !(shdr.sh_flags & SHF_ALLOC) ||
+      shdr.sh_type == SHT_NOBITS || is_c_identifier(name))
+    return false;
 
-  return is_alloc && is_exec && is_readonly && !is_bss && !is_empty &&
-         !is_init && !is_fini && !is_enumerable && !is_addr_taken;
+  if (shdr.sh_flags & SHF_EXECINSTR) {
+    return (ctx.arg.icf_all || !isec.address_taken) &&
+           name != ".init" && name != ".fini";
+  } else {
+    bool is_readonly = !(shdr.sh_flags & SHF_WRITE);
+    bool is_relro = isec.output_section && isec.output_section->is_relro;
+    return (ctx.arg.ignore_data_address_equality || !isec.address_taken) &&
+           (is_readonly || is_relro);
+  }
 }
 
 static Digest digest_final(blake3_hasher *hasher) {

elf/input-files.cc

@@ -249,15 +249,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
       break;
     }
     case SHT_REL:
-      if constexpr (E::is_rela)
-        Fatal(ctx) << *this << ": REL-type relocation table is not supported"
-                   << " for this target";
-      break;
     case SHT_RELA:
-      if constexpr (!E::is_rela)
-        Fatal(ctx) << *this <<": RELA-type relocation table is not supported"
-                   << " for this target";
-      break;
     case SHT_SYMTAB:
     case SHT_SYMTAB_SHNDX:
     case SHT_STRTAB:
@@ -314,7 +306,7 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
       if (ctx.arg.oformat_binary && !(shdr.sh_flags & SHF_ALLOC))
         continue;
 
-      this->sections[i] = std::make_unique<InputSection<E>>(ctx, *this, name, i);
+      this->sections[i] = std::make_unique<InputSection<E>>(ctx, *this, i);
 
       // Save .llvm_addrsig for --icf=safe.
       if (shdr.sh_type == SHT_LLVM_ADDRSIG && !ctx.arg.relocatable) {
@@ -335,10 +327,6 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
 
         if (name == ".debug_info")
           debug_info = isec;
-        if (name == ".debug_ranges")
-          debug_ranges = isec;
-        if (name == ".debug_rnglists")
-          debug_rnglists = isec;
 
         // If --gdb-index is given, contents of .debug_gnu_pubnames and
         // .debug_gnu_pubtypes are copied to .gdb_index, so keeping them
@@ -649,9 +637,16 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
   if (!(shdr.sh_flags & SHF_MERGE))
     return nullptr;
 
+  i64 entsize = shdr.sh_entsize;
+  if (entsize == 0)
+    entsize = (shdr.sh_flags & SHF_STRINGS) ? 1 : (int)shdr.sh_addralign;
+
+  if (entsize == 0)
+    return nullptr;
+
   std::unique_ptr<MergeableSection<E>> rec(new MergeableSection<E>);
   rec->parent = MergedSection<E>::get_instance(ctx, sec.name(), shdr.sh_type,
-                                               shdr.sh_flags);
+                                               shdr.sh_flags, entsize);
   rec->p2align = sec.p2align;
 
   if (sec.sh_size == 0)
@@ -662,18 +657,10 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
 
   std::string_view data = sec.contents;
   const char *begin = data.data();
-  u64 entsize = shdr.sh_entsize;
   HyperLogLog estimator;
 
   // Split sections
   if (shdr.sh_flags & SHF_STRINGS) {
-    if (entsize == 0) {
-      // GHC (Glasgow Haskell Compiler) sometimes creates a mergeable
-      // string section with entsize of 0 instead of 1, though such
-      // entsize is technically wrong. This is a workaround for the issue.
-      entsize = 1;
-    }
-
     while (!data.empty()) {
       size_t end = find_null(data, entsize);
       if (end == data.npos)
@@ -690,11 +677,6 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
       estimator.insert(hash);
     }
   } else {
-    // OCaml compiler seems to create a mergeable non-string section with
-    // entisze of 0. Such section is malformed. We do not split such section.
-    if (entsize == 0)
-      return nullptr;
-
     if (data.size() % entsize)
       Fatal(ctx) << sec << ": section size is not multiple of sh_entsize";
 
@@ -866,33 +848,6 @@ void ObjectFile<E>::resolve_section_pieces(Context<E> &ctx) {
     this->symbols.push_back(&sym);
 }
 
-template <typename E>
-void ObjectFile<E>::mark_addrsig(Context<E> &ctx) {
-  // Parse a .llvm_addrsig section.
-  if (llvm_addrsig) {
-    u8 *cur = (u8 *)llvm_addrsig->contents.data();
-    u8 *end = cur + llvm_addrsig->contents.size();
-
-    while (cur != end) {
-      Symbol<E> &sym = *this->symbols[read_uleb(&cur)];
-      if (sym.file == this)
-        if (InputSection<E> *isec = sym.get_input_section())
-          isec->address_significant = true;
-    }
-  }
-
-  // We treat a symbol's address as significant if
-  //
-  // 1. we have no address significance information for the symbol, or
-  // 2. the symbol can be referenced from the outside in an address-
-  //    significant manner.
-  for (Symbol<E> *sym : this->symbols)
-    if (sym->file == this)
-      if (InputSection<E> *isec = sym->get_input_section())
-        if (!llvm_addrsig || sym->is_exported)
-          isec->address_significant = true;
-}
-
 template <typename E>
 void ObjectFile<E>::parse(Context<E> &ctx) {
   sections.resize(this->elf_sections.size());
@@ -1124,13 +1079,9 @@ void ObjectFile<E>::convert_common_symbols(Context<E> &ctx) {
     ElfShdr<E> &shdr = elf_sections2.back();
     memset(&shdr, 0, sizeof(shdr));
 
-    std::string_view name;
-
     if (sym.get_type() == STT_TLS) {
-      name = ".tls_common";
       shdr.sh_flags = SHF_ALLOC | SHF_WRITE | SHF_TLS;
     } else {
-      name = ".common";
       shdr.sh_flags = SHF_ALLOC | SHF_WRITE;
     }
 
@@ -1140,7 +1091,7 @@ void ObjectFile<E>::convert_common_symbols(Context<E> &ctx) {
 
     i64 idx = this->elf_sections.size() + elf_sections2.size() - 1;
     std::unique_ptr<InputSection<E>> isec =
-      std::make_unique<InputSection<E>>(ctx, *this, name, idx);
+      std::make_unique<InputSection<E>>(ctx, *this, idx);
 
     sym.file = this;
     sym.set_input_section(isec.get());

elf/input-sections.cc

@@ -7,7 +7,8 @@
 namespace mold::elf {
 
 typedef enum {
-  NONE, ERROR, COPYREL, DYN_COPYREL, PLT, CPLT, DYN_CPLT, DYNREL, BASEREL, IFUNC,
+  NONE, ERROR, COPYREL, DYN_COPYREL, PLT, CPLT, DYN_CPLT, DYNREL,
+  BASEREL, IFUNC_DYNREL,
 } Action;
 
 template <typename E>
@@ -37,8 +38,7 @@ static i64 to_p2align(u64 alignment) {
 }
 
 template <typename E>
-InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file,
-                              std::string_view name, i64 shndx)
+InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx)
   : file(file), shndx(shndx) {
   if (shndx < file.elf_sections.size())
     contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};
@@ -204,7 +204,7 @@ static void scan_rel(Context<E> &ctx, InputSection<E> &isec, Symbol<E> &sym,
     if (!isec.is_relr_reloc(ctx, rel))
       isec.file.num_dynrel++;
     break;
-  case IFUNC:
+  case IFUNC_DYNREL:
     // Create an IRELATIVE relocation for a GNU ifunc symbol.
     //
     // We usually create an IRELATIVE relocation in .got for each ifunc.
@@ -274,7 +274,7 @@ static Action get_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
 template <typename E>
 static Action get_dyn_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
   if (sym.is_ifunc())
-    return IFUNC;
+    return ctx.arg.pic ? IFUNC_DYNREL : NONE;
 
   // This is a decision table for absolute relocations for the pointer
   // size data (e.g. R_X86_64_64). Unlike the absrel_table, we can emit
@@ -292,7 +292,7 @@ static Action get_dyn_absrel_action(Context<E> &ctx, Symbol<E> &sym) {
 template <typename E>
 static Action get_ppc64_toc_action(Context<E> &ctx, Symbol<E> &sym) {
   if (sym.is_ifunc())
-    return IFUNC;
+    return IFUNC_DYNREL;
 
   // As a special case, we do not create copy relocations nor canonical
   // PLTs for .toc sections. PPC64's .toc is a compiler-generated
@@ -372,7 +372,7 @@ static void apply_absrel(Context<E> &ctx, InputSection<E> &isec,
   bool writable = (isec.shdr().sh_flags & SHF_WRITE);
 
   auto emit_abs_dynrel = [&] {
-    *dynrel++ = ElfRel<E>(P, E::R_DYNAMIC, sym.get_dynsym_idx(ctx), A);
+    *dynrel++ = ElfRel<E>(P, E::R_ABS, sym.get_dynsym_idx(ctx), A);
     if (ctx.arg.apply_dynamic_relocs)
       *(Word<E> *)loc = A;
   };
@@ -407,7 +407,7 @@ static void apply_absrel(Context<E> &ctx, InputSection<E> &isec,
   case DYNREL:
     emit_abs_dynrel();
     break;
-  case IFUNC:
+  case IFUNC_DYNREL:
     if constexpr (supports_ifunc<E>) {
       u64 addr = sym.get_addr(ctx, NO_PLT) + A;
       *dynrel++ = ElfRel<E>(P, E::R_IRELATIVE, 0, addr);
@@ -457,6 +457,18 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
       apply_reloc_alloc(ctx, buf);
     else
       apply_reloc_nonalloc(ctx, buf);
+
+    if constexpr (is_x86_64<E>) {
+      u8 endbr[] = {0xf3, 0x0f, 0x1e, 0xfa};
+      u8 nop[] = {0x0f, 0x1f, 0x40, 0x00};
+
+      // Rewrite the leading endbr instruction with a nop if the section
+      // is not address-taken.
+      if (ctx.arg.z_rewrite_endbr && (shdr().sh_flags & SHF_EXECINSTR) &&
+          !address_taken && sh_size >= 4 && memcmp(buf, endbr, 4) == 0) {
+        memcpy(buf, nop, 4);
+      }
+    }
   }
 }
 

elf/jobs.cc

@@ -1,3 +1,14 @@
+// Many build systems attempt to invoke as many linker processes as there
+// are cores, based on the assumption that the linker is single-threaded.
+// However, since mold is multi-threaded, such build systems' behavior is
+// not beneficial and just increases the overall peak memory usage.
+// On machines with limited memory, this could lead to an out-of-memory
+// error.
+//
+// This file implements a feature that limits the number of concurrent
+// mold processes to just 1 for each user. It is intended to be used as
+// `MOLD_JOBS=1 ninja` or `MOLD_JOBS=1 make -j$(nproc)`.
+
 #include "mold.h"
 
 #ifndef _WIN32
@@ -14,14 +25,15 @@ template <typename E>
 void acquire_global_lock(Context<E> &ctx) {
 #ifndef _WIN32
   char *jobs = getenv("MOLD_JOBS");
-  if (!jobs || std::string(jobs) != "1")
+  if (!jobs || jobs != "1"s)
     return;
 
-  char *home = getenv("HOME");
-  if (!home)
-    home = getpwuid(getuid())->pw_dir;
+  std::string path;
+  if (char *dir = getenv("XDG_RUNTIME_DIR"))
+    path = dir + "/mold-lock"s;
+  else
+    path = "/tmp/mold-lock-"s + getpwuid(getuid())->pw_name;
 
-  std::string path = std::string(home) + "/.mold-lock";
   int fd = open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600);
   if (fd == -1)
     return;

elf/main.cc

@@ -191,7 +191,8 @@ void read_file(Context<E> &ctx, MappedFile<Context<E>> *mf) {
         break;
       }
     }
-    ctx.visited.insert(mf->name);
+    if (!ctx.whole_archive)
+      ctx.visited.insert(mf->name);
     return;
   case FileType::TEXT:
     parse_linker_script(ctx, mf);
@@ -338,47 +339,6 @@ static void read_input_files(Context<E> &ctx, std::span<std::string> args) {
   ctx.tg.wait();
 }
 
-// Since elf_main is a template, we can't run it without a type parameter.
-// We speculatively run elf_main with X86_64, and if the speculation was
-// wrong, re-run it with an actual machine type.
-static int redo_main(int argc, char **argv, std::string_view target) {
-  if (target == I386::target_name)
-    return elf_main<I386>(argc, argv);
-  if (target == ARM64::target_name)
-    return elf_main<ARM64>(argc, argv);
-  if (target == ARM32::target_name)
-    return elf_main<ARM32>(argc, argv);
-  if (target == RV64LE::target_name)
-    return elf_main<RV64LE>(argc, argv);
-  if (target == RV64BE::target_name)
-    return elf_main<RV64BE>(argc, argv);
-  if (target == RV32LE::target_name)
-    return elf_main<RV32LE>(argc, argv);
-  if (target == RV32BE::target_name)
-    return elf_main<RV32BE>(argc, argv);
-  if (target == PPC32::target_name)
-    return elf_main<PPC32>(argc, argv);
-  if (target == PPC64V1::target_name)
-    return elf_main<PPC64V1>(argc, argv);
-  if (target == PPC64V2::target_name)
-    return elf_main<PPC64V2>(argc, argv);
-  if (target == S390X::target_name)
-    return elf_main<S390X>(argc, argv);
-  if (target == SPARC64::target_name)
-    return elf_main<SPARC64>(argc, argv);
-  if (target == M68K::target_name)
-    return elf_main<M68K>(argc, argv);
-  if (target == SH4::target_name)
-    return elf_main<SH4>(argc, argv);
-  if (target == ALPHA::target_name)
-    return elf_main<ALPHA>(argc, argv);
-  if (target == LOONGARCH32::target_name)
-    return elf_main<LOONGARCH32>(argc, argv);
-  if (target == LOONGARCH64::target_name)
-    return elf_main<LOONGARCH64>(argc, argv);
-  unreachable();
-}
-
 template <typename E>
 int elf_main(int argc, char **argv) {
   Context<E> ctx;
@@ -402,7 +362,7 @@ int elf_main(int argc, char **argv) {
   // Redo if -m is not x86-64.
   if constexpr (is_x86_64<E>)
     if (ctx.arg.emulation != X86_64::target_name)
-      return redo_main(argc, argv, ctx.arg.emulation);
+      return redo_main(ctx, argc, argv);
 
   Timer t_all(ctx, "all");
 
@@ -496,9 +456,9 @@ int elf_main(int argc, char **argv) {
   // Set is_imported and is_exported bits for each symbol.
   compute_import_export(ctx);
 
-  // Read address-significant section information.
-  if (ctx.arg.icf && !ctx.arg.icf_all)
-    mark_addrsig(ctx);
+  // Set "address-taken" bits for input sections.
+  if (ctx.arg.icf || ctx.arg.z_rewrite_endbr)
+    compute_address_significance(ctx);
 
   // Garbage-collect unreachable sections.
   if (ctx.arg.gc_sections)
@@ -637,10 +597,6 @@ int elf_main(int argc, char **argv) {
   // Here, we construct output .eh_frame contents.
   ctx.eh_frame->construct(ctx);
 
-  // Handle --gdb-index.
-  if (ctx.arg.gdb_index)
-    ctx.gdb_index->construct(ctx);
-
   // If --emit-relocs is given, we'll copy relocation sections from input
   // files to an output file.
   if (ctx.arg.emit_relocs)
@@ -686,12 +642,6 @@ int elf_main(int argc, char **argv) {
   // Copy input sections to the output file and apply relocations.
   copy_chunks(ctx);
 
-  // Some part of .gdb_index couldn't be computed until other debug
-  // sections are complete. We have complete debug sections now, so
-  // write the rest of .gdb_index.
-  if (ctx.gdb_index)
-    ctx.gdb_index->write_address_areas(ctx);
-
   // Dynamic linker works better with sorted .rela.dyn section,
   // so we sort them.
   ctx.reldyn->sort(ctx);
@@ -699,6 +649,12 @@ int elf_main(int argc, char **argv) {
   // Zero-clear paddings between sections
   clear_padding(ctx);
 
+  // .gdb_index's contents cannot be constructed before applying
+  // relocations to other debug sections. We have relocated debug
+  // sections now, so write the .gdb_index section.
+  if (ctx.gdb_index)
+    write_gdb_index(ctx);
+
   // .note.gnu.build-id section contains a cryptographic hash of the
   // entire output file. Now that we wrote everything except build-id,
   // we can compute it.
@@ -747,35 +703,13 @@ int elf_main(int argc, char **argv) {
 }
 
 #ifdef MOLD_X86_64
-
-extern template int elf_main<I386>(int, char **);
-extern template int elf_main<ARM32>(int, char **);
-extern template int elf_main<ARM64>(int, char **);
-extern template int elf_main<RV32BE>(int, char **);
-extern template int elf_main<RV32LE>(int, char **);
-extern template int elf_main<RV64LE>(int, char **);
-extern template int elf_main<RV64BE>(int, char **);
-extern template int elf_main<PPC32>(int, char **);
-extern template int elf_main<PPC64V1>(int, char **);
-extern template int elf_main<PPC64V2>(int, char **);
-extern template int elf_main<S390X>(int, char **);
-extern template int elf_main<SPARC64>(int, char **);
-extern template int elf_main<M68K>(int, char **);
-extern template int elf_main<SH4>(int, char **);
-extern template int elf_main<ALPHA>(int, char **);
-extern template int elf_main<LOONGARCH32>(int, char **);
-extern template int elf_main<LOONGARCH64>(int, char **);
-
 int main(int argc, char **argv) {
   return elf_main<X86_64>(argc, argv);
 }
-
-#else
+#endif
 
 using E = MOLD_TARGET;
 
 template int elf_main<E>(int, char **);
 
-#endif
-
 } // namespace mold::elf

elf/mold.h

@@ -224,8 +224,7 @@ struct InputSectionExtras<E> {
 template <typename E>
 class InputSection {
 public:
-  InputSection(Context<E> &ctx, ObjectFile<E> &file, std::string_view name,
-               i64 shndx);
+  InputSection(Context<E> &ctx, ObjectFile<E> &file, i64 shndx);
 
   void uncompress(Context<E> &ctx);
   void uncompress_to(Context<E> &ctx, u8 *buf);
@@ -258,17 +257,19 @@ public:
   i32 fde_begin = -1;
   i32 fde_end = -1;
 
-  u32 offset = -1;
+  u64 offset = -1;
   u32 shndx = -1;
   u32 relsec_idx = -1;
   u32 reldyn_offset = 0;
 
+  bool uncompressed = false;
+
   // For COMDAT de-duplication and garbage collection
   std::atomic_bool is_alive = true;
   u8 p2align = 0;
 
-  bool address_significant : 1 = false;
-  bool uncompressed : 1 = false;
+  // For ICF
+  Atomic<bool> address_taken = false;
 
   // For garbage collection
   Atomic<bool> is_visited = false;
@@ -354,14 +355,15 @@ public:
   virtual void write_to(Context<E> &ctx, u8 *buf) { unreachable(); }
   virtual void update_shdr(Context<E> &ctx) {}
 
-  // For --gdb-index
-  virtual u8 *get_uncompressed_data() { return nullptr; }
-
   std::string_view name;
   ElfShdr<E> shdr = { .sh_addralign = 1 };
   i64 shndx = 0;
   bool is_relro = false;
 
+  // For --gdb-index
+  bool is_compressed = false;
+  std::vector<u8> uncompressed_data;
+
   // Some synethetic sections add local symbols to the output.
   // For example, range extension thunks adds function_name@thunk
   // symbol for each thunk entry. The following members are used
@@ -620,6 +622,12 @@ public:
   }
 
   void update_shdr(Context<E> &ctx) override;
+  void copy_buf(Context<E> &ctx) override;
+
+  // Offsets in .strtab for ARM32 mapping symbols
+  static constexpr i64 ARM = 1;
+  static constexpr i64 THUMB = 4;
+  static constexpr i64 DATA = 7;
 };
 
 template <typename E>
@@ -759,7 +767,8 @@ template <typename E>
 class MergedSection : public Chunk<E> {
 public:
   static MergedSection<E> *
-  get_instance(Context<E> &ctx, std::string_view name, u64 type, u64 flags);
+  get_instance(Context<E> &ctx, std::string_view name, i64 type, i64 flags,
+               i64 entsize);
 
   SectionFragment<E> *insert(Context<E> &ctx, std::string_view data,
                              u64 hash, i64 p2align);
@@ -772,7 +781,7 @@ public:
   HyperLogLog estimator;
 
 private:
-  MergedSection(std::string_view name, u64 flags, u32 type);
+  MergedSection(std::string_view name, i64 flags, i64 type, i64 entsize);
 
   ConcurrentMap<SectionFragment<E>> map;
   std::vector<i64> shard_offsets;
@@ -948,13 +957,6 @@ private:
   std::map<u32, u32> properties;
 };
 
-struct GdbIndexName {
-  std::string_view name;
-  u32 hash = 0;
-  u32 attr = 0;
-  u32 entry_idx = 0;
-};
-
 template <typename E>
 class GdbIndexSection : public Chunk<E> {
 public:
@@ -963,38 +965,6 @@ public:
     this->shdr.sh_type = SHT_PROGBITS;
     this->shdr.sh_addralign = 4;
   }
-
-  void construct(Context<E> &ctx);
-  void copy_buf(Context<E> &ctx) override;
-  void write_address_areas(Context<E> &ctx);
-
-private:
-  struct SectionHeader {
-    ul32 version = 7;
-    ul32 cu_list_offset = 0;
-    ul32 cu_types_offset = 0;
-    ul32 areas_offset = 0;
-    ul32 symtab_offset = 0;
-    ul32 const_pool_offset = 0;
-  };
-
-  struct MapEntry {
-    MapEntry(ObjectFile<E> *owner, u32 hash) : owner(owner), hash(hash) {}
-
-    MapEntry(const MapEntry &other)
-      : owner(other.owner.load()), num_attrs(other.num_attrs.load()),
-        hash(other.hash), name_offset(other.name_offset),
-        attr_offset(other.attr_offset) {}
-
-    std::atomic<ObjectFile<E> *> owner;
-    std::atomic_uint32_t num_attrs = 0;
-    u32 hash = 0;
-    u32 name_offset = -1;
-    u32 attr_offset = -1;
-  };
-
-  SectionHeader header;
-  ConcurrentMap<MapEntry> map;
 };
 
 template <typename E>
@@ -1002,12 +972,10 @@ class CompressedSection : public Chunk<E> {
 public:
   CompressedSection(Context<E> &ctx, Chunk<E> &chunk);
   void copy_buf(Context<E> &ctx) override;
-  u8 *get_uncompressed_data() override { return uncompressed.get(); }
 
 private:
   ElfChdr<E> chdr = {};
   std::unique_ptr<Compressor> compressed;
-  std::unique_ptr<u8[]> uncompressed;
 };
 
 template <typename E>
@@ -1059,22 +1027,10 @@ private:
 };
 
 //
-// dwarf.cc
+// gdb-index.cc
 //
 
-template <typename E>
-std::vector<std::string_view>
-read_compunits(Context<E> &ctx, ObjectFile<E> &file);
-
-template <typename E>
-std::vector<GdbIndexName> read_pubnames(Context<E> &ctx, ObjectFile<E> &file);
-
-template <typename E>
-i64 estimate_address_areas(Context<E> &ctx, ObjectFile<E> &file);
-
-template <typename E>
-std::vector<u64>
-read_address_areas(Context<E> &ctx, ObjectFile<E> &file, i64 offset);
+template <typename E> void write_gdb_index(Context<E> &ctx);
 
 //
 // input-files.cc
@@ -1210,7 +1166,6 @@ public:
   void mark_live_objects(Context<E> &ctx,
                          std::function<void(InputFile<E> *)> feeder) override;
   void convert_undefined_weak_symbols(Context<E> &ctx);
-  void mark_addrsig(Context<E> &ctx);
   void scan_relocations(Context<E> &ctx);
   void convert_common_symbols(Context<E> &ctx);
   void compute_symtab_size(Context<E> &ctx);
@@ -1246,19 +1201,8 @@ public:
 
   // For .gdb_index
   InputSection<E> *debug_info = nullptr;
-  InputSection<E> *debug_ranges = nullptr;
-  InputSection<E> *debug_rnglists = nullptr;
   InputSection<E> *debug_pubnames = nullptr;
   InputSection<E> *debug_pubtypes = nullptr;
-  std::vector<std::string_view> compunits;
-  std::vector<GdbIndexName> gdb_names;
-  i64 compunits_idx = 0;
-  i64 attrs_size = 0;
-  i64 attrs_offset = 0;
-  i64 names_size = 0;
-  i64 names_offset = 0;
-  i64 num_areas = 0;
-  i64 area_offset = 0;
 
   // For LTO
   std::vector<std::string_view> lto_symbol_versions;
@@ -1409,6 +1353,7 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx);
 // passes.cc
 //
 
+template <typename E> int redo_main(Context<E> &, int argc, char **argv);
 template <typename E> void create_internal_file(Context<E> &);
 template <typename E> void apply_exclude_libs(Context<E> &);
 template <typename E> void create_synthetic_sections(Context<E> &);
@@ -1440,7 +1385,7 @@ template <typename E> void copy_chunks(Context<E> &);
 template <typename E> void apply_version_script(Context<E> &);
 template <typename E> void parse_symbol_version(Context<E> &);
 template <typename E> void compute_import_export(Context<E> &);
-template <typename E> void mark_addrsig(Context<E> &);
+template <typename E> void compute_address_significance(Context<E> &);
 template <typename E> void clear_padding(Context<E> &);
 template <typename E> void compute_section_headers(Context<E> &);
 template <typename E> i64 set_osec_offsets(Context<E> &);
@@ -1741,6 +1686,7 @@ struct Context {
     bool z_now = false;
     bool z_origin = false;
     bool z_relro = true;
+    bool z_rewrite_endbr = false;
     bool z_sectionheader = true;
     bool z_shstk = false;
     bool z_text = false;
@@ -1883,11 +1829,11 @@ struct Context {
   [[no_unique_address]] ContextExtras<E> extra;
 
   // For --gdb-index
-  Chunk<E> *debug_info = nullptr;
-  Chunk<E> *debug_abbrev = nullptr;
-  Chunk<E> *debug_ranges = nullptr;
-  Chunk<E> *debug_addr = nullptr;
-  Chunk<E> *debug_rnglists = nullptr;
+  std::span<u8> debug_info;
+  std::span<u8> debug_abbrev;
+  std::span<u8> debug_ranges;
+  std::span<u8> debug_addr;
+  std::span<u8> debug_rnglists;
 
   // For thread-local variables
   u64 tls_begin = 0;
@@ -1995,6 +1941,7 @@ public:
   u64 get_tlsdesc_addr(Context<E> &ctx) const;
   u64 get_plt_addr(Context<E> &ctx) const;
   u64 get_opd_addr(Context<E> &ctx) const;
+  u64 get_got_pltgot_addr(Context<E> &ctx) const;
 
   void set_got_idx(Context<E> &ctx, i32 idx);
   void set_gottp_idx(Context<E> &ctx, i32 idx);
@@ -2028,6 +1975,7 @@ public:
   bool is_relative() const { return !is_absolute(); }
   bool is_local(Context<E> &ctx) const;
   bool is_ifunc() const { return get_type() == STT_GNU_IFUNC; }
+  bool is_pde_ifunc(Context<E> &ctx) const;
   bool is_remaining_undef_weak() const;
 
   bool is_pcrel_linktime_const(Context<E> &ctx) const;
@@ -2629,6 +2577,25 @@ inline u64 Symbol<E>::get_opd_addr(Context<E> &ctx) const {
          get_opd_idx(ctx) * PPC64OpdSection::ENTRY_SIZE;
 }
 
+template <typename E>
+inline u64 Symbol<E>::get_got_pltgot_addr(Context<E> &ctx) const {
+  // An ifunc symbol occupies two consecutive GOT slots in a
+  // position-dependent executable (PDE). The first slot contains the
+  // symbol's PLT address, and the second slot holds the resolved
+  // address. A PDE uses the ifunc symbol's PLT entry as the address
+  // for the symbol, akin to a canonical PLT.
+  //
+  // This function returns the address that the PLT entry should use
+  // to jump to the resolved address.
+  //
+  // Note that we don't use this function for PPC64. In PPC64, symbols
+  // are always accessed through the TOC table regardless of the
+  // -fno-PIE setting. We don't need canonical PLTs on the psABIs too.
+  if (is_pde_ifunc(ctx))
+    return get_got_addr(ctx) + sizeof(Word<E>);
+  return get_got_addr(ctx);
+}
+
 template <typename E>
 inline void Symbol<E>::set_got_idx(Context<E> &ctx, i32 idx) {
   assert(aux_idx != -1);
@@ -2757,6 +2724,12 @@ inline bool Symbol<E>::is_local(Context<E> &ctx) const {
   return !is_imported && !is_exported;
 }
 
+template <typename E>
+inline bool Symbol<E>::is_pde_ifunc(Context<E> &ctx) const {
+  // Returns true if this is an ifunc tha uses two GOT slots
+  return is_ifunc() && !ctx.arg.pic && !is_ppc64<E>;
+}
+
 // A remaining weak undefined symbol is promoted to a dynamic symbol
 // in DSO and resolved to 0 in an executable. This function returns
 // true if it's latter.

elf/output-chunks.cc

@@ -462,6 +462,14 @@ template <typename E>
 void StrtabSection<E>::update_shdr(Context<E> &ctx) {
   i64 offset = 1;
 
+  // ARM32 uses $a, $t and $t mapping symbols to mark the beginning of
+  // ARM, Thumb and data in text, respectively. These symbols don't
+  // affect correctness of the program but helps disassembler to
+  // disassemble machine code appropriately.
+  if constexpr (is_arm32<E>)
+    if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file)
+      offset += sizeof("$a\0$t\0$d");
+
   for (Chunk<E> *chunk : ctx.chunks) {
     chunk->strtab_offset = offset;
     offset += chunk->strtab_size;
@@ -480,6 +488,16 @@ void StrtabSection<E>::update_shdr(Context<E> &ctx) {
   this->shdr.sh_size = (offset == 1) ? 0 : offset;
 }
 
+template <typename E>
+void StrtabSection<E>::copy_buf(Context<E> &ctx) {
+  u8 *buf = ctx.buf + this->shdr.sh_offset;
+  buf[0] = '\0';
+
+  if constexpr (is_arm32<E>)
+    if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file)
+      memcpy(buf + 1, "$a\0$t\0$d", 9);
+}
+
 template <typename E>
 void ShstrtabSection<E>::update_shdr(Context<E> &ctx) {
   std::unordered_map<std::string_view, i64> map;
@@ -591,9 +609,6 @@ void SymtabSection<E>::copy_buf(Context<E> &ctx) {
   ElfSym<E> *buf = (ElfSym<E> *)(ctx.buf + this->shdr.sh_offset);
   memset(buf, 0, sizeof(ElfSym<E>));
 
-  // Write the initial NUL byte to .strtab.
-  ctx.buf[ctx.strtab->shdr.sh_offset] = '\0';
-
   if (ctx.symtab_shndx) {
     ElfShdr<E> &shdr = ctx.symtab_shndx->shdr;
     memset(ctx.buf + shdr.sh_offset, 0, shdr.sh_size);
@@ -632,6 +647,27 @@ void SymtabSection<E>::copy_buf(Context<E> &ctx) {
   });
 }
 
+// An ARM64 function with a non-standard calling convention is marked with
+// STO_AARCH64_VARIANT_PCS bit in the symbol table.
+//
+// A function with that bit is not safe to be called through a lazy PLT
+// stub because the PLT resolver may clobber registers that should be
+// preserved in a non-standard calling convention.
+//
+// To solve the problem, the dynamic linker scans the dynamic symbol table
+// at process startup time and resolve symbols with STO_AARCH64_VARIANT_PCS
+// bit eagerly, so that the PLT resolver won't be called for that symbol
+// lazily. As an optimization, it does so only when DT_AARCH64_VARIANT_PCS
+// is set in the dynamic section.
+//
+// This function returns true if DT_AARCH64_VARIANT_PCS needs to be set.
+static bool contains_variant_pcs(Context<ARM64> &ctx) {
+  for (Symbol<ARM64> *sym : ctx.plt->symbols)
+    if (sym->esym().arm64_variant_pcs)
+      return true;
+  return false;
+}
+
 template <typename E>
 static std::vector<Word<E>> create_dynamic_section(Context<E> &ctx) {
   std::vector<Word<E>> vec;
@@ -781,6 +817,10 @@ static std::vector<Word<E>> create_dynamic_section(Context<E> &ctx) {
   if (flags1)
     define(DT_FLAGS_1, flags1);
 
+  if constexpr (is_arm64<E>)
+    if (contains_variant_pcs(ctx))
+      define(DT_AARCH64_VARIANT_PCS, 1);
+
   if constexpr (is_ppc32<E>)
     define(DT_PPC_GOT, ctx.gotplt->shdr.sh_addr);
 
@@ -825,7 +865,7 @@ OutputSection<E>::OutputSection(Context<E> &ctx, std::string_view name,
                                 u32 type, u64 flags) {
   this->name = name;
   this->shdr.sh_type = type;
-  this->shdr.sh_flags = flags;
+  this->shdr.sh_flags = flags & ~SHF_MERGE & ~SHF_STRINGS;
 
   if (auto it = ctx.arg.section_align.find(name);
       it != ctx.arg.section_align.end())
@@ -858,31 +898,30 @@ void OutputSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void OutputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
-  auto clear = [&](u8 *loc, i64 size) {
-    // As a special case, .init and .fini are filled with NOPs for s390x
-    // because the runtime executes the sections as if they were a single
-    // function. .init and .fini are superceded by .init_array and
-    // .fini_array but being actively used only on s390x.
-    if constexpr (is_s390x<E>) {
-      if (this->name == ".init" || this->name == ".fini") {
-        for (i64 i = 0; i < size; i += 2)
-          *(ub16 *)(loc + i) = 0x0700; // nop
-        return;
-      }
-    }
-    memset(loc, 0, size);
-  };
-
   tbb::parallel_for((i64)0, (i64)members.size(), [&](i64 i) {
-    // Copy section contents to an output file
+    // Copy section contents to an output file.
     InputSection<E> &isec = *members[i];
     isec.write_to(ctx, buf + isec.offset);
 
-    // Clear trailing padding
+    // Clear trailing padding. We write trap or nop instructions for
+    // an executable segment so that a disassembler wouldn't try to
+    // disassemble garbage as instructions.
     u64 this_end = isec.offset + isec.sh_size;
-    u64 next_start = (i == members.size() - 1) ?
-      (u64)this->shdr.sh_size : members[i + 1]->offset;
-    clear(buf + this_end, next_start - this_end);
+    u64 next_start;
+    if (i + 1 < members.size())
+      next_start = members[i + 1]->offset;
+    else
+      next_start = this->shdr.sh_size;
+
+    u8 *loc = buf + this_end;
+    i64 size = next_start - this_end;
+
+    if (this->shdr.sh_flags & SHF_EXECINSTR) {
+      for (i64 i = 0; i + sizeof(E::filler) <= size; i += sizeof(E::filler))
+        memcpy(loc + i, E::filler, sizeof(E::filler));
+    } else {
+      memset(loc, 0, size);
+    }
   });
 
   if constexpr (needs_thunk<E>) {
@@ -976,16 +1015,10 @@ void OutputSection<E>::construct_relr(Context<E> &ctx) {
 // Compute spaces needed for thunk symbols
 template <typename E>
 void OutputSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file || ctx.arg.relocatable)
-    return;
-
   if constexpr (needs_thunk<E>) {
     this->strtab_size = 0;
     this->num_local_symtab = 0;
 
-    if constexpr (is_arm32<E>)
-      this->strtab_size = 9; // for "$t", "$a" and "$d" symbols
-
     for (std::unique_ptr<RangeExtensionThunk<E>> &thunk : thunks) {
       // For ARM32, we emit additional symbol "$t", "$a" and "$d" for
       // each thunk to mark the beginning of Thumb code, ARM code and
@@ -1006,7 +1039,7 @@ void OutputSection<E>::compute_symtab_size(Context<E> &ctx) {
 // disassembling and/or debugging our output.
 template <typename E>
 void OutputSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   if constexpr (needs_thunk<E>) {
@@ -1016,37 +1049,30 @@ void OutputSection<E>::populate_symtab(Context<E> &ctx) {
     u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
     u8 *strtab = strtab_base + this->strtab_offset;
 
-    if constexpr (is_arm32<E>) {
-      // ARM uses these symbols to mark the begining of Thumb code, ARM
-      // code and data, respectively. Our thunk contains all of them.
-      strtab += write_string(strtab, "$t");
-      strtab += write_string(strtab, "$a");
-      strtab += write_string(strtab, "$d");
-    }
+    auto write_esym = [&](u64 addr, i64 st_name) {
+      memset(esym, 0, sizeof(*esym));
+      esym->st_name = st_name;
+      esym->st_type = STT_FUNC;
+      esym->st_shndx = this->shndx;
+      esym->st_value = addr;
+      esym++;
+    };
 
     for (std::unique_ptr<RangeExtensionThunk<E>> &thunk : thunks) {
       for (i64 i = 0; i < thunk->symbols.size(); i++) {
         Symbol<E> &sym = *thunk->symbols[i];
+        u64 addr = thunk->get_addr(i);
 
-        auto write_esym = [&](i64 st_name, i64 off) {
-          memset(esym, 0, sizeof(*esym));
-          esym->st_name = st_name;
-          esym->st_type = STT_FUNC;
-          esym->st_shndx = this->shndx;
-          esym->st_value = thunk->get_addr(i) + off;
-          esym++;
-        };
-
-        write_esym(strtab - strtab_base, 0);
+        write_esym(addr, strtab - strtab_base);
 
         strtab += write_string(strtab, sym.name()) - 1;
         strtab += write_string(strtab, "$thunk");
 
         // Emit "$t", "$a" and "$d" if ARM32.
         if constexpr (is_arm32<E>) {
-          write_esym(this->strtab_offset, 0);
-          write_esym(this->strtab_offset + 3, 4);
-          write_esym(this->strtab_offset + 6, 12);
+          write_esym(addr, ctx.strtab->THUMB);
+          write_esym(addr + 4, ctx.strtab->ARM);
+          write_esym(addr + 12, ctx.strtab->DATA);
         }
       }
     }
@@ -1056,7 +1082,14 @@ void OutputSection<E>::populate_symtab(Context<E> &ctx) {
 template <typename E>
 void GotSection<E>::add_got_symbol(Context<E> &ctx, Symbol<E> *sym) {
   sym->set_got_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
-  this->shdr.sh_size += sizeof(Word<E>);
+
+  // An IFUNC symbol uses two GOT slots in a position-dependent
+  // executable.
+  if (sym->is_pde_ifunc(ctx))
+    this->shdr.sh_size += sizeof(Word<E>) * 2;
+  else
+    this->shdr.sh_size += sizeof(Word<E>);
+
   got_syms.push_back(sym);
 }
 
@@ -1080,10 +1113,10 @@ void GotSection<E>::add_tlsdesc_symbol(Context<E> &ctx, Symbol<E> *sym) {
   // always emit a dynamic relocation for each TLSDESC entry.
   //
   // If dynamic relocation is not available (i.e. if we are creating a
-  // non-PIC executable), we always relax TLSDESC relocations so that
-  // no TLSDESC relocation exist at runtime.
+  // statically-linked executable), we always relax TLSDESC relocations
+  // so that no TLSDESC relocation exist at runtime.
   assert(supports_tlsdesc<E>);
-  assert(ctx.arg.pic);
+  assert(!ctx.arg.is_static);
 
   sym->set_tlsdesc_idx(ctx, this->shdr.sh_size / sizeof(Word<E>));
   this->shdr.sh_size += sizeof(Word<E>) * 2;
@@ -1150,7 +1183,12 @@ static std::vector<GotEntry<E>> get_got_entries(Context<E> &ctx) {
     // IFUNC always needs to be fixed up by the dynamic linker.
     if constexpr (supports_ifunc<E>) {
       if (sym->is_ifunc()) {
-        add({idx, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        if (sym->is_pde_ifunc(ctx)) {
+          add({idx, sym->get_plt_addr(ctx)});
+          add({idx + 1, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        } else {
+          add({idx, sym->get_addr(ctx, NO_PLT), E::R_IRELATIVE});
+        }
         continue;
       }
     }
@@ -1305,9 +1343,6 @@ void GotSection<E>::construct_relr(Context<E> &ctx) {
 
 template <typename E>
 void GotSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->strtab_size = 0;
   this->num_local_symtab = 0;
 
@@ -1339,7 +1374,7 @@ void GotSection<E>::compute_symtab_size(Context<E> &ctx) {
 
 template <typename E>
 void GotSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1430,19 +1465,19 @@ void PltSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void PltSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->num_local_symtab = symbols.size();
   this->strtab_size = 0;
 
   for (Symbol<E> *sym : symbols)
     this->strtab_size += sym->name().size() + sizeof("$plt");
+
+  if constexpr (is_arm32<E>)
+    this->num_local_symtab += symbols.size() * 2 + 2;
 }
 
 template <typename E>
 void PltSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1451,16 +1486,30 @@ void PltSection<E>::populate_symtab(Context<E> &ctx) {
   u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
   u8 *strtab = strtab_base + this->strtab_offset;
 
-  for (Symbol<E> *sym : symbols) {
+  auto write_esym = [&](u64 addr, i64 st_name) {
     memset(esym, 0, sizeof(*esym));
-    esym->st_name = strtab - strtab_base;
+    esym->st_name = st_name;
     esym->st_type = STT_FUNC;
     esym->st_shndx = this->shndx;
-    esym->st_value = sym->get_plt_addr(ctx);
+    esym->st_value = addr;
     esym++;
+  };
 
+  if constexpr (is_arm32<E>) {
+    write_esym(this->shdr.sh_addr, ctx.strtab->ARM);
+    write_esym(this->shdr.sh_addr + 16, ctx.strtab->DATA);
+  }
+
+  for (Symbol<E> *sym : symbols) {
+    u64 addr = sym->get_plt_addr(ctx);
+    write_esym(addr, strtab - strtab_base);
     strtab += write_string(strtab, sym->name()) - 1;
     strtab += write_string(strtab, "$plt");
+
+    if constexpr (is_arm32<E>) {
+      write_esym(addr, ctx.strtab->ARM);
+      write_esym(addr + 12, ctx.strtab->DATA);
+    }
   }
 }
 
@@ -1483,19 +1532,19 @@ void PltGotSection<E>::copy_buf(Context<E> &ctx) {
 
 template <typename E>
 void PltGotSection<E>::compute_symtab_size(Context<E> &ctx) {
-  if (ctx.arg.strip_all || ctx.arg.retain_symbols_file)
-    return;
-
   this->num_local_symtab = symbols.size();
   this->strtab_size = 0;
 
   for (Symbol<E> *sym : symbols)
     this->strtab_size += sym->name().size() + sizeof("$pltgot");
+
+  if constexpr (is_arm32<E>)
+    this->num_local_symtab += symbols.size() * 2;
 }
 
 template <typename E>
 void PltGotSection<E>::populate_symtab(Context<E> &ctx) {
-  if (this->strtab_size == 0)
+  if (this->num_local_symtab == 0)
     return;
 
   ElfSym<E> *esym =
@@ -1504,16 +1553,25 @@ void PltGotSection<E>::populate_symtab(Context<E> &ctx) {
   u8 *strtab_base = ctx.buf + ctx.strtab->shdr.sh_offset;
   u8 *strtab = strtab_base + this->strtab_offset;
 
-  for (Symbol<E> *sym : symbols) {
+  auto write_esym = [&](u64 addr, i64 st_name) {
     memset(esym, 0, sizeof(*esym));
-    esym->st_name = strtab - strtab_base;
+    esym->st_name = st_name;
     esym->st_type = STT_FUNC;
     esym->st_shndx = this->shndx;
-    esym->st_value = sym->get_plt_addr(ctx);
+    esym->st_value = addr;
     esym++;
+  };
 
+  for (Symbol<E> *sym : symbols) {
+    u64 addr = sym->get_plt_addr(ctx);
+    write_esym(addr, strtab - strtab_base);
     strtab += write_string(strtab, sym->name()) - 1;
     strtab += write_string(strtab, "$pltgot");
+
+    if constexpr (is_arm32<E>) {
+      write_esym(addr, ctx.strtab->ARM);
+      write_esym(addr + 12, ctx.strtab->DATA);
+    }
   }
 }
 
@@ -1561,6 +1619,9 @@ ElfSym<E> to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name,
   else
     esym.st_bind = sym.esym().st_bind;
 
+  if constexpr (is_arm64<E>)
+    esym.arm64_variant_pcs = sym.esym().arm64_variant_pcs;
+
   if constexpr (is_ppc64v2<E>)
     esym.ppc_local_entry = sym.esym().ppc_local_entry;
 
@@ -1607,8 +1668,15 @@ ElfSym<E> to_output_esym(Context<E> &ctx, Symbol<E> &sym, u32 st_name,
     esym.st_shndx = SHN_ABS;
     esym.st_value = sym.get_addr(ctx);
   } else if (sym.get_type() == STT_TLS) {
+    // TLS symbol
     shndx = get_st_shndx(sym);
     esym.st_value = sym.get_addr(ctx) - ctx.tls_begin;
+  } else if (sym.is_pde_ifunc(ctx)) {
+    // IFUNC symbol in PDE that uses two GOT slots
+    shndx = get_st_shndx(sym);
+    esym.st_type = STT_FUNC;
+    esym.st_visibility = sym.visibility;
+    esym.st_value = sym.get_addr(ctx);
   } else {
     shndx = get_st_shndx(sym);
     esym.st_visibility = sym.visibility;
@@ -1849,29 +1917,30 @@ get_merged_output_name(Context<E> &ctx, std::string_view name, u64 flags) {
     return name;
   if (ctx.arg.unique && ctx.arg.unique->match(name))
     return name;
-  if (name == ".rodata" || name.starts_with(".rodata."))
-    return (flags & SHF_STRINGS) ? ".rodata.str" : ".rodata.cst";
   return name;
 }
 
 template <typename E>
-MergedSection<E>::MergedSection(std::string_view name, u64 flags, u32 type) {
+MergedSection<E>::MergedSection(std::string_view name, i64 flags, i64 type,
+                                i64 entsize) {
   this->name = name;
   this->shdr.sh_flags = flags;
   this->shdr.sh_type = type;
+  this->shdr.sh_entsize = entsize;
 }
 
 template <typename E>
 MergedSection<E> *
 MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
-                               u64 type, u64 flags) {
+                               i64 type, i64 flags, i64 entsize) {
   name = get_merged_output_name(ctx, name, flags);
   flags = flags & ~(u64)SHF_GROUP & ~(u64)SHF_COMPRESSED;
 
   auto find = [&]() -> MergedSection * {
     for (std::unique_ptr<MergedSection<E>> &osec : ctx.merged_sections)
-      if (std::tuple(name, flags, type) ==
-          std::tuple(osec->name, osec->shdr.sh_flags, osec->shdr.sh_type))
+      if (std::tuple(name, flags, type, entsize) ==
+          std::tuple(osec->name, osec->shdr.sh_flags, osec->shdr.sh_type,
+                     osec->shdr.sh_entsize))
         return osec.get();
     return nullptr;
   };
@@ -1889,7 +1958,7 @@ MergedSection<E>::get_instance(Context<E> &ctx, std::string_view name,
   if (MergedSection *osec = find())
     return osec;
 
-  MergedSection *osec = new MergedSection(name, flags, type);
+  MergedSection *osec = new MergedSection(name, flags, type, entsize);
   ctx.merged_sections.emplace_back(osec);
   return osec;
 }
@@ -1919,67 +1988,49 @@ MergedSection<E>::insert(Context<E> &ctx, std::string_view data, u64 hash,
 template <typename E>
 void MergedSection<E>::assign_offsets(Context<E> &ctx) {
   std::vector<i64> sizes(map.NUM_SHARDS);
-  std::vector<i64> max_p2aligns(map.NUM_SHARDS);
-  shard_offsets.resize(map.NUM_SHARDS + 1);
-
-  i64 shard_size = map.nbuckets / map.NUM_SHARDS;
+  Atomic<i64> alignment = 1;
 
   tbb::parallel_for((i64)0, map.NUM_SHARDS, [&](i64 i) {
-    struct KeyVal {
-      std::string_view key;
-      SectionFragment<E> *val;
-    };
+    using Entry = typename decltype(map)::Entry;
+    std::vector<Entry *> entries = map.get_sorted_entries(i);
 
-    std::vector<KeyVal> fragments;
-    fragments.reserve(shard_size);
-
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (const char *key = map.get_key(j))
-        if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
-          fragments.push_back({{key, map.entries[j].keylen}, &frag});
-
-    // Sort fragments to make output deterministic.
-    tbb::parallel_sort(fragments.begin(), fragments.end(),
-                       [](const KeyVal &a, const KeyVal &b) {
-      return std::tuple{(u32)a.val->p2align, a.key.size(), a.key} <
-             std::tuple{(u32)b.val->p2align, b.key.size(), b.key};
-    });
-
-    // Assign offsets.
     i64 offset = 0;
     i64 p2align = 0;
 
-    for (KeyVal &kv : fragments) {
-      SectionFragment<E> &frag = *kv.val;
-      offset = align_to(offset, 1 << frag.p2align);
-      frag.offset = offset;
-      offset += kv.key.size();
-      p2align = std::max<i64>(p2align, frag.p2align);
+    for (Entry *ent : entries) {
+      SectionFragment<E> &frag = ent->value;
+      if (frag.is_alive) {
+        offset = align_to(offset, 1 << frag.p2align);
+        frag.offset = offset;
+        offset += ent->keylen;
+        p2align = std::max<i64>(p2align, frag.p2align);
+      }
     }
 
     sizes[i] = offset;
-    max_p2aligns[i] = p2align;
+    update_maximum(alignment, 1 << p2align);
 
     static Counter merged_strings("merged_strings");
-    merged_strings += fragments.size();
+    merged_strings += entries.size();
   });
 
-  i64 p2align = 0;
-  for (i64 x : max_p2aligns)
-    p2align = std::max(p2align, x);
+  i64 shard_size = map.nbuckets / map.NUM_SHARDS;
+  shard_offsets.resize(map.NUM_SHARDS + 1);
 
   for (i64 i = 1; i < map.NUM_SHARDS + 1; i++)
     shard_offsets[i] =
-      align_to(shard_offsets[i - 1] + sizes[i - 1], 1 << p2align);
+      align_to(shard_offsets[i - 1] + sizes[i - 1], alignment);
 
   tbb::parallel_for((i64)1, map.NUM_SHARDS, [&](i64 i) {
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
+    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
+      SectionFragment<E> &frag = map.entries[j].value;
+      if (frag.is_alive)
         frag.offset += shard_offsets[i];
+    }
   });
 
   this->shdr.sh_size = shard_offsets[map.NUM_SHARDS];
-  this->shdr.sh_addralign = 1 << p2align;
+  this->shdr.sh_addralign = alignment;
 }
 
 template <typename E>
@@ -1995,9 +2046,11 @@ void MergedSection<E>::write_to(Context<E> &ctx, u8 *buf) {
     memset(buf + shard_offsets[i], 0, shard_offsets[i + 1] - shard_offsets[i]);
 
     for (i64 j = shard_size * i; j < shard_size * (i + 1); j++)
-      if (const char *key = map.get_key(j))
-        if (SectionFragment<E> &frag = map.entries[j].value; frag.is_alive)
+      if (const char *key = map.entries[j].key) {
+        SectionFragment<E> &frag = map.entries[j].value;
+        if (frag.is_alive)
           memcpy(buf + frag.offset, key, map.entries[j].keylen);
+      }
   });
 }
 
@@ -2005,7 +2058,7 @@ template <typename E>
 void MergedSection<E>::print_stats(Context<E> &ctx) {
   i64 used = 0;
   for (i64 i = 0; i < map.nbuckets; i++)
-    if (map.get_key(i))
+    if (map.entries[i].key)
       used++;
 
   SyncOut(ctx) << this->name
@@ -2287,8 +2340,7 @@ void CopyrelSection<E>::update_shdr(Context<E> &ctx) {
   // a segment, so a .copyrel.rel.ro usually requires one extra
   // segment for it. We turn a .copyrel.rel.ro into a regular section
   // if it is very small to avoid the cost of the extra segment.
-  constexpr i64 threshold = 4096;
-  if (this->is_relro && ctx.arg.z_relro && this->shdr.sh_size < threshold)
+  if (this->is_relro && ctx.arg.z_relro && this->shdr.sh_size < E::page_size)
     this->shdr.sh_type = SHT_PROGBITS;
 }
 
@@ -2704,316 +2756,25 @@ void NotePropertySection<E>::copy_buf(Context<E> &ctx) {
   }
 }
 
-// This page explains the format of .gdb_index:
-// https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html
-template <typename E>
-void GdbIndexSection<E>::construct(Context<E> &ctx) {
-  Timer t(ctx, "GdbIndexSection::construct");
-
-  std::atomic_bool has_debug_info = false;
-
-  // Read debug sections
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    if (file->debug_info) {
-      // Read compilation units from .debug_info.
-      file->compunits = read_compunits(ctx, *file);
-
-      // Count the number of address areas contained in this file.
-      file->num_areas = estimate_address_areas(ctx, *file);
-      has_debug_info = true;
-    }
-  });
-
-  if (!has_debug_info)
-    return;
-
-  // Initialize `area_offset` and `compunits_idx`.
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++) {
-    ctx.objs[i + 1]->area_offset =
-      ctx.objs[i]->area_offset + ctx.objs[i]->num_areas * 20;
-    ctx.objs[i + 1]->compunits_idx =
-      ctx.objs[i]->compunits_idx + ctx.objs[i]->compunits.size();
-  }
-
-  // Read .debug_gnu_pubnames and .debug_gnu_pubtypes.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    file->gdb_names = read_pubnames(ctx, *file);
-  });
-
-  // Estimate the unique number of pubnames.
-  HyperLogLog estimator;
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    HyperLogLog e;
-    for (GdbIndexName &name : file->gdb_names)
-      e.insert(name.hash);
-    estimator.merge(e);
-  });
-
-  // Uniquify pubnames by inserting all name strings into a concurrent
-  // hashmap.
-  map.resize(estimator.get_cardinality() * 2);
-  tbb::enumerable_thread_specific<i64> num_names;
-
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (GdbIndexName &name : file->gdb_names) {
-      MapEntry *ent;
-      bool inserted;
-      std::tie(ent, inserted) = map.insert(name.name, name.hash, {file, name.hash});
-      if (inserted)
-        num_names.local()++;
-
-      ObjectFile<E> *old_val = ent->owner;
-      while (file->priority < old_val->priority &&
-             !ent->owner.compare_exchange_weak(old_val, file));
-
-      ent->num_attrs++;
-      name.entry_idx = map.get_idx(ent);
-    }
-  });
-
-  // Assign offsets for names and attributes within each file.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (GdbIndexName &name : file->gdb_names) {
-      MapEntry &ent = map.entries[name.entry_idx].value;
-      if (ent.owner == file) {
-        ent.attr_offset = file->attrs_size;
-        file->attrs_size += (ent.num_attrs + 1) * 4;
-        ent.name_offset = file->names_size;
-        file->names_size += name.name.size() + 1;
-      }
-    }
-  });
-
-  // Compute per-file name and attributes offsets.
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
-    ctx.objs[i + 1]->attrs_offset =
-      ctx.objs[i]->attrs_offset + ctx.objs[i]->attrs_size;
-
-  ctx.objs[0]->names_offset =
-    ctx.objs.back()->attrs_offset + ctx.objs.back()->attrs_size;
-
-  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
-    ctx.objs[i + 1]->names_offset =
-      ctx.objs[i]->names_offset + ctx.objs[i]->names_size;
-
-  // .gdb_index contains an on-disk hash table for pubnames and
-  // pubtypes. We aim 75% utilization. As per the format specification,
-  // It must be a power of two.
-  i64 num_symtab_entries =
-    std::max<i64>(bit_ceil(num_names.combine(std::plus()) * 4 / 3), 16);
-
-  // Now that we can compute the size of this section.
-  ObjectFile<E> &last = *ctx.objs.back();
-  i64 compunits_size = (last.compunits_idx + last.compunits.size()) * 16;
-  i64 areas_size = last.area_offset + last.num_areas * 20;
-  i64 offset = sizeof(header);
-
-  header.cu_list_offset = offset;
-  offset += compunits_size;
-
-  header.cu_types_offset = offset;
-  header.areas_offset = offset;
-  offset += areas_size;
-
-  header.symtab_offset = offset;
-  offset += num_symtab_entries * 8;
-
-  header.const_pool_offset = offset;
-  offset += last.names_offset + last.names_size;
-
-  this->shdr.sh_size = offset;
-}
-
-template <typename E>
-void GdbIndexSection<E>::copy_buf(Context<E> &ctx) {
-  u8 *buf = ctx.buf + this->shdr.sh_offset;
-
-  // Write section header.
-  memcpy(buf, &header, sizeof(header));
-  buf += sizeof(header);
-
-  // Write compilation unit list.
-  for (ObjectFile<E> *file : ctx.objs) {
-    if (file->debug_info) {
-      u64 offset = file->debug_info->offset;
-      for (std::string_view cu : file->compunits) {
-        *(ul64 *)buf = offset;
-        *(ul64 *)(buf + 8) = cu.size();
-        buf += 16;
-        offset += cu.size();
-      }
-    }
-  }
-
-  // Skip address areas. It'll be filled by write_address_areas.
-  buf += header.symtab_offset - header.areas_offset;
-
-  // Write an on-disk hash table for names.
-  u32 symtab_size = header.const_pool_offset - header.symtab_offset;
-  memset(buf, 0, symtab_size);
-
-  assert(has_single_bit(symtab_size / 8));
-  u32 mask = symtab_size / 8 - 1;
-
-  for (i64 i = 0; i < map.nbuckets; i++) {
-    if (map.get_key(i)) {
-      MapEntry &ent = map.entries[i].value;
-      u32 hash = ent.hash;
-      u32 step = (hash & mask) | 1;
-      u32 j = hash & mask;
-
-      while (*(U32<E> *)(buf + j * 8))
-        j = (j + step) & mask;
-
-      ObjectFile<E> &file = *ent.owner;
-      *(ul32 *)(buf + j * 8) = file.names_offset + ent.name_offset;
-      *(ul32 *)(buf + j * 8 + 4) = file.attrs_offset + ent.attr_offset;
-    }
-  }
-
-  buf += symtab_size;
-
-  // Write CU vector
-  memset(buf, 0, ctx.objs[0]->names_offset);
-
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    std::atomic_uint32_t *attrs = (std::atomic_uint32_t *)buf;
-
-    for (GdbIndexName &name : file->gdb_names) {
-      MapEntry &ent = map.entries[name.entry_idx].value;
-      u32 idx = (ent.owner.load()->attrs_offset + ent.attr_offset) / 4;
-      attrs[idx + ++attrs[idx]] = name.attr;
-    }
-  });
-
-  // Sort CU vector for build reproducibility
-  const i64 shard_size = map.nbuckets / map.NUM_SHARDS;
-
-  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
-    u32 *attrs = (u32 *)buf;
-
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
-      if (map.get_key(j)) {
-        MapEntry &ent = map.entries[j].value;
-        u32 idx = (ent.owner.load()->attrs_offset + ent.attr_offset) / 4;
-        u32 *start = attrs + idx + 1;
-        std::sort(start, start + attrs[idx]);
-      }
-    }
-  });
-
-  // .gdb_index contents are little-endian, so swap bytes if big-endian.
-  if constexpr (std::endian::native == std::endian::big)
-    for (i64 i = 0; i < ctx.objs[0]->names_offset; i += 4)
-      *(u32 *)(buf + i) = bswap(*(u32 *)(buf + i));
-
-  // Write pubnames and pubtypes.
-  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
-      if (const char *key = map.get_key(j)) {
-        MapEntry &ent = map.entries[j].value;
-        ObjectFile<E> &file = *ent.owner;
-        std::string_view name{key, map.entries[j].keylen};
-        write_string(buf + file.names_offset + ent.name_offset, name);
-      }
-    }
-  });
-}
-
-template <typename E>
-void GdbIndexSection<E>::write_address_areas(Context<E> &ctx) {
-  Timer t(ctx, "GdbIndexSection::write_address_areas");
-
-  if (this->shdr.sh_size == 0)
-    return;
-
-  u8 *base = ctx.buf + this->shdr.sh_offset;
-
-  for (Chunk<E> *chunk : ctx.chunks) {
-    std::string_view name = chunk->name;
-    if (name == ".debug_info")
-      ctx.debug_info = chunk;
-    if (name == ".debug_abbrev")
-      ctx.debug_abbrev = chunk;
-    if (name == ".debug_ranges")
-      ctx.debug_ranges = chunk;
-    if (name == ".debug_addr")
-      ctx.debug_addr = chunk;
-    if (name == ".debug_rnglists")
-      ctx.debug_rnglists = chunk;
-  }
-
-  assert(ctx.debug_info);
-  assert(ctx.debug_abbrev);
-
-  struct Entry {
-    ul64 start;
-    ul64 end;
-    ul32 attr;
-  };
-
-  // Read address ranges from debug sections and copy them to .gdb_index.
-  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    if (!file->debug_info)
-      return;
-
-    Entry *begin = (Entry *)(base + header.areas_offset + file->area_offset);
-    Entry *e = begin;
-    u64 offset = file->debug_info->offset;
-
-    for (i64 i = 0; i < file->compunits.size(); i++) {
-      std::vector<u64> addrs = read_address_areas(ctx, *file, offset);
-
-      for (i64 j = 0; j < addrs.size(); j += 2) {
-        // Skip an empty range
-        if (addrs[j] == addrs[j + 1])
-          continue;
-
-        // Gdb crashes if there are entries with address 0.
-        if (addrs[j] == 0)
-          continue;
-
-        assert(e < begin + file->num_areas);
-        e->start = addrs[j];
-        e->end = addrs[j + 1];
-        e->attr = file->compunits_idx + i;
-        e++;
-      }
-      offset += file->compunits[i].size();
-    }
-
-    // Fill trailing null entries with dummy values because gdb
-    // crashes if there are entries with address 0.
-    u64 filler;
-    if (e == begin)
-      filler = ctx.etext->get_addr(ctx) - 1;
-    else
-      filler = e[-1].start;
-
-    for (; e < begin + file->num_areas; e++) {
-      e->start = filler;
-      e->end = filler;
-      e->attr = file->compunits_idx;
-    }
-  });
-}
-
 template <typename E>
 CompressedSection<E>::CompressedSection(Context<E> &ctx, Chunk<E> &chunk) {
   assert(chunk.name.starts_with(".debug"));
   this->name = chunk.name;
+  this->is_compressed = true;
 
-  uncompressed.reset(new u8[chunk.shdr.sh_size]);
-  chunk.write_to(ctx, uncompressed.get());
+  this->uncompressed_data.resize(chunk.shdr.sh_size);
+  u8 *buf = this->uncompressed_data.data();
+
+  chunk.write_to(ctx, buf);
 
   switch (ctx.arg.compress_debug_sections) {
   case COMPRESS_ZLIB:
     chdr.ch_type = ELFCOMPRESS_ZLIB;
-    compressed.reset(new ZlibCompressor(uncompressed.get(), chunk.shdr.sh_size));
+    compressed.reset(new ZlibCompressor(buf, chunk.shdr.sh_size));
     break;
   case COMPRESS_ZSTD:
     chdr.ch_type = ELFCOMPRESS_ZSTD;
-    compressed.reset(new ZstdCompressor(uncompressed.get(), chunk.shdr.sh_size));
+    compressed.reset(new ZstdCompressor(buf, chunk.shdr.sh_size));
     break;
   default:
     unreachable();
@@ -3029,8 +2790,10 @@ CompressedSection<E>::CompressedSection(Context<E> &ctx, Chunk<E> &chunk) {
   this->shndx = chunk.shndx;
 
   // We don't need to keep the original data unless --gdb-index is given.
-  if (!ctx.arg.gdb_index)
-    uncompressed.reset(nullptr);
+  if (!ctx.arg.gdb_index) {
+    this->uncompressed_data.clear();
+    this->uncompressed_data.shrink_to_fit();
+  }
 }
 
 template <typename E>

elf/passes.cc

@@ -13,6 +13,50 @@
 
 namespace mold::elf {
 
+// Since elf_main is a template, we can't run it without a type parameter.
+// We speculatively run elf_main with X86_64, and if the speculation was
+// wrong, re-run it with an actual machine type.
+template <typename E>
+int redo_main(Context<E> &ctx, int argc, char **argv) {
+  std::string_view target = ctx.arg.emulation;
+
+  if (target == I386::target_name)
+    return elf_main<I386>(argc, argv);
+  if (target == ARM64::target_name)
+    return elf_main<ARM64>(argc, argv);
+  if (target == ARM32::target_name)
+    return elf_main<ARM32>(argc, argv);
+  if (target == RV64LE::target_name)
+    return elf_main<RV64LE>(argc, argv);
+  if (target == RV64BE::target_name)
+    return elf_main<RV64BE>(argc, argv);
+  if (target == RV32LE::target_name)
+    return elf_main<RV32LE>(argc, argv);
+  if (target == RV32BE::target_name)
+    return elf_main<RV32BE>(argc, argv);
+  if (target == PPC32::target_name)
+    return elf_main<PPC32>(argc, argv);
+  if (target == PPC64V1::target_name)
+    return elf_main<PPC64V1>(argc, argv);
+  if (target == PPC64V2::target_name)
+    return elf_main<PPC64V2>(argc, argv);
+  if (target == S390X::target_name)
+    return elf_main<S390X>(argc, argv);
+  if (target == SPARC64::target_name)
+    return elf_main<SPARC64>(argc, argv);
+  if (target == M68K::target_name)
+    return elf_main<M68K>(argc, argv);
+  if (target == SH4::target_name)
+    return elf_main<SH4>(argc, argv);
+  if (target == ALPHA::target_name)
+    return elf_main<ALPHA>(argc, argv);
+  if (target == LOONGARCH32::target_name)
+    return elf_main<LOONGARCH32>(argc, argv);
+  if (target == LOONGARCH64::target_name)
+    return elf_main<LOONGARCH64>(argc, argv);
+  unreachable();
+}
+
 template <typename E>
 void apply_exclude_libs(Context<E> &ctx) {
   Timer t(ctx, "apply_exclude_libs");
@@ -31,6 +75,14 @@ void apply_exclude_libs(Context<E> &ctx) {
   }
 }
 
+template <typename E>
+static bool has_debug_info_section(Context<E> &ctx) {
+  for (ObjectFile<E> *file : ctx.objs)
+    if (file->debug_info)
+      return true;
+  return false;
+}
+
 template <typename E>
 void create_synthetic_sections(Context<E> &ctx) {
   auto push = [&](auto *x) {
@@ -91,7 +143,7 @@ void create_synthetic_sections(Context<E> &ctx) {
     ctx.buildid = push(new BuildIdSection<E>);
   if (ctx.arg.eh_frame_hdr)
     ctx.eh_frame_hdr = push(new EhFrameHdrSection<E>);
-  if (ctx.arg.gdb_index)
+  if (ctx.arg.gdb_index && has_debug_info_section(ctx))
     ctx.gdb_index = push(new GdbIndexSection<E>);
   if (ctx.arg.z_relro && ctx.arg.section_order.empty() &&
       ctx.arg.z_separate_code != SEPARATE_LOADABLE_SEGMENTS)
@@ -345,7 +397,7 @@ template <typename E>
 void add_comment_string(Context<E> &ctx, std::string str) {
   MergedSection<E> *sec =
     MergedSection<E>::get_instance(ctx, ".comment", SHT_PROGBITS,
-                                   SHF_MERGE | SHF_STRINGS);
+                                   SHF_MERGE | SHF_STRINGS, 1);
 
   std::string_view buf = save_string(ctx, str);
   std::string_view data(buf.data(), buf.size() + 1);
@@ -982,8 +1034,8 @@ void check_symbol_types(Context<E> &ctx) {
       const ElfSym<E> &esym1 = sym.esym();
       const ElfSym<E> &esym2 = file->elf_syms[i];
 
-      u32 ty1 = (esym1.st_type == STT_GNU_IFUNC) ? STT_FUNC : esym1.st_type;
-      u32 ty2 = (esym2.st_type == STT_GNU_IFUNC) ? STT_FUNC : esym2.st_type;
+      u32 ty1 = (esym1.st_type == STT_GNU_IFUNC) ? (u32)STT_FUNC : esym1.st_type;
+      u32 ty2 = (esym2.st_type == STT_GNU_IFUNC) ? (u32)STT_FUNC : esym2.st_type;
 
       if (ty1 != STT_NOTYPE && ty2 != STT_NOTYPE && ty1 != ty2)
         Warn(ctx) << "symbol type mismatch: " << sym << '\n'
@@ -1198,12 +1250,15 @@ void compute_section_sizes(Context<E> &ctx) {
   // inserting thunks. This pass cannot be parallelized. That is,
   // create_range_extension_thunks is parallelized internally, but the
   // function itself is not thread-safe.
-  if constexpr (needs_thunk<E>)
+  if constexpr (needs_thunk<E>) {
+    Timer t2(ctx, "create_range_extension_thunks");
+
     if (!ctx.arg.relocatable)
       for (Chunk<E> *chunk : ctx.chunks)
         if (OutputSection<E> *osec = chunk->to_osec())
           if (osec->shdr.sh_flags & SHF_EXECINSTR)
             osec->create_range_extension_thunks(ctx);
+  }
 }
 
 // Find all unresolved symbols and attach them to the most appropriate files.
@@ -1480,9 +1535,11 @@ template <typename E>
 void create_output_symtab(Context<E> &ctx) {
   Timer t(ctx, "compute_symtab_size");
 
-  tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
-    chunk->compute_symtab_size(ctx);
-  });
+  if (!ctx.arg.strip_all && !ctx.arg.retain_symbols_file) {
+    tbb::parallel_for_each(ctx.chunks, [&](Chunk<E> *chunk) {
+      chunk->compute_symtab_size(ctx);
+    });
+  }
 
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
     file->compute_symtab_size(ctx);
@@ -1672,12 +1729,85 @@ void compute_import_export(Context<E> &ctx) {
   });
 }
 
+// Compute the "address-taken" bit for each input section.
+//
+// As a space-saving optimization, we want to merge two read-only objects
+// into a single object if their contents are equivalent. That
+// optimization is called the Identical Code Folding or ICF.
+//
+// A catch is that comparing object contents is not enough to determine if
+// two objects can be merged safely; we need to take care of pointer
+// equivalence.
+//
+// In C/C++, two pointers are equivalent if and only if they are taken for
+// the same object. Merging two objects into a single object can break
+// this assumption because two distinctive pointers would become
+// equivalent as a result of merging. We can still merge one object with
+// another if no pointer to the object was taken in code, because without
+// a pointer, comparing its address becomes moot.
+//
+// In mold, each input section has an "address-taken" bit. If there is a
+// pointer-taking reference to the object, it's set to true. At the ICF
+// stage, we merge only objects whose addresses were not taken.
+//
+// For functions, address-taking relocations are separated from
+// non-address-taking ones. For example, x86-64 uses R_X86_64_PLT32 for
+// direct function calls (e.g., "call foo" to call the function foo) while
+// R_X86_64_PC32 or R_X86_64_GOT32 are used for pointer-taking operations.
+//
+// Unfortunately, for data, we can't distinguish between address-taking
+// relocations and non-address-taking ones. LLVM generates an "address
+// significance" table in the ".llvm_addrsig" section to mark symbols
+// whose addresses are taken in code. If that table is available, we use
+// that information in this function. Otherwise, we conservatively assume
+// that all data items are address-taken.
 template <typename E>
-void mark_addrsig(Context<E> &ctx) {
-  Timer t(ctx, "mark_addrsig");
+void compute_address_significance(Context<E> &ctx) {
+  Timer t(ctx, "compute_address_significance");
 
+  // Flip address-taken bit for executable sections first.
   tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    file->mark_addrsig(ctx);
+    for (std::unique_ptr<InputSection<E>> &src : file->sections)
+      if (src && src->is_alive && (src->shdr().sh_flags & SHF_ALLOC))
+        for (const ElfRel<E> &r : src->get_rels(ctx))
+          if (!is_func_call_rel(r))
+            if (InputSection<E> *dst = file->symbols[r.r_sym]->get_input_section())
+              if (dst->shdr().sh_flags & SHF_EXECINSTR)
+                dst->address_taken = true;
+  });
+
+  auto mark = [](Symbol<E> *sym) {
+    if (sym)
+      if (InputSection<E> *isec = sym->get_input_section())
+        isec->address_taken = true;
+  };
+
+  // Some symbols' pointer values are leaked to the dynamic section.
+  mark(get_symbol(ctx, ctx.arg.entry));
+  mark(get_symbol(ctx, ctx.arg.init));
+  mark(get_symbol(ctx, ctx.arg.fini));
+
+  // Exported symbols are conservatively considered address-taken.
+  if (ctx.dynsym)
+    for (Symbol<E> *sym : ctx.dynsym->symbols)
+      if (sym->is_exported)
+        mark(sym);
+
+  // Handle data objects.
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    if (InputSection<E> *sec = file->llvm_addrsig.get()) {
+      u8 *p = (u8 *)sec->contents.data();
+      u8 *end = p + sec->contents.size();
+      while (p != end) {
+        Symbol<E> *sym = file->symbols[read_uleb(&p)];
+        if (InputSection<E> *isec = sym->get_input_section())
+          isec->address_taken = true;
+      }
+    } else {
+      for (std::unique_ptr<InputSection<E>> &isec : file->sections)
+        if (isec && !(isec->shdr().sh_flags & SHF_EXECINSTR))
+          isec->address_taken = true;
+    }
   });
 }
 
@@ -1729,6 +1859,7 @@ void clear_padding(Context<E> &ctx) {
 //   <writable non-RELRO bss>
 //   <non-memory-allocated sections>
 //   <section header>
+//   .gdb_index
 //
 // .interp and some other linker-synthesized sections are placed at the
 // beginning of a file because they are needed by loader. Especially on
@@ -1740,6 +1871,10 @@ void clear_padding(Context<E> &ctx) {
 // particular, if .note.gnu.build-id is in a truncated core file, you
 // can at least identify which executable has crashed.
 //
+// .gdb_index cannot be constructed before applying relocations to
+// other debug sections, so we create it after completing other part
+// of the output file and append it to the very end of the file.
+//
 // A PT_NOTE segment will contain multiple .note sections if exists,
 // but there's no way to represent a gap between .note sections.
 // Therefore, we sort .note sections by decreasing alignment
@@ -1785,6 +1920,8 @@ void sort_output_sections_regular(Context<E> &ctx) {
     if (chunk == ctx.relplt)
       return 11;
     if (chunk == ctx.shdr)
+      return INT32_MAX - 1;
+    if (chunk == ctx.gdb_index)
       return INT32_MAX;
 
     bool alloc = (flags & SHF_ALLOC);
@@ -1800,23 +1937,31 @@ void sort_output_sections_regular(Context<E> &ctx) {
 
   // Ties are broken by additional rules
   auto get_rank2 = [&](Chunk<E> *chunk) -> i64 {
-    if (chunk->shdr.sh_type == SHT_NOTE)
-      return -chunk->shdr.sh_addralign;
+    ElfShdr<E> &shdr = chunk->shdr;
+    if (shdr.sh_type == SHT_NOTE)
+      return -shdr.sh_addralign;
 
     if (chunk == ctx.got)
-      return 1;
-    if (chunk->name == ".toc")
       return 2;
-    if (chunk->name == ".alpha_got")
+    if (chunk->name == ".toc")
       return 3;
+    if (chunk->name == ".alpha_got")
+      return 4;
+
+    if (shdr.sh_flags & SHF_MERGE) {
+      if (shdr.sh_flags & SHF_STRINGS)
+        return (5LL << 32) | shdr.sh_entsize;
+      return (6LL << 32) | shdr.sh_entsize;
+    }
+
     if (chunk == ctx.relro_padding)
-      return INT_MAX;
+      return INT64_MAX;
     return 0;
   };
 
   sort(ctx.chunks, [&](Chunk<E> *a, Chunk<E> *b) {
-    return std::tuple{get_rank1(a), get_rank2(a)} <
-           std::tuple{get_rank1(b), get_rank2(b)};
+    return std::tuple{get_rank1(a), get_rank2(a), a->name} <
+           std::tuple{get_rank1(b), get_rank2(b), b->name};
   });
 }
 
@@ -2188,8 +2333,9 @@ void compute_section_headers(Context<E> &ctx) {
     chunk->update_shdr(ctx);
 
   // Remove empty chunks.
-  std::erase_if(ctx.chunks, [](Chunk<E> *chunk) {
-    return chunk->kind() != OUTPUT_SECTION && chunk->shdr.sh_size == 0;
+  std::erase_if(ctx.chunks, [&](Chunk<E> *chunk) {
+    return chunk->kind() != OUTPUT_SECTION && chunk != ctx.gdb_index &&
+           chunk->shdr.sh_size == 0;
   });
 
   // Set section indices.
@@ -2592,6 +2738,7 @@ void show_stats(Context<E> &ctx) {
 
 using E = MOLD_TARGET;
 
+template int redo_main(Context<E> &, int, char **);
 template void create_internal_file(Context<E> &);
 template void apply_exclude_libs(Context<E> &);
 template void create_synthetic_sections(Context<E> &);
@@ -2622,7 +2769,7 @@ template void create_output_symtab(Context<E> &);
 template void apply_version_script(Context<E> &);
 template void parse_symbol_version(Context<E> &);
 template void compute_import_export(Context<E> &);
-template void mark_addrsig(Context<E> &);
+template void compute_address_significance(Context<E> &);
 template void clear_padding(Context<E> &);
 template void compute_section_headers(Context<E> &);
 template i64 set_osec_offsets(Context<E> &);

elf/thunks.cc

@@ -68,27 +68,6 @@ static constexpr i64 max_thunk_size = 102400;
 
 static_assert(max_thunk_size / E::thunk_size < INT16_MAX);
 
-// Returns true if a given relocation is of type used for function calls.
-template <typename E>
-static bool needs_thunk_rel(const ElfRel<E> &r) {
-  u32 ty = r.r_type;
-
-  if constexpr (is_arm64<E>) {
-    return ty == R_AARCH64_JUMP26 || ty == R_AARCH64_CALL26;
-  } else if constexpr (is_arm32<E>) {
-    return ty == R_ARM_JUMP24 || ty == R_ARM_THM_JUMP24 ||
-           ty == R_ARM_CALL   || ty == R_ARM_THM_CALL   ||
-           ty == R_ARM_PLT32;
-  } else if constexpr (is_ppc32<E>) {
-    return ty == R_PPC_REL24  || ty == R_PPC_PLTREL24 || ty == R_PPC_LOCAL24PC;
-  } else if constexpr (is_ppc64<E>) {
-    return ty == R_PPC64_REL24 || ty == R_PPC64_REL24_NOTOC;
-  } else {
-    static_assert(is_loongarch<E>);
-    return ty == R_LARCH_B26;
-  }
-}
-
 template <typename E>
 static bool is_reachable(Context<E> &ctx, InputSection<E> &isec,
                          Symbol<E> &sym, const ElfRel<E> &rel) {
@@ -157,7 +136,7 @@ static void scan_rels(Context<E> &ctx, InputSection<E> &isec,
 
   for (i64 i = 0; i < rels.size(); i++) {
     const ElfRel<E> &rel = rels[i];
-    if (!needs_thunk_rel(rel))
+    if (!is_func_call_rel(rel))
       continue;
 
     // Skip if the symbol is undefined. apply_reloc() will report an error.
@@ -191,17 +170,17 @@ static void scan_rels(Context<E> &ctx, InputSection<E> &isec,
 
 template <>
 void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
+  using Thunk = RangeExtensionThunk<E>;
+
   std::span<InputSection<E> *> m = members;
   if (m.empty())
     return;
 
-  m[0]->offset = 0;
-
   // Initialize input sections with a dummy offset so that we can
   // distinguish sections that have got an address with the one who
   // haven't.
-  tbb::parallel_for((i64)1, (i64)m.size(), [&](i64 i) {
-    m[i]->offset = -1;
+  tbb::parallel_for_each(m, [](InputSection<E> *isec) {
+    isec->offset = -1;
   });
 
   // We create thunks from the beginning of the section to the end.
@@ -211,7 +190,7 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   // Input sections between B and C are in the current batch.
   //
   // A is the input section with the smallest address than can reach
-  // anywhere from the current batch.
+  // from the current batch.
   //
   // D is the input section with the largest address such that the thunk
   // is reachable from the current batch if it's inserted right before D.
@@ -219,9 +198,9 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   //  ................................ <input sections> ............
   //     A    B    C    D
   //                    ^ We insert a thunk for the current batch just before D
-  //          <--->       The current batch, which is smaller than batch_size
-  //     <-------->       Smaller than max_distance
-  //          <-------->  Smaller than max_distance
+  //          <--->       The current batch, which is smaller than BATCH_SIZE
+  //     <-------->       Smaller than MAX_DISTANCE
+  //          <-------->  Smaller than MAX_DISTANCE
   //     <------------->  Reachable from the current batch
   i64 a = 0;
   i64 b = 0;
@@ -233,10 +212,14 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
   i64 t = 0;
 
   while (b < m.size()) {
-    // Move D foward as far as we can jump from B to anywhere in a thunk at D.
+    // Move D foward as far as we can jump from B to a thunk at D.
+    auto d_thunk_end = [&] {
+      u64 d_end = align_to(offset, 1 << m[d]->p2align) + m[d]->sh_size;
+      return align_to(d_end, Thunk::alignment) + max_thunk_size;
+    };
+
     while (d < m.size() &&
-           align_to(offset, 1 << m[d]->p2align) + m[d]->sh_size + max_thunk_size <
-           m[b]->offset + max_distance()) {
+           (b == d || d_thunk_end() <= m[b]->offset + max_distance())) {
       offset = align_to(offset, 1 << m[d]->p2align);
       m[d]->offset = offset;
       offset += m[d]->sh_size;
@@ -247,13 +230,12 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
     // to make sure that there's at least one section between B and C
     // to ensure progress.
     c = b + 1;
-    while (c < m.size() &&
-           m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
+    while (c < d && m[c]->offset + m[c]->sh_size < m[b]->offset + batch_size)
       c++;
 
     // Move A forward so that A is reachable from C.
     i64 c_offset = (c == m.size()) ? offset : m[c]->offset;
-    while (a < m.size() && m[a]->offset + max_distance() < c_offset)
+    while (a < b && m[a]->offset + max_distance() < c_offset)
       a++;
 
     // Erase references to out-of-range thunks.
@@ -261,9 +243,9 @@ void OutputSection<E>::create_range_extension_thunks(Context<E> &ctx) {
       reset_thunk(*thunks[t++]);
 
     // Create a new thunk and place it at D.
-    offset = align_to(offset, RangeExtensionThunk<E>::alignment);
+    offset = align_to(offset, Thunk::alignment);
     i64 thunk_idx = thunks.size();
-    RangeExtensionThunk<E> *thunk = new RangeExtensionThunk<E>(*this, offset);
+    Thunk *thunk = new Thunk(*this, offset);
     thunks.emplace_back(thunk);
 
     // Scan relocations between B and C to collect symbols that need

install-build-deps.sh

@@ -36,7 +36,7 @@ gentoo-*)
   emerge-webrsync
   emerge dev-util/cmake
   ;;
-arch-* | archarm-*)
+arch-* | archarm-* | artix-*)
   pacman -Sy
   pacman -S --needed --noconfirm base-devel cmake util-linux
   ;;

test/elf/CMakeLists.txt

@@ -1,84 +1,91 @@
 option(MOLD_ENABLE_QEMU_TESTS "Enable tests on non-native targets" OFF)
 
-function(add_target TRIPLE)
-  set(HOST ${CMAKE_HOST_SYSTEM_PROCESSOR})
+set(HOST ${CMAKE_HOST_SYSTEM_PROCESSOR})
 
-  if(${HOST} MATCHES "amd64")
-    set(HOST x86_64)
-  elseif(${HOST} MATCHES "arm.*")
-    set(HOST arm)
-  elseif(${HOST} STREQUAL "ppc64")
-    set(HOST powerpc64)
-  elseif(${HOST} STREQUAL "ppc64le")
-    set(HOST powerpc64le)
-  endif()
+# CMAKE_HOST_SYSTEM_PROCESSOR returns "aarch64" on ARM32 userland
+# with ARM64 kernel. Here, we run `cc -dumpmachine` to determine the
+# compiler's default target.
+execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpmachine
+  RESULT_VARIABLE EXIT_CODE
+  OUTPUT_VARIABLE ARCH
+  ERROR_QUIET)
+if(NOT EXIT_CODE AND ARCH MATCHES "([^-]+).*")
+  set(HOST ${CMAKE_MATCH_1})
+endif()
 
-  if(${TRIPLE} MATCHES "${HOST}-.*")
+if(${HOST} MATCHES "amd64")
+  set(HOST x86_64)
+elseif(${HOST} MATCHES "i386")
+  set(HOST i686)
+elseif(${HOST} MATCHES "arm.*")
+  set(HOST arm)
+elseif(${HOST} STREQUAL "powerpc64")
+  set(HOST ppc64)
+elseif(${HOST} STREQUAL "powerpc64le")
+  set(HOST ppc64le)
+endif()
+
+function(add_target ARCH TRIPLE)
+  if(${ARCH} STREQUAL ${HOST})
     set(IS_NATIVE 1)
   endif()
 
-  if(${TRIPLE} MATCHES "([^-]+)-.")
-    set(MACHINE ${CMAKE_MATCH_1})
-  endif()
-
   if(IS_NATIVE OR MOLD_ENABLE_QEMU_TESTS)
-    file(GLOB ALL_TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
-      "*.sh")
+    file(GLOB ALL_TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS "*.sh")
 
     list(FILTER ALL_TESTS EXCLUDE REGEX "_")
 
-    file(GLOB TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS
-      "${MACHINE}_*.sh")
+    file(GLOB TESTS RELATIVE ${CMAKE_CURRENT_LIST_DIR} CONFIGURE_DEPENDS "${ARCH}_*.sh")
 
     list(APPEND TESTS ${ALL_TESTS})
 
     foreach(TEST IN LISTS TESTS)
-      string(REGEX REPLACE "\\.sh$" "" TESTNAME "${MACHINE}-${TEST}")
+      string(REGEX REPLACE "\\.sh$" "" TESTNAME "${ARCH}-${TEST}")
 
       add_test(NAME ${TESTNAME}
         COMMAND bash -x ${CMAKE_CURRENT_LIST_DIR}/${TEST}
         WORKING_DIRECTORY ${mold_BINARY_DIR})
 
+      set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT "HOST=${HOST}")
+
       if(IS_NATIVE)
-        set_tests_properties(${TESTNAME} PROPERTIES
-          SKIP_REGULAR_EXPRESSION "skipped")
+        set_property(TEST ${TESTNAME} APPEND PROPERTY SKIP_REGULAR_EXPRESSION "skipped")
       else()
-        set_tests_properties(${TESTNAME} PROPERTIES
-          ENVIRONMENT "TRIPLE=${TRIPLE}")
+        set_property(TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT "TRIPLE=${TRIPLE}")
       endif()
     endforeach()
   endif()
 endfunction()
 
-add_target(x86_64-linux-gnu)
-add_target(i686-linux-gnu)
-add_target(aarch64-linux-gnu)
-add_target(arm-linux-gnueabihf)
-add_target(riscv64-linux-gnu)
-add_target(powerpc-linux-gnu)
-add_target(powerpc64-linux-gnu)
-add_target(powerpc64le-linux-gnu)
-add_target(sparc64-linux-gnu)
-add_target(s390x-linux-gnu)
-add_target(sh4-linux-gnu)
-add_target(alpha-linux-gnu)
+add_target(x86_64 x86_64-linux-gnu)
+add_target(i686 i686-linux-gnu)
+add_target(aarch64 aarch64-linux-gnu)
+add_target(arm arm-linux-gnueabihf)
+add_target(riscv64 riscv64-linux-gnu)
+add_target(ppc powerpc-linux-gnu)
+add_target(ppc64 powerpc64-linux-gnu)
+add_target(ppc64le powerpc64le-linux-gnu)
+add_target(sparc64 sparc64-linux-gnu)
+add_target(s390x s390x-linux-gnu)
+add_target(sh4 sh4-linux-gnu)
+add_target(alpha alpha-linux-gnu)
 
 option(MOLD_ENABLE_QEMU_TESTS_RV32 "Enable tests for RV32" OFF)
 if(MOLD_ENABLE_QEMU_TESTS_RV32)
-  add_target(riscv32-linux-gnu)
+  add_target(riscv32 riscv32-linux-gnu)
 endif()
 
 option(MOLD_ENABLE_QEMU_TESTS_POWER10 "Enable tests for Power10" OFF)
 if(MOLD_ENABLE_QEMU_TESTS_POWER10)
-  add_target(powerpc64le_power10-linux-gnu)
+  add_target(ppc64le powerpc64le_power10-linux-gnu)
 endif()
 
 option(MOLD_ENABLE_QEMU_TESTS_M68K "Enable tests for m68k" OFF)
 if(MOLD_ENABLE_QEMU_TESTS_M68K)
-  add_target(m68k-linux-gnu)
+  add_target(m68k m68k-linux-gnu)
 endif()
 
 option(MOLD_ENABLE_QEMU_TESTS_LOONGARCH "Enable tests for LoongArch" OFF)
-if(MOLD_ENABLE_QEMU_TESTS_LOONGARCH)
-  add_target(loongarch64-linux-gnu)
+if(MOLD_ENABLE_QEMU_TESTS_LOONGARCH OR ${HOST} STREQUAL loongarch64)
+  add_target(loongarch64 loongarch64-linux-gnu)
 endif()

test/elf/aarch64_variant-pcs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = aarch64 ] || skip
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -xassembler - 2> /dev/null || skip
+.global foo
+.type foo, %function
+.variant_pcs foo
+foo:
+  ret
+EOF
+
+$CC -B. -shared -o $t/b.so $t/a.o
+readelf -W --dyn-syms $t/b.so | grep foo | grep -q '[VARIANT_PCS]'
+
+cat <<EOF | $CC -c -o $t/c.o -xc -
+void foo();
+int main() { foo(); }
+EOF
+
+$CC -B. -o $t/exe $t/c.o $t/b.so
+readelf -W --dynamic $t/exe | grep -q AARCH64_VARIANT_PCS

test/elf/arm_tlsdesc.sh

@@ -2,6 +2,7 @@
 . $(dirname $0)/common.inc
 
 [ $MACHINE = arm ] || skip
+is_musl && skip
 
 echo 'int main() {}' | $GCC -c -o /dev/null -xc - -O0 -mthumb >& /dev/null \
   || skip

test/elf/common.inc

@@ -14,10 +14,12 @@ canonical_name() {
   esac
 }
 
-HOST=$(canonical_name $(uname -m))
+if [ -z "$HOST" ]; then
+  HOST=$(canonical_name $(uname -m))
+fi
 
 # Set tool names
-if [ "$TRIPLE" = "" ]; then
+if [ -z "$TRIPLE" ]; then
   MACHINE=$HOST
   TESTDIR=out/test/elf/$HOST
   CC="${TEST_CC:-cc}"
@@ -59,6 +61,13 @@ elif [ $MACHINE = aarch64 ]; then
   tlsdesc_opt=-mtls-dialect=desc
 fi
 
+if [ $MACHINE = ppc64le -a -e /proc/cpuinfo ] && grep -q POWER10 /proc/cpuinfo; then
+  CC="$CC -mcpu=power10"
+  CXX="$CXX -mcpu=power10"
+  GCC="$GCC -mcpu=power10"
+  GXX="$GXX -mcpu=power10"
+fi
+
 # Common functions
 test_cflags() {
   echo 'int main() {}' | $CC "$@" -o /dev/null -xc - >& /dev/null
@@ -69,7 +78,14 @@ supports_ifunc() {
     $CC -c -o /dev/null -xc - >& /dev/null
 }
 
+is_musl() {
+  ldd --help 2>&1 | grep -q musl
+}
+
 supports_tlsdesc() {
+  # musl's tlsdesc on arm32 seems to be broken
+  [ $MACHINE = arm ] && is_musl && return 1
+
   [ -n "$tlsdesc_opt" ]
 }
 

test/elf/default-symver.sh

@@ -6,8 +6,8 @@ void foo() {}
 EOF
 
 $CC -B. -o $t/b.so -shared $t/a.o -Wl,-default-symver
-readelf --dyn-syms $t/b.so | grep -q ' foo@@b\.so$'
+readelf --dyn-syms $t/b.so | grep -q ' foo@@b\.so'
 
 $CC -B. -o $t/b.so -shared $t/a.o \
   -Wl,--soname=bar -Wl,-default-symver
-readelf --dyn-syms $t/b.so | grep -q ' foo@@bar$'
+readelf --dyn-syms $t/b.so | grep -q ' foo@@bar'

test/elf/dt-init.sh

@@ -6,7 +6,7 @@
 
 # musl libc does not support init/fini on ARM
 # https://github.com/rui314/mold/issues/951
-[ $MACHINE = arm -o $MACHINE = aarch64 ] && ldd --help 2>&1 | grep -q musl && skip
+[ $MACHINE = arm -o $MACHINE = aarch64 ] && is_musl && skip
 
 cat <<EOF | $CC -c -fPIC -o $t/a.o -xc -
 void keep();

test/elf/duplicate-error-archive.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc -
+void foo() {}
+EOF
+
+rm -f $t/b.a
+ar rcs $t/b.a $t/a.o
+
+cat <<EOF | $CC -o $t/c.o -c -xc -
+void foo();
+int main() { foo(); }
+EOF
+
+$CC -B. -o $t/exe $t/c.o $t/b.a $t/b.a
+
+! $CC -B. -o $t/exe $t/c.o -Wl,--push-state,--whole-archive \
+  $t/b.a $t/b.a -Wl,--pop-state 2> $t/log || false
+
+grep -q 'duplicate symbol:.* foo' $t/log

test/elf/dynamic-list.sh

@@ -10,8 +10,8 @@ EOF
 $CC -B. -o $t/exe $t/a.o
 
 readelf --dyn-syms $t/exe > $t/log
-! grep -q ' foo$' $t/log || false
-! grep -q ' bar$' $t/log || false
+! grep -q ' foo' $t/log || false
+! grep -q ' bar' $t/log || false
 
 cat <<EOF > $t/dyn
 { foo; bar; };
@@ -20,17 +20,17 @@ EOF
 $CC -B. -o $t/exe1 $t/a.o -Wl,-dynamic-list=$t/dyn
 
 readelf --dyn-syms $t/exe1 > $t/log1
-grep -q ' foo$' $t/log1
-grep -q ' bar$' $t/log1
+grep -q ' foo' $t/log1
+grep -q ' bar' $t/log1
 
 $CC -B. -o $t/exe2 $t/a.o -Wl,--export-dynamic-symbol-list=$t/dyn
 
 readelf --dyn-syms $t/exe2 > $t/log2
-grep -q ' foo$' $t/log2
-grep -q ' bar$' $t/log2
+grep -q ' foo' $t/log2
+grep -q ' bar' $t/log2
 
 $CC -B. -o $t/exe3 $t/a.o -Wl,--export-dynamic-symbol=foo,--export-dynamic-symbol=bar
 
 readelf --dyn-syms $t/exe3 > $t/log3
-grep -q ' foo$' $t/log3
-grep -q ' bar$' $t/log3
+grep -q ' foo' $t/log3
+grep -q ' bar' $t/log3

test/elf/dynamic-list2.sh

@@ -14,8 +14,8 @@ EOF
 $CXX -B. -o $t/exe $t/a.o $t/b.o
 
 readelf --dyn-syms $t/exe > $t/log
-! grep -q ' foo$' $t/log || false
-! grep -q ' bar$' $t/log || false
+! grep -q ' foo' $t/log || false
+! grep -q ' bar' $t/log || false
 
 cat <<EOF > $t/dyn
 { foo; extern "C++" { "baz(int)"; }; };
@@ -24,13 +24,13 @@ EOF
 $CC -B. -o $t/exe1 $t/a.o $t/b.o -Wl,-dynamic-list=$t/dyn
 
 readelf --dyn-syms $t/exe1 > $t/log1
-grep -q ' foo$' $t/log1
-! grep -q ' bar$' $t/log1 || false
-grep -q ' _Z3bazi$' $t/log1
+grep -q ' foo' $t/log1
+! grep -q ' bar' $t/log1 || false
+grep -q ' _Z3bazi' $t/log1
 
 $CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,--export-dynamic-symbol-list=$t/dyn
 
 readelf --dyn-syms $t/exe2 > $t/log2
-grep -q ' foo$' $t/log2
-! grep -q ' bar$' $t/log2 || false
-grep -q ' _Z3bazi$' $t/log2
+grep -q ' foo' $t/log2
+! grep -q ' bar' $t/log2 || false
+grep -q ' _Z3bazi' $t/log2

test/elf/dynamic-list3.sh

@@ -21,19 +21,19 @@ EOF
 $CC -B. -Wl,--dynamic-list=$t/dyn -o $t/exe1 $t/b.o
 
 readelf --dyn-syms $t/exe1 > $t/log1
-grep -q ' xyz$' $t/log1
-! grep -q ' foobarzx$' $t/log1 || false
-grep -q ' foobarcx$' $t/log1
-grep -q ' foo123bar456bx$' $t/log1
-! grep -q ' foo123bar456c$' $t/log1 || false
-! grep -q ' foo123bar456x$' $t/log1 || false
+grep -q ' xyz' $t/log1
+! grep -q ' foobarzx' $t/log1 || false
+grep -q ' foobarcx' $t/log1
+grep -q ' foo123bar456bx' $t/log1
+! grep -q ' foo123bar456c' $t/log1 || false
+! grep -q ' foo123bar456x' $t/log1 || false
 
 $CC -B. -Wl,--export-dynamic-symbol-list=$t/dyn -o $t/exe2 $t/b.o
 
 readelf --dyn-syms $t/exe2 > $t/log2
-grep -q ' xyz$' $t/log2
-! grep -q ' foobarzx$' $t/log2 || false
-grep -q ' foobarcx$' $t/log2
-grep -q ' foo123bar456bx$' $t/log2
-! grep -q ' foo123bar456c$' $t/log2 || false
-! grep -q ' foo123bar456x$' $t/log2 || false
+grep -q ' xyz' $t/log2
+! grep -q ' foobarzx' $t/log2 || false
+grep -q ' foobarcx' $t/log2
+grep -q ' foo123bar456bx' $t/log2
+! grep -q ' foo123bar456c' $t/log2 || false
+! grep -q ' foo123bar456x' $t/log2 || false

test/elf/empty-version.sh

@@ -11,5 +11,5 @@ EOF
 
 $CC -B. -shared -o $t/b.so $t/a.o
 
-readelf --dyn-syms $t/b.so | grep -q 'bar1$'
-readelf --dyn-syms $t/b.so | grep -q 'bar2$'
+readelf --dyn-syms $t/b.so | grep -q ' bar1'
+readelf --dyn-syms $t/b.so | grep -q ' bar2'

test/elf/gdb-index-dwarf64.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = $HOST ] || skip
+[ $MACHINE = riscv64 -o $MACHINE = riscv32 -o $MACHINE = sparc64 ] && skip
+
+command -v gdb >& /dev/null || skip
+
+test_cflags -gdwarf-5 -g -gdwarf64 || skip
+
+cat <<EOF > $t/a.c
+void fn3();
+
+static void fn2() {
+  fn3();
+}
+
+void fn1() {
+  fn2();
+}
+EOF
+
+cat <<EOF > $t/b.c
+void fn5();
+
+static void fn4() {
+  fn5();
+}
+
+void fn3() {
+  fn4();
+}
+EOF
+
+cat <<EOF > $t/c.c
+void fn7();
+
+static void fn6() {
+  fn7();
+}
+
+void fn5() {
+  fn6();
+}
+EOF
+
+cat <<EOF > $t/d.c
+#include <stdio.h>
+void trap() {}
+
+static void fn8() {
+  printf("Hello world\n");
+  trap();
+}
+
+void fn7() {
+  fn8();
+}
+EOF
+
+$CC -c -o $t/a.o $t/a.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -ffunction-sections
+$CC -c -o $t/b.o $t/b.c -fPIC -g -ggnu-pubnames -gdwarf-4 -gdwarf64 -ffunction-sections
+$CC -c -o $t/c.o $t/c.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64
+$CC -c -o $t/d.o $t/d.c -fPIC -g -ggnu-pubnames -gdwarf-5 -gdwarf64 -ffunction-sections
+
+$CC -B. -shared -o $t/e.so $t/a.o $t/b.o $t/c.o $t/d.o
+
+$CC -B. -shared -o $t/f.so $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--gdb-index
+readelf -WS $t/f.so 2> /dev/null | grep -Fq .gdb_index
+
+cat <<EOF | $CC -c -o $t/g.o -fPIC -g -ggnu-pubnames -gdwarf-5 -xc - -gz
+void fn1();
+
+int main() {
+  fn1();
+}
+EOF
+
+# Older versions of gdb are buggy that they complain DWARF64 debug sections
+# even without .gdb_index. Skip if such version.
+$CC -B. -o $t/exe1 $t/e.so $t/g.o
+DEBUGINFOD_URLS= gdb $t/exe1 -nx -batch -ex 'b main' -ex r -ex quit >& $t/log1
+grep -q 'DW_FORM_line_strp pointing outside of .debug_line_str' $t/log1 && skip
+
+# We are using a recent version of gdb.
+$CC -B. -o $t/exe2 $t/f.so $t/g.o -Wl,--gdb-index
+readelf -WS $t/exe2 2> /dev/null | grep -Fq .gdb_index
+
+$QEMU $t/exe2 | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe2 -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log2
+
+grep -q 'fn8 () at .*/d.c:6' $t/log2
+grep -q 'fn7 () at .*/d.c:10' $t/log2
+grep -q 'fn6 () at .*/c.c:4' $t/log2
+grep -q 'fn5 () at .*/c.c:8' $t/log2
+grep -q 'fn4 () at .*/b.c:4' $t/log2
+grep -q 'fn3 () at .*/b.c:8' $t/log2
+grep -q 'fn2 () at .*/a.c:4' $t/log2
+grep -q 'fn1 () at .*/a.c:8' $t/log2

test/elf/icf-safe.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+# GCC 11 or older uses R_390_PLT32DBL to take an address of a function
+# instead of R_390_PC32DBL
+if [ $MACHINE = s390x ]; then
+  echo 'void *foo() { return foo; }' | $CC -c -o $t/a.o -xc -
+  readelf -r $t/a.o | grep -q R_390_PLT32DBL && skip
+fi
+
+cat <<EOF | $CC -c -o $t/a.o -ffunction-sections -fdata-sections -xc -
+int bar() {
+  return 5;
+}
+
+int foo1(int x) {
+  return bar() + x;
+}
+
+int foo2(int x) {
+  return bar() + x;
+}
+
+int foo3(int x) {
+  return bar() + x;
+}
+EOF
+
+cat <<EOF | $CC -c -o $t/b.o -ffunction-sections -fdata-sections -xc -
+#include <stdio.h>
+
+int foo1();
+int foo2();
+int foo3();
+
+int main() {
+  printf("%d %d\n", foo1 == foo2, foo2 == foo3);
+}
+EOF
+
+$CC -B. -o $t/exe1 -Wl,-icf=safe $t/a.o $t/b.o
+$QEMU $t/exe1 | grep -q '^0 0$'
+
+cat <<EOF | $CC -c -o $t/c.o -ffunction-sections -fdata-sections -xc -
+int foo1();
+int foo2();
+int foo3();
+
+int main() {
+  foo1();
+  foo2();
+  foo3();
+}
+EOF
+
+$CC -B. -o $t/exe2 -Wl,-icf=safe $t/a.o $t/c.o
+$QEMU $t/exe2 > $t/log2
+! grep foo2 $t/log2 || false
+

test/elf/ifunc-address-equality-exported.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+supports_ifunc || skip
+
+cat <<EOF | $CC -c -fPIC -o $t/a.o -xc -
+typedef void Func();
+void foo();
+Func *get_foo() { return foo; }
+EOF
+
+$CC -B. -shared -o $t/b.so $t/a.o
+
+cat <<EOF | $CC -c -fno-PIE -o $t/c.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+__attribute__((ifunc("resolve_foo"))) void foo(void);
+void real_foo(void) { printf("foo "); }
+Func *resolve_foo() { return real_foo; }
+
+Func *get_foo();
+
+int main() {
+  printf("%p %p\n", foo, get_foo());
+  foo();
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe1 $t/c.o $t/b.so -no-pie
+$QEMU $t/exe1 | grep -Eq '^(\S+) \1'

test/elf/ifunc-address-equality.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+supports_ifunc || skip
+
+cat <<EOF | $CC -c -fno-PIE -o $t/a.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+__attribute__((ifunc("resolve_foo"))) void foo(void);
+void real_foo(void) { printf("foo "); }
+Func *resolve_foo() { return real_foo; }
+
+__attribute__((ifunc("resolve_bar"))) void bar(void);
+void real_bar(void) { printf("bar "); }
+Func *resolve_bar() { return real_bar; }
+EOF
+
+cat <<EOF | $CC -c -fPIC -o $t/b.o -xc -
+typedef void Func();
+
+void foo();
+void bar();
+
+Func *get_foo() { return foo; }
+Func *get_bar() { return bar; }
+EOF
+
+cat <<EOF | $CC -c -fno-PIE -o $t/c.o -xc -
+#include <stdio.h>
+
+typedef void Func();
+
+void foo();
+void bar();
+Func *get_foo();
+Func *get_bar();
+
+int main() {
+  printf("%p %p %p %p\n", foo, get_foo(), bar, get_bar());
+  foo();
+  bar();
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe1 $t/a.o $t/b.o $t/c.o -no-pie
+$QEMU $t/exe1 | grep -Eq '^(\S+) \1 (\S+) \2'
+
+readelf --dynamic $t/exe1 > $t/log1
+! grep -q TEXTREL $t/log1 || false

test/elf/init-array-priorities.sh

@@ -2,7 +2,7 @@
 . $(dirname $0)/common.inc
 
 # musl does not support GNU-style init/fini priorities
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 cat <<'EOF' | $CC -c -o $t/a.o -xc -
 #include <stdio.h>

test/elf/loongarch64_mcmodel-extreme.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -mcmodel=extreme
+#include <stdio.h>
+char msg[] = "Hello world\n";
+int main() { printf(msg); }
+EOF
+
+$CC -B. -o $t/exe1 $t/a.o
+$QEMU $t/exe1 | grep -q 'Hello world'

test/elf/lto-nostdlib.sh

@@ -6,4 +6,4 @@ void _start() {}
 EOF
 
 $CC -B. -o $t/exe -flto $t/a.o -nostdlib
-readelf -Ws $t/exe | grep -Eq ' _start$'
+readelf -Ws $t/exe | grep -Eq ' _start'

test/elf/range-extension-thunk2.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections
+#include <stdio.h>
+
+void f0();
+void f1();
+void f2();
+void f3();
+void f4();
+void f5();
+void f6();
+void f7();
+void f8();
+void f9();
+void f10();
+void f11();
+void f12();
+void f13();
+void f14();
+void f15();
+void f16();
+void f17();
+void f18();
+void f19();
+
+void f0(int x) { printf("0 "); if (!x) f9(); }
+void space0() { __asm__(".space 1024*1024"); }
+
+void f1(int x) { printf("1 "); f8(x); }
+void space1() { __asm__(".space 1024*1024"); }
+
+void f2(int x) { printf("2 "); f7(x); }
+void space2() { __asm__(".space 1024*1024"); }
+
+void f3(int x) { printf("3 "); f6(x); }
+void space3() { __asm__(".space 1024*1024"); }
+
+void f4(int x) { printf("4 "); f5(x); }
+void space4() { __asm__(".space 1024*1024"); }
+
+void f5(int x) { printf("5 "); f10(x); }
+void space5() { __asm__(".space 1024*1024"); }
+
+void f6(int x) { printf("6 "); f4(x); }
+void space6() { __asm__(".space 1024*1024"); }
+
+void f7(int x) { printf("7 "); f3(x); }
+void space7() { __asm__(".space 1024*1024"); }
+
+void f8(int x) { printf("8 "); f2(x); }
+void space8() { __asm__(".space 1024*1024"); }
+
+void f9(int x) { printf("9 "); f1(x); }
+void space9() { __asm__(".space 1024*1024"); }
+
+void f10(int x) { printf("10 "); f19(x); }
+void space10() { __asm__(".space 8*1024*1024"); }
+
+void f11(int x) { printf("11 "); f18(x); }
+void space11() { __asm__(".space 8*1024*1024"); }
+
+void f12(int x) { printf("12 "); f17(x); }
+void space12() { __asm__(".space 8*1024*1024"); }
+
+void f13(int x) { printf("13 "); f16(x); }
+void space13() { __asm__(".space 8*1024*1024"); }
+
+void f14(int x) { printf("14 "); f15(x); }
+void space14() { __asm__(".space 8*1024*1024"); }
+
+void f15(int x) { printf("15 "); f0(x + 1); }
+void space15() { __asm__(".space 8*1024*1024"); }
+
+void f16(int x) { printf("16 "); f14(x); }
+void space16() { __asm__(".space 8*1024*1024"); }
+
+void f17(int x) { printf("17 "); f13(x); }
+void space17() { __asm__(".space 8*1024*1024"); }
+
+void f18(int x) { printf("18 "); f12(x); }
+void space18() { __asm__(".space 8*1024*1024"); }
+
+void f19(int x) { printf("19 "); f11(x); }
+void space19() { __asm__(".space 8*1024*1024"); }
+
+int main() {
+  f0(0);
+  printf("\n");
+}
+EOF
+
+$CC -B. -o $t/exe $t/a.o
+$QEMU $t/exe | grep -Eq '^0 9 1 8 2 7 3 6 4 5 10 19 11 18 12 17 13 16 14 15 0 $'

test/elf/relocatable-mergeable-sections.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+# OneTBB isn't tsan-clean
+nm mold | grep -q '__tsan_init' && skip
+
+cat <<EOF | $CC -c -o $t/a.o -xassembler -
+.section .rodata.str1.1,"aMS",@progbits,1
+val1:
+.ascii "Hello \0"
+
+.section .rodata.str1.1,"aMS",@progbits,1
+val5:
+.ascii "World \0"
+EOF
+
+./mold --relocatable -o $t/b.o $t/a.o
+
+readelf -W -p .rodata.str1.1 $t/b.o | grep -Eq '\b0\b.*Hello'
+readelf -W -p .rodata.str1.1 $t/b.o | grep -Eq '\b7\b.*World'

test/elf/response-file.sh

@@ -17,4 +17,4 @@ EOF
 
 echo "'$t/b.o' '$t/c.o'" > $t/rsp
 
-$CC -o $t/exe $t/a.o -Wl,@$t/rsp
+$CC -B. -o $t/exe $t/a.o -Wl,@$t/rsp

test/elf/response-file2.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -c -o $t/a.o -xc -
+void foo();
+int main() { foo(); }
+EOF
+
+cat <<EOF | $CC -c -o $t/b.o -xc -
+void foo() {}
+EOF
+
+echo "'$t/b.o'" > $t/rsp1
+echo "@$t/rsp1" > $t/rsp2
+
+$CC -B. -o $t/exe $t/a.o -Wl,@$t/rsp2

test/elf/rodata-name.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-. $(dirname $0)/common.inc
-
-# ARM assembler has a differnet grammar than the others.
-# Concretely speaking, ARM as uses "@" as a start of a comment.
-[ $MACHINE = arm ] && skip
-
-cat <<'EOF' | $CC -c -o $t/a.o -x assembler -
-.globl val1, val2, val3, val4, val5
-
-.section .rodata.str1.1,"aMS",@progbits,1
-val1:
-.ascii "Hello \0"
-
-.section .rodata.str4.4,"aMS",@progbits,4
-.align 4
-val2:
-.ascii "world   \0\0\0\0"
-
-.section .rodata.foo,"aMS",@progbits,1
-val3:
-.ascii "foobar\0"
-
-.section .rodata.cst8,"aM",@progbits,8
-.align 8
-val4:
-.ascii "abcdefgh"
-
-.section .rodatabar,"aMS",@progbits,1
-val5:
-.ascii "bar\0"
-EOF
-
-cat <<'EOF' | $CC -c -o $t/b.o -xc -
-#include <stdio.h>
-
-extern char val1, val2, val3, val4, val5;
-
-int main() {
-  printf("%p %p %p %p %p\n", &val1, &val2, &val3, &val4, &val5);
-}
-EOF
-
-$CC -B. -o $t/exe $t/a.o $t/b.o
-
-readelf -p .rodata.str $t/exe | grep -q Hello
-readelf -p .rodata.str $t/exe | grep -q world
-readelf -p .rodata.str $t/exe | grep -q foobar
-readelf -p .rodata.cst $t/exe | grep -q abcdefgh
-readelf -p .rodatabar $t/exe | grep -q bar

test/elf/run.sh

@@ -4,7 +4,7 @@
 [ "$CC" = cc ] || skip
 
 # ASAN doesn't work with LD_PRELOAD
-nm mold-wrapper.so | grep -q '__[at]san_init' && skip
+nm mold | grep -q '__[at]san_init' && skip
 
 cat <<'EOF' | $CC -xc -c -o $t/a.o -
 #include <stdio.h>

test/elf/strip-debug.sh

@@ -12,4 +12,4 @@ $CC -B. -o $t/exe $t/a.o -Wl,--strip-debug
 readelf -W --sections $t/exe > $t/log
 ! grep -Fq .debug_info $t/log || false
 
-readelf -W --symbols $t/exe | grep -q ' bar$'
+readelf -W --symbols $t/exe | grep -q ' bar'

test/elf/symbol-version2.sh

@@ -16,7 +16,7 @@ EOF
 $CC -B. -o $t/c.so -shared $t/a.o -Wl,--version-script=$t/b.version
 readelf -W --dyn-syms $t/c.so > $t/log
 
-grep -q ' foo@TEST$' $t/log
-grep -q ' bar@TEST$' $t/log
-grep -q ' bar1$' $t/log
-! grep -q ' foo@@TEST$' $t/log || false
+grep -q ' foo@TEST' $t/log
+grep -q ' bar@TEST' $t/log
+grep -q ' bar1' $t/log
+! grep -q ' foo@@TEST' $t/log || false

test/elf/symbol-version3.sh

@@ -19,7 +19,7 @@ EOF
 $CC -B. -o $t/c.so -shared $t/a.o -Wl,--version-script=$t/b.version
 readelf -W --dyn-syms $t/c.so > $t/log
 
-grep -q ' foo@@TEST1$' $t/log
-grep -q ' foo@TEST2$' $t/log
-grep -q ' foo@TEST3$' $t/log
+grep -q ' foo@@TEST1' $t/log
+grep -q ' foo@TEST2' $t/log
+grep -q ' foo@TEST3' $t/log
 ! grep -q ' foo$' $t/log || false

test/elf/version-script11.sh

@@ -14,6 +14,6 @@ EOF
 $CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 readelf --dyn-syms $t/c.so > $t/log
-grep -q 'foo@@VER_X1$' $t/log
-! grep -q ' bar$' $t/log || false
-! grep -q ' baz$' $t/log || false
+grep -q 'foo@@VER_X1' $t/log
+! grep -q ' bar' $t/log || false
+! grep -q ' baz' $t/log || false

test/elf/version-script12.sh

@@ -20,6 +20,6 @@ EOF
 $CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 readelf --dyn-syms $t/c.so > $t/log
-grep -q ' xyz$' $t/log
-grep -q ' foo_bar$' $t/log
+grep -q ' xyz' $t/log
+grep -q ' foo_bar' $t/log
 ! grep -q ' foo$' $t/log || false

test/elf/version-script13.sh

@@ -16,5 +16,5 @@ EOF
 $CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 readelf --dyn-syms $t/c.so > $t/log
-grep -q ' foobar$' $t/log
+grep -q ' foobar' $t/log
 ! grep -q ' foo$' $t/log || false

test/elf/version-script14.sh

@@ -23,9 +23,9 @@ EOF
 $CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 readelf --dyn-syms $t/c.so > $t/log
-grep -q ' xyz$' $t/log
-! grep -q ' foobarzx$' $t/log || false
-grep -q ' foobarcx$' $t/log
-grep -q ' foo123bar456bx$' $t/log
-! grep -q ' foo123bar456c$' $t/log || false
-! grep -q ' foo123bar456x$' $t/log || false
+grep -q ' xyz' $t/log
+! grep -q ' foobarzx' $t/log || false
+grep -q ' foobarcx' $t/log
+grep -q ' foo123bar456bx' $t/log
+! grep -q ' foo123bar456c' $t/log || false
+! grep -q ' foo123bar456x' $t/log || false

test/elf/version-script15.sh

@@ -20,7 +20,7 @@ EOF
 $CC -B. -shared -Wl,--version-script=$t/a.ver -o $t/c.so $t/b.o
 
 readelf --dyn-syms $t/c.so > $t/log
-grep -q ' azZ$' $t/log
-grep -q ' czZ$' $t/log
-! grep -q ' azC$' $t/log || false
-! grep -q ' aaZ$' $t/log || false
+grep -q ' azZ' $t/log
+grep -q ' czZ' $t/log
+! grep -q ' azC' $t/log || false
+! grep -q ' aaZ' $t/log || false

test/elf/versioned-undef.sh

@@ -3,7 +3,7 @@
 
 # Skip if libc is musl because musl does not fully support GNU-style
 # symbol versioning.
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 cat <<EOF | $CC -fPIC -c -o $t/a.o -xc -
 int foo1() { return 1; }

test/elf/x86_64_empty-mergeable-section.sh

@@ -3,6 +3,7 @@
 
 cat <<EOF | $CC -o $t/a.o -c -xassembler -
 .section .rodata.str1.1, "aMS", @progbits, 1
+.section .rodata.foo, "aMS", @progbits, 1
 EOF
 
 cat <<EOF | $CC -o $t/b.o -c -xassembler -
@@ -19,4 +20,5 @@ $CC -B. -o $t/exe $t/a.o $t/b.o $t/c.o
 $t/exe
 
 readelf -SW $t/exe > $t/log
-! grep -Fq .rodata.str1.1 $t/log || false
+grep -q '.rodata.str1.1 .* AMS ' $t/log
+! grep -q '.rodata.str1.1 .* AM ' $t/log || false

test/elf/x86_64_endbr.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+[ $MACHINE = x86_64 ] || skip
+test_cflags -fcf-protection || skip
+
+cat <<EOF | $CC -o $t/a.o -c -xc - -ffunction-sections -O -fcf-protection
+int foo() { return 3; }
+int bar() { return foo(); }
+EOF
+
+cat <<EOF | $CC -o $t/b.o -c -xc - -ffunction-sections -O -fcf-protection
+int main() {}
+EOF
+
+$CC -B. -o $t/exe1 $t/a.o $t/b.o
+$OBJDUMP -dr $t/exe1 > $t/log1
+
+grep -A1 '<foo>:' $t/log1 | grep -q endbr64
+grep -A1 '<bar>:' $t/log1 | grep -q endbr64
+grep -A1 '<main>:' $t/log1 | grep -q endbr64
+
+$CC -B. -o $t/exe2 $t/a.o $t/b.o -Wl,-z,rewrite-endbr
+$OBJDUMP -dr $t/exe2 > $t/log2
+
+grep -A1 '<foo>:' $t/log2 | grep -q nop
+grep -A1 '<bar>:' $t/log2 | grep -q nop
+grep -A1 '<main>:' $t/log2 | grep -q endbr64

test/elf/x86_64_large-bss.sh

@@ -3,12 +3,18 @@
 
 [ $MACHINE = x86_64 ] || skip
 
-cat <<EOF | $CC -o $t/a.o -c -xc - 2> /dev/null || skip
-volatile char arr[0x100000000];
+cat <<EOF | $CC -o $t/a.o -c -xc - -mcmodel=large
+#include <stdio.h>
+char arr1[0xc0000000];
+extern char arr2[0xc0000000];
 int main() {
-  return arr[2000];
+  printf("%d %lx\n", (void *)arr1 < (void *)arr2, arr2 - arr1);
 }
 EOF
 
-$CC -B. -o $t/exe $t/a.o
-$QEMU $t/exe
+cat <<EOF | $CC -o $t/b.o -c -xc - -mcmodel=large
+char arr2[0xc0000000];
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o
+$QEMU $t/exe | grep -Eq '^1 c0000000$'

test/elf/x86_64_note-property2.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
+# OneTBB isn't tsan-clean
+nm mold | grep -q '__tsan_init' && skip
+
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip
 

test/elf/x86_64_pltgot.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-. $(dirname $0)/common.inc
-
-[ $MACHINE = x86_64 ] || skip
-
-cat <<EOF | $CC -fPIC -shared -Wl,-z,noexecstack -o $t/a.so -x assembler -
-.globl ext1, ext2
-ext1:
-  nop
-ext2:
-  nop
-EOF
-
-cat <<EOF | $CC -c -o $t/b.o -x assembler -
-.globl _start
-_start:
-  call ext1@PLT
-  call ext2@PLT
-  mov ext2@GOTPCREL(%rip), %rax
-  ret
-EOF
-
-./mold -z separate-loadable-segments -pie -o $t/exe $t/b.o $t/a.so
-
-$OBJDUMP -d -j .plt.got $t/exe > $t/log
-
-grep -Eq '1034:.*jmp.* <ext2>' $t/log

test/elf/x86_64_preinit-array.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 . $(dirname $0)/common.inc
 
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 [ $MACHINE = x86_64 ] || skip
 

test/elf/x86_64_warn-shared-textrel.sh

@@ -2,7 +2,7 @@
 . $(dirname $0)/common.inc
 
 # Skip if libc is musl
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip

test/elf/x86_64_warn-textrel.sh

@@ -2,7 +2,7 @@
 . $(dirname $0)/common.inc
 
 # Skip if libc is musl
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip

test/elf/x86_64_z-text.sh

@@ -2,7 +2,7 @@
 . $(dirname $0)/common.inc
 
 # Skip if libc is musl
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 # Skip if target is not x86-64
 [ $MACHINE = x86_64 ] || skip

test/elf/z-separate-code.sh

@@ -2,7 +2,7 @@
 . $(dirname $0)/common.inc
 
 # musl doesn't work with `-z noseparate-code`
-ldd --help 2>&1 | grep -q musl && skip
+is_musl && skip
 
 cat <<EOF | $CC -o $t/a.o -c -xc -
 #include <stdio.h>

third-party/blake3/c/CMakeLists.txt

@@ -6,6 +6,7 @@ project(libblake3
   LANGUAGES C ASM
 )
 
+include(CheckCCompilerFlag)
 include(FeatureSummary)
 include(GNUInstallDirs)
 
@@ -131,7 +132,12 @@ elseif(CMAKE_SYSTEM_PROCESSOR IN_LIST BLAKE3_ARMv8_NAMES
   target_sources(blake3 PRIVATE
     blake3_neon.c
   )
-  set_source_files_properties(blake3_dispatch.c PROPERTIES COMPILE_DEFINITIONS BLAKE3_USE_NEON=1)
+  target_compile_options(blake3 PRIVATE -DBLAKE3_USE_NEON=1)
+
+  check_c_compiler_flag(-mfpu=neon BLAKE3_MFPU_NEON_SUPPORTED)
+  if (BLAKE3_MFPU_NEON_SUPPORTED)
+    target_compile_options(blake3 PRIVATE -mfpu=neon)
+  endif()
 
   if (DEFINED BLAKE3_CFLAGS_NEON)
     set_source_files_properties(blake3_neon.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_NEON}")

third-party/rust-demangle/README.md

@@ -18,21 +18,22 @@ As this C port was originally (see the [History](#history) section) *only* for t
 it may lag behind `rustc-demangle` in functionality, for now.
 
 The current port status by category is:
-* **(UNPORTED)** `legacy` (pre-RFC2603 Rust symbols) demangling
+* **ported** `legacy` (pre-RFC2603 Rust symbols) demangling
 * `v0` ([RFC2603](https://rust-lang.github.io/rfcs/2603-rust-symbol-name-mangling-v0.html) Rust symbols) demangling
   * **ported** PRs:
     * [[#23] Support demangling the new Rust mangling scheme (v0).](https://github.com/rust-lang/rustc-demangle/pull/23)
     * [[#26] v0: allow identifiers to start with a digit.](https://github.com/rust-lang/rustc-demangle/pull/26)
     * [[#53] v0: replace `skip_*` methods with `print_*` methods in a "skip printing" mode.](https://github.com/rust-lang/rustc-demangle/pull/53)
       * arguably backported to Rust, as the C port always took this approach
-  * **(UNPORTED)** symbol prefix flexibility (`__R` and `R`, instead of `_R`)
-  * **(UNPORTED)** `min_const_generics` constants (`bool`, `char`, negative signed integers)
-    * this arguably also includes `p` as an *untyped* placeholder constant
-  * **(UNPORTED)** [`str` and structural constants](https://github.com/rust-lang/rfcs/pull/3161)
-    (only usable in `const` generics on unstable Rust)
+    * symbol prefix flexibility (`__R` and `R`, instead of `_R`)
+    * [[#39] Add support for `min_const_generics` constants](https://github.com/rust-lang/rustc-demangle/pull/39)
+      * [[#40] Elide the type when the const value is a placeholder `p`](https://github.com/rust-lang/rustc-demangle/pull/40)
+    * [[#55] v0: demangle structural constants and &str.](https://github.com/rust-lang/rustc-demangle/pull/55)
+      (only usable in `const` generics on unstable Rust)
   * **(UNPORTED)** recursion limits
 * miscellaneous
-  * **(UNPORTED)** extraneous symbol suffix (e.g. `.llvm.*`) removal
+  * **ported** PRs:
+    * [[#30] v0: also support preserving extra suffixes found after mangled symbol.](https://github.com/rust-lang/rustc-demangle/pull/30)
   * **(UNPORTED)** output size limits
 
 Notable differences (intentionally) introduced by porting:

third-party/rust-demangle/rust-demangle.c

@@ -70,6 +70,54 @@ static char next(struct rust_demangler *rdm) {
     return c;
 }
 
+struct hex_nibbles {
+    const char *nibbles;
+    size_t nibbles_len;
+};
+
+static struct hex_nibbles parse_hex_nibbles(struct rust_demangler *rdm) {
+    struct hex_nibbles hex;
+
+    hex.nibbles = NULL;
+    hex.nibbles_len = 0;
+
+    size_t start = rdm->next, hex_len = 0;
+    while (!eat(rdm, '_')) {
+        char c = next(rdm);
+        CHECK_OR(IS_DIGIT(c) || (c >= 'a' && c <= 'f'), return hex);
+        hex_len++;
+    }
+
+    hex.nibbles = rdm->sym + start;
+    hex.nibbles_len = hex_len;
+    return hex;
+}
+
+static struct hex_nibbles
+parse_hex_nibbles_for_const_uint(struct rust_demangler *rdm) {
+    struct hex_nibbles hex = parse_hex_nibbles(rdm);
+    CHECK_OR(!rdm->errored, return hex);
+
+    // Trim leading `0`s.
+    while (hex.nibbles_len > 0 && *hex.nibbles == '0') {
+        hex.nibbles++;
+        hex.nibbles_len--;
+    }
+
+    return hex;
+}
+
+static struct hex_nibbles
+parse_hex_nibbles_for_const_bytes(struct rust_demangler *rdm) {
+    struct hex_nibbles hex = parse_hex_nibbles(rdm);
+    CHECK_OR(!rdm->errored && (hex.nibbles_len % 2 == 0), return hex);
+    return hex;
+}
+
+static uint8_t decode_hex_nibble(char nibble) {
+    return nibble >= 'a' ? 10 + (nibble - 'a') : nibble - '0';
+}
+
 static uint64_t parse_integer_62(struct rust_demangler *rdm) {
     if (eat(rdm, '_'))
         return 0;
@@ -118,7 +166,10 @@ static struct rust_mangled_ident parse_ident(struct rust_demangler *rdm) {
     ident.punycode = NULL;
     ident.punycode_len = 0;
 
-    bool is_punycode = eat(rdm, 'u');
+    bool is_punycode = false;
+    if (rdm->version != -1) {
+        is_punycode = eat(rdm, 'u');
+    }
 
     char c = next(rdm);
     CHECK_OR(IS_DIGIT(c), return ident);
@@ -128,8 +179,10 @@ static struct rust_mangled_ident parse_ident(struct rust_demangler *rdm) {
         while (IS_DIGIT(peek(rdm)))
             len = len * 10 + (next(rdm) - '0');
 
-    // Skip past the optional `_` separator.
-    eat(rdm, '_');
+    if (rdm->version != -1) {
+        // Skip past the optional `_` separator.
+        eat(rdm, '_');
+    }
 
     size_t start = rdm->next;
     rdm->next += len;
@@ -182,6 +235,63 @@ static void print_uint64_hex(struct rust_demangler *rdm, uint64_t x) {
     PRINT(s);
 }
 
+static void
+print_quoted_escaped_char(struct rust_demangler *rdm, char quote, uint32_t c) {
+    CHECK_OR(c < 0xd800 || (c > 0xdfff && c < 0x10ffff), return);
+
+    switch (c) {
+    case '\0':
+        PRINT("\\0");
+        break;
+
+    case '\t':
+        PRINT("\\t");
+        break;
+
+    case '\r':
+        PRINT("\\r");
+        break;
+
+    case '\n':
+        PRINT("\\n");
+        break;
+
+    case '\\':
+        PRINT("\\\\");
+        break;
+
+    case '"':
+        if (quote == '"') {
+            PRINT("\\\"");
+        } else {
+            PRINT("\"");
+        }
+        break;
+
+    case '\'':
+        if (quote == '\'') {
+            PRINT("\\'");
+        } else {
+            PRINT("'");
+        }
+        break;
+
+    default:
+        if (c >= 0x20 && c <= 0x7e) {
+            // Printable ASCII
+            char v = (char)c;
+            print_str(rdm, &v, 1);
+        } else {
+            // FIXME show printable unicode characters without hex encoding
+            PRINT("\\u{");
+            char s[9] = {0};
+            sprintf(s, "%" PRIx32, c);
+            PRINT(s);
+            PRINT("}");
+        }
+    }
+}
+
 static void
 print_ident(struct rust_demangler *rdm, struct rust_mangled_ident ident) {
     if (rdm->errored || rdm->skipping_printing)
@@ -197,12 +307,12 @@ print_ident(struct rust_demangler *rdm, struct rust_mangled_ident ident) {
     while (cap < ident.ascii_len) {
         cap *= 2;
         // Check for overflows.
-        CHECK_OR((cap * 4) / 4 == cap, return );
+        CHECK_OR((cap * 4) / 4 == cap, return);
     }
 
     // Store the output codepoints as groups of 4 UTF-8 bytes.
     uint8_t *out = (uint8_t *)malloc(cap * 4);
-    CHECK_OR(out, return );
+    CHECK_OR(out, return);
 
     // Populate initial output from ASCII fragment.
     for (len = 0; len < ident.ascii_len; len++) {
@@ -342,14 +452,15 @@ static void demangle_generic_arg(struct rust_demangler *rdm);
 static void demangle_type(struct rust_demangler *rdm);
 static bool demangle_path_maybe_open_generics(struct rust_demangler *rdm);
 static void demangle_dyn_trait(struct rust_demangler *rdm);
-static void demangle_const(struct rust_demangler *rdm);
-static void demangle_const_uint(struct rust_demangler *rdm);
+static void demangle_const(struct rust_demangler *rdm, bool in_value);
+static void demangle_const_uint(struct rust_demangler *rdm, char ty_tag);
+static void demangle_const_str_literal(struct rust_demangler *rdm);
 
 /// Optionally enter a binder ('G') for late-bound lifetimes,
 /// printing e.g. `for<'a, 'b> `, and make those lifetimes visible
 /// to the caller (via depth level, which the caller should reset).
 static void demangle_binder(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     uint64_t bound_lifetimes = parse_opt_integer_62(rdm, 'G');
     if (bound_lifetimes > 0) {
@@ -365,7 +476,7 @@ static void demangle_binder(struct rust_demangler *rdm) {
 }
 
 static void demangle_path(struct rust_demangler *rdm, bool in_value) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     char tag = next(rdm);
     switch (tag) {
@@ -383,7 +494,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
     }
     case 'N': {
         char ns = next(rdm);
-        CHECK_OR(IS_LOWER(ns) || IS_UPPER(ns), return );
+        CHECK_OR(IS_LOWER(ns) || IS_UPPER(ns), return);
 
         demangle_path(rdm, in_value);
 
@@ -428,6 +539,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
         rdm->skipping_printing = true;
         demangle_path(rdm, in_value);
         rdm->skipping_printing = was_skipping_printing;
+        __attribute__((fallthrough));
     case 'Y':
         PRINT("<");
         demangle_type(rdm);
@@ -460,7 +572,7 @@ static void demangle_path(struct rust_demangler *rdm, bool in_value) {
         break;
     }
     default:
-        ERROR_AND(return );
+        ERROR_AND(return);
     }
 }
 
@@ -469,7 +581,7 @@ static void demangle_generic_arg(struct rust_demangler *rdm) {
         uint64_t lt = parse_integer_62(rdm);
         print_lifetime_from_index(rdm, lt);
     } else if (eat(rdm, 'K'))
-        demangle_const(rdm);
+        demangle_const(rdm, false);
     else
         demangle_type(rdm);
 }
@@ -525,7 +637,7 @@ static const char *basic_type(char tag) {
 }
 
 static void demangle_type(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     char tag = next(rdm);
 
@@ -565,11 +677,11 @@ static void demangle_type(struct rust_demangler *rdm) {
         demangle_type(rdm);
         if (tag == 'A') {
             PRINT("; ");
-            demangle_const(rdm);
+            demangle_const(rdm, true);
         }
         PRINT("]");
         break;
-    case 'T':
+    case 'T': {
         PRINT("(");
         size_t i;
         for (i = 0; !rdm->errored && !eat(rdm, 'E'); i++) {
@@ -581,6 +693,7 @@ static void demangle_type(struct rust_demangler *rdm) {
             PRINT(",");
         PRINT(")");
         break;
+    }
     case 'F': {
         uint64_t old_bound_lifetime_depth = rdm->bound_lifetime_depth;
         demangle_binder(rdm);
@@ -652,7 +765,7 @@ static void demangle_type(struct rust_demangler *rdm) {
         // Restore `bound_lifetime_depth` to outside the binder.
         rdm->bound_lifetime_depth = old_bound_lifetime_depth;
 
-        CHECK_OR(eat(rdm, 'L'), return );
+        CHECK_OR(eat(rdm, 'L'), return);
         uint64_t lt = parse_integer_62(rdm);
         if (lt) {
             PRINT(" + ");
@@ -709,7 +822,7 @@ static bool demangle_path_maybe_open_generics(struct rust_demangler *rdm) {
 }
 
 static void demangle_dyn_trait(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+    CHECK_OR(!rdm->errored, return);
 
     bool open = demangle_path_maybe_open_generics(rdm);
 
@@ -730,22 +843,17 @@ static void demangle_dyn_trait(struct rust_demangler *rdm) {
         PRINT(">");
 }
 
-static void demangle_const(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+static void demangle_const(struct rust_demangler *rdm, bool in_value) {
+    CHECK_OR(!rdm->errored, return);
 
-    if (eat(rdm, 'B')) {
-        size_t backref = parse_integer_62(rdm);
-        if (!rdm->skipping_printing) {
-            size_t old_next = rdm->next;
-            rdm->next = backref;
-            demangle_const(rdm);
-            rdm->next = old_next;
-        }
-        return;
-    }
+    bool opened_brace = false;
 
     char ty_tag = next(rdm);
     switch (ty_tag) {
+    case 'p':
+        PRINT("_");
+        break;
+
     // Unsigned integer types.
     case 'h':
     case 't':
@@ -753,66 +861,464 @@ static void demangle_const(struct rust_demangler *rdm) {
     case 'y':
     case 'o':
     case 'j':
+        demangle_const_uint(rdm, ty_tag);
         break;
 
-    default:
-        ERROR_AND(return );
+    case 'a':
+    case 's':
+    case 'l':
+    case 'x':
+    case 'n':
+    case 'i':
+        if (eat(rdm, 'n')) {
+            PRINT("-");
+        }
+        demangle_const_uint(rdm, ty_tag);
+        break;
+
+    case 'b': {
+        struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+        CHECK_OR(!rdm->errored && hex.nibbles_len <= 1, return);
+        uint8_t v = hex.nibbles_len > 0 ? decode_hex_nibble(hex.nibbles[0]) : 0;
+        CHECK_OR(v <= 1, return);
+        PRINT(v == 1 ? "true" : "false");
+        break;
     }
 
-    if (eat(rdm, 'p'))
-        PRINT("_");
-    else {
-        demangle_const_uint(rdm);
-        if (rdm->verbose)
-            PRINT(basic_type(ty_tag));
+    case 'c': {
+        struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+        CHECK_OR(!rdm->errored && hex.nibbles_len <= 6, return);
+
+        uint32_t c = 0;
+        for (size_t i = 0; i < hex.nibbles_len; i++)
+            c = (c << 4) | decode_hex_nibble(hex.nibbles[i]);
+
+        PRINT("'");
+        print_quoted_escaped_char(rdm, '\'', c);
+        PRINT("'");
+
+        break;
+    }
+
+    case 'e':
+        // NOTE(eddyb) a string literal `"..."` has type `&str`, so
+        // to get back the type `str`, `*"..."` syntax is needed
+        // (even if that may not be valid in Rust itself).
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+        PRINT("*");
+
+        demangle_const_str_literal(rdm);
+        break;
+
+    case 'R':
+    case 'Q':
+        if (ty_tag == 'R' && eat(rdm, 'e')) {
+            // NOTE(eddyb) this prints `"..."` instead of `&*"..."`, which
+            // is what `Re..._` would imply (see comment for `str` above).
+            demangle_const_str_literal(rdm);
+            break;
+        }
+
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("&");
+        if (ty_tag != 'R') {
+            PRINT("mut ");
+        }
+
+        demangle_const(rdm, true);
+        break;
+
+    case 'A': {
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("[");
+
+        size_t i = 0;
+        while (!eat(rdm, 'E')) {
+            CHECK_OR(!rdm->errored, return);
+
+            if (i > 0)
+                PRINT(", ");
+
+            demangle_const(rdm, true);
+
+            i += 1;
+        }
+
+        PRINT("]");
+        break;
+    }
+
+    case 'T': {
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        PRINT("(");
+
+        size_t i = 0;
+        while (!eat(rdm, 'E')) {
+            CHECK_OR(!rdm->errored, return);
+
+            if (i > 0)
+                PRINT(", ");
+
+            demangle_const(rdm, true);
+
+            i += 1;
+        }
+
+        if (i == 1)
+            PRINT(",");
+
+        PRINT(")");
+        break;
+    }
+
+    case 'V':
+        if (!in_value) {
+            opened_brace = true;
+            PRINT("{");
+        }
+
+        demangle_path(rdm, true);
+
+        switch (next(rdm)) {
+        case 'U':
+            break;
+
+        case 'T': {
+            PRINT("(");
+
+            size_t i = 0;
+            while (!eat(rdm, 'E')) {
+                CHECK_OR(!rdm->errored, return);
+
+                if (i > 0)
+                    PRINT(", ");
+
+                demangle_const(rdm, true);
+
+                i += 1;
+            }
+
+            PRINT(")");
+            break;
+        }
+
+        case 'S': {
+            PRINT(" { ");
+
+            size_t i = 0;
+            while (!eat(rdm, 'E')) {
+                CHECK_OR(!rdm->errored, return);
+
+                if (i > 0)
+                    PRINT(", ");
+
+                parse_disambiguator(rdm);
+
+                struct rust_mangled_ident name = parse_ident(rdm);
+                print_ident(rdm, name);
+
+                PRINT(": ");
+
+                demangle_const(rdm, true);
+
+                i += 1;
+            }
+
+            PRINT(" }");
+            break;
+        }
+
+        default:
+            ERROR_AND(return);
+        }
+
+        break;
+
+    case 'B': {
+        size_t backref = parse_integer_62(rdm);
+        if (!rdm->skipping_printing) {
+            size_t old_next = rdm->next;
+            rdm->next = backref;
+            demangle_const(rdm, in_value);
+            rdm->next = old_next;
+        }
+        break;
+    }
+
+    default:
+        ERROR_AND(return);
+    }
+
+    if (opened_brace) {
+        PRINT("}");
     }
 }
 
-static void demangle_const_uint(struct rust_demangler *rdm) {
-    CHECK_OR(!rdm->errored, return );
+static void demangle_const_uint(struct rust_demangler *rdm, char ty_tag) {
+    CHECK_OR(!rdm->errored, return);
 
-    uint64_t value = 0;
-    size_t hex_len = 0;
-    while (!eat(rdm, '_')) {
-        value <<= 4;
-
-        char c = next(rdm);
-        if (IS_DIGIT(c))
-            value |= c - '0';
-        else if (c >= 'a' && c <= 'f')
-            value |= 10 + (c - 'a');
-        else
-            ERROR_AND(return );
-        hex_len++;
-    }
+    struct hex_nibbles hex = parse_hex_nibbles_for_const_uint(rdm);
+    CHECK_OR(!rdm->errored, return);
 
     // Print anything that doesn't fit in `uint64_t` verbatim.
-    if (hex_len > 16) {
+    if (hex.nibbles_len > 16) {
         PRINT("0x");
-        print_str(rdm, rdm->sym + (rdm->next - hex_len), hex_len);
-        return;
+        print_str(rdm, hex.nibbles, hex.nibbles_len);
+    } else {
+        uint64_t v = 0;
+        for (size_t i = 0; i < hex.nibbles_len; i++)
+            v = (v << 4) | decode_hex_nibble(hex.nibbles[i]);
+        print_uint64(rdm, v);
     }
 
-    print_uint64(rdm, value);
+    if (rdm->verbose)
+        PRINT(basic_type(ty_tag));
+}
+
+// UTF-8 uses an unary encoding for its "length" field (`1`s followed by a `0`).
+struct utf8_byte {
+    // Decoded "length" field of an UTF-8 byte, including the special cases:
+    // - `0` indicates this is a lone ASCII byte
+    // - `1` indicates a continuation byte (cannot start an UTF-8 sequence)
+    size_t seq_len;
+
+    // Remaining (`payload_width`) bits in the UTF-8 byte, contributing to
+    // the Unicode scalar value being encoded in the UTF-8 sequence.
+    uint8_t payload;
+    size_t payload_width;
+};
+static struct utf8_byte utf8_decode(uint8_t byte) {
+    struct utf8_byte utf8;
+
+    utf8.seq_len = 0;
+    utf8.payload = byte;
+    utf8.payload_width = 8;
+
+    // FIXME(eddyb) figure out if using "count leading ones/zeros" is an option.
+    while (utf8.seq_len <= 6) {
+        uint8_t msb = 0x80 >> utf8.seq_len;
+        utf8.payload &= ~msb;
+        utf8.payload_width--;
+        if ((byte & msb) == 0)
+            break;
+        utf8.seq_len++;
+    }
+
+    return utf8;
+}
+
+static void demangle_const_str_literal(struct rust_demangler *rdm) {
+    CHECK_OR(!rdm->errored, return);
+
+    struct hex_nibbles hex = parse_hex_nibbles_for_const_bytes(rdm);
+    CHECK_OR(!rdm->errored, return);
+
+    PRINT("\"");
+    for (size_t i = 0; i < hex.nibbles_len; i += 2) {
+        struct utf8_byte utf8 = utf8_decode(
+            (decode_hex_nibble(hex.nibbles[i]) << 4) |
+            decode_hex_nibble(hex.nibbles[i + 1])
+        );
+        uint32_t c = utf8.payload;
+        if (utf8.seq_len > 0) {
+            CHECK_OR(utf8.seq_len >= 2 && utf8.seq_len <= 4, return);
+            for (size_t extra = utf8.seq_len - 1; extra > 0; extra--) {
+                i += 2;
+                utf8 = utf8_decode(
+                    (decode_hex_nibble(hex.nibbles[i]) << 4) |
+                    decode_hex_nibble(hex.nibbles[i + 1])
+                );
+                CHECK_OR(utf8.seq_len == 1, return);
+                c = (c << utf8.payload_width) | utf8.payload;
+            }
+        }
+        print_quoted_escaped_char(rdm, '"', c);
+    }
+    PRINT("\"");
+}
+
+static bool is_rust_hash(struct rust_mangled_ident name) {
+    if (name.ascii[0] != 'h') {
+        return false;
+    }
+    for (size_t i = 1; i < name.ascii_len; i++) {
+        if (!IS_DIGIT(name.ascii[i]) &&
+            !(name.ascii[i] >= 'a' && name.ascii[i] <= 'f')) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static void print_legacy_ident(
+    struct rust_demangler *rdm, struct rust_mangled_ident ident
+) {
+    if (rdm->errored || rdm->skipping_printing)
+        return;
+
+    CHECK_OR(!ident.punycode, return);
+
+    if (ident.ascii[0] == '_' && ident.ascii[1] == '$') {
+        ident.ascii += 1;
+        ident.ascii_len -= 1;
+    }
+
+    while (1) {
+        if (ident.ascii_len == 0) {
+            break;
+        } else if (ident.ascii[0] == '.') {
+            if (ident.ascii_len >= 2 && ident.ascii[1] == '.') {
+                PRINT("::");
+                ident.ascii += 2;
+                ident.ascii_len -= 2;
+            } else {
+                PRINT(".");
+                ident.ascii += 1;
+                ident.ascii_len -= 1;
+            }
+        } else if (ident.ascii[0] == '$') {
+            const char *end_ptr =
+                (const char *)memchr(&ident.ascii[1], '$', ident.ascii_len - 1);
+            if (!end_ptr)
+                break;
+            const char *escape = &ident.ascii[1];
+            size_t escape_len = end_ptr - escape;
+
+            if (strncmp(escape, "SP", 2) == 0) {
+                PRINT("@");
+            } else if (strncmp(escape, "BP", 2) == 0) {
+                PRINT("*");
+            } else if (strncmp(escape, "RF", 2) == 0) {
+                PRINT("&");
+            } else if (strncmp(escape, "LT", 2) == 0) {
+                PRINT("<");
+            } else if (strncmp(escape, "GT", 2) == 0) {
+                PRINT(">");
+            } else if (strncmp(escape, "LP", 2) == 0) {
+                PRINT("(");
+            } else if (strncmp(escape, "RP", 2) == 0) {
+                PRINT(")");
+            } else if (strncmp(escape, "C", 1) == 0) {
+                PRINT(",");
+            } else {
+                if (escape[0] != 'u') {
+                    break;
+                }
+
+                const char *digits = &escape[1];
+                size_t digits_len = escape_len - 1;
+
+                bool invalid = false;
+                for (size_t i = 1; i < digits_len; i++) {
+                    if (!IS_DIGIT(digits[i]) &&
+                        !(digits[i] >= 'a' && digits[i] <= 'f')) {
+                        invalid = true;
+                        break;
+                    }
+                }
+                if (invalid)
+                    break;
+
+                struct hex_nibbles hex;
+
+                hex.nibbles = digits;
+                hex.nibbles_len = digits_len;
+
+                uint32_t c = 0;
+                for (size_t i = 0; i < hex.nibbles_len; i++)
+                    c = (c << 4) | decode_hex_nibble(hex.nibbles[i]);
+
+                if (!(c < 0xd800 || (c > 0xdfff && c < 0x10ffff))) {
+                    break; // Not a valid unicode scalar
+                }
+
+                if (c >= 0x20 && c <= 0x7e) {
+                    // Printable ASCII
+                    char v = (char)c;
+                    print_str(rdm, &v, 1);
+                } else {
+                    // FIXME show printable unicode characters without hex
+                    // encoding
+                    PRINT("\\u{");
+                    char s[9] = {0};
+                    sprintf(s, "%" PRIx32, c);
+                    PRINT(s);
+                    PRINT("}");
+                }
+            }
+
+            ident.ascii += escape_len + 2;
+            ident.ascii_len -= escape_len + 2;
+        } else {
+            bool found = false;
+            for (size_t i = 0; i < ident.ascii_len; i++) {
+                if (ident.ascii[i] == '$' || ident.ascii[i] == '.') {
+                    print_str(rdm, ident.ascii, i);
+                    ident.ascii += i;
+                    ident.ascii_len -= i;
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                break;
+            }
+        }
+    }
+
+    print_str(rdm, ident.ascii, ident.ascii_len);
+}
+
+static void demangle_legacy_path(struct rust_demangler *rdm) {
+    bool first = true;
+
+    while (1) {
+        if (eat(rdm, 'E')) {
+            // FIXME Maybe check if at end of symbol?
+            return;
+        }
+
+        struct rust_mangled_ident name = parse_ident(rdm);
+
+        if (!rdm->verbose && peek(rdm) == 'E' && is_rust_hash(name)) {
+            // Skip printing the hash if verbose mode is disabled.
+            eat(rdm, 'E');
+            break;
+        }
+
+        if (!first) {
+            PRINT("::");
+        }
+        first = false;
+
+        print_legacy_ident(rdm, name);
+
+        CHECK_OR(!rdm->errored, return);
+    }
 }
 
 bool rust_demangle_with_callback(
-    const char *mangled, int flags,
+    const char *whole_mangled_symbol, int flags,
     void (*callback)(const char *data, size_t len, void *opaque), void *opaque
 ) {
-    // Rust symbols always start with _R.
-    if (mangled[0] == '_' && mangled[1] == 'R')
-        mangled += 2;
-    else
-        return false;
-
-    // Paths always start with uppercase characters.
-    if (!IS_UPPER(mangled[0]))
-        return false;
-
     struct rust_demangler rdm;
 
-    rdm.sym = mangled;
+    rdm.sym = whole_mangled_symbol;
     rdm.sym_len = 0;
 
     rdm.callback_opaque = opaque;
@@ -822,26 +1328,84 @@ bool rust_demangle_with_callback(
     rdm.errored = false;
     rdm.skipping_printing = false;
     rdm.verbose = (flags & RUST_DEMANGLE_FLAG_VERBOSE) != 0;
-    rdm.version = 0;
+    rdm.version = -2; // Invalid version
     rdm.bound_lifetime_depth = 0;
 
-    // Rust symbols use only [_0-9a-zA-Z] characters.
-    for (const char *p = mangled; *p; p++) {
-        if (!(*p == '_' || IS_DIGIT(*p) || IS_LOWER(*p) || IS_UPPER(*p)))
+    // Rust symbols always start with R, _R or __R for the v0 scheme or ZN, _ZN
+    // or __ZN for the legacy scheme.
+    if (strncmp(rdm.sym, "_R", 2) == 0) {
+        rdm.sym += 2;
+        rdm.version = 0; // v0
+    } else if (rdm.sym[0] == 'R') {
+        // On Windows, dbghelp strips leading underscores, so we accept "R..."
+        // form too.
+        rdm.sym += 1;
+        rdm.version = 0; // v0
+    } else if (strncmp(rdm.sym, "__R", 3) == 0) {
+        // On OSX, symbols are prefixed with an extra _
+        rdm.sym += 3;
+        rdm.version = 0; // v0
+    } else if (strncmp(rdm.sym, "_ZN", 3) == 0) {
+        rdm.sym += 3;
+        rdm.version = -1; // legacy
+    } else if (strncmp(rdm.sym, "ZN", 2) == 0) {
+        // On Windows, dbghelp strips leading underscores, so we accept "R..."
+        // form too.
+        rdm.sym += 2;
+        rdm.version = -1; // legacy
+    } else if (strncmp(rdm.sym, "__ZN", 4) == 0) {
+        // On OSX, symbols are prefixed with an extra _
+        rdm.sym += 4;
+        rdm.version = -1; // legacy
+    } else {
+        return false;
+    }
+
+    if (rdm.version != -1) {
+        // Paths always start with uppercase characters.
+        if (!IS_UPPER(rdm.sym[0]))
             return false;
+    }
+
+    // Rust symbols only use ASCII characters.
+    for (const char *p = rdm.sym; *p; p++) {
+        if ((*p & 0x80) != 0)
+            return false;
+
+        if (*p == '.' && strncmp(p, ".llvm.", 6) == 0) {
+            // Ignore .llvm.<hash> suffixes
+            break;
+        }
+
         rdm.sym_len++;
     }
 
-    demangle_path(&rdm, true);
+    if (rdm.version == -1) {
+        demangle_legacy_path(&rdm);
+    } else {
+        demangle_path(&rdm, true);
 
-    // Skip instantiating crate.
-    if (!rdm.errored && rdm.next < rdm.sym_len) {
-        rdm.skipping_printing = true;
-        demangle_path(&rdm, false);
+        // Skip instantiating crate.
+        if (!rdm.errored && rdm.next < rdm.sym_len && peek(&rdm) >= 'A' &&
+            peek(&rdm) <= 'Z') {
+            rdm.skipping_printing = true;
+            demangle_path(&rdm, false);
+        }
     }
 
-    // It's an error to not reach the end.
-    rdm.errored |= rdm.next != rdm.sym_len;
+    if (!rdm.errored && (rdm.sym_len - rdm.next > 0)) {
+        for (const char *p = rdm.sym + rdm.next; *p; p++) {
+            // FIXME match is_symbol_like from rustc-demangle
+            if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
+                  (*p >= '0' && *p <= '9') || *p == '.')) {
+                // Suffix is not a symbol like string
+                return false;
+            }
+        }
+
+        // Print LLVM produced suffix
+        print_str(&rdm, rdm.sym + rdm.next, rdm.sym_len - rdm.next);
+    }
 
     return !rdm.errored;
 }

third-party/rust-demangle/test-harness/src/lib.rs

@@ -54,8 +54,7 @@ pub fn try_demangle(s: &str) -> Result<Demangle<'_>, rustc_demangle::TryDemangle
             rustc_demangle: d,
         }),
         Err(e) => {
-            // HACK(eddyb) ensure the C port also errors.
-            assert_eq!(demangle_via_c(s, false), "");
+            assert!(demangle_via_c(s, false).is_err());
 
             Err(e)
         }
@@ -68,6 +67,36 @@ impl fmt::Display for Demangle<'_> {
     }
 }
 
+// HACK(eddyb) the C port doesn't have the "is printable Unicode" heuristic,
+// to avoid having to include the non-trivial amount of data that requires,
+// so instead we allow mismatches when `b` has more `\u{...}` escapes than `a`.
+fn equal_modulo_unicode_escapes(a: &str, b: &str) -> bool {
+    let mut a_chars = a.chars();
+    let mut a_active_escape: Option<std::char::EscapeUnicode> = None;
+    let mut b_chars = b.chars();
+    loop {
+        let a_ch = a_active_escape
+            .as_mut()
+            .and_then(|escape| escape.next())
+            .or_else(|| {
+                a_active_escape = None;
+                a_chars.next()
+            });
+        let b_ch = b_chars.next();
+        match (a_ch, b_ch) {
+            (Some(a_ch), Some(b_ch)) if a_ch == b_ch => {}
+            // Compare with the `\u{...}` escape instead, if possible.
+            (Some(a_ch), Some('\\')) if a_active_escape.is_none() => {
+                let mut escape = a_ch.escape_unicode();
+                assert_eq!(escape.next(), Some('\\'));
+                a_active_escape = Some(escape);
+            }
+            (None, None) => return true,
+            _ => return false,
+        }
+    }
+}
+
 impl Demangle<'_> {
     pub fn to_string_maybe_verbose(&self, verbose: bool) -> String {
         let rust = if verbose {
@@ -75,8 +104,8 @@ impl Demangle<'_> {
         } else {
             format!("{:#}", self.rustc_demangle)
         };
-        let c = demangle_via_c(self.original, verbose);
-        if rust != c {
+        let c = demangle_via_c(self.original, verbose).unwrap_or_else(|_| self.original.to_owned());
+        if rust != c && !equal_modulo_unicode_escapes(&rust, &c) {
             panic!(
                 "Rust vs C demangling difference:\
             \n mangled: {mangled:?}\
@@ -91,7 +120,7 @@ impl Demangle<'_> {
     }
 }
 
-fn demangle_via_c(mangled: &str, verbose: bool) -> String {
+fn demangle_via_c(mangled: &str, verbose: bool) -> Result<String, ()> {
     use std::ffi::{CStr, CString};
     use std::os::raw::c_char;
 
@@ -101,15 +130,18 @@ fn demangle_via_c(mangled: &str, verbose: bool) -> String {
     }
 
     let flags = if verbose { 1 } else { 0 };
-    let mangled = CString::new(mangled).unwrap();
+    let Ok(mangled) = CString::new(mangled) else {
+        // C can't handle strings containing nul bytes
+        return Err(());
+    };
     let out = unsafe { rust_demangle(mangled.as_ptr(), flags) };
     if out.is_null() {
-        String::new()
+        Err(())
     } else {
         unsafe {
             let s = CStr::from_ptr(out).to_string_lossy().into_owned();
             free(out);
-            s
+            Ok(s)
         }
     }
 }

third-party/rust-demangle/test-harness/tests/legacy.rs

@@ -53,8 +53,6 @@ fn ok_err(sym: &str) -> bool {
     }
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle() {
     t_err!("test");
@@ -63,8 +61,6 @@ fn demangle() {
     t!("_ZN4test1a2bcE", "test::a::bc");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_dollars() {
     t!("_ZN4$RP$E", ")");
@@ -74,16 +70,12 @@ fn demangle_dollars() {
     t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_many_dollars() {
     t!("_ZN13test$u20$test4foobE", "test test::foob");
     t!("_ZN12test$BP$test4foobE", "test*test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_osx() {
     t!(
@@ -94,8 +86,6 @@ fn demangle_osx() {
     t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::<impl core::iter::traits::IntoIterator for &'a [T]>::into_iter::h450e234d27262170");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_windows() {
     t!("ZN4testE", "test");
@@ -103,8 +93,6 @@ fn demangle_windows() {
     t!("ZN12test$RF$test4foobE", "test&test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_elements_beginning_with_underscore() {
     t!("_ZN13_$LT$test$GT$E", "<test>");
@@ -112,8 +100,6 @@ fn demangle_elements_beginning_with_underscore() {
     t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_trait_impls() {
     t!(
@@ -122,8 +108,6 @@ fn demangle_trait_impls() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash() {
     let s = "_ZN3foo17h05af221e174051e9E";
@@ -131,8 +115,6 @@ fn demangle_without_hash() {
     t_nohash!(s, "foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash_edgecases() {
     // One element, no hash.
@@ -151,8 +133,6 @@ fn demangle_without_hash_edgecases() {
     t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     // One element, no hash.
@@ -164,23 +144,17 @@ fn demangle_thinlto() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_llvm_ir_branch_labels() {
     t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i");
     t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut.exit.i.i");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() {
     t_err!("_ZN3fooE.llvm moocow");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn dont_panic() {
     rust_demangle_c_test_harness::demangle("_ZN2222222222222222222222EE").to_string();
@@ -196,22 +170,16 @@ fn dont_panic() {
     .to_string();
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn invalid_no_chop() {
     t_err!("_ZNfooE");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_assoc_types() {
     t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", "<alloc::boxed::Box<alloc::boxed::FnBox<A, Output=R> + 'a> as core::ops::function::FnOnce<A>>::call_once::h69e8f44b3723e1ca");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_bang() {
     t!(
@@ -220,8 +188,6 @@ fn handle_bang() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_utf8_idents() {
     t_nohash!(
@@ -230,8 +196,6 @@ fn demangle_utf8_idents() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_issue_60925() {
     t_nohash!(

third-party/rust-demangle/test-harness/tests/top_level.rs

@@ -56,8 +56,6 @@ fn ok_err(sym: &str) -> bool {
     }
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle() {
     t_err!("test");
@@ -66,8 +64,6 @@ fn demangle() {
     t!("_ZN4test1a2bcE", "test::a::bc");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_dollars() {
     t!("_ZN4$RP$E", ")");
@@ -77,16 +73,12 @@ fn demangle_dollars() {
     t!("_ZN35Bar$LT$$u5b$u32$u3b$$u20$4$u5d$$GT$E", "Bar<[u32; 4]>");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_many_dollars() {
     t!("_ZN13test$u20$test4foobE", "test test::foob");
     t!("_ZN12test$BP$test4foobE", "test*test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_osx() {
     t!(
@@ -97,8 +89,6 @@ fn demangle_osx() {
     t!("__ZN4core5slice89_$LT$impl$u20$core..iter..traits..IntoIterator$u20$for$u20$$RF$$u27$a$u20$$u5b$T$u5d$$GT$9into_iter17h450e234d27262170E", "core::slice::<impl core::iter::traits::IntoIterator for &'a [T]>::into_iter::h450e234d27262170");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_windows() {
     t!("ZN4testE", "test");
@@ -106,8 +96,6 @@ fn demangle_windows() {
     t!("ZN12test$RF$test4foobE", "test&test::foob");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_elements_beginning_with_underscore() {
     t!("_ZN13_$LT$test$GT$E", "<test>");
@@ -115,8 +103,6 @@ fn demangle_elements_beginning_with_underscore() {
     t!("_ZN15__STATIC_FMTSTRE", "__STATIC_FMTSTR");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_trait_impls() {
     t!(
@@ -125,8 +111,6 @@ fn demangle_trait_impls() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash() {
     let s = "_ZN3foo17h05af221e174051e9E";
@@ -134,8 +118,6 @@ fn demangle_without_hash() {
     t_nohash!(s, "foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_without_hash_edgecases() {
     // One element, no hash.
@@ -154,8 +136,6 @@ fn demangle_without_hash_edgecases() {
     t_nohash!("_ZN3foo17hg5af221e174051e9E", "foo::hg5af221e174051e9");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     // One element, no hash.
@@ -167,23 +147,17 @@ fn demangle_thinlto() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_llvm_ir_branch_labels() {
     t!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut::haf9727c2edfbc47b.exit.i.i");
     t_nohash!("_ZN4core5slice77_$LT$impl$u20$core..ops..index..IndexMut$LT$I$GT$$u20$for$u20$$u5b$T$u5d$$GT$9index_mut17haf9727c2edfbc47bE.exit.i.i", "core::slice::<impl core::ops::index::IndexMut<I> for [T]>::index_mut.exit.i.i");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_ignores_suffix_that_doesnt_look_like_a_symbol() {
     t_err!("_ZN3fooE.llvm moocow");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn dont_panic() {
     rust_demangle_c_test_harness::demangle("_ZN2222222222222222222222EE").to_string();
@@ -199,22 +173,16 @@ fn dont_panic() {
     .to_string();
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn invalid_no_chop() {
     t_err!("_ZNfooE");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_assoc_types() {
     t!("_ZN151_$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$9call_once17h69e8f44b3723e1caE", "<alloc::boxed::Box<alloc::boxed::FnBox<A, Output=R> + 'a> as core::ops::function::FnOnce<A>>::call_once::h69e8f44b3723e1ca");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn handle_bang() {
     t!(
@@ -238,14 +206,20 @@ fn limit_recursion() {
 }
 
 // FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
+#[ignore = "would slowly use up all RAM before being OOM-killed"]
 #[test]
-fn limit_output() {
+fn limit_output_oom_hazard() {
     assert_ends_with!(
         rust_demangle_c_test_harness::demangle("RYFG_FGyyEvRYFF_EvRYFFEvERLB_B_B_ERLRjB_B_B_")
             .to_string(),
         "{size limit reached}"
     );
+}
+
+// FIXME(eddyb) port the relevant functionality to C.
+#[should_panic]
+#[test]
+fn limit_output() {
     // NOTE(eddyb) somewhat reduced version of the above, effectively
     // `<for<...> fn()>` with a larger number of lifetimes in `...`.
     assert_ends_with!(

third-party/rust-demangle/test-harness/tests/v0.rs

@@ -92,8 +92,6 @@ fn demangle_const_generics_preview() {
     t_const_suffixed!("j7b_", "123", "usize");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_min_const_generics() {
     t_const!("p", "_");
@@ -109,8 +107,6 @@ fn demangle_min_const_generics() {
     t_const!("c2202_", "'∂'");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_str() {
     t_const!("e616263_", "{*\"abc\"}");
@@ -129,8 +125,6 @@ fn demangle_const_str() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 // NOTE(eddyb) this uses the same strings as `demangle_const_str` and should
 // be kept in sync with it - while a macro could be used to generate both
 // `str` and `&str` tests, from a single list of strings, this seems clearer.
@@ -152,8 +146,6 @@ fn demangle_const_ref_str() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_ref() {
     t_const!("Rp", "{&_}");
@@ -165,8 +157,6 @@ fn demangle_const_ref() {
     t_const!("QAE", "{&mut []}");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_array() {
     t_const!("AE", "{[]}");
@@ -176,8 +166,6 @@ fn demangle_const_array() {
     t_const!("AAh1_h2_EAh3_h4_EE", "{[[1, 2], [3, 4]]}");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_tuple() {
     t_const!("TE", "{()}");
@@ -189,8 +177,6 @@ fn demangle_const_tuple() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_const_adt() {
     t_const!(
@@ -223,8 +209,6 @@ fn demangle_exponential_explosion() {
     );
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_thinlto() {
     t_nohash!("_RC3foo.llvm.9D1C9369", "foo");
@@ -232,8 +216,6 @@ fn demangle_thinlto() {
     t_nohash!("_RNvC9backtrace3foo.llvm.A5310EB9", "backtrace::foo");
 }
 
-// FIXME(eddyb) port the relevant functionality to C.
-#[should_panic]
 #[test]
 fn demangle_extra_suffix() {
     // From alexcrichton/rustc-demangle#27: