Bump mold version to 1.2.1

[ELF] Support object files with more than 2^19 symbols
Previously, Symbol::sym_idx overflows if there are more than 524,288 symbols in a single file. That's a very large number but is not impossible. Fixes https://github.com/rui314/mold/issues/405
2026-11-12 14:59:43 -10:00 · 2022-04-28 17:51:29 +08:00 · 2022-04-28 17:08:42 +08:00 · 2022-04-28 13:40:11 +08:00 · 2022-04-28 13:37:16 +08:00 · 2022-04-28 11:40:31 +08:00
77 changed files with 2163 additions and 1139 deletions
@@ -13,7 +13,7 @@ jobs:
      with:
        make-default: false
    - name: apt-get
-      run: sudo dpkg --add-architecture i386 && sudo apt-get update && sudo apt-get install git build-essential libstdc++-10-dev cmake clang libssl-dev zlib1g-dev libxxhash-dev libtbb-dev git bsdmainutils dwarfdump pkg-config
+      run: sudo dpkg --add-architecture i386 && sudo apt-get update && sudo apt-get install git build-essential libstdc++-10-dev cmake clang libssl-dev zlib1g-dev libtbb-dev git bsdmainutils dwarfdump pkg-config
    - name: ccache
      uses: hendrikmuhs/ccache-action@v1
    - name: make
@@ -27,7 +27,7 @@ jobs:
    - uses: actions/checkout@v2
    - name: apt-get
      run: |
-        dpkg --add-architecture i386 && apt-get update && apt-get install -y sudo libssl-dev zlib1g-dev libxxhash-dev cmake clang bsdmainutils dwarfdump qemu-user gcc-10-i686-linux-gnu gcc-10-aarch64-linux-gnu gcc-10-riscv64-linux-gnu gcc-10-arm-linux-gnueabihf g++-10-i686-linux-gnu g++-10-aarch64-linux-gnu g++-10-riscv64-linux-gnu g++-10-arm-linux-gnueabihf
+        dpkg --add-architecture i386 && apt-get update && apt-get install -y sudo libssl-dev zlib1g-dev cmake clang bsdmainutils dwarfdump qemu-user gcc-10-i686-linux-gnu gcc-10-aarch64-linux-gnu gcc-10-riscv64-linux-gnu gcc-10-arm-linux-gnueabihf g++-10-i686-linux-gnu g++-10-aarch64-linux-gnu g++-10-riscv64-linux-gnu g++-10-arm-linux-gnueabihf
        ln -sf /usr/bin/i686-linux-gnu-gcc-10 /usr/bin/i686-linux-gnu-gcc
        ln -sf /usr/bin/i686-linux-gnu-g++-10 /usr/bin/i686-linux-gnu-g++
        ln -sf /usr/bin/aarch64-linux-gnu-gcc-10 /usr/bin/aarch64-linux-gnu-gcc
@@ -46,8 +46,6 @@ jobs:
    runs-on: macos-11
    steps:
    - uses: actions/checkout@v2
-    - name: brew
-      run: brew install xxhash
    - name: ccache
      uses: hendrikmuhs/ccache-action@v1
    - name: make
@@ -2,7 +2,7 @@
 #
 # make CXXFLAGS='-fsanitize=address -g' LDFLAGS=-fsanitize=address USE_MIMALLOC=0

-VERSION = 1.2.0
+VERSION = 1.2.1

 PREFIX = /usr/local
 BINDIR = $(PREFIX)/bin
@@ -187,7 +187,6 @@ test-riscv64: all
 	@echo riscv64
 	CC=riscv64-linux-gnu-gcc CXX=riscv64-linux-gnu-g++ GCC=riscv64-linux-gnu-gcc GXX=riscv64-linux-gnu-g++ OBJDUMP=riscv64-linux-gnu-objdump MACHINE=riscv64 QEMU='qemu-riscv64 -L /usr/riscv64-linux-gnu' $(MAKE) test

-
 test-all:
 	$(MAKE) test-x86-64
 	$(MAKE) test-i386
@@ -205,7 +204,12 @@ install: all
 	$(STRIP) $D$(LIBDIR)/mold/mold-wrapper.so

 	$(INSTALL) -d $D$(LIBEXECDIR)/mold
-	ln -sf $(BINDIR)/mold $D$(LIBEXECDIR)/mold/ld
+
+# We want to make a symblink with a relative path, so that users can
+# move the entire directory to other place without breaking the reference.
+# GNU ln supports `--relative` to do that, but that's not supported by
+# non-GNU systems. So we use Python to compute a relative path.
+	ln -sf `python3 -c "import os.path; print(os.path.relpath('$(BINDIR)/mold', '$(LIBEXECDIR)/mold'))"` $D$(LIBEXECDIR)/mold/ld

 	$(INSTALL) -d $D$(MANDIR)/man1
 	$(INSTALL_DATA) docs/mold.1 $D$(MANDIR)/man1
@@ -27,8 +27,8 @@ mold currently supports x86-64, i386, ARM32, ARM64 and 64-bit RISC-V.
 ## Why does the speed of linking matter?

 If you are using a compiled language such as C, C++ or Rust, a build
-consists of two phases. In the first phase, a compiler compiles a
-source file into an object file (`.o` files). In the second phase,
+consists of two phases. In the first phase, a compiler compiles
+source files into object files (`.o` files). In the second phase,
 a linker takes all object files to combine them into a single executable
 or a shared library file.

@@ -47,32 +47,21 @@ Binary packages for the following distros are currently available.

 mold is written in C++20, so if you build mold yourself, you need a
 recent version of a C++ compiler and a C++ standard library. GCC 10.2
-or Clang 12.0.0 as well as libstdc++ 10 or libc++ 7 are recommended.
-
-I'm using Ubuntu 20.04 as a development platform. In that environment,
-you can build mold by the following commands.
+or Clang 12.0.0 (or later) as well as libstdc++ 10 or libc++ 7 (or
+later) are recommended.

 ### Install dependencies

-#### Ubuntu 20.04 and later / Debian 11 and later
-
-```shell
-sudo apt-get update
-sudo apt-get install -y build-essential git clang cmake libstdc++-10-dev libssl-dev libxxhash-dev zlib1g-dev pkg-config
-```
-
-#### Fedora 34 and later
-
-```shell
-sudo dnf install -y git clang cmake openssl-devel xxhash-devel zlib-devel libstdc++-devel
-```
+To install build dependencies, run `./install-build-deps.sh` in this
+directory. It recognizes your Linux distribution and tries to install
+necessary packages. You may want to run it as root.

 ### Compile mold

 ```shell
 git clone https://github.com/rui314/mold.git
 cd mold
-git checkout v1.2.0
+git checkout v1.2.1
 make -j$(nproc) CXX=clang++
 sudo make install
 ```
@@ -86,11 +75,10 @@ By default, `mold` is installed to `/usr/local/bin`.
 If you don't use a recent enough Linux distribution, or if for any reason
 `make` in the above commands doesn't work for you, you can use Docker to
 build it in a Docker environment. To do so, just run `./dist.sh` in this
-directory instead of running `make -j$(nproc)`. The shell script creates a
-Ubuntu 18.04 Docker image, installs necessary tools and libraries to it,
-builds mold and auxiliary files, and packs them into a single tar file
-`mold-$version-$arch-linux.tar.gz`. You can extract the tar file anywhere
-and use `mold` executable in it.
+directory instead of running `make -j$(nproc)`. The shell script pulls a
+Docker image, builds mold and auxiliary files inside it, and packs
+them into a single tar file `mold-$version-$arch-linux.tar.gz`.
+You can extract the tar file anywhere and use `mold` executable in it.

 `make test` depends on a few more packages. To install, run the following commands:

@@ -123,7 +111,7 @@ following flags to use `mold` instead of `/usr/bin/ld`:

  If you have installed mold with `make install`, there should be a
  directory named `/usr/libexec/mold` (or `/usr/local/libexec/mold`,
-  depending on your $PREFIX), and `ld` command should be there. The
+  depending on your `$PREFIX`), and `ld` command should be there. The
  `ld` is actually a symlink to `mold`. So, all you need is to pass
  `-B/usr/libexec/mold` (or `-B/usr/local/libexec/mold`) to GCC.

@@ -184,7 +172,7 @@ mold leaves its identification string in `.comment` section in an output
 file. You can print it out to verify that you are actually using mold.

 ```shell
-readelf -p .comment <executable-file>
+$ readelf -p .comment <executable-file>

 String dump of section '.comment':
  [     0]  GCC: (Ubuntu 10.2.0-5ubuntu1~20.04) 10.2.0
@@ -213,7 +201,3 @@ cores most of the time. In this demo, the maximum parallelism is
 artificially capped to 16 so that the bars fit in the GIF.

 For details, please read [design notes](docs/design.md).
-
-# Logo
-
-![mold image](docs/mold.jpg)
@@ -48,6 +48,7 @@ read_response_file(C &ctx, std::string_view path) {
  return vec;
 }

+// Replace "@path/to/some/text/file" with its file contents.
 template <typename C>
 std::vector<std::string_view> expand_response_files(C &ctx, char **argv) {
  std::vector<std::string_view> vec;
@@ -89,7 +89,8 @@ static std::vector<u8> do_compress(std::string_view input) {
  return buf;
 }

-ZlibCompressor::ZlibCompressor(std::string_view input) {
+ZlibCompressor::ZlibCompressor(u8 *buf, i64 size) {
+  std::string_view input{(char *)buf, (size_t)size};
  std::vector<std::string_view> inputs = split(input);
  std::vector<u64> adlers(inputs.size());
  shards.resize(inputs.size());
@@ -9,19 +9,19 @@

 set -e

-version=$(grep '^VERSION =' $(dirname $0)/Makefile | sed 's/.* = //')
+# Unlike Linux, macOS's uname returns arm64 for aarch64.
 arch=$(uname -m)
-dest=mold-$version-$arch-linux
+[ $arch = arm64 ] && arch=aarch64

-if [ $arch = x86_64 ]; then
-  image=rui314/mold-builder:v1-x86_64
-elif [ $arch = aarch64 -o $arch = arm64 ]; then
-  image=rui314/mold-builder:v1-aarch64
-else
+if [ $arch != x86_64 -a $arch != aarch64 ]; then
  echo "Error: no docker image for $arch"
  exit 1
 fi

+version=$(grep '^VERSION =' $(dirname $0)/Makefile | sed 's/.* = //')
+dest=mold-$version-$arch-linux
+image=rui314/mold-builder:v1-$arch
+
 docker images -q $image 2> /dev/null || docker pull $image

 docker run -it --rm -v "$(pwd):/mold:Z" -u "$(id -u):$(id -g)" $image \
@@ -30,6 +30,5 @@ cd /tmp/mold &&
 make clean &&
 make -j\$(nproc) CXX=clang++-14 CXXFLAGS='-I/openssl/include -O2 $CXXFLAGS' LDFLAGS='-static-libstdc++ /openssl/libcrypto.a' NEEDS_LIBCRYPTO=0 LTO=${LTO:-0} &&
 make install PREFIX=/ DESTDIR=$dest &&
-ln -sfr $dest/bin/mold $dest/libexec/mold/ld &&
 tar czf /mold/$dest.tar.gz $dest &&
 cp mold /mold"
@@ -473,6 +473,20 @@ Set
 .Ar dir
 to root directory.
 .Pp
+.It Fl -color-diagnostics Ns = Ns Op Sy auto | always | never
+.It Fl -color-diagnostics
+.It Fl -no-color-diagnostics
+.Pp
+Show diagnostics messages in color using ANSI escape sequences.
+.Ar auto
+means that
+.Nm
+prints out messages in color only if the standard output is connected to a TTY. \
+Default is
+.Ar auto .
+.Pp
+.It Fl -defsym Ns = Ns Ar symbol Ns = Ns Ar value
+.Pp
 .It Fl -compress-debug-sections Ns = Ns Op Sy none | zlib | zlib-gabi | zlib-gnu
 Compress DWARF debug info
 .Pf ( Sy .debug_*
@@ -555,6 +569,10 @@ Mark all symbols in the given
 .Ar libraries
 hidden.
 .Pp
+.It Fl -fatal-warnings
+.It Fl -no-fatal-warnings
+Treat warnings as errors.
+.Pp
 .It Fl -fini Ns = Ns Ar symbol
 Call
 .Ar symbol
@@ -574,6 +592,14 @@ hides that latency.
 .It Fl -no-gc-sections
 Remove unreferenced sections.
 .Pp
+.It Fl -gdb-index
+Create a
+.Li .gdb_index
+section to speed up GNU debugger. To use this, you need to compile source files \
+with the
+.Fl -ggnu-pubnames
+compiler flag.
+.Pp
 .It Fl -hash-style Ns = Ns Op Sy sysv | gnu | both
 Set hash style.
 .Pp
@@ -1044,20 +1070,15 @@ Mark object to interpose all DSOs but executable.
 .It Fl EL
 .It Fl O Ns Ar number
 .It Fl -allow-shlib-undefined
-.It Fl -color-diagnostics
 .It Fl -dc
 .It Fl -dp
 .It Fl -end-group
-.It Fl -fatal-warnings
-.It Fl -gdb-index
 .It Fl -no-add-needed
 .It Fl -no-allow-shlib-undefined
 .It Fl -no-copy-dt-needed-entries
 .It Fl -no-fatal-warnings
 .It Fl -no-undefined-version
 .It Fl -nostdlib
-.It Fl -plugin
-.It Fl -plugin-opt
 .It Fl -rpath-link Ns = Ns Ar dir
 .It Fl -sort-common
 .It Fl -sort-section
@@ -490,8 +490,8 @@ template <>
 void InputSection<E>::copy_contents_riscv(Context<E> &ctx, u8 *buf) {
  // A non-alloc section isn't relaxed, so just copy it as one big chunk.
  if (!(shdr().sh_flags & SHF_ALLOC)) {
-    if (is_compressed())
-      uncompress(ctx, buf);
+    if (compressed)
+      uncompress_to(ctx, buf);
    else
      memcpy(buf, contents.data(), contents.size());
    return;
@@ -77,8 +77,8 @@ Options:
    --no-eh-frame-hdr
  --enable-new-dtags          Ignored
  --exclude-libs LIB,LIB,..   Mark all symbols in given libraries hidden
-  --fatal-warnings            Ignored
-    --no-fatal-warnings       Ignored
+  --fatal-warnings            Treat warnings as errors
+    --no-fatal-warnings       Do not treat warnings as errors (default)
  --fini SYMBOL               Call SYMBOL at unload-time
  --fork                      Spawn a child process (default)
    --no-fork
@@ -153,6 +153,7 @@ Options:
  -z common-page-size=VALUE   Ignored
  -z execstack                Require executable stack
    -z noexecstack
+  -z execstack-if-needed      Make the stack area execuable if an input file explicitly requests it
  -z initfirst                Mark DSO to be initialized first at runtime
  -z interpose                Mark object to interpose all DSOs but executable
  -z keep-text-section-prefix Keep .text.{hot,unknown,unlikely,startup,exit} as separate sections in the final binary
@@ -722,6 +723,8 @@ void parse_nonpositional_args(Context<E> &ctx,
      ctx.arg.z_cet_report = CET_REPORT_ERROR;
    } else if (read_z_flag(args, "execstack")) {
      ctx.arg.z_execstack = true;
+    } else if (read_z_flag(args, "execstack-if-needed")) {
+      ctx.arg.z_execstack_if_needed = true;
    } else if (read_z_arg(ctx, args, arg, "max-page-size")) {
      ctx.page_size = parse_number(ctx, "-z max-page-size", arg);
      if (std::popcount<u64>(ctx.page_size) != 1)
@@ -868,6 +871,10 @@ void parse_nonpositional_args(Context<E> &ctx,
      ctx.arg.plugin_opt.push_back("thinlto-object-suffix-replace=" + std::string(arg));
    } else if (read_arg(ctx, args, arg, "thinlto-prefix-replace")) {
      ctx.arg.plugin_opt.push_back("thinlto-prefix-replace=" + std::string(arg));
+    } else if (read_arg(ctx, args, arg, "thinlto-cache-dir")) {
+      ctx.arg.plugin_opt.push_back("cache-dir=" + std::string(arg));
+    } else if (read_arg(ctx, args, arg, "thinlto-cache-policy")) {
+      ctx.arg.plugin_opt.push_back("cache-policy=" + std::string(arg));
    } else if (read_arg(ctx, args, arg, "thinlto-jobs")) {
      ctx.arg.plugin_opt.push_back("jobs=" + std::string(arg));
    } else if (read_arg(ctx, args, arg, "thread-count")) {
@@ -1105,10 +1112,6 @@ void parse_nonpositional_args(Context<E> &ctx,
  void parse_nonpositional_args(Context<E> &ctx,                        \
                                std::vector<std::string_view> &remaining)

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -0,0 +1,512 @@
+#include "mold.h"
+
+namespace mold::elf {
+
+// The hash function for .gdb_index.
+static u32 gdb_hash(std::string_view name) {
+  u32 h = 0;
+  for (u8 c : name) {
+    if ('A' <= c && c <= 'Z')
+      c = 'a' + c - 'A';
+    h = h * 67 + c - 113;
+  }
+  return h;
+}
+
+// Split .debug_info into so-called "compilation units". A .debug_info
+// section usually contains one compunit unless it was created by `ld -r`.
+// This is for --gdb-index.
+template <typename E>
+std::vector<std::string_view>
+read_compunits(Context<E> &ctx, ObjectFile<E> &file) {
+  file.debug_info->uncompress(ctx);
+  std::string_view data = file.debug_info->contents;
+  std::vector<std::string_view> vec;
+
+  while (!data.empty()) {
+    if (data.size() < 4)
+      Fatal(ctx) << *file.debug_info << ": corrupted .debug_info";
+    if (*(u32 *)data.data() == 0xffffffff)
+      Fatal(ctx) << *file.debug_info << ": --gdb-index: DWARF64 not supported";
+    i64 len = *(u32 *)data.data() + 4;
+    vec.push_back(data.substr(0, len));
+    data = data.substr(len);
+  }
+  return vec;
+}
+
+// Parses .debug_gnu_pubnames and .debug_gnu_pubtypes. These sections
+// start with a 14 bytes header followed by (4-byte offset, 1-byte type,
+// null-terminated string) tuples.
+//
+// The 4-byte offset is an offset into .debug_info that contains details
+// about the name. The 1-byte type is a type of the corresponding name
+// (e.g. function, variable or datatype). The string is a name of a
+// function, a variable or a type.
+template <typename E>
+std::vector<GdbIndexName> read_pubnames(Context<E> &ctx, ObjectFile<E> &file) {
+  std::vector<GdbIndexName> vec;
+
+  auto get_cu_idx = [&](InputSection<E> &isec, i64 offset) {
+    i64 off = 0;
+    for (i64 i = 0; i < file.compunits.size(); i++) {
+      if (offset == off)
+        return file.compunits_idx + i;
+      off += file.compunits[i].size();
+    }
+    Fatal(ctx) << isec << ": corrupted debug_info_offset";
+  };
+
+  auto read = [&](InputSection<E> &isec) {
+    isec.uncompress(ctx);
+    std::string_view contents = isec.contents;
+
+    while (!contents.empty()) {
+      if (contents.size() < 14)
+        Fatal(ctx) << isec << ": corrupted header";
+
+      u32 len = *(u32 *)contents.data() + 4;
+      u32 debug_info_offset = *(u32 *)(contents.data() + 6);
+      u32 cu_idx = get_cu_idx(isec, debug_info_offset);
+
+      std::string_view data = contents.substr(14, len - 14);
+      contents = contents.substr(len);
+
+      while (!data.empty()) {
+        u32 offset = *(u32 *)data.data();
+        data = data.substr(4);
+        if (offset == 0)
+          break;
+
+        u8 type = data[0];
+        data = data.substr(1);
+
+        std::string_view name = data.data();
+        data = data.substr(name.size() + 1);
+
+        vec.push_back({name, gdb_hash(name), (type << 24) | cu_idx});
+      }
+    }
+  };
+
+  if (file.debug_pubnames)
+    read(*file.debug_pubnames);
+  if (file.debug_pubtypes)
+    read(*file.debug_pubtypes);
+
+  // Uniquify elements because GCC 11 seems to emit one record for each
+  // comdat group which results in having a lot of duplicate records.
+  std::sort(vec.begin(), vec.end(),
+            [](const GdbIndexName &a, const GdbIndexName &b) {
+    return std::tuple{a.hash, a.attr, a.name} <
+           std::tuple{b.hash, b.attr, b.name};
+  });
+
+  auto last = std::unique(vec.begin(), vec.end(),
+                          [](const GdbIndexName &a, const GdbIndexName &b) {
+    return std::tuple{a.hash, a.attr, a.name} ==
+           std::tuple{b.hash, b.attr, b.name};
+  });
+
+  vec.erase(last, vec.end());
+  return vec;
+}
+
+template <typename E>
+static u8 *get_buffer(Context<E> &ctx, Chunk<E> *chunk) {
+  if (u8 *buf = chunk->get_uncompressed_data())
+    return buf;
+  return ctx.buf + chunk->shdr.sh_offset;
+}
+
+// Try to find a compilation unit from .debug_info and its
+// corresponding record from .debug_abbrev and returns them.
+template <typename E>
+static std::tuple<u8 *, u8 *, u32>
+find_compunit(Context<E> &ctx, ObjectFile<E> &file, i64 offset) {
+  // Read .debug_info to find the record at a given offset.
+  u8 *cu = get_buffer(ctx, ctx.debug_info) + offset;
+  u32 dwarf_version = *(u16 *)(cu + 4);
+  u32 abbrev_offset;
+
+  // Skip a header.
+  switch (dwarf_version) {
+  case 2:
+  case 3:
+  case 4:
+    abbrev_offset = *(u32 *)(cu + 6);
+    if (u32 address_size = cu[10]; address_size != E::word_size)
+      Fatal(ctx) << file << ": --gdb-index: unsupported address size "
+                 << address_size;
+    cu += 11;
+    break;
+  case 5: {
+    abbrev_offset = *(u32 *)(cu + 8);
+    if (u32 address_size = cu[7]; address_size != E::word_size)
+      Fatal(ctx) << file << ": --gdb-index: unsupported address size "
+                 << address_size;
+
+    switch (u32 unit_type = cu[6]; unit_type) {
+    case DW_UT_compile:
+    case DW_UT_partial:
+      cu += 12;
+      break;
+    case DW_UT_skeleton:
+    case DW_UT_split_compile:
+      cu += 20;
+      break;
+    default:
+      Fatal(ctx) << file << ": --gdb-index: unknown DW_UT_* value: 0x"
+                 << std::hex << unit_type;
+    }
+    break;
+  }
+  default:
+    Fatal(ctx) << file << ": --gdb-index: unknown DWARF version: "
+               << dwarf_version;
+  }
+
+  u32 abbrev_code = read_uleb(cu);
+
+  // Find a .debug_abbrev record corresponding to the .debug_info record.
+  // We assume the .debug_info record at a given offset is of
+  // DW_TAG_compile_unit which describes a compunit.
+  u8 *abbrev = get_buffer(ctx, ctx.debug_abbrev) + abbrev_offset;
+
+  for (;;) {
+    u32 code = read_uleb(abbrev);
+    if (code == 0)
+      Fatal(ctx) << file << ": --gdb-index: .debug_abbrev does not contain"
+                 << " a record for the first .debug_info record";
+
+    if (code == abbrev_code) {
+      // Found a record
+      u64 abbrev_tag = read_uleb(abbrev);
+      if (abbrev_tag != DW_TAG_compile_unit && abbrev_tag != DW_TAG_skeleton_unit)
+        Fatal(ctx) << file << ": --gdb-index: the first entry's tag is not"
+                   << " DW_TAG_compile_unit/DW_TAG_skeleton_unit but 0x"
+                   << std::hex << abbrev_tag;
+      break;
+    }
+
+    // Skip an uninteresting record
+    read_uleb(abbrev); // tag
+    abbrev++; // has_children byte
+    for (;;) {
+      u64 name = read_uleb(abbrev);
+      u64 form = read_uleb(abbrev);
+      if (name == 0 && form == 0)
+        break;
+      if (form == DW_FORM_implicit_const)
+        read_uleb(abbrev);
+    }
+  }
+
+  abbrev++; // skip has_children byte
+  return {cu, abbrev, dwarf_version};
+}
+
+// Estimate the number of address ranges contained in a given file.
+// It may over-estimate but never under-estimate.
+template <typename E>
+i64 estimate_address_areas(Context<E> &ctx, ObjectFile<E> &file) {
+  // Each CU contains zero or one address area.
+  i64 ret = file.compunits.size();
+
+  // In DWARF 4, a CU can refer address ranges in .debug_ranges.
+  // .debug_ranges contains a vector of [begin, end) address pairs.
+  // The last entry must be a null terminator, so we do -1.
+  if (file.debug_ranges)
+    ret += file.debug_ranges->sh_size / E::word_size / 2 - 1;
+
+  // In DWARF 5, a CU can refer address ranges in .debug_rnglists, which
+  // contains variable-length entries. The smallest possible range entry
+  // is one byte for the code and two ULEB128 values (each can be as
+  // small as one byte), so 3 bytes.
+  if (file.debug_rnglists)
+    ret += file.debug_rnglists->sh_size / 3;
+  return ret;
+}
+
+// .debug_info contains variable-length fields. This class reads them.
+template <typename E>
+class DebugInfoReader {
+public:
+  DebugInfoReader(Context<E> &ctx, ObjectFile<E> &file, u8 *cu)
+    : ctx(ctx), file(file), cu(cu) {}
+
+  u64 read(u64 form);
+
+  Context<E> &ctx;
+  ObjectFile<E> &file;
+  u8 *cu;
+};
+
+// Read value of the given DW_FORM_* form. If a value is not scalar,
+// returns a dummy value 0.
+template <typename E>
+inline u64 DebugInfoReader<E>::read(u64 form) {
+  switch (form) {
+  case DW_FORM_flag_present:
+    return 0;
+  case DW_FORM_data1:
+  case DW_FORM_flag:
+  case DW_FORM_strx1:
+  case DW_FORM_addrx1:
+  case DW_FORM_ref1:
+    return *cu++;
+  case DW_FORM_data2:
+  case DW_FORM_strx2:
+  case DW_FORM_addrx2:
+  case DW_FORM_ref2: {
+    u64 val = *(u16 *)cu;
+    cu += 2;
+    return val;
+  }
+  case DW_FORM_strx3:
+  case DW_FORM_addrx3: {
+    u64 val = 0;
+    memcpy(&val, cu, 3); // This assumes little-endian.
+    cu += 3;
+    return val;
+  }
+  case DW_FORM_data4:
+  case DW_FORM_strp:
+  case DW_FORM_sec_offset:
+  case DW_FORM_line_strp:
+  case DW_FORM_strx4:
+  case DW_FORM_addrx4:
+  case DW_FORM_ref4: {
+    u64 val = *(u32 *)cu;
+    cu += 4;
+    return val;
+  }
+  case DW_FORM_data8:
+  case DW_FORM_ref8: {
+    u64 val = *(u64 *)cu;
+    cu += 8;
+    return val;
+  }
+  case DW_FORM_addr:
+  case DW_FORM_ref_addr: {
+    u64 val = *(typename E::WordTy *)cu;
+    cu += E::word_size;
+    return val;
+  }
+  case DW_FORM_strx:
+  case DW_FORM_addrx:
+  case DW_FORM_udata:
+  case DW_FORM_ref_udata:
+  case DW_FORM_loclistx:
+  case DW_FORM_rnglistx:
+    return read_uleb(cu);
+  case DW_FORM_string:
+    cu += strlen((char *)cu) + 1;
+    return 0;
+  default:
+    Fatal(ctx) << file << ": --gdb-index: unhandled debug info form: 0x"
+               << std::hex << form;
+    return 0;
+  }
+}
+
+// Read a range list from .debug_ranges starting at the given offset
+// (until an end of list entry).
+template <typename E>
+static std::vector<u64>
+read_debug_range(Context<E> &ctx, ObjectFile<E> &file, typename E::WordTy *range) {
+  std::vector<u64> vec;
+  u64 base = 0;
+
+  for (i64 i = 0; range[i] || range[i + 1]; i += 2) {
+    if (range[i] == (typename E::WordTy)-1) {
+      // base address selection entry
+      base = range[i + 1];
+    } else {
+      vec.push_back(range[i] + base);
+      vec.push_back(range[i + 1] + base);
+    }
+  }
+  return vec;
+}
+
+// Read a range list from .debug_rnglists starting at the given offset
+// (until an end of list entry).
+template <typename E>
+static std::vector<u64>
+read_rnglist_range(Context<E> &ctx, ObjectFile<E> &file, u8 *rnglist,
+                   typename E::WordTy *addrx) {
+  std::vector<u64> vec;
+  u64 base = 0;
+
+  for (;;) {
+    switch (*rnglist++) {
+    case DW_RLE_end_of_list:
+      return vec;
+    case DW_RLE_base_addressx:
+      base = addrx[read_uleb(rnglist)];
+      break;
+    case DW_RLE_startx_endx:
+      vec.push_back(addrx[read_uleb(rnglist)]);
+      vec.push_back(addrx[read_uleb(rnglist)]);
+      break;
+    case DW_RLE_startx_length:
+      vec.push_back(addrx[read_uleb(rnglist)]);
+      vec.push_back(vec.back() + read_uleb(rnglist));
+      break;
+    case DW_RLE_offset_pair:
+      vec.push_back(base + read_uleb(rnglist));
+      vec.push_back(base + read_uleb(rnglist));
+      break;
+    case DW_RLE_base_address:
+      base = *(typename E::WordTy *)rnglist;
+      rnglist += E::word_size;
+      break;
+    case DW_RLE_start_end:
+      vec.push_back(*(typename E::WordTy *)rnglist);
+      rnglist += E::word_size;
+      vec.push_back(*(typename E::WordTy *)rnglist);
+      rnglist += E::word_size;
+      break;
+    case DW_RLE_start_length:
+      vec.push_back(*(typename E::WordTy *)rnglist);
+      rnglist += E::word_size;
+      vec.push_back(vec.back() + read_uleb(rnglist));
+      break;
+    }
+  }
+}
+
+// Returns a list of address ranges explained by a compunit at the
+// `offset` in an output .debug_info section.
+//
+// .debug_info contains DWARF debug info records, so this function
+// parses DWARF. If a designated compunit contains multiple ranges, the
+// ranges are read from .debug_ranges (or .debug_rnglists for DWARF5).
+// Otherwise, a range is read directly from .debug_info (or possibly
+// from .debug_addr for DWARF5).
+template <typename E>
+std::vector<u64>
+read_address_areas(Context<E> &ctx, ObjectFile<E> &file, i64 offset) {
+  u8 *cu;
+  u8 *abbrev;
+  u32 dwarf_version;
+  std::tie(cu, abbrev, dwarf_version) = find_compunit(ctx, file, offset);
+
+  DebugInfoReader<E> reader{ctx, file, cu};
+
+  struct Record {
+    u64 form = 0;
+    u64 value = 0;
+  };
+
+  Record low_pc;
+  Record high_pc;
+  Record ranges;
+  std::optional<u64> rnglists_base;
+  typename E::WordTy *addrx = nullptr;
+
+  // Read all interesting debug records.
+  for (;;) {
+    u64 name = read_uleb(abbrev);
+    u64 form = read_uleb(abbrev);
+    if (name == 0 && form == 0)
+      break;
+
+    u64 val = reader.read(form);
+
+    switch (name) {
+    case DW_AT_low_pc:
+      low_pc = {form, val};
+      break;
+    case DW_AT_high_pc:
+      high_pc = {form, val};
+      break;
+    case DW_AT_rnglists_base:
+      rnglists_base = val;
+      break;
+    case DW_AT_addr_base:
+      addrx = (typename E::WordTy *)(get_buffer(ctx, ctx.debug_addr) + val);
+      break;
+    case DW_AT_ranges:
+      ranges = {form, val};
+      break;
+    }
+  }
+
+  // Handle non-contiguous address ranges.
+  if (ranges.form) {
+    if (dwarf_version <= 4) {
+      typename E::WordTy *range_begin =
+        (typename E::WordTy *)(get_buffer(ctx, ctx.debug_ranges) + ranges.value);
+      return read_debug_range(ctx, file, range_begin);
+    }
+
+    assert(dwarf_version == 5);
+
+    u8 *buf = get_buffer(ctx, ctx.debug_rnglists);
+    if (ranges.form == DW_FORM_sec_offset)
+      return read_rnglist_range(ctx, file, buf + ranges.value, addrx);
+
+    if (!rnglists_base)
+      Fatal(ctx) << file << ": --gdb-index: missing DW_AT_rnglists_base";
+
+    u8 *base = buf + *rnglists_base;
+    return read_rnglist_range(ctx, file, base + *(u32 *)base, addrx);
+  }
+
+  // Handle a contiguous address range.
+  if (low_pc.form && high_pc.form) {
+    u64 lo;
+
+    switch (low_pc.form) {
+    case DW_FORM_addr:
+      lo = low_pc.value;
+      break;
+    case DW_FORM_addrx:
+    case DW_FORM_addrx1:
+    case DW_FORM_addrx2:
+    case DW_FORM_addrx4:
+      lo = addrx[low_pc.value];
+      break;
+    default:
+      Fatal(ctx) << file << ": --gdb-index: unhandled form for DW_AT_low_pc: 0x"
+                 << std::hex << high_pc.form;
+    }
+
+    switch (high_pc.form) {
+    case DW_FORM_addr:
+      return {lo, high_pc.value};
+    case DW_FORM_addrx:
+    case DW_FORM_addrx1:
+    case DW_FORM_addrx2:
+    case DW_FORM_addrx4:
+      return {lo, addrx[high_pc.value]};
+    case DW_FORM_udata:
+    case DW_FORM_data1:
+    case DW_FORM_data2:
+    case DW_FORM_data4:
+    case DW_FORM_data8:
+      return {lo, lo + high_pc.value};
+    default:
+      Fatal(ctx) << file << ": --gdb-index: unhandled form for DW_AT_high_pc: 0x"
+                 << std::hex << high_pc.form;
+    }
+  }
+
+  return {};
+}
+
+#define INSTANTIATE(E)                                                  \
+  template std::vector<std::string_view>                                \
+  read_compunits(Context<E> &, ObjectFile<E> &);                        \
+  template std::vector<GdbIndexName>                                    \
+  read_pubnames(Context<E> &, ObjectFile<E> &);                         \
+  template i64                                                          \
+  estimate_address_areas(Context<E> &, ObjectFile<E> &);                \
+  template std::vector<u64>                                             \
+  read_address_areas(Context<E> &, ObjectFile<E> &, i64)
+
+INSTANTIATE_ALL;
+
+} // namespace mold::elf
@@ -1054,8 +1054,16 @@ static constexpr u32 DW_AT_low_pc = 0x11;
 static constexpr u32 DW_AT_high_pc = 0x12;
 static constexpr u32 DW_AT_producer = 0x25;
 static constexpr u32 DW_AT_ranges = 0x55;
+static constexpr u32 DW_AT_addr_base = 0x73;
+static constexpr u32 DW_AT_rnglists_base = 0x74;

 static constexpr u32 DW_TAG_compile_unit = 0x11;
+static constexpr u32 DW_TAG_skeleton_unit = 0x4a;
+
+static constexpr u32 DW_UT_compile = 0x01;
+static constexpr u32 DW_UT_partial = 0x03;
+static constexpr u32 DW_UT_skeleton = 0x04;
+static constexpr u32 DW_UT_split_compile = 0x05;

 static constexpr u32 DW_FORM_addr = 0x01;
 static constexpr u32 DW_FORM_block2 = 0x03;
@@ -1081,6 +1089,34 @@ static constexpr u32 DW_FORM_indirect = 0x16;
 static constexpr u32 DW_FORM_sec_offset = 0x17;
 static constexpr u32 DW_FORM_exprloc = 0x18;
 static constexpr u32 DW_FORM_flag_present = 0x19;
+static constexpr u32 DW_FORM_strx = 0x1a;
+static constexpr u32 DW_FORM_addrx = 0x1b;
+static constexpr u32 DW_FORM_ref_sup4 = 0x1c;
+static constexpr u32 DW_FORM_strp_sup = 0x1d;
+static constexpr u32 DW_FORM_data16 = 0x1e;
+static constexpr u32 DW_FORM_line_strp = 0x1f;
+static constexpr u32 DW_FORM_ref_sig8 = 0x20;
+static constexpr u32 DW_FORM_implicit_const = 0x21;
+static constexpr u32 DW_FORM_loclistx = 0x22;
+static constexpr u32 DW_FORM_rnglistx = 0x23;
+static constexpr u32 DW_FORM_ref_sup8 = 0x24;
+static constexpr u32 DW_FORM_strx1 = 0x25;
+static constexpr u32 DW_FORM_strx2 = 0x26;
+static constexpr u32 DW_FORM_strx3 = 0x27;
+static constexpr u32 DW_FORM_strx4 = 0x28;
+static constexpr u32 DW_FORM_addrx1 = 0x29;
+static constexpr u32 DW_FORM_addrx2 = 0x2a;
+static constexpr u32 DW_FORM_addrx3 = 0x2b;
+static constexpr u32 DW_FORM_addrx4 = 0x2c;
+
+static constexpr u32 DW_RLE_end_of_list = 0x00;
+static constexpr u32 DW_RLE_base_addressx = 0x01;
+static constexpr u32 DW_RLE_startx_endx = 0x02;
+static constexpr u32 DW_RLE_startx_length = 0x03;
+static constexpr u32 DW_RLE_offset_pair = 0x04;
+static constexpr u32 DW_RLE_base_address = 0x05;
+static constexpr u32 DW_RLE_start_end = 0x06;
+static constexpr u32 DW_RLE_start_length = 0x07;

 struct Elf64Sym {
  bool is_defined() const { return !is_undef(); }
@@ -1318,7 +1354,6 @@ struct X86_64 {
  static constexpr u32 e_machine = EM_X86_64;
  static constexpr u32 pltgot_size = 8;
  static constexpr bool is_rel = false;
-  static constexpr bool is_le = true;
  static constexpr bool supports_tlsdesc = true;
 };

@@ -1350,7 +1385,6 @@ struct I386 {
  static constexpr u32 e_machine = EM_386;
  static constexpr u32 pltgot_size = 8;
  static constexpr bool is_rel = true;
-  static constexpr bool is_le = true;
  static constexpr bool supports_tlsdesc = true;
 };

@@ -1382,7 +1416,6 @@ struct ARM64 {
  static constexpr u32 e_machine = EM_AARCH64;
  static constexpr u32 pltgot_size = 16;
  static constexpr bool is_rel = false;
-  static constexpr bool is_le = true;
  static constexpr bool supports_tlsdesc = true;
 };

@@ -1414,7 +1447,6 @@ struct ARM32 {
  static constexpr u32 e_machine = EM_ARM;
  static constexpr u32 pltgot_size = 16;
  static constexpr bool is_rel = true;
-  static constexpr bool is_le = true;
  static constexpr bool supports_tlsdesc = true;
 };

@@ -1445,7 +1477,6 @@ struct RISCV64 {
  static constexpr u32 e_machine = EM_RISCV;
  static constexpr u32 pltgot_size = 16;
  static constexpr bool is_rel = false;
-  static constexpr bool is_le = true;
  static constexpr bool supports_tlsdesc = false;
 };

@@ -201,10 +201,6 @@ void gc_sections(Context<E> &ctx) {
 #define INSTANTIATE(E)                                  \
  template void gc_sections(Context<E> &ctx);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -588,10 +588,6 @@ void icf_sections(Context<E> &ctx) {
 #define INSTANTIATE(E)                          \
  template void icf_sections(Context<E> &ctx);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -133,6 +133,11 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
      const ElfSym<E> &sym = this->elf_syms[shdr.sh_info];
      std::string_view signature = symbol_strtab.data() + sym.st_name;

+      // Ignore a broken comdat group GCC emits for .debug_macros.
+      // https://github.com/rui314/mold/issues/438
+      if (signature.starts_with("wm4."))
+        continue;
+
      // Get comdat group members.
      std::span<u32> entries = this->template get_data<u32>(ctx, shdr);

@@ -167,11 +172,14 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
      // making the stack area executable is too dangerous. Tell our
      // users about the difference if that matters.
      if (name == ".note.GNU-stack") {
-        if (shdr.sh_flags & SHF_EXECINSTR)
-          Warn(ctx) << *this << ": this file may cause a segmentation"
-            " fault because it requires an executable stack. See"
-            " https://github.com/rui314/mold/tree/main/docs/execstack.md"
-            " for more info.";
+        if (shdr.sh_flags & SHF_EXECINSTR) {
+          if (!ctx.arg.z_execstack && !ctx.arg.z_execstack_if_needed)
+            Warn(ctx) << *this << ": this file may cause a segmentation"
+              " fault because it requires an executable stack. See"
+              " https://github.com/rui314/mold/tree/main/docs/execstack.md"
+              " for more info.";
+          needs_executable_stack = true;
+        }
        continue;
      }

@@ -205,10 +213,10 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {

        if (name == ".debug_info")
          debug_info = isec;
-        if (name == ".debug_abbrev")
-          debug_abbrev = isec;
        if (name == ".debug_ranges")
          debug_ranges = isec;
+        if (name == ".debug_rnglists")
+          debug_rnglists = isec;

        // If --gdb-index is given, contents of .debug_gnu_pubnames and
        // .debug_gnu_pubtypes are copied to .gdb_index, so keeping them
@@ -222,6 +230,16 @@ void ObjectFile<E>::initialize_sections(Context<E> &ctx) {
          debug_pubtypes = isec;
          isec->is_alive = false;
        }
+
+        // .debug_types is similar to .debug_info but contains type info
+        // only. It exists only in DWARF 4, has been removed in DWARF 5 and
+        // neither GCC nor Clang generate it by default
+        // (-fdebug-types-section is needed). As such there is probably
+        // little need to support it.
+        if (name == ".debug_types")
+          Fatal(ctx) << *this << ": mold's --gdb-index is not compatible"
+            " with .debug_types; to fix this error, remove"
+            " -fdebug-types-section and recompile";
      }

      static Counter counter("regular_sections");
@@ -524,16 +542,10 @@ split_section(Context<E> &ctx, InputSection<E> &sec) {
                                               sec.shdr().sh_flags);
  rec->p2align = sec.p2align;

-  std::string_view data = sec.contents;
-
  // If thes section contents are compressed, uncompress them.
-  if (sec.is_compressed()) {
-    u8 *buf = new u8[sec.sh_size];
-    sec.uncompress(ctx, buf);
-    data = {(char *)buf, sec.sh_size};
-    ctx.string_pool.emplace_back(buf);
-  }
+  sec.uncompress(ctx);

+  std::string_view data = sec.contents;
  const char *begin = data.data();
  u64 entsize = sec.shdr().sh_entsize;
  HyperLogLog estimator;
@@ -734,6 +746,8 @@ void ObjectFile<E>::parse(Context<E> &ctx) {
  symtab_sec = this->find_section(SHT_SYMTAB);

  if (symtab_sec) {
+    // In ELF, all local symbols precede global symbols in the symbol table.
+    // sh_info has an index of the first global symbol.
    this->first_global = symtab_sec->sh_info;
    this->elf_syms = this->template get_data<ElfSym<E>>(ctx, *symtab_sec);
    symbol_strtab = this->get_string(ctx, symtab_sec->sh_link);
@@ -1063,10 +1077,14 @@ void ObjectFile<E>::convert_common_symbols(Context<E> &ctx) {
  if (!has_common_symbol)
    return;

-  OutputSection<E> *osec =
+  OutputSection<E> *common =
    OutputSection<E>::get_instance(ctx, ".common", SHT_NOBITS,
                                   SHF_WRITE | SHF_ALLOC);

+  OutputSection<E> *tls_common =
+    OutputSection<E>::get_instance(ctx, ".tls_common", SHT_NOBITS,
+                                   SHF_WRITE | SHF_ALLOC | SHF_TLS);
+
  for (i64 i = this->first_global; i < this->elf_syms.size(); i++) {
    if (!this->elf_syms[i].is_common())
      continue;
@@ -1082,17 +1100,20 @@ void ObjectFile<E>::convert_common_symbols(Context<E> &ctx) {

    elf_sections2.push_back({});
    ElfShdr<E> &shdr = elf_sections2.back();
-
    memset(&shdr, 0, sizeof(shdr));
-    shdr.sh_flags = SHF_ALLOC;
+
+    bool is_tls = (sym.get_type() == STT_TLS);
+    shdr.sh_flags = is_tls ? (SHF_ALLOC | SHF_TLS) : SHF_ALLOC;
    shdr.sh_type = SHT_NOBITS;
    shdr.sh_size = this->elf_syms[i].st_size;
    shdr.sh_addralign = this->elf_syms[i].st_value;

    i64 idx = this->elf_sections.size() + elf_sections2.size() - 1;
    std::unique_ptr<InputSection<E>> isec =
-      std::make_unique<InputSection<E>>(ctx, *this, ".common", idx);
-    isec->output_section = osec;
+      std::make_unique<InputSection<E>>(ctx, *this,
+                                        is_tls ? ".tls_common" : ".common",
+                                        idx);
+    isec->output_section = is_tls ? tls_common : common;

    sym.file = this;
    sym.shndx = idx;
@@ -1187,9 +1208,9 @@ void ObjectFile<E>::write_symtab(Context<E> &ctx) {
    esym.st_name = strtab_off;

    if (sym.get_type() == STT_TLS)
-      esym.st_value = sym.get_addr(ctx) - ctx.tls_begin;
+      esym.st_value = sym.get_addr(ctx, false) - ctx.tls_begin;
    else
-      esym.st_value = sym.get_addr(ctx);
+      esym.st_value = sym.get_addr(ctx, false);

    if (InputSection<E> *isec = sym.get_input_section())
      esym.st_shndx = isec->output_section->shndx;
@@ -1251,6 +1272,7 @@ SharedFile<E>::create(Context<E> &ctx, MappedFile<Context<E>> *mf) {
 template <typename E>
 SharedFile<E>::SharedFile(Context<E> &ctx, MappedFile<Context<E>> *mf)
  : InputFile<E>(ctx, mf) {
+  this->is_needed = ctx.as_needed;
  this->is_alive = !ctx.as_needed;
 }

@@ -1460,10 +1482,6 @@ void SharedFile<E>::write_symtab(Context<E> &ctx) {
  template class SharedFile<E>;                                         \
  template std::ostream &operator<<(std::ostream &, const InputFile<E> &)

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -36,9 +36,7 @@ InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file,
                              std::string_view name, i64 shndx)
  : file(file), shndx(shndx) {
  if (shndx < file.elf_sections.size())
-    contents = {(char *)file.mf->data + shdr().sh_offset, shdr().sh_size};
-
-  bool compressed;
+    contents = {(char *)file.mf->data + shdr().sh_offset, (size_t)shdr().sh_size};

  if (name.starts_with(".zdebug")) {
    sh_size = *(ubig64 *)&contents[4];
@@ -55,28 +53,37 @@ InputSection<E>::InputSection(Context<E> &ctx, ObjectFile<E> &file,
    compressed = false;
  }

-  // Uncompress early if the relocation is REL-type so that we can read
-  // addends from section contents. If RELA-type, we don't need to do this
-  // because addends are in relocations.
-  if (compressed && E::is_rel) {
-    u8 *buf = new u8[sh_size];
-    uncompress(ctx, buf);
-    contents = {(char *)buf, sh_size};
-    ctx.string_pool.emplace_back(buf);
-  }
+  // Sections may have been compressed. We usually uncompress them
+  // directly into the mmap'ed output file, but we want to uncompress
+  // early for REL-type ELF types to read relocation addends from
+  // section contents. For RELA-type, we don't need to do this because
+  // addends are in relocations.
+  if (E::is_rel)
+    uncompress(ctx);

  output_section =
    OutputSection<E>::get_instance(ctx, name, shdr().sh_type, shdr().sh_flags);
 }

 template <typename E>
-bool InputSection<E>::is_compressed() {
-  return !E::is_rel &&
-         (name().starts_with(".zdebug") || (shdr().sh_flags & SHF_COMPRESSED));
+void InputSection<E>::uncompress(Context<E> &ctx) {
+  if (!compressed || uncompressed)
+    return;
+
+  u8 *buf = new u8[sh_size];
+  uncompress_to(ctx, buf);
+  contents = {(char *)buf, sh_size};
+  ctx.string_pool.emplace_back(buf);
+  uncompressed = true;
 }

 template <typename E>
-void InputSection<E>::uncompress(Context<E> &ctx, u8 *buf) {
+void InputSection<E>::uncompress_to(Context<E> &ctx, u8 *buf) {
+  if (!compressed || uncompressed) {
+    memcpy(buf, contents.data(), contents.size());
+    return;
+  }
+
  auto do_uncompress = [&](std::string_view data) {
    unsigned long size = sh_size;
    if (::uncompress(buf, &size, (u8 *)data.data(), data.size()) != Z_OK)
@@ -100,7 +107,8 @@ void InputSection<E>::uncompress(Context<E> &ctx, u8 *buf) {

  ElfChdr<E> &hdr = *(ElfChdr<E> *)&contents[0];
  if (hdr.ch_type != ELFCOMPRESS_ZLIB)
-    Fatal(ctx) << *this << ": unsupported compression type";
+    Fatal(ctx) << *this << ": unsupported compression type: 0x"
+               << std::hex << hdr.ch_type;
  do_uncompress(contents.substr(sizeof(ElfChdr<E>)));
 }

@@ -213,8 +221,8 @@ void InputSection<E>::write_to(Context<E> &ctx, u8 *buf) {
  // Copy data
  if constexpr (std::is_same_v<E, RISCV64>) {
    copy_contents_riscv(ctx, buf);
-  } else if (is_compressed()) {
-    uncompress(ctx, buf);
+  } else if (compressed) {
+    uncompress_to(ctx, buf);
  } else {
    memcpy(buf, contents.data(), contents.size());
  }
@@ -249,10 +257,6 @@ void report_undef(Context<E> &ctx, InputFile<E> &file, Symbol<E> &sym) {
  template void report_undef(Context<E> &, InputFile<E> &, Symbol<E> &)


-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -405,10 +405,6 @@ void parse_dynamic_list(Context<E> &ctx, std::string path) {
  template                                                              \
  void parse_dynamic_list(Context<E> &ctx, std::string path)

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -86,6 +86,7 @@
 #include <dlfcn.h>
 #include <fcntl.h>
 #include <sstream>
+#include <tbb/parallel_for_each.h>
 #include <unistd.h>

 #if 0
@@ -614,21 +615,21 @@ std::vector<ObjectFile<E> *> do_lto(Context<E> &ctx) {
  phase = 2;

  // Set `referenced_by_regular_obj` bit.
-  for (ObjectFile<E> *file : ctx.objs) {
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    if (file->is_lto_obj)
-      continue;
+      return;

    for (i64 i = file->first_global; i < file->symbols.size(); i++) {
      ElfSym<E> &esym = file->elf_syms[i];
      Symbol<E> &sym = *file->symbols[i];

-      if (esym.is_undef() && sym.file && sym.file != file &&
-          !sym.file->is_dso && ((ObjectFile<E> *)sym.file)->is_lto_obj) {
+      if (sym.file && !sym.file->is_dso &&
+          ((ObjectFile<E> *)sym.file)->is_lto_obj) {
        std::scoped_lock lock(sym.mu);
        sym.referenced_by_regular_obj = true;
      }
    }
-  }
+  });

  // all_symbols_read_hook() calls add_input_file() and add_input_library()
  LOG << "all symbols read\n";
@@ -652,10 +653,6 @@ void lto_cleanup(Context<E> &ctx) {
  template std::vector<ObjectFile<E> *> do_lto(Context<E> &);           \
  template void lto_cleanup(Context<E> &)

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -382,6 +382,7 @@ static int elf_main(int argc, char **argv) {

  // Redo if -m is not x86-64.
  if (ctx.arg.emulation != E::e_machine) {
+#if !MOLD_DEBUG_X86_64_ONLY && !MOLD_DEBUG_ARM64_ONLY
    switch (ctx.arg.emulation) {
    case EM_386:
      return elf_main<I386>(argc, argv);
@@ -392,7 +393,8 @@ static int elf_main(int argc, char **argv) {
    case EM_RISCV:
      return elf_main<RISCV64>(argc, argv);
    }
-    unreachable();
+#endif
+    Fatal(ctx) << "unknown emulation: " << ctx.arg.emulation;
  }

  Timer t_all(ctx, "all");
@@ -521,6 +523,12 @@ static int elf_main(int argc, char **argv) {
  if (ctx.arg.z_cet_report != CET_REPORT_NONE)
    check_cet_errors(ctx);

+  // Handle `-z execstack-if-needed`.
+  if (ctx.arg.z_execstack_if_needed)
+    for (ObjectFile<E> *file : ctx.objs)
+      if (file->needs_executable_stack)
+        ctx.arg.z_execstack = true;
+
  // If we are linking a .so file, remaining undefined symbols does
  // not cause a linker error. Instead, they are treated as if they
  // were imported symbols.
@@ -532,6 +540,12 @@ static int elf_main(int argc, char **argv) {

  // Beyond this point, no new symbols will be added to the result.

+  // Handle --print-dependencies
+  if (ctx.arg.print_dependencies == 1)
+    print_dependencies(ctx);
+  else if (ctx.arg.print_dependencies == 2)
+    print_dependencies_full(ctx);
+
  // Handle -repro
  if (ctx.arg.repro)
    write_repro_file(ctx);
@@ -707,12 +721,11 @@ static int elf_main(int argc, char **argv) {
  // Zero-clear paddings between sections
  clear_padding(ctx);

-  if (ctx.buildid) {
-    Timer t(ctx, "build_id");
+  if (ctx.buildid)
    ctx.buildid->write_buildid(ctx);
-  }

  t_copy.stop();
+  ctx.checkpoint();

  // Close the output file. This is the end of the linker's main job.
  ctx.output_file->close(ctx);
@@ -721,12 +734,6 @@ static int elf_main(int argc, char **argv) {
  if (!ctx.arg.dependency_file.empty())
    write_dependency_file(ctx);

-  // Handle --print-dependencies
-  if (ctx.arg.print_dependencies == 1)
-    print_dependencies(ctx);
-  else if (ctx.arg.print_dependencies == 2)
-    print_dependencies_full(ctx);
-
  if (ctx.has_lto_object)
    lto_cleanup(ctx);

@@ -753,20 +760,21 @@ static int elf_main(int argc, char **argv) {

  for (std::function<void()> &fn : ctx.on_exit)
    fn();
+  ctx.checkpoint();
  return 0;
 }

 int main(int argc, char **argv) {
+#if MOLD_DEBUG_ARM64_ONLY
+  return elf_main<ARM64>(argc, argv);
+#else
  return elf_main<X86_64>(argc, argv);
+#endif
 }

 #define INSTANTIATE(E)                                                  \
  template void read_file(Context<E> &, MappedFile<Context<E>> *);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -109,10 +109,6 @@ void print_map(Context<E> &ctx) {
 #define INSTANTIATE(E)                          \
  template void print_map(Context<E> &ctx);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -31,12 +31,24 @@ static void debug_print(char *fmt, ...) {
  va_end(ap);
 }

-static void get_args(va_list *ap, int argc, char **argv) {
-  for (int i = 1; i < argc - 1; i++) {
+static int count_args(va_list *ap) {
+  va_list aq;
+  va_copy(aq, *ap);
+
+  int i = 0;
+  while (va_arg(aq, char *))
+    i++;
+  va_end(aq);
+  return i;
+}
+
+static void copy_args(va_list *ap, char **argv) {
+  int i = 1;
+  for (;;) {
    char *arg = va_arg(*ap, char *);
    if (!arg)
      break;
-    argv[i] = arg;
+    argv[i++] = arg;
  }
 }

@@ -52,10 +64,8 @@ static bool is_ld(const char *path) {
 int execvpe(const char *file, char *const *argv, char *const *envp) {
  debug_print("execvpe %s\n", file);

-  if (!strcmp(file, "ld") || is_ld(file)) {
+  if (!strcmp(file, "ld") || is_ld(file))
    file = get_mold_path();
-    ((const char **)argv)[0] = file;
-  }

  for (int i = 0; envp[i]; i++)
    putenv(envp[i]);
@@ -66,12 +76,8 @@ int execvpe(const char *file, char *const *argv, char *const *envp) {

 int execve(const char *path, char *const *argv, char *const *envp) {
  debug_print("execve %s\n", path);
-
-  if (is_ld(path)) {
+  if (is_ld(path))
    path = get_mold_path();
-    ((const char **)argv)[0] = path;
-  }
-
  typeof(execve) *real = dlsym(RTLD_NEXT, "execve");
  return real(path, argv, envp);
 }
@@ -79,8 +85,9 @@ int execve(const char *path, char *const *argv, char *const *envp) {
 int execl(const char *path, const char *arg0, ...) {
  va_list ap;
  va_start(ap, arg0);
-  char *argv[4096] = {(char *)arg0};
-  get_args(&ap, 4096, argv);
+  char **argv = calloc(sizeof(char *), count_args(&ap) + 1);
+  ((const char **)argv)[0] = arg0;
+  copy_args(&ap, argv);
  va_end(ap);
  return execve(path, argv, environ);
 }
@@ -88,8 +95,9 @@ int execl(const char *path, const char *arg0, ...) {
 int execlp(const char *file, const char *arg0, ...) {
  va_list ap;
  va_start(ap, arg0);
-  char *argv[4096] = {(char *)arg0};
-  get_args(&ap, 4096, argv);
+  char **argv = calloc(sizeof(char *), count_args(&ap) + 1);
+  ((const char **)argv)[0] = arg0;
+  copy_args(&ap, argv);
  va_end(ap);
  return execvpe(file, argv, environ);
 }
@@ -97,8 +105,9 @@ int execlp(const char *file, const char *arg0, ...) {
 int execle(const char *path, const char *arg0, ...) {
  va_list ap;
  va_start(ap, arg0);
-  char *argv[4096] = {(char *)arg0};
-  get_args(&ap, 4096, argv);
+  char **argv = calloc(sizeof(char *), count_args(&ap) + 1);
+  ((const char **)argv)[0] = arg0;
+  copy_args(&ap, argv);
  char **env = va_arg(ap, char **);
  va_end(ap);
  return execve(path, argv, env);
@@ -117,12 +126,8 @@ int posix_spawn(pid_t *pid, const char *path,
                const posix_spawnattr_t *attrp,
                char *const *argv, char *const *envp) {
  debug_print("posix_spawn %s\n", path);
-
-  if (is_ld(path)) {
+  if (is_ld(path))
    path = get_mold_path();
-    ((const char **)argv)[0] = path;
-  }
-
  typeof(posix_spawn) *real = dlsym(RTLD_NEXT, "posix_spawn");
  return real(pid, path, file_actions, attrp, argv, envp);
 }
@@ -33,6 +33,29 @@
 #define XXH_INLINE_ALL 1
 #include <xxhash.h>

+// MOLD_DEBUG_{X86_64,ARM64}_ONLY are macros to speed up builds.
+// This should be used only for debugging. When you use this flag,
+// you need to compile mold C++ files with `-ffunction-sections
+// -fdata-sections` and link them with -Wl,-gc-sections.
+#if MOLD_DEBUG_X86_64_ONLY
+# ifdef __OPTIMIZE__
+#  error "MOLD_DEBUG_X86_64_ONLY is for debugging only"
+# endif
+# define INSTANTIATE_ALL INSTANTIATE(X86_64)
+#elif MOLD_DEBUG_ARM64_ONLY
+# ifdef __OPTIMIZE__
+#  error "MOLD_DEBUG_ARM64_ONLY is for debugging only"
+# endif
+# define INSTANTIATE_ALL INSTANTIATE(ARM64)
+#else
+# define INSTANTIATE_ALL                       \
+  INSTANTIATE(X86_64);                          \
+  INSTANTIATE(I386);                            \
+  INSTANTIATE(ARM64);                           \
+  INSTANTIATE(ARM32);                           \
+  INSTANTIATE(RISCV64)
+#endif
+
 namespace mold::elf {

 static constexpr i32 SHA256_SIZE = 32;
@@ -266,8 +289,8 @@ public:
  InputSection(Context<E> &ctx, ObjectFile<E> &file, std::string_view name,
               i64 shndx);

-  bool is_compressed();
-  void uncompress(Context<E> &ctx, u8 *buf);
+  void uncompress(Context<E> &ctx);
+  void uncompress_to(Context<E> &ctx, u8 *buf);
  void scan_relocations(Context<E> &ctx);
  void write_to(Context<E> &ctx, u8 *buf);
  void apply_reloc_alloc(Context<E> &ctx, u8 *base);
@@ -307,10 +330,12 @@ public:

  // For COMDAT de-duplication and garbage collection
  std::atomic_bool is_alive = true;
-  bool killed_by_icf = false;
-
  u8 p2align = 0;

+  u8 compressed : 1 = false;
+  u8 uncompressed : 1 = false;
+  u8 killed_by_icf : 1 = false;
+
 private:
  typedef enum : u8 { NONE, ERROR, COPYREL, PLT, CPLT, DYNREL, BASEREL } Action;

@@ -349,6 +374,9 @@ public:
  virtual void write_to(Context<E> &ctx, u8 *buf);
  virtual void update_shdr(Context<E> &ctx) {}

+  // For --gdb-index
+  virtual u8 *get_uncompressed_data() { return nullptr; }
+
  std::string_view name;
  ElfShdr<E> shdr = {};
  i64 shndx = 0;
@@ -868,7 +896,6 @@ public:
 struct GdbIndexName {
  std::string_view name;
  u32 hash = 0;
-  u32 offset = 0;
  u32 attr = 0;
  u32 entry_idx = 0;
 };
@@ -879,6 +906,7 @@ public:
  GdbIndexSection() {
    this->name = ".gdb_index";
    this->shdr.sh_type = SHT_PROGBITS;
+    this->shdr.sh_addralign = 4;
  }

  void construct(Context<E> &ctx);
@@ -896,33 +924,21 @@ private:
  };

  struct MapEntry {
-    MapEntry(u32 hash) : hash(hash) {}
+    MapEntry(ObjectFile<E> *owner, u32 hash) : owner(owner), hash(hash) {}

-    MapEntry(const MapEntry &other) {
-      hash = other.hash;
-      name_offset = other.name_offset;
-      attr_offset = other.attr_offset;
-      num_attrs = other.num_attrs.load();
-    }
+    MapEntry(const MapEntry &other)
+      : owner(other.owner.load()), num_attrs(other.num_attrs.load()),
+        hash(other.hash), name_offset(other.name_offset),
+        attr_offset(other.attr_offset) {}

-    u32 hash = 0;
-    u32 name_offset = 0;
-    u32 attr_offset = 0;
+    std::atomic<ObjectFile<E> *> owner;
    std::atomic_uint32_t num_attrs = 0;
+    u32 hash = 0;
+    u32 name_offset = -1;
+    u32 attr_offset = -1;
  };

  SectionHeader header;
-  i64 num_symtab_entries = 0;
-  i64 attrs_size = 0;
-
-  std::vector<std::string_view>
-  read_compunits(Context<E> &ctx, ObjectFile<E> &file);
-
-  std::vector<GdbIndexName> read_pubnames(Context<E> &ctx, ObjectFile<E> &file);
-
-  std::vector<u64> read_address_areas(Context<E> &ctx, ObjectFile<E> &file,
-                                      i64 offset);
-
  ConcurrentMap<MapEntry> map;
 };

@@ -931,10 +947,12 @@ class GabiCompressedSection : public Chunk<E> {
 public:
  GabiCompressedSection(Context<E> &ctx, Chunk<E> &chunk);
  void copy_buf(Context<E> &ctx) override;
+  u8 *get_uncompressed_data() override { return uncompressed.get(); }

 private:
  ElfChdr<E> chdr = {};
-  std::unique_ptr<ZlibCompressor> contents;
+  std::unique_ptr<ZlibCompressor> compressed;
+  std::unique_ptr<u8[]> uncompressed;
 };

 template <typename E>
@@ -942,11 +960,13 @@ class GnuCompressedSection : public Chunk<E> {
 public:
  GnuCompressedSection(Context<E> &ctx, Chunk<E> &chunk);
  void copy_buf(Context<E> &ctx) override;
+  u8 *get_uncompressed_data() override { return uncompressed.get(); }

 private:
  static constexpr i64 HEADER_SIZE = 12;
  i64 original_size = 0;
-  std::unique_ptr<ZlibCompressor> contents;
+  std::unique_ptr<ZlibCompressor> compressed;
+  std::unique_ptr<u8[]> uncompressed;
 };

 template <typename E>
@@ -966,6 +986,24 @@ private:

 bool is_c_identifier(std::string_view name);

+//
+// dwarf.cc
+//
+
+template <typename E>
+std::vector<std::string_view>
+read_compunits(Context<E> &ctx, ObjectFile<E> &file);
+
+template <typename E>
+std::vector<GdbIndexName> read_pubnames(Context<E> &ctx, ObjectFile<E> &file);
+
+template <typename E>
+i64 estimate_address_areas(Context<E> &ctx, ObjectFile<E> &file);
+
+template <typename E>
+std::vector<u64>
+read_address_areas(Context<E> &ctx, ObjectFile<E> &file, i64 offset);
+
 //
 // input-files.cc
 //
@@ -1099,6 +1137,7 @@ public:
  bool exclude_libs = false;
  u32 features = 0;
  bool is_lto_obj = false;
+  bool needs_executable_stack = false;

  u64 num_dynrel = 0;
  u64 reldyn_offset = 0;
@@ -1117,15 +1156,19 @@ public:

  // For .gdb_index
  InputSection<E> *debug_info = nullptr;
-  InputSection<E> *debug_abbrev = nullptr;
  InputSection<E> *debug_ranges = nullptr;
+  InputSection<E> *debug_rnglists = nullptr;
  InputSection<E> *debug_pubnames = nullptr;
  InputSection<E> *debug_pubtypes = nullptr;
  std::vector<std::string_view> compunits;
+  std::vector<GdbIndexName> gdb_names;
+  i64 compunits_idx = 0;
+  i64 attrs_size = 0;
+  i64 attrs_offset = 0;
+  i64 names_size = 0;
+  i64 names_offset = 0;
  i64 num_areas = 0;
  i64 area_offset = 0;
-  i64 compunits_idx = 0;
-  std::vector<GdbIndexName> pubnames;

 private:
  ObjectFile(Context<E> &ctx, MappedFile<Context<E>> *mf,
@@ -1142,10 +1185,6 @@ private:
                       const ElfSym<E> &esym, i64 symidx);
  void merge_visibility(Context<E> &ctx, Symbol<E> &sym, u8 visibility);

-  std::pair<std::string_view, const ElfShdr<E> *>
-  uncompress_contents(Context<E> &ctx, const ElfShdr<E> &shdr,
-                      std::string_view name);
-
  bool has_common_symbol = false;

  std::string_view symbol_strtab;
@@ -1170,6 +1209,7 @@ public:
  void compute_symtab(Context<E> &ctx);
  void write_symtab(Context<E> &ctx);

+  bool is_needed = false;
  std::string soname;
  std::vector<std::string_view> version_strings;
  std::vector<ElfSym<E>> elf_syms2;
@@ -1588,6 +1628,7 @@ struct Context {
    bool z_dlopen = true;
    bool z_dump = true;
    bool z_execstack = false;
+    bool z_execstack_if_needed = false;
    bool z_ibt = false;
    bool z_ibtplt = false;
    bool z_initfirst = false;
@@ -1728,6 +1769,13 @@ struct Context {
  std::unique_ptr<ThumbToArmSection> thumb_to_arm;
  std::unique_ptr<TlsTrampolineSection> tls_trampoline;

+  // For --gdb-index
+  Chunk<E> *debug_info = nullptr;
+  Chunk<E> *debug_abbrev = nullptr;
+  Chunk<E> *debug_ranges = nullptr;
+  Chunk<E> *debug_addr = nullptr;
+  Chunk<E> *debug_rnglists = nullptr;
+
  // For --relocatable
  std::vector<RChunk<E> *> r_chunks;
  ROutputEhdr<E> *r_ehdr = nullptr;
@@ -1736,8 +1784,8 @@ struct Context {
  RStrtabSection<E> *r_strtab = nullptr;
  RSymtabSection<E> *r_symtab = nullptr;

-  u64 tls_begin = -1;
-  u64 tls_end = -1;
+  u64 tls_begin = 0;
+  u64 tls_end = 0;
  bool relax_tlsdesc = false;

  // Linker-synthesized symbols
@@ -1871,15 +1919,15 @@ public:
  u64 value = 0;

  const char *nameptr = nullptr;
-  u64 namelen : 20 = 0;
+  i32 namelen = 0;

  // Index into the symbol table of the owner file.
-  i64 sym_idx : 20 = -1;
+  i32 sym_idx = -1;

  // shndx > 0  : symbol is in file's shndx'th section
  // shndx == 0 : absolute symbol
  // shndx < 0  : symbol is in the -shndx'th output section
-  i64 shndx : 20 = 0;
+  i32 shndx = 0;

  i32 aux_idx = -1;
  u16 ver_idx = 0;
@@ -31,17 +31,6 @@ static u32 djb_hash(std::string_view name) {
  return h;
 }

-// The hash function for .gdb_index.
-static u32 gdb_hash(std::string_view name) {
-  u32 h = 0;
-  for (u8 c : name) {
-    if ('A' <= c && c <= 'Z')
-      c = 'a' + c - 'A';
-    h = h * 67 + c - 113;
-  }
-  return h;
-}
-
 template <typename E>
 void Chunk<E>::write_to(Context<E> &ctx, u8 *buf) {
  Fatal(ctx) << name << ": write_to is called on an invalid section";
@@ -50,9 +39,9 @@ void Chunk<E>::write_to(Context<E> &ctx, u8 *buf) {
 template <typename E>
 u64 get_entry_addr(Context<E> &ctx) {
  if (!ctx.arg.entry.empty())
-    if (Symbol<E> *sym = get_symbol(ctx, ctx.arg.entry))
-      if (sym->file && !sym->file->is_dso)
-        return sym->get_addr(ctx);
+    if (Symbol<E> *sym = get_symbol(ctx, ctx.arg.entry);
+        sym->file && !sym->file->is_dso)
+      return sym->get_addr(ctx);

  for (std::unique_ptr<OutputSection<E>> &osec : ctx.output_sections)
    if (osec->name == ".text")
@@ -89,7 +78,7 @@ void OutputEhdr<E>::copy_buf(Context<E> &ctx) {

  memcpy(&hdr.e_ident, "\177ELF", 4);
  hdr.e_ident[EI_CLASS] = (E::word_size == 8) ? ELFCLASS64 : ELFCLASS32;
-  hdr.e_ident[EI_DATA] = E::is_le ? ELFDATA2LSB : ELFDATA2MSB;
+  hdr.e_ident[EI_DATA] = ELFDATA2LSB;
  hdr.e_ident[EI_VERSION] = EV_CURRENT;
  hdr.e_type = ctx.arg.pic ? ET_DYN : ET_EXEC;
  hdr.e_machine = E::e_machine;
@@ -434,8 +423,8 @@ void RelDynSection<E>::sort(Context<E> &ctx) {
  //   file. This tends to optimize paging and caching when there are two
  //   references from the same page.
  //
-  // We place IFUNC relocations at the beginning of .rel.dyn because
-  // we set `__rel_iplt_start` to that address.
+  // We group IFUNC relocations at the end of .rel.dyn because we need to
+  // mark them with `__rel_iplt_start and `__rel_iplt_end`.
  tbb::parallel_sort(begin, end, [&](const ElfRel<E> &a, const ElfRel<E> &b) {
    return std::tuple(get_rank(a.r_type), a.r_sym, a.r_offset) <
           std::tuple(get_rank(b.r_type), b.r_sym, b.r_offset);
@@ -810,29 +799,8 @@ static std::string_view get_output_name(Context<E> &ctx, std::string_view name)
      return stem;
  }

-  if (name.starts_with(".zdebug_")) {
-    if (name == ".zdebug_aranges")
-      return ".debug_aranges";
-    if (name == ".zdebug_frame")
-      return ".debug_frame";
-    if (name == ".zdebug_info")
-      return ".debug_info";
-    if (name == ".zdebug_line")
-      return ".debug_line";
-    if (name == ".zdebug_loc")
-      return ".debug_loc";
-    if (name == ".zdebug_pubnames")
-      return ".debug_pubnames";
-    if (name == ".zdebug_pubtypes")
-      return ".debug_pubtypes";
-    if (name == ".zdebug_ranges")
-      return ".debug_ranges";
-    if (name == ".zdebug_str")
-      return ".debug_str";
-    if (name == ".zdebug_types")
-      return ".debug_types";
+  if (name.starts_with(".zdebug_"))
    return save_string(ctx, "."s + std::string(name.substr(2)));
-  }

  return name;
 }
@@ -2064,6 +2032,8 @@ static std::vector<u8> get_uuid_v4(Context<E> &ctx) {

 template <typename E>
 void BuildIdSection<E>::write_buildid(Context<E> &ctx) {
+  Timer t(ctx, "build_id");
+
  switch (ctx.arg.build_id.kind) {
  case BuildId::HEX:
    write_vector(ctx.buf + this->shdr.sh_offset + HEADER_SIZE,
@@ -2106,7 +2076,7 @@ void NotePropertySection<E>::copy_buf(Context<E> &ctx) {
  memset(buf, 0, this->shdr.sh_size);

  buf[0] = 4;                              // Name size
-  buf[1] = (E::word_size == 8) ? 16 : 12;   // Content size
+  buf[1] = (E::word_size == 8) ? 16 : 12;  // Content size
  buf[2] = NT_GNU_PROPERTY_TYPE_0;         // Type
  memcpy(buf + 3, "GNU", 4);               // Name
  buf[4] = GNU_PROPERTY_X86_FEATURE_1_AND; // Feature type
@@ -2120,16 +2090,16 @@ void NotePropertySection<E>::copy_buf(Context<E> &ctx) {
 // gdb uses these maps to quickly find a compunit given a name or an
 // instruction pointer.
 //
-// .gdb_index is not mandatory. All the information in .gdb_index is
-// also in other debug info sections. You can actually create an
-// executable without .gdb_index and later add that using
-// `gdb-add-index` post-processing tool that comes with gdb.
-//
 // (Terminology: a compilation unit, which often abbreviated as compunit
 // or cu, is a unit of debug info. An input .debug_info section usually
 // contains one compunit, and thus an output .debug_info contains as
 // many compunits as the number of input files.)
 //
+// .gdb_index is not mandatory. All the information in .gdb_index is
+// also in other debug info sections. You can actually create an
+// executable without .gdb_index and later add it using `gdb-add-index`
+// post-processing tool that comes with gdb.
+//
 // The mapping from names to compunits is 1:n while the mapping from
 // address ranges to compunits is 1:1. That is, two object files may
 // define the same type name (with the same definition), while there
@@ -2144,18 +2114,22 @@ void NotePropertySection<E>::copy_buf(Context<E> &ctx) {
 // Besides names, these sections contains attributes for each name so
 // that gdb can distinguish type names from function names, for example.
 //
-// Function address ranges are in .debug_info and .debug_ranges
-// sections. If an object file is compiled without -ffunction-sections,
-// there's only one .text section in that object file, and in that case
-// its address range is directly stored to .debug_info. If an object
-// file is compiled with -ffunction-sections, it contains multiple .text
-// sections, and address ranges for them are stored to .debug_ranges.
+// A compunit contains one or more function address ranges. If an
+// object file is compiled without -ffunction-sections, it contains
+// only one .text section and therefore contains a single address range.
+// Such range is typically stored directly to the compunit.
+//
+// If an object file is compiled with -fucntion-sections, it contains
+// more than one .text section, and it has as many address ranges as
+// the number of .text sections. Such discontiguous address ranges are
+// stored to .debug_ranges in DWARF 2/3/4/5 and
+// .debug_rnglists/.debug_addr in DWARF 5.
 //
 // .debug_info section contains DWARF debug info. Although we don't need
 // to parse the whole .debug_info section to read address ranges, we
 // have to do a little bit. DWARF is complicated and often handled using
 // a library such as libdwarf. But we don't use any library because we
-// didn't want to add an extra run-time dependency just for --gdb-index.
+// don't want to add an extra run-time dependency just for --gdb-index.
 //
 // This page explains the format of .gdb_index:
 // https://sourceware.org/gdb/onlinedocs/gdb/Index-Section-Format.html
@@ -2165,20 +2139,13 @@ void GdbIndexSection<E>::construct(Context<E> &ctx) {

  // Read debug sections
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    if (file->debug_info == nullptr)
-      return;
+    if (file->debug_info) {
+      // Read compilation units from .debug_info.
+      file->compunits = read_compunits(ctx, *file);

-    // Read compilation units from .debug_info.
-    file->compunits = read_compunits(ctx, *file);
-
-    // Initialize `num_areas`. Each CU contains zero or one address area.
-    file->num_areas = file->compunits.size();
-
-    // Optionally, a CU can refer an address area list in .debug_ranges.
-    // .debug_ranges contains a vector of [begin, end) address pairs.
-    // The last entry must be a null terminator, so we do -1.
-    if (file->debug_ranges)
-      file->num_areas += file->debug_ranges->sh_size / E::word_size / 2 - 1;
+      // Count the number of address areas contained in this file.
+      file->num_areas = estimate_address_areas(ctx, *file);
+    }
  });

  // Initialize `area_offset` and `compunits_idx`.
@@ -2191,14 +2158,14 @@ void GdbIndexSection<E>::construct(Context<E> &ctx) {

  // Read .debug_gnu_pubnames and .debug_gnu_pubtypes.
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    file->pubnames = read_pubnames(ctx, *file);
+    file->gdb_names = read_pubnames(ctx, *file);
  });

  // Estimate the unique number of pubnames.
  HyperLogLog estimator;
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    HyperLogLog e;
-    for (GdbIndexName &name : file->pubnames)
+    for (GdbIndexName &name : file->gdb_names)
      e.insert(name.hash);
    estimator.merge(e);
  });
@@ -2206,66 +2173,55 @@ void GdbIndexSection<E>::construct(Context<E> &ctx) {
  // Uniquify pubnames by inserting all name strings into a concurrent
  // hashmap.
  map.resize(estimator.get_cardinality() * 2);
+  tbb::enumerable_thread_specific<i64> num_names;

  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (GdbIndexName &name : file->pubnames) {
-      MapEntry *ent = map.insert(name.name, name.hash, {name.hash}).first;
+    for (GdbIndexName &name : file->gdb_names) {
+      MapEntry *ent;
+      bool inserted;
+      std::tie(ent, inserted) = map.insert(name.name, name.hash, {file, name.hash});
+      if (inserted)
+        num_names.local()++;
+
+      ObjectFile<E> *old_val = ent->owner;
+      while (file->priority < old_val->priority &&
+             !ent->owner.compare_exchange_weak(old_val, file));
+
      ent->num_attrs++;
      name.entry_idx = ent - map.values;
    }
  });

-  // Assign in-shard output offsets to names and attrs.
-  std::vector<u32> attr_sizes(map.NUM_SHARDS);
-  std::vector<u32> name_sizes(map.NUM_SHARDS);
-  const i64 shard_size = map.nbuckets / map.NUM_SHARDS;
-  std::atomic_uint32_t num_names = 0;
-
-  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
-    i64 attr_offset = 0;
-    i64 name_offset = 0;
-    i64 count = 0;
-
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
-      if (map.keys[j]) {
-        map.values[j].name_offset = name_offset;
-        map.values[j].attr_offset = attr_offset;
-        attr_offset += (map.values[j].num_attrs + 1) * 4;
-        name_offset += map.key_sizes[j] + 1;
-        count++;
-      }
-    }
-
-    attr_sizes[i] = attr_offset;
-    name_sizes[i] = name_offset;
-    num_names += count;
-  });
-
-  // Fix name and attr offsets so that they are relative to
-  // `const_pool_offset`.
-  std::vector<u32> attr_offsets(map.NUM_SHARDS + 1);
-  for (i64 i = 0; i < map.NUM_SHARDS; i++)
-    attr_offsets[i + 1] = attr_offsets[i] + attr_sizes[i];
-  attrs_size = attr_offsets.back();
-
-  std::vector<u32> name_offsets(map.NUM_SHARDS + 1);
-  name_offsets[0] = attrs_size;
-  for (i64 i = 0; i < map.NUM_SHARDS; i++)
-    name_offsets[i + 1] = name_offsets[i] + name_sizes[i];
-
-  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
-    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
-      if (map.keys[j]) {
-        map.values[j].attr_offset += attr_offsets[i];
-        map.values[j].name_offset += name_offsets[i];
+  // Assign offsets for names and attributes within each file.
+  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
+    for (GdbIndexName &name : file->gdb_names) {
+      MapEntry &ent = map.values[name.entry_idx];
+      if (ent.owner == file) {
+        ent.attr_offset = file->attrs_size;
+        file->attrs_size += (ent.num_attrs + 1) * 4;
+        ent.name_offset = file->names_size;
+        file->names_size += name.name.size() + 1;
      }
    }
  });

+  // Compute per-file name and attributes offsets.
+  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
+    ctx.objs[i + 1]->attrs_offset =
+      ctx.objs[i]->attrs_offset + ctx.objs[i]->attrs_size;
+
+  ctx.objs[0]->names_offset =
+    ctx.objs.back()->attrs_offset + ctx.objs.back()->attrs_size;
+
+  for (i64 i = 0; i < ctx.objs.size() - 1; i++)
+    ctx.objs[i + 1]->names_offset =
+      ctx.objs[i]->names_offset + ctx.objs[i]->names_size;
+
  // .gdb_index contains an on-disk hash table for pubnames and
  // pubtypes. We aim 75% utilization. As per the format specification,
  // It must be a power of two.
-  num_symtab_entries = std::max<i64>(next_power_of_two(num_names * 4 / 3), 16);
+  i64 num_symtab_entries =
+    std::max<i64>(next_power_of_two(num_names.combine(std::plus()) * 4 / 3), 16);

  // Now that we can compute the size of this section.
  ObjectFile<E> &last = *ctx.objs.back();
@@ -2284,7 +2240,7 @@ void GdbIndexSection<E>::construct(Context<E> &ctx) {
  offset += num_symtab_entries * 8;

  header.const_pool_offset = offset;
-  offset += name_offsets.back();
+  offset += last.names_offset + last.names_size;

  this->shdr.sh_size = offset;
 }
@@ -2313,45 +2269,67 @@ void GdbIndexSection<E>::copy_buf(Context<E> &ctx) {
  // Skip address areas. It'll be filled by write_address_areas.
  buf += header.symtab_offset - header.areas_offset;

-  // Write symbol table.
-  memset(buf, 0, num_symtab_entries * 8);
+  // Write an on-disk hash table for names.
+  u32 symtab_size = header.const_pool_offset - header.symtab_offset;
+  memset(buf, 0, symtab_size);

-  assert(std::popcount<u64>(num_symtab_entries) == 1);
-  u32 mask = num_symtab_entries - 1;
+  assert(std::popcount(symtab_size / 8) == 1);
+  u32 mask = symtab_size / 8 - 1;

  for (i64 i = 0; i < map.nbuckets; i++) {
    if (map.has_key(i)) {
      u32 hash = map.values[i].hash;
-      u32 step = ((hash * 17) & mask) | 1;
+      u32 step = (hash & mask) | 1;
      u32 j = hash & mask;

      while (*(u32 *)(buf + j * 8))
        j = (j + step) & mask;

-      *(u32 *)(buf + j * 8) = map.values[i].name_offset;
-      *(u32 *)(buf + j * 8 + 4) = map.values[i].attr_offset;
+      ObjectFile<E> &file = *map.values[i].owner;
+      *(u32 *)(buf + j * 8) = file.names_offset + map.values[i].name_offset;
+      *(u32 *)(buf + j * 8 + 4) = file.attrs_offset + map.values[i].attr_offset;
    }
  }

-  buf += num_symtab_entries * 8;
+  buf += symtab_size;

  // Write CU vector
-  memset(buf, 0, attrs_size);
-  std::atomic_uint32_t *attrs = (std::atomic_uint32_t *)buf;
+  memset(buf, 0, ctx.objs[0]->names_offset);

  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
-    for (GdbIndexName &name : file->pubnames) {
+    std::atomic_uint32_t *attrs = (std::atomic_uint32_t *)buf;
+
+    for (GdbIndexName &name : file->gdb_names) {
      MapEntry &ent = map.values[name.entry_idx];
-      u32 idx = ent.attr_offset / 4;
+      u32 idx = (ent.owner.load()->attrs_offset + ent.attr_offset) / 4;
      attrs[idx + ++attrs[idx]] = name.attr;
    }
  });

+  // Sort CU vector for build reproducibility
+  const i64 shard_size = map.nbuckets / map.NUM_SHARDS;
+
+  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
+    u32 *attrs = (u32 *)buf;
+
+    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
+      if (map.has_key(j)) {
+        MapEntry &ent = map.values[j];
+        u32 idx = (ent.owner.load()->attrs_offset + ent.attr_offset) / 4;
+        u32 *start = attrs + idx + 1;
+        std::sort(start, start + attrs[idx]);
+      }
+    }
+  });
+
  // Write pubnames and pubtypes.
-  tbb::parallel_for((i64)0, (i64)map.nbuckets, [&](i64 i) {
-    if (map.has_key(i)) {
-      std::string_view name{map.keys[i], map.key_sizes[i]};
-      write_string(buf + map.values[i].name_offset, name);
+  tbb::parallel_for((i64)0, (i64)map.NUM_SHARDS, [&](i64 i) {
+    for (i64 j = shard_size * i; j < shard_size * (i + 1); j++) {
+      if (map.has_key(j)) {
+        ObjectFile<E> &file = *map.values[j].owner;
+        std::string_view name{map.keys[j], map.key_sizes[j]};
+        write_string(buf + file.names_offset + map.values[j].name_offset, name);
+      }
    }
  });
 }
@@ -2361,14 +2339,30 @@ void GdbIndexSection<E>::write_address_areas(Context<E> &ctx) {
  Timer t(ctx, "GdbIndexSection::write_address_areas");
  u8 *base = ctx.buf + this->shdr.sh_offset;

+  for (Chunk<E> *chunk : ctx.chunks) {
+    std::string_view name = chunk->name;
+    if (name == ".debug_info" || name == ".zdebug_info")
+      ctx.debug_info = chunk;
+    if (name == ".debug_abbrev" || name == ".zdebug_abbrev")
+      ctx.debug_abbrev = chunk;
+    if (name == ".debug_ranges" || name == ".zdebug_ranges")
+      ctx.debug_ranges = chunk;
+    if (name == ".debug_addr" || name == ".zdebug_addr")
+      ctx.debug_addr = chunk;
+    if (name == ".debug_rnglists" || name == ".zdebug_rnglists")
+      ctx.debug_rnglists = chunk;
+  }
+
+  assert(ctx.debug_info);
+  assert(ctx.debug_abbrev);
+
  struct __attribute__((packed)) Entry {
    u64 start;
    u64 end;
    u32 attr;
  };

-  // Read .debug_info and .debug_ranges to copy address ranges
-  // to .gdb_index.
+  // Read address ranges from debug sections and copy them to .gdb_index.
  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    if (!file->debug_info)
      return;
@@ -2381,10 +2375,15 @@ void GdbIndexSection<E>::write_address_areas(Context<E> &ctx) {
      std::vector<u64> addrs = read_address_areas(ctx, *file, offset);

      for (i64 j = 0; j < addrs.size(); j += 2) {
-        // Skip an address area for a dead function
-        if (addrs[j] == 1 && addrs[j + 1] == 1)
+        // Skip an empty range
+        if (addrs[j] == addrs[j + 1])
          continue;

+        // Gdb crashes if there are entries with address 0.
+        if (addrs[j] == 0)
+          continue;
+
+        assert(e < begin + file->num_areas);
        e->start = addrs[j];
        e->end = addrs[j + 1];
        e->attr = file->compunits_idx + i;
@@ -2396,247 +2395,44 @@ void GdbIndexSection<E>::write_address_areas(Context<E> &ctx) {
    // Fill trailing null entries with dummy values because gdb
    // crashes if there are entries with address 0.
    u64 filler = (e == begin) ? ctx.etext->get_addr(ctx) - 1 : e[-1].start;
-    for (; e < begin + file->num_areas; e++)
+    for (; e < begin + file->num_areas; e++) {
      e->start = e->end = filler;
+      e->attr = file->compunits_idx;
+    }
  });
 }

-// Returns the list of compilation units in .gdb_index. A .gdb_index
-// usually contains only one compilatation unit unless the object was
-// built by `ld -r`.
-template <typename E>
-std::vector<std::string_view>
-GdbIndexSection<E>::read_compunits(Context<E> &ctx, ObjectFile<E> &file) {
-  std::string_view data = file.debug_info->contents;
-  std::vector<std::string_view> vec;
-
-  while (!data.empty()) {
-    if (data.size() < 4)
-      Fatal(ctx) << *file.debug_info << ": corrupted .debug_info";
-    i64 len = *(u32 *)data.data() + 4;
-    vec.push_back(data.substr(0, len));
-    data = data.substr(len);
-  }
-  return vec;
-}
-
-// Parses .debug_gnu_pubnames and .debug_gnu_pubtypes. These sections
-// start with a 14 bytes header followed by (4-byte offset, 1-byte type,
-// null-terminated string) tuples.
-//
-// The 4-byte offset is an offset into .debug_info that contains details
-// about the name. The 1-byte type is a type of the corresponding name
-// (e.g. function, variable or datatype). The string is a name of a
-// function, a variable or a type.
-template <typename E>
-std::vector<GdbIndexName>
-GdbIndexSection<E>::read_pubnames(Context<E> &ctx, ObjectFile<E> &file) {
-  std::vector<GdbIndexName> vec;
-
-  auto get_cu_idx = [&](InputSection<E> &isec, i64 offset) {
-    i64 off = 0;
-    for (i64 i = 0; i < file.compunits.size(); i++) {
-      if (offset == off)
-        return file.compunits_idx + i;
-      off += file.compunits[i].size();
-    }
-    Fatal(ctx) << isec << ": corrupted debug_info_offset";
-  };
-
-  auto read = [&](InputSection<E> &isec) {
-    std::string_view contents = isec.contents;
-
-    while (!contents.empty()) {
-      if (contents.size() < 14)
-        Fatal(ctx) << isec << ": corrupted header";
-
-      u32 len = *(u32 *)contents.data() + 4;
-      u32 debug_info_offset = *(u32 *)(contents.data() + 6);
-      u32 cu_idx = get_cu_idx(isec, debug_info_offset);
-
-      std::string_view data = contents.substr(14, len - 14);
-      contents = contents.substr(len);
-
-      while (!data.empty()) {
-        u32 offset = *(u32 *)data.data();
-        data = data.substr(4);
-        if (offset == 0)
-          break;
-
-        u8 type = data[0];
-        data = data.substr(1);
-
-        std::string_view name = data.data();
-        data = data.substr(name.size() + 1);
-
-        vec.push_back({name, gdb_hash(name), offset + debug_info_offset,
-                       (type << 24) | cu_idx});
-      }
-    }
-  };
-
-  if (file.debug_pubnames)
-    read(*file.debug_pubnames);
-  if (file.debug_pubtypes)
-    read(*file.debug_pubtypes);
-  return vec;
-}
-
-// Returns a list of address ranges explained by a compunit at the
-// `offset` in an output .debug_info section.
-//
-// .debug_info contains DWARF debug info records, so this function
-// parses DWARF. If a designated compunit contains multiple ranges, the
-// ranges are read from .debug_ranges. Otherwise, a range is read
-// directly from .debug_info.
-template <typename E>
-std::vector<u64>
-GdbIndexSection<E>::read_address_areas(Context<E> &ctx, ObjectFile<E> &file,
-                                       i64 offset) {
-  OutputSection<E> *debug_info = nullptr;
-  OutputSection<E> *debug_abbrev = nullptr;
-  OutputSection<E> *debug_ranges = nullptr;
-
-  for (std::unique_ptr<OutputSection<E>> &osec : ctx.output_sections) {
-    if (osec->name == ".debug_info")
-      debug_info = osec.get();
-    if (osec->name == ".debug_abbrev")
-      debug_abbrev = osec.get();
-    if (osec->name == ".debug_ranges")
-      debug_ranges = osec.get();
-  }
-
-  assert(debug_info);
-  assert(debug_abbrev);
-
-  // Read .debug_abbrev to learn the format of .debug_info.
-  u8 *cu = (u8 *)(ctx.buf + debug_info->shdr.sh_offset + offset);
-  u32 abbrev_offset = *(u32 *)(cu + 6);
-  cu += 11;
-  read_uleb(cu); // skip abbrev code
-
-  u8 *abbrev = (u8 *)(ctx.buf + debug_abbrev->shdr.sh_offset + abbrev_offset);
-  read_uleb(abbrev); // skip abbrev code
-  u64 abbrev_tag = read_uleb(abbrev);
-
-  if (abbrev_tag != DW_TAG_compile_unit)
-    Error(ctx) << file << ": --gdb-index: .debug_abbrev does not contain"
-               << " DW_TAG_compile_unit";
-
-  abbrev++; // skip has_children byte
-
-  std::optional<u64> low_pc;
-
-  for (;;) {
-    u64 name = read_uleb(abbrev);
-    u64 form = read_uleb(abbrev);
-    if (name == 0 && form == 0)
-      break;
-
-    auto read_value = [&]() -> u64 {
-      switch (form) {
-      case DW_FORM_flag_present:
-        return 0;
-      case DW_FORM_data1:
-      case DW_FORM_flag:
-        return *cu++;
-      case DW_FORM_data2: {
-        u64 val = *(u16 *)cu;
-        cu += 2;
-        return val;
-      }
-      case DW_FORM_data4:
-      case DW_FORM_strp:
-      case DW_FORM_sec_offset: {
-        u64 val = *(u32 *)cu;
-        cu += 4;
-        return val;
-      }
-      case DW_FORM_data8: {
-        u64 val = *(u64 *)cu;
-        cu += 8;
-        return val;
-      }
-      case DW_FORM_addr: {
-        u64 val = *(typename E::WordTy *)cu;
-        cu += E::word_size;
-        return val;
-      }
-      case DW_FORM_string: {
-        while (*cu)
-          cu++;
-        cu++;
-        return 0;
-      }
-      default:
-        Error(ctx) << file << ": --gdb-index: unknown debug info form: 0x"
-                   << std::hex << form;
-        return 0;
-      }
-    };
-
-    switch (name) {
-    case DW_AT_low_pc:
-      *low_pc = read_value();
-      break;
-    case DW_AT_high_pc:
-      if (low_pc)
-        Error(ctx) << file << ": --gdb-index: missing DW_AT_low_pc";
-
-      if (form == DW_FORM_addr)
-        return {*low_pc, read_value()};
-      return {*low_pc, *low_pc + read_value()};
-    case DW_AT_ranges: {
-      if (!debug_ranges)
-        Fatal(ctx) << file << ": --gdb-index: .missing debug_ranges";
-
-      u64 offset = read_value();
-      typename E::WordTy *range =
-        (typename E::WordTy *)(ctx.buf + debug_ranges->shdr.sh_offset + offset);
-
-      std::vector<u64> vec;
-      for (i64 i = 0; range[i] || range[i + 1]; i += 2) {
-        vec.push_back(range[i]);
-        vec.push_back(range[i + 1]);
-      }
-      return vec;
-    }
-    default:
-      read_value();
-      break;
-    }
-  }
-
-  return {};
-}
-
 template <typename E>
 GabiCompressedSection<E>::GabiCompressedSection(Context<E> &ctx,
                                                Chunk<E> &chunk) {
  assert(chunk.name.starts_with(".debug"));
  this->name = chunk.name;

-  std::unique_ptr<u8[]> buf(new u8[chunk.shdr.sh_size]);
-  chunk.write_to(ctx, buf.get());
+  uncompressed.reset(new u8[chunk.shdr.sh_size]);
+  chunk.write_to(ctx, uncompressed.get());

  chdr.ch_type = ELFCOMPRESS_ZLIB;
  chdr.ch_size = chunk.shdr.sh_size;
  chdr.ch_addralign = chunk.shdr.sh_addralign;

-  contents.reset(new ZlibCompressor({(char *)buf.get(), chunk.shdr.sh_size}));
+  compressed.reset(new ZlibCompressor(uncompressed.get(), chunk.shdr.sh_size));

  this->shdr = chunk.shdr;
  this->shdr.sh_flags |= SHF_COMPRESSED;
  this->shdr.sh_addralign = 1;
-  this->shdr.sh_size = sizeof(chdr) + contents->size();
+  this->shdr.sh_size = sizeof(chdr) + compressed->size();
  this->shndx = chunk.shndx;
+
+  // We don't need to keep the original data unless --gdb-index is given.
+  if (!ctx.arg.gdb_index)
+    uncompressed.reset(nullptr);
 }

 template <typename E>
 void GabiCompressedSection<E>::copy_buf(Context<E> &ctx) {
  u8 *base = ctx.buf + this->shdr.sh_offset;
  memcpy(base, &chdr, sizeof(chdr));
-  contents->write_to(base + sizeof(chdr));
+  compressed->write_to(base + sizeof(chdr));
 }

 template <typename E>
@@ -2645,15 +2441,19 @@ GnuCompressedSection<E>::GnuCompressedSection(Context<E> &ctx,
  assert(chunk.name.starts_with(".debug"));
  this->name = save_string(ctx, ".zdebug" + std::string(chunk.name.substr(6)));

-  std::unique_ptr<u8[]> buf(new u8[chunk.shdr.sh_size]);
-  chunk.write_to(ctx, buf.get());
+  uncompressed.reset(new u8[chunk.shdr.sh_size]);
+  chunk.write_to(ctx, uncompressed.get());

-  contents.reset(new ZlibCompressor({(char *)buf.get(), chunk.shdr.sh_size}));
+  compressed.reset(new ZlibCompressor(uncompressed.get(), chunk.shdr.sh_size));

  this->shdr = chunk.shdr;
-  this->shdr.sh_size = HEADER_SIZE + contents->size();
+  this->shdr.sh_size = HEADER_SIZE + compressed->size();
  this->shndx = chunk.shndx;
  this->original_size = chunk.shdr.sh_size;
+
+  // We don't need to keep the original data unless --gdb-index is given.
+  if (!ctx.arg.gdb_index)
+    uncompressed.reset(nullptr);
 }

 template <typename E>
@@ -2661,7 +2461,7 @@ void GnuCompressedSection<E>::copy_buf(Context<E> &ctx) {
  u8 *base = ctx.buf + this->shdr.sh_offset;
  memcpy(base, "ZLIB", 4);
  *(ubig64 *)(base + 4) = this->original_size;
-  contents->write_to(base + 12);
+  compressed->write_to(base + 12);
 }

 template <typename E>
@@ -2779,10 +2579,6 @@ void RelocSection<E>::copy_buf(Context<E> &ctx) {
  template i64 BuildId::size(Context<E> &) const;                       \
  template bool is_relro(Context<E> &, Chunk<E> *);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -135,10 +135,6 @@ OutputFile<E>::open(Context<E> &ctx, std::string path, i64 filesize, i64 perm) {
 #define INSTANTIATE(E)                          \
  template class OutputFile<E>;

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -128,8 +128,6 @@ static void mark_live_objects(Context<E> &ctx) {

 template <typename E>
 void do_resolve_symbols(Context<E> &ctx) {
-  Timer t(ctx, "do_resolve_symbols");
-
  auto for_each_file = [&](std::function<void(InputFile<E> *)> fn) {
    tbb::parallel_for_each(ctx.objs, fn);
    tbb::parallel_for_each(ctx.dsos, fn);
@@ -163,7 +161,7 @@ void do_resolve_symbols(Context<E> &ctx) {

 template <typename E>
 void resolve_symbols(Context<E> &ctx) {
-  Timer t(ctx, "do_resolve_symbols");
+  Timer t(ctx, "resolve_symbols");

  std::vector<ObjectFile<E> *> objs = ctx.objs;
  std::vector<SharedFile<E> *> dsos = ctx.dsos;
@@ -207,7 +205,7 @@ void resolve_symbols(Context<E> &ctx) {

    tbb::parallel_for_each(ctx.dsos, [&](SharedFile<E> *file) {
      file->clear_symbols();
-      file->is_alive = true;
+      file->is_alive = !file->is_needed;
    });

    do_resolve_symbols(ctx);
@@ -1093,6 +1091,8 @@ void parse_symbol_version(Context<E> &ctx) {

  tbb::parallel_for_each(ctx.objs, [&](ObjectFile<E> *file) {
    for (i64 i = 0; i < file->symbols.size() - file->first_global; i++) {
+      // Match VERSION part of symbol foo@VERSION with version definitions.
+      // The symbols' VERSION parts are in file->symvers.
      if (!file->symvers[i])
        continue;

@@ -1118,6 +1118,16 @@ void parse_symbol_version(Context<E> &ctx) {
      sym->ver_idx = it->second;
      if (!is_default)
        sym->ver_idx |= VERSYM_HIDDEN;
+
+      // If both symbol `foo` and `foo@VERSION` are defined, `foo@VERSION`
+      // hides `foo` so that all references to `foo` are resolved to a
+      // versioned symbol. Likewise, if `foo@VERSION` and `foo@@VERSION` are
+      // defined, the default one takes precedence.
+      Symbol<E> *sym2 = get_symbol(ctx, sym->name());
+      if (sym2->file == file && !file->symvers[sym2->sym_idx - file->first_global])
+        if (sym2->ver_idx == ctx.default_version ||
+            (sym2->ver_idx & ~VERSYM_HIDDEN) == (sym->ver_idx & ~VERSYM_HIDDEN))
+          sym2->ver_idx = VER_NDX_LOCAL;
    }
  });
 }
@@ -1144,12 +1154,14 @@ void compute_import_export(Context<E> &ctx) {
          sym->ver_idx == VER_NDX_LOCAL)
        continue;

+      // If we are using a symbol in a DSO, we need to import it at runtime.
      if (sym->file != file && sym->file->is_dso) {
        std::scoped_lock lock(sym->mu);
        sym->is_imported = true;
        continue;
      }

+      // If we are creating a DSO, all global symbols are exported by default.
      if (sym->file == file) {
        std::scoped_lock lock(sym->mu);
        sym->is_exported = true;
@@ -1672,10 +1684,6 @@ void write_dependency_file(Context<E> &ctx) {
  template i64 compress_debug_sections(Context<E> &);                   \
  template void write_dependency_file(Context<E> &);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -298,7 +298,7 @@ void ROutputEhdr<E>::write_to(Context<E> &ctx) {
  ElfEhdr<E> &hdr = *(ElfEhdr<E> *)(ctx.buf + this->out_shdr.sh_offset);
  memcpy(&hdr.e_ident, "\177ELF", 4);
  hdr.e_ident[EI_CLASS] = (E::word_size == 8) ? ELFCLASS64 : ELFCLASS32;
-  hdr.e_ident[EI_DATA] = E::is_le ? ELFDATA2LSB : ELFDATA2MSB;
+  hdr.e_ident[EI_DATA] = ELFDATA2LSB;
  hdr.e_ident[EI_VERSION] = EV_CURRENT;
  hdr.e_type = ET_REL;
  hdr.e_machine = E::e_machine;
@@ -588,10 +588,6 @@ void combine_objects(Context<E> &ctx, std::span<std::string_view> file_args) {
 #define INSTANTIATE(E)                                                  \
  template void combine_objects(Context<E> &, std::span<std::string_view>);

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -312,10 +312,6 @@ void process_run_subcommand(Context<E> &ctx, int argc, char **argv) {
                          std::function<void()> *);                     \
  template void process_run_subcommand(Context<E> &, int, char **)

-INSTANTIATE(X86_64);
-INSTANTIATE(I386);
-INSTANTIATE(ARM64);
-INSTANTIATE(ARM32);
-INSTANTIATE(RISCV64);
+INSTANTIATE_ALL;

 } // namespace mold::elf
@@ -41,7 +41,7 @@ inline bool is_gcc_lto_obj(MappedFile<C> *mf) {
      continue;

    std::span<ElfSym<E>> elf_syms{(ElfSym<E> *)(data + sec.sh_offset),
-                                  sec.sh_size / sizeof(ElfSym<E>)};
+                                  (size_t)sec.sh_size / sizeof(ElfSym<E>)};

    auto skip = [](u8 type) {
      return type == STT_NOTYPE || type == STT_FILE || type == STT_SECTION;
@@ -0,0 +1,19 @@
+#!/bin/bash -x
+set -e
+
+source /etc/os-release
+
+case "$ID$VERSION_ID" in
+ubuntu20.04)
+  apt-get install -y build-essential git g++-10 cmake libssl-dev zlib1g-dev pkg-config git
+  ;;
+ubuntu22.04)
+  apt-get install -y build-essential git g++ cmake libssl-dev zlib1g-dev pkg-config git
+  ;;
+fedora*)
+  dnf install -y git gcc-g++ cmake openssl-devel zlib-devel
+  ;;
+*)
+  echo "Error: don't know anything about build dependencies on $ID $VERSION_ID"
+  exit 1
+esac
@@ -432,7 +432,7 @@ std::string_view demangle(std::string_view name);

 class ZlibCompressor {
 public:
-  ZlibCompressor(std::string_view input);
+  ZlibCompressor(u8 *buf, i64 size);
  void write_to(u8 *buf);
  i64 size() const;

@@ -0,0 +1,39 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+which dwarfdump >& /dev/null || { echo skipped; exit; }
+
+# Clang may not support -gz=zlib-gnu
+echo 'int main() {}' | $CXX -o /dev/null -gz=zlib-gnu -xc++ ->& /dev/null ||
+  { echo skipped; exit; }
+
+cat <<EOF | $CXX -c -o $t/a.o -g -gz=zlib-gnu -xc++ -
+int main() {
+  return 0;
+}
+EOF
+
+cat <<EOF | $CXX -c -o $t/b.o -g -gz=zlib-gnu -xc++ -
+int foo() {
+  return 0;
+}
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o
+dwarfdump $t/exe > /dev/null
+readelf --sections $t/exe | fgrep -q .debug_info
+
+echo ' OK'
@@ -16,7 +16,7 @@ mkdir -p $t

 which dwarfdump >& /dev/null || { echo skipped; exit; }

-cat <<EOF | $CXX -c -o $t/a.o -g -gz=zlib-gnu -xc++ -
+cat <<EOF | $CXX -c -o $t/a.o -g -gz=zlib -xc++ -
 int main() {
  return 0;
 }
@@ -14,6 +14,8 @@ mold="$(pwd)/mold"
 t=out/test/elf/$testname
 mkdir -p $t

+which dwarfdump >& /dev/null || { echo skipped; exit; }
+
 cat <<EOF | $CXX -c -o $t/a.o -g -xc++ -
 #include <iostream>
 struct Foo {
@@ -0,0 +1,37 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+cat <<EOF > $t/a.h
+#define A 23
+#define B 99
+EOF
+
+cat <<EOF | $GCC -o $t/b.o -c -xc - -I$t -g3
+#include "a.h"
+extern int z();
+int main () { return z() - 122; }
+EOF
+
+cat <<EOF | $GCC -o $t/c.o -c -xc - -I$t -g3
+#include "a.h"
+int z()  { return A + B; }
+EOF
+
+$GCC -B. -o $t/exe $t/b.o $t/c.o
+$OBJDUMP --dwarf=macro $t/exe > $t/log
+! grep 'DW_MACRO_import -.* 0x0$' $t/log || false
+
+echo OK
@@ -0,0 +1,32 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = x86_64 ] || { echo skipped; exit; }
+
+cat <<EOF | $CC -c -xassembler -o $t/a.o -
+.globl main
+main:
+  ret
+.section .note.GNU-stack, "x", @progbits
+EOF
+
+$CC -B. -o $t/exe $t/a.o >& /dev/null
+readelf --segments -W $t/exe | grep -q 'GNU_STACK.* RW '
+
+$CC -B. -o $t/exe $t/a.o -Wl,-z,execstack-if-needed
+readelf --segments -W $t/exe | grep -q 'GNU_STACK.* RWE '
+
+echo OK
@@ -19,16 +19,12 @@ int main() {}
 EOF

 $CC -B. -o $t/exe $t/a.o -Wl,-z,execstack
-readelf --segments -W $t/exe > $t/log
-grep -q 'GNU_STACK.* RWE ' $t/log
+readelf --segments -W $t/exe | grep -q 'GNU_STACK.* RWE '

-$CC -B. -o $t/exe $t/a.o -Wl,-z,execstack \
-  -Wl,-z,noexecstack
-readelf --segments -W $t/exe > $t/log
-grep -q 'GNU_STACK.* RW ' $t/log
+$CC -B. -o $t/exe $t/a.o -Wl,-z,execstack -Wl,-z,noexecstack
+readelf --segments -W $t/exe | grep -q 'GNU_STACK.* RW '

 $CC -B. -o $t/exe $t/a.o
-readelf --segments -W $t/exe > $t/log
-grep -q 'GNU_STACK.* RW ' $t/log
+readelf --segments -W $t/exe | grep -q 'GNU_STACK.* RW '

 echo OK
@@ -1,45 +0,0 @@
-#!/bin/bash
-export LC_ALL=C
-set -e
-CC="${CC:-cc}"
-CXX="${CXX:-c++}"
-GCC="${GCC:-gcc}"
-GXX="${GXX:-g++}"
-OBJDUMP="${OBJDUMP:-objdump}"
-MACHINE="${MACHINE:-$(uname -m)}"
-testname=$(basename "$0" .sh)
-echo -n "Testing $testname ... "
-cd "$(dirname "$0")"/../..
-mold="$(pwd)/mold"
-t=out/test/elf/$testname
-mkdir -p $t
-
-echo 'int main() {}' | $CC -o $t/exe -xc -
-dyld=$(readelf -Wl $t/exe | grep 'program interpreter:' | sed -e 's!.*: \(.*\)\]!\1!')
-
-cat <<EOF | $CC -o $t/a.o -c -xc -fPIC -
-#include <stdio.h>
-#include <stdlib.h>
-
-void hello() {
-  puts("Hello world");
-}
-
-void _start() {
-  hello();
-  exit(0);
-}
-EOF
-
-$CC -B. -o $t/b.so $t/a.o -shared -Wl,--dynamic-linker=$dyld
-$QEMU $t/b.so | grep -q 'Hello world'
-
-cat <<EOF | $CC -o $t/c.o -c -xc -
-void hello();
-int main() { hello(); }
-EOF
-
-$CC -B. -o $t/exe $t/c.o $t/b.so
-$QEMU $t/exe | grep -q 'Hello world'
-
-echo OK
@@ -0,0 +1,59 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
+which gdb >& /dev/null || { echo skipped; exit; }
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -g -ggnu-pubnames -gdwarf-4 -xc - -ffunction-sections
+#include <stdio.h>
+
+void trap() {}
+
+static void hello() {
+  printf("Hello world\n");
+  trap();
+}
+
+void greet() {
+  hello();
+}
+EOF
+
+$CC -B. -shared -o $t/b.so $t/a.o -Wl,--gdb-index -Wl,--compress-debug-sections=zlib-gabi
+readelf -WS $t/b.so 2> /dev/null | fgrep -q .gdb_index
+
+cat <<EOF | $CC -c -o $t/c.o -fPIC -g -ggnu-pubnames -gdwarf-4 -xc - -gz
+void greet();
+
+int main() {
+  greet();
+}
+EOF
+
+$CC -B. -o $t/exe $t/b.so $t/c.o -Wl,--gdb-index -Wl,--compress-debug-sections=zlib-gnu
+readelf -WS $t/exe 2> /dev/null | fgrep -q .gdb_index
+
+$QEMU $t/exe | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log
+
+grep -Pq 'hello \(\) at .*<stdin>:7' $t/log
+grep -Pq 'greet \(\) at .*<stdin>:11' $t/log
+grep -Pq 'main \(\) at .*<stdin>:4' $t/log
+
+echo OK
@@ -0,0 +1,71 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
+which gdb >& /dev/null || { echo skipped; exit; }
+
+echo 'int main() {}' | $CC -o /dev/null -xc -gdwarf-2 -g - >& /dev/null ||
+  { echo skipped; exit; }
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -g -ggnu-pubnames -gdwarf-2 -xc - -ffunction-sections
+void hello2();
+
+static void hello() {
+  hello2();
+}
+
+void greet() {
+  hello();
+}
+EOF
+
+cat <<EOF | $CC -c -o $t/b.o -fPIC -g -ggnu-pubnames -gdwarf-2 -xc - -ffunction-sections
+#include <stdio.h>
+
+void trap() {}
+
+void hello2() {
+  printf("Hello world\n");
+  trap();
+}
+EOF
+
+$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index
+readelf -WS $t/c.so 2> /dev/null | fgrep -q .gdb_index
+
+cat <<EOF | $CC -c -o $t/d.o -fPIC -g -ggnu-pubnames -gdwarf-2 -xc - -gz
+void greet();
+
+int main() {
+  greet();
+}
+EOF
+
+$CC -B. -o $t/exe $t/c.so $t/d.o -Wl,--gdb-index
+readelf -WS $t/exe 2> /dev/null | fgrep -q .gdb_index
+
+$QEMU $t/exe | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log
+
+grep -Pq 'hello2 \(\) at .*<stdin>:7' $t/log
+grep -Pq 'hello \(\) at .*<stdin>:4' $t/log
+grep -Pq 'greet \(\) at .*<stdin>:8' $t/log
+grep -Pq 'main \(\) at .*<stdin>:4' $t/log
+
+echo OK
@@ -0,0 +1,71 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
+which gdb >& /dev/null || { echo skipped; exit; }
+
+echo 'int main() {}' | $CC -o /dev/null -xc -gdwarf-3 -g - >& /dev/null ||
+  { echo skipped; exit; }
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -g -ggnu-pubnames -gdwarf-3 -xc - -ffunction-sections
+void hello2();
+
+static void hello() {
+  hello2();
+}
+
+void greet() {
+  hello();
+}
+EOF
+
+cat <<EOF | $CC -c -o $t/b.o -fPIC -g -ggnu-pubnames -gdwarf-3 -xc - -ffunction-sections
+#include <stdio.h>
+
+void trap() {}
+
+void hello2() {
+  printf("Hello world\n");
+  trap();
+}
+EOF
+
+$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index
+readelf -WS $t/c.so 2> /dev/null | fgrep -q .gdb_index
+
+cat <<EOF | $CC -c -o $t/d.o -fPIC -g -ggnu-pubnames -gdwarf-3 -xc - -gz
+void greet();
+
+int main() {
+  greet();
+}
+EOF
+
+$CC -B. -o $t/exe $t/c.so $t/d.o -Wl,--gdb-index
+readelf -WS $t/exe 2> /dev/null | fgrep -q .gdb_index
+
+$QEMU $t/exe | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log
+
+grep -Pq 'hello2 \(\) at .*<stdin>:7' $t/log
+grep -Pq 'hello \(\) at .*<stdin>:4' $t/log
+grep -Pq 'greet \(\) at .*<stdin>:8' $t/log
+grep -Pq 'main \(\) at .*<stdin>:4' $t/log
+
+echo OK
@@ -0,0 +1,71 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
+which gdb >& /dev/null || { echo skipped; exit; }
+
+echo 'int main() {}' | $CC -o /dev/null -xc -gdwarf-4 -g - >& /dev/null ||
+  { echo skipped; exit; }
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -g -ggnu-pubnames -gdwarf-4 -xc - -ffunction-sections
+void hello2();
+
+static void hello() {
+  hello2();
+}
+
+void greet() {
+  hello();
+}
+EOF
+
+cat <<EOF | $CC -c -o $t/b.o -fPIC -g -ggnu-pubnames -gdwarf-4 -xc - -ffunction-sections
+#include <stdio.h>
+
+void trap() {}
+
+void hello2() {
+  printf("Hello world\n");
+  trap();
+}
+EOF
+
+$CC -B. -shared -o $t/c.so $t/a.o $t/b.o -Wl,--gdb-index
+readelf -WS $t/c.so 2> /dev/null | fgrep -q .gdb_index
+
+cat <<EOF | $CC -c -o $t/d.o -fPIC -g -ggnu-pubnames -gdwarf-4 -xc - -gz
+void greet();
+
+int main() {
+  greet();
+}
+EOF
+
+$CC -B. -o $t/exe $t/c.so $t/d.o -Wl,--gdb-index
+readelf -WS $t/exe 2> /dev/null | fgrep -q .gdb_index
+
+$QEMU $t/exe | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log
+
+grep -Pq 'hello2 \(\) at .*<stdin>:7' $t/log
+grep -Pq 'hello \(\) at .*<stdin>:4' $t/log
+grep -Pq 'greet \(\) at .*<stdin>:8' $t/log
+grep -Pq 'main \(\) at .*<stdin>:4' $t/log
+
+echo OK
@@ -0,0 +1,107 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
+which gdb >& /dev/null || { echo skipped; exit; }
+
+echo 'int main() {}' | $CC -o /dev/null -xc -gdwarf-5 -g - >& /dev/null ||
+  { echo skipped; exit; }
+
+cat <<EOF > $t/a.c
+void fn3();
+
+static void fn2() {
+  fn3();
+}
+
+void fn1() {
+  fn2();
+}
+EOF
+
+cat <<EOF > $t/b.c
+void fn5();
+
+static void fn4() {
+  fn5();
+}
+
+void fn3() {
+  fn4();
+}
+EOF
+
+cat <<EOF > $t/c.c
+void fn7();
+
+static void fn6() {
+  fn7();
+}
+
+void fn5() {
+  fn6();
+}
+EOF
+
+cat <<EOF > $t/d.c
+#include <stdio.h>
+void trap() {}
+
+static void fn8() {
+  printf("Hello world\n");
+  trap();
+}
+
+void fn7() {
+  fn8();
+}
+EOF
+
+$CC -c -o $t/a.o $t/a.c -fPIC -g -ggnu-pubnames -gdwarf-5 -ffunction-sections
+$CC -c -o $t/b.o $t/b.c -fPIC -g -ggnu-pubnames -gdwarf-4 -ffunction-sections
+$CC -c -o $t/c.o $t/c.c -fPIC -g -ggnu-pubnames -gdwarf-5
+$CC -c -o $t/d.o $t/d.c -fPIC -g -ggnu-pubnames -gdwarf-5 -ffunction-sections
+
+$CC -B. -shared -o $t/e.so $t/a.o $t/b.o $t/c.o $t/d.o -Wl,--gdb-index
+readelf -WS $t/e.so 2> /dev/null | fgrep -q .gdb_index
+
+cat <<EOF | $CC -c -o $t/f.o -fPIC -g -ggnu-pubnames -gdwarf-5 -xc - -gz
+void fn1();
+
+int main() {
+  fn1();
+}
+EOF
+
+$CC -B. -o $t/exe $t/e.so $t/f.o -Wl,--gdb-index
+readelf -WS $t/exe 2> /dev/null | fgrep -q .gdb_index
+
+$QEMU $t/exe | grep -q 'Hello world'
+
+DEBUGINFOD_URLS= gdb $t/exe -nx -batch -ex 'b main' -ex r -ex 'b trap' \
+  -ex c -ex bt -ex quit >& $t/log
+
+grep -Pq 'fn8 \(\) at .*/d.c:6' $t/log
+grep -Pq 'fn7 \(\) at .*/d.c:10' $t/log
+grep -Pq 'fn6 \(\) at .*/c.c:4' $t/log
+grep -Pq 'fn5 \(\) at .*/c.c:8' $t/log
+grep -Pq 'fn4 \(\) at .*/b.c:4' $t/log
+grep -Pq 'fn3 \(\) at .*/b.c:8' $t/log
+grep -Pq 'fn2 \(\) at .*/a.c:4' $t/log
+grep -Pq 'fn1 \(\) at .*/a.c:8' $t/log
+
+echo OK
@@ -1,48 +0,0 @@
-#!/bin/bash
-export LC_ALL=C
-set -e
-CC="${CC:-cc}"
-CXX="${CXX:-c++}"
-GCC="${GCC:-gcc}"
-GXX="${GXX:-g++}"
-OBJDUMP="${OBJDUMP:-objdump}"
-MACHINE="${MACHINE:-$(uname -m)}"
-testname=$(basename "$0" .sh)
-echo -n "Testing $testname ... "
-cd "$(dirname "$0")"/../..
-mold="$(pwd)/mold"
-t=out/test/elf/$testname
-mkdir -p $t
-
-[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
-
-which gdb >& /dev/null || { echo skipped; exit; }
-
-cat <<EOF | $CC -o $t/a.o -c -xc -ggnu-pubnames -g - -ffunction-sections
-#include <stdio.h>
-
-void hello() {
-  printf("Hello world\n");
-}
-
-void greet() {
-  hello();
-}
-EOF
-
-cat <<EOF | $CC -o $t/b.o -c -xc -ggnu-pubnames -g -
-void greet();
-
-int main() {
-  greet();
-}
-EOF
-
-$CC -B. -o $t/exe $t/a.o $t/b.o -Wl,--gdb-index
-$QEMU $t/exe | grep -q 'Hello world'
-
-readelf -WS $t/exe | fgrep -q .gdb_index
-
-gdb $t/exe -ex 'b main' -ex run -ex cont -ex quit >& /dev/null
-
-echo OK
@@ -14,6 +14,8 @@ mold="$(pwd)/mold"
 t=out/test/elf/$testname
 mkdir -p $t

+[ $MACHINE = $(uname -m) ] || { echo skipped; exit; }
+
 which clang >& /dev/null || { echo skipped; exit; }

 cat <<EOF | clang -flto -c -o $t/a.o -xc -
@@ -15,17 +15,17 @@ t=out/test/elf/$testname
 mkdir -p $t

 cat <<EOF | $CC -o $t/a.o -c -xc -
-#include <stdio.h>
-int main() {
-  printf("Hello world\n");
-}
+void foo() {}
 EOF

-rm -f $t/exe
+cat <<EOF | $CC -o $t/b.o -c -xc -
+void foo();
+int main() { foo(); }
+EOF

-$CC -B. -o $t/exe $t/a.o -Wl,-preload 2> /dev/null
-! test -e $t/exe || false
-$CC -B. -o $t/exe $t/a.o 2> /dev/null
-$QEMU $t/exe | grep -q 'Hello world'
+! $CC -B. -o $t/exe $t/a.o $t/b.o \
+  -Wl,--print-dependencies=full > $t/log 2> /dev/null
+
+grep -q 'b\.o.*a\.o.*foo$' $t/log

 echo OK
@@ -0,0 +1,41 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+cat <<EOF | $CC -o $t/a.o -c -xc -
+void foo() {}
+void bar1() {}
+
+__asm__(".symver foo, foo@TEST");
+__asm__(".symver bar1, bar@TEST");
+EOF
+
+cat <<EOF > $t/b.version
+TEST {
+global:
+  foo;
+  bar;
+};
+EOF
+
+$CC -B. -o $t/c.so -shared $t/a.o -Wl,--version-script=$t/b.version
+readelf -W --dyn-syms $t/c.so > $t/log
+
+grep -q ' foo@TEST$' $t/log
+grep -q ' bar@TEST$' $t/log
+grep -q ' bar1$' $t/log
+! grep -q ' foo@@TEST$' $t/log || false
+
+echo OK
@@ -0,0 +1,40 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+cat <<EOF | $CC -o $t/a.o -c -xc -
+void foo() {}
+void foo2() {}
+void foo3() {}
+
+__asm__(".symver foo2, foo@TEST2");
+__asm__(".symver foo3, foo@TEST3");
+EOF
+
+cat <<EOF > $t/b.version
+TEST1 { global: foo; };
+TEST2 {};
+TEST3 {};
+EOF
+
+$CC -B. -o $t/c.so -shared $t/a.o -Wl,--version-script=$t/b.version
+readelf -W --dyn-syms $t/c.so > $t/log
+
+grep -q ' foo@@TEST1$' $t/log
+grep -q ' foo@TEST2$' $t/log
+grep -q ' foo@TEST3$' $t/log
+! grep -q ' foo$' $t/log || false
+
+echo OK
@@ -0,0 +1,35 @@
+#!/bin/bash
+export LC_ALL=C
+set -e
+CC="${CC:-cc}"
+CXX="${CXX:-c++}"
+GCC="${GCC:-gcc}"
+GXX="${GXX:-g++}"
+OBJDUMP="${OBJDUMP:-objdump}"
+MACHINE="${MACHINE:-$(uname -m)}"
+testname=$(basename "$0" .sh)
+echo -n "Testing $testname ... "
+cd "$(dirname "$0")"/../..
+mold="$(pwd)/mold"
+t=out/test/elf/$testname
+mkdir -p $t
+
+cat <<EOF | $GCC -o $t/a.o -c -xassembler -
+.globl foo
+.tls_common foo,4,4
+EOF
+
+cat <<EOF | $CC -o $t/b.o -c -xc -std=c11 -
+#include <stdio.h>
+
+extern _Thread_local int foo;
+
+int main() {
+  printf("foo=%d\n", foo);
+}
+EOF
+
+$CC -B. -o $t/exe $t/a.o $t/b.o
+$QEMU $t/exe | grep -q '^foo=0$'
+
+echo OK
@@ -11,7 +11,9 @@
 # docker run --rm mold-gentoo emerge --color n -s '' | \
 #   perl -ne 'next unless m!^\*\s+(\S+/\S+)!; print "$1\n"'

-if [ "$1" = "" ]; then
+package="$1"
+
+if [ "$package" = "" ]; then
  echo "Usage: $0 gentoo-package-name"
  exit 1
 fi
@@ -24,51 +26,42 @@ if ! docker image ls mold-gentoo | grep -q mold-gentoo; then
  cat <<EOF | docker build -t mold-gentoo -
 FROM gentoo/stage3
 RUN emerge-webrsync
-RUN echo 'USE="elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary"' >> /etc/portage/make.conf && \
+RUN echo 'USE="X ssl elogind -systemd corefonts truetype jpeg jpeg2k tiff zstd static-libs binary"' >> /etc/portage/make.conf && \
    echo 'ACCEPT_LICENSE="* -@EULA"' >> /etc/portage/make.conf && \
    echo 'FEATURES="\${FEATURE} noclean nostrip ccache -ipc-sandbox -network-sandbox -pid-sandbox -sandbox"' >> /etc/portage/make.conf && \
    echo 'CCACHE_DIR="/ccache"' >> /etc/portage/make.conf
-RUN emerge gdb lld vim strace ccache && rm -rf /var/tmp/portage
+RUN emerge gdb lld clang vim emacs strace ccache xeyes dev-util/cmake dev-vcs/git && rm -rf /var/tmp/portage
 EOF
  set +e
 fi

-# Build mold as a portable executable
-./dist.sh
-
 git_hash=$(./mold --version | perl -ne '/\((\w+)/; print $1;')

-# Build a given package in Docker
-build() {
-  package="$1"
-  cmd1='(cd /usr/bin; ln -sf /mold/mold $(realpath ld))'
-  cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
-  cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
-  filename=`echo "$package" | sed 's!/!_!g'`
-  docker="docker run --rm --cap-add=SYS_PTRACE -v `pwd`:/mold -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 1h"
-  dir=gentoo/$git_hash
-
-  mkdir -p "$dir"/success "$dir"/failure
-
-  $docker nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
-  if [ $? = 0 ]; then
-    mv "$dir"/"$filename".mold "$dir"/success
-  else
-    mv "$dir"/"$filename".mold "$dir"/failure
-  fi
-
-  $docker nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
-  if [ $? = 0 ]; then
-    mv "$dir"/"$filename".ld "$dir"/success
-  else
-    mv "$dir"/"$filename".ld "$dir"/failure
-  fi
-}
-
-if [ "$1" = dev-libs/concurrencykit ]; then
-  echo "Skipping known broken package: $1"
+if [ "$package" = dev-libs/concurrencykit ]; then
+  echo "Skipping known broken package: $package"
  exit 0
 fi

-# Build a package
-build "$1"
+# Build a given package in Docker
+cmd1='(cd /usr/bin; ln -sf /mold/mold $(realpath ld))'
+cmd2="MAKEOPTS=-'j$(nproc) --load-average=100' emerge --onlydeps $package"
+cmd3="MAKEOPTS='-j$(nproc) --load-average=100' FEATURES=test emerge $package"
+filename=`echo "$package" | sed 's!/!_!g'`
+docker="docker run --rm --cap-add=SYS_PTRACE -v `pwd`:/mold -v /var/cache/ccache-gentoo:/ccache mold-gentoo timeout -v -k 15s 1h"
+dir=gentoo/$git_hash
+
+mkdir -p "$dir"/success "$dir"/failure
+
+$docker nice -n 19 bash -c "$cmd1 && $cmd2 && $cmd3" >& "$dir"/"$filename".mold
+if [ $? = 0 ]; then
+  mv "$dir"/"$filename".mold "$dir"/success
+else
+  mv "$dir"/"$filename".mold "$dir"/failure
+fi
+
+$docker nice -n 19 bash -c "$cmd2 && $cmd3" >& "$dir"/"$filename".ld
+if [ $? = 0 ]; then
+  mv "$dir"/"$filename".ld "$dir"/success
+else
+  mv "$dir"/"$filename".ld "$dir"/failure
+fi
@@ -97,8 +97,8 @@ if(MI_OVERRIDE)
        message(STATUS "  WARNING: interpose usually also needs zone overriding (use -DMI_OSX_INTERPOSE=ON)")
      endif()
    endif()
-    if((NOT MI_USE_CXX) AND MI_OVERRIDE)
-      message(STATUS "  WARNING: if overriding C++ new/delete, it is best to build mimalloc with a C++ compiler (use -DMI_USE_CXX=ON)")
+    if(MI_USE_CXX AND MI_OSX_INTERPOSE)
+      message(STATUS "  WARNING: if dynamically overriding malloc/free, it is more reliable to build mimalloc as C code (use -DMI_USE_CXX=OFF)")
    endif()
  endif()
 endif()
@@ -18,7 +18,7 @@ jobs:
  displayName: Windows
  pool:
    vmImage:
-      windows-2019
+      windows-2022
  strategy:
    matrix:
      Debug:
@@ -1,6 +1,6 @@
 set(mi_version_major 2)
 set(mi_version_minor 0)
-set(mi_version_patch 5)
+set(mi_version_patch 6)
 set(mi_version ${mi_version_major}.${mi_version_minor})

 set(PACKAGE_VERSION ${mi_version})
@@ -56,7 +56,7 @@ Notable aspects of the design include:
 - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions.
  A heap can be destroyed at once instead of deallocating each object separately.
 - __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation
-  times (_wcat_), bounded space overhead (~0.2% meta-data, with at most 12.5% waste in allocation sizes),
+  times (_wcat_), bounded space overhead (~0.2% meta-data, with low internal fragmentation),
  and has no internal points of contention using only atomic operations.
 - __fast__: In our benchmarks (see [below](#performance)),
  _mimalloc_ outperforms all other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc),
@@ -118,7 +118,7 @@ $(document).ready(function(){initNavTree('overrides.html',''); initResizable();
 <h3>Windows</h3>
 <p>Overriding on Windows is robust and has the particular advantage to be able to redirect all malloc/free calls that go through the (dynamic) C runtime allocator, including those from other DLL's or libraries.</p>
 <p>The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the <code>/MD</code> or <code>/MDd</code> switch). Also, the <code>mimalloc-redirect.dll</code> (or <code>mimalloc-redirect32.dll</code>) must be available in the same folder as the main <code>mimalloc-override.dll</code> at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in <code>mimalloc-override.dll</code>).</p>
-<p>To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the <code>main</code> function, like <code>mi_version()</code> (or use the <code>/INCLUDE:mi_version</code> switch on the linker). See the <code>mimalloc-override-test</code> project for an example on how to use this. For best performance on Windows with C++, it is also recommended to also override the <code>new</code>/<code>delete</code> operations (by including <a href="https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h"><code>mimalloc-new-delete.h</code></a> a single(!) source file in your project).</p>
+<p>To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the <code>main</code> function, like <code>mi_version()</code> (or use the <code>/INCLUDE:mi_version</code> switch on the linker). See the <code>mimalloc-override-test</code> project for an example on how to use this. For best performance on Windows with C++, it is also recommended to also override the <code>new</code>/<code>delete</code> operations (by including <a href="https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h"><code>mimalloc-new-delete.h</code></a> a single(!) source file in your project without linking to the mimalloc library).</p>
 <p>The environment variable <code>MIMALLOC_DISABLE_REDIRECT=1</code> can be used to disable dynamic overriding at run-time. Use <code>MIMALLOC_VERBOSE=1</code> to check if mimalloc was successfully redirected.</p>
 <p>(Note: in principle, it is possible to even patch existing executables without any recompilation if they are linked with the dynamic C runtime (<code>ucrtbase.dll</code>) &ndash; just put the <code>mimalloc-override.dll</code> into the import table (and put <code>mimalloc-redirect.dll</code> in the same folder) Such patching can be done for example with <a href="https://ntcore.com/?page_id=388">CFF Explorer</a>).</p>
 <h2>Static override</h2>
@@ -105,7 +105,7 @@ $(document).ready(function(){initNavTree('using.html',''); initResizable(); });
 </div><!-- fragment --><p> to link with the shared (dynamic) library, or: </p><div class="fragment"><div class="line">target_link_libraries(myapp PUBLIC mimalloc-<span class="keyword">static</span>)</div>
 </div><!-- fragment --><p> to link with the static library. See <code>test\CMakeLists.txt</code> for an example.</p>
 <h3>C++</h3>
-<p>For best performance in C++ programs, it is also recommended to override the global <code>new</code> and <code>delete</code> operators. For convience, mimalloc provides <a href="https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h"><code>mimalloc-new-delete.h</code></a> which does this for you &ndash; just include it in a single(!) source file in your project.</p>
+<p>For best performance in C++ programs, it is also recommended to override the global <code>new</code> and <code>delete</code> operators. For convience, mimalloc provides <a href="https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h"><code>mimalloc-new-delete.h</code></a> which does this for you &ndash; just include it in a single(!) source file in your project without linking to the mimalloc's library.</p>
 <p>In C++, mimalloc also provides the <code><a class="el" href="group__cpp.html#structmi__stl__allocator" title="std::allocator implementation for mimalloc for use in STL containers.">mi_stl_allocator</a></code> struct which implements the <code>std::allocator</code> interface. For example: </p><div class="fragment"><div class="line">std::vector&lt;some_struct, mi_stl_allocator&lt;some_struct&gt;&gt; vec;</div>
 <div class="line">vec.push_back(some_struct());</div>
 </div><!-- fragment --><h3>Statistics</h3>
@@ -23,10 +23,15 @@ terms of the MIT license. A copy of the license can be found in the file
 #define  _Atomic(tp)            std::atomic<tp>
 #define  mi_atomic(name)        std::atomic_##name
 #define  mi_memory_order(name)  std::memory_order_##name
+#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571
+ #define MI_ATOMIC_VAR_INIT(x)  x
+#else
+ #define MI_ATOMIC_VAR_INIT(x)  ATOMIC_VAR_INIT(x)
+#endif
 #elif defined(_MSC_VER)
 // Use MSVC C wrapper for C11 atomics
 #define  _Atomic(tp)            tp 
-#define  ATOMIC_VAR_INIT(x)     x
+#define  MI_ATOMIC_VAR_INIT(x)  x
 #define  mi_atomic(name)        mi_atomic_##name
 #define  mi_memory_order(name)  mi_memory_order_##name
 #else
@@ -34,6 +39,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #include <stdatomic.h>
 #define  mi_atomic(name)        atomic_##name
 #define  mi_memory_order(name)  memory_order_##name
+#define  MI_ATOMIC_VAR_INIT(x)  ATOMIC_VAR_INIT(x)
 #endif

 // Various defines for all used memory orders in mimalloc
@@ -148,8 +148,8 @@ mi_msecs_t  _mi_clock_start(void);

 // "alloc.c"
 void*       _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept;  // called from `_mi_malloc_generic`
-void*       _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero);
-void*       _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero);
+void*       _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
+void*       _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept;
 mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p);
 bool        _mi_free_delayed_block(mi_block_t* block);
 void        _mi_block_zero_init(const mi_page_t* page, void* p, size_t size);
@@ -1035,7 +1035,7 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
  mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
  void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
  const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
-  memcpy(adst, asrc, n);
+  _mi_memcpy(adst, asrc, n);
 }
 #else
 // Default fallback on `_mi_memcpy`
@@ -25,6 +25,9 @@ terms of the MIT license. A copy of the license can be found in the file
  void operator delete(void* p) noexcept              { mi_free(p); };
  void operator delete[](void* p) noexcept            { mi_free(p); };

+  void operator delete  (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
+  void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
+
  void* operator new(std::size_t n) noexcept(false)   { return mi_new(n); }
  void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }

@@ -41,9 +44,11 @@ terms of the MIT license. A copy of the license can be found in the file
  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-
-  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t& tag) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t& tag) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  
+  void* operator new  (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
+  void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
  #endif
@@ -156,7 +156,6 @@ typedef int32_t  mi_ssize_t;
 #define MI_MEDIUM_OBJ_WSIZE_MAX           (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)   
 #define MI_LARGE_OBJ_SIZE_MAX             (MI_SEGMENT_SIZE/2)      // 32MiB on 64-bit
 #define MI_LARGE_OBJ_WSIZE_MAX            (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
-#define MI_HUGE_OBJ_SIZE_MAX              (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE)        // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)

 // Maximum number of size classes. (spaced exponentially in 12.5% increments)
 #define MI_BIN_HUGE  (73U)
@@ -175,7 +174,7 @@ typedef int32_t  mi_ssize_t;
 #define MI_MAX_SLICE_OFFSET               ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1)

 // Used as a special value to encode block sizes in 32 bits.
-#define MI_HUGE_BLOCK_SIZE                ((uint32_t)MI_HUGE_OBJ_SIZE_MAX)
+#define MI_HUGE_BLOCK_SIZE                ((uint32_t)(2*MI_GiB))

 // blocks up to this size are always allocated aligned
 #define MI_MAX_ALIGN_GUARANTEE            (8*MI_MAX_ALIGN_SIZE)  
@@ -581,9 +580,6 @@ typedef struct mi_segments_tld_s {
  size_t              peak_count;   // peak number of segments
  size_t              current_size; // current size of all segments
  size_t              peak_size;    // peak size of all segments
-  size_t              cache_count;  // number of segments in the cache
-  size_t              cache_size;   // total size of all segments in the cache
-  mi_segment_t*       cache;        // (small) cache of segments
  mi_stats_t*         stats;        // points to tld stats
  mi_os_tld_t*        os;           // points to os stats
 } mi_segments_tld_t;
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #ifndef MIMALLOC_H
 #define MIMALLOC_H

-#define MI_MALLOC_VERSION 205   // major + 2 digits minor
+#define MI_MALLOC_VERSION 206   // major + 2 digits minor

 // ------------------------------------------------------
 // Compiler specific attributes
@@ -256,6 +256,7 @@ typedef struct mi_heap_area_s {
  size_t committed;   // current available bytes for this area
  size_t used;        // number of allocated blocks
  size_t block_size;  // size in bytes of each block
+  size_t full_block_size; // size in bytes of a full block including padding and metadata.
 } mi_heap_area_t;

 typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
@@ -315,7 +316,7 @@ typedef enum mi_option_e {
  mi_option_reserve_huge_os_pages,    // reserve N huge OS pages (1GiB) at startup
  mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
  mi_option_reserve_os_memory,        // reserve specified amount of OS memory at startup
-  mi_option_segment_cache,
+  mi_option_deprecated_segment_cache,
  mi_option_page_reset,
  mi_option_abandoned_page_decommit,
  mi_option_deprecated_segment_reset,
@@ -326,6 +327,7 @@ typedef enum mi_option_e {
  mi_option_os_tag,
  mi_option_max_errors,
  mi_option_max_warnings,
+  mi_option_max_segment_reclaim,
  mi_option_allow_decommit,
  mi_option_segment_decommit_delay,  
  mi_option_decommit_extend_delay,
@@ -340,6 +342,7 @@ mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
 mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);

 mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
+mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
 mi_decl_export void mi_option_set(mi_option_t option, long value);
 mi_decl_export void mi_option_set_default(mi_option_t option, long value);

@@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac
 Initially developed by Daan Leijen for the run-time systems of the
 [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages.

-Latest release tag: `v2.0.5` (alpha, 2022-02-14).  
-Latest stable  tag: `v1.7.5` (2022-02-14).
+Latest release tag: `v2.0.6` (2022-04-14).  
+Latest stable  tag: `v1.7.6` (2022-02-14).

 mimalloc is a drop-in replacement for `malloc` and can be used in other programs
 without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as:
@@ -52,7 +52,7 @@ It also has an easy way to override the default allocator in [Windows](#override
 - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions.
  A heap can be destroyed at once instead of deallocating each object separately.  
 - __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation
-  times (_wcat_), bounded space overhead (~0.2% meta-data, with at most 12.5% waste in allocation sizes),
+  times (_wcat_), bounded space overhead (~0.2% meta-data, with low internal fragmentation),
  and has no internal points of contention using only atomic operations.
 - __fast__: In our benchmarks (see [below](#performance)),
  _mimalloc_ outperforms other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc),
@@ -67,16 +67,21 @@ Enjoy!

 ### Branches

-* `master`: latest stable release.
-* `dev`: development branch for mimalloc v1.
-* `dev-slice`: development branch for mimalloc v2 with a new algorithm for managing internal mimalloc pages.
+* `master`: latest stable release (based on `dev-slice`).
+* `dev`: development branch for mimalloc v1. Use this branch for submitting PR's.
+* `dev-slice`: development branch for mimalloc v2. This branch is downstream of `dev`.

 ### Releases

-Note: the `v2.x` beta has a new algorithm for managing internal mimalloc pages that tends to use reduce memory usage
+Note: the `v2.x` version has a new algorithm for managing internal mimalloc pages that tends to use reduce memory usage
  and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance
  (see [below](#performance)); please report if you observe any significant performance regression.

+* 2022-04-14, `v1.7.6`, `v2.0.6`: fix fallback path for aligned OS allocation on Windows, improve Windows aligned allocation
+  even when compiling with older SDK's, fix dynamic overriding on macOS Monterey, fix MSVC C++ dynamic overriding, fix
+  warnings under Clang 14, improve performance if many OS threads are created and destroyed, fix statistics for large object
+  allocations, using MIMALLOC_VERBOSE=1 has no maximum on the number of error messages, various small fixes.
+
 * 2022-02-14, `v1.7.5`, `v2.0.5` (alpha): fix malloc override on
  Windows 11, fix compilation with musl, potentially reduced
  committed memory, add `bin/minject` for Windows, 
@@ -301,7 +306,7 @@ or via environment variables:

 Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write
 for all pages in the original process including the huge OS pages. When any memory is now written in that area, the
-OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments.
+OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments.

 [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5
 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017
@@ -64,7 +64,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) {

 static void zone_free(malloc_zone_t* zone, void* p) {
  MI_UNUSED(zone);
-  mi_free(p);
+  mi_cfree(p);
 }

 static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
@@ -373,7 +373,7 @@ __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[]
  MI_INTERPOSE_MI(_malloc_fork_child),
  MI_INTERPOSE_MI(_malloc_fork_parent),
  MI_INTERPOSE_MI(_malloc_fork_prepare),
-
+  
  MI_INTERPOSE_ZONE(zone_batch_free),
  MI_INTERPOSE_ZONE(zone_batch_malloc),
  MI_INTERPOSE_ZONE(zone_calloc),
@@ -16,6 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) 

 #if defined(__APPLE__)
+#include <AvailabilityMacros.h>
 mi_decl_externc void   vfree(void* p);
 mi_decl_externc size_t malloc_size(const void* p);
 mi_decl_externc size_t malloc_good_size(size_t size);
@@ -77,7 +78,9 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
    MI_INTERPOSE_MI(valloc),
    MI_INTERPOSE_MI(malloc_size),
    MI_INTERPOSE_MI(malloc_good_size),
+    #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 
    MI_INTERPOSE_MI(aligned_alloc),
+    #endif
    #ifdef MI_OSX_ZONE
    // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely
    MI_INTERPOSE_MI(free),
@@ -91,15 +94,18 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;

  #ifdef __cplusplus
  extern "C" {
-    void  _ZdlPv(void* p);   // delete
-    void  _ZdaPv(void* p);   // delete[]
-    void  _ZdlPvm(void* p, size_t n);  // delete
-    void  _ZdaPvm(void* p, size_t n);  // delete[]
-    void* _Znwm(size_t n);  // new
-    void* _Znam(size_t n);  // new[]
-    void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
-    void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
-  }  
+  #endif
+  void  _ZdlPv(void* p);   // delete
+  void  _ZdaPv(void* p);   // delete[]
+  void  _ZdlPvm(void* p, size_t n);  // delete
+  void  _ZdaPvm(void* p, size_t n);  // delete[]
+  void* _Znwm(size_t n);  // new
+  void* _Znam(size_t n);  // new[]
+  void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow
+  void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow
+  #ifdef __cplusplus
+  }
+  #endif
  __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[]  __attribute__((section("__DATA, __interpose"))) =
  {
    MI_INTERPOSE_FUN(_ZdlPv,mi_free),
@@ -111,7 +117,6 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
    MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),
    MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),
  };
-  #endif // __cplusplus

 #elif defined(_MSC_VER)
  // cannot override malloc unless using a dll.
@@ -161,7 +166,9 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
  void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-
+  void operator delete  (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
+  
  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
@@ -32,17 +32,17 @@ terms of the MIT license. A copy of the license can be found in the file
 #endif


-size_t mi_malloc_size(const void* p) mi_attr_noexcept {
+mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
  //if (!mi_is_in_heap_region(p)) return 0;
  return mi_usable_size(p);
 }

-size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
+mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
  //if (!mi_is_in_heap_region(p)) return 0;
  return mi_usable_size(p);
 }

-size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
+mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
  return mi_good_size(size);
 }

@@ -65,24 +65,24 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept
  return 0;
 }

-mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
  void* p = mi_malloc_aligned(size, alignment);
  mi_assert_internal(((uintptr_t)p % alignment) == 0);
  return p;
 }

-mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {
  return mi_memalign( _mi_os_page_size(), size );
 }

-mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
  size_t psize = _mi_os_page_size();
  if (size >= SIZE_MAX - psize) return NULL; // overflow
  size_t asize = _mi_align_up(size, psize);
  return mi_malloc_aligned(asize, psize);
 }

-mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
  if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
    #if MI_DEBUG > 0
    _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
@@ -95,13 +95,13 @@ mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_n
  return p;
 }

-void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept {  // BSD
+mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept {  // BSD
  void* newp = mi_reallocn(p,count,size);
  if (newp==NULL) { errno = ENOMEM; }
  return newp;
 }

-int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
+mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
  mi_assert(p != NULL);
  if (p == NULL) {
    errno = EINVAL;
@@ -120,7 +120,7 @@ void* mi__expand(void* p, size_t newsize) mi_attr_noexcept {  // Microsoft
  return res;
 }

-mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
  if (s==NULL) return NULL;
  size_t len;
  for(len = 0; s[len] != 0; len++) { }
@@ -132,7 +132,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex
  return p;
 }

-mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s)  mi_attr_noexcept {
  return (unsigned char*)mi_strdup((const char*)s);
 }

@@ -172,10 +172,10 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name)
 #endif
 }

-void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
+mi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
  return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
 }

-void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
+mi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
  return mi_recalloc_aligned(p, newcount, size, alignment);
 }
@@ -45,7 +45,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz

 #if (MI_STAT>0)
  const size_t bsize = mi_page_usable_block_size(page);
-  if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
+  if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {
    mi_heap_stat_increase(heap, normal, bsize);
    mi_heap_stat_counter_increase(heap, normal_count, 1);
 #if (MI_STAT>1)
@@ -147,7 +147,7 @@ mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
  return p;
 }

-void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) {
+void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
  void* p = mi_heap_malloc(heap,size);
  if (zero && p != NULL) {
    _mi_block_zero_init(_mi_ptr_page(p),p,size);  // todo: can we avoid getting the page again?
@@ -297,20 +297,26 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co
 // only maintain stats for smaller objects if requested
 #if (MI_STAT>0)
 static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
-#if (MI_STAT < 2)  
+  #if (MI_STAT < 2)  
  MI_UNUSED(block);
-#endif
+  #endif
  mi_heap_t* const heap = mi_heap_get_default();
-  const size_t bsize = mi_page_usable_block_size(page);  
-#if (MI_STAT>1)
+  const size_t bsize = mi_page_usable_block_size(page);
+  #if (MI_STAT>1)
  const size_t usize = mi_page_usable_size_of(page, block);
  mi_heap_stat_decrease(heap, malloc, usize);
-#endif  
-  if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
+  #endif  
+  if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {
    mi_heap_stat_decrease(heap, normal, bsize);
-#if (MI_STAT > 1)
+    #if (MI_STAT > 1)
    mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1);
-#endif
+    #endif
+  }
+  else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
+    mi_heap_stat_decrease(heap, large, bsize);
+  }
+  else {
+    mi_heap_stat_decrease(heap, huge, bsize);
  }
 }
 #else
@@ -470,6 +476,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms
 #if (MI_DEBUG>0 || MI_SECURE>=4)
  if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) {
    _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p);
+    return NULL;
  }
 #endif
  return segment;
@@ -529,20 +536,25 @@ bool _mi_free_delayed_block(mi_block_t* block) {
 }

 // Bytes available in a block
-static size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept {
-  const mi_segment_t* const segment = mi_checked_ptr_segment(p,msg);
-  if (segment==NULL) return 0;
-  const mi_page_t* const page = _mi_segment_page_of(segment, p);
-  const mi_block_t* block = (const mi_block_t*)p;
-  if (mi_unlikely(mi_page_has_aligned(page))) {
-    block = _mi_page_ptr_unalign(segment, page, p);
-    size_t size = mi_page_usable_size_of(page, block);
-    ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)block;
-    mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
-    return (size - adjust);
+mi_decl_noinline static size_t mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept {
+  const mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+  const size_t size = mi_page_usable_size_of(page, block);
+  const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
+  mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
+  return (size - adjust);
+}
+
+static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept {
+  const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg);
+  if (segment==NULL) return 0;  // also returns 0 if `p == NULL`
+  const mi_page_t* const page = _mi_segment_page_of(segment, p);  
+  if (mi_likely(!mi_page_has_aligned(page))) {
+    const mi_block_t* block = (const mi_block_t*)p;
+    return mi_page_usable_size_of(page, block);
  }
  else {
-    return mi_page_usable_size_of(page, block);
+    // split out to separate routine for improved code generation
+    return mi_page_usable_aligned_size_of(segment, page, p);
  }
 }

@@ -611,35 +623,43 @@ mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
  return mi_heap_mallocn(mi_get_default_heap(),count,size);
 }

-// Expand in place or fail
+// Expand (or shrink) in place (or fail)
 void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
+  #if MI_PADDING
+  // we do not shrink/expand with padding enabled 
+  MI_UNUSED(p); MI_UNUSED(newsize);
+  return NULL;
+  #else
  if (p == NULL) return NULL;
-  size_t size = _mi_usable_size(p,"mi_expand");
+  const size_t size = _mi_usable_size(p,"mi_expand");
  if (newsize > size) return NULL;
  return p; // it fits
+  #endif
 }

-void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) {
-  if (p == NULL) return _mi_heap_malloc_zero(heap,newsize,zero);
-  size_t size = _mi_usable_size(p,"mi_realloc");
-  if (newsize <= size && newsize >= (size / 2)) {
+void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
+  const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL
+  if (mi_unlikely(newsize <= size && newsize >= (size / 2))) {
+    // todo: adjust potential padding to reflect the new size?
    return p;  // reallocation still fits and not more than 50% waste
  }
  void* newp = mi_heap_malloc(heap,newsize);
  if (mi_likely(newp != NULL)) {
    if (zero && newsize > size) {
      // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
-      size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
+      const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
      memset((uint8_t*)newp + start, 0, newsize - start);
    }
-    _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
-    mi_free(p); // only free if successful
+    if (mi_likely(p != NULL)) {
+      _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
+      mi_free(p); // only free the original pointer if successful
+    }
  }
  return newp;
 }

 void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
-  return _mi_heap_realloc_zero(heap, p, newsize, false);
+  return _mi_heap_realloc_zero(heap, p, newsize, false);  
 }

 void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
@@ -481,13 +481,14 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
  if (page->used == 0) return true;

  const size_t bsize = mi_page_block_size(page);
+  const size_t ubsize = mi_page_usable_block_size(page); // without padding
  size_t   psize;
  uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize);

  if (page->capacity == 1) {
    // optimize page with one block
    mi_assert_internal(page->used == 1 && page->free == NULL);
-    return visitor(mi_page_heap(page), area, pstart, bsize, arg);
+    return visitor(mi_page_heap(page), area, pstart, ubsize, arg);
  }

  // create a bitmap of free blocks.
@@ -521,7 +522,7 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
    else if ((m & ((uintptr_t)1 << bit)) == 0) {
      used_count++;
      uint8_t* block = pstart + (i * bsize);
-      if (!visitor(mi_page_heap(page), area, block, bsize, arg)) return false;
+      if (!visitor(mi_page_heap(page), area, block, ubsize, arg)) return false;
    }
  }
  mi_assert_internal(page->used == used_count);
@@ -537,12 +538,14 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
  mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun;
  mi_heap_area_ex_t xarea;
  const size_t bsize = mi_page_block_size(page);
+  const size_t ubsize = mi_page_usable_block_size(page);
  xarea.page = page;
  xarea.area.reserved = page->reserved * bsize;
  xarea.area.committed = page->capacity * bsize;
  xarea.area.blocks = _mi_page_start(_mi_page_segment(page), page, NULL);
-  xarea.area.used = page->used;
-  xarea.area.block_size = bsize;
+  xarea.area.used = page->used * bsize;
+  xarea.area.block_size = ubsize;
+  xarea.area.full_block_size = bsize;
  return fun(heap, &xarea, arg);
 }

@@ -25,8 +25,8 @@ const mi_page_t _mi_page_empty = {
  0,       // used
  0,       // xblock_size
  NULL,    // local_free
-  ATOMIC_VAR_INIT(0), // xthread_free
-  ATOMIC_VAR_INIT(0), // xheap
+  MI_ATOMIC_VAR_INIT(0), // xthread_free
+  MI_ATOMIC_VAR_INIT(0), // xheap
  NULL, NULL
  #if MI_INTPTR_SIZE==8
  , { 0 }  // padding
@@ -106,7 +106,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
  NULL,
  MI_SMALL_PAGES_EMPTY,
  MI_PAGE_QUEUES_EMPTY,
-  ATOMIC_VAR_INIT(NULL),
+  MI_ATOMIC_VAR_INIT(NULL),
  0,                // tid
  0,                // cookie
  { 0, 0 },         // keys
@@ -124,7 +124,7 @@ mi_decl_cache_align static const mi_tld_t tld_empty = {
  0,
  false,
  NULL, NULL,
-  { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_empty_stats, tld_empty_os }, // segments
+  { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, tld_empty_stats, tld_empty_os }, // segments
  { 0, tld_empty_stats }, // os
  { MI_STATS_NULL }       // stats
 };
@@ -137,7 +137,7 @@ extern mi_heap_t _mi_heap_main;
 static mi_tld_t tld_main = {
  0, false,
  &_mi_heap_main, & _mi_heap_main,
-  { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, &tld_main.stats, &tld_main.os }, // segments
+  { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, &tld_main.stats, &tld_main.os }, // segments
  { 0, &tld_main.stats },  // os
  { MI_STATS_NULL }       // stats
 };
@@ -146,7 +146,7 @@ mi_heap_t _mi_heap_main = {
  &tld_main,
  MI_SMALL_PAGES_EMPTY,
  MI_PAGE_QUEUES_EMPTY,
-  ATOMIC_VAR_INIT(NULL),
+  MI_ATOMIC_VAR_INIT(NULL),
  0,                // thread id
  0,                // initial cookie
  { 0, 0 },         // the key of the main heap can be fixed (unlike page keys that need to be secure!)
@@ -188,6 +188,68 @@ typedef struct mi_thread_data_s {
  mi_tld_t   tld;
 } mi_thread_data_t;

+
+// Thread meta-data is allocated directly from the OS. For
+// some programs that do not use thread pools and allocate and
+// destroy many OS threads, this may causes too much overhead 
+// per thread so we maintain a small cache of recently freed metadata.
+
+#define TD_CACHE_SIZE (8)
+static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE];
+
+static mi_thread_data_t* mi_thread_data_alloc(void) {
+  // try to find thread metadata in the cache
+  mi_thread_data_t* td;
+  for (int i = 0; i < TD_CACHE_SIZE; i++) {
+    td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);
+    if (td != NULL) {
+      td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); 
+      if (td != NULL) {
+        return td;
+      }
+    }
+  }
+  // if that fails, allocate directly from the OS
+  td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
+  if (td == NULL) {
+    // if this fails, try once more. (issue #257)
+    td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
+    if (td == NULL) {
+      // really out of memory
+      _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
+    }
+  }
+  return td;
+}
+
+static void mi_thread_data_free( mi_thread_data_t* tdfree ) {
+  // try to add the thread metadata to the cache
+  for (int i = 0; i < TD_CACHE_SIZE; i++) {
+    mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);
+    if (td == NULL) {
+      mi_thread_data_t* expected = NULL;
+      if (mi_atomic_cas_ptr_weak_acq_rel(mi_thread_data_t, &td_cache[i], &expected, tdfree)) {
+        return;
+      }
+    }
+  }
+  // if that fails, just free it directly
+  _mi_os_free(tdfree, sizeof(mi_thread_data_t), &_mi_stats_main);
+}
+
+static void mi_thread_data_collect(void) {
+  // free all thread metadata from the cache
+  for (int i = 0; i < TD_CACHE_SIZE; i++) {
+    mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);
+    if (td != NULL) {
+      td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL);
+      if (td != NULL) {
+        _mi_os_free( td, sizeof(mi_thread_data_t), &_mi_stats_main );
+      }
+    }
+  }
+}
+
 // Initialize the thread local default heap, called from `mi_thread_init`
 static bool _mi_heap_init(void) {
  if (mi_heap_is_initialized(mi_get_default_heap())) return true;
@@ -200,16 +262,9 @@ static bool _mi_heap_init(void) {
  }
  else {
    // use `_mi_os_alloc` to allocate directly from the OS
-    mi_thread_data_t* td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation?
-    if (td == NULL) {
-      // if this fails, try once more. (issue #257)
-      td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
-      if (td == NULL) {
-        // really out of memory
-        _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
-        return false;
-      }
-    }
+    mi_thread_data_t* td = mi_thread_data_alloc();
+    if (td == NULL) return false;
+
    // OS allocated so already zero initialized
    mi_tld_t*  tld = &td->tld;
    mi_heap_t* heap = &td->heap;
@@ -269,16 +324,17 @@ static bool _mi_heap_done(mi_heap_t* heap) {
    // as abondened: one may allocate it in one thread, but deallocate in another in which case
    // the count can be too large or negative. todo: perhaps not count huge segments? see issue #363
    // mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());
-    _mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main);
+    mi_thread_data_free((mi_thread_data_t*)heap);
  }
-#if 0  
-  // never free the main thread even in debug mode; if a dll is linked statically with mimalloc,
-  // there may still be delete/free calls after the mi_fls_done is called. Issue #207
  else {
+    mi_thread_data_collect(); // free cached thread metadata  
+    #if 0  
+    // never free the main thread even in debug mode; if a dll is linked statically with mimalloc,
+    // there may still be delete/free calls after the mi_fls_done is called. Issue #207
    _mi_heap_destroy_pages(heap);
    mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main);
+    #endif
  }
-#endif
  return false;
 }

@@ -352,7 +408,7 @@ bool _mi_is_main_thread(void) {
  return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id());
 }

-static _Atomic(size_t) thread_count = ATOMIC_VAR_INIT(1);
+static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1);

 size_t  _mi_current_thread_count(void) {
  return mi_atomic_load_relaxed(&thread_count);
@@ -429,7 +485,7 @@ bool _mi_preloading(void) {
  return os_preloading;
 }

-bool mi_is_redirected(void) mi_attr_noexcept {
+mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
  return mi_redirected;
 }

@@ -533,7 +589,7 @@ void mi_process_init(void) mi_attr_noexcept {
  mi_stats_reset();  // only call stat reset *after* thread init (or the heap tld == NULL)

  if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
-    size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
+    size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);
    long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at);
    if (reserve_at != -1) {
      mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);
@@ -19,8 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file
 #endif


-static size_t mi_max_error_count   = 16; // stop outputting errors after this
-static size_t mi_max_warning_count = 16; // stop outputting warnings after this
+static long mi_max_error_count   = 16; // stop outputting errors after this (use < 0 for no limit)
+static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)

 static void mi_add_stderr_output(void);

@@ -74,7 +74,7 @@ static mi_option_desc_t options[_mi_option_last] =
  { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },  // per 1GiB huge pages
  { -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
  { 0, UNINIT, MI_OPTION(reserve_os_memory)     },
-  { 0, UNINIT, MI_OPTION(segment_cache) },       // cache N segments per thread
+  { 0, UNINIT, MI_OPTION(deprecated_segment_cache) },  // cache N segments per thread
  { 0, UNINIT, MI_OPTION(page_reset) },          // reset page memory on free
  { 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_decommit, abandoned_page_reset) },// decommit free page memory when a thread terminates  
  { 0, UNINIT, MI_OPTION(deprecated_segment_reset) },
@@ -91,9 +91,10 @@ static mi_option_desc_t options[_mi_option_last] =
  { 100,  UNINIT, MI_OPTION(os_tag) },            // only apple specific for now but might serve more or less related purpose
  { 16,   UNINIT, MI_OPTION(max_errors) },        // maximum errors that are output
  { 16,   UNINIT, MI_OPTION(max_warnings) },      // maximum warnings that are output
+  { 8,    UNINIT, MI_OPTION(max_segment_reclaim)},// max. number of segment reclaims from the abandoned segments per try.  
  { 1,    UNINIT, MI_OPTION(allow_decommit) },    // decommit slices when no longer used (after decommit_delay milli-seconds)
  { 500,  UNINIT, MI_OPTION(segment_decommit_delay) }, // decommit delay in milli-seconds for freed segments
-  { 2,    UNINIT, MI_OPTION(decommit_extend_delay) }  
+  { 2,    UNINIT, MI_OPTION(decommit_extend_delay) }
 };

 static void mi_option_init(mi_option_desc_t* desc);
@@ -116,6 +117,7 @@ void _mi_options_init(void) {

 mi_decl_nodiscard long mi_option_get(mi_option_t option) {
  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return 0;
  mi_option_desc_t* desc = &options[option];
  mi_assert(desc->option == option);  // index should match the option
  if (mi_unlikely(desc->init == UNINIT)) {
@@ -124,8 +126,14 @@ mi_decl_nodiscard long mi_option_get(mi_option_t option) {
  return desc->value;
 }

+mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {
+  long x = mi_option_get(option);
+  return (x < min ? min : (x > max ? max : x));
+}
+
 void mi_option_set(mi_option_t option, long value) {
  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return;
  mi_option_desc_t* desc = &options[option];
  mi_assert(desc->option == option);  // index should match the option
  desc->value = value;
@@ -134,6 +142,7 @@ void mi_option_set(mi_option_t option, long value) {

 void mi_option_set_default(mi_option_t option, long value) {
  mi_assert(option >= 0 && option < _mi_option_last);
+  if (option < 0 || option >= _mi_option_last) return;
  mi_option_desc_t* desc = &options[option];
  if (desc->init != INITIALIZED) {
    desc->value = value;
@@ -163,10 +172,22 @@ void mi_option_disable(mi_option_t option) {

 static void mi_out_stderr(const char* msg, void* arg) {
  MI_UNUSED(arg);
+  if (msg == NULL) return;
  #ifdef _WIN32
  // on windows with redirection, the C runtime cannot handle locale dependent output
  // after the main thread closes so we use direct console output.
-  if (!_mi_preloading()) { _cputs(msg); }
+  if (!_mi_preloading()) { 
+    // _cputs(msg);  // _cputs cannot be used at is aborts if it fails to lock the console
+    static HANDLE hcon = INVALID_HANDLE_VALUE;
+    if (hcon == INVALID_HANDLE_VALUE) {
+      hcon = GetStdHandle(STD_ERROR_HANDLE);
+    }
+    const size_t len = strlen(msg);
+    if (hcon != INVALID_HANDLE_VALUE && len > 0 && len < UINT32_MAX) {
+      DWORD written = 0;
+      WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
+    }
+  }
  #else
  fputs(msg, stderr);
  #endif
@@ -322,11 +343,22 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
  va_end(args);
 }

+static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
+  if (prefix != NULL && strlen(prefix) <= 32 && !_mi_is_main_thread()) {
+    char tprefix[64];
+    snprintf(tprefix, sizeof(tprefix), "%sthread 0x%zx: ", prefix, _mi_thread_id());
+    mi_vfprintf(out, arg, tprefix, fmt, args);
+  }
+  else {
+    mi_vfprintf(out, arg, prefix, fmt, args);
+  }
+}
+
 void _mi_trace_message(const char* fmt, ...) {
  if (mi_option_get(mi_option_verbose) <= 1) return;  // only with verbose level 2 or higher
  va_list args;
  va_start(args, fmt);
-  mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
  va_end(args);
 }

@@ -339,17 +371,21 @@ void _mi_verbose_message(const char* fmt, ...) {
 }

 static void mi_show_error_message(const char* fmt, va_list args) {
-  if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
-  if (mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
-  mi_vfprintf(NULL, NULL, "mimalloc: error: ", fmt, args);
+  if (!mi_option_is_enabled(mi_option_verbose)) {
+    if (!mi_option_is_enabled(mi_option_show_errors)) return;
+    if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
+  }
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args);
 }

 void _mi_warning_message(const char* fmt, ...) {
-  if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
-  if (mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
+  if (!mi_option_is_enabled(mi_option_verbose)) {
+    if (!mi_option_is_enabled(mi_option_show_errors)) return;
+    if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
+  }
  va_list args;
  va_start(args,fmt);
-  mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args);
+  mi_vfprintf_thread(NULL, NULL, "mimalloc: warning: ", fmt, args);
  va_end(args);
 }

@@ -67,7 +67,8 @@ terms of the MIT license. A copy of the license can be found in the file
  On windows initializes support for aligned allocation and
  large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
 ----------------------------------------------------------- */
-bool    _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
+bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
+bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats);

 static void* mi_align_up_ptr(void* p, size_t alignment) {
  return (void*)_mi_align_up((uintptr_t)p, alignment);
@@ -96,7 +97,7 @@ bool _mi_os_has_overcommit(void) {
 }

 // OS (small) page size
-size_t _mi_os_page_size() {
+size_t _mi_os_page_size(void) {
  return os_page_size;
 }

@@ -129,26 +130,47 @@ size_t _mi_os_good_alloc_size(size_t size) {
 // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
 // So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
 // NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
-//
-// We hide MEM_EXTENDED_PARAMETER to compile with older SDK's.
+// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.
+typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {
+  MiMemExtendedParameterInvalidType = 0,
+  MiMemExtendedParameterAddressRequirements,
+  MiMemExtendedParameterNumaNode,
+  MiMemExtendedParameterPartitionHandle,
+  MiMemExtendedParameterUserPhysicalHandle,
+  MiMemExtendedParameterAttributeFlags,
+  MiMemExtendedParameterMax
+} MI_MEM_EXTENDED_PARAMETER_TYPE; 
+
+typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {
+  struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;
+  union  { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;
+} MI_MEM_EXTENDED_PARAMETER;
+
+typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {
+  PVOID  LowestStartingAddress;
+  PVOID  HighestEndingAddress;
+  SIZE_T Alignment;
+} MI_MEM_ADDRESS_REQUIREMENTS;
+
+#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE   0x00000010
+
 #include <winternl.h>
-typedef PVOID    (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
-typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG);
+typedef PVOID    (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
+typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
 static PVirtualAlloc2 pVirtualAlloc2 = NULL;
 static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;

 // Similarly, GetNumaProcesorNodeEx is only supported since Windows 7
-#if (_WIN32_WINNT < 0x601)  // before Win7
-typedef struct _PROCESSOR_NUMBER { WORD Group; BYTE Number; BYTE Reserved; } PROCESSOR_NUMBER, *PPROCESSOR_NUMBER;
-#endif
-typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(PPROCESSOR_NUMBER ProcNumber);
-typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(PPROCESSOR_NUMBER Processor, PUSHORT NodeNumber);
+typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;
+
+typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);
+typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);
 typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);
 static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;
 static PGetNumaProcessorNodeEx      pGetNumaProcessorNodeEx = NULL;
 static PGetNumaNodeProcessorMaskEx  pGetNumaNodeProcessorMaskEx = NULL;

-static bool mi_win_enable_large_os_pages()
+static bool mi_win_enable_large_os_pages(void)
 {
  if (large_os_page_size > 0) return true;

@@ -219,7 +241,7 @@ void _mi_os_init(void)
  }
 }
 #elif defined(__wasi__)
-void _mi_os_init() {
+void _mi_os_init(void) {
  os_overcommit = false;
  os_page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
  os_alloc_granularity = 16;
@@ -250,7 +272,7 @@ static void os_detect_overcommit(void) {
 #endif
 }

-void _mi_os_init() {
+void _mi_os_init(void) {
  // get the page size
  long result = sysconf(_SC_PAGESIZE);
  if (result > 0) {
@@ -275,7 +297,57 @@ static int mi_madvise(void* addr, size_t length, int advice) {


 /* -----------------------------------------------------------
-  free memory
+  aligned hinting
+-------------------------------------------------------------- */
+
+// On 64-bit systems, we can do efficient aligned allocation by using
+// the 2TiB to 30TiB area to allocate those.
+#if (MI_INTPTR_SIZE >= 8)
+static mi_decl_cache_align _Atomic(uintptr_t)aligned_base;
+
+// Return a MI_SEGMENT_SIZE aligned address that is probably available.
+// If this returns NULL, the OS will determine the address but on some OS's that may not be 
+// properly aligned which can be more costly as it needs to be adjusted afterwards.
+// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; 
+// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses 
+//  in the middle of the 2TiB - 6TiB address range (see issue #372))
+
+#define MI_HINT_BASE ((uintptr_t)2 << 40)  // 2TiB start
+#define MI_HINT_AREA ((uintptr_t)4 << 40)  // upto 6TiB   (since before win8 there is "only" 8TiB available to processes)
+#define MI_HINT_MAX  ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
+
+static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size)
+{
+  if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
+  size = _mi_align_up(size, MI_SEGMENT_SIZE);
+  if (size > 1*MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
+  #if (MI_SECURE>0)
+  size += MI_SEGMENT_SIZE;        // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
+  #endif
+
+  uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
+  if (hint == 0 || hint > MI_HINT_MAX) {   // wrap or initialize
+    uintptr_t init = MI_HINT_BASE;
+    #if (MI_SECURE>0 || MI_DEBUG==0)       // security: randomize start of aligned allocations unless in debug mode
+    uintptr_t r = _mi_heap_random_next(mi_get_default_heap());
+    init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
+    #endif
+    uintptr_t expected = hint + size;
+    mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
+    hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
+  }
+  if (hint%try_alignment != 0) return NULL;
+  return (void*)hint;
+}
+#else
+static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
+  MI_UNUSED(try_alignment); MI_UNUSED(size);
+  return NULL;
+}
+#endif
+
+/* -----------------------------------------------------------
+  Free memory
 -------------------------------------------------------------- */

 static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* stats)
@@ -283,26 +355,37 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats
  if (addr == NULL || size == 0) return true; // || _mi_os_is_huge_reserved(addr)
  bool err = false;
 #if defined(_WIN32)
+  DWORD errcode = 0;
  err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
+  if (err) { errcode = GetLastError(); }
+  if (errcode == ERROR_INVALID_ADDRESS) {
+    // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside
+    // the memory region returned by VirtualAlloc; in that case we need to free using
+    // the start of the region.
+    MEMORY_BASIC_INFORMATION info = { 0, 0 };
+    VirtualQuery(addr, &info, sizeof(info));
+    if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < MI_SEGMENT_SIZE) {
+      errcode = 0;
+      err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
+      if (err) { errcode = GetLastError(); }
+    }
+  }
+  if (errcode != 0) {
+    _mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size);
+  }
 #elif defined(MI_USE_SBRK) || defined(__wasi__)
-  err = 0; // sbrk heap cannot be shrunk
+  err = false; // sbrk heap cannot be shrunk
 #else
  err = (munmap(addr, size) == -1);
-#endif
-  if (was_committed) _mi_stat_decrease(&stats->committed, size);
-  _mi_stat_decrease(&stats->reserved, size);
  if (err) {
-    _mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size);
-    return false;
-  }
-  else {
-    return true;
+    _mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size);
  }
+#endif
+  if (was_committed) { _mi_stat_decrease(&stats->committed, size); }
+  _mi_stat_decrease(&stats->reserved, size);
+  return !err;  
 }

-#if !(defined(__wasi__) || defined(MI_USE_SBRK) || defined(MAP_ALIGNED))
-static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size);
-#endif

 /* -----------------------------------------------------------
  Raw allocation on Windows (VirtualAlloc) 
@@ -320,32 +403,23 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
    if (hint != NULL) {
      void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
      if (p != NULL) return p;
-      // for robustness always fall through in case of an error
-      /*
-      DWORD err = GetLastError();
-      if (err != ERROR_INVALID_ADDRESS &&   // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210)
-          err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230)
-        return NULL;
-      }
-      */
-      _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags);
+      _mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags);
+      // fall through on error
    }
  } 
 #endif
-#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
  // on modern Windows try use VirtualAlloc2 for aligned allocation
  if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
-    MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
+    MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
    reqs.Alignment = try_alignment;
-    MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
-    param.Type = MemExtendedParameterAddressRequirements;
-    param.Pointer = &reqs;
+    MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
+    param.Type.Type = MiMemExtendedParameterAddressRequirements;
+    param.Arg.Pointer = &reqs;
    void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, &param, 1);
    if (p != NULL) return p;
-    _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), addr, try_alignment, flags);
+    _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
    // fall through on error
  }
-#endif
  // last resort
  return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
 }
@@ -354,6 +428,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
  mi_assert_internal(!(large_only && !allow_large));
  static _Atomic(size_t) large_page_try_ok; // = 0;
  void* p = NULL;
+  // Try to allocate large OS pages (2MiB) if allowed or required.
  if ((large_only || use_large_os_page(size, try_alignment))
      && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) {
    size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
@@ -373,12 +448,13 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
      }
    }
  }
+  // Fall back to regular page allocation
  if (p == NULL) {
    *is_large = ((flags&MEM_LARGE_PAGES) != 0);
    p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
  }
  if (p == NULL) {
-    _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, GetLastError(), addr, large_only, allow_large);
+    _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large);
  }
  return p;
 }
@@ -620,53 +696,6 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
 }
 #endif

-// On 64-bit systems, we can do efficient aligned allocation by using
-// the 2TiB to 30TiB area to allocate them.
-#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || defined(MI_OS_USE_MMAP))
-static mi_decl_cache_align _Atomic(uintptr_t) aligned_base;
-
-// Return a 4MiB aligned address that is probably available.
-// If this returns NULL, the OS will determine the address but on some OS's that may not be 
-// properly aligned which can be more costly as it needs to be adjusted afterwards.
-// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; 
-// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses 
-//  in the middle of the 2TiB - 6TiB address range (see issue #372))
-
-#define MI_HINT_BASE ((uintptr_t)2 << 40)  // 2TiB start
-#define MI_HINT_AREA ((uintptr_t)4 << 40)  // upto 6TiB   (since before win8 there is "only" 8TiB available to processes)
-#define MI_HINT_MAX  ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
-
-static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) 
-{
-  if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
-  size = _mi_align_up(size, MI_SEGMENT_SIZE);
-  if (size > 1*MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
-  #if (MI_SECURE>0)
-  size += MI_SEGMENT_SIZE;        // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
-  #endif
-
-  uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
-  if (hint == 0 || hint > MI_HINT_MAX) {   // wrap or initialize
-    uintptr_t init = MI_HINT_BASE;
-    #if (MI_SECURE>0 || MI_DEBUG==0)       // security: randomize start of aligned allocations unless in debug mode
-    uintptr_t r = _mi_heap_random_next(mi_get_default_heap());
-    init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
-    #endif
-    uintptr_t expected = hint + size;
-    mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
-    hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
-  }
-  if (hint%try_alignment != 0) return NULL;
-  return (void*)hint;
-}
-#elif defined(__wasi__) || defined(MI_USE_SBRK) || defined(MAP_ALIGNED)
-// no need for mi_os_get_aligned_hint
-#else
-static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
-  MI_UNUSED(try_alignment); MI_UNUSED(size);
-  return NULL;
-}
-#endif

 /* -----------------------------------------------------------
   Primitive allocation from the OS.
@@ -692,7 +721,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo

  #if defined(_WIN32)
    int flags = MEM_RESERVE;
-    if (commit) flags |= MEM_COMMIT;
+    if (commit) { flags |= MEM_COMMIT; }
    p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
  #elif defined(MI_USE_SBRK) || defined(__wasi__)
    MI_UNUSED(allow_large);
@@ -716,6 +745,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo
 static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
  mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(is_large != NULL);
  if (!commit) allow_large = false;
  if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
  size = _mi_align_up(size, _mi_os_page_size());
@@ -723,45 +753,27 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
  // try first with a hint (this will be aligned directly on Win 10+ or BSD)
  void* p = mi_os_mem_alloc(size, alignment, commit, allow_large, is_large, stats);
  if (p == NULL) return NULL;
-
+  
  // if not aligned, free it, overallocate, and unmap around it
  if (((uintptr_t)p % alignment != 0)) {
    mi_os_mem_free(p, size, commit, stats);
+    _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (%zu bytes, address: %p, alignment: %zu, commit: %d)\n", size, p, alignment, commit);
    if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
-    size_t over_size = size + alignment;
+    const size_t over_size = size + alignment;

 #if _WIN32
-    // over-allocate and than re-allocate exactly at an aligned address in there.
-    // this may fail due to threads allocating at the same time so we
-    // retry this at most 3 times before giving up.
-    // (we can not decommit around the overallocation on Windows, because we can only
-    //  free the original pointer, not one pointing inside the area)
-    int flags = MEM_RESERVE;
-    if (commit) flags |= MEM_COMMIT;
-    for (int tries = 0; tries < 3; tries++) {
-      // over-allocate to determine a virtual memory range
-      p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
-      if (p == NULL) return NULL; // error
-      if (((uintptr_t)p % alignment) == 0) {
-        // if p happens to be aligned, just decommit the left-over area
-        _mi_os_decommit((uint8_t*)p + size, over_size - size, stats);
-        break;
-      }
-      else {
-        // otherwise free and allocate at an aligned address in there
-        mi_os_mem_free(p, over_size, commit, stats);
-        void* aligned_p = mi_align_up_ptr(p, alignment);
-        p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large);
-        if (p != NULL) {
-          _mi_stat_increase(&stats->reserved, size);
-          if (commit) { _mi_stat_increase(&stats->committed, size); }
-        }
-        if (p == aligned_p) break; // success!
-        if (p != NULL) { // should not happen?
-          mi_os_mem_free(p, size, commit, stats);
-          p = NULL;
-        }
-      }
+    // over-allocate uncommitted (virtual) memory
+    p = mi_os_mem_alloc(over_size, 0 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, stats);
+    if (p == NULL) return NULL;
+    
+    // set p to the aligned part in the full region
+    // note: this is dangerous on Windows as VirtualFree needs the actual region pointer
+    // but in mi_os_mem_free we handle this (hopefully exceptional) situation.
+    p = mi_align_up_ptr(p, alignment);
+
+    // explicitly commit only the aligned part
+    if (commit) {
+      _mi_os_commit(p, size, NULL, stats);
    }
 #else
    // overallocate...
@@ -784,6 +796,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
  return p;
 }

+
 /* -----------------------------------------------------------
  OS API: alloc, free, alloc_aligned
 ----------------------------------------------------------- */
@@ -811,6 +824,7 @@ void  _mi_os_free(void* p, size_t size, mi_stats_t* stats) {

 void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats)
 {
+  MI_UNUSED(&mi_os_get_aligned_hint); // suppress unused warnings
  MI_UNUSED(tld_stats);
  if (size == 0) return NULL;
  size = _mi_os_good_alloc_size(size);
@@ -990,7 +1004,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
  if (p != start) return false;
 #else
 #if defined(MADV_FREE)
-  static _Atomic(size_t) advice = ATOMIC_VAR_INIT(MADV_FREE);
+  static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
  int oadvice = (int)mi_atomic_load_relaxed(&advice);
  int err;
  while ((err = mi_madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0;  };
@@ -1112,21 +1126,17 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)

  mi_win_enable_large_os_pages();

-  #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
-  MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
+  MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
  static bool mi_huge_pages_available = true;
  if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
-    #ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
-    #define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE  (0x10)
-    #endif
-    params[0].Type = 5; // == MemExtendedParameterAttributeFlags;
-    params[0].ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
+    params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
+    params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
    ULONG param_count = 1;
    if (numa_node >= 0) {
      param_count++;
-      params[1].Type = MemExtendedParameterNumaNode;
-      params[1].ULong = (unsigned)numa_node;
+      params[1].Type.Type = MiMemExtendedParameterNumaNode;
+      params[1].Arg.ULong = (unsigned)numa_node;
    }
    SIZE_T psize = size;
    void* base = addr;
@@ -1142,13 +1152,11 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
  }
  // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation
  if (pVirtualAlloc2 != NULL && numa_node >= 0) {
-    params[0].Type = MemExtendedParameterNumaNode;
-    params[0].ULong = (unsigned)numa_node;
+    params[0].Type.Type = MiMemExtendedParameterNumaNode;
+    params[0].Arg.ULong = (unsigned)numa_node;
    return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
  }
-  #else
-    MI_UNUSED(numa_node);
-  #endif
+  
  // otherwise use regular virtual alloc on older windows
  return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
 }
@@ -1298,11 +1306,11 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
 Support NUMA aware allocation
 -----------------------------------------------------------------------------*/
 #ifdef _WIN32  
-static size_t mi_os_numa_nodex() {
+static size_t mi_os_numa_nodex(void) {
  USHORT numa_node = 0;
  if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
    // Extended API is supported
-    PROCESSOR_NUMBER pnum;
+    MI_PROCESSOR_NUMBER pnum;
    (*pGetCurrentProcessorNumberEx)(&pnum);
    USHORT nnode = 0;
    BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);
@@ -252,7 +252,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
  }
  mi_assert_internal(pq==NULL || _mi_page_segment(page)->kind != MI_SEGMENT_HUGE);
  mi_page_init(heap, page, block_size, heap->tld);
-  _mi_stat_increase(&heap->tld->stats.pages, 1);
+  mi_heap_stat_increase(heap, pages, 1);
  if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL
  mi_assert_expensive(_mi_page_is_valid(page));
  return page;
@@ -368,17 +368,6 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
  mi_page_set_has_aligned(page, false);

  mi_heap_t* heap = mi_page_heap(page);
-  const size_t bsize = mi_page_block_size(page);
-  if (bsize > MI_MEDIUM_OBJ_SIZE_MAX) {
-    if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {      
-      _mi_stat_decrease(&heap->tld->stats.large, bsize);
-    }
-    else {
-      // not strictly necessary as we never get here for a huge page
-      mi_assert_internal(false);
-      _mi_stat_decrease(&heap->tld->stats.huge, bsize);      
-    }
-  }

  // remove from the page list
  // (no need to do _mi_heap_delayed_free first as all blocks are already free)
@@ -705,7 +694,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
    page = next;
  } // for each page

-  mi_stat_counter_increase(heap->tld->stats.searches, count);
+  mi_heap_stat_counter_increase(heap, searches, count);

  if (page == NULL) {
    _mi_heap_collect_retired(heap, false); // perhaps make a page available?
@@ -791,10 +780,8 @@ static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size) {
  mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap, block_size));
  mi_page_t* page = mi_page_fresh_alloc(heap, pq, block_size);
  if (page != NULL) {
-    const size_t bsize = mi_page_block_size(page);  // note: not `mi_page_usable_block_size` as `size` includes padding
    mi_assert_internal(mi_page_immediate_available(page));
-    mi_assert_internal(bsize >= size);
-
+    
    if (pq == NULL) {
      // huge pages are directly abandoned
      mi_assert_internal(_mi_page_segment(page)->kind == MI_SEGMENT_HUGE);
@@ -805,13 +792,15 @@ static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size) {
    else {
      mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE);
    }
+    
+    const size_t bsize = mi_page_usable_block_size(page);  // note: not `mi_page_block_size` to account for padding
    if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
-      _mi_stat_increase(&heap->tld->stats.large, bsize);
-      _mi_stat_counter_increase(&heap->tld->stats.large_count, 1);
+      mi_heap_stat_increase(heap, large, bsize);
+      mi_heap_stat_counter_increase(heap, large_count, 1);
    }
    else {
-      _mi_stat_increase(&heap->tld->stats.huge, bsize);
-      _mi_stat_counter_increase(&heap->tld->stats.huge_count, 1);
+      mi_heap_stat_increase(heap, huge, bsize);
+      mi_heap_stat_counter_increase(heap, huge_count, 1);
    }
  }
  return page;
@@ -168,16 +168,10 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim

 #if defined(_WIN32)

-#if !defined(MI_USE_RTLGENRANDOM)
-// We prefer to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock 
-// under the VS debugger when using dynamic overriding.
-#pragma comment (lib,"bcrypt.lib")
-#include <bcrypt.h>
-static bool os_random_buf(void* buf, size_t buf_len) {
-  return (BCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
-}
-#else
-// Use (unofficial) RtlGenRandom
+#if defined(MI_USE_RTLGENRANDOM) || defined(__cplusplus)
+// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using 
+// dynamic overriding, we observed it can raise an exception when compiled with C++, and 
+// sometimes deadlocks when also running under the VS debugger.
 #pragma comment (lib,"advapi32.lib")
 #define RtlGenRandom  SystemFunction036
 #ifdef __cplusplus
@@ -190,11 +184,18 @@ BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);
 static bool os_random_buf(void* buf, size_t buf_len) {
  return (RtlGenRandom(buf, (ULONG)buf_len) != 0);
 }
+#else
+#pragma comment (lib,"bcrypt.lib")
+#include <bcrypt.h>
+static bool os_random_buf(void* buf, size_t buf_len) {
+  return (BCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
+}
 #endif

 #elif defined(__APPLE__)
 #include <AvailabilityMacros.h>
 #if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10
+#include <CommonCrypto/CommonCryptoError.h>
 #include <CommonCrypto/CommonRandom.h>
 #endif
 static bool os_random_buf(void* buf, size_t buf_len) {
@@ -122,7 +122,7 @@ static size_t mi_good_commit_size(size_t size) {
 */

 // Return if a pointer points into a region reserved by us.
-bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
+mi_decl_nodiscard bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
  if (p==NULL) return false;
  size_t count = mi_atomic_load_relaxed(&regions_count);
  for (size_t i = 0; i < count; i++) {
@@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MI_CACHE_FIELDS     (16)
 #define MI_CACHE_MAX        (MI_BITMAP_FIELD_BITS*MI_CACHE_FIELDS)       // 1024 on 64-bit

-#define BITS_SET()          ATOMIC_VAR_INIT(UINTPTR_MAX)
+#define BITS_SET()          MI_ATOMIC_VAR_INIT(UINTPTR_MAX)
 #define MI_CACHE_BITS_SET   MI_INIT16(BITS_SET)                          // note: update if MI_CACHE_FIELDS changes

 typedef struct mi_cache_slot_s {
@@ -394,69 +394,13 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
  }
 }

-
-// The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use,
-#define MI_SEGMENT_CACHE_FRACTION (8)
-
-// note: returned segment may be partially reset
-static mi_segment_t* mi_segment_cache_pop(size_t segment_slices, mi_segments_tld_t* tld) {
-  if (segment_slices != 0 && segment_slices != MI_SLICES_PER_SEGMENT) return NULL;
-  mi_segment_t* segment = tld->cache;
-  if (segment == NULL) return NULL;
-  tld->cache_count--;
-  tld->cache = segment->next;
-  segment->next = NULL;
-  mi_assert_internal(segment->segment_slices == MI_SLICES_PER_SEGMENT);
-  _mi_stat_decrease(&tld->stats->segments_cache, 1);
-  return segment;
-}
-
-static bool mi_segment_cache_full(mi_segments_tld_t* tld)
-{
-  // if (tld->count == 1 && tld->cache_count==0) return false; // always cache at least the final segment of a thread
-  size_t max_cache = mi_option_get(mi_option_segment_cache);
-  if (tld->cache_count < max_cache
-       && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION)) // at least allow a 1 element cache
-     ) {
-    return false;
-  }
-  // take the opportunity to reduce the segment cache if it is too large (now)
-  // TODO: this never happens as we check against peak usage, should we use current usage instead?
-  while (tld->cache_count > max_cache) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) {
-    mi_segment_t* segment = mi_segment_cache_pop(0,tld);
-    mi_assert_internal(segment != NULL);
-    if (segment != NULL) mi_segment_os_free(segment, tld);
-  }
-  return true;
-}
-
-static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) {
-  mi_assert_internal(segment->next == NULL);
-  if (segment->segment_slices != MI_SLICES_PER_SEGMENT || mi_segment_cache_full(tld)) {
-    return false;
-  }
-  // mi_segment_delayed_decommit(segment, true, tld->stats);  
-  mi_assert_internal(segment->segment_slices == MI_SLICES_PER_SEGMENT);
-  mi_assert_internal(segment->next == NULL);  
-  segment->next = tld->cache;
-  tld->cache = segment;
-  tld->cache_count++;
-  _mi_stat_increase(&tld->stats->segments_cache,1);
-  return true;
-}
-
-// called by threads that are terminating to free cached segments
+// called by threads that are terminating 
 void _mi_segment_thread_collect(mi_segments_tld_t* tld) {
-  mi_segment_t* segment;
-  while ((segment = mi_segment_cache_pop(0,tld)) != NULL) {
-    mi_segment_os_free(segment, tld);
-  }
-  mi_assert_internal(tld->cache_count == 0);
-  mi_assert_internal(tld->cache == NULL);  
+  MI_UNUSED(tld);
+  // nothing to do
 }


-
 /* -----------------------------------------------------------
   Span management
 ----------------------------------------------------------- */
@@ -927,7 +871,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
  size_t guard_slices = 0;
  if (MI_SECURE>0) {
    // in secure mode, we set up a protected page in between the segment info
-    // and the page data
+    // and the page data, and at the end of the segment.
    size_t os_pagesize = _mi_os_page_size();    
    mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size);
    _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize);
@@ -969,6 +913,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_segments_tld_t* tld, m


 static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) {
+  MI_UNUSED(force);
  mi_assert_internal(segment != NULL);
  mi_assert_internal(segment->next == NULL);
  mi_assert_internal(segment->used == 0);
@@ -992,13 +937,8 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t
  // stats
  _mi_stat_decrease(&tld->stats->page_committed, mi_segment_info_size(segment));

-  if (!force && mi_segment_cache_push(segment, tld)) {
-    // it is put in our cache
-  }
-  else {
-    // otherwise return it to the OS
-    mi_segment_os_free(segment,  tld);
-  }
+  // return it to the OS
+  mi_segment_os_free(segment, tld);
 }


@@ -1424,7 +1364,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice
 {
  *reclaimed = false;
  mi_segment_t* segment;
-  int max_tries = 8;     // limit the work to bound allocation times
+  long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 8, 1024);     // limit the work to bound allocation times  
  while ((max_tries-- > 0) && ((segment = mi_abandoned_pop()) != NULL)) {
    segment->abandoned_visits++;
    bool has_page = mi_segment_check_free(segment,needed_slices,block_size,tld); // try to free up pages (due to concurrent frees)
@@ -1488,15 +1428,10 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_
 {
  mi_assert_internal(block_size < MI_HUGE_BLOCK_SIZE);
  mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX);
-  // 1. try to get a segment from our cache
-  mi_segment_t* segment = mi_segment_cache_pop(MI_SEGMENT_SIZE, tld);
-  if (segment != NULL) {
-    mi_segment_init(segment, 0, tld, os_tld, NULL);
-    return segment;
-  }
-  // 2. try to reclaim an abandoned segment
+  
+  // 1. try to reclaim an abandoned segment
  bool reclaimed;
-  segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld);
+  mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld);
  if (reclaimed) {
    // reclaimed the right page right into the heap
    mi_assert_internal(segment != NULL);
@@ -1506,7 +1441,7 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_
    // reclaimed a segment with a large enough empty span in it
    return segment;
  }
-  // 3. otherwise allocate a fresh segment
+  // 2. otherwise allocate a fresh segment
  return mi_segment_alloc(0, tld, os_tld, NULL);  
 }

@@ -18,6 +18,7 @@ static void test_process_info(void);
 static void test_reserved(void);
 static void negative_stat(void);
 static void alloc_huge(void);
+static void test_heap_walk(void);

 int main() {
  mi_version();
@@ -31,7 +32,8 @@ int main() {
  // invalid_free();
  // test_reserved();
  // negative_stat();
-  alloc_huge();
+  // alloc_huge();
+  test_heap_walk();
  
  void* p1 = malloc(78);
  void* p2 = malloc(24);
@@ -51,8 +53,10 @@ int main() {
  //free(p1);
  //p2 = malloc(32);
  //mi_free(p2);
-  mi_collect(true);
-  mi_stats_print(NULL);
+  
+  //mi_collect(true);
+  //mi_stats_print(NULL);
+  
  // test_process_info();
  return 0;
 }
@@ -189,6 +193,24 @@ static void alloc_huge(void) {
  mi_free(p);
 }

+static bool test_visit(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) {
+  if (block == NULL) {
+    printf("visiting an area with blocks of size %zu (including padding)\n", area->full_block_size);
+  }
+  else {
+    printf("  block of size %zu (allocated size is %zu)\n", block_size, mi_usable_size(block));
+  }
+  return true;
+}
+
+static void test_heap_walk(void) {
+  mi_heap_t* heap = mi_heap_new();
+  //mi_heap_malloc(heap, 2097152);
+  mi_heap_malloc(heap, 2067152);
+  mi_heap_malloc(heap, 2097160);
+  mi_heap_malloc(heap, 24576);
+  mi_heap_visit_blocks(heap, true, &test_visit, NULL);
+}

 // ----------------------------
 // bin size experiments
@@ -360,4 +382,4 @@ static void mi_bins(void) {
    last_bsize = bsize;
  }
 }
-#endif
+#endif