Release Cosmopolitan v3.5.2

This fixes a regression in mmap(MAP_FIXED) on Windows caused by a recent
revision. This change also fixes ZipOS so it no longer needs a MAP_FIXED
mapping to open files from the PKZIP store. The memory mapping mutex was
implemented incorrectly earlier which meant that ftrace and strace could
cause cause crashes. This lock and other recursive mutexes are rewritten
so that it should be provable that recursive mutexes in cosmopolitan are
asynchronous signal safe.

ape/aarch64.lds

@@ -286,12 +286,7 @@ SECTIONS {
 
 ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
 ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
-ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
 ape_stack_prot = PROT_READ | PROT_WRITE;
-ape_stack_round = -ape_stack_align;
-
-ASSERT(ape_stack_align < 1048576, "stack align too big");
-ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
 
 _tls_size = _tbss_end - _tdata_start;
 _tdata_size = _tdata_end - _tdata_start;

ape/ape.S

@@ -761,7 +761,7 @@ ape_phdrs:
 	.quad	0
 	.quad	0
 	.quad	0
-	.stub	ape_stack_align,quad
+	.quad	16
 
 #if SupportsOpenbsd() || SupportsNetbsd()
 	.long	PT_NOTE

ape/ape.lds

@@ -588,11 +588,6 @@ ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
 ape_stack_paddr = ape_ram_paddr + ape_ram_filesz;
 ape_stack_filesz = 0;
 ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
-ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
-ape_stack_round = -ape_stack_align;
-
-ASSERT(ape_stack_align < 1048576, "stack align too big");
-ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
 
 ape_note_offset = ape_cod_offset + (ape_note - ape_cod_vaddr);
 ape_note_filesz = ape_note_end - ape_note;

ctl/BUILD.mk

@@ -36,6 +36,7 @@ $(CTL_A_OBJS): private					\
 			-Walloca-larger-than=4096	\
 			-ffunction-sections		\
 			-fdata-sections			\
+			-fexceptions			\
 
 CTL_LIBS = $(foreach x,$(CTL_ARTIFACTS),$($(x)))
 CTL_SRCS = $(foreach x,$(CTL_ARTIFACTS),$($(x)_SRCS))

ctl/accumulate.h

@@ -0,0 +1,28 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ACCUMULATE_H_
+#define CTL_ACCUMULATE_H_
+
+namespace ctl {
+
+template<class InputIt, class T>
+constexpr T
+accumulate(InputIt first, InputIt last, T init)
+{
+    for (; first != last; ++first)
+        init = init + *first;
+    return init;
+}
+
+template<class InputIt, class T, class BinaryOperation>
+constexpr T
+accumulate(InputIt first, InputIt last, T init, BinaryOperation op)
+{
+    for (; first != last; ++first)
+        init = op(init, *first);
+    return init;
+}
+
+} // namespace ctl
+
+#endif // CTL_ACCUMULATE_H_

ctl/add_lvalue_reference.h

@@ -0,0 +1,55 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ADD_LVALUE_REFERENCE_H_
+#define CTL_ADD_LVALUE_REFERENCE_H_
+
+namespace ctl {
+
+template<typename T>
+struct add_lvalue_reference
+{
+    typedef T& type;
+};
+
+template<typename T>
+struct add_lvalue_reference<T&>
+{
+    typedef T& type;
+};
+
+template<typename T>
+struct add_lvalue_reference<T&&>
+{
+    typedef T& type;
+};
+
+template<>
+struct add_lvalue_reference<void>
+{
+    typedef void type;
+};
+
+template<>
+struct add_lvalue_reference<const void>
+{
+    typedef const void type;
+};
+
+template<>
+struct add_lvalue_reference<volatile void>
+{
+    typedef volatile void type;
+};
+
+template<>
+struct add_lvalue_reference<const volatile void>
+{
+    typedef const volatile void type;
+};
+
+template<typename T>
+using add_lvalue_reference_t = typename add_lvalue_reference<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_ADD_LVALUE_REFERENCE_H_

ctl/add_pointer.h

@@ -0,0 +1,43 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ADD_POINTER_H_
+#define CTL_ADD_POINTER_H_
+#include "remove_reference.h"
+
+namespace ctl {
+
+// Primary template
+template<typename T>
+struct add_pointer
+{
+    typedef typename remove_reference<T>::type* type;
+};
+
+// Specialization for functions
+template<typename R, typename... Args>
+struct add_pointer<R(Args...)>
+{
+    typedef R (*type)(Args...);
+};
+
+// Specialization for function references
+template<typename R, typename... Args>
+struct add_pointer<R (&)(Args...)>
+{
+    typedef R (*type)(Args...);
+};
+
+// Specialization for rvalue references to functions
+template<typename R, typename... Args>
+struct add_pointer<R (&&)(Args...)>
+{
+    typedef R (*type)(Args...);
+};
+
+// Helper alias template (C++14 and later)
+template<typename T>
+using add_pointer_t = typename add_pointer<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_ADD_POINTER_H_

ctl/addressof.h

@@ -0,0 +1,29 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ADDRESSOF_H_
+#define CTL_ADDRESSOF_H_
+
+namespace ctl {
+
+template<typename T>
+T*
+addressof(T& arg) noexcept
+{
+    return reinterpret_cast<T*>(
+      &const_cast<char&>(reinterpret_cast<const volatile char&>(arg)));
+}
+
+template<typename R, typename... Args>
+R (*addressof(R (*&arg)(Args...)) noexcept)
+(Args...)
+{
+    return arg;
+}
+
+template<typename T>
+T*
+addressof(T&&) = delete;
+
+} // namespace ctl
+
+#endif // CTL_ADDRESSOF_H_

ctl/advance.h

@@ -0,0 +1,24 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ADVANCE_H_
+#define CTL_ADVANCE_H_
+
+namespace ctl {
+
+template<class InputIt, class Distance>
+constexpr void
+advance(InputIt& it, Distance n)
+{
+    while (n > 0) {
+        --n;
+        ++it;
+    }
+    while (n < 0) {
+        ++n;
+        --it;
+    }
+}
+
+} // namespace ctl
+
+#endif // CTL_ADVANCE_H_

ctl/all_of.h

@@ -0,0 +1,20 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ALL_OF_H_
+#define CTL_ALL_OF_H_
+
+namespace ctl {
+
+template<class InputIt, class UnaryPredicate>
+constexpr bool
+all_of(InputIt first, InputIt last, UnaryPredicate p)
+{
+    for (; first != last; ++first)
+        if (!p(*first))
+            return false;
+    return true;
+}
+
+} // namespace ctl
+
+#endif // CTL_ALL_OF_H_

ctl/allocator.h

@@ -0,0 +1,94 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ALLOCATOR_H_
+#define CTL_ALLOCATOR_H_
+#include "bad_alloc.h"
+#include "integral_constant.h"
+#include "new.h"
+#include "utility.h"
+
+namespace ctl {
+
+template<typename T>
+class allocator
+{
+  public:
+    using value_type = T;
+    using size_type = size_t;
+    using difference_type = ptrdiff_t;
+    using propagate_on_container_move_assignment = ctl::true_type;
+    using is_always_equal = ctl::true_type;
+    using pointer = T*;
+    using const_pointer = const T*;
+    using reference = T&;
+    using const_reference = const T&;
+
+    constexpr allocator() noexcept = default;
+
+    constexpr allocator(const allocator&) noexcept = default;
+
+    template<class U>
+    constexpr allocator(const allocator<U>&) noexcept
+    {
+    }
+
+    constexpr ~allocator() = default;
+
+    [[nodiscard]] T* allocate(size_type n)
+    {
+        if (n > __SIZE_MAX__ / sizeof(T))
+            throw ctl::bad_alloc();
+        if (auto p = static_cast<T*>(::operator new(
+              n * sizeof(T), ctl::align_val_t(alignof(T)), ctl::nothrow)))
+            return p;
+        throw ctl::bad_alloc();
+    }
+
+    void deallocate(T* p, size_type n) noexcept
+    {
+        ::operator delete(p, n * sizeof(T), ctl::align_val_t(alignof(T)));
+    }
+
+    template<typename U, typename... Args>
+    void construct(U* p, Args&&... args)
+    {
+        ::new (static_cast<void*>(p)) U(ctl::forward<Args>(args)...);
+    }
+
+    template<typename U>
+    void destroy(U* p)
+    {
+        p->~U();
+    }
+
+    size_type max_size() const noexcept
+    {
+        return __SIZE_MAX__ / sizeof(T);
+    }
+
+    allocator& operator=(const allocator&) = default;
+
+    template<typename U>
+    struct rebind
+    {
+        using other = allocator<U>;
+    };
+};
+
+template<class T, class U>
+bool
+operator==(const allocator<T>&, const allocator<U>&) noexcept
+{
+    return true;
+}
+
+template<class T, class U>
+bool
+operator!=(const allocator<T>&, const allocator<U>&) noexcept
+{
+    return false;
+}
+
+} // namespace ctl
+
+#endif // CTL_ALLOCATOR_H_

ctl/allocator_traits.h

@@ -0,0 +1,74 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ALLOCATOR_TRAITS_H_
+#define CTL_ALLOCATOR_TRAITS_H_
+#include "integral_constant.h"
+
+namespace ctl {
+
+template<typename Alloc>
+struct allocator_traits
+{
+    using allocator_type = Alloc;
+    using value_type = typename Alloc::value_type;
+    using pointer = typename Alloc::pointer;
+    using const_pointer = typename Alloc::const_pointer;
+    using void_pointer = void*;
+    using const_void_pointer = const void*;
+    using difference_type = typename Alloc::difference_type;
+    using size_type = typename Alloc::size_type;
+
+    using propagate_on_container_copy_assignment = false_type;
+    using propagate_on_container_move_assignment = true_type;
+    using propagate_on_container_swap = false_type;
+    using is_always_equal = true_type;
+
+    template<typename T>
+    using rebind_alloc = typename Alloc::template rebind<T>::other;
+
+    template<typename T>
+    using rebind_traits = allocator_traits<rebind_alloc<T>>;
+
+    __attribute__((__always_inline__)) static pointer allocate(Alloc& a,
+                                                               size_type n)
+    {
+        return a.allocate(n);
+    }
+
+    __attribute__((__always_inline__)) static void deallocate(Alloc& a,
+                                                              pointer p,
+                                                              size_type n)
+    {
+        a.deallocate(p, n);
+    }
+
+    template<typename T, typename... Args>
+    __attribute__((__always_inline__)) static void construct(Alloc& a,
+                                                             T* p,
+                                                             Args&&... args)
+    {
+        ::new ((void*)p) T(static_cast<Args&&>(args)...);
+    }
+
+    template<typename T>
+    __attribute__((__always_inline__)) static void destroy(Alloc& a, T* p)
+    {
+        p->~T();
+    }
+
+    __attribute__((__always_inline__)) static size_type max_size(
+      const Alloc& a) noexcept
+    {
+        return __PTRDIFF_MAX__ / sizeof(value_type);
+    }
+
+    __attribute__((__always_inline__)) static Alloc
+    select_on_container_copy_construction(const Alloc& a)
+    {
+        return a;
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_ALLOCATOR_TRAITS_H_

ctl/any_of.h

@@ -0,0 +1,20 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ANY_OF_H_
+#define CTL_ANY_OF_H_
+
+namespace ctl {
+
+template<class InputIt, class UnaryPredicate>
+constexpr bool
+any_of(InputIt first, InputIt last, UnaryPredicate p)
+{
+    for (; first != last; ++first)
+        if (p(*first))
+            return true;
+    return false;
+}
+
+} // namespace ctl
+
+#endif // CTL_ANY_OF_H_

ctl/array.h

@@ -0,0 +1,244 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ARRAY_H_
+#define CTL_ARRAY_H_
+#include "initializer_list.h"
+#include "out_of_range.h"
+#include "reverse_iterator.h"
+
+namespace ctl {
+
+template<typename T, size_t N>
+struct array
+{
+    using value_type = T;
+    using size_type = size_t;
+    using difference_type = ptrdiff_t;
+    using reference = value_type&;
+    using const_reference = const value_type&;
+    using pointer = value_type*;
+    using const_pointer = const value_type*;
+    using iterator = value_type*;
+    using const_iterator = const value_type*;
+    using reverse_iterator = ctl::reverse_iterator<iterator>;
+    using const_reverse_iterator = ctl::reverse_iterator<const_iterator>;
+
+    T elems[N];
+
+    constexpr array() = default;
+
+    constexpr array(std::initializer_list<T> init)
+    {
+        auto it = init.begin();
+        for (size_t i = 0; i < N && it != init.end(); ++i, ++it)
+            elems[i] = *it;
+    }
+
+    constexpr reference at(size_type pos)
+    {
+        if (pos >= N)
+            throw ctl::out_of_range("out of range");
+        return elems[pos];
+    }
+
+    constexpr const_reference at(size_type pos) const
+    {
+        if (pos >= N)
+            throw ctl::out_of_range("out of range");
+        return elems[pos];
+    }
+
+    constexpr reference operator[](size_type pos)
+    {
+        return elems[pos];
+    }
+
+    constexpr const_reference operator[](size_type pos) const
+    {
+        return elems[pos];
+    }
+
+    constexpr reference front()
+    {
+        return elems[0];
+    }
+
+    constexpr const_reference front() const
+    {
+        return elems[0];
+    }
+
+    constexpr reference back()
+    {
+        return elems[N - 1];
+    }
+
+    constexpr const_reference back() const
+    {
+        return elems[N - 1];
+    }
+
+    constexpr T* data() noexcept
+    {
+        return elems;
+    }
+
+    constexpr const T* data() const noexcept
+    {
+        return elems;
+    }
+
+    constexpr iterator begin() noexcept
+    {
+        return elems;
+    }
+
+    constexpr const_iterator begin() const noexcept
+    {
+        return elems;
+    }
+
+    constexpr iterator end() noexcept
+    {
+        return elems + N;
+    }
+
+    constexpr const_iterator end() const noexcept
+    {
+        return elems + N;
+    }
+
+    constexpr reverse_iterator rbegin() noexcept
+    {
+        return reverse_iterator(end());
+    }
+
+    constexpr const_reverse_iterator rbegin() const noexcept
+    {
+        return const_reverse_iterator(end());
+    }
+
+    constexpr reverse_iterator rend() noexcept
+    {
+        return reverse_iterator(begin());
+    }
+
+    constexpr const_reverse_iterator rend() const noexcept
+    {
+        return const_reverse_iterator(begin());
+    }
+
+    constexpr const_iterator cbegin() const noexcept
+    {
+        return begin();
+    }
+
+    constexpr const_iterator cend() const noexcept
+    {
+        return end();
+    }
+
+    constexpr const_reverse_iterator crbegin() const noexcept
+    {
+        return rbegin();
+    }
+
+    constexpr const_reverse_iterator crend() const noexcept
+    {
+        return rend();
+    }
+
+    constexpr bool empty() const noexcept
+    {
+        return N == 0;
+    }
+
+    constexpr size_type size() const noexcept
+    {
+        return N;
+    }
+
+    constexpr size_type max_size() const noexcept
+    {
+        return N;
+    }
+
+    void fill(const T& value)
+    {
+        for (size_type i = 0; i < N; ++i) {
+            elems[i] = value;
+        }
+    }
+
+    void swap(array& other) noexcept
+    {
+        for (size_type i = 0; i < N; ++i) {
+            T temp = elems[i];
+            elems[i] = other.elems[i];
+            other.elems[i] = temp;
+        }
+    }
+};
+
+template<typename T, size_t N>
+bool
+operator==(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    for (size_t i = 0; i < N; ++i) {
+        if (!(lhs[i] == rhs[i]))
+            return false;
+    }
+    return true;
+}
+
+template<typename T, size_t N>
+bool
+operator!=(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    return !(lhs == rhs);
+}
+
+template<typename T, size_t N>
+bool
+operator<(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    for (size_t i = 0; i < N; ++i) {
+        if (lhs[i] < rhs[i])
+            return true;
+        if (rhs[i] < lhs[i])
+            return false;
+    }
+    return false;
+}
+
+template<typename T, size_t N>
+bool
+operator<=(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    return !(rhs < lhs);
+}
+
+template<typename T, size_t N>
+bool
+operator>(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    return rhs < lhs;
+}
+
+template<typename T, size_t N>
+bool
+operator>=(const array<T, N>& lhs, const array<T, N>& rhs)
+{
+    return !(lhs < rhs);
+}
+
+template<typename T, size_t N>
+void
+swap(array<T, N>& lhs, array<T, N>& rhs) noexcept
+{
+    lhs.swap(rhs);
+}
+
+} // namespace ctl
+
+#endif // CTL_ARRAY_H_

ctl/back_inserter.h

@@ -0,0 +1,64 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_BACK_INSERTER_H_
+#define CTL_BACK_INSERTER_H_
+#include "iterator.h"
+#include "utility.h"
+
+namespace ctl {
+
+template<class Container>
+class back_insert_iterator
+{
+  protected:
+    Container* container;
+
+  public:
+    using iterator_category = ctl::output_iterator_tag;
+    using value_type = void;
+    using difference_type = void;
+    using pointer = void;
+    using reference = void;
+
+    explicit back_insert_iterator(Container& c) : container(&c)
+    {
+    }
+
+    back_insert_iterator& operator=(const typename Container::value_type& value)
+    {
+        container->push_back(value);
+        return *this;
+    }
+
+    back_insert_iterator& operator=(typename Container::value_type&& value)
+    {
+        container->push_back(ctl::move(value));
+        return *this;
+    }
+
+    back_insert_iterator& operator*()
+    {
+        return *this;
+    }
+
+    back_insert_iterator& operator++()
+    {
+        return *this;
+    }
+
+    back_insert_iterator operator++(int)
+    {
+        return *this;
+    }
+};
+
+template<class Container>
+back_insert_iterator<Container>
+back_inserter(Container& c)
+{
+    return back_insert_iterator<Container>(c);
+}
+
+} // namespace ctl
+
+#endif // CTL_BACK_INSERTER_H_

ctl/bad_alloc.h

@@ -0,0 +1,23 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_BAD_ALLOC_H_
+#define CTL_BAD_ALLOC_H_
+#include "exception.h"
+
+namespace ctl {
+
+class bad_alloc : public exception
+{
+  public:
+    bad_alloc() noexcept = default;
+    ~bad_alloc() override = default;
+
+    const char* what() const noexcept override
+    {
+        return "ctl::bad_alloc";
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_BAD_ALLOC_H_

ctl/conditional.h

@@ -0,0 +1,22 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_CONDITIONAL_H_
+#define CTL_CONDITIONAL_H_
+
+namespace ctl {
+
+template<bool B, class T, class F>
+struct conditional
+{
+    typedef T type;
+};
+
+template<class T, class F>
+struct conditional<false, T, F>
+{
+    typedef F type;
+};
+
+} // namespace ctl
+
+#endif // CTL_CONDITIONAL_H_

ctl/copy.h

@@ -0,0 +1,22 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_COPY_H_
+#define CTL_COPY_H_
+
+namespace ctl {
+
+template<class InputIt, class OutputIt>
+OutputIt
+copy(InputIt first, InputIt last, OutputIt d_first)
+{
+    while (first != last) {
+        *d_first = *first;
+        ++d_first;
+        ++first;
+    }
+    return d_first;
+}
+
+} // namespace ctl
+
+#endif // CTL_COPY_H_

ctl/decay.h

@@ -0,0 +1,35 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_DECAY_H_
+#define CTL_DECAY_H_
+#include "add_pointer.h"
+#include "conditional.h"
+#include "is_array.h"
+#include "is_function.h"
+#include "remove_cv.h"
+#include "remove_extent.h"
+#include "remove_reference.h"
+
+namespace ctl {
+
+template<typename T>
+struct decay
+{
+  private:
+    typedef typename remove_reference<T>::type U;
+
+  public:
+    typedef typename conditional<
+      is_array<U>::value,
+      typename remove_extent<U>::type*,
+      typename conditional<is_function<U>::value,
+                           typename add_pointer<U>::type,
+                           typename remove_cv<U>::type>::type>::type type;
+};
+
+template<typename T>
+using decay_t = typename decay<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_DECAY_H_

ctl/default_delete.h

@@ -0,0 +1,44 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_DEFAULT_DELETE_H_
+#define CTL_DEFAULT_DELETE_H_
+#include "enable_if.h"
+#include "is_convertible.h"
+
+namespace ctl {
+
+template<typename T>
+struct default_delete
+{
+    constexpr default_delete() noexcept = default;
+    template<typename U,
+             typename = typename ctl::enable_if<
+               ctl::is_convertible<U*, T*>::value>::type>
+    constexpr default_delete(const default_delete<U>&) noexcept
+    {
+    }
+    constexpr void operator()(T* const p) const noexcept
+    {
+        delete p;
+    }
+};
+
+template<typename T>
+struct default_delete<T[]>
+{
+    constexpr default_delete() noexcept = default;
+    template<typename U,
+             typename = typename ctl::enable_if<
+               ctl::is_convertible<U (*)[], T (*)[]>::value>::type>
+    constexpr default_delete(const default_delete<U[]>&) noexcept
+    {
+    }
+    constexpr void operator()(T* const p) const noexcept
+    {
+        delete[] p;
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_DEFAULT_DELETE_H_

ctl/distance.h

@@ -0,0 +1,37 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_DISTANCE_H_
+#define CTL_DISTANCE_H_
+#include "iterator.h"
+#include "iterator_traits.h"
+
+namespace ctl {
+
+template<class InputIter>
+constexpr typename iterator_traits<InputIter>::difference_type
+distance_impl(InputIter first, InputIter last, input_iterator_tag)
+{
+    typename iterator_traits<InputIter>::difference_type res(0);
+    for (; first != last; ++first)
+        ++res;
+    return res;
+}
+
+template<class RandIter>
+constexpr typename iterator_traits<RandIter>::difference_type
+distance_impl(RandIter first, RandIter last, random_access_iterator_tag)
+{
+    return last - first;
+}
+
+template<class InputIter>
+constexpr typename iterator_traits<InputIter>::difference_type
+distance(InputIter first, InputIter last)
+{
+    return distance_impl(
+      first, last, typename iterator_traits<InputIter>::iterator_category());
+}
+
+} // namespace ctl
+
+#endif // CTL_DISTANCE_H_

ctl/enable_if.h

@@ -0,0 +1,23 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ENABLE_IF_H_
+#define CTL_ENABLE_IF_H_
+
+namespace ctl {
+
+template<bool B, class T = void>
+struct enable_if
+{};
+
+template<class T>
+struct enable_if<true, T>
+{
+    typedef T type;
+};
+
+template<bool B, class T = void>
+using enable_if_t = typename enable_if<B, T>::type;
+
+} // namespace ctl
+
+#endif // CTL_ENABLE_IF_H_

ctl/exception.h

@@ -0,0 +1,22 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_EXCEPTION_H_
+#define CTL_EXCEPTION_H_
+
+namespace ctl {
+
+class exception
+{
+  public:
+    exception() noexcept = default;
+    virtual ~exception() = default;
+
+    virtual const char* what() const noexcept
+    {
+        return "ctl::exception";
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_EXCEPTION_H_

ctl/fill.h

@@ -0,0 +1,18 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_FILL_H_
+#define CTL_FILL_H_
+
+namespace ctl {
+
+template<typename ForwardIt, typename T>
+void
+fill(ForwardIt first, ForwardIt last, const T& value)
+{
+    for (; first != last; ++first)
+        *first = value;
+}
+
+} // namespace ctl
+
+#endif // CTL_FILL_H_

ctl/fill_n.h

@@ -0,0 +1,19 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_FILL_N_H_
+#define CTL_FILL_N_H_
+
+namespace ctl {
+
+template<typename OutputIt, typename Size, typename T>
+OutputIt
+fill_n(OutputIt first, Size count, const T& value)
+{
+    for (; count > 0; --count, ++first)
+        *first = value;
+    return first;
+}
+
+} // namespace ctl
+
+#endif // CTL_FILL_N_H_

ctl/integral_constant.h

@@ -0,0 +1,29 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_INTEGRAL_CONSTANT_H_
+#define CTL_INTEGRAL_CONSTANT_H_
+
+namespace ctl {
+
+template<class T, T v>
+struct integral_constant
+{
+    static constexpr T value = v;
+    typedef T value_type;
+    typedef integral_constant type;
+    constexpr operator value_type() const noexcept
+    {
+        return value;
+    }
+    constexpr value_type operator()() const noexcept
+    {
+        return value;
+    }
+};
+
+typedef integral_constant<bool, true> true_type;
+typedef integral_constant<bool, false> false_type;
+
+} // namespace ctl
+
+#endif // CTL_INTEGRAL_CONSTANT_H_

ctl/is_array.h

@@ -0,0 +1,26 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_ARRAY_H_
+#define CTL_IS_ARRAY_H_
+#include "integral_constant.h"
+
+namespace ctl {
+
+template<typename T>
+struct is_array : false_type
+{};
+
+template<typename T, size_t N>
+struct is_array<T[N]> : true_type
+{};
+
+template<typename T>
+struct is_array<T[]> : true_type
+{};
+
+template<typename T>
+inline constexpr bool is_array_v = is_array<T>::value;
+
+} // namespace ctl
+
+#endif // CTL_IS_ARRAY_H_

ctl/is_convertible.h

@@ -0,0 +1,54 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_CONVERTIBLE_H_
+#define CTL_IS_CONVERTIBLE_H_
+
+#include "ctl/integral_constant.h"
+#include "ctl/void_t.h"
+
+namespace ctl {
+
+// Declaration of declval
+template<typename T>
+T&&
+declval() noexcept;
+
+namespace detail {
+
+template<typename From, typename To, typename = void>
+struct is_convertible_impl : false_type
+{};
+
+template<typename From, typename To>
+struct is_convertible_impl<From,
+                           To,
+                           void_t<decltype(static_cast<To>(declval<From>()))>>
+  : true_type
+{};
+
+// Handle void types separately
+template<>
+struct is_convertible_impl<void, void> : true_type
+{};
+
+template<typename To>
+struct is_convertible_impl<void, To> : false_type
+{};
+
+template<typename From>
+struct is_convertible_impl<From, void> : false_type
+{};
+
+} // namespace detail
+
+template<typename From, typename To>
+struct is_convertible : detail::is_convertible_impl<From, To>
+{};
+
+// Helper variable template (C++17 and later)
+template<typename From, typename To>
+inline constexpr bool is_convertible_v = is_convertible<From, To>::value;
+
+} // namespace ctl
+
+#endif // CTL_IS_CONVERTIBLE_H_

ctl/is_function.h

@@ -0,0 +1,120 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_FUNCTION_H_
+#define CTL_IS_FUNCTION_H_
+#include "integral_constant.h"
+
+namespace ctl {
+
+// Primary template
+template<class>
+struct is_function : false_type
+{};
+
+// Specializations for various function types
+
+// Regular functions
+template<class Ret, class... Args>
+struct is_function<Ret(Args...)> : true_type
+{};
+
+// Functions with cv-qualifiers
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) volatile> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const volatile> : true_type
+{};
+
+// Functions with ref-qualifiers
+template<class Ret, class... Args>
+struct is_function<Ret(Args...)&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) volatile&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const volatile&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) &&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const&&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) volatile&&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args...) const volatile&&> : true_type
+{};
+
+// Variadic functions
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...)> : true_type
+{};
+
+// Variadic functions with cv-qualifiers
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) volatile> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const volatile> : true_type
+{};
+
+// Variadic functions with ref-qualifiers
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...)&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) volatile&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const volatile&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) &&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const&&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) volatile&&> : true_type
+{};
+
+template<class Ret, class... Args>
+struct is_function<Ret(Args..., ...) const volatile&&> : true_type
+{};
+
+} // namespace ctl
+
+#endif // CTL_IS_FUNCTION_H_

ctl/is_integral.h

@@ -0,0 +1,78 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_INTEGRAL_H_
+#define CTL_IS_INTEGRAL_H_
+#include "integral_constant.h"
+
+namespace ctl {
+
+template<typename T>
+struct is_integral : false_type
+{};
+
+template<>
+struct is_integral<bool> : true_type
+{};
+
+template<>
+struct is_integral<char> : true_type
+{};
+
+template<>
+struct is_integral<signed char> : true_type
+{};
+
+template<>
+struct is_integral<unsigned char> : true_type
+{};
+
+template<>
+struct is_integral<short> : true_type
+{};
+
+template<>
+struct is_integral<unsigned short> : true_type
+{};
+
+template<>
+struct is_integral<int> : true_type
+{};
+
+template<>
+struct is_integral<unsigned int> : true_type
+{};
+
+template<>
+struct is_integral<long> : true_type
+{};
+
+template<>
+struct is_integral<unsigned long> : true_type
+{};
+
+template<>
+struct is_integral<long long> : true_type
+{};
+
+template<>
+struct is_integral<unsigned long long> : true_type
+{};
+
+template<>
+struct is_integral<char16_t> : true_type
+{};
+
+template<>
+struct is_integral<char32_t> : true_type
+{};
+
+template<>
+struct is_integral<wchar_t> : true_type
+{};
+
+template<typename T>
+inline constexpr bool is_integral_v = is_integral<T>::value;
+
+} // namespace ctl
+
+#endif // CTL_IS_INTEGRAL_H_

ctl/is_reference.h

@@ -0,0 +1,35 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_REFERENCE_H_
+#define CTL_IS_REFERENCE_H_
+
+namespace ctl {
+
+// Primary template
+template<class T>
+struct is_reference
+{
+    static constexpr bool value = false;
+};
+
+// Specialization for lvalue reference
+template<class T>
+struct is_reference<T&>
+{
+    static constexpr bool value = true;
+};
+
+// Specialization for rvalue reference
+template<class T>
+struct is_reference<T&&>
+{
+    static constexpr bool value = true;
+};
+
+// Helper variable template (C++14 and later)
+template<class T>
+inline constexpr bool is_reference_v = is_reference<T>::value;
+
+} // namespace ctl
+
+#endif // CTL_IS_REFERENCE_H_

ctl/is_same.h

@@ -0,0 +1,25 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_SAME_H_
+#define CTL_IS_SAME_H_
+
+namespace ctl {
+
+template<typename T, typename U>
+struct is_same
+{
+    static constexpr bool value = false;
+};
+
+template<typename T>
+struct is_same<T, T>
+{
+    static constexpr bool value = true;
+};
+
+template<class T, class U>
+inline constexpr bool is_same_v = is_same<T, U>::value;
+
+} // namespace ctl
+
+#endif // CTL_IS_SAME_H_

ctl/is_signed.h

@@ -0,0 +1,16 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_SIGNED_H_
+#define CTL_IS_SIGNED_H_
+
+namespace ctl {
+
+template<typename T>
+struct is_signed
+{
+    static constexpr bool value = T(0) > T(-1);
+};
+
+} // namespace ctl
+
+#endif // CTL_IS_SIGNED_H_

ctl/is_sorted.h

@@ -0,0 +1,25 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_SORTED_H_
+#define CTL_IS_SORTED_H_
+
+namespace ctl {
+
+template<typename It, typename Compare>
+bool
+is_sorted(It first, It last, Compare comp)
+{
+    if (first == last)
+        return true;
+    It next = first;
+    while (++next != last) {
+        if (comp(*next, *first))
+            return false;
+        first = next;
+    }
+    return true;
+}
+
+} // namespace ctl
+
+#endif // CTL_IS_SORTED_H_

ctl/is_unsigned.h

@@ -0,0 +1,16 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_IS_UNSIGNED_H_
+#define CTL_IS_UNSIGNED_H_
+
+namespace ctl {
+
+template<typename T>
+struct is_unsigned
+{
+    static constexpr bool value = T(0) < T(-1);
+};
+
+} // namespace ctl
+
+#endif // CTL_IS_UNSIGNED_H_

ctl/iterator.h

@@ -0,0 +1,28 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_OUTPUT_ITERATOR_TAG_H_
+#define CTL_OUTPUT_ITERATOR_TAG_H_
+
+namespace ctl {
+
+struct input_iterator_tag
+{};
+
+struct output_iterator_tag
+{};
+
+struct forward_iterator_tag : public input_iterator_tag
+{};
+
+struct bidirectional_iterator_tag : public forward_iterator_tag
+{};
+
+struct random_access_iterator_tag : public bidirectional_iterator_tag
+{};
+
+struct contiguous_iterator_tag : public random_access_iterator_tag
+{};
+
+} // namespace ctl
+
+#endif // CTL_OUTPUT_ITERATOR_TAG_H_

ctl/iterator_traits.h

@@ -0,0 +1,31 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_ITERATOR_TRAITS_H_
+#define CTL_ITERATOR_TRAITS_H_
+#include "utility.h"
+
+namespace ctl {
+
+template<class Iterator>
+struct iterator_traits
+{
+    using iterator_category = typename Iterator::iterator_category;
+    using value_type = typename Iterator::value_type;
+    using difference_type = typename Iterator::difference_type;
+    using pointer = typename Iterator::pointer;
+    using reference = typename Iterator::reference;
+};
+
+template<class T>
+struct iterator_traits<T*>
+{
+    using iterator_category = void*; // We don't actually use this
+    using value_type = T;
+    using difference_type = ptrdiff_t;
+    using pointer = T*;
+    using reference = T&;
+};
+
+} // namespace ctl
+
+#endif // CTL_ITERATOR_TRAITS_H_

ctl/lexicographical_compare.h

@@ -0,0 +1,43 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_LEXICOGRAPHICAL_COMPARE_H_
+#define CTL_LEXICOGRAPHICAL_COMPARE_H_
+
+namespace ctl {
+
+template<class InputIt1, class InputIt2>
+bool
+lexicographical_compare(InputIt1 first1,
+                        InputIt1 last1,
+                        InputIt2 first2,
+                        InputIt2 last2)
+{
+    for (; (first1 != last1) && (first2 != last2); ++first1, ++first2) {
+        if (*first1 < *first2)
+            return true;
+        if (*first2 < *first1)
+            return false;
+    }
+    return (first1 == last1) && (first2 != last2);
+}
+
+template<class InputIt1, class InputIt2, class Compare>
+bool
+lexicographical_compare(InputIt1 first1,
+                        InputIt1 last1,
+                        InputIt2 first2,
+                        InputIt2 last2,
+                        Compare comp)
+{
+    for (; (first1 != last1) && (first2 != last2); ++first1, ++first2) {
+        if (comp(*first1, *first2))
+            return true;
+        if (comp(*first2, *first1))
+            return false;
+    }
+    return (first1 == last1) && (first2 != last2);
+}
+
+} // namespace ctl
+
+#endif /* CTL_LEXICOGRAPHICAL_COMPARE_H_ */

ctl/map.h

@@ -2,6 +2,7 @@
 // vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
 #ifndef CTL_MAP_H_
 #define CTL_MAP_H_
+#include "out_of_range.h"
 #include "set.h"
 
 namespace ctl {
@@ -164,7 +165,7 @@ class map
     {
         auto it = find(key);
         if (it == end())
-            __builtin_trap();
+            throw ctl::out_of_range("out of range");
         return it->second;
     }
 
@@ -172,7 +173,7 @@ class map
     {
         auto it = find(key);
         if (it == end())
-            __builtin_trap();
+            throw ctl::out_of_range("out of range");
         return it->second;
     }
 

ctl/max.h

@@ -0,0 +1,17 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_MAX_H_
+#define CTL_MAX_H_
+
+namespace ctl {
+
+template<class T>
+inline constexpr const T&
+max(const T& a, const T& b)
+{
+    return a < b ? b : a;
+}
+
+} // namespace ctl
+
+#endif /* CTL_MAX_H_ */

ctl/min.h

@@ -0,0 +1,17 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_MIN_H_
+#define CTL_MIN_H_
+
+namespace ctl {
+
+template<class T>
+inline constexpr const T&
+min(const T& a, const T& b)
+{
+    return b < a ? b : a;
+}
+
+} // namespace ctl
+
+#endif /* CTL_MIN_H_ */

ctl/move_backward.h

@@ -0,0 +1,20 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_MOVE_BACKWARD_H_
+#define CTL_MOVE_BACKWARD_H_
+#include "utility.h"
+
+namespace ctl {
+
+template<typename BidirIt1, typename BidirIt2>
+BidirIt2
+move_backward(BidirIt1 first, BidirIt1 last, BidirIt2 d_last)
+{
+    while (first != last)
+        *(--d_last) = ctl::move(*(--last));
+    return d_last;
+}
+
+} // namespace ctl
+
+#endif // CTL_MOVE_BACKWARD_H_

ctl/move_iterator.h

@@ -0,0 +1,172 @@
+#ifndef CTL_MOVE_ITERATOR_H_
+#define CTL_MOVE_ITERATOR_H_
+#include "iterator_traits.h"
+
+namespace ctl {
+
+template<typename Iterator>
+class move_iterator
+{
+  public:
+    using iterator_type = Iterator;
+    using iterator_category =
+      typename ctl::iterator_traits<Iterator>::iterator_category;
+    using value_type = typename ctl::iterator_traits<Iterator>::value_type;
+    using difference_type =
+      typename ctl::iterator_traits<Iterator>::difference_type;
+    using pointer = Iterator;
+    using reference = value_type&&;
+
+    constexpr move_iterator() : current()
+    {
+    }
+
+    explicit constexpr move_iterator(Iterator i) : current(i)
+    {
+    }
+
+    template<class U>
+    constexpr move_iterator(const move_iterator<U>& u) : current(u.base())
+    {
+    }
+
+    constexpr Iterator base() const
+    {
+        return current;
+    }
+
+    constexpr reference operator*() const
+    {
+        return static_cast<reference>(*current);
+    }
+
+    constexpr pointer operator->() const
+    {
+        return current;
+    }
+
+    constexpr move_iterator& operator++()
+    {
+        ++current;
+        return *this;
+    }
+
+    constexpr move_iterator operator++(int)
+    {
+        move_iterator tmp = *this;
+        ++current;
+        return tmp;
+    }
+
+    constexpr move_iterator& operator--()
+    {
+        --current;
+        return *this;
+    }
+
+    constexpr move_iterator operator--(int)
+    {
+        move_iterator tmp = *this;
+        --current;
+        return tmp;
+    }
+
+    constexpr move_iterator operator+(difference_type n) const
+    {
+        return move_iterator(current + n);
+    }
+
+    constexpr move_iterator& operator+=(difference_type n)
+    {
+        current += n;
+        return *this;
+    }
+
+    constexpr move_iterator operator-(difference_type n) const
+    {
+        return move_iterator(current - n);
+    }
+
+    constexpr move_iterator& operator-=(difference_type n)
+    {
+        current -= n;
+        return *this;
+    }
+
+    constexpr reference operator[](difference_type n) const
+    {
+        return ctl::move(current[n]);
+    }
+
+  private:
+    Iterator current;
+};
+
+template<typename Iterator>
+__attribute__((__always_inline__)) constexpr move_iterator<Iterator>
+make_move_iterator(Iterator i)
+{
+    return move_iterator<Iterator>(i);
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator==(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return x.base() == y.base();
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator!=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return !(x == y);
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator<(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return x.base() < y.base();
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator<=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return !(y < x);
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator>(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return y < x;
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr bool
+operator>=(const move_iterator<Iterator1>& x, const move_iterator<Iterator2>& y)
+{
+    return !(x < y);
+}
+
+template<typename Iterator>
+constexpr move_iterator<Iterator>
+operator+(typename move_iterator<Iterator>::difference_type n,
+          const move_iterator<Iterator>& x)
+{
+    return x + n;
+}
+
+template<typename Iterator1, typename Iterator2>
+constexpr auto
+operator-(const move_iterator<Iterator1>& x,
+          const move_iterator<Iterator2>& y) -> decltype(x.base() - y.base())
+{
+    return x.base() - y.base();
+}
+
+} // namespace ctl
+
+#endif // CTL_MOVE_ITERATOR_H_

ctl/mutex.h

@@ -0,0 +1,47 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_MUTEX_H_
+#define CTL_MUTEX_H_
+#include "libc/thread/thread.h"
+
+namespace ctl {
+
+class mutex
+{
+  public:
+    mutex()
+    {
+        pthread_mutex_init(&m_, nullptr);
+    }
+
+    ~mutex()
+    {
+        pthread_mutex_destroy(&m_);
+    }
+
+    void lock()
+    {
+        pthread_mutex_lock(&m_);
+    }
+
+    bool try_lock()
+    {
+        return pthread_mutex_trylock(&m_) == 0;
+    }
+
+    void unlock()
+    {
+        pthread_mutex_unlock(&m_);
+    }
+
+    // Delete copy constructor and assignment operator
+    mutex(const mutex&) = delete;
+    mutex& operator=(const mutex&) = delete;
+
+  private:
+    pthread_mutex_t m_;
+};
+
+} // namespace ctl
+
+#endif // CTL_MUTEX_H_

ctl/new.cc

@@ -23,7 +23,7 @@
 COSMOPOLITAN_C_START_
 
 static void*
-_ctl_alloc(size_t n, size_t a)
+_ctl_alloc(size_t n, size_t a) noexcept
 {
     void* p;
     if (!(p = memalign(a, n)))
@@ -32,7 +32,16 @@ _ctl_alloc(size_t n, size_t a)
 }
 
 static void*
-_ctl_alloc1(size_t n)
+_ctl_alloc_nothrow(size_t n, size_t a, const ctl::nothrow_t&)
+{
+    void* p;
+    if (!(p = memalign(a, n)))
+        __builtin_trap();
+    return p;
+}
+
+static void*
+_ctl_alloc1(size_t n) noexcept
 {
     void* p;
     if (!(p = malloc(n)))
@@ -41,19 +50,28 @@ _ctl_alloc1(size_t n)
 }
 
 static void*
-_ctl_ret(size_t, void* p)
+_ctl_alloc1_nothrow(size_t n, const ctl::nothrow_t&) noexcept
+{
+    void* p;
+    if (!(p = malloc(n)))
+        __builtin_trap();
+    return p;
+}
+
+static void*
+_ctl_ret(size_t, void* p) noexcept
 {
     return p;
 }
 
 static void
-_ctl_free(void* p)
+_ctl_free(void* p) noexcept
 {
     free(p);
 }
 
 static void
-_ctl_nop(void*, void*)
+_ctl_nop(void*, void*) noexcept
 {
 }
 
@@ -75,23 +93,43 @@ __weak_reference(void*
                  operator new(size_t, ctl::align_val_t),
                  _ctl_alloc);
 
+__weak_reference(void*
+                 operator new(size_t,
+                              ctl::align_val_t,
+                              const ctl::nothrow_t&) noexcept,
+                 _ctl_alloc_nothrow);
+
 __weak_reference(void*
                  operator new[](size_t, ctl::align_val_t),
                  _ctl_alloc);
 
+__weak_reference(void*
+                 operator new[](size_t,
+                                ctl::align_val_t,
+                                const ctl::nothrow_t&) noexcept,
+                 _ctl_alloc_nothrow);
+
 __weak_reference(void*
                  operator new(size_t),
                  _ctl_alloc1);
 
+__weak_reference(void*
+                 operator new(size_t, const ctl::nothrow_t&) noexcept,
+                 _ctl_alloc1_nothrow);
+
 __weak_reference(void*
                  operator new[](size_t),
                  _ctl_alloc1);
 
+__weak_reference(void*
+                 operator new[](size_t, const ctl::nothrow_t&) noexcept,
+                 _ctl_alloc1_nothrow);
+
 // XXX clang-format currently mutilates these for some reason.
 // clang-format off
 
-__weak_reference(void* operator new(size_t, void*), _ctl_ret);
-__weak_reference(void* operator new[](size_t, void*), _ctl_ret);
+__weak_reference(void* operator new(size_t, void*) noexcept, _ctl_ret);
+__weak_reference(void* operator new[](size_t, void*) noexcept, _ctl_ret);
 
 __weak_reference(void operator delete(void*) noexcept, _ctl_free);
 __weak_reference(void operator delete[](void*) noexcept, _ctl_free);

ctl/new.h

@@ -13,18 +13,30 @@ enum class align_val_t : size_t
 {
 };
 
-} // namespace ctl
+struct nothrow_t
+{
+    explicit nothrow_t() = default;
+};
 
-void* operator new(size_t);
-void* operator new[](size_t);
-void* operator new(size_t, ctl::align_val_t);
-void* operator new[](size_t, ctl::align_val_t);
+inline constexpr nothrow_t nothrow{};
+
+} // namespace ctl
 
 // XXX clang-format currently mutilates these for some reason.
 // clang-format off
 
-void* operator new(size_t, void*);
-void* operator new[](size_t, void*);
+void* operator new(size_t);
+void* operator new(size_t, const ctl::nothrow_t&) noexcept;
+void* operator new(size_t, ctl::align_val_t);
+void* operator new(size_t, ctl::align_val_t, const ctl::nothrow_t&) noexcept;
+
+void* operator new[](size_t);
+void* operator new[](size_t, const ctl::nothrow_t&) noexcept;
+void* operator new[](size_t, ctl::align_val_t);
+void* operator new[](size_t, ctl::align_val_t, const ctl::nothrow_t&) noexcept;
+
+void* operator new(size_t, void*) noexcept;
+void* operator new[](size_t, void*) noexcept;
 
 void operator delete(void*) noexcept;
 void operator delete[](void*) noexcept;

ctl/numeric_limits.h

@@ -0,0 +1,123 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_NUMERIC_LIMITS_H_
+#define CTL_NUMERIC_LIMITS_H_
+#include "is_integral.h"
+#include "is_unsigned.h"
+
+namespace ctl {
+
+template<typename T>
+class numeric_limits
+{
+    static constexpr T max_signed()
+    {
+        return T(((uintmax_t)1 << (sizeof(T) * 8 - 1)) - 1);
+    }
+
+    static constexpr T max_unsigned()
+    {
+        return T(~T(0));
+    }
+
+  public:
+    static constexpr bool is_specialized = ctl::is_integral<T>::value;
+    static constexpr bool is_signed = !ctl::is_unsigned<T>::value;
+    static constexpr bool is_integer = ctl::is_integral<T>::value;
+    static constexpr bool is_exact = ctl::is_integral<T>::value;
+    static constexpr int digits = sizeof(T) * 8 - (is_signed ? 1 : 0);
+
+    static constexpr T min() noexcept
+    {
+        return is_signed ? T(-max_signed() - 1) : T(0);
+    }
+
+    static constexpr T max() noexcept
+    {
+        return is_signed ? max_signed() : max_unsigned();
+    }
+
+    static constexpr T lowest() noexcept
+    {
+        return min();
+    }
+};
+
+template<>
+struct numeric_limits<bool>
+{
+    static constexpr bool is_specialized = true;
+    static constexpr bool is_signed = false;
+    static constexpr bool is_integer = true;
+    static constexpr bool is_exact = true;
+    static constexpr int digits = 1;
+
+    static constexpr bool min() noexcept
+    {
+        return false;
+    }
+
+    static constexpr bool max() noexcept
+    {
+        return true;
+    }
+
+    static constexpr bool lowest() noexcept
+    {
+        return false;
+    }
+};
+
+template<>
+struct numeric_limits<float>
+{
+    static constexpr bool is_specialized = true;
+    static constexpr bool is_signed = true;
+    static constexpr bool is_integer = false;
+    static constexpr bool is_exact = false;
+    static constexpr int digits = 24;
+
+    static constexpr float min() noexcept
+    {
+        return __FLT_MIN__;
+    }
+
+    static constexpr float max() noexcept
+    {
+        return __FLT_MAX__;
+    }
+
+    static constexpr float lowest() noexcept
+    {
+        return -max();
+    }
+};
+
+template<>
+struct numeric_limits<double>
+{
+    static constexpr bool is_specialized = true;
+    static constexpr bool is_signed = true;
+    static constexpr bool is_integer = false;
+    static constexpr bool is_exact = false;
+    static constexpr int digits = 53;
+
+    static constexpr double min() noexcept
+    {
+        return __DBL_MIN__;
+    }
+
+    static constexpr double max() noexcept
+    {
+        return __DBL_MAX__;
+    }
+
+    static constexpr double lowest() noexcept
+    {
+        return -max();
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_NUMERIC_LIMITS_H_

ctl/out_of_range.h

@@ -0,0 +1,23 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_OUT_OF_RANGE_H_
+#define CTL_OUT_OF_RANGE_H_
+#include "exception.h"
+
+namespace ctl {
+
+class out_of_range : public exception
+{
+  public:
+    out_of_range() noexcept = default;
+    ~out_of_range() override = default;
+
+    const char* what() const noexcept override
+    {
+        return "ctl::out_of_range";
+    }
+};
+
+} // namespace ctl
+
+#endif // CTL_OUT_OF_RANGE_H_

ctl/partition.h

@@ -0,0 +1,29 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_PARTITION_H_
+#define CTL_PARTITION_H_
+#include "utility.h"
+
+namespace ctl {
+
+template<typename RandomIt, typename Compare>
+RandomIt
+partition(RandomIt first, RandomIt last, Compare comp)
+{
+    auto pivot = *ctl::move(last - 1);
+    auto i = first - 1;
+
+    for (auto j = first; j < last - 1; ++j) {
+        if (comp(*j, pivot)) {
+            ++i;
+            ctl::swap(*i, *j);
+        }
+    }
+
+    ctl::swap(*(i + 1), *(last - 1));
+    return i + 1;
+}
+
+} // namespace ctl
+
+#endif // CTL_PARTITION_H_

ctl/remove_cv.h

@@ -0,0 +1,49 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_REMOVE_CV_H_
+#define CTL_REMOVE_CV_H_
+
+namespace ctl {
+
+template<class T>
+struct remove_const
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_const<const T>
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_volatile
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_volatile<volatile T>
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_cv
+{
+    typedef typename remove_volatile<typename remove_const<T>::type>::type type;
+};
+
+template<class T>
+using remove_const_t = typename remove_const<T>::type;
+
+template<class T>
+using remove_volatile_t = typename remove_volatile<T>::type;
+
+template<class T>
+using remove_cv_t = typename remove_cv<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_REMOVE_CV_H_

ctl/remove_extent.h

@@ -0,0 +1,31 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_REMOVE_EXTENT_H_
+#define CTL_REMOVE_EXTENT_H_
+
+namespace ctl {
+
+template<typename T>
+struct remove_extent
+{
+    typedef T type;
+};
+
+template<typename T, size_t N>
+struct remove_extent<T[N]>
+{
+    typedef T type;
+};
+
+template<typename T>
+struct remove_extent<T[]>
+{
+    typedef T type;
+};
+
+template<typename T>
+using remove_extent_t = typename remove_extent<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_REMOVE_EXTENT_H_

ctl/remove_reference.h

@@ -0,0 +1,31 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_REMOVE_REFERENCE_H_
+#define CTL_REMOVE_REFERENCE_H_
+
+namespace ctl {
+
+template<class T>
+struct remove_reference
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_reference<T&>
+{
+    typedef T type;
+};
+
+template<class T>
+struct remove_reference<T&&>
+{
+    typedef T type;
+};
+
+template<class T>
+using remove_reference_t = typename remove_reference<T>::type;
+
+} // namespace ctl
+
+#endif // CTL_REMOVE_REFERENCE_H_

ctl/reverse_iterator.h

@@ -0,0 +1,183 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_REVERSE_ITERATOR_H_
+#define CTL_REVERSE_ITERATOR_H_
+#include "iterator_traits.h"
+#include "utility.h"
+
+namespace ctl {
+
+template<class Iterator>
+class reverse_iterator
+{
+  public:
+    using iterator_type = Iterator;
+    using iterator_category =
+      typename iterator_traits<Iterator>::iterator_category;
+    using value_type = typename iterator_traits<Iterator>::value_type;
+    using difference_type = typename iterator_traits<Iterator>::difference_type;
+    using pointer = typename iterator_traits<Iterator>::pointer;
+    using reference = typename iterator_traits<Iterator>::reference;
+
+    constexpr reverse_iterator() : current()
+    {
+    }
+
+    constexpr explicit reverse_iterator(Iterator x) : current(x)
+    {
+    }
+
+    template<class U>
+    constexpr reverse_iterator(const reverse_iterator<U>& other)
+      : current(other.base())
+    {
+    }
+
+    template<class U>
+    constexpr reverse_iterator& operator=(const reverse_iterator<U>& other)
+    {
+        current = other.base();
+        return *this;
+    }
+
+    constexpr Iterator base() const
+    {
+        return current;
+    }
+
+    constexpr reference operator*() const
+    {
+        Iterator tmp = current;
+        return *--tmp;
+    }
+
+    constexpr pointer operator->() const
+    {
+        return &(operator*());
+    }
+
+    constexpr reverse_iterator& operator++()
+    {
+        --current;
+        return *this;
+    }
+
+    constexpr reverse_iterator operator++(int)
+    {
+        reverse_iterator tmp = *this;
+        --current;
+        return tmp;
+    }
+
+    constexpr reverse_iterator& operator--()
+    {
+        ++current;
+        return *this;
+    }
+
+    constexpr reverse_iterator operator--(int)
+    {
+        reverse_iterator tmp = *this;
+        ++current;
+        return tmp;
+    }
+
+    constexpr reverse_iterator operator+(difference_type n) const
+    {
+        return reverse_iterator(current - n);
+    }
+
+    constexpr reverse_iterator& operator+=(difference_type n)
+    {
+        current -= n;
+        return *this;
+    }
+
+    constexpr reverse_iterator operator-(difference_type n) const
+    {
+        return reverse_iterator(current + n);
+    }
+
+    constexpr reverse_iterator& operator-=(difference_type n)
+    {
+        current += n;
+        return *this;
+    }
+
+    constexpr reference operator[](difference_type n) const
+    {
+        return *(*this + n);
+    }
+
+  protected:
+    Iterator current;
+};
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator==(const reverse_iterator<Iterator1>& lhs,
+           const reverse_iterator<Iterator2>& rhs)
+{
+    return lhs.base() == rhs.base();
+}
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator!=(const reverse_iterator<Iterator1>& lhs,
+           const reverse_iterator<Iterator2>& rhs)
+{
+    return lhs.base() != rhs.base();
+}
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator<(const reverse_iterator<Iterator1>& lhs,
+          const reverse_iterator<Iterator2>& rhs)
+{
+    return rhs.base() < lhs.base();
+}
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator<=(const reverse_iterator<Iterator1>& lhs,
+           const reverse_iterator<Iterator2>& rhs)
+{
+    return !(rhs < lhs);
+}
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator>(const reverse_iterator<Iterator1>& lhs,
+          const reverse_iterator<Iterator2>& rhs)
+{
+    return rhs < lhs;
+}
+
+template<class Iterator1, class Iterator2>
+constexpr bool
+operator>=(const reverse_iterator<Iterator1>& lhs,
+           const reverse_iterator<Iterator2>& rhs)
+{
+    return !(lhs < rhs);
+}
+
+template<class Iterator>
+constexpr reverse_iterator<Iterator>
+operator+(typename reverse_iterator<Iterator>::difference_type n,
+          const reverse_iterator<Iterator>& it)
+{
+    return reverse_iterator<Iterator>(it.base() - n);
+}
+
+template<class Iterator1, class Iterator2>
+constexpr auto
+operator-(const reverse_iterator<Iterator1>& lhs,
+          const reverse_iterator<Iterator2>& rhs) -> decltype(rhs.base() -
+                                                              lhs.base())
+{
+    return rhs.base() - lhs.base();
+}
+
+} // namespace ctl
+
+#endif // CTL_REVERSE_ITERATOR_H_

ctl/runtime_error.h

@@ -0,0 +1,33 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_RUNTIME_ERROR_H_
+#define CTL_RUNTIME_ERROR_H_
+#include "ctl/string.h"
+
+namespace ctl {
+
+class runtime_error
+{
+  public:
+    explicit runtime_error(const string& what_arg) : msg_(what_arg)
+    {
+    }
+
+    explicit runtime_error(const char* what_arg) : msg_(what_arg)
+    {
+    }
+
+    virtual ~runtime_error() noexcept = default;
+
+    virtual const char* what() const noexcept
+    {
+        return msg_.c_str();
+    }
+
+  private:
+    string msg_;
+};
+
+} // namespace ctl
+
+#endif // CTL_RUNTIME_ERROR_H_

ctl/sort.h

@@ -0,0 +1,44 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_SORT_H_
+#define CTL_SORT_H_
+#include "iterator_traits.h"
+#include "less.h"
+#include "partition.h"
+
+namespace ctl {
+
+namespace detail {
+
+template<typename RandomIt, typename Compare>
+void
+quicksort(RandomIt first, RandomIt last, Compare comp)
+{
+    if (first < last) {
+        auto pivot = ctl::partition(first, last, comp);
+        quicksort(first, pivot, comp);
+        quicksort(pivot + 1, last, comp);
+    }
+}
+
+} // namespace detail
+
+template<typename RandomIt, typename Compare>
+void
+sort(RandomIt first, RandomIt last, Compare comp)
+{
+    detail::quicksort(first, last, comp);
+}
+
+template<typename RandomIt>
+void
+sort(RandomIt first, RandomIt last)
+{
+    sort(first,
+         last,
+         ctl::less<typename ctl::iterator_traits<RandomIt>::value_type>());
+}
+
+} // namespace ctl
+
+#endif // CTL_SORT_H_

ctl/strcat.cc

@@ -38,9 +38,9 @@ strcat(const string_view lhs, const string_view rhs) noexcept
     if (rhs.n)
         memcpy(res.data() + lhs.n, rhs.p, rhs.n);
     if (res.isbig()) {
-        res.big()->n = lhs.n + rhs.n;
+        res.__b.n = lhs.n + rhs.n;
     } else {
-        res.small()->rem = __::sso_max - lhs.n - rhs.n;
+        res.__s.rem = __::sso_max - lhs.n - rhs.n;
     }
     res.data()[res.size()] = 0;
     return res;

ctl/string.cc

@@ -26,7 +26,7 @@ namespace ctl {
 void
 string::destroy_big() noexcept
 {
-    auto* b = big();
+    auto* b = &__b;
     if (b->n) {
         if (b->n >= b->c)
             __builtin_trap();
@@ -42,24 +42,20 @@ void
 string::init_big(const string& s) noexcept
 {
     char* p2;
-#ifndef NDEBUG
-    if (!s.isbig())
+    size_t size = s.size();
+    size_t need = size + 1;
+    size_t capacity = need;
+    if (!(p2 = (char*)malloc(capacity)))
         __builtin_trap();
-#endif
-    if (s.size() >= s.capacity() >> 1) {
-        if (!(p2 = (char*)malloc(s.capacity())))
-            __builtin_trap();
-        set_big_string(p2, s.size(), s.capacity());
-    } else {
-        init_big(string_view(s));
-    }
+    memcpy(p2, s.data(), need);
+    set_big_string(p2, size, capacity);
 }
 
 void
 string::init_big(const string_view s) noexcept
 {
-    size_t need;
     char* p2;
+    size_t need;
     if (ckd_add(&need, s.n, 1 /* nul */ + 15))
         __builtin_trap();
     need &= -16;
@@ -114,7 +110,7 @@ string::reserve(size_t c2) noexcept
             __builtin_trap();
         memcpy(p2, data(), __::string_size);
     } else {
-        if (!(p2 = (char*)realloc(big()->p, c2)))
+        if (!(p2 = (char*)realloc(__b.p, c2)))
             __builtin_trap();
     }
     std::atomic_signal_fence(std::memory_order_seq_cst);
@@ -131,7 +127,7 @@ string::resize(const size_t n2, const char ch) noexcept
     if (n2 > size())
         memset(data() + size(), ch, n2 - size());
     if (isbig()) {
-        big()->p[big()->n = n2] = 0;
+        __b.p[__b.n = n2] = 0;
     } else {
         set_small_size(n2);
         data()[size()] = 0;
@@ -152,9 +148,9 @@ string::append(const char ch) noexcept
     }
     data()[size()] = ch;
     if (isbig()) {
-        ++big()->n;
+        ++__b.n;
     } else {
-        --small()->rem;
+        --__s.rem;
     }
     data()[size()] = 0;
 }
@@ -185,9 +181,9 @@ string::append(const char ch, const size_t size) noexcept
     if (size)
         memset(data() + this->size(), ch, size);
     if (isbig()) {
-        big()->n += size;
+        __b.n += size;
     } else {
-        small()->rem -= size;
+        __s.rem -= size;
     }
     data()[this->size()] = 0;
 }
@@ -199,9 +195,9 @@ string::append(const void* data, const size_t size) noexcept
     if (size)
         memcpy(this->data() + this->size(), data, size);
     if (isbig()) {
-        big()->n += size;
+        __b.n += size;
     } else {
-        small()->rem -= size;
+        __s.rem -= size;
     }
     this->data()[this->size()] = 0;
 }
@@ -212,9 +208,9 @@ string::pop_back() noexcept
     if (!size())
         __builtin_trap();
     if (isbig()) {
-        --big()->n;
+        --__b.n;
     } else {
-        ++small()->rem;
+        ++__s.rem;
     }
     data()[size()] = 0;
 }
@@ -333,7 +329,7 @@ string::replace(const size_t pos,
         memmove(data() + pos + s.n, data() + last, extra);
     memcpy(data() + pos, s.p, s.n);
     if (isbig()) {
-        big()->p[big()->n = need] = 0;
+        __b.p[__b.n = need] = 0;
     } else {
         set_small_size(need);
         data()[size()] = 0;
@@ -357,9 +353,9 @@ string::insert(const size_t i, const string_view s) noexcept
         memmove(data() + i + s.n, data() + i, extra);
     memcpy(data() + i, s.p, s.n);
     if (isbig()) {
-        big()->n += s.n;
+        __b.n += s.n;
     } else {
-        small()->rem -= s.n;
+        __s.rem -= s.n;
     }
     data()[size()] = 0;
     return *this;
@@ -376,7 +372,7 @@ string::erase(const size_t pos, size_t count) noexcept
     if (extra)
         memmove(data() + pos, data() + pos + count, extra);
     if (isbig()) {
-        big()->n = pos + extra;
+        __b.n = pos + extra;
     } else {
         set_small_size(pos + extra);
     }

ctl/string.h

@@ -2,6 +2,7 @@
 // vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
 #ifndef CTL_STRING_H_
 #define CTL_STRING_H_
+#include "reverse_iterator.h"
 #include "string_view.h"
 
 namespace ctl {
@@ -44,8 +45,18 @@ struct big_string
 class string
 {
   public:
-    using iterator = char*;
-    using const_iterator = const char*;
+    using value_type = char;
+    using size_type = size_t;
+    using difference_type = ptrdiff_t;
+    using pointer = value_type*;
+    using const_pointer = const value_type*;
+    using reference = value_type&;
+    using const_reference = const value_type&;
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+    using reverse_iterator = ctl::reverse_iterator<iterator>;
+    using const_reverse_iterator = ctl::reverse_iterator<const_iterator>;
+
     static constexpr size_t npos = -1;
 
     string() noexcept
@@ -140,7 +151,7 @@ class string
     void clear() noexcept
     {
         if (isbig()) {
-            big()->n = 0;
+            __b.n = 0;
         } else {
             set_small_size(0);
         }
@@ -148,26 +159,26 @@ class string
 
     bool empty() const noexcept
     {
-        return isbig() ? !big()->n : small()->rem >= __::sso_max;
+        return isbig() ? !__b.n : __s.rem >= __::sso_max;
     }
 
-    char* data() noexcept
+    __attribute__((__always_inline__)) char* data() noexcept
     {
-        return isbig() ? big()->p : small()->buf;
+        return isbig() ? __b.p : __s.buf;
     }
 
-    const char* data() const noexcept
+    __attribute__((__always_inline__)) const char* data() const noexcept
     {
-        return isbig() ? big()->p : small()->buf;
+        return isbig() ? __b.p : __s.buf;
     }
 
-    size_t size() const noexcept
+    __attribute__((__always_inline__)) size_t size() const noexcept
     {
 #if 0
-        if (!isbig() && small()->rem > __::sso_max)
+        if (!isbig() && __s.rem > __::sso_max)
             __builtin_trap();
 #endif
-        return isbig() ? big()->n : __::sso_max - small()->rem;
+        return isbig() ? __b.n : __::sso_max - __s.rem;
     }
 
     size_t length() const noexcept
@@ -178,10 +189,10 @@ class string
     size_t capacity() const noexcept
     {
 #if 0
-        if (isbig() && big()->c <= __::sso_max)
+        if (isbig() && __b.c <= __::sso_max)
             __builtin_trap();
 #endif
-        return isbig() ? __::big_mask & big()->c : __::string_size;
+        return isbig() ? __::big_mask & __b.c : __::string_size;
     }
 
     iterator begin() noexcept
@@ -189,9 +200,9 @@ class string
         return data();
     }
 
-    iterator end() noexcept
+    const_iterator begin() const noexcept
     {
-        return data() + size();
+        return data();
     }
 
     const_iterator cbegin() const noexcept
@@ -199,11 +210,51 @@ class string
         return data();
     }
 
+    reverse_iterator rbegin() noexcept
+    {
+        return reverse_iterator(end());
+    }
+
+    const_reverse_iterator rbegin() const noexcept
+    {
+        return const_reverse_iterator(end());
+    }
+
+    const_reverse_iterator crbegin() const noexcept
+    {
+        return const_reverse_iterator(end());
+    }
+
+    iterator end() noexcept
+    {
+        return data() + size();
+    }
+
+    const_iterator end() const noexcept
+    {
+        return data() + size();
+    }
+
     const_iterator cend() const noexcept
     {
         return data() + size();
     }
 
+    reverse_iterator rend() noexcept
+    {
+        return reverse_iterator(begin());
+    }
+
+    const_reverse_iterator rend() const noexcept
+    {
+        return const_reverse_iterator(cbegin());
+    }
+
+    const_reverse_iterator crend() const noexcept
+    {
+        return const_reverse_iterator(cbegin());
+    }
+
     char& front()
     {
         if (!size())
@@ -323,7 +374,7 @@ class string
     void init_big(string_view) noexcept;
     void init_big(size_t, char) noexcept;
 
-    bool isbig() const noexcept
+    __attribute__((__always_inline__)) bool isbig() const noexcept
     {
         return *(blob + __::sso_max) & 0x80;
     }
@@ -344,34 +395,6 @@ class string
         __b.c = c2 | ~__::big_mask;
     }
 
-    __::small_string* small() noexcept
-    {
-        if (isbig())
-            __builtin_trap();
-        return &__s;
-    }
-
-    const __::small_string* small() const noexcept
-    {
-        if (isbig())
-            __builtin_trap();
-        return &__s;
-    }
-
-    __::big_string* big() noexcept
-    {
-        if (!isbig())
-            __builtin_trap();
-        return &__b;
-    }
-
-    const __::big_string* big() const noexcept
-    {
-        if (!isbig())
-            __builtin_trap();
-        return &__b;
-    }
-
     friend string strcat(string_view, string_view) noexcept;
 
     union

ctl/tuple.h

@@ -0,0 +1,138 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_TUPLE_H_
+#define CTL_TUPLE_H_
+#include "decay.h"
+#include "enable_if.h"
+#include "is_same.h"
+#include "utility.h"
+
+namespace ctl {
+
+// Forward declaration
+template<typename... Types>
+class tuple;
+
+// Base case for tuple
+template<>
+class tuple<>
+{};
+
+// Recursive case for tuple
+template<typename Head, typename... Tail>
+class tuple<Head, Tail...> : public tuple<Tail...>
+{
+    using Base = tuple<Tail...>;
+
+  public:
+    Head head;
+
+    constexpr tuple() : Base(), head()
+    {
+    }
+
+    template<typename H,
+             typename... T,
+             typename = typename ctl::enable_if<
+               !ctl::is_same<typename ctl::decay<H>::type, tuple>::value>::type>
+    constexpr tuple(H&& h, T&&... t)
+      : Base(ctl::forward<T>(t)...), head(ctl::forward<H>(h))
+    {
+    }
+
+    template<typename... UTypes>
+    constexpr tuple(const tuple<UTypes...>& other)
+      : Base(static_cast<const tuple<UTypes...>&>(other)), head(other.head)
+    {
+    }
+
+    template<typename... UTypes>
+    constexpr tuple(tuple<UTypes...>&& other)
+      : Base(static_cast<tuple<UTypes...>&&>(other))
+      , head(ctl::forward<Head>(other.head))
+    {
+    }
+
+    tuple(const tuple&) = default;
+    tuple(tuple&&) = default;
+    tuple& operator=(const tuple&) = default;
+    tuple& operator=(tuple&&) = default;
+};
+
+// Helper struct for getting element type
+template<size_t I, typename T>
+struct tuple_element;
+
+template<size_t I, typename Head, typename... Tail>
+struct tuple_element<I, tuple<Head, Tail...>>
+  : tuple_element<I - 1, tuple<Tail...>>
+{};
+
+template<typename Head, typename... Tail>
+struct tuple_element<0, tuple<Head, Tail...>>
+{
+    using type = Head;
+};
+
+// Helper function to get element
+template<size_t I, typename Head, typename... Tail>
+constexpr typename tuple_element<I, tuple<Head, Tail...>>::type&
+get(tuple<Head, Tail...>& t)
+{
+    if constexpr (I == 0) {
+        return t.head;
+    } else {
+        return get<I - 1>(static_cast<tuple<Tail...>&>(t));
+    }
+}
+
+template<size_t I, typename Head, typename... Tail>
+constexpr const typename tuple_element<I, tuple<Head, Tail...>>::type&
+get(const tuple<Head, Tail...>& t)
+{
+    if constexpr (I == 0) {
+        return t.head;
+    } else {
+        return get<I - 1>(static_cast<const tuple<Tail...>&>(t));
+    }
+}
+
+// Helper function to create a tuple
+template<typename... Types>
+constexpr tuple<typename ctl::decay<Types>::type...>
+make_tuple(Types&&... args)
+{
+    return tuple<typename ctl::decay<Types>::type...>(
+      ctl::forward<Types>(args)...);
+}
+
+// Helper function for tuple comparison
+template<size_t I = 0, typename... TTypes, typename... UTypes>
+constexpr bool
+tuple_equals(const tuple<TTypes...>& t, const tuple<UTypes...>& u)
+{
+    if constexpr (I == sizeof...(TTypes)) {
+        return true;
+    } else {
+        return get<I>(t) == get<I>(u) && tuple_equals<I + 1>(t, u);
+    }
+}
+
+// Equality comparison
+template<typename... TTypes, typename... UTypes>
+constexpr bool
+operator==(const tuple<TTypes...>& t, const tuple<UTypes...>& u)
+{
+    return tuple_equals(t, u);
+}
+
+template<typename... TTypes, typename... UTypes>
+constexpr bool
+operator!=(const tuple<TTypes...>& t, const tuple<UTypes...>& u)
+{
+    return !(t == u);
+}
+
+} // namespace ctl
+
+#endif // CTL_TUPLE_H_

ctl/uninitialized_fill.h

@@ -0,0 +1,29 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_UNINITIALIZED_FILL_H_
+#define CTL_UNINITIALIZED_FILL_H_
+#include "addressof.h"
+#include "iterator_traits.h"
+
+namespace ctl {
+
+template<typename ForwardIt, typename T>
+void
+uninitialized_fill(ForwardIt first, ForwardIt last, const T& value)
+{
+    using ValueType = typename ctl::iterator_traits<ForwardIt>::value_type;
+    ForwardIt current = first;
+    try {
+        for (; current != last; ++current)
+            ::new (static_cast<void*>(ctl::addressof(*current)))
+              ValueType(value);
+    } catch (...) {
+        for (; first != current; ++first)
+            first->~ValueType();
+        throw;
+    }
+}
+
+} // namespace ctl
+
+#endif // CTL_UNINITIALIZED_FILL_H_

ctl/uninitialized_fill_n.h

@@ -0,0 +1,52 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_UNINITIALIZED_FILL_N_H_
+#define CTL_UNINITIALIZED_FILL_N_H_
+#include "addressof.h"
+#include "allocator_traits.h"
+#include "iterator_traits.h"
+
+namespace ctl {
+
+template<typename ForwardIt, typename Size, typename T>
+ForwardIt
+uninitialized_fill_n(ForwardIt first, Size n, const T& value)
+{
+    using ValueType = typename ctl::iterator_traits<ForwardIt>::value_type;
+    ForwardIt current = first;
+    try {
+        for (; n > 0; ++current, --n) {
+            ::new (static_cast<void*>(ctl::addressof(*current)))
+              ValueType(value);
+        }
+        return current;
+    } catch (...) {
+        for (; first != current; ++first) {
+            first->~ValueType();
+        }
+        throw;
+    }
+}
+
+template<typename ForwardIt, typename Size, typename T, typename Alloc>
+ForwardIt
+uninitialized_fill_n(ForwardIt first, Size n, const T& value, Alloc& alloc)
+{
+    using AllocTraits = ctl::allocator_traits<Alloc>;
+    ForwardIt current = first;
+    try {
+        for (; n > 0; ++current, --n) {
+            AllocTraits::construct(alloc, ctl::addressof(*current), value);
+        }
+        return current;
+    } catch (...) {
+        for (; first != current; ++first) {
+            AllocTraits::destroy(alloc, ctl::addressof(*first));
+        }
+        throw;
+    }
+}
+
+} // namespace ctl
+
+#endif // CTL_UNINITIALIZED_FILL_N_H_

ctl/uninitialized_move_n.h

@@ -0,0 +1,52 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_UNINITIALIZED_MOVE_N_H_
+#define CTL_UNINITIALIZED_MOVE_N_H_
+#include "addressof.h"
+#include "allocator_traits.h"
+#include "iterator_traits.h"
+#include "utility.h"
+
+namespace ctl {
+
+template<typename InputIt, typename Size, typename ForwardIt>
+ForwardIt
+uninitialized_move_n(InputIt first, Size n, ForwardIt d_first)
+{
+    using T = typename ctl::iterator_traits<ForwardIt>::value_type;
+    ForwardIt current = d_first;
+    try {
+        for (; n > 0; ++first, (void)++current, --n) {
+            ::new (static_cast<void*>(ctl::addressof(*current)))
+              T(ctl::move(*first));
+        }
+    } catch (...) {
+        for (; d_first != current; ++d_first) {
+            d_first->~T();
+        }
+        throw;
+    }
+    return current;
+}
+
+template<typename InputIt, typename Size, typename ForwardIt, typename Alloc>
+ForwardIt
+uninitialized_move_n(InputIt first, Size n, ForwardIt d_first, Alloc& alloc)
+{
+    using AllocTraits = ctl::allocator_traits<Alloc>;
+    ForwardIt current = d_first;
+    try {
+        for (; n > 0; ++first, (void)++current, --n)
+            AllocTraits::construct(
+              alloc, ctl::addressof(*current), ctl::move(*first));
+    } catch (...) {
+        for (; d_first != current; ++d_first)
+            AllocTraits::destroy(alloc, ctl::addressof(*d_first));
+        throw;
+    }
+    return current;
+}
+
+} // namespace ctl
+
+#endif // CTL_UNINITIALIZED_MOVE_N_H_

ctl/unique_lock.h

@@ -0,0 +1,147 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_UNIQUE_LOCK_H_
+#define CTL_UNIQUE_LOCK_H_
+#include "mutex.h"
+#include "utility.h"
+
+namespace ctl {
+
+struct defer_lock_t
+{
+    explicit defer_lock_t() = default;
+};
+
+struct try_to_lock_t
+{
+    explicit try_to_lock_t() = default;
+};
+
+struct adopt_lock_t
+{
+    explicit adopt_lock_t() = default;
+};
+
+inline constexpr defer_lock_t defer_lock{};
+inline constexpr try_to_lock_t try_to_lock{};
+inline constexpr adopt_lock_t adopt_lock{};
+
+class unique_lock
+{
+  public:
+    unique_lock() noexcept : mutex_(nullptr), owns_lock_(false)
+    {
+    }
+
+    explicit unique_lock(ctl::mutex& m) : mutex_(&m), owns_lock_(false)
+    {
+        lock();
+    }
+
+    unique_lock(ctl::mutex& m, defer_lock_t) noexcept
+      : mutex_(&m), owns_lock_(false)
+    {
+    }
+
+    unique_lock(ctl::mutex& m, try_to_lock_t)
+      : mutex_(&m), owns_lock_(mutex_->try_lock())
+    {
+    }
+
+    unique_lock(ctl::mutex& m, adopt_lock_t) : mutex_(&m), owns_lock_(true)
+    {
+    }
+
+    ~unique_lock()
+    {
+        if (owns_lock_)
+            mutex_->unlock();
+    }
+
+    unique_lock(const unique_lock&) = delete;
+    unique_lock& operator=(const unique_lock&) = delete;
+
+    unique_lock(unique_lock&& other) noexcept
+      : mutex_(other.mutex_), owns_lock_(other.owns_lock_)
+    {
+        other.mutex_ = nullptr;
+        other.owns_lock_ = false;
+    }
+
+    unique_lock& operator=(unique_lock&& other) noexcept
+    {
+        if (owns_lock_)
+            mutex_->unlock();
+        mutex_ = other.mutex_;
+        owns_lock_ = other.owns_lock_;
+        other.mutex_ = nullptr;
+        other.owns_lock_ = false;
+        return *this;
+    }
+
+    void lock()
+    {
+        if (!mutex_)
+            __builtin_trap();
+        if (owns_lock_)
+            __builtin_trap();
+        mutex_->lock();
+        owns_lock_ = true;
+    }
+
+    bool try_lock()
+    {
+        if (!mutex_)
+            __builtin_trap();
+        if (owns_lock_)
+            __builtin_trap();
+        owns_lock_ = mutex_->try_lock();
+        return owns_lock_;
+    }
+
+    void unlock()
+    {
+        if (!owns_lock_)
+            __builtin_trap();
+        mutex_->unlock();
+        owns_lock_ = false;
+    }
+
+    void swap(unique_lock& other) noexcept
+    {
+        using ctl::swap;
+        swap(mutex_, other.mutex_);
+        swap(owns_lock_, other.owns_lock_);
+    }
+
+    ctl::mutex* release() noexcept
+    {
+        ctl::mutex* result = mutex_;
+        mutex_ = nullptr;
+        owns_lock_ = false;
+        return result;
+    }
+
+    bool owns_lock() const noexcept
+    {
+        return owns_lock_;
+    }
+
+    explicit operator bool() const noexcept
+    {
+        return owns_lock_;
+    }
+
+    ctl::mutex* mutex() const noexcept
+    {
+        return mutex_;
+    }
+
+  private:
+    ctl::mutex* mutex_;
+    bool owns_lock_;
+};
+
+} // namespace ctl
+
+#endif // CTL_UNIQUE_LOCK_H_

ctl/unique_ptr.h

@@ -1,41 +1,17 @@
 // -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
 // vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#ifndef COSMOPOLITAN_CTL_UNIQUE_PTR_H_
-#define COSMOPOLITAN_CTL_UNIQUE_PTR_H_
+#ifndef CTL_UNIQUE_PTR_H_
+#define CTL_UNIQUE_PTR_H_
+#include "add_lvalue_reference.h"
+#include "default_delete.h"
+#include "is_convertible.h"
+#include "is_reference.h"
+#include "is_same.h"
 #include "utility.h"
-#include <__type_traits/is_convertible.h>
 
 namespace ctl {
 
-template<typename T>
-struct default_delete
-{
-    constexpr default_delete() noexcept = default;
-    template<typename U>
-    constexpr default_delete(default_delete<U>&&) noexcept
-    {
-    }
-    constexpr void operator()(T* const p) const noexcept
-    {
-        delete p;
-    }
-};
-
-template<typename T>
-struct default_delete<T[]>
-{
-    constexpr default_delete() noexcept = default;
-    template<typename U>
-    constexpr default_delete(default_delete<U>&&) noexcept
-    {
-    }
-    constexpr void operator()(T* const p) const noexcept
-    {
-        delete[] p;
-    }
-};
-
-template<typename T, typename D = default_delete<T>>
+template<typename T, typename D = ctl::default_delete<T>>
 struct unique_ptr
 {
     using pointer = T*;
@@ -49,26 +25,34 @@ struct unique_ptr
     {
     }
 
-    constexpr unique_ptr(const pointer p) noexcept : p(p), d()
+    constexpr explicit unique_ptr(pointer p) noexcept : p(p), d()
     {
     }
 
-    constexpr unique_ptr(const pointer p, auto&& d) noexcept
-      : p(p), d(ctl::forward<decltype(d)>(d))
+    constexpr unique_ptr(pointer p, const D& d) noexcept : p(p), d(d)
+    {
+    }
+
+    constexpr unique_ptr(pointer p, D&& d) noexcept : p(p), d(ctl::move(d))
     {
     }
 
     template<typename U, typename E>
-        requires std::is_convertible_v<U, T> && std::is_convertible_v<E, D>
     constexpr unique_ptr(unique_ptr<U, E>&& u) noexcept
-      : p(u.p), d(ctl::move(u.d))
+      : p(u.release()), d(ctl::forward<E>(u.get_deleter()))
     {
-        u.p = nullptr;
+        static_assert(ctl::is_convertible<typename unique_ptr<U, E>::pointer,
+                                          pointer>::value,
+                      "U* must be implicitly convertible to T*");
+        static_assert(
+          (ctl::is_reference<D>::value && ctl::is_same<D, E>::value) ||
+            (!ctl::is_reference<D>::value && ctl::is_convertible<E, D>::value),
+          "The deleter must be convertible to the target deleter type");
     }
 
     unique_ptr(const unique_ptr&) = delete;
 
-    constexpr ~unique_ptr() /* noexcept */
+    constexpr ~unique_ptr() noexcept
     {
         if (p)
             d(p);
@@ -80,6 +64,20 @@ struct unique_ptr
         return *this;
     }
 
+    template<typename U, typename E>
+    constexpr unique_ptr& operator=(unique_ptr<U, E>&& r) noexcept
+    {
+        reset(r.release());
+        d = ctl::forward<E>(r.get_deleter());
+        return *this;
+    }
+
+    constexpr unique_ptr& operator=(nullptr_t) noexcept
+    {
+        reset();
+        return *this;
+    }
+
     constexpr pointer release() noexcept
     {
         pointer r = p;
@@ -87,12 +85,12 @@ struct unique_ptr
         return r;
     }
 
-    constexpr void reset(const pointer p2 = pointer()) noexcept
+    constexpr void reset(pointer p2 = pointer()) noexcept
     {
-        const pointer r = p;
+        pointer old = p;
         p = p2;
-        if (r)
-            d(r);
+        if (old)
+            d(old);
     }
 
     constexpr void swap(unique_ptr& r) noexcept
@@ -119,20 +117,16 @@ struct unique_ptr
 
     constexpr explicit operator bool() const noexcept
     {
-        return p;
+        return p != nullptr;
     }
 
-    element_type& operator*() const noexcept(noexcept(*ctl::declval<pointer>()))
+    constexpr typename ctl::add_lvalue_reference<T>::type operator*() const
     {
-        if (!p)
-            __builtin_trap();
         return *p;
     }
 
-    pointer operator->() const noexcept
+    constexpr pointer operator->() const noexcept
     {
-        if (!p)
-            __builtin_trap();
         return p;
     }
 };
@@ -148,16 +142,9 @@ template<typename T>
 constexpr unique_ptr<T>
 make_unique_for_overwrite()
 {
-#if 0
-    // You'd think that it'd work like this, but std::unique_ptr does not.
-    return unique_ptr<T>(
-      static_cast<T*>(::operator new(sizeof(T), align_val_t(alignof(T)))));
-#else
     return unique_ptr<T>(new T);
-#endif
 }
 
-// TODO(mrdomino): specializations for T[]
-
 } // namespace ctl
-#endif // COSMOPOLITAN_CTL_UNIQUE_PTR_H_
+
+#endif // CTL_UNIQUE_PTR_H_

ctl/utility.h

@@ -2,6 +2,7 @@
 // vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
 #ifndef COSMOPOLITAN_CTL_UTILITY_H_
 #define COSMOPOLITAN_CTL_UTILITY_H_
+#include "remove_reference.h"
 
 namespace ctl {
 
@@ -18,6 +19,14 @@ using no_infer = typename no_infer_<T>::type;
 
 } // namespace __
 
+template<typename T>
+constexpr T&&
+move(T&& t) noexcept
+{
+    typedef remove_reference_t<T> U;
+    return static_cast<U&&>(t);
+}
+
 template<typename T>
 constexpr T&&
 move(T& t) noexcept
@@ -47,9 +56,8 @@ template<typename T, size_t N>
 constexpr void
 swap(T (&a)[N], T (&b)[N]) noexcept
 {
-    for (size_t i = 0; i < N; ++i) {
+    for (size_t i = 0; i < N; ++i)
         swap(a[i], b[i]);
-    }
 }
 
 template<typename T>

ctl/vector.h

@@ -1,245 +1,594 @@
 // -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
 // vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#ifndef COSMOPOLITAN_CTL_OPTIONAL_H_
-#define COSMOPOLITAN_CTL_OPTIONAL_H_
-#include "new.h"
-#include "utility.h"
+#ifndef CTL_VECTOR_H_
+#define CTL_VECTOR_H_
+#include "allocator.h"
+#include "allocator_traits.h"
+#include "copy.h"
+#include "distance.h"
+#include "equal.h"
+#include "fill_n.h"
+#include "initializer_list.h"
+#include "iterator_traits.h"
+#include "lexicographical_compare.h"
+#include "max.h"
+#include "move_backward.h"
+#include "move_iterator.h"
+#include "out_of_range.h"
+#include "reverse_iterator.h"
+#include "uninitialized_fill.h"
+#include "uninitialized_fill_n.h"
+#include "uninitialized_move_n.h"
 
 namespace ctl {
 
-template<typename T>
-struct vector
+template<typename T, typename Allocator = ctl::allocator<T>>
+class vector
 {
-    size_t n = 0;
-    size_t c = 0;
-    T* p = nullptr;
+  public:
+    using value_type = T;
+    using allocator_type = Allocator;
+    using size_type = size_t;
+    using difference_type = ptrdiff_t;
+    using reference = value_type&;
+    using const_reference = const value_type&;
+    using pointer = typename ctl::allocator_traits<Allocator>::pointer;
+    using const_pointer =
+      typename ctl::allocator_traits<Allocator>::const_pointer;
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+    using reverse_iterator = ctl::reverse_iterator<iterator>;
+    using const_reverse_iterator = ctl::reverse_iterator<const_iterator>;
 
-    using iterator = T*;
-    using const_iterator = const T*;
-
-    vector() = default;
-
-    ~vector()
+  public:
+    vector() noexcept(noexcept(Allocator()))
+      : alloc_(), data_(nullptr), size_(0), capacity_(0)
     {
-        delete[] p;
+    }
+
+    explicit vector(const Allocator& alloc) noexcept
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+    }
+
+    vector(size_type count,
+           const T& value,
+           const Allocator& alloc = Allocator())
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+        assign(count, value);
+    }
+
+    explicit vector(size_type count, const Allocator& alloc = Allocator())
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+        resize(count);
+    }
+
+    template<class InputIt>
+    vector(InputIt first, InputIt last, const Allocator& alloc = Allocator())
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+        assign(first, last);
     }
 
     vector(const vector& other)
+      : alloc_(ctl::allocator_traits<
+               Allocator>::select_on_container_copy_construction(other.alloc_))
+      , data_(nullptr)
+      , size_(0)
+      , capacity_(0)
     {
-        n = other.n;
-        c = other.c;
-        p = new T[c];
-        for (size_t i = 0; i < n; ++i)
-            new (&p[i]) T(other.p[i]);
+        assign(other.begin(), other.end());
+    }
+
+    vector(const vector& other, const Allocator& alloc)
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+        assign(other.begin(), other.end());
     }
 
     vector(vector&& other) noexcept
+      : alloc_(ctl::move(other.alloc_))
+      , data_(other.data_)
+      , size_(other.size_)
+      , capacity_(other.capacity_)
     {
-        n = other.n;
-        c = other.c;
-        p = other.p;
-        other.n = 0;
-        other.c = 0;
-        other.p = nullptr;
+        other.data_ = nullptr;
+        other.size_ = 0;
+        other.capacity_ = 0;
     }
 
-    explicit vector(size_t count, const T& value = T())
+    vector(vector&& other, const Allocator& alloc)
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
     {
-        n = count;
-        c = count;
-        p = new T[c];
-        for (size_t i = 0; i < n; ++i)
-            new (&p[i]) T(value);
+        if (alloc_ == other.alloc_) {
+            data_ = other.data_;
+            size_ = other.size_;
+            capacity_ = other.capacity_;
+            other.data_ = nullptr;
+            other.size_ = 0;
+            other.capacity_ = 0;
+        } else {
+            assign(ctl::make_move_iterator(other.begin()),
+                   ctl::make_move_iterator(other.end()));
+        }
+    }
+
+    vector(std::initializer_list<T> init, const Allocator& alloc = Allocator())
+      : alloc_(alloc), data_(nullptr), size_(0), capacity_(0)
+    {
+        assign(init.begin(), init.end());
+    }
+
+    ~vector()
+    {
+        clear();
+        ctl::allocator_traits<Allocator>::deallocate(alloc_, data_, capacity_);
     }
 
     vector& operator=(const vector& other)
     {
         if (this != &other) {
-            T* newData = new T[other.c];
-            for (size_t i = 0; i < other.n; ++i) {
-                newData[i] = other.p[i];
-            }
-            delete[] p;
-            p = newData;
-            n = other.n;
-            c = other.c;
+            if (ctl::allocator_traits<
+                  Allocator>::propagate_on_container_copy_assignment::value)
+                alloc_ = other.alloc_;
+            assign(other.begin(), other.end());
         }
         return *this;
     }
 
-    vector& operator=(vector&& other) noexcept
+    vector& operator=(vector&& other) noexcept(
+      ctl::allocator_traits<Allocator>::is_always_equal::value)
     {
         if (this != &other) {
-            delete[] p;
-            p = other.p;
-            n = other.n;
-            c = other.c;
-            other.p = nullptr;
-            other.n = 0;
-            other.c = 0;
+            clear();
+            ctl::allocator_traits<Allocator>::deallocate(
+              alloc_, data_, capacity_);
+            if (ctl::allocator_traits<
+                  Allocator>::propagate_on_container_move_assignment::value)
+                alloc_ = ctl::move(other.alloc_);
+            data_ = other.data_;
+            size_ = other.size_;
+            capacity_ = other.capacity_;
+            other.data_ = nullptr;
+            other.size_ = 0;
+            other.capacity_ = 0;
         }
         return *this;
     }
 
-    bool empty() const
+    vector& operator=(std::initializer_list<T> ilist)
     {
-        return !n;
+        assign(ilist.begin(), ilist.end());
+        return *this;
     }
 
-    size_t size() const
+    void assign(size_type count, const T& value)
     {
-        return n;
+        clear();
+        if (count > capacity_)
+            reallocate(count);
+        ctl::uninitialized_fill_n(data_, count, value);
+        size_ = count;
     }
 
-    size_t capacity() const
+    template<class InputIt>
+    void assign(InputIt first, InputIt last)
     {
-        return c;
+        clear();
+        for (; first != last; ++first)
+            push_back(*first);
     }
 
-    T& operator[](size_t i)
+    void assign(std::initializer_list<T> ilist)
     {
-        if (i >= n)
-            __builtin_trap();
-        return p[i];
+        assign(ilist.begin(), ilist.end());
     }
 
-    const T& operator[](size_t i) const
+    reference at(size_type pos)
     {
-        if (i >= n)
-            __builtin_trap();
-        return p[i];
+        if (pos >= size_)
+            throw ctl::out_of_range("out of range");
+        return data_[pos];
     }
 
-    iterator begin()
+    const_reference at(size_type pos) const
     {
-        return p;
+        if (pos >= size_)
+            throw ctl::out_of_range("out of range");
+        return data_[pos];
     }
 
-    iterator end()
+    reference operator[](size_type pos)
     {
-        return p + n;
+        return data_[pos];
     }
 
-    const_iterator cbegin() const
+    const_reference operator[](size_type pos) const
     {
-        return p;
+        return data_[pos];
     }
 
-    const_iterator cend() const
+    reference front()
     {
-        return p + n;
+        return data_[0];
     }
 
-    T& front()
+    const_reference front() const
     {
-        if (!n)
-            __builtin_trap();
-        return p[0];
+        return data_[0];
     }
 
-    const T& front() const
+    reference back()
     {
-        if (!n)
-            __builtin_trap();
-        return p[0];
+        return data_[size_ - 1];
     }
 
-    T& back()
+    const_reference back() const
     {
-        if (!n)
-            __builtin_trap();
-        return p[n - 1];
+        return data_[size_ - 1];
     }
 
-    const T& back() const
+    T* data() noexcept
     {
-        if (!n)
-            __builtin_trap();
-        return p[n - 1];
+        return data_;
     }
 
-    void clear()
+    const T* data() const noexcept
     {
-        for (size_t i = 0; i < n; ++i)
-            p[i].~T();
-        n = 0;
+        return data_;
     }
 
-    void reserve(size_t c2)
+    iterator begin() noexcept
     {
-        if (c2 <= c)
-            return;
-        T* newP = new T[c2];
-        for (size_t i = 0; i < n; ++i)
-            newP[i] = ctl::move(p[i]);
-        delete[] p;
-        p = newP;
-        c = c2;
+        return data_;
     }
 
-    void push_back(const T& e)
+    const_iterator begin() const noexcept
     {
-        if (n == c) {
-            size_t c2 = c + 1;
-            c2 += c2 >> 1;
-            reserve(c2);
-        }
-        new (&p[n]) T(e);
-        ++n;
+        return data_;
     }
 
-    void push_back(T&& e)
+    const_iterator cbegin() const noexcept
     {
-        if (n == c) {
-            size_t c2 = c + 1;
-            c2 += c2 >> 1;
-            reserve(c2);
-        }
-        new (&p[n]) T(ctl::forward<T>(e));
-        ++n;
+        return data_;
     }
 
-    template<typename... Args>
-    void emplace_back(Args&&... args)
+    iterator end() noexcept
     {
-        if (n == c) {
-            size_t c2 = c + 1;
-            c2 += c2 >> 1;
-            reserve(c2);
-        }
-        new (&p[n]) T(ctl::forward<Args>(args)...);
-        ++n;
+        return data_ + size_;
+    }
+
+    const_iterator end() const noexcept
+    {
+        return data_ + size_;
+    }
+
+    const_iterator cend() const noexcept
+    {
+        return data_ + size_;
+    }
+
+    reverse_iterator rbegin() noexcept
+    {
+        return reverse_iterator(end());
+    }
+
+    const_reverse_iterator rbegin() const noexcept
+    {
+        return const_reverse_iterator(end());
+    }
+
+    const_reverse_iterator crbegin() const noexcept
+    {
+        return const_reverse_iterator(end());
+    }
+
+    reverse_iterator rend() noexcept
+    {
+        return reverse_iterator(begin());
+    }
+
+    const_reverse_iterator rend() const noexcept
+    {
+        return const_reverse_iterator(begin());
+    }
+
+    const_reverse_iterator crend() const noexcept
+    {
+        return const_reverse_iterator(begin());
+    }
+
+    bool empty() const noexcept
+    {
+        return size_ == 0;
+    }
+
+    size_type size() const noexcept
+    {
+        return size_;
+    }
+
+    size_type max_size() const noexcept
+    {
+        return __PTRDIFF_MAX__;
+    }
+
+    size_type capacity() const noexcept
+    {
+        return capacity_;
+    }
+
+    void reserve(size_type new_cap)
+    {
+        if (new_cap > capacity_)
+            reallocate(new_cap);
+    }
+
+    void shrink_to_fit()
+    {
+        if (size_ < capacity_)
+            reallocate(size_);
+    }
+
+    void clear() noexcept
+    {
+        for (size_type i = 0; i < size_; ++i)
+            ctl::allocator_traits<Allocator>::destroy(alloc_, data_ + i);
+        size_ = 0;
+    }
+
+    iterator insert(const_iterator pos, const T& value)
+    {
+        return insert(pos, 1, value);
+    }
+
+    iterator insert(const_iterator pos, T&& value)
+    {
+        difference_type index = pos - begin();
+        if (size_ == capacity_)
+            grow();
+        iterator it = begin() + index;
+        ctl::move_backward(it, end(), end() + 1);
+        *it = ctl::move(value);
+        ++size_;
+        return it;
+    }
+
+    iterator insert(const_iterator pos, size_type count, const T& value)
+    {
+        difference_type index = pos - begin();
+        if (size_ + count > capacity_)
+            reallocate(ctl::max(size_ + count, capacity_ * 2));
+        iterator it = begin() + index;
+        ctl::move_backward(it, end(), end() + count);
+        ctl::fill_n(it, count, value);
+        size_ += count;
+        return it;
+    }
+
+    template<class InputIt>
+    iterator insert(const_iterator pos, InputIt first, InputIt last)
+    {
+        difference_type count = ctl::distance(first, last);
+        difference_type index = pos - begin();
+        if (size_ + count > capacity_)
+            reallocate(ctl::max(size_ + count, capacity_ * 2));
+        iterator it = begin() + index;
+        ctl::move_backward(it, end(), end() + count);
+        ctl::copy(first, last, it);
+        size_ += count;
+        return it;
+    }
+
+    iterator insert(const_iterator pos, std::initializer_list<T> ilist)
+    {
+        return insert(pos, ilist.begin(), ilist.end());
+    }
+
+    template<class... Args>
+    iterator emplace(const_iterator pos, Args&&... args)
+    {
+        difference_type index = pos - begin();
+        if (size_ == capacity_)
+            grow();
+        iterator it = begin() + index;
+        ctl::move_backward(it, end(), end() + 1);
+        ctl::allocator_traits<Allocator>::construct(
+          alloc_, it, ctl::forward<Args>(args)...);
+        ++size_;
+        return it;
+    }
+
+    iterator erase(const_iterator pos)
+    {
+        return erase(pos, pos + 1);
+    }
+
+    iterator erase(const_iterator first, const_iterator last)
+    {
+        difference_type index = first - begin();
+        difference_type count = last - first;
+        iterator it = begin() + index;
+        ctl::move(it + count, end(), it);
+        for (difference_type i = 0; i < count; ++i)
+            ctl::allocator_traits<Allocator>::destroy(alloc_, end() - i - 1);
+        size_ -= count;
+        return it;
+    }
+
+    void push_back(const T& value)
+    {
+        if (size_ == capacity_)
+            grow();
+        ctl::allocator_traits<Allocator>::construct(
+          alloc_, data_ + size_, value);
+        ++size_;
+    }
+
+    void push_back(T&& value)
+    {
+        if (size_ == capacity_)
+            grow();
+        ctl::allocator_traits<Allocator>::construct(
+          alloc_, data_ + size_, ctl::move(value));
+        ++size_;
+    }
+
+    template<class... Args>
+    reference emplace_back(Args&&... args)
+    {
+        if (size_ == capacity_)
+            grow();
+        ctl::allocator_traits<Allocator>::construct(
+          alloc_, data_ + size_, ctl::forward<Args>(args)...);
+        return data_[size_++];
     }
 
     void pop_back()
     {
-        if (n > 0) {
-            --n;
-            p[n].~T();
+        if (!empty()) {
+            ctl::allocator_traits<Allocator>::destroy(alloc_,
+                                                      data_ + size_ - 1);
+            --size_;
         }
     }
 
-    void resize(size_t n2)
+    void resize(size_type count)
     {
-        if (n2 > n) {
-            reserve(n2);
-            for (size_t i = n; i < n2; ++i)
-                new (&p[i]) T();
-        } else if (n2 < n) {
-            for (size_t i = n2; i < n; ++i)
-                p[i].~T();
-        }
-        n = n2;
+        resize(count, T());
     }
 
-    void swap(vector& other) noexcept
+    void resize(size_type count, const value_type& value)
     {
-        ctl::swap(n, other.n);
-        ctl::swap(c, other.c);
-        ctl::swap(p, other.p);
+        if (count > size_) {
+            if (count > capacity_)
+                reallocate(count);
+            ctl::uninitialized_fill(data_ + size_, data_ + count, value);
+        } else if (count < size_) {
+            for (size_type i = count; i < size_; ++i)
+                ctl::allocator_traits<Allocator>::destroy(alloc_, data_ + i);
+        }
+        size_ = count;
     }
+
+    void swap(vector& other) noexcept(
+      ctl::allocator_traits<Allocator>::is_always_equal::value)
+    {
+        using ctl::swap;
+        swap(alloc_, other.alloc_);
+        swap(data_, other.data_);
+        swap(size_, other.size_);
+        swap(capacity_, other.capacity_);
+    }
+
+    allocator_type get_allocator() const noexcept
+    {
+        return alloc_;
+    }
+
+  private:
+    void grow()
+    {
+        size_type c2;
+        c2 = capacity_;
+        if (c2 < 4)
+            c2 = 4;
+        c2 += c2 >> 1;
+        reallocate(c2);
+    }
+
+    void reallocate(size_type new_capacity)
+    {
+        pointer new_data =
+          ctl::allocator_traits<Allocator>::allocate(alloc_, new_capacity);
+        size_type new_size = size_ < new_capacity ? size_ : new_capacity;
+        try {
+            ctl::uninitialized_move_n(data_, new_size, new_data);
+        } catch (...) {
+            ctl::allocator_traits<Allocator>::deallocate(
+              alloc_, new_data, new_capacity);
+            throw;
+        }
+        for (size_type i = 0; i < size_; ++i)
+            ctl::allocator_traits<Allocator>::destroy(alloc_, data_ + i);
+        ctl::allocator_traits<Allocator>::deallocate(alloc_, data_, capacity_);
+        data_ = new_data;
+        size_ = new_size;
+        capacity_ = new_capacity;
+    }
+
+    Allocator alloc_;
+    pointer data_;
+    size_type size_;
+    size_type capacity_;
 };
 
+template<class T, class Alloc>
+bool
+operator==(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return lhs.size() == rhs.size() &&
+           ctl::equal(lhs.begin(), lhs.end(), rhs.begin());
+}
+
+template<class T, class Alloc>
+bool
+operator!=(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return !(lhs == rhs);
+}
+
+template<class T, class Alloc>
+bool
+operator<(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return ctl::lexicographical_compare(
+      lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+}
+
+template<class T, class Alloc>
+bool
+operator<=(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return !(rhs < lhs);
+}
+
+template<class T, class Alloc>
+bool
+operator>(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return rhs < lhs;
+}
+
+template<class T, class Alloc>
+bool
+operator>=(const vector<T, Alloc>& lhs, const vector<T, Alloc>& rhs)
+{
+    return !(lhs < rhs);
+}
+
+template<class T, class Alloc>
+void
+swap(vector<T, Alloc>& lhs,
+     vector<T, Alloc>& rhs) noexcept(noexcept(lhs.swap(rhs)))
+{
+    lhs.swap(rhs);
+}
+
+template<class InputIt,
+         class Alloc =
+           ctl::allocator<typename ctl::iterator_traits<InputIt>::value_type>>
+vector(InputIt, InputIt, Alloc = Alloc())
+  -> vector<typename ctl::iterator_traits<InputIt>::value_type, Alloc>;
+
+template<class Alloc>
+vector(size_t,
+       const typename ctl::allocator_traits<Alloc>::value_type&,
+       Alloc = Alloc())
+  -> vector<typename ctl::allocator_traits<Alloc>::value_type, Alloc>;
+
 } // namespace ctl
 
-#endif // COSMOPOLITAN_CTL_OPTIONAL_H_
+#endif // CTL_VECTOR_H_

ctl/void_t.h

@@ -0,0 +1,13 @@
+// -*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#ifndef CTL_VOID_T_H_
+#define CTL_VOID_T_H_
+
+namespace ctl {
+
+template<typename... Ts>
+using void_t = void;
+
+} // namespace ctl
+
+#endif // CTL_VOID_T_H_

libc/calls/BUILD.mk

@@ -163,8 +163,6 @@ o/$(MODE)/libc/calls/tailcontext.o: libc/calls/tailcontext.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/calls/stackjump.o: libc/calls/stackjump.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
-o/$(MODE)/libc/calls/sched_yield.o: libc/calls/sched_yield.S
-	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 
 o/$(MODE)/libc/calls/uname.o: libc/integral/normalize.inc
 

libc/calls/struct/sigset.internal.h

@@ -2,17 +2,30 @@
 #define COSMOPOLITAN_LIBC_CALLS_STRUCT_SIGSET_INTERNAL_H_
 #include "libc/calls/struct/sigset.h"
 #include "libc/mem/alloca.h"
+#include "libc/sysv/consts/sig.h"
 COSMOPOLITAN_C_START_
 
+#ifndef MODE_DBG
+/* block sigs because theoretical edge cases */
 #define BLOCK_SIGNALS  \
   do {                 \
     sigset_t _SigMask; \
   _SigMask = __sig_block()
-
 #define ALLOW_SIGNALS      \
   __sig_unblock(_SigMask); \
   }                        \
   while (0)
+#else
+/* doesn't block signals so we can get a crash
+   report, when a core runtime library crashes */
+#define BLOCK_SIGNALS  \
+  do {                 \
+    sigset_t _SigMask; \
+  sigprocmask(SIG_SETMASK, 0, &_SigMask)
+#define ALLOW_SIGNALS \
+  }                   \
+  while (0)
+#endif
 
 sigset_t __sig_block(void);
 void __sig_unblock(sigset_t);

libc/crt/crt.S

@@ -79,7 +79,7 @@ _start:
 
 //	setup stack
 	xor	%ebp,%ebp
-	and	$ape_stack_round,%rsp
+	and	$-16,%rsp
 
 #if SupportsWindows() && !IsTiny()
 //	make win32 imps crash
@@ -136,9 +136,7 @@ _start:
 	mov	x28,#0
 	mov	x29,#0
 	mov	x30,#0
-	ldr	x1,=ape_stack_round
-	and	x1,x0,x1
-	mov	sp,x1
+	and	sp,x0,#-16
 
 //	second arg shall be struct Syslib passed by ape-m1.c
 //	used to talk to apple's authoritarian libraries

libc/integral/normalize.inc

@@ -4,7 +4,7 @@
 
 #define __COSMOPOLITAN_MAJOR__ 3
 #define __COSMOPOLITAN_MINOR__ 5
-#define __COSMOPOLITAN_PATCH__ 0
+#define __COSMOPOLITAN_PATCH__ 2
 #define __COSMOPOLITAN__                                                   \
   (100000000 * __COSMOPOLITAN_MAJOR__ + 1000000 * __COSMOPOLITAN_MINOR__ + \
    __COSMOPOLITAN_PATCH__)

libc/intrin/BUILD.mk

@@ -135,6 +135,10 @@ o/$(MODE)/libc/intrin/kweekdayname.o: libc/intrin/kweekdayname.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/intrin/kweekdaynameshort.o: libc/intrin/kweekdaynameshort.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
+o/$(MODE)/libc/intrin/sched_yield.o: libc/intrin/sched_yield.S
+	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
+o/$(MODE)/libc/intrin/dsohandle.o: libc/intrin/dsohandle.S
+	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 
 LIBC_INTRIN_LIBS = $(foreach x,$(LIBC_INTRIN_ARTIFACTS),$($(x)))
 LIBC_INTRIN_HDRS = $(foreach x,$(LIBC_INTRIN_ARTIFACTS),$($(x)_HDRS))

libc/intrin/fds_lock_obj.c

@@ -19,4 +19,4 @@
 #include "libc/calls/state.internal.h"
 #include "libc/thread/thread.h"
 
-pthread_mutex_t __fds_lock_obj = {._type = PTHREAD_MUTEX_RECURSIVE};
+pthread_mutex_t __fds_lock_obj = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;

libc/intrin/maps.c

@@ -20,14 +20,10 @@
 #include "ape/sections.internal.h"
 #include "libc/dce.h"
 #include "libc/intrin/dll.h"
-#include "libc/intrin/kprintf.h"
-#include "libc/intrin/strace.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
 #include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/prot.h"
-#include "libc/thread/thread.h"
-#include "libc/thread/tls.h"
 
 #ifdef __x86_64__
 __static_yoink("_init_maps");
@@ -73,7 +69,7 @@ privileged void __maps_lock(void) {
   if (!__tls_enabled)
     return;
   tib = __get_tls_privileged();
-  if (tib->tib_flags & TIB_FLAG_MAPLOCK)
+  if (tib->tib_relock_maps++)
     return;
   while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
 #if defined(__GNUC__) && defined(__aarch64__)
@@ -82,14 +78,15 @@ privileged void __maps_lock(void) {
     __asm__ volatile("pause");
 #endif
   }
-  tib->tib_flags |= TIB_FLAG_MAPLOCK;
 }
 
 privileged void __maps_unlock(void) {
   struct CosmoTib *tib;
-  atomic_store_explicit(&__maps.lock, 0, memory_order_release);
-  if (__tls_enabled) {
-    tib = __get_tls_privileged();
-    tib->tib_flags &= ~TIB_FLAG_MAPLOCK;
-  }
+  if (!__threaded)
+    return;
+  if (!__tls_enabled)
+    return;
+  tib = __get_tls_privileged();
+  if (!--tib->tib_relock_maps)
+    atomic_store_explicit(&__maps.lock, 0, memory_order_release);
 }

libc/intrin/maps.h

@@ -8,24 +8,28 @@ COSMOPOLITAN_C_START_
 #define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
 
 struct Map {
-  _Atomic(struct Map *) next; /* for __maps.maps */
-  char *addr;                 /* granule aligned */
-  size_t size;                /* must be nonzero */
-  struct Dll elem;            /* for __maps.free */
-  int64_t off;                /* -1 if anonymous */
-  int prot;                   /* memory protects */
-  int flags;                  /* memory map flag */
-  bool iscow;                 /* windows nt only */
-  bool readonlyfile;          /* windows nt only */
-  intptr_t h;                 /* windows nt only */
+  struct Map *next;  /* for __maps.maps */
+  char *addr;        /* granule aligned */
+  size_t size;       /* must be nonzero */
+  struct Dll elem;   /* for __maps.free */
+  int64_t off;       /* -1 if anonymous */
+  int prot;          /* memory protects */
+  int flags;         /* memory map flag */
+  bool iscow;        /* windows nt only */
+  bool readonlyfile; /* windows nt only */
+  unsigned visited;  /* used for checks */
+  intptr_t h;        /* windows nt only */
 };
 
 struct Maps {
+  unsigned mono;
   atomic_int lock;
-  _Atomic(struct Map *) maps;
+  struct Map *maps;
   struct Dll *free;
   struct Map stack;
   struct Dll *used;
+  size_t count;
+  size_t pages;
 };
 
 struct AddrSize {
@@ -35,13 +39,15 @@ struct AddrSize {
 
 extern struct Maps __maps;
 
-int maps_check(void);
+void *randaddr(void);
 void __maps_init(void);
 void __maps_lock(void);
+void __maps_check(void);
 void __maps_unlock(void);
 struct Map *__maps_alloc(void);
 void __maps_free(struct Map *);
 void __maps_insert(struct Map *);
+int __munmap(char *, size_t, bool);
 void *__mmap(char *, size_t, int, int, int, int64_t);
 struct AddrSize __get_main_stack(void);
 

libc/intrin/mmap.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "ape/sections.internal.h"
 #include "libc/atomic.h"
+#include "libc/calls/blockcancel.internal.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/struct/sigset.internal.h"
@@ -25,9 +26,11 @@
 #include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/describebacktrace.internal.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
@@ -44,6 +47,7 @@
 #include "libc/sysv/errfuns.h"
 #include "libc/thread/thread.h"
 
+#define MMDEBUG 0               // this code is too slow for openbsd/windows
 #define WINBASE 0x100080040000  // TODO: Can we support Windows Vista again?
 #define WINMAXX 0x200080000000
 
@@ -51,21 +55,89 @@
 
 #define PGUP(x) (((x) + granularity - 1) & -granularity)
 
+#if MMDEBUG
+#define ASSERT(x) (void)0
+#else
+#define ASSERT(x)                                                         \
+  do {                                                                    \
+    if (!(x)) {                                                           \
+      char bt[160];                                                       \
+      struct StackFrame *bp = __builtin_frame_address(0);                 \
+      kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \
+      kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp));                   \
+      __print_maps();                                                     \
+      _Exit(99);                                                          \
+    }                                                                     \
+  } while (0)
+#endif
+
 static atomic_ulong rollo;
 
+static bool overlaps_existing_map(const char *addr, size_t size) {
+  int granularity = __granularity();
+  for (struct Map *map = __maps.maps; map; map = map->next)
+    if (MAX(addr, map->addr) <
+        MIN(addr + PGUP(size), map->addr + PGUP(map->size)))
+      return true;
+  return false;
+}
+
+void __maps_check(void) {
+#if MMDEBUG
+  size_t maps = 0;
+  size_t pages = 0;
+  int granularity = getauxval(AT_PAGESZ);
+  unsigned id = __maps.mono++;
+  for (struct Map *map = __maps.maps; map; map = map->next) {
+    ASSERT(map->addr != MAP_FAILED);
+    ASSERT(map->visited != id);
+    ASSERT(map->size);
+    map->visited = id;
+    pages += PGUP(map->size) / granularity;
+    maps += 1;
+  }
+  ASSERT(maps = __maps.count);
+  ASSERT(pages == __maps.pages);
+  for (struct Dll *e = dll_first(__maps.used); e;
+       e = dll_next(__maps.used, e)) {
+    ASSERT(MAP_CONTAINER(e)->visited == id);
+    --maps;
+  }
+  ASSERT(maps == 0);
+  for (struct Map *m1 = __maps.maps; m1; m1 = m1->next)
+    for (struct Map *m2 = m1->next; m2; m2 = m2->next)
+      ASSERT(MAX(m1->addr, m2->addr) >=
+             MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size)));
+#endif
+}
+
 void __maps_free(struct Map *map) {
+  map->next = 0;
+  map->size = 0;
+  map->addr = MAP_FAILED;
+  ASSERT(dll_is_alone(&map->elem));
   dll_make_last(&__maps.free, &map->elem);
 }
 
 void __maps_insert(struct Map *map) {
   struct Map *last = __maps.maps;
-  if (last &&                                  //
+  int granularity = getauxval(AT_PAGESZ);
+  __maps.pages += PGUP(map->size) / granularity;
+  if (last && !IsWindows() &&                  //
       map->addr == last->addr + last->size &&  //
+      (map->flags & MAP_ANONYMOUS) &&          //
       map->flags == last->flags &&             //
-      map->prot == last->prot &&               //
-      map->off == last->off &&                 //
-      map->h == last->h &&                     //
-      map->off == -1) {
+      map->prot == last->prot) {
+    last->size += map->size;
+    dll_remove(&__maps.used, &last->elem);
+    dll_make_first(&__maps.used, &last->elem);
+    __maps_free(map);
+  } else if (last && !IsWindows() &&                 //
+             map->addr + map->size == last->addr &&  //
+             (map->flags & MAP_ANONYMOUS) &&         //
+             map->flags == last->flags &&            //
+             map->prot == last->prot) {
+    last->addr -= map->size;
     last->size += map->size;
     dll_remove(&__maps.used, &last->elem);
     dll_make_first(&__maps.used, &last->elem);
@@ -74,7 +146,9 @@ void __maps_insert(struct Map *map) {
     dll_make_first(&__maps.used, &map->elem);
     map->next = __maps.maps;
     __maps.maps = map;
+    ++__maps.count;
   }
+  __maps_check();
 }
 
 struct Map *__maps_alloc(void) {
@@ -104,21 +178,10 @@ struct Map *__maps_alloc(void) {
   return map;
 }
 
-static bool __overlaps_existing_map(const char *addr, size_t size) {
-  for (struct Map *map = __maps.maps; map; map = map->next) {
-    if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size))
-      return true;
-  }
-  return false;
-}
-
-static int __munmap_chunk(void *addr, size_t size) {
-  return sys_munmap(addr, size);
-}
-
-static int __munmap(char *addr, size_t size, bool untrack_only) {
+int __munmap(char *addr, size_t size, bool untrack_only) {
 
   // validate arguments
+  int pagesz = getauxval(AT_PAGESZ);
   int granularity = __granularity();
   if (((uintptr_t)addr & (granularity - 1)) ||  //
       !size || (uintptr_t)addr + size < size)
@@ -127,10 +190,9 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
   // untrack and delete mapping
   int rc = 0;
   __maps_lock();
-  // we can't call strace, kprintf, or nothing
 StartOver:;
   struct Map *map = __maps.maps;
-  _Atomic(struct Map *) *prev = &__maps.maps;
+  struct Map **prev = &__maps.maps;
   while (map) {
     char *map_addr = map->addr;
     size_t map_size = map->size;
@@ -141,13 +203,17 @@ StartOver:;
         // remove mapping completely
         dll_remove(&__maps.used, &map->elem);
         *prev = next;
-        map->size = 0;
-        map->addr = MAP_FAILED;
+        __maps.pages -= (map_size + pagesz - 1) / pagesz;
+        __maps.count -= 1;
         if (untrack_only) {
           __maps_free(map);
+          __maps_check();
         } else {
+          __maps_unlock();
           if (!IsWindows()) {
-            if (__munmap_chunk(map_addr, map_size))
+            ASSERT(addr <= map_addr);
+            ASSERT(map_addr + PGUP(map_size) <= addr + PGUP(size));
+            if (sys_munmap(map_addr, map_size))
               rc = -1;
           } else {
             if (!UnmapViewOfFile(map_addr))
@@ -155,7 +221,9 @@ StartOver:;
             if (!CloseHandle(map->h))
               rc = -1;
           }
+          __maps_lock();
           __maps_free(map);
+          __maps_check();
           goto StartOver;
         }
         map = next;
@@ -167,15 +235,24 @@ StartOver:;
         rc = einval();
       } else if (addr <= map_addr) {
         // shave off lefthand side of mapping
-        size_t left = addr + size - map_addr;
-        size_t right = map_addr + map_size - (addr + size);
+        ASSERT(addr + size < map_addr + PGUP(map_size));
+        size_t left = PGUP(addr + size - map_addr);
+        size_t right = map_size - left;
+        ASSERT(right > 0);
+        ASSERT(left > 0);
         map->addr += left;
         map->size = right;
-        if (map->off != -1)
+        if (!(map->flags & MAP_ANONYMOUS))
           map->off += left;
+        __maps.pages -= (left + pagesz - 1) / pagesz;
+        __maps_check();
         if (!untrack_only) {
-          if (__munmap_chunk(map_addr, left) == -1)
+          __maps_unlock();
+          ASSERT(addr <= map_addr);
+          ASSERT(map_addr + PGUP(left) <= addr + PGUP(size));
+          if (sys_munmap(map_addr, left) == -1)
             rc = -1;
+          __maps_lock();
           goto StartOver;
         }
       } else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
@@ -183,9 +260,14 @@ StartOver:;
         size_t left = addr - map_addr;
         size_t right = map_addr + map_size - addr;
         map->size = left;
+        __maps.pages -= (right + pagesz - 1) / pagesz;
+        __maps_check();
         if (!untrack_only) {
-          if (__munmap_chunk(addr, right) == -1)
+          __maps_unlock();
+          ASSERT(PGUP(right) <= PGUP(size));
+          if (sys_munmap(addr, right) == -1)
             rc = -1;
+          __maps_lock();
           goto StartOver;
         }
       } else {
@@ -203,13 +285,18 @@ StartOver:;
           leftmap->flags = map->flags;
           map->addr += left + middle;
           map->size = right;
-          if (map->off != -1)
+          if (!(map->flags & MAP_ANONYMOUS))
             map->off += left + middle;
           dll_make_first(&__maps.used, &leftmap->elem);
           *prev = leftmap;
+          __maps.pages -= (middle + pagesz - 1) / pagesz;
+          __maps.count += 1;
+          __maps_check();
           if (!untrack_only) {
-            if (__munmap_chunk(addr, size) == -1)
+            __maps_unlock();
+            if (sys_munmap(addr, size) == -1)
               rc = -1;
+            __maps_lock();
             goto StartOver;
           }
         } else {
@@ -241,7 +328,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
       sysflags |= MAP_FIXED_NOREPLACE_linux;
     } else if (IsFreebsd() || IsNetbsd()) {
       sysflags |= MAP_FIXED;
-      if (__overlaps_existing_map(addr, size))
+      if (overlaps_existing_map(addr, size))
         return (void *)eexist();
     } else {
       noreplace = true;
@@ -258,6 +345,11 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
   if (!map)
     return MAP_FAILED;
 
+  // remove mapping we blew away
+  if (IsWindows() && should_untrack)
+    if (__munmap(addr, size, false))
+      return MAP_FAILED;
+
   // obtain mapping from operating system
   int olderr = errno;
   struct DirectMap res;
@@ -268,9 +360,7 @@ TryAgain:
       if (noreplace) {
         errno = EEXIST;
       } else if (should_untrack) {
-        sys_munmap(res.addr, size);
-        errno = olderr;
-        goto TryAgain;
+        errno = ENOMEM;
       } else {
         addr += granularity;
         errno = olderr;
@@ -287,7 +377,12 @@ TryAgain:
   // we assume non-linux gives us addr if it's free
   // that's what linux (e.g. rhel7) did before noreplace
   if (noreplace && res.addr != addr) {
-    sys_munmap(res.addr, size);
+    if (!IsWindows()) {
+      sys_munmap(res.addr, size);
+    } else {
+      UnmapViewOfFile(res.addr);
+      CloseHandle(res.maphandle);
+    }
     __maps_lock();
     __maps_free(map);
     __maps_unlock();
@@ -295,8 +390,8 @@ TryAgain:
   }
 
   // untrack mapping we blew away
-  if (should_untrack)
-    __munmap(addr, size, true);
+  if (!IsWindows() && should_untrack)
+    __munmap(res.addr, size, true);
 
   // track Map object
   map->addr = res.addr;
@@ -344,9 +439,9 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
   if (size <= granularity || size > 100 * 1024 * 1024)
     return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
 
-  // so we create an separate map for each granule in the mapping
+  // so we create a separate map for each granule in the mapping
   if (!(flags & MAP_FIXED)) {
-    while (__overlaps_existing_map(addr, size)) {
+    while (overlaps_existing_map(addr, size)) {
       if (flags & MAP_FIXED_NOREPLACE)
         return (void *)eexist();
       addr += granularity;
@@ -405,7 +500,11 @@ void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
 
 void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
   void *res;
+  BLOCK_SIGNALS;
+  BLOCK_CANCELATION;
   res = __mmap(addr, size, prot, flags, fd, off);
+  ALLOW_CANCELATION;
+  ALLOW_SIGNALS;
   STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m", addr, size,
          DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res);
   return res;
@@ -413,7 +512,11 @@ void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
 
 int munmap(void *addr, size_t size) {
   int rc;
+  BLOCK_SIGNALS;
+  BLOCK_CANCELATION;
   rc = __munmap(addr, size, false);
+  ALLOW_CANCELATION;
+  ALLOW_SIGNALS;
   STRACE("munmap(%p, %'zu) → %d% m", addr, size, rc);
   return rc;
 }

libc/intrin/mprotect.c

@@ -30,6 +30,7 @@
 #include "libc/runtime/runtime.h"
 #include "libc/stdio/sysparam.h"
 #include "libc/sysv/consts/auxv.h"
+#include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
 #include "libc/sysv/errfuns.h"
 
@@ -67,7 +68,7 @@ int __mprotect(char *addr, size_t size, int prot) {
   __maps_lock();
   bool found = false;
   struct Map *map = __maps.maps;
-  _Atomic(struct Map *) *prev = &__maps.maps;
+  struct Map **prev = &__maps.maps;
   while (map) {
     char *map_addr = map->addr;
     size_t map_size = map->size;
@@ -91,8 +92,8 @@ int __mprotect(char *addr, size_t size, int prot) {
           rc = -1;
       } else if (addr <= map_addr) {
         // cleave lefthand side of mapping
-        size_t left = addr + size - map_addr;
-        size_t right = map_addr + map_size - (addr + size);
+        size_t left = PGUP(addr + size - map_addr);
+        size_t right = map_size - left;
         struct Map *leftmap;
         if ((leftmap = __maps_alloc())) {
           if (!__mprotect_chunk(map_addr, left, prot, false)) {
@@ -104,10 +105,12 @@ int __mprotect(char *addr, size_t size, int prot) {
             leftmap->flags = map->flags;
             map->addr += left;
             map->size = right;
-            if (map->off != -1)
+            if (!(map->flags & MAP_ANONYMOUS))
               map->off += left;
             dll_make_first(&__maps.used, &leftmap->elem);
             *prev = leftmap;
+            __maps.count += 1;
+            __maps_check();
           } else {
             __maps_free(leftmap);
             rc = -1;
@@ -131,10 +134,12 @@ int __mprotect(char *addr, size_t size, int prot) {
             map->addr += left;
             map->size = right;
             map->prot = prot;
-            if (map->off != -1)
+            if (!(map->flags & MAP_ANONYMOUS))
               map->off += left;
             dll_make_first(&__maps.used, &leftmap->elem);
             *prev = leftmap;
+            __maps.count += 1;
+            __maps_check();
           } else {
             __maps_free(leftmap);
             rc = -1;
@@ -161,16 +166,18 @@ int __mprotect(char *addr, size_t size, int prot) {
               midlmap->next = map;
               midlmap->addr = map_addr + left;
               midlmap->size = middle;
-              midlmap->off = map->off == -1 ? -1 : map->off + left;
+              midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left;
               midlmap->prot = prot;
               midlmap->flags = map->flags;
               map->addr += left + middle;
               map->size = right;
-              if (map->off != -1)
+              if (!(map->flags & MAP_ANONYMOUS))
                 map->off += left + middle;
               dll_make_first(&__maps.used, &leftmap->elem);
               dll_make_first(&__maps.used, &midlmap->elem);
               *prev = leftmap;
+              __maps.count += 2;
+              __maps_check();
             } else {
               __maps_free(midlmap);
               __maps_free(leftmap);

libc/intrin/msync-nt.c

@@ -34,6 +34,7 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
     return einval();
 
   int rc = 0;
+  __maps_lock();
   for (struct Map *map = __maps.maps; map; map = map->next) {
     char *beg = MAX(addr, map->addr);
     char *end = MIN(addr + size, map->addr + map->size);
@@ -42,6 +43,7 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
         rc = -1;
     // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
   }
+  __maps_unlock();
 
   return rc;
 }

libc/intrin/printmaps.c

@@ -24,15 +24,20 @@
 #include "libc/macros.internal.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/auxv.h"
 
 /**
  * Prints memory mappings.
  */
 void __print_maps(void) {
-  int limit = 10;
+  int limit = 13;
   long maptally = 0;
   char mappingbuf[8], sb[16];
-  for (struct Map *map = __maps.maps; map; map = map->next) {
+  __maps_lock();
+  struct Dll *e, *e2;
+  for (e = dll_first(__maps.used); e; e = e2) {
+    e2 = dll_next(__maps.used, e);
+    struct Map *map = MAP_CONTAINER(e);
     maptally += map->size;
     kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
             (DescribeMapping)(mappingbuf, map->prot, map->flags));
@@ -45,11 +50,10 @@ void __print_maps(void) {
     if (map->readonlyfile)
       kprintf(" readonlyfile");
     kprintf("\n");
-    if (!--limit) {
-      kprintf("...\n");
+    if (!--limit)
       break;
-    }
   }
-  sizefmt(sb, maptally, 1024);
-  kprintf("# %!sb mapped memory\n", sb);
+  kprintf("# %'zu bytes in %'zu mappings\n",
+          __maps.pages * getauxval(AT_PAGESZ), __maps.count);
+  __maps_unlock();
 }

libc/intrin/pthread_mutex_init.c

@@ -35,9 +35,6 @@
  */
 int pthread_mutex_init(pthread_mutex_t *mutex,
                        const pthread_mutexattr_t *attr) {
-  *mutex = (pthread_mutex_t){
-      ._type = attr ? attr->_type : 0,
-      ._pshared = attr ? attr->_pshared : 0,
-  };
+  *mutex = (pthread_mutex_t){._word = attr ? attr->_word : 0};
   return 0;
 }

libc/intrin/pthread_mutex_lock.c

@@ -23,6 +23,7 @@
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/runtime/internal.h"
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 #include "third_party/nsync/mu.h"
@@ -65,49 +66,65 @@
  * @vforksafe
  */
 errno_t pthread_mutex_lock(pthread_mutex_t *mutex) {
-  int t;
+  int me;
+  uint64_t word, lock;
 
   LOCKTRACE("pthread_mutex_lock(%t)", mutex);
 
-  if (__vforked) {
+  if (__vforked)
     return 0;
-  }
 
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL &&        //
-      mutex->_pshared == PTHREAD_PROCESS_PRIVATE &&  //
+  // get current state of lock
+  word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
+
+  // use fancy nsync mutex if possible
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&        //
+      MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&  //
       _weaken(nsync_mu_lock)) {
     _weaken(nsync_mu_lock)((nsync_mu *)mutex);
     return 0;
   }
 
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL) {
-    while (atomic_exchange_explicit(&mutex->_lock, 1, memory_order_acquire)) {
+  // implement barebones normal mutexes
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
+    for (;;) {
+      word = MUTEX_UNLOCK(word);
+      lock = MUTEX_LOCK(word);
+      if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                                memory_order_acquire,
+                                                memory_order_relaxed))
+        return 0;
       pthread_pause_np();
     }
-    return 0;
   }
 
-  t = gettid();
-  if (mutex->_owner == t) {
-    if (mutex->_type != PTHREAD_MUTEX_ERRORCHECK) {
-      if (mutex->_depth < 63) {
-        ++mutex->_depth;
-        return 0;
+  // implement recursive mutexes
+  me = gettid();
+  for (;;) {
+    if (MUTEX_OWNER(word) == me) {
+      if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
+        if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
+          if (atomic_compare_exchange_weak_explicit(
+                  &mutex->_word, &word, MUTEX_INC_DEPTH(word),
+                  memory_order_relaxed, memory_order_relaxed))
+            return 0;
+          continue;
+        } else {
+          return EAGAIN;
+        }
       } else {
-        return EAGAIN;
+        return EDEADLK;
       }
-    } else {
-      return EDEADLK;
     }
-  }
-
-  while (atomic_exchange_explicit(&mutex->_lock, 1, memory_order_acquire)) {
+    word = MUTEX_UNLOCK(word);
+    lock = MUTEX_LOCK(word);
+    lock = MUTEX_SET_OWNER(lock, me);
+    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                              memory_order_acquire,
+                                              memory_order_relaxed)) {
+      mutex->_pid = __pid;
+      return 0;
+    }
     pthread_pause_np();
   }
-
-  mutex->_depth = 0;
-  mutex->_owner = t;
-  mutex->_pid = __pid;
-
-  return 0;
 }

libc/intrin/pthread_mutex_trylock.c

@@ -21,6 +21,7 @@
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/weaken.h"
 #include "libc/runtime/internal.h"
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -38,11 +39,15 @@
  *     current thread already holds this mutex
  */
 errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) {
-  int t;
+  int me;
+  uint64_t word, lock;
+
+  // get current state of lock
+  word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
 
   // delegate to *NSYNC if possible
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL &&
-      mutex->_pshared == PTHREAD_PROCESS_PRIVATE &&  //
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&
+      MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&  //
       _weaken(nsync_mu_trylock)) {
     if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex)) {
       return 0;
@@ -52,36 +57,43 @@ errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) {
   }
 
   // handle normal mutexes
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL) {
-    if (!atomic_exchange_explicit(&mutex->_lock, 1, memory_order_acquire)) {
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
+    word = MUTEX_UNLOCK(word);
+    lock = MUTEX_LOCK(word);
+    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                              memory_order_acquire,
+                                              memory_order_relaxed))
       return 0;
-    } else {
-      return EBUSY;
-    }
-  }
-
-  // handle recursive and error check mutexes
-  t = gettid();
-  if (mutex->_owner == t) {
-    if (mutex->_type != PTHREAD_MUTEX_ERRORCHECK) {
-      if (mutex->_depth < 63) {
-        ++mutex->_depth;
-        return 0;
-      } else {
-        return EAGAIN;
-      }
-    } else {
-      return EDEADLK;
-    }
-  }
-
-  if (atomic_exchange_explicit(&mutex->_lock, 1, memory_order_acquire)) {
     return EBUSY;
   }
 
-  mutex->_depth = 0;
-  mutex->_owner = t;
-  mutex->_pid = __pid;
-
-  return 0;
+  // handle recursive and error check mutexes
+  me = gettid();
+  for (;;) {
+    if (MUTEX_OWNER(word) == me) {
+      if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
+        if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
+          if (atomic_compare_exchange_weak_explicit(
+                  &mutex->_word, &word, MUTEX_INC_DEPTH(word),
+                  memory_order_relaxed, memory_order_relaxed))
+            return 0;
+          continue;
+        } else {
+          return EAGAIN;
+        }
+      } else {
+        return EDEADLK;
+      }
+    }
+    word = MUTEX_UNLOCK(word);
+    lock = MUTEX_LOCK(word);
+    lock = MUTEX_SET_OWNER(lock, me);
+    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                              memory_order_acquire,
+                                              memory_order_relaxed)) {
+      mutex->_pid = __pid;
+      return 0;
+    }
+    return EBUSY;
+  }
 }

libc/intrin/pthread_mutex_unlock.c

@@ -22,6 +22,7 @@
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/runtime/internal.h"
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -35,38 +36,52 @@
  * @vforksafe
  */
 errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) {
-  int t;
+  int me;
+  uint64_t word, lock;
 
   LOCKTRACE("pthread_mutex_unlock(%t)", mutex);
 
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL &&        //
-      mutex->_pshared == PTHREAD_PROCESS_PRIVATE &&  //
+  // get current state of lock
+  word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
+
+  // use fancy nsync mutex if possible
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&        //
+      MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&  //
       _weaken(nsync_mu_unlock)) {
     _weaken(nsync_mu_unlock)((nsync_mu *)mutex);
     return 0;
   }
 
-  if (mutex->_type == PTHREAD_MUTEX_NORMAL) {
-    atomic_store_explicit(&mutex->_lock, 0, memory_order_release);
+  // implement barebones normal mutexes
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
+    lock = MUTEX_UNLOCK(word);
+    atomic_store_explicit(&mutex->_word, lock, memory_order_release);
     return 0;
   }
 
-  t = gettid();
+  // implement recursive mutex unlocking
+  me = gettid();
+  for (;;) {
 
-  // we allow unlocking an initialized lock that wasn't locked, but we
-  // don't allow unlocking a lock held by another thread, or unlocking
-  // recursive locks from a forked child, since it should be re-init'd
-  if (mutex->_owner && (mutex->_owner != t || mutex->_pid != __pid)) {
-    return EPERM;
+    // we allow unlocking an initialized lock that wasn't locked, but we
+    // don't allow unlocking a lock held by another thread, or unlocking
+    // recursive locks from a forked child, since it should be re-init'd
+    if (MUTEX_OWNER(word) && (MUTEX_OWNER(word) != me || mutex->_pid != __pid))
+      return EPERM;
+
+    // check if this is a nested lock with signal safety
+    if (MUTEX_DEPTH(word)) {
+      if (atomic_compare_exchange_weak_explicit(
+              &mutex->_word, &word, MUTEX_DEC_DEPTH(word), memory_order_relaxed,
+              memory_order_relaxed))
+        return 0;
+      continue;
+    }
+
+    // actually unlock the mutex
+    if (atomic_compare_exchange_weak_explicit(
+            &mutex->_word, &word, MUTEX_UNLOCK(word), memory_order_release,
+            memory_order_relaxed))
+      return 0;
   }
-
-  if (mutex->_depth) {
-    --mutex->_depth;
-    return 0;
-  }
-
-  mutex->_owner = 0;
-  atomic_store_explicit(&mutex->_lock, 0, memory_order_release);
-
-  return 0;
 }

libc/intrin/pthread_mutexattr_getpshared.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -28,6 +29,6 @@
  */
 errno_t pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr,
                                      int *pshared) {
-  *pshared = attr->_pshared;
+  *pshared = MUTEX_PSHARED(attr->_word);
   return 0;
 }

libc/intrin/pthread_mutexattr_gettype.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -28,6 +29,6 @@
  * @return 0 on success, or error on failure
  */
 errno_t pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) {
-  *type = attr->_type;
+  *type = MUTEX_TYPE(attr->_word);
   return 0;
 }

libc/intrin/pthread_mutexattr_setpshared.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -32,7 +33,7 @@ errno_t pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) {
   switch (pshared) {
     case PTHREAD_PROCESS_SHARED:
     case PTHREAD_PROCESS_PRIVATE:
-      attr->_pshared = pshared;
+      attr->_word = MUTEX_SET_PSHARED(attr->_word, pshared);
       return 0;
     default:
       return EINVAL;

libc/intrin/pthread_mutexattr_settype.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
+#include "libc/thread/lock.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -35,7 +36,7 @@ errno_t pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) {
     case PTHREAD_MUTEX_NORMAL:
     case PTHREAD_MUTEX_RECURSIVE:
     case PTHREAD_MUTEX_ERRORCHECK:
-      attr->_type = type;
+      attr->_word = MUTEX_SET_TYPE(attr->_word, type);
       return 0;
     default:
       return EINVAL;

libc/intrin/randaddr.c

@@ -0,0 +1,25 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+
+void *randaddr(void) {
+  static unsigned long lcg = 1;
+  lcg *= 6364136223846793005;
+  lcg += 1442695040888963407;
+  return (void *)(lcg >> 48 << 28);
+}

libc/intrin/sched_yield.S

@@ -19,13 +19,12 @@
 #include "libc/dce.h"
 #include "libc/sysv/consts/nr.h"
 #include "libc/macros.internal.h"
+.privileged
 
 //	Relinquishes scheduled quantum.
 //
 //	@return	0 on success, or -1 w/ errno
-	.ftrace1
 sched_yield:
-	.ftrace2
 #ifdef __x86_64__
 	push	%rbp
 	mov	%rsp,%rbp

libc/proc/fork-nt.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "ape/sections.internal.h"
 #include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/sig.internal.h"
 #include "libc/calls/state.internal.h"
@@ -66,6 +67,7 @@
 #ifdef __x86_64__
 
 extern long __klog_handle;
+extern atomic_uint free_waiters_mu;
 void WipeKeystrokes(void);
 __msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId;
 
@@ -93,11 +95,9 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n,
                                                      struct NtOverlapped *)) {
   size_t i;
   uint32_t x;
-  for (i = 0; i < n; i += x) {
-    if (!f(h, p + i, n - i, &x, NULL)) {
+  for (i = 0; i < n; i += x)
+    if (!f(h, p + i, n - i, &x, NULL))
       return __winerr();
-    }
-  }
   return i;
 }
 
@@ -151,9 +151,8 @@ static textwindows dontinline void ReadOrDie(int64_t h, void *buf, size_t n) {
 static textwindows int64_t MapOrDie(uint32_t prot, uint64_t size) {
   int64_t h;
   for (;;) {
-    if ((h = CreateFileMapping(-1, 0, prot, size >> 32, size, 0))) {
+    if ((h = CreateFileMapping(-1, 0, prot, size >> 32, size, 0)))
       return h;
-    }
     if (GetLastError() == kNtErrorAccessDenied) {
       switch (prot) {
         case kNtPageExecuteWritecopy:
@@ -262,21 +261,23 @@ textwindows void WinMainForked(void) {
   __maps.free = 0;
   __maps.used = 0;
   __maps.maps = maps;
+  __maps.count = 0;
+  __maps.pages = 0;
   dll_init(&__maps.stack.elem);
   dll_make_first(&__maps.used, &__maps.stack.elem);
   for (struct Map *map = maps; map; map = map->next) {
+    __maps.count += 1;
+    __maps.pages += (map->size + 4095) / 4096;
     dll_make_last(&__maps.used, &map->elem);
     if (!VirtualProtect(map->addr, map->size, __prot2nt(map->prot, map->iscow),
-                        &oldprot)) {
+                        &oldprot))
       AbortFork("VirtualProtect");
-    }
   }
   __maps_init();
 
   // mitosis complete
-  if (!CloseHandle(reader)) {
+  if (!CloseHandle(reader))
     AbortFork("CloseHandle");
-  }
 
   // rewrap the stdin named pipe hack
   // since the handles closed on fork
@@ -422,9 +423,8 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
     atomic_store_explicit(&_pthread_static.ptid, GetCurrentThreadId(),
                           memory_order_release);
   }
-  if (rc == -1) {
+  if (rc == -1)
     dll_make_first(&__proc.free, &proc->elem);
-  }
   ftrace_enabled(+1);
   strace_enabled(+1);
   return rc;

libc/proc/fork.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/struct/sigset.h"
@@ -44,15 +45,20 @@
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/tls.h"
 
+extern atomic_uint free_waiters_mu;
+
 static void _onfork_prepare(void) {
   if (_weaken(_pthread_onfork_prepare))
     _weaken(_pthread_onfork_prepare)();
   _pthread_lock();
   __maps_lock();
   __fds_lock();
+  while (atomic_exchange_explicit(&free_waiters_mu, 1, memory_order_acquire)) {
+  }
 }
 
 static void _onfork_parent(void) {
+  atomic_store_explicit(&free_waiters_mu, 0, memory_order_release);
   __fds_unlock();
   __maps_unlock();
   _pthread_unlock();
@@ -65,12 +71,13 @@ static void _onfork_child(void) {
   pthread_mutexattr_init(&attr);
   pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
   pthread_mutex_init(&__fds_lock_obj, &attr);
+  atomic_store_explicit(&free_waiters_mu, 0, memory_order_relaxed);
   pthread_mutexattr_destroy(&attr);
   _pthread_init();
-  __maps_unlock();
-  if (_weaken(_pthread_onfork_child)) {
+  atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed);
+  atomic_store_explicit(&__get_tls()->tib_relock_maps, 0, memory_order_relaxed);
+  if (_weaken(_pthread_onfork_child))
     _weaken(_pthread_onfork_child)();
-  }
 }
 
 int _fork(uint32_t dwCreationFlags) {

libc/runtime/BUILD.mk

@@ -107,8 +107,6 @@ o/$(MODE)/libc/runtime/clone-linux.o: libc/runtime/clone-linux.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/runtime/ftrace-hook.o: libc/runtime/ftrace-hook.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
-o/$(MODE)/libc/runtime/dsohandle.o: libc/runtime/dsohandle.S
-	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/runtime/zipos.o: libc/runtime/zipos.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/runtime/switchstacks.o: libc/runtime/switchstacks.S

libc/runtime/cxa_thread_atexit.c

@@ -32,6 +32,16 @@ struct Dtor {
 
 static _Thread_local struct Dtor *__cxa_thread_atexit_list;
 
+static void _pthread_unwind(struct CosmoTib *tib) {
+  struct PosixThread *pt;
+  struct _pthread_cleanup_buffer *cb;
+  pt = (struct PosixThread *)tib->tib_pthread;
+  while ((cb = pt->pt_cleanup)) {
+    pt->pt_cleanup = cb->__prev;
+    cb->__routine(cb->__arg);
+  }
+}
+
 static void __cxa_thread_unkey(struct CosmoTib *tib) {
   void *val;
   int i, j, gotsome;
@@ -67,16 +77,18 @@ static void _pthread_ungarbage(struct CosmoTib *tib) {
 
 void __cxa_thread_finalize(void) {
   struct Dtor *dtor;
+  struct CosmoTib *tib;
+  tib = __get_tls();
+  _pthread_unwind(tib);
+  if (tib->tib_nsync)
+    _weaken(nsync_waiter_destroy)(tib->tib_nsync);
+  __cxa_thread_unkey(tib);
+  _pthread_ungarbage(tib);
   while ((dtor = __cxa_thread_atexit_list)) {
     __cxa_thread_atexit_list = dtor->next;
     ((void (*)(void *))dtor->fun)(dtor->arg);
     _weaken(free)(dtor);
   }
-  struct CosmoTib *tib = __get_tls();
-  __cxa_thread_unkey(tib);
-  if (tib->tib_nsync)
-    _weaken(nsync_waiter_destroy)(tib->tib_nsync);
-  _pthread_ungarbage(tib);
 }
 
 int __cxa_thread_atexit_impl(void *fun, void *arg, void *dso_symbol) {

libc/runtime/mapstack.c

@@ -44,7 +44,7 @@
  */
 void *NewCosmoStack(void) {
   char *p;
-  size_t n = GetStackSize() + (uintptr_t)ape_stack_align;
+  size_t n = GetStackSize();
   if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
                 0)) != MAP_FAILED) {
     if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,

libc/runtime/memtrack.internal.h

@@ -3,19 +3,15 @@
 COSMOPOLITAN_C_START_
 
 #ifndef __SANITIZE_ADDRESS__
-#define kFixedmapStart      0x300000000
-#define kFixedmapSize       (0x400000000 - kFixedmapStart)
-#define kMemtrackFdsStart   0x6fe000000
-#define kMemtrackFdsSize    (0x6ff000000 - kMemtrackFdsStart)
-#define kMemtrackZiposStart 0x6fd000000
-#define kMemtrackZiposSize  (0xafe000000 - kMemtrackZiposStart)
+#define kFixedmapStart    0x300000000
+#define kFixedmapSize     (0x400000000 - kFixedmapStart)
+#define kMemtrackFdsStart 0x6fe000000
+#define kMemtrackFdsSize  (0x6ff000000 - kMemtrackFdsStart)
 #else
-#define kFixedmapStart      0x300000040000
-#define kFixedmapSize       (0x400000040000 - kFixedmapStart)
-#define kMemtrackFdsStart   0x6fe000040000
-#define kMemtrackFdsSize    (0x6feffffc0000 - kMemtrackFdsStart)
-#define kMemtrackZiposStart 0x6fd000040000
-#define kMemtrackZiposSize  (0x6fdffffc0000 - kMemtrackZiposStart)
+#define kFixedmapStart    0x300000040000
+#define kFixedmapSize     (0x400000040000 - kFixedmapStart)
+#define kMemtrackFdsStart 0x6fe000040000
+#define kMemtrackFdsSize  (0x6feffffc0000 - kMemtrackFdsStart)
 #endif
 
 COSMOPOLITAN_C_END_

libc/runtime/stack.h

@@ -14,9 +14,6 @@
  */
 #define GetGuardSize() 16384
 
-#define STATIC_STACK_ALIGN(BYTES) \
-  _STACK_SYMBOL("ape_stack_align", _STACK_STRINGIFY(BYTES) _STACK_EXTRA)
-
 /**
  * Makes program stack executable if declared, e.g.
  *
@@ -50,8 +47,6 @@ COSMOPOLITAN_C_START_
 
 extern char ape_stack_prot[] __attribute__((__weak__));
 extern char ape_stack_memsz[] __attribute__((__weak__));
-extern char ape_stack_align[] __attribute__((__weak__));
-extern char ape_stack_round[] __attribute__((__weak__));
 
 uintptr_t GetStackBottom(void) pureconst;
 

libc/runtime/winmain.greg.c

@@ -212,6 +212,8 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
   __maps.stack.size = stacksize;
   __maps.stack.prot = prot;
   __maps.maps = &__maps.stack;
+  __maps.pages = (stacksize + 4095) / 4096;
+  __maps.count = 1;
   dll_init(&__maps.stack.elem);
   dll_make_first(&__maps.used, &__maps.stack.elem);
   struct WinArgs *wa =

libc/runtime/zipos-get.c

@@ -69,9 +69,8 @@ static void __zipos_dismiss(uint8_t *map, const uint8_t *cdir, long pg) {
   // this is supposed to reduce our rss usage but does it really?
   lo = ROUNDDOWN(lo, pg);
   hi = MIN(ROUNDUP(hi, pg), ROUNDDOWN(c, pg));
-  if (hi > lo) {
+  if (hi > lo)
     posix_madvise(map + lo, hi - lo, POSIX_MADV_DONTNEED);
-  }
 }
 
 static int __zipos_compare_names(const void *a, const void *b, void *c) {
@@ -96,9 +95,8 @@ static void __zipos_generate_index(struct Zipos *zipos) {
   zipos->records = GetZipCdirRecords(zipos->cdir);
   zipos->index = _mapanon(zipos->records * sizeof(size_t));
   for (i = 0, c = GetZipCdirOffset(zipos->cdir); i < zipos->records;
-       ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) {
+       ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c))
     zipos->index[i] = c;
-  }
   // smoothsort() isn't the fastest algorithm, but it guarantees
   // o(nlogn) won't smash the stack and doesn't depend on malloc
   smoothsort_r(zipos->index, zipos->records, sizeof(size_t),
@@ -122,9 +120,8 @@ static void __zipos_init(void) {
     }
     if (fd != -1 || PLEDGED(RPATH)) {
       if (fd == -1) {
-        if (!progpath) {
+        if (!progpath)
           progpath = GetProgramExecutableName();
-        }
         fd = open(progpath, O_RDONLY);
       }
       if (fd != -1) {

libc/runtime/zipos-open.c

@@ -16,112 +16,50 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/calls/calls.h"
+#include "libc/calls/blockcancel.internal.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/state.internal.h"
-#include "libc/calls/struct/sigset.h"
+#include "libc/calls/struct/fd.internal.h"
 #include "libc/calls/struct/sigset.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/calls/syscall_support-sysv.internal.h"
-#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
-#include "libc/intrin/cmpxchg.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/extend.internal.h"
-#include "libc/intrin/likely.h"
-#include "libc/intrin/strace.internal.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/weaken.h"
-#include "libc/limits.h"
 #include "libc/runtime/internal.h"
-#include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/runtime/zipos.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/f.h"
-#include "libc/sysv/consts/fd.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/prot.h"
 #include "libc/sysv/consts/s.h"
-#include "libc/sysv/consts/sig.h"
 #include "libc/sysv/errfuns.h"
-#include "libc/thread/thread.h"
-#include "libc/thread/tls.h"
 #include "libc/zip.internal.h"
 
-#define MAX_REFS SSIZE_MAX
-
-static char *__zipos_mapend;
-static size_t __zipos_maptotal;
-static pthread_mutex_t __zipos_lock_obj;
-
-static void __zipos_wipe(void) {
-  pthread_mutex_init(&__zipos_lock_obj, 0);
-}
-
-static void __zipos_lock(void) {
-  pthread_mutex_lock(&__zipos_lock_obj);
-}
-
-static void __zipos_unlock(void) {
-  pthread_mutex_unlock(&__zipos_lock_obj);
-}
-
-static void *__zipos_mmap_space(size_t mapsize) {
-  char *start;
-  size_t offset;
-  unassert(mapsize);
-  offset = __zipos_maptotal;
-  __zipos_maptotal += mapsize;
-  start = (char *)kMemtrackZiposStart;
-  if (!__zipos_mapend)
-    __zipos_mapend = start;
-  __zipos_mapend = _extend(start, __zipos_maptotal, __zipos_mapend, MAP_PRIVATE,
-                           kMemtrackZiposStart + kMemtrackZiposSize);
-  return start + offset;
-}
-
 struct ZiposHandle *__zipos_keep(struct ZiposHandle *h) {
-  size_t refs = atomic_fetch_add_explicit(&h->refs, 1, memory_order_relaxed);
-  unassert(!VERY_UNLIKELY(refs > MAX_REFS));
+  atomic_fetch_add_explicit(&h->refs, 1, memory_order_relaxed);
   return h;
 }
 
 void __zipos_drop(struct ZiposHandle *h) {
-  if (atomic_fetch_sub_explicit(&h->refs, 1, memory_order_release)) {
+  if (atomic_fetch_sub_explicit(&h->refs, 1, memory_order_release))
     return;
-  }
   atomic_thread_fence(memory_order_acquire);
-  __zipos_lock();
-  do
-    h->next = h->zipos->freelist;
-  while (!_cmpxchg(&h->zipos->freelist, h->next, h));
-  __zipos_unlock();
+  __munmap((char *)h, h->mapsize, false);
 }
 
 static struct ZiposHandle *__zipos_alloc(struct Zipos *zipos, size_t size) {
   size_t mapsize;
-  struct ZiposHandle *h, **ph;
-  __zipos_lock();
+  int granularity;
+  struct ZiposHandle *h;
+  granularity = __granularity();
   mapsize = sizeof(struct ZiposHandle) + size;
-  mapsize = ROUNDUP(mapsize, 4096);
-StartOver:
-  ph = &zipos->freelist;
-  while ((h = *ph)) {
-    if (h->mapsize >= mapsize) {
-      if (!_cmpxchg(ph, h, h->next))
-        goto StartOver;
-      break;
-    }
-    ph = &h->next;
-  }
-  if (!h) {
-    h = __zipos_mmap_space(mapsize);
-  }
-  __zipos_unlock();
-  if (h) {
-    atomic_store_explicit(&h->refs, 0, memory_order_relaxed);
+  mapsize = (mapsize + granularity - 1) & -granularity;
+  if ((h = __mmap(randaddr(), mapsize, PROT_READ | PROT_WRITE,
+                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) != MAP_FAILED) {
     h->size = size;
     h->zipos = zipos;
     h->mapsize = mapsize;
@@ -158,6 +96,7 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, int flags,
   size_t size;
   int fd, minfd;
   struct ZiposHandle *h;
+
   if (cf == ZIPOS_SYNTHETIC_DIRECTORY) {
     size = name->len;
     if (!(h = __zipos_alloc(zipos, size + 1)))
@@ -168,7 +107,6 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, int flags,
     h->mem = h->data;
   } else {
     lf = GetZipCfileOffset(zipos->map + cf);
-    npassert((ZIP_LFILE_MAGIC(zipos->map + lf) == kZipLfileHdrMagic));
     size = GetZipLfileUncompressedSize(zipos->map + lf);
     switch (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
       case kZipCompressionNone:
@@ -191,18 +129,17 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, int flags,
         return eio();
     }
   }
+
   atomic_store_explicit(&h->pos, 0, memory_order_relaxed);
   h->cfile = cf;
-  unassert(size < SIZE_MAX);
   h->size = size;
   if (h->mem) {
     minfd = 3;
     __fds_lock();
   TryAgain:
     if (IsWindows() || IsMetal()) {
-      if ((fd = __reservefd_unlocked(-1)) != -1) {
+      if ((fd = __reservefd_unlocked(-1)) != -1)
         return __zipos_setfd(fd, h, flags);
-      }
     } else if ((fd = __zipos_mkfd(minfd)) != -1) {
       if (__ensurefds_unlocked(fd) != -1) {
         if (g_fds.p[fd].kind) {
@@ -221,16 +158,15 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, int flags,
 }
 
 void __zipos_postdup(int oldfd, int newfd) {
-  if (oldfd == newfd) {
+  if (oldfd == newfd)
     return;
-  }
   BLOCK_SIGNALS;
+  BLOCK_CANCELATION;
   __fds_lock();
   if (__isfdkind(newfd, kFdZip)) {
     __zipos_drop((struct ZiposHandle *)(intptr_t)g_fds.p[newfd].handle);
-    if (!__isfdkind(oldfd, kFdZip)) {
+    if (!__isfdkind(oldfd, kFdZip))
       bzero(g_fds.p + newfd, sizeof(*g_fds.p));
-    }
   }
   if (__isfdkind(oldfd, kFdZip)) {
     __zipos_keep((struct ZiposHandle *)(intptr_t)g_fds.p[oldfd].handle);
@@ -238,6 +174,7 @@ void __zipos_postdup(int oldfd, int newfd) {
     g_fds.p[newfd] = g_fds.p[oldfd];
   }
   __fds_unlock();
+  ALLOW_CANCELATION;
   ALLOW_SIGNALS;
 }
 
@@ -260,45 +197,36 @@ int __zipos_open(struct ZiposUri *name, int flags) {
   // validate api usage
   if ((flags & O_CREAT) ||  //
       (flags & O_TRUNC) ||  //
-      (flags & O_ACCMODE) != O_RDONLY) {
+      (flags & O_ACCMODE) != O_RDONLY)
     return erofs();
-  }
 
   // get the zipos global singleton
   struct Zipos *zipos;
-  if (!(zipos = __zipos_get())) {
+  if (!(zipos = __zipos_get()))
     return enoexec();
-  }
 
   // most open() calls are due to languages path searching assets. the
   // majority of these calls will return ENOENT or ENOTDIR. we need to
   // perform two extremely costly sigprocmask() calls below. thanks to
   // zipos being a read-only filesystem, we can avoid it in many cases
   ssize_t cf;
-  if ((cf = __zipos_find(zipos, name)) == -1) {
+  if ((cf = __zipos_find(zipos, name)) == -1)
     return -1;
-  }
-  if (flags & O_EXCL) {
+  if (flags & O_EXCL)
     return eexist();
-  }
   if (cf != ZIPOS_SYNTHETIC_DIRECTORY) {
     int mode = GetZipCfileMode(zipos->map + cf);
-    if ((flags & O_DIRECTORY) && !S_ISDIR(mode)) {
+    if ((flags & O_DIRECTORY) && !S_ISDIR(mode))
       return enotdir();
-    }
-    if (!(mode & 0444)) {
+    if (!(mode & 0444))
       return eacces();
-    }
   }
 
   // now do the heavy lifting
   BLOCK_SIGNALS;
+  BLOCK_CANCELATION;
   rc = __zipos_load(zipos, cf, flags, name);
+  ALLOW_CANCELATION;
   ALLOW_SIGNALS;
   return rc;
 }
-
-__attribute__((__constructor__(60))) static textstartup void zipos_ctor(void) {
-  __zipos_wipe();
-  pthread_atfork(__zipos_lock, __zipos_unlock, __zipos_wipe);
-}