Get xtrace working again (for multithreaded programs)

We basically have to reimplement locking, memory allocation, and TLS for xtrace directly on top of Linux syscalls because it can't use almost anything external (it can pretty much only use libsystem_kernel).

The one exception is that we use `pthread_key`s from libpthread to make TLS easier to implement. The only side effect that has is that it's one less available key for the application we're tracing to use, but we shouldn't ever see that cause problems (and if we do see it cause problems later on, then we can revise that).

Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/darling/emulation/base.h

@@ -0,0 +1 @@
+../../../../../../../../../../src/kernel/emulation/linux/base.h

Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/darling/emulation/ext/for-xtrace.h

@@ -0,0 +1 @@
+../../../../../../../../../../../src/kernel/emulation/linux/ext/for-xtrace.h

Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/darling/emulation/ext/futex.h

@@ -0,0 +1 @@
+../../../../../../../../../../../src/kernel/emulation/linux/ext/futex.h

Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/darling/emulation/mman/duct_mman.h

@@ -0,0 +1 @@
+../../../../../../../../../../../src/kernel/emulation/linux/mman/duct_mman.h

src/kernel/emulation/linux/CMakeLists.txt

@@ -263,6 +263,7 @@ set(emulation_sources
 	ext/mremap.c
 	ext/file_handle.c
 	ext/fanotify.c
+	ext/for-xtrace.c
 	bsdthread/bsdthread_register.c
 	bsdthread/bsdthread_create.c
 	bsdthread/bsdthread_terminate.c

src/kernel/emulation/linux/ext/for-xtrace.c

@@ -0,0 +1,18 @@
+#include "for-xtrace.h"
+#include "../mman/mman.h"
+#include "../misc/abort_with_payload.h"
+
+VISIBLE
+void* _mmap_for_xtrace(void* start, unsigned long len, int prot, int flags, int fd, long pos) {
+	return sys_mmap(start, len, prot, flags, fd, pos);
+};
+
+VISIBLE
+long _munmap_for_xtrace(void* addr, unsigned long len) {
+	return sys_munmap(addr, len);
+};
+
+VISIBLE
+long _abort_with_payload_for_xtrace(unsigned int reason_namespace, unsigned long long reason_code, void *payload, unsigned int payload_size, const char *reason_string, unsigned long long reason_flags) {
+	return sys_abort_with_payload(reason_namespace, reason_code, payload, payload_size, reason_string, reason_flags);
+};

src/kernel/emulation/linux/ext/for-xtrace.h

@@ -0,0 +1,16 @@
+#ifndef _DARLING_EMULATION_FOR_XTRACE_H_
+#define _DARLING_EMULATION_FOR_XTRACE_H_
+
+#include <darling/emulation/base.h>
+#include <darling/emulation/mman/duct_mman.h>
+
+VISIBLE
+void* _mmap_for_xtrace(void* start, unsigned long len, int prot, int flags, int fd, long pos);
+
+VISIBLE
+long _munmap_for_xtrace(void* addr, unsigned long len);
+
+VISIBLE
+long _abort_with_payload_for_xtrace(unsigned int reason_namespace, unsigned long long reason_code, void *payload, unsigned int payload_size, const char *reason_string, unsigned long long reason_flags);
+
+#endif // _DARLING_EMULATION_FOR_XTRACE_H_

src/xtrace/CMakeLists.txt

@@ -15,6 +15,9 @@ set(xtrace_sources
 	mach_trace.cpp
 	bsd_trace.cpp
 	mig_trace.c
+	tls.c
+	malloc.c
+	lock.c
 )
 
 if (TARGET_x86_64)

src/xtrace/base.h

@@ -0,0 +1,20 @@
+#ifndef _XTRACE_BASE_H_
+#define _XTRACE_BASE_H_
+
+// common utilities for xtrace code
+
+#define XTRACE_INLINE static __attribute__((always_inline))
+
+#ifdef __cplusplus
+	#define XTRACE_DECLARATIONS_BEGIN extern "C" {
+#else
+	#define XTRACE_DECLARATIONS_BEGIN
+#endif
+
+#ifdef __cplusplus
+	#define XTRACE_DECLARATIONS_END }
+#else
+	#define XTRACE_DECLARATIONS_END
+#endif
+
+#endif // _XTRACE_BASE_H_

src/xtrace/lock.c

@@ -0,0 +1,105 @@
+#include <stdatomic.h>
+
+#include "lock.h"
+#include "simple.h"
+
+#ifndef XTRACE_LOCK_DEBUG
+	#define XTRACE_LOCK_DEBUG 0
+#endif
+
+#if XTRACE_LOCK_DEBUG
+	#define xtrace_lock_debug(x, ...) __simple_printf(x "\n", ## __VA_ARGS__)
+	#undef XTRACE_INLINE
+	#define XTRACE_INLINE
+#else
+	#define xtrace_lock_debug(x, ...)
+#endif
+
+// simple lock implementation on top of futexes, based on https://github.com/eliben/code-for-blog/blob/master/2018/futex-basics/mutex-using-futex.cpp
+
+// used to hide the use of C11 atomics from external code (e.g. C++ code, which can't use C11 atomics).
+// it's *technically* undefined behavior to cast the external one to this one, but that's okay;
+// we know our exact environment and it works in it
+typedef struct xtrace_lock_internal {
+	_Atomic uint32_t state;
+} xtrace_lock_internal_t;
+
+typedef struct xtrace_once_internal {
+	_Atomic uint8_t state;
+} xtrace_once_internal_t;
+
+enum xtrace_lock_state {
+	// the lock is unlocked (duh)
+	xtrace_lock_state_unlocked = 0,
+
+	// the lock is locked, but no one is waiting for it
+	xtrace_lock_state_locked_uncontended = 1,
+
+	// the locked is locked and there are waiters
+	xtrace_lock_state_locked_contended = 2,
+};
+
+enum xtrace_once_state {
+	// the once hasn't been run yet
+	xtrace_once_state_untriggered = 0,
+
+	// the once has already been run at least once
+	xtrace_once_state_triggered = 1,
+};
+
+void xtrace_lock_lock(xtrace_lock_t* _lock) {
+	xtrace_lock_internal_t* lock = (xtrace_lock_internal_t*)_lock;
+
+	xtrace_lock_debug("lock %p: locking", lock);
+
+	// we do the initial exchange using `xtrace_lock_state_locked_uncontended` because
+	// if it was previously unlocked, it is now locked but nobody is waiting for it
+	uint32_t prev = atomic_exchange(&lock->state, xtrace_lock_state_locked_uncontended);
+
+	xtrace_lock_debug("lock %p: previous state was %u", lock, prev);
+
+	// if it was not unlocked, we need to do some waiting
+	if (prev != xtrace_lock_state_unlocked) {
+		do {
+			// we can skip the atomic_exchange if the lock was already contended
+			// also, when we do the atomic_exchange, we need to make sure it's still locked
+			// we use `xtrace_lock_state_locked_contended` because it was either uncontended (but it is now) or it was already contended
+			if (prev == xtrace_lock_state_locked_contended || atomic_exchange(&lock->state, xtrace_lock_state_locked_contended) != xtrace_lock_state_unlocked) {
+				xtrace_lock_debug("lock %p: going to wait", lock);
+				// do the actual sleeping
+				// we expect `xtrace_lock_state_locked_contended` because we don't want to sleep if it's not contended
+				__linux_futex(&lock->state, FUTEX_WAIT, xtrace_lock_state_locked_contended, NULL, 0, 0);
+				xtrace_lock_debug("lock %p: awoken", lock);
+			}
+
+			// loop as long as the lock was not unlocked
+			// we use `xtrace_lock_state_locked_contended` for the same reason as before
+		} while ((prev = atomic_exchange(&lock->state, xtrace_lock_state_locked_contended)) != xtrace_lock_state_unlocked);
+		xtrace_lock_debug("lock %p: lock acquired after sleeping", lock);
+	}
+
+	xtrace_lock_debug("lock %p: lock acquired", lock);
+};
+
+void xtrace_lock_unlock(xtrace_lock_t* _lock) {
+	xtrace_lock_internal_t* lock = (xtrace_lock_internal_t*)_lock;
+
+	xtrace_lock_debug("lock %p: unlocking", lock);
+
+	uint32_t prev = atomic_exchange(&lock->state, xtrace_lock_state_unlocked);
+
+	xtrace_lock_debug("lock %p: previous state was %u", prev);
+
+	// if it was previously contended, then we need to wake someone up
+	if (prev == xtrace_lock_state_locked_contended) {
+		xtrace_lock_debug("lock %p: waking someone up", lock);
+		__linux_futex(&lock->state, FUTEX_WAKE, 1, NULL, 0, 0);
+	}
+};
+
+void xtrace_once(xtrace_once_t* _once, xtrace_once_callback callback) {
+	xtrace_lock_internal_t* once = (xtrace_lock_internal_t*)_once;
+	if (atomic_exchange(&once->state, xtrace_once_state_triggered) == xtrace_once_state_untriggered) {
+		callback();
+	}
+};

src/xtrace/lock.h

@@ -0,0 +1,53 @@
+#ifndef _XTRACE_LOCK_H_
+#define _XTRACE_LOCK_H_
+
+#include <darling/emulation/ext/futex.h>
+#include <stdint.h>
+
+#include "base.h"
+
+// we also have to implement our own locks, because we can't rely on libplatform's or libpthread's locks
+// so we implement our own on top of Linux futexes
+
+XTRACE_DECLARATIONS_BEGIN;
+
+//
+// lock
+//
+
+typedef struct xtrace_lock {
+	uint32_t state;
+} xtrace_lock_t;
+
+#define XTRACE_LOCK_INITIALIZER {0}
+
+XTRACE_INLINE
+void xtrace_lock_init(xtrace_lock_t* lock) {
+	lock->state = 0;
+};
+
+void xtrace_lock_lock(xtrace_lock_t* lock);
+void xtrace_lock_unlock(xtrace_lock_t* lock);
+
+//
+// once
+//
+
+typedef struct xtrace_once {
+	uint8_t state;
+} xtrace_once_t;
+
+typedef void (*xtrace_once_callback)(void);
+
+#define XTRACE_ONCE_INITIALIZER {0}
+
+XTRACE_INLINE
+void xtrace_once_init(xtrace_once_t* once) {
+	once->state = 0;
+};
+
+void xtrace_once(xtrace_once_t* once, xtrace_once_callback callback);
+
+XTRACE_DECLARATIONS_END;
+
+#endif // _XTRACE_LOCK_H_

src/xtrace/mach_trace.cpp

@@ -11,9 +11,11 @@
 #include "xtracelib.h"
 #include "mach_trace.h"
 #include "mig_trace.h"
+#include "tls.h"
 
-_Thread_local int mach_call_nr = -1;
-_Thread_local void* argument_ptr = NULL;
+DEFINE_XTRACE_TLS_VAR(int, mach_call_nr);
+DEFINE_XTRACE_TLS_VAR(void*, argument_ptr);
+DEFINE_XTRACE_TLS_VAR(mach_port_name_t, request_port);
 
 static void print_kern_return(char* buf, int nr, uintptr_t rv);
 static void print_port_return(char* buf, int nr, uintptr_t rv);
@@ -207,8 +209,8 @@ static const char* const bootstrap_errors[] = {
 extern "C"
 void darling_mach_syscall_entry_print(int nr, void* args[])
 {
-	mach_call_nr = nr;
-	handle_generic_entry(mach_defs, "mach", mach_call_nr, args);
+	set_mach_call_nr(nr);
+	handle_generic_entry(mach_defs, "mach", nr, args);
 	if (nr == 31 || nr == 32)
 		print_mach_msg_entry(args);
 }
@@ -216,11 +218,12 @@ void darling_mach_syscall_entry_print(int nr, void* args[])
 extern "C"
 void darling_mach_syscall_exit_print(uintptr_t retval)
 {
-	int is_msg = mach_call_nr == 31 || mach_call_nr == 32;
+	int nr = get_mach_call_nr();
+	int is_msg = nr == 31 || nr == 32;
 	handle_generic_exit(mach_defs, "mach", retval, is_msg);
 	if (retval == KERN_SUCCESS && is_msg)
 		print_mach_msg_exit();
-	mach_call_nr = -1;
+	set_mach_call_nr(-1);
 }
 
 int xtrace_kern_return_to_str(char* buf, kern_return_t kr)
@@ -264,16 +267,16 @@ static void print_port_ptr_return(char* buf, int nr, uintptr_t rv)
 	if (rv != KERN_SUCCESS)
 	{
 		print_kern_return(buf, nr, rv);
-		argument_ptr = NULL;
+		set_argument_ptr(NULL);
 		return;
 	}
-	if (argument_ptr == NULL)
+	if (get_argument_ptr() == NULL)
 	{
 		*buf = 0;
 		return;
 	}
-	__simple_sprintf(buf, "port right %d", *(mach_port_name_t*) argument_ptr);
-	argument_ptr = NULL;
+	__simple_sprintf(buf, "port right %d", *(mach_port_name_t*)get_argument_ptr());
+	set_argument_ptr(NULL);
 }
 
 
@@ -290,7 +293,7 @@ static void print_mach_port_allocate_args(char* buf, int nr, void* args[])
 {
 	mach_port_name_t target = (mach_port_name_t) (long) args[0];
 	mach_port_right_t right = (mach_port_right_t) (long) args[1];
-	argument_ptr = args[2];
+	set_argument_ptr(args[2]);
 
 	const char* right_name;
 	if (right > MACH_PORT_RIGHT_NUMBER)
@@ -342,8 +345,8 @@ static void print_mach_port_member_args(char* buf, int nr, void* args[])
 
 static void print_mach_timebase_info_args(char* buf, int nr, void* args[])
 {
-	argument_ptr = args[0];
-	if (argument_ptr == NULL)
+	set_argument_ptr(args[0]);
+	if (get_argument_ptr() == NULL)
 		__simple_sprintf(buf, "NULL");
 	else
 		*buf = 0;
@@ -354,25 +357,25 @@ static void print_mach_timebase_info_res(char* buf, int nr, uintptr_t rv)
 	if (rv != KERN_SUCCESS)
 	{
 		print_kern_return(buf, nr, rv);
-		argument_ptr = NULL;
+		set_argument_ptr(NULL);
 		return;
 	}
-	if (argument_ptr != NULL)
+	if (get_argument_ptr() != NULL)
 	{
-		mach_timebase_info_t timebase = (mach_timebase_info_t) argument_ptr;
+		mach_timebase_info_t timebase = (mach_timebase_info_t)get_argument_ptr();
 		__simple_sprintf(buf, "numer = %d, denom = %d", timebase->numer, timebase->denom);
 	}
 	else
 		*buf = 0;
 
-	argument_ptr = NULL;
+	set_argument_ptr(NULL);
 }
 
 static void print_task_for_pid_args(char* buf, int nr, void* args[])
 {
 	mach_port_name_t target = (mach_port_name_t) (long) args[0];
 	int pid = (int) (long) args[1];
-	argument_ptr = args[2];
+	set_argument_ptr(args[2]);
 
 	__simple_sprintf(buf, "task %d, pid %d", target, pid);
 }
@@ -380,7 +383,7 @@ static void print_task_for_pid_args(char* buf, int nr, void* args[])
 static void print_pid_for_task_args(char* buf, int nr, void* args[])
 {
 	mach_port_name_t task = (mach_port_name_t) (long) args[0];
-	argument_ptr = args[1];
+	set_argument_ptr(args[1]);
 
 	__simple_sprintf(buf, "task %d", task);
 }
@@ -390,15 +393,15 @@ static void print_pid_for_task_res(char* buf, int nr, uintptr_t rv)
 	if (rv != KERN_SUCCESS)
 	{
 		print_kern_return(buf, nr, rv);
-		argument_ptr = NULL;
+		set_argument_ptr(NULL);
 		return;
 	}
-	if (argument_ptr != NULL)
-		__simple_sprintf(buf, "pid %d", * (int*) argument_ptr);
+	if (get_argument_ptr() != NULL)
+		__simple_sprintf(buf, "pid %d", * (int*)get_argument_ptr());
 	else
 		*buf = 0;
 
-	argument_ptr = NULL;
+	set_argument_ptr(NULL);
 }
 
 const char* xtrace_msg_type_to_str(mach_msg_type_name_t type_name, int full)
@@ -480,8 +483,6 @@ static void print_mach_msg(const mach_msg_header_t* msg, mach_msg_size_t size)
 	__simple_printf(", %lu bytes of inline data\n", size - ((const char*) ptr - (const char*) msg));
 }
 
-_Thread_local mach_port_name_t request_port;
-
 static void print_mach_msg_entry(void* args[])
 {
 	const mach_msg_header_t* message = (const mach_msg_header_t*) args[0];
@@ -490,28 +491,28 @@ static void print_mach_msg_entry(void* args[])
 
 	if (options & MACH_SEND_MSG)
 	{
-		request_port = message->msgh_remote_port;
+		set_request_port(message->msgh_remote_port);
 		__simple_printf("\n");
 		xtrace_start_line(8);
 		print_mach_msg(message, send_size);
 		xtrace_start_line(8);
-		xtrace_print_mig_message(message, request_port);
+		xtrace_print_mig_message(message, get_request_port());
 		__simple_printf("\n");
 	}
 
 	if (options & MACH_RCV_MSG)
 	{
-		switch (mach_call_nr)
+		switch (get_mach_call_nr())
 		{
 			case 31:
 				// mach_msg_trap
-				argument_ptr = args[0];
+				set_argument_ptr(args[0]);
 				break;
 			case 32:
 				// mach_msg_overwrite_trap
-				argument_ptr = args[7];
-				if (argument_ptr == NULL)
-					argument_ptr = args[0];
+				set_argument_ptr(args[7]);
+				if (get_argument_ptr() == NULL)
+					set_argument_ptr(args[0]);
 				break;
 			default:
 				__simple_printf("Unexpected mach_call_nr");
@@ -522,17 +523,17 @@ static void print_mach_msg_entry(void* args[])
 
 static void print_mach_msg_exit()
 {
-	if (argument_ptr == NULL)
+	if (get_argument_ptr() == NULL)
 		return;
 
-	const mach_msg_header_t* message = (const mach_msg_header_t*) argument_ptr;
+	const mach_msg_header_t* message = (const mach_msg_header_t*)get_argument_ptr();
 	xtrace_start_line(8);
 	print_mach_msg(message, message->msgh_size);
 	xtrace_start_line(8);
-	xtrace_print_mig_message(message, request_port);
+	xtrace_print_mig_message(message, get_request_port());
 	__simple_printf("\n");
-	argument_ptr = NULL;
-	request_port = MACH_PORT_NULL;
+	set_argument_ptr(NULL);
+	set_request_port(MACH_PORT_NULL);
 }
 
 static void print_mach_msg_args(char* buf, int nr, void* args[])

src/xtrace/malloc.c

@@ -0,0 +1,482 @@
+#define PRIVATE 1
+#include <sys/cdefs.h>
+
+#include <darling/emulation/ext/for-xtrace.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/queue.h>
+
+#include "malloc.h"
+#include "lock.h"
+#include "simple.h"
+
+#ifndef XTRACE_MALLOC_DEBUG
+	#define XTRACE_MALLOC_DEBUG 0
+#endif
+
+#if XTRACE_MALLOC_DEBUG
+	#define xtrace_malloc_debug(x, ...) __simple_printf(x "\n", ## __VA_ARGS__)
+	#undef XTRACE_INLINE
+	#define XTRACE_INLINE
+#else
+	#define xtrace_malloc_debug(x, ...)
+#endif
+
+// we can't rely on libmalloc because:
+// 1. we should be invisible to everyone but libkernel
+// 2. libmalloc might need to `thread_switch(2)`, which will recurse back into xtrace, blowing up the stack
+// so let's roll our own malloc! (using Linux's mmap)
+
+// all blocks will be allocated in multiples of this size
+#define BLOCK_SIZE_MULTIPLE (4096ULL)
+
+#define LIST_PREV(elm, link) ({ \
+		__typeof(elm) elm_cache = elm; \
+		__typeof((elm_cache)->link)* as_link = __container_of(elm_cache->link.le_prev, __typeof((elm_cache)->link), le_next); \
+		elm_cache = __container_of(as_link, __typeof(*elm_cache), link); \
+		elm_cache; \
+	})
+
+typedef LIST_HEAD(xtrace_memory_fragment_head, xtrace_memory_fragment) xtrace_memory_fragment_head_t;
+typedef LIST_ENTRY(xtrace_memory_fragment) xtrace_memory_fragment_entry_t;
+
+//
+// xtrace_memory_fragment
+//
+
+#define XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE (1ULL << 63)
+#define XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE (1ULL << 62)
+#define XTRACE_MEMORY_FRAGMENT_FLAG_BIT_COUNT (2)
+#define XTRACE_MEMORY_FRAGMENT_SIZE_MASK (0xffffffffffffffffULL >> XTRACE_MEMORY_FRAGMENT_FLAG_BIT_COUNT)
+
+typedef struct xtrace_memory_fragment* xtrace_memory_fragment_t;
+struct xtrace_memory_fragment {
+	uint64_t flags;
+	xtrace_memory_fragment_entry_t block_link;
+	xtrace_memory_fragment_entry_t free_link;
+	char memory[];
+};
+
+XTRACE_INLINE
+bool xtrace_memory_fragment_in_use(xtrace_memory_fragment_t fragment) {
+	return fragment->flags & XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
+};
+
+XTRACE_INLINE
+void xtrace_memory_fragment_set_in_use(xtrace_memory_fragment_t fragment, bool in_use) {
+	if (in_use) {
+		fragment->flags |= XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
+	} else {
+		fragment->flags &= ~XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
+	}
+};
+
+XTRACE_INLINE
+bool xtrace_memory_fragment_is_mmap_base(xtrace_memory_fragment_t fragment) {
+	return fragment->flags & XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
+};
+
+XTRACE_INLINE
+void xtrace_memory_fragment_set_is_mmap_base(xtrace_memory_fragment_t fragment, bool is_mmap_base) {
+	if (is_mmap_base) {
+		fragment->flags |= XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
+	} else {
+		fragment->flags &= ~XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
+	}
+};
+
+XTRACE_INLINE
+size_t xtrace_memory_fragment_size(xtrace_memory_fragment_t fragment) {
+	return fragment->flags & XTRACE_MEMORY_FRAGMENT_SIZE_MASK;
+};
+
+XTRACE_INLINE
+void xtrace_memory_fragment_set_size(xtrace_memory_fragment_t fragment, size_t new_size) {
+	fragment->flags = (fragment->flags & ~XTRACE_MEMORY_FRAGMENT_SIZE_MASK) | (new_size & XTRACE_MEMORY_FRAGMENT_SIZE_MASK);
+};
+
+XTRACE_INLINE
+void xtrace_memory_fragment_init(xtrace_memory_fragment_t fragment, size_t size) {
+	xtrace_malloc_debug("initializing fragment of %llu bytes at %p", size, fragment);
+	fragment->flags = size & XTRACE_MEMORY_FRAGMENT_SIZE_MASK;
+	fragment->block_link.le_next = NULL;
+	fragment->block_link.le_prev = NULL;
+	fragment->free_link.le_next = NULL;
+	fragment->free_link.le_prev = NULL;
+};
+
+//
+// xtrace_memory_block
+//
+
+typedef struct xtrace_memory_block* xtrace_memory_block_t;
+struct xtrace_memory_block {
+	uint64_t size;
+	xtrace_memory_fragment_head_t fragment_list;
+	struct xtrace_memory_fragment fragment;
+};
+
+XTRACE_INLINE
+bool xtrace_memory_block_in_use(xtrace_memory_block_t block) {
+	// the block is in use if either:
+	//   * the first fragment is in use, or
+	//   * the block has been fragmented
+	return xtrace_memory_fragment_in_use(&block->fragment) || (block->size != xtrace_memory_fragment_size(&block->fragment));
+};
+
+XTRACE_INLINE
+void xtrace_memory_block_init(xtrace_memory_block_t block, size_t size) {
+	block->size = size;
+	LIST_INIT(&block->fragment_list);
+	xtrace_memory_fragment_init(&block->fragment, block->size);
+	xtrace_memory_fragment_set_is_mmap_base(&block->fragment, true);
+	LIST_INSERT_HEAD(&block->fragment_list, &block->fragment, block_link);
+};
+
+//
+// global variables
+//
+
+// list of available fragments
+static xtrace_memory_fragment_head_t free_list = LIST_HEAD_INITIALIZER(free_list);
+// lock for free_list
+static xtrace_lock_t free_lock = XTRACE_LOCK_INITIALIZER;
+
+//
+// internal functions
+//
+
+static xtrace_memory_fragment_t allocate_fragment(size_t required_size);
+static void release_fragment(xtrace_memory_fragment_t fragment);
+static bool shrink_fragment(xtrace_memory_fragment_t fragment, size_t new_size);
+static bool expand_fragment(xtrace_memory_fragment_t fragment, size_t new_size);
+
+// borrowed from libgmalloc
+XTRACE_INLINE
+size_t round_up(size_t size, size_t increment) {
+	if ((size & (increment - 1)) == 0) {
+		return size;
+	}
+	return (size | (increment - 1)) + 1;
+}
+
+static xtrace_memory_fragment_t allocate_fragment(size_t required_size) {
+	xtrace_memory_fragment_t viable_fragment = NULL;
+	size_t viable_fragment_size = 0;
+	xtrace_memory_fragment_t loop_var = NULL;
+	xtrace_lock_lock(&free_lock);
+
+	// look for the smallest free fragment
+	LIST_FOREACH(loop_var, &free_list, free_link) {
+		size_t loop_var_size = xtrace_memory_fragment_size(loop_var);
+
+		// 1. always make sure the current fragment is large enough
+		// 2. we'll prefer the current fragment over the previous candidate if either:
+		//      * we don't currently have a candidate fragment, or
+		//      * the current fragment is smaller than the previous candidate
+		if (loop_var_size > required_size && (viable_fragment == NULL || loop_var_size < viable_fragment_size)) {
+			viable_fragment = loop_var;
+			viable_fragment_size = loop_var_size;
+			xtrace_malloc_debug("found viable fragment with address %p and size %llu", viable_fragment->memory, viable_fragment_size);
+		}
+	}
+
+	// if we didn't find a viable candidate...
+	if (viable_fragment == NULL) {
+		// ...allocate some memory
+		size_t allocation_size = round_up(required_size, BLOCK_SIZE_MULTIPLE);
+		xtrace_memory_block_t block = _mmap_for_xtrace(NULL, allocation_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
+
+		xtrace_malloc_debug("mmap for %llu bytes returned %p", allocation_size, block);
+
+		// check if return value is an error code
+		// see similar check in libkernel's `mman.c`
+		if ((unsigned long)block > (unsigned long)-4096) {
+			// we failed to allocate the memory, we can't do anything else
+			xtrace_malloc_debug("mmap result was failure");
+			goto out;
+		}
+
+		// initialize it
+		xtrace_memory_block_init(block, allocation_size - sizeof(struct xtrace_memory_block));
+
+		// add it to the free list
+		LIST_INSERT_HEAD(&free_list, &block->fragment, free_link);
+
+		// and set `viable_fragment` so our code below can handle it the same way as the no-mmap case
+		viable_fragment = &block->fragment;
+	}
+
+	// at this point, `viable_fragment` is guaranteed to have a value
+	xtrace_malloc_debug("final fragment for allocation: %llu bytes at %p", xtrace_memory_fragment_size(viable_fragment), viable_fragment->memory);
+
+	// shrink the fragment (if possible)
+	shrink_fragment(viable_fragment, required_size);
+
+	// set the in_use bit
+	xtrace_memory_fragment_set_in_use(viable_fragment, true);
+
+	// finally, remove the viable fragment from the free list
+	LIST_REMOVE(viable_fragment, free_link);
+
+out:
+	xtrace_lock_unlock(&free_lock);
+
+	return viable_fragment;
+};
+
+static void release_fragment(xtrace_memory_fragment_t fragment) {
+	xtrace_lock_lock(&free_lock);
+
+	xtrace_malloc_debug("releasing fragment of %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
+
+	// clear the in_use bit
+	xtrace_memory_fragment_set_in_use(fragment, false);
+
+	// add it to the free list
+	LIST_INSERT_HEAD(&free_list, fragment, free_link);
+
+	// try to defragment the block
+	while (true) {
+		xtrace_memory_fragment_t prev_frag = xtrace_memory_fragment_is_mmap_base(fragment) ? NULL : LIST_PREV(fragment, block_link);
+		xtrace_memory_fragment_t next_frag = LIST_NEXT(fragment, block_link);
+		bool defragged = false;
+
+		if (prev_frag && !xtrace_memory_fragment_in_use(prev_frag)) {
+			xtrace_malloc_debug("defragmenting with previous fragment (%llu bytes at %p)", xtrace_memory_fragment_size(prev_frag), prev_frag->memory);
+			defragged = true;
+
+			// remove the current fragment from the free list
+			LIST_REMOVE(fragment, free_link);
+
+			// remove the current fragment from the block list
+			LIST_REMOVE(fragment, block_link);
+
+			// expand the previous fragment
+			xtrace_memory_fragment_set_size(prev_frag, xtrace_memory_fragment_size(prev_frag) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(fragment));
+
+			// `fragment` is now gone; set it to `prev_frag`
+			fragment = prev_frag;
+		}
+
+		if (next_frag && !xtrace_memory_fragment_in_use(next_frag)) {
+			xtrace_malloc_debug("defragmenting with next fragment (%llu bytes at %p)", xtrace_memory_fragment_size(fragment), fragment->memory);
+			defragged = true;
+
+			// remove the next fragment from the free list
+			LIST_REMOVE(next_frag, free_link);
+
+			// remove the next fragment from the block list
+			LIST_REMOVE(next_frag, block_link);
+
+			// expand the current fragment
+			xtrace_memory_fragment_set_size(fragment, xtrace_memory_fragment_size(fragment) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(next_frag));
+
+			// `next_frag` is now gone
+		}
+
+		// if we didn't defragment anything on this loop, we can't defragment any more
+		if (!defragged) {
+			xtrace_malloc_debug("cannot defragment any further");
+			break;
+		} else {
+			// otherwise, keep looping to try to defragment more
+			xtrace_malloc_debug("doing another defragmentation iteration");
+		}
+	}
+
+	xtrace_malloc_debug("final releasing fragment is %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
+
+	// if `fragment` is the mmap base fragment...
+	if (xtrace_memory_fragment_is_mmap_base(fragment)) {
+		xtrace_memory_block_t block = __container_of(fragment, struct xtrace_memory_block, fragment);
+		xtrace_malloc_debug("releasing fragment is base fragment");
+
+		// ...and we managed to defragment the entire block...
+		if (!xtrace_memory_block_in_use(block)) {
+			// ...unmap it!
+			xtrace_malloc_debug("unmapping completely freed block (%llu bytes at %p)", block->size, block->fragment.memory);
+
+			// start off by removing all traces of this block in the free list.
+			// at this point, only the base fragment should be in the free list, so remove it
+			LIST_REMOVE(fragment, free_link);
+
+			// and now, just unmap it
+			_munmap_for_xtrace(block, block->size + sizeof(struct xtrace_memory_block));
+			// and we ignore errors
+		}
+	}
+
+out:
+	xtrace_lock_unlock(&free_lock);
+};
+
+// must be called with free_lock held
+static bool shrink_fragment(xtrace_memory_fragment_t fragment, size_t new_size) {
+	xtrace_malloc_debug("shrink to %llu bytes requested for fragment of %llu bytes at %p", new_size, xtrace_memory_fragment_size(fragment), fragment->memory);
+
+	// calculate the remaining size if we were to shrink this fragment
+	size_t remaining_size = xtrace_memory_fragment_size(fragment) - new_size;
+
+	// if we would have enough space to create another fragment after shrinking this one...
+	if (remaining_size > sizeof(struct xtrace_memory_fragment)) {
+		xtrace_malloc_debug("fragment eligible; shrinking...");
+
+		// ...shrink it...
+		xtrace_memory_fragment_set_size(fragment, new_size);
+
+		// ...and create the new fragment
+		xtrace_memory_fragment_t new_fragment = (xtrace_memory_fragment_t)(fragment->memory + new_size);
+		xtrace_memory_fragment_init(new_fragment, remaining_size - sizeof(struct xtrace_memory_fragment));
+
+		// add it to the block list
+		LIST_INSERT_AFTER(fragment, new_fragment, block_link);
+
+		xtrace_malloc_debug("old fragment shrunk to %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
+		xtrace_malloc_debug("new fragment of %llu bytes created at %p", xtrace_memory_fragment_size(new_fragment), new_fragment->memory);
+
+		// insert it into the free list
+		LIST_INSERT_HEAD(&free_list, new_fragment, free_link);
+
+		return true;
+	} else {
+		// otherwise, we don't want to shrink this one to avoid holes in our block
+		xtrace_malloc_debug("fragment ineligible for shrinking (not enough space after it)");
+	}
+
+	return false;
+};
+
+// must be called with free_lock held
+static bool expand_fragment(xtrace_memory_fragment_t fragment, size_t new_size) {
+	xtrace_malloc_debug("expansion to %llu bytes requested for fragment of %llu bytes at %p", new_size, xtrace_memory_fragment_size(fragment), fragment->memory);
+
+	size_t available_size = xtrace_memory_fragment_size(fragment);
+	xtrace_memory_fragment_t loop_var = fragment;
+	xtrace_memory_fragment_t end_frag = NULL;
+
+	while ((end_frag = LIST_NEXT(loop_var, block_link)) != NULL) {
+		// if the fragment is in use...
+		if (xtrace_memory_fragment_in_use(end_frag)) {
+			// ...we've reached a limit
+			xtrace_malloc_debug("insufficient free fragments for expansion");
+			break;
+		}
+
+		available_size += xtrace_memory_fragment_size(end_frag) + sizeof(struct xtrace_memory_fragment);
+		loop_var = end_frag;
+
+		// if we've got enough...
+		if (available_size >= new_size) {
+			xtrace_malloc_debug("found enough free fragments for expansion");
+			// ...we're done; we don't need to keep looking for more
+			break;
+		}
+	}
+
+	if (end_frag && available_size >= new_size) {
+		xtrace_memory_fragment_t terminating_value = LIST_NEXT(end_frag, block_link);
+
+		while (LIST_NEXT(fragment, block_link) != terminating_value) {
+			xtrace_memory_fragment_t dying_fragment = LIST_NEXT(fragment, block_link);
+
+			xtrace_malloc_debug("absorbing fragment of %llu bytes at %p", xtrace_memory_fragment_size(dying_fragment), dying_fragment->memory);
+
+			// remove the dying fragment from the free list
+			LIST_REMOVE(dying_fragment, free_link);
+
+			// remove the dying fragment from the block list
+			LIST_REMOVE(dying_fragment, block_link);
+
+			// expand the surviving fragment
+			xtrace_memory_fragment_set_size(fragment, xtrace_memory_fragment_size(fragment) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(dying_fragment));
+
+			// `dying_fragment` is now gone
+		}
+
+		xtrace_malloc_debug("expanded fragment is %llu bytes at %p; trying to shrink...", xtrace_memory_fragment_size(fragment), fragment->memory);
+
+		// try to shrink it so that we leave space for other possible fragments
+		shrink_fragment(fragment, new_size);
+
+		xtrace_malloc_debug("final fragment is %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
+
+		return true;
+	}
+
+	xtrace_malloc_debug("ineligible for expansion");
+
+	return false;
+};
+
+//
+// api functions
+//
+
+void* xtrace_malloc(size_t size) {
+	if (size == 0) {
+		return NULL;
+	}
+	xtrace_memory_fragment_t fragment = allocate_fragment(size);
+	if (fragment == NULL) {
+		return NULL;
+	}
+	return fragment->memory;
+};
+
+void xtrace_free(void* pointer) {
+	if (pointer == NULL) {
+		return;
+	}
+	xtrace_memory_fragment_t fragment = __container_of(pointer, struct xtrace_memory_fragment, memory);
+	release_fragment(fragment);
+};
+
+void* xtrace_realloc(void* old_pointer, size_t new_size) {
+	xtrace_memory_fragment_t fragment = __container_of(old_pointer, struct xtrace_memory_fragment, memory);
+	size_t old_size = xtrace_memory_fragment_size(fragment);
+
+	xtrace_lock_lock(&free_lock);
+
+	// if we're shrinking, we can always do that
+	if (new_size < old_size) {
+		// we don't really care if it succeeds or not
+		shrink_fragment(fragment, new_size);
+		goto out;
+	} else if (new_size == old_size) {
+		// likewise, we can always keep it the same
+		goto out;
+	}
+
+	// but otherwise, we're expanding
+
+	// try to see if we can expand the fragment first
+	if (expand_fragment(fragment, new_size)) {
+		// if we expanded it successfully, we're done!
+		goto out;
+	} else {
+		// otherwise, we need to allocate a new fragment
+		xtrace_lock_unlock(&free_lock); // `allocate_fragment` takes the lock, so we need to drop it
+		xtrace_memory_fragment_t new_fragment = allocate_fragment(new_size);
+		if (new_fragment == NULL) {
+			return NULL;
+		}
+
+		// copy the old contents to the new fragment
+		memcpy(new_fragment->memory, fragment->memory, old_size);
+
+		// and release the old fragment
+		release_fragment(fragment);
+
+		// assign the new pointer to the return variable and we're done!
+		old_pointer = new_fragment->memory;
+		goto out_unlocked;
+	}
+
+out:
+	xtrace_lock_unlock(&free_lock);
+
+out_unlocked:
+	return old_pointer;
+};

src/xtrace/malloc.h

@@ -0,0 +1,15 @@
+#ifndef _XTRACE_MALLOC_H_
+#define _XTRACE_MALLOC_H_
+
+#include <stddef.h>
+#include "base.h"
+
+XTRACE_DECLARATIONS_BEGIN;
+
+void* xtrace_malloc(size_t size);
+void xtrace_free(void* pointer);
+void* xtrace_realloc(void* old_pointer, size_t new_size);
+
+XTRACE_DECLARATIONS_END;
+
+#endif // _XTRACE_MALLOC_H_

src/xtrace/mig_trace.c

@@ -3,6 +3,7 @@
 #include <sys/types.h>
 #include <dirent.h>
 #include <dlfcn.h>
+#include <stdbool.h>
 
 #include <mach/mach.h>
 
@@ -11,6 +12,7 @@
 #include "mach_trace.h"
 #include "bsd_trace.h"
 #include "xtrace/xtrace-mig-types.h"
+#include "tls.h"
 
 #define XTRACE_MIG_DIR_PATH "/usr/lib/darling/xtrace-mig"
 
@@ -74,10 +76,10 @@ void xtrace_setup_mig_tracing(void)
 	closedir(xtrace_mig_dir);
 }
 
-_Thread_local int is_first_arg;
+DEFINE_XTRACE_TLS_VAR(bool, is_first_arg);
 
-#define BEFORE if (!is_first_arg) __simple_printf(", ")
-#define AFTER is_first_arg = 0
+#define BEFORE if (!get_is_first_arg()) __simple_printf(", ")
+#define AFTER set_is_first_arg(false)
 
 static void add_raw_arg(const char* format, ...)
 {
@@ -349,7 +351,7 @@ void xtrace_print_mig_message(const mach_msg_header_t* message, mach_port_name_t
 		xtrace_reset_color();
 	}
 
-	is_first_arg = 1;
+	set_is_first_arg(true);
 	r->routine(message, is_reply, &callbacks);
 
 	if (!is_reply)

src/xtrace/tls.c

@@ -0,0 +1,86 @@
+#include <pthread/pthread.h>
+#include <stdlib.h>
+#include <darling/emulation/ext/for-xtrace.h>
+#include "tls.h"
+#include "malloc.h"
+#include "lock.h"
+#include "simple.h"
+
+#ifndef XTRACE_TLS_DEBUG
+	#define XTRACE_TLS_DEBUG 0
+#endif
+
+#if XTRACE_TLS_DEBUG
+	#define xtrace_tls_debug(x, ...) __simple_printf(x "\n", ## __VA_ARGS__)
+#else
+	#define xtrace_tls_debug(x, ...)
+#endif
+
+// 100 TLS vars should be enough, right?
+#define TLS_TABLE_MAX_SIZE 100
+
+typedef struct tls_table* tls_table_t;
+struct tls_table {
+	size_t size;
+	void* table[TLS_TABLE_MAX_SIZE][2];
+};
+
+static xtrace_once_t tls_key_initialized = XTRACE_ONCE_INITIALIZER;
+static pthread_key_t tls_key;
+
+static void tls_table_destructor(void* _table) {
+	tls_table_t table = _table;
+	xtrace_tls_debug("destroying table %p", table);
+	for (size_t i = 0; i < table->size; ++i) {
+		xtrace_tls_debug("freeing value %p for key %p", table->table[i][1], table->table[i][0]);
+		xtrace_free(table->table[i][1]);
+	}
+	xtrace_tls_debug("freeing table %p", table);
+	xtrace_free(table);
+};
+
+static void tls_key_initializer(void) {
+	xtrace_tls_debug("initializing tls key");
+	if (pthread_key_create(&tls_key, tls_table_destructor) != 0) {
+		_abort_with_payload_for_xtrace(0, 0, NULL, 0, "xtrace: failed pthread key creation", 0);
+	}
+};
+
+void* xtrace_tls(void* key, size_t size) {
+	xtrace_tls_debug("looking up tls variable for key %p", key);
+
+	xtrace_once(&tls_key_initialized, tls_key_initializer);
+	tls_table_t table = pthread_getspecific(tls_key);
+
+	xtrace_tls_debug("got %p as table pointer from pthread", table);
+
+	// if the table doesn't exist yet, create it
+	if (table == NULL) {
+		xtrace_tls_debug("table is NULL, creating now...");
+		table = xtrace_malloc(sizeof(struct tls_table));
+		if (table == NULL) {
+			_abort_with_payload_for_xtrace(0, 0, NULL, 0, "xtrace: failed TLS table memory allocation", 0);
+		}
+		table->size = 0;
+		pthread_setspecific(tls_key, table);
+	}
+
+	// check if the key is already present
+	for (size_t i = 0; i < table->size; ++i) {
+		if (table->table[i][0] == key) {
+			xtrace_tls_debug("found entry in table for key %p with value %p", key, table->table[i][1]);
+			return table->table[i][1];
+		}
+	}
+
+	// otherwise, create it
+	xtrace_tls_debug("creating new entry in table for key %p", key);
+	size_t index = table->size++;
+	table->table[index][0] = key;
+	table->table[index][1] = xtrace_malloc(size);
+	if (table->table[index][1] == NULL) {
+		_abort_with_payload_for_xtrace(0, 0, NULL, 0, "xtrace: failed TLS variable memory allocation", 0);
+	}
+	xtrace_tls_debug("new table entry created for key %p with value %p", key, table->table[index][1]);
+	return table->table[index][1];
+};

src/xtrace/tls.h

@@ -0,0 +1,28 @@
+#ifndef _XTRACE_TLS_H_
+#define _XTRACE_TLS_H_
+
+#include <mach/port.h>
+#include "base.h"
+
+XTRACE_DECLARATIONS_BEGIN;
+
+#define DEFINE_XTRACE_TLS_VAR(type, name) \
+	char name ## _key = '\0'; \
+	XTRACE_INLINE \
+	type get_ ## name(void) { \
+		return *(type*)xtrace_tls(&(name ## _key), sizeof(type)); \
+	}; \
+	XTRACE_INLINE \
+	void set_ ## name(type value) { \
+		*(type*)xtrace_tls(&(name ## _key), sizeof(type)) = value; \
+	}; \
+	XTRACE_INLINE \
+	type* get_ptr_ ## name(void) { \
+		return (type*)xtrace_tls(&(name ## _key), sizeof(type)); \
+	};
+
+void* xtrace_tls(void* key, size_t size);
+
+XTRACE_DECLARATIONS_END;
+
+#endif // _XTRACE_TLS_H_

src/xtrace/xtracelib.c

@@ -7,6 +7,7 @@
 #include "simple.h"
 #include "xtracelib.h"
 #include "mig_trace.h"
+#include "tls.h"
 
 // Defined in assembly
 extern void darling_mach_syscall_entry_trampoline(void);
@@ -152,7 +153,7 @@ static void print_call(const struct calldef* defs, const char* type, int nr, int
 }
 
 
-_Thread_local struct {
+struct nested_call_struct {
 	// We're inside this many calls. In other words, we have printed this many
 	// call entries without matching exits.
 	int current_level;
@@ -161,21 +162,23 @@ _Thread_local struct {
 	int previous_level;
 	// Call numbers, indexed by current level.
 	int nrs[64];
-} nested_call;
+};
+
+DEFINE_XTRACE_TLS_VAR(struct nested_call_struct, nested_call);
 
 void handle_generic_entry(const struct calldef* defs, const char* type, int nr, void* args[])
 {
 	if (xtrace_ignore)
 		return;
 
-	if (nested_call.previous_level < nested_call.current_level && !xtrace_split_entry_and_exit)
+	if (get_ptr_nested_call()->previous_level < get_ptr_nested_call()->current_level && !xtrace_split_entry_and_exit)
 	{
 		// We are after an earlier entry without an exit.
 		__simple_printf("\n");
 	}
 
-	int indent = 4 * nested_call.current_level;
-	nested_call.nrs[nested_call.current_level] = nr;
+	int indent = 4 * get_ptr_nested_call()->current_level;
+	get_ptr_nested_call()->nrs[get_ptr_nested_call()->current_level] = nr;
 
 	print_call(defs, type, nr, indent, 0);
 
@@ -194,7 +197,7 @@ void handle_generic_entry(const struct calldef* defs, const char* type, int nr,
 	if (xtrace_split_entry_and_exit)
 		__simple_printf("\n");
 
-	nested_call.previous_level = nested_call.current_level++;
+	get_ptr_nested_call()->previous_level = get_ptr_nested_call()->current_level++;
 }
 
 
@@ -203,17 +206,17 @@ void handle_generic_exit(const struct calldef* defs, const char* type, uintptr_t
 	if (xtrace_ignore)
 		return;
 
-	if (nested_call.previous_level > nested_call.current_level)
+	if (get_ptr_nested_call()->previous_level > get_ptr_nested_call()->current_level)
 	{
 		// We are after an exit, so our call has been split up.
 		force_split = 1;
 	}
-	nested_call.previous_level = nested_call.current_level--;
-	int nr = nested_call.nrs[nested_call.current_level];
+	get_ptr_nested_call()->previous_level = get_ptr_nested_call()->current_level--;
+	int nr = get_ptr_nested_call()->nrs[get_ptr_nested_call()->current_level];
 
 	if (xtrace_split_entry_and_exit || force_split)
 	{
-		int indent = 4 * nested_call.current_level;
+		int indent = 4 * get_ptr_nested_call()->current_level;
 		print_call(defs, type, nr, indent, 1);
 		__simple_printf("()");
 	}