From b8fb573849038c16a16f472902a493c7d6de89e5 Mon Sep 17 00:00:00 2001 From: David Mandelin Date: Fri, 30 Oct 2009 11:15:03 -0700 Subject: [PATCH] Backed out changeset 109b74e8e902 due to tinderbox bustage (was bug 515211) --- build/wince/shunt/mozce_shunt.def.in | 9 + memory/jemalloc/Makefile.in | 6 +- memory/jemalloc/jemalloc.c | 853 ++++++++++++++++++++++++++- memory/jemalloc/jemalloc.h | 122 ++++ 4 files changed, 983 insertions(+), 7 deletions(-) diff --git a/build/wince/shunt/mozce_shunt.def.in b/build/wince/shunt/mozce_shunt.def.in index ac7273a4b1f9..5968921c2249 100644 --- a/build/wince/shunt/mozce_shunt.def.in +++ b/build/wince/shunt/mozce_shunt.def.in @@ -47,6 +47,15 @@ posix_memalign memalign malloc_usable_size jemalloc_stats +xmalloc +xcalloc +xrealloc +xmemalign +reserve_cb_register +reserve_cb_unregister +reserve_cur_get +reserve_min_get +reserve_min_set _strdup _wcsdup _strndup diff --git a/memory/jemalloc/Makefile.in b/memory/jemalloc/Makefile.in index 593520d6ea33..b10dc4ab414f 100644 --- a/memory/jemalloc/Makefile.in +++ b/memory/jemalloc/Makefile.in @@ -92,9 +92,9 @@ $(CRT_OBJ_DIR)/jemalloc.c: $(srcdir)/$(CRTDIFF) $(MOZ_CRT_DLL): \ $(CRT_OBJ_DIR)/jemalloc.c $(srcdir)/jemalloc.c $(srcdir)/jemalloc.h \ - $(srcdir)/rb.h - cp $(srcdir)/jemalloc.c $(srcdir)/jemalloc.h $(srcdir)/rb.h \ - $(CRT_OBJ_DIR) + $(srcdir)/ql.h $(srcdir)/qr.h $(srcdir)/rb.h + cp $(srcdir)/jemalloc.c $(srcdir)/jemalloc.h $(srcdir)/ql.h \ + $(srcdir)/qr.h $(srcdir)/rb.h $(CRT_OBJ_DIR) # this pretty much sucks, but nmake and make don't play well together $(PYTHON) $(srcdir)/build-crt.py $(CRT_OBJ_DIR) # XXX: these don't link right for some reason; the problem is likely diff --git a/memory/jemalloc/jemalloc.c b/memory/jemalloc/jemalloc.c index ff3b0955482b..a3952dee01c4 100644 --- a/memory/jemalloc/jemalloc.c +++ b/memory/jemalloc/jemalloc.c @@ -369,9 +369,7 @@ __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z ja #include "jemalloc.h" #undef bool -#ifdef _MSC_VER -#define bool BOOL -#endif +#define bool jemalloc_bool #ifdef MOZ_MEMORY_DARWIN static const bool __isthreaded = true; @@ -387,6 +385,8 @@ static const bool __isthreaded = true; #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) +#include "qr.h" +#include "ql.h" #ifdef MOZ_MEMORY_WINDOWS /* MSVC++ does not support C99 variable-length arrays. */ # define RB_NO_C99_VARARRAYS @@ -490,6 +490,18 @@ static const bool __isthreaded = true; /* Maximum number of dirty pages per arena. */ #define DIRTY_MAX_DEFAULT (1U << 10) +/* Default reserve chunks. */ +#define RESERVE_MIN_2POW_DEFAULT 1 +/* + * Default range (in chunks) between reserve_min and reserve_max, in addition + * to the mandatory one chunk per arena. + */ +#ifdef MALLOC_PAGEFILE +# define RESERVE_RANGE_2POW_DEFAULT 5 +#else +# define RESERVE_RANGE_2POW_DEFAULT 0 +#endif + /* * Maximum size of L1 cache line. This is used to avoid cache line aliasing, * so over-estimates are okay (up to a point), but under-estimates will @@ -752,6 +764,30 @@ struct malloc_rtree_s { }; #endif +/******************************************************************************/ +/* + * Reserve data structures. + */ + +/* Callback registration. */ +typedef struct reserve_reg_s reserve_reg_t; +struct reserve_reg_s { + /* Linkage for list of all registered callbacks. */ + ql_elm(reserve_reg_t) link; + + /* Callback function pointer. */ + reserve_cb_t *cb; + + /* Opaque application data pointer. */ + void *ctx; + + /* + * Sequence number of condition notification most recently sent to this + * callback. + */ + uint64_t seq; +}; + /******************************************************************************/ /* * Arena data structures. @@ -917,6 +953,12 @@ struct arena_s { arena_stats_t stats; #endif + /* + * Chunk allocation sequence number, used to detect races with other + * threads during chunk allocation, and then discard unnecessary chunks. + */ + uint64_t chunk_seq; + /* Tree of dirty-page-containing chunks this arena manages. */ arena_chunk_tree_t chunks_dirty; @@ -1034,10 +1076,43 @@ static uint64_t huge_ndalloc; static size_t huge_allocated; #endif +/****************/ +/* + * Memory reserve. + */ + #ifdef MALLOC_PAGEFILE static char pagefile_templ[PATH_MAX]; #endif +/* Protects reserve-related data structures. */ +static malloc_mutex_t reserve_mtx; + +/* + * Bounds on acceptable reserve size, and current reserve size. Reserve + * depletion may cause (reserve_cur < reserve_min). + */ +static size_t reserve_min; +static size_t reserve_cur; +static size_t reserve_max; + +/* List of registered callbacks. */ +static ql_head(reserve_reg_t) reserve_regs; + +/* + * Condition notification sequence number, used to determine whether all + * registered callbacks have been notified of the most current condition. + */ +static uint64_t reserve_seq; + +/* + * Trees of chunks currently in the memory reserve. Depending on function, + * different tree orderings are needed, which is why there are two trees with + * the same contents. + */ +static extent_tree_t reserve_chunks_szad; +static extent_tree_t reserve_chunks_ad; + /****************************/ /* * base (internal allocation). @@ -1055,6 +1130,7 @@ static void *base_next_decommitted; #endif static void *base_past_addr; /* Addr immediately past base_pages. */ static extent_node_t *base_nodes; +static reserve_reg_t *base_reserve_regs; static malloc_mutex_t base_mtx; #ifdef MALLOC_STATS static size_t base_mapped; @@ -1071,6 +1147,7 @@ static size_t base_mapped; */ static arena_t **arenas; static unsigned narenas; +static unsigned narenas_2pow; #ifndef NO_TLS # ifdef MALLOC_BALANCE static unsigned narenas_2pow; @@ -1124,6 +1201,8 @@ static bool opt_print_stats = false; static size_t opt_quantum_2pow = QUANTUM_2POW_MIN; static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT; static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT; +static int opt_reserve_min_lshift = 0; +static int opt_reserve_range_lshift = 0; #ifdef MALLOC_PAGEFILE static bool opt_pagefile = false; #endif @@ -1183,6 +1262,8 @@ static void *base_alloc(size_t size); static void *base_calloc(size_t number, size_t size); static extent_node_t *base_node_alloc(void); static void base_node_dealloc(extent_node_t *node); +static reserve_reg_t *base_reserve_reg_alloc(void); +static void base_reserve_reg_dealloc(reserve_reg_t *reg); #ifdef MALLOC_STATS static void stats_print(arena_t *arena); #endif @@ -1193,7 +1274,9 @@ static void *chunk_alloc_mmap(size_t size, bool pagefile); static int pagefile_init(size_t size); static void pagefile_close(int pfd); #endif +static void *chunk_recycle_reserve(size_t size, bool zero); static void *chunk_alloc(size_t size, bool zero, bool pagefile); +static extent_node_t *chunk_dealloc_reserve(void *chunk, size_t size); static void chunk_dealloc_mmap(void *chunk, size_t size); static void chunk_dealloc(void *chunk, size_t size); #ifndef NO_TLS @@ -1240,6 +1323,10 @@ static void malloc_print_stats(void); static #endif bool malloc_init_hard(void); +static void reserve_shrink(void); +static uint64_t reserve_notify(reserve_cnd_t cnd, size_t size, uint64_t seq); +static uint64_t reserve_crit(size_t size, const char *fname, uint64_t seq); +static void reserve_fail(size_t size, const char *fname); void _malloc_prefork(void); void _malloc_postfork(void); @@ -1860,6 +1947,38 @@ base_node_dealloc(extent_node_t *node) malloc_mutex_unlock(&base_mtx); } +static reserve_reg_t * +base_reserve_reg_alloc(void) +{ + reserve_reg_t *ret; + + malloc_mutex_lock(&base_mtx); + if (base_reserve_regs != NULL) { + ret = base_reserve_regs; + base_reserve_regs = *(reserve_reg_t **)ret; + VALGRIND_FREELIKE_BLOCK(ret, 0); + VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(reserve_reg_t), 0, false); + malloc_mutex_unlock(&base_mtx); + } else { + malloc_mutex_unlock(&base_mtx); + ret = (reserve_reg_t *)base_alloc(sizeof(reserve_reg_t)); + } + + return (ret); +} + +static void +base_reserve_reg_dealloc(reserve_reg_t *reg) +{ + + malloc_mutex_lock(&base_mtx); + VALGRIND_FREELIKE_BLOCK(reg, 0); + VALGRIND_MALLOCLIKE_BLOCK(reg, sizeof(reserve_reg_t *), 0, false); + *(reserve_reg_t **)reg = base_reserve_regs; + base_reserve_regs = reg; + malloc_mutex_unlock(&base_mtx); +} + /******************************************************************************/ #ifdef MALLOC_STATS @@ -2503,6 +2622,104 @@ pagefile_close(int pfd) } #endif +static void * +chunk_recycle_reserve(size_t size, bool zero) +{ + extent_node_t *node, key; + +#ifdef MALLOC_DECOMMIT + if (size != chunksize) + return (NULL); +#endif + + key.addr = NULL; + key.size = size; + malloc_mutex_lock(&reserve_mtx); + node = extent_tree_szad_nsearch(&reserve_chunks_szad, &key); + if (node != NULL) { + void *ret = node->addr; + + /* Remove node from the tree. */ + extent_tree_szad_remove(&reserve_chunks_szad, node); +#ifndef MALLOC_DECOMMIT + if (node->size == size) { +#else + assert(node->size == size); +#endif + extent_tree_ad_remove(&reserve_chunks_ad, node); + base_node_dealloc(node); +#ifndef MALLOC_DECOMMIT + } else { + /* + * Insert the remainder of node's address range as a + * smaller chunk. Its position within reserve_chunks_ad + * does not change. + */ + assert(node->size > size); + node->addr = (void *)((uintptr_t)node->addr + size); + node->size -= size; + extent_tree_szad_insert(&reserve_chunks_szad, node); + } +#endif + reserve_cur -= size; + /* + * Try to replenish the reserve if this allocation depleted it. + */ +#ifndef MALLOC_DECOMMIT + if (reserve_cur < reserve_min) { + size_t diff = reserve_min - reserve_cur; +#else + while (reserve_cur < reserve_min) { +# define diff chunksize +#endif + void *chunk; + + malloc_mutex_unlock(&reserve_mtx); + chunk = chunk_alloc_mmap(diff, true); + malloc_mutex_lock(&reserve_mtx); + if (chunk == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_notify(RESERVE_CND_LOW, + size, seq); + if (seq == 0) + goto MALLOC_OUT; + } while (reserve_cur < reserve_min); + } else { + extent_node_t *node; + + node = chunk_dealloc_reserve(chunk, diff); + if (node == NULL) { + uint64_t seq = 0; + + pages_unmap(chunk, diff); + do { + seq = reserve_notify( + RESERVE_CND_LOW, size, seq); + if (seq == 0) + goto MALLOC_OUT; + } while (reserve_cur < reserve_min); + } + } + } +MALLOC_OUT: + malloc_mutex_unlock(&reserve_mtx); + +#ifdef MALLOC_DECOMMIT + pages_commit(ret, size); +# undef diff +#else + if (zero) + memset(ret, 0, size); +#endif + return (ret); + } + malloc_mutex_unlock(&reserve_mtx); + + return (NULL); +} + static void * chunk_alloc(size_t size, bool zero, bool pagefile) { @@ -2511,6 +2728,10 @@ chunk_alloc(size_t size, bool zero, bool pagefile) assert(size != 0); assert((size & chunksize_mask) == 0); + ret = chunk_recycle_reserve(size, zero); + if (ret != NULL) + goto RETURN; + ret = chunk_alloc_mmap(size, pagefile); if (ret != NULL) { goto RETURN; @@ -2539,6 +2760,77 @@ RETURN: return (ret); } +static extent_node_t * +chunk_dealloc_reserve(void *chunk, size_t size) +{ + extent_node_t *node; + +#ifdef MALLOC_DECOMMIT + if (size != chunksize) + return (NULL); +#else + extent_node_t *prev, key; + + key.addr = (void *)((uintptr_t)chunk + size); + node = extent_tree_ad_nsearch(&reserve_chunks_ad, &key); + /* Try to coalesce forward. */ + if (node != NULL && node->addr == key.addr) { + /* + * Coalesce chunk with the following address range. This does + * not change the position within reserve_chunks_ad, so only + * remove/insert from/into reserve_chunks_szad. + */ + extent_tree_szad_remove(&reserve_chunks_szad, node); + node->addr = chunk; + node->size += size; + extent_tree_szad_insert(&reserve_chunks_szad, node); + } else { +#endif + /* Coalescing forward failed, so insert a new node. */ + node = base_node_alloc(); + if (node == NULL) + return (NULL); + node->addr = chunk; + node->size = size; + extent_tree_ad_insert(&reserve_chunks_ad, node); + extent_tree_szad_insert(&reserve_chunks_szad, node); +#ifndef MALLOC_DECOMMIT + } + + /* Try to coalesce backward. */ + prev = extent_tree_ad_prev(&reserve_chunks_ad, node); + if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == + chunk) { + /* + * Coalesce chunk with the previous address range. This does + * not change the position within reserve_chunks_ad, so only + * remove/insert node from/into reserve_chunks_szad. + */ + extent_tree_szad_remove(&reserve_chunks_szad, prev); + extent_tree_ad_remove(&reserve_chunks_ad, prev); + + extent_tree_szad_remove(&reserve_chunks_szad, node); + node->addr = prev->addr; + node->size += prev->size; + extent_tree_szad_insert(&reserve_chunks_szad, node); + + base_node_dealloc(prev); + } +#endif + +#ifdef MALLOC_DECOMMIT + pages_decommit(chunk, size); +#else + madvise(chunk, size, MADV_FREE); +#endif + + reserve_cur += size; + if (reserve_cur > reserve_max) + reserve_shrink(); + + return (node); +} + static void chunk_dealloc_mmap(void *chunk, size_t size) { @@ -2549,6 +2841,7 @@ chunk_dealloc_mmap(void *chunk, size_t size) static void chunk_dealloc(void *chunk, size_t size) { + extent_node_t *node; assert(chunk != NULL); assert(CHUNK_ADDR2BASE(chunk) == chunk); @@ -2562,7 +2855,12 @@ chunk_dealloc(void *chunk, size_t size) malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL); #endif - chunk_dealloc_mmap(chunk, size); + /* Try to merge chunk into the reserve. */ + malloc_mutex_lock(&reserve_mtx); + node = chunk_dealloc_reserve(chunk, size); + malloc_mutex_unlock(&reserve_mtx); + if (node == NULL) + chunk_dealloc_mmap(chunk, size); } /* @@ -3170,8 +3468,49 @@ arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large, * the run. */ if (chunk == NULL) { + uint64_t chunk_seq; + + /* + * Record the chunk allocation sequence number in order + * to detect races. + */ + arena->chunk_seq++; + chunk_seq = arena->chunk_seq; + + /* + * Drop the arena lock while allocating a chunk, since + * reserve notifications may cause recursive + * allocation. Dropping the lock here opens an + * allocataion race, but we recover. + */ + malloc_mutex_unlock(&arena->lock); chunk = (arena_chunk_t *)chunk_alloc(chunksize, true, true); + malloc_mutex_lock(&arena->lock); + + /* + * Check whether a race allowed a usable run to appear. + */ + if (bin != NULL && (run = bin->runcur) != NULL && + run->nfree > 0) { + if (chunk != NULL) + chunk_dealloc(chunk, chunksize); + return (run); + } + + /* + * If this thread raced with another such that multiple + * chunks were allocated, make sure that there is still + * inadequate space before using this chunk. + */ + if (chunk_seq != arena->chunk_seq) + continue; + + /* + * Check for an error *after* checking for a race, + * since a race could also cause a transient OOM + * condition. + */ if (chunk == NULL) return (NULL); } @@ -4451,6 +4790,8 @@ arena_new(arena_t *arena) memset(&arena->stats, 0, sizeof(arena_stats_t)); #endif + arena->chunk_seq = 0; + /* Initialize chunks. */ arena_chunk_tree_dirty_new(&arena->chunks_dirty); arena->spare = NULL; @@ -5099,6 +5440,22 @@ malloc_print_stats(void) allocated, mapped); #endif + malloc_mutex_lock(&reserve_mtx); + malloc_printf("Reserve: min " + "cur max\n"); +#ifdef MOZ_MEMORY_WINDOWS + malloc_printf(" %12lu %12lu %12lu\n", + CHUNK_CEILING(reserve_min) >> opt_chunk_2pow, + reserve_cur >> opt_chunk_2pow, + reserve_max >> opt_chunk_2pow); +#else + malloc_printf(" %12zu %12zu %12zu\n", + CHUNK_CEILING(reserve_min) >> opt_chunk_2pow, + reserve_cur >> opt_chunk_2pow, + reserve_max >> opt_chunk_2pow); +#endif + malloc_mutex_unlock(&reserve_mtx); + #ifdef MALLOC_BALANCE malloc_printf("Arena balance reassignments: %llu\n", nbalance); @@ -5369,6 +5726,12 @@ MALLOC_OUT: else if ((opt_dirty_max << 1) != 0) opt_dirty_max <<= 1; break; + case 'g': + opt_reserve_range_lshift--; + break; + case 'G': + opt_reserve_range_lshift++; + break; #ifdef MALLOC_FILL case 'j': opt_junk = false; @@ -5422,6 +5785,12 @@ MALLOC_OUT: 1) opt_quantum_2pow++; break; + case 'r': + opt_reserve_min_lshift--; + break; + case 'R': + opt_reserve_min_lshift++; + break; case 's': if (opt_small_max_2pow > QUANTUM_2POW_MIN) @@ -5571,6 +5940,7 @@ MALLOC_OUT: base_mapped = 0; #endif base_nodes = NULL; + base_reserve_regs = NULL; malloc_mutex_init(&base_mtx); #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE @@ -5697,6 +6067,27 @@ MALLOC_OUT: return (true); #endif + /* + * Configure and initialize the memory reserve. This needs to happen + * late during initialization, since chunks are allocated. + */ + malloc_mutex_init(&reserve_mtx); + reserve_min = 0; + reserve_cur = 0; + reserve_max = 0; + if (RESERVE_RANGE_2POW_DEFAULT + opt_reserve_range_lshift >= 0) { + reserve_max += chunksize << (RESERVE_RANGE_2POW_DEFAULT + + opt_reserve_range_lshift); + } + ql_new(&reserve_regs); + reserve_seq = 0; + extent_tree_szad_new(&reserve_chunks_szad); + extent_tree_ad_new(&reserve_chunks_ad); + if (RESERVE_MIN_2POW_DEFAULT + opt_reserve_min_lshift >= 0) { + reserve_min_set(chunksize << (RESERVE_MIN_2POW_DEFAULT + + opt_reserve_min_lshift)); + } + malloc_initialized = true; #ifndef MOZ_MEMORY_WINDOWS malloc_mutex_unlock(&init_lock); @@ -6068,6 +6459,12 @@ jemalloc_stats(jemalloc_stats_t *stats) stats->chunksize = chunksize; stats->dirty_max = opt_dirty_max; + malloc_mutex_lock(&reserve_mtx); + stats->reserve_min = reserve_min; + stats->reserve_max = reserve_max; + stats->reserve_cur = reserve_cur; + malloc_mutex_unlock(&reserve_mtx); + /* * Gather current memory usage statistics. */ @@ -6125,6 +6522,454 @@ jemalloc_stats(jemalloc_stats_t *stats) #endif } +void * +xmalloc(size_t size) +{ + void *ret; + + if (malloc_init()) + reserve_fail(size, "xmalloc"); + + if (size == 0) { +#ifdef MALLOC_SYSV + if (opt_sysv == false) +#endif + size = 1; +#ifdef MALLOC_SYSV + else { + _malloc_message(_getprogname(), + ": (malloc) Error in xmalloc(): ", + "invalid size 0", "\n"); + abort(); + } +#endif + } + + ret = imalloc(size); + if (ret == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_crit(size, "xmalloc", seq); + ret = imalloc(size); + } while (ret == NULL); + } + + UTRACE(0, size, ret); + return (ret); +} + +void * +xcalloc(size_t num, size_t size) +{ + void *ret; + size_t num_size; + + num_size = num * size; + if (malloc_init()) + reserve_fail(num_size, "xcalloc"); + + if (num_size == 0) { +#ifdef MALLOC_SYSV + if ((opt_sysv == false) && ((num == 0) || (size == 0))) +#endif + num_size = 1; +#ifdef MALLOC_SYSV + else { + _malloc_message(_getprogname(), + ": (malloc) Error in xcalloc(): ", + "invalid size 0", "\n"); + abort(); + } +#endif + /* + * Try to avoid division here. We know that it isn't possible to + * overflow during multiplication if neither operand uses any of the + * most significant half of the bits in a size_t. + */ + } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) + && (num_size / size != num)) { + /* size_t overflow. */ + _malloc_message(_getprogname(), + ": (malloc) Error in xcalloc(): ", + "size overflow", "\n"); + abort(); + } + + ret = icalloc(num_size); + if (ret == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_crit(num_size, "xcalloc", seq); + ret = icalloc(num_size); + } while (ret == NULL); + } + + UTRACE(0, num_size, ret); + return (ret); +} + +void * +xrealloc(void *ptr, size_t size) +{ + void *ret; + + if (size == 0) { +#ifdef MALLOC_SYSV + if (opt_sysv == false) +#endif + size = 1; +#ifdef MALLOC_SYSV + else { + if (ptr != NULL) + idalloc(ptr); + _malloc_message(_getprogname(), + ": (malloc) Error in xrealloc(): ", + "invalid size 0", "\n"); + abort(); + } +#endif + } + + if (ptr != NULL) { + assert(malloc_initialized); + + ret = iralloc(ptr, size); + if (ret == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_crit(size, "xrealloc", seq); + ret = iralloc(ptr, size); + } while (ret == NULL); + } + } else { + if (malloc_init()) + reserve_fail(size, "xrealloc"); + + ret = imalloc(size); + if (ret == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_crit(size, "xrealloc", seq); + ret = imalloc(size); + } while (ret == NULL); + } + } + + UTRACE(ptr, size, ret); + return (ret); +} + +void * +xmemalign(size_t alignment, size_t size) +{ + void *ret; + + assert(((alignment - 1) & alignment) == 0 && alignment >= + sizeof(void *)); + + if (malloc_init()) + reserve_fail(size, "xmemalign"); + + ret = ipalloc(alignment, size); + if (ret == NULL) { + uint64_t seq = 0; + + do { + seq = reserve_crit(size, "xmemalign", seq); + ret = ipalloc(alignment, size); + } while (ret == NULL); + } + + UTRACE(0, size, ret); + return (ret); +} + +static void +reserve_shrink(void) +{ + extent_node_t *node; + + assert(reserve_cur > reserve_max); +#ifdef MALLOC_DEBUG + { + extent_node_t *node; + size_t reserve_size; + + reserve_size = 0; + rb_foreach_begin(extent_node_t, link_szad, &reserve_chunks_szad, + node) { + reserve_size += node->size; + } rb_foreach_end(extent_node_t, link_szad, &reserve_chunks_szad, + node) + assert(reserve_size == reserve_cur); + + reserve_size = 0; + rb_foreach_begin(extent_node_t, link_ad, &reserve_chunks_ad, + node) { + reserve_size += node->size; + } rb_foreach_end(extent_node_t, link_ad, &reserve_chunks_ad, + node) + assert(reserve_size == reserve_cur); + } +#endif + + /* Discard chunks until the the reserve is below the size limit. */ + rb_foreach_reverse_begin(extent_node_t, link_ad, &reserve_chunks_ad, + node) { +#ifndef MALLOC_DECOMMIT + if (node->size <= reserve_cur - reserve_max) { +#endif + extent_node_t *tnode = extent_tree_ad_prev( + &reserve_chunks_ad, node); + +#ifdef MALLOC_DECOMMIT + assert(node->size <= reserve_cur - reserve_max); +#endif + + /* Discard the entire [multi-]chunk. */ + extent_tree_szad_remove(&reserve_chunks_szad, node); + extent_tree_ad_remove(&reserve_chunks_ad, node); + reserve_cur -= node->size; + pages_unmap(node->addr, node->size); +#ifdef MALLOC_STATS + stats_chunks.curchunks -= (node->size / chunksize); +#endif + base_node_dealloc(node); + if (reserve_cur == reserve_max) + break; + + rb_foreach_reverse_prev(extent_node_t, link_ad, + extent_ad_comp, &reserve_chunks_ad, tnode); +#ifndef MALLOC_DECOMMIT + } else { + /* Discard the end of the multi-chunk. */ + extent_tree_szad_remove(&reserve_chunks_szad, node); + node->size -= reserve_cur - reserve_max; + extent_tree_szad_insert(&reserve_chunks_szad, node); + pages_unmap((void *)((uintptr_t)node->addr + + node->size), reserve_cur - reserve_max); +#ifdef MALLOC_STATS + stats_chunks.curchunks -= ((reserve_cur - reserve_max) / + chunksize); +#endif + reserve_cur = reserve_max; + break; + } +#endif + assert(reserve_cur > reserve_max); + } rb_foreach_reverse_end(extent_node_t, link_ad, &reserve_chunks_ad, + node) +} + +/* Send a condition notification. */ +static uint64_t +reserve_notify(reserve_cnd_t cnd, size_t size, uint64_t seq) +{ + reserve_reg_t *reg; + + /* seq is used to keep track of distinct condition-causing events. */ + if (seq == 0) { + /* Allocate new sequence number. */ + reserve_seq++; + seq = reserve_seq; + } + + /* + * Advance to the next callback registration and send a notification, + * unless one has already been sent for this condition-causing event. + */ + reg = ql_first(&reserve_regs); + if (reg == NULL) + return (0); + ql_first(&reserve_regs) = ql_next(&reserve_regs, reg, link); + if (reg->seq == seq) + return (0); + reg->seq = seq; + malloc_mutex_unlock(&reserve_mtx); + reg->cb(reg->ctx, cnd, size); + malloc_mutex_lock(&reserve_mtx); + + return (seq); +} + +/* Allocation failure due to OOM. Try to free some memory via callbacks. */ +static uint64_t +reserve_crit(size_t size, const char *fname, uint64_t seq) +{ + + /* + * Send one condition notification. Iteration is handled by the + * caller of this function. + */ + malloc_mutex_lock(&reserve_mtx); + seq = reserve_notify(RESERVE_CND_CRIT, size, seq); + malloc_mutex_unlock(&reserve_mtx); + + /* If no notification could be sent, then no further recourse exists. */ + if (seq == 0) + reserve_fail(size, fname); + + return (seq); +} + +/* Permanent allocation failure due to OOM. */ +static void +reserve_fail(size_t size, const char *fname) +{ + uint64_t seq = 0; + + /* Send fail notifications. */ + malloc_mutex_lock(&reserve_mtx); + do { + seq = reserve_notify(RESERVE_CND_FAIL, size, seq); + } while (seq != 0); + malloc_mutex_unlock(&reserve_mtx); + + /* Terminate the application. */ + _malloc_message(_getprogname(), + ": (malloc) Error in ", fname, "(): out of memory\n"); + abort(); +} + +bool +reserve_cb_register(reserve_cb_t *cb, void *ctx) +{ + reserve_reg_t *reg = base_reserve_reg_alloc(); + if (reg == NULL) + return (true); + + ql_elm_new(reg, link); + reg->cb = cb; + reg->ctx = ctx; + reg->seq = 0; + + malloc_mutex_lock(&reserve_mtx); + ql_head_insert(&reserve_regs, reg, link); + malloc_mutex_unlock(&reserve_mtx); + + return (false); +} + +bool +reserve_cb_unregister(reserve_cb_t *cb, void *ctx) +{ + reserve_reg_t *reg = NULL; + + malloc_mutex_lock(&reserve_mtx); + ql_foreach(reg, &reserve_regs, link) { + if (reg->cb == cb && reg->ctx == ctx) { + ql_remove(&reserve_regs, reg, link); + break; + } + } + malloc_mutex_unlock(&reserve_mtx); + + if (reg != NULL) + base_reserve_reg_dealloc(reg); + return (false); + return (true); +} + +size_t +reserve_cur_get(void) +{ + size_t ret; + + malloc_mutex_lock(&reserve_mtx); + ret = reserve_cur; + malloc_mutex_unlock(&reserve_mtx); + + return (ret); +} + +size_t +reserve_min_get(void) +{ + size_t ret; + + malloc_mutex_lock(&reserve_mtx); + ret = reserve_min; + malloc_mutex_unlock(&reserve_mtx); + + return (ret); +} + +bool +reserve_min_set(size_t min) +{ + + min = CHUNK_CEILING(min); + + malloc_mutex_lock(&reserve_mtx); + /* Keep |reserve_max - reserve_min| the same. */ + if (min < reserve_min) { + reserve_max -= reserve_min - min; + reserve_min = min; + } else { + /* Protect against wrap-around. */ + if (reserve_max + min - reserve_min < reserve_max) { + reserve_min = SIZE_T_MAX - (reserve_max - reserve_min) + - chunksize + 1; + reserve_max = SIZE_T_MAX - chunksize + 1; + } else { + reserve_max += min - reserve_min; + reserve_min = min; + } + } + + /* Resize the reserve if necessary. */ + if (reserve_cur < reserve_min) { + size_t size = reserve_min - reserve_cur; + + /* Force the reserve to grow by allocating/deallocating. */ + malloc_mutex_unlock(&reserve_mtx); +#ifdef MALLOC_DECOMMIT + { + void **chunks; + size_t i, n; + + n = size >> opt_chunk_2pow; + chunks = (void**)imalloc(n * sizeof(void *)); + if (chunks == NULL) + return (true); + for (i = 0; i < n; i++) { + chunks[i] = huge_malloc(chunksize, false); + if (chunks[i] == NULL) { + size_t j; + + for (j = 0; j < i; j++) { + huge_dalloc(chunks[j]); + } + idalloc(chunks); + return (true); + } + } + for (i = 0; i < n; i++) + huge_dalloc(chunks[i]); + idalloc(chunks); + } +#else + { + void *x = huge_malloc(size, false); + if (x == NULL) { + return (true); + } + huge_dalloc(x); + } +#endif + } else if (reserve_cur > reserve_max) { + reserve_shrink(); + malloc_mutex_unlock(&reserve_mtx); + } else + malloc_mutex_unlock(&reserve_mtx); + + return (false); +} + #ifdef MOZ_MEMORY_WINDOWS void* _recalloc(void *ptr, size_t count, size_t size) diff --git a/memory/jemalloc/jemalloc.h b/memory/jemalloc/jemalloc.h index 399895deb577..4419bd0ff56c 100644 --- a/memory/jemalloc/jemalloc.h +++ b/memory/jemalloc/jemalloc.h @@ -69,6 +69,8 @@ typedef struct { size_t large_max; /* Max sub-chunksize allocation size. */ size_t chunksize; /* Size of each virtual memory mapping. */ size_t dirty_max; /* Max dirty pages per arena. */ + size_t reserve_min; /* reserve_low callback threshold. */ + size_t reserve_max; /* Maximum reserve size before unmapping. */ /* * Current memory usage statistics. @@ -77,6 +79,7 @@ typedef struct { size_t committed; /* Bytes committed (readable/writable). */ size_t allocated; /* Bytes allocted (in use by application). */ size_t dirty; /* Bytes dirty (committed unused pages). */ + size_t reserve_cur; /* Current memory reserve. */ } jemalloc_stats_t; #ifndef MOZ_MEMORY_DARWIN @@ -92,6 +95,125 @@ void *memalign(size_t alignment, size_t size); size_t malloc_usable_size(const void *ptr); void jemalloc_stats(jemalloc_stats_t *stats); +/* The x*() functions never return NULL. */ +void *xmalloc(size_t size); +void *xcalloc(size_t num, size_t size); +void *xrealloc(void *ptr, size_t size); +void *xmemalign(size_t alignment, size_t size); + +/* + * The allocator maintains a memory reserve that is used to satisfy allocation + * requests when no additional memory can be acquired from the operating + * system. Under normal operating conditions, the reserve size is at least + * reserve_min bytes. If the reserve is depleted or insufficient to satisfy an + * allocation request, then condition notifications are sent to one or more of + * the registered callback functions: + * + * RESERVE_CND_LOW: The reserve had to be used to satisfy an allocation + * request, which dropped the reserve size below the + * minimum. The callee should try to free memory in order + * to restore the reserve. + * + * RESERVE_CND_CRIT: The reserve was not large enough to satisfy a pending + * allocation request. Some callee must free adequate + * memory in order to prevent application failure (unless + * the condition spontaneously desists due to concurrent + * deallocation). + * + * RESERVE_CND_FAIL: An allocation request could not be satisfied, despite all + * attempts. The allocator is about to terminate the + * application. + * + * The order in which the callback functions are called is only loosely + * specified: in the absence of interposing callback + * registrations/unregistrations, enabled callbacks will be called in an + * arbitrary round-robin order. + * + * Condition notifications are sent to callbacks only while conditions exist. + * For example, just before the allocator sends a RESERVE_CND_LOW condition + * notification to a callback, the reserve is in fact depleted. However, due + * to allocator concurrency, the reserve may have been restored by the time the + * callback function executes. Furthermore, if the reserve is restored at some + * point during the delivery of condition notifications to callbacks, no + * further deliveries will occur, since the condition no longer exists. + * + * Callback functions can freely call back into the allocator (i.e. the + * allocator releases all internal resources before calling each callback + * function), though allocation is discouraged, since recursive callbacks are + * likely to result, which places extra burden on the application to avoid + * deadlock. + * + * Callback functions must be thread-safe, since it is possible that multiple + * threads will call into the same callback function concurrently. + */ + +/* Memory reserve condition types. */ +typedef enum { + RESERVE_CND_LOW, + RESERVE_CND_CRIT, + RESERVE_CND_FAIL +} reserve_cnd_t; + +/* + * Reserve condition notification callback function type definition. + * + * Inputs: + * ctx: Opaque application data, as passed to reserve_cb_register(). + * cnd: Condition type being delivered. + * size: Allocation request size for the allocation that caused the condition. + */ +typedef void reserve_cb_t(void *ctx, reserve_cnd_t cnd, size_t size); + +/* + * Register a callback function. + * + * Inputs: + * cb: Callback function pointer. + * ctx: Opaque application data, passed to cb(). + * + * Output: + * ret: If true, failure due to OOM; success otherwise. + */ +jemalloc_bool reserve_cb_register(reserve_cb_t *cb, void *ctx); + +/* + * Unregister a callback function. + * + * Inputs: + * cb: Callback function pointer. + * ctx: Opaque application data, same as that passed to reserve_cb_register(). + * + * Output: + * ret: False upon success, true if the {cb,ctx} registration could not be + * found. + */ +jemalloc_bool reserve_cb_unregister(reserve_cb_t *cb, void *ctx); + +/* + * Get the current reserve size. + * + * ret: Current reserve size. + */ +size_t reserve_cur_get(void); + +/* + * Get the minimum acceptable reserve size. If the reserve drops below this + * value, the RESERVE_CND_LOW condition notification is sent to the callbacks. + * + * ret: Minimum acceptable reserve size. + */ +size_t reserve_min_get(void); + +/* + * Set the minimum acceptable reserve size. + * + * min: Reserve threshold. This value may be internally rounded up. + * ret: False if the reserve was successfully resized; true otherwise. Note + * that failure to resize the reserve also results in a RESERVE_CND_LOW + * condition. + */ +jemalloc_bool reserve_min_set(size_t min); + #ifdef __cplusplus } /* extern "C" */ #endif