diff --git a/memory/build/mozjemalloc.cpp b/memory/build/mozjemalloc.cpp index d6377d00a6f5..0a9155716c3e 100644 --- a/memory/build/mozjemalloc.cpp +++ b/memory/build/mozjemalloc.cpp @@ -4,108 +4,106 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/* - * Portions of this file were originally under the following license: - * - * Copyright (C) 2006-2008 Jason Evans . - * All rights reserved. - * Copyright (C) 2007-2017 Mozilla Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice(s), this list of conditions and the following disclaimer as - * the first lines of this file unmodified other than the possible - * addition of one or more copyright notices. - * 2. Redistributions in binary form must reproduce the above copyright - * notice(s), this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************* - * - * This allocator implementation is designed to provide scalable performance - * for multi-threaded programs on multi-processor systems. The following - * features are included for this purpose: - * - * + Multiple arenas are used if there are multiple CPUs, which reduces lock - * contention and cache sloshing. - * - * + Cache line sharing between arenas is avoided for internal data - * structures. - * - * + Memory is managed in chunks and runs (chunks can be split into runs), - * rather than as individual pages. This provides a constant-time - * mechanism for associating allocations with particular arenas. - * - * Allocation requests are rounded up to the nearest size class, and no record - * of the original request size is maintained. Allocations are broken into - * categories according to size class. Assuming runtime defaults, 4 kB pages - * and a 16 byte quantum on a 32-bit system, the size classes in each category - * are as follows: - * - * |=====================================| - * | Category | Subcategory | Size | - * |=====================================| - * | Small | Tiny | 2 | - * | | | 4 | - * | | | 8 | - * | |----------------+---------| - * | | Quantum-spaced | 16 | - * | | | 32 | - * | | | 48 | - * | | | ... | - * | | | 480 | - * | | | 496 | - * | | | 512 | - * | |----------------+---------| - * | | Sub-page | 1 kB | - * | | | 2 kB | - * |=====================================| - * | Large | 4 kB | - * | | 8 kB | - * | | 12 kB | - * | | ... | - * | | 1012 kB | - * | | 1016 kB | - * | | 1020 kB | - * |=====================================| - * | Huge | 1 MB | - * | | 2 MB | - * | | 3 MB | - * | | ... | - * |=====================================| - * - * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an - * allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is - * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes. - * - * A different mechanism is used for each category: - * - * Small : Each size class is segregated into its own set of runs. Each run - * maintains a bitmap of which regions are free/allocated. - * - * Large : Each allocation is backed by a dedicated run. Metadata are stored - * in the associated arena chunk header maps. - * - * Huge : Each allocation is backed by a dedicated contiguous set of chunks. - * Metadata are stored in a separate red-black tree. - * - ******************************************************************************* - */ +// Portions of this file were originally under the following license: +// +// Copyright (C) 2006-2008 Jason Evans . +// All rights reserved. +// Copyright (C) 2007-2017 Mozilla Foundation. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice(s), this list of conditions and the following disclaimer as +// the first lines of this file unmodified other than the possible +// addition of one or more copyright notices. +// 2. Redistributions in binary form must reproduce the above copyright +// notice(s), this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// ***************************************************************************** +// +// This allocator implementation is designed to provide scalable performance +// for multi-threaded programs on multi-processor systems. The following +// features are included for this purpose: +// +// + Multiple arenas are used if there are multiple CPUs, which reduces lock +// contention and cache sloshing. +// +// + Cache line sharing between arenas is avoided for internal data +// structures. +// +// + Memory is managed in chunks and runs (chunks can be split into runs), +// rather than as individual pages. This provides a constant-time +// mechanism for associating allocations with particular arenas. +// +// Allocation requests are rounded up to the nearest size class, and no record +// of the original request size is maintained. Allocations are broken into +// categories according to size class. Assuming runtime defaults, 4 kB pages +// and a 16 byte quantum on a 32-bit system, the size classes in each category +// are as follows: +// +// |=====================================| +// | Category | Subcategory | Size | +// |=====================================| +// | Small | Tiny | 2 | +// | | | 4 | +// | | | 8 | +// | |----------------+---------| +// | | Quantum-spaced | 16 | +// | | | 32 | +// | | | 48 | +// | | | ... | +// | | | 480 | +// | | | 496 | +// | | | 512 | +// | |----------------+---------| +// | | Sub-page | 1 kB | +// | | | 2 kB | +// |=====================================| +// | Large | 4 kB | +// | | 8 kB | +// | | 12 kB | +// | | ... | +// | | 1012 kB | +// | | 1016 kB | +// | | 1020 kB | +// |=====================================| +// | Huge | 1 MB | +// | | 2 MB | +// | | 3 MB | +// | | ... | +// |=====================================| +// +// NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an +// allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is +// 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes. +// +// A different mechanism is used for each category: +// +// Small : Each size class is segregated into its own set of runs. Each run +// maintains a bitmap of which regions are free/allocated. +// +// Large : Each allocation is backed by a dedicated run. Metadata are stored +// in the associated arena chunk header maps. +// +// Huge : Each allocation is backed by a dedicated contiguous set of chunks. +// Metadata are stored in a separate red-black tree. +// +// ***************************************************************************** #include "mozmemory_wrap.h" #include "mozjemalloc.h" @@ -117,27 +115,26 @@ #include "mozilla/UniquePtr.h" #include "mozilla/Unused.h" -/* - * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the - * operating system. If we release 1MB of live pages with MADV_DONTNEED, our - * RSS will decrease by 1MB (almost) immediately. - * - * On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE - * on Mac doesn't cause the OS to release the specified pages immediately; the - * OS keeps them in our process until the machine comes under memory pressure. - * - * It's therefore difficult to measure the process's RSS on Mac, since, in the - * absence of memory pressure, the contribution from the heap to RSS will not - * decrease due to our madvise calls. - * - * We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to - * track which pages have been MADV_FREE'd. You can then call - * jemalloc_purge_freed_pages(), which will force the OS to release those - * MADV_FREE'd pages, making the process's RSS reflect its true memory usage. - * - * The jemalloc_purge_freed_pages definition in memory/build/mozmemory.h needs - * to be adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux. - */ +// On Linux, we use madvise(MADV_DONTNEED) to release memory back to the +// operating system. If we release 1MB of live pages with MADV_DONTNEED, our +// RSS will decrease by 1MB (almost) immediately. +// +// On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE +// on Mac doesn't cause the OS to release the specified pages immediately; the +// OS keeps them in our process until the machine comes under memory pressure. +// +// It's therefore difficult to measure the process's RSS on Mac, since, in the +// absence of memory pressure, the contribution from the heap to RSS will not +// decrease due to our madvise calls. +// +// We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to +// track which pages have been MADV_FREE'd. You can then call +// jemalloc_purge_freed_pages(), which will force the OS to release those +// MADV_FREE'd pages, making the process's RSS reflect its true memory usage. +// +// The jemalloc_purge_freed_pages definition in memory/build/mozmemory.h needs +// to be adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux. + #ifdef XP_DARWIN #define MALLOC_DOUBLE_PURGE #endif @@ -154,7 +151,7 @@ #ifdef XP_WIN -/* Some defines from the CRT internal headers that we need here. */ +// Some defines from the CRT internal headers that we need here. #define _CRT_SPINCOUNT 5000 #include #include @@ -163,7 +160,7 @@ #define SIZE_T_MAX SIZE_MAX #define STDERR_FILENO 2 -/* use MSVC intrinsics */ +// Use MSVC intrinsics. #pragma intrinsic(_BitScanForward) static __forceinline int ffs(int x) @@ -176,7 +173,7 @@ ffs(int x) return 0; } -/* Implement getenv without using malloc */ +// Implement getenv without using malloc. static char mozillaMallocOptionsBuf[64]; #define getenv xgetenv @@ -251,18 +248,17 @@ typedef long ssize_t; #include "mozilla/ThreadLocal.h" #include "mozjemalloc_types.h" -/* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that - * happen to override mmap() and call dlsym() from their overridden - * mmap(). The problem is that dlsym() calls malloc(), and this ends - * up in a dead lock in jemalloc. - * On these systems, we prefer to directly use the system call. - * We do that for Linux systems and kfreebsd with GNU userland. - * Note sanity checks are not done (alignment of offset, ...) because - * the uses of mmap are pretty limited, in jemalloc. - * - * On Alpha, glibc has a bug that prevents syscall() to work for system - * calls with 6 arguments - */ +// Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that +// happen to override mmap() and call dlsym() from their overridden +// mmap(). The problem is that dlsym() calls malloc(), and this ends +// up in a dead lock in jemalloc. +// On these systems, we prefer to directly use the system call. +// We do that for Linux systems and kfreebsd with GNU userland. +// Note sanity checks are not done (alignment of offset, ...) because +// the uses of mmap are pretty limited, in jemalloc. +// +// On Alpha, glibc has a bug that prevents syscall() to work for system +// calls with 6 arguments. #if (defined(XP_LINUX) && !defined(__alpha__)) || \ (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) #include @@ -271,8 +267,8 @@ static inline void *_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { -/* S390 only passes one argument to the mmap system call, which is a - * pointer to a structure containing the arguments */ +// S390 only passes one argument to the mmap system call, which is a +// pointer to a structure containing the arguments. #ifdef __s390__ struct { void *addr; @@ -285,7 +281,7 @@ void *_mmap(void *addr, size_t length, int prot, int flags, return (void *) syscall(SYS_mmap, &args); #else #if defined(ANDROID) && defined(__aarch64__) && defined(SYS_mmap2) -/* Android NDK defines SYS_mmap2 for AArch64 despite it not supporting mmap2 */ +// Android NDK defines SYS_mmap2 for AArch64 despite it not supporting mmap2. #undef SYS_mmap2 #endif #ifdef SYS_mmap2 @@ -302,10 +298,10 @@ void *_mmap(void *addr, size_t length, int prot, int flags, #endif #endif -/* Size of stack-allocated buffer passed to strerror_r(). */ +// Size of stack-allocated buffer passed to strerror_r(). #define STRERROR_BUF 64 -/* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */ +// Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. # define QUANTUM_2POW_MIN 4 #if defined(_WIN64) || defined(__LP64__) # define SIZEOF_PTR_2POW 3 @@ -317,104 +313,88 @@ void *_mmap(void *addr, size_t length, int prot, int flags, #include "rb.h" -/* sizeof(int) == (1U << SIZEOF_INT_2POW). */ +// sizeof(int) == (1U << SIZEOF_INT_2POW). #ifndef SIZEOF_INT_2POW # define SIZEOF_INT_2POW 2 #endif -/* - * Size and alignment of memory chunks that are allocated by the OS's virtual - * memory system. - */ +// Size and alignment of memory chunks that are allocated by the OS's virtual +// memory system. #define CHUNK_2POW_DEFAULT 20 -/* Maximum number of dirty pages per arena. */ +// Maximum number of dirty pages per arena. #define DIRTY_MAX_DEFAULT (1U << 8) -/* - * Maximum size of L1 cache line. This is used to avoid cache line aliasing, - * so over-estimates are okay (up to a point), but under-estimates will - * negatively affect performance. - */ +// Maximum size of L1 cache line. This is used to avoid cache line aliasing, +// so over-estimates are okay (up to a point), but under-estimates will +// negatively affect performance. #define CACHELINE_2POW 6 #define CACHELINE ((size_t)(1U << CACHELINE_2POW)) -/* - * Smallest size class to support. On Windows the smallest allocation size - * must be 8 bytes on 32-bit, 16 bytes on 64-bit. On Linux and Mac, even - * malloc(1) must reserve a word's worth of memory (see Mozilla bug 691003). - */ +// Smallest size class to support. On Windows the smallest allocation size +// must be 8 bytes on 32-bit, 16 bytes on 64-bit. On Linux and Mac, even +// malloc(1) must reserve a word's worth of memory (see Mozilla bug 691003). #ifdef XP_WIN #define TINY_MIN_2POW (sizeof(void*) == 8 ? 4 : 3) #else #define TINY_MIN_2POW (sizeof(void*) == 8 ? 3 : 2) #endif -/* - * Maximum size class that is a multiple of the quantum, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ +// Maximum size class that is a multiple of the quantum, but not (necessarily) +// a power of 2. Above this size, allocations are rounded up to the nearest +// power of 2. #define SMALL_MAX_2POW_DEFAULT 9 #define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT) -/* - * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized - * as small as possible such that this setting is still honored, without - * violating other constraints. The goal is to make runs as small as possible - * without exceeding a per run external fragmentation threshold. - * - * We use binary fixed point math for overhead computations, where the binary - * point is implicitly RUN_BFP bits to the left. - * - * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be - * honored for some/all object sizes, since there is one bit of header overhead - * per object (plus a constant). This constraint is relaxed (ignored) for runs - * that are so small that the per-region overhead is greater than: - * - * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) - */ +// RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized +// as small as possible such that this setting is still honored, without +// violating other constraints. The goal is to make runs as small as possible +// without exceeding a per run external fragmentation threshold. +// +// We use binary fixed point math for overhead computations, where the binary +// point is implicitly RUN_BFP bits to the left. +// +// Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be +// honored for some/all object sizes, since there is one bit of header overhead +// per object (plus a constant). This constraint is relaxed (ignored) for runs +// that are so small that the per-region overhead is greater than: +// +// (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) #define RUN_BFP 12 -/* \/ Implicit binary fixed point. */ +// \/ Implicit binary fixed point. #define RUN_MAX_OVRHD 0x0000003dU #define RUN_MAX_OVRHD_RELAX 0x00001800U -/* - * When MALLOC_STATIC_PAGESIZE is defined, the page size is fixed at - * compile-time for better performance, as opposed to determined at - * runtime. Some platforms can have different page sizes at runtime - * depending on kernel configuration, so they are opted out by default. - * Debug builds are opted out too, for test coverage. - */ +// When MALLOC_STATIC_PAGESIZE is defined, the page size is fixed at +// compile-time for better performance, as opposed to determined at +// runtime. Some platforms can have different page sizes at runtime +// depending on kernel configuration, so they are opted out by default. +// Debug builds are opted out too, for test coverage. #ifndef MOZ_DEBUG #if !defined(__ia64__) && !defined(__sparc__) && !defined(__mips__) && !defined(__aarch64__) #define MALLOC_STATIC_PAGESIZE 1 #endif #endif -/* Various quantum-related settings. */ - +// Various quantum-related settings. #define QUANTUM_DEFAULT (size_t(1) << QUANTUM_2POW_MIN) static const size_t quantum = QUANTUM_DEFAULT; static const size_t quantum_mask = QUANTUM_DEFAULT - 1; -/* Various bin-related settings. */ - +// Various bin-related settings. static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1; static const size_t small_max = size_t(SMALL_MAX_DEFAULT); -/* Number of (2^n)-spaced tiny bins. */ +// Number of (2^n)-spaced tiny bins. static const unsigned ntbins = unsigned(QUANTUM_2POW_MIN - TINY_MIN_2POW); - /* Number of quantum-spaced bins. */ +// Number of quantum-spaced bins. static const unsigned nqbins = unsigned(SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN); #ifdef MALLOC_STATIC_PAGESIZE -/* - * VM page size. It must divide the runtime CPU page size or the code - * will abort. - * Platform specific page size conditions copied from js/public/HeapAPI.h - */ +// VM page size. It must divide the runtime CPU page size or the code +// will abort. +// Platform specific page size conditions copied from js/public/HeapAPI.h #if (defined(SOLARIS) || defined(__FreeBSD__)) && \ (defined(__sparc) || defined(__sparcv9) || defined(__ia64)) #define pagesize_2pow (size_t(13)) @@ -426,32 +406,30 @@ static const unsigned nqbins = unsigned(SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN); #define pagesize (size_t(1) << pagesize_2pow) #define pagesize_mask (pagesize - 1) -/* Max size class for bins. */ +// Max size class for bins. static const size_t bin_maxclass = pagesize >> 1; -/* Number of (2^n)-spaced sub-page bins. */ +// Number of (2^n)-spaced sub-page bins. static const unsigned nsbins = unsigned(pagesize_2pow - SMALL_MAX_2POW_DEFAULT - 1); -#else /* !MALLOC_STATIC_PAGESIZE */ +#else // !MALLOC_STATIC_PAGESIZE -/* VM page size. */ +// VM page size. static size_t pagesize; static size_t pagesize_mask; static size_t pagesize_2pow; -/* Various bin-related settings. */ -static size_t bin_maxclass; /* Max size class for bins. */ -static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */ +// Various bin-related settings. +static size_t bin_maxclass; // Max size class for bins. +static unsigned nsbins; // Number of (2^n)-spaced sub-page bins. #endif -/* Various chunk-related settings. */ +// Various chunk-related settings. -/* - * Compute the header size such that it is large enough to contain the page map - * and enough nodes for the worst case: one node per non-header page plus one - * extra for situations where we briefly have one more node allocated than we - * will need. - */ +// Compute the header size such that it is large enough to contain the page map +// and enough nodes for the worst case: one node per non-header page plus one +// extra for situations where we briefly have one more node allocated than we +// will need. #define calculate_arena_header_size() \ (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1)) @@ -459,7 +437,7 @@ static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */ ((calculate_arena_header_size() >> pagesize_2pow) + \ ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0)) -/* Max size class for arenas. */ +// Max size class for arenas. #define calculate_arena_maxclass() \ (chunksize - (arena_chunk_header_npages << pagesize_2pow)) @@ -474,32 +452,27 @@ static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow; #else static size_t chunk_npages; static size_t arena_chunk_header_npages; -static size_t arena_maxclass; /* Max size class for arenas. */ +static size_t arena_maxclass; // Max size class for arenas. #endif -/* - * Recycle at most 128 chunks. With 1 MiB chunks, this means we retain at most - * 6.25% of the process address space on a 32-bit OS for later use. - */ +// Recycle at most 128 chunks. With 1 MiB chunks, this means we retain at most +// 6.25% of the process address space on a 32-bit OS for later use. #define CHUNK_RECYCLE_LIMIT 128 static const size_t gRecycleLimit = CHUNK_RECYCLE_LIMIT * CHUNKSIZE_DEFAULT; -/* The current amount of recycled bytes, updated atomically. */ +// The current amount of recycled bytes, updated atomically. static mozilla::Atomic gRecycledSize; -/******************************************************************************/ - -/* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */ +// *************************************************************************** +// MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE) #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. #endif -/* - * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all - * places, because they require malloc()ed memory, which causes bootstrapping - * issues in some cases. - */ +// Mutexes based on spinlocks. We can't use normal pthread spinlocks in all +// places, because they require malloc()ed memory, which causes bootstrapping +// issues in some cases. struct Mutex { #if defined(XP_WIN) @@ -533,11 +506,11 @@ private: Mutex& mMutex; }; -/* Set to true once the allocator has been initialized. */ +// Set to true once the allocator has been initialized. static bool malloc_initialized = false; #if defined(XP_WIN) -/* No init lock for Windows. */ +// No init lock for Windows. #elif defined(XP_DARWIN) static Mutex gInitLock = { OS_SPINLOCK_INIT }; #elif defined(XP_LINUX) && !defined(ANDROID) @@ -546,57 +519,53 @@ static Mutex gInitLock = { PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP }; static Mutex gInitLock = { PTHREAD_MUTEX_INITIALIZER }; #endif -/******************************************************************************/ -/* - * Statistics data structures. - */ +// *************************************************************************** +// Statistics data structures. struct malloc_bin_stats_t { - /* Current number of runs in this bin. */ + // Current number of runs in this bin. unsigned long curruns; }; struct arena_stats_t { - /* Number of bytes currently mapped. */ + // Number of bytes currently mapped. size_t mapped; - /* Current number of committed pages. */ + // Current number of committed pages. size_t committed; - /* Per-size-category statistics. */ + // Per-size-category statistics. size_t allocated_small; size_t allocated_large; }; -/******************************************************************************/ -/* - * Extent data structures. - */ +// *************************************************************************** +// Extent data structures. enum ChunkType { UNKNOWN_CHUNK, - ZEROED_CHUNK, // chunk only contains zeroes - ARENA_CHUNK, // used to back arena runs created by arena_t::AllocRun - HUGE_CHUNK, // used to back huge allocations (e.g. huge_malloc) - RECYCLED_CHUNK, // chunk has been stored for future use by chunk_recycle + ZEROED_CHUNK, // chunk only contains zeroes. + ARENA_CHUNK, // used to back arena runs created by arena_t::AllocRun. + HUGE_CHUNK, // used to back huge allocations (e.g. huge_malloc). + RECYCLED_CHUNK, // chunk has been stored for future use by chunk_recycle. }; -/* Tree of extents. */ +// Tree of extents. struct extent_node_t { - /* Linkage for the size/address-ordered tree. */ + // Linkage for the size/address-ordered tree. RedBlackTreeNode link_szad; - /* Linkage for the address-ordered tree. */ + // Linkage for the address-ordered tree. RedBlackTreeNode link_ad; - /* Pointer to the extent that this tree node is responsible for. */ + // Pointer to the extent that this tree node is responsible for. void *addr; - /* Total region size. */ + // Total region size. size_t size; - /* What type of chunk is there; used by chunk recycling code. */ + // What type of chunk is there; used by chunk recycling code. ChunkType chunk_type; }; @@ -654,22 +623,19 @@ struct ExtentTreeBoundsTrait : public ExtentTreeTrait } }; -/******************************************************************************/ -/* - * Radix tree data structures. - * - * The number of bits passed to the template is the number of significant bits - * in an address to do a radix lookup with. - * - * An address is looked up by splitting it in kBitsPerLevel bit chunks, except - * the most significant bits, where the bit chunk is kBitsAtLevel1 which can be - * different if Bits is not a multiple of kBitsPerLevel. - * - * With e.g. sizeof(void*)=4, Bits=16 and kBitsPerLevel=8, an address is split - * like the following: - * 0x12345678 -> mRoot[0x12][0x34] - */ - +// *************************************************************************** +// Radix tree data structures. +// +// The number of bits passed to the template is the number of significant bits +// in an address to do a radix lookup with. +// +// An address is looked up by splitting it in kBitsPerLevel bit chunks, except +// the most significant bits, where the bit chunk is kBitsAtLevel1 which can be +// different if Bits is not a multiple of kBitsPerLevel. +// +// With e.g. sizeof(void*)=4, Bits=16 and kBitsPerLevel=8, an address is split +// like the following: +// 0x12345678 -> mRoot[0x12][0x34] template class AddressRadixTree { // Size of each radix tree node (as a power of 2). @@ -694,7 +660,7 @@ public: inline void* Get(void* aAddr); - // Returns whether the value was properly set + // Returns whether the value was properly set. inline bool Set(void* aAddr, void* aValue); inline bool Unset(void* aAddr) @@ -706,85 +672,78 @@ private: inline void** GetSlot(void* aAddr, bool aCreate = false); }; -/******************************************************************************/ -/* - * Arena data structures. - */ +// *************************************************************************** +// Arena data structures. struct arena_t; struct arena_bin_t; -/* Each element of the chunk map corresponds to one page within the chunk. */ +// Each element of the chunk map corresponds to one page within the chunk. struct arena_chunk_map_t { - /* - * Linkage for run trees. There are two disjoint uses: - * - * 1) arena_t's tree or available runs. - * 2) arena_run_t conceptually uses this linkage for in-use non-full - * runs, rather than directly embedding linkage. - */ + // Linkage for run trees. There are two disjoint uses: + // + // 1) arena_t's tree or available runs. + // 2) arena_run_t conceptually uses this linkage for in-use non-full + // runs, rather than directly embedding linkage. RedBlackTreeNode link; - /* - * Run address (or size) and various flags are stored together. The bit - * layout looks like (assuming 32-bit system): - * - * ???????? ???????? ????---- -mckdzla - * - * ? : Unallocated: Run address for first/last pages, unset for internal - * pages. - * Small: Run address. - * Large: Run size for first page, unset for trailing pages. - * - : Unused. - * m : MADV_FREE/MADV_DONTNEED'ed? - * c : decommitted? - * k : key? - * d : dirty? - * z : zeroed? - * l : large? - * a : allocated? - * - * Following are example bit patterns for the three types of runs. - * - * r : run address - * s : run size - * x : don't care - * - : 0 - * [cdzla] : bit set - * - * Unallocated: - * ssssssss ssssssss ssss---- --c----- - * xxxxxxxx xxxxxxxx xxxx---- ----d--- - * ssssssss ssssssss ssss---- -----z-- - * - * Small: - * rrrrrrrr rrrrrrrr rrrr---- -------a - * rrrrrrrr rrrrrrrr rrrr---- -------a - * rrrrrrrr rrrrrrrr rrrr---- -------a - * - * Large: - * ssssssss ssssssss ssss---- ------la - * -------- -------- -------- ------la - * -------- -------- -------- ------la - */ + // Run address (or size) and various flags are stored together. The bit + // layout looks like (assuming 32-bit system): + // + // ???????? ???????? ????---- -mckdzla + // + // ? : Unallocated: Run address for first/last pages, unset for internal + // pages. + // Small: Run address. + // Large: Run size for first page, unset for trailing pages. + // - : Unused. + // m : MADV_FREE/MADV_DONTNEED'ed? + // c : decommitted? + // k : key? + // d : dirty? + // z : zeroed? + // l : large? + // a : allocated? + // + // Following are example bit patterns for the three types of runs. + // + // r : run address + // s : run size + // x : don't care + // - : 0 + // [cdzla] : bit set + // + // Unallocated: + // ssssssss ssssssss ssss---- --c----- + // xxxxxxxx xxxxxxxx xxxx---- ----d--- + // ssssssss ssssssss ssss---- -----z-- + // + // Small: + // rrrrrrrr rrrrrrrr rrrr---- -------a + // rrrrrrrr rrrrrrrr rrrr---- -------a + // rrrrrrrr rrrrrrrr rrrr---- -------a + // + // Large: + // ssssssss ssssssss ssss---- ------la + // -------- -------- -------- ------la + // -------- -------- -------- ------la size_t bits; -/* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether - * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined. - * - * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be - * re-committed with pages_commit() before it may be touched. If - * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined. - * - * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which - * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with - * CHUNK_MAP_MADVISED. - * - * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is - * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED. - * When it's finally freed with jemalloc_purge_freed_pages, the page is marked - * as CHUNK_MAP_DECOMMITTED. - */ +// Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether +// MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined. +// +// If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be +// re-committed with pages_commit() before it may be touched. If +// MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined. +// +// If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which +// are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with +// CHUNK_MAP_MADVISED. +// +// Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is +// defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED. +// When it's finally freed with jemalloc_purge_freed_pages, the page is marked +// as CHUNK_MAP_DECOMMITTED. #define CHUNK_MAP_MADVISED ((size_t)0x40U) #define CHUNK_MAP_DECOMMITTED ((size_t)0x20U) #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED) @@ -794,6 +753,7 @@ struct arena_chunk_map_t { #define CHUNK_MAP_LARGE ((size_t)0x02U) #define CHUNK_MAP_ALLOCATED ((size_t)0x01U) }; + struct ArenaChunkMapLink { static RedBlackTreeNode& GetTreeNode(arena_chunk_map_t* aThis) @@ -823,29 +783,29 @@ struct ArenaAvailTreeTrait : public ArenaChunkMapLink } }; -/* Arena chunk header. */ +// Arena chunk header. struct arena_chunk_t { - /* Arena that owns the chunk. */ + // Arena that owns the chunk. arena_t *arena; - /* Linkage for the arena's tree of dirty chunks. */ + // Linkage for the arena's tree of dirty chunks. RedBlackTreeNode link_dirty; #ifdef MALLOC_DOUBLE_PURGE - /* If we're double-purging, we maintain a linked list of chunks which - * have pages which have been madvise(MADV_FREE)'d but not explicitly - * purged. - * - * We're currently lazy and don't remove a chunk from this list when - * all its madvised pages are recommitted. */ + // If we're double-purging, we maintain a linked list of chunks which + // have pages which have been madvise(MADV_FREE)'d but not explicitly + // purged. + // + // We're currently lazy and don't remove a chunk from this list when + // all its madvised pages are recommitted. mozilla::DoublyLinkedListElement chunks_madvised_elem; #endif - /* Number of dirty pages. */ + // Number of dirty pages. size_t ndirty; - /* Map of pages within chunk that keeps track of free/large/small. */ - arena_chunk_map_t map[1]; /* Dynamically sized. */ + // Map of pages within chunk that keeps track of free/large/small. + arena_chunk_map_t map[1]; // Dynamically sized. }; struct ArenaDirtyChunkTrait @@ -884,51 +844,47 @@ struct arena_run_t { # define ARENA_RUN_MAGIC 0x384adf93 #endif - /* Bin this run is associated with. */ + // Bin this run is associated with. arena_bin_t *bin; - /* Index of first element that might have a free region. */ + // Index of first element that might have a free region. unsigned regs_minelm; - /* Number of free regions in run. */ + // Number of free regions in run. unsigned nfree; - /* Bitmask of in-use regions (0: in use, 1: free). */ - unsigned regs_mask[1]; /* Dynamically sized. */ + // Bitmask of in-use regions (0: in use, 1: free). + unsigned regs_mask[1]; // Dynamically sized. }; struct arena_bin_t { - /* - * Current run being used to service allocations of this bin's size - * class. - */ + // Current run being used to service allocations of this bin's size + // class. arena_run_t *runcur; - /* - * Tree of non-full runs. This tree is used when looking for an - * existing run when runcur is no longer usable. We choose the - * non-full run that is lowest in memory; this policy tends to keep - * objects packed well, and it can also help reduce the number of - * almost-empty chunks. - */ + // Tree of non-full runs. This tree is used when looking for an + // existing run when runcur is no longer usable. We choose the + // non-full run that is lowest in memory; this policy tends to keep + // objects packed well, and it can also help reduce the number of + // almost-empty chunks. RedBlackTree runs; - /* Size of regions in a run for this bin's size class. */ + // Size of regions in a run for this bin's size class. size_t reg_size; - /* Total size of a run for this bin's size class. */ + // Total size of a run for this bin's size class. size_t run_size; - /* Total number of regions in a run for this bin's size class. */ + // Total number of regions in a run for this bin's size class. uint32_t nregs; - /* Number of elements in a run's regs_mask for this bin's size class. */ + // Number of elements in a run's regs_mask for this bin's size class. uint32_t regs_mask_nelms; - /* Offset of first region in a run for this bin's size class. */ + // Offset of first region in a run for this bin's size class. uint32_t reg0_offset; - /* Bin statistics. */ + // Bin statistics. malloc_bin_stats_t stats; }; @@ -939,81 +895,72 @@ struct arena_t { #endif arena_id_t mId; - /* Linkage for the tree of arenas by id. */ + // Linkage for the tree of arenas by id. RedBlackTreeNode mLink; - /* All operations on this arena require that lock be locked. */ + // All operations on this arena require that lock be locked. Mutex mLock; arena_stats_t mStats; private: - /* Tree of dirty-page-containing chunks this arena manages. */ + // Tree of dirty-page-containing chunks this arena manages. RedBlackTree mChunksDirty; #ifdef MALLOC_DOUBLE_PURGE - /* Head of a linked list of MADV_FREE'd-page-containing chunks this - * arena manages. */ + // Head of a linked list of MADV_FREE'd-page-containing chunks this + // arena manages. mozilla::DoublyLinkedList mChunksMAdvised; #endif - /* - * In order to avoid rapid chunk allocation/deallocation when an arena - * oscillates right on the cusp of needing a new chunk, cache the most - * recently freed chunk. The spare is left in the arena's chunk trees - * until it is deleted. - * - * There is one spare chunk per arena, rather than one spare total, in - * order to avoid interactions between multiple threads that could make - * a single spare inadequate. - */ + // In order to avoid rapid chunk allocation/deallocation when an arena + // oscillates right on the cusp of needing a new chunk, cache the most + // recently freed chunk. The spare is left in the arena's chunk trees + // until it is deleted. + // + // There is one spare chunk per arena, rather than one spare total, in + // order to avoid interactions between multiple threads that could make + // a single spare inadequate. arena_chunk_t* mSpare; public: - /* - * Current count of pages within unused runs that are potentially - * dirty, and for which madvise(... MADV_FREE) has not been called. By - * tracking this, we can institute a limit on how much dirty unused - * memory is mapped for each arena. - */ + // Current count of pages within unused runs that are potentially + // dirty, and for which madvise(... MADV_FREE) has not been called. By + // tracking this, we can institute a limit on how much dirty unused + // memory is mapped for each arena. size_t mNumDirty; - /* - * Maximum value allowed for mNumDirty. - */ + + // Maximum value allowed for mNumDirty. size_t mMaxDirty; private: - /* - * Size/address-ordered tree of this arena's available runs. This tree - * is used for first-best-fit run allocation. - */ + // Size/address-ordered tree of this arena's available runs. This tree + // is used for first-best-fit run allocation. RedBlackTree mRunsAvail; public: - /* - * mBins is used to store rings of free regions of the following sizes, - * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS. - * - * mBins[i] | size | - * --------+------+ - * 0 | 2 | - * 1 | 4 | - * 2 | 8 | - * --------+------+ - * 3 | 16 | - * 4 | 32 | - * 5 | 48 | - * 6 | 64 | - * : : - * : : - * 33 | 496 | - * 34 | 512 | - * --------+------+ - * 35 | 1024 | - * 36 | 2048 | - * --------+------+ - */ - arena_bin_t mBins[1]; /* Dynamically sized. */ + // mBins is used to store rings of free regions of the following sizes, + // assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS. + // + // mBins[i] | size | + // --------+------+ + // 0 | 2 | + // 1 | 4 | + // 2 | 8 | + // --------+------+ + // 3 | 16 | + // 4 | 32 | + // 5 | 48 | + // 6 | 64 | + // : : + // : : + // 33 | 496 | + // 34 | 512 | + // --------+------+ + // 35 | 1024 | + // 36 | 2048 | + // --------+------+ + arena_bin_t mBins[1]; // Dynamically sized. bool Init(); @@ -1077,72 +1024,60 @@ struct ArenaTreeTrait } }; -/********/ -/* - * Chunks. - */ - +// ****** +// Chunks. static AddressRadixTree<(SIZEOF_PTR << 3) - CHUNK_2POW_DEFAULT> gChunkRTree; -/* Protects chunk-related data structures. */ +// Protects chunk-related data structures. static Mutex chunks_mtx; -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ +// Trees of chunks that were previously allocated (trees differ only in node +// ordering). These are used when allocating chunks, in an attempt to re-use +// address space. Depending on function, different tree orderings are needed, +// which is why there are two trees with the same contents. static RedBlackTree gChunksBySize; static RedBlackTree gChunksByAddress; -/* Protects huge allocation-related data structures. */ +// Protects huge allocation-related data structures. static Mutex huge_mtx; -/* Tree of chunks that are stand-alone huge allocations. */ +// Tree of chunks that are stand-alone huge allocations. static RedBlackTree huge; -/* Huge allocation statistics. */ +// Huge allocation statistics. static size_t huge_allocated; static size_t huge_mapped; -/****************************/ -/* - * base (internal allocation). - */ +// ************************** +// base (internal allocation). + +// Current pages that are being used for internal memory allocations. These +// pages are carved up in cacheline-size quanta, so that there is no chance of +// false cache line sharing. -/* - * Current pages that are being used for internal memory allocations. These - * pages are carved up in cacheline-size quanta, so that there is no chance of - * false cache line sharing. - */ static void *base_pages; static void *base_next_addr; static void *base_next_decommitted; -static void *base_past_addr; /* Addr immediately past base_pages. */ +static void *base_past_addr; // Addr immediately past base_pages. static extent_node_t *base_nodes; static Mutex base_mtx; static size_t base_mapped; static size_t base_committed; -/********/ -/* - * Arenas. - */ +// ****** +// Arenas. // A tree of all available arenas, arranged by id. // TODO: Move into arena_t as a static member when rb_tree doesn't depend on // the type being defined anymore. static RedBlackTree gArenaTree; static unsigned narenas; -static Mutex arenas_lock; /* Protects arenas initialization. */ +static Mutex arenas_lock; // Protects arenas initialization. -/* - * The arena associated with the current thread (per jemalloc_thread_local_arena) - * On OSX, __thread/thread_local circles back calling malloc to allocate storage - * on first access on each thread, which leads to an infinite loop, but - * pthread-based TLS somehow doesn't have this problem. - */ +// The arena associated with the current thread (per jemalloc_thread_local_arena) +// On OSX, __thread/thread_local circles back calling malloc to allocate storage +// on first access on each thread, which leads to an infinite loop, but +// pthread-based TLS somehow doesn't have this problem. #if !defined(XP_DARWIN) static MOZ_THREAD_LOCAL(arena_t*) thread_arena; #else @@ -1153,10 +1088,9 @@ static mozilla::detail::ThreadLocal= s. */ +// Return the smallest chunk multiple that is >= s. #define CHUNK_CEILING(s) \ (((s) + chunksize_mask) & ~chunksize_mask) -/* Return the smallest cacheline multiple that is >= s. */ +// Return the smallest cacheline multiple that is >= s. #define CACHELINE_CEILING(s) \ (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1)) -/* Return the smallest quantum multiple that is >= a. */ +// Return the smallest quantum multiple that is >= a. #define QUANTUM_CEILING(a) \ (((a) + quantum_mask) & ~quantum_mask) -/* Return the smallest pagesize multiple that is >= s. */ +// Return the smallest pagesize multiple that is >= s. #define PAGE_CEILING(s) \ (((s) + pagesize_mask) & ~pagesize_mask) -/* Compute the smallest power of 2 that is >= x. */ +// Compute the smallest power of 2 that is >= x. static inline size_t pow2_ceil(size_t x) { @@ -1361,18 +1285,16 @@ _getprogname(void) return ""; } -/******************************************************************************/ +// *************************************************************************** static inline void pages_decommit(void* aAddr, size_t aSize) { #ifdef XP_WIN - /* - * The region starting at addr may have been allocated in multiple calls - * to VirtualAlloc and recycled, so decommitting the entire region in one - * go may not be valid. However, since we allocate at least a chunk at a - * time, we may touch any region in chunksized increments. - */ + // The region starting at addr may have been allocated in multiple calls + // to VirtualAlloc and recycled, so decommitting the entire region in one + // go may not be valid. However, since we allocate at least a chunk at a + // time, we may touch any region in chunksized increments. size_t pages_size = std::min(aSize, chunksize - GetChunkOffsetForPtr(aAddr)); while (aSize > 0) { if (!VirtualFree(aAddr, pages_size, MEM_DECOMMIT)) { @@ -1391,17 +1313,15 @@ pages_decommit(void* aAddr, size_t aSize) #endif } -/* Commit pages. Returns whether pages were committed. */ +// Commit pages. Returns whether pages were committed. MOZ_MUST_USE static inline bool pages_commit(void* aAddr, size_t aSize) { #ifdef XP_WIN - /* - * The region starting at addr may have been allocated in multiple calls - * to VirtualAlloc and recycled, so committing the entire region in one - * go may not be valid. However, since we allocate at least a chunk at a - * time, we may touch any region in chunksized increments. - */ + // The region starting at addr may have been allocated in multiple calls + // to VirtualAlloc and recycled, so committing the entire region in one + // go may not be valid. However, since we allocate at least a chunk at a + // time, we may touch any region in chunksized increments. size_t pages_size = std::min(aSize, chunksize - GetChunkOffsetForPtr(aAddr)); while (aSize > 0) { if (!VirtualAlloc(aAddr, pages_size, MEM_COMMIT, PAGE_READWRITE)) { @@ -1439,10 +1359,8 @@ base_pages_alloc(size_t minsize) } base_next_addr = base_pages; base_past_addr = (void *)((uintptr_t)base_pages + csize); - /* - * Leave enough pages for minsize committed, since otherwise they would - * have to be immediately recommitted. - */ + // Leave enough pages for minsize committed, since otherwise they would + // have to be immediately recommitted. pminsize = PAGE_CEILING(minsize); base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize); # if defined(MALLOC_DECOMMIT) @@ -1462,20 +1380,20 @@ base_alloc(size_t aSize) void* ret; size_t csize; - /* Round size up to nearest multiple of the cacheline size. */ + // Round size up to nearest multiple of the cacheline size. csize = CACHELINE_CEILING(aSize); MutexAutoLock lock(base_mtx); - /* Make sure there's enough space for the allocation. */ + // Make sure there's enough space for the allocation. if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) { if (base_pages_alloc(csize)) { return nullptr; } } - /* Allocate. */ + // Allocate. ret = base_next_addr; base_next_addr = (void*)((uintptr_t)base_next_addr + csize); - /* Make sure enough pages are committed for the new allocation. */ + // Make sure enough pages are committed for the new allocation. if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) { void* pbase_next_addr = (void*)(PAGE_CEILING((uintptr_t)base_next_addr)); @@ -1537,13 +1455,9 @@ struct BaseNodeFreePolicy using UniqueBaseNode = mozilla::UniquePtr; -/* - * End Utility functions/macros. - */ -/******************************************************************************/ -/* - * Begin chunk management functions. - */ +// End Utility functions/macros. +// *************************************************************************** +// Begin chunk management functions. #ifdef XP_WIN @@ -1584,19 +1498,17 @@ pages_map(void *aAddr, size_t aSize) { void *ret; #if defined(__ia64__) || (defined(__sparc__) && defined(__arch64__) && defined(__linux__)) - /* - * The JS engine assumes that all allocated pointers have their high 17 bits clear, - * which ia64's mmap doesn't support directly. However, we can emulate it by passing - * mmap an "addr" parameter with those bits clear. The mmap will return that address, - * or the nearest available memory above that address, providing a near-guarantee - * that those bits are clear. If they are not, we return nullptr below to indicate - * out-of-memory. - * - * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual - * address space. - * - * See Bug 589735 for more information. - */ + // The JS engine assumes that all allocated pointers have their high 17 bits clear, + // which ia64's mmap doesn't support directly. However, we can emulate it by passing + // mmap an "addr" parameter with those bits clear. The mmap will return that address, + // or the nearest available memory above that address, providing a near-guarantee + // that those bits are clear. If they are not, we return nullptr below to indicate + // out-of-memory. + // + // The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual + // address space. + // + // See Bug 589735 for more information. bool check_placement = true; if (!aAddr) { aAddr = (void*)0x0000070000000000; @@ -1608,8 +1520,7 @@ pages_map(void *aAddr, size_t aSize) const uintptr_t start = 0x0000070000000000ULL; const uintptr_t end = 0x0000800000000000ULL; - /* Copied from js/src/gc/Memory.cpp and adapted for this source */ - + // Copied from js/src/gc/Memory.cpp and adapted for this source uintptr_t hint; void* region = MAP_FAILED; for (hint = start; region == MAP_FAILED && hint + aSize <= end; hint += chunksize) { @@ -1625,11 +1536,8 @@ pages_map(void *aAddr, size_t aSize) } ret = region; #else - - /* - * We don't use MAP_FIXED here, because it can cause the *replacement* - * of existing mappings, and we only want to create new mappings. - */ + // We don't use MAP_FIXED here, because it can cause the *replacement* + // of existing mappings, and we only want to create new mappings. ret = mmap(aAddr, aSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); MOZ_ASSERT(ret); @@ -1638,22 +1546,18 @@ pages_map(void *aAddr, size_t aSize) ret = nullptr; } #if defined(__ia64__) || (defined(__sparc__) && defined(__arch64__) && defined(__linux__)) - /* - * If the allocated memory doesn't have its upper 17 bits clear, consider it - * as out of memory. - */ + // If the allocated memory doesn't have its upper 17 bits clear, consider it + // as out of memory. else if ((long long)ret & 0xffff800000000000) { munmap(ret, aSize); ret = nullptr; } - /* If the caller requested a specific memory location, verify that's what mmap returned. */ + // If the caller requested a specific memory location, verify that's what mmap returned. else if (check_placement && ret != aAddr) { #else else if (aAddr && ret != aAddr) { #endif - /* - * We succeeded in mapping memory, but not in the right place. - */ + // We succeeded in mapping memory, but not in the right place. pages_unmap(ret, aSize); ret = nullptr; } @@ -1723,10 +1627,8 @@ AddressRadixTree::GetSlot(void* aKey, bool aCreate) } } - /* - * node is a leaf, so it contains values rather than node - * pointers. - */ + // node is a leaf, so it contains values rather than node + // pointers. bits = i ? kBitsPerLevel : kBitsAtLevel1; subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); return &node[subkey]; @@ -1745,15 +1647,14 @@ AddressRadixTree::Get(void* aKey) } #ifdef MOZ_DEBUG MutexAutoLock lock(mLock); - /* - * Suppose that it were possible for a jemalloc-allocated chunk to be - * munmap()ped, followed by a different allocator in another thread re-using - * overlapping virtual memory, all without invalidating the cached rtree - * value. The result would be a false positive (the rtree would claim that - * jemalloc owns memory that it had actually discarded). I don't think this - * scenario is possible, but the following assertion is a prudent sanity - * check. - */ + + // Suppose that it were possible for a jemalloc-allocated chunk to be + // munmap()ped, followed by a different allocator in another thread re-using + // overlapping virtual memory, all without invalidating the cached rtree + // value. The result would be a false positive (the rtree would claim that + // jemalloc owns memory that it had actually discarded). I don't think this + // scenario is possible, but the following assertion is a prudent sanity + // check. if (!slot) { // In case a slot has been created in the meantime. slot = GetSlot(aKey); @@ -1774,21 +1675,21 @@ bool AddressRadixTree::Set(void* aKey, void* aValue) { MutexAutoLock lock(mLock); - void** slot = GetSlot(aKey, /* create */ true); + void** slot = GetSlot(aKey, /* create = */ true); if (slot) { *slot = aValue; } return slot; } -/* pages_trim, chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked - * from upstream jemalloc 3.4.1 to fix Mozilla bug 956501. */ +// pages_trim, chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked +// from upstream jemalloc 3.4.1 to fix Mozilla bug 956501. -/* Return the offset between a and the nearest aligned address at or below a. */ +// Return the offset between a and the nearest aligned address at or below a. #define ALIGNMENT_ADDR2OFFSET(a, alignment) \ ((size_t)((uintptr_t)(a) & (alignment - 1))) -/* Return the smallest alignment multiple that is >= s. */ +// Return the smallest alignment multiple that is >= s. #define ALIGNMENT_CEILING(s, alignment) \ (((s) + (alignment - 1)) & (~(alignment - 1))) @@ -1834,7 +1735,7 @@ chunk_alloc_mmap_slow(size_t size, size_t alignment) size_t alloc_size, leadsize; alloc_size = size + alignment - pagesize; - /* Beware size_t wrap-around. */ + // Beware size_t wrap-around. if (alloc_size < size) { return nullptr; } @@ -1858,19 +1759,16 @@ chunk_alloc_mmap(size_t size, size_t alignment) void *ret; size_t offset; - /* - * Ideally, there would be a way to specify alignment to mmap() (like - * NetBSD has), but in the absence of such a feature, we have to work - * hard to efficiently create aligned mappings. The reliable, but - * slow method is to create a mapping that is over-sized, then trim the - * excess. However, that always results in one or two calls to - * pages_unmap(). - * - * Optimistically try mapping precisely the right amount before falling - * back to the slow method, with the expectation that the optimistic - * approach works most of the time. - */ - + // Ideally, there would be a way to specify alignment to mmap() (like + // NetBSD has), but in the absence of such a feature, we have to work + // hard to efficiently create aligned mappings. The reliable, but + // slow method is to create a mapping that is over-sized, then trim the + // excess. However, that always results in one or two calls to + // pages_unmap(). + // + // Optimistically try mapping precisely the right amount before falling + // back to the slow method, with the expectation that the optimistic + // approach works most of the time. ret = pages_map(nullptr, size); if (!ret) { return nullptr; @@ -1885,13 +1783,12 @@ chunk_alloc_mmap(size_t size, size_t alignment) return ret; } -/* Purge and release the pages in the chunk of length `length` at `addr` to - * the OS. - * Returns whether the pages are guaranteed to be full of zeroes when the - * function returns. - * The force_zero argument explicitly requests that the memory is guaranteed - * to be full of zeroes when the function returns. - */ +// Purge and release the pages in the chunk of length `length` at `addr` to +// the OS. +// Returns whether the pages are guaranteed to be full of zeroes when the +// function returns. +// The force_zero argument explicitly requests that the memory is guaranteed +// to be full of zeroes when the function returns. static bool pages_purge(void *addr, size_t length, bool force_zero) { @@ -1905,12 +1802,10 @@ pages_purge(void *addr, size_t length, bool force_zero) } # endif # ifdef XP_WIN - /* - * The region starting at addr may have been allocated in multiple calls - * to VirtualAlloc and recycled, so resetting the entire region in one - * go may not be valid. However, since we allocate at least a chunk at a - * time, we may touch any region in chunksized increments. - */ + // The region starting at addr may have been allocated in multiple calls + // to VirtualAlloc and recycled, so resetting the entire region in one + // go may not be valid. However, since we allocate at least a chunk at a + // time, we may touch any region in chunksized increments. size_t pages_size = std::min(length, chunksize - GetChunkOffsetForPtr(addr)); while (length > 0) { @@ -1924,7 +1819,7 @@ pages_purge(void *addr, size_t length, bool force_zero) # ifdef XP_LINUX # define JEMALLOC_MADV_PURGE MADV_DONTNEED # define JEMALLOC_MADV_ZEROS true -# else /* FreeBSD and Darwin. */ +# else // FreeBSD and Darwin. # define JEMALLOC_MADV_PURGE MADV_FREE # define JEMALLOC_MADV_ZEROS force_zero # endif @@ -1942,7 +1837,7 @@ chunk_recycle(size_t aSize, size_t aAlignment, bool* aZeroed) extent_node_t key; size_t alloc_size = aSize + aAlignment - chunksize; - /* Beware size_t wrap-around. */ + // Beware size_t wrap-around. if (alloc_size < aSize) { return nullptr; } @@ -1963,26 +1858,24 @@ chunk_recycle(size_t aSize, size_t aAlignment, bool* aZeroed) if (aZeroed) { *aZeroed = (chunk_type == ZEROED_CHUNK); } - /* Remove node from the tree. */ + // Remove node from the tree. gChunksBySize.Remove(node); gChunksByAddress.Remove(node); if (leadsize != 0) { - /* Insert the leading space as a smaller chunk. */ + // Insert the leading space as a smaller chunk. node->size = leadsize; gChunksBySize.Insert(node); gChunksByAddress.Insert(node); node = nullptr; } if (trailsize != 0) { - /* Insert the trailing space as a smaller chunk. */ + // Insert the trailing space as a smaller chunk. if (!node) { - /* - * An additional node is required, but - * base_node_alloc() can cause a new base chunk to be - * allocated. Drop chunks_mtx in order to avoid - * deadlock, and if node allocation fails, deallocate - * the result before returning an error. - */ + // An additional node is required, but + // base_node_alloc() can cause a new base chunk to be + // allocated. Drop chunks_mtx in order to avoid + // deadlock, and if node allocation fails, deallocate + // the result before returning an error. chunks_mtx.Unlock(); node = base_node_alloc(); if (!node) { @@ -2019,24 +1912,21 @@ chunk_recycle(size_t aSize, size_t aAlignment, bool* aZeroed) } #ifdef XP_WIN -/* - * On Windows, calls to VirtualAlloc and VirtualFree must be matched, making it - * awkward to recycle allocations of varying sizes. Therefore we only allow - * recycling when the size equals the chunksize, unless deallocation is entirely - * disabled. - */ +// On Windows, calls to VirtualAlloc and VirtualFree must be matched, making it +// awkward to recycle allocations of varying sizes. Therefore we only allow +// recycling when the size equals the chunksize, unless deallocation is entirely +// disabled. #define CAN_RECYCLE(size) (size == chunksize) #else #define CAN_RECYCLE(size) true #endif -/* Allocates `size` bytes of system memory aligned for `alignment`. - * `base` indicates whether the memory will be used for the base allocator - * (e.g. base_alloc). - * `zeroed` is an outvalue that returns whether the allocated memory is - * guaranteed to be full of zeroes. It can be omitted when the caller doesn't - * care about the result. - */ +// Allocates `size` bytes of system memory aligned for `alignment`. +// `base` indicates whether the memory will be used for the base allocator +// (e.g. base_alloc). +// `zeroed` is an outvalue that returns whether the allocated memory is +// guaranteed to be full of zeroes. It can be omitted when the caller doesn't +// care about the result. static void* chunk_alloc(size_t aSize, size_t aAlignment, bool aBase, bool* aZeroed) { @@ -2098,28 +1988,24 @@ chunk_record(void* aChunk, size_t aSize, ChunkType aType) } } - /* - * Allocate a node before acquiring chunks_mtx even though it might not - * be needed, because base_node_alloc() may cause a new base chunk to - * be allocated, which could cause deadlock if chunks_mtx were already - * held. - */ + // Allocate a node before acquiring chunks_mtx even though it might not + // be needed, because base_node_alloc() may cause a new base chunk to + // be allocated, which could cause deadlock if chunks_mtx were already + // held. UniqueBaseNode xnode(base_node_alloc()); - /* Use xprev to implement conditional deferred deallocation of prev. */ + // Use xprev to implement conditional deferred deallocation of prev. UniqueBaseNode xprev; - /* RAII deallocates xnode and xprev defined above after unlocking - * in order to avoid potential dead-locks */ + // RAII deallocates xnode and xprev defined above after unlocking + // in order to avoid potential dead-locks MutexAutoLock lock(chunks_mtx); key.addr = (void*)((uintptr_t)aChunk + aSize); extent_node_t* node = gChunksByAddress.SearchOrNext(&key); - /* Try to coalesce forward. */ + // Try to coalesce forward. if (node && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. This does - * not change the position within gChunksByAddress, so only - * remove/insert from/into gChunksBySize. - */ + // Coalesce chunk with the following address range. This does + // not change the position within gChunksByAddress, so only + // remove/insert from/into gChunksBySize. gChunksBySize.Remove(node); node->addr = aChunk; node->size += aSize; @@ -2128,14 +2014,12 @@ chunk_record(void* aChunk, size_t aSize, ChunkType aType) } gChunksBySize.Insert(node); } else { - /* Coalescing forward failed, so insert a new node. */ + // Coalescing forward failed, so insert a new node. if (!xnode) { - /* - * base_node_alloc() failed, which is an exceedingly - * unlikely failure. Leak chunk; its pages have - * already been purged, so this is only a virtual - * memory leak. - */ + // base_node_alloc() failed, which is an exceedingly + // unlikely failure. Leak chunk; its pages have + // already been purged, so this is only a virtual + // memory leak. return; } node = xnode.release(); @@ -2146,14 +2030,12 @@ chunk_record(void* aChunk, size_t aSize, ChunkType aType) gChunksBySize.Insert(node); } - /* Try to coalesce backward. */ + // Try to coalesce backward. extent_node_t* prev = gChunksByAddress.Prev(node); if (prev && (void*)((uintptr_t)prev->addr + prev->size) == aChunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within gChunksByAddress, so only - * remove/insert node from/into gChunksBySize. - */ + // Coalesce chunk with the previous address range. This does + // not change the position within gChunksByAddress, so only + // remove/insert node from/into gChunksBySize. gChunksBySize.Remove(prev); gChunksByAddress.Remove(prev); @@ -2204,13 +2086,9 @@ chunk_dealloc(void* aChunk, size_t aSize, ChunkType aType) #undef CAN_RECYCLE -/* - * End chunk management functions. - */ -/******************************************************************************/ -/* - * Begin arena. - */ +// End chunk management functions. +// *************************************************************************** +// Begin arena. static inline arena_t * thread_local_arena(bool enabled) @@ -2218,10 +2096,10 @@ thread_local_arena(bool enabled) arena_t *arena; if (enabled) { - /* The arena will essentially be leaked if this function is - * called with `false`, but it doesn't matter at the moment. - * because in practice nothing actually calls this function - * with `false`, except maybe at shutdown. */ + // The arena will essentially be leaked if this function is + // called with `false`, but it doesn't matter at the moment. + // because in practice nothing actually calls this function + // with `false`, except maybe at shutdown. arena = arenas_extend(); } else { arena = gMainArena; @@ -2236,19 +2114,16 @@ MozJemalloc::jemalloc_thread_local_arena(bool aEnabled) thread_local_arena(aEnabled); } -/* - * Choose an arena based on a per-thread value. - */ +// Choose an arena based on a per-thread value. static inline arena_t * choose_arena(size_t size) { arena_t *ret = nullptr; - /* - * We can only use TLS if this is a PIC library, since for the static - * library version, libc's malloc is used by TLS allocation, which - * introduces a bootstrapping issue. - */ + // We can only use TLS if this is a PIC library, since for the static + // library version, libc's malloc is used by TLS allocation, which + // introduces a bootstrapping issue. + // Only use a thread local arena for small sizes. if (size <= small_max) { ret = thread_arena.get(); @@ -2270,15 +2145,13 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) MOZ_ASSERT(run->magic == ARENA_RUN_MAGIC); MOZ_ASSERT(run->regs_minelm < bin->regs_mask_nelms); - /* - * Move the first check outside the loop, so that run->regs_minelm can - * be updated unconditionally, without the possibility of updating it - * multiple times. - */ + // Move the first check outside the loop, so that run->regs_minelm can + // be updated unconditionally, without the possibility of updating it + // multiple times. i = run->regs_minelm; mask = run->regs_mask[i]; if (mask != 0) { - /* Usable allocation found. */ + // Usable allocation found. bit = ffs((int)mask) - 1; regind = ((i << (SIZEOF_INT_2POW + 3)) + bit); @@ -2286,7 +2159,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) ret = (void *)(((uintptr_t)run) + bin->reg0_offset + (bin->reg_size * regind)); - /* Clear bit. */ + // Clear bit. mask ^= (1U << bit); run->regs_mask[i] = mask; @@ -2296,7 +2169,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) for (i++; i < bin->regs_mask_nelms; i++) { mask = run->regs_mask[i]; if (mask != 0) { - /* Usable allocation found. */ + // Usable allocation found. bit = ffs((int)mask) - 1; regind = ((i << (SIZEOF_INT_2POW + 3)) + bit); @@ -2304,20 +2177,18 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) ret = (void *)(((uintptr_t)run) + bin->reg0_offset + (bin->reg_size * regind)); - /* Clear bit. */ + // Clear bit. mask ^= (1U << bit); run->regs_mask[i] = mask; - /* - * Make a note that nothing before this element - * contains a free region. - */ - run->regs_minelm = i; /* Low payoff: + (mask == 0); */ + // Make a note that nothing before this element + // contains a free region. + run->regs_minelm = i; // Low payoff: + (mask == 0); return ret; } } - /* Not reached. */ + // Not reached. MOZ_DIAGNOSTIC_ASSERT(0); return nullptr; } @@ -2325,16 +2196,15 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) static inline void arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) { - /* - * To divide by a number D that is not a power of two we multiply - * by (2^21 / D) and then right shift by 21 positions. - * - * X / D - * - * becomes - * - * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT - */ + // To divide by a number D that is not a power of two we multiply + // by (2^21 / D) and then right shift by 21 positions. + // + // X / D + // + // becomes + // + // (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT + #define SIZE_INV_SHIFT 21 #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1) static const unsigned size_invs[] = { @@ -2364,18 +2234,14 @@ arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) MOZ_ASSERT(((sizeof(size_invs)) / sizeof(unsigned)) + 3 >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN)); - /* - * Avoid doing division with a variable divisor if possible. Using - * actual division here can reduce allocator throughput by over 20%! - */ + // Avoid doing division with a variable divisor if possible. Using + // actual division here can reduce allocator throughput by over 20%! diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset); if ((size & (size - 1)) == 0) { - /* - * log2_table allows fast division of a power of two in the - * [1..128] range. - * - * (x / divisor) becomes (x >> log2_table[divisor - 1]). - */ + // log2_table allows fast division of a power of two in the + // [1..128] range. + // + // (x / divisor) becomes (x >> log2_table[divisor - 1]). static const unsigned char log2_table[] = { 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, @@ -2392,10 +2258,8 @@ arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) } else if (size <= 32768) { regind = diff >> (8 + log2_table[(size >> 8) - 1]); } else { - /* - * The run size is too large for us to use the lookup - * table. Use real division. - */ + // The run size is too large for us to use the lookup + // table. Use real division. regind = diff / size; } } else if (size <= ((sizeof(size_invs) / sizeof(unsigned)) @@ -2403,12 +2267,10 @@ arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff; regind >>= SIZE_INV_SHIFT; } else { - /* - * size_invs isn't large enough to handle this size class, so - * calculate regind using actual division. This only happens - * if the user increases small_max via the 'S' runtime - * configuration option. - */ + // size_invs isn't large enough to handle this size class, so + // calculate regind using actual division. This only happens + // if the user increases small_max via the 'S' runtime + // configuration option. regind = diff / size; }; MOZ_DIAGNOSTIC_ASSERT(diff == regind * size); @@ -2441,23 +2303,19 @@ arena_t::SplitRun(arena_run_t* aRun, size_t aSize, bool aLarge, bool aZero) rem_pages = total_pages - need_pages; for (i = 0; i < need_pages; i++) { - /* - * Commit decommitted pages if necessary. If a decommitted - * page is encountered, commit all needed adjacent decommitted - * pages in one operation, in order to reduce system call - * overhead. - */ + // Commit decommitted pages if necessary. If a decommitted + // page is encountered, commit all needed adjacent decommitted + // pages in one operation, in order to reduce system call + // overhead. if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) { size_t j; - /* - * Advance i+j to just past the index of the last page - * to commit. Clear CHUNK_MAP_DECOMMITTED and - * CHUNK_MAP_MADVISED along the way. - */ + // Advance i+j to just past the index of the last page + // to commit. Clear CHUNK_MAP_DECOMMITTED and + // CHUNK_MAP_MADVISED along the way. for (j = 0; i + j < need_pages && (chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) { - /* DECOMMITTED and MADVISED are mutually exclusive. */ + // DECOMMITTED and MADVISED are mutually exclusive. MOZ_ASSERT(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED && chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED)); @@ -2486,7 +2344,7 @@ arena_t::SplitRun(arena_run_t* aRun, size_t aSize, bool aLarge, bool aZero) mRunsAvail.Remove(&chunk->map[run_ind]); - /* Keep track of trailing unused pages for later use. */ + // Keep track of trailing unused pages for later use. if (rem_pages > 0) { chunk->map[run_ind+need_pages].bits = (rem_pages << pagesize_2pow) | (chunk->map[run_ind+need_pages].bits & @@ -2498,23 +2356,23 @@ arena_t::SplitRun(arena_run_t* aRun, size_t aSize, bool aLarge, bool aZero) } for (i = 0; i < need_pages; i++) { - /* Zero if necessary. */ + // Zero if necessary. if (aZero) { if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED) == 0) { memset((void*)(uintptr_t(chunk) + ((run_ind + i) << pagesize_2pow)), 0, pagesize); - /* CHUNK_MAP_ZEROED is cleared below. */ + // CHUNK_MAP_ZEROED is cleared below. } } - /* Update dirty page accounting. */ + // Update dirty page accounting. if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) { chunk->ndirty--; mNumDirty--; - /* CHUNK_MAP_DIRTY is cleared below. */ + // CHUNK_MAP_DIRTY is cleared below. } - /* Initialize the chunk map. */ + // Initialize the chunk map. if (aLarge) { chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; } else { @@ -2522,12 +2380,10 @@ arena_t::SplitRun(arena_run_t* aRun, size_t aSize, bool aLarge, bool aZero) } } - /* - * Set the run size only in the first element for large runs. This is - * primarily a debugging aid, since the lack of size info for trailing - * pages only matters if the application tries to operate on an - * interior pointer. - */ + // Set the run size only in the first element for large runs. This is + // primarily a debugging aid, since the lack of size info for trailing + // pages only matters if the application tries to operate on an + // interior pointer. if (aLarge) { chunk->map[run_ind].bits |= aSize; } @@ -2542,16 +2398,16 @@ void arena_t::InitChunk(arena_chunk_t* aChunk, bool aZeroed) { size_t i; - /* WARNING: The following relies on !aZeroed meaning "used to be an arena - * chunk". - * When the chunk we're initializating as an arena chunk is zeroed, we - * mark all runs are decommitted and zeroed. - * When it is not, which we can assume means it's a recycled arena chunk, - * all it can contain is an arena chunk header (which we're overwriting), - * and zeroed or poisoned memory (because a recycled arena chunk will - * have been emptied before being recycled). In that case, we can get - * away with reusing the chunk as-is, marking all runs as madvised. - */ + // WARNING: The following relies on !aZeroed meaning "used to be an arena + // chunk". + // When the chunk we're initializating as an arena chunk is zeroed, we + // mark all runs are decommitted and zeroed. + // When it is not, which we can assume means it's a recycled arena chunk, + // all it can contain is an arena chunk header (which we're overwriting), + // and zeroed or poisoned memory (because a recycled arena chunk will + // have been emptied before being recycled). In that case, we can get + // away with reusing the chunk as-is, marking all runs as madvised. + size_t flags = aZeroed ? CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED : CHUNK_MAP_MADVISED; @@ -2559,12 +2415,10 @@ arena_t::InitChunk(arena_chunk_t* aChunk, bool aZeroed) aChunk->arena = this; - /* - * Claim that no pages are in use, since the header is merely overhead. - */ + // Claim that no pages are in use, since the header is merely overhead. aChunk->ndirty = 0; - /* Initialize the map to contain one maximal free untouched run. */ + // Initialize the map to contain one maximal free untouched run. #ifdef MALLOC_DECOMMIT arena_run_t* run = (arena_run_t*)(uintptr_t(aChunk) + (arena_chunk_header_npages << pagesize_2pow)); @@ -2580,15 +2434,13 @@ arena_t::InitChunk(arena_chunk_t* aChunk, bool aZeroed) aChunk->map[chunk_npages-1].bits = arena_maxclass | flags; #ifdef MALLOC_DECOMMIT - /* - * Start out decommitted, in order to force a closer correspondence - * between dirty pages and committed untouched pages. - */ + // Start out decommitted, in order to force a closer correspondence + // between dirty pages and committed untouched pages. pages_decommit(run, arena_maxclass); #endif mStats.committed += arena_chunk_header_npages; - /* Insert the run into the tree of available runs. */ + // Insert the run into the tree of available runs. mRunsAvail.Insert(&aChunk->map[arena_chunk_header_npages]); #ifdef MALLOC_DOUBLE_PURGE @@ -2617,11 +2469,9 @@ arena_t::DeallocChunk(arena_chunk_t* aChunk) mStats.committed -= arena_chunk_header_npages; } - /* - * Remove run from the tree of available runs, so that the arena does not use it. - * Dirty page flushing only uses the tree of dirty chunks, so leaving this - * chunk in the chunks_* trees is sufficient for that purpose. - */ + // Remove run from the tree of available runs, so that the arena does not use it. + // Dirty page flushing only uses the tree of dirty chunks, so leaving this + // chunk in the chunks_* trees is sufficient for that purpose. mRunsAvail.Remove(&aChunk->map[arena_chunk_header_npages]); mSpare = aChunk; @@ -2637,7 +2487,7 @@ arena_t::AllocRun(arena_bin_t* aBin, size_t aSize, bool aLarge, bool aZero) MOZ_ASSERT(aSize <= arena_maxclass); MOZ_ASSERT((aSize & pagesize_mask) == 0); - /* Search the arena's chunks for the lowest best fit. */ + // Search the arena's chunks for the lowest best fit. key.bits = aSize | CHUNK_MAP_KEY; mapelm = mRunsAvail.SearchOrNext(&key); if (mapelm) { @@ -2647,17 +2497,15 @@ arena_t::AllocRun(arena_bin_t* aBin, size_t aSize, bool aLarge, bool aZero) run = (arena_run_t*)(uintptr_t(chunk) + (pageind << pagesize_2pow)); } else if (mSpare) { - /* Use the spare. */ + // Use the spare. arena_chunk_t* chunk = mSpare; mSpare = nullptr; run = (arena_run_t*)(uintptr_t(chunk) + (arena_chunk_header_npages << pagesize_2pow)); - /* Insert the run into the tree of available runs. */ + // Insert the run into the tree of available runs. mRunsAvail.Insert(&chunk->map[arena_chunk_header_npages]); } else { - /* - * No usable runs. Create a new chunk from which to allocate - * the run. - */ + // No usable runs. Create a new chunk from which to allocate + // the run. bool zeroed; arena_chunk_t* chunk = (arena_chunk_t*) chunk_alloc(chunksize, chunksize, false, &zeroed); @@ -2668,7 +2516,7 @@ arena_t::AllocRun(arena_bin_t* aBin, size_t aSize, bool aLarge, bool aZero) InitChunk(chunk, zeroed); run = (arena_run_t*)(uintptr_t(chunk) + (arena_chunk_header_npages << pagesize_2pow)); } - /* Update page map. */ + // Update page map. return SplitRun(run, aSize, aLarge, aZero) ? run : nullptr; } @@ -2677,7 +2525,7 @@ arena_t::Purge(bool aAll) { arena_chunk_t* chunk; size_t i, npages; - /* If all is set purge all dirty pages. */ + // If all is set purge all dirty pages. size_t dirty_max = aAll ? 1 : mMaxDirty; #ifdef MOZ_DEBUG size_t ndirty = 0; @@ -2688,12 +2536,10 @@ arena_t::Purge(bool aAll) #endif MOZ_DIAGNOSTIC_ASSERT(aAll || (mNumDirty > mMaxDirty)); - /* - * Iterate downward through chunks until enough dirty memory has been - * purged. Terminate as soon as possible in order to minimize the - * number of system calls, even if a chunk has only been partially - * purged. - */ + // Iterate downward through chunks until enough dirty memory has been + // purged. Terminate as soon as possible in order to minimize the + // number of system calls, even if a chunk has only been partially + // purged. while (mNumDirty > (dirty_max >> 1)) { #ifdef MALLOC_DOUBLE_PURGE bool madvised = false; @@ -2713,7 +2559,7 @@ arena_t::Purge(bool aAll) MOZ_ASSERT((chunk->map[i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0); chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY; - /* Find adjacent dirty run(s). */ + // Find adjacent dirty run(s). for (npages = 1; i > arena_chunk_header_npages && (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY); @@ -2750,8 +2596,8 @@ arena_t::Purge(bool aAll) } #ifdef MALLOC_DOUBLE_PURGE if (madvised) { - /* The chunk might already be in the list, but this - * makes sure it's at the front. */ + // The chunk might already be in the list, but this + // makes sure it's at the front. if (mChunksMAdvised.ElementProbablyInList(chunk)) { mChunksMAdvised.remove(chunk); } @@ -2778,7 +2624,7 @@ arena_t::DallocRun(arena_run_t* aRun, bool aDirty) } run_pages = (size >> pagesize_2pow); - /* Mark pages as unallocated in the chunk map. */ + // Mark pages as unallocated in the chunk map. if (aDirty) { size_t i; @@ -2806,16 +2652,14 @@ arena_t::DallocRun(arena_run_t* aRun, bool aDirty) chunk->map[run_ind+run_pages-1].bits = size | (chunk->map[run_ind+run_pages-1].bits & pagesize_mask); - /* Try to coalesce forward. */ + // Try to coalesce forward. if (run_ind + run_pages < chunk_npages && (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) { size_t nrun_size = chunk->map[run_ind+run_pages].bits & ~pagesize_mask; - /* - * Remove successor from tree of available runs; the coalesced run is - * inserted later. - */ + // Remove successor from tree of available runs; the coalesced run is + // inserted later. mRunsAvail.Remove(&chunk->map[run_ind+run_pages]); size += nrun_size; @@ -2829,17 +2673,15 @@ arena_t::DallocRun(arena_run_t* aRun, bool aDirty) (chunk->map[run_ind+run_pages-1].bits & pagesize_mask); } - /* Try to coalesce backward. */ + // Try to coalesce backward. if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits & CHUNK_MAP_ALLOCATED) == 0) { size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask; run_ind -= prun_size >> pagesize_2pow; - /* - * Remove predecessor from tree of available runs; the coalesced run is - * inserted later. - */ + // Remove predecessor from tree of available runs; the coalesced run is + // inserted later. mRunsAvail.Remove(&chunk->map[run_ind]); size += prun_size; @@ -2853,16 +2695,16 @@ arena_t::DallocRun(arena_run_t* aRun, bool aDirty) (chunk->map[run_ind+run_pages-1].bits & pagesize_mask); } - /* Insert into tree of available runs, now that coalescing is complete. */ + // Insert into tree of available runs, now that coalescing is complete. mRunsAvail.Insert(&chunk->map[run_ind]); - /* Deallocate chunk if it is now completely unused. */ + // Deallocate chunk if it is now completely unused. if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask | CHUNK_MAP_ALLOCATED)) == arena_maxclass) { DeallocChunk(chunk); } - /* Enforce mMaxDirty. */ + // Enforce mMaxDirty. if (mNumDirty > mMaxDirty) { Purge(false); } @@ -2877,10 +2719,8 @@ arena_t::TrimRunHead(arena_chunk_t* aChunk, arena_run_t* aRun, size_t aOldSize, MOZ_ASSERT(aOldSize > aNewSize); - /* - * Update the chunk map so that arena_t::RunDalloc() can treat the - * leading run as separately allocated. - */ + // Update the chunk map so that arena_t::RunDalloc() can treat the + // leading run as separately allocated. aChunk->map[pageind].bits = (aOldSize - aNewSize) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; aChunk->map[pageind+head_npages].bits = aNewSize | CHUNK_MAP_LARGE | @@ -2898,10 +2738,8 @@ arena_t::TrimRunTail(arena_chunk_t* aChunk, arena_run_t* aRun, size_t aOldSize, MOZ_ASSERT(aOldSize > aNewSize); - /* - * Update the chunk map so that arena_t::RunDalloc() can treat the - * trailing run as separately allocated. - */ + // Update the chunk map so that arena_t::RunDalloc() can treat the + // trailing run as separately allocated. aChunk->map[pageind].bits = aNewSize | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; aChunk->map[pageind+npages].bits = (aOldSize - aNewSize) | CHUNK_MAP_LARGE @@ -2917,30 +2755,28 @@ arena_t::GetNonFullBinRun(arena_bin_t* aBin) arena_run_t* run; unsigned i, remainder; - /* Look for a usable run. */ + // Look for a usable run. mapelm = aBin->runs.First(); if (mapelm) { - /* run is guaranteed to have available space. */ + // run is guaranteed to have available space. aBin->runs.Remove(mapelm); run = (arena_run_t*)(mapelm->bits & ~pagesize_mask); return run; } - /* No existing runs have any space available. */ + // No existing runs have any space available. - /* Allocate a new run. */ + // Allocate a new run. run = AllocRun(aBin, aBin->run_size, false, false); if (!run) { return nullptr; } - /* - * Don't initialize if a race in arena_t::RunAlloc() allowed an existing - * run to become usable. - */ + // Don't initialize if a race in arena_t::RunAlloc() allowed an existing + // run to become usable. if (run == aBin->runcur) { return run; } - /* Initialize run internals. */ + // Initialize run internals. run->bin = aBin; for (i = 0; i < aBin->regs_mask_nelms - 1; i++) { @@ -2950,7 +2786,7 @@ arena_t::GetNonFullBinRun(arena_bin_t* aBin) if (remainder == 0) { run->regs_mask[i] = UINT_MAX; } else { - /* The last element has spare bits that need to be unset. */ + // The last element has spare bits that need to be unset. run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3)) - remainder)); } @@ -2966,7 +2802,7 @@ arena_t::GetNonFullBinRun(arena_bin_t* aBin) return run; } -/* bin->runcur must have space available before this function is called. */ +// bin->runcur must have space available before this function is called. void* arena_t::MallocBinEasy(arena_bin_t* aBin, arena_run_t* aRun) { @@ -2982,7 +2818,7 @@ arena_t::MallocBinEasy(arena_bin_t* aBin, arena_run_t* aRun) return ret; } -/* Re-fill aBin->runcur, then call arena_t::MallocBinEasy(). */ +// Re-fill aBin->runcur, then call arena_t::MallocBinEasy(). void* arena_t::MallocBinHard(arena_bin_t* aBin) { @@ -2996,17 +2832,15 @@ arena_t::MallocBinHard(arena_bin_t* aBin) return MallocBinEasy(aBin, aBin->runcur); } -/* - * Calculate bin->run_size such that it meets the following constraints: - * - * *) bin->run_size >= min_run_size - * *) bin->run_size <= arena_maxclass - * *) bin->run_size <= RUN_MAX_SMALL - * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). - * - * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are - * also calculated here, since these settings are all interdependent. - */ +// Calculate bin->run_size such that it meets the following constraints: +// +// *) bin->run_size >= min_run_size +// *) bin->run_size <= arena_maxclass +// *) bin->run_size <= RUN_MAX_SMALL +// *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). +// +// bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are +// also calculated here, since these settings are all interdependent. static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) { @@ -3017,19 +2851,17 @@ arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) MOZ_ASSERT(min_run_size >= pagesize); MOZ_ASSERT(min_run_size <= arena_maxclass); - /* - * Calculate known-valid settings before entering the run_size - * expansion loop, so that the first part of the loop always copies - * valid settings. - * - * The do..while loop iteratively reduces the number of regions until - * the run header and the regions no longer overlap. A closed formula - * would be quite messy, since there is an interdependency between the - * header's mask length and the number of regions. - */ + // Calculate known-valid settings before entering the run_size + // expansion loop, so that the first part of the loop always copies + // valid settings. + // + // The do..while loop iteratively reduces the number of regions until + // the run header and the regions no longer overlap. A closed formula + // would be quite messy, since there is an interdependency between the + // header's mask length and the number of regions. try_run_size = min_run_size; try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size) - + 1; /* Counter-act try_nregs-- in loop. */ + + 1; // Counter-act try_nregs-- in loop. do { try_nregs--; try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) + @@ -3038,20 +2870,18 @@ arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1)) > try_reg0_offset); - /* run_size expansion loop. */ + // run_size expansion loop. do { - /* - * Copy valid settings before trying more aggressive settings. - */ + // Copy valid settings before trying more aggressive settings. good_run_size = try_run_size; good_nregs = try_nregs; good_mask_nelms = try_mask_nelms; good_reg0_offset = try_reg0_offset; - /* Try more aggressive settings. */ + // Try more aggressive settings. try_run_size += pagesize; try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ + bin->reg_size) + 1; // Counter-act try_nregs-- in loop. do { try_nregs--; try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) + @@ -3069,7 +2899,7 @@ arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) <= good_reg0_offset); MOZ_ASSERT((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs); - /* Copy final settings. */ + // Copy final settings. bin->run_size = good_run_size; bin->nregs = good_nregs; bin->regs_mask_nelms = good_mask_nelms; @@ -3086,23 +2916,22 @@ arena_t::MallocSmall(size_t aSize, bool aZero) arena_run_t* run; if (aSize < small_min) { - /* Tiny. */ + // Tiny. aSize = pow2_ceil(aSize); bin = &mBins[ffs((int)(aSize >> (TINY_MIN_2POW + 1)))]; - /* - * Bin calculation is always correct, but we may need - * to fix size for the purposes of assertions and/or - * stats accuracy. - */ + + // Bin calculation is always correct, but we may need + // to fix size for the purposes of assertions and/or + // stats accuracy. if (aSize < (1U << TINY_MIN_2POW)) { aSize = 1U << TINY_MIN_2POW; } } else if (aSize <= small_max) { - /* Quantum-spaced. */ + // Quantum-spaced. aSize = QUANTUM_CEILING(aSize); bin = &mBins[ntbins + (aSize >> QUANTUM_2POW_MIN) - 1]; } else { - /* Sub-page. */ + // Sub-page. aSize = pow2_ceil(aSize); bin = &mBins[ntbins + nqbins + (ffs((int)(aSize >> SMALL_MAX_2POW_DEFAULT)) - 2)]; @@ -3142,7 +2971,7 @@ arena_t::MallocLarge(size_t aSize, bool aZero) { void* ret; - /* Large allocation. */ + // Large allocation. aSize = PAGE_CEILING(aSize); { @@ -3188,7 +3017,7 @@ imalloc(size_t aSize, bool aZero, arena_t* aArena) return huge_malloc(aSize, aZero); } -/* Only handles large allocations that require more than page alignment. */ +// Only handles large allocations that require more than page alignment. void* arena_t::Palloc(size_t aAlignment, size_t aSize, size_t aAllocSize) { @@ -3224,7 +3053,7 @@ arena_t::Palloc(size_t aAlignment, size_t aSize, size_t aAllocSize) trailsize = aAllocSize - leadsize - aSize; if (trailsize != 0) { - /* Trim trailing space. */ + // Trim trailing space. MOZ_ASSERT(trailsize < aAllocSize); TrimRunTail(chunk, (arena_run_t*)ret, aSize + trailsize, aSize, false); } @@ -3247,31 +3076,28 @@ ipalloc(size_t aAlignment, size_t aSize, arena_t* aArena) void* ret; size_t ceil_size; - /* - * Round size up to the nearest multiple of alignment. - * - * This done, we can take advantage of the fact that for each small - * size class, every object is aligned at the smallest power of two - * that is non-zero in the base two representation of the size. For - * example: - * - * Size | Base 2 | Minimum alignment - * -----+----------+------------------ - * 96 | 1100000 | 32 - * 144 | 10100000 | 32 - * 192 | 11000000 | 64 - * - * Depending on runtime settings, it is possible that arena_malloc() - * will further round up to a power of two, but that never causes - * correctness issues. - */ + // Round size up to the nearest multiple of alignment. + // + // This done, we can take advantage of the fact that for each small + // size class, every object is aligned at the smallest power of two + // that is non-zero in the base two representation of the size. For + // example: + // + // Size | Base 2 | Minimum alignment + // -----+----------+------------------ + // 96 | 1100000 | 32 + // 144 | 10100000 | 32 + // 192 | 11000000 | 64 + // + // Depending on runtime settings, it is possible that arena_malloc() + // will further round up to a power of two, but that never causes + // correctness issues. ceil_size = ALIGNMENT_CEILING(aSize, aAlignment); - /* - * (ceil_size < aSize) protects against the combination of maximal - * alignment and size greater than maximal alignment. - */ + + // (ceil_size < aSize) protects against the combination of maximal + // alignment and size greater than maximal alignment. if (ceil_size < aSize) { - /* size_t overflow. */ + // size_t overflow. return nullptr; } @@ -3282,45 +3108,38 @@ ipalloc(size_t aAlignment, size_t aSize, arena_t* aArena) } else { size_t run_size; - /* - * We can't achieve sub-page alignment, so round up alignment - * permanently; it makes later calculations simpler. - */ + // We can't achieve sub-page alignment, so round up alignment + // permanently; it makes later calculations simpler. aAlignment = PAGE_CEILING(aAlignment); ceil_size = PAGE_CEILING(aSize); - /* - * (ceil_size < aSize) protects against very large sizes within - * pagesize of SIZE_T_MAX. - * - * (ceil_size + aAlignment < ceil_size) protects against the - * combination of maximal alignment and ceil_size large enough - * to cause overflow. This is similar to the first overflow - * check above, but it needs to be repeated due to the new - * ceil_size value, which may now be *equal* to maximal - * alignment, whereas before we only detected overflow if the - * original size was *greater* than maximal alignment. - */ + + // (ceil_size < aSize) protects against very large sizes within + // pagesize of SIZE_T_MAX. + // + // (ceil_size + aAlignment < ceil_size) protects against the + // combination of maximal alignment and ceil_size large enough + // to cause overflow. This is similar to the first overflow + // check above, but it needs to be repeated due to the new + // ceil_size value, which may now be *equal* to maximal + // alignment, whereas before we only detected overflow if the + // original size was *greater* than maximal alignment. if (ceil_size < aSize || ceil_size + aAlignment < ceil_size) { - /* size_t overflow. */ + // size_t overflow. return nullptr; } - /* - * Calculate the size of the over-size run that arena_palloc() - * would need to allocate in order to guarantee the alignment. - */ + // Calculate the size of the over-size run that arena_palloc() + // would need to allocate in order to guarantee the alignment. if (ceil_size >= aAlignment) { run_size = ceil_size + aAlignment - pagesize; } else { - /* - * It is possible that (aAlignment << 1) will cause - * overflow, but it doesn't matter because we also - * subtract pagesize, which in the case of overflow - * leaves us with a very large run_size. That causes - * the first conditional below to fail, which means - * that the bogus run_size value never gets used for - * anything important. - */ + // It is possible that (aAlignment << 1) will cause + // overflow, but it doesn't matter because we also + // subtract pagesize, which in the case of overflow + // leaves us with a very large run_size. That causes + // the first conditional below to fail, which means + // that the bogus run_size value never gets used for + // anything important. run_size = (aAlignment << 1) - pagesize; } @@ -3338,7 +3157,7 @@ ipalloc(size_t aAlignment, size_t aSize, arena_t* aArena) return ret; } -/* Return the size of the allocation pointed to by ptr. */ +// Return the size of the allocation pointed to by ptr. static size_t arena_salloc(const void *ptr) { @@ -3365,18 +3184,16 @@ arena_salloc(const void *ptr) return ret; } -/* - * Validate ptr before assuming that it points to an allocation. Currently, - * the following validation is performed: - * - * + Check that ptr is not nullptr. - * - * + Check that ptr lies within a mapped chunk. - */ +// Validate ptr before assuming that it points to an allocation. Currently, +// the following validation is performed: +// +// + Check that ptr is not nullptr. +// +// + Check that ptr lies within a mapped chunk. static inline size_t isalloc_validate(const void* aPtr) { - /* If the allocator is not initialized, the pointer can't belong to it. */ + // If the allocator is not initialized, the pointer can't belong to it. if (malloc_initialized == false) { return 0; } @@ -3397,7 +3214,7 @@ isalloc_validate(const void* aPtr) extent_node_t key; - /* Chunk. */ + // Chunk. key.addr = (void*)chunk; MutexAutoLock lock(huge_mtx); extent_node_t* node = huge.Search(&key); @@ -3414,7 +3231,7 @@ isalloc(const void* aPtr) auto chunk = GetChunkForPtr(aPtr); if (chunk != aPtr) { - /* Region. */ + // Region. MOZ_ASSERT(chunk->arena->mMagic == ARENA_MAGIC); return arena_salloc(aPtr); @@ -3422,11 +3239,10 @@ isalloc(const void* aPtr) extent_node_t key; - /* Chunk (huge allocation). */ - + // Chunk (huge allocation). MutexAutoLock lock(huge_mtx); - /* Extract from tree of huge allocations. */ + // Extract from tree of huge allocations. key.addr = const_cast(aPtr); extent_node_t* node = huge.Search(&key); MOZ_DIAGNOSTIC_ASSERT(node); @@ -3511,7 +3327,6 @@ MozJemalloc::jemalloc_ptr_info(const void* aPtr, jemalloc_ptr_info_t* aInfo) // The following two return paths shouldn't occur in // practice unless there is heap corruption. - pageind--; MOZ_DIAGNOSTIC_ASSERT(pageind >= arena_chunk_header_npages); if (pageind < arena_chunk_header_npages) { @@ -3533,7 +3348,6 @@ MozJemalloc::jemalloc_ptr_info(const void* aPtr, jemalloc_ptr_info_t* aInfo) } // It must be a small allocation. - auto run = (arena_run_t *)(mapbits & ~pagesize_mask); MOZ_DIAGNOSTIC_ASSERT(run->magic == ARENA_RUN_MAGIC); @@ -3595,17 +3409,16 @@ arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr, arena_chunk_map_t* aMapE run->nfree++; if (run->nfree == bin->nregs) { - /* Deallocate run. */ + // Deallocate run. if (run == bin->runcur) { bin->runcur = nullptr; } else if (bin->nregs != 1) { size_t run_pageind = (uintptr_t(run) - uintptr_t(aChunk)) >> pagesize_2pow; arena_chunk_map_t* run_mapelm = &aChunk->map[run_pageind]; - /* - * This block's conditional is necessary because if the - * run only contains one region, then it never gets - * inserted into the non-full runs tree. - */ + + // This block's conditional is necessary because if the + // run only contains one region, then it never gets + // inserted into the non-full runs tree. MOZ_DIAGNOSTIC_ASSERT(bin->runs.Search(run_mapelm) == run_mapelm); bin->runs.Remove(run_mapelm); } @@ -3615,20 +3428,18 @@ arena_t::DallocSmall(arena_chunk_t* aChunk, void* aPtr, arena_chunk_map_t* aMapE DallocRun(run, true); bin->stats.curruns--; } else if (run->nfree == 1 && run != bin->runcur) { - /* - * Make sure that bin->runcur always refers to the lowest - * non-full run, if one exists. - */ + // Make sure that bin->runcur always refers to the lowest + // non-full run, if one exists. if (!bin->runcur) { bin->runcur = run; } else if (uintptr_t(run) < uintptr_t(bin->runcur)) { - /* Switch runcur. */ + // Switch runcur. if (bin->runcur->nfree > 0) { arena_chunk_t* runcur_chunk = GetChunkForPtr(bin->runcur); size_t runcur_pageind = (uintptr_t(bin->runcur) - uintptr_t(runcur_chunk)) >> pagesize_2pow; arena_chunk_map_t* runcur_mapelm = &runcur_chunk->map[runcur_pageind]; - /* Insert runcur. */ + // Insert runcur. MOZ_DIAGNOSTIC_ASSERT(!bin->runs.Search(runcur_mapelm)); bin->runs.Insert(runcur_mapelm); } @@ -3674,10 +3485,10 @@ arena_dalloc(void* aPtr, size_t aOffset) arena_chunk_map_t* mapelm = &chunk->map[pageind]; MOZ_DIAGNOSTIC_ASSERT((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { - /* Small allocation. */ + // Small allocation. arena->DallocSmall(chunk, aPtr, mapelm); } else { - /* Large allocation. */ + // Large allocation. arena->DallocLarge(chunk, aPtr); } } @@ -3703,16 +3514,14 @@ arena_t::RallocShrinkLarge(arena_chunk_t* aChunk, void* aPtr, size_t aSize, { MOZ_ASSERT(aSize < aOldSize); - /* - * Shrink the run, and make trailing pages available for other - * allocations. - */ + // Shrink the run, and make trailing pages available for other + // allocations. MutexAutoLock lock(mLock); TrimRunTail(aChunk, (arena_run_t*)aPtr, aOldSize, aSize, true); mStats.allocated_large -= aOldSize - aSize; } -/* Returns whether reallocation was successful. */ +// Returns whether reallocation was successful. bool arena_t::RallocGrowLarge(arena_chunk_t* aChunk, void* aPtr, size_t aSize, size_t aOldSize) @@ -3723,16 +3532,14 @@ arena_t::RallocGrowLarge(arena_chunk_t* aChunk, void* aPtr, size_t aSize, MutexAutoLock lock(mLock); MOZ_DIAGNOSTIC_ASSERT(aOldSize == (aChunk->map[pageind].bits & ~pagesize_mask)); - /* Try to extend the run. */ + // Try to extend the run. MOZ_ASSERT(aSize > aOldSize); if (pageind + npages < chunk_npages && (aChunk->map[pageind+npages].bits & CHUNK_MAP_ALLOCATED) == 0 && (aChunk->map[pageind+npages].bits & ~pagesize_mask) >= aSize - aOldSize) { - /* - * The next run is available and sufficiently large. Split the - * following run, then merge the first part with the existing - * allocation. - */ + // The next run is available and sufficiently large. Split the + // following run, then merge the first part with the existing + // allocation. if (!SplitRun((arena_run_t*)(uintptr_t(aChunk) + ((pageind + npages) << pagesize_2pow)), aSize - aOldSize, @@ -3753,11 +3560,9 @@ arena_t::RallocGrowLarge(arena_chunk_t* aChunk, void* aPtr, size_t aSize, return false; } -/* - * Try to resize a large allocation, in order to avoid copying. This will - * always fail if growing an object, and the following run is already in use. - * Returns whether reallocation was successful. - */ +// Try to resize a large allocation, in order to avoid copying. This will +// always fail if growing an object, and the following run is already in use. +// Returns whether reallocation was successful. static bool arena_ralloc_large(void* aPtr, size_t aSize, size_t aOldSize) { @@ -3765,7 +3570,7 @@ arena_ralloc_large(void* aPtr, size_t aSize, size_t aOldSize) psize = PAGE_CEILING(aSize); if (psize == aOldSize) { - /* Same size class. */ + // Same size class. if (aSize < aOldSize) { memset((void*)((uintptr_t)aPtr + aSize), kAllocPoison, aOldSize - aSize); } @@ -3777,7 +3582,7 @@ arena_ralloc_large(void* aPtr, size_t aSize, size_t aOldSize) MOZ_DIAGNOSTIC_ASSERT(arena->mMagic == ARENA_MAGIC); if (psize < aOldSize) { - /* Fill before shrinking in order avoid a race. */ + // Fill before shrinking in order avoid a race. memset((void*)((uintptr_t)aPtr + aSize), kAllocPoison, aOldSize - aSize); arena->RallocShrinkLarge(chunk, aPtr, psize, aOldSize); return true; @@ -3796,23 +3601,23 @@ arena_ralloc(void* aPtr, size_t aSize, size_t aOldSize, arena_t* aArena) void* ret; size_t copysize; - /* Try to avoid moving the allocation. */ + // Try to avoid moving the allocation. if (aSize < small_min) { if (aOldSize < small_min && ffs((int)(pow2_ceil(aSize) >> (TINY_MIN_2POW + 1))) == ffs((int)(pow2_ceil(aOldSize) >> (TINY_MIN_2POW + 1)))) { - goto IN_PLACE; /* Same size class. */ + goto IN_PLACE; // Same size class. } } else if (aSize <= small_max) { if (aOldSize >= small_min && aOldSize <= small_max && (QUANTUM_CEILING(aSize) >> QUANTUM_2POW_MIN) == (QUANTUM_CEILING(aOldSize) >> QUANTUM_2POW_MIN)) { - goto IN_PLACE; /* Same size class. */ + goto IN_PLACE; // Same size class. } } else if (aSize <= bin_maxclass) { if (aOldSize > small_max && aOldSize <= bin_maxclass && pow2_ceil(aSize) == pow2_ceil(aOldSize)) { - goto IN_PLACE; /* Same size class. */ + goto IN_PLACE; // Same size class. } } else if (aOldSize > bin_maxclass && aOldSize <= arena_maxclass) { MOZ_ASSERT(aSize > bin_maxclass); @@ -3821,18 +3626,16 @@ arena_ralloc(void* aPtr, size_t aSize, size_t aOldSize, arena_t* aArena) } } - /* - * If we get here, then aSize and aOldSize are different enough that we - * need to move the object. In that case, fall back to allocating new - * space and copying. - */ + // If we get here, then aSize and aOldSize are different enough that we + // need to move the object. In that case, fall back to allocating new + // space and copying. aArena = aArena ? aArena : choose_arena(aSize); ret = aArena->Malloc(aSize, false); if (!ret) { return nullptr; } - /* Junk/zero-filling were already done by arena_t::Malloc(). */ + // Junk/zero-filling were already done by arena_t::Malloc(). copysize = (aSize < aOldSize) ? aSize : aOldSize; #ifdef VM_COPY_MIN if (copysize >= VM_COPY_MIN) { @@ -3882,7 +3685,7 @@ arena_t::Init() memset(&mLink, 0, sizeof(mLink)); memset(&mStats, 0, sizeof(arena_stats_t)); - /* Initialize chunks. */ + // Initialize chunks. mChunksDirty.Init(); #ifdef MALLOC_DOUBLE_PURGE new (&mChunksMAdvised) mozilla::DoublyLinkedList(); @@ -3896,10 +3699,10 @@ arena_t::Init() mRunsAvail.Init(); - /* Initialize bins. */ + // Initialize bins. prev_run_size = pagesize; - /* (2^n)-spaced tiny bins. */ + // (2^n)-spaced tiny bins. for (i = 0; i < ntbins; i++) { bin = &mBins[i]; bin->runcur = nullptr; @@ -3912,7 +3715,7 @@ arena_t::Init() memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } - /* Quantum-spaced bins. */ + // Quantum-spaced bins. for (; i < ntbins + nqbins; i++) { bin = &mBins[i]; bin->runcur = nullptr; @@ -3925,7 +3728,7 @@ arena_t::Init() memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } - /* (2^n)-spaced sub-page bins. */ + // (2^n)-spaced sub-page bins. for (; i < ntbins + nqbins + nsbins; i++) { bin = &mBins[i]; bin->runcur = nullptr; @@ -3948,27 +3751,25 @@ arena_t::Init() static inline arena_t * arenas_fallback() { - /* Only reached if there is an OOM error. */ + // Only reached if there is an OOM error. - /* - * OOM here is quite inconvenient to propagate, since dealing with it - * would require a check for failure in the fast path. Instead, punt - * by using the first arena. - * In practice, this is an extremely unlikely failure. - */ + // OOM here is quite inconvenient to propagate, since dealing with it + // would require a check for failure in the fast path. Instead, punt + // by using the first arena. + // In practice, this is an extremely unlikely failure. _malloc_message(_getprogname(), ": (malloc) Error initializing arena\n"); return gMainArena; } -/* Create a new arena and return it. */ +// Create a new arena and return it. static arena_t* arenas_extend() { arena_t* ret; - /* Allocate enough space for trailing bins. */ + // Allocate enough space for trailing bins. ret = (arena_t*)base_alloc(sizeof(arena_t) + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1))); if (!ret || !ret->Init()) { @@ -3984,13 +3785,9 @@ arenas_extend() return ret; } -/* - * End arena. - */ -/******************************************************************************/ -/* - * Begin general internal functions. - */ +// End arena. +// *************************************************************************** +// Begin general internal functions. static void * huge_malloc(size_t size, bool zero) @@ -4007,15 +3804,14 @@ huge_palloc(size_t aSize, size_t aAlignment, bool aZero) extent_node_t* node; bool zeroed; - /* Allocate one or more contiguous chunks for this request. */ - + // Allocate one or more contiguous chunks for this request. csize = CHUNK_CEILING(aSize); if (csize == 0) { - /* size is large enough to cause size_t wrap-around. */ + // size is large enough to cause size_t wrap-around. return nullptr; } - /* Allocate an extent node with which to track the chunk. */ + // Allocate an extent node with which to track the chunk. node = base_node_alloc(); if (!node) { return nullptr; @@ -4030,7 +3826,7 @@ huge_palloc(size_t aSize, size_t aAlignment, bool aZero) chunk_ensure_zero(ret, csize, zeroed); } - /* Insert node into huge. */ + // Insert node into huge. node->addr = ret; psize = PAGE_CEILING(aSize); node->size = psize; @@ -4039,24 +3835,24 @@ huge_palloc(size_t aSize, size_t aAlignment, bool aZero) MutexAutoLock lock(huge_mtx); huge.Insert(node); - /* Although we allocated space for csize bytes, we indicate that we've - * allocated only psize bytes. - * - * If DECOMMIT is defined, this is a reasonable thing to do, since - * we'll explicitly decommit the bytes in excess of psize. - * - * If DECOMMIT is not defined, then we're relying on the OS to be lazy - * about how it allocates physical pages to mappings. If we never - * touch the pages in excess of psize, the OS won't allocate a physical - * page, and we won't use more than psize bytes of physical memory. - * - * A correct program will only touch memory in excess of how much it - * requested if it first calls malloc_usable_size and finds out how - * much space it has to play with. But because we set node->size = - * psize above, malloc_usable_size will return psize, not csize, and - * the program will (hopefully) never touch bytes in excess of psize. - * Thus those bytes won't take up space in physical memory, and we can - * reasonably claim we never "allocated" them in the first place. */ + // Although we allocated space for csize bytes, we indicate that we've + // allocated only psize bytes. + // + // If DECOMMIT is defined, this is a reasonable thing to do, since + // we'll explicitly decommit the bytes in excess of psize. + // + // If DECOMMIT is not defined, then we're relying on the OS to be lazy + // about how it allocates physical pages to mappings. If we never + // touch the pages in excess of psize, the OS won't allocate a physical + // page, and we won't use more than psize bytes of physical memory. + // + // A correct program will only touch memory in excess of how much it + // requested if it first calls malloc_usable_size and finds out how + // much space it has to play with. But because we set node->size = + // psize above, malloc_usable_size will return psize, not csize, and + // the program will (hopefully) never touch bytes in excess of psize. + // Thus those bytes won't take up space in physical memory, and we can + // reasonably claim we never "allocated" them in the first place. huge_allocated += psize; huge_mapped += csize; } @@ -4092,8 +3888,7 @@ huge_ralloc(void* aPtr, size_t aSize, size_t aOldSize) void* ret; size_t copysize; - /* Avoid moving the allocation if the size class would not change. */ - + // Avoid moving the allocation if the size class would not change. if (aOldSize > arena_maxclass && CHUNK_CEILING(aSize) == CHUNK_CEILING(aOldSize)) { size_t psize = PAGE_CEILING(aSize); @@ -4106,14 +3901,14 @@ huge_ralloc(void* aPtr, size_t aSize, size_t aOldSize) pages_decommit((void*)((uintptr_t)aPtr + psize), aOldSize - psize); - /* Update recorded size. */ + // Update recorded size. MutexAutoLock lock(huge_mtx); key.addr = const_cast(aPtr); extent_node_t* node = huge.Search(&key); MOZ_ASSERT(node); MOZ_ASSERT(node->size == aOldSize); huge_allocated -= aOldSize - psize; - /* No need to change huge_mapped, because we didn't (un)map anything. */ + // No need to change huge_mapped, because we didn't (un)map anything. node->size = psize; } else if (psize > aOldSize) { if (!pages_commit((void*)((uintptr_t)aPtr + aOldSize), @@ -4123,13 +3918,13 @@ huge_ralloc(void* aPtr, size_t aSize, size_t aOldSize) } #endif - /* Although we don't have to commit or decommit anything if - * DECOMMIT is not defined and the size class didn't change, we - * do need to update the recorded size if the size increased, - * so malloc_usable_size doesn't return a value smaller than - * what was requested via realloc(). */ + // Although we don't have to commit or decommit anything if + // DECOMMIT is not defined and the size class didn't change, we + // do need to update the recorded size if the size increased, + // so malloc_usable_size doesn't return a value smaller than + // what was requested via realloc(). if (psize > aOldSize) { - /* Update recorded size. */ + // Update recorded size. extent_node_t key; MutexAutoLock lock(huge_mtx); key.addr = const_cast(aPtr); @@ -4137,8 +3932,8 @@ huge_ralloc(void* aPtr, size_t aSize, size_t aOldSize) MOZ_ASSERT(node); MOZ_ASSERT(node->size == aOldSize); huge_allocated += psize - aOldSize; - /* No need to change huge_mapped, because we didn't - * (un)map anything. */ + // No need to change huge_mapped, because we didn't + // (un)map anything. node->size = psize; } @@ -4148,11 +3943,9 @@ huge_ralloc(void* aPtr, size_t aSize, size_t aOldSize) return aPtr; } - /* - * If we get here, then aSize and aOldSize are different enough that we - * need to use a different size class. In that case, fall back to - * allocating new space and copying. - */ + // If we get here, then aSize and aOldSize are different enough that we + // need to use a different size class. In that case, fall back to + // allocating new space and copying. ret = huge_malloc(aSize, false); if (!ret) { return nullptr; @@ -4179,7 +3972,7 @@ huge_dalloc(void* aPtr) extent_node_t key; MutexAutoLock lock(huge_mtx); - /* Extract from tree of huge allocations. */ + // Extract from tree of huge allocations. key.addr = aPtr; node = huge.Search(&key); MOZ_ASSERT(node); @@ -4190,17 +3983,15 @@ huge_dalloc(void* aPtr) huge_mapped -= CHUNK_CEILING(node->size); } - /* Unmap chunk. */ + // Unmap chunk. chunk_dealloc(node->addr, CHUNK_CEILING(node->size), HUGE_CHUNK); base_node_dealloc(node); } -/* - * FreeBSD's pthreads implementation calls malloc(3), so the malloc - * implementation has to take pains to avoid infinite recursion during - * initialization. - */ +// FreeBSD's pthreads implementation calls malloc(3), so the malloc +// implementation has to take pains to avoid infinite recursion during +// initialization. #if defined(XP_WIN) #define malloc_init() false #else @@ -4248,10 +4039,8 @@ malloc_init_hard(void) #endif if (malloc_initialized) { - /* - * Another thread initialized the allocator before this one - * acquired gInitLock. - */ + // Another thread initialized the allocator before this one + // acquired gInitLock. return false; } @@ -4259,9 +4048,9 @@ malloc_init_hard(void) return false; } - /* Get page size and number of CPUs */ + // Get page size and number of CPUs result = GetKernelPageSize(); - /* We assume that the page size is a power of 2. */ + // We assume that the page size is a power of 2. MOZ_ASSERT(((result - 1) & result) == 0); #ifdef MALLOC_STATIC_PAGESIZE if (pagesize % (size_t) result) { @@ -4275,13 +4064,13 @@ malloc_init_hard(void) pagesize_2pow = ffs((int)result) - 1; #endif - /* Get runtime configuration. */ + // Get runtime configuration. if ((opts = getenv("MALLOC_OPTIONS"))) { for (i = 0; opts[i] != '\0'; i++) { unsigned j, nreps; bool nseen; - /* Parse repetition count, if any. */ + // Parse repetition count, if any. for (nreps = 0, nseen = false;; i++, nseen = true) { switch (opts[i]) { case '0': case '1': case '2': case '3': @@ -4343,7 +4132,7 @@ MALLOC_OUT: } #ifndef MALLOC_STATIC_PAGESIZE - /* Set bin-related variables. */ + // Set bin-related variables. bin_maxclass = (pagesize >> 1); nsbins = pagesize_2pow - SMALL_MAX_2POW_DEFAULT - 1; @@ -4355,24 +4144,24 @@ MALLOC_OUT: gRecycledSize = 0; - /* Various sanity checks that regard configuration. */ + // Various sanity checks that regard configuration. MOZ_ASSERT(quantum >= sizeof(void *)); MOZ_ASSERT(quantum <= pagesize); MOZ_ASSERT(chunksize >= pagesize); MOZ_ASSERT(quantum * 4 <= chunksize); - /* Initialize chunks data. */ + // Initialize chunks data. chunks_mtx.Init(); gChunksBySize.Init(); gChunksByAddress.Init(); - /* Initialize huge allocation data. */ + // Initialize huge allocation data. huge_mtx.Init(); huge.Init(); huge_allocated = 0; huge_mapped = 0; - /* Initialize base allocation data structures. */ + // Initialize base allocation data structures. base_mapped = 0; base_committed = 0; base_nodes = nullptr; @@ -4380,22 +4169,18 @@ MALLOC_OUT: arenas_lock.Init(); - /* - * Initialize one arena here. - */ + // Initialize one arena here. gArenaTree.Init(); arenas_extend(); gMainArena = gArenaTree.First(); if (!gMainArena) { return true; } - /* arena_t::Init() sets this to a lower value for thread local arenas; - * reset to the default value for the main arena. */ + // arena_t::Init() sets this to a lower value for thread local arenas; + // reset to the default value for the main arena. gMainArena->mMaxDirty = opt_dirty_max; - /* - * Assign the initial arena to the initial thread. - */ + // Assign the initial arena to the initial thread. thread_arena.set(gMainArena); if (!gChunkRTree.Init()) { @@ -4408,24 +4193,20 @@ MALLOC_OUT: Debug::jemalloc_ptr_info(nullptr); #if !defined(XP_WIN) && !defined(XP_DARWIN) - /* Prevent potential deadlock on malloc locks after fork. */ + // Prevent potential deadlock on malloc locks after fork. pthread_atfork(_malloc_prefork, _malloc_postfork_parent, _malloc_postfork_child); #endif return false; } -/* - * End general internal functions. - */ -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ +// End general internal functions. +// *************************************************************************** +// Begin malloc(3)-compatible functions. -/* The BaseAllocator class is a helper class that implements the base allocator - * functions (malloc, calloc, realloc, free, memalign) for a given arena, - * or an appropriately chosen arena (per choose_arena()) when none is given. */ +// The BaseAllocator class is a helper class that implements the base allocator +// functions (malloc, calloc, realloc, free, memalign) for a given arena, +// or an appropriately chosen arena (per choose_arena()) when none is given. struct BaseAllocator { #define MALLOC_DECL(name, return_type, ...) \ inline return_type name(__VA_ARGS__); @@ -4507,14 +4288,13 @@ BaseAllocator::calloc(size_t aNum, size_t aSize) size_t num_size = aNum * aSize; if (num_size == 0) { num_size = 1; - /* - * Try to avoid division here. We know that it isn't possible to - * overflow during multiplication if neither operand uses any of the - * most significant half of the bits in a size_t. - */ + + // Try to avoid division here. We know that it isn't possible to + // overflow during multiplication if neither operand uses any of the + // most significant half of the bits in a size_t. } else if (((aNum | aSize) & (SIZE_T_MAX << (sizeof(size_t) << 2))) && (num_size / aSize != aNum)) { - /* size_t overflow. */ + // size_t overflow. ret = nullptr; goto RETURN; } @@ -4583,13 +4363,12 @@ struct AlignedAllocator { void* result; - /* alignment must be a power of two and a multiple of sizeof(void*) */ + // alignment must be a power of two and a multiple of sizeof(void*) if (((aAlignment - 1) & aAlignment) != 0 || aAlignment < sizeof(void*)) { return EINVAL; } - /* The 0-->1 size promotion is done in the memalign() call below */ - + // The 0-->1 size promotion is done in the memalign() call below result = memalign(aAlignment, aSize); if (!result) { @@ -4634,49 +4413,40 @@ MozJemalloc::valloc(size_t aSize) return AlignedAllocator::valloc(aSize); } -/* - * End malloc(3)-compatible functions. - */ -/******************************************************************************/ -/* - * Begin non-standard functions. - */ +// End malloc(3)-compatible functions. +// *************************************************************************** +// Begin non-standard functions. -/* This was added by Mozilla for use by SQLite. */ +// This was added by Mozilla for use by SQLite. template<> inline size_t MozJemalloc::malloc_good_size(size_t aSize) { - /* - * This duplicates the logic in imalloc(), arena_malloc() and - * arena_t::MallocSmall(). - */ + // This duplicates the logic in imalloc(), arena_malloc() and + // arena_t::MallocSmall(). if (aSize < small_min) { - /* Small (tiny). */ + // Small (tiny). aSize = pow2_ceil(aSize); - /* - * We omit the #ifdefs from arena_t::MallocSmall() -- - * it can be inaccurate with its size in some cases, but this - * function must be accurate. - */ + + // We omit the #ifdefs from arena_t::MallocSmall() -- + // it can be inaccurate with its size in some cases, but this + // function must be accurate. if (aSize < (1U << TINY_MIN_2POW)) { aSize = (1U << TINY_MIN_2POW); } } else if (aSize <= small_max) { - /* Small (quantum-spaced). */ + // Small (quantum-spaced). aSize = QUANTUM_CEILING(aSize); } else if (aSize <= bin_maxclass) { - /* Small (sub-page). */ + // Small (sub-page). aSize = pow2_ceil(aSize); } else if (aSize <= arena_maxclass) { - /* Large. */ + // Large. aSize = PAGE_CEILING(aSize); } else { - /* - * Huge. We use PAGE_CEILING to get psize, instead of using - * CHUNK_CEILING to get csize. This ensures that this - * malloc_usable_size(malloc(n)) always matches - * malloc_good_size(n). - */ + // Huge. We use PAGE_CEILING to get psize, instead of using + // CHUNK_CEILING to get csize. This ensures that this + // malloc_usable_size(malloc(n)) always matches + // malloc_good_size(n). aSize = PAGE_CEILING(aSize); } return aSize; @@ -4696,9 +4466,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) MOZ_ASSERT(aStats); - /* - * Gather runtime settings. - */ + // Gather runtime settings. aStats->opt_junk = opt_junk; aStats->opt_zero = opt_zero; aStats->narenas = narenas; @@ -4709,9 +4477,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) aStats->page_size = pagesize; aStats->dirty_max = opt_dirty_max; - /* - * Gather current memory usage statistics. - */ + // Gather current memory usage statistics. aStats->mapped = 0; aStats->allocated = 0; aStats->waste = 0; @@ -4721,7 +4487,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) non_arena_mapped = 0; - /* Get huge mapped/allocated. */ + // Get huge mapped/allocated. { MutexAutoLock lock(huge_mtx); non_arena_mapped += huge_mapped; @@ -4729,7 +4495,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) MOZ_ASSERT(huge_mapped >= huge_allocated); } - /* Get base mapped/allocated. */ + // Get base mapped/allocated. { MutexAutoLock lock(base_mtx); non_arena_mapped += base_mapped; @@ -4738,7 +4504,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) } arenas_lock.Lock(); - /* Iterate over arenas. */ + // Iterate over arenas. for (auto arena : gArenaTree.iter()) { size_t arena_mapped, arena_allocated, arena_committed, arena_dirty, j, arena_unused, arena_headers; @@ -4756,7 +4522,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) arena_mapped = arena->mStats.mapped; - /* "committed" counts dirty and allocated memory. */ + // "committed" counts dirty and allocated memory. arena_committed = arena->mStats.committed << pagesize_2pow; arena_allocated = arena->mStats.allocated_small + @@ -4785,8 +4551,8 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) MOZ_ASSERT(arena_mapped >= arena_committed); MOZ_ASSERT(arena_committed >= arena_allocated + arena_dirty); - /* "waste" is committed memory that is neither dirty nor - * allocated. */ + // "waste" is committed memory that is neither dirty nor + // allocated. aStats->mapped += arena_mapped; aStats->allocated += arena_allocated; aStats->page_cache += arena_dirty; @@ -4797,7 +4563,7 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) } arenas_lock.Unlock(); - /* Account for arena chunk headers in bookkeeping rather than waste. */ + // Account for arena chunk headers in bookkeeping rather than waste. chunk_header_size = ((aStats->mapped / aStats->chunksize) * arena_chunk_header_npages) << pagesize_2pow; @@ -4812,26 +4578,26 @@ MozJemalloc::jemalloc_stats(jemalloc_stats_t* aStats) #ifdef MALLOC_DOUBLE_PURGE -/* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */ +// Explicitly remove all of this chunk's MADV_FREE'd pages from memory. static void hard_purge_chunk(arena_chunk_t* aChunk) { - /* See similar logic in arena_t::Purge(). */ + // See similar logic in arena_t::Purge(). for (size_t i = arena_chunk_header_npages; i < chunk_npages; i++) { - /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */ + // Find all adjacent pages with CHUNK_MAP_MADVISED set. size_t npages; for (npages = 0; aChunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages; npages++) { - /* Turn off the chunk's MADV_FREED bit and turn on its - * DECOMMITTED bit. */ + // Turn off the chunk's MADV_FREED bit and turn on its + // DECOMMITTED bit. MOZ_DIAGNOSTIC_ASSERT( !(aChunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED)); aChunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED; } - /* We could use mincore to find out which pages are actually - * present, but it's not clear that's better. */ + // We could use mincore to find out which pages are actually + // present, but it's not clear that's better. if (npages > 0) { pages_decommit(((char*)aChunk) + (i << pagesize_2pow), npages << pagesize_2pow); @@ -4842,7 +4608,7 @@ hard_purge_chunk(arena_chunk_t* aChunk) } } -/* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */ +// Explicitly remove all of this arena's MADV_FREE'd pages from memory. void arena_t::HardPurge() { @@ -4863,15 +4629,15 @@ MozJemalloc::jemalloc_purge_freed_pages() } } -#else /* !defined MALLOC_DOUBLE_PURGE */ +#else // !defined MALLOC_DOUBLE_PURGE template<> inline void MozJemalloc::jemalloc_purge_freed_pages() { - /* Do nothing. */ + // Do nothing. } -#endif /* defined MALLOC_DOUBLE_PURGE */ +#endif // defined MALLOC_DOUBLE_PURGE template<> inline void @@ -4937,25 +4703,19 @@ MozJemalloc::moz_dispose_arena(arena_id_t aArenaId) #endif -/* - * End non-standard functions. - */ -/******************************************************************************/ -/* - * Begin library-private functions, used by threading libraries for protection - * of malloc during fork(). These functions are only called if the program is - * running in threaded mode, so there is no need to check whether the program - * is threaded here. - */ - +// End non-standard functions. +// *************************************************************************** +// Begin library-private functions, used by threading libraries for protection +// of malloc during fork(). These functions are only called if the program is +// running in threaded mode, so there is no need to check whether the program +// is threaded here. #ifndef XP_DARWIN static #endif void _malloc_prefork(void) { - /* Acquire all mutexes in a safe order. */ - + // Acquire all mutexes in a safe order. arenas_lock.Lock(); for (auto arena : gArenaTree.iter()) { @@ -4973,8 +4733,7 @@ static void _malloc_postfork_parent(void) { - /* Release all mutexes, now that fork() has completed. */ - + // Release all mutexes, now that fork() has completed. huge_mtx.Unlock(); base_mtx.Unlock(); @@ -4991,8 +4750,7 @@ static void _malloc_postfork_child(void) { - /* Reinitialize all mutexes, now that fork() has completed. */ - + // Reinitialize all mutexes, now that fork() has completed. huge_mtx.Init(); base_mtx.Init(); @@ -5003,22 +4761,17 @@ _malloc_postfork_child(void) arenas_lock.Init(); } -/* - * End library-private functions. - */ -/******************************************************************************/ +// End library-private functions. +// *************************************************************************** #ifdef MOZ_REPLACE_MALLOC - -/* - * Windows doesn't come with weak imports as they are possible with - * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform, - * the replacement functions are defined as variable pointers to the - * function resolved with GetProcAddress() instead of weak definitions - * of functions. On Android, the same needs to happen as well, because - * the Android linker doesn't handle weak linking with non LD_PRELOADed - * libraries, but LD_PRELOADing is not very convenient on Android, with - * the zygote. - */ +// Windows doesn't come with weak imports as they are possible with +// LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform, +// the replacement functions are defined as variable pointers to the +// function resolved with GetProcAddress() instead of weak definitions +// of functions. On Android, the same needs to happen as well, because +// the Android linker doesn't handle weak linking with non LD_PRELOADed +// libraries, but LD_PRELOADing is not very convenient on Android, with +// the zygote. #ifdef XP_DARWIN # define MOZ_REPLACE_WEAK __attribute__((weak_import)) #elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID) @@ -5098,11 +4851,8 @@ replace_malloc_handle() static void replace_malloc_init_funcs(); -/* - * Below is the malloc implementation overriding jemalloc and calling the - * replacement functions if they exist. - */ - +// Below is the malloc implementation overriding jemalloc and calling the +// replacement functions if they exist. static int replace_malloc_initialized = 0; static void init() @@ -5139,13 +4889,11 @@ get_bridge(void) return replace_get_bridge(); } -/* - * posix_memalign, aligned_alloc, memalign and valloc all implement some kind - * of aligned memory allocation. For convenience, a replace-malloc library can - * skip defining replace_posix_memalign, replace_aligned_alloc and - * replace_valloc, and default implementations will be automatically derived - * from replace_memalign. - */ +// posix_memalign, aligned_alloc, memalign and valloc all implement some kind +// of aligned memory allocation. For convenience, a replace-malloc library can +// skip defining replace_posix_memalign, replace_aligned_alloc and +// replace_valloc, and default implementations will be automatically derived +// from replace_memalign. static void replace_malloc_init_funcs() { @@ -5187,9 +4935,9 @@ replace_malloc_init_funcs() #include "malloc_decls.h" } -#endif /* MOZ_REPLACE_MALLOC */ -/******************************************************************************/ -/* Definition of all the _impl functions */ +#endif // MOZ_REPLACE_MALLOC +// *************************************************************************** +// Definition of all the _impl functions #define GENERIC_MALLOC_DECL2(name, name_impl, return_type, ...) \ return_type name_impl(ARGS_HELPER(TYPED_ARGS, ##__VA_ARGS__)) \ @@ -5211,22 +4959,21 @@ replace_malloc_init_funcs() #define MALLOC_DECL(...) MOZ_JEMALLOC_API MACRO_CALL(GENERIC_MALLOC_DECL, (__VA_ARGS__)) #define MALLOC_FUNCS (MALLOC_FUNCS_JEMALLOC | MALLOC_FUNCS_ARENA) #include "malloc_decls.h" -/******************************************************************************/ +// *************************************************************************** #ifdef HAVE_DLOPEN # include #endif #if defined(__GLIBC__) && !defined(__UCLIBC__) -/* - * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible - * to inconsistently reference libc's malloc(3)-compatible functions - * (bug 493541). - * - * These definitions interpose hooks in glibc. The functions are actually - * passed an extra argument for the caller return address, which will be - * ignored. - */ +// glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible +// to inconsistently reference libc's malloc(3)-compatible functions +// (bug 493541). +// +// These definitions interpose hooks in glibc. The functions are actually +// passed an extra argument for the caller return address, which will be +// ignored. + extern "C" { MOZ_EXPORT void (*__free_hook)(void*) = free_impl; MOZ_EXPORT void* (*__malloc_hook)(size_t) = malloc_impl; @@ -5235,11 +4982,9 @@ MOZ_EXPORT void* (*__memalign_hook)(size_t, size_t) = memalign_impl; } #elif defined(RTLD_DEEPBIND) -/* - * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their - * implementations permit similar inconsistencies? Should STV_SINGLETON - * visibility be used for interposition where available? - */ +// XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their +// implementations permit similar inconsistencies? Should STV_SINGLETON +// visibility be used for interposition where available? # error "Interposing malloc is unsafe on this system without libc malloc hooks." #endif @@ -5250,14 +4995,11 @@ _recalloc(void* aPtr, size_t aCount, size_t aSize) size_t oldsize = aPtr ? isalloc(aPtr) : 0; size_t newsize = aCount * aSize; - /* - * In order for all trailing bytes to be zeroed, the caller needs to - * use calloc(), followed by recalloc(). However, the current calloc() - * implementation only zeros the bytes requested, so if recalloc() is - * to work 100% correctly, calloc() will need to change to zero - * trailing bytes. - */ - + // In order for all trailing bytes to be zeroed, the caller needs to + // use calloc(), followed by recalloc(). However, the current calloc() + // implementation only zeros the bytes requested, so if recalloc() is + // to work 100% correctly, calloc() will need to change to zero + // trailing bytes. aPtr = DefaultMalloc::realloc(aPtr, newsize); if (aPtr && oldsize < newsize) { memset((void*)((uintptr_t)aPtr + oldsize), 0, newsize - oldsize); @@ -5266,10 +5008,8 @@ _recalloc(void* aPtr, size_t aCount, size_t aSize) return aPtr; } -/* - * This impl of _expand doesn't ever actually expand or shrink blocks: it - * simply replies that you may continue using a shrunk block. - */ +// This impl of _expand doesn't ever actually expand or shrink blocks: it +// simply replies that you may continue using a shrunk block. void* _expand(void* aPtr, size_t newsize) { @@ -5286,22 +5026,20 @@ _msize(void* aPtr) return DefaultMalloc::malloc_usable_size(aPtr); } -/* - * In the new style jemalloc integration jemalloc is built as a separate - * shared library. Since we're no longer hooking into the CRT binary, - * we need to initialize the heap at the first opportunity we get. - * DLL_PROCESS_ATTACH in DllMain is that opportunity. - */ +// In the new style jemalloc integration jemalloc is built as a separate +// shared library. Since we're no longer hooking into the CRT binary, +// we need to initialize the heap at the first opportunity we get. +// DLL_PROCESS_ATTACH in DllMain is that opportunity. BOOL APIENTRY DllMain(HINSTANCE hModule, DWORD reason, LPVOID lpReserved) { switch (reason) { case DLL_PROCESS_ATTACH: - /* Don't force the system to page DllMain back in every time - * we create/destroy a thread */ + // Don't force the system to page DllMain back in every time + // we create/destroy a thread DisableThreadLibraryCalls(hModule); - /* Initialize the heap */ + // Initialize the heap malloc_init_hard(); break;