From 63c70be95d57c0d3de4d7b5ed4c48e6f53c95fda Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 30 May 2021 13:09:02 +0800 Subject: [PATCH 01/19] Distinguish SI and Binary Prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SI prefixes [the decimal prefixes] refer strictly to powers of 10. They should not be used to indicate powers of 2. e.g., one kilobit represents 1000 bits instead of 1024 bits. IEC 60027‐2 symbols are formed adding a "i" to the SI symbol (e.g. G + i = Gi). --- include/mimalloc-types.h | 22 +++++++++++----------- src/arena.c | 6 +++--- src/os.c | 6 +++--- src/segment.c | 8 ++++---- src/stats.c | 10 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..502e41f8 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -105,10 +105,10 @@ terms of the MIT license. A copy of the license can be found in the file // Main tuning parameters for segment and page sizes // Sizes for 64-bit, divide by two for 32-bit -#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb -#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb -#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb -#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb +#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB +#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB // Derived constants #define MI_SEGMENT_SIZE (1UL<512kb) are put into a single page in a segment of the exact size (but still 2mb aligned) + MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned) } mi_page_kind_t; -// Segments are large allocated memory blocks (2mb on 64 bit) from +// Segments are large allocated memory blocks (2MiB on 64 bit) from // the OS. Inside segments we allocated fixed size _pages_ that // contain blocks. typedef struct mi_segment_s { diff --git a/src/arena.c b/src/arena.c index 0e6615a4..fb25a89d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -330,7 +330,7 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noe _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; } - _mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); return 0; } @@ -347,10 +347,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec size_t pages_reserved = 0; void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize); if (p==NULL || pages_reserved==0) { - _mi_warning_message("failed to reserve %zu gb huge pages\n", pages); + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); return ENOMEM; } - _mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages); + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); diff --git a/src/os.c b/src/os.c index 2e2ec88d..4b87054e 100644 --- a/src/os.c +++ b/src/os.c @@ -210,7 +210,7 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KB + os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } #else @@ -964,7 +964,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) else { // fall back to regular large pages mi_huge_pages_available = false; // don't try further huge pages - _mi_warning_message("unable to allocate using huge (1gb) pages, trying large (2mb) pages instead (status 0x%lx)\n", err); + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); } } // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation @@ -1007,7 +1007,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) // see: long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1gb) pages to numa node %d: %s\n", numa_node, strerror(errno)); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); } } return p; diff --git a/src/segment.c b/src/segment.c index 1d59be9d..bdf97019 100644 --- a/src/segment.c +++ b/src/segment.c @@ -17,14 +17,14 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_ /* -------------------------------------------------------------------------------- Segment allocation - We allocate pages inside bigger "segments" (4mb on 64-bit). This is to avoid + We allocate pages inside bigger "segments" (4MiB on 64-bit). This is to avoid splitting VMA's on Linux and reduce fragmentation on other OS's. Each thread owns its own segments. Currently we have: - - small pages (64kb), 64 in one segment - - medium pages (512kb), 8 in one segment - - large pages (4mb), 1 in one segment + - small pages (64KiB), 64 in one segment + - medium pages (512KiB), 8 in one segment + - large pages (4MiB), 1 in one segment - huge blocks > MI_LARGE_OBJ_SIZE_MAX become large segment with 1 page In any case the memory for a segment is virtual and usually committed on demand. diff --git a/src/stats.c b/src/stats.c index c94fbde9..07fce5e2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -135,7 +135,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { char buf[32]; int len = 32; - const char* suffix = (unit <= 0 ? " " : "b"); + const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); if (unit>0) n *= unit; @@ -145,13 +145,13 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* } else { int64_t divider = base; - const char* magnitude = "k"; - if (pos >= divider*base) { divider *= base; magnitude = "m"; } - if (pos >= divider*base) { divider *= base; magnitude = "g"; } + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix); + snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } From 060adf6b035ab543dcb233c56419e6371f56637f Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 19 Jun 2021 09:14:43 +0000 Subject: [PATCH 02/19] haiku biuld fix proposal, warning suppression. --- src/random.c | 4 +++- src/stats.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/random.c b/src/random.c index 255bede4..723d3507 100644 --- a/src/random.c +++ b/src/random.c @@ -194,8 +194,10 @@ static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); return true; } -#elif defined(__linux__) +#elif defined(__linux__) || defined(__HAIKU__) +#if defined(__linux__) #include +#endif #include #include #include diff --git a/src/stats.c b/src/stats.c index c94fbde9..74b783e9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -520,6 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } + *page_faults = 0; #elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; From 8d11dbe4e9315601129525855e44b93c300c4484 Mon Sep 17 00:00:00 2001 From: hank Date: Mon, 21 Jun 2021 22:36:47 +0800 Subject: [PATCH 03/19] fix typo --- include/mimalloc-internal.h | 2 +- src/page.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..825b0ed0 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -427,7 +427,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) { return _mi_segment_page_of(_mi_ptr_segment(p), p); } -// Get the block size of a page (special cased for huge objects) +// Get the block size of a page (special case for huge objects) static inline size_t mi_page_block_size(const mi_page_t* page) { const size_t bsize = page->xblock_size; mi_assert_internal(bsize > 0); diff --git a/src/page.c b/src/page.c index c08be9c0..033976dc 100644 --- a/src/page.c +++ b/src/page.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- The core of the allocator. Every segment contains - pages of a {certain block size. The main function + pages of a certain block size. The main function exported is `mi_malloc_generic`. ----------------------------------------------------------- */ From 3f9adc429ca2381d05fa99a3da23b540dd3f05fd Mon Sep 17 00:00:00 2001 From: Artur Sinila Date: Tue, 29 Jun 2021 21:07:13 +0300 Subject: [PATCH 04/19] Fix 'malloc-nomem1' test for 32-bit architectures --- test/test-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-api.c b/test/test-api.c index d3344928..739b1d18 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -83,7 +83,7 @@ int main() { void* p = mi_malloc(0); mi_free(p); }); CHECK_BODY("malloc-nomem1",{ - result = (mi_malloc(SIZE_MAX/2) == NULL); + result = (mi_malloc((size_t)PTRDIFF_MAX + (size_t)1) == NULL); }); CHECK_BODY("malloc-null",{ mi_free(NULL); From b39960e2a86133fc3644107e245903a5b8d76034 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:31:48 -0400 Subject: [PATCH 05/19] Fixed typo in headers --- include/mimalloc-types.h | 2 +- src/heap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index caf161d6..66b311c1 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -319,7 +319,7 @@ typedef struct mi_random_cxt_s { } mi_random_ctx_t; -// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows #if (MI_PADDING) typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) diff --git a/src/heap.c b/src/heap.c index bda10699..bbf4cef9 100644 --- a/src/heap.c +++ b/src/heap.c @@ -333,7 +333,7 @@ void mi_heap_destroy(mi_heap_t* heap) { Safe Heap delete ----------------------------------------------------------- */ -// Tranfer the pages from one heap to the other +// Transfer the pages from one heap to the other static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; From bfa83d65c2839feab973fad139530445d6713d7e Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:46:58 -0400 Subject: [PATCH 06/19] Fixed stats in mi_os_mem_alloc_aligned on windows when re-allocate In the last try the previous mi_os_mem_free decrease stat and mi_win_virtual_alloc doesn't increase it --- src/os.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/os.c b/src/os.c index 85415232..89d166ef 100644 --- a/src/os.c +++ b/src/os.c @@ -593,6 +593,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); + if (p != NULL) + { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } + } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? mi_os_mem_free(p, size, commit, stats); From db7199ac067c0ddacaede02cea4e55b7dfef31d1 Mon Sep 17 00:00:00 2001 From: Masashi Fujita Date: Thu, 16 Sep 2021 03:35:56 +0900 Subject: [PATCH 07/19] fix: Fix a typo --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..385d2071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ set(mi_sources src/init.c) # ----------------------------------------------------------------------------- -# Converience: set default build type depending on the build directory +# Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- if (NOT CMAKE_BUILD_TYPE) From b895bd832e3487c6d4863e1226e89b4e6b19c7d6 Mon Sep 17 00:00:00 2001 From: Jeremy Lorelli Date: Tue, 5 Oct 2021 08:41:03 -0700 Subject: [PATCH 08/19] Fix missing parameter in mi_free error message --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..1154e70a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -465,7 +465,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #endif #if (MI_DEBUG>0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { - _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", p); + _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); } #endif return segment; From aeb1fe56c28dd32eb8ac00fa934226edbb47623a Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 8 Oct 2021 23:59:35 -0700 Subject: [PATCH 09/19] Add an option to disable automatic use of `getenv` inside options.c --- src/options.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 30025db2..2dc7bfee 100644 --- a/src/options.c +++ b/src/options.c @@ -409,6 +409,14 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } +#ifdef MI_NO_GETENV +static bool mi_getenv(const char* name, char* result, size_t result_size) { + UNUSED(name); + UNUSED(result); + UNUSED(result_size); + return false; +} +#else static inline int mi_strnicmp(const char* s, const char* t, size_t n) { if (n==0) return 0; for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { @@ -416,7 +424,6 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { } return (n==0 ? 0 : *s - *t); } - #if defined _WIN32 // On Windows use GetEnvironmentVariable instead of getenv to work // reliably even when this is invoked before the C runtime is initialized. @@ -485,6 +492,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } } #endif +#endif static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From af5433179d2a36c39156b57335c7a2f3fcc4fd33 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 10:38:57 +0200 Subject: [PATCH 10/19] Fix strict function prototype warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix warning ``warning: function declaration isn’t a prototype`` when building mimalloc with ``-Wstrict-prototypes`` flag. In C argumentless functions should be declared as ``func(void)``. Reproducer: ```shell $ cmake ../.. -DCMAKE_C_FLAGS="-Wstrict-prototypes" $ make VERBOSE=1 ``` Co-authored-by: Sam Gross Co-authored-by: Neil Schemenauer Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/alloc.c | 8 ++++---- src/options.c | 2 +- src/os.c | 2 +- src/region.c | 2 +- test/test-api.c | 10 +++++----- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..9b097b8e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -53,7 +53,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); -bool _mi_preloading(); // true while the C runtime is not ready +bool _mi_preloading(void); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); diff --git a/src/alloc.c b/src/alloc.c index 8acff783..d2ba5c68 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -747,7 +747,7 @@ mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char } #else #include // pathconf -static size_t mi_path_max() { +static size_t mi_path_max(void) { static size_t path_max = 0; if (path_max <= 0) { long m = pathconf("/",_PC_PATH_MAX); @@ -807,13 +807,13 @@ static bool mi_try_new_handler(bool nothrow) { } } #else -typedef void (*std_new_handler_t)(); +typedef void (*std_new_handler_t)(void); #if (defined(__GNUC__) || defined(__clang__)) -std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { +std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv(void) { return NULL; } -static std_new_handler_t mi_get_new_handler() { +static std_new_handler_t mi_get_new_handler(void) { return _ZSt15get_new_handlerv(); } #else diff --git a/src/options.c b/src/options.c index 30025db2..4274af97 100644 --- a/src/options.c +++ b/src/options.c @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file static uintptr_t mi_max_error_count = 16; // stop outputting errors after this static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this -static void mi_add_stderr_output(); +static void mi_add_stderr_output(void); int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; diff --git a/src/os.c b/src/os.c index 85415232..16659b33 100644 --- a/src/os.c +++ b/src/os.c @@ -95,7 +95,7 @@ size_t _mi_os_page_size() { } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) -size_t _mi_os_large_page_size() { +size_t _mi_os_large_page_size(void) { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } diff --git a/src/region.c b/src/region.c index 79540730..2f68b140 100644 --- a/src/region.c +++ b/src/region.c @@ -40,7 +40,7 @@ Possible issues: #include "bitmap.h" // Internal raw OS interface -size_t _mi_os_large_page_size(); +size_t _mi_os_large_page_size(void); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); diff --git a/test/test-api.c b/test/test-api.c index d3344928..53e7f3d8 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -64,15 +64,15 @@ static int failed = 0; // --------------------------------------------------------------------------- // Test functions // --------------------------------------------------------------------------- -bool test_heap1(); -bool test_heap2(); -bool test_stl_allocator1(); -bool test_stl_allocator2(); +bool test_heap1(void); +bool test_heap2(void); +bool test_stl_allocator1(void); +bool test_stl_allocator2(void); // --------------------------------------------------------------------------- // Main testing // --------------------------------------------------------------------------- -int main() { +int main(void) { mi_option_disable(mi_option_verbose); // --------------------------------------------------- From d0d4e1a34590d6763db8d073784c0c42ce038b17 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 15:07:52 +0200 Subject: [PATCH 11/19] Define _DEFAULT_SOURCE for syscall and realpath Define ``_DEFAULT_SOURCE`` in ``random.c`` and ``alloc.c``. The macro is required for ``syscall()`` and ``realpath()``. Other files like ``os.c`` already define the macro. Signed-off-by: Christian Heimes --- src/alloc.c | 4 ++++ src/random.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..5e45e76f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" diff --git a/src/random.c b/src/random.c index 255bede4..0e2331bd 100644 --- a/src/random.c +++ b/src/random.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for syscall() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" From ee343f523dc3ac993c2c122e47e84688201427c5 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:39:33 -0700 Subject: [PATCH 12/19] add -Wstrict-prototypes flag during compilation --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..ac4c0569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,7 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() From fc18c8a479da092bfc82baaea113367c61a2281f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:52:20 -0700 Subject: [PATCH 13/19] add comment on #if ending --- src/options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/options.c b/src/options.c index 8d5f1556..c40a187b 100644 --- a/src/options.c +++ b/src/options.c @@ -491,8 +491,8 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { return false; } } -#endif -#endif +#endif // !MI_USE_ENVIRON +#endif // !MI_NO_GETENV static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 533e964d502eafa70ac24c75959aad6860b3b6a4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:17:30 -0700 Subject: [PATCH 14/19] add space after _Atomic to prevent errors on msvc without /TP (see PR #452) --- include/mimalloc-atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index dc48f0a2..e07df84d 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_memory_order(name) std::memory_order_##name #elif defined(_MSC_VER) // Use MSVC C wrapper for C11 atomics -#define _Atomic(tp) tp +#define _Atomic(tp) tp #define ATOMIC_VAR_INIT(x) x #define mi_atomic(name) mi_atomic_##name #define mi_memory_order(name) mi_memory_order_##name @@ -173,7 +173,7 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt } static inline void mi_atomic_thread_fence(mi_memory_order mo) { (void)(mo); - _Atomic(uintptr_t)x = 0; + _Atomic(uintptr_t) x = 0; mi_atomic_exchange_explicit(&x, 1, mo); } static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { From 10085583ef65fedb390771a7304aecf2b39e6405 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:31:49 -0700 Subject: [PATCH 15/19] formatting --- src/os.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index d7d3d33d..ba6cc5e3 100644 --- a/src/os.c +++ b/src/os.c @@ -641,10 +641,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); - if (p != NULL) - { - _mi_stat_increase(&stats->reserved, size); - if (commit) { _mi_stat_increase(&stats->committed, size); } + if (p != NULL) { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? From 125eb18d9974064e796768df7fcbc40c7b5b9a77 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:56:13 -0700 Subject: [PATCH 16/19] don't add prefix if build type is None (PR #427) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d1d9191..abab5a05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ else() endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel)$")) +if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")) set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() if(MI_BUILD_SHARED) From a36e9c10123dfbe86510d3eb949d2d210380e2e5 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 12:50:16 -0700 Subject: [PATCH 17/19] refine stat output with new SI units --- src/stats.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/stats.c b/src/stats.c index f85285e1..dda55cda 100644 --- a/src/stats.c +++ b/src/stats.c @@ -133,7 +133,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit == 0: count as decimal // unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { - char buf[32]; + char buf[32]; buf[0] = 0; int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); @@ -141,17 +141,21 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t pos = (n < 0 ? -n : n); if (pos < base) { - snprintf(buf, len, "%d %s ", (int)n, suffix); + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); + } } else { - int64_t divider = base; + int64_t divider = base; const char* magnitude = "K"; if (pos >= divider*base) { divider *= base; magnitude = "M"; } if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); + char unitdesc[16]; + snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } @@ -221,7 +225,7 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* static void mi_print_header(mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); + _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); } #if MI_STAT>1 From 1899c03f1c79ada3c017f85ae102df9a0669bc08 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 13:09:43 -0700 Subject: [PATCH 18/19] remove MADV_FREE_REUSABLE from mi_os_reset as it needs MADV_FREE_REUSE to reuse the memory again. --- src/os.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/os.c b/src/os.c index 08068c9d..bf1d4716 100644 --- a/src/os.c +++ b/src/os.c @@ -796,9 +796,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } - #if defined(MADV_FREE_REUSE) - while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } - #endif + //#if defined(MADV_FREE_REUSE) + // while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + //#endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -860,17 +860,12 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - #if defined(MADV_FREE_REUSABLE) - #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE - #else - #define KK_MADV_FREE_INITIAL MADV_FREE - #endif - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; - if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { - // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } From 08f59c82dafa33d951a834a9ecbee28dcb3e9ff4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 14:00:40 -0700 Subject: [PATCH 19/19] allow decommit in arena separate from initial commit --- src/arena.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/arena.c b/src/arena.c index fb25a89d..2f5a349f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -62,11 +62,11 @@ typedef struct mi_arena_s { size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool is_committed; // is the memory fully committed? (if so, block_committed == NULL) + bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) } mi_arena_t; @@ -129,8 +129,8 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n *memid = mi_arena_id_create(arena_index, bitmap_index); *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); *large = arena->is_large; - *is_pinned = (arena->is_large || arena->is_committed); - if (arena->is_committed) { + *is_pinned = (arena->is_large || !arena->allow_decommit); + if (arena->blocks_committed == NULL) { // always committed *commit = true; } @@ -245,12 +245,13 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s return; } // potentially decommit - if (arena->is_committed) { - mi_assert_internal(all_committed); + if (!arena->allow_decommit || arena->blocks_committed == NULL) { + mi_assert_internal(all_committed); // note: may be not true as we may "pretend" to be not committed (in segment.c) } else { mi_assert_internal(arena->blocks_committed != NULL); _mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails + // todo: use reset instead of decommit on windows? _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); } // and make it available to others again @@ -300,12 +301,16 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->is_large = is_large; arena->is_zero_init = is_zero; - arena->is_committed = is_committed; + arena->allow_decommit = !is_large && !is_committed; // only allow decommit for initially uncommitted memory arena->search_idx = 0; arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap - arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + arena->blocks_committed = (!arena->allow_decommit ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap // the bitmaps are already zero initialized due to os_alloc - // just claim leftover blocks if needed + // initialize committed bitmap? + if (arena->blocks_committed != NULL && is_committed) { + memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + } + // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; mi_assert_internal(post >= 0); if (post > 0) {