From 52116d808c0894a6ef66075aaa08449e3474cec8 Mon Sep 17 00:00:00 2001 From: Sangheon Kim Date: Wed, 13 Nov 2019 10:49:12 -0800 Subject: [PATCH] 8220310: Implementation: NUMA-Aware Memory Allocation for G1, Mutator (1/3) Reviewed-by: kbarrett, sjohanss, tschatzl, pliden --- src/hotspot/os/bsd/os_bsd.cpp | 4 + src/hotspot/os/linux/os_linux.cpp | 17 ++ src/hotspot/os/solaris/os_solaris.cpp | 6 +- src/hotspot/os/windows/os_windows.cpp | 4 + src/hotspot/share/gc/g1/g1AllocRegion.cpp | 10 +- src/hotspot/share/gc/g1/g1AllocRegion.hpp | 18 +- src/hotspot/share/gc/g1/g1Allocator.cpp | 49 +++- src/hotspot/share/gc/g1/g1Allocator.hpp | 28 +- .../share/gc/g1/g1Allocator.inline.hpp | 24 +- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 49 +++- src/hotspot/share/gc/g1/g1CollectedHeap.hpp | 14 +- src/hotspot/share/gc/g1/g1NUMA.cpp | 227 ++++++++++++++++ src/hotspot/share/gc/g1/g1NUMA.hpp | 118 +++++++++ .../share/gc/g1/g1PageBasedVirtualSpace.cpp | 5 + .../share/gc/g1/g1PageBasedVirtualSpace.hpp | 6 +- .../share/gc/g1/g1RegionToSpaceMapper.cpp | 40 ++- .../share/gc/g1/g1RegionToSpaceMapper.hpp | 2 + src/hotspot/share/gc/g1/heapRegion.cpp | 15 +- src/hotspot/share/gc/g1/heapRegion.hpp | 5 + src/hotspot/share/gc/g1/heapRegionManager.cpp | 59 +++++ src/hotspot/share/gc/g1/heapRegionManager.hpp | 17 +- src/hotspot/share/gc/g1/heapRegionSet.hpp | 4 + .../share/gc/g1/heapRegionSet.inline.hpp | 62 +++++ .../gc/g1/heterogeneousHeapRegionManager.cpp | 4 +- .../gc/g1/heterogeneousHeapRegionManager.hpp | 2 +- src/hotspot/share/logging/logPrefix.hpp | 1 + src/hotspot/share/logging/logTag.hpp | 1 + src/hotspot/share/prims/whitebox.cpp | 25 ++ src/hotspot/share/runtime/arguments.cpp | 13 +- src/hotspot/share/runtime/os.hpp | 1 + .../gc/g1/numa/TestG1NUMATouchRegions.java | 245 ++++++++++++++++++ test/lib/sun/hotspot/WhiteBox.java | 3 + 32 files changed, 990 insertions(+), 88 deletions(-) create mode 100644 src/hotspot/share/gc/g1/g1NUMA.cpp create mode 100644 src/hotspot/share/gc/g1/g1NUMA.hpp create mode 100644 test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp index d9fa9d4149..6b31ad62ac 100644 --- a/src/hotspot/os/bsd/os_bsd.cpp +++ b/src/hotspot/os/bsd/os_bsd.cpp @@ -2005,6 +2005,10 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) { return 0; } +int os::numa_get_group_id_for_address(const void* address) { + return 0; +} + bool os::get_page_info(char *start, page_info* info) { return false; } diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp index 2083a5dc36..7460723093 100644 --- a/src/hotspot/os/linux/os_linux.cpp +++ b/src/hotspot/os/linux/os_linux.cpp @@ -3007,6 +3007,23 @@ int os::numa_get_group_id() { return 0; } +int os::numa_get_group_id_for_address(const void* address) { +#ifndef MPOL_F_NODE +#define MPOL_F_NODE (1<<0) // Return next IL mode instead of node mask +#endif + +#ifndef MPOL_F_ADDR +#define MPOL_F_ADDR (1<<1) // Look up VMA using address +#endif + + int id = 0; + + if (syscall(SYS_get_mempolicy, &id, NULL, 0, const_cast(address), MPOL_F_NODE | MPOL_F_ADDR) == -1) { + return -1; + } + return id; +} + int os::Linux::get_existing_num_nodes() { int node; int highest_node_number = Linux::numa_max_node(); diff --git a/src/hotspot/os/solaris/os_solaris.cpp b/src/hotspot/os/solaris/os_solaris.cpp index 12a8b14f4f..c117cb0aa0 100644 --- a/src/hotspot/os/solaris/os_solaris.cpp +++ b/src/hotspot/os/solaris/os_solaris.cpp @@ -2072,7 +2072,7 @@ int os::Solaris::commit_memory_impl(char* addr, size_t bytes, bool exec) { char *res = Solaris::mmap_chunk(addr, size, MAP_PRIVATE|MAP_FIXED, prot); if (res != NULL) { if (UseNUMAInterleaving) { - numa_make_global(addr, bytes); + numa_make_global(addr, bytes); } return 0; } @@ -2267,6 +2267,10 @@ int os::numa_get_group_id() { return ids[os::random() % r]; } +int os::numa_get_group_id_for_address(const void* address) { + return 0; +} + // Request information about the page. bool os::get_page_info(char *start, page_info* info) { const uint_t info_types[] = { MEMINFO_VLGRP, MEMINFO_VPAGESIZE }; diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index 782329cddc..d1d551f5cd 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -3447,6 +3447,10 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) { } } +int os::numa_get_group_id_for_address(const void* address) { + return 0; +} + bool os::get_page_info(char *start, page_info* info) { return false; } diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.cpp b/src/hotspot/share/gc/g1/g1AllocRegion.cpp index 32265bb2af..8bacc1a514 100644 --- a/src/hotspot/share/gc/g1/g1AllocRegion.cpp +++ b/src/hotspot/share/gc/g1/g1AllocRegion.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -250,17 +250,19 @@ void G1AllocRegion::trace(const char* str, size_t min_word_size, size_t desired_ #endif // PRODUCT G1AllocRegion::G1AllocRegion(const char* name, - bool bot_updates) + bool bot_updates, + uint node_index) : _alloc_region(NULL), _count(0), _used_bytes_before(0), _bot_updates(bot_updates), - _name(name) + _name(name), + _node_index(node_index) { } HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size, bool force) { - return _g1h->new_mutator_alloc_region(word_size, force); + return _g1h->new_mutator_alloc_region(word_size, force, _node_index); } void MutatorAllocRegion::retire_region(HeapRegion* alloc_region, diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.hpp index 93ba0bf10b..05fd041b84 100644 --- a/src/hotspot/share/gc/g1/g1AllocRegion.hpp +++ b/src/hotspot/share/gc/g1/g1AllocRegion.hpp @@ -28,6 +28,7 @@ #include "gc/g1/heapRegion.hpp" #include "gc/g1/g1EvacStats.hpp" #include "gc/g1/g1HeapRegionAttr.hpp" +#include "gc/g1/g1NUMA.hpp" class G1CollectedHeap; @@ -38,7 +39,7 @@ class G1CollectedHeap; // and a lock will need to be taken when the active region needs to be // replaced. -class G1AllocRegion { +class G1AllocRegion : public CHeapObj { private: // The active allocating region we are currently allocating out @@ -91,6 +92,9 @@ private: HeapWord* new_alloc_region_and_allocate(size_t word_size, bool force); protected: + // The memory node index this allocation region belongs to. + uint _node_index; + // Reset the alloc region to point a the dummy region. void reset_alloc_region(); @@ -131,7 +135,7 @@ protected: virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes) = 0; - G1AllocRegion(const char* name, bool bot_updates); + G1AllocRegion(const char* name, bool bot_updates, uint node_index); public: static void setup(G1CollectedHeap* g1h, HeapRegion* dummy_region); @@ -220,8 +224,8 @@ protected: virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); virtual size_t retire(bool fill_up); public: - MutatorAllocRegion() - : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */), + MutatorAllocRegion(uint node_index) + : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */, node_index), _wasted_bytes(0), _retained_alloc_region(NULL) { } @@ -245,6 +249,7 @@ public: virtual void init(); }; + // Common base class for allocation regions used during GC. class G1GCAllocRegion : public G1AllocRegion { protected: @@ -256,8 +261,9 @@ protected: virtual size_t retire(bool fill_up); - G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats, G1HeapRegionAttr::region_type_t purpose) - : G1AllocRegion(name, bot_updates), _stats(stats), _purpose(purpose) { + G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats, + G1HeapRegionAttr::region_type_t purpose, uint node_index = G1NUMA::AnyNodeIndex) + : G1AllocRegion(name, bot_updates, node_index), _stats(stats), _purpose(purpose) { assert(stats != NULL, "Must pass non-NULL PLAB statistics"); } }; diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp index 07e1a5c05b..a39c8eee5a 100644 --- a/src/hotspot/share/gc/g1/g1Allocator.cpp +++ b/src/hotspot/share/gc/g1/g1Allocator.cpp @@ -28,6 +28,7 @@ #include "gc/g1/g1EvacStats.inline.hpp" #include "gc/g1/g1EvacuationInfo.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1NUMA.hpp" #include "gc/g1/g1Policy.hpp" #include "gc/g1/heapRegion.inline.hpp" #include "gc/g1/heapRegionSet.inline.hpp" @@ -36,22 +37,47 @@ G1Allocator::G1Allocator(G1CollectedHeap* heap) : _g1h(heap), + _numa(heap->numa()), _survivor_is_full(false), _old_is_full(false), - _mutator_alloc_region(), + _num_alloc_regions(_numa->num_active_nodes()), + _mutator_alloc_regions(NULL), _survivor_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Young)), _old_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Old)), _retained_old_gc_alloc_region(NULL) { + + _mutator_alloc_regions = NEW_C_HEAP_ARRAY(MutatorAllocRegion, _num_alloc_regions, mtGC); + for (uint i = 0; i < _num_alloc_regions; i++) { + ::new(_mutator_alloc_regions + i) MutatorAllocRegion(i); + } } -void G1Allocator::init_mutator_alloc_region() { - assert(_mutator_alloc_region.get() == NULL, "pre-condition"); - _mutator_alloc_region.init(); +G1Allocator::~G1Allocator() { + for (uint i = 0; i < _num_alloc_regions; i++) { + _mutator_alloc_regions[i].~MutatorAllocRegion(); + } + FREE_C_HEAP_ARRAY(MutatorAllocRegion, _mutator_alloc_regions); } -void G1Allocator::release_mutator_alloc_region() { - _mutator_alloc_region.release(); - assert(_mutator_alloc_region.get() == NULL, "post-condition"); +#ifdef ASSERT +bool G1Allocator::has_mutator_alloc_region() { + uint node_index = current_node_index(); + return mutator_alloc_region(node_index)->get() != NULL; +} +#endif + +void G1Allocator::init_mutator_alloc_regions() { + for (uint i = 0; i < _num_alloc_regions; i++) { + assert(mutator_alloc_region(i)->get() == NULL, "pre-condition"); + mutator_alloc_region(i)->init(); + } +} + +void G1Allocator::release_mutator_alloc_regions() { + for (uint i = 0; i < _num_alloc_regions; i++) { + mutator_alloc_region(i)->release(); + assert(mutator_alloc_region(i)->get() == NULL, "post-condition"); + } } bool G1Allocator::is_retained_old_region(HeapRegion* hr) { @@ -146,7 +172,8 @@ size_t G1Allocator::unsafe_max_tlab_alloc() { // since we can't allow tlabs to grow big enough to accommodate // humongous objects. - HeapRegion* hr = mutator_alloc_region()->get(); + uint node_index = current_node_index(); + HeapRegion* hr = mutator_alloc_region(node_index)->get(); size_t max_tlab = _g1h->max_tlab_size() * wordSize; if (hr == NULL) { return max_tlab; @@ -157,7 +184,11 @@ size_t G1Allocator::unsafe_max_tlab_alloc() { size_t G1Allocator::used_in_alloc_regions() { assert(Heap_lock->owner() != NULL, "Should be owned on this thread's behalf."); - return mutator_alloc_region()->used_in_alloc_regions(); + size_t used = 0; + for (uint i = 0; i < _num_alloc_regions; i++) { + used += mutator_alloc_region(i)->used_in_alloc_regions(); + } + return used; } diff --git a/src/hotspot/share/gc/g1/g1Allocator.hpp b/src/hotspot/share/gc/g1/g1Allocator.hpp index a99a256fea..5c2fc8c75f 100644 --- a/src/hotspot/share/gc/g1/g1Allocator.hpp +++ b/src/hotspot/share/gc/g1/g1Allocator.hpp @@ -31,6 +31,7 @@ #include "gc/shared/plab.hpp" class G1EvacuationInfo; +class G1NUMA; // Interface to keep track of which regions G1 is currently allocating into. Provides // some accessors (e.g. allocating into them, or getting their occupancy). @@ -40,12 +41,16 @@ class G1Allocator : public CHeapObj { private: G1CollectedHeap* _g1h; + G1NUMA* _numa; bool _survivor_is_full; bool _old_is_full; + // The number of MutatorAllocRegions used, one per memory node. + size_t _num_alloc_regions; + // Alloc region used to satisfy mutator allocation requests. - MutatorAllocRegion _mutator_alloc_region; + MutatorAllocRegion* _mutator_alloc_regions; // Alloc region used to satisfy allocation requests by the GC for // survivor objects. @@ -68,29 +73,34 @@ private: HeapRegion** retained); // Accessors to the allocation regions. - inline MutatorAllocRegion* mutator_alloc_region(); + inline MutatorAllocRegion* mutator_alloc_region(uint node_index); inline SurvivorGCAllocRegion* survivor_gc_alloc_region(); inline OldGCAllocRegion* old_gc_alloc_region(); // Allocation attempt during GC for a survivor object / PLAB. HeapWord* survivor_attempt_allocation(size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size); + size_t desired_word_size, + size_t* actual_word_size); // Allocation attempt during GC for an old object / PLAB. HeapWord* old_attempt_allocation(size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size); + size_t desired_word_size, + size_t* actual_word_size); + + // Node index of current thread. + inline uint current_node_index() const; + public: G1Allocator(G1CollectedHeap* heap); + ~G1Allocator(); #ifdef ASSERT // Do we currently have an active mutator region to allocate into? - bool has_mutator_alloc_region() { return mutator_alloc_region()->get() != NULL; } + bool has_mutator_alloc_region(); #endif - void init_mutator_alloc_region(); - void release_mutator_alloc_region(); + void init_mutator_alloc_regions(); + void release_mutator_alloc_regions(); void init_gc_alloc_regions(G1EvacuationInfo& evacuation_info); void release_gc_alloc_regions(G1EvacuationInfo& evacuation_info); diff --git a/src/hotspot/share/gc/g1/g1Allocator.inline.hpp b/src/hotspot/share/gc/g1/g1Allocator.inline.hpp index 39f3e32f0a..2f562cd958 100644 --- a/src/hotspot/share/gc/g1/g1Allocator.inline.hpp +++ b/src/hotspot/share/gc/g1/g1Allocator.inline.hpp @@ -30,8 +30,13 @@ #include "gc/shared/plab.inline.hpp" #include "memory/universe.hpp" -inline MutatorAllocRegion* G1Allocator::mutator_alloc_region() { - return &_mutator_alloc_region; +inline uint G1Allocator::current_node_index() const { + return _numa->index_of_current_thread(); +} + +inline MutatorAllocRegion* G1Allocator::mutator_alloc_region(uint node_index) { + assert(node_index < _num_alloc_regions, "Invalid index: %u", node_index); + return &_mutator_alloc_regions[node_index]; } inline SurvivorGCAllocRegion* G1Allocator::survivor_gc_alloc_region() { @@ -45,22 +50,25 @@ inline OldGCAllocRegion* G1Allocator::old_gc_alloc_region() { inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size) { - HeapWord* result = mutator_alloc_region()->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size); + uint node_index = current_node_index(); + HeapWord* result = mutator_alloc_region(node_index)->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size); if (result != NULL) { return result; } - return mutator_alloc_region()->attempt_allocation(min_word_size, desired_word_size, actual_word_size); + return mutator_alloc_region(node_index)->attempt_allocation(min_word_size, desired_word_size, actual_word_size); } inline HeapWord* G1Allocator::attempt_allocation_locked(size_t word_size) { - HeapWord* result = mutator_alloc_region()->attempt_allocation_locked(word_size); - assert(result != NULL || mutator_alloc_region()->get() == NULL, - "Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region()->get())); + uint node_index = current_node_index(); + HeapWord* result = mutator_alloc_region(node_index)->attempt_allocation_locked(word_size); + assert(result != NULL || mutator_alloc_region(node_index)->get() == NULL, + "Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region(node_index)->get())); return result; } inline HeapWord* G1Allocator::attempt_allocation_force(size_t word_size) { - return mutator_alloc_region()->attempt_allocation_force(word_size); + uint node_index = current_node_index(); + return mutator_alloc_region(node_index)->attempt_allocation_force(word_size); } inline PLAB* G1PLABAllocator::alloc_buffer(G1HeapRegionAttr dest) { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index 8e8f878d2e..82e43a4a2c 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -169,12 +169,15 @@ HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, // Private methods. -HeapRegion* G1CollectedHeap::new_region(size_t word_size, HeapRegionType type, bool do_expand) { +HeapRegion* G1CollectedHeap::new_region(size_t word_size, + HeapRegionType type, + bool do_expand, + uint node_index) { assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords, "the only time we use this to allocate a humongous region is " "when we are allocating a single humongous region"); - HeapRegion* res = _hrm->allocate_free_region(type); + HeapRegion* res = _hrm->allocate_free_region(type, node_index); if (res == NULL && do_expand && _expand_heap_after_alloc_failure) { // Currently, only attempts to allocate GC alloc regions set @@ -186,12 +189,15 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size, HeapRegionType type, b log_debug(gc, ergo, heap)("Attempt heap expansion (region allocation request failed). Allocation request: " SIZE_FORMAT "B", word_size * HeapWordSize); - if (expand(word_size * HeapWordSize)) { - // Given that expand() succeeded in expanding the heap, and we + assert(word_size * HeapWordSize < HeapRegion::GrainBytes, + "This kind of expansion should never be more than one region. Size: " SIZE_FORMAT, + word_size * HeapWordSize); + if (expand_single_region(node_index)) { + // Given that expand_single_region() succeeded in expanding the heap, and we // always expand the heap by an amount aligned to the heap // region size, the free list should in theory not be empty. // In either case allocate_free_region() will check for NULL. - res = _hrm->allocate_free_region(type); + res = _hrm->allocate_free_region(type, node_index); } else { _expand_heap_after_alloc_failure = false; } @@ -1020,7 +1026,7 @@ void G1CollectedHeap::abort_concurrent_cycle() { void G1CollectedHeap::prepare_heap_for_full_collection() { // Make sure we'll choose a new allocation region afterwards. - _allocator->release_mutator_alloc_region(); + _allocator->release_mutator_alloc_regions(); _allocator->abandon_gc_alloc_regions(); // We may have added regions to the current incremental collection @@ -1064,7 +1070,7 @@ void G1CollectedHeap::prepare_heap_for_mutators() { // Start a new incremental collection set for the next pause start_new_collection_set(); - _allocator->init_mutator_alloc_region(); + _allocator->init_mutator_alloc_regions(); // Post collection state updates. MetaspaceGC::compute_new_size(); @@ -1381,6 +1387,19 @@ bool G1CollectedHeap::expand(size_t expand_bytes, WorkGang* pretouch_workers, do return regions_to_expand > 0; } +bool G1CollectedHeap::expand_single_region(uint node_index) { + uint expanded_by = _hrm->expand_on_preferred_node(node_index); + + if (expanded_by == 0) { + assert(is_maximal_no_gc(), "Should be no regions left, available: %u", _hrm->available()); + log_debug(gc, ergo, heap)("Did not expand the heap (heap already fully expanded)"); + return false; + } + + policy()->record_new_heap_size(num_regions()); + return true; +} + void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { size_t aligned_shrink_bytes = ReservedSpace::page_align_size_down(shrink_bytes); @@ -1391,7 +1410,6 @@ void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { uint num_regions_removed = _hrm->shrink_by(num_regions_to_remove); size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes; - log_debug(gc, ergo, heap)("Shrink the heap. requested shrinking amount: " SIZE_FORMAT "B aligned shrinking amount: " SIZE_FORMAT "B attempted shrinking amount: " SIZE_FORMAT "B", shrink_bytes, aligned_shrink_bytes, shrunk_bytes); if (num_regions_removed > 0) { @@ -1493,6 +1511,7 @@ G1CollectedHeap::G1CollectedHeap() : _humongous_set("Humongous Region Set", new HumongousRegionSetChecker()), _bot(NULL), _listener(), + _numa(G1NUMA::create()), _hrm(NULL), _allocator(NULL), _verifier(NULL), @@ -1775,6 +1794,8 @@ jint G1CollectedHeap::initialize() { } _workers->initialize_workers(); + _numa->set_region_info(HeapRegion::GrainBytes, page_size); + // Create the G1ConcurrentMark data structure and thread. // (Must do this late, so that "max_regions" is defined.) _cm = new G1ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage); @@ -1822,7 +1843,7 @@ jint G1CollectedHeap::initialize() { dummy_region->set_top(dummy_region->end()); G1AllocRegion::setup(this, dummy_region); - _allocator->init_mutator_alloc_region(); + _allocator->init_mutator_alloc_regions(); // Do create of the monitoring and management support so that // values in the heap have been properly initialized. @@ -3005,7 +3026,7 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ // Forget the current allocation region (we might even choose it to be part // of the collection set!). - _allocator->release_mutator_alloc_region(); + _allocator->release_mutator_alloc_regions(); calculate_collection_set(evacuation_info, target_pause_time_ms); @@ -3042,7 +3063,7 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ allocate_dummy_regions(); - _allocator->init_mutator_alloc_region(); + _allocator->init_mutator_alloc_regions(); expand_heap_after_young_collection(); @@ -4538,13 +4559,15 @@ void G1CollectedHeap::rebuild_region_sets(bool free_list_only) { // Methods for the mutator alloc region HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size, - bool force) { + bool force, + uint node_index) { assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); bool should_allocate = policy()->should_allocate_mutator_region(); if (force || should_allocate) { HeapRegion* new_alloc_region = new_region(word_size, HeapRegionType::Eden, - false /* do_expand */); + false /* do_expand */, + node_index); if (new_alloc_region != NULL) { set_region_short_lived_locked(new_alloc_region); _hr_printer.alloc(new_alloc_region, !should_allocate); diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index 149dcf26f7..750b43e647 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -41,6 +41,7 @@ #include "gc/g1/g1HRPrinter.hpp" #include "gc/g1/g1HeapRegionAttr.hpp" #include "gc/g1/g1MonitoringSupport.hpp" +#include "gc/g1/g1NUMA.hpp" #include "gc/g1/g1RedirtyCardsQueue.hpp" #include "gc/g1/g1SurvivorRegions.hpp" #include "gc/g1/g1YCTypes.hpp" @@ -191,6 +192,9 @@ private: // Callback for region mapping changed events. G1RegionMappingChangedListener _listener; + // Handle G1 NUMA support. + G1NUMA* _numa; + // The sequence of all heap regions in the heap. HeapRegionManager* _hrm; @@ -387,7 +391,10 @@ private: // attempt to expand the heap if necessary to satisfy the allocation // request. 'type' takes the type of region to be allocated. (Use constants // Old, Eden, Humongous, Survivor defined in HeapRegionType.) - HeapRegion* new_region(size_t word_size, HeapRegionType type, bool do_expand); + HeapRegion* new_region(size_t word_size, + HeapRegionType type, + bool do_expand, + uint node_index = G1NUMA::AnyNodeIndex); // Initialize a contiguous set of free regions of length num_regions // and starting at index first so that they appear as a single @@ -462,7 +469,7 @@ private: // These methods are the "callbacks" from the G1AllocRegion class. // For mutator alloc regions. - HeapRegion* new_mutator_alloc_region(size_t word_size, bool force); + HeapRegion* new_mutator_alloc_region(size_t word_size, bool force, uint node_index); void retire_mutator_alloc_region(HeapRegion* alloc_region, size_t allocated_bytes); @@ -547,11 +554,14 @@ public: void resize_heap_if_necessary(); + G1NUMA* numa() const { return _numa; } + // Expand the garbage-first heap by at least the given size (in bytes!). // Returns true if the heap was expanded by the requested amount; // false otherwise. // (Rounds up to a HeapRegion boundary.) bool expand(size_t expand_bytes, WorkGang* pretouch_workers = NULL, double* expand_time_ms = NULL); + bool expand_single_region(uint node_index); // Returns the PLAB statistics for a given destination. inline G1EvacStats* alloc_buffer_stats(G1HeapRegionAttr dest); diff --git a/src/hotspot/share/gc/g1/g1NUMA.cpp b/src/hotspot/share/gc/g1/g1NUMA.cpp new file mode 100644 index 0000000000..931a1c606d --- /dev/null +++ b/src/hotspot/share/gc/g1/g1NUMA.cpp @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1NUMA.hpp" +#include "gc/g1/heapRegion.hpp" +#include "logging/log.hpp" +#include "runtime/globals.hpp" +#include "runtime/os.hpp" + +G1NUMA* G1NUMA::_inst = NULL; + +size_t G1NUMA::region_size() const { + assert(_region_size > 0, "Heap region size is not yet set"); + return _region_size; +} + +size_t G1NUMA::page_size() const { + assert(_page_size > 0, "Page size not is yet set"); + return _page_size; +} + +bool G1NUMA::is_enabled() const { return num_active_nodes() > 1; } + +G1NUMA* G1NUMA::create() { + guarantee(_inst == NULL, "Should be called once."); + _inst = new G1NUMA(); + + // NUMA only supported on Linux. +#ifdef LINUX + _inst->initialize(UseNUMA); +#else + _inst->initialize(false); +#endif /* LINUX */ + + return _inst; +} + + // Returns memory node ids +const int* G1NUMA::node_ids() const { + return _node_ids; +} + +uint G1NUMA::index_of_node_id(int node_id) const { + assert(node_id >= 0, "invalid node id %d", node_id); + assert(node_id < _len_node_id_to_index_map, "invalid node id %d", node_id); + uint node_index = _node_id_to_index_map[node_id]; + assert(node_index != G1NUMA::UnknownNodeIndex, + "invalid node id %d", node_id); + return node_index; +} + +G1NUMA::G1NUMA() : + _node_id_to_index_map(NULL), _len_node_id_to_index_map(0), + _node_ids(NULL), _num_active_node_ids(0), + _region_size(0), _page_size(0) { +} + +void G1NUMA::initialize_without_numa() { + // If NUMA is not enabled or supported, initialize as having a singel node. + _num_active_node_ids = 1; + _node_ids = NEW_C_HEAP_ARRAY(int, _num_active_node_ids, mtGC); + _node_ids[0] = 0; + // Map index 0 to node 0 + _len_node_id_to_index_map = 1; + _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC); + _node_id_to_index_map[0] = 0; +} + +void G1NUMA::initialize(bool use_numa) { + if (!use_numa) { + initialize_without_numa(); + return; + } + + assert(UseNUMA, "Invariant"); + size_t num_node_ids = os::numa_get_groups_num(); + + // Create an array of active node ids. + _node_ids = NEW_C_HEAP_ARRAY(int, num_node_ids, mtGC); + _num_active_node_ids = (uint)os::numa_get_leaf_groups(_node_ids, num_node_ids); + + int max_node_id = 0; + for (uint i = 0; i < _num_active_node_ids; i++) { + max_node_id = MAX2(max_node_id, _node_ids[i]); + } + + // Create a mapping between node_id and index. + _len_node_id_to_index_map = max_node_id + 1; + _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC); + + // Set all indices with unknown node id. + for (int i = 0; i < _len_node_id_to_index_map; i++) { + _node_id_to_index_map[i] = G1NUMA::UnknownNodeIndex; + } + + // Set the indices for the actually retrieved node ids. + for (uint i = 0; i < _num_active_node_ids; i++) { + _node_id_to_index_map[_node_ids[i]] = i; + } +} + +G1NUMA::~G1NUMA() { + FREE_C_HEAP_ARRAY(int, _node_id_to_index_map); + FREE_C_HEAP_ARRAY(int, _node_ids); +} + +void G1NUMA::set_region_info(size_t region_size, size_t page_size) { + _region_size = region_size; + _page_size = page_size; +} + +uint G1NUMA::num_active_nodes() const { + assert(_num_active_node_ids > 0, "just checking"); + return _num_active_node_ids; +} + +uint G1NUMA::index_of_current_thread() const { + if (!is_enabled()) { + return 0; + } + return index_of_node_id(os::numa_get_group_id()); +} + +uint G1NUMA::preferred_node_index_for_index(uint region_index) const { + if (region_size() >= page_size()) { + // Simple case, pages are smaller than the region so we + // can just alternate over the nodes. + return region_index % _num_active_node_ids; + } else { + // Multiple regions in one page, so we need to make sure the + // regions within a page is preferred on the same node. + size_t regions_per_page = page_size() / region_size(); + return (region_index / regions_per_page) % _num_active_node_ids; + } +} + +int G1NUMA::numa_id(int index) const { + assert(index < _len_node_id_to_index_map, "Index %d out of range: [0,%d)", + index, _len_node_id_to_index_map); + return _node_ids[index]; +} + +uint G1NUMA::index_of_address(HeapWord *address) const { + int numa_id = os::numa_get_group_id_for_address((const void*)address); + if (numa_id == -1) { + return UnknownNodeIndex; + } else { + return index_of_node_id(numa_id); + } +} + +uint G1NUMA::index_for_region(HeapRegion* hr) const { + if (!is_enabled()) { + return 0; + } + + if (AlwaysPreTouch) { + // If we already pretouched, we can check actual node index here. + // However, if node index is still unknown, use preferred node index. + uint node_index = index_of_address(hr->bottom()); + if (node_index != UnknownNodeIndex) { + return node_index; + } + } + + return preferred_node_index_for_index(hr->hrm_index()); +} + +// Request to spread the given memory evenly across the available NUMA +// nodes. Which node to request for a given address is given by the +// region size and the page size. Below are two examples on 4 NUMA nodes system: +// 1. G1HeapRegionSize(_region_size) is larger than or equal to page size. +// * Page #: |-0--||-1--||-2--||-3--||-4--||-5--||-6--||-7--||-8--||-9--||-10-||-11-||-12-||-13-||-14-||-15-| +// * HeapRegion #: |----#0----||----#1----||----#2----||----#3----||----#4----||----#5----||----#6----||----#7----| +// * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----| +// 2. G1HeapRegionSize(_region_size) is smaller than page size. +// Memory will be touched one page at a time because G1RegionToSpaceMapper commits +// pages one by one. +// * Page #: |-----0----||-----1----||-----2----||-----3----||-----4----||-----5----||-----6----||-----7----| +// * HeapRegion #: |-#0-||-#1-||-#2-||-#3-||-#4-||-#5-||-#6-||-#7-||-#8-||-#9-||#10-||#11-||#12-||#13-||#14-||#15-| +// * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----| +void G1NUMA::request_memory_on_node(void* aligned_address, size_t size_in_bytes, uint region_index) { + if (!is_enabled()) { + return; + } + + if (size_in_bytes == 0) { + return; + } + + uint node_index = preferred_node_index_for_index(region_index); + + assert(is_aligned(aligned_address, page_size()), "Given address (" PTR_FORMAT ") should be aligned.", p2i(aligned_address)); + assert(is_aligned(size_in_bytes, page_size()), "Given size (" SIZE_FORMAT ") should be aligned.", size_in_bytes); + + log_debug(gc, heap, numa)("Request memory [" PTR_FORMAT ", " PTR_FORMAT ") to be numa id (%d).", + p2i(aligned_address), p2i((char*)aligned_address + size_in_bytes), _node_ids[node_index]); + os::numa_make_local((char*)aligned_address, size_in_bytes, _node_ids[node_index]); +} + +uint G1NUMA::max_search_depth() const { + // Multiple of 3 is just random number to limit iterations. + // There would be some cases that 1 page may be consisted of multiple HeapRegions. + return 3 * MAX2((uint)(page_size() / region_size()), (uint)1) * num_active_nodes(); +} diff --git a/src/hotspot/share/gc/g1/g1NUMA.hpp b/src/hotspot/share/gc/g1/g1NUMA.hpp new file mode 100644 index 0000000000..3749f719de --- /dev/null +++ b/src/hotspot/share/gc/g1/g1NUMA.hpp @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_NUMA_HPP +#define SHARE_VM_GC_G1_NUMA_HPP + +#include "memory/allocation.hpp" +#include "runtime/os.hpp" + +class HeapRegion; + +class G1NUMA: public CHeapObj { + // Mapping of available node ids to 0-based index which can be used for + // fast resource management. I.e. for every node id provides a unique value in + // the range from [0, {# of nodes-1}]. + // For invalid node id, return UnknownNodeIndex. + uint* _node_id_to_index_map; + // Length of _num_active_node_ids_id to index map. + int _len_node_id_to_index_map; + + // Current active node ids. + int* _node_ids; + // Total number of node ids. + uint _num_active_node_ids; + + // HeapRegion size + size_t _region_size; + // Necessary when touching memory. + size_t _page_size; + + size_t region_size() const; + size_t page_size() const; + + // Returns node index of the given node id. + // Precondition: node_id is an active node id. + inline uint index_of_node_id(int node_id) const; + + // Creates node id and node index mapping table of _node_id_to_index_map. + void init_node_id_to_index_map(const int* node_ids, uint num_node_ids); + + static G1NUMA* _inst; + + G1NUMA(); + void initialize(bool use_numa); + void initialize_without_numa(); + +public: + static const uint UnknownNodeIndex = UINT_MAX; + static const uint AnyNodeIndex = UnknownNodeIndex - 1; + + static G1NUMA* numa() { return _inst; } + + static G1NUMA* create(); + + ~G1NUMA(); + + // Sets heap region size and page size after those values + // are determined at G1CollectedHeap::initialize(). + void set_region_info(size_t region_size, size_t page_size); + + // Returns active memory node count. + uint num_active_nodes() const; + + bool is_enabled() const; + + int numa_id(int index) const; + + // Returns memory node ids + const int* node_ids() const; + + // Returns node index of current calling thread. + uint index_of_current_thread() const; + + // Returns the preferred index for the given HeapRegion index. + // This assumes that HeapRegions are evenly spit, so we can decide preferred index + // with the given HeapRegion index. + // Result is less than num_active_nodes(). + uint preferred_node_index_for_index(uint region_index) const; + + // Retrieves node index of the given address. + // Result is less than num_active_nodes() or is UnknownNodeIndex. + // Precondition: address is in reserved range for heap. + uint index_of_address(HeapWord* address) const; + + // If AlwaysPreTouch is enabled, return actual node index via system call. + // If disabled, return preferred node index of the given heap region. + uint index_for_region(HeapRegion* hr) const; + + // Requests the given memory area to be located at the given node index. + void request_memory_on_node(void* aligned_address, size_t size_in_bytes, uint region_index); + + // Returns maximum search depth which is used to limit heap region search iterations. + // The number of active nodes, page size and heap region size are considered. + uint max_search_depth() const; +}; + +#endif // SHARE_VM_GC_G1_NUMA_HPP diff --git a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp index 9a9c3894b4..58680e460d 100644 --- a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp +++ b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp @@ -124,6 +124,11 @@ char* G1PageBasedVirtualSpace::page_start(size_t index) const { return _low_boundary + index * _page_size; } +size_t G1PageBasedVirtualSpace::page_size() const { + assert(_page_size > 0, "Page size is not yet initialized."); + return _page_size; +} + bool G1PageBasedVirtualSpace::is_after_last_page(size_t index) const { guarantee(index <= _committed.size(), "Given boundary page " SIZE_FORMAT " is beyond managed page count " SIZE_FORMAT, index, _committed.size()); diff --git a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp index 3a1319dde4..cdcea4f28f 100644 --- a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp +++ b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp @@ -92,8 +92,6 @@ class G1PageBasedVirtualSpace { // Returns the index of the page which contains the given address. size_t addr_to_page_index(char* addr) const; - // Returns the address of the given page index. - char* page_start(size_t index) const; // Is the given page index the last page? bool is_last_page(size_t index) const { return index == (_committed.size() - 1); } @@ -147,6 +145,10 @@ class G1PageBasedVirtualSpace { void check_for_contiguity() PRODUCT_RETURN; + // Returns the address of the given page index. + char* page_start(size_t index) const; + size_t page_size() const; + // Debugging void print_on(outputStream* out) PRODUCT_RETURN; void print(); diff --git a/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp b/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp index 61d7332d8b..b187f90ac7 100644 --- a/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp +++ b/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "gc/g1/g1BiasedArray.hpp" +#include "gc/g1/g1NUMA.hpp" #include "gc/g1/g1RegionToSpaceMapper.hpp" #include "logging/log.hpp" #include "memory/allocation.inline.hpp" @@ -44,7 +45,8 @@ G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs, _listener(NULL), _storage(rs, used_size, page_size), _region_granularity(region_granularity), - _commit_map(rs.size() * commit_factor / region_granularity, mtGC) { + _commit_map(rs.size() * commit_factor / region_granularity, mtGC), + _memory_type(type) { guarantee(is_power_of_2(page_size), "must be"); guarantee(is_power_of_2(region_granularity), "must be"); @@ -72,10 +74,18 @@ class G1RegionsLargerThanCommitSizeMapper : public G1RegionToSpaceMapper { } virtual void commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) { - size_t const start_page = (size_t)start_idx * _pages_per_region; - bool zero_filled = _storage.commit(start_page, num_regions * _pages_per_region); + const size_t start_page = (size_t)start_idx * _pages_per_region; + const size_t size_in_pages = num_regions * _pages_per_region; + bool zero_filled = _storage.commit(start_page, size_in_pages); + if (_memory_type == mtJavaHeap) { + for (uint region_index = start_idx; region_index < start_idx + num_regions; region_index++ ) { + void* address = _storage.page_start(region_index * _pages_per_region); + size_t size_in_bytes = _storage.page_size() * _pages_per_region; + G1NUMA::numa()->request_memory_on_node(address, size_in_bytes, region_index); + } + } if (AlwaysPreTouch) { - _storage.pretouch(start_page, num_regions * _pages_per_region, pretouch_gang); + _storage.pretouch(start_page, size_in_pages, pretouch_gang); } _commit_map.set_range(start_idx, start_idx + num_regions); fire_on_commit(start_idx, num_regions, zero_filled); @@ -126,26 +136,32 @@ class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper { size_t num_committed = 0; bool all_zero_filled = true; + G1NUMA* numa = G1NUMA::numa(); - for (uint i = start_idx; i < start_idx + num_regions; i++) { - assert(!_commit_map.at(i), "Trying to commit storage at region %u that is already committed", i); - size_t idx = region_idx_to_page_idx(i); - uint old_refcount = _refcounts.get_by_index(idx); + for (uint region_idx = start_idx; region_idx < start_idx + num_regions; region_idx++) { + assert(!_commit_map.at(region_idx), "Trying to commit storage at region %u that is already committed", region_idx); + size_t page_idx = region_idx_to_page_idx(region_idx); + uint old_refcount = _refcounts.get_by_index(page_idx); bool zero_filled = false; if (old_refcount == 0) { if (first_committed == NoPage) { - first_committed = idx; + first_committed = page_idx; num_committed = 1; } else { num_committed++; } - zero_filled = _storage.commit(idx, 1); + zero_filled = _storage.commit(page_idx, 1); + if (_memory_type == mtJavaHeap) { + void* address = _storage.page_start(page_idx); + size_t size_in_bytes = _storage.page_size(); + numa->request_memory_on_node(address, size_in_bytes, region_idx); + } } all_zero_filled &= zero_filled; - _refcounts.set_by_index(idx, old_refcount + 1); - _commit_map.set_bit(i); + _refcounts.set_by_index(page_idx, old_refcount + 1); + _commit_map.set_bit(region_idx); } if (AlwaysPreTouch && num_committed > 0) { _storage.pretouch(first_committed, num_committed, pretouch_gang); diff --git a/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp b/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp index 9db83350ef..7ac80020e9 100644 --- a/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp +++ b/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp @@ -53,6 +53,8 @@ class G1RegionToSpaceMapper : public CHeapObj { // Mapping management CHeapBitMap _commit_map; + MemoryType _memory_type; + G1RegionToSpaceMapper(ReservedSpace rs, size_t used_size, size_t page_size, size_t region_granularity, size_t commit_factor, MemoryType type); void fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled); diff --git a/src/hotspot/share/gc/g1/heapRegion.cpp b/src/hotspot/share/gc/g1/heapRegion.cpp index e60388d802..f31565a326 100644 --- a/src/hotspot/share/gc/g1/heapRegion.cpp +++ b/src/hotspot/share/gc/g1/heapRegion.cpp @@ -28,6 +28,7 @@ #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectionSet.hpp" #include "gc/g1/g1HeapRegionTraceType.hpp" +#include "gc/g1/g1NUMA.hpp" #include "gc/g1/g1OopClosures.inline.hpp" #include "gc/g1/heapRegion.inline.hpp" #include "gc/g1/heapRegionBounds.inline.hpp" @@ -252,7 +253,8 @@ HeapRegion::HeapRegion(uint hrm_index, _index_in_opt_cset(InvalidCSetIndex), _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), _prev_top_at_mark_start(NULL), _next_top_at_mark_start(NULL), - _recorded_rs_length(0), _predicted_elapsed_time_ms(0) + _recorded_rs_length(0), _predicted_elapsed_time_ms(0), + _node_index(G1NUMA::UnknownNodeIndex) { _rem_set = new HeapRegionRemSet(bot, this); @@ -470,8 +472,17 @@ void HeapRegion::print_on(outputStream* st) const { } else { st->print("| "); } - st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ", + st->print("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ", p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()), rem_set()->get_state_str()); + if (UseNUMA) { + G1NUMA* numa = G1NUMA::numa(); + if (node_index() < numa->num_active_nodes()) { + st->print("|%02d", numa->numa_id(node_index())); + } else { + st->print("|--"); + } + } + st->print_cr(""); } class G1VerificationClosure : public BasicOopIterateClosure { diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/heapRegion.hpp index 038458ca94..4ac426bb33 100644 --- a/src/hotspot/share/gc/g1/heapRegion.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.hpp @@ -253,6 +253,8 @@ private: // for the collection set. double _predicted_elapsed_time_ms; + uint _node_index; + // Iterate over the references covered by the given MemRegion in a humongous // object and apply the given closure to them. // Humongous objects are allocated directly in the old-gen. So we need special @@ -643,6 +645,9 @@ public: // the strong code roots list for this region void strong_code_roots_do(CodeBlobClosure* blk) const; + uint node_index() const { return _node_index; } + void set_node_index(uint node_index) { _node_index = node_index; } + // Verify that the entries on the strong code root list for this // region are live and include at least one pointer into this region. void verify_strong_code_roots(VerifyOption vo, bool* failures) const; diff --git a/src/hotspot/share/gc/g1/heapRegionManager.cpp b/src/hotspot/share/gc/g1/heapRegionManager.cpp index 81cd7553ef..40635cb47e 100644 --- a/src/hotspot/share/gc/g1/heapRegionManager.cpp +++ b/src/hotspot/share/gc/g1/heapRegionManager.cpp @@ -30,6 +30,7 @@ #include "gc/g1/heapRegionManager.inline.hpp" #include "gc/g1/heapRegionSet.inline.hpp" #include "gc/g1/heterogeneousHeapRegionManager.hpp" +#include "logging/logStream.hpp" #include "memory/allocation.hpp" #include "utilities/bitMap.inline.hpp" @@ -103,6 +104,29 @@ bool HeapRegionManager::is_available(uint region) const { return _available_map.at(region); } +HeapRegion* HeapRegionManager::allocate_free_region(HeapRegionType type, uint requested_node_index) { + HeapRegion* hr = NULL; + bool from_head = !type.is_young(); + + if (requested_node_index != G1NUMA::AnyNodeIndex && G1NUMA::numa()->is_enabled()) { + // Try to allocate with requested node index. + hr = _free_list.remove_region_with_node_index(from_head, requested_node_index, NULL); + } + + if (hr == NULL) { + // If there's a single active node or we did not get a region from our requested node, + // try without requested node index. + hr = _free_list.remove_region(from_head); + } + + if (hr != NULL) { + assert(hr->next() == NULL, "Single region should not have next"); + assert(is_available(hr->hrm_index()), "Must be committed"); + } + + return hr; +} + #ifdef ASSERT bool HeapRegionManager::is_free(HeapRegion* hr) const { return _free_list.contains(hr); @@ -139,6 +163,11 @@ void HeapRegionManager::uncommit_regions(uint start, size_t num_regions) { guarantee(num_regions >= 1, "Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start); guarantee(_num_committed >= num_regions, "pre-condition"); + // Reset node index to distinguish with committed regions. + for (uint i = start; i < start + num_regions; i++) { + at(i)->set_node_index(G1NUMA::UnknownNodeIndex); + } + // Print before uncommitting. if (G1CollectedHeap::heap()->hr_printer()->is_active()) { for (uint i = start; i < start + num_regions; i++) { @@ -186,6 +215,7 @@ void HeapRegionManager::make_regions_available(uint start, uint num_regions, Wor MemRegion mr(bottom, bottom + HeapRegion::GrainWords); hr->initialize(mr); + hr->set_node_index(G1NUMA::numa()->index_for_region(hr)); insert_into_free_list(at(i)); } } @@ -235,6 +265,35 @@ uint HeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretou return expanded; } +uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) { + uint expand_candidate = UINT_MAX; + for (uint i = 0; i < max_length(); i++) { + if (is_available(i)) { + // Already in use continue + continue; + } + // Always save the candidate so we can expand later on. + expand_candidate = i; + if (is_on_preferred_index(expand_candidate, preferred_index)) { + // We have found a candidate on the preffered node, break. + break; + } + } + + if (expand_candidate == UINT_MAX) { + // No regions left, expand failed. + return 0; + } + + make_regions_available(expand_candidate, 1, NULL); + return 1; +} + +bool HeapRegionManager::is_on_preferred_index(uint region_index, uint preferred_node_index) { + uint region_node_index = G1NUMA::numa()->preferred_node_index_for_index(region_index); + return region_node_index == preferred_node_index; +} + uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) { uint found = 0; size_t length_found = 0; diff --git a/src/hotspot/share/gc/g1/heapRegionManager.hpp b/src/hotspot/share/gc/g1/heapRegionManager.hpp index 2bc5d29851..750c8fd13d 100644 --- a/src/hotspot/share/gc/g1/heapRegionManager.hpp +++ b/src/hotspot/share/gc/g1/heapRegionManager.hpp @@ -108,6 +108,9 @@ class HeapRegionManager: public CHeapObj { // sequence could be found, otherwise res_idx contains the start index of this range. uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const; + // Checks the G1MemoryNodeManager to see if this region is on the preferred node. + bool is_on_preferred_index(uint region_index, uint preferred_node_index); + protected: G1HeapRegionTable _regions; G1RegionToSpaceMapper* _heap_mapper; @@ -174,15 +177,8 @@ public: _free_list.add_ordered(list); } - virtual HeapRegion* allocate_free_region(HeapRegionType type) { - HeapRegion* hr = _free_list.remove_region(!type.is_young()); - - if (hr != NULL) { - assert(hr->next() == NULL, "Single region should not have next"); - assert(is_available(hr->hrm_index()), "Must be committed"); - } - return hr; - } + // Allocate a free region with specific node index. If fails allocate with next node index. + virtual HeapRegion* allocate_free_region(HeapRegionType type, uint requested_node_index); inline void allocate_free_regions_starting_at(uint first, uint num_regions); @@ -227,6 +223,9 @@ public: // this. virtual uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers); + // Try to expand on the given node index. + virtual uint expand_on_preferred_node(uint node_index); + // Find a contiguous set of empty regions of length num. Returns the start index of // that set, or G1_NO_HRM_INDEX. virtual uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); } diff --git a/src/hotspot/share/gc/g1/heapRegionSet.hpp b/src/hotspot/share/gc/g1/heapRegionSet.hpp index f92fa08c8b..425948687d 100644 --- a/src/hotspot/share/gc/g1/heapRegionSet.hpp +++ b/src/hotspot/share/gc/g1/heapRegionSet.hpp @@ -181,6 +181,10 @@ public: // Removes from head or tail based on the given argument. HeapRegion* remove_region(bool from_head); + HeapRegion* remove_region_with_node_index(bool from_head, + const uint requested_node_index, + uint* region_node_index); + // Merge two ordered lists. The result is also ordered. The order is // determined by hrm_index. void add_ordered(FreeRegionList* from_list); diff --git a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp index 01da9314ad..7f73696ada 100644 --- a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp +++ b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP #define SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP +#include "gc/g1/g1NUMA.hpp" #include "gc/g1/heapRegionSet.hpp" inline void HeapRegionSetBase::add(HeapRegion* hr) { @@ -147,4 +148,65 @@ inline HeapRegion* FreeRegionList::remove_region(bool from_head) { return hr; } +inline HeapRegion* FreeRegionList::remove_region_with_node_index(bool from_head, + const uint requested_node_index, + uint* allocated_node_index) { + assert(UseNUMA, "Invariant"); + + const uint max_search_depth = G1NUMA::numa()->max_search_depth(); + HeapRegion* cur; + + // Find the region to use, searching from _head or _tail as requested. + size_t cur_depth = 0; + if (from_head) { + for (cur = _head; + cur != NULL && cur_depth < max_search_depth; + cur = cur->next(), ++cur_depth) { + if (requested_node_index == cur->node_index()) { + break; + } + } + } else { + for (cur = _tail; + cur != NULL && cur_depth < max_search_depth; + cur = cur->prev(), ++cur_depth) { + if (requested_node_index == cur->node_index()) { + break; + } + } + } + + // Didn't find a region to use. + if (cur == NULL || cur_depth >= max_search_depth) { + return NULL; + } + + // Splice the region out of the list. + HeapRegion* prev = cur->prev(); + HeapRegion* next = cur->next(); + if (prev == NULL) { + _head = next; + } else { + prev->set_next(next); + } + if (next == NULL) { + _tail = prev; + } else { + next->set_prev(prev); + } + cur->set_prev(NULL); + cur->set_next(NULL); + + if (_last == cur) { + _last = NULL; + } + + remove(cur); + if (allocated_node_index != NULL) { + *allocated_node_index = cur->node_index(); + } + + return cur; +} + #endif // SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP diff --git a/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp b/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp index 56261b6984..ce3c802208 100644 --- a/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp +++ b/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -263,7 +263,7 @@ uint HeterogeneousHeapRegionManager::find_empty_in_range_reverse(uint start_idx, return num_regions_found; } -HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type) { +HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type, uint node_index) { // We want to prevent mutators from proceeding when we have borrowed regions from the last collection. This // will force a full collection to remedy the situation. diff --git a/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp b/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp index 634edebfd4..fd4061f6e3 100644 --- a/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp +++ b/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp @@ -119,7 +119,7 @@ public: void prepare_for_full_collection_start(); void prepare_for_full_collection_end(); - virtual HeapRegion* allocate_free_region(HeapRegionType type); + virtual HeapRegion* allocate_free_region(HeapRegionType type, uint node_index); // Return maximum number of regions that heap can expand to. uint max_expandable_length() const; diff --git a/src/hotspot/share/logging/logPrefix.hpp b/src/hotspot/share/logging/logPrefix.hpp index ee76dfdc6d..c3323b5c91 100644 --- a/src/hotspot/share/logging/logPrefix.hpp +++ b/src/hotspot/share/logging/logPrefix.hpp @@ -57,6 +57,7 @@ DEBUG_ONLY(size_t Test_log_prefix_prefixer(char* buf, size_t len);) LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, ihop)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, refine)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap)) \ + LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, numa)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, region)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, freelist)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, humongous)) \ diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp index 2a641badbb..e9ce26fae9 100644 --- a/src/hotspot/share/logging/logTag.hpp +++ b/src/hotspot/share/logging/logTag.hpp @@ -108,6 +108,7 @@ LOG_TAG(nestmates) \ LOG_TAG(nmethod) \ LOG_TAG(normalize) \ + LOG_TAG(numa) \ LOG_TAG(objecttagging) \ LOG_TAG(obsolete) \ LOG_TAG(oldobject) \ diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp index 74846f6976..5d67c9c004 100644 --- a/src/hotspot/share/prims/whitebox.cpp +++ b/src/hotspot/share/prims/whitebox.cpp @@ -619,6 +619,29 @@ WB_ENTRY(jobject, WB_G1AuxiliaryMemoryUsage(JNIEnv* env)) THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1AuxiliaryMemoryUsage: G1 GC is not enabled"); WB_END +WB_ENTRY(jint, WB_G1ActiveMemoryNodeCount(JNIEnv* env, jobject o)) + if (UseG1GC) { + G1NUMA* numa = G1NUMA::numa(); + return (jint)numa->num_active_nodes(); + } + THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1ActiveMemoryNodeCount: G1 GC is not enabled"); +WB_END + +WB_ENTRY(jintArray, WB_G1MemoryNodeIds(JNIEnv* env, jobject o)) + if (UseG1GC) { + G1NUMA* numa = G1NUMA::numa(); + int num_node_ids = (int)numa->num_active_nodes(); + const int* node_ids = numa->node_ids(); + + typeArrayOop result = oopFactory::new_intArray(num_node_ids, CHECK_NULL); + for (int i = 0; i < num_node_ids; i++) { + result->int_at_put(i, (jint)node_ids[i]); + } + return (jintArray) JNIHandles::make_local(env, result); + } + THROW_MSG_NULL(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1MemoryNodeIds: G1 GC is not enabled"); +WB_END + class OldRegionsLivenessClosure: public HeapRegionClosure { private: @@ -2195,6 +2218,8 @@ static JNINativeMethod methods[] = { {CC"g1StartConcMarkCycle", CC"()Z", (void*)&WB_G1StartMarkCycle }, {CC"g1AuxiliaryMemoryUsage", CC"()Ljava/lang/management/MemoryUsage;", (void*)&WB_G1AuxiliaryMemoryUsage }, + {CC"g1ActiveMemoryNodeCount", CC"()I", (void*)&WB_G1ActiveMemoryNodeCount }, + {CC"g1MemoryNodeIds", CC"()[I", (void*)&WB_G1MemoryNodeIds }, {CC"g1GetMixedGCInfo", CC"(I)[J", (void*)&WB_G1GetMixedGCInfo }, #endif // INCLUDE_G1GC #if INCLUDE_G1GC || INCLUDE_PARALLELGC diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index ec886981b9..0946b654a1 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -4219,14 +4219,11 @@ jint Arguments::adjust_after_os() { FLAG_SET_DEFAULT(MinHeapDeltaBytes, 64*M); } } - // UseNUMAInterleaving is set to ON for all collectors and - // platforms when UseNUMA is set to ON. NUMA-aware collectors - // such as the parallel collector for Linux and Solaris will - // interleave old gen and survivor spaces on top of NUMA - // allocation policy for the eden space. - // Non NUMA-aware collectors such as G1 and Serial-GC on - // all platforms and ParallelGC on Windows will interleave all - // of the heap spaces across NUMA nodes. + // UseNUMAInterleaving is set to ON for all collectors and platforms when + // UseNUMA is set to ON. NUMA-aware collectors will interleave old gen and + // survivor spaces on top of NUMA allocation policy for the eden space. + // Non NUMA-aware collectors will interleave all of the heap spaces across + // NUMA nodes. if (FLAG_IS_DEFAULT(UseNUMAInterleaving)) { FLAG_SET_ERGO(UseNUMAInterleaving, true); } diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index d3394d8af3..cb0aa86457 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -374,6 +374,7 @@ class os: AllStatic { static size_t numa_get_leaf_groups(int *ids, size_t size); static bool numa_topology_changed(); static int numa_get_group_id(); + static int numa_get_group_id_for_address(const void* address); // Page manipulation struct page_info { diff --git a/test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java b/test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java new file mode 100644 index 0000000000..102de8c66d --- /dev/null +++ b/test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package gc.g1; + +/** + * @test TestG1NUMATouchRegions + * @summary Ensure the bottom of the given heap regions are properly touched with requested NUMA id. + * @key gc + * @requires vm.gc.G1 + * @requires os.family == "linux" + * @library /test/lib + * @modules java.base/jdk.internal.misc + * java.management + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main/othervm -XX:+UseG1GC -Xbootclasspath/a:. -XX:+UseNUMA -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.TestG1NUMATouchRegions + */ + +import java.util.LinkedList; +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import sun.hotspot.WhiteBox; + +public class TestG1NUMATouchRegions { + enum NUMASupportStatus { + NOT_CHECKED, + SUPPORT, + NOT_SUPPORT + }; + + static int G1HeapRegionSize1MB = 1; + static int G1HeapRegionSize8MB = 8; + + static NUMASupportStatus status = NUMASupportStatus.NOT_CHECKED; + + public static void main(String[] args) throws Exception { + // 1. Page size < G1HeapRegionSize + // Test default page with 1MB heap region size + testMemoryTouch("-XX:-UseLargePages", G1HeapRegionSize1MB); + // 2. Page size > G1HeapRegionSize + // Test large page with 1MB heap region size. + testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize1MB); + // 3. Page size < G1HeapRegionSize + // Test large page with 8MB heap region size. + testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize8MB); + } + + // On Linux, always UseNUMA is enabled if there is multiple active numa nodes. + static NUMASupportStatus checkNUMAIsEnabled(OutputAnalyzer output) { + boolean supportNUMA = Boolean.parseBoolean(output.firstMatch("\\bUseNUMA\\b.*?=.*?([a-z]+)", 1)); + System.out.println("supportNUMA=" + supportNUMA); + return supportNUMA ? NUMASupportStatus.SUPPORT : NUMASupportStatus.NOT_SUPPORT; + } + + static long parseSizeString(String size) { + long multiplier = 1; + + if (size.endsWith("B")) { + multiplier = 1; + } else if (size.endsWith("K")) { + multiplier = 1024; + } else if (size.endsWith("M")) { + multiplier = 1024 * 1024; + } else if (size.endsWith("G")) { + multiplier = 1024 * 1024 * 1024; + } else { + throw new IllegalArgumentException("Expected memory string '" + size + "'to end with either of: B, K, M, G"); + } + + long longSize = Long.parseUnsignedLong(size.substring(0, size.length() - 1)); + + return longSize * multiplier; + } + + static long heapPageSize(OutputAnalyzer output) { + String HeapPageSizePattern = "Heap: .*page_size=([^ ]+)"; + String str = output.firstMatch(HeapPageSizePattern, 1); + + if (str == null) { + output.reportDiagnosticSummary(); + throw new RuntimeException("Match from '" + HeapPageSizePattern + "' got 'null'"); + } + + return parseSizeString(str); + } + + // 1. -UseLargePages: default page, page size < G1HeapRegionSize + // +UseLargePages: large page size <= G1HeapRegionSize + // + // Each 'int' represents a numa id of single HeapRegion (bottom page). + // e.g. 1MB heap region, 2MB page size and 2 NUMA nodes system + // Check the first set(2 regions) + // 0| ...omitted..| 00 + // 1| ...omitted..| 01 + static void checkCase1Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception { + StringBuilder sb = new StringBuilder(); + + // Append index which means heap region index. + sb.append(String.format("%6d", index)); + sb.append("| .* | "); + + // Append page node id. + sb.append(String.format("%02d", memoryNodeIds[index])); + + output.shouldMatch(sb.toString()); + } + + // 3. +UseLargePages: large page size > G1HeapRegionSize + // + // As a OS page is consist of multiple heap regions, log also should be + // printed multiple times for same numa id. + // e.g. 1MB heap region, 2MB page size and 2 NUMA nodes system + // Check the first set(4 regions) + // 0| ...omitted..| 00 + // 1| ...omitted..| 00 + // 2| ...omitted..| 01 + // 3| ...omitted..| 01 + static void checkCase2Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception { + StringBuilder sb = new StringBuilder(); + + // Append page range. + int lines_to_print = (int)(actualPageSize / g1HeapRegionSize); + for (int i = 0; i < lines_to_print; i++) { + // Append index which means heap region index. + sb.append(String.format("%6d", index * lines_to_print + i)); + sb.append("| .* | "); + + // Append page node id. + sb.append(String.format("%02d", memoryNodeIds[index])); + + output.shouldMatch(sb.toString()); + sb.setLength(0); + } + } + + static void checkNUMALog(OutputAnalyzer output, int regionSizeInMB) throws Exception { + WhiteBox wb = WhiteBox.getWhiteBox(); + long g1HeapRegionSize = regionSizeInMB * 1024 * 1024; + long actualPageSize = heapPageSize(output); + long defaultPageSize = (long)wb.getVMPageSize(); + int memoryNodeCount = wb.g1ActiveMemoryNodeCount(); + int[] memoryNodeIds = wb.g1MemoryNodeIds(); + + System.out.println("node count=" + memoryNodeCount + ", actualPageSize=" + actualPageSize); + // Check for the first set of active numa nodes. + for (int index = 0; index < memoryNodeCount; index++) { + if (actualPageSize <= defaultPageSize) { + checkCase1Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds); + } else { + checkCase2Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds); + } + } + } + + static void testMemoryTouch(String largePagesSetting, int regionSizeInMB) throws Exception { + // Skip testing with message. + if (status == NUMASupportStatus.NOT_SUPPORT) { + System.out.println("NUMA is not supported"); + return; + } + + ProcessBuilder pb_enabled = ProcessTools.createJavaProcessBuilder( + "-Xbootclasspath/a:.", + "-Xlog:pagesize,gc+heap+region=trace", + "-XX:+UseG1GC", + "-Xmx128m", + "-Xms128m", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+WhiteBoxAPI", + "-XX:+PrintFlagsFinal", + "-XX:+UseNUMA", + "-XX:+AlwaysPreTouch", + largePagesSetting, + "-XX:G1HeapRegionSize=" + regionSizeInMB + "m", + GCTest.class.getName()); + OutputAnalyzer output = new OutputAnalyzer(pb_enabled.start()); + + // Check NUMA availability. + if (status == NUMASupportStatus.NOT_CHECKED) { + status = checkNUMAIsEnabled(output); + } + + if (status == NUMASupportStatus.SUPPORT) { + checkNUMALog(output, regionSizeInMB); + } else { + // Exit with message for the first test. + System.out.println("NUMA is not supported"); + } + } + + static class GCTest { + public static final int M = 1024*1024; + public static LinkedList garbageList = new LinkedList(); + // A large object referenced by a static. + static int[] filler = new int[10 * M]; + + public static void genGarbage() { + for (int i = 0; i < 32*1024; i++) { + garbageList.add(new int[100]); + } + garbageList.clear(); + } + + public static void main(String[] args) { + + int[] large = new int[M]; + Object ref = large; + + System.out.println("Creating garbage"); + for (int i = 0; i < 100; i++) { + // A large object that will be reclaimed eagerly. + large = new int[6*M]; + genGarbage(); + // Make sure that the compiler cannot completely remove + // the allocation of the large object until here. + System.out.println(large); + } + + // Keep the reference to the first object alive. + System.out.println(ref); + System.out.println("Done"); + } + } +} diff --git a/test/lib/sun/hotspot/WhiteBox.java b/test/lib/sun/hotspot/WhiteBox.java index f2c95ada2c..2cec9ec8c2 100644 --- a/test/lib/sun/hotspot/WhiteBox.java +++ b/test/lib/sun/hotspot/WhiteBox.java @@ -193,6 +193,9 @@ public class WhiteBox { return parseCommandLine0(commandline, delim, args); } + public native int g1ActiveMemoryNodeCount(); + public native int[] g1MemoryNodeIds(); + // Parallel GC public native long psVirtualSpaceAlignment(); public native long psHeapGenerationAlignment();