darlinghq/darling-openjdk
1
0
Fork 0
mirror of https://github.com/darlinghq/darling-openjdk.git synced 2024-11-30 15:50:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

8220310: Implementation: NUMA-Aware Memory Allocation for G1, Mutator (1/3)

Browse Source 
Reviewed-by: kbarrett, sjohanss, tschatzl, pliden
This commit is contained in:
Sangheon Kim 2019-11-13 10:49:12 -08:00
parent b171594072
commit 52116d808c
32 changed files with 990 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/hotspot/os/bsd/os_bsd.cpp
View File

@ -2005,6 +2005,10 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) {
return 0;
}
int os::numa_get_group_id_for_address(const void* address) {
return 0;
}
bool os::get_page_info(char *start, page_info* info) {
return false;
}

17
src/hotspot/os/linux/os_linux.cpp
View File

@ -3007,6 +3007,23 @@ int os::numa_get_group_id() {
return 0;
}
int os::numa_get_group_id_for_address(const void* address) {
#ifndef MPOL_F_NODE
#define MPOL_F_NODE (1<<0) // Return next IL mode instead of node mask
#endif
#ifndef MPOL_F_ADDR
#define MPOL_F_ADDR (1<<1) // Look up VMA using address
#endif
int id = 0;
if (syscall(SYS_get_mempolicy, &id, NULL, 0, const_cast<void*>(address), MPOL_F_NODE | MPOL_F_ADDR) == -1) {
return -1;
}
return id;
}
int os::Linux::get_existing_num_nodes() {
int node;
int highest_node_number = Linux::numa_max_node();

6
src/hotspot/os/solaris/os_solaris.cpp
View File

@ -2072,7 +2072,7 @@ int os::Solaris::commit_memory_impl(char* addr, size_t bytes, bool exec) {
char *res = Solaris::mmap_chunk(addr, size, MAP_PRIVATE|MAP_FIXED, prot);
if (res != NULL) {
if (UseNUMAInterleaving) {
numa_make_global(addr, bytes);
numa_make_global(addr, bytes);
}
return 0;
}
@ -2267,6 +2267,10 @@ int os::numa_get_group_id() {
return ids[os::random() % r];
}
int os::numa_get_group_id_for_address(const void* address) {
return 0;
}
// Request information about the page.
bool os::get_page_info(char *start, page_info* info) {
const uint_t info_types[] = { MEMINFO_VLGRP, MEMINFO_VPAGESIZE };

4
src/hotspot/os/windows/os_windows.cpp
View File

@ -3447,6 +3447,10 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) {
}
}
int os::numa_get_group_id_for_address(const void* address) {
return 0;
}
bool os::get_page_info(char *start, page_info* info) {
return false;
}

10
src/hotspot/share/gc/g1/g1AllocRegion.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -250,17 +250,19 @@ void G1AllocRegion::trace(const char* str, size_t min_word_size, size_t desired_
#endif // PRODUCT
G1AllocRegion::G1AllocRegion(const char* name,
bool bot_updates)
bool bot_updates,
uint node_index)
: _alloc_region(NULL),
_count(0),
_used_bytes_before(0),
_bot_updates(bot_updates),
_name(name)
_name(name),
_node_index(node_index)
{ }
HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size,
bool force) {
return _g1h->new_mutator_alloc_region(word_size, force);
return _g1h->new_mutator_alloc_region(word_size, force, _node_index);
}
void MutatorAllocRegion::retire_region(HeapRegion* alloc_region,

18
src/hotspot/share/gc/g1/g1AllocRegion.hpp
View File

@ -28,6 +28,7 @@
#include "gc/g1/heapRegion.hpp"
#include "gc/g1/g1EvacStats.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "gc/g1/g1NUMA.hpp"
class G1CollectedHeap;
@ -38,7 +39,7 @@ class G1CollectedHeap;
// and a lock will need to be taken when the active region needs to be
// replaced.
class G1AllocRegion {
class G1AllocRegion : public CHeapObj<mtGC> {
private:
// The active allocating region we are currently allocating out
@ -91,6 +92,9 @@ private:
HeapWord* new_alloc_region_and_allocate(size_t word_size, bool force);
protected:
// The memory node index this allocation region belongs to.
uint _node_index;
// Reset the alloc region to point a the dummy region.
void reset_alloc_region();
@ -131,7 +135,7 @@ protected:
virtual void retire_region(HeapRegion* alloc_region,
size_t allocated_bytes) = 0;
G1AllocRegion(const char* name, bool bot_updates);
G1AllocRegion(const char* name, bool bot_updates, uint node_index);
public:
static void setup(G1CollectedHeap* g1h, HeapRegion* dummy_region);
@ -220,8 +224,8 @@ protected:
virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes);
virtual size_t retire(bool fill_up);
public:
MutatorAllocRegion()
: G1AllocRegion("Mutator Alloc Region", false /* bot_updates */),
MutatorAllocRegion(uint node_index)
: G1AllocRegion("Mutator Alloc Region", false /* bot_updates */, node_index),
_wasted_bytes(0),
_retained_alloc_region(NULL) { }
@ -245,6 +249,7 @@ public:
virtual void init();
};
// Common base class for allocation regions used during GC.
class G1GCAllocRegion : public G1AllocRegion {
protected:
@ -256,8 +261,9 @@ protected:
virtual size_t retire(bool fill_up);
G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats, G1HeapRegionAttr::region_type_t purpose)
: G1AllocRegion(name, bot_updates), _stats(stats), _purpose(purpose) {
G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats,
G1HeapRegionAttr::region_type_t purpose, uint node_index = G1NUMA::AnyNodeIndex)
: G1AllocRegion(name, bot_updates, node_index), _stats(stats), _purpose(purpose) {
assert(stats != NULL, "Must pass non-NULL PLAB statistics");
}
};

49
src/hotspot/share/gc/g1/g1Allocator.cpp
View File

@ -28,6 +28,7 @@
#include "gc/g1/g1EvacStats.inline.hpp"
#include "gc/g1/g1EvacuationInfo.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionSet.inline.hpp"
@ -36,22 +37,47 @@
G1Allocator::G1Allocator(G1CollectedHeap* heap) :
_g1h(heap),
_numa(heap->numa()),
_survivor_is_full(false),
_old_is_full(false),
_mutator_alloc_region(),
_num_alloc_regions(_numa->num_active_nodes()),
_mutator_alloc_regions(NULL),
_survivor_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Young)),
_old_gc_alloc_region(heap->alloc_buffer_stats(G1HeapRegionAttr::Old)),
_retained_old_gc_alloc_region(NULL) {
_mutator_alloc_regions = NEW_C_HEAP_ARRAY(MutatorAllocRegion, _num_alloc_regions, mtGC);
for (uint i = 0; i < _num_alloc_regions; i++) {
::new(_mutator_alloc_regions + i) MutatorAllocRegion(i);
}
}
void G1Allocator::init_mutator_alloc_region() {
assert(_mutator_alloc_region.get() == NULL, "pre-condition");
_mutator_alloc_region.init();
G1Allocator::~G1Allocator() {
for (uint i = 0; i < _num_alloc_regions; i++) {
_mutator_alloc_regions[i].~MutatorAllocRegion();
}
FREE_C_HEAP_ARRAY(MutatorAllocRegion, _mutator_alloc_regions);
}
void G1Allocator::release_mutator_alloc_region() {
_mutator_alloc_region.release();
assert(_mutator_alloc_region.get() == NULL, "post-condition");
#ifdef ASSERT
bool G1Allocator::has_mutator_alloc_region() {
uint node_index = current_node_index();
return mutator_alloc_region(node_index)->get() != NULL;
}
#endif
void G1Allocator::init_mutator_alloc_regions() {
for (uint i = 0; i < _num_alloc_regions; i++) {
assert(mutator_alloc_region(i)->get() == NULL, "pre-condition");
mutator_alloc_region(i)->init();
}
}
void G1Allocator::release_mutator_alloc_regions() {
for (uint i = 0; i < _num_alloc_regions; i++) {
mutator_alloc_region(i)->release();
assert(mutator_alloc_region(i)->get() == NULL, "post-condition");
}
}
bool G1Allocator::is_retained_old_region(HeapRegion* hr) {
@ -146,7 +172,8 @@ size_t G1Allocator::unsafe_max_tlab_alloc() {
// since we can't allow tlabs to grow big enough to accommodate
// humongous objects.
HeapRegion* hr = mutator_alloc_region()->get();
uint node_index = current_node_index();
HeapRegion* hr = mutator_alloc_region(node_index)->get();
size_t max_tlab = _g1h->max_tlab_size() * wordSize;
if (hr == NULL) {
return max_tlab;
@ -157,7 +184,11 @@ size_t G1Allocator::unsafe_max_tlab_alloc() {
size_t G1Allocator::used_in_alloc_regions() {
assert(Heap_lock->owner() != NULL, "Should be owned on this thread's behalf.");
return mutator_alloc_region()->used_in_alloc_regions();
size_t used = 0;
for (uint i = 0; i < _num_alloc_regions; i++) {
used += mutator_alloc_region(i)->used_in_alloc_regions();
}
return used;
}

28
src/hotspot/share/gc/g1/g1Allocator.hpp
View File

@ -31,6 +31,7 @@
#include "gc/shared/plab.hpp"
class G1EvacuationInfo;
class G1NUMA;
// Interface to keep track of which regions G1 is currently allocating into. Provides
// some accessors (e.g. allocating into them, or getting their occupancy).
@ -40,12 +41,16 @@ class G1Allocator : public CHeapObj<mtGC> {
private:
G1CollectedHeap* _g1h;
G1NUMA* _numa;
bool _survivor_is_full;
bool _old_is_full;
// The number of MutatorAllocRegions used, one per memory node.
size_t _num_alloc_regions;
// Alloc region used to satisfy mutator allocation requests.
MutatorAllocRegion _mutator_alloc_region;
MutatorAllocRegion* _mutator_alloc_regions;
// Alloc region used to satisfy allocation requests by the GC for
// survivor objects.
@ -68,29 +73,34 @@ private:
HeapRegion** retained);
// Accessors to the allocation regions.
inline MutatorAllocRegion* mutator_alloc_region();
inline MutatorAllocRegion* mutator_alloc_region(uint node_index);
inline SurvivorGCAllocRegion* survivor_gc_alloc_region();
inline OldGCAllocRegion* old_gc_alloc_region();
// Allocation attempt during GC for a survivor object / PLAB.
HeapWord* survivor_attempt_allocation(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size);
size_t desired_word_size,
size_t* actual_word_size);
// Allocation attempt during GC for an old object / PLAB.
HeapWord* old_attempt_allocation(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size);
size_t desired_word_size,
size_t* actual_word_size);
// Node index of current thread.
inline uint current_node_index() const;
public:
G1Allocator(G1CollectedHeap* heap);
~G1Allocator();
#ifdef ASSERT
// Do we currently have an active mutator region to allocate into?
bool has_mutator_alloc_region() { return mutator_alloc_region()->get() != NULL; }
bool has_mutator_alloc_region();
#endif
void init_mutator_alloc_region();
void release_mutator_alloc_region();
void init_mutator_alloc_regions();
void release_mutator_alloc_regions();
void init_gc_alloc_regions(G1EvacuationInfo& evacuation_info);
void release_gc_alloc_regions(G1EvacuationInfo& evacuation_info);

24
src/hotspot/share/gc/g1/g1Allocator.inline.hpp
View File

@ -30,8 +30,13 @@
#include "gc/shared/plab.inline.hpp"
#include "memory/universe.hpp"
inline MutatorAllocRegion* G1Allocator::mutator_alloc_region() {
return &_mutator_alloc_region;
inline uint G1Allocator::current_node_index() const {
return _numa->index_of_current_thread();
}
inline MutatorAllocRegion* G1Allocator::mutator_alloc_region(uint node_index) {
assert(node_index < _num_alloc_regions, "Invalid index: %u", node_index);
return &_mutator_alloc_regions[node_index];
}
inline SurvivorGCAllocRegion* G1Allocator::survivor_gc_alloc_region() {
@ -45,22 +50,25 @@ inline OldGCAllocRegion* G1Allocator::old_gc_alloc_region() {
inline HeapWord* G1Allocator::attempt_allocation(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
HeapWord* result = mutator_alloc_region()->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
uint node_index = current_node_index();
HeapWord* result = mutator_alloc_region(node_index)->attempt_retained_allocation(min_word_size, desired_word_size, actual_word_size);
if (result != NULL) {
return result;
}
return mutator_alloc_region()->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
return mutator_alloc_region(node_index)->attempt_allocation(min_word_size, desired_word_size, actual_word_size);
}
inline HeapWord* G1Allocator::attempt_allocation_locked(size_t word_size) {
HeapWord* result = mutator_alloc_region()->attempt_allocation_locked(word_size);
assert(result != NULL || mutator_alloc_region()->get() == NULL,
"Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region()->get()));
uint node_index = current_node_index();
HeapWord* result = mutator_alloc_region(node_index)->attempt_allocation_locked(word_size);
assert(result != NULL || mutator_alloc_region(node_index)->get() == NULL,
"Must not have a mutator alloc region if there is no memory, but is " PTR_FORMAT, p2i(mutator_alloc_region(node_index)->get()));
return result;
}
inline HeapWord* G1Allocator::attempt_allocation_force(size_t word_size) {
return mutator_alloc_region()->attempt_allocation_force(word_size);
uint node_index = current_node_index();
return mutator_alloc_region(node_index)->attempt_allocation_force(word_size);
}
inline PLAB* G1PLABAllocator::alloc_buffer(G1HeapRegionAttr dest) {

49
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -169,12 +169,15 @@ HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index,
// Private methods.
HeapRegion* G1CollectedHeap::new_region(size_t word_size, HeapRegionType type, bool do_expand) {
HeapRegion* G1CollectedHeap::new_region(size_t word_size,
HeapRegionType type,
bool do_expand,
uint node_index) {
assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords,
"the only time we use this to allocate a humongous region is "
"when we are allocating a single humongous region");
HeapRegion* res = _hrm->allocate_free_region(type);
HeapRegion* res = _hrm->allocate_free_region(type, node_index);
if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {
// Currently, only attempts to allocate GC alloc regions set
@ -186,12 +189,15 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size, HeapRegionType type, b
log_debug(gc, ergo, heap)("Attempt heap expansion (region allocation request failed). Allocation request: " SIZE_FORMAT "B",
word_size * HeapWordSize);
if (expand(word_size * HeapWordSize)) {
// Given that expand() succeeded in expanding the heap, and we
assert(word_size * HeapWordSize < HeapRegion::GrainBytes,
"This kind of expansion should never be more than one region. Size: " SIZE_FORMAT,
word_size * HeapWordSize);
if (expand_single_region(node_index)) {
// Given that expand_single_region() succeeded in expanding the heap, and we
// always expand the heap by an amount aligned to the heap
// region size, the free list should in theory not be empty.
// In either case allocate_free_region() will check for NULL.
res = _hrm->allocate_free_region(type);
res = _hrm->allocate_free_region(type, node_index);
} else {
_expand_heap_after_alloc_failure = false;
}
@ -1020,7 +1026,7 @@ void G1CollectedHeap::abort_concurrent_cycle() {
void G1CollectedHeap::prepare_heap_for_full_collection() {
// Make sure we'll choose a new allocation region afterwards.
_allocator->release_mutator_alloc_region();
_allocator->release_mutator_alloc_regions();
_allocator->abandon_gc_alloc_regions();
// We may have added regions to the current incremental collection
@ -1064,7 +1070,7 @@ void G1CollectedHeap::prepare_heap_for_mutators() {
// Start a new incremental collection set for the next pause
start_new_collection_set();
_allocator->init_mutator_alloc_region();
_allocator->init_mutator_alloc_regions();
// Post collection state updates.
MetaspaceGC::compute_new_size();
@ -1381,6 +1387,19 @@ bool G1CollectedHeap::expand(size_t expand_bytes, WorkGang* pretouch_workers, do
return regions_to_expand > 0;
}
bool G1CollectedHeap::expand_single_region(uint node_index) {
uint expanded_by = _hrm->expand_on_preferred_node(node_index);
if (expanded_by == 0) {
assert(is_maximal_no_gc(), "Should be no regions left, available: %u", _hrm->available());
log_debug(gc, ergo, heap)("Did not expand the heap (heap already fully expanded)");
return false;
}
policy()->record_new_heap_size(num_regions());
return true;
}
void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
size_t aligned_shrink_bytes =
ReservedSpace::page_align_size_down(shrink_bytes);
@ -1391,7 +1410,6 @@ void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
uint num_regions_removed = _hrm->shrink_by(num_regions_to_remove);
size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes;
log_debug(gc, ergo, heap)("Shrink the heap. requested shrinking amount: " SIZE_FORMAT "B aligned shrinking amount: " SIZE_FORMAT "B attempted shrinking amount: " SIZE_FORMAT "B",
shrink_bytes, aligned_shrink_bytes, shrunk_bytes);
if (num_regions_removed > 0) {
@ -1493,6 +1511,7 @@ G1CollectedHeap::G1CollectedHeap() :
_humongous_set("Humongous Region Set", new HumongousRegionSetChecker()),
_bot(NULL),
_listener(),
_numa(G1NUMA::create()),
_hrm(NULL),
_allocator(NULL),
_verifier(NULL),
@ -1775,6 +1794,8 @@ jint G1CollectedHeap::initialize() {
}
_workers->initialize_workers();
_numa->set_region_info(HeapRegion::GrainBytes, page_size);
// Create the G1ConcurrentMark data structure and thread.
// (Must do this late, so that "max_regions" is defined.)
_cm = new G1ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage);
@ -1822,7 +1843,7 @@ jint G1CollectedHeap::initialize() {
dummy_region->set_top(dummy_region->end());
G1AllocRegion::setup(this, dummy_region);
_allocator->init_mutator_alloc_region();
_allocator->init_mutator_alloc_regions();
// Do create of the monitoring and management support so that
// values in the heap have been properly initialized.
@ -3005,7 +3026,7 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_
// Forget the current allocation region (we might even choose it to be part
// of the collection set!).
_allocator->release_mutator_alloc_region();
_allocator->release_mutator_alloc_regions();
calculate_collection_set(evacuation_info, target_pause_time_ms);
@ -3042,7 +3063,7 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_
allocate_dummy_regions();
_allocator->init_mutator_alloc_region();
_allocator->init_mutator_alloc_regions();
expand_heap_after_young_collection();
@ -4538,13 +4559,15 @@ void G1CollectedHeap::rebuild_region_sets(bool free_list_only) {
// Methods for the mutator alloc region
HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size,
bool force) {
bool force,
uint node_index) {
assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
bool should_allocate = policy()->should_allocate_mutator_region();
if (force || should_allocate) {
HeapRegion* new_alloc_region = new_region(word_size,
HeapRegionType::Eden,
false /* do_expand */);
false /* do_expand */,
node_index);
if (new_alloc_region != NULL) {
set_region_short_lived_locked(new_alloc_region);
_hr_printer.alloc(new_alloc_region, !should_allocate);

14
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -41,6 +41,7 @@
#include "gc/g1/g1HRPrinter.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "gc/g1/g1MonitoringSupport.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
#include "gc/g1/g1YCTypes.hpp"
@ -191,6 +192,9 @@ private:
// Callback for region mapping changed events.
G1RegionMappingChangedListener _listener;
// Handle G1 NUMA support.
G1NUMA* _numa;
// The sequence of all heap regions in the heap.
HeapRegionManager* _hrm;
@ -387,7 +391,10 @@ private:
// attempt to expand the heap if necessary to satisfy the allocation
// request. 'type' takes the type of region to be allocated. (Use constants
// Old, Eden, Humongous, Survivor defined in HeapRegionType.)
HeapRegion* new_region(size_t word_size, HeapRegionType type, bool do_expand);
HeapRegion* new_region(size_t word_size,
HeapRegionType type,
bool do_expand,
uint node_index = G1NUMA::AnyNodeIndex);
// Initialize a contiguous set of free regions of length num_regions
// and starting at index first so that they appear as a single
@ -462,7 +469,7 @@ private:
// These methods are the "callbacks" from the G1AllocRegion class.
// For mutator alloc regions.
HeapRegion* new_mutator_alloc_region(size_t word_size, bool force);
HeapRegion* new_mutator_alloc_region(size_t word_size, bool force, uint node_index);
void retire_mutator_alloc_region(HeapRegion* alloc_region,
size_t allocated_bytes);
@ -547,11 +554,14 @@ public:
void resize_heap_if_necessary();
G1NUMA* numa() const { return _numa; }
// Expand the garbage-first heap by at least the given size (in bytes!).
// Returns true if the heap was expanded by the requested amount;
// false otherwise.
// (Rounds up to a HeapRegion boundary.)
bool expand(size_t expand_bytes, WorkGang* pretouch_workers = NULL, double* expand_time_ms = NULL);
bool expand_single_region(uint node_index);
// Returns the PLAB statistics for a given destination.
inline G1EvacStats* alloc_buffer_stats(G1HeapRegionAttr dest);

227
src/hotspot/share/gc/g1/g1NUMA.cpp Normal file
View File

@ -0,0 +1,227 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/heapRegion.hpp"
#include "logging/log.hpp"
#include "runtime/globals.hpp"
#include "runtime/os.hpp"
G1NUMA* G1NUMA::_inst = NULL;
size_t G1NUMA::region_size() const {
assert(_region_size > 0, "Heap region size is not yet set");
return _region_size;
}
size_t G1NUMA::page_size() const {
assert(_page_size > 0, "Page size not is yet set");
return _page_size;
}
bool G1NUMA::is_enabled() const { return num_active_nodes() > 1; }
G1NUMA* G1NUMA::create() {
guarantee(_inst == NULL, "Should be called once.");
_inst = new G1NUMA();
// NUMA only supported on Linux.
#ifdef LINUX
_inst->initialize(UseNUMA);
#else
_inst->initialize(false);
#endif /* LINUX */
return _inst;
}
// Returns memory node ids
const int* G1NUMA::node_ids() const {
return _node_ids;
}
uint G1NUMA::index_of_node_id(int node_id) const {
assert(node_id >= 0, "invalid node id %d", node_id);
assert(node_id < _len_node_id_to_index_map, "invalid node id %d", node_id);
uint node_index = _node_id_to_index_map[node_id];
assert(node_index != G1NUMA::UnknownNodeIndex,
"invalid node id %d", node_id);
return node_index;
}
G1NUMA::G1NUMA() :
_node_id_to_index_map(NULL), _len_node_id_to_index_map(0),
_node_ids(NULL), _num_active_node_ids(0),
_region_size(0), _page_size(0) {
}
void G1NUMA::initialize_without_numa() {
// If NUMA is not enabled or supported, initialize as having a singel node.
_num_active_node_ids = 1;
_node_ids = NEW_C_HEAP_ARRAY(int, _num_active_node_ids, mtGC);
_node_ids[0] = 0;
// Map index 0 to node 0
_len_node_id_to_index_map = 1;
_node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC);
_node_id_to_index_map[0] = 0;
}
void G1NUMA::initialize(bool use_numa) {
if (!use_numa) {
initialize_without_numa();
return;
}
assert(UseNUMA, "Invariant");
size_t num_node_ids = os::numa_get_groups_num();
// Create an array of active node ids.
_node_ids = NEW_C_HEAP_ARRAY(int, num_node_ids, mtGC);
_num_active_node_ids = (uint)os::numa_get_leaf_groups(_node_ids, num_node_ids);
int max_node_id = 0;
for (uint i = 0; i < _num_active_node_ids; i++) {
max_node_id = MAX2(max_node_id, _node_ids[i]);
}
// Create a mapping between node_id and index.
_len_node_id_to_index_map = max_node_id + 1;
_node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC);
// Set all indices with unknown node id.
for (int i = 0; i < _len_node_id_to_index_map; i++) {
_node_id_to_index_map[i] = G1NUMA::UnknownNodeIndex;
}
// Set the indices for the actually retrieved node ids.
for (uint i = 0; i < _num_active_node_ids; i++) {
_node_id_to_index_map[_node_ids[i]] = i;
}
}
G1NUMA::~G1NUMA() {
FREE_C_HEAP_ARRAY(int, _node_id_to_index_map);
FREE_C_HEAP_ARRAY(int, _node_ids);
}
void G1NUMA::set_region_info(size_t region_size, size_t page_size) {
_region_size = region_size;
_page_size = page_size;
}
uint G1NUMA::num_active_nodes() const {
assert(_num_active_node_ids > 0, "just checking");
return _num_active_node_ids;
}
uint G1NUMA::index_of_current_thread() const {
if (!is_enabled()) {
return 0;
}
return index_of_node_id(os::numa_get_group_id());
}
uint G1NUMA::preferred_node_index_for_index(uint region_index) const {
if (region_size() >= page_size()) {
// Simple case, pages are smaller than the region so we
// can just alternate over the nodes.
return region_index % _num_active_node_ids;
} else {
// Multiple regions in one page, so we need to make sure the
// regions within a page is preferred on the same node.
size_t regions_per_page = page_size() / region_size();
return (region_index / regions_per_page) % _num_active_node_ids;
}
}
int G1NUMA::numa_id(int index) const {
assert(index < _len_node_id_to_index_map, "Index %d out of range: [0,%d)",
index, _len_node_id_to_index_map);
return _node_ids[index];
}
uint G1NUMA::index_of_address(HeapWord *address) const {
int numa_id = os::numa_get_group_id_for_address((const void*)address);
if (numa_id == -1) {
return UnknownNodeIndex;
} else {
return index_of_node_id(numa_id);
}
}
uint G1NUMA::index_for_region(HeapRegion* hr) const {
if (!is_enabled()) {
return 0;
}
if (AlwaysPreTouch) {
// If we already pretouched, we can check actual node index here.
// However, if node index is still unknown, use preferred node index.
uint node_index = index_of_address(hr->bottom());
if (node_index != UnknownNodeIndex) {
return node_index;
}
}
return preferred_node_index_for_index(hr->hrm_index());
}
// Request to spread the given memory evenly across the available NUMA
// nodes. Which node to request for a given address is given by the
// region size and the page size. Below are two examples on 4 NUMA nodes system:
// 1. G1HeapRegionSize(_region_size) is larger than or equal to page size.
// * Page #: |-0--||-1--||-2--||-3--||-4--||-5--||-6--||-7--||-8--||-9--||-10-||-11-||-12-||-13-||-14-||-15-|
// * HeapRegion #: |----#0----||----#1----||----#2----||----#3----||----#4----||----#5----||----#6----||----#7----|
// * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----|
// 2. G1HeapRegionSize(_region_size) is smaller than page size.
// Memory will be touched one page at a time because G1RegionToSpaceMapper commits
// pages one by one.
// * Page #: |-----0----||-----1----||-----2----||-----3----||-----4----||-----5----||-----6----||-----7----|
// * HeapRegion #: |-#0-||-#1-||-#2-||-#3-||-#4-||-#5-||-#6-||-#7-||-#8-||-#9-||#10-||#11-||#12-||#13-||#14-||#15-|
// * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----|
void G1NUMA::request_memory_on_node(void* aligned_address, size_t size_in_bytes, uint region_index) {
if (!is_enabled()) {
return;
}
if (size_in_bytes == 0) {
return;
}
uint node_index = preferred_node_index_for_index(region_index);
assert(is_aligned(aligned_address, page_size()), "Given address (" PTR_FORMAT ") should be aligned.", p2i(aligned_address));
assert(is_aligned(size_in_bytes, page_size()), "Given size (" SIZE_FORMAT ") should be aligned.", size_in_bytes);
log_debug(gc, heap, numa)("Request memory [" PTR_FORMAT ", " PTR_FORMAT ") to be numa id (%d).",
p2i(aligned_address), p2i((char*)aligned_address + size_in_bytes), _node_ids[node_index]);
os::numa_make_local((char*)aligned_address, size_in_bytes, _node_ids[node_index]);
}
uint G1NUMA::max_search_depth() const {
// Multiple of 3 is just random number to limit iterations.
// There would be some cases that 1 page may be consisted of multiple HeapRegions.
return 3 * MAX2((uint)(page_size() / region_size()), (uint)1) * num_active_nodes();
}

118
src/hotspot/share/gc/g1/g1NUMA.hpp Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_GC_G1_NUMA_HPP
#define SHARE_VM_GC_G1_NUMA_HPP
#include "memory/allocation.hpp"
#include "runtime/os.hpp"
class HeapRegion;
class G1NUMA: public CHeapObj<mtGC> {
// Mapping of available node ids to 0-based index which can be used for
// fast resource management. I.e. for every node id provides a unique value in
// the range from [0, {# of nodes-1}].
// For invalid node id, return UnknownNodeIndex.
uint* _node_id_to_index_map;
// Length of _num_active_node_ids_id to index map.
int _len_node_id_to_index_map;
// Current active node ids.
int* _node_ids;
// Total number of node ids.
uint _num_active_node_ids;
// HeapRegion size
size_t _region_size;
// Necessary when touching memory.
size_t _page_size;
size_t region_size() const;
size_t page_size() const;
// Returns node index of the given node id.
// Precondition: node_id is an active node id.
inline uint index_of_node_id(int node_id) const;
// Creates node id and node index mapping table of _node_id_to_index_map.
void init_node_id_to_index_map(const int* node_ids, uint num_node_ids);
static G1NUMA* _inst;
G1NUMA();
void initialize(bool use_numa);
void initialize_without_numa();
public:
static const uint UnknownNodeIndex = UINT_MAX;
static const uint AnyNodeIndex = UnknownNodeIndex - 1;
static G1NUMA* numa() { return _inst; }
static G1NUMA* create();
~G1NUMA();
// Sets heap region size and page size after those values
// are determined at G1CollectedHeap::initialize().
void set_region_info(size_t region_size, size_t page_size);
// Returns active memory node count.
uint num_active_nodes() const;
bool is_enabled() const;
int numa_id(int index) const;
// Returns memory node ids
const int* node_ids() const;
// Returns node index of current calling thread.
uint index_of_current_thread() const;
// Returns the preferred index for the given HeapRegion index.
// This assumes that HeapRegions are evenly spit, so we can decide preferred index
// with the given HeapRegion index.
// Result is less than num_active_nodes().
uint preferred_node_index_for_index(uint region_index) const;
// Retrieves node index of the given address.
// Result is less than num_active_nodes() or is UnknownNodeIndex.
// Precondition: address is in reserved range for heap.
uint index_of_address(HeapWord* address) const;
// If AlwaysPreTouch is enabled, return actual node index via system call.
// If disabled, return preferred node index of the given heap region.
uint index_for_region(HeapRegion* hr) const;
// Requests the given memory area to be located at the given node index.
void request_memory_on_node(void* aligned_address, size_t size_in_bytes, uint region_index);
// Returns maximum search depth which is used to limit heap region search iterations.
// The number of active nodes, page size and heap region size are considered.
uint max_search_depth() const;
};
#endif // SHARE_VM_GC_G1_NUMA_HPP

5
src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp
View File

@ -124,6 +124,11 @@ char* G1PageBasedVirtualSpace::page_start(size_t index) const {
return _low_boundary + index * _page_size;
}
size_t G1PageBasedVirtualSpace::page_size() const {
assert(_page_size > 0, "Page size is not yet initialized.");
return _page_size;
}
bool G1PageBasedVirtualSpace::is_after_last_page(size_t index) const {
guarantee(index <= _committed.size(),
"Given boundary page " SIZE_FORMAT " is beyond managed page count " SIZE_FORMAT, index, _committed.size());

6
src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.hpp
View File

@ -92,8 +92,6 @@ class G1PageBasedVirtualSpace {
// Returns the index of the page which contains the given address.
size_t addr_to_page_index(char* addr) const;
// Returns the address of the given page index.
char* page_start(size_t index) const;
// Is the given page index the last page?
bool is_last_page(size_t index) const { return index == (_committed.size() - 1); }
@ -147,6 +145,10 @@ class G1PageBasedVirtualSpace {
void check_for_contiguity() PRODUCT_RETURN;
// Returns the address of the given page index.
char* page_start(size_t index) const;
size_t page_size() const;
// Debugging
void print_on(outputStream* out) PRODUCT_RETURN;
void print();

40
src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp
View File

@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1BiasedArray.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
#include "logging/log.hpp"
#include "memory/allocation.inline.hpp"
@ -44,7 +45,8 @@ G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs,
_listener(NULL),
_storage(rs, used_size, page_size),
_region_granularity(region_granularity),
_commit_map(rs.size() * commit_factor / region_granularity, mtGC) {
_commit_map(rs.size() * commit_factor / region_granularity, mtGC),
_memory_type(type) {
guarantee(is_power_of_2(page_size), "must be");
guarantee(is_power_of_2(region_granularity), "must be");
@ -72,10 +74,18 @@ class G1RegionsLargerThanCommitSizeMapper : public G1RegionToSpaceMapper {
}
virtual void commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) {
size_t const start_page = (size_t)start_idx * _pages_per_region;
bool zero_filled = _storage.commit(start_page, num_regions * _pages_per_region);
const size_t start_page = (size_t)start_idx * _pages_per_region;
const size_t size_in_pages = num_regions * _pages_per_region;
bool zero_filled = _storage.commit(start_page, size_in_pages);
if (_memory_type == mtJavaHeap) {
for (uint region_index = start_idx; region_index < start_idx + num_regions; region_index++ ) {
void* address = _storage.page_start(region_index * _pages_per_region);
size_t size_in_bytes = _storage.page_size() * _pages_per_region;
G1NUMA::numa()->request_memory_on_node(address, size_in_bytes, region_index);
}
}
if (AlwaysPreTouch) {
_storage.pretouch(start_page, num_regions * _pages_per_region, pretouch_gang);
_storage.pretouch(start_page, size_in_pages, pretouch_gang);
}
_commit_map.set_range(start_idx, start_idx + num_regions);
fire_on_commit(start_idx, num_regions, zero_filled);
@ -126,26 +136,32 @@ class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper {
size_t num_committed = 0;
bool all_zero_filled = true;
G1NUMA* numa = G1NUMA::numa();
for (uint i = start_idx; i < start_idx + num_regions; i++) {
assert(!_commit_map.at(i), "Trying to commit storage at region %u that is already committed", i);
size_t idx = region_idx_to_page_idx(i);
uint old_refcount = _refcounts.get_by_index(idx);
for (uint region_idx = start_idx; region_idx < start_idx + num_regions; region_idx++) {
assert(!_commit_map.at(region_idx), "Trying to commit storage at region %u that is already committed", region_idx);
size_t page_idx = region_idx_to_page_idx(region_idx);
uint old_refcount = _refcounts.get_by_index(page_idx);
bool zero_filled = false;
if (old_refcount == 0) {
if (first_committed == NoPage) {
first_committed = idx;
first_committed = page_idx;
num_committed = 1;
} else {
num_committed++;
}
zero_filled = _storage.commit(idx, 1);
zero_filled = _storage.commit(page_idx, 1);
if (_memory_type == mtJavaHeap) {
void* address = _storage.page_start(page_idx);
size_t size_in_bytes = _storage.page_size();
numa->request_memory_on_node(address, size_in_bytes, region_idx);
}
}
all_zero_filled &= zero_filled;
_refcounts.set_by_index(idx, old_refcount + 1);
_commit_map.set_bit(i);
_refcounts.set_by_index(page_idx, old_refcount + 1);
_commit_map.set_bit(region_idx);
}
if (AlwaysPreTouch && num_committed > 0) {
_storage.pretouch(first_committed, num_committed, pretouch_gang);

2
src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp
View File

@ -53,6 +53,8 @@ class G1RegionToSpaceMapper : public CHeapObj<mtGC> {
// Mapping management
CHeapBitMap _commit_map;
MemoryType _memory_type;
G1RegionToSpaceMapper(ReservedSpace rs, size_t used_size, size_t page_size, size_t region_granularity, size_t commit_factor, MemoryType type);
void fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled);

15
src/hotspot/share/gc/g1/heapRegion.cpp
View File

@ -28,6 +28,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionBounds.inline.hpp"
@ -252,7 +253,8 @@ HeapRegion::HeapRegion(uint hrm_index,
_index_in_opt_cset(InvalidCSetIndex), _young_index_in_cset(-1),
_surv_rate_group(NULL), _age_index(-1),
_prev_top_at_mark_start(NULL), _next_top_at_mark_start(NULL),
_recorded_rs_length(0), _predicted_elapsed_time_ms(0)
_recorded_rs_length(0), _predicted_elapsed_time_ms(0),
_node_index(G1NUMA::UnknownNodeIndex)
{
_rem_set = new HeapRegionRemSet(bot, this);
@ -470,8 +472,17 @@ void HeapRegion::print_on(outputStream* st) const {
} else {
st->print("| ");
}
st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ",
st->print("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ",
p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()), rem_set()->get_state_str());
if (UseNUMA) {
G1NUMA* numa = G1NUMA::numa();
if (node_index() < numa->num_active_nodes()) {
st->print("|%02d", numa->numa_id(node_index()));
} else {
st->print("|--");
}
}
st->print_cr("");
}
class G1VerificationClosure : public BasicOopIterateClosure {

5
src/hotspot/share/gc/g1/heapRegion.hpp
View File

@ -253,6 +253,8 @@ private:
// for the collection set.
double _predicted_elapsed_time_ms;
uint _node_index;
// Iterate over the references covered by the given MemRegion in a humongous
// object and apply the given closure to them.
// Humongous objects are allocated directly in the old-gen. So we need special
@ -643,6 +645,9 @@ public:
// the strong code roots list for this region
void strong_code_roots_do(CodeBlobClosure* blk) const;
uint node_index() const { return _node_index; }
void set_node_index(uint node_index) { _node_index = node_index; }
// Verify that the entries on the strong code root list for this
// region are live and include at least one pointer into this region.
void verify_strong_code_roots(VerifyOption vo, bool* failures) const;

59
src/hotspot/share/gc/g1/heapRegionManager.cpp
View File

@ -30,6 +30,7 @@
#include "gc/g1/heapRegionManager.inline.hpp"
#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/g1/heterogeneousHeapRegionManager.hpp"
#include "logging/logStream.hpp"
#include "memory/allocation.hpp"
#include "utilities/bitMap.inline.hpp"
@ -103,6 +104,29 @@ bool HeapRegionManager::is_available(uint region) const {
return _available_map.at(region);
}
HeapRegion* HeapRegionManager::allocate_free_region(HeapRegionType type, uint requested_node_index) {
HeapRegion* hr = NULL;
bool from_head = !type.is_young();
if (requested_node_index != G1NUMA::AnyNodeIndex && G1NUMA::numa()->is_enabled()) {
// Try to allocate with requested node index.
hr = _free_list.remove_region_with_node_index(from_head, requested_node_index, NULL);
}
if (hr == NULL) {
// If there's a single active node or we did not get a region from our requested node,
// try without requested node index.
hr = _free_list.remove_region(from_head);
}
if (hr != NULL) {
assert(hr->next() == NULL, "Single region should not have next");
assert(is_available(hr->hrm_index()), "Must be committed");
}
return hr;
}
#ifdef ASSERT
bool HeapRegionManager::is_free(HeapRegion* hr) const {
return _free_list.contains(hr);
@ -139,6 +163,11 @@ void HeapRegionManager::uncommit_regions(uint start, size_t num_regions) {
guarantee(num_regions >= 1, "Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start);
guarantee(_num_committed >= num_regions, "pre-condition");
// Reset node index to distinguish with committed regions.
for (uint i = start; i < start + num_regions; i++) {
at(i)->set_node_index(G1NUMA::UnknownNodeIndex);
}
// Print before uncommitting.
if (G1CollectedHeap::heap()->hr_printer()->is_active()) {
for (uint i = start; i < start + num_regions; i++) {
@ -186,6 +215,7 @@ void HeapRegionManager::make_regions_available(uint start, uint num_regions, Wor
MemRegion mr(bottom, bottom + HeapRegion::GrainWords);
hr->initialize(mr);
hr->set_node_index(G1NUMA::numa()->index_for_region(hr));
insert_into_free_list(at(i));
}
}
@ -235,6 +265,35 @@ uint HeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretou
return expanded;
}
uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) {
uint expand_candidate = UINT_MAX;
for (uint i = 0; i < max_length(); i++) {
if (is_available(i)) {
// Already in use continue
continue;
}
// Always save the candidate so we can expand later on.
expand_candidate = i;
if (is_on_preferred_index(expand_candidate, preferred_index)) {
// We have found a candidate on the preffered node, break.
break;
}
}
if (expand_candidate == UINT_MAX) {
// No regions left, expand failed.
return 0;
}
make_regions_available(expand_candidate, 1, NULL);
return 1;
}
bool HeapRegionManager::is_on_preferred_index(uint region_index, uint preferred_node_index) {
uint region_node_index = G1NUMA::numa()->preferred_node_index_for_index(region_index);
return region_node_index == preferred_node_index;
}
uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) {
uint found = 0;
size_t length_found = 0;

17
src/hotspot/share/gc/g1/heapRegionManager.hpp
View File

@ -108,6 +108,9 @@ class HeapRegionManager: public CHeapObj<mtGC> {
// sequence could be found, otherwise res_idx contains the start index of this range.
uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const;
// Checks the G1MemoryNodeManager to see if this region is on the preferred node.
bool is_on_preferred_index(uint region_index, uint preferred_node_index);
protected:
G1HeapRegionTable _regions;
G1RegionToSpaceMapper* _heap_mapper;
@ -174,15 +177,8 @@ public:
_free_list.add_ordered(list);
}
virtual HeapRegion* allocate_free_region(HeapRegionType type) {
HeapRegion* hr = _free_list.remove_region(!type.is_young());
if (hr != NULL) {
assert(hr->next() == NULL, "Single region should not have next");
assert(is_available(hr->hrm_index()), "Must be committed");
}
return hr;
}
// Allocate a free region with specific node index. If fails allocate with next node index.
virtual HeapRegion* allocate_free_region(HeapRegionType type, uint requested_node_index);
inline void allocate_free_regions_starting_at(uint first, uint num_regions);
@ -227,6 +223,9 @@ public:
// this.
virtual uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers);
// Try to expand on the given node index.
virtual uint expand_on_preferred_node(uint node_index);
// Find a contiguous set of empty regions of length num. Returns the start index of
// that set, or G1_NO_HRM_INDEX.
virtual uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); }

4
src/hotspot/share/gc/g1/heapRegionSet.hpp
View File

@ -181,6 +181,10 @@ public:
// Removes from head or tail based on the given argument.
HeapRegion* remove_region(bool from_head);
HeapRegion* remove_region_with_node_index(bool from_head,
const uint requested_node_index,
uint* region_node_index);
// Merge two ordered lists. The result is also ordered. The order is
// determined by hrm_index.
void add_ordered(FreeRegionList* from_list);

62
src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
View File

@ -25,6 +25,7 @@
#ifndef SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
#define SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/heapRegionSet.hpp"
inline void HeapRegionSetBase::add(HeapRegion* hr) {
@ -147,4 +148,65 @@ inline HeapRegion* FreeRegionList::remove_region(bool from_head) {
return hr;
}
inline HeapRegion* FreeRegionList::remove_region_with_node_index(bool from_head,
const uint requested_node_index,
uint* allocated_node_index) {
assert(UseNUMA, "Invariant");
const uint max_search_depth = G1NUMA::numa()->max_search_depth();
HeapRegion* cur;
// Find the region to use, searching from _head or _tail as requested.
size_t cur_depth = 0;
if (from_head) {
for (cur = _head;
cur != NULL && cur_depth < max_search_depth;
cur = cur->next(), ++cur_depth) {
if (requested_node_index == cur->node_index()) {
break;
}
}
} else {
for (cur = _tail;
cur != NULL && cur_depth < max_search_depth;
cur = cur->prev(), ++cur_depth) {
if (requested_node_index == cur->node_index()) {
break;
}
}
}
// Didn't find a region to use.
if (cur == NULL || cur_depth >= max_search_depth) {
return NULL;
}
// Splice the region out of the list.
HeapRegion* prev = cur->prev();
HeapRegion* next = cur->next();
if (prev == NULL) {
_head = next;
} else {
prev->set_next(next);
}
if (next == NULL) {
_tail = prev;
} else {
next->set_prev(prev);
}
cur->set_prev(NULL);
cur->set_next(NULL);
if (_last == cur) {
_last = NULL;
}
remove(cur);
if (allocated_node_index != NULL) {
*allocated_node_index = cur->node_index();
}
return cur;
}
#endif // SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP

4
src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -263,7 +263,7 @@ uint HeterogeneousHeapRegionManager::find_empty_in_range_reverse(uint start_idx,
return num_regions_found;
}
HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type) {
HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType type, uint node_index) {
// We want to prevent mutators from proceeding when we have borrowed regions from the last collection. This
// will force a full collection to remedy the situation.

2
src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp
View File

@ -119,7 +119,7 @@ public:
void prepare_for_full_collection_start();
void prepare_for_full_collection_end();
virtual HeapRegion* allocate_free_region(HeapRegionType type);
virtual HeapRegion* allocate_free_region(HeapRegionType type, uint node_index);
// Return maximum number of regions that heap can expand to.
uint max_expandable_length() const;

1
src/hotspot/share/logging/logPrefix.hpp
View File

@ -57,6 +57,7 @@ DEBUG_ONLY(size_t Test_log_prefix_prefixer(char* buf, size_t len);)
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, ihop)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ergo, refine)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, numa)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, heap, region)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, freelist)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, humongous)) \

1
src/hotspot/share/logging/logTag.hpp
View File

@ -108,6 +108,7 @@
LOG_TAG(nestmates) \
LOG_TAG(nmethod) \
LOG_TAG(normalize) \
LOG_TAG(numa) \
LOG_TAG(objecttagging) \
LOG_TAG(obsolete) \
LOG_TAG(oldobject) \

25
src/hotspot/share/prims/whitebox.cpp
View File

@ -619,6 +619,29 @@ WB_ENTRY(jobject, WB_G1AuxiliaryMemoryUsage(JNIEnv* env))
THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1AuxiliaryMemoryUsage: G1 GC is not enabled");
WB_END
WB_ENTRY(jint, WB_G1ActiveMemoryNodeCount(JNIEnv* env, jobject o))
if (UseG1GC) {
G1NUMA* numa = G1NUMA::numa();
return (jint)numa->num_active_nodes();
}
THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1ActiveMemoryNodeCount: G1 GC is not enabled");
WB_END
WB_ENTRY(jintArray, WB_G1MemoryNodeIds(JNIEnv* env, jobject o))
if (UseG1GC) {
G1NUMA* numa = G1NUMA::numa();
int num_node_ids = (int)numa->num_active_nodes();
const int* node_ids = numa->node_ids();
typeArrayOop result = oopFactory::new_intArray(num_node_ids, CHECK_NULL);
for (int i = 0; i < num_node_ids; i++) {
result->int_at_put(i, (jint)node_ids[i]);
}
return (jintArray) JNIHandles::make_local(env, result);
}
THROW_MSG_NULL(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1MemoryNodeIds: G1 GC is not enabled");
WB_END
class OldRegionsLivenessClosure: public HeapRegionClosure {
private:
@ -2195,6 +2218,8 @@ static JNINativeMethod methods[] = {
{CC"g1StartConcMarkCycle", CC"()Z", (void*)&WB_G1StartMarkCycle },
{CC"g1AuxiliaryMemoryUsage", CC"()Ljava/lang/management/MemoryUsage;",
(void*)&WB_G1AuxiliaryMemoryUsage },
{CC"g1ActiveMemoryNodeCount", CC"()I", (void*)&WB_G1ActiveMemoryNodeCount },
{CC"g1MemoryNodeIds", CC"()[I", (void*)&WB_G1MemoryNodeIds },
{CC"g1GetMixedGCInfo", CC"(I)[J", (void*)&WB_G1GetMixedGCInfo },
#endif // INCLUDE_G1GC
#if INCLUDE_G1GC || INCLUDE_PARALLELGC

13
src/hotspot/share/runtime/arguments.cpp
View File

@ -4219,14 +4219,11 @@ jint Arguments::adjust_after_os() {
FLAG_SET_DEFAULT(MinHeapDeltaBytes, 64*M);
}
}
// UseNUMAInterleaving is set to ON for all collectors and
// platforms when UseNUMA is set to ON. NUMA-aware collectors
// such as the parallel collector for Linux and Solaris will
// interleave old gen and survivor spaces on top of NUMA
// allocation policy for the eden space.
// Non NUMA-aware collectors such as G1 and Serial-GC on
// all platforms and ParallelGC on Windows will interleave all
// of the heap spaces across NUMA nodes.
// UseNUMAInterleaving is set to ON for all collectors and platforms when
// UseNUMA is set to ON. NUMA-aware collectors will interleave old gen and
// survivor spaces on top of NUMA allocation policy for the eden space.
// Non NUMA-aware collectors will interleave all of the heap spaces across
// NUMA nodes.
if (FLAG_IS_DEFAULT(UseNUMAInterleaving)) {
FLAG_SET_ERGO(UseNUMAInterleaving, true);
}

1
src/hotspot/share/runtime/os.hpp
View File

@ -374,6 +374,7 @@ class os: AllStatic {
static size_t numa_get_leaf_groups(int *ids, size_t size);
static bool numa_topology_changed();
static int numa_get_group_id();
static int numa_get_group_id_for_address(const void* address);
// Page manipulation
struct page_info {

245
test/hotspot/jtreg/gc/g1/numa/TestG1NUMATouchRegions.java Normal file
View File

@ -0,0 +1,245 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package gc.g1;
/**
* @test TestG1NUMATouchRegions
* @summary Ensure the bottom of the given heap regions are properly touched with requested NUMA id.
* @key gc
* @requires vm.gc.G1
* @requires os.family == "linux"
* @library /test/lib
* @modules java.base/jdk.internal.misc
* java.management
* @build sun.hotspot.WhiteBox
* @run driver ClassFileInstaller sun.hotspot.WhiteBox
* @run main/othervm -XX:+UseG1GC -Xbootclasspath/a:. -XX:+UseNUMA -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.TestG1NUMATouchRegions
*/
import java.util.LinkedList;
import jdk.test.lib.process.OutputAnalyzer;
import jdk.test.lib.process.ProcessTools;
import sun.hotspot.WhiteBox;
public class TestG1NUMATouchRegions {
enum NUMASupportStatus {
NOT_CHECKED,
SUPPORT,
NOT_SUPPORT
};
static int G1HeapRegionSize1MB = 1;
static int G1HeapRegionSize8MB = 8;
static NUMASupportStatus status = NUMASupportStatus.NOT_CHECKED;
public static void main(String[] args) throws Exception {
// 1. Page size < G1HeapRegionSize
// Test default page with 1MB heap region size
testMemoryTouch("-XX:-UseLargePages", G1HeapRegionSize1MB);
// 2. Page size > G1HeapRegionSize
// Test large page with 1MB heap region size.
testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize1MB);
// 3. Page size < G1HeapRegionSize
// Test large page with 8MB heap region size.
testMemoryTouch("-XX:+UseLargePages", G1HeapRegionSize8MB);
}
// On Linux, always UseNUMA is enabled if there is multiple active numa nodes.
static NUMASupportStatus checkNUMAIsEnabled(OutputAnalyzer output) {
boolean supportNUMA = Boolean.parseBoolean(output.firstMatch("\\bUseNUMA\\b.*?=.*?([a-z]+)", 1));
System.out.println("supportNUMA=" + supportNUMA);
return supportNUMA ? NUMASupportStatus.SUPPORT : NUMASupportStatus.NOT_SUPPORT;
}
static long parseSizeString(String size) {
long multiplier = 1;
if (size.endsWith("B")) {
multiplier = 1;
} else if (size.endsWith("K")) {
multiplier = 1024;
} else if (size.endsWith("M")) {
multiplier = 1024 * 1024;
} else if (size.endsWith("G")) {
multiplier = 1024 * 1024 * 1024;
} else {
throw new IllegalArgumentException("Expected memory string '" + size + "'to end with either of: B, K, M, G");
}
long longSize = Long.parseUnsignedLong(size.substring(0, size.length() - 1));
return longSize * multiplier;
}
static long heapPageSize(OutputAnalyzer output) {
String HeapPageSizePattern = "Heap: .*page_size=([^ ]+)";
String str = output.firstMatch(HeapPageSizePattern, 1);
if (str == null) {
output.reportDiagnosticSummary();
throw new RuntimeException("Match from '" + HeapPageSizePattern + "' got 'null'");
}
return parseSizeString(str);
}
// 1. -UseLargePages: default page, page size < G1HeapRegionSize
// +UseLargePages: large page size <= G1HeapRegionSize
//
// Each 'int' represents a numa id of single HeapRegion (bottom page).
// e.g. 1MB heap region, 2MB page size and 2 NUMA nodes system
// Check the first set(2 regions)
// 0| ...omitted..| 00
// 1| ...omitted..| 01
static void checkCase1Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception {
StringBuilder sb = new StringBuilder();
// Append index which means heap region index.
sb.append(String.format("%6d", index));
sb.append("| .* | ");
// Append page node id.
sb.append(String.format("%02d", memoryNodeIds[index]));
output.shouldMatch(sb.toString());
}
// 3. +UseLargePages: large page size > G1HeapRegionSize
//
// As a OS page is consist of multiple heap regions, log also should be
// printed multiple times for same numa id.
// e.g. 1MB heap region, 2MB page size and 2 NUMA nodes system
// Check the first set(4 regions)
// 0| ...omitted..| 00
// 1| ...omitted..| 00
// 2| ...omitted..| 01
// 3| ...omitted..| 01
static void checkCase2Pattern(OutputAnalyzer output, int index, long g1HeapRegionSize, long actualPageSize, int[] memoryNodeIds) throws Exception {
StringBuilder sb = new StringBuilder();
// Append page range.
int lines_to_print = (int)(actualPageSize / g1HeapRegionSize);
for (int i = 0; i < lines_to_print; i++) {
// Append index which means heap region index.
sb.append(String.format("%6d", index * lines_to_print + i));
sb.append("| .* | ");
// Append page node id.
sb.append(String.format("%02d", memoryNodeIds[index]));
output.shouldMatch(sb.toString());
sb.setLength(0);
}
}
static void checkNUMALog(OutputAnalyzer output, int regionSizeInMB) throws Exception {
WhiteBox wb = WhiteBox.getWhiteBox();
long g1HeapRegionSize = regionSizeInMB * 1024 * 1024;
long actualPageSize = heapPageSize(output);
long defaultPageSize = (long)wb.getVMPageSize();
int memoryNodeCount = wb.g1ActiveMemoryNodeCount();
int[] memoryNodeIds = wb.g1MemoryNodeIds();
System.out.println("node count=" + memoryNodeCount + ", actualPageSize=" + actualPageSize);
// Check for the first set of active numa nodes.
for (int index = 0; index < memoryNodeCount; index++) {
if (actualPageSize <= defaultPageSize) {
checkCase1Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds);
} else {
checkCase2Pattern(output, index, g1HeapRegionSize, actualPageSize, memoryNodeIds);
}
}
}
static void testMemoryTouch(String largePagesSetting, int regionSizeInMB) throws Exception {
// Skip testing with message.
if (status == NUMASupportStatus.NOT_SUPPORT) {
System.out.println("NUMA is not supported");
return;
}
ProcessBuilder pb_enabled = ProcessTools.createJavaProcessBuilder(
"-Xbootclasspath/a:.",
"-Xlog:pagesize,gc+heap+region=trace",
"-XX:+UseG1GC",
"-Xmx128m",
"-Xms128m",
"-XX:+UnlockDiagnosticVMOptions",
"-XX:+WhiteBoxAPI",
"-XX:+PrintFlagsFinal",
"-XX:+UseNUMA",
"-XX:+AlwaysPreTouch",
largePagesSetting,
"-XX:G1HeapRegionSize=" + regionSizeInMB + "m",
GCTest.class.getName());
OutputAnalyzer output = new OutputAnalyzer(pb_enabled.start());
// Check NUMA availability.
if (status == NUMASupportStatus.NOT_CHECKED) {
status = checkNUMAIsEnabled(output);
}
if (status == NUMASupportStatus.SUPPORT) {
checkNUMALog(output, regionSizeInMB);
} else {
// Exit with message for the first test.
System.out.println("NUMA is not supported");
}
}
static class GCTest {
public static final int M = 1024*1024;
public static LinkedList<Object> garbageList = new LinkedList<Object>();
// A large object referenced by a static.
static int[] filler = new int[10 * M];
public static void genGarbage() {
for (int i = 0; i < 32*1024; i++) {
garbageList.add(new int[100]);
}
garbageList.clear();
}
public static void main(String[] args) {
int[] large = new int[M];
Object ref = large;
System.out.println("Creating garbage");
for (int i = 0; i < 100; i++) {
// A large object that will be reclaimed eagerly.
large = new int[6*M];
genGarbage();
// Make sure that the compiler cannot completely remove
// the allocation of the large object until here.
System.out.println(large);
}
// Keep the reference to the first object alive.
System.out.println(ref);
System.out.println("Done");
}
}
}

3
test/lib/sun/hotspot/WhiteBox.java
View File

@ -193,6 +193,9 @@ public class WhiteBox {
return parseCommandLine0(commandline, delim, args);
}
public native int g1ActiveMemoryNodeCount();
public native int[] g1MemoryNodeIds();
// Parallel GC
public native long psVirtualSpaceAlignment();
public native long psHeapGenerationAlignment();