This commit is contained in:
Erik Trimble 2009-03-12 18:16:36 -07:00
commit 8d6035660e
223 changed files with 5079 additions and 2219 deletions

View file

@ -393,7 +393,7 @@ class CMSAdaptiveSizePolicy : public AdaptiveSizePolicy {
// Restarts the concurrent phases timer.
void concurrent_phases_resume();
// Time begining and end of the marking phase for
// Time beginning and end of the marking phase for
// a synchronous MS collection. A MS collection
// that finishes in the foreground can have started
// in the background. These methods capture the

View file

@ -69,7 +69,7 @@ class CMSGCAdaptivePolicyCounters : public GCAdaptivePolicyCounters {
// end of the sweep of the tenured generation.
PerfVariable* _avg_cms_free_counter;
// Average of the free space in the tenured generation at the
// start of the sweep of the tenured genertion.
// start of the sweep of the tenured generation.
PerfVariable* _avg_cms_free_at_sweep_counter;
// Average of the free space in the tenured generation at the
// after any resizing of the tenured generation at the end

View file

@ -4178,7 +4178,7 @@ bool CMSCollector::do_marking_mt(bool asynch) {
// and is deferred for now; see CR# TBF. 07252005YSR. XXX
assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
// If _restart_addr is non-NULL, a marking stack overflow
// occured; we need to do a fresh marking iteration from the
// occurred; we need to do a fresh marking iteration from the
// indicated restart address.
if (_foregroundGCIsActive && asynch) {
// We may be running into repeated stack overflows, having
@ -4221,7 +4221,7 @@ bool CMSCollector::do_marking_st(bool asynch) {
// should be incremental with periodic yields.
_markBitMap.iterate(&markFromRootsClosure);
// If _restart_addr is non-NULL, a marking stack overflow
// occured; we need to do a fresh iteration from the
// occurred; we need to do a fresh iteration from the
// indicated restart address.
while (_restart_addr != NULL) {
if (_foregroundGCIsActive && asynch) {

View file

@ -133,14 +133,12 @@ void ConcurrentG1RefineThread::queueBasedRefinement() {
_co_tracker.update(false);
if (G1SmoothConcRefine) {
start_vtime_sec = os::elapsedVTime();
prev_buffer_num = curr_buffer_num;
_sts.leave();
os::sleep(Thread::current(), (jlong) _interval_ms, false);
_sts.join();
start_vtime_sec = os::elapsedVTime();
}
n_logs++;
}
// Make sure we harvest the PYA, if any.

View file

@ -420,6 +420,10 @@ ConcurrentMark::ConcurrentMark(ReservedSpace rs,
_has_overflown(false),
_concurrent(false),
_has_aborted(false),
_restart_for_overflow(false),
_concurrent_marking_in_progress(false),
_should_gray_objects(false),
// _verbose_level set below

View file

@ -107,7 +107,7 @@ void ConcurrentMarkThread::run() {
if (PrintGC) {
gclog_or_tty->date_stamp(PrintGCDateStamps);
gclog_or_tty->stamp(PrintGCTimeStamps);
tty->print_cr("[GC concurrent-mark-start]");
gclog_or_tty->print_cr("[GC concurrent-mark-start]");
}
if (!g1_policy->in_young_gc_mode()) {
@ -320,8 +320,6 @@ void ConcurrentMarkThread::sleepBeforeNextCycle() {
set_in_progress();
clear_started();
if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-starting");
return;
}
// Note: this method, although exported by the ConcurrentMarkSweepThread,

View file

@ -78,8 +78,8 @@ size_t DirtyCardQueueSet::num_par_ids() {
void DirtyCardQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock,
int max_completed_queue,
Mutex* lock) {
PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue);
Mutex* lock, PtrQueueSet* fl_owner) {
PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue, fl_owner);
set_buffer_size(DCQBarrierQueueBufferSize);
set_process_completed_threshold(DCQBarrierProcessCompletedThreshold);

View file

@ -88,7 +88,7 @@ public:
void initialize(Monitor* cbl_mon, Mutex* fl_lock,
int max_completed_queue = 0,
Mutex* lock = NULL);
Mutex* lock = NULL, PtrQueueSet* fl_owner = NULL);
// The number of parallel ids that can be claimed to allow collector or
// mutator threads to do card-processing work.

View file

@ -136,6 +136,14 @@ public:
int calls() { return _calls; }
};
class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {
public:
bool do_card_ptr(jbyte* card_ptr, int worker_i) {
*card_ptr = CardTableModRefBS::dirty_card_val();
return true;
}
};
YoungList::YoungList(G1CollectedHeap* g1h)
: _g1h(g1h), _head(NULL),
_scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),
@ -812,6 +820,40 @@ public:
}
};
class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
G1CollectedHeap* _g1h;
UpdateRSOopClosure _cl;
int _worker_i;
public:
RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :
_cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),
_worker_i(worker_i),
_g1h(g1)
{ }
bool doHeapRegion(HeapRegion* r) {
if (!r->continuesHumongous()) {
_cl.set_from(r);
r->oop_iterate(&_cl);
}
return false;
}
};
class ParRebuildRSTask: public AbstractGangTask {
G1CollectedHeap* _g1;
public:
ParRebuildRSTask(G1CollectedHeap* g1)
: AbstractGangTask("ParRebuildRSTask"),
_g1(g1)
{ }
void work(int i) {
RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);
_g1->heap_region_par_iterate_chunked(&rebuild_rs, i,
HeapRegion::RebuildRSClaimValue);
}
};
void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
size_t word_size) {
ResourceMark rm;
@ -918,24 +960,35 @@ void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
reset_gc_time_stamp();
// Since everything potentially moved, we will clear all remembered
// sets, and clear all cards. Later we will also cards in the used
// portion of the heap after the resizing (which could be a shrinking.)
// We will also reset the GC time stamps of the regions.
// sets, and clear all cards. Later we will rebuild remebered
// sets. We will also reset the GC time stamps of the regions.
PostMCRemSetClearClosure rs_clear(mr_bs());
heap_region_iterate(&rs_clear);
// Resize the heap if necessary.
resize_if_necessary_after_full_collection(full ? 0 : word_size);
// Since everything potentially moved, we will clear all remembered
// sets, but also dirty all cards corresponding to used regions.
PostMCRemSetInvalidateClosure rs_invalidate(mr_bs());
heap_region_iterate(&rs_invalidate);
if (_cg1r->use_cache()) {
_cg1r->clear_and_record_card_counts();
_cg1r->clear_hot_cache();
}
// Rebuild remembered sets of all regions.
if (ParallelGCThreads > 0) {
ParRebuildRSTask rebuild_rs_task(this);
assert(check_heap_region_claim_values(
HeapRegion::InitialClaimValue), "sanity check");
set_par_threads(workers()->total_workers());
workers()->run_task(&rebuild_rs_task);
set_par_threads(0);
assert(check_heap_region_claim_values(
HeapRegion::RebuildRSClaimValue), "sanity check");
reset_heap_region_claim_values();
} else {
RebuildRSOutOfRegionClosure rebuild_rs(this);
heap_region_iterate(&rebuild_rs);
}
if (PrintGC) {
print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());
}
@ -961,7 +1014,8 @@ void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
// dirtied, so this should abandon those logs, and set "do_traversal"
// to true.
concurrent_g1_refine()->set_pya_restart();
assert(!G1DeferredRSUpdate
|| (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");
assert(regions_accounted_for(), "Region leakage!");
}
@ -1466,6 +1520,13 @@ jint G1CollectedHeap::initialize() {
G1DirtyCardQueueMax,
Shared_DirtyCardQ_lock);
}
if (G1DeferredRSUpdate) {
dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,
DirtyCardQ_FL_lock,
0,
Shared_DirtyCardQ_lock,
&JavaThread::dirty_card_queue_set());
}
// In case we're keeping closure specialization stats, initialize those
// counts and that mechanism.
SpecializationStats::clear();
@ -2316,7 +2377,6 @@ class VerifyMarkedObjsClosure: public ObjectClosure {
void
G1CollectedHeap::checkConcurrentMark() {
VerifyMarkedObjsClosure verifycl(this);
doConcurrentMark();
// MutexLockerEx x(getMarkBitMapLock(),
// Mutex::_no_safepoint_check_flag);
object_iterate(&verifycl);
@ -2493,7 +2553,7 @@ G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) {
guarantee(_in_cset_fast_test == NULL, "invariant");
guarantee(_in_cset_fast_test_base == NULL, "invariant");
_in_cset_fast_test_length = n_regions();
_in_cset_fast_test_length = max_regions();
_in_cset_fast_test_base =
NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
memset(_in_cset_fast_test_base, false,
@ -2513,7 +2573,7 @@ G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) {
}
save_marks();
// We must do this before any possible evacuation that should propogate
// We must do this before any possible evacuation that should propagate
// marks, including evacuation of popular objects in a popular pause.
if (mark_in_progress()) {
double start_time_sec = os::elapsedTime();
@ -2626,9 +2686,8 @@ G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) {
#endif // SCAN_ONLY_VERBOSE
double end_time_sec = os::elapsedTime();
if (!evacuation_failed()) {
g1_policy()->record_pause_time((end_time_sec - start_time_sec)*1000.0);
}
double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
g1_policy()->record_pause_time_ms(pause_time_ms);
GCOverheadReporter::recordSTWEnd(end_time_sec);
g1_policy()->record_collection_pause_end(popular_region != NULL,
abandoned);
@ -2919,27 +2978,51 @@ public:
}
};
class RecreateRSetEntriesClosure: public OopClosure {
class UpdateRSetImmediate : public OopsInHeapRegionClosure {
private:
G1CollectedHeap* _g1;
G1RemSet* _g1_rem_set;
HeapRegion* _from;
public:
RecreateRSetEntriesClosure(G1CollectedHeap* g1, HeapRegion* from) :
_g1(g1), _g1_rem_set(g1->g1_rem_set()), _from(from)
{}
UpdateRSetImmediate(G1CollectedHeap* g1) :
_g1(g1), _g1_rem_set(g1->g1_rem_set()) {}
void do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
void do_oop(oop* p) {
assert(_from->is_in_reserved(p), "paranoia");
if (*p != NULL) {
_g1_rem_set->write_ref(_from, p);
if (*p != NULL && !_from->is_survivor()) {
_g1_rem_set->par_write_ref(_from, p, 0);
}
}
};
class UpdateRSetDeferred : public OopsInHeapRegionClosure {
private:
G1CollectedHeap* _g1;
DirtyCardQueue *_dcq;
CardTableModRefBS* _ct_bs;
public:
UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
_g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
void do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
void do_oop(oop* p) {
assert(_from->is_in_reserved(p), "paranoia");
if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
size_t card_index = _ct_bs->index_for(p);
if (_ct_bs->mark_card_deferred(card_index)) {
_dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
}
}
}
};
class RemoveSelfPointerClosure: public ObjectClosure {
private:
G1CollectedHeap* _g1;
@ -2947,11 +3030,11 @@ private:
HeapRegion* _hr;
size_t _prev_marked_bytes;
size_t _next_marked_bytes;
OopsInHeapRegionClosure *_cl;
public:
RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr) :
_g1(g1), _cm(_g1->concurrent_mark()), _hr(hr),
_prev_marked_bytes(0), _next_marked_bytes(0)
{}
RemoveSelfPointerClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* cl) :
_g1(g1), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0),
_next_marked_bytes(0), _cl(cl) {}
size_t prev_marked_bytes() { return _prev_marked_bytes; }
size_t next_marked_bytes() { return _next_marked_bytes; }
@ -2989,8 +3072,7 @@ public:
// that, if evacuation fails, we might have remembered set
// entries missing given that we skipped cards on the
// collection set. So, we'll recreate such entries now.
RecreateRSetEntriesClosure cl(_g1, _hr);
obj->oop_iterate(&cl);
obj->oop_iterate(_cl);
assert(_cm->isPrevMarked(obj), "Should be marked!");
} else {
// The object has been either evacuated or is dead. Fill it with a
@ -3003,14 +3085,23 @@ public:
};
void G1CollectedHeap::remove_self_forwarding_pointers() {
UpdateRSetImmediate immediate_update(_g1h);
DirtyCardQueue dcq(&_g1h->dirty_card_queue_set());
UpdateRSetDeferred deferred_update(_g1h, &dcq);
OopsInHeapRegionClosure *cl;
if (G1DeferredRSUpdate) {
cl = &deferred_update;
} else {
cl = &immediate_update;
}
HeapRegion* cur = g1_policy()->collection_set();
while (cur != NULL) {
assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
RemoveSelfPointerClosure rspc(_g1h, cl);
if (cur->evacuation_failed()) {
RemoveSelfPointerClosure rspc(_g1h, cur);
assert(cur->in_collection_set(), "bad CS");
cl->set_region(cur);
cur->object_iterate(&rspc);
// A number of manipulations to make the TAMS be the current top,
@ -3519,6 +3610,9 @@ class G1ParScanThreadState : public StackObj {
protected:
G1CollectedHeap* _g1h;
RefToScanQueue* _refs;
DirtyCardQueue _dcq;
CardTableModRefBS* _ct_bs;
G1RemSet* _g1_rem;
typedef GrowableArray<oop*> OverflowQueue;
OverflowQueue* _overflowed_refs;
@ -3560,10 +3654,32 @@ protected:
void add_to_undo_waste(size_t waste) { _undo_waste += waste; }
DirtyCardQueue& dirty_card_queue() { return _dcq; }
CardTableModRefBS* ctbs() { return _ct_bs; }
void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
_g1_rem->par_write_ref(from, p, tid);
}
void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
// If the new value of the field points to the same region or
// is the to-space, we don't need to include it in the Rset updates.
if (!from->is_in_reserved(*p) && !from->is_survivor()) {
size_t card_index = ctbs()->index_for(p);
// If the card hasn't been added to the buffer, do it.
if (ctbs()->mark_card_deferred(card_index)) {
dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
}
}
}
public:
G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
: _g1h(g1h),
_refs(g1h->task_queue(queue_num)),
_dcq(&g1h->dirty_card_queue_set()),
_ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
_g1_rem(g1h->g1_rem_set()),
_hash_seed(17), _queue_num(queue_num),
_term_attempts(0),
_age_table(false),
@ -3641,6 +3757,14 @@ public:
int refs_to_scan() { return refs()->size(); }
int overflowed_refs_to_scan() { return overflowed_refs()->length(); }
void update_rs(HeapRegion* from, oop* p, int tid) {
if (G1DeferredRSUpdate) {
deferred_rs_update(from, p, tid);
} else {
immediate_rs_update(from, p, tid);
}
}
HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
HeapWord* obj = NULL;
@ -3809,7 +3933,6 @@ public:
}
};
G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
_g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
_par_scan_state(par_scan_state) { }
@ -3835,7 +3958,7 @@ void G1ParScanClosure::do_oop_nv(oop* p) {
assert(obj == *p, "the value of *p should not have changed");
_par_scan_state->push_on_queue(p);
} else {
_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
_par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
}
}
}
@ -3973,13 +4096,13 @@ void G1ParCopyClosure<do_gen_barrier, barrier,
}
// When scanning the RS, we only care about objs in CS.
if (barrier == G1BarrierRS) {
_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
_par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
}
}
// When scanning moved objs, must look at all oops.
if (barrier == G1BarrierEvac && obj != NULL) {
_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
_par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
}
if (do_gen_barrier && obj != NULL) {
@ -4128,6 +4251,7 @@ public:
G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss);
G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss);
G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss);
G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss);
G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss);
G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss);
@ -4383,7 +4507,6 @@ void G1CollectedHeap::evacuate_collection_set() {
g1_rem_set()->prepare_for_oops_into_collection_set_do();
concurrent_g1_refine()->set_use_cache(false);
int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
set_par_threads(n_workers);
G1ParTask g1_par_task(this, n_workers, _task_queues);
@ -4391,8 +4514,9 @@ void G1CollectedHeap::evacuate_collection_set() {
change_strong_roots_parity(); // In preparation for parallel strong roots.
rem_set()->prepare_for_younger_refs_iterate(true);
double start_par = os::elapsedTime();
assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
double start_par = os::elapsedTime();
if (ParallelGCThreads > 0) {
// The individual threads will set their evac-failure closures.
workers()->run_task(&g1_par_task);
@ -4412,8 +4536,8 @@ void G1CollectedHeap::evacuate_collection_set() {
G1KeepAliveClosure keep_alive(this);
JNIHandles::weak_oops_do(&is_alive, &keep_alive);
}
g1_rem_set()->cleanup_after_oops_into_collection_set_do();
concurrent_g1_refine()->set_use_cache(true);
finalize_for_evac_failure();
@ -4424,7 +4548,6 @@ void G1CollectedHeap::evacuate_collection_set() {
if (evacuation_failed()) {
remove_self_forwarding_pointers();
if (PrintGCDetails) {
gclog_or_tty->print(" (evacuation failed)");
} else if (PrintGC) {
@ -4432,6 +4555,14 @@ void G1CollectedHeap::evacuate_collection_set() {
}
}
if (G1DeferredRSUpdate) {
RedirtyLoggedCardTableEntryFastClosure redirty;
dirty_card_queue_set().set_closure(&redirty);
dirty_card_queue_set().apply_closure_to_all_completed_buffers();
JavaThread::dirty_card_queue_set().merge_bufferlists(&dirty_card_queue_set());
assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
}
COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
}

View file

@ -457,6 +457,10 @@ protected:
// And it's mod ref barrier set, used to track updates for the above.
ModRefBarrierSet* _mr_bs;
// A set of cards that cover the objects for which the Rsets should be updated
// concurrently after the collection.
DirtyCardQueueSet _dirty_card_queue_set;
// The Heap Region Rem Set Iterator.
HeapRegionRemSetIterator** _rem_set_iterator;
@ -666,6 +670,9 @@ public:
RefToScanQueue *task_queue(int i);
// A set of cards where updates happened during the GC
DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; }
// Create a G1CollectedHeap with the specified policy.
// Must call the initialize method afterwards.
// May not return if something goes wrong.

View file

@ -1014,7 +1014,7 @@ void G1CollectorPolicy::record_full_collection_end() {
_all_full_gc_times_ms->add(full_gc_time_ms);
update_recent_gc_times(end_sec, full_gc_time_sec);
update_recent_gc_times(end_sec, full_gc_time_ms);
_g1->clear_full_collection();
@ -1475,6 +1475,7 @@ void G1CollectorPolicy::record_collection_pause_end(bool popular,
size_t cur_used_bytes = _g1->used();
assert(cur_used_bytes == _g1->recalculate_used(), "It should!");
bool last_pause_included_initial_mark = false;
bool update_stats = !abandoned && !_g1->evacuation_failed();
#ifndef PRODUCT
if (G1YoungSurvRateVerbose) {
@ -1535,7 +1536,7 @@ void G1CollectorPolicy::record_collection_pause_end(bool popular,
_n_pauses++;
if (!abandoned) {
if (update_stats) {
_recent_CH_strong_roots_times_ms->add(_cur_CH_strong_roots_dur_ms);
_recent_G1_strong_roots_times_ms->add(_cur_G1_strong_roots_dur_ms);
_recent_evac_times_ms->add(evac_ms);
@ -1636,7 +1637,7 @@ void G1CollectorPolicy::record_collection_pause_end(bool popular,
double termination_time = avg_value(_par_last_termination_times_ms);
double parallel_other_time;
if (!abandoned) {
if (update_stats) {
MainBodySummary* body_summary = summary->main_body_summary();
guarantee(body_summary != NULL, "should not be null!");
@ -1852,7 +1853,7 @@ void G1CollectorPolicy::record_collection_pause_end(bool popular,
// <NEW PREDICTION>
if (!popular && !abandoned) {
if (!popular && update_stats) {
double pause_time_ms = elapsed_ms;
size_t diff = 0;

View file

@ -966,7 +966,7 @@ public:
record_termination_time(0, ms);
}
void record_pause_time(double ms) {
void record_pause_time_ms(double ms) {
_last_pause_time_ms = ms;
}

View file

@ -105,33 +105,6 @@ StupidG1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
_g1->heap_region_iterate(&rc);
}
class UpdateRSOopClosure: public OopClosure {
HeapRegion* _from;
HRInto_G1RemSet* _rs;
int _worker_i;
public:
UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
_from(NULL), _rs(rs), _worker_i(worker_i) {
guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
}
void set_from(HeapRegion* from) {
assert(from != NULL, "from region must be non-NULL");
_from = from;
}
virtual void do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
virtual void do_oop(oop* p) {
assert(_from != NULL, "from region must be non-NULL");
_rs->par_write_ref(_from, p, _worker_i);
}
// Override: this closure is idempotent.
// bool idempotent() { return true; }
bool apply_to_weak_ref_discovered_field() { return true; }
};
class UpdateRSOutOfRegionClosure: public HeapRegionClosure {
G1CollectedHeap* _g1h;
ModRefBarrierSet* _mr_bs;
@ -177,11 +150,19 @@ HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
_cards_scanned(NULL), _total_cards_scanned(0)
{
_seq_task = new SubTasksDone(NumSeqTasks);
_new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, ParallelGCThreads);
guarantee(n_workers() > 0, "There should be some workers");
_new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, n_workers());
for (uint i = 0; i < n_workers(); i++) {
_new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
}
}
HRInto_G1RemSet::~HRInto_G1RemSet() {
delete _seq_task;
for (uint i = 0; i < n_workers(); i++) {
delete _new_refs[i];
}
FREE_C_HEAP_ARRAY(GrowableArray<oop*>*, _new_refs);
}
void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
@ -281,8 +262,9 @@ public:
if (!_ct_bs->is_card_claimed(card_index) &&
!_ct_bs->is_card_dirty(card_index)) {
assert(_ct_bs->is_card_clean(card_index) ||
_ct_bs->is_card_claimed(card_index),
"Card is either dirty, clean, or claimed");
_ct_bs->is_card_claimed(card_index) ||
_ct_bs->is_card_deferred(card_index),
"Card is either clean, claimed or deferred");
if (_ct_bs->claim_card(card_index))
scanCard(card_index, card_region);
}
@ -338,14 +320,12 @@ void HRInto_G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
_g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0);
_g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
if (ParallelGCThreads > 0) {
// In this case, we called scanNewRefsRS and recorded the corresponding
// time.
double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
if (scan_new_refs_time_ms > 0.0) {
closure_app_time_ms += scan_new_refs_time_ms;
}
double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
if (scan_new_refs_time_ms > 0.0) {
closure_app_time_ms += scan_new_refs_time_ms;
}
_g1p->record_obj_copy_time(worker_i, closure_app_time_ms);
}
@ -469,8 +449,8 @@ HRInto_G1RemSet::scanNewRefsRS(OopsInHeapRegionClosure* oc,
double scan_new_refs_start_sec = os::elapsedTime();
G1CollectedHeap* g1h = G1CollectedHeap::heap();
CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
while (_new_refs[worker_i]->is_nonempty()) {
oop* p = _new_refs[worker_i]->pop();
for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
oop* p = _new_refs[worker_i]->at(i);
oop obj = *p;
// *p was in the collection set when p was pushed on "_new_refs", but
// another thread may have processed this location from an RS, so it
@ -480,10 +460,6 @@ HRInto_G1RemSet::scanNewRefsRS(OopsInHeapRegionClosure* oc,
HeapRegion* r = g1h->heap_region_containing(p);
DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj));
assert(ParallelGCThreads > 1
|| to->rem_set()->contains_reference(p),
"Invariant: pushed after being added."
"(Not reliable in parallel code.)");
oc->set_region(r);
// If "p" has already been processed concurrently, this is
// idempotent.
@ -538,8 +514,8 @@ HRInto_G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
}
} else {
assert(worker_i == 0, "invariant");
updateRS(0);
scanNewRefsRS(oc, 0);
scanRS(oc, 0);
}
}
@ -559,11 +535,7 @@ prepare_for_oops_into_collection_set_do() {
assert(!_par_traversal_in_progress, "Invariant between iterations.");
if (ParallelGCThreads > 0) {
set_par_traversal(true);
int n_workers = _g1->workers()->total_workers();
_seq_task->set_par_threads(n_workers);
for (uint i = 0; i < ParallelGCThreads; i++)
_new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
_seq_task->set_par_threads((int)n_workers());
if (cg1r->do_traversal()) {
updateRS(0);
// Have to do this again after updaters
@ -587,6 +559,53 @@ class cleanUpIteratorsClosure : public HeapRegionClosure {
}
};
class UpdateRSetOopsIntoCSImmediate : public OopClosure {
G1CollectedHeap* _g1;
public:
UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { }
virtual void do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
virtual void do_oop(oop* p) {
HeapRegion* to = _g1->heap_region_containing(*p);
if (to->in_collection_set()) {
if (to->rem_set()->add_reference(p, 0)) {
_g1->schedule_popular_region_evac(to);
}
}
}
};
class UpdateRSetOopsIntoCSDeferred : public OopClosure {
G1CollectedHeap* _g1;
CardTableModRefBS* _ct_bs;
DirtyCardQueue* _dcq;
public:
UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
_g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { }
virtual void do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
virtual void do_oop(oop* p) {
oop obj = *p;
if (_g1->obj_in_cs(obj)) {
size_t card_index = _ct_bs->index_for(p);
if (_ct_bs->mark_card_deferred(card_index)) {
_dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
}
}
}
};
void HRInto_G1RemSet::new_refs_iterate(OopClosure* cl) {
for (size_t i = 0; i < n_workers(); i++) {
for (int j = 0; j < _new_refs[i]->length(); j++) {
oop* p = _new_refs[i]->at(j);
cl->do_oop(p);
}
}
}
void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
guarantee( _cards_scanned != NULL, "invariant" );
_total_cards_scanned = 0;
@ -609,11 +628,25 @@ void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
if (cg1r->do_traversal()) {
cg1r->cg1rThread()->set_do_traversal(false);
}
for (uint i = 0; i < ParallelGCThreads; i++) {
delete _new_refs[i];
}
set_par_traversal(false);
}
if (_g1->evacuation_failed()) {
// Restore remembered sets for the regions pointing into
// the collection set.
if (G1DeferredRSUpdate) {
DirtyCardQueue dcq(&_g1->dirty_card_queue_set());
UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq);
new_refs_iterate(&deferred_update);
} else {
UpdateRSetOopsIntoCSImmediate immediate_update(_g1);
new_refs_iterate(&immediate_update);
}
}
for (uint i = 0; i < n_workers(); i++) {
_new_refs[i]->clear();
}
assert(!_par_traversal_in_progress, "Invariant between iterations.");
}
@ -683,7 +716,8 @@ public:
bool doHeapRegion(HeapRegion* r) {
if (!r->in_collection_set() &&
!r->continuesHumongous() &&
!r->is_young()) {
!r->is_young() &&
!r->is_survivor()) {
_update_rs_oop_cl.set_from(r);
UpdateRSObjectClosure update_rs_obj_cl(&_update_rs_oop_cl);
@ -820,7 +854,7 @@ void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) {
// before all the cards on the region are dirtied. This is unlikely,
// and it doesn't happen often, but it can happen. So, the extra
// check below filters out those cards.
if (r->is_young()) {
if (r->is_young() || r->is_survivor()) {
return;
}
// While we are processing RSet buffers during the collection, we

View file

@ -155,6 +155,7 @@ protected:
bool _par_traversal_in_progress;
void set_par_traversal(bool b);
GrowableArray<oop*>** _new_refs;
void new_refs_iterate(OopClosure* cl);
public:
// This is called to reset dual hash tables after the gc pause
@ -214,3 +215,27 @@ public:
int n() { return _n; };
HeapWord* start_first() { return _start_first; }
};
class UpdateRSOopClosure: public OopClosure {
HeapRegion* _from;
HRInto_G1RemSet* _rs;
int _worker_i;
public:
UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
_from(NULL), _rs(rs), _worker_i(worker_i) {
guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
}
void set_from(HeapRegion* from) {
assert(from != NULL, "from region must be non-NULL");
_from = from;
}
virtual void do_oop(narrowOop* p);
virtual void do_oop(oop* p);
// Override: this closure is idempotent.
// bool idempotent() { return true; }
bool apply_to_weak_ref_discovered_field() { return true; }
};

View file

@ -31,24 +31,7 @@ inline size_t G1RemSet::n_workers() {
}
inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, oop* p) {
oop obj = *p;
assert(from != NULL && from->is_in_reserved(p),
"p is not in a from");
HeapRegion* to = _g1->heap_region_containing(obj);
if (from != to && to != NULL) {
if (!to->popular() && !from->is_survivor()) {
#if G1_REM_SET_LOGGING
gclog_or_tty->print_cr("Adding " PTR_FORMAT " (" PTR_FORMAT ") to RS"
" for region [" PTR_FORMAT ", " PTR_FORMAT ")",
p, obj,
to->bottom(), to->end());
#endif
assert(to->rem_set() != NULL, "Need per-region 'into' remsets.");
if (to->rem_set()->add_reference(p)) {
_g1->schedule_popular_region_evac(to);
}
}
}
par_write_ref(from, p, 0);
}
inline void HRInto_G1RemSet::write_ref(HeapRegion* from, oop* p) {
@ -82,7 +65,22 @@ inline void HRInto_G1RemSet::par_write_ref(HeapRegion* from, oop* p, int tid) {
HeapRegion* to = _g1->heap_region_containing(obj);
// The test below could be optimized by applying a bit op to to and from.
if (to != NULL && from != NULL && from != to) {
if (!to->popular() && !from->is_survivor()) {
bool update_delayed = false;
// There is a tricky infinite loop if we keep pushing
// self forwarding pointers onto our _new_refs list.
// The _par_traversal_in_progress flag is true during the collection pause,
// false during the evacuation failure handing.
if (_par_traversal_in_progress &&
to->in_collection_set() && !self_forwarded(obj)) {
_new_refs[tid]->push(p);
// Deferred updates to the Cset are either discarded (in the normal case),
// or processed (if an evacuation failure occurs) at the end
// of the collection.
// See HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do().
update_delayed = true;
}
if (!to->popular() && !update_delayed) {
#if G1_REM_SET_LOGGING
gclog_or_tty->print_cr("Adding " PTR_FORMAT " (" PTR_FORMAT ") to RS"
" for region [" PTR_FORMAT ", " PTR_FORMAT ")",
@ -94,11 +92,14 @@ inline void HRInto_G1RemSet::par_write_ref(HeapRegion* from, oop* p, int tid) {
_g1->schedule_popular_region_evac(to);
}
}
// There is a tricky infinite loop if we keep pushing
// self forwarding pointers onto our _new_refs list.
if (_par_traversal_in_progress &&
to->in_collection_set() && !self_forwarded(obj)) {
_new_refs[tid]->push(p);
}
}
}
inline void UpdateRSOopClosure::do_oop(narrowOop* p) {
guarantee(false, "NYI");
}
inline void UpdateRSOopClosure::do_oop(oop* p) {
assert(_from != NULL, "from region must be non-NULL");
_rs->par_write_ref(_from, p, _worker_i);
}

View file

@ -172,6 +172,9 @@
develop(bool, G1RSBarrierUseQueue, true, \
"If true, use queueing RS barrier") \
\
develop(bool, G1DeferredRSUpdate, true, \
"If true, use deferred RS updates") \
\
develop(bool, G1RSLogCheckCardTable, false, \
"If true, verify that no dirty cards remain after RS log " \
"processing.") \

View file

@ -318,7 +318,8 @@ class HeapRegion: public G1OffsetTableContigSpace {
FinalCountClaimValue = 1,
NoteEndClaimValue = 2,
ScrubRemSetClaimValue = 3,
ParVerifyClaimValue = 4
ParVerifyClaimValue = 4,
RebuildRSClaimValue = 5
};
// Concurrent refinement requires contiguous heap regions (in which TLABs

View file

@ -91,15 +91,17 @@ PtrQueueSet::PtrQueueSet(bool notify_when_complete) :
_n_completed_buffers(0),
_process_completed_threshold(0), _process_completed(false),
_buf_free_list(NULL), _buf_free_list_sz(0)
{}
{
_fl_owner = this;
}
void** PtrQueueSet::allocate_buffer() {
assert(_sz > 0, "Didn't set a buffer size.");
MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
if (_buf_free_list != NULL) {
void** res = _buf_free_list;
_buf_free_list = (void**)_buf_free_list[0];
_buf_free_list_sz--;
MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
if (_fl_owner->_buf_free_list != NULL) {
void** res = _fl_owner->_buf_free_list;
_fl_owner->_buf_free_list = (void**)_fl_owner->_buf_free_list[0];
_fl_owner->_buf_free_list_sz--;
// Just override the next pointer with NULL, just in case we scan this part
// of the buffer.
res[0] = NULL;
@ -111,10 +113,10 @@ void** PtrQueueSet::allocate_buffer() {
void PtrQueueSet::deallocate_buffer(void** buf) {
assert(_sz > 0, "Didn't set a buffer size.");
MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
buf[0] = (void*)_buf_free_list;
_buf_free_list = buf;
_buf_free_list_sz++;
MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
buf[0] = (void*)_fl_owner->_buf_free_list;
_fl_owner->_buf_free_list = buf;
_fl_owner->_buf_free_list_sz++;
}
void PtrQueueSet::reduce_free_list() {
@ -207,3 +209,58 @@ void PtrQueueSet::set_buffer_size(size_t sz) {
void PtrQueueSet::set_process_completed_threshold(size_t sz) {
_process_completed_threshold = sz;
}
// Merge lists of buffers. Notify waiting threads if the length of the list
// exceeds threshold. The source queue is emptied as a result. The queues
// must share the monitor.
void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) {
assert(_cbl_mon == src->_cbl_mon, "Should share the same lock");
MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
if (_completed_buffers_tail == NULL) {
assert(_completed_buffers_head == NULL, "Well-formedness");
_completed_buffers_head = src->_completed_buffers_head;
_completed_buffers_tail = src->_completed_buffers_tail;
} else {
assert(_completed_buffers_head != NULL, "Well formedness");
if (src->_completed_buffers_head != NULL) {
_completed_buffers_tail->next = src->_completed_buffers_head;
_completed_buffers_tail = src->_completed_buffers_tail;
}
}
_n_completed_buffers += src->_n_completed_buffers;
src->_n_completed_buffers = 0;
src->_completed_buffers_head = NULL;
src->_completed_buffers_tail = NULL;
assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL ||
_completed_buffers_head != NULL && _completed_buffers_tail != NULL,
"Sanity");
if (!_process_completed &&
_n_completed_buffers >= _process_completed_threshold) {
_process_completed = true;
if (_notify_when_complete)
_cbl_mon->notify_all();
}
}
// Merge free lists of the two queues. The free list of the source
// queue is emptied as a result. The queues must share the same
// mutex that guards free lists.
void PtrQueueSet::merge_freelists(PtrQueueSet* src) {
assert(_fl_lock == src->_fl_lock, "Should share the same lock");
MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
if (_buf_free_list != NULL) {
void **p = _buf_free_list;
while (*p != NULL) {
p = (void**)*p;
}
*p = src->_buf_free_list;
} else {
_buf_free_list = src->_buf_free_list;
}
_buf_free_list_sz += src->_buf_free_list_sz;
src->_buf_free_list = NULL;
src->_buf_free_list_sz = 0;
}

View file

@ -155,6 +155,9 @@ protected:
Mutex* _fl_lock;
void** _buf_free_list;
size_t _buf_free_list_sz;
// Queue set can share a freelist. The _fl_owner variable
// specifies the owner. It is set to "this" by default.
PtrQueueSet* _fl_owner;
// The size of all buffers in the set.
size_t _sz;
@ -188,10 +191,13 @@ public:
// Because of init-order concerns, we can't pass these as constructor
// arguments.
void initialize(Monitor* cbl_mon, Mutex* fl_lock,
int max_completed_queue = 0) {
int max_completed_queue = 0,
PtrQueueSet *fl_owner = NULL) {
_max_completed_queue = max_completed_queue;
assert(cbl_mon != NULL && fl_lock != NULL, "Init order issue?");
_cbl_mon = cbl_mon; _fl_lock = fl_lock;
_cbl_mon = cbl_mon;
_fl_lock = fl_lock;
_fl_owner = (fl_owner != NULL) ? fl_owner : this;
}
// Return an empty oop array of size _sz (required to be non-zero).
@ -228,4 +234,7 @@ public:
void reduce_free_list();
size_t completed_buffers_num() { return _n_completed_buffers; }
void merge_bufferlists(PtrQueueSet* src);
void merge_freelists(PtrQueueSet* src);
};

View file

@ -504,6 +504,7 @@ void SparsePRT::cleanup() {
// Make sure that the current and next tables agree. (Another mechanism
// takes care of deleting now-unused tables.)
_cur = _next;
set_expanded(false);
}
void SparsePRT::expand() {

View file

@ -274,7 +274,7 @@ public:
// Clean up all tables on the expanded list. Called single threaded.
static void cleanup_all();
RSHashTable* next() const { return _next; }
RSHashTable* cur() const { return _cur; }
void init_iterator(SparsePRTIter* sprt_iter);
@ -300,7 +300,7 @@ public:
{}
void init(const SparsePRT* sprt) {
RSHashTableIter::init(sprt->next());
RSHashTableIter::init(sprt->cur());
}
bool has_next(size_t& card_index) {
return RSHashTableIter::has_next(card_index);

View file

@ -78,7 +78,7 @@ class CheckForUnmarkedObjects : public ObjectClosure {
}
// Card marks are not precise. The current system can leave us with
// a mismash of precise marks and begining of object marks. This means
// a mismash of precise marks and beginning of object marks. This means
// we test for missing precise marks first. If any are found, we don't
// fail unless the object head is also unmarked.
virtual void do_object(oop obj) {
@ -258,7 +258,7 @@ void CardTableExtension::scavenge_contents_parallel(ObjectStartArray* start_arra
if (!start_array->object_starts_in_range(slice_start, slice_end)) {
continue;
}
// Update our begining addr
// Update our beginning addr
HeapWord* first_object = start_array->object_start(slice_start);
debug_only(oop* first_object_within_slice = (oop*) first_object;)
if (first_object < slice_start) {

View file

@ -127,7 +127,7 @@ class ObjectStartArray : public CHeapObj {
// Optimized for finding the first object that crosses into
// a given block. The blocks contain the offset of the last
// object in that block. Scroll backwards by one, and the first
// object hit should be at the begining of the block
// object hit should be at the beginning of the block
HeapWord* object_start(HeapWord* addr) const {
assert(_covered_region.contains(addr), "Must be in covered region");
jbyte* block = block_for_addr(addr);

View file

@ -26,7 +26,7 @@
// PrefetchQueue is a FIFO queue of variable length (currently 8).
//
// We need to examine the performance penalty of variable lengths.
// We may also want to split this into cpu dependant bits.
// We may also want to split this into cpu dependent bits.
//
const int PREFETCH_QUEUE_SIZE = 8;

View file

@ -125,6 +125,8 @@ void PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) {
perm_gen->verify_object_start_array();
}
heap->pre_full_gc_dump();
// Filled in below to track the state of the young gen after the collection.
bool eden_empty;
bool survivors_empty;
@ -363,6 +365,8 @@ void PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) {
Universe::print_heap_after_gc();
}
heap->post_full_gc_dump();
#ifdef TRACESPINNING
ParallelTaskTerminator::print_termination_counts();
#endif

View file

@ -1982,6 +1982,8 @@ void PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) {
heap->record_gen_tops_before_GC();
}
heap->pre_full_gc_dump();
_print_phases = PrintGCDetails && PrintParallelOldGCPhaseTimes;
// Make sure data structures are sane, make the heap parsable, and do other
@ -2204,6 +2206,8 @@ void PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) {
gc_task_manager()->print_task_time_stamps();
}
heap->post_full_gc_dump();
#ifdef TRACESPINNING
ParallelTaskTerminator::print_termination_counts();
#endif

View file

@ -74,7 +74,7 @@ void MutableNUMASpace::ensure_parsability() {
for (int i = 0; i < lgrp_spaces()->length(); i++) {
LGRPSpace *ls = lgrp_spaces()->at(i);
MutableSpace *s = ls->space();
if (s->top() < top()) { // For all spaces preceeding the one containing top()
if (s->top() < top()) { // For all spaces preceding the one containing top()
if (s->free_in_words() > 0) {
size_t area_touched_words = pointer_delta(s->end(), s->top());
CollectedHeap::fill_with_object(s->top(), area_touched_words);

View file

@ -121,7 +121,7 @@ void VM_GC_HeapInspection::doit() {
// make the heap parsable (no need to retire TLABs)
ch->ensure_parsability(false);
}
HeapInspection::heap_inspection(_out);
HeapInspection::heap_inspection(_out, _need_prologue /* need_prologue */);
}

View file

@ -112,13 +112,16 @@ class VM_GC_HeapInspection: public VM_GC_Operation {
private:
outputStream* _out;
bool _full_gc;
bool _need_prologue;
public:
VM_GC_HeapInspection(outputStream* out, bool request_full_gc) :
VM_GC_HeapInspection(outputStream* out, bool request_full_gc,
bool need_prologue) :
VM_GC_Operation(0 /* total collections, dummy, ignored */,
0 /* total full collections, dummy, ignored */,
request_full_gc) {
_out = out;
_full_gc = request_full_gc;
_need_prologue = need_prologue;
}
~VM_GC_HeapInspection() {}