diff -r 102466e70deb src/share/vm/gc_implementation/g1/collectionSetChooser.cpp --- a/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp Fri Apr 01 14:13:53 2011 -0700 @@ -21,17 +21,20 @@ * questions. * */ #include "precompiled.hpp" #include "gc_implementation/g1/collectionSetChooser.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" +#include "gc_implementation/g1/heapRegionRemSet.hpp" #include "memory/space.inline.hpp" +#include "memory/heapInspection.hpp" + CSetChooserCache::CSetChooserCache() { for (int i = 0; i < CacheLength; ++i) _cache[i] = NULL; clear(); } void CSetChooserCache::clear() { @@ -254,49 +257,164 @@ CollectionSetChooser::sortMarkedHeapRegi || _markedRegions.at(_numMarkedRegions-1) != NULL, "Testing _numMarkedRegions"); assert(_numMarkedRegions == _markedRegions.length() || _markedRegions.at(_numMarkedRegions) == NULL, "Testing _numMarkedRegions"); if (G1PrintParCleanupStats) { gclog_or_tty->print_cr(" Sorted %d marked regions.", _numMarkedRegions); } + + size_t tot_used = 0; + size_t tot_max_live = 0; for (int i = 0; i < _numMarkedRegions; i++) { assert(_markedRegions.at(i) != NULL, "Should be true by sorting!"); _markedRegions.at(i)->set_sort_index(i); if (G1PrintRegionLivenessInfo > 0) { if (i == 0) gclog_or_tty->print_cr("Sorted marked regions:"); if (i < G1PrintRegionLivenessInfo || (_numMarkedRegions-i) < G1PrintRegionLivenessInfo) { HeapRegion* hr = _markedRegions.at(i); size_t u = hr->used(); + tot_used += u; + tot_max_live += hr->max_live_bytes(); gclog_or_tty->print_cr(" Region %d: %d used, %d max live, %5.2f%%.", i, u, hr->max_live_bytes(), 100.0*(float)hr->max_live_bytes()/(float)u); } } } + + if (G1PrintRegionLivenessInfo > 0) { + gclog_or_tty->print_cr(" Overall (among printed): " + "" SIZE_FORMAT " used, " SIZE_FORMAT " max live, %5.2f%%.", + tot_used, tot_max_live, + 100.0*(float)tot_max_live/(float)tot_used); + } + if (G1PolicyVerbose > 1) printSortedHeapRegions(); assert(verify(), "should now be sorted"); } +class HistoClosure : public KlassInfoClosure { + private: + KlassInfoHisto* _cih; + public: + HistoClosure(KlassInfoHisto* cih) : _cih(cih) {} + + void do_cinfo(KlassInfoEntry* cie) { + _cih->add(cie); + } +}; + +class RecordInstanceClosure : public ObjectClosure { + private: + G1CollectedHeap *_g1; + KlassInfoTable* _cit; + size_t _missed_count; + size_t _instance_count; + public: + RecordInstanceClosure(KlassInfoTable* cit) : + _cit(cit), _missed_count(0), _instance_count(0) { + _g1 = G1CollectedHeap::heap(); + } + + void do_object(oop obj) { + if (obj == NULL) return; + if (!_g1->isMarkedNext(obj)) { + // dead + return; + } + + _instance_count++; + if (!_cit->record_instance(obj)) { + _missed_count++; + } + } + + size_t missed_count() { return _missed_count; } + size_t instance_count() { return _instance_count; } +}; + + +size_t +printHeapRegionLiveClassHistogram(outputStream *st, HeapRegion *hr) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + KlassInfoTable cit(KlassInfoTable::cit_size, g1h->perm_gen()->used_region().start()); + if (!cit.allocation_failed()) { + RecordInstanceClosure ric(&cit); + hr->object_iterate(&ric); + size_t missed_count = ric.missed_count(); + if (missed_count != 0) { + st->print_cr("WARNING: Ran out of C-heap; undercounted " SIZE_FORMAT + " total instances in data below", + missed_count); + } + // Sort and print klass instance info + KlassInfoHisto histo("\n" + " num #instances #bytes class name\n" + "----------------------------------------------", + KlassInfoHisto::histo_initial_size); + HistoClosure hc(&histo); + cit.iterate(&hc); + histo.sort(); + histo.print_on(st); + return ric.instance_count(); + + } else { + st->print_cr("WARNING: Ran out of C-heap; histogram not generated"); + return 0; + } + +} + +void printHeapRegion(HeapRegion *hr, bool details); + + +class PrintHeapRegionClosure : public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion *hr) { + printHeapRegion(hr, false); + return false; + } +}; + void -printHeapRegion(HeapRegion *hr) { +printHeapRegion(HeapRegion *hr, bool details) { if (hr->isHumongous()) gclog_or_tty->print("H: "); if (hr->in_collection_set()) gclog_or_tty->print("CS: "); gclog_or_tty->print_cr("Region " PTR_FORMAT " (%s%s) " "[" PTR_FORMAT ", " PTR_FORMAT"] " - "Used: " SIZE_FORMAT "K, garbage: " SIZE_FORMAT "K.", + "Used: " SIZE_FORMAT "K, garbage: " SIZE_FORMAT "K. " + "Eff: %f K/ms", hr, hr->is_young() ? "Y " : " ", hr->is_marked()? "M1" : "M0", hr->bottom(), hr->end(), - hr->used()/K, hr->garbage_bytes()/K); + hr->used()/K, hr->garbage_bytes()/K, + hr->gc_efficiency()/K); + double garbage_pct = (double)hr->garbage_bytes() / hr->used(); + if (details && garbage_pct > 0.98 && hr->gc_efficiency() < 100*K) { + HeapRegionRemSet *rem_set = hr->rem_set(); + gclog_or_tty->print_cr(" Very low-occupancy low-efficiency region."); + gclog_or_tty->print_cr(" RSet: coarse: %d fine: %d sparse: %d", + rem_set->occ_coarse(), + rem_set->occ_fine(), + rem_set->occ_sparse()); + size_t num_instances = printHeapRegionLiveClassHistogram(gclog_or_tty, hr); + if (num_instances == 1) { + gclog_or_tty->print_cr("Coarse region references:\n" + "--------------------------"); + PrintHeapRegionClosure closure; + rem_set->iterate_coarse_regions(&closure); + gclog_or_tty->print_cr("--------------------------"); + } + } } void CollectionSetChooser::addMarkedHeapRegion(HeapRegion* hr) { assert(!hr->isHumongous(), "Humongous regions shouldn't be added to the collection set"); assert(!hr->is_young(), "should not be young!"); _markedRegions.append(hr); @@ -359,17 +477,17 @@ void CollectionSetChooser::printSortedHeapRegions() { gclog_or_tty->print_cr("Printing %d Heap Regions sorted by amount of known garbage", _numMarkedRegions); DEBUG_ONLY(int marked_count = 0;) for (int i = 0; i < _markedRegions.length(); i++) { HeapRegion* r = _markedRegions.at(i); if (r != NULL) { - printHeapRegion(r); + printHeapRegion(r, true); DEBUG_ONLY(marked_count++;) } } assert(marked_count == _numMarkedRegions, "must be"); gclog_or_tty->print_cr("Done sorted heap region print"); } void CollectionSetChooser::removeRegion(HeapRegion *hr) { @@ -392,31 +510,51 @@ HeapRegion* CollectionSetChooser::getNextMarkedRegion(double time_remaining, double avg_prediction) { G1CollectedHeap* g1h = G1CollectedHeap::heap(); G1CollectorPolicy* g1p = g1h->g1_policy(); fillCache(); if (_cache.is_empty()) { assert(_curMarkedIndex == _numMarkedRegions, "if cache is empty, list should also be empty"); + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr(" (no more marked regions; cache is empty)"); + } return NULL; } HeapRegion *hr = _cache.get_first(); assert(hr != NULL, "if cache not empty, first entry should be non-null"); double predicted_time = g1h->predict_region_elapsed_time_ms(hr, false); if (g1p->adaptive_young_list_length()) { if (time_remaining - predicted_time < 0.0) { g1h->check_if_region_is_too_expensive(predicted_time); + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr(" (no more marked regions; next region too " + "expensive (adaptive; predicted %lfms > remaining %lfms))", + predicted_time, time_remaining); + } return NULL; } } else { if (predicted_time > 2.0 * avg_prediction) { + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr(" (no more marked regions; next region too expensive (static))"); + } return NULL; } } + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr(" (picked region; %lfms predicted; %lfms remaining; " + "" SIZE_FORMAT "kb marked; " SIZE_FORMAT "kb maxlive; %d-%d%% liveness)", + predicted_time, time_remaining, + hr->marked_bytes()/K, hr->max_live_bytes()/K, + (int)((float)hr->marked_bytes()/(float)hr->used()*100.0), + (int)((float)hr->max_live_bytes()/(float)hr->used()*100.0)); + } + HeapRegion *hr2 = _cache.remove_first(); assert(hr == hr2, "cache contents should not have changed"); return hr; } diff -r 102466e70deb src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Apr 01 14:13:53 2011 -0700 @@ -41,16 +41,19 @@ #include "memory/gcLocker.inline.hpp" #include "memory/genOopClosures.inline.hpp" #include "memory/generationSpec.hpp" #include "oops/oop.inline.hpp" #include "oops/oop.pcgc.inline.hpp" #include "runtime/aprofiler.hpp" #include "runtime/vmThread.hpp" +#include +#include + size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0; // turn it on so that the contents of the young list (scan-only / // to-be-collected) are printed at "strategic" points before / during // / after the collection --- this is useful for debugging #define YOUNG_LIST_VERBOSE 0 // CURRENT STATUS // This file is under construction. Search for "FIXME". @@ -3340,17 +3343,33 @@ G1CollectedHeap::do_collection_pause_at_ concurrent_mark()->newCSet(); #if YOUNG_LIST_VERBOSE gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:"); _young_list->print(); g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); #endif // YOUNG_LIST_VERBOSE + struct rusage rusage_before, rusage_after; + if (getrusage(RUSAGE_SELF, &rusage_before) != 0) { + vm_exit_out_of_memory(0, + "Unable to get memory usage"); + } g1_policy()->choose_collection_set(target_pause_time_ms); + if (getrusage(RUSAGE_SELF, &rusage_after) != 0) { + vm_exit_out_of_memory(0, + "Unable to get memory usage"); + } + gclog_or_tty->print_cr( + "Resource usage of choose_cset:" + "majflt: %ld\tnswap: %ld\tnvcsw: %ld\tnivcsw: %ld", + rusage_after.ru_majflt - rusage_before.ru_majflt, + rusage_after.ru_nswap - rusage_before.ru_nswap, + rusage_after.ru_nvcsw - rusage_before.ru_nvcsw, + rusage_after.ru_nivcsw - rusage_before.ru_nivcsw); // Nothing to do if we were unable to choose a collection set. #if G1_REM_SET_LOGGING gclog_or_tty->print_cr("\nAfter pause, heap:"); print(); #endif PrepareForRSScanningClosure prepare_for_rs_scan; collection_set_iterate(&prepare_for_rs_scan); diff -r 102466e70deb src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Fri Apr 01 14:13:53 2011 -0700 @@ -30,17 +30,17 @@ #include "gc_implementation/g1/g1CollectorPolicy.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/shared/gcPolicyCounters.hpp" #include "runtime/arguments.hpp" #include "runtime/java.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/debug.hpp" -#define PREDICTIONS_VERBOSE 0 +#define PREDICTIONS_VERBOSE 1 // // Different defaults for different number of GC threads // They were chosen by running GCOld and SPECjbb on debris with different // numbers of GC threads and choosing them based on the results // all the same @@ -1414,17 +1414,18 @@ void G1CollectorPolicy::record_collectio print_stats(1, "Cum Clear CC", _cum_clear_cc_time_ms); print_stats(1, "Min Clear CC", _min_clear_cc_time_ms); print_stats(1, "Max Clear CC", _max_clear_cc_time_ms); if (_num_cc_clears > 0) { print_stats(1, "Avg Clear CC", _cum_clear_cc_time_ms / ((double)_num_cc_clears)); } #endif print_stats(1, "Other", other_time_ms); - print_stats(2, "Choose CSet", _recorded_young_cset_choice_time_ms); + print_stats(2, "Choose Young CSet", _recorded_young_cset_choice_time_ms); + print_stats(2, "Choose Non-Young CSet", _recorded_non_young_cset_choice_time_ms); for (int i = 0; i < _aux_num; ++i) { if (_cur_aux_times_set[i]) { char buffer[96]; sprintf(buffer, "Aux%d", i); print_stats(1, buffer, _cur_aux_times_ms[i]); } } @@ -1464,25 +1465,39 @@ void G1CollectorPolicy::record_collectio bool new_in_marking_window_im = false; if (during_initial_mark_pause()) { new_in_marking_window = true; new_in_marking_window_im = true; } if (in_young_gc_mode()) { if (_last_full_young_gc) { + if (G1PolicyVerbose > 1) { + gclog_or_tty->print_cr("Did a young GC last time"); + } set_full_young_gcs(false); _last_full_young_gc = false; } if ( !_last_young_gc_full ) { if ( _should_revert_to_full_young_gcs || _known_garbage_ratio < 0.05 || - (adaptive_young_list_length() && + (adaptive_young_list_length() && /* false &&*/ // scodetodo (get_gc_eff_factor() * cur_efficiency < predict_young_gc_eff())) ) { + if (G1PolicyVerbose > 1) { + gclog_or_tty->print_cr("Reverting to young GCs:\n" + " should_revert: %s\n" + " known_garbage: %.2f\n" + " cur_efficiency: %.4f\n" + " predicted eff: %.4f\n", + _should_revert_to_full_young_gcs ? "true":"false", + _known_garbage_ratio, + cur_efficiency, + predict_young_gc_eff()); + } set_full_young_gcs(true); } } _should_revert_to_full_young_gcs = false; if (_last_young_gc_full && !_during_marking) _young_gc_eff_seq->add(cur_efficiency); } @@ -1555,16 +1570,29 @@ void G1CollectorPolicy::record_collectio double non_young_other_time_ms = 0.0; if (_recorded_non_young_regions > 0) { non_young_other_time_ms = _recorded_non_young_cset_choice_time_ms + _recorded_non_young_free_cset_time_ms; _non_young_other_cost_per_region_ms_seq->add(non_young_other_time_ms / (double) _recorded_non_young_regions); + } else { + // no non-young gen collections - if our prediction is high enough, we would + // never collect non-young again, so push it back towards zero so we give it + // another try. + double predicted_other_time = predict_non_young_other_time_ms(1); + if (predicted_other_time > MaxGCPauseMillis/2.0) { + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr( + "Predicted non-young other time %.1f is too large compared to max pause time. Weighting down.", + predicted_other_time); + } + _non_young_other_cost_per_region_ms_seq->add(0.0); + } } double constant_other_time_ms = all_other_time_ms - (young_other_time_ms + non_young_other_time_ms); _constant_other_time_ms_seq->add(constant_other_time_ms); double survival_ratio = 0.0; if (_bytes_in_collection_set_before_gc > 0) { @@ -1591,16 +1619,17 @@ void G1CollectorPolicy::record_collectio "REGIONS %d %d %d " "PENDING_CARDS %d %d " "CARDS_SCANNED %d %d " "RS_LENGTHS %d %d " "RS_UPDATE %1.6lf %1.6lf RS_SCAN %1.6lf %1.6lf " "SURVIVAL_RATIO %1.6lf %1.6lf " "OBJECT_COPY %1.6lf %1.6lf OTHER_CONSTANT %1.6lf %1.6lf " "OTHER_YOUNG %1.6lf %1.6lf " + " REC_CSET_CHOICE=%1.6lf REC_CSET_FREE=%1.6lf " "OTHER_NON_YOUNG %1.6lf %1.6lf " "VTIME_DIFF %1.6lf TERMINATION %1.6lf " "ELAPSED %1.6lf %1.6lf ", _cur_collection_start_sec, (!_last_young_gc_full) ? 2 : (last_pause_included_initial_mark) ? 1 : 0, _recorded_region_num, _recorded_young_regions, @@ -1609,18 +1638,18 @@ void G1CollectorPolicy::record_collectio _predicted_cards_scanned, cards_scanned, _predicted_rs_lengths, _max_rs_lengths, _predicted_rs_update_time_ms, update_rs_time, _predicted_rs_scan_time_ms, scan_rs_time, _predicted_survival_ratio, survival_ratio, _predicted_object_copy_time_ms, obj_copy_time, _predicted_constant_other_time_ms, constant_other_time_ms, _predicted_young_other_time_ms, young_other_time_ms, - _predicted_non_young_other_time_ms, - non_young_other_time_ms, + _recorded_non_young_cset_choice_time_ms, _recorded_non_young_free_cset_time_ms, + _predicted_non_young_other_time_ms, non_young_other_time_ms, _vtime_diff_ms, termination_time, _predicted_pause_time_ms, elapsed_ms); } if (G1PolicyVerbose > 0) { gclog_or_tty->print_cr("Pause Time, predicted: %1.4lfms (predicted %s), actual: %1.4lfms", _predicted_pause_time_ms, (_within_target) ? "within" : "outside", @@ -1744,24 +1773,31 @@ G1CollectorPolicy::predict_region_elapse size_t rs_length = hr->rem_set()->occupied(); size_t card_num; if (full_young_gcs()) card_num = predict_young_card_num(rs_length); else card_num = predict_non_young_card_num(rs_length); size_t bytes_to_copy = predict_bytes_to_copy(hr); - double region_elapsed_time_ms = - predict_rs_scan_time_ms(card_num) + - predict_object_copy_time_ms(bytes_to_copy); + double rs_scan_time_ms = predict_rs_scan_time_ms(card_num); + double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy); + double other_time_ms; if (young) - region_elapsed_time_ms += predict_young_other_time_ms(1); + other_time_ms = predict_young_other_time_ms(1); else - region_elapsed_time_ms += predict_non_young_other_time_ms(1); + other_time_ms = predict_non_young_other_time_ms(1); + + double region_elapsed_time_ms = rs_scan_time_ms + copy_time_ms + other_time_ms; + + if (G1PolicyVerbose > 0 && region_elapsed_time_ms > 10.0) { // scodetodo + gclog_or_tty->print_cr("predict_region_elapsed_time_ms: %lfms total, %lfms rs scan (" SIZE_FORMAT " cnum), %lf copy time (" SIZE_FORMAT " bytes), %lf other time", + region_elapsed_time_ms, rs_scan_time_ms, card_num, copy_time_ms, bytes_to_copy, other_time_ms); + } return region_elapsed_time_ms; } size_t G1CollectorPolicy::predict_bytes_to_copy(HeapRegion* hr) { size_t bytes_to_copy; if (hr->is_marked()) @@ -1881,16 +1917,19 @@ void G1CollectorPolicy::check_if_region_ // right now we don't support the non-generational G1 mode // (and in fact we are planning to remove the associated code, // see CR 6814390). So, let's leave it as is and this will be // removed some time in the future ShouldNotReachHere(); set_during_initial_mark_pause(); } else // no point in doing another partial one + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr("Reverting to young gc:s due to expensive region"); // scodetodo + } _should_revert_to_full_young_gcs = true; } } // void G1CollectorPolicy::update_recent_gc_times(double end_time_sec, diff -r 102466e70deb src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Apr 01 14:13:53 2011 -0700 @@ -216,16 +216,19 @@ "-1 means print all.") \ \ develop(bool, G1ScrubRemSets, true, \ "When true, do RS scrubbing after cleanup.") \ \ develop(bool, G1RSScrubVerbose, false, \ "When true, do RS scrubbing with verbose output.") \ \ + develop(bool, G1ScrubSplitCoarse, false, \ + "When true, split coarse rset entries into fine ones.") \ + \ develop(bool, G1YoungSurvRateVerbose, false, \ "print out the survival rate of young regions according to age.") \ \ develop(intx, G1YoungSurvRateNumRegionsSummary, 0, \ "the number of regions for which we'll print a surv rate " \ "summary.") \ \ product(intx, G1ReservePercent, 10, \ diff -r 102466e70deb src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Fri Apr 01 14:13:53 2011 -0700 @@ -190,16 +190,25 @@ public: HeapWord* hr_bot = hr()->bottom(); size_t hr_first_card_index = ctbs->index_for(hr_bot); bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); #if PRT_COUNT_OCCUPIED recount_occupied(); #endif } + void merge(CardTableModRefBS* ctbs, BitMap* card_bm) { + HeapWord* hr_bot = hr()->bottom(); + size_t hr_first_card_index = ctbs->index_for(hr_bot); + bm()->set_union_at_offset(*card_bm, hr_first_card_index); +#if PRT_COUNT_OCCUPIED + recount_occupied(); +#endif + } + void add_card(CardIdx_t from_card_index) { add_card_work(from_card_index, /*parallel*/ true); } void seq_add_card(CardIdx_t from_card_index) { add_card_work(from_card_index, /*parallel*/ false); } @@ -426,16 +435,21 @@ public: } } void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { assert(_par_tables == NULL, "Precondition"); PerRegionTable::scrub(ctbs, card_bm); } + void merge(CardTableModRefBS* ctbs, BitMap* card_bm) { + assert(_par_tables == NULL, "Precondition"); + PerRegionTable::merge(ctbs, card_bm); + } + size_t mem_size() const { size_t res = PerRegionTable::mem_size() + sizeof(this) - sizeof(PerRegionTable); if (_par_tables != NULL) { for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { res += _par_tables[i]->mem_size(); } } @@ -500,16 +514,19 @@ size_t OtherRegionsTable::_fine_eviction OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) : _g1h(G1CollectedHeap::heap()), _m(Mutex::leaf, "An OtherRegionsTable lock", true), _hr(hr), _coarse_map(G1CollectedHeap::heap()->max_regions(), false /* in-resource-area */), _fine_grain_regions(NULL), _n_fine_entries(0), _n_coarse_entries(0), +#if ACCOUNT_COARSE + _coarse_card_count(0), +#endif #if SAMPLE_FOR_EVICTION _fine_eviction_start(0), #endif _sparse_table(hr) { typedef PosParPRT* PosParPRTPtr; if (_max_fine_entries == 0) { assert(_mod_max_fine_entries_mask == 0, "Both or none."); @@ -565,16 +582,39 @@ void OtherRegionsTable::print_from_card_ for (size_t j = 0; j < _from_card_cache_max_regions; j++) { gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.", i, j, _from_card_cache[i][j]); } } } #endif +class BitMapToRegionClosure : public BitMapClosure { +private: + HeapRegionClosure *_hrc; + G1CollectedHeap *_g1h; + +public: + BitMapToRegionClosure(HeapRegionClosure *hrc, + G1CollectedHeap *g1h) : + _hrc(hrc), _g1h(g1h) {} + bool do_bit(BitMap::idx_t offset) { + HeapRegion* hr = _g1h->region_at(offset); + assert(hr != NULL, "bad region"); + bool shouldTerminate = _hrc->doHeapRegion(hr); + return !shouldTerminate; // bitmap closure return false means terminate + } +}; + + +void OtherRegionsTable::iterate_coarse_regions(HeapRegionClosure *heap_closure) { + BitMapToRegionClosure closure(heap_closure, _g1h); + _coarse_map.iterate(&closure); +} + void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { size_t cur_hrs_ind = hr()->hrs_index(); #if HRRS_VERBOSE gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", from, UseCompressedOops ? oopDesc::load_decode_heap_oop((narrowOop*)from) @@ -834,16 +874,19 @@ PosParPRT* OtherRegionsTable::delete_reg } #endif // Set the corresponding coarse bit. int max_hrs_index = max->hr()->hrs_index(); if (!_coarse_map.at(max_hrs_index)) { _coarse_map.at_put(max_hrs_index, true); _n_coarse_entries++; +#if ACCOUNT_COARSE + _coarse_card_count += max->occupied(); +#endif #if 0 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " "for region [" PTR_FORMAT "...] (%d coarse entries).\n", hr()->bottom(), max->hr()->bottom(), _n_coarse_entries); #endif } @@ -902,16 +945,67 @@ void OtherRegionsTable::scrub(CardTableM PosParPRT::free(cur); } else { prev = cur->next_addr(); } } cur = nxt; } } + +#if ACCOUNT_COARSE + _coarse_card_count = 0; + int coarse_idx = -1; + for (size_t i = 0; i < _n_coarse_entries; i++) { + // Find next region in course map + coarse_idx = _coarse_map.get_next_one_offset(coarse_idx + 1); + assert(_coarse_map.at(coarse_idx), "should get 1-bit"); + HeapRegion *hr = _g1h->region_at(coarse_idx); + assert(hr != NULL, "bad region"); + + size_t off = ctbs->index_for(hr->bottom()); + _coarse_card_count += card_bm->count_one_bits(off, off + HeapRegion::CardsPerRegion); + } +#else + if (G1ScrubSplitCoarse) { + // Try to un-coarsen the course maps into fine: + size_t avail_fine_entries = _max_fine_entries - _n_fine_entries; + int coarse_idx = -1; + while (avail_fine_entries > 0 && _n_coarse_entries > 0) { + // Find next region in course map + coarse_idx = _coarse_map.get_next_one_offset(coarse_idx + 1); + assert(_coarse_map.at(coarse_idx), "should get 1-bit"); + HeapRegion *hr = _g1h->region_at(coarse_idx); + assert(hr != NULL, "bad region"); + + // Remove from coarse map + _coarse_map.at_put(coarse_idx, 0); + _n_coarse_entries--; + assert(_n_coarse_entries >= 0, "invariant"); + + // Add to fine + PosParPRT *prt = PosParPRT::alloc(hr); + prt->merge(ctbs, card_bm); + RegionIdx_t from_hrs_ind = (RegionIdx_t) hr->hrs_index(); + size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; + + PosParPRT* first_prt = _fine_grain_regions[ind]; + prt->set_next(first_prt); // XXX Maybe move to init? + _fine_grain_regions[ind] = prt; + _n_fine_entries++; + avail_fine_entries--; + assert(_n_fine_entries <= _max_fine_entries, "invariant"); + + gclog_or_tty->print_cr("Split coarse region " PTR_FORMAT + " into fine entry with %d occupied cards.", + hr, + prt->occupied()); + } + } +#endif // Since we may have deleted a from_card_cache entry from the RS, clear // the FCC. clear_fcc(); } size_t OtherRegionsTable::occupied() const { // Cast away const in this case. @@ -930,17 +1024,21 @@ size_t OtherRegionsTable::occ_fine() con sum += cur->occupied(); cur = cur->next(); } } return sum; } size_t OtherRegionsTable::occ_coarse() const { +#if ACCOUNT_COARSE + return _coarse_card_count; +#else return (_n_coarse_entries * HeapRegion::CardsPerRegion); +#endif } size_t OtherRegionsTable::occ_sparse() const { return _sparse_table.occupied(); } size_t OtherRegionsTable::mem_size() const { // Cast away const in this case. @@ -984,16 +1082,19 @@ void OtherRegionsTable::clear() { cur = nxt; } _fine_grain_regions[i] = NULL; } _sparse_table.clear(); _coarse_map.clear(); _n_fine_entries = 0; _n_coarse_entries = 0; +#if ACCOUNT_COARSE + _coarse_card_count = 0; +#endif clear_fcc(); } void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); size_t hrs_ind = (size_t)from_hr->hrs_index(); size_t ind = hrs_ind & _mod_max_fine_entries_mask; diff -r 102466e70deb src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Fri Apr 01 14:13:53 2011 -0700 @@ -68,16 +68,20 @@ class OtherRegionsTable VALUE_OBJ_CLASS_ G1CollectedHeap* _g1h; Mutex _m; HeapRegion* _hr; // These are protected by "_m". BitMap _coarse_map; size_t _n_coarse_entries; +#define ACCOUNT_COARSE 1 +#if ACCOUNT_COARSE + size_t _coarse_card_count; +#endif static jint _n_coarsenings; PosParPRT** _fine_grain_regions; size_t _n_fine_entries; #define SAMPLE_FOR_EVICTION 1 #if SAMPLE_FOR_EVICTION size_t _fine_eviction_start; @@ -161,16 +165,17 @@ public: static void init_from_card_cache(size_t max_regions); // Declares that only regions i s.t. 0 <= i < new_n_regs are in use. // Make sure any entries for higher regions are invalid. static void shrink_from_card_cache(size_t new_n_regs); static void print_from_card_cache(); + void iterate_coarse_regions(HeapRegionClosure *closure); }; class HeapRegionRemSet : public CHeapObj { friend class VMStructs; friend class HeapRegionRemSetIterator; public: @@ -222,16 +227,20 @@ public: } size_t occ_coarse() const { return _other_regions.occ_coarse(); } size_t occ_sparse() const { return _other_regions.occ_sparse(); } + void iterate_coarse_regions(HeapRegionClosure *closure) { + _other_regions.iterate_coarse_regions(closure); + } + static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); } /* Used in the sequential case. Returns "true" iff this addition causes the size limit to be reached. */ void add_reference(OopOrNarrowOopStar from) { _other_regions.add_reference(from); } diff -r 102466e70deb src/share/vm/utilities/bitMap.cpp --- a/src/share/vm/utilities/bitMap.cpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/utilities/bitMap.cpp Fri Apr 01 14:13:53 2011 -0700 @@ -396,16 +396,31 @@ void BitMap::set_intersection_at_offset( bm_word_t* other_map = other.map(); idx_t offset_word_ind = word_index(offset); idx_t size = size_in_words(); for (idx_t index = 0; index < size; index++) { dest_map[index] = dest_map[index] & other_map[offset_word_ind + index]; } } +void BitMap::set_union_at_offset(BitMap other, idx_t offset) { + assert(other.size() >= offset, "offset not in range"); + assert(other.size() - offset >= size(), "other not large enough"); + // XXX Ideally, we would remove this restriction. + guarantee((offset % (sizeof(bm_word_t) * BitsPerByte)) == 0, + "Only handle aligned cases so far."); + bm_word_t* dest_map = map(); + bm_word_t* other_map = other.map(); + idx_t offset_word_ind = word_index(offset); + idx_t size = size_in_words(); + for (idx_t index = 0; index < size; index++) { + dest_map[index] = dest_map[index] | other_map[offset_word_ind + index]; + } +} + bool BitMap::set_union_with_result(BitMap other) { assert(size() == other.size(), "must have same size"); bool changed = false; bm_word_t* dest_map = map(); bm_word_t* other_map = other.map(); idx_t size = size_in_words(); for (idx_t index = 0; index < size; index++) { idx_t temp = map(index) | other_map[index]; @@ -552,21 +567,26 @@ BitMap::idx_t BitMap::num_set_bits(bm_wo return bits; } BitMap::idx_t BitMap::num_set_bits_from_table(unsigned char c) { assert(_pop_count_table != NULL, "precondition"); return _pop_count_table[c]; } -BitMap::idx_t BitMap::count_one_bits() const { +BitMap::idx_t BitMap::count_one_bits(idx_t l_index, idx_t r_index) const { + guarantee((l_index % (sizeof(bm_word_t) * BitsPerByte)) == 0, + "Only handle aligned cases so far."); + guarantee((r_index % (sizeof(bm_word_t) * BitsPerByte)) == 0, + "Only handle aligned cases so far."); + init_pop_count_table(); // If necessary. idx_t sum = 0; typedef unsigned char uchar; - for (idx_t i = 0; i < size_in_words(); i++) { + for (idx_t i = word_index(l_index); i < word_index(r_index); i++) { bm_word_t w = map()[i]; for (size_t j = 0; j < sizeof(bm_word_t); j++) { sum += num_set_bits_from_table(uchar(w & 255)); w >>= 8; } } return sum; } diff -r 102466e70deb src/share/vm/utilities/bitMap.hpp --- a/src/share/vm/utilities/bitMap.hpp Thu Jan 20 15:52:05 2011 -0800 +++ b/src/share/vm/utilities/bitMap.hpp Fri Apr 01 14:13:53 2011 -0700 @@ -248,17 +248,21 @@ class BitMap VALUE_OBJ_CLASS_SPEC { idx_t get_next_one_offset(idx_t offset) const { return get_next_one_offset(offset, size()); } idx_t get_next_zero_offset(idx_t offset) const { return get_next_zero_offset(offset, size()); } // Returns the number of bits set in the bitmap. - idx_t count_one_bits() const; + idx_t count_one_bits(idx_t l_index, idx_t r_index) const; + + idx_t count_one_bits() const { + return count_one_bits(0, size()); + } // Set operations. void set_union(BitMap bits); void set_difference(BitMap bits); void set_intersection(BitMap bits); // Returns true iff "this" is a superset of "bits". bool contains(const BitMap bits) const; // Returns true iff "this and "bits" have a non-empty intersection. @@ -273,16 +277,17 @@ class BitMap VALUE_OBJ_CLASS_SPEC { // Requires the submap of "bits" starting at offset to be at least as // large as "this". Modifies "this" to be the intersection of its // current contents and the submap of "bits" starting at "offset" of the // same length as "this." // (For expedience, currently requires the offset to be aligned to the // bitsize of a uintptr_t. This should go away in the future though it // will probably remain a good case to optimize.) void set_intersection_at_offset(BitMap bits, idx_t offset); + void set_union_at_offset(BitMap bits, idx_t offset); void set_from(BitMap bits); bool is_same(BitMap bits); // Test if all bits are set or cleared bool is_full() const; bool is_empty() const;