8151386: Extract card live data out of G1ConcurrentMark

Move card live data management out of G1ConcurrentMark into extra class G1CardLiveData managed by G1RemSet

Reviewed-by: mgerdin, kbarrett

hotspot/src/share/vm/gc/g1/concurrentMarkThread.cpp

@@ -28,6 +28,7 @@
 #include "gc/g1/g1Analytics.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectorPolicy.hpp"
+#include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1MMUTracker.hpp"
 #include "gc/g1/suspendibleThreadSet.hpp"
 #include "gc/g1/vm_operations_g1.hpp"

hotspot/src/share/vm/gc/g1/g1CardLiveData.cpp

@@ -0,0 +1,552 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1ConcurrentMark.inline.hpp"
+#include "gc/g1/g1CardLiveData.inline.hpp"
+#include "gc/g1/suspendibleThreadSet.hpp"
+#include "gc/shared/workgroup.hpp"
+#include "memory/universe.hpp"
+#include "runtime/atomic.inline.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+G1CardLiveData::G1CardLiveData() :
+  _max_capacity(0),
+  _cards_per_region(0),
+  _live_regions(NULL),
+  _live_regions_size_in_bits(0),
+  _live_cards(NULL),
+  _live_cards_size_in_bits(0) {
+}
+
+G1CardLiveData::~G1CardLiveData()  {
+  free_large_bitmap(_live_cards, _live_cards_size_in_bits);
+  free_large_bitmap(_live_regions, _live_regions_size_in_bits);
+}
+
+G1CardLiveData::bm_word_t* G1CardLiveData::allocate_large_bitmap(size_t size_in_bits) {
+  size_t size_in_words = BitMap::calc_size_in_words(size_in_bits);
+
+  bm_word_t* map = MmapArrayAllocator<bm_word_t, mtGC>::allocate(size_in_words);
+
+  return map;
+}
+
+void G1CardLiveData::free_large_bitmap(bm_word_t* bitmap, size_t size_in_bits) {
+  MmapArrayAllocator<bm_word_t, mtGC>::free(bitmap, size_in_bits / BitsPerWord);
+}
+
+void G1CardLiveData::initialize(size_t max_capacity, uint num_max_regions) {
+  assert(max_capacity % num_max_regions == 0,
+         "Given capacity must be evenly divisible by region size.");
+  size_t region_size = max_capacity / num_max_regions;
+  assert(region_size % (G1SATBCardTableModRefBS::card_size * BitsPerWord) == 0,
+         "Region size must be evenly divisible by area covered by a single word.");
+  _max_capacity = max_capacity;
+  _cards_per_region = region_size / G1SATBCardTableModRefBS::card_size;
+
+  _live_regions_size_in_bits = live_region_bitmap_size_in_bits();
+  _live_regions = allocate_large_bitmap(_live_regions_size_in_bits);
+  _live_cards_size_in_bits = live_card_bitmap_size_in_bits();
+  _live_cards = allocate_large_bitmap(_live_cards_size_in_bits);
+}
+
+void G1CardLiveData::pretouch() {
+  live_cards_bm().pretouch();
+  live_regions_bm().pretouch();
+}
+
+size_t G1CardLiveData::live_region_bitmap_size_in_bits() const {
+  return _max_capacity / (_cards_per_region << G1SATBCardTableModRefBS::card_shift);
+}
+
+size_t G1CardLiveData::live_card_bitmap_size_in_bits() const {
+  return _max_capacity >> G1SATBCardTableModRefBS::card_shift;
+}
+
+// Helper class that provides functionality to generate the Live Data Count
+// information.
+class G1CardLiveDataHelper VALUE_OBJ_CLASS_SPEC {
+private:
+  BitMap _region_bm;
+  BitMap _card_bm;
+
+  // The card number of the bottom of the G1 heap.
+  // Used in biasing indices into accounting card bitmaps.
+  BitMap::idx_t _heap_card_bias;
+
+  // Utility routine to set an exclusive range of bits on the given
+  // bitmap, optimized for very small ranges.
+  // There must be at least one bit to set.
+  void set_card_bitmap_range(BitMap::idx_t start_idx,
+                             BitMap::idx_t end_idx) {
+
+    // Set the exclusive bit range [start_idx, end_idx).
+    assert((end_idx - start_idx) > 0, "at least one bit");
+
+    // For small ranges use a simple loop; otherwise use set_range.
+    // The range is made up of the cards that are spanned by an object/mem
+    // region so 8 cards will allow up to object sizes up to 4K to be handled
+    // using the loop.
+    if ((end_idx - start_idx) <= 8) {
+      for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
+        _card_bm.set_bit(i);
+      }
+    } else {
+      _card_bm.set_range(start_idx, end_idx);
+    }
+  }
+
+  // We cache the last mark set. This avoids setting the same bit multiple times.
+  // This is particularly interesting for dense bitmaps, as this avoids doing
+  // lots of work most of the time.
+  BitMap::idx_t _last_marked_bit_idx;
+
+  // Mark the card liveness bitmap for the object spanning from start to end.
+  void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
+    BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
+    BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
+
+    assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
+
+    if (start_idx == _last_marked_bit_idx) {
+      start_idx++;
+    }
+    if (start_idx == end_idx) {
+      return;
+    }
+
+    // Set the bits in the card bitmap for the cards spanned by this object.
+    set_card_bitmap_range(start_idx, end_idx);
+    _last_marked_bit_idx = end_idx - 1;
+  }
+
+  void reset_mark_cache() {
+    _last_marked_bit_idx = (BitMap::idx_t)-1;
+  }
+
+public:
+  // Returns the index in the per-card liveness count bitmap
+  // for the given address
+  inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
+    // Below, the term "card num" means the result of shifting an address
+    // by the card shift -- address 0 corresponds to card number 0.  One
+    // must subtract the card num of the bottom of the heap to obtain a
+    // card table index.
+    BitMap::idx_t card_num = uintptr_t(addr) >> CardTableModRefBS::card_shift;
+    return card_num - _heap_card_bias;
+  }
+
+  // Takes a region that's not empty (i.e., it has at least one
+  // live object in it and sets its corresponding bit on the region
+  // bitmap to 1.
+  void set_bit_for_region(HeapRegion* hr) {
+    _region_bm.par_set_bit(hr->hrm_index());
+  }
+
+  // Mark the range of bits covered by allocations done since the last marking
+  // in the given heap region, i.e. from NTAMS to top of the given region.
+  // Returns if there has been some allocation in this region since the last marking.
+  bool mark_allocated_since_marking(HeapRegion* hr) {
+    reset_mark_cache();
+
+    HeapWord* ntams = hr->next_top_at_mark_start();
+    HeapWord* top   = hr->top();
+
+    assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
+
+    // Mark the allocated-since-marking portion...
+    if (ntams < top) {
+      mark_card_bitmap_range(ntams, top);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  // Mark the range of bits covered by live objects on the mark bitmap between
+  // bottom and NTAMS of the given region.
+  // Returns the number of live bytes marked within that area for the given
+  // heap region.
+  size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) {
+    reset_mark_cache();
+
+    size_t marked_bytes = 0;
+
+    HeapWord* ntams = hr->next_top_at_mark_start();
+    HeapWord* start = hr->bottom();
+
+    if (ntams <= start) {
+      // Skip empty regions.
+      return 0;
+    }
+    if (hr->is_humongous()) {
+      mark_card_bitmap_range(start, hr->top());
+      return pointer_delta(hr->top(), start, 1);
+    }
+
+    assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
+           "Preconditions not met - "
+           "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
+           p2i(start), p2i(ntams), p2i(hr->end()));
+
+    // Find the first marked object at or after "start".
+    start = mark_bitmap->getNextMarkedWordAddress(start, ntams);
+    while (start < ntams) {
+      oop obj = oop(start);
+      size_t obj_size = obj->size();
+      HeapWord* obj_end = start + obj_size;
+
+      assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
+
+      mark_card_bitmap_range(start, obj_end);
+
+      // Add the size of this object to the number of marked bytes.
+      marked_bytes += obj_size * HeapWordSize;
+
+      // Find the next marked object after this one.
+      start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams);
+    }
+
+    return marked_bytes;
+  }
+
+  G1CardLiveDataHelper(G1CardLiveData* live_data, HeapWord* base_address) :
+    _region_bm(live_data->live_regions_bm()),
+    _card_bm(live_data->live_cards_bm()) {
+    // Calculate the card number for the bottom of the heap. Used
+    // in biasing indexes into the accounting card bitmaps.
+    _heap_card_bias =
+      uintptr_t(base_address) >> CardTableModRefBS::card_shift;
+  }
+};
+
+class G1CreateCardLiveDataTask: public AbstractGangTask {
+  // Aggregate the counting data that was constructed concurrently
+  // with marking.
+  class G1CreateLiveDataClosure : public HeapRegionClosure {
+    G1CardLiveDataHelper _helper;
+
+    G1CMBitMap* _mark_bitmap;
+
+    G1ConcurrentMark* _cm;
+  public:
+    G1CreateLiveDataClosure(G1CollectedHeap* g1h,
+                            G1ConcurrentMark* cm,
+                            G1CMBitMap* mark_bitmap,
+                            G1CardLiveData* live_data) :
+      HeapRegionClosure(),
+      _helper(live_data, g1h->reserved_region().start()),
+      _mark_bitmap(mark_bitmap),
+      _cm(cm) { }
+
+    bool doHeapRegion(HeapRegion* hr) {
+      size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr);
+      if (marked_bytes > 0) {
+        hr->add_to_marked_bytes(marked_bytes);
+      }
+
+      return (_cm->do_yield_check() && _cm->has_aborted());
+    }
+  };
+
+  G1ConcurrentMark* _cm;
+  G1CardLiveData* _live_data;
+  HeapRegionClaimer _hr_claimer;
+
+public:
+  G1CreateCardLiveDataTask(G1CMBitMap* bitmap,
+                           G1CardLiveData* live_data,
+                           uint n_workers) :
+      AbstractGangTask("G1 Create Live Data"),
+      _live_data(live_data),
+      _hr_claimer(n_workers) {
+  }
+
+  void work(uint worker_id) {
+    SuspendibleThreadSetJoiner sts_join;
+
+    G1CollectedHeap* g1h = G1CollectedHeap::heap();
+    G1ConcurrentMark* cm = g1h->concurrent_mark();
+    G1CreateLiveDataClosure cl(g1h, cm, cm->nextMarkBitMap(), _live_data);
+    g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
+  }
+};
+
+void G1CardLiveData::create(WorkGang* workers, G1CMBitMap* mark_bitmap) {
+  uint n_workers = workers->active_workers();
+
+  G1CreateCardLiveDataTask cl(mark_bitmap,
+                              this,
+                              n_workers);
+  workers->run_task(&cl);
+}
+
+class G1FinalizeCardLiveDataTask: public AbstractGangTask {
+  // Finalizes the liveness counting data.
+  // Sets the bits corresponding to the interval [NTAMS, top]
+  // (which contains the implicitly live objects) in the
+  // card liveness bitmap. Also sets the bit for each region
+  // containing live data, in the region liveness bitmap.
+  class G1FinalizeCardLiveDataClosure: public HeapRegionClosure {
+  private:
+    G1CardLiveDataHelper _helper;
+  public:
+    G1FinalizeCardLiveDataClosure(G1CollectedHeap* g1h,
+                                  G1CMBitMap* bitmap,
+                                  G1CardLiveData* live_data) :
+      HeapRegionClosure(),
+      _helper(live_data, g1h->reserved_region().start()) { }
+
+    bool doHeapRegion(HeapRegion* hr) {
+      bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
+      if (allocated_since_marking || hr->next_marked_bytes() > 0) {
+        _helper.set_bit_for_region(hr);
+      }
+      return false;
+    }
+  };
+
+  G1CMBitMap* _bitmap;
+
+  G1CardLiveData* _live_data;
+
+  HeapRegionClaimer _hr_claimer;
+
+public:
+  G1FinalizeCardLiveDataTask(G1CMBitMap* bitmap, G1CardLiveData* live_data, uint n_workers) :
+    AbstractGangTask("G1 Finalize Card Live Data"),
+    _bitmap(bitmap),
+    _live_data(live_data),
+    _hr_claimer(n_workers) {
+  }
+
+  void work(uint worker_id) {
+    G1FinalizeCardLiveDataClosure cl(G1CollectedHeap::heap(), _bitmap, _live_data);
+
+    G1CollectedHeap::heap()->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
+  }
+};
+
+void G1CardLiveData::finalize(WorkGang* workers, G1CMBitMap* mark_bitmap) {
+  // Finalize the live data.
+  G1FinalizeCardLiveDataTask cl(mark_bitmap,
+                                this,
+                                workers->active_workers());
+  workers->run_task(&cl);
+}
+
+class G1ClearCardLiveDataTask : public AbstractGangTask {
+  BitMap _bitmap;
+  size_t _num_chunks;
+  size_t _cur_chunk;
+public:
+  G1ClearCardLiveDataTask(BitMap bitmap, size_t num_tasks) :
+    AbstractGangTask("G1 Clear Card Live Data"),
+    _bitmap(bitmap),
+    _num_chunks(num_tasks),
+    _cur_chunk(0) {
+  }
+
+  static size_t chunk_size() { return M; }
+
+  virtual void work(uint worker_id) {
+    while (true) {
+      size_t to_process = Atomic::add(1, &_cur_chunk) - 1;
+      if (to_process >= _num_chunks) {
+        break;
+      }
+
+      BitMap::idx_t start = M * BitsPerByte * to_process;
+      BitMap::idx_t end = MIN2(start + M * BitsPerByte, _bitmap.size());
+      _bitmap.clear_range(start, end);
+    }
+  }
+};
+
+void G1CardLiveData::clear(WorkGang* workers) {
+  guarantee(Universe::is_fully_initialized(), "Should not call this during initialization.");
+
+  size_t const num_chunks = align_size_up(live_cards_bm().size_in_bytes(), G1ClearCardLiveDataTask::chunk_size()) / G1ClearCardLiveDataTask::chunk_size();
+
+  G1ClearCardLiveDataTask cl(live_cards_bm(), num_chunks);
+  workers->run_task(&cl);
+
+  // The region live bitmap is always very small, even for huge heaps. Clear
+  // directly.
+  live_regions_bm().clear();
+}
+
+class G1VerifyCardLiveDataTask: public AbstractGangTask {
+  // Heap region closure used for verifying the live count data
+  // that was created concurrently and finalized during
+  // the remark pause. This closure is applied to the heap
+  // regions during the STW cleanup pause.
+  class G1VerifyCardLiveDataClosure: public HeapRegionClosure {
+  private:
+    G1CollectedHeap* _g1h;
+    G1CMBitMap* _mark_bitmap;
+    G1CardLiveDataHelper _helper;
+
+    G1CardLiveData* _act_live_data;
+
+    G1CardLiveData* _exp_live_data;
+
+    int _failures;
+
+    // Completely recreates the live data count for the given heap region and
+    // returns the number of bytes marked.
+    size_t create_live_data_count(HeapRegion* hr) {
+      size_t bytes_marked = _helper.mark_marked_during_marking(_mark_bitmap, hr);
+      bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
+      if (allocated_since_marking || bytes_marked > 0) {
+        _helper.set_bit_for_region(hr);
+      }
+      return bytes_marked;
+    }
+  public:
+    G1VerifyCardLiveDataClosure(G1CollectedHeap* g1h,
+                                G1CMBitMap* mark_bitmap,
+                                G1CardLiveData* act_live_data,
+                                G1CardLiveData* exp_live_data) :
+      _g1h(g1h),
+      _mark_bitmap(mark_bitmap),
+      _helper(exp_live_data, g1h->reserved_region().start()),
+      _act_live_data(act_live_data),
+      _exp_live_data(exp_live_data),
+      _failures(0) { }
+
+    int failures() const { return _failures; }
+
+    bool doHeapRegion(HeapRegion* hr) {
+      int failures = 0;
+
+      // Walk the marking bitmap for this region and set the corresponding bits
+      // in the expected region and card bitmaps.
+      size_t exp_marked_bytes = create_live_data_count(hr);
+      size_t act_marked_bytes = hr->next_marked_bytes();
+      // Verify the marked bytes for this region.
+
+      if (exp_marked_bytes != act_marked_bytes) {
+        failures += 1;
+      } else if (exp_marked_bytes > HeapRegion::GrainBytes) {
+        failures += 1;
+      }
+
+      // Verify the bit, for this region, in the actual and expected
+      // (which was just calculated) region bit maps.
+      // We're not OK if the bit in the calculated expected region
+      // bitmap is set and the bit in the actual region bitmap is not.
+      uint index = hr->hrm_index();
+
+      bool expected = _exp_live_data->is_region_live(index);
+      bool actual = _act_live_data->is_region_live(index);
+      if (expected && !actual) {
+        failures += 1;
+      }
+
+      // Verify that the card bit maps for the cards spanned by the current
+      // region match. We have an error if we have a set bit in the expected
+      // bit map and the corresponding bit in the actual bitmap is not set.
+
+      BitMap::idx_t start_idx = _helper.card_live_bitmap_index_for(hr->bottom());
+      BitMap::idx_t end_idx = _helper.card_live_bitmap_index_for(hr->top());
+
+      for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
+        expected = _exp_live_data->is_card_live_at(i);
+        actual = _act_live_data->is_card_live_at(i);
+
+        if (expected && !actual) {
+          failures += 1;
+        }
+      }
+
+      _failures += failures;
+
+      // We could stop iteration over the heap when we
+      // find the first violating region by returning true.
+      return false;
+    }
+  };
+protected:
+  G1CollectedHeap* _g1h;
+  G1CMBitMap* _mark_bitmap;
+
+  G1CardLiveData* _act_live_data;
+
+  G1CardLiveData _exp_live_data;
+
+  int  _failures;
+
+  HeapRegionClaimer _hr_claimer;
+
+public:
+  G1VerifyCardLiveDataTask(G1CMBitMap* bitmap,
+                           G1CardLiveData* act_live_data,
+                           uint n_workers)
+  : AbstractGangTask("G1 Verify Card Live Data"),
+    _g1h(G1CollectedHeap::heap()),
+    _mark_bitmap(bitmap),
+    _act_live_data(act_live_data),
+    _exp_live_data(),
+    _failures(0),
+    _hr_claimer(n_workers) {
+    assert(VerifyDuringGC, "don't call this otherwise");
+    _exp_live_data.initialize(_g1h->max_capacity(), _g1h->max_regions());
+  }
+
+  void work(uint worker_id) {
+    G1VerifyCardLiveDataClosure cl(_g1h,
+                                   _mark_bitmap,
+                                   _act_live_data,
+                                   &_exp_live_data);
+    _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
+
+    Atomic::add(cl.failures(), &_failures);
+  }
+
+  int failures() const { return _failures; }
+};
+
+void G1CardLiveData::verify(WorkGang* workers, G1CMBitMap* actual_bitmap) {
+    ResourceMark rm;
+
+    G1VerifyCardLiveDataTask cl(actual_bitmap,
+                                this,
+                                workers->active_workers());
+    workers->run_task(&cl);
+
+    guarantee(cl.failures() == 0, "Unexpected accounting failures");
+}
+
+#ifndef PRODUCT
+void G1CardLiveData::verify_is_clear() {
+  assert(live_cards_bm().count_one_bits() == 0, "Live cards bitmap must be clear.");
+  assert(live_regions_bm().count_one_bits() == 0, "Live regions bitmap must be clear.");
+}
+#endif

hotspot/src/share/vm/gc/g1/g1CardLiveData.hpp

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP
+#define SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP
+
+#include "gc/g1/g1CollectedHeap.hpp"
+#include "utilities/bitMap.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class G1CollectedHeap;
+class G1CMBitMap;
+class WorkGang;
+
+// Information about object liveness on the Java heap on a "card" basis.
+// Can be used for various purposes, like as remembered set for completely
+// coarsened remembered sets, scrubbing remembered sets or estimating liveness.
+// This information is created as part of the concurrent marking cycle.
+class G1CardLiveData VALUE_OBJ_CLASS_SPEC {
+  friend class G1CardLiveDataHelper;
+  friend class G1VerifyCardLiveDataTask;
+private:
+  typedef BitMap::bm_word_t bm_word_t;
+  // Store some additional information about the covered area to be able to test.
+  size_t _max_capacity;
+  size_t _cards_per_region;
+
+  // The per-card liveness bitmap.
+  bm_word_t* _live_cards;
+  size_t _live_cards_size_in_bits;
+  // The per-region liveness bitmap.
+  bm_word_t* _live_regions;
+  size_t _live_regions_size_in_bits;
+  // The bits in this bitmap contain for every card whether it contains
+  // at least part of at least one live object.
+  BitMap live_cards_bm() const { return BitMap(_live_cards, _live_cards_size_in_bits); }
+  // The bits in this bitmap indicate that a given region contains some live objects.
+  BitMap live_regions_bm() const { return BitMap(_live_regions, _live_regions_size_in_bits); }
+
+  // Allocate a "large" bitmap from virtual memory with the given size in bits.
+  bm_word_t* allocate_large_bitmap(size_t size_in_bits);
+  void free_large_bitmap(bm_word_t* map, size_t size_in_bits);
+
+  inline BitMap live_card_bitmap(uint region);
+
+  inline bool is_card_live_at(BitMap::idx_t idx) const;
+
+  size_t live_region_bitmap_size_in_bits() const;
+  size_t live_card_bitmap_size_in_bits() const;
+public:
+  inline bool is_region_live(uint region) const;
+
+  inline void remove_nonlive_cards(uint region, BitMap* bm);
+  inline void remove_nonlive_regions(BitMap* bm);
+
+  G1CardLiveData();
+  ~G1CardLiveData();
+
+  void initialize(size_t max_capacity, uint num_max_regions);
+  void pretouch();
+
+  // Create the initial liveness data based on the marking result from the bottom
+  // to the ntams of every region in the heap and the marks in the given bitmap.
+  void create(WorkGang* workers, G1CMBitMap* mark_bitmap);
+  // Finalize the liveness data.
+  void finalize(WorkGang* workers, G1CMBitMap* mark_bitmap);
+
+  // Verify that the liveness count data created concurrently matches one created
+  // during this safepoint.
+  void verify(WorkGang* workers, G1CMBitMap* actual_bitmap);
+  // Clear all data structures, prepare for next processing.
+  void clear(WorkGang* workers);
+
+  void verify_is_clear() PRODUCT_RETURN;
+};
+
+#endif /* SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP */
+

hotspot/src/share/vm/gc/g1/g1CardLiveData.inline.hpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP
+#define SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP
+
+#include "gc/g1/g1CardLiveData.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+inline BitMap G1CardLiveData::live_card_bitmap(uint region) {
+  return BitMap(_live_cards + ((size_t)region * _cards_per_region >> LogBitsPerWord), _cards_per_region);
+}
+
+inline bool G1CardLiveData::is_card_live_at(BitMap::idx_t idx) const {
+  return live_cards_bm().at(idx);
+}
+
+inline bool G1CardLiveData::is_region_live(uint region) const {
+  return live_regions_bm().at(region);
+}
+
+inline void G1CardLiveData::remove_nonlive_cards(uint region, BitMap* bm) {
+  bm->set_intersection(live_card_bitmap(region));
+}
+
+inline void G1CardLiveData::remove_nonlive_regions(BitMap* bm) {
+  bm->set_intersection(live_regions_bm());
+}
+
+#endif /* SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP */

hotspot/src/share/vm/gc/g1/g1CollectedHeap.cpp

@@ -1425,6 +1425,7 @@ bool G1CollectedHeap::do_full_collection(bool explicit_gc,
       // the full GC has compacted objects and updated TAMS but not updated
       // the prev bitmap.
       if (G1VerifyBitmaps) {
+        GCTraceTime(Debug, gc)("Clear Bitmap for Verification");
         _cm->clear_prev_bitmap(workers());
       }
       _verifier->check_bitmaps("Full GC End");
@@ -1944,7 +1945,7 @@ jint G1CollectedHeap::initialize() {
   const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;
   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");
 
-  G1RemSet::initialize(max_regions());
+  g1_rem_set()->initialize(max_capacity(), max_regions());
 
   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;
   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");
@@ -4787,27 +4788,23 @@ public:
 class G1ParScrubRemSetTask: public AbstractGangTask {
 protected:
   G1RemSet* _g1rs;
-  BitMap* _region_bm;
-  BitMap* _card_bm;
   HeapRegionClaimer _hrclaimer;
 
 public:
-  G1ParScrubRemSetTask(G1RemSet* g1_rs, BitMap* region_bm, BitMap* card_bm, uint num_workers) :
+  G1ParScrubRemSetTask(G1RemSet* g1_rs, uint num_workers) :
     AbstractGangTask("G1 ScrubRS"),
     _g1rs(g1_rs),
-    _region_bm(region_bm),
-    _card_bm(card_bm),
     _hrclaimer(num_workers) {
   }
 
   void work(uint worker_id) {
-    _g1rs->scrub(_region_bm, _card_bm, worker_id, &_hrclaimer);
+    _g1rs->scrub(worker_id, &_hrclaimer);
   }
 };
 
-void G1CollectedHeap::scrub_rem_set(BitMap* region_bm, BitMap* card_bm) {
+void G1CollectedHeap::scrub_rem_set() {
   uint num_workers = workers()->active_workers();
-  G1ParScrubRemSetTask g1_par_scrub_rs_task(g1_rem_set(), region_bm, card_bm, num_workers);
+  G1ParScrubRemSetTask g1_par_scrub_rs_task(g1_rem_set(), num_workers);
   workers()->run_task(&g1_par_scrub_rs_task);
 }
 

hotspot/src/share/vm/gc/g1/g1CollectedHeap.hpp

@@ -992,7 +992,8 @@ public:
   // The rem set and barrier set.
   G1RemSet* g1_rem_set() const { return _g1_rem_set; }
 
-  void scrub_rem_set(BitMap* region_bm, BitMap* card_bm);
+  // Try to minimize the remembered set.
+  void scrub_rem_set();
 
   unsigned get_gc_time_stamp() {
     return _gc_time_stamp;

hotspot/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp

@@ -28,7 +28,7 @@
 #include "gc/g1/g1CollectedHeap.hpp"
 #include "gc/g1/g1CollectorPolicy.hpp"
 #include "gc/g1/g1CollectorState.hpp"
-#include "gc/g1/g1ConcurrentMark.hpp"
+#include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"

hotspot/src/share/vm/gc/g1/g1ConcurrentMark.cpp

@@ -33,6 +33,7 @@
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/g1/g1OopClosures.inline.hpp"
+#include "gc/g1/g1CardLiveData.inline.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionRemSet.hpp"
@@ -48,7 +49,6 @@
 #include "gc/shared/taskqueue.inline.hpp"
 #include "gc/shared/vmGCOperations.hpp"
 #include "logging/log.hpp"
-#include "logging/logTag.hpp"
 #include "memory/allocation.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/oop.inline.hpp"
@@ -356,8 +356,6 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper*
   _sleep_factor(0.0),
   _marking_task_overhead(1.0),
   _cleanup_list("Cleanup List"),
-  _region_live_bm(),
-  _card_live_bm(),
 
   _prevMarkBitMap(&_markBitMap1),
   _nextMarkBitMap(&_markBitMap2),
@@ -499,12 +497,6 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper*
     return;
   }
 
-  allocate_internal_bitmaps();
-
-  if (G1PretouchAuxiliaryMemory) {
-    pretouch_internal_bitmaps();
-  }
-
   _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_worker_id, mtGC);
   _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
 
@@ -701,8 +693,8 @@ void G1ConcurrentMark::cleanup_for_next_mark() {
   // Clear the live count data. If the marking has been aborted, the abort()
   // call already did that.
   if (!has_aborted()) {
-    clear_all_live_data(_parallel_workers);
+    clear_live_data(_parallel_workers);
-    DEBUG_ONLY(verify_all_live_data());
+    DEBUG_ONLY(verify_live_data_clear());
   }
 
   // Repeat the asserts from above.
@@ -884,7 +876,7 @@ public:
           double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
           _cm->clear_has_overflown();
 
-          _cm->do_yield_check(worker_id);
+          _cm->do_yield_check();
 
           jlong sleep_time_ms;
           if (!_cm->has_aborted() && the_task->has_aborted()) {
@@ -934,10 +926,10 @@ uint G1ConcurrentMark::calc_parallel_marking_threads() {
   return n_conc_workers;
 }
 
-void G1ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) {
+void G1ConcurrentMark::scanRootRegion(HeapRegion* hr) {
   // Currently, only survivors can be root regions.
   assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
-  G1RootRegionScanClosure cl(_g1h, this, worker_id);
+  G1RootRegionScanClosure cl(_g1h, this);
 
   const uintx interval = PrefetchScanIntervalInBytes;
   HeapWord* curr = hr->bottom();
@@ -966,7 +958,7 @@ public:
     G1CMRootRegions* root_regions = _cm->root_regions();
     HeapRegion* hr = root_regions->claim_next();
     while (hr != NULL) {
-      _cm->scanRootRegion(hr, worker_id);
+      _cm->scanRootRegion(hr);
       hr = root_regions->claim_next();
     }
   }
@@ -1125,363 +1117,6 @@ void G1ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
   _gc_tracer_cm->report_object_count_after_gc(&is_alive);
 }
 
-// Helper class that provides functionality to generate the Live Data Count
-// information.
-class G1LiveDataHelper VALUE_OBJ_CLASS_SPEC {
-private:
-  BitMap* _region_bm;
-  BitMap* _card_bm;
-
-  // The card number of the bottom of the G1 heap. Used for converting addresses
-  // to bitmap indices quickly.
-  BitMap::idx_t _heap_card_bias;
-
-  // Utility routine to set an exclusive range of bits on the given
-  // bitmap, optimized for very small ranges.
-  // There must be at least one bit to set.
-  inline void set_card_bitmap_range(BitMap* bm,
-                                    BitMap::idx_t start_idx,
-                                    BitMap::idx_t end_idx) {
-
-    // Set the exclusive bit range [start_idx, end_idx).
-    assert((end_idx - start_idx) > 0, "at least one bit");
-    assert(end_idx <= bm->size(), "sanity");
-
-    // For small ranges use a simple loop; otherwise use set_range or
-    // use par_at_put_range (if parallel). The range is made up of the
-    // cards that are spanned by an object/mem region so 8 cards will
-    // allow up to object sizes up to 4K to be handled using the loop.
-    if ((end_idx - start_idx) <= 8) {
-      for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
-        bm->set_bit(i);
-      }
-    } else {
-      bm->set_range(start_idx, end_idx);
-    }
-  }
-
-  // We cache the last mark set. This avoids setting the same bit multiple times.
-  // This is particularly interesting for dense bitmaps, as this avoids doing
-  // lots of work most of the time.
-  BitMap::idx_t _last_marked_bit_idx;
-
-  // Mark the card liveness bitmap for the object spanning from start to end.
-  void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
-    BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
-    BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
-
-    assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
-
-    if (start_idx == _last_marked_bit_idx) {
-      start_idx++;
-    }
-    if (start_idx == end_idx) {
-      return;
-    }
-
-    // Set the bits in the card bitmap for the cards spanned by this object.
-    set_card_bitmap_range(_card_bm, start_idx, end_idx);
-    _last_marked_bit_idx = end_idx - 1;
-  }
-
-  void reset_mark_cache() {
-    _last_marked_bit_idx = (BitMap::idx_t)-1;
-  }
-
-public:
-  // Returns the index in the per-card liveness count bitmap
-  // for the given address
-  inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
-    // Below, the term "card num" means the result of shifting an address
-    // by the card shift -- address 0 corresponds to card number 0.  One
-    // must subtract the card num of the bottom of the heap to obtain a
-    // card table index.
-    BitMap::idx_t card_num = (BitMap::idx_t)(uintptr_t(addr) >> CardTableModRefBS::card_shift);
-    return card_num - _heap_card_bias;
-  }
-
-  // Takes a region that's not empty (i.e., it has at least one
-  // live object in it and sets its corresponding bit on the region
-  // bitmap to 1.
-  void set_bit_for_region(HeapRegion* hr) {
-    BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
-    _region_bm->par_at_put(index, true);
-  }
-
-  // Mark the range of bits covered by allocations done since the last marking
-  // in the given heap region, i.e. from NTAMS to top of the given region.
-  // Returns if there has been some allocation in this region since the last marking.
-  bool mark_allocated_since_marking(HeapRegion* hr) {
-    reset_mark_cache();
-
-    HeapWord* ntams = hr->next_top_at_mark_start();
-    HeapWord* top   = hr->top();
-
-    assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
-
-    // Mark the allocated-since-marking portion...
-    if (ntams < top) {
-      mark_card_bitmap_range(ntams, top);
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  // Mark the range of bits covered by live objects on the mark bitmap between
-  // bottom and NTAMS of the given region.
-  // Returns the number of live bytes marked within that area for the given
-  // heap region.
-  size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) {
-    reset_mark_cache();
-
-    size_t marked_bytes = 0;
-
-    HeapWord* ntams = hr->next_top_at_mark_start();
-    HeapWord* start = hr->bottom();
-
-    if (ntams <= start) {
-      // Skip empty regions.
-      return 0;
-    } else if (hr->is_humongous()) {
-      mark_card_bitmap_range(start, hr->top());
-      return pointer_delta(hr->top(), start, 1);
-    }
-
-    assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
-           "Preconditions not met - "
-           "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
-           p2i(start), p2i(ntams), p2i(hr->end()));
-
-    // Find the first marked object at or after "start".
-    start = mark_bitmap->getNextMarkedWordAddress(start, ntams);
-    while (start < ntams) {
-      oop obj = oop(start);
-      int obj_sz = obj->size();
-      HeapWord* obj_end = start + obj_sz;
-
-      assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
-
-      mark_card_bitmap_range(start, obj_end);
-
-      // Add the size of this object to the number of marked bytes.
-      marked_bytes += (size_t)obj_sz * HeapWordSize;
-
-      // Find the next marked object after this one.
-      start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams);
-    }
-
-    return marked_bytes;
-  }
-
-  G1LiveDataHelper(BitMap* region_bm,
-                   BitMap* card_bm):
-    _region_bm(region_bm),
-    _card_bm(card_bm) {
-    //assert(region_bm != NULL, "");
-    assert(card_bm != NULL, "");
-    // Calculate the card number for the bottom of the heap. Used
-    // in biasing indexes into the accounting card bitmaps.
-    _heap_card_bias =
-      (BitMap::idx_t)(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >> CardTableModRefBS::card_shift);
-  }
-};
-
-// Heap region closure used for verifying the live count data
-// that was created concurrently and finalized during
-// the remark pause. This closure is applied to the heap
-// regions during the STW cleanup pause.
-class G1VerifyLiveDataHRClosure: public HeapRegionClosure {
-private:
-  G1CollectedHeap* _g1h;
-  G1CMBitMap* _mark_bitmap;
-  G1LiveDataHelper _calc_helper;
-
-  BitMap* _act_region_bm; // Region BM to be verified
-  BitMap* _act_card_bm;   // Card BM to be verified
-
-  BitMap* _exp_region_bm; // Expected Region BM values
-  BitMap* _exp_card_bm;   // Expected card BM values
-
-  int _failures;
-
-  // Updates the live data count for the given heap region and returns the number
-  // of bytes marked.
-  size_t create_live_data_count(HeapRegion* hr) {
-    size_t bytes_marked = _calc_helper.mark_marked_during_marking(_mark_bitmap, hr);
-    bool allocated_since_marking = _calc_helper.mark_allocated_since_marking(hr);
-    if (allocated_since_marking || bytes_marked > 0) {
-      _calc_helper.set_bit_for_region(hr);
-    }
-    return bytes_marked;
-  }
-
-public:
-  G1VerifyLiveDataHRClosure(G1CollectedHeap* g1h,
-                            G1CMBitMap* mark_bitmap,
-                            BitMap* act_region_bm,
-                            BitMap* act_card_bm,
-                            BitMap* exp_region_bm,
-                            BitMap* exp_card_bm) :
-    _g1h(g1h),
-    _mark_bitmap(mark_bitmap),
-    _calc_helper(exp_region_bm, exp_card_bm),
-    _act_region_bm(act_region_bm),
-    _act_card_bm(act_card_bm),
-    _exp_region_bm(exp_region_bm),
-    _exp_card_bm(exp_card_bm),
-    _failures(0) { }
-
-  int failures() const { return _failures; }
-
-  bool doHeapRegion(HeapRegion* hr) {
-    int failures = 0;
-
-    // Walk the marking bitmap for this region and set the corresponding bits
-    // in the expected region and card bitmaps.
-    size_t exp_marked_bytes = create_live_data_count(hr);
-    size_t act_marked_bytes = hr->next_marked_bytes();
-    // Verify the marked bytes for this region.
-
-    if (exp_marked_bytes != act_marked_bytes) {
-      failures += 1;
-    } else if (exp_marked_bytes > HeapRegion::GrainBytes) {
-      failures += 1;
-    }
-
-    // Verify the bit, for this region, in the actual and expected
-    // (which was just calculated) region bit maps.
-    // We're not OK if the bit in the calculated expected region
-    // bitmap is set and the bit in the actual region bitmap is not.
-    BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
-
-    bool expected = _exp_region_bm->at(index);
-    bool actual = _act_region_bm->at(index);
-    if (expected && !actual) {
-      failures += 1;
-    }
-
-    // Verify that the card bit maps for the cards spanned by the current
-    // region match. We have an error if we have a set bit in the expected
-    // bit map and the corresponding bit in the actual bitmap is not set.
-
-    BitMap::idx_t start_idx = _calc_helper.card_live_bitmap_index_for(hr->bottom());
-    BitMap::idx_t end_idx = _calc_helper.card_live_bitmap_index_for(hr->top());
-
-    for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
-      expected = _exp_card_bm->at(i);
-      actual = _act_card_bm->at(i);
-
-      if (expected && !actual) {
-        failures += 1;
-      }
-    }
-
-    _failures += failures;
-
-    // We could stop iteration over the heap when we
-    // find the first violating region by returning true.
-    return false;
-  }
-};
-
-class G1VerifyLiveDataTask: public AbstractGangTask {
-protected:
-  G1CollectedHeap* _g1h;
-  G1CMBitMap* _mark_bitmap;
-  BitMap* _actual_region_bm;
-  BitMap* _actual_card_bm;
-
-  BitMap _expected_region_bm;
-  BitMap _expected_card_bm;
-
-  int  _failures;
-
-  HeapRegionClaimer _hr_claimer;
-
-public:
-  G1VerifyLiveDataTask(G1CollectedHeap* g1h,
-                       G1CMBitMap* bitmap,
-                       BitMap* region_bm,
-                       BitMap* card_bm,
-                       uint n_workers)
-  : AbstractGangTask("G1 verify final counting"),
-    _g1h(g1h),
-    _mark_bitmap(bitmap),
-    _actual_region_bm(region_bm),
-    _actual_card_bm(card_bm),
-    _expected_region_bm(region_bm->size(), true /* in_resource_area */),
-    _expected_card_bm(card_bm->size(), true /* in_resource_area */),
-    _failures(0),
-    _hr_claimer(n_workers) {
-    assert(VerifyDuringGC, "don't call this otherwise");
-  }
-
-  void work(uint worker_id) {
-    G1VerifyLiveDataHRClosure cl(_g1h,
-                                 _mark_bitmap,
-                                 _actual_region_bm,
-                                 _actual_card_bm,
-                                 &_expected_region_bm,
-                                 &_expected_card_bm);
-    _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
-
-    Atomic::add(cl.failures(), &_failures);
-  }
-
-  int failures() const { return _failures; }
-};
-
-class G1FinalizeLiveDataTask: public AbstractGangTask {
-  // Finalizes the liveness counting data.
-  // Sets the bits corresponding to the interval [NTAMS, top]
-  // (which contains the implicitly live objects) in the
-  // card liveness bitmap. Also sets the bit for each region
-  // containing live data, in the region liveness bitmap.
-  class G1FinalizeCountDataClosure: public HeapRegionClosure {
-  private:
-    G1LiveDataHelper _helper;
-  public:
-    G1FinalizeCountDataClosure(G1CMBitMap* bitmap,
-                               BitMap* region_bm,
-                               BitMap* card_bm) :
-      HeapRegionClosure(),
-      _helper(region_bm, card_bm) { }
-
-    bool doHeapRegion(HeapRegion* hr) {
-      bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
-      if (allocated_since_marking || hr->next_marked_bytes() > 0) {
-        _helper.set_bit_for_region(hr);
-      }
-      return false;
-    }
-  };
-
-  G1CMBitMap* _bitmap;
-
-  BitMap* _actual_region_bm;
-  BitMap* _actual_card_bm;
-
-  HeapRegionClaimer _hr_claimer;
-
-public:
-  G1FinalizeLiveDataTask(G1CMBitMap* bitmap, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
-    AbstractGangTask("G1 final counting"),
-    _bitmap(bitmap),
-    _actual_region_bm(region_bm),
-    _actual_card_bm(card_bm),
-    _hr_claimer(n_workers) {
-  }
-
-  void work(uint worker_id) {
-    G1FinalizeCountDataClosure cl(_bitmap,
-                                  _actual_region_bm,
-                                  _actual_card_bm);
-
-    G1CollectedHeap::heap()->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
-  }
-};
-
 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
   G1CollectedHeap* _g1;
   size_t _freed_bytes;
@@ -1613,26 +1248,13 @@ void G1ConcurrentMark::cleanup() {
   HeapRegionRemSet::reset_for_cleanup_tasks();
 
   {
-    // Finalize the live data.
+    GCTraceTime(Debug, gc)("Finalize Live Data");
-    G1FinalizeLiveDataTask cl(_nextMarkBitMap,
+    finalize_live_data();
-                              &_region_live_bm,
-                              &_card_live_bm,
-                              g1h->workers()->active_workers());
-    g1h->workers()->run_task(&cl);
   }
 
   if (VerifyDuringGC) {
-    // Verify that the liveness count data created concurrently matches one created
+    GCTraceTime(Debug, gc)("Verify Live Data");
-    // during this safepoint.
+    verify_live_data();
-    ResourceMark rm;
-    G1VerifyLiveDataTask cl(G1CollectedHeap::heap(),
-                            _nextMarkBitMap,
-                            &_region_live_bm,
-                            &_card_live_bm,
-                            g1h->workers()->active_workers());
-    g1h->workers()->run_task(&cl);
-
-    guarantee(cl.failures() == 0, "Unexpected accounting failures");
   }
 
   g1h->collector_state()->set_mark_in_progress(false);
@@ -1669,7 +1291,7 @@ void G1ConcurrentMark::cleanup() {
   // regions.
   if (G1ScrubRemSets) {
     double rs_scrub_start = os::elapsedTime();
-    g1h->scrub_rem_set(&_region_live_bm, &_card_live_bm);
+    g1h->scrub_rem_set();
     _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
   }
 
@@ -2115,35 +1737,6 @@ void G1ConcurrentMark::swapMarkBitMaps() {
   _nextMarkBitMap    = (G1CMBitMap*)  temp;
 }
 
-BitMap G1ConcurrentMark::allocate_large_bitmap(BitMap::idx_t size_in_bits) {
-  size_t size_in_words = BitMap::size_in_words(size_in_bits);
-
-  BitMap::bm_word_t* map = MmapArrayAllocator<BitMap::bm_word_t, mtGC>::allocate(size_in_words);
-
-  return BitMap(map, size_in_bits);
-}
-
-void G1ConcurrentMark::allocate_internal_bitmaps() {
-  double start_time = os::elapsedTime();
-
-  _region_live_bm = allocate_large_bitmap(_g1h->max_regions());
-
-  guarantee(_g1h->max_capacity() % CardTableModRefBS::card_size == 0,
-            "Heap capacity must be aligned to card size.");
-  _card_live_bm = allocate_large_bitmap(_g1h->max_capacity() / CardTableModRefBS::card_size);
-
-  log_debug(gc, marking)("Allocating internal bitmaps took %1.2f seconds.", os::elapsedTime() - start_time);
-}
-
-void G1ConcurrentMark::pretouch_internal_bitmaps() {
-  double start_time = os::elapsedTime();
-
-  _region_live_bm.pretouch();
-  _card_live_bm.pretouch();
-
-  log_debug(gc, marking)("Pre-touching internal bitmaps took %1.2f seconds.", os::elapsedTime() - start_time);
-}
-
 // Closure for marking entries in SATB buffers.
 class G1CMSATBBufferClosure : public SATBBufferClosure {
 private:
@@ -2403,120 +1996,28 @@ void G1ConcurrentMark::verify_no_cset_oops() {
   }
 }
 #endif // PRODUCT
-
-class G1CreateLiveDataTask: public AbstractGangTask {
-  // Aggregate the counting data that was constructed concurrently
-  // with marking.
-  class G1CreateLiveDataHRClosure: public HeapRegionClosure {
-    G1LiveDataHelper _helper;
-
-    G1CMBitMap* _mark_bitmap;
-
-    G1ConcurrentMark* _cm;
-  public:
-    G1CreateLiveDataHRClosure(G1ConcurrentMark* cm,
-                              G1CMBitMap* mark_bitmap,
-                              BitMap* cm_card_bm) :
-      HeapRegionClosure(),
-      _helper(NULL, cm_card_bm),
-      _mark_bitmap(mark_bitmap),
-      _cm(cm) { }
-
-    bool doHeapRegion(HeapRegion* hr) {
-      size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr);
-      if (marked_bytes > 0) {
-        hr->add_to_marked_bytes(marked_bytes);
-      }
-
-      if (_cm->do_yield_check() && _cm->has_aborted()) {
-        return true;
-      }
-      return false;
-    }
-  };
-
-  G1CollectedHeap* _g1h;
-  G1ConcurrentMark* _cm;
-  BitMap* _cm_card_bm;
-  HeapRegionClaimer _hr_claimer;
-
-public:
-  G1CreateLiveDataTask(G1CollectedHeap* g1h,
-                       BitMap* cm_card_bm,
-                       uint n_workers) :
-      AbstractGangTask("Create Live Data"),
-      _g1h(g1h),
-      _cm_card_bm(cm_card_bm),
-      _hr_claimer(n_workers) {
-  }
-
-  void work(uint worker_id) {
-    SuspendibleThreadSetJoiner sts_join;
-
-    G1CreateLiveDataHRClosure cl(_g1h->concurrent_mark(), _g1h->concurrent_mark()->nextMarkBitMap(), _cm_card_bm);
-    _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
-  }
-};
-
-
 void G1ConcurrentMark::create_live_data() {
-  uint n_workers = _parallel_workers->active_workers();
+  _g1h->g1_rem_set()->create_card_live_data(_parallel_workers, _nextMarkBitMap);
-
-  G1CreateLiveDataTask cl(_g1h,
-                          &_card_live_bm,
-                          n_workers);
-  _parallel_workers->run_task(&cl);
 }
 
-class G1ClearAllLiveDataTask : public AbstractGangTask {
+void G1ConcurrentMark::finalize_live_data() {
-  BitMap* _bitmap;
+  _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _nextMarkBitMap);
-  size_t _num_tasks;
-  size_t _cur_task;
-public:
-  G1ClearAllLiveDataTask(BitMap* bitmap, size_t num_tasks) :
-    AbstractGangTask("Clear All Live Data"),
-    _bitmap(bitmap),
-    _num_tasks(num_tasks),
-    _cur_task(0) {
-  }
-
-  virtual void work(uint worker_id) {
-    while (true) {
-      size_t to_process = Atomic::add(1, &_cur_task) - 1;
-      if (to_process >= _num_tasks) {
-        break;
-      }
-
-      BitMap::idx_t start = M * BitsPerByte * to_process;
-      BitMap::idx_t end = MIN2(start + M * BitsPerByte, _bitmap->size());
-      _bitmap->clear_range(start, end);
-    }
-  }
-};
-
-void G1ConcurrentMark::clear_all_live_data(WorkGang* workers) {
-  double start_time = os::elapsedTime();
-
-  guarantee(Universe::is_fully_initialized(), "Should not call this during initialization.");
-
-  size_t const num_chunks = align_size_up(_card_live_bm.size_in_words() * HeapWordSize, M) / M;
-
-  G1ClearAllLiveDataTask cl(&_card_live_bm, num_chunks);
-  workers->run_task(&cl);
-
-  // The region live bitmap is always very small, even for huge heaps. Clear
-  // directly.
-  _region_live_bm.clear();
-
-
-  log_debug(gc, marking)("Clear Live Data took %.3fms", (os::elapsedTime() - start_time) * 1000.0);
 }
 
-void G1ConcurrentMark::verify_all_live_data() {
+void G1ConcurrentMark::verify_live_data() {
-  assert(_card_live_bm.count_one_bits() == 0, "Master card bitmap not clear");
+  _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _nextMarkBitMap);
-  assert(_region_live_bm.count_one_bits() == 0, "Master region bitmap not clear");
 }
 
+void G1ConcurrentMark::clear_live_data(WorkGang* workers) {
+  _g1h->g1_rem_set()->clear_card_live_data(workers);
+}
+
+#ifdef ASSERT
+void G1ConcurrentMark::verify_live_data_clear() {
+  _g1h->g1_rem_set()->verify_card_live_data_is_clear();
+}
+#endif
+
 void G1ConcurrentMark::print_stats() {
   if (!log_is_enabled(Debug, gc, stats)) {
     return;
@@ -2536,14 +2037,22 @@ void G1ConcurrentMark::abort() {
 
   // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
   // concurrent bitmap clearing.
-  clear_bitmap(_nextMarkBitMap, _g1h->workers(), false);
+  {
-
+    GCTraceTime(Debug, gc)("Clear Next Bitmap");
+    clear_bitmap(_nextMarkBitMap, _g1h->workers(), false);
+  }
   // Note we cannot clear the previous marking bitmap here
   // since VerifyDuringGC verifies the objects marked during
   // a full GC against the previous bitmap.
 
-  clear_all_live_data(_g1h->workers());
+  {
-  DEBUG_ONLY(verify_all_live_data());
+    GCTraceTime(Debug, gc)("Clear Live Data");
+    clear_live_data(_g1h->workers());
+  }
+  DEBUG_ONLY({
+    GCTraceTime(Debug, gc)("Verify Live Data Clear");
+    verify_live_data_clear();
+  })
   // Empty mark stack
   reset_marking_state();
   for (uint i = 0; i < _max_worker_id; ++i) {
@@ -2610,16 +2119,6 @@ void G1ConcurrentMark::print_on_error(outputStream* st) const {
   _nextMarkBitMap->print_on_error(st, " Next Bits: ");
 }
 
-// We take a break if someone is trying to stop the world.
-bool G1ConcurrentMark::do_yield_check(uint worker_id) {
-  if (SuspendibleThreadSet::should_yield()) {
-    SuspendibleThreadSet::yield();
-    return true;
-  } else {
-    return false;
-  }
-}
-
 // Closure for iteration over bitmaps
 class G1CMBitMapClosure : public BitMapClosure {
 private:

hotspot/src/share/vm/gc/g1/g1ConcurrentMark.hpp

@@ -298,15 +298,6 @@ protected:
   G1CMBitMapRO*           _prevMarkBitMap; // Completed mark bitmap
   G1CMBitMap*             _nextMarkBitMap; // Under-construction mark bitmap
 
-  // Liveness count data. After marking G1 iterates over the recently gathered mark
-  // bitmap and records rough information about liveness on card and region basis.
-  // This information can be used for e.g. remembered set scrubbing.
-
-  // A set bit indicates whether the given region contains any live object.
-  BitMap                  _region_live_bm;
-  // A set bit indicates that the given card contains a live object.
-  BitMap                  _card_live_bm;
-
   // Heap bounds
   HeapWord*               _heap_start;
   HeapWord*               _heap_end;
@@ -379,14 +370,6 @@ protected:
 
   void swapMarkBitMaps();
 
-  // Allocates and returns a zero-ed out "large" bitmap of the given size in bits.
-  // It is always allocated using virtual memory.
-  BitMap allocate_large_bitmap(BitMap::idx_t size_in_bits);
-  // Allocates the memory for all bitmaps used by the concurrent marking.
-  void allocate_internal_bitmaps();
-  // Pre-touches the internal bitmaps.
-  void pretouch_internal_bitmaps();
-
   // It resets the global marking data structures, as well as the
   // task local ones; should be called during initial mark.
   void reset();
@@ -592,7 +575,7 @@ public:
   void scan_root_regions();
 
   // Scan a single root region and mark everything reachable from it.
-  void scanRootRegion(HeapRegion* hr, uint worker_id);
+  void scanRootRegion(HeapRegion* hr);
 
   // Do concurrent phase of marking, to a tentative transitive closure.
   void mark_from_roots();
@@ -628,7 +611,7 @@ public:
 
   inline bool isPrevMarked(oop p) const;
 
-  inline bool do_yield_check(uint worker_i = 0);
+  inline bool do_yield_check();
 
   // Abandon current marking iteration due to a Full GC.
   void abort();
@@ -654,16 +637,19 @@ public:
 
 private:
   // Clear (Reset) all liveness count data.
-  void clear_all_live_data(WorkGang* workers);
+  void clear_live_data(WorkGang* workers);
 
+#ifdef ASSERT
   // Verify all of the above data structures that they are in initial state.
-  void verify_all_live_data();
+  void verify_live_data_clear();
+#endif
 
   // Aggregates the per-card liveness data based on the current marking. Also sets
   // the amount of marked bytes for each region.
   void create_live_data();
 
-  // Verification routine
+  void finalize_live_data();
+
   void verify_live_data();
 };
 
@@ -852,7 +838,7 @@ public:
 
   // Grey the object by marking it.  If not already marked, push it on
   // the local queue if below the finger.
-  // Precondition: obj is below region's NTAMS.
+  // obj is below its region's NTAMS.
   inline void make_reference_grey(oop obj);
 
   // Grey the object (by calling make_grey_reference) if required,

hotspot/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentMark.hpp"
+#include "gc/g1/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 
 inline bool G1ConcurrentMark::par_mark(oop obj) {
@@ -258,4 +259,13 @@ inline void G1ConcurrentMark::grayRoot(oop obj, HeapRegion* hr) {
   }
 }
 
+inline bool G1ConcurrentMark::do_yield_check() {
+  if (SuspendibleThreadSet::should_yield()) {
+    SuspendibleThreadSet::yield();
+    return true;
+  } else {
+    return false;
+  }
+}
+
 #endif // SHARE_VM_GC_G1_G1CONCURRENTMARK_INLINE_HPP

hotspot/src/share/vm/gc/g1/g1OopClosures.hpp

@@ -186,11 +186,9 @@ class G1RootRegionScanClosure : public MetadataAwareOopClosure {
 private:
   G1CollectedHeap* _g1h;
   G1ConcurrentMark* _cm;
-  uint _worker_id;
 public:
-  G1RootRegionScanClosure(G1CollectedHeap* g1h, G1ConcurrentMark* cm,
+  G1RootRegionScanClosure(G1CollectedHeap* g1h, G1ConcurrentMark* cm) :
-                          uint worker_id) :
+    _g1h(g1h), _cm(cm) { }
-    _g1h(g1h), _cm(cm), _worker_id(worker_id) { }
   template <class T> void do_oop_nv(T* p);
   virtual void do_oop(      oop* p) { do_oop_nv(p); }
   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }

hotspot/src/share/vm/gc/g1/g1RemSet.cpp

@@ -38,6 +38,7 @@
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
 #include "memory/iterator.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/oop.inline.hpp"
@@ -84,8 +85,16 @@ uint G1RemSet::num_par_rem_sets() {
   return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads);
 }
 
-void G1RemSet::initialize(uint max_regions) {
+void G1RemSet::initialize(size_t capacity, uint max_regions) {
   G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
+  {
+    GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
+    _card_live_data.initialize(capacity, max_regions);
+  }
+  if (G1PretouchAuxiliaryMemory) {
+    GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
+    _card_live_data.pretouch();
+  }
 }
 
 ScanRSClosure::ScanRSClosure(G1ParPushHeapRSClosure* oc,
@@ -312,27 +321,24 @@ void G1RemSet::cleanup_after_oops_into_collection_set_do() {
   _into_cset_dirty_card_queue_set.clear_n_completed_buffers();
 }
 
-class ScrubRSClosure: public HeapRegionClosure {
+class G1ScrubRSClosure: public HeapRegionClosure {
   G1CollectedHeap* _g1h;
-  BitMap* _region_bm;
+  G1CardLiveData* _live_data;
-  BitMap* _card_bm;
-  CardTableModRefBS* _ctbs;
 public:
-  ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) :
+  G1ScrubRSClosure(G1CardLiveData* live_data) :
     _g1h(G1CollectedHeap::heap()),
-    _region_bm(region_bm), _card_bm(card_bm),
+    _live_data(live_data) { }
-    _ctbs(_g1h->g1_barrier_set()) {}
 
   bool doHeapRegion(HeapRegion* r) {
     if (!r->is_continues_humongous()) {
-      r->rem_set()->scrub(_ctbs, _region_bm, _card_bm);
+      r->rem_set()->scrub(_live_data);
     }
     return false;
   }
 };
 
-void G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer *hrclaimer) {
+void G1RemSet::scrub(uint worker_num, HeapRegionClaimer *hrclaimer) {
-  ScrubRSClosure scrub_cl(region_bm, card_bm);
+  G1ScrubRSClosure scrub_cl(&_card_live_data);
   _g1->heap_region_par_iterate(&scrub_cl, worker_num, hrclaimer);
 }
 
@@ -580,3 +586,25 @@ void G1RemSet::prepare_for_verify() {
     assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
   }
 }
+
+void G1RemSet::create_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap) {
+  _card_live_data.create(workers, mark_bitmap);
+}
+
+void G1RemSet::finalize_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap) {
+  _card_live_data.finalize(workers, mark_bitmap);
+}
+
+void G1RemSet::verify_card_live_data(WorkGang* workers, G1CMBitMap* bitmap) {
+  _card_live_data.verify(workers, bitmap);
+}
+
+void G1RemSet::clear_card_live_data(WorkGang* workers) {
+  _card_live_data.clear(workers);
+}
+
+#ifdef ASSERT
+void G1RemSet::verify_card_live_data_is_clear() {
+  _card_live_data.verify_is_clear();
+}
+#endif

hotspot/src/share/vm/gc/g1/g1RemSet.hpp

@@ -26,6 +26,7 @@
 #define SHARE_VM_GC_G1_G1REMSET_HPP
 
 #include "gc/g1/dirtyCardQueue.hpp"
+#include "gc/g1/g1CardLiveData.hpp"
 #include "gc/g1/g1RemSetSummary.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "memory/allocation.hpp"
@@ -48,9 +49,10 @@ class HeapRegionClaimer;
 // A G1RemSet in which each heap region has a rem set that records the
 // external heap references into it.  Uses a mod ref bs to track updates,
 // so that they can be used to update the individual region remsets.
-
 class G1RemSet: public CHeapObj<mtGC> {
 private:
+  G1CardLiveData _card_live_data;
+
   G1RemSetSummary _prev_period_summary;
 
   // A DirtyCardQueueSet that is used to hold cards that contain
@@ -83,7 +85,7 @@ public:
   static uint num_par_rem_sets();
 
   // Initialize data that depends on the heap size being known.
-  static void initialize(uint max_regions);
+  void initialize(size_t capacity, uint max_regions);
 
   // This is called to reset dual hash tables after the gc pause
   // is finished and the initial hash table is no longer being
@@ -140,7 +142,7 @@ public:
   // set entries that correspond to dead heap ranges. "worker_num" is the
   // parallel thread id of the current thread, and "hrclaimer" is the
   // HeapRegionClaimer that should be used.
-  void scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer* hrclaimer);
+  void scrub(uint worker_num, HeapRegionClaimer* hrclaimer);
 
   // Refine the card corresponding to "card_ptr".
   // If check_for_refs_into_cset is true, a true result is returned
@@ -162,6 +164,19 @@ public:
   size_t conc_refine_cards() const {
     return _conc_refine_cards;
   }
+
+  void create_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap);
+  void finalize_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap);
+
+  // Verify that the liveness count data created concurrently matches one created
+  // during this safepoint.
+  void verify_card_live_data(WorkGang* workers, G1CMBitMap* actual_bitmap);
+
+  void clear_card_live_data(WorkGang* workers);
+
+#ifdef ASSERT
+  void verify_card_live_data_is_clear();
+#endif
 };
 
 class ScanRSClosure : public HeapRegionClosure {

hotspot/src/share/vm/gc/g1/heapRegionRemSet.cpp

@@ -26,6 +26,7 @@
 #include "gc/g1/concurrentG1Refine.hpp"
 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1CardLiveData.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.hpp"
 #include "gc/shared/space.inline.hpp"
@@ -141,10 +142,8 @@ public:
     add_reference_work(from, /*parallel*/ false);
   }
 
-  void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) {
+  void scrub(G1CardLiveData* live_data) {
-    HeapWord* hr_bot = hr()->bottom();
+    live_data->remove_nonlive_cards(hr()->hrm_index(), &_bm);
-    size_t hr_first_card_index = ctbs->index_for(hr_bot);
-    bm()->set_intersection_at_offset(*card_bm, hr_first_card_index);
     recount_occupied();
   }
 
@@ -515,14 +514,12 @@ PerRegionTable* OtherRegionsTable::delete_region_table() {
   return max;
 }
 
-void OtherRegionsTable::scrub(CardTableModRefBS* ctbs,
+void OtherRegionsTable::scrub(G1CardLiveData* live_data) {
-                              BitMap* region_bm, BitMap* card_bm) {
   // First eliminated garbage regions from the coarse map.
   log_develop_trace(gc, remset, scrub)("Scrubbing region %u:", _hr->hrm_index());
 
-  assert(_coarse_map.size() == region_bm->size(), "Precondition");
   log_develop_trace(gc, remset, scrub)("   Coarse map: before = " SIZE_FORMAT "...", _n_coarse_entries);
-  _coarse_map.set_intersection(*region_bm);
+  live_data->remove_nonlive_regions(&_coarse_map);
   _n_coarse_entries = _coarse_map.count_one_bits();
   log_develop_trace(gc, remset, scrub)("   after = " SIZE_FORMAT ".", _n_coarse_entries);
 
@@ -534,7 +531,7 @@ void OtherRegionsTable::scrub(CardTableModRefBS* ctbs,
       PerRegionTable* nxt = cur->collision_list_next();
       // If the entire region is dead, eliminate.
       log_develop_trace(gc, remset, scrub)("     For other region %u:", cur->hr()->hrm_index());
-      if (!region_bm->at((size_t) cur->hr()->hrm_index())) {
+      if (!live_data->is_region_live(cur->hr()->hrm_index())) {
         *prev = nxt;
         cur->set_collision_list_next(NULL);
         _n_fine_entries--;
@@ -544,7 +541,7 @@ void OtherRegionsTable::scrub(CardTableModRefBS* ctbs,
       } else {
         // Do fine-grain elimination.
         log_develop_trace(gc, remset, scrub)("          occ: before = %4d.", cur->occupied());
-        cur->scrub(ctbs, card_bm);
+        cur->scrub(live_data);
         log_develop_trace(gc, remset, scrub)("          after = %4d.", cur->occupied());
         // Did that empty the table completely?
         if (cur->occupied() == 0) {
@@ -773,9 +770,8 @@ void HeapRegionRemSet::reset_for_par_iteration() {
   assert(verify_ready_for_par_iteration(), "post-condition");
 }
 
-void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs,
+void HeapRegionRemSet::scrub(G1CardLiveData* live_data) {
-                             BitMap* region_bm, BitMap* card_bm) {
+  _other_regions.scrub(live_data);
-  _other_regions.scrub(ctbs, region_bm, card_bm);
 }
 
 // Code roots support

hotspot/src/share/vm/gc/g1/heapRegionRemSet.hpp

@@ -35,6 +35,7 @@
 
 class G1CollectedHeap;
 class G1BlockOffsetTable;
+class G1CardLiveData;
 class HeapRegion;
 class HeapRegionRemSetIterator;
 class PerRegionTable;
@@ -143,7 +144,7 @@ public:
   // Removes any entries shown by the given bitmaps to contain only dead
   // objects. Not thread safe.
   // Set bits in the bitmaps indicate that the given region or card is live.
-  void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
+  void scrub(G1CardLiveData* live_data);
 
   // Returns whether this remembered set (and all sub-sets) does not contain any entry.
   bool is_empty() const;
@@ -230,10 +231,9 @@ public:
     _other_regions.add_reference(from, tid);
   }
 
-  // Removes any entries in the remembered set shown by the given bitmaps to
+  // Removes any entries in the remembered set shown by the given card live data to
   // contain only dead objects. Not thread safe.
-  // One bits in the bitmaps indicate that the given region or card is live.
+  void scrub(G1CardLiveData* live_data);
-  void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
 
   // The region is being reclaimed; clear its remset, and any mention of
   // entries for this region in other remsets.

hotspot/src/share/vm/utilities/bitMap.cpp

@@ -69,7 +69,7 @@ void BitMap::resize(idx_t size_in_bits, bool in_resource_area) {
 }
 
 void BitMap::pretouch() {
-  os::pretouch_memory((char*)word_addr(0), (char*)word_addr(size()));
+  os::pretouch_memory(word_addr(0), word_addr(size()));
 }
 
 void BitMap::set_range_within_word(idx_t beg, idx_t end) {

hotspot/src/share/vm/utilities/bitMap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -140,11 +140,12 @@ class BitMap VALUE_OBJ_CLASS_SPEC {
 
   // Accessing
   idx_t size() const                    { return _size; }
+  idx_t size_in_bytes() const           { return size_in_words() * BytesPerWord; }
   idx_t size_in_words() const           {
-    return word_index(size() + BitsPerWord - 1);
+    return calc_size_in_words(size());
   }
 
-  static idx_t size_in_words(size_t size_in_bits) {
+  static idx_t calc_size_in_words(size_t size_in_bits) {
     return word_index(size_in_bits + BitsPerWord - 1);
   }