8275320: NMT should perform buffer overrun checks

8275320: NMT should perform buffer overrun checks
8275301: Unify C-heap buffer overrun checks into NMT

Reviewed-by: simonis, zgu

src/hotspot/share/services/mallocTracker.cpp

@@ -23,10 +23,13 @@
  */
 #include "precompiled.hpp"
 
+#include "runtime/os.hpp"
 #include "services/mallocSiteTable.hpp"
 #include "services/mallocTracker.hpp"
 #include "services/mallocTracker.inline.hpp"
 #include "services/memTracker.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/ostream.hpp"
 
 size_t MallocMemorySummary::_snapshot[CALC_OBJ_SIZE_IN_TYPE(MallocMemorySnapshot, size_t)];
 
@@ -103,15 +106,118 @@ void MallocMemorySummary::initialize() {
   ::new ((void*)_snapshot)MallocMemorySnapshot();
 }
 
-void MallocHeader::release() const {
+void MallocHeader::mark_block_as_dead() {
+  _canary = _header_canary_dead_mark;
+  NOT_LP64(_alt_canary = _header_alt_canary_dead_mark);
+  set_footer(_footer_canary_dead_mark);
+}
+
+void MallocHeader::release() {
   // Tracking already shutdown, no housekeeping is needed anymore
   if (MemTracker::tracking_level() <= NMT_minimal) return;
 
+  check_block_integrity();
+
   MallocMemorySummary::record_free(size(), flags());
   MallocMemorySummary::record_free_malloc_header(sizeof(MallocHeader));
   if (MemTracker::tracking_level() == NMT_detail) {
     MallocSiteTable::deallocation_at(size(), _bucket_idx, _pos_idx);
   }
+
+  mark_block_as_dead();
+}
+
+void MallocHeader::print_block_on_error(outputStream* st, address bad_address) const {
+  assert(bad_address >= (address)this, "sanity");
+
+  // This function prints block information, including hex dump, in case of a detected
+  // corruption. The hex dump should show both block header and corruption site
+  // (which may or may not be close together or identical). Plus some surrounding area.
+  //
+  // Note that we use os::print_hex_dump(), which is able to cope with unmapped
+  // memory (it uses SafeFetch).
+
+  st->print_cr("NMT Block at " PTR_FORMAT ", corruption at: " PTR_FORMAT ": ",
+               p2i(this), p2i(bad_address));
+  static const size_t min_dump_length = 256;
+  address from1 = align_down((address)this, sizeof(void*)) - (min_dump_length / 2);
+  address to1 = from1 + min_dump_length;
+  address from2 = align_down(bad_address, sizeof(void*)) - (min_dump_length / 2);
+  address to2 = from2 + min_dump_length;
+  if (from2 > to1) {
+    // Dump gets too large, split up in two sections.
+    os::print_hex_dump(st, from1, to1, 1);
+    st->print_cr("...");
+    os::print_hex_dump(st, from2, to2, 1);
+  } else {
+    // print one hex dump
+    os::print_hex_dump(st, from1, to2, 1);
+  }
+}
+
+// Check block integrity. If block is broken, print out a report
+// to tty (optionally with hex dump surrounding the broken block),
+// then trigger a fatal error.
+void MallocHeader::check_block_integrity() const {
+
+#define PREFIX "NMT corruption: "
+  // Note: if you modify the error messages here, make sure you
+  // adapt the associated gtests too.
+
+  // Weed out obviously wrong block addresses of NULL or very low
+  // values. Note that we should not call this for ::free(NULL),
+  // which should be handled by os::free() above us.
+  if (((size_t)p2i(this)) < K) {
+    fatal(PREFIX "Block at " PTR_FORMAT ": invalid block address", p2i(this));
+  }
+
+  // From here on we assume the block pointer to be valid. We could
+  // use SafeFetch but since this is a hot path we don't. If we are
+  // wrong, we will crash when accessing the canary, which hopefully
+  // generates distinct crash report.
+
+  // Weed out obviously unaligned addresses. NMT blocks, being the result of
+  // malloc calls, should adhere to malloc() alignment. Malloc alignment is
+  // specified by the standard by this requirement:
+  // "malloc returns a pointer which is suitably aligned for any built-in type"
+  // For us it means that it is *at least* 64-bit on all of our 32-bit and
+  // 64-bit platforms since we have native 64-bit types. It very probably is
+  // larger than that, since there exist scalar types larger than 64bit. Here,
+  // we test the smallest alignment we know.
+  // Should we ever start using std::max_align_t, this would be one place to
+  // fix up.
+  if (!is_aligned(this, sizeof(uint64_t))) {
+    print_block_on_error(tty, (address)this);
+    fatal(PREFIX "Block at " PTR_FORMAT ": block address is unaligned", p2i(this));
+  }
+
+  // Check header canary
+  if (_canary != _header_canary_life_mark) {
+    print_block_on_error(tty, (address)this);
+    fatal(PREFIX "Block at " PTR_FORMAT ": header canary broken.", p2i(this));
+  }
+
+#ifndef _LP64
+  // On 32-bit we have a second canary, check that one too.
+  if (_alt_canary != _header_alt_canary_life_mark) {
+    print_block_on_error(tty, (address)this);
+    fatal(PREFIX "Block at " PTR_FORMAT ": header alternate canary broken.", p2i(this));
+  }
+#endif
+
+  // Does block size seems reasonable?
+  if (_size >= max_reasonable_malloc_size) {
+    print_block_on_error(tty, (address)this);
+    fatal(PREFIX "Block at " PTR_FORMAT ": header looks invalid (weirdly large block size)", p2i(this));
+  }
+
+  // Check footer canary
+  if (get_footer() != _footer_canary_life_mark) {
+    print_block_on_error(tty, footer_address());
+    fatal(PREFIX "Block at " PTR_FORMAT ": footer canary broken at " PTR_FORMAT " (buffer overflow?)",
+          p2i(this), p2i(footer_address()));
+  }
+#undef PREFIX
 }
 
 bool MallocHeader::record_malloc_site(const NativeCallStack& stack, size_t size,