8205924: ZGC: Premature OOME due to failure to expand backing file

Reviewed-by: ehelin

src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.cpp

@@ -28,7 +28,6 @@
 #include "gc/z/zErrno.hpp"
 #include "gc/z/zLargePages.inline.hpp"
 #include "logging/log.hpp"
-#include "runtime/init.hpp"
 #include "runtime/os.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
@@ -83,9 +82,12 @@ static int z_memfd_create(const char *name, unsigned int flags) {
   return syscall(__NR_memfd_create, name, flags);
 }
 
+bool ZBackingFile::_hugetlbfs_mmap_retry = true;
+
 ZBackingFile::ZBackingFile() :
     _fd(-1),
     _filesystem(0),
+    _available(0),
     _initialized(false) {
 
   // Create backing file
@@ -94,39 +96,47 @@ ZBackingFile::ZBackingFile() :
     return;
   }
 
-  // Get filesystem type
+  // Get filesystem statistics
   struct statfs statfs_buf;
   if (fstatfs(_fd, &statfs_buf) == -1) {
     ZErrno err;
-    log_error(gc, init)("Failed to determine filesystem type for backing file (%s)", err.to_string());
+    log_error(gc, init)("Failed to determine filesystem type for backing file (%s)",
+                        err.to_string());
     return;
   }
+
   _filesystem = statfs_buf.f_type;
+  _available = statfs_buf.f_bavail * statfs_buf.f_bsize;
 
   // Make sure we're on a supported filesystem
   if (!is_tmpfs() && !is_hugetlbfs()) {
-    log_error(gc, init)("Backing file must be located on a %s or a %s filesystem", ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS);
+    log_error(gc, init)("Backing file must be located on a %s or a %s filesystem",
+                        ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS);
     return;
   }
 
   // Make sure the filesystem type matches requested large page type
   if (ZLargePages::is_transparent() && !is_tmpfs()) {
-    log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", ZFILESYSTEM_TMPFS);
+    log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem",
+                        ZFILESYSTEM_TMPFS);
     return;
   }
 
   if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
-    log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", ZFILESYSTEM_TMPFS);
+    log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
+                        ZFILESYSTEM_TMPFS);
     return;
   }
 
   if (ZLargePages::is_explicit() && !is_hugetlbfs()) {
-    log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);
+    log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem",
+                        ZFILESYSTEM_HUGETLBFS);
     return;
   }
 
   if (!ZLargePages::is_explicit() && is_hugetlbfs()) {
-    log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);
+    log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem",
+                        ZFILESYSTEM_HUGETLBFS);
     return;
   }
 
@@ -149,7 +159,7 @@ int ZBackingFile::create_mem_fd(const char* name) const {
     return -1;
   }
 
-  log_debug(gc, init)("Heap backed by file /memfd:%s", filename);
+  log_info(gc, init)("Heap backed by file: /memfd:%s", filename);
 
   return fd;
 }
@@ -181,7 +191,7 @@ int ZBackingFile::create_file_fd(const char* name) const {
       return -1;
     }
 
-    log_debug(gc, init)("Heap backed by file %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino);
+    log_info(gc, init)("Heap backed by file: %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino);
 
     return fd_anon;
   }
@@ -207,7 +217,7 @@ int ZBackingFile::create_file_fd(const char* name) const {
     return -1;
   }
 
-  log_debug(gc, init)("Heap backed by file %s", filename);
+  log_info(gc, init)("Heap backed by file: %s", filename);
 
   return fd;
 }
@@ -238,6 +248,10 @@ int ZBackingFile::fd() const {
   return _fd;
 }
 
+size_t ZBackingFile::available() const {
+  return _available;
+}
+
 bool ZBackingFile::is_tmpfs() const {
   return _filesystem == TMPFS_MAGIC;
 }
@@ -292,12 +306,12 @@ bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignme
   return true;
 }
 
-bool ZBackingFile::expand_tmpfs(size_t offset, size_t length) const {
+bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length) const {
   assert(is_tmpfs(), "Wrong filesystem");
   return try_expand_tmpfs(offset, length, os::vm_page_size());
 }
 
-bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const {
+bool ZBackingFile::try_expand_hugetlbfs(size_t offset, size_t length) const {
   assert(is_hugetlbfs(), "Wrong filesystem");
 
   // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate().
@@ -320,11 +334,11 @@ bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const {
   // process being returned to the huge page pool and made available for new
   // allocations.
   void* addr = MAP_FAILED;
-  const int max_attempts = 3;
+  const int max_attempts = 5;
   for (int attempt = 1; attempt <= max_attempts; attempt++) {
     addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
-    if (addr != MAP_FAILED || is_init_completed()) {
-      // Mapping was successful or initialization phase has completed
+    if (addr != MAP_FAILED || !_hugetlbfs_mmap_retry) {
+      // Mapping was successful or mmap retry is disabled
       break;
     }
 
@@ -337,6 +351,11 @@ bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const {
     sleep(1);
   }
 
+  // Disable mmap retry from now on
+  if (_hugetlbfs_mmap_retry) {
+    _hugetlbfs_mmap_retry = false;
+  }
+
   if (addr == MAP_FAILED) {
     // Not enough huge pages left
     ZErrno err;
@@ -355,6 +374,39 @@ bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const {
   return true;
 }
 
-bool ZBackingFile::expand(size_t offset, size_t length) const {
-  return is_hugetlbfs() ? expand_hugetlbfs(offset, length) : expand_tmpfs(offset, length);
+bool ZBackingFile::try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const {
+  assert(is_aligned(offset, alignment), "Invalid offset");
+  assert(is_aligned(length, alignment), "Invalid length");
+
+  log_debug(gc)("Expanding heap from " SIZE_FORMAT "M to " SIZE_FORMAT "M", offset / M, (offset + length) / M);
+
+  return is_hugetlbfs() ? try_expand_hugetlbfs(offset, length) : try_expand_tmpfs(offset, length);
+}
+
+size_t ZBackingFile::try_expand(size_t offset, size_t length, size_t alignment) const {
+  size_t start = offset;
+  size_t end = offset + length;
+
+  // Try to expand
+  if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) {
+    // Success
+    return end;
+  }
+
+  // Failed, try to expand as much as possible
+  for (;;) {
+    length = align_down((end - start) / 2, alignment);
+    if (length < alignment) {
+      // Done, don't expand more
+      return start;
+    }
+
+    if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) {
+      // Success, try expand more
+      start += length;
+    } else {
+      // Failed, try expand less
+      end -= length;
+    }
+  }
 }

src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,8 +28,11 @@
 
 class ZBackingFile {
 private:
+  static bool _hugetlbfs_mmap_retry;
+
   int      _fd;
   uint64_t _filesystem;
+  size_t   _available;
   bool     _initialized;
 
   int create_mem_fd(const char* name) const;
@@ -42,9 +45,9 @@ private:
 
   bool try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const;
   bool try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const;
-  bool expand_tmpfs(size_t offset, size_t length) const;
-
-  bool expand_hugetlbfs(size_t offset, size_t length) const;
+  bool try_expand_tmpfs(size_t offset, size_t length) const;
+  bool try_expand_hugetlbfs(size_t offset, size_t length) const;
+  bool try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const;
 
 public:
   ZBackingFile();
@@ -52,7 +55,9 @@ public:
   bool is_initialized() const;
 
   int fd() const;
-  bool expand(size_t offset, size_t length) const;
+  size_t available() const;
+
+  size_t try_expand(size_t offset, size_t length, size_t alignment) const;
 };
 
 #endif // OS_CPU_LINUX_X86_ZBACKINGFILE_LINUX_X86_HPP

src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp

@@ -52,8 +52,15 @@ ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity, size_t granu
     _file(),
     _granule_size(granule_size) {
 
-  // Check and warn if max map count seems too low
+  if (!_file.is_initialized()) {
+    return;
+  }
+
+  // Check and warn if max map count is too low
   check_max_map_count(max_capacity, granule_size);
+
+  // Check and warn if available space on filesystem is too low
+  check_available_space_on_filesystem(max_capacity);
 }
 
 void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t granule_size) const {
@@ -61,7 +68,7 @@ void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t gra
   FILE* const file = fopen(filename, "r");
   if (file == NULL) {
     // Failed to open file, skip check
-    log_debug(gc)("Failed to open %s", filename);
+    log_debug(gc, init)("Failed to open %s", filename);
     return;
   }
 
@@ -70,7 +77,7 @@ void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t gra
   fclose(file);
   if (result != 1) {
     // Failed to read file, skip check
-    log_debug(gc)("Failed to read %s", filename);
+    log_debug(gc, init)("Failed to read %s", filename);
     return;
   }
 
@@ -81,15 +88,43 @@ void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t gra
   // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
   const size_t required_max_map_count = (max_capacity / granule_size) * 3 * 1.2;
   if (actual_max_map_count < required_max_map_count) {
-    log_warning(gc)("The system limit on number of memory mappings "
-                    "per process might be too low for the given");
-    log_warning(gc)("Java heap size (" SIZE_FORMAT "M). Please "
-                    "adjust %s to allow for at least", max_capacity / M, filename);
-    log_warning(gc)(SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). "
-                    "Continuing execution with the current limit could",
-                    required_max_map_count, actual_max_map_count);
-    log_warning(gc)("lead to a fatal error down the line, due to failed "
-                    "attempts to map memory.");
+    log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+    log_warning(gc, init)("The system limit on number of memory mappings per process might be too low "
+                          "for the given");
+    log_warning(gc, init)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at",
+                          max_capacity / M, filename);
+    log_warning(gc, init)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing "
+                          "execution with the current", required_max_map_count, actual_max_map_count);
+    log_warning(gc, init)("limit could lead to a fatal error, due to failure to map memory.");
+  }
+}
+
+void ZPhysicalMemoryBacking::check_available_space_on_filesystem(size_t max_capacity) const {
+  // Note that the available space on a tmpfs or a hugetlbfs filesystem
+  // will be zero if no size limit was specified when it was mounted.
+  const size_t available = _file.available();
+  if (available == 0) {
+    // No size limit set, skip check
+    log_info(gc, init)("Available space on backing filesystem: N/A");
+    return;
+  }
+
+  log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M",
+                     available / M);
+
+  // Warn if the filesystem doesn't currently have enough space available to hold
+  // the max heap size. The max heap size will be capped if we later hit this limit
+  // when trying to expand the heap.
+  if (available < max_capacity) {
+    log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+    log_warning(gc, init)("Not enough space available on the backing filesystem to hold the current "
+                          "max Java heap");
+    log_warning(gc, init)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem "
+                          "accordingly (available", max_capacity / M);
+    log_warning(gc, init)("space is currently " SIZE_FORMAT "M). Continuing execution with the current "
+                          "filesystem size could", available / M);
+    log_warning(gc, init)("lead to a premature OutOfMemoryError being thrown, due to failure to map "
+                          "memory.");
   }
 }
 
@@ -97,18 +132,16 @@ bool ZPhysicalMemoryBacking::is_initialized() const {
   return _file.is_initialized();
 }
 
-bool ZPhysicalMemoryBacking::expand(size_t from, size_t to) {
-  const size_t size = to - from;
+size_t ZPhysicalMemoryBacking::try_expand(size_t old_capacity, size_t new_capacity) {
+  assert(old_capacity < new_capacity, "Invalid old/new capacity");
 
-  // Expand
-  if (!_file.expand(from, size)) {
-    return false;
+  const size_t capacity = _file.try_expand(old_capacity, new_capacity - old_capacity, _granule_size);
+  if (capacity > old_capacity) {
+    // Add expanded capacity to free list
+    _manager.free(old_capacity, capacity - old_capacity);
   }
 
-  // Add expanded space to free list
-  _manager.free(from, size);
-
-  return true;
+  return capacity;
 }
 
 ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) {

src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,7 @@ private:
   const size_t   _granule_size;
 
   void check_max_map_count(size_t max_capacity, size_t granule_size) const;
+  void check_available_space_on_filesystem(size_t max_capacity) const;
   void map_failed(ZErrno err) const;
 
   void advise_view(uintptr_t addr, size_t size) const;
@@ -49,7 +50,8 @@ public:
 
   bool is_initialized() const;
 
-  bool expand(size_t from, size_t to);
+  size_t try_expand(size_t old_capacity, size_t new_capacity);
+
   ZPhysicalMemory alloc(size_t size);
   void free(ZPhysicalMemory pmem);
 

src/hotspot/share/gc/z/zDirector.cpp

@@ -81,7 +81,7 @@ bool ZDirector::rule_warmup() const {
   // Perform GC if heap usage passes 10/20/30% and no other GC has been
   // performed yet. This allows us to get some early samples of the GC
   // duration, which is needed by the other rules.
-  const size_t max_capacity = ZHeap::heap()->max_capacity();
+  const size_t max_capacity = ZHeap::heap()->current_max_capacity();
   const size_t used = ZHeap::heap()->used();
   const double used_threshold_percent = (ZStatCycle::ncycles() + 1) * 0.1;
   const size_t used_threshold = max_capacity * used_threshold_percent;
@@ -107,7 +107,7 @@ bool ZDirector::rule_allocation_rate() const {
   // Calculate amount of free memory available to Java threads. Note that
   // the heap reserve is not available to Java threads and is therefore not
   // considered part of the free memory.
-  const size_t max_capacity = ZHeap::heap()->max_capacity();
+  const size_t max_capacity = ZHeap::heap()->current_max_capacity();
   const size_t max_reserve = ZHeap::heap()->max_reserve();
   const size_t used = ZHeap::heap()->used();
   const size_t free_with_reserve = max_capacity - used;
@@ -155,7 +155,7 @@ bool ZDirector::rule_proactive() const {
   // passed since the previous GC. This helps avoid superfluous GCs when running
   // applications with very low allocation rate.
   const size_t used_after_last_gc = ZStatHeap::used_at_relocate_end();
-  const size_t used_increase_threshold = ZHeap::heap()->max_capacity() * 0.10; // 10%
+  const size_t used_increase_threshold = ZHeap::heap()->current_max_capacity() * 0.10; // 10%
   const size_t used_threshold = used_after_last_gc + used_increase_threshold;
   const size_t used = ZHeap::heap()->used();
   const double time_since_last_gc = ZStatCycle::time_since_last();

src/hotspot/share/gc/z/zHeap.cpp

@@ -107,6 +107,10 @@ size_t ZHeap::max_capacity() const {
   return _page_allocator.max_capacity();
 }
 
+size_t ZHeap::current_max_capacity() const {
+  return _page_allocator.current_max_capacity();
+}
+
 size_t ZHeap::capacity() const {
   return _page_allocator.capacity();
 }

src/hotspot/share/gc/z/zHeap.hpp

@@ -79,6 +79,7 @@ public:
   // Heap metrics
   size_t min_capacity() const;
   size_t max_capacity() const;
+  size_t current_max_capacity() const;
   size_t capacity() const;
   size_t max_reserve() const;
   size_t used_high() const;

src/hotspot/share/gc/z/zPageAllocator.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -84,11 +84,12 @@ public:
 ZPage* const ZPageAllocator::gc_marker = (ZPage*)-1;
 
 ZPageAllocator::ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t max_reserve) :
+    _lock(),
     _virtual(),
     _physical(max_capacity, ZPageSizeMin),
     _cache(),
-    _pre_mapped(_virtual, _physical, min_capacity),
     _max_reserve(max_reserve),
+    _pre_mapped(_virtual, _physical, try_ensure_unused_for_pre_mapped(min_capacity)),
     _used_high(0),
     _used_low(0),
     _used(0),
@@ -107,6 +108,10 @@ size_t ZPageAllocator::max_capacity() const {
   return _physical.max_capacity();
 }
 
+size_t ZPageAllocator::current_max_capacity() const {
+  return _physical.current_max_capacity();
+}
+
 size_t ZPageAllocator::capacity() const {
   return _physical.capacity();
 }
@@ -169,18 +174,43 @@ void ZPageAllocator::decrease_used(size_t size, bool reclaimed) {
   }
 }
 
-size_t ZPageAllocator::available(ZAllocationFlags flags) const {
-  size_t available = max_capacity() - used();
-  assert(_physical.available() + _pre_mapped.available() + _cache.available()  == available, "Should be equal");
+size_t ZPageAllocator::max_available(bool no_reserve) const {
+  size_t available = current_max_capacity() - used();
 
-  if (flags.no_reserve()) {
-    // The memory reserve should not be considered free
+  if (no_reserve) {
+    // The reserve should not be considered available
     available -= MIN2(available, max_reserve());
   }
 
   return available;
 }
 
+size_t ZPageAllocator::try_ensure_unused(size_t size, bool no_reserve) {
+  // Ensure that we always have space available for the reserve. This
+  // is needed to avoid losing the reserve because of failure to map
+  // more memory before reaching max capacity.
+  _physical.try_ensure_unused_capacity(size + max_reserve());
+
+  size_t unused = _physical.unused_capacity();
+
+  if (no_reserve) {
+    // The reserve should not be considered unused
+    unused -= MIN2(unused, max_reserve());
+  }
+
+  return MIN2(size, unused);
+}
+
+size_t ZPageAllocator::try_ensure_unused_for_pre_mapped(size_t size) {
+  // This function is called during construction, where the
+  // physical memory manager might have failed to initialied.
+  if (!_physical.is_initialized()) {
+    return 0;
+  }
+
+  return try_ensure_unused(size, true /* no_reserve */);
+}
+
 ZPage* ZPageAllocator::create_page(uint8_t type, size_t size) {
   // Allocate physical memory
   const ZPhysicalMemory pmem = _physical.alloc(size);
@@ -259,8 +289,8 @@ void ZPageAllocator::check_out_of_memory_during_initialization() {
 }
 
 ZPage* ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZAllocationFlags flags) {
-  const size_t available_total = available(flags);
-  if (available_total < size) {
+  const size_t max = max_available(flags.no_reserve());
+  if (max < size) {
     // Not enough free memory
     return NULL;
   }
@@ -281,11 +311,11 @@ ZPage* ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZAlloc
   // subsequent allocations can use the physical memory.
   flush_pre_mapped();
 
-  // Check if physical memory is available
-  const size_t available_physical = _physical.available();
-  if (available_physical < size) {
+  // Try ensure that physical memory is available
+  const size_t unused = try_ensure_unused(size, flags.no_reserve());
+  if (unused < size) {
     // Flush cache to free up more physical memory
-    flush_cache(size - available_physical);
+    flush_cache(size - unused);
   }
 
   // Create new page and allocate physical memory
@@ -303,7 +333,7 @@ ZPage* ZPageAllocator::alloc_page_common(uint8_t type, size_t size, ZAllocationF
   increase_used(size, flags.relocation());
 
   // Send trace event
-  ZTracer::tracer()->report_page_alloc(size, used(), available(flags), _cache.available(), flags);
+  ZTracer::tracer()->report_page_alloc(size, used(), max_available(flags.no_reserve()), _cache.available(), flags);
 
   return page;
 }

src/hotspot/share/gc/z/zPageAllocator.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,8 +43,8 @@ private:
   ZVirtualMemoryManager    _virtual;
   ZPhysicalMemoryManager   _physical;
   ZPageCache               _cache;
-  ZPreMappedMemory         _pre_mapped;
   const size_t             _max_reserve;
+  ZPreMappedMemory         _pre_mapped;
   size_t                   _used_high;
   size_t                   _used_low;
   size_t                   _used;
@@ -58,7 +58,9 @@ private:
   void increase_used(size_t size, bool relocation);
   void decrease_used(size_t size, bool reclaimed);
 
-  size_t available(ZAllocationFlags flags) const;
+  size_t max_available(bool no_reserve) const;
+  size_t try_ensure_unused(size_t size, bool no_reserve);
+  size_t try_ensure_unused_for_pre_mapped(size_t size);
 
   ZPage* create_page(uint8_t type, size_t size);
   void map_page(ZPage* page);
@@ -83,6 +85,7 @@ public:
   bool is_initialized() const;
 
   size_t max_capacity() const;
+  size_t current_max_capacity() const;
   size_t capacity() const;
   size_t max_reserve() const;
   size_t used_high() const;

src/hotspot/share/gc/z/zPhysicalMemory.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 #include "memory/allocation.inline.hpp"
 #include "services/memTracker.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 ZPhysicalMemory::ZPhysicalMemory() :
     _nsegments(0),
@@ -93,6 +94,7 @@ void ZPhysicalMemory::clear() {
 ZPhysicalMemoryManager::ZPhysicalMemoryManager(size_t max_capacity, size_t granule_size) :
     _backing(max_capacity, granule_size),
     _max_capacity(max_capacity),
+    _current_max_capacity(max_capacity),
     _capacity(0),
     _used(0) {}
 
@@ -100,31 +102,34 @@ bool ZPhysicalMemoryManager::is_initialized() const {
   return _backing.is_initialized();
 }
 
-bool ZPhysicalMemoryManager::ensure_available(size_t size) {
-  const size_t unused_capacity = _capacity - _used;
-  if (unused_capacity >= size) {
-    // Enough unused capacity available
-    return true;
+void ZPhysicalMemoryManager::try_ensure_unused_capacity(size_t size) {
+  const size_t unused = unused_capacity();
+  if (unused >= size) {
+    // Don't try to expand, enough unused capacity available
+    return;
   }
 
-  const size_t expand_with = size - unused_capacity;
-  const size_t new_capacity = _capacity + expand_with;
-  if (new_capacity > _max_capacity) {
-    // Can not expand beyond max capacity
-    return false;
+  const size_t current_max = current_max_capacity();
+  if (_capacity == current_max) {
+    // Don't try to expand, current max capacity reached
+    return;
   }
 
-  // Expand
-  if (!_backing.expand(_capacity, new_capacity)) {
-    log_error(gc)("Failed to expand Java heap with " SIZE_FORMAT "%s",
-                  byte_size_in_proper_unit(expand_with),
-                  proper_unit_for_byte_size(expand_with));
-    return false;
+  // Try to expand
+  const size_t old_capacity = capacity();
+  const size_t new_capacity = MIN2(old_capacity + size - unused, current_max);
+  _capacity = _backing.try_expand(old_capacity, new_capacity);
+
+  if (_capacity != new_capacity) {
+    // Failed, or partly failed, to expand
+    log_error(gc, init)("Not enough space available on the backing filesystem to hold the current max");
+    log_error(gc, init)("Java heap size (" SIZE_FORMAT "M). Forcefully lowering max Java heap size to "
+                        SIZE_FORMAT "M (%.0lf%%).", current_max / M, _capacity / M,
+                        percent_of(_capacity, current_max));
+
+    // Adjust current max capacity to avoid further expand attempts
+    _current_max_capacity = _capacity;
   }
-
-  _capacity = new_capacity;
-
-  return true;
 }
 
 void ZPhysicalMemoryManager::nmt_commit(ZPhysicalMemory pmem, uintptr_t offset) {
@@ -144,7 +149,7 @@ void ZPhysicalMemoryManager::nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset
 }
 
 ZPhysicalMemory ZPhysicalMemoryManager::alloc(size_t size) {
-  if (!ensure_available(size)) {
+  if (unused_capacity() < size) {
     // Not enough memory available
     return ZPhysicalMemory();
   }

src/hotspot/share/gc/z/zPhysicalMemory.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -70,11 +70,10 @@ class ZPhysicalMemoryManager {
 private:
   ZPhysicalMemoryBacking _backing;
   const size_t           _max_capacity;
+  size_t                 _current_max_capacity;
   size_t                 _capacity;
   size_t                 _used;
 
-  bool ensure_available(size_t size);
-
   void nmt_commit(ZPhysicalMemory pmem, uintptr_t offset);
   void nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset);
 
@@ -84,9 +83,11 @@ public:
   bool is_initialized() const;
 
   size_t max_capacity() const;
+  size_t current_max_capacity() const;
   size_t capacity() const;
-  size_t used() const;
-  size_t available() const;
+  size_t unused_capacity() const;
+
+  void try_ensure_unused_capacity(size_t size);
 
   ZPhysicalMemory alloc(size_t size);
   void free(ZPhysicalMemory pmem);

src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,16 +71,16 @@ inline size_t ZPhysicalMemoryManager::max_capacity() const {
   return _max_capacity;
 }
 
+inline size_t ZPhysicalMemoryManager::current_max_capacity() const {
+  return _current_max_capacity;
+}
+
 inline size_t ZPhysicalMemoryManager::capacity() const {
   return _capacity;
 }
 
-inline size_t ZPhysicalMemoryManager::used() const {
-  return _used;
-}
-
-inline size_t ZPhysicalMemoryManager::available() const {
-  return _max_capacity - _used;
+inline size_t ZPhysicalMemoryManager::unused_capacity() const {
+  return _capacity - _used;
 }
 
 #endif // SHARE_GC_Z_ZPHYSICALMEMORY_INLINE_HPP

src/hotspot/share/gc/z/zPreMappedMemory.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,21 +42,25 @@ ZPreMappedMemory::ZPreMappedMemory(ZVirtualMemoryManager &vmm, ZPhysicalMemoryMa
   log_info(gc, init)("Pre-touching: %s", AlwaysPreTouch ? "Enabled" : "Disabled");
   log_info(gc, init)("Pre-mapping: " SIZE_FORMAT "M", size / M);
 
+  if (size > 0) {
     _pmem = pmm.alloc(size);
     if (_pmem.is_null()) {
       // Out of memory
+      log_error(gc, init)("Failed to pre-map Java heap (Cannot allocate physical memory)");
       return;
     }
 
     _vmem = vmm.alloc(size, true /* alloc_from_front */);
     if (_vmem.is_null()) {
       // Out of address space
+      log_error(gc, init)("Failed to pre-map Java heap (Cannot allocate virtual memory)");
       pmm.free(_pmem);
       return;
     }
 
     // Map physical memory
     pmm.map(_pmem, _vmem.start());
+  }
 
   _initialized = true;
 }

test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,9 @@ TEST(ZPhysicalMemoryTest, split) {
 
   ZPhysicalMemoryManager pmem_manager(10 * SegmentSize, SegmentSize);
 
+  pmem_manager.try_ensure_unused_capacity(10 * SegmentSize);
+  EXPECT_EQ(pmem_manager.unused_capacity(), 10 * SegmentSize);
+
   ZPhysicalMemory pmem = pmem_manager.alloc(8 * SegmentSize);
   EXPECT_EQ(pmem.nsegments(), 1u) << "wrong number of segments";