archive/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2025-09-18 01:54:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

8262291: Refactor reserve_memory_special_huge_tlbfs

Browse source 
Reviewed-by: iwalulya, stuefe
This commit is contained in:
Stefan Johansson 2021-04-13 08:59:09 +00:00
parent 008fc75a29
commit f2f7aa3bec
3 changed files with 103 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

186
src/hotspot/os/linux/os_linux.cpp
View file

@ -3898,8 +3898,8 @@ char* os::Linux::reserve_memory_special_shm(size_t bytes, size_t alignment,
return addr;
}
static void warn_on_large_pages_failure(char* req_addr, size_t bytes,
int error) {
static void warn_on_commit_special_failure(char* req_addr, size_t bytes,
size_t page_size, int error) {
assert(error == ENOMEM, "Only expect to fail if no memory is available");
bool warn_on_failure = UseLargePages &&
@ -3909,122 +3909,45 @@ static void warn_on_large_pages_failure(char* req_addr, size_t bytes,
if (warn_on_failure) {
char msg[128];
jio_snprintf(msg, sizeof(msg), "Failed to reserve large pages memory req_addr: "
PTR_FORMAT " bytes: " SIZE_FORMAT " (errno = %d).", req_addr, bytes, error);
jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory. req_addr: "
PTR_FORMAT " bytes: " SIZE_FORMAT " page size: "
SIZE_FORMAT " (errno = %d).",
req_addr, bytes, page_size, error);
warning("%s", msg);
}
}
char* os::Linux::reserve_memory_special_huge_tlbfs_only(size_t bytes,
char* req_addr,
bool exec) {
assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
assert(is_aligned(bytes, os::large_page_size()), "Unaligned size");
assert(is_aligned(req_addr, os::large_page_size()), "Unaligned address");
int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
int flags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB;
// Ensure the correct page size flag is used when needed.
flags |= hugetlbfs_page_size_flag(os::large_page_size());
char* addr = (char*)::mmap(req_addr, bytes, prot, flags, -1, 0);
if (addr == MAP_FAILED) {
warn_on_large_pages_failure(req_addr, bytes, errno);
return NULL;
}
assert(is_aligned(addr, os::large_page_size()), "Must be");
return addr;
}
// Reserve memory using mmap(MAP_HUGETLB).
// - bytes shall be a multiple of alignment.
// - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
// - alignment sets the alignment at which memory shall be allocated.
// It must be a multiple of allocation granularity.
// Returns address of memory or NULL. If req_addr was not NULL, will only return
// req_addr or NULL.
char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
size_t alignment,
char* req_addr,
bool exec) {
size_t large_page_size = os::large_page_size();
assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
assert(is_aligned(req_addr, alignment), "Must be");
assert(is_aligned(bytes, alignment), "Must be");
// First reserve - but not commit - the address range in small pages.
char* const start = anon_mmap_aligned(req_addr, bytes, alignment);
if (start == NULL) {
return NULL;
}
assert(is_aligned(start, alignment), "Must be");
char* end = start + bytes;
// Find the regions of the allocated chunk that can be promoted to large pages.
char* lp_start = align_up(start, large_page_size);
char* lp_end = align_down(end, large_page_size);
size_t lp_bytes = lp_end - lp_start;
assert(is_aligned(lp_bytes, large_page_size), "Must be");
if (lp_bytes == 0) {
// The mapped region doesn't even span the start and the end of a large page.
// Fall back to allocate a non-special area.
::munmap(start, end - start);
return NULL;
}
bool os::Linux::commit_memory_special(size_t bytes,
size_t page_size,
char* req_addr,
bool exec) {
assert(UseLargePages && UseHugeTLBFS, "Should only get here when HugeTLBFS large pages are used");
assert(is_aligned(bytes, page_size), "Unaligned size");
assert(is_aligned(req_addr, page_size), "Unaligned address");
assert(req_addr != NULL, "Must have a requested address for special mappings");
int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
int flags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED;
void* result;
// Commit small-paged leading area.
if (start != lp_start) {
result = ::mmap(start, lp_start - start, prot, flags, -1, 0);
if (result == MAP_FAILED) {
::munmap(lp_start, end - lp_start);
return NULL;
}
// For large pages additional flags are required.
if (page_size > (size_t) os::vm_page_size()) {
flags |= MAP_HUGETLB | hugetlbfs_page_size_flag(page_size);
}
char* addr = (char*)::mmap(req_addr, bytes, prot, flags, -1, 0);
if (addr == MAP_FAILED) {
warn_on_commit_special_failure(req_addr, bytes, page_size, errno);
return false;
}
// Commit large-paged area.
flags |= MAP_HUGETLB | hugetlbfs_page_size_flag(os::large_page_size());
result = ::mmap(lp_start, lp_bytes, prot, flags, -1, 0);
if (result == MAP_FAILED) {
warn_on_large_pages_failure(lp_start, lp_bytes, errno);
// If the mmap above fails, the large pages region will be unmapped and we
// have regions before and after with small pages. Release these regions.
//
// | mapped | unmapped | mapped |
// ^ ^ ^ ^
// start lp_start lp_end end
//
::munmap(start, lp_start - start);
::munmap(lp_end, end - lp_end);
return NULL;
}
// Commit small-paged trailing area.
if (lp_end != end) {
result = ::mmap(lp_end, end - lp_end, prot,
MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,
-1, 0);
if (result == MAP_FAILED) {
::munmap(start, lp_end - start);
return NULL;
}
}
return start;
log_debug(pagesize)("Commit special mapping: " PTR_FORMAT ", size=" SIZE_FORMAT "%s, page size="
SIZE_FORMAT "%s",
p2i(addr), byte_size_in_exact_unit(bytes),
exact_unit_for_byte_size(bytes),
byte_size_in_exact_unit(page_size),
exact_unit_for_byte_size(page_size));
assert(is_aligned(addr, page_size), "Must be");
return true;
}
char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes,
@ -4033,15 +3956,54 @@ char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes,
bool exec) {
assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
assert(is_aligned(req_addr, alignment), "Must be");
assert(is_aligned(req_addr, os::large_page_size()), "Must be");
assert(is_aligned(alignment, os::vm_allocation_granularity()), "Must be");
assert(is_power_of_2(os::large_page_size()), "Must be");
assert(bytes >= os::large_page_size(), "Shouldn't allocate large pages for small sizes");
if (is_aligned(bytes, os::large_page_size()) && alignment <= os::large_page_size()) {
return reserve_memory_special_huge_tlbfs_only(bytes, req_addr, exec);
} else {
return reserve_memory_special_huge_tlbfs_mixed(bytes, alignment, req_addr, exec);
// We only end up here when at least 1 large page can be used.
// If the size is not a multiple of the large page size, we
// will mix the type of pages used, but in a decending order.
// Start off by reserving a range of the given size that is
// properly aligned. At this point no pages are committed. If
// a requested address is given it will be used and it must be
// aligned to both the large page size and the given alignment.
// The larger of the two will be used.
size_t required_alignment = MAX(os::large_page_size(), alignment);
char* const aligned_start = anon_mmap_aligned(req_addr, bytes, required_alignment);
if (aligned_start == NULL) {
return NULL;
}
// First commit using large pages.
size_t large_bytes = align_down(bytes, os::large_page_size());
bool large_committed = commit_memory_special(large_bytes, os::large_page_size(), aligned_start, exec);
if (large_committed && bytes == large_bytes) {
// The size was large page aligned so no additional work is
// needed even if the commit failed.
return aligned_start;
}
// The requested size requires some small pages as well.
char* small_start = aligned_start + large_bytes;
size_t small_size = bytes - large_bytes;
if (!large_committed) {
// Failed to commit large pages, so we need to unmap the
// reminder of the orinal reservation.
::munmap(small_start, small_size);
return NULL;
}
// Commit the remaining bytes using small pages.
bool small_committed = commit_memory_special(small_size, os::vm_page_size(), small_start, exec);
if (!small_committed) {
// Failed to commit the remaining size, need to unmap
// the large pages part of the reservation.
::munmap(aligned_start, large_bytes);
return NULL;
}
return aligned_start;
}
char* os::pd_reserve_memory_special(size_t bytes, size_t alignment,