8320368: Per-CPU optimization of Klass range reservation

Reviewed-by: rkennke, rehn

Author: tstuefe

src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "logging/log.hpp"
+#include "oops/compressedKlass.hpp"
+#include "memory/metaspace.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Helper function; reserve at an address that is compatible with EOR
+static char* reserve_at_eor_compatible_address(size_t size, bool aslr) {
+  char* result = nullptr;
+
+  log_debug(metaspace, map)("Trying to reserve at an EOR-compatible address");
+
+  // We need immediates that are 32-bit aligned, since they should not intersect nKlass
+  // bits. They should not be larger than the addressable space either, but we still
+  // lack a good abstraction for that (see JDK-8320584), therefore we assume and hard-code
+  // 2^48 as a reasonable higher ceiling.
+  static const uint16_t immediates[] = {
+      0x0001, 0x0002, 0x0003, 0x0004, 0x0006, 0x0007, 0x0008, 0x000c, 0x000e,
+      0x000f, 0x0010, 0x0018, 0x001c, 0x001e, 0x001f, 0x0020, 0x0030, 0x0038,
+      0x003c, 0x003e, 0x003f, 0x0040, 0x0060, 0x0070, 0x0078, 0x007c, 0x007e,
+      0x007f, 0x0080, 0x00c0, 0x00e0, 0x00f0, 0x00f8, 0x00fc, 0x00fe, 0x00ff,
+      0x0100, 0x0180, 0x01c0, 0x01e0, 0x01f0, 0x01f8, 0x01fc, 0x01fe, 0x01ff,
+      0x0200, 0x0300, 0x0380, 0x03c0, 0x03e0, 0x03f0, 0x03f8, 0x03fc, 0x03fe,
+      0x03ff, 0x0400, 0x0600, 0x0700, 0x0780, 0x07c0, 0x07e0, 0x07f0, 0x07f8,
+      0x07fc, 0x07fe, 0x07ff, 0x0800, 0x0c00, 0x0e00, 0x0f00, 0x0f80, 0x0fc0,
+      0x0fe0, 0x0ff0, 0x0ff8, 0x0ffc, 0x0ffe, 0x0fff, 0x1000, 0x1800, 0x1c00,
+      0x1e00, 0x1f00, 0x1f80, 0x1fc0, 0x1fe0, 0x1ff0, 0x1ff8, 0x1ffc, 0x1ffe,
+      0x1fff, 0x2000, 0x3000, 0x3800, 0x3c00, 0x3e00, 0x3f00, 0x3f80, 0x3fc0,
+      0x3fe0, 0x3ff0, 0x3ff8, 0x3ffc, 0x3ffe, 0x3fff, 0x4000, 0x6000, 0x7000,
+      0x7800, 0x7c00, 0x7e00, 0x7f00, 0x7f80, 0x7fc0, 0x7fe0, 0x7ff0, 0x7ff8,
+      0x7ffc, 0x7ffe, 0x7fff
+  };
+  static constexpr int num_immediates = sizeof(immediates) / sizeof(immediates[0]);
+  const int start_index = aslr ? os::random() : 0;
+  constexpr int max_tries = 64;
+  for (int ntry = 0; result == nullptr && ntry < max_tries; ntry ++) {
+    // As in os::attempt_reserve_memory_between, we alternate between higher and lower
+    // addresses; this maximizes the chance of early success if part of the address space
+    // is not accessible (e.g. 39-bit address space).
+    const int alt_index = (ntry & 1) ? 0 : num_immediates / 2;
+    const int index = (start_index + ntry + alt_index) % num_immediates;
+    const uint64_t immediate = ((uint64_t)immediates[index]) << 32;
+    assert(immediate > 0 && Assembler::operand_valid_for_logical_immediate(/*is32*/false, immediate),
+           "Invalid immediate %d " UINT64_FORMAT, index, immediate);
+    result = os::attempt_reserve_memory_at((char*)immediate, size, false);
+    if (result == nullptr) {
+      log_trace(metaspace, map)("Failed to attach at " UINT64_FORMAT_X, immediate);
+    }
+  }
+  if (result == nullptr) {
+    log_debug(metaspace, map)("Failed to reserve at any EOR-compatible address");
+  }
+  return result;
+}
+char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
+
+  char* result = nullptr;
+
+  // Optimize for base=0 shift=0
+  if (optimize_for_zero_base) {
+    result = reserve_address_space_for_unscaled_encoding(size, aslr);
+  }
+
+  // If this fails, we don't bother aiming for zero-based encoding (base=0 shift>0), since it has no
+  // advantages over EOR or movk mode.
+
+  // EOR-compatible reservation
+  if (result == nullptr) {
+    result = reserve_at_eor_compatible_address(size, aslr);
+  }
+
+  // Movk-compatible reservation via probing.
+  if (result == nullptr) {
+    result = reserve_address_space_for_16bit_move(size, aslr);
+  }
+
+  // Movk-compatible reservation via overallocation.
+  // If that failed, attempt to allocate at any 4G-aligned address. Let the system decide where. For ASLR,
+  // we now rely on the system.
+  // Compared with the probing done above, this has two disadvantages:
+  // - on a kernel with 52-bit address space we may get an address that has bits set between [48, 52).
+  //   In that case, we may need two movk moves (not yet implemented).
+  // - this technique leads to temporary over-reservation of address space; it will spike the vsize of
+  //   the process. Therefore it may fail if a vsize limit is in place (e.g. ulimit -v).
+  if (result == nullptr) {
+    constexpr size_t alignment = nth_bit(32);
+    log_debug(metaspace, map)("Trying to reserve at a 32-bit-aligned address");
+    result = os::reserve_memory_aligned(size, alignment, false);
+  }
+
+  return result;
+}
+
+void CompressedKlassPointers::initialize(address addr, size_t len) {
+  constexpr uintptr_t unscaled_max = nth_bit(32);
+  assert(len <= unscaled_max, "Klass range larger than 32 bits?");
+
+  // Shift is always 0 on aarch64.
+  _shift = 0;
+
+  // On aarch64, we don't bother with zero-based encoding (base=0 shift>0).
+  address const end = addr + len;
+  _base = (end <= (address)unscaled_max) ? nullptr : addr;
+
+  _range = end - _base;
+}

src/hotspot/cpu/ppc/compressedKlass_ppc.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/compressedKlass.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
+
+  char* result = nullptr;
+
+  // Optimize for base=0 shift=0; failing that, for base=0 shift>0
+  if (optimize_for_zero_base) {
+    result = reserve_address_space_for_unscaled_encoding(size, aslr);
+    if (result == nullptr) {
+      result = reserve_address_space_for_zerobased_encoding(size, aslr);
+    }
+  }
+
+  // Optimize for a single 16-bit move: a base that has only bits set in its third quadrant [32..48).
+  if (result == nullptr) {
+    result = reserve_address_space_for_16bit_move(size, aslr);
+  }
+
+  return result;
+}

src/hotspot/cpu/riscv/compressedKlass_riscv.cpp

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/compressedKlass.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
+
+  char* result = nullptr;
+
+  // RiscV loads a 64-bit immediate in up to four separate steps, splitting it into four different sections
+  // (two 32-bit sections, each split into two subsections of 20/12 bits).
+  //
+  // 63 ....... 44 43 ... 32 31 ....... 12 11 ... 0
+  //       D           C          B           A
+  //
+  // A "good" base is, in this order:
+  // 1) only bits in A; this would be an address < 4KB, which is unrealistic on normal Linux boxes since
+  //    the typical default for vm.mmap_min_address is 64KB. We ignore that.
+  // 2) only bits in B: a 12-bit-aligned address below 4GB. 12 bit = 4KB, but since mmap reserves at
+  //    page boundaries, we can ignore the alignment.
+  // 3) only bits in C: a 4GB-aligned address that is lower than 16TB.
+  // 4) only bits in D: a 16TB-aligned address.
+
+  // First, attempt to allocate < 4GB. We do this unconditionally:
+  // - if can_optimize_for_zero_base, a <4GB mapping start would allow us to run unscaled (base = 0, shift = 0)
+  // - if !can_optimize_for_zero_base, a <4GB mapping start is still good, the resulting immediate can be encoded
+  //   with one instruction (2)
+  result = reserve_address_space_for_unscaled_encoding(size, aslr);
+
+  // Failing that, attempt to reserve for base=zero shift>0
+  if (result == nullptr && optimize_for_zero_base) {
+    result = reserve_address_space_for_zerobased_encoding(size, aslr);
+  }
+
+  // Failing that, optimize for case (3) - a base with only bits set between [33-44)
+  if (result == nullptr) {
+    const uintptr_t from = nth_bit(32 + (optimize_for_zero_base ? LogKlassAlignmentInBytes : 0));
+    constexpr uintptr_t to = nth_bit(44);
+    constexpr size_t alignment = nth_bit(32);
+    result = reserve_address_space_X(from, to, size, alignment, aslr);
+  }
+
+  // Failing that, optimize for case (4) - a base with only bits set between [44-64)
+  if (result == nullptr) {
+    constexpr uintptr_t from = nth_bit(44);
+    constexpr uintptr_t to = UINT64_MAX;
+    constexpr size_t alignment = nth_bit(44);
+    result = reserve_address_space_X(from, to, size, alignment, aslr);
+  }
+
+  return result;
+}

src/hotspot/cpu/s390/compressedKlass_s390.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/compressedKlass.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
+
+  char* result = nullptr;
+
+  uintptr_t tried_below = 0;
+
+  // First, attempt to allocate < 4GB. We do this unconditionally:
+  // - if optimize_for_zero_base, a <4GB mapping start allows us to use base=0 shift=0
+  // - if !optimize_for_zero_base, a <4GB mapping start allows us to use algfi
+  result = reserve_address_space_for_unscaled_encoding(size, aslr);
+
+  // Failing that, try optimized for base=0 shift>0
+  if (result == nullptr && optimize_for_zero_base) {
+    result = reserve_address_space_for_zerobased_encoding(size, aslr);
+  }
+
+  // Failing that, aim for a base that is 4G-aligned; such a base can be set with aih.
+  if (result == nullptr) {
+    result = reserve_address_space_for_16bit_move(size, aslr);
+  }
+
+  return result;
+}

src/hotspot/cpu/x86/compressedKlass_x86.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#ifdef _LP64
+
+#include "oops/compressedKlass.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
+
+  char* result = nullptr;
+
+  // Optimize for unscaled encoding; failing that, for zero-based encoding:
+  if (optimize_for_zero_base) {
+    result = reserve_address_space_for_unscaled_encoding(size, aslr);
+    if (result == nullptr) {
+      result = reserve_address_space_for_zerobased_encoding(size, aslr);
+    }
+  } // end: low-address reservation
+
+  // Nothing more to optimize for on x64. If base != 0, we will always emit the full 64-bit immediate.
+  return result;
+}
+
+#endif // _LP64

src/hotspot/os/posix/os_posix.cpp

@@ -343,9 +343,10 @@ char* os::replace_existing_mapping_with_file_mapping(char* base, size_t size, in
 }
 
 static size_t calculate_aligned_extra_size(size_t size, size_t alignment) {
-  assert((alignment & (os::vm_allocation_granularity() - 1)) == 0,
+  assert(is_aligned(alignment, os::vm_allocation_granularity()),
       "Alignment must be a multiple of allocation granularity (page size)");
-  assert((size & (alignment -1)) == 0, "size must be 'alignment' aligned");
+  assert(is_aligned(size, os::vm_allocation_granularity()),
+      "Size must be a multiple of allocation granularity (page size)");
 
   size_t extra_size = size + alignment;
   assert(extra_size >= size, "overflow, size is too large to allow alignment");

src/hotspot/os/windows/os_windows.cpp

@@ -3331,9 +3331,10 @@ char* os::replace_existing_mapping_with_file_mapping(char* base, size_t size, in
 // virtual space to get requested alignment, like posix-like os's.
 // Windows prevents multiple thread from remapping over each other so this loop is thread-safe.
 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int file_desc) {
-  assert((alignment & (os::vm_allocation_granularity() - 1)) == 0,
-         "Alignment must be a multiple of allocation granularity (page size)");
-  assert((size & (alignment -1)) == 0, "size must be 'alignment' aligned");
+  assert(is_aligned(alignment, os::vm_allocation_granularity()),
+      "Alignment must be a multiple of allocation granularity (page size)");
+  assert(is_aligned(size, os::vm_allocation_granularity()),
+      "Size must be a multiple of allocation granularity (page size)");
 
   size_t extra_size = size + alignment;
   assert(extra_size >= size, "overflow, size is too large to allow alignment");