8267341: macos attempt_reserve_memory_at(arg1, arg2, true) failure

8271898: disable os.release_multi_mappings_vm on macOS-X64

Reviewed-by: lucy
Backport-of: d1efb0cc56127a40f2b23ccc608b27be2b39a389

test/hotspot/gtest/runtime/test_os.cpp

@@ -352,6 +352,23 @@ TEST_VM(os, jio_snprintf) {
   test_snprintf(jio_snprintf, false);
 }
 
+#ifdef __APPLE__
+// Not all macOS versions can use os::reserve_memory (i.e. anon_mmap) API
+// to reserve executable memory, so before attempting to use it,
+// we need to verify that we can do so by asking for a tiny executable
+// memory chunk.
+static inline bool can_reserve_executable_memory(void) {
+  bool executable = true;
+  size_t len = 128;
+  char* p = os::reserve_memory(len, executable);
+  bool exec_supported = (p != NULL);
+  if (exec_supported) {
+    os::release_memory(p, len);
+  }
+  return exec_supported;
+}
+#endif
+
 // Test that os::release_memory() can deal with areas containing multiple mappings.
 #define PRINT_MAPPINGS(s) { tty->print_cr("%s", s); os::print_memory_mappings((char*)p, total_range_len, tty); }
 //#define PRINT_MAPPINGS
@@ -361,6 +378,13 @@ TEST_VM(os, jio_snprintf) {
 //  (from multiple calls to os::reserve_memory)
 static address reserve_multiple(int num_stripes, size_t stripe_len) {
   assert(is_aligned(stripe_len, os::vm_allocation_granularity()), "Sanity");
+
+#ifdef __APPLE__
+  // Workaround: try reserving executable memory to figure out
+  // if such operation is supported on this macOS version
+  const bool exec_supported = can_reserve_executable_memory();
+#endif
+
   size_t total_range_len = num_stripes * stripe_len;
   // Reserve a large contiguous area to get the address space...
   address p = (address)os::reserve_memory(total_range_len);
@@ -372,7 +396,11 @@ static address reserve_multiple(int num_stripes, size_t stripe_len) {
     address q = p + (stripe * stripe_len);
     // Commit, alternatingly with or without exec permission,
     //  to prevent kernel from folding these mappings.
+#ifdef __APPLE__
+    const bool executable = exec_supported ? (stripe % 2 == 0) : false;
+#else
     const bool executable = stripe % 2 == 0;
+#endif
     q = (address)os::attempt_reserve_memory_at((char*)q, stripe_len, executable);
     EXPECT_NE(q, (address)NULL);
     EXPECT_TRUE(os::commit_memory((char*)q, stripe_len, executable));
@@ -414,11 +442,7 @@ struct NUMASwitcher {
 #endif
 
 #ifndef _AIX // JDK-8257041
-#if defined(__APPLE__) && !defined(AARCH64) // JDK-8267339
-  TEST_VM(os, DISABLED_release_multi_mappings) {
-#else
   TEST_VM(os, release_multi_mappings) {
-#endif
 
   // With NMT enabled, this will trigger JDK-8263464. For now disable the test if NMT=on.
   if (MemTracker::tracking_level() > NMT_off) {