8339587: runtime/reflect/ReflectOutOfMemoryError.java fails with "bootstrap method initialization exception"

Reviewed-by: lmesnik, ccheung

Author: David Holmes

test/hotspot/jtreg/runtime/reflect/ReflectOutOfMemoryError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -88,15 +88,24 @@ public static void main(java.lang.String[] unused) throws Exception {
             Object junk = testMethod.invoke(null, new Object [0]);
             throw new RuntimeException("InvocationTargetException should be thrown");
         } catch (InvocationTargetException ite) {
-            Throwable targetException = ite.getTargetException();
-            if (targetException instanceof OutOfMemoryError) {
-                System.out.println("OutOfMemoryError thrown as expected.");
-                System.out.println("Test passed.");
-            } else {
-                throw new RuntimeException("Unexpected InvocationTargetException: " + targetException);
+            // We may not directly get OOME but it could have caused
+            // secondary exceptions, so walk the chain of exceptions
+            // and see if there is an OOME somewhere.
+            for (Throwable cause = ite.getTargetException();
+                 cause != null;
+                 cause = cause.getCause()) {
+                if (cause instanceof OutOfMemoryError) {
+                    System.out.println("OutOfMemoryError thrown as expected.");
+                    ite.printStackTrace(System.out);
+                    System.out.println("Test passed.");
+                    return;
+                }
             }
+
+            throw new RuntimeException("Unexpected InvocationTargetException: ",
+                                       ite.getTargetException());
         } catch (Exception exception) {
-            throw new RuntimeException("Unexpected exception: " + exception);
+            throw new RuntimeException("Unexpected exception: ", exception);
         }
     }
 }