8274911: testlibrary_tests/ir_framework/tests/TestIRMatching.java fails with "java.lang.RuntimeException: Should have thrown exception"

Backport-of: f62346066869b681d1cc9f63775393b11a48722a

Author: GoeLin

test/hotspot/jtreg/compiler/lib/ir_framework/driver/IRMatcher.java

@@ -495,7 +495,7 @@ private void reportFailuresIfAny() {
             // Do not throw an exception in this case (i.e. bailout).
             String compilations = compilationsBuilder.toString();
             if (!compilations.contains(SAFEPOINT_WHILE_PRINTING_MESSAGE)) {
-                throw new IRViolationException(failuresBuilder.toString(), compilationsBuilder.toString());
+                throw new IRViolationException(failuresBuilder.toString(), compilations);
             } else {
                 System.out.println("Found " + SAFEPOINT_WHILE_PRINTING_MESSAGE + ", bail out of IR matching");
             }

test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestIRMatching.java

@@ -31,9 +31,7 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -51,22 +49,32 @@
 
 public class TestIRMatching {
 
-    private static final List<Exception> exceptions = new ArrayList<>();
+    private static final Map<Exception, String> exceptions = new LinkedHashMap<>();
+    private static final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    private static final ByteArrayOutputStream baosErr = new ByteArrayOutputStream();
+    private static final PrintStream ps = new PrintStream(baos);
+    private static final PrintStream psErr = new PrintStream(baosErr);
+    private static final PrintStream oldOut = System.out;
+    private static final PrintStream oldErr = System.err;
 
     private static void addException(Exception e) {
-        System.out.println(TestFramework.getLastTestVMOutput());
-        exceptions.add(e);
+        System.out.flush();
+        System.err.flush();
+        exceptions.put(e, baos.toString() + System.lineSeparator() + baosErr.toString());
     }
 
     public static void main(String[] args) {
-        runFailOnTestsArgs(BadFailOnConstraint.create(AndOr1.class, "test1(int)", 1, "CallStaticJava"), "-XX:TLABRefillWasteFraction=50", "-XX:+UsePerfData", "-XX:+UseTLAB");
-        runFailOnTestsArgs(BadFailOnConstraint.create(AndOr1.class, "test2()", 1, "CallStaticJava"), "-XX:TLABRefillWasteFraction=50", "-XX:-UsePerfData", "-XX:+UseTLAB");
-
+        // Redirect System.out and System.err to reduce noise.
+        System.setOut(ps);
+        System.setErr(psErr);
         runWithArguments(AndOr1.class, "-XX:TLABRefillWasteFraction=52", "-XX:+UsePerfData", "-XX:+UseTLAB");
         runWithArguments(CountComparisons.class, "-XX:TLABRefillWasteFraction=50");
         runWithArguments(GoodCount.class, "-XX:TLABRefillWasteFraction=50");
         runWithArguments(MultipleFailOnGood.class, "-XX:TLABRefillWasteFraction=50");
 
+        runCheck(new String[] {"-XX:TLABRefillWasteFraction=50", "-XX:+UsePerfData", "-XX:+UseTLAB"}, BadFailOnConstraint.create(AndOr1.class, "test1(int)", 1, "CallStaticJava"));
+        runCheck(new String[] {"-XX:TLABRefillWasteFraction=50", "-XX:-UsePerfData", "-XX:+UseTLAB"}, BadFailOnConstraint.create(AndOr1.class, "test2()", 1, "CallStaticJava"));
+
         String[] allocMatches = { "MyClass", "wrapper for: _new_instance_Java" };
         runCheck(BadFailOnConstraint.create(MultipleFailOnBad.class, "fail1()", 1, 1, "Store"),
                  BadFailOnConstraint.create(MultipleFailOnBad.class, "fail1()", 1,  3, "Store"),
@@ -223,55 +231,49 @@ public static void main(String[] args) {
                      : BadFailOnConstraint.create(CheckCastArray.class, "arrayCopy(java.lang.Object[],java.lang.Class)", 1, "checkcast_arraycopy")
         );
 
-        // Redirect stdout to stream and then check if we find required IR encoding read from socket.
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        PrintStream ps = new PrintStream(baos);
-        PrintStream old = System.out;
-        System.setOut(ps);
-
         try {
             runWithArgumentsFail(CompilationOutputOfFails.class);
-            Utils.shouldHaveThrownException();
+            Utils.shouldHaveThrownException(baos.toString());
         } catch (IRViolationException e) {
             try {
-                boolean failed = false;
+                StringBuilder failures = new StringBuilder();
                 System.out.flush();
                 String output = baos.toString();
                 baos.reset();
                 Pattern pattern = Pattern.compile(">>> Compilation.*both\\d.*\\RPrintIdeal:(?:(?!PrintOpto|>>> Compilation)[\\S\\s])+PrintOptoAssembly");
                 Matcher matcher = pattern.matcher(output);
                 long bothCount = matcher.results().count();
                 if (bothCount != 7L) {
-                    exceptions.add(new RuntimeException("Could not find all both() methods, expected 7 but found " + bothCount));
-                    failed = true;
+                    failures.append("- Could not find all both() methods, expected 7 but found ").append(bothCount).append(System.lineSeparator());
                 }
                 pattern = Pattern.compile(">>> Compilation.*ideal\\d.*\\RPrintIdeal:(?:(?!>>> Compilation)[\\S\\s])+");
                 matcher = pattern.matcher(output);
                 int count = 0;
                 while (matcher.find()) {
                     String match = matcher.group();
-                    Asserts.assertFalse(match.contains("PrintOptoAssembly"), "Cannot contain opto assembly: " + output);
+                    if (match.contains("PrintOptoAssembly")) {
+                        failures.append("Cannot contain opto assembly: ").append(System.lineSeparator()).append(match);
+                    }
                     count++;
                 }
                 if (count != 7) {
-                    exceptions.add(new RuntimeException("Could not find all ideal() methods, expected 7 but found " + count));
-                    failed = true;
+                    failures.append("- Could not find all ideal() methods, expected 7 but found ").append(count).append(System.lineSeparator());
                 }
                 pattern = Pattern.compile(">>> Compilation.*opto\\d.*\\RPrintOptoAssembly:(?:(?!>>> Compilation)[\\S\\s])+");
                 matcher = pattern.matcher(output);
                 count = 0;
                 while (matcher.find()) {
                     String match = matcher.group();
-                    Asserts.assertFalse(match.contains("PrintIdeal"), "Cannot contain opto assembly: " + output);
+                    if (match.contains("PrintIdeal")) {
+                        failures.append("Cannot contain print assembly: ").append(System.lineSeparator()).append(match);
+                    }
                     count++;
                 }
                 if (count != 7) {
-                    exceptions.add(new RuntimeException("Could not find all opto() methods, expected 7 but found " + count));
-                    failed = true;
+                    failures.append("- Could not find all opto() methods, expected 7 but found ").append(count).append(System.lineSeparator());
                 }
-                if (failed) {
-                    System.err.println(TestFramework.getLastTestVMOutput());
-                    System.err.println(output);
+                if (!failures.isEmpty()) {
+                    addException(new RuntimeException(failures.toString()));
                 }
             } catch (Exception e1) {
                 addException(e1);
@@ -283,52 +285,80 @@ public static void main(String[] args) {
         runWithArguments(FlagComparisons.class, "-XX:TLABRefillWasteFraction=50");
         System.out.flush();
         String output = baos.toString();
-        baos.reset();
         findIrIds(output, "testMatchAllIf50", 0, 21);
         findIrIds(output, "testMatchNoneIf50", -1, -1);
 
         runWithArguments(FlagComparisons.class, "-XX:TLABRefillWasteFraction=49");
         System.out.flush();
         output = baos.toString();
-        baos.reset();
         findIrIds(output, "testMatchAllIf50", 4, 6, 13, 18);
         findIrIds(output, "testMatchNoneIf50", 0, 3, 8, 10, 17, 22);
 
         runWithArguments(FlagComparisons.class, "-XX:TLABRefillWasteFraction=51");
         System.out.flush();
         output = baos.toString();
-        baos.reset();
         findIrIds(output, "testMatchAllIf50", 7, 12, 19, 21);
         findIrIds(output, "testMatchNoneIf50", 4, 7, 11, 16, 20, 22);
-        System.setOut(old);
+        System.setOut(oldOut);
+        System.setErr(oldErr);
 
         if (!exceptions.isEmpty()) {
-            System.err.println("TestIRMatching failed with one or more exceptions:");
-            for (Exception e : exceptions) {
-                System.err.println(e.getMessage());
+            System.err.println("TestIRMatching failed with " + exceptions.size() + " exception(s):");
+            int i = 1;
+            System.err.println("************************");
+            for (Map.Entry<Exception, String> entry : exceptions.entrySet()) {
+                System.err.println("***** Exception " + String.format("%02d", i++) +" *****");
+                System.err.println("************************");
+
+                Exception e = entry.getKey();
                 e.printStackTrace(System.err);
-                System.err.println("---------");
+                System.err.println();
+                System.err.println("===== OUTPUT ======");
+                System.err.println(entry.getValue());
+                System.err.println("MESSAGE: " + e.getMessage());
+                System.err.println("************************");
             }
-            throw new RuntimeException("TestIRMatching failed with one or more exceptions - check stderr and stdout");
+            i = 1;
+            System.err.println("====================================");
+            System.err.println("********************");
+            System.err.println("***** OVERVIEW *****");
+            System.err.println("********************");
+            for (Map.Entry<Exception, String> entry : exceptions.entrySet()) {
+                Exception e = entry.getKey();
+                System.err.print((i++) + ") ");
+                entry.getKey().printStackTrace(System.err);
+                System.err.println("********************");
+            }
+            throw new RuntimeException("TestIRMatching failed with " + exceptions.size() + " exception(s) - check stderr and stdout");
         }
     }
 
+    private static void runFramework(TestFramework framework) {
+        baos.reset();
+        baosErr.reset();
+        framework.start();
+    }
+
     private static void runWithArguments(Class<?> clazz, String... args) {
         try {
-            new TestFramework(clazz).addFlags(args).start();
+            runFramework(new TestFramework(clazz).addFlags(args));
         } catch (Exception e) {
             addException(e);
         }
     }
 
     private static void runWithArgumentsFail(Class<?> clazz, String... args) {
-        new TestFramework(clazz).addFlags(args).start();
+        runFramework(new TestFramework(clazz).addFlags(args));
     }
 
     private static void runCheck(String[] args , Constraint... constraints) {
         try {
-            new TestFramework(constraints[0].getKlass()).addFlags(args).start(); // All constraints have the same class.
-            Utils.shouldHaveThrownException();
+            TestFramework framework = new TestFramework(constraints[0].getKlass()); // All constraints have the same class.
+            if (args != null) {
+                framework.addFlags(args);
+            }
+            runFramework(framework);
+            Utils.shouldHaveThrownException(baos.toString());
         } catch (IRViolationException e) {
             checkConstraints(e, constraints);
         } catch (Exception e) {
@@ -337,14 +367,7 @@ private static void runCheck(String[] args , Constraint... constraints) {
     }
 
     private static void runCheck(Constraint... constraints) {
-        try {
-            TestFramework.run(constraints[0].getKlass()); // All constraints have the same class.
-            Utils.shouldHaveThrownException();
-        } catch (IRViolationException e) {
-            checkConstraints(e, constraints);
-        } catch (Exception e) {
-            addException(e);
-        }
+        runCheck(null, constraints);
     }
 
     private static void checkConstraints(IRViolationException e, Constraint[] constraints) {
@@ -354,25 +377,9 @@ private static void checkConstraints(IRViolationException e, Constraint[] constr
                 constraint.checkConstraint(e);
             }
         } catch (Exception e1) {
-            System.out.println(TestFramework.getLastTestVMOutput());
+            System.out.println(e.getCompilations());
             System.out.println(message);
-            exceptions.add(e1);
-        }
-    }
-
-    // Single constraint
-    private static void runFailOnTestsArgs(Constraint constraint, String... args) {
-        try {
-            new TestFramework(constraint.getKlass()).addFlags(args).start(); // All constraints have the same class.
-            Utils.shouldHaveThrownException();
-        } catch (IRViolationException e) {
-            try {
-                constraint.checkConstraint(e);
-            } catch (Exception e1) {
-                addException(e);
-            }
-        } catch (Exception e) {
-            addException(e);
+            addException(e1);
         }
     }
 
@@ -387,8 +394,9 @@ public static void findIrIds(String output, String method, int... numbers) {
                 builder.append(j);
             }
         }
-        Asserts.assertTrue(output.contains(builder.toString()), "Could not find encoding: \"" + builder.toString()
-                                                                + System.lineSeparator());
+        if (!output.contains(builder.toString())) {
+            addException(new RuntimeException("Could not find encoding: \"" + builder.toString() + System.lineSeparator()));
+        }
     }
 }
 

test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestRunTests.java

@@ -28,6 +28,8 @@
 import compiler.lib.ir_framework.shared.TestRunException;
 import jdk.test.lib.Asserts;
 
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.Arrays;
 
 /*
@@ -41,15 +43,22 @@
 public class TestRunTests {
 
     public static void main(String[] args) {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        PrintStream ps = new PrintStream(baos);
+        PrintStream oldOut = System.out;
+        System.setOut(ps);
+
         TestFramework.run();
         try {
             TestFramework.run(BadStandalone.class);
-            Utils.shouldHaveThrownException();
+            Utils.shouldHaveThrownException(baos.toString());
         } catch (IRViolationException e) {
+            System.setOut(oldOut);
             String[] matches = { "test(int)", "test2(int)", "Failed IR Rules (2)"};
             Arrays.stream(matches).forEach(m -> Asserts.assertTrue(e.getExceptionInfo().contains(m)));
             Asserts.assertEQ(e.getExceptionInfo().split("STANDALONE mode", -1).length - 1, 2);
         }
+        System.setOut(oldOut);
         new TestFramework(SkipCompilation.class).addFlags("-XX:-UseCompiler").start();
         new TestFramework(SkipCompilation.class).addFlags("-Xint").start();
         new TestFramework(SkipC2Compilation.class).addFlags("-XX:TieredStopAtLevel=1").start();

test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/Utils.java

@@ -31,10 +31,10 @@
 import java.util.Arrays;
 
 public class Utils {
-    public static void shouldHaveThrownException() {
+    public static void shouldHaveThrownException(String s) {
         // Do not throw an exception if we hit a safepoint while printing which could possibly let the IR matching fail.
         // This happens very rarely. If there is a problem with the test, then we will catch that on the next test invocation.
-        if (!TestVMProcess.getLastTestVMOutput().contains(IRMatcher.SAFEPOINT_WHILE_PRINTING_MESSAGE)) {
+        if (!s.contains(IRMatcher.SAFEPOINT_WHILE_PRINTING_MESSAGE)) {
             Asserts.fail("Should have thrown exception");
         }
     }