8311557: [JVMCI] deadlock with JVMTI thread suspension

Reviewed-by: thartmann, dnsimon

Author: Tom Rodriguez

src/hotspot/share/compiler/abstractCompiler.hpp

@@ -150,6 +150,7 @@ class AbstractCompiler : public CHeapObj<mtCompiler> {
   bool is_c2() const                     { return _type == compiler_c2; }
   bool is_jvmci() const                  { return _type == compiler_jvmci; }
   CompilerType type() const              { return _type; }
+  virtual bool is_hidden_from_external_view() const { return false; }
 
   // Customization
   virtual void initialize () = 0;

src/hotspot/share/compiler/compilerThread.cpp

@@ -61,3 +61,8 @@ void CompilerThread::thread_entry(JavaThread* thread, TRAPS) {
 bool CompilerThread::can_call_java() const {
   return _compiler != nullptr && _compiler->is_jvmci();
 }
+
+// Hide native compiler threads from external view.
+bool CompilerThread::is_hidden_from_external_view() const {
+  return _compiler == nullptr || _compiler->is_hidden_from_external_view();
+}

src/hotspot/share/compiler/compilerThread.hpp

@@ -72,8 +72,9 @@ class CompilerThread : public JavaThread {
 
   virtual bool can_call_java() const;
 
-  // Hide native compiler threads from external view.
-  bool is_hidden_from_external_view() const      { return !can_call_java(); }
+  // Returns true if this CompilerThread is hidden from JVMTI and FlightRecorder.  C1 and C2 are
+  // always hidden but JVMCI compiler threads might be hidden.
+  virtual bool is_hidden_from_external_view() const;
 
   void set_compiler(AbstractCompiler* c)         { _compiler = c; }
   AbstractCompiler* compiler() const             { return _compiler; }

src/hotspot/share/jvmci/jvmciCompiler.hpp

@@ -103,6 +103,9 @@ class JVMCICompiler : public AbstractCompiler {
   bool is_c1   ()                                { return false; }
   bool is_c2   ()                                { return false; }
 
+  virtual bool is_hidden_from_external_view() const { return UseJVMCINativeLibrary && LibJVMCICompilerThreadHidden; }
+
+
   bool needs_stubs            () { return false; }
 
   // Initialization

src/hotspot/share/jvmci/jvmci_globals.cpp

@@ -73,10 +73,11 @@ bool JVMCIGlobals::check_jvmci_flags_are_consistent() {
   JVMCI_FLAG_CHECKED(EnableJVMCIProduct)
   JVMCI_FLAG_CHECKED(UseGraalJIT)
 
-  CHECK_NOT_SET(BootstrapJVMCI,   UseJVMCICompiler)
-  CHECK_NOT_SET(PrintBootstrap,   UseJVMCICompiler)
-  CHECK_NOT_SET(JVMCIThreads,     UseJVMCICompiler)
-  CHECK_NOT_SET(JVMCIHostThreads, UseJVMCICompiler)
+  CHECK_NOT_SET(BootstrapJVMCI,               UseJVMCICompiler)
+  CHECK_NOT_SET(PrintBootstrap,               UseJVMCICompiler)
+  CHECK_NOT_SET(JVMCIThreads,                 UseJVMCICompiler)
+  CHECK_NOT_SET(JVMCIHostThreads,             UseJVMCICompiler)
+  CHECK_NOT_SET(LibJVMCICompilerThreadHidden, UseJVMCICompiler)
 
   if (UseJVMCICompiler) {
     if (FLAG_IS_DEFAULT(UseJVMCINativeLibrary) && !UseJVMCINativeLibrary) {
@@ -185,6 +186,7 @@ bool JVMCIGlobals::enable_jvmci_product_mode(JVMFlagOrigin origin, bool use_graa
     "UseJVMCINativeLibrary",
     "JVMCINativeLibraryThreadFraction",
     "JVMCINativeLibraryErrorFile",
+    "LibJVMCICompilerThreadHidden",
     nullptr
   };
 

src/hotspot/share/jvmci/jvmci_globals.hpp

@@ -157,6 +157,11 @@ class fileStream;
           "error data to this file"                                         \
           "[default: ./" LIBJVMCI_ERR_FILE "] (%p replaced with pid)")      \
                                                                             \
+  product(bool, LibJVMCICompilerThreadHidden, true, EXPERIMENTAL,           \
+          "If true then native JVMCI compiler threads are hidden from "     \
+          "JVMTI and FlightRecorder.  This must be set to false if you "    \
+          "wish to use a Java debugger against JVMCI threads.")             \
+                                                                            \
   NOT_COMPILER2(product(bool, UseMultiplyToLenIntrinsic, false, DIAGNOSTIC, \
           "Enables intrinsification of BigInteger.multiplyToLen()"))        \
                                                                             \