8291873: Implement return value normalization in Java

Reviewed-by: mcimadamore

src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp

@@ -208,23 +208,7 @@ void DowncallStubGenerator::generate() {
   __ blr(_abi._target_addr_reg);
   // this call is assumed not to have killed rthread
 
-  if (!_needs_return_buffer) {
-    // Unpack native results.
-    switch (_ret_bt) {
-      case T_BOOLEAN: __ c2bool(r0);                     break;
-      case T_CHAR   : __ ubfx(r0, r0, 0, 16);            break;
-      case T_BYTE   : __ sbfx(r0, r0, 0, 8);             break;
-      case T_SHORT  : __ sbfx(r0, r0, 0, 16);            break;
-      case T_INT    : __ sbfx(r0, r0, 0, 32);            break;
-      case T_DOUBLE :
-      case T_FLOAT  :
-        // Result is in v0 we'll save as needed
-        break;
-      case T_VOID: break;
-      case T_LONG: break;
-      default       : ShouldNotReachHere();
-    }
-  } else {
+  if (_needs_return_buffer) {
     assert(ret_buf_addr_sp_offset != -1, "no return buffer addr spill");
     __ ldr(tmp1, Address(sp, ret_buf_addr_sp_offset));
     int offset = 0;

src/hotspot/cpu/x86/downcallLinker_x86_64.cpp

@@ -209,24 +209,7 @@ void DowncallStubGenerator::generate() {
   __ call(_abi._target_addr_reg);
   // this call is assumed not to have killed r15_thread
 
-  if (!_needs_return_buffer) {
-    // FIXME: this assumes we return in rax/xmm0, which might not be the case
-    // Unpack native results.
-    switch (_ret_bt) {
-      case T_BOOLEAN: __ c2bool(rax);            break;
-      case T_CHAR   : __ movzwl(rax, rax);       break;
-      case T_BYTE   : __ sign_extend_byte (rax); break;
-      case T_SHORT  : __ sign_extend_short(rax); break;
-      case T_INT    : /* nothing to do */        break;
-      case T_DOUBLE :
-      case T_FLOAT  :
-        // Result is in xmm0 we'll save as needed
-        break;
-      case T_VOID: break;
-      case T_LONG: break;
-      default       : ShouldNotReachHere();
-    }
-  } else {
+  if (_needs_return_buffer) {
     assert(ret_buf_addr_rsp_offset != -1, "no return buffer addr spill");
     __ movptr(rscratch1, Address(rsp, ret_buf_addr_rsp_offset));
     int offset = 0;

src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java

@@ -52,24 +52,32 @@ private NativeMethodHandle(MethodType type, LambdaForm form, NativeEntryPoint ne
      */
     public static MethodHandle make(NativeEntryPoint nep) {
         MethodType type = nep.type();
-        if (!allTypesPrimitive(type))
-            throw new IllegalArgumentException("Type must only contain primitives: " + type);
+        if (hasIllegalType(type))
+            throw new IllegalArgumentException("Illegal type(s) found: " + type);
 
 
         LambdaForm lform = preparedLambdaForm(type);
         return new NativeMethodHandle(type, lform, nep);
     }
 
-    private static boolean allTypesPrimitive(MethodType type) {
-        if (!type.returnType().isPrimitive())
-            return false;
+    private static boolean hasIllegalType(MethodType type) {
+        if (isIllegalType(type.returnType()))
+            return true;
 
         for (Class<?> pType : type.parameterArray()) {
-            if (!pType.isPrimitive())
-                return false;
+            if (isIllegalType(pType))
+                return true;
         }
 
-        return true;
+        return false;
+    }
+
+    private static boolean isIllegalType(Class<?> pType) {
+        return !(pType == long.class
+              || pType == int.class
+              || pType == float.class
+              || pType == double.class
+              || pType == void.class);
     }
 
     private static final MemberName.Factory IMPL_NAMES = MemberName.getFactory();

src/java.base/share/classes/jdk/internal/foreign/Utils.java

@@ -128,7 +128,7 @@ static VarHandle put(ValueLayout layout, VarHandle handle) {
         return VarHandleCache.put(layout, handle);
     }
 
-    private static boolean byteToBoolean(byte b) {
+    public static boolean byteToBoolean(byte b) {
         return b != 0;
     }
 

src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java

@@ -30,6 +30,9 @@
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.MemorySession;
 import java.lang.foreign.SegmentAllocator;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
@@ -281,7 +284,8 @@ enum Tag {
         ALLOC_BUFFER,
         BOX_ADDRESS,
         UNBOX_ADDRESS,
-        DUP
+        DUP,
+        CAST
     }
 
     Tag tag();
@@ -351,6 +355,10 @@ static Dup dup() {
         return Dup.INSTANCE;
     }
 
+    static Binding cast(Class<?> fromType, Class<?> toType) {
+        return new Cast(fromType, toType);
+    }
+
 
     static Binding.Builder builder() {
         return new Binding.Builder();
@@ -362,13 +370,28 @@ static Binding.Builder builder() {
     class Builder {
         private final List<Binding> bindings = new ArrayList<>();
 
+        private static boolean isSubIntType(Class<?> type) {
+            return type == boolean.class || type == byte.class || type == short.class || type == char.class;
+        }
+
         public Binding.Builder vmStore(VMStorage storage, Class<?> type) {
+            if (isSubIntType(type)) {
+                bindings.add(Binding.cast(type, int.class));
+                type = int.class;
+            }
             bindings.add(Binding.vmStore(storage, type));
             return this;
         }
 
         public Binding.Builder vmLoad(VMStorage storage, Class<?> type) {
-            bindings.add(Binding.vmLoad(storage, type));
+            Class<?> loadType = type;
+            if (isSubIntType(type)) {
+                loadType = int.class;
+            }
+            bindings.add(Binding.vmLoad(storage, loadType));
+            if (isSubIntType(type)) {
+                bindings.add(Binding.cast(int.class, type));
+            }
             return this;
         }
 
@@ -672,4 +695,39 @@ public void interpret(Deque<Object> stack, BindingInterpreter.StoreFunc storeFun
             stack.push(stack.peekLast());
         }
     }
+
+    /**
+     * CAST([fromType], [toType])
+     *   Pop a [fromType] from the stack, convert it to [toType], and push the resulting
+     *   value onto the stack.
+     *
+     */
+    record Cast(Class<?> fromType, Class<?> toType) implements Binding {
+
+        @Override
+        public Tag tag() {
+            return Tag.CAST;
+        }
+
+        @Override
+        public void verify(Deque<Class<?>> stack) {
+            Class<?> actualType = stack.pop();
+            SharedUtils.checkType(actualType, fromType);
+            stack.push(toType);
+        }
+
+        @Override
+        public void interpret(Deque<Object> stack, BindingInterpreter.StoreFunc storeFunc,
+                              BindingInterpreter.LoadFunc loadFunc, Context context) {
+            Object arg = stack.pop();
+            MethodHandle converter = MethodHandles.explicitCastArguments(MethodHandles.identity(toType),
+                    MethodType.methodType(toType, fromType));
+            try {
+                Object result = converter.invoke(arg);
+                stack.push(result);
+            } catch (Throwable e) {
+                throw new InternalError(e);
+            }
+        }
+    }
 }

src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java

@@ -27,6 +27,7 @@
 import jdk.internal.foreign.MemorySessionImpl;
 import jdk.internal.foreign.NativeMemorySegmentImpl;
 import jdk.internal.foreign.Scoped;
+import jdk.internal.foreign.Utils;
 import jdk.internal.misc.VM;
 import jdk.internal.org.objectweb.asm.ClassReader;
 import jdk.internal.org.objectweb.asm.ClassWriter;
@@ -60,7 +61,6 @@
 import java.util.Deque;
 import java.util.List;
 
-import static java.lang.foreign.ValueLayout.*;
 import static java.lang.invoke.MethodType.methodType;
 import static jdk.internal.org.objectweb.asm.Opcodes.*;
 
@@ -474,6 +474,7 @@ private void doBindings(List<Binding> bindings) {
                 case BOX_ADDRESS -> emitBoxAddress((Binding.BoxAddress) binding);
                 case UNBOX_ADDRESS -> emitUnboxAddress();
                 case DUP -> emitDupBinding();
+                case CAST -> emitCast((Binding.Cast) binding);
             }
         }
     }
@@ -617,7 +618,6 @@ private void emitBufferStore(Binding.BufferStore bufferStore) {
         emitInvokeInterface(MemorySegment.class, "set", descriptor);
     }
 
-
     // VM_STORE and VM_LOAD are emulated, which is different for down/upcalls
     private void emitVMStore(Binding.VMStore vmStore) {
         Class<?> storeType = vmStore.type();
@@ -668,12 +668,53 @@ private void emitVMLoad(Binding.VMLoad vmLoad) {
             emitGetInput();
         }
     }
+
     private void emitDupBinding() {
         Class<?> dupType = typeStack.peek();
         emitDup(dupType);
         pushType(dupType);
     }
 
+    private void emitCast(Binding.Cast cast) {
+        Class<?> fromType = cast.fromType();
+        Class<?> toType = cast.toType();
+
+        if (fromType == int.class) {
+            popType(int.class);
+
+            if (toType == boolean.class) {
+                // implement least significant byte non-zero test
+
+                // select first byte
+                emitConst(0xFF);
+                mv.visitInsn(IAND);
+
+                // convert to boolean
+                emitInvokeStatic(Utils.class, "byteToBoolean", "(B)Z");
+            } else if (toType == byte.class) {
+                mv.visitInsn(I2B);
+            } else if (toType == short.class) {
+                mv.visitInsn(I2S);
+            } else {
+                assert toType == char.class;
+                mv.visitInsn(I2C);
+            }
+
+            pushType(toType);
+        } else {
+            popType(fromType);
+
+            assert fromType == boolean.class
+                    || fromType == byte.class
+                    || fromType == short.class
+                    || fromType == char.class;
+            // no-op in bytecode
+
+            assert toType == int.class;
+            pushType(int.class);
+        }
+    }
+
     private void emitUnboxAddress() {
         popType(MemorySegment.class);
         emitAddress();

src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java

@@ -193,7 +193,8 @@ private void verifyBindings(boolean forArguments, Class<?> carrier, List<Binding
         //ALLOC_BUFFER,
         //BOX_ADDRESS,
         UNBOX_ADDRESS,
-        DUP
+        DUP,
+        CAST
     );
 
     private static void verifyUnboxBindings(Class<?> inType, List<Binding> bindings) {
@@ -220,7 +221,8 @@ private static void verifyUnboxBindings(Class<?> inType, List<Binding> bindings)
         ALLOC_BUFFER,
         BOX_ADDRESS,
         //UNBOX_ADDRESS,
-        DUP
+        DUP,
+        CAST
     );
 
     private static void verifyBoxBindings(Class<?> expectedOutType, List<Binding> bindings) {