8344989: Test java/foreign/TestLinker.java failed with zero: IllegalStateException: libffi call failed with status: FFI_BAD_TYPEDEF

src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java

@@ -31,6 +31,7 @@
 import java.lang.foreign.MemoryLayout;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.SegmentAllocator;
+import java.lang.foreign.SequenceLayout;
 import java.lang.foreign.ValueLayout;
 import static java.lang.foreign.ValueLayout.ADDRESS;
 import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN;
@@ -85,6 +86,7 @@ public static boolean isSupported() {
 
     @Override
     protected MethodHandle arrangeDowncall(MethodType inferredMethodType, FunctionDescriptor function, LinkerOptions options) {
+        assertNotEmpty(function);
         MemorySegment cif = makeCif(inferredMethodType, function, options, Arena.ofAuto());
 
         int capturedStateMask = options.capturedCallState()
@@ -111,6 +113,7 @@ protected MethodHandle arrangeDowncall(MethodType inferredMethodType, FunctionDe
 
     @Override
     protected UpcallStubFactory arrangeUpcall(MethodType targetType, FunctionDescriptor function, LinkerOptions options) {
+        assertNotEmpty(function);
         MemorySegment cif = makeCif(targetType, function, options, Arena.ofAuto());
 
         UpcallData invData = new UpcallData(function.returnLayout().orElse(null), function.argumentLayouts(), cif);
@@ -325,4 +328,35 @@ class Holder {
 
         return Holder.CANONICAL_LAYOUTS;
     }
+
+    private static void assertNotEmpty(FunctionDescriptor fd) {
+        fd.returnLayout().ifPresent(FallbackLinker::assertNotEmpty);
+        fd.argumentLayouts().forEach(FallbackLinker::assertNotEmpty);
+    }
+
+    // Recursively tests for emptiness
+    private static void assertNotEmpty(MemoryLayout layout) {
+        switch (layout) {
+            case GroupLayout gl -> {
+                if (gl.memberLayouts().isEmpty()) {
+                    throw empty(gl);
+                } else {
+                    gl.memberLayouts().forEach(FallbackLinker::assertNotEmpty);
+                }
+            }
+            case SequenceLayout sl -> {
+                if (sl.elementCount() == 0) {
+                    throw empty(sl);
+                } else {
+                    assertNotEmpty(sl.elementLayout());
+                }
+            }
+            default -> { /* do nothing */ }
+        }
+    }
+
+    private static IllegalArgumentException empty(MemoryLayout layout) {
+        return new IllegalArgumentException("The layout " + layout + " is empty");
+    }
+
 }

test/jdk/java/foreign/TestLinker.java

@@ -23,12 +23,13 @@
 
 /*
  * @test
- * @modules java.base/jdk.internal.foreign
+ * @modules java.base/jdk.internal.foreign java.base/jdk.internal.foreign.abi.fallback
  * @run testng TestLinker
  * @run testng/othervm TestLinker
  */
 
 import jdk.internal.foreign.CABI;
+import jdk.internal.foreign.abi.fallback.FallbackLinker;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
@@ -250,7 +251,13 @@ public void sequenceOfZeroElements() {
         var padding5a1 = MemoryLayout.paddingLayout(5);
         var struct8a8 = MemoryLayout.structLayout(sequence0a8, sequence3a1, padding5a1);
         var fd = FunctionDescriptor.of(struct8a8, struct8a8, struct8a8);
-        linker.downcallHandle(fd);
+        if (linker.getClass().equals(FallbackLinker.class)) {
+            // The fallback linker does not support empty layouts (FFI_BAD_TYPEDEF)
+            var iae = expectThrows(IllegalArgumentException.class, () -> linker.downcallHandle(fd));
+            assertTrue(iae.getMessage().contains("is empty"));
+        } else {
+            linker.downcallHandle(fd);
+        }
     }
 
     @DataProvider