8320646: RISC-V: C2 VectorCastHF2F

8320647: RISC-V: C2 VectorCastF2HF

Reviewed-by: luhenry, fyang

Author: Hamlin Li

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -1191,7 +1191,7 @@ static Assembler::SEW elemtype_to_sew(BasicType etype) {
   void NAME(Register Rd, uint32_t imm, SEW sew, LMUL lmul = m1,           \
             VMA vma = mu, VTA vta = tu, bool vill = false) {              \
     unsigned insn = 0;                                                    \
-    guarantee(is_uimm5(imm), "imm is invalid");                           \
+    guarantee(is_uimm5(imm), "uimm is invalid");                           \
     patch((address)&insn, 6, 0, op);                                      \
     patch((address)&insn, 14, 12, funct3);                                \
     patch((address)&insn, 19, 15, imm);                                   \
@@ -1327,7 +1327,7 @@ enum VectorMask {
 // r_vm
 #define INSN(NAME, op, funct3, funct6)                                                             \
   void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {       \
-    guarantee(is_uimm5(imm), "imm is invalid");                                                    \
+    guarantee(is_uimm5(imm), "uimm is invalid");                                                    \
     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
   }
 
@@ -1340,6 +1340,9 @@ enum VectorMask {
   INSN(vslideup_vi,   0b1010111, 0b011, 0b001110);
   INSN(vslidedown_vi, 0b1010111, 0b011, 0b001111);
 
+  // Vector Narrowing Integer Right Shift Instructions
+  INSN(vnsra_wi, 0b1010111, 0b011, 0b101101);
+
 #undef INSN
 
 #define INSN(NAME, op, funct3, funct6)                                                             \
@@ -1505,6 +1508,9 @@ enum VectorMask {
   INSN(vmulh_vx,   0b1010111, 0b110, 0b100111);
   INSN(vmul_vx,    0b1010111, 0b110, 0b100101);
 
+  // Vector Widening Integer Add/Subtract
+  INSN(vwadd_vx, 0b1010111, 0b110, 0b110001);
+
   // Vector Integer Min/Max Instructions
   INSN(vmax_vx,  0b1010111, 0b100, 0b000111);
   INSN(vmaxu_vx, 0b1010111, 0b100, 0b000110);
@@ -1538,6 +1544,8 @@ enum VectorMask {
   // Vector Single-Width Integer Add and Subtract
   INSN(vsub_vx,  0b1010111, 0b100, 0b000010);
   INSN(vadd_vx,  0b1010111, 0b100, 0b000000);
+
+  // Vector Integer reverse subtract
   INSN(vrsub_vx, 0b1010111, 0b100, 0b000011);
 
   // Vector Slide Instructions
@@ -1600,16 +1608,23 @@ enum VectorMask {
     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
   }
 
+  // Vector Integer Comparison Instructions
   INSN(vmsgt_vi,  0b1010111, 0b011, 0b011111);
   INSN(vmsgtu_vi, 0b1010111, 0b011, 0b011110);
   INSN(vmsle_vi,  0b1010111, 0b011, 0b011101);
   INSN(vmsleu_vi, 0b1010111, 0b011, 0b011100);
   INSN(vmsne_vi,  0b1010111, 0b011, 0b011001);
   INSN(vmseq_vi,  0b1010111, 0b011, 0b011000);
+
+  // Vector Bitwise Logical Instructions
   INSN(vxor_vi,   0b1010111, 0b011, 0b001011);
   INSN(vor_vi,    0b1010111, 0b011, 0b001010);
   INSN(vand_vi,   0b1010111, 0b011, 0b001001);
+
+  // Vector Single-Width Integer Add and Subtract
   INSN(vadd_vi,   0b1010111, 0b011, 0b000000);
+
+  // Vector Integer reverse subtract
   INSN(vrsub_vi,  0b1010111, 0b011, 0b000011);
 
 #undef INSN

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -2064,8 +2064,8 @@ static void float16_to_float_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Fl
 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
   auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
 
-  // in riscv, NaN needs a special process as fcvt does not work in that case.
-  // in riscv, Inf does not need a special process as fcvt can handle it correctly.
+  // On riscv, NaN needs a special process as fcvt does not work in that case.
+  // On riscv, Inf does not need a special process as fcvt can handle it correctly.
   // but we consider to get the slow path to process NaN and Inf at the same time,
   // as both of them are rare cases, and if we try to get the slow path to handle
   // only NaN case it would sacrifise the performance for normal cases,
@@ -2112,7 +2112,7 @@ static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Re
 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
   auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 130, float_to_float16_slow_path);
 
-  // in riscv, NaN needs a special process as fcvt does not work in that case.
+  // On riscv, NaN needs a special process as fcvt does not work in that case.
 
   // check whether it's a NaN.
   // replace fclass with feq as performance optimization.
@@ -2127,6 +2127,117 @@ void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatR
   bind(stub->continuation());
 }
 
+static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
+#define __ masm.
+  VectorRegister dst = stub.data<0>();
+  VectorRegister src = stub.data<1>();
+  uint vector_length = stub.data<2>();
+  __ bind(stub.entry());
+
+  // following instructions mainly focus on NaN, as riscv does not handle
+  // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
+  //
+  // construct NaN's in 32 bits from the NaN's in 16 bits,
+  // we need the payloads of non-canonical NaNs to be preserved.
+
+  // adjust vector type to 2 * SEW.
+  __ vsetvli_helper(T_FLOAT, vector_length, Assembler::m1);
+  // widen and sign-extend src data.
+  __ vsext_vf2(dst, src, Assembler::v0_t);
+  __ mv(t0, 0x7f800000);
+  // sign-bit was already set via sign-extension if necessary.
+  __ vsll_vi(dst, dst, 13, Assembler::v0_t);
+  __ vor_vx(dst, dst, t0, Assembler::v0_t);
+
+  __ j(stub.continuation());
+#undef __
+}
+
+// j.l.Float.float16ToFloat
+void C2_MacroAssembler::float16_to_float_v(VectorRegister dst, VectorRegister src, uint vector_length) {
+  auto stub = C2CodeStub::make<VectorRegister, VectorRegister, uint>
+              (dst, src, vector_length, 24, float16_to_float_v_slow_path);
+  assert_different_registers(dst, src);
+
+  // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
+  // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
+  // but we consider to get the slow path to process NaN and Inf at the same time,
+  // as both of them are rare cases, and if we try to get the slow path to handle
+  // only NaN case it would sacrifise the performance for normal cases,
+  // i.e. non-NaN and non-Inf cases.
+
+  vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
+
+  // check whether there is a NaN or +/- Inf.
+  mv(t0, 0x7c00);
+  vand_vx(v0, src, t0);
+  // v0 will be used as mask in slow path.
+  vmseq_vx(v0, v0, t0);
+  vcpop_m(t0, v0);
+
+  // For non-NaN or non-Inf cases, just use built-in instructions.
+  vfwcvt_f_f_v(dst, src);
+
+  // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
+  bnez(t0, stub->entry());
+
+  bind(stub->continuation());
+}
+
+static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
+                                         C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
+#define __ masm.
+  VectorRegister dst = stub.data<0>();
+  VectorRegister src = stub.data<1>();
+  VectorRegister tmp = stub.data<2>();
+  __ bind(stub.entry());
+
+  // mul is already set to mf2 in float_to_float16_v.
+
+  // preserve the payloads of non-canonical NaNs.
+  __ vnsra_wi(dst, src, 13, Assembler::v0_t);
+
+  // preserve the sign bit.
+  __ vnsra_wi(tmp, src, 26, Assembler::v0_t);
+  __ vsll_vi(tmp, tmp, 10, Assembler::v0_t);
+  __ mv(t0, 0x3ff);
+  __ vor_vx(tmp, tmp, t0, Assembler::v0_t);
+
+  // get the result by merging sign bit and payloads of preserved non-canonical NaNs.
+  __ vand_vv(dst, dst, tmp, Assembler::v0_t);
+
+  __ j(stub.continuation());
+#undef __
+}
+
+// j.l.Float.float16ToFloat
+void C2_MacroAssembler::float_to_float16_v(VectorRegister dst, VectorRegister src, VectorRegister vtmp,
+                                           Register tmp, uint vector_length) {
+  assert_different_registers(dst, src, vtmp);
+
+  auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
+              (dst, src, vtmp, 28, float_to_float16_v_slow_path);
+
+  // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
+
+  vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
+
+  // check whether there is a NaN.
+  // replace v_fclass with vmseq_vv as performance optimization.
+  vmfne_vv(v0, src, src);
+  vcpop_m(t0, v0);
+
+  vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
+
+  // For non-NaN cases, just use built-in instructions.
+  vfncvt_f_f_w(dst, src);
+
+  // jump to stub processing NaN cases.
+  bnez(t0, stub->entry());
+
+  bind(stub->continuation());
+}
+
 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
   vsetvli_helper(bt, vlen);
 

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp

@@ -188,6 +188,9 @@
   void expand_bits_i_v(Register dst, Register src, Register mask);
   void expand_bits_l_v(Register dst, Register src, Register mask);
 
+  void float16_to_float_v(VectorRegister dst, VectorRegister src, uint vector_length);
+  void float_to_float16_v(VectorRegister dst, VectorRegister src, VectorRegister vtmp, Register tmp, uint vector_length);
+
   void string_equals_v(Register r1, Register r2,
                        Register result, Register cnt1);
 

src/hotspot/cpu/riscv/globals_riscv.hpp

@@ -115,6 +115,7 @@ define_pd_global(intx, InlineSmallCode,          1000);
   product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model")        \
   product(bool, UseZihintpause, false, EXPERIMENTAL,                             \
           "Use Zihintpause instructions")                                        \
+  product(bool, UseZvfh, false, EXPERIMENTAL, "Use Zvfh instructions")           \
   product(bool, UseZvkn, false, EXPERIMENTAL,                                    \
           "Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt")                \
   product(bool, UseRVVForBigIntegerShiftIntrinsics, true,                        \

src/hotspot/cpu/riscv/riscv_v.ad

@@ -73,6 +73,9 @@ source %{
           return false;
         }
         break;
+      case Op_VectorCastHF2F:
+      case Op_VectorCastF2HF:
+        return UseZvfh;
       default:
         break;
     }
@@ -3660,6 +3663,37 @@ instruct vsignum_reg(vReg dst, vReg zero, vReg one, vRegMask_V0 v0) %{
   ins_pipe(pipe_slow);
 %}
 
+// ---------------- Convert Half Floating to Floating Vector Operations ----------------
+
+// half precision -> single
+
+instruct vconvHF2F(vReg dst, vReg src, vRegMask_V0 v0) %{
+  predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
+  match(Set dst (VectorCastHF2F src));
+  effect(TEMP_DEF dst, TEMP v0);
+  format %{ "vfwcvt.f.f.v $dst, $src\t# convert half to single precision" %}
+  ins_encode %{
+    __ float16_to_float_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
+                          Matcher::vector_length(this));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+// single precision -> half
+
+instruct vconvF2HF(vReg dst, vReg src, vReg vtmp, vRegMask_V0 v0, iRegINoSp tmp) %{
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
+  match(Set dst (VectorCastF2HF src));
+  effect(TEMP_DEF dst, TEMP v0, TEMP vtmp, TEMP tmp);
+  format %{ "vfncvt.f.f.w $dst, $src\t# convert single to half precision" %}
+  ins_encode %{
+    __ float_to_float16_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
+                          as_VectorRegister($vtmp$$reg), $tmp$$Register,
+                          Matcher::vector_length(this));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Vector Load Gather ---------------------------
 
 instruct gather_load(vReg dst, indirect mem, vReg idx) %{

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -152,6 +152,7 @@ class VM_Version : public Abstract_VM_Version {
   decl(ext_Ztso        , "Ztso"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))        \
   decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
   decl(ext_Zacas       , "Zacas"       , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas))       \
+  decl(ext_Zvfh        , "Zvfh"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZvfh))        \
   decl(mvendorid       , "VendorId"    , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \
   decl(marchid         , "ArchId"      , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \
   decl(mimpid          , "ImpId"       , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \

src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp

@@ -252,6 +252,8 @@ void VM_Version::rivos_features() {
   ext_Ztso.enable_feature();
   ext_Zihintpause.enable_feature();
 
+  ext_Zvfh.enable_feature();
+
   unaligned_access.enable_feature(MISALIGNED_FAST);
   satp_mode.enable_feature(VM_SV48);
 

test/hotspot/jtreg/compiler/vectorization/TestFloatConversionsVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,9 @@
  * @bug 8294588
  * @summary Auto-vectorize Float.floatToFloat16, Float.float16ToFloat APIs
  * @requires vm.compiler2.enabled
- * @requires (os.simpleArch == "x64" & (vm.cpu.features ~= ".*avx512f.*" | vm.cpu.features ~= ".*f16c.*")) | os.arch == "aarch64"
+ * @requires (os.simpleArch == "x64" & (vm.cpu.features ~= ".*avx512f.*" | vm.cpu.features ~= ".*f16c.*")) |
+ *           os.arch == "aarch64" |
+ *           (os.arch == "riscv64" & vm.cpu.features ~= ".*zvfh.*")
  * @library /test/lib /
  * @run driver compiler.vectorization.TestFloatConversionsVector
  */

test/hotspot/jtreg/compiler/vectorization/TestFloatConversionsVectorNaN.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8320646
+ * @summary Auto-vectorize Float.floatToFloat16, Float.float16ToFloat APIs, with NaN
+ * @requires vm.compiler2.enabled
+ * @requires (os.arch == "riscv64" & vm.cpu.features ~= ".*zvfh.*")
+ * @library /test/lib /
+ * @run driver compiler.vectorization.TestFloatConversionsVectorNaN
+ */
+
+package compiler.vectorization;
+
+import java.util.HexFormat;
+
+import compiler.lib.ir_framework.*;
+import jdk.test.lib.Asserts;
+
+public class TestFloatConversionsVectorNaN {
+    private static final int ARRLEN = 1024;
+    private static final int ITERS  = 11000;
+    private static float  [] finp;
+    private static short  [] sout;
+    private static short  [] sinp;
+    private static float  [] fout;
+
+    public static void main(String args[]) {
+        TestFramework.runWithFlags("-XX:-TieredCompilation",
+                                   "-XX:CompileThresholdScaling=0.3");
+        System.out.println("PASSED");
+    }
+
+    @Test
+    @IR(counts = {IRNode.VECTOR_CAST_F2HF, IRNode.VECTOR_SIZE + "min(max_float, max_short)", "> 0"})
+    public void test_float_float16(short[] sout, float[] finp) {
+        for (int i = 0; i < finp.length; i++) {
+            sout[i] = Float.floatToFloat16(finp[i]);
+        }
+    }
+
+    @Run(test = {"test_float_float16"}, mode = RunMode.STANDALONE)
+    public void kernel_test_float_float16() {
+        int errno = 0;
+        finp = new float[ARRLEN];
+        sout = new short[ARRLEN];
+
+        // Setup
+        for (int i = 0; i < ARRLEN; i++) {
+            if (i%39 == 0) {
+                int x = 0x7f800000 + ((i/39) << 13);
+                x = (i%2 == 0) ? x : (x | 0x80000000);
+                finp[i] = Float.intBitsToFloat(x);
+            } else {
+                finp[i] = (float) i * 1.4f;
+            }
+        }
+        int ranges[][] = {
+            {128, 64},
+            {256, 19},
+            {384-19, 19},
+            {512-19, 17},
+            {640+19, 19},
+            {768+19, 32},
+            {896-19, 32}
+        };
+        for (int range[] : ranges) {
+            int start = range[0];
+            int offset = range[1];
+            for (int i = start; i < start+offset; i++) {
+                int x = 0x7f800000 + (i << 13);
+                finp[i] = Float.intBitsToFloat(x);
+            }
+        }
+
+        // Test
+        for (int i = 0; i < ITERS; i++) {
+            test_float_float16(sout, finp);
+        }
+
+        // Verifying the result
+        for (int i = 0; i < ARRLEN; i++) {
+            errno += assertEquals(i, finp[i], Float.floatToFloat16(finp[i]), sout[i]);
+        }
+
+        if (errno > 0) {
+            throw new RuntimeException("errors occur");
+        }
+    }
+
+    static int assertEquals(int idx, float f, short expected, short actual) {
+        HexFormat hf = HexFormat.of();
+        String msg = "floatToFloat16 wrong result: idx: " + idx + ", \t" + f +
+                     ",\t expected: " + hf.toHexDigits(expected) +
+                     ",\t actual: " + hf.toHexDigits(actual);
+        if ((expected & 0x7c00) != 0x7c00) {
+            if (expected != actual) {
+                System.err.println(msg);
+                return 1;
+            }
+        } else if ((expected & 0x3ff) != 0) {
+            if (((actual & 0x7c00) != 0x7c00) || (actual & 0x3ff) == 0) {
+                System.err.println(msg);
+                return 1;
+            }
+        }
+        return 0;
+    }
+
+    @Test
+    @IR(counts = {IRNode.VECTOR_CAST_HF2F, IRNode.VECTOR_SIZE + "min(max_float, max_short)", "> 0"})
+    public void test_float16_float(float[] fout, short[] sinp) {
+        for (int i = 0; i < sinp.length; i++) {
+            fout[i] = Float.float16ToFloat(sinp[i]);
+        }
+    }
+
+    @Run(test = {"test_float16_float"}, mode = RunMode.STANDALONE)
+    public void kernel_test_float16_float() {
+        int errno = 0;
+        sinp = new short[ARRLEN];
+        fout = new float[ARRLEN];
+
+        // Setup
+        for (int i = 0; i < ARRLEN; i++) {
+            if (i%39 == 0) {
+                int x = 0x7c00 + i;
+                x = (i%2 == 0) ? x : (x | 0x8000);
+                sinp[i] = (short)x;
+            } else {
+                sinp[i] = (short)i;
+            }
+        }
+
+        int ranges[][] = {
+            {128, 64},
+            {256, 19},
+            {384-19, 19},
+            {512-19, 17},
+            {640+19, 19},
+            {768+19, 32},
+            {896-19, 32}
+        };
+        for (int range[] : ranges) {
+            int start = range[0];
+            int offset = range[1];
+            for (int i = start; i < start+offset; i++) {
+                int x = 0x7c00 + i;
+                x = (i%2 == 0) ? x : (x | 0x8000);
+                sinp[i] = (short)x;
+            }
+        }
+
+        // Test
+        for (int i = 0; i < ITERS; i++) {
+            test_float16_float(fout, sinp);
+        }
+
+        // Verifying the result
+        for (int i = 0; i < ARRLEN; i++) {
+            errno += assertEquals(i, sinp[i], Float.float16ToFloat(sinp[i]), fout[i]);
+        }
+
+        if (errno > 0) {
+            throw new RuntimeException("errors occur");
+        }
+    }
+
+    static int assertEquals(int idx, short s, float expected, float actual) {
+        String msg = "float16ToFloat wrong result: idx: " + idx + ", \t" + s +
+                     ",\t expected: " + expected + ",\t" + Integer.toHexString(Float.floatToIntBits(expected)) +
+                     ",\t actual: " + actual + ",\t" + Integer.toHexString(Float.floatToIntBits(actual));
+        if (!Float.isNaN(expected) || !Float.isNaN(actual)) {
+            if (expected != actual) {
+                System.err.println(msg);
+                return 1;
+            }
+        }
+        return 0;
+    }
+}