8349637: Integer.numberOfLeadingZeros outputs incorrectly in certain cases

jaskarth · jaskarth · commit 3657e92ead1e · 2025-03-03T05:18:55.000Z
Reviewed-by: thartmann, qamai, jbhateja
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -6227,15 +6227,21 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR
   // Since IEEE 754 floating point format represents mantissa in 1.0 format
   // hence biased exponent can be used to compute leading zero count as per
   // following formula:-
-  // LZCNT = 32 - (biased_exp - 127)
+  // LZCNT = 31 - (biased_exp - 127)
   // Special handling has been introduced for Zero, Max_Int and -ve source values.
 
   // Broadcast 0xFF
   vpcmpeqd(xtmp1, xtmp1, xtmp1, vec_enc);
   vpsrld(xtmp1, xtmp1, 24, vec_enc);
 
+  // Remove the bit to the right of the highest set bit ensuring that the conversion to float cannot round up to a higher
+  // power of 2, which has a higher exponent than the input. This transformation is valid as only the highest set bit
+  // contributes to the leading number of zeros.
+  vpsrld(xtmp2, src, 1, vec_enc);
+  vpandn(xtmp3, xtmp2, src, vec_enc);
+
   // Extract biased exponent.
-  vcvtdq2ps(dst, src, vec_enc);
+  vcvtdq2ps(dst, xtmp3, vec_enc);
   vpsrld(dst, dst, 23, vec_enc);
   vpand(dst, dst, xtmp1, vec_enc);
 
@@ -6244,7 +6250,7 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR
   // Exponent = biased_exp - 127
   vpsubd(dst, dst, xtmp1, vec_enc);
 
-  // Exponent = Exponent  + 1
+  // Exponent_plus_one = Exponent + 1
   vpsrld(xtmp3, xtmp1, 6, vec_enc);
   vpaddd(dst, dst, xtmp3, vec_enc);
 
@@ -6257,7 +6263,7 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR
   vpslld(xtmp1, xtmp3, 5, vec_enc);
   // Exponent is 32 if corresponding source lane contains max_int value.
   vpcmpeqd(xtmp2, dst, xtmp1, vec_enc);
-  // LZCNT = 32 - exponent
+  // LZCNT = 32 - exponent_plus_one
   vpsubd(dst, xtmp1, dst, vec_enc);
 
   // Replace LZCNT with a value 1 if corresponding source lane
diff --git a/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java b/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2022, 2023, Arm Limited. All rights reserved.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,40 +24,45 @@
 
 /**
 * @test
+* @bug 8297172 8331993 8349637
 * @key randomness
 * @summary Test vectorization of numberOfTrailingZeros/numberOfLeadingZeros for Long
 * @requires vm.compiler2.enabled
 * @requires (os.simpleArch == "x64" & vm.cpu.features ~= ".*avx2.*") |
 *           (os.simpleArch == "aarch64" & vm.cpu.features ~= ".*sve.*") |
 *           (os.simpleArch == "riscv64" & vm.cpu.features ~= ".*zvbb.*")
 * @library /test/lib /
+* @modules jdk.incubator.vector
 * @run driver compiler.vectorization.TestNumberOfContinuousZeros
 */
 
 package compiler.vectorization;
 
+import jdk.incubator.vector.*;
 import compiler.lib.ir_framework.*;
 import java.util.Random;
 import jdk.test.lib.Asserts;
+import jdk.test.lib.Utils;
 
 public class TestNumberOfContinuousZeros {
+    private static final int[] SPECIAL = { 0x01FFFFFF, 0x03FFFFFE, 0x07FFFFFC, 0x0FFFFFF8, 0x1FFFFFF0, 0x3FFFFFE0, 0xFFFFFFFF };
     private long[] inputLong;
     private int[] outputLong;
     private int[] inputInt;
     private int[] outputInt;
     private static final int LEN = 1024;
     private Random rng;
 
-    public static void main(String args[]) {
-        TestFramework.run();
+    public static void main(String[] args) {
+        TestFramework.runWithFlags("--add-modules=jdk.incubator.vector");
     }
 
     public TestNumberOfContinuousZeros() {
         inputLong = new long[LEN];
         outputLong = new int[LEN];
         inputInt = new int[LEN];
         outputInt = new int[LEN];
-        rng = new Random(42);
+        rng = Utils.getRandomInstance();
         for (int i = 0; i < LEN; ++i) {
             inputLong[i] = rng.nextLong();
             inputInt[i] = rng.nextInt();
@@ -119,5 +125,80 @@ public void checkResultInt() {
             Asserts.assertEquals(outputInt[i], Integer.numberOfLeadingZeros(inputInt[i]));
         }
     }
+
+    @Setup
+    static Object[] setupSpecialIntArray() {
+        int[] res = new int[LEN];
+
+        for (int i = 0; i < LEN; i++) {
+            res[i] = SPECIAL[i % SPECIAL.length];
+        }
+
+        return new Object[] { res };
+    }
+
+    @Test
+    @IR(counts = {IRNode.COUNT_LEADING_ZEROS_VI, "> 0"})
+    @Arguments(setup = "setupSpecialIntArray")
+    public Object[] testSpecialIntLeadingZeros(int[] ints) {
+        int[] res = new int[LEN];
+
+        for (int i = 0; i < LEN; ++i) {
+            res[i] = Integer.numberOfLeadingZeros(ints[i]);
+        }
+
+        return new Object[] { ints, res };
+    }
+
+    @Check(test = "testSpecialIntLeadingZeros")
+    public void checkSpecialIntLeadingZeros(Object[] vals) {
+        int[] in = (int[]) vals[0];
+        int[] out = (int[]) vals[1];
+
+        for (int i = 0; i < LEN; ++i) {
+            int value = Integer.numberOfLeadingZeros(in[i]);
+
+            if (out[i] != value) {
+                throw new IllegalStateException("Expected lzcnt(" + in[i] + ") to be " + value + " but got " + out[i]);
+            }
+        }
+    }
+
+    private static final VectorSpecies<Integer> SPECIES = IntVector.SPECIES_PREFERRED;
+
+    @Test
+    @IR(counts = {IRNode.COUNT_LEADING_ZEROS_VI, "> 0"})
+    @Arguments(setup = "setupSpecialIntArray")
+    public Object[] checkSpecialIntLeadingZerosVector(int[] ints) {
+        int[] res = new int[LEN];
+
+        for (int i = 0; i < ints.length; i += SPECIES.length()) {
+            IntVector av = IntVector.fromArray(SPECIES, ints, i);
+            av.lanewise(VectorOperators.LEADING_ZEROS_COUNT).intoArray(res, i);
+        }
+
+        return new Object[] { ints, res };
+    }
+
+    @Check(test = "checkSpecialIntLeadingZerosVector")
+    public void checkSpecialIntLeadingZerosVector(Object[] vals) {
+        int[] ints = (int[]) vals[0];
+        int[] res = (int[]) vals[1];
+
+        // Verification
+
+        int[] check = new int[LEN];
+
+        for (int i = 0; i < ints.length; i += SPECIES.length()) {
+            IntVector av = IntVector.fromArray(SPECIES, ints, i);
+            av.lanewise(VectorOperators.LEADING_ZEROS_COUNT).intoArray(check, i);
+        }
+
+        for (int i = 0; i < LEN; i++) {
+            if (res[i] != check[i]) {
+                throw new IllegalStateException("Expected " + check[i] + " but got " + res[i]);
+            }
+        }
+    }
 }
 
diff --git a/test/hotspot/jtreg/compiler/vectorization/TestVectorZeroCount.java b/test/hotspot/jtreg/compiler/vectorization/TestVectorZeroCount.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @bug 8349637
+ * @summary Ensure that vectorization of numberOfLeadingZeros and numberOfTrailingZeros outputs correct values
+ * @library /test/lib /
+ * @run main/othervm compiler.vectorization.TestVectorZeroCount
+ */
+
+package compiler.vectorization;
+
+import java.util.Random;
+
+import jdk.test.lib.Utils;
+
+public class TestVectorZeroCount {
+    private static final int SIZE = 1024;
+    private static final Random RANDOM = Utils.getRandomInstance();
+
+    private static final int[] INT_VALUES = new int[SIZE];
+    private static final int[] INT_EXPECTED_LEADING = new int[SIZE];
+    private static final int[] INT_RESULT_LEADING = new int[SIZE];
+    private static final int[] INT_EXPECTED_TRAILING = new int[SIZE];
+    private static final int[] INT_RESULT_TRAILING = new int[SIZE];
+
+    private static final long[] LONG_VALUES = new long[SIZE];
+    private static final long[] LONG_EXPECTED_LEADING = new long[SIZE];
+    private static final long[] LONG_RESULT_LEADING = new long[SIZE];
+    private static final long[] LONG_EXPECTED_TRAILING = new long[SIZE];
+    private static final long[] LONG_RESULT_TRAILING = new long[SIZE];
+
+    private static int intCounter = Integer.MIN_VALUE;
+    private static int longIterations = 100_000_000;
+
+    public static boolean testInt() {
+        boolean done = false;
+
+        // Non-vectorized loop as baseline (not vectorized because source array is initialized)
+        for (int i = 0; i < SIZE; ++i) {
+            INT_VALUES[i] = intCounter++;
+            if (intCounter == Integer.MAX_VALUE) {
+                done = true;
+            }
+            INT_EXPECTED_LEADING[i] = Integer.numberOfLeadingZeros(INT_VALUES[i]);
+            INT_EXPECTED_TRAILING[i] = Integer.numberOfTrailingZeros(INT_VALUES[i]);
+        }
+        // Vectorized loop
+        for (int i = 0; i < SIZE; ++i) {
+            INT_RESULT_LEADING[i] = Integer.numberOfLeadingZeros(INT_VALUES[i]);
+        }
+        for (int i = 0; i < SIZE; ++i) {
+            INT_RESULT_TRAILING[i] = Integer.numberOfTrailingZeros(INT_VALUES[i]);
+        }
+
+        // Compare results
+        for (int i = 0; i < SIZE; ++i) {
+            if (INT_RESULT_LEADING[i] != INT_EXPECTED_LEADING[i]) {
+                throw new RuntimeException("Unexpected result for Integer.numberOfLeadingZeros(" + INT_VALUES[i] + "): " + INT_RESULT_LEADING[i] + ", expected " + INT_EXPECTED_LEADING[i]);
+            }
+            if (INT_RESULT_TRAILING[i] != INT_EXPECTED_TRAILING[i]) {
+                throw new RuntimeException("Unexpected result for Integer.numberOfTrailingZeros(" + INT_VALUES[i] + "): " + INT_RESULT_TRAILING[i] + ", expected " + INT_EXPECTED_TRAILING[i]);
+            }
+        }
+        return done;
+    }
+
+    public static boolean testLong() {
+        boolean done = false;
+
+        // Non-vectorized loop as baseline (not vectorized because source array is initialized)
+        for (int i = 0; i < SIZE; ++i) {
+            // Use random values because the long range is too large to iterate over it
+            LONG_VALUES[i] = RANDOM.nextLong();
+            if (longIterations-- == 0) {
+                done = true;
+            }
+            LONG_EXPECTED_LEADING[i] = Long.numberOfLeadingZeros(LONG_VALUES[i]);
+            LONG_EXPECTED_TRAILING[i] = Long.numberOfTrailingZeros(LONG_VALUES[i]);
+        }
+        // Vectorized loop
+        for (int i = 0; i < SIZE; ++i) {
+            LONG_RESULT_LEADING[i] = Long.numberOfLeadingZeros(LONG_VALUES[i]);
+        }
+        for (int i = 0; i < SIZE; ++i) {
+            LONG_RESULT_TRAILING[i] = Long.numberOfTrailingZeros(LONG_VALUES[i]);
+        }
+
+        // Compare results
+        for (int i = 0; i < SIZE; ++i) {
+            if (LONG_RESULT_LEADING[i] != LONG_EXPECTED_LEADING[i]) {
+                throw new RuntimeException("Unexpected result for Long.numberOfLeadingZeros(" + LONG_VALUES[i] + "): " + LONG_RESULT_LEADING[i] + ", expected " + LONG_EXPECTED_LEADING[i]);
+            }
+            if (LONG_RESULT_TRAILING[i] != LONG_EXPECTED_TRAILING[i]) {
+                throw new RuntimeException("Unexpected result for Long.numberOfTrailingZeros(" + LONG_VALUES[i] + "): " + LONG_RESULT_TRAILING[i] + ", expected " + LONG_EXPECTED_TRAILING[i]);
+            }
+        }
+        return done;
+    }
+
+    public static void main(String[] args) {
+        // Run twice to make sure compiled code is used from the beginning
+        for (int i = 0; i < 2; ++i) {
+            while (!testLong()) ;
+            while (!testInt()) ;
+        }
+    }
+}
