8288508: Enhance ECDSA usage

Reviewed-by: ascarpino, ahgross, rhalade

src/java.base/share/classes/sun/security/util/ECUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -347,5 +347,40 @@ public static ECPrivateKey checkPrivateKey(ECPrivateKey prv)
         return prv;
     }
 
+    // Partial Public key validation as described in NIST SP 800-186 Appendix D.1.1.1.
+    // The extra step in the full validation (described in Appendix D.1.1.2) is implemented
+    // as sun.security.ec.ECOperations#checkOrder inside the jdk.crypto.ec module.
+    public static void validatePublicKey(ECPoint point, ECParameterSpec spec)
+            throws InvalidKeyException {
+        BigInteger p;
+        if (spec.getCurve().getField() instanceof ECFieldFp f) {
+            p = f.getP();
+        } else {
+            throw new InvalidKeyException("Only curves over prime fields are supported");
+        }
+
+        // 1. If Q is the point at infinity, output REJECT
+        if (point.equals(ECPoint.POINT_INFINITY)) {
+            throw new InvalidKeyException("Public point is at infinity");
+        }
+        // 2. Verify that x and y are integers in the interval [0, p-1]. Output REJECT if verification fails.
+        BigInteger x = point.getAffineX();
+        if (x.signum() < 0 || x.compareTo(p) >= 0) {
+            throw new InvalidKeyException("Public point x is not in the interval [0, p-1]");
+        }
+        BigInteger y = point.getAffineY();
+        if (y.signum() < 0 || y.compareTo(p) >= 0) {
+            throw new InvalidKeyException("Public point y is not in the interval [0, p-1]");
+        }
+        // 3. Verify that (x, y) is a point on the W_a,b by checking that (x, y) satisfies the defining
+        // equation y^2 = x^3 + a x + b where computations are carried out in GF(p). Output REJECT
+        // if verification fails.
+        BigInteger left = y.modPow(BigInteger.TWO, p);
+        BigInteger right = x.pow(3).add(spec.getCurve().getA().multiply(x)).add(spec.getCurve().getB()).mod(p);
+        if (!left.equals(right)) {
+            throw new InvalidKeyException("Public point is not on the curve");
+        }
+    }
+
     private ECUtil() {}
 }

src/jdk.crypto.ec/share/classes/sun/security/ec/ECDSASignature.java

@@ -463,8 +463,7 @@ protected byte[] engineSign() throws SignatureException {
         Optional<ECDSAOperations> opsOpt =
             ECDSAOperations.forParameters(params);
         if (opsOpt.isEmpty()) {
-            throw new SignatureException("Curve not supported: " +
-                params.toString());
+            throw new SignatureException("Curve not supported: " + params);
         }
         byte[] sig = signDigestImpl(opsOpt.get(), seedBits, digest, privateKey,
             random);
@@ -480,22 +479,33 @@ protected byte[] engineSign() throws SignatureException {
     @Override
     protected boolean engineVerify(byte[] signature) throws SignatureException {
 
+        ECPoint w = publicKey.getW();
+        ECParameterSpec params = publicKey.getParams();
+
+        // Partial public key validation
+        try {
+            ECUtil.validatePublicKey(w, params);
+        } catch (InvalidKeyException e) {
+            return false;
+        }
+
+        ECDSAOperations ops = ECDSAOperations.forParameters(params)
+                .orElseThrow(() -> new SignatureException("Curve not supported: " + params));
+
+        // Full public key validation, only necessary when h != 1.
+        if (params.getCofactor() != 1) {
+            if (!ops.getEcOperations().checkOrder(w)) {
+                return false;
+            }
+        }
+
         byte[] sig;
         if (p1363Format) {
             sig = signature;
         } else {
             sig = ECUtil.decodeSignature(signature);
         }
-
-        byte[] digest = getDigestValue();
-
-        Optional<ECDSAOperations> opsOpt =
-            ECDSAOperations.forParameters(publicKey.getParams());
-        if (opsOpt.isEmpty()) {
-            throw new SignatureException("Curve not supported: " +
-                publicKey.getParams().toString());
-        }
-        return opsOpt.get().verifySignedDigest(digest, sig, publicKey.getW());
+        return ops.verifySignedDigest(getDigestValue(), sig, w);
     }
 
     // set parameter, not supported. See JCA doc

src/jdk.crypto.ec/share/classes/sun/security/ec/ECOperations.java

@@ -26,13 +26,15 @@
 package sun.security.ec;
 
 import sun.security.ec.point.*;
+import sun.security.util.ArrayUtil;
 import sun.security.util.math.*;
 import sun.security.util.math.intpoly.*;
 
 import java.math.BigInteger;
 import java.security.ProviderException;
 import java.security.spec.ECFieldFp;
 import java.security.spec.ECParameterSpec;
+import java.security.spec.ECPoint;
 import java.security.spec.EllipticCurve;
 import java.util.Map;
 import java.util.Optional;
@@ -470,5 +472,19 @@ private void setSum(ProjectivePoint.Mutable p, ProjectivePoint.Mutable p2,
         p.getZ().setSum(t3);
 
     }
+
+    // The extra step in the Full Public key validation as described in
+    // NIST SP 800-186 Appendix D.1.1.2
+    public boolean checkOrder(ECPoint point) {
+        BigInteger x = point.getAffineX();
+        BigInteger y = point.getAffineY();
+
+        // Verify that n Q = INFINITY. Output REJECT if verification fails.
+        IntegerFieldModuloP field = this.getField();
+        AffinePoint ap = new AffinePoint(field.getElement(x), field.getElement(y));
+        byte[] scalar = this.orderField.getSize().toByteArray();
+        ArrayUtil.reverse(scalar);
+        return isNeutral(this.multiply(ap, scalar));
+    }
 }