8336499: Failure when creating non-CRT RSA private keys in SunPKCS11

Co-authored-by: Francisco Ferrari Bihurriet <fferrari@openjdk.org>
Co-authored-by: Martin Balao <mbalao@openjdk.org>
Reviewed-by: fferrari, valeriep

Author: martinuy

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Key.java

@@ -561,47 +561,73 @@ static class P11RSAPrivateKeyInternal extends P11PrivateKey {
         static P11RSAPrivateKeyInternal of(Session session, long keyID,
                 String algorithm, int keyLength, CK_ATTRIBUTE[] attrs,
                 boolean keySensitive) {
-            if (keySensitive) {
-                return new P11RSAPrivateKeyInternal(session, keyID, algorithm,
+            P11RSAPrivateKeyInternal p11Key = null;
+            if (!keySensitive) {
+                // Key is not sensitive: try to interpret as CRT or non-CRT.
+                p11Key = asCRT(session, keyID, algorithm, keyLength, attrs);
+                if (p11Key == null) {
+                    p11Key = asNonCRT(session, keyID, algorithm, keyLength,
+                            attrs);
+                }
+            }
+            if (p11Key == null) {
+                // Key is sensitive or there was a failure while querying its
+                // attributes: handle as opaque.
+                p11Key = new P11RSAPrivateKeyInternal(session, keyID, algorithm,
                         keyLength, attrs);
-            } else {
-                CK_ATTRIBUTE[] rsaAttrs = new CK_ATTRIBUTE[] {
-                        new CK_ATTRIBUTE(CKA_MODULUS),
-                        new CK_ATTRIBUTE(CKA_PRIVATE_EXPONENT),
-                        new CK_ATTRIBUTE(CKA_PUBLIC_EXPONENT),
-                        new CK_ATTRIBUTE(CKA_PRIME_1),
-                        new CK_ATTRIBUTE(CKA_PRIME_2),
-                        new CK_ATTRIBUTE(CKA_EXPONENT_1),
-                        new CK_ATTRIBUTE(CKA_EXPONENT_2),
-                        new CK_ATTRIBUTE(CKA_COEFFICIENT),
-                };
-                boolean isCRT = true;
-                Session tempSession = null;
-                try {
-                    tempSession = session.token.getOpSession();
-                    session.token.p11.C_GetAttributeValue(tempSession.id(),
-                            keyID, rsaAttrs);
-                    for (CK_ATTRIBUTE attr : rsaAttrs) {
-                        isCRT &= (attr.pValue instanceof byte[]);
-                        if (!isCRT) break;
+            }
+            return p11Key;
+        }
+
+        private static CK_ATTRIBUTE[] tryFetchAttributes(Session session,
+                long keyID, long... attrTypes) {
+            int i = 0;
+            CK_ATTRIBUTE[] attrs = new CK_ATTRIBUTE[attrTypes.length];
+            for (long attrType : attrTypes) {
+                attrs[i++] = new CK_ATTRIBUTE(attrType);
+            }
+            try {
+                session.token.p11.C_GetAttributeValue(session.id(), keyID,
+                        attrs);
+                for (CK_ATTRIBUTE attr : attrs) {
+                    if (!(attr.pValue instanceof byte[])) {
+                        return null;
                     }
-                } catch (PKCS11Exception e) {
-                    // ignore, assume not available
-                    isCRT = false;
-                } finally {
-                    session.token.releaseSession(tempSession);
-                }
-                BigInteger n = rsaAttrs[0].getBigInteger();
-                BigInteger d = rsaAttrs[1].getBigInteger();
-                if (isCRT) {
-                    return new P11RSAPrivateKey(session, keyID, algorithm,
-                           keyLength, attrs, n, d,
-                           Arrays.copyOfRange(rsaAttrs, 2, rsaAttrs.length));
-                } else {
-                    return new P11RSAPrivateNonCRTKey(session, keyID,
-                           algorithm, keyLength, attrs, n, d);
                 }
+                return attrs;
+            } catch (PKCS11Exception ignored) {
+                // ignore, assume not available
+                return null;
+            }
+        }
+
+        private static P11RSAPrivateKeyInternal asCRT(Session session,
+                long keyID, String algorithm, int keyLength,
+                CK_ATTRIBUTE[] attrs) {
+            CK_ATTRIBUTE[] rsaCRTAttrs = tryFetchAttributes(session, keyID,
+                    CKA_MODULUS, CKA_PRIVATE_EXPONENT, CKA_PUBLIC_EXPONENT,
+                    CKA_PRIME_1, CKA_PRIME_2, CKA_EXPONENT_1, CKA_EXPONENT_2,
+                    CKA_COEFFICIENT);
+            if (rsaCRTAttrs == null) {
+                return null;
+            }
+            return new P11RSAPrivateKey(session, keyID, algorithm, keyLength,
+                    attrs, rsaCRTAttrs[0].getBigInteger(),
+                    rsaCRTAttrs[1].getBigInteger(),
+                    Arrays.copyOfRange(rsaCRTAttrs, 2, rsaCRTAttrs.length));
+        }
+
+        private static P11RSAPrivateKeyInternal asNonCRT(Session session,
+                long keyID, String algorithm, int keyLength,
+                CK_ATTRIBUTE[] attrs) {
+            CK_ATTRIBUTE[] rsaNonCRTAttrs = tryFetchAttributes(session, keyID,
+                    CKA_MODULUS, CKA_PRIVATE_EXPONENT);
+            if (rsaNonCRTAttrs == null) {
+                return null;
             }
+            return new P11RSAPrivateNonCRTKey(session, keyID, algorithm,
+                    keyLength, attrs, rsaNonCRTAttrs[0].getBigInteger(),
+                    rsaNonCRTAttrs[1].getBigInteger());
         }
 
         protected transient BigInteger n;