8225392: Comparison builds are failing due to cacerts file

Reviewed-by: erikj, martin, mullan

make/jdk/src/classes/build/tools/generatecacerts/GenerateCacerts.java

@@ -25,12 +25,23 @@
 
 package build.tools.generatecacerts;
 
+import java.io.DataOutputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.KeyStore;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Generate cacerts
@@ -39,23 +50,99 @@ import java.security.cert.CertificateFactory;
  */
 public class GenerateCacerts {
     public static void main(String[] args) throws Exception {
-        KeyStore ks = KeyStore.getInstance("JKS");
-        ks.load(null, null);
-        CertificateFactory cf = CertificateFactory.getInstance("X509");
-        Files.list(Path.of(args[0]))
-                .filter(p -> !p.getFileName().toString().equals("README"))
-                .forEach(p -> {
-                    try {
-                        String alias = p.getFileName().toString() + " [jdk]";
-                        try (InputStream fis = Files.newInputStream(p)) {
-                            ks.setCertificateEntry(alias, cf.generateCertificate(fis));
-                        }
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
-                });
         try (FileOutputStream fos = new FileOutputStream(args[1])) {
-            ks.store(fos, "changeit".toCharArray());
+            store(args[0], fos, "changeit".toCharArray());
         }
     }
+
+    // The following code are copied from JavaKeyStore.java.
+
+    private static final int MAGIC = 0xfeedfeed;
+    private static final int VERSION_2 = 0x02;
+
+    // This method is a simplified version of JavaKeyStore::engineStore.
+    // A new "dir" argument is added. All cert names in "dir" is collected into
+    // a sorted array. Each cert is stored with a creation date set to its
+    // notBefore value. Thus the output is determined as long as the certs
+    // are the same.
+    public static void store(String dir, OutputStream stream, char[] password)
+            throws IOException, NoSuchAlgorithmException, CertificateException
+    {
+        byte[] encoded; // the certificate encoding
+        CertificateFactory cf = CertificateFactory.getInstance("X509");
+
+        MessageDigest md = getPreKeyedHash(password);
+        DataOutputStream dos
+                = new DataOutputStream(new DigestOutputStream(stream, md));
+
+        dos.writeInt(MAGIC);
+        // always write the latest version
+        dos.writeInt(VERSION_2);
+
+        // All file names in dir sorted.
+        // README is excluded. Name starting with "." excluded.
+        List<String> entries = Files.list(Path.of(dir))
+                .map(p -> p.getFileName().toString())
+                .filter(s -> !s.equals("README") && !s.startsWith("."))
+                .collect(Collectors.toList());
+
+        entries.sort(String::compareTo);
+
+        dos.writeInt(entries.size());
+
+        for (String entry : entries) {
+
+            String alias = entry + " [jdk]";
+            X509Certificate cert;
+            try (InputStream fis = Files.newInputStream(Path.of(dir, entry))) {
+                cert = (X509Certificate) cf.generateCertificate(fis);
+            }
+
+            dos.writeInt(2);
+
+            // Write the alias
+            dos.writeUTF(alias);
+
+            // Write the (entry creation) date, which is notBefore of the cert
+            dos.writeLong(cert.getNotBefore().getTime());
+
+            // Write the trusted certificate
+            encoded = cert.getEncoded();
+            dos.writeUTF(cert.getType());
+            dos.writeInt(encoded.length);
+            dos.write(encoded);
+        }
+
+        /*
+         * Write the keyed hash which is used to detect tampering with
+         * the keystore (such as deleting or modifying key or
+         * certificate entries).
+         */
+        byte[] digest = md.digest();
+
+        dos.write(digest);
+        dos.flush();
+    }
+
+    private static MessageDigest getPreKeyedHash(char[] password)
+            throws NoSuchAlgorithmException, UnsupportedEncodingException
+    {
+
+        MessageDigest md = MessageDigest.getInstance("SHA");
+        byte[] passwdBytes = convertToBytes(password);
+        md.update(passwdBytes);
+        Arrays.fill(passwdBytes, (byte) 0x00);
+        md.update("Mighty Aphrodite".getBytes("UTF8"));
+        return md;
+    }
+
+    private static byte[] convertToBytes(char[] password) {
+        int i, j;
+        byte[] passwdBytes = new byte[password.length * 2];
+        for (i=0, j=0; i<password.length; i++) {
+            passwdBytes[j++] = (byte)(password[i] >> 8);
+            passwdBytes[j++] = (byte)password[i];
+        }
+        return passwdBytes;
+    }
 }

src/java.base/share/classes/sun/security/tools/keytool/Main.java

@@ -2418,9 +2418,9 @@ public final class Main {
         out.println(form.format(source));
         out.println();
 
-        for (Enumeration<String> e = keyStore.aliases();
+        List<String> aliases = Collections.list(keyStore.aliases());
-                                        e.hasMoreElements(); ) {
+        aliases.sort(String::compareTo);
-            String alias = e.nextElement();
+        for (String alias : aliases) {
             doPrintEntry("<" + alias + ">", alias, out);
             if (verbose || rfc) {
                 out.println(rb.getString("NEWLINE"));

test/jdk/sun/security/lib/cacerts/VerifyCACerts.java

@@ -26,11 +26,13 @@
  * @test
  * @bug 8189131 8198240 8191844 8189949 8191031 8196141 8204923 8195774 8199779
  *      8209452 8209506 8210432 8195793 8216577 8222089 8222133 8222137 8222136
- *      8223499
+ *      8223499 8225392
  * @summary Check root CA entries in cacerts file
  */
+import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.KeyStore;
 import java.security.MessageDigest;
 import java.security.cert.Certificate;
@@ -52,6 +54,11 @@ public class VerifyCACerts {
     // The numbers of certs now.
     private static final int COUNT = 88;
 
+    // SHA-256 of cacerts, can be generated with
+    // shasum -a 256 cacerts | sed -e 's/../&:/g' | tr '[:lower:]' '[:upper:]' | cut -c1-95
+    private static final String CHECKSUM
+            = "4E:21:94:7C:1D:49:28:BB:34:B0:40:DF:AE:19:B4:41:C6:B5:8A:EE:EB:D5:DE:B4:EF:07:AF:63:18:73:A6:FE";
+
     // map of cert alias to SHA-256 fingerprint
     @SuppressWarnings("serial")
     private static final Map<String, String> FINGERPRINT_MAP = new HashMap<>() {
@@ -255,8 +262,16 @@ public class VerifyCACerts {
     public static void main(String[] args) throws Exception {
         System.out.println("cacerts file: " + CACERTS);
         md = MessageDigest.getInstance("SHA-256");
+
+        byte[] data = Files.readAllBytes(Path.of(CACERTS));
+        String checksum = toHexString(md.digest(data));
+        if (!checksum.equals(CHECKSUM)) {
+            atLeastOneFailed = true;
+            System.err.println("ERROR: wrong checksum\n" + checksum);
+        }
+
         KeyStore ks = KeyStore.getInstance("JKS");
-        ks.load(new FileInputStream(CACERTS), "changeit".toCharArray());
+        ks.load(new ByteArrayInputStream(data), "changeit".toCharArray());
 
         // check the count of certs inside
         if (ks.size() != COUNT) {

test/jdk/sun/security/tools/keytool/ListOrder.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8225392
+ * @summary Comparison builds are failing due to cacerts file
+ * @library /test/lib
+ */
+
+import jdk.test.lib.SecurityTools;
+
+import java.util.Random;
+
+public class ListOrder {
+
+    public static void main(String[] args) throws Throwable {
+
+        Random rand = new Random();
+        for (int i = 0; i < 10; i++) {
+            gen(String.format("a%02d", rand.nextInt(100)));
+        }
+
+        String last = "";
+        for (String line : SecurityTools.keytool(
+                "-keystore ks -storepass changeit -list").asLines()) {
+            if (line.contains("PrivateKeyEntry")) {
+                // This is the line starting with the alias
+                System.out.println(line);
+                if (line.compareTo(last) <= 0) {
+                    throw new RuntimeException("Not ordered");
+                } else {
+                    last = line;
+                }
+            }
+        }
+    }
+
+    static void gen(String a) throws Exception {
+        // Do not check result, there might be duplicated alias(es).
+        SecurityTools.keytool("-keystore ks -storepass changeit "
+                + "-keyalg ec -genkeypair -alias " + a + " -dname CN=" + a);
+    }
+}