[ruby/openssl] pkey: avoid calling i2d_PUBKEY family on an incomplete key

Call ossl_pkey_check_public_key() to ensure that
EVP_PKEY_missing_parameters() passes. This check should be cheap.

DSA#{to_der,to_pem,export,to_s} and PKey#{public_to_der,public_to_pem}
cause a segfault if the receiver is an empty DSA instance with no
parameters set.

Fixes <https://github.com/ruby/openssl/issues/845>.

5aeed935e5

ext/openssl/ossl_pkey.c

@@ -919,6 +919,7 @@ ossl_pkey_export_spki(VALUE self, int to_der)
     BIO *bio;
 
     GetPKey(self, pkey);
+    ossl_pkey_check_public_key(pkey);
     bio = BIO_new(BIO_s_mem());
     if (!bio)
 	ossl_raise(ePKeyError, "BIO_new");

test/openssl/test_pkey_dsa.rb

@@ -33,6 +33,12 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
     end
   end
 
+  def test_new_empty
+    key = OpenSSL::PKey::DSA.new
+    assert_nil(key.p)
+    assert_raise(OpenSSL::PKey::PKeyError) { key.to_der }
+  end
+
   def test_generate
     # DSA.generate used to call DSA_generate_parameters_ex(), which adjusts the
     # size of q according to the size of p