* ext/openssl/ossl_cipher.c: remove the encryption key initialization

from Cipher#initialize. This is effectively a revert of r32723
  ("Avoid possible SEGV from AES encryption/decryption", 2011-07-28).
  the patch is derived from 8108e0a6db,
  written by Kazuki Yamaguchi.
  [Backport #8221]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_3@59267 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,12 @@
+Wed Jul  5 15:55:35 2017  NAKAMURA Usaku  <usa@ruby-lang.org>
+
+	* ext/openssl/ossl_cipher.c: remove the encryption key initialization
+	  from Cipher#initialize. This is effectively a revert of r32723
+	  ("Avoid possible SEGV from AES encryption/decryption", 2011-07-28).
+	  the patch is derived from https://github.com/ruby/openssl/commit/8108e0a6db133f3375608303fdd2083eb5115062,
+	  written by Kazuki Yamaguchi.
+	  [Backport #8221]
+
 Sat Jul  1 00:28:22 2017  SHIBATA Hiroshi  <hsbt@ruby-lang.org>
 
 	* ext/psych/yaml: update libyaml to 0.1.7.

ext/openssl/ossl_cipher.c

@@ -34,6 +34,7 @@
  */
 VALUE cCipher;
 VALUE eCipherError;
+static ID id_key_set;
 
 static VALUE ossl_cipher_alloc(VALUE klass);
 static void ossl_cipher_free(void *ptr);
@@ -114,7 +115,6 @@ ossl_cipher_initialize(VALUE self, VALUE str)
     EVP_CIPHER_CTX *ctx;
     const EVP_CIPHER *cipher;
     char *name;
-    unsigned char key[EVP_MAX_KEY_LENGTH];
 
     name = StringValuePtr(str);
     GetCipherInit(self, ctx);
@@ -126,14 +126,7 @@ ossl_cipher_initialize(VALUE self, VALUE str)
     if (!(cipher = EVP_get_cipherbyname(name))) {
 	ossl_raise(rb_eRuntimeError, "unsupported cipher algorithm (%s)", name);
     }
-    /*
-     * The EVP which has EVP_CIPH_RAND_KEY flag (such as DES3) allows
-     * uninitialized key, but other EVPs (such as AES) does not allow it.
-     * Calling EVP_CipherUpdate() without initializing key causes SEGV so we
-     * set the data filled with "\0" as the key by default.
-     */
-    memset(key, 0, EVP_MAX_KEY_LENGTH);
-    if (EVP_CipherInit_ex(ctx, cipher, NULL, key, NULL, -1) != 1)
+    if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, -1) != 1)
 	ossl_raise(eCipherError, NULL);
 
     return self;
@@ -252,6 +245,9 @@ ossl_cipher_init(int argc, VALUE *argv, VALUE self, int mode)
 	ossl_raise(eCipherError, NULL);
     }
 
+    if (p_key)
+	rb_ivar_set(self, id_key_set, Qtrue);
+
     return self;
 }
 
@@ -338,6 +334,8 @@ ossl_cipher_pkcs5_keyivgen(int argc, VALUE *argv, VALUE self)
     OPENSSL_cleanse(key, sizeof key);
     OPENSSL_cleanse(iv, sizeof iv);
 
+    rb_ivar_set(self, id_key_set, Qtrue);
+
     return Qnil;
 }
 
@@ -391,6 +389,9 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
 
     rb_scan_args(argc, argv, "11", &data, &str);
 
+    if (!RTEST(rb_attr_get(self, id_key_set)))
+	ossl_raise(eCipherError, "key not set");
+
     StringValue(data);
     in = (unsigned char *)RSTRING_PTR(data);
     if ((in_len = RSTRING_LEN(data)) == 0)
@@ -490,6 +491,8 @@ ossl_cipher_set_key(VALUE self, VALUE key)
     if (EVP_CipherInit_ex(ctx, NULL, NULL, (unsigned char *)RSTRING_PTR(key), NULL, -1) != 1)
         ossl_raise(eCipherError, NULL);
 
+    rb_ivar_set(self, id_key_set, Qtrue);
+
     return key;
 }
 
@@ -1008,4 +1011,6 @@ Init_ossl_cipher(void)
     rb_define_method(cCipher, "iv_len", ossl_cipher_iv_length, 0);
     rb_define_method(cCipher, "block_size", ossl_cipher_block_size, 0);
     rb_define_method(cCipher, "padding=", ossl_cipher_set_padding, 1);
+
+    id_key_set = rb_intern_const("key_set");
 }

test/openssl/test_cipher.rb

@@ -81,6 +81,7 @@ class OpenSSL::TestCipher < Test::Unit::TestCase
 
   def test_empty_data
     @c1.encrypt
+    @c1.random_key
     assert_raise(ArgumentError){ @c1.update("") }
   end
 
@@ -129,12 +130,10 @@ class OpenSSL::TestCipher < Test::Unit::TestCase
       }
     end
 
-    def test_AES_crush
-      500.times do
-        assert_nothing_raised("[Bug #2768]") do
-          # it caused OpenSSL SEGV by uninitialized key
-          OpenSSL::Cipher::AES128.new("ECB").update "." * 17
-        end
+    def test_update_raise_if_key_not_set
+      assert_raise(OpenSSL::Cipher::CipherError) do
+        # it caused OpenSSL SEGV by uninitialized key [Bug #2768]
+        OpenSSL::Cipher::AES128.new("ECB").update "." * 17
       end
     end
   end
@@ -236,6 +235,24 @@ class OpenSSL::TestCipher < Test::Unit::TestCase
       end
     end
 
+    def test_aes_gcm_key_iv_order_issue
+      pt = "[ruby/openssl#49]"
+      cipher = OpenSSL::Cipher.new("aes-128-gcm").encrypt
+      cipher.key = "x" * 16
+      cipher.iv = "a" * 12
+      ct1 = cipher.update(pt) << cipher.final
+      tag1 = cipher.auth_tag
+
+      cipher = OpenSSL::Cipher.new("aes-128-gcm").encrypt
+      cipher.iv = "a" * 12
+      cipher.key = "x" * 16
+      ct2 = cipher.update(pt) << cipher.final
+      tag2 = cipher.auth_tag
+
+      assert_equal ct1, ct2
+      assert_equal tag1, tag2
+    end if has_cipher?("aes-128-gcm")
+
   end
 
   private

version.h

@@ -1,10 +1,10 @@
 #define RUBY_VERSION "2.3.5"
-#define RUBY_RELEASE_DATE "2017-07-01"
-#define RUBY_PATCHLEVEL 339
+#define RUBY_RELEASE_DATE "2017-07-05"
+#define RUBY_PATCHLEVEL 340
 
 #define RUBY_RELEASE_YEAR 2017
 #define RUBY_RELEASE_MONTH 7
-#define RUBY_RELEASE_DAY 1
+#define RUBY_RELEASE_DAY 5
 
 #include "ruby/version.h"