check ar_table after #hash call

ar_table can be converted to st_table just after `ar_do_hash()`
function which calls `#hash` method. We need to check
the representation to detect this mutation.
[Bug #16676]

(cherry picked from commit 4c019f5a62)

hash.c

@@ -996,6 +996,11 @@ ar_update(VALUE hash, st_data_t key,
     st_data_t value = 0, old_key;
     st_hash_t hash_value = ar_do_hash(key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        // `#hash` changes ar_table -> st_table
+        return -1;
+    }
+
     if (RHASH_AR_TABLE_SIZE(hash) > 0) {
         bin = ar_find_entry(hash, hash_value, key);
         existing = (bin != RHASH_AR_TABLE_MAX_BOUND) ? TRUE : FALSE;
@@ -1045,6 +1050,11 @@ ar_insert(VALUE hash, st_data_t key, st_data_t value)
     unsigned bin = RHASH_AR_TABLE_BOUND(hash);
     st_hash_t hash_value = ar_do_hash(key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        // `#hash` changes ar_table -> st_table
+        return -1;
+    }
+
     hash_ar_table(hash); /* prepare ltbl */
 
     bin = ar_find_entry(hash, hash_value, key);
@@ -1077,6 +1087,10 @@ ar_lookup(VALUE hash, st_data_t key, st_data_t *value)
     }
     else {
         st_hash_t hash_value = ar_do_hash(key);
+        if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+            // `#hash` changes ar_table -> st_table
+            return st_lookup(RHASH_ST_TABLE(hash), key, value);
+        }
         unsigned bin = ar_find_entry(hash, hash_value, key);
 
         if (bin == RHASH_AR_TABLE_MAX_BOUND) {
@@ -1098,6 +1112,10 @@ ar_delete(VALUE hash, st_data_t *key, st_data_t *value)
     unsigned bin;
     st_hash_t hash_value = ar_do_hash(*key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        // `#hash` changes ar_table -> st_table
+        return st_delete(RHASH_ST_TABLE(hash), key, value);
+    }
 
     bin = ar_find_entry(hash, hash_value, *key);
 

test/ruby/test_hash.rb

@@ -1764,4 +1764,62 @@ class TestHash < Test::Unit::TestCase
     assert_equal(1, check_flagged_hash(*[hash]))
     assert_raise(TypeError) { Hash.ruby2_keywords_hash(1) }
   end
+
+  def test_ar2st
+    # insert
+    obj = Object.new
+    obj.instance_variable_set(:@h, h = {})
+    def obj.hash
+      10.times{|i| @h[i] = i}
+      0
+    end
+    def obj.inspect
+      'test'
+    end
+    h[obj] = true
+    assert_equal '{0=>0, 1=>1, 2=>2, 3=>3, 4=>4, 5=>5, 6=>6, 7=>7, 8=>8, 9=>9, test=>true}', h.inspect
+
+    # delete
+    obj = Object.new
+    obj.instance_variable_set(:@h, h = {})
+    def obj.hash
+      10.times{|i| @h[i] = i}
+      0
+    end
+    def obj.inspect
+      'test'
+    end
+    def obj.eql? other
+      other.class == Object
+    end
+    obj2 = Object.new
+    def obj2.hash
+      0
+    end
+
+    h[obj2] = true
+    h.delete obj
+    assert_equal '{0=>0, 1=>1, 2=>2, 3=>3, 4=>4, 5=>5, 6=>6, 7=>7, 8=>8, 9=>9}', h.inspect
+
+    # lookup
+    obj = Object.new
+    obj.instance_variable_set(:@h, h = {})
+    def obj.hash
+      10.times{|i| @h[i] = i}
+      0
+    end
+    def obj.inspect
+      'test'
+    end
+    def obj.eql? other
+      other.class == Object
+    end
+    obj2 = Object.new
+    def obj2.hash
+      0
+    end
+
+    h[obj2] = true
+    assert_equal true, h[obj]
+  end
 end