[pull] master from ruby:master

lib/prism/translation/ruby_parser.rb

@@ -1151,8 +1151,8 @@ def visit_keyword_rest_parameter_node(node)
         def visit_lambda_node(node)
           parameters =
             case node.parameters
-            when nil, NumberedParametersNode
-              s(node, :args)
+            when nil, ItParametersNode, NumberedParametersNode
+              0
             else
               visit(node.parameters)
             end

ractor_sync.c

@@ -1221,7 +1221,6 @@ ractor_try_send(rb_execution_context_t *ec, const struct ractor_port *rp, VALUE
 // Ractor::Selector
 
 struct ractor_selector {
-    rb_ractor_t *r;
     struct st_table *ports; // rpv -> rp
 
 };

test/prism/fixtures/it.txt

@@ -1,3 +1,5 @@
 x do
   it
 end
+
+-> { it }

test/prism/ruby/parser_test.rb

@@ -181,9 +181,13 @@ def test_it_block_parameter_syntax
       actual_ast = Prism::Translation::Parser34.new.tokenize(buffer)[0]
 
       it_block_parameter_sexp = parse_sexp {
+        s(:begin,
         s(:itblock,
           s(:send, nil, :x), :it,
-          s(:lvar, :it))
+          s(:lvar, :it)),
+        s(:itblock,
+          s(:lambda), :it,
+          s(:lvar, :it)))
       }
 
       assert_equal(it_block_parameter_sexp, actual_ast.to_sexp)

test/prism/ruby/ruby_parser_test.rb

@@ -13,39 +13,22 @@
   return
 end
 
-# We want to also compare lines and files to make sure we're setting them
-# correctly.
-Sexp.prepend(
-  Module.new do
-    def ==(other)
-      super && line == other.line && file == other.file # && line_max == other.line_max
-    end
-  end
-)
-
 module Prism
   class RubyParserTest < TestCase
     todos = [
       "encoding_euc_jp.txt",
-      "newline_terminated.txt",
       "regex_char_width.txt",
-      "seattlerb/bug169.txt",
       "seattlerb/masgn_colon3.txt",
       "seattlerb/messy_op_asgn_lineno.txt",
       "seattlerb/op_asgn_primary_colon_const_command_call.txt",
       "seattlerb/regexp_esc_C_slash.txt",
       "seattlerb/str_lit_concat_bad_encodings.txt",
       "strings.txt",
       "unescaping.txt",
-      "unparser/corpus/literal/kwbegin.txt",
-      "unparser/corpus/literal/send.txt",
       "whitequark/masgn_const.txt",
       "whitequark/pattern_matching_constants.txt",
-      "whitequark/pattern_matching_implicit_array_match.txt",
       "whitequark/pattern_matching_single_match.txt",
       "whitequark/ruby_bug_12402.txt",
-      "whitequark/ruby_bug_14690.txt",
-      "whitequark/space_args_block.txt"
     ]
 
     # https://github.com/seattlerb/ruby_parser/issues/344
@@ -105,10 +88,16 @@ def assert_ruby_parser(fixture, allowed_failure)
       source = fixture.read
       expected = ignore_warnings { ::RubyParser.new.parse(source, fixture.path) }
       actual = Prism::Translation::RubyParser.new.parse(source, fixture.path)
+      on_failure = -> { message(expected, actual) }
 
       if !allowed_failure
-        assert_equal(expected, actual, -> { message(expected, actual) })
-      elsif expected == actual
+        assert_equal(expected, actual, on_failure)
+
+        unless actual.nil?
+          assert_equal(expected.line, actual.line, on_failure)
+          assert_equal(expected.file, actual.file, on_failure)
+        end
+      elsif expected == actual && expected.line && actual.line && expected.file == actual.file
         puts "#{name} now passes"
       end
     end

test/ruby/test_marshal.rb

@@ -469,6 +469,30 @@ def test_marshal_private_class
     assert_equal(o1.foo, o2.foo)
   end
 
+  class TooComplex
+    def initialize
+      @marshal_too_complex = 1
+    end
+  end
+
+  def test_complex_shape_object_id_not_dumped
+    if defined?(RubyVM::Shape::SHAPE_MAX_VARIATIONS)
+      assert_equal 8, RubyVM::Shape::SHAPE_MAX_VARIATIONS
+    end
+    8.times do |i|
+      TooComplex.new.instance_variable_set("@TestObjectIdTooComplex#{i}", 1)
+    end
+    obj = TooComplex.new
+    ivar = "@a#{rand(10_000).to_s.rjust(5, '0')}"
+    obj.instance_variable_set(ivar, 1)
+
+    if defined?(RubyVM::Shape)
+      assert_predicate(RubyVM::Shape.of(obj), :too_complex?)
+    end
+    obj.object_id
+    assert_equal "\x04\bo:\x1CTestMarshal::TooComplex\a:\x19@marshal_too_complexi\x06:\f#{ivar}i\x06".b, Marshal.dump(obj)
+  end
+
   def test_marshal_complex
     assert_raise(ArgumentError){Marshal.load("\x04\bU:\fComplex[\x05")}
     assert_raise(ArgumentError){Marshal.load("\x04\bU:\fComplex[\x06i\x00")}

variable.c

@@ -2255,6 +2255,9 @@ each_hash_iv(st_data_t id, st_data_t val, st_data_t data)
 {
     struct iv_itr_data * itr_data = (struct iv_itr_data *)data;
     rb_ivar_foreach_callback_func *callback = itr_data->func;
+    if (is_internal_id((ID)id)) {
+        return ST_CONTINUE;
+    }
     return callback((ID)id, (VALUE)val, itr_data->arg);
 }
 

vm_callinfo.h

@@ -585,9 +585,15 @@ struct rb_class_cc_entries {
         unsigned int argc;
         unsigned int flag;
         const struct rb_callcache *cc;
-    } *entries;
+    } entries[FLEX_ARY_LEN];
 };
 
+static inline size_t
+vm_ccs_alloc_size(size_t capa)
+{
+    return offsetof(struct rb_class_cc_entries, entries) + (sizeof(struct rb_class_cc_entries_entry) * capa);
+}
+
 #if VM_CHECK_MODE > 0
 
 const rb_callable_method_entry_t *rb_vm_lookup_overloaded_cme(const rb_callable_method_entry_t *cme);

vm_insnhelper.c

@@ -1981,37 +1981,37 @@ static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
 static struct rb_class_cc_entries *
 vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
 {
-    struct rb_class_cc_entries *ccs = ALLOC(struct rb_class_cc_entries);
+    int initial_capa = 2;
+    struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
 #if VM_CHECK_MODE > 0
     ccs->debug_sig = ~(VALUE)ccs;
 #endif
-    ccs->capa = 0;
+    ccs->capa = initial_capa;
     ccs->len = 0;
     ccs->cme = cme;
     METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
-    ccs->entries = NULL;
 
     rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
     RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
     return ccs;
 }
 
 static void
-vm_ccs_push(VALUE cc_tbl, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
+vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
 {
     if (! vm_cc_markable(cc)) {
         return;
     }
 
     if (UNLIKELY(ccs->len == ccs->capa)) {
-        if (ccs->capa == 0) {
-            ccs->capa = 1;
-            ccs->entries = ALLOC_N(struct rb_class_cc_entries_entry, ccs->capa);
-        }
-        else {
-            ccs->capa *= 2;
-            REALLOC_N(ccs->entries, struct rb_class_cc_entries_entry, ccs->capa);
-        }
+        RUBY_ASSERT(ccs->capa > 0);
+        ccs->capa *= 2;
+        ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
+#if VM_CHECK_MODE > 0
+        ccs->debug_sig = ~(VALUE)ccs;
+#endif
+        // GC?
+        rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
     }
     VM_ASSERT(ccs->len < ccs->capa);
 
@@ -2143,7 +2143,7 @@ vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
     cme = rb_check_overloaded_cme(cme, ci);
 
     const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
-    vm_ccs_push(cc_tbl, ccs, ci, cc);
+    vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
 
     VM_ASSERT(vm_cc_cme(cc) != NULL);
     VM_ASSERT(cme->called_id == mid);

vm_method.c

@@ -22,15 +22,6 @@ static inline rb_method_entry_t *lookup_method_table(VALUE klass, ID id);
 #define ruby_running (GET_VM()->running)
 /* int ruby_running = 0; */
 
-static void
-vm_ccs_free(struct rb_class_cc_entries *ccs)
-{
-    if (ccs->entries) {
-        ruby_xfree(ccs->entries);
-    }
-    ruby_xfree(ccs);
-}
-
 static enum rb_id_table_iterator_result
 mark_cc_entry_i(VALUE ccs_ptr, void *data)
 {
@@ -39,7 +30,7 @@ mark_cc_entry_i(VALUE ccs_ptr, void *data)
     VM_ASSERT(vm_ccs_p(ccs));
 
     if (METHOD_ENTRY_INVALIDATED(ccs->cme)) {
-        vm_ccs_free(ccs);
+        ruby_xfree(ccs);
         return ID_TABLE_DELETE;
     }
     else {
@@ -69,7 +60,7 @@ cc_table_free_i(VALUE ccs_ptr, void *data)
     struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
     VM_ASSERT(vm_ccs_p(ccs));
 
-    vm_ccs_free(ccs);
+    ruby_xfree(ccs);
 
     return ID_TABLE_CONTINUE;
 }
@@ -146,13 +137,13 @@ static enum rb_id_table_iterator_result
 vm_cc_table_dup_i(ID key, VALUE old_ccs_ptr, void *data)
 {
     struct rb_class_cc_entries *old_ccs = (struct rb_class_cc_entries *)old_ccs_ptr;
-    struct rb_class_cc_entries *new_ccs = ALLOC(struct rb_class_cc_entries);
-    MEMCPY(new_ccs, old_ccs, struct rb_class_cc_entries, 1);
+    size_t memsize = vm_ccs_alloc_size(old_ccs->capa);
+    struct rb_class_cc_entries *new_ccs = ruby_xmalloc(memsize);
+    memcpy(new_ccs, old_ccs, memsize);
+
 #if VM_CHECK_MODE > 0
     new_ccs->debug_sig = ~(VALUE)new_ccs;
 #endif
-    new_ccs->entries = ALLOC_N(struct rb_class_cc_entries_entry, new_ccs->capa);
-    MEMCPY(new_ccs->entries, old_ccs->entries, struct rb_class_cc_entries_entry, new_ccs->capa);
 
     VALUE new_table = (VALUE)data;
     rb_managed_id_table_insert(new_table, key, (VALUE)new_ccs);
@@ -173,12 +164,10 @@ rb_vm_cc_table_dup(VALUE old_table)
 static void
 vm_ccs_invalidate(struct rb_class_cc_entries *ccs)
 {
-    if (ccs->entries) {
-        for (int i=0; i<ccs->len; i++) {
-            const struct rb_callcache *cc = ccs->entries[i].cc;
-            VM_ASSERT(!vm_cc_super_p(cc) && !vm_cc_refinement_p(cc));
-            vm_cc_invalidate(cc);
-        }
+    for (int i=0; i<ccs->len; i++) {
+        const struct rb_callcache *cc = ccs->entries[i].cc;
+        VM_ASSERT(!vm_cc_super_p(cc) && !vm_cc_refinement_p(cc));
+        vm_cc_invalidate(cc);
     }
 }
 
@@ -187,7 +176,7 @@ rb_vm_ccs_invalidate_and_free(struct rb_class_cc_entries *ccs)
 {
     RB_DEBUG_COUNTER_INC(ccs_free);
     vm_ccs_invalidate(ccs);
-    vm_ccs_free(ccs);
+    ruby_xfree(ccs);
 }
 
 void