[pull] master from ruby:master

darray.h

@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include <stddef.h>
 #include <stdlib.h>
+#include "ruby/ruby.h"
 
 // Type for a dynamic array. Use to declare a dynamic array.
 // It is a pointer so it fits in st_table nicely. Designed
@@ -147,6 +148,9 @@ rb_darray_size(const void *ary)
     return meta ? meta->size : 0;
 }
 
+/* Estimate of the amount of memory used by this darray.
+ * Useful for TypedData objects. */
+#define rb_darray_memsize(ary) (sizeof(*(ary)) + (rb_darray_size(ary) * sizeof((ary)->data[0])))
 
 static inline void
 rb_darray_pop(void *ary, size_t count)

depend

@@ -17138,6 +17138,7 @@ symbol.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h
 symbol.$(OBJEXT): {$(VPATH)}builtin.h
 symbol.$(OBJEXT): {$(VPATH)}config.h
 symbol.$(OBJEXT): {$(VPATH)}constant.h
+symbol.$(OBJEXT): {$(VPATH)}darray.h
 symbol.$(OBJEXT): {$(VPATH)}debug_counter.h
 symbol.$(OBJEXT): {$(VPATH)}defines.h
 symbol.$(OBJEXT): {$(VPATH)}encoding.h

gc/default/default.c

@@ -6872,12 +6872,6 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
       case T_ZOMBIE:
         return FALSE;
       case T_SYMBOL:
-        // TODO: restore original behavior
-        // if (RSYMBOL(obj)->id & ~ID_SCOPE_MASK) {
-        //     return FALSE;
-        // }
-        return false;
-        /* fall through */
       case T_STRING:
       case T_OBJECT:
       case T_FLOAT:

internal/imemo.h

@@ -256,8 +256,8 @@ struct rb_fields {
 #define OBJ_FIELD_HEAP ROBJECT_HEAP
 STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_HEAP == IMEMO_FL_USER0);
 STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields));
-STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr));
-STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table));
+STATIC_ASSERT(imemo_fields_external_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr));
+STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table));
 
 #define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields)
 

lib/prism/translation/ripper/lexer.rb

@@ -6,7 +6,7 @@
 module Prism
   module Translation
     class Ripper
-      class Lexer # :nodoc:
+      class Lexer < Ripper # :nodoc:
         # :stopdoc:
         class State
 
@@ -39,6 +39,92 @@ def allbits?(i) to_int.allbits?(i) end
           def anybits?(i) to_int.anybits?(i) end
           def nobits?(i) to_int.nobits?(i) end
         end
+
+        class Elem
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State.new(state)
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        def initialize(...)
+          super
+          @lex_compat = Prism.lex_compat(@source, filepath: filename, line: lineno)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(raise_errors: false)
+          if @lex_compat.failure? && raise_errors
+            raise SyntaxError, @lex_compat.errors.first.message
+          else
+            @lex_compat.value.map do |position, event, token, state|
+              Elem.new(position, event, token, state.to_int)
+            end
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
+        end
+
         # :startdoc:
       end
     end

prism/prism.c

@@ -12438,7 +12438,7 @@ expect1_opening(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t di
 
     pm_parser_err(parser, opening->start, opening->end, diag_id);
 
-    parser->previous.start = opening->end;
+    parser->previous.start = parser->previous.end;
     parser->previous.type = PM_TOKEN_MISSING;
 }