[pull] master from ruby:master

NEWS.md

@@ -46,7 +46,7 @@ releases.
 
 * RubyGems 4.1.0.dev
 * bundler 4.1.0.dev
-* prism 1.8.0
+* prism 1.9.0
 * stringio 3.2.1.dev
 * strscan 3.1.7.dev
 * syntax_suggest 2.0.3

gems/bundled_gems

@@ -34,7 +34,7 @@ drb                 2.2.3   https://github.com/ruby/drb
 nkf                 0.2.0   https://github.com/ruby/nkf
 syslog              0.3.0   https://github.com/ruby/syslog
 csv                 3.3.5   https://github.com/ruby/csv
-repl_type_completor 0.1.12  https://github.com/ruby/repl_type_completor
+repl_type_completor 0.1.12  https://github.com/ruby/repl_type_completor   26b8e964557690c0b539cff8940bcfb1591f1fe6
 ostruct             0.6.3   https://github.com/ruby/ostruct
 pstore              0.2.0   https://github.com/ruby/pstore
 benchmark           0.5.0   https://github.com/ruby/benchmark

hash.c

@@ -4492,21 +4492,21 @@ flatten_i(VALUE key, VALUE val, VALUE ary)
  *  Examples; note that entry <tt>foo: {bar: 1, baz: 2}</tt> is never flattened.
  *
  *   h = {foo: {bar: 1, baz: 2}, bat: [:bam, [:bap, [:bah]]]}
- *   h.flatten(1) # => [:foo, {:bar=>1, :baz=>2}, :bat, [:bam, [:bap, [:bah]]]]
- *   h.flatten(2) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, [:bap, [:bah]]]
- *   h.flatten(3) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, [:bah]]
- *   h.flatten(4) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah]
- *   h.flatten(5) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah]
+ *   h.flatten(1) # => [:foo, {bar: 1, baz: 2}, :bat, [:bam, [:bap, [:bah]]]]
+ *   h.flatten(2) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, [:bap, [:bah]]]
+ *   h.flatten(3) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, [:bah]]
+ *   h.flatten(4) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah]
+ *   h.flatten(5) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah]
  *
  *  With negative integer +depth+,
  *  flattens all levels:
  *
- *    h.flatten(-1) # => [:foo, {:bar=>1, :baz=>2}, :bat, :bam, :bap, :bah]
+ *    h.flatten(-1) # => [:foo, {bar: 1, baz: 2}, :bat, :bam, :bap, :bah]
  *
  *  With +depth+ zero,
  *  returns the equivalent of #to_a:
  *
- *    h.flatten(0) # => [[:foo, {:bar=>1, :baz=>2}], [:bat, [:bam, [:bap, [:bah]]]]]
+ *    h.flatten(0) # => [[:foo, {bar: 1, baz: 2}], [:bat, [:bam, [:bap, [:bah]]]]]
  *
  *  Related: see {Methods for Converting}[rdoc-ref:Hash@Methods+for+Converting].
  */

iseq.c

@@ -1142,7 +1142,7 @@ pm_iseq_new_with_opt(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpa
     int32_t start_line = node->parser->start_line;
 
     pm_line_column_t start = pm_newline_list_line_column(&node->parser->newline_list, location->start, start_line);
-    pm_line_column_t end = pm_newline_list_line_column(&node->parser->newline_list, location->end, start_line);
+    pm_line_column_t end = pm_newline_list_line_column(&node->parser->newline_list, location->start + location->length, start_line);
 
     rb_code_location_t code_location = (rb_code_location_t) {
         .beg_pos = { .lineno = (int) start.line, .column = (int) start.column },

lib/prism/lex_compat.rb

@@ -639,7 +639,7 @@ def result
 
         event = RIPPER.fetch(token.type)
         value = token.value
-        lex_state = Translation::Ripper::Lexer::State.cached(lex_state)
+        lex_state = Translation::Ripper::Lexer::State[lex_state]
 
         token =
           case event
@@ -691,7 +691,7 @@ def result
                   counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
                 end
 
-                Translation::Ripper::Lexer::State.cached(result_value[current_index][1])
+                Translation::Ripper::Lexer::State[result_value[current_index][1]]
               else
                 previous_state
               end
@@ -816,25 +816,29 @@ def result
       # Manually implemented instead of `sort_by!(&:location)` for performance.
       tokens.sort_by! do |token|
         line, column = token.location
-        source.line_to_byte_offset(line) + column
+        source.byte_offset(line, column)
       end
 
       # Add :on_sp tokens
-      tokens = add_on_sp_tokens(tokens, source, result.data_loc, bom, eof_token)
+      tokens = insert_on_sp(tokens, source, result.data_loc, bom, eof_token)
 
       Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source)
     end
 
-    def add_on_sp_tokens(tokens, source, data_loc, bom, eof_token)
+    private
+
+    def insert_on_sp(tokens, source, data_loc, bom, eof_token)
       new_tokens = []
 
-      prev_token_state = Translation::Ripper::Lexer::State.cached(Translation::Ripper::EXPR_BEG)
+      prev_token_state = Translation::Ripper::Lexer::State[Translation::Ripper::EXPR_BEG]
       prev_token_end = bom ? 3 : 0
 
       tokens.each do |token|
         line, column = token.location
-        start_offset = source.line_to_byte_offset(line) + column
-        # Ripper reports columns on line 1 without counting the BOM, so we adjust to get the real offset
+        start_offset = source.byte_offset(line, column)
+
+        # Ripper reports columns on line 1 without counting the BOM, so we
+        # adjust to get the real offset
         start_offset += 3 if line == 1 && bom
 
         if start_offset > prev_token_end

lib/prism/parse_result.rb

@@ -76,13 +76,13 @@ def slice(byte_offset, length)
       source.byteslice(byte_offset, length) or raise
     end
 
-    # Converts the line number to a byte offset corresponding to the start of that line
-    def line_to_byte_offset(line)
-      l = line - @start_line
-      if l < 0 || l >= offsets.size
-        raise ArgumentError, "line #{line} is out of range"
-      end
-      offsets[l]
+    # Converts the line number and column in bytes to a byte offset.
+    def byte_offset(line, column)
+      normal = line - @start_line
+      raise IndexError if normal < 0
+      offsets.fetch(normal) + column
+    rescue IndexError
+      raise ArgumentError, "line #{line} is out of range"
     end
 
     # Binary search through the offsets to find the line number for the given
@@ -103,7 +103,7 @@ def line_end(byte_offset)
       offsets[find_line(byte_offset) + 1] || source.bytesize
     end
 
-    # Return the column number for the given byte offset.
+    # Return the column in bytes for the given byte offset.
     def column(byte_offset)
       byte_offset - line_start(byte_offset)
     end
@@ -113,7 +113,7 @@ def character_offset(byte_offset)
       (source.byteslice(0, byte_offset) or raise).length
     end
 
-    # Return the column number in characters for the given byte offset.
+    # Return the column in characters for the given byte offset.
     def character_column(byte_offset)
       character_offset(byte_offset) - character_offset(line_start(byte_offset))
     end
@@ -146,7 +146,7 @@ def code_units_cache(encoding)
       CodeUnitsCache.new(source, encoding)
     end
 
-    # Returns the column number in code units for the given encoding for the
+    # Returns the column in code units for the given encoding for the
     # given byte offset.
     def code_units_column(byte_offset, encoding)
       code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
@@ -253,7 +253,7 @@ def character_offset(byte_offset)
       byte_offset
     end
 
-    # Return the column number in characters for the given byte offset.
+    # Return the column in characters for the given byte offset.
     def character_column(byte_offset)
       byte_offset - line_start(byte_offset)
     end
@@ -428,19 +428,19 @@ def end_line
       source.line(end_offset)
     end
 
-    # The column number in bytes where this location starts from the start of
+    # The column in bytes where this location starts from the start of
     # the line.
     def start_column
       source.column(start_offset)
     end
 
-    # The column number in characters where this location ends from the start of
+    # The column in characters where this location ends from the start of
     # the line.
     def start_character_column
       source.character_column(start_offset)
     end
 
-    # The column number in code units of the given encoding where this location
+    # The column in code units of the given encoding where this location
     # starts from the start of the line.
     def start_code_units_column(encoding = Encoding::UTF_16LE)
       source.code_units_column(start_offset, encoding)
@@ -452,19 +452,19 @@ def cached_start_code_units_column(cache)
       cache[start_offset] - cache[source.line_start(start_offset)]
     end
 
-    # The column number in bytes where this location ends from the start of the
+    # The column in bytes where this location ends from the start of the
     # line.
     def end_column
       source.column(end_offset)
     end
 
-    # The column number in characters where this location ends from the start of
+    # The column in characters where this location ends from the start of
     # the line.
     def end_character_column
       source.character_column(end_offset)
     end
 
-    # The column number in code units of the given encoding where this location
+    # The column in code units of the given encoding where this location
     # ends from the start of the line.
     def end_code_units_column(encoding = Encoding::UTF_16LE)
       source.code_units_column(end_offset, encoding)

lib/prism/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.8.0"
+  spec.version = "1.9.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 

lib/prism/translation/parser/compiler.rb

@@ -1767,7 +1767,7 @@ def visit_symbol_node(node)
             end
           else
             parts =
-              if node.value == ""
+              if node.value_loc.nil?
                 []
               elsif node.value.include?("\n")
                 string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening)

lib/prism/translation/parser/lexer.rb

@@ -18,8 +18,6 @@ class Lexer
         # The direct translating of types between the two lexers.
         TYPES = {
           # These tokens should never appear in the output of the lexer.
-          MISSING: nil,
-          NOT_PROVIDED: nil,
           EMBDOC_END: nil,
           EMBDOC_LINE: nil,
 

lib/prism/translation/ripper.rb

@@ -475,7 +475,7 @@ def self.lex_state_name(state)
       # The current line number of the parser.
       attr_reader :lineno
 
-      # The current column number of the parser.
+      # The current column in bytes of the parser.
       attr_reader :column
 
       # Create a new Translation::Ripper object with the given source.
@@ -3152,14 +3152,13 @@ def visit_super_node(node)
       # :foo
       # ^^^^
       def visit_symbol_node(node)
-        if (opening = node.opening)&.match?(/^%s|['"]:?$/)
+        if node.value_loc.nil?
+          bounds(node.location)
+          on_dyna_symbol(on_string_content)
+        elsif (opening = node.opening)&.match?(/^%s|['"]:?$/)
           bounds(node.value_loc)
-          content = on_string_content
-
-          if !(value = node.value).empty?
-            content = on_string_add(content, on_tstring_content(value))
-          end
-
+          content = on_string_add(on_string_content, on_tstring_content(node.value))
+          bounds(node.location)
           on_dyna_symbol(content)
         elsif (closing = node.closing) == ":"
           bounds(node.location)

lib/prism/translation/ripper/lexer.rb

@@ -9,7 +9,6 @@ class Ripper
       class Lexer < Ripper # :nodoc:
         # :stopdoc:
         class State
-
           attr_reader :to_int, :to_s
 
           def initialize(i)
@@ -39,10 +38,12 @@ def allbits?(i) to_int.allbits?(i) end
           def anybits?(i) to_int.anybits?(i) end
           def nobits?(i) to_int.nobits?(i) end
 
-          # Instances are frozen and there are only a handful of them so we cache them here.
-          STATES = Hash.new { |h,k| h[k] = State.new(k) }
+          # Instances are frozen and there are only a handful of them so we
+          # cache them here.
+          STATES = Hash.new { |hash, key| hash[key] = State.new(key) }
+          private_constant :STATES
 
-          def self.cached(i)
+          def self.[](i)
             STATES[i]
           end
         end
@@ -54,7 +55,7 @@ def initialize(pos, event, tok, state, message = nil)
             @pos = pos
             @event = event
             @tok = tok
-            @state = State.cached(state)
+            @state = State[state]
             @message = message
           end
 

prism/config.yml

@@ -653,10 +653,6 @@ tokens:
     comment: "a separator between words in a list"
   - name: __END__
     comment: "marker for the point in the file at which the parser should stop"
-  - name: MISSING
-    comment: "a token that was expected but not found"
-  - name: NOT_PROVIDED
-    comment: "a token that was not present but it is okay"
 flags:
   - name: ArgumentsNodeFlags
     values:

prism/defines.h

@@ -257,4 +257,37 @@
     #define PRISM_FALLTHROUGH
 #endif
 
+/**
+ * We need to align nodes in the AST to a pointer boundary so that it can be
+ * safely cast to different node types. Use PRISM_ALIGNAS/PRISM_ALIGNOF to
+ * specify alignment in a compiler-agnostic way.
+ */
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L /* C11 or later */
+    #include <stdalign.h>
+
+    /** Specify alignment for a type or variable. */
+    #define PRISM_ALIGNAS(size) alignas(size)
+
+    /** Get the alignment requirement of a type. */
+    #define PRISM_ALIGNOF(type) alignof(type)
+#elif defined(__GNUC__) || defined(__clang__)
+    /** Specify alignment for a type or variable. */
+    #define PRISM_ALIGNAS(size) __attribute__((aligned(size)))
+
+    /** Get the alignment requirement of a type. */
+    #define PRISM_ALIGNOF(type) __alignof__(type)
+#elif defined(_MSC_VER)
+    /** Specify alignment for a type or variable. */
+    #define PRISM_ALIGNAS(size) __declspec(align(size))
+
+    /** Get the alignment requirement of a type. */
+    #define PRISM_ALIGNOF(type) __alignof(type)
+#else
+    /** Void because this platform does not support specifying alignment. */
+    #define PRISM_ALIGNAS(size)
+
+    /** Fallback to sizeof as alignment requirement of a type. */
+    #define PRISM_ALIGNOF(type) sizeof(type)
+#endif
+
 #endif