[pull] master from ruby:master

NEWS.md

@@ -182,7 +182,7 @@ The following bundled gems are updated.
 * minitest 5.25.5
 * rake 13.3.0
 * test-unit 3.7.0
-* rexml 3.4.1
+* rexml 3.4.2
 * net-imap 0.5.9
 * net-smtp 0.5.1
 * matrix 0.4.3

gems/bundled_gems

@@ -10,7 +10,7 @@ minitest            5.25.5  https://github.com/minitest/minitest
 power_assert        2.0.5   https://github.com/ruby/power_assert f88e406e7c9e0810cc149869582afbae1fb84c4a
 rake                13.3.0  https://github.com/ruby/rake
 test-unit           3.7.0   https://github.com/test-unit/test-unit
-rexml               3.4.1   https://github.com/ruby/rexml
+rexml               3.4.2   https://github.com/ruby/rexml
 rss                 0.3.1   https://github.com/ruby/rss
 net-ftp             0.3.8   https://github.com/ruby/net-ftp
 net-imap            0.5.9   https://github.com/ruby/net-imap

imemo.c

@@ -116,11 +116,11 @@ imemo_fields_new(VALUE owner, size_t capa)
     if (rb_gc_size_allocatable_p(embedded_size)) {
         VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size);
         RUBY_ASSERT(IMEMO_TYPE_P(fields, imemo_fields));
+        FL_SET_RAW(fields, OBJ_FIELD_EMBED);
         return fields;
     }
     else {
         VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields));
-        FL_SET_RAW(fields, OBJ_FIELD_EXTERNAL);
         IMEMO_OBJ_FIELDS(fields)->as.external.ptr = ALLOC_N(VALUE, capa);
         return fields;
     }
@@ -135,8 +135,9 @@ rb_imemo_fields_new(VALUE owner, size_t capa)
 static VALUE
 imemo_fields_new_complex(VALUE owner, size_t capa)
 {
-    VALUE fields = imemo_fields_new(owner, sizeof(struct rb_fields));
+    VALUE fields = imemo_fields_new(owner, 1);
     IMEMO_OBJ_FIELDS(fields)->as.complex.table = st_init_numtable_with_size(capa);
+    FL_UNSET_RAW(fields, OBJ_FIELD_EMBED);
     return fields;
 }
 
@@ -165,6 +166,7 @@ VALUE
 rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl)
 {
     VALUE fields = imemo_fields_new(owner, sizeof(struct rb_fields));
+    FL_UNSET_RAW(fields, OBJ_FIELD_EMBED);
     IMEMO_OBJ_FIELDS(fields)->as.complex.table = tbl;
     st_foreach(tbl, imemo_fields_trigger_wb_i, (st_data_t)fields);
     return fields;
@@ -258,7 +260,7 @@ rb_imemo_memsize(VALUE obj)
         if (rb_shape_obj_too_complex_p(obj)) {
             size += st_memsize(IMEMO_OBJ_FIELDS(obj)->as.complex.table);
         }
-        else if (FL_TEST_RAW(obj, OBJ_FIELD_EXTERNAL)) {
+        else if (!FL_TEST_RAW(obj, OBJ_FIELD_EMBED)) {
             size += RSHAPE_CAPACITY(RBASIC_SHAPE_ID(obj)) * sizeof(VALUE);
         }
         break;
@@ -536,7 +538,7 @@ imemo_fields_free(struct rb_fields *fields)
     if (rb_shape_obj_too_complex_p((VALUE)fields)) {
         st_free_table(fields->as.complex.table);
     }
-    else if (FL_TEST_RAW((VALUE)fields, OBJ_FIELD_EXTERNAL)) {
+    else if (!FL_TEST_RAW((VALUE)fields, OBJ_FIELD_EMBED)) {
         xfree(fields->as.external.ptr);
     }
 }

include/ruby/internal/core/robject.h

@@ -71,7 +71,7 @@ enum ruby_robject_flags {
      * 3rd parties must  not be aware that  there even is more than  one way to
      * store instance variables.  Might better be hidden.
      */
-    ROBJECT_EMBED = RUBY_FL_USER1
+    ROBJECT_EMBED = RUBY_FL_USER4
 };
 
 struct st_table;

internal/imemo.h

@@ -270,7 +270,14 @@ struct rb_fields {
     } as;
 };
 
-#define OBJ_FIELD_EXTERNAL IMEMO_FL_USER0
+// IMEMO/fields and T_OBJECT have exactly the same layout.
+// This is useful for JIT and common codepaths.
+#define OBJ_FIELD_EMBED ROBJECT_EMBED
+STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_EMBED == IMEMO_FL_USER0);
+STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields));
+STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr));
+STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table));
+
 #define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields)
 
 VALUE rb_imemo_fields_new(VALUE owner, size_t capa);
@@ -282,36 +289,43 @@ void rb_imemo_fields_clear(VALUE fields_obj);
 static inline VALUE
 rb_imemo_fields_owner(VALUE fields_obj)
 {
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields));
+
     return CLASS_OF(fields_obj);
 }
 
 static inline VALUE *
-rb_imemo_fields_ptr(VALUE obj_fields)
+rb_imemo_fields_ptr(VALUE fields_obj)
 {
-    if (!obj_fields) {
+    if (!fields_obj) {
         return NULL;
     }
 
-    RUBY_ASSERT(IMEMO_TYPE_P(obj_fields, imemo_fields));
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
 
-    if (RB_UNLIKELY(FL_TEST_RAW(obj_fields, OBJ_FIELD_EXTERNAL))) {
-        return IMEMO_OBJ_FIELDS(obj_fields)->as.external.ptr;
+    if (RB_LIKELY(FL_TEST_RAW(fields_obj, OBJ_FIELD_EMBED))) {
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.embed.fields;
     }
     else {
-        return IMEMO_OBJ_FIELDS(obj_fields)->as.embed.fields;
+        return IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr;
     }
 }
 
 static inline st_table *
-rb_imemo_fields_complex_tbl(VALUE obj_fields)
+rb_imemo_fields_complex_tbl(VALUE fields_obj)
 {
-    if (!obj_fields) {
+    if (!fields_obj) {
         return NULL;
     }
 
-    RUBY_ASSERT(IMEMO_TYPE_P(obj_fields, imemo_fields));
+    RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT));
+    RUBY_ASSERT(!FL_TEST_RAW(fields_obj, OBJ_FIELD_EMBED));
+
+    // Some codepaths unconditionally access the fields_ptr, and assume it can be used as st_table if the
+    // shape is too_complex.
+    RUBY_ASSERT((st_table *)rb_imemo_fields_ptr(fields_obj) == IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table);
 
-    return IMEMO_OBJ_FIELDS(obj_fields)->as.complex.table;
+    return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table;
 }
 
 #endif /* INTERNAL_IMEMO_H */

object.c

@@ -46,7 +46,7 @@
 
 /* Flags of RObject
  *
- * 1:    ROBJECT_EMBED
+ * 4:    ROBJECT_EMBED
  *           The object has its instance variables embedded (the array of
  *           instance variables directly follow the object, rather than being
  *           on a separately allocated buffer).

pathname.c

@@ -3,7 +3,6 @@
 static VALUE rb_cPathname;
 static ID id_at_path;
 static ID id_sub;
-static ID id_realdirpath;
 
 static VALUE
 get_strpath(VALUE obj)
@@ -84,22 +83,6 @@ path_sub(int argc, VALUE *argv, VALUE self)
     return rb_class_new_instance(1, &str, rb_obj_class(self));
 }
 
-/*
- * Returns the real (absolute) pathname of +self+ in the actual filesystem.
- *
- * Does not contain symlinks or useless dots, +..+ and +.+.
- *
- * The last component of the real pathname can be nonexistent.
- */
-static VALUE
-path_realdirpath(int argc, VALUE *argv, VALUE self)
-{
-    VALUE basedir, str;
-    rb_scan_args(argc, argv, "01", &basedir);
-    str = rb_funcall(rb_cFile, id_realdirpath, 2, get_strpath(self), basedir);
-    return rb_class_new_instance(1, &str, rb_obj_class(self));
-}
-
 #include "pathname_builtin.rbinc"
 
 static void init_ids(void);
@@ -121,7 +104,6 @@ InitVM_pathname(void)
     rb_cPathname = rb_define_class("Pathname", rb_cObject);
     rb_define_method(rb_cPathname, "<=>", path_cmp, 1);
     rb_define_method(rb_cPathname, "sub", path_sub, -1);
-    rb_define_method(rb_cPathname, "realdirpath", path_realdirpath, -1);
 
     rb_provide("pathname.so");
 }
@@ -132,5 +114,4 @@ init_ids(void)
 #undef rb_intern
     id_at_path = rb_intern("@path");
     id_sub = rb_intern("sub");
-    id_realdirpath = rb_intern("realdirpath");
 }

pathname_builtin.rb

@@ -130,6 +130,7 @@
 # - #read(*args)
 # - #binread(*args)
 # - #readlines(*args)
+# - #sysopen(*args)
 # - #write(*args)
 # - #binwrite(*args)
 # - #atime
@@ -170,11 +171,6 @@
 # - #mkdir(*args)
 # - #opendir(*args)
 #
-# === IO
-#
-# This method is a facade for IO:
-# - #sysopen(*args)
-#
 # === Utilities
 #
 # These methods are a mixture of Find, FileUtils, and others:
@@ -219,9 +215,10 @@ class Pathname
   # If +path+ contains a NUL character (<tt>\0</tt>), an ArgumentError is raised.
   #
   def initialize(path)
-    path = path.to_path if path.respond_to? :to_path
-
-    raise TypeError unless path.is_a?(String) # Compatibility for C version
+    unless String === path
+      path = path.to_path if path.respond_to? :to_path
+      raise TypeError unless String === path
+    end
 
     if path.include?("\0")
       raise ArgumentError, "pathname contains \\0: #{path.inspect}"
@@ -864,11 +861,6 @@ def relative_path_from(base_directory)
   end
 end
 
-class Pathname    # * IO *
-  # See <tt>IO.sysopen</tt>.
-  def sysopen(...) IO.sysopen(@path, ...) end
-end
-
 class Pathname    # * File *
   #
   # #each_line iterates over the line in the file.  It yields a String object
@@ -891,6 +883,9 @@ def binread(...) File.binread(@path, ...) end
   # See <tt>File.readlines</tt>.  Returns all the lines from the file.
   def readlines(...) File.readlines(@path, ...) end
 
+  # See <tt>File.sysopen</tt>.
+  def sysopen(...) File.sysopen(@path, ...) end
+
   # Writes +contents+ to the file. See <tt>File.write</tt>.
   def write(...) File.write(@path, ...) end
 
@@ -966,6 +961,13 @@ def truncate(length) File.truncate(@path, length) end
   # See <tt>File.utime</tt>.  Update the access and modification times.
   def utime(atime, mtime) File.utime(atime, mtime, @path) end
 
+  # Update the access and modification times of the file.
+  #
+  # Same as Pathname#utime, but does not follow symbolic links.
+  #
+  # See File.lutime.
+  def lutime(atime, mtime) File.lutime(atime, mtime, @path) end
+
   # See <tt>File.basename</tt>.  Returns the last component of the path.
   def basename(...) self.class.new(File.basename(@path, ...)) end
 
@@ -992,6 +994,13 @@ def split()
   #
   # All components of the pathname must exist when this method is called.
   def realpath(...) self.class.new(File.realpath(@path, ...)) end
+
+  # Returns the real (absolute) pathname of +self+ in the actual filesystem.
+  #
+  # Does not contain symlinks or useless dots, +..+ and +.+.
+  #
+  # The last component of the real pathname can be nonexistent.
+  def realdirpath(...) self.class.new(File.realdirpath(@path, ...)) end
 end
 
 
@@ -1158,12 +1167,8 @@ module Kernel
   #
   # This method is available since 1.8.5.
   def Pathname(path) # :doc:
-    Kernel.Pathname(path)
-  end
-  private :Pathname
-
-  def self.Pathname(path) # Compatibility for C version
     return path if Pathname === path
     Pathname.new(path)
   end
+  module_function :Pathname
 end

shape.c

@@ -1270,6 +1270,11 @@ rb_shape_verify_consistency(VALUE obj, shape_id_t shape_id)
     // Make sure SHAPE_ID_HAS_IVAR_MASK is valid.
     if (rb_shape_too_complex_p(shape_id)) {
         RUBY_ASSERT(shape_id & SHAPE_ID_HAS_IVAR_MASK);
+
+        // Ensure complex object don't appear as embedded
+        if (RB_TYPE_P(obj, T_OBJECT) || IMEMO_TYPE_P(obj, imemo_fields)) {
+            RUBY_ASSERT(!FL_TEST_RAW(obj, ROBJECT_EMBED));
+        }
     }
     else {
         attr_index_t ivar_count = RSHAPE_LEN(shape_id);

shape.h

@@ -351,6 +351,8 @@ ROBJECT_FIELDS_HASH(VALUE obj)
 {
     RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
     RUBY_ASSERT(rb_shape_obj_too_complex_p(obj));
+    RUBY_ASSERT(!FL_TEST_RAW(obj, ROBJECT_EMBED));
+
     return (st_table *)ROBJECT(obj)->as.heap.fields;
 }
 
@@ -359,6 +361,8 @@ ROBJECT_SET_FIELDS_HASH(VALUE obj, const st_table *tbl)
 {
     RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
     RUBY_ASSERT(rb_shape_obj_too_complex_p(obj));
+    RUBY_ASSERT(!FL_TEST_RAW(obj, ROBJECT_EMBED));
+
     ROBJECT(obj)->as.heap.fields = (VALUE *)tbl;
 }
 

test/pathname/test_pathname.rb

@@ -348,7 +348,7 @@ def has_symlink?
     rescue NotImplementedError
       return false
     rescue Errno::ENOENT
-      return false
+      return true
     rescue Errno::EACCES
       return false
     end
@@ -370,10 +370,11 @@ def has_hardlink?
   end
 
   def realpath(path, basedir=nil)
-    Pathname.new(path).realpath(basedir).to_s
+    Pathname.new(path).realpath(*basedir).to_s
   end
 
   def test_realpath
+    omit "not working yet" if RUBY_ENGINE == "jruby"
     return if !has_symlink?
     with_tmpchdir('rubytest-pathname') {|dir|
       assert_raise(Errno::ENOENT) { realpath("#{dir}/not-exist") }
@@ -434,6 +435,7 @@ def realdirpath(path)
   end
 
   def test_realdirpath
+    omit "not working yet" if RUBY_ENGINE == "jruby"
     return if !has_symlink?
     Dir.mktmpdir('rubytest-pathname') {|dir|
       rdir = realpath(dir)
@@ -1054,7 +1056,11 @@ def test_lutime
       latime = Time.utc(2000)
       lmtime = Time.utc(1999)
       File.symlink("a", "l")
-      Pathname("l").utime(latime, lmtime)
+      begin
+        Pathname("l").lutime(latime, lmtime)
+      rescue NotImplementedError
+        next
+      end
       s = File.lstat("a")
       ls = File.lstat("l")
       assert_equal(atime, s.atime)