diff --git a/NEWS.md b/NEWS.md
index 476d5ba88c5884..ea0f4c28bf0ce9 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -243,6 +243,23 @@ The following bundled gems are updated.
## Implementation improvements
+### Ractor
+
+A lot of work has gone into making Ractors more stable, performant, and usable. These improvements bring Ractors implementation closer to leaving experimental status.
+
+* Performance improvements
+ * Frozen strings and the symbol table internally use a lock-free hash set
+ * Method cache lookups avoid locking in most cases
+ * Class (and geniv) instance variable access is faster and avoids locking
+ * Cache contention is avoided during object allocation
+ * `object_id` avoids locking in most cases
+* Bug fixes and stability
+ * Fixed possible deadlocks when combining Ractors and Threads
+ * Fixed issues with require and autoload in a Ractor
+ * Fixed encoding/transcoding issues across Ractors
+ * Fixed race conditions in GC operations and method invalidation
+ * Fixed issues with processes forking after starting a Ractor
+
## JIT
* YJIT
diff --git a/ext/json/fbuffer/fbuffer.h b/ext/json/fbuffer/fbuffer.h
index d32371476c85a2..d5fd8ac6d71f78 100644
--- a/ext/json/fbuffer/fbuffer.h
+++ b/ext/json/fbuffer/fbuffer.h
@@ -24,6 +24,14 @@ typedef unsigned char _Bool;
#endif
#endif
+#ifndef NOINLINE
+#if defined(__has_attribute) && __has_attribute(noinline)
+#define NOINLINE() __attribute__((noinline))
+#else
+#define NOINLINE()
+#endif
+#endif
+
#ifndef RB_UNLIKELY
#define RB_UNLIKELY(expr) expr
#endif
@@ -169,12 +177,17 @@ static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested)
}
}
-static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len)
+static inline void fbuffer_append_reserved(FBuffer *fb, const char *newstr, unsigned long len)
+{
+ MEMCPY(fb->ptr + fb->len, newstr, char, len);
+ fbuffer_consumed(fb, len);
+}
+
+static inline void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len)
{
if (len > 0) {
fbuffer_inc_capa(fb, len);
- MEMCPY(fb->ptr + fb->len, newstr, char, len);
- fbuffer_consumed(fb, len);
+ fbuffer_append_reserved(fb, newstr, len);
}
}
@@ -197,11 +210,24 @@ static void fbuffer_append_str(FBuffer *fb, VALUE str)
const char *newstr = StringValuePtr(str);
unsigned long len = RSTRING_LEN(str);
- RB_GC_GUARD(str);
-
fbuffer_append(fb, newstr, len);
}
+static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat)
+{
+ const char *newstr = StringValuePtr(str);
+ unsigned long len = RSTRING_LEN(str);
+
+ fbuffer_inc_capa(fb, repeat * len);
+ while (repeat) {
+#ifdef JSON_DEBUG
+ fb->requested = len;
+#endif
+ fbuffer_append_reserved(fb, newstr, len);
+ repeat--;
+ }
+}
+
static inline void fbuffer_append_char(FBuffer *fb, char newchr)
{
fbuffer_inc_capa(fb, 1);
diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c
index 9c6ed930495a51..d810927c58087c 100644
--- a/ext/json/generator/generator.c
+++ b/ext/json/generator/generator.c
@@ -9,6 +9,12 @@
/* ruby api and some helpers */
+enum duplicate_key_action {
+ JSON_DEPRECATED = 0,
+ JSON_IGNORE,
+ JSON_RAISE,
+};
+
typedef struct JSON_Generator_StateStruct {
VALUE indent;
VALUE space;
@@ -21,6 +27,8 @@ typedef struct JSON_Generator_StateStruct {
long depth;
long buffer_initial_length;
+ enum duplicate_key_action on_duplicate_key;
+
bool allow_nan;
bool ascii_only;
bool script_safe;
@@ -34,7 +42,7 @@ typedef struct JSON_Generator_StateStruct {
static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8;
static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode;
-static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan,
+static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key,
sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json;
@@ -987,8 +995,11 @@ static inline VALUE vstate_get(struct generate_json_data *data)
}
struct hash_foreach_arg {
+ VALUE hash;
struct generate_json_data *data;
- int iter;
+ int first_key_type;
+ bool first;
+ bool mixed_keys_encountered;
};
static VALUE
@@ -1006,6 +1017,22 @@ convert_string_subclass(VALUE key)
return key_to_s;
}
+NOINLINE()
+static void
+json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg)
+{
+ if (arg->mixed_keys_encountered) {
+ return;
+ }
+ arg->mixed_keys_encountered = true;
+
+ JSON_Generator_State *state = arg->data->state;
+ if (state->on_duplicate_key != JSON_IGNORE) {
+ VALUE do_raise = state->on_duplicate_key == JSON_RAISE ? Qtrue : Qfalse;
+ rb_funcall(mJSON, rb_intern("on_mixed_keys_hash"), 2, arg->hash, do_raise);
+ }
+}
+
static int
json_object_i(VALUE key, VALUE val, VALUE _arg)
{
@@ -1016,21 +1043,33 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
JSON_Generator_State *state = data->state;
long depth = state->depth;
- int j;
+ int key_type = rb_type(key);
+
+ if (arg->first) {
+ arg->first = false;
+ arg->first_key_type = key_type;
+ }
+ else {
+ fbuffer_append_char(buffer, ',');
+ }
- if (arg->iter > 0) fbuffer_append_char(buffer, ',');
if (RB_UNLIKELY(data->state->object_nl)) {
fbuffer_append_str(buffer, data->state->object_nl);
}
if (RB_UNLIKELY(data->state->indent)) {
- for (j = 0; j < depth; j++) {
- fbuffer_append_str(buffer, data->state->indent);
- }
+ fbuffer_append_str_repeat(buffer, data->state->indent, depth);
}
VALUE key_to_s;
- switch (rb_type(key)) {
+ bool as_json_called = false;
+
+ start:
+ switch (key_type) {
case T_STRING:
+ if (RB_UNLIKELY(arg->first_key_type != T_STRING)) {
+ json_inspect_hash_with_mixed_keys(arg);
+ }
+
if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) {
key_to_s = key;
} else {
@@ -1038,9 +1077,23 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
}
break;
case T_SYMBOL:
+ if (RB_UNLIKELY(arg->first_key_type != T_SYMBOL)) {
+ json_inspect_hash_with_mixed_keys(arg);
+ }
+
key_to_s = rb_sym2str(key);
break;
default:
+ if (data->state->strict) {
+ if (RTEST(data->state->as_json) && !as_json_called) {
+ key = rb_proc_call_with_block(data->state->as_json, 1, &key, Qnil);
+ key_type = rb_type(key);
+ as_json_called = true;
+ goto start;
+ } else {
+ raise_generator_error(key, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(key));
+ }
+ }
key_to_s = rb_convert_type(key, T_STRING, "String", "to_s");
break;
}
@@ -1055,7 +1108,6 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, data->state->space);
generate_json(buffer, data, val);
- arg->iter++;
return ST_CONTINUE;
}
@@ -1071,7 +1123,6 @@ static inline long increase_depth(struct generate_json_data *data)
static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
{
- int j;
long depth = increase_depth(data);
if (RHASH_SIZE(obj) == 0) {
@@ -1083,8 +1134,9 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat
fbuffer_append_char(buffer, '{');
struct hash_foreach_arg arg = {
+ .hash = obj,
.data = data,
- .iter = 0,
+ .first = true,
};
rb_hash_foreach(obj, json_object_i, (VALUE)&arg);
@@ -1092,9 +1144,7 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat
if (RB_UNLIKELY(data->state->object_nl)) {
fbuffer_append_str(buffer, data->state->object_nl);
if (RB_UNLIKELY(data->state->indent)) {
- for (j = 0; j < depth; j++) {
- fbuffer_append_str(buffer, data->state->indent);
- }
+ fbuffer_append_str_repeat(buffer, data->state->indent, depth);
}
}
fbuffer_append_char(buffer, '}');
@@ -1102,7 +1152,6 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat
static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
{
- int i, j;
long depth = increase_depth(data);
if (RARRAY_LEN(obj) == 0) {
@@ -1113,15 +1162,13 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data
fbuffer_append_char(buffer, '[');
if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
- for (i = 0; i < RARRAY_LEN(obj); i++) {
+ for (int i = 0; i < RARRAY_LEN(obj); i++) {
if (i > 0) {
fbuffer_append_char(buffer, ',');
if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
}
if (RB_UNLIKELY(data->state->indent)) {
- for (j = 0; j < depth; j++) {
- fbuffer_append_str(buffer, data->state->indent);
- }
+ fbuffer_append_str_repeat(buffer, data->state->indent, depth);
}
generate_json(buffer, data, RARRAY_AREF(obj, i));
}
@@ -1129,9 +1176,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data
if (RB_UNLIKELY(data->state->array_nl)) {
fbuffer_append_str(buffer, data->state->array_nl);
if (RB_UNLIKELY(data->state->indent)) {
- for (j = 0; j < depth; j++) {
- fbuffer_append_str(buffer, data->state->indent);
- }
+ fbuffer_append_str_repeat(buffer, data->state->indent, depth);
}
}
fbuffer_append_char(buffer, ']');
@@ -1793,6 +1838,19 @@ static VALUE cState_ascii_only_set(VALUE self, VALUE enable)
return Qnil;
}
+static VALUE cState_allow_duplicate_key_p(VALUE self)
+{
+ GET_STATE(self);
+ switch (state->on_duplicate_key) {
+ case JSON_IGNORE:
+ return Qtrue;
+ case JSON_DEPRECATED:
+ return Qnil;
+ case JSON_RAISE:
+ return Qfalse;
+ }
+}
+
/*
* call-seq: depth
*
@@ -1882,6 +1940,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
else if (key == sym_script_safe) { state->script_safe = RTEST(val); }
else if (key == sym_escape_slash) { state->script_safe = RTEST(val); }
else if (key == sym_strict) { state->strict = RTEST(val); }
+ else if (key == sym_allow_duplicate_key) { state->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
else if (key == sym_as_json) {
VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse;
state_write_value(data, &state->as_json, proc);
@@ -2007,6 +2066,8 @@ void Init_generator(void)
rb_define_method(cState, "generate", cState_generate, -1);
rb_define_alias(cState, "generate_new", "generate"); // :nodoc:
+ rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
+
rb_define_singleton_method(cState, "generate", cState_m_generate, 3);
VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
@@ -2071,6 +2132,7 @@ void Init_generator(void)
sym_escape_slash = ID2SYM(rb_intern("escape_slash"));
sym_strict = ID2SYM(rb_intern("strict"));
sym_as_json = ID2SYM(rb_intern("as_json"));
+ sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
usascii_encindex = rb_usascii_encindex();
utf8_encindex = rb_utf8_encindex();
diff --git a/ext/json/lib/json.rb b/ext/json/lib/json.rb
index 0ebff2f948af56..43d96afa95835c 100644
--- a/ext/json/lib/json.rb
+++ b/ext/json/lib/json.rb
@@ -307,6 +307,25 @@
#
# ---
#
+# Option +allow_duplicate_key+ (boolean) specifies whether
+# hashes with duplicate keys should be allowed or produce an error.
+# defaults to emit a deprecation warning.
+#
+# With the default, (not set):
+# Warning[:deprecated] = true
+# JSON.generate({ foo: 1, "foo" => 2 })
+# # warning: detected duplicate key "foo" in {foo: 1, "foo" => 2}.
+# # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`
+# # => '{"foo":1,"foo":2}'
+#
+# With false
+# JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: false)
+# # detected duplicate key "foo" in {foo: 1, "foo" => 2} (JSON::GeneratorError)
+#
+# In version 3.0, false will become the default.
+#
+# ---
+#
# Option +max_nesting+ (\Integer) specifies the maximum nesting depth
# in +obj+; defaults to +100+.
#
@@ -384,6 +403,9 @@
#
# == \JSON Additions
#
+# Note that JSON Additions must only be used with trusted data, and is
+# deprecated.
+#
# When you "round trip" a non-\String object from Ruby to \JSON and back,
# you have a new \String, instead of the object you began with:
# ruby0 = Range.new(0, 2)
diff --git a/ext/json/lib/json/common.rb b/ext/json/lib/json/common.rb
index e99d152a884898..45200a83bceda6 100644
--- a/ext/json/lib/json/common.rb
+++ b/ext/json/lib/json/common.rb
@@ -186,6 +186,25 @@ def generator=(generator) # :nodoc:
private
+ # Called from the extension when a hash has both string and symbol keys
+ def on_mixed_keys_hash(hash, do_raise)
+ set = {}
+ hash.each_key do |key|
+ key_str = key.to_s
+
+ if set[key_str]
+ message = "detected duplicate key #{key_str.inspect} in #{hash.inspect}"
+ if do_raise
+ raise GeneratorError, message
+ else
+ deprecation_warning("#{message}.\nThis will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`")
+ end
+ else
+ set[key_str] = true
+ end
+ end
+ end
+
def deprecated_singleton_attr_accessor(*attrs)
args = RUBY_VERSION >= "3.0" ? ", category: :deprecated" : ""
attrs.each do |attr|
@@ -391,7 +410,7 @@ def load_file!(filespec, opts = nil)
#
# Returns a \String containing the generated \JSON data.
#
- # See also JSON.fast_generate, JSON.pretty_generate.
+ # See also JSON.pretty_generate.
#
# Argument +obj+ is the Ruby object to be converted to \JSON.
#
diff --git a/ext/json/lib/json/ext/generator/state.rb b/ext/json/lib/json/ext/generator/state.rb
index d40c3b5ec35a30..1f56e6c682cc6d 100644
--- a/ext/json/lib/json/ext/generator/state.rb
+++ b/ext/json/lib/json/ext/generator/state.rb
@@ -8,20 +8,8 @@ class State
#
# Instantiates a new State object, configured by _opts_.
#
- # _opts_ can have the following keys:
- #
- # * *indent*: a string used to indent levels (default: ''),
- # * *space*: a string that is put after, a : or , delimiter (default: ''),
- # * *space_before*: a string that is put before a : pair delimiter (default: ''),
- # * *object_nl*: a string that is put at the end of a JSON object (default: ''),
- # * *array_nl*: a string that is put at the end of a JSON array (default: ''),
- # * *allow_nan*: true if NaN, Infinity, and -Infinity should be
- # generated, otherwise an exception is thrown, if these values are
- # encountered. This options defaults to false.
- # * *ascii_only*: true if only ASCII characters should be generated. This
- # option defaults to false.
- # * *buffer_initial_length*: sets the initial length of the generator's
- # internal buffer.
+ # Argument +opts+, if given, contains a \Hash of options for the generation.
+ # See {Generating Options}[#module-JSON-label-Generating+Options].
def initialize(opts = nil)
if opts && !opts.empty?
configure(opts)
@@ -68,6 +56,11 @@ def to_h
buffer_initial_length: buffer_initial_length,
}
+ allow_duplicate_key = allow_duplicate_key?
+ unless allow_duplicate_key.nil?
+ result[:allow_duplicate_key] = allow_duplicate_key
+ end
+
instance_variables.each do |iv|
iv = iv.to_s[1..-1]
result[iv.to_sym] = self[iv]
diff --git a/ext/json/parser/parser.c b/ext/json/parser/parser.c
index 1e6ee753f0cf4b..e34f1999d5f191 100644
--- a/ext/json/parser/parser.c
+++ b/ext/json/parser/parser.c
@@ -975,7 +975,7 @@ static inline bool FORCE_INLINE string_scan(JSON_ParserState *state)
if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) {
return 1;
}
- *state->cursor++;
+ state->cursor++;
}
return 0;
}
@@ -1314,7 +1314,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
else if (key == sym_freeze) { config->freeze = RTEST(val); }
else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; }
- else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
+ else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
else if (key == sym_decimal_class) {
if (RTEST(val)) {
if (rb_respond_to(val, i_try_convert)) {
diff --git a/test/json/fixtures/pass1.json b/test/json/fixtures/pass1.json
index 7828fcc1374e9f..fa9058b1366bb5 100644
--- a/test/json/fixtures/pass1.json
+++ b/test/json/fixtures/pass1.json
@@ -12,7 +12,7 @@
"real": -9876.543210,
"e": 0.123456789e-12,
"E": 1.234567890E+34,
- "": 23456789012E666,
+ "": 23456789012E66,
"zero": 0,
"one": 1,
"space": " ",
diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb
index 9861181910cefd..fc4aba296858ae 100755
--- a/test/json/json_coder_test.rb
+++ b/test/json/json_coder_test.rb
@@ -18,6 +18,18 @@ def test_json_coder_with_proc_with_unsupported_value
assert_raise(JSON::GeneratorError) { coder.dump([Object.new]) }
end
+ def test_json_coder_hash_key
+ obj = Object.new
+ coder = JSON::Coder.new(&:to_s)
+ assert_equal %({#{obj.to_s.inspect}:1}), coder.dump({ obj => 1 })
+
+ coder = JSON::Coder.new { 42 }
+ error = assert_raise JSON::GeneratorError do
+ coder.dump({ obj => 1 })
+ end
+ assert_equal "Integer not allowed as object key in JSON", error.message
+ end
+
def test_json_coder_options
coder = JSON::Coder.new(array_nl: "\n") do |object|
42
diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb
index 914b3f4ed0a904..4315d109d8ec68 100755
--- a/test/json/json_generator_test.rb
+++ b/test/json/json_generator_test.rb
@@ -234,6 +234,24 @@ def test_state_defaults
:space => "",
:space_before => "",
}.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+
+ state = JSON::State.new(allow_duplicate_key: true)
+ assert_equal({
+ :allow_duplicate_key => true,
+ :allow_nan => false,
+ :array_nl => "",
+ :as_json => false,
+ :ascii_only => false,
+ :buffer_initial_length => 1024,
+ :depth => 0,
+ :script_safe => false,
+ :strict => false,
+ :indent => "",
+ :max_nesting => 100,
+ :object_nl => "",
+ :space => "",
+ :space_before => "",
+ }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
end
def test_allow_nan
@@ -404,6 +422,18 @@ def test_json_generate_unsupported_types
assert_raise JSON::GeneratorError do
generate(Object.new, strict: true)
end
+
+ assert_raise JSON::GeneratorError do
+ generate([Object.new], strict: true)
+ end
+
+ assert_raise JSON::GeneratorError do
+ generate({ "key" => Object.new }, strict: true)
+ end
+
+ assert_raise JSON::GeneratorError do
+ generate({ Object.new => "value" }, strict: true)
+ end
end
def test_nesting
@@ -816,4 +846,24 @@ def test_numbers_of_various_sizes
assert_equal "[#{number}]", JSON.generate([number])
end
end
+
+ def test_generate_duplicate_keys_allowed
+ hash = { foo: 1, "foo" => 2 }
+ assert_equal %({"foo":1,"foo":2}), JSON.generate(hash, allow_duplicate_key: true)
+ end
+
+ def test_generate_duplicate_keys_deprecated
+ hash = { foo: 1, "foo" => 2 }
+ assert_deprecated_warning(/allow_duplicate_key/) do
+ assert_equal %({"foo":1,"foo":2}), JSON.generate(hash)
+ end
+ end
+
+ def test_generate_duplicate_keys_disallowed
+ hash = { foo: 1, "foo" => 2 }
+ error = assert_raise JSON::GeneratorError do
+ JSON.generate(hash, allow_duplicate_key: false)
+ end
+ assert_equal %(detected duplicate key "foo" in #{hash.inspect}), error.message
+ end
end
diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb
index 6455d2971ae3c0..3c60fec8753ea5 100644
--- a/test/json/json_parser_test.rb
+++ b/test/json/json_parser_test.rb
@@ -128,6 +128,7 @@ def test_parse_numbers
assert_equal(1.0/0, parse('Infinity', :allow_nan => true))
assert_raise(ParserError) { parse('-Infinity') }
assert_equal(-1.0/0, parse('-Infinity', :allow_nan => true))
+ capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) }
end
def test_parse_bigdecimals
diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb
index 0ebdb0e91a1cfc..53e1099ce9b76c 100644
--- a/test/json/ractor_test.rb
+++ b/test/json/ractor_test.rb
@@ -20,6 +20,7 @@ module RactorBackport
def test_generate
pid = fork do
+ Warning[:experimental] = false
r = Ractor.new do
json = JSON.generate({
'a' => 2,