diff --git a/NEWS.md b/NEWS.md index a4f3c723df79fd..4679ef9a289380 100644 --- a/NEWS.md +++ b/NEWS.md @@ -167,7 +167,7 @@ The following default gems are updated. * io-wait 0.3.2 * json 2.13.2 * optparse 0.7.0.dev.2 -* prism 1.4.0 +* prism 1.5.0 * psych 5.2.6 * resolv 0.6.2 * stringio 3.1.8.dev diff --git a/depend b/depend index b9d91faa2a05a2..dbc002d5861080 100644 --- a/depend +++ b/depend @@ -1415,6 +1415,7 @@ compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_strncasecmp.h compile.$(OBJEXT): $(top_srcdir)/prism/util/pm_strpbrk.h compile.$(OBJEXT): $(top_srcdir)/prism/version.h compile.$(OBJEXT): $(top_srcdir)/prism_compile.c +compile.$(OBJEXT): $(top_srcdir)/version.h compile.$(OBJEXT): {$(VPATH)}assert.h compile.$(OBJEXT): {$(VPATH)}atomic.h compile.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -1605,6 +1606,7 @@ compile.$(OBJEXT): {$(VPATH)}prism_compile.h compile.$(OBJEXT): {$(VPATH)}ractor.h compile.$(OBJEXT): {$(VPATH)}re.h compile.$(OBJEXT): {$(VPATH)}regex.h +compile.$(OBJEXT): {$(VPATH)}revision.h compile.$(OBJEXT): {$(VPATH)}ruby_assert.h compile.$(OBJEXT): {$(VPATH)}ruby_atomic.h compile.$(OBJEXT): {$(VPATH)}rubyparser.h diff --git a/jit.c b/jit.c index efecbef35455ca..f233a2f01f1c8e 100644 --- a/jit.c +++ b/jit.c @@ -533,6 +533,131 @@ for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data) return 0; } +uint32_t +rb_jit_get_page_size(void) +{ +#if defined(_SC_PAGESIZE) + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) rb_bug("jit: failed to get page size"); + + // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. + // Though our design sort of assume we have fine grained control over memory protection + // which require small page sizes. + if (page_size > 0x40000000l) rb_bug("jit page size too large"); + + return (uint32_t)page_size; +#else +#error "JIT supports POSIX only for now" +#endif +} + +#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) +// Align the current write position to a multiple of bytes +static uint8_t * +align_ptr(uint8_t *ptr, uint32_t multiple) +{ + // Compute the pointer modulo the given alignment boundary + uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; + + // If the pointer is already aligned, stop + if (rem == 0) + return ptr; + + // Pad the pointer by the necessary amount to align it + uint32_t pad = multiple - rem; + + return ptr + pad; +} +#endif + +// Address space reservation. Memory pages are mapped on an as needed basis. +// See the Rust mm module for details. +uint8_t * +rb_jit_reserve_addr_space(uint32_t mem_size) +{ +#ifndef _WIN32 + uint8_t *mem_block; + + // On Linux + #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) + uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); + uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space; + uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; + // Align the requested address to page size + uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); + + // Probe for addresses close to this function using MAP_FIXED_NOREPLACE + // to improve odds of being in range for 32-bit relative call instructions. + do { + mem_block = mmap( + req_addr, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, + 0 + ); + + // If we succeeded, stop + if (mem_block != MAP_FAILED) { + ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space"); + break; + } + + // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux + // main_code_addr < heap_addr, and in case we are in a shared + // library mapped higher than the heap, downwards is still better + // since it's towards the end of the heap rather than the stack.) + req_addr -= 4 * 1024 * 1024; + } while (req_addr < probe_region_end); + + // On MacOS and other platforms + #else + // Try to map a chunk of memory as executable + mem_block = mmap( + (void *)rb_jit_reserve_addr_space, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0 + ); + #endif + + // Fallback + if (mem_block == MAP_FAILED) { + // Try again without the address hint (e.g., valgrind) + mem_block = mmap( + NULL, + mem_size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0 + ); + + if (mem_block != MAP_FAILED) { + ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space:fallback"); + } + } + + // Check that the memory mapping was successful + if (mem_block == MAP_FAILED) { + perror("ruby: jit: mmap:"); + if(errno == ENOMEM) { + // No crash report if it's only insufficient memory + exit(EXIT_FAILURE); + } + rb_bug("mmap failed"); + } + + return mem_block; +#else + // Windows not supported for now + return NULL; +#endif +} + // Walk all ISEQs in the heap and invoke the callback - shared between YJIT and ZJIT void rb_jit_for_each_iseq(rb_iseq_callback callback, void *data) @@ -540,3 +665,52 @@ rb_jit_for_each_iseq(rb_iseq_callback callback, void *data) struct iseq_callback_data callback_data = { .callback = callback, .data = data }; rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data); } + +bool +rb_jit_mark_writable(void *mem_block, uint32_t mem_size) +{ + return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; +} + +void +rb_jit_mark_executable(void *mem_block, uint32_t mem_size) +{ + // Do not call mprotect when mem_size is zero. Some platforms may return + // an error for it. https://github.com/Shopify/ruby/issues/450 + if (mem_size == 0) { + return; + } + if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { + rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", + mem_block, (unsigned long)mem_size, strerror(errno)); + } +} + +// Free the specified memory block. +bool +rb_jit_mark_unused(void *mem_block, uint32_t mem_size) +{ + // On Linux, you need to use madvise MADV_DONTNEED to free memory. + // We might not need to call this on macOS, but it's not really documented. + // We generally prefer to do the same thing on both to ease testing too. + madvise(mem_block, mem_size, MADV_DONTNEED); + + // On macOS, mprotect PROT_NONE seems to reduce RSS. + // We also call this on Linux to avoid executing unused pages. + return mprotect(mem_block, mem_size, PROT_NONE) == 0; +} + +// Invalidate icache for arm64. +// `start` is inclusive and `end` is exclusive. +void +rb_jit_icache_invalidate(void *start, void *end) +{ + // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 + // but required on ARM before running freshly written code. + // On Darwin it's the same as calling sys_icache_invalidate(). +#ifdef __GNUC__ + __builtin___clear_cache(start, end); +#elif defined(__aarch64__) +#error No instruction cache clear available with this compiler on Aarch64! +#endif +} diff --git a/lib/erb.rb b/lib/erb.rb index ebf91e4792d67e..c8aae4fd15239c 100644 --- a/lib/erb.rb +++ b/lib/erb.rb @@ -17,246 +17,506 @@ require 'erb/def_method' require 'erb/util' +# :markup: markdown # -# = ERB -- Ruby Templating +# Class **ERB** (the name stands for **Embedded Ruby**) +# is an easy-to-use, but also very powerful, [template processor][template processor]. # -# == Introduction +# Like method [sprintf][sprintf], \ERB can format run-time data into a string. +# \ERB, however,s is *much more powerful*. # -# ERB provides an easy to use but powerful templating system for Ruby. Using -# ERB, actual Ruby code can be added to any plain text document for the -# purposes of generating document information details and/or flow control. +# In simplest terms: # -# A very simple example is this: +# - You can create an \ERB object to store a text *template* that includes specially formatted *tags*; +# each tag specifies data that at run-time is to replace the tag in the produced result. +# - You can call instance method ERB#result to get the result, +# which is the string formed by replacing each tag with run-time data. # -# require 'erb' +# \ERB is commonly used to produce: # -# x = 42 -# template = ERB.new <<-EOF -# The value of x is: <%= x %> -# EOF -# puts template.result(binding) +# - Customized or personalized email messages. +# - Customized or personalized web pages. +# - Software code (in code-generating applications). # -# Prints: The value of x is: 42 +# ## Usage # -# More complex examples are given below. +# Before you can use \ERB, you must first require it +# (examples on this page assume that this has been done): # +# ``` +# require 'erb' +# ``` # -# == Recognized Tags +# ## In Brief # -# ERB recognizes certain tags in the provided template and converts them based -# on the rules below: +# ``` +# # Expression tag: begins with '<%', ends with '%>'. +# # This expression does not need the local binding. +# ERB.new('Today is <%= Date::DAYNAMES[Date.today.wday] %>.').result # => "Today is Monday." +# # This expression tag does need the local binding. +# magic_word = 'xyzzy' +# template.result(binding) # => "The magic word is xyzzy." # -# <% Ruby code -- inline with output %> -# <%= Ruby expression -- replace with result %> -# <%# comment -- ignored -- useful in testing %> (`<% #` doesn't work. Don't use Ruby comments.) -# % a line of Ruby code -- treated as <% line %> (optional -- see ERB.new) -# %% replaced with % if first thing on a line and % processing is used -# <%% or %%> -- replace with <% or %> respectively +# Execution tag: begins with '<%=', ends with '%>'. +# s = '<% File.write("t.txt", "Some stuff.") %>' +# ERB.new(s).result +# File.read('t.txt') # => "Some stuff." # -# All other text is passed through ERB filtering unchanged. +# # Comment tag: begins with '<%#', ends with '%>'. +# s = 'Some stuff;<%# Note to self: figure out what the stuff is. %> more stuff.' +# ERB.new(s).result # => "Some stuff; more stuff." +# ``` # # -# == Options +# ## Some Simple Examples # -# There are several settings you can change when you use ERB: -# * the nature of the tags that are recognized; -# * the binding used to resolve local variables in the template. +# Here's a simple example of \ERB in action: # -# See the ERB.new and ERB#result methods for more detail. +# ``` +# s = 'The time is <%= Time.now %>.' +# template = ERB.new(s) +# template.result +# # => "The time is 2025-09-09 10:49:26 -0500." +# ``` # -# == Character encodings +# Details: # -# ERB (or Ruby code generated by ERB) returns a string in the same -# character encoding as the input string. When the input string has -# a magic comment, however, it returns a string in the encoding specified -# by the magic comment. +# 1. A plain-text string is assigned to variable `s`. +# Its embedded [expression tag][expression tags] `'<%= Time.now %>'` includes a Ruby expression, `Time.now`. +# 2. The string is put into a new \ERB object, and stored in variable `template`. +# 4. Method call `template.result` generates a string that contains the run-time value of `Time.now`, +# as computed at the time of the call. # -# # -*- coding: utf-8 -*- -# require 'erb' +# The template may be re-used: # -# template = ERB.new < -# \_\_ENCODING\_\_ is <%= \_\_ENCODING\_\_ %>. -# EOF -# puts template.result +# ``` +# template.result +# # => "The time is 2025-09-09 10:49:33 -0500." +# ``` # -# Prints: \_\_ENCODING\_\_ is Big5. +# Another example: # +# ``` +# s = 'The magic word is <%= magic_word %>.' +# template = ERB.new(s) +# magic_word = 'abracadabra' +# # => "abracadabra" +# template.result(binding) +# # => "The magic word is abracadabra." +# ``` # -# == Examples +# Details: # -# === Plain Text +# 1. As before, a plain-text string is assigned to variable `s`. +# Its embedded [expression tag][expression tags] `'<%= magic_word %>'` has a variable *name*, `magic_word`. +# 2. The string is put into a new \ERB object, and stored in variable `template`; +# note that `magic_word` need not be defined before the \ERB object is created. +# 3. `magic_word = 'abracadabra'` assigns a value to variable `magic_word`. +# 4. Method call `template.result(binding)` generates a string +# that contains the *value* of `magic_word`. # -# ERB is useful for any generic templating situation. Note that in this example, we use the -# convenient "% at start of line" tag, and we quote the template literally with -# %q{...} to avoid trouble with the backslash. +# As before, the template may be re-used: # -# require "erb" +# ``` +# magic_word = 'xyzzy' +# template.result(binding) +# # => "The magic word is xyzzy." +# ``` +# +# ## Bindings +# +# The first example above passed no argument to method `result`; +# the second example passed argument `binding`. +# +# Here's why: +# +# - The first example has tag `<%= Time.now %>`, +# which cites *globally-defined* constant `Time`; +# the default `binding` (details not needed here) includes the binding of global constant `Time` to its value. +# - The second example has tag `<%= magic_word %>`, +# which cites *locally-defined* variable `magic_word`; +# the passed argument `binding` (which is simply a call to method [Kernel#binding][kernel#binding]) +# includes the binding of local variable `magic_word` to its value. +# +# ## Tags +# +# The examples above use expression tags. +# These are the tags available in \ERB: +# +# - [Expression tag][expression tags]: the tag contains a Ruby exprssion; +# in the result, the entire tag is to be replaced with the run-time value of the expression. +# - [Execution tag][execution tags]: the tag contains Ruby code; +# in the result, the entire tag is to be replaced with the run-time value of the code. +# - [Comment tag][comment tags]: the tag contains comment code; +# in the result, the entire tag is to be omitted. +# +# ### Expression Tags +# +# You can embed a Ruby expression in a template using an *expression tag*. +# +# Its syntax is `<%= _expression_ %>`, +# where *expression* is any valid Ruby expression. +# +# When you call method #result, +# the method evaluates the expression and replaces the entire expression tag with the expression's value: +# +# ``` +# ERB.new('Today is <%= Date::DAYNAMES[Date.today.wday] %>.').result +# # => "Today is Monday." +# ERB.new('Tomorrow will be <%= Date::DAYNAMES[Date.today.wday + 1] %>.').result +# # => "Tomorrow will be Tuesday." +# ERB.new('Yesterday was <%= Date::DAYNAMES[Date.today.wday - 1] %>.').result +# # => "Yesterday was Sunday." +# ``` +# +# Note that whitespace before and after the expression +# is allowed but not required, +# and that such whitespace is stripped from the result. +# +# ``` +# ERB.new('My appointment is on <%=Date::DAYNAMES[Date.today.wday + 2]%>.').result +# # => "My appointment is on Wednesday." +# ERB.new('My appointment is on <%= Date::DAYNAMES[Date.today.wday + 2] %>.').result +# # => "My appointment is on Wednesday." +# ``` +# +# ### Execution Tags +# +# You can embed Ruby executable code in template using an *execution tag*. +# +# Its syntax is `<% _code_ %>`, +# where *code* is any valid Ruby code. +# +# When you call method #result, +# the method executes the code and removes the entire execution tag +# (generating no text in the result). +# +# You can interleave text with execution tags to form a control structure +# such as a conditional, a loop, or a `case` statements. +# +# Conditional: +# +# ``` +# s = < +# An error has occurred. +# <% else %> +# Oops! +# <% end %> +# EOT +# template = ERB.new(s) +# verbosity = true +# template.result(binding) +# # => "\nAn error has occurred.\n\n" +# verbosity = false +# template.result(binding) +# # => "\nOops!\n\n" +# ``` +# +# Note that the interleaved text may itself contain expression tags: +# +# Loop: +# +# ``` +# s = < +# <%= dayname %> +# <% end %> +# EOT +# ERB.new(s).result +# # => "\nSun\n\nMon\n\nTue\n\nWed\n\nThu\n\nFri\n\nSat\n\n" +# ``` +# +# Other, non-control, lines of Ruby code may be interleaved with the text, +# and the Ruby code may itself contain regular Ruby comments: +# +# ``` +# s = < +# <%= Time.now %> +# <% sleep(1) # Let's make the times different. %> +# <% end %> +# EOT +# ERB.new(s).result +# # => "\n2025-09-09 11:36:02 -0500\n\n\n2025-09-09 11:36:03 -0500\n\n\n2025-09-09 11:36:04 -0500\n\n\n" +# ``` +# +# The execution tag may also contain multiple lines of code: +# +# ``` +# s = < +# * <%=i%>,<%=j%> +# <% +# end +# end +# %> +# EOT +# ERB.new(s).result +# # => "\n* 0,0\n\n* 0,1\n\n* 0,2\n\n* 1,0\n\n* 1,1\n\n* 1,2\n\n* 2,0\n\n* 2,1\n\n* 2,2\n\n" +# ``` +# +# You can use keyword argument `trim_mode` to make certain adjustments to the processing; +# see ERB.new. +# +# In particular, you can give `trim_mode: '%'` to enable a shorthand format for execution tags; +# this example uses the shorthand format `% _code_` instead of `<% _code_ %>`: +# +# ``` +# s = < +# % end +# EOT +# template = ERB.new(s, trim_mode: '%') +# priorities = [ 'Run Ruby Quiz', +# 'Document Modules', +# 'Answer Questions on Ruby Talk' ] +# puts template.result(binding) +# * Run Ruby Quiz +# * Document Modules +# * Answer Questions on Ruby Talk +# ``` +# +# Note that in the shorthand format, the character `'%'` must be the first character in the code line +# (no leading whitespace). # -# # Create template. -# template = %q{ -# From: James Edward Gray II -# To: <%= to %> -# Subject: Addressing Needs +# ### Comment Tags # -# <%= to[/\w+/] %>: +# You can embed a comment in a template using a *comment tag*; +# its syntax is `<%# _text_ %>`, +# where *text* is the text of the comment. # -# Just wanted to send a quick note assuring that your needs are being -# addressed. +# When you call method #result, +# it removes the entire comment tag +# (generating no text in the result). # -# I want you to know that my team will keep working on the issues, -# especially: +# Example: # -# <%# ignore numerous minor requests -- focus on priorities %> -# % priorities.each do |priority| -# * <%= priority %> -# % end +# ``` +# s = 'Some stuff;<%# Note to self: figure out what the stuff is. %> more stuff.' +# ERB.new(s).result # => "Some stuff; more stuff." +# ``` # -# Thanks for your patience. +# A comment tag may appear anywhere in the template text. # -# James Edward Gray II -# }.gsub(/^ /, '') +# Note that the beginning of the tag must be `'<%#'`, not `'<% #'`. # -# message = ERB.new(template, trim_mode: "%<>") +# In this example, the tag begins with `'<% #'`, and so is an execution tag, not a comment tag; +# the cited code consists entirely of a Ruby-style comment (which is of course ignored): # -# # Set up template data. -# to = "Community Spokesman " -# priorities = [ "Run Ruby Quiz", -# "Document Modules", -# "Answer Questions on Ruby Talk" ] +# ``` +# ERB.new('Some stuff;<% # Note to self: figure out what the stuff is. %> more stuff.').result +# # => "Some stuff;" +# ``` # -# # Produce result. -# email = message.result -# puts email +# ## Encodings # -# Generates: +# In general, an \ERB result string (or Ruby code generated by \ERB) +# has the same encoding as the string originally passed to ERB.new; +# see [Encoding][encoding]. # -# From: James Edward Gray II -# To: Community Spokesman -# Subject: Addressing Needs +# You can specify the output encoding by adding a [magic comment][magic comments] +# at the top of the given string: # -# Community: +# ``` +# s = < # -# Just wanted to send a quick note assuring that your needs are being addressed. +# Some text. +# EOF +# # => "<%#-*- coding: Big5 -*-%>\n\nSome text.\n" +# s.encoding +# # => # +# ERB.new(s).result.encoding +# # => # +# ``` # -# I want you to know that my team will keep working on the issues, especially: +# ## Plain Text Example # -# * Run Ruby Quiz -# * Document Modules -# * Answer Questions on Ruby Talk +# Here's a plain-text string; +# it uses the literal notation `'%q{ ... }'` to define the string +# (see [%q literals][%q literals]); +# this avoids problems with backslashes. # -# Thanks for your patience. +# ``` +# s = %q{ +# From: James Edward Gray II +# To: <%= to %> +# Subject: Addressing Needs # -# James Edward Gray II +# <%= to[/\w+/] %>: # -# === Ruby in HTML +# Just wanted to send a quick note assuring that your needs are being +# addressed. # -# ERB is often used in .rhtml files (HTML with embedded Ruby). Notice the need in -# this example to provide a special binding when the template is run, so that the instance -# variables in the Product object can be resolved. +# I want you to know that my team will keep working on the issues, +# especially: # -# require "erb" +# <%# ignore numerous minor requests -- focus on priorities %> +# % priorities.each do |priority| +# * <%= priority %> +# % end # -# # Build template data class. -# class Product -# def initialize( code, name, desc, cost ) -# @code = code -# @name = name -# @desc = desc -# @cost = cost +# Thanks for your patience. # -# @features = [ ] -# end +# James Edward Gray II +# } +# ``` # -# def add_feature( feature ) -# @features << feature -# end +# The template will need these: # -# # Support templating of member data. -# def get_binding -# binding -# end +# ``` +# to = 'Community Spokesman ' +# priorities = [ 'Run Ruby Quiz', +# 'Document Modules', +# 'Answer Questions on Ruby Talk' ] +# ``` # -# # ... -# end +# Finally, make the template and get the result # -# # Create template. -# template = %{ -# -# Ruby Toys -- <%= @name %> -# +# ``` +# template = ERB.new(s, trim_mode: '%<>') +# puts template.result(binding) # -#

<%= @name %> (<%= @code %>)

-#

<%= @desc %>

+# From: James Edward Gray II +# To: Community Spokesman +# Subject: Addressing Needs # -#
    -# <% @features.each do |f| %> -#
  • <%= f %>
  • -# <% end %> -#
+# Community: # -#

-# <% if @cost < 10 %> -# Only <%= @cost %>!!! -# <% else %> -# Call for a price, today! -# <% end %> -#

+# Just wanted to send a quick note assuring that your needs are being +# addressed. # -# -# -# }.gsub(/^ /, '') +# I want you to know that my team will keep working on the issues, +# especially: # -# rhtml = ERB.new(template) +# * Run Ruby Quiz +# * Document Modules +# * Answer Questions on Ruby Talk # -# # Set up template data. -# toy = Product.new( "TZ-1002", -# "Rubysapien", -# "Geek's Best Friend! Responds to Ruby commands...", -# 999.95 ) -# toy.add_feature("Listens for verbal commands in the Ruby language!") -# toy.add_feature("Ignores Perl, Java, and all C variants.") -# toy.add_feature("Karate-Chop Action!!!") -# toy.add_feature("Matz signature on left leg.") -# toy.add_feature("Gem studded eyes... Rubies, of course!") +# Thanks for your patience. # -# # Produce result. -# rhtml.run(toy.get_binding) +# James Edward Gray II +# ``` # -# Generates (some blank lines removed): +# ## HTML Example # -# -# Ruby Toys -- Rubysapien -# +# This example shows an HTML template. # -#

Rubysapien (TZ-1002)

-#

Geek's Best Friend! Responds to Ruby commands...

-# -#
    -#
  • Listens for verbal commands in the Ruby language!
  • -#
  • Ignores Perl, Java, and all C variants.
  • -#
  • Karate-Chop Action!!!
  • -#
  • Matz signature on left leg.
  • -#
  • Gem studded eyes... Rubies, of course!
  • -#
-# -#

-# Call for a price, today! -#

-# -# -# +# First, here's a custom class, `Product`: # +# ``` +# class Product +# def initialize(code, name, desc, cost) +# @code = code +# @name = name +# @desc = desc +# @cost = cost +# @features = [] +# end # -# == Notes +# def add_feature(feature) +# @features << feature +# end # -# There are a variety of templating solutions available in various Ruby projects. -# For example, RDoc, distributed with Ruby, uses its own template engine, which -# can be reused elsewhere. +# # Support templating of member data. +# def get_binding +# binding +# end # -# Other popular engines could be found in the corresponding -# {Category}[https://www.ruby-toolbox.com/categories/template_engines] of -# The Ruby Toolbox. +# end +# ``` +# +# The template below will need these values: +# +# ``` +# toy = Product.new('TZ-1002', +# 'Rubysapien', +# "Geek's Best Friend! Responds to Ruby commands...", +# 999.95 +# ) +# toy.add_feature('Listens for verbal commands in the Ruby language!') +# toy.add_feature('Ignores Perl, Java, and all C variants.') +# toy.add_feature('Karate-Chop Action!!!') +# toy.add_feature('Matz signature on left leg.') +# toy.add_feature('Gem studded eyes... Rubies, of course!') +# ``` +# +# Here's the HTML: +# +# ``` +# s = < +# Ruby Toys -- <%= @name %> +# +#

<%= @name %> (<%= @code %>)

+#

<%= @desc %>

+#
    +# <% @features.each do |f| %> +#
  • <%= f %>
  • +# <% end %> +#
+#

+# <% if @cost < 10 %> +# Only <%= @cost %>!!! +# <% else %> +# Call for a price, today! +# <% end %> +#

+# +# +# EOT +# ``` +# +# Finally, build the template and get the result (omitting some blank lines): +# +# ``` +# template = ERB.new(s) +# puts template.result(toy.get_binding) +# +# Ruby Toys -- Rubysapien +# +#

Rubysapien (TZ-1002)

+#

Geek's Best Friend! Responds to Ruby commands...

+#
    +#
  • Listens for verbal commands in the Ruby language!
  • +#
  • Ignores Perl, Java, and all C variants.
  • +#
  • Karate-Chop Action!!!
  • +#
  • Matz signature on left leg.
  • +#
  • Gem studded eyes... Rubies, of course!
  • +#
+#

+# Call for a price, today! +#

+# +# +# ``` +# +# +# ## Other Template Processors +# +# Various Ruby projects have their own template processors. +# The Ruby Processing System [RDoc][rdoc], for example, has one that can be used elsewhere. +# +# Other popular template processors may found in the [Template Engines][template engines] page +# of the Ruby Toolbox. +# +# [binding object]: https://docs.ruby-lang.org/en/master/Binding.html +# [comment tags]: rdoc-ref:ERB@Comment+Tags +# [encoding]: https://docs.ruby-lang.org/en/master/Encoding.html +# [execution tags]: rdoc-ref:ERB@Execution+Tags +# [expression tags]: rdoc-ref:ERB@Expression+Tags +# [kernel#binding]: https://docs.ruby-lang.org/en/master/Kernel.html#method-i-binding +# [%q literals]: https://docs.ruby-lang.org/en/master/syntax/literals_rdoc.html#label-25q-3A+Non-Interpolable+String+Literals +# [magic comments]: https://docs.ruby-lang.org/en/master/syntax/comments_rdoc.html#label-Magic+Comments +# [rdoc]: https://ruby.github.io/rdoc +# [sprintf]: https://docs.ruby-lang.org/en/master/Kernel.html#method-i-sprintf +# [template engines]: https://www.ruby-toolbox.com/categories/template_engines +# [template processor]: https://en.wikipedia.org/wiki/Template_processor # class ERB Revision = '$Date:: $' # :nodoc: #' @@ -286,7 +546,7 @@ def self.version # templates through the same binding and/or when you want to control where # output ends up. Pass the name of the variable to be used inside a String. # - # === Example + # ### Example # # require "erb" # diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 4daa5113007086..74b9971a00a3e8 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "1.4.0" + spec.version = "1.5.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] diff --git a/prism/extension.h b/prism/extension.h index 506da2fd6f079e..f4cb9c438d5ed8 100644 --- a/prism/extension.h +++ b/prism/extension.h @@ -1,7 +1,7 @@ #ifndef PRISM_EXT_NODE_H #define PRISM_EXT_NODE_H -#define EXPECTED_PRISM_VERSION "1.4.0" +#define EXPECTED_PRISM_VERSION "1.5.0" #include #include diff --git a/prism/options.h b/prism/options.h index 092fda4f07878a..1a92c470f1ea7d 100644 --- a/prism/options.h +++ b/prism/options.h @@ -237,6 +237,8 @@ static const uint8_t PM_OPTIONS_COMMAND_LINE_X = 0x20; * @param shebang_callback The shebang callback to set. * @param shebang_callback_data Any additional data that should be passed along * to the callback. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *options, pm_options_shebang_callback_t shebang_callback, void *shebang_callback_data); @@ -245,6 +247,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *optio * * @param options The options struct to set the filepath on. * @param filepath The filepath to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath); @@ -253,6 +257,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons * * @param options The options struct to set the line on. * @param line The line to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line); @@ -261,6 +267,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t * * @param options The options struct to set the encoding on. * @param encoding The encoding to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding); @@ -269,6 +277,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons * * @param options The options struct to set the encoding_locked value on. * @param encoding_locked The encoding_locked value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked); @@ -277,6 +287,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *option * * @param options The options struct to set the frozen string literal value on. * @param frozen_string_literal The frozen string literal value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal); @@ -285,6 +297,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t * * * @param options The options struct to set the command line option on. * @param command_line The command_line value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options, uint8_t command_line); @@ -297,6 +311,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options, * @param version The version to set. * @param length The length of the version string. * @return Whether or not the version was parsed successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length); @@ -305,6 +321,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const * * @param options The options struct to set the main script value on. * @param main_script The main script value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script); @@ -313,6 +331,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, b * * @param options The options struct to set the partial script value on. * @param partial_script The partial script value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script); @@ -321,6 +341,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options * * @param options The options struct to set the freeze value on. * @param freeze The freeze value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze); @@ -330,6 +352,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool f * @param options The options struct to initialize the scopes array on. * @param scopes_count The number of scopes to allocate. * @return Whether or not the scopes array was initialized successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_scopes_init(pm_options_t *options, size_t scopes_count); @@ -339,6 +363,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_scopes_init(pm_options_t *options, size_ * @param options The options struct to get the scope from. * @param index The index of the scope to get. * @return A pointer to the scope at the given index. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index); @@ -349,6 +375,8 @@ PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm * @param scope The scope struct to initialize. * @param locals_count The number of locals to allocate. * @return Whether or not the scope was initialized successfully. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count); @@ -358,6 +386,8 @@ PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, si * @param scope The scope struct to get the local from. * @param index The index of the local to get. * @return A pointer to the local at the given index. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index); @@ -366,6 +396,8 @@ PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_ * * @param scope The scope struct to set the forwarding on. * @param forwarding The forwarding value to set. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding); @@ -373,6 +405,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t * Free the internal memory associated with the options. * * @param options The options struct whose internal memory should be freed. + * + * \public \memberof pm_options */ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); diff --git a/prism/prism.c b/prism/prism.c index daf69d2ef97553..337b77637b748f 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19585,13 +19585,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_push(parser, false); statements = (pm_node_t *) pm_statements_node_create(parser); - // In endless method bodies, we need to handle command calls carefully. - // We want to allow command calls in assignment context but maintain - // the same binding power to avoid changing how operators are parsed. - // Note that we're intentionally NOT allowing code like `private def foo = puts "Hello"` - // because the original parser, parse.y, can't handle it and we want to maintain the same behavior - bool allow_command_call = (binding_power == PM_BINDING_POWER_ASSIGNMENT) || - (binding_power < PM_BINDING_POWER_COMPOSITION); + bool allow_command_call; + if (parser->version >= PM_OPTIONS_VERSION_CRUBY_3_5) { + allow_command_call = accepts_command_call; + } else { + // Allow `def foo = puts "Hello"` but not `private def foo = puts "Hello"` + allow_command_call = binding_power == PM_BINDING_POWER_ASSIGNMENT || binding_power < PM_BINDING_POWER_COMPOSITION; + } pm_node_t *statement = parse_expression(parser, PM_BINDING_POWER_DEFINED + 1, allow_command_call, false, PM_ERR_DEF_ENDLESS, (uint16_t) (depth + 1)); diff --git a/prism/prism.h b/prism/prism.h index 555bda08515fa9..dc31f26e786a5c 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -56,6 +56,8 @@ PRISM_EXPORTED_FUNCTION const char * pm_version(void); * @param size The size of the source. * @param options The optional options to use when parsing. These options must * live for the whole lifetime of this parser. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options); @@ -65,6 +67,8 @@ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t * * * @param parser The parser to register the callback with. * @param callback The callback to register. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback); @@ -75,6 +79,8 @@ PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_par * parser. * * @param parser The parser to free. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser); @@ -83,11 +89,13 @@ PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser); * * @param parser The parser to use. * @return The AST representing the source. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser); /** - * This function is used in pm_parse_stream to retrieve a line of input from a + * This function is used in pm_parse_stream() to retrieve a line of input from a * stream. It closely mirrors that of fgets so that fgets can be used as the * default implementation. */ @@ -110,6 +118,8 @@ typedef int (pm_parse_stream_feof_t)(void *stream); * @param stream_feof The function to use to determine if the stream has hit eof. * @param options The optional options to use when parsing. * @return The AST representing the source. + * + * \public \memberof pm_parser */ PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t *parser, pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *stream_fgets, pm_parse_stream_feof_t *stream_feof, const pm_options_t *options); diff --git a/prism/regexp.h b/prism/regexp.h index c0b3163e93b649..5366b5a5a0d359 100644 --- a/prism/regexp.h +++ b/prism/regexp.h @@ -17,12 +17,12 @@ #include /** - * This callback is called when a named capture group is found. + * This callback is called by pm_regexp_parse() when a named capture group is found. */ typedef void (*pm_regexp_name_callback_t)(const pm_string_t *name, void *data); /** - * This callback is called when a parse error is found. + * This callback is called by pm_regexp_parse() when a parse error is found. */ typedef void (*pm_regexp_error_callback_t)(const uint8_t *start, const uint8_t *end, const char *message, void *data); diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 104b60f4842015..a5909e15bb67fe 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -10,7 +10,7 @@ module Prism # The minor version of prism that we are expecting to find in the serialized # strings. - MINOR_VERSION = 4 + MINOR_VERSION = 5 # The patch version of prism that we are expecting to find in the serialized # strings. diff --git a/prism/util/pm_buffer.h b/prism/util/pm_buffer.h index f3c20ab2a5ad10..cb80f8b3ce7d01 100644 --- a/prism/util/pm_buffer.h +++ b/prism/util/pm_buffer.h @@ -51,6 +51,8 @@ bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity); * * @param buffer The buffer to initialize. * @returns True if the buffer was initialized successfully, false otherwise. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer); @@ -59,6 +61,8 @@ PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer); * * @param buffer The buffer to get the value of. * @returns The value of the buffer. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer); @@ -67,6 +71,8 @@ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer); * * @param buffer The buffer to get the length of. * @returns The length of the buffer. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(const pm_buffer_t *buffer); @@ -222,6 +228,8 @@ void pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size * Free the memory associated with the buffer. * * @param buffer The buffer to free. + * + * \public \memberof pm_buffer_t */ PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer); diff --git a/prism/util/pm_integer.h b/prism/util/pm_integer.h index a9e2966703408b..304665e6205dea 100644 --- a/prism/util/pm_integer.h +++ b/prism/util/pm_integer.h @@ -112,6 +112,8 @@ void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator); * * @param buffer The buffer to append the string to. * @param integer The integer to convert to a string. + * + * \public \memberof pm_integer_t */ PRISM_EXPORTED_FUNCTION void pm_integer_string(pm_buffer_t *buffer, const pm_integer_t *integer); @@ -120,6 +122,8 @@ PRISM_EXPORTED_FUNCTION void pm_integer_string(pm_buffer_t *buffer, const pm_int * the integer exceeds the size of a single node in the linked list. * * @param integer The integer to free. + * + * \public \memberof pm_integer_t */ PRISM_EXPORTED_FUNCTION void pm_integer_free(pm_integer_t *integer); diff --git a/prism/util/pm_list.h b/prism/util/pm_list.h index 3512dee979aa52..f544bb2943d3ff 100644 --- a/prism/util/pm_list.h +++ b/prism/util/pm_list.h @@ -68,6 +68,8 @@ typedef struct { * * @param list The list to check. * @return True if the given list is empty, otherwise false. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list); @@ -76,6 +78,8 @@ PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list); * * @param list The list to check. * @return The size of the list. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION size_t pm_list_size(pm_list_t *list); @@ -91,6 +95,8 @@ void pm_list_append(pm_list_t *list, pm_list_node_t *node); * Deallocate the internal state of the given list. * * @param list The list to free. + * + * \public \memberof pm_list_t */ PRISM_EXPORTED_FUNCTION void pm_list_free(pm_list_t *list); diff --git a/prism/util/pm_string.h b/prism/util/pm_string.h index ccf6648d263bc0..d8456ff2947eb8 100644 --- a/prism/util/pm_string.h +++ b/prism/util/pm_string.h @@ -130,6 +130,8 @@ typedef enum { * @param string The string to initialize. * @param filepath The filepath to read. * @return The success of the read, indicated by the value of the enum. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_mapped_init(pm_string_t *string, const char *filepath); @@ -141,6 +143,8 @@ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_mapped_init(pm_string_ * @param string The string to initialize. * @param filepath The filepath to read. * @return The success of the read, indicated by the value of the enum. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_file_init(pm_string_t *string, const char *filepath); @@ -169,6 +173,8 @@ int pm_string_compare(const pm_string_t *left, const pm_string_t *right); * * @param string The string to get the length of. * @return The length of the string. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string); @@ -177,6 +183,8 @@ PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string); * * @param string The string to get the start pointer of. * @return The start pointer of the string. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *string); @@ -184,6 +192,8 @@ PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *stri * Free the associated memory of the given string. * * @param string The string to free. + * + * \public \memberof pm_string_t */ PRISM_EXPORTED_FUNCTION void pm_string_free(pm_string_t *string); diff --git a/prism/version.h b/prism/version.h index 0a2a8c8fce5a25..697c7b5ad6f37e 100644 --- a/prism/version.h +++ b/prism/version.h @@ -14,7 +14,7 @@ /** * The minor version of the Prism library as an int. */ -#define PRISM_VERSION_MINOR 4 +#define PRISM_VERSION_MINOR 5 /** * The patch version of the Prism library as an int. @@ -24,6 +24,6 @@ /** * The version of the Prism library as a constant string. */ -#define PRISM_VERSION "1.4.0" +#define PRISM_VERSION "1.5.0" #endif diff --git a/prism_compile.c b/prism_compile.c index 37909e49e01444..70081f3d95ad1f 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -1,4 +1,5 @@ #include "prism.h" +#include "version.h" /** * This compiler defines its own concept of the location of a node. We do this @@ -11352,6 +11353,8 @@ pm_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lines) pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); + pm_options_version_for_current_ruby_set(&result->options); + pm_parser_init(&result->parser, pm_string_source(&result->input), pm_string_length(&result->input), &result->options); pm_node_t *node = pm_parse(&result->parser); @@ -11410,6 +11413,8 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath, VALUE * pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); + pm_options_version_for_current_ruby_set(&result->options); + pm_parser_init(&result->parser, pm_string_source(&result->input), pm_string_length(&result->input), &result->options); pm_node_t *node = pm_parse(&result->parser); @@ -11492,6 +11497,13 @@ pm_parse_stdin(pm_parse_result_t *result) return pm_parse_process(result, node, NULL); } +#define PM_VERSION_FOR_RELEASE(major, minor) PM_VERSION_FOR_RELEASE_IMPL(major, minor) +#define PM_VERSION_FOR_RELEASE_IMPL(major, minor) PM_OPTIONS_VERSION_CRUBY_##major##_##minor + +void pm_options_version_for_current_ruby_set(pm_options_t *options) { + options->version = PM_VERSION_FOR_RELEASE(RUBY_VERSION_MAJOR, RUBY_VERSION_MINOR); +} + #undef NEW_ISEQ #define NEW_ISEQ OLD_ISEQ diff --git a/prism_compile.h b/prism_compile.h index c032449bd65ca9..e588122205948b 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -94,6 +94,7 @@ VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lin VALUE pm_load_parse_file(pm_parse_result_t *result, VALUE filepath, VALUE *script_lines); VALUE pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath, VALUE *script_lines); VALUE pm_parse_stdin(pm_parse_result_t *result); +void pm_options_version_for_current_ruby_set(pm_options_t *options); void pm_parse_result_free(pm_parse_result_t *result); rb_iseq_t *pm_iseq_new(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, enum rb_iseq_type, int *error_state); diff --git a/test/.excludes-parsey/Prism/FixturesTest.rb b/test/.excludes-parsey/Prism/FixturesTest.rb new file mode 100644 index 00000000000000..452ff4f5058746 --- /dev/null +++ b/test/.excludes-parsey/Prism/FixturesTest.rb @@ -0,0 +1 @@ +exclude(:"test_leading_logical.txt", "Requires Feature #20925 to be implemented on parse.y") diff --git a/test/.excludes-parsey/Prism/LocalsTest.rb b/test/.excludes-parsey/Prism/LocalsTest.rb new file mode 100644 index 00000000000000..452ff4f5058746 --- /dev/null +++ b/test/.excludes-parsey/Prism/LocalsTest.rb @@ -0,0 +1 @@ +exclude(:"test_leading_logical.txt", "Requires Feature #20925 to be implemented on parse.y") diff --git a/test/.excludes-parsey/TestSyntax.rb b/test/.excludes-parsey/TestSyntax.rb new file mode 100644 index 00000000000000..ff7307060dd2a4 --- /dev/null +++ b/test/.excludes-parsey/TestSyntax.rb @@ -0,0 +1 @@ +exclude(:test_methoddef_endless_command, "[Bug #17398]") diff --git a/test/prism/errors/endless_method_command_call.txt b/test/prism/errors/endless_method_command_call.txt new file mode 100644 index 00000000000000..e6a328c2944a68 --- /dev/null +++ b/test/prism/errors/endless_method_command_call.txt @@ -0,0 +1,3 @@ +private :m, def hello = puts "Hello" + ^ unexpected string literal, expecting end-of-input + diff --git a/test/prism/errors/private_endless_method.txt b/test/prism/errors/private_endless_method.txt deleted file mode 100644 index 8aae5e0cd39035..00000000000000 --- a/test/prism/errors/private_endless_method.txt +++ /dev/null @@ -1,3 +0,0 @@ -private def foo = puts "Hello" - ^ unexpected string literal, expecting end-of-input - diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 62bbd8458b2ca9..9dd4aea72865d9 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -100,6 +100,15 @@ def foo(bar: bar) = 42 end end + def test_private_endless_method + source = <<~RUBY + private def foo = puts "Hello" + RUBY + + assert_predicate Prism.parse(source, version: "3.4"), :failure? + assert_predicate Prism.parse(source), :success? + end + private def assert_errors(filepath) diff --git a/test/prism/fixtures/endless_methods_command_call.txt b/test/prism/fixtures/endless_methods_command_call.txt new file mode 100644 index 00000000000000..91a9d156d5538b --- /dev/null +++ b/test/prism/fixtures/endless_methods_command_call.txt @@ -0,0 +1,8 @@ +private def foo = puts "Hello" +private def foo = puts "Hello", "World" +private def foo = puts "Hello" do expr end +private def foo() = puts "Hello" +private def foo(x) = puts x +private def obj.foo = puts "Hello" +private def obj.foo() = puts "Hello" +private def obj.foo(x) = puts x diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb index 124a834317498b..b4b656fcf49b48 100644 --- a/test/prism/fixtures_test.rb +++ b/test/prism/fixtures_test.rb @@ -8,7 +8,6 @@ module Prism class FixturesTest < TestCase except = [] - if RUBY_VERSION < "3.3.0" # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace # characters in the heredoc start. @@ -25,7 +24,9 @@ class FixturesTest < TestCase except << "whitequark/ruby_bug_19281.txt" end - except << "leading_logical.txt" if RUBY_VERSION < "3.5.0" + # Leaving these out until they are supported by parse.y. + except << "leading_logical.txt" + except << "endless_methods_command_call.txt" Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_valid_syntax(fixture.read) } diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index abce18a0ad3387..4eacbab3e1170d 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -45,6 +45,9 @@ class LexTest < TestCase # https://bugs.ruby-lang.org/issues/20925 except << "leading_logical.txt" + # https://bugs.ruby-lang.org/issues/17398#note-12 + except << "endless_methods_command_call.txt" + Fixture.each(except: except) do |fixture| define_method(fixture.test_name) { assert_lex(fixture) } end diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb index e0e9a458559759..950e7118af526a 100644 --- a/test/prism/locals_test.rb +++ b/test/prism/locals_test.rb @@ -29,7 +29,11 @@ class LocalsTest < TestCase except = [ # Skip this fixture because it has a different number of locals because # CRuby is eliminating dead code. - "whitequark/ruby_bug_10653.txt" + "whitequark/ruby_bug_10653.txt", + + # Leaving these out until they are supported by parse.y. + "leading_logical.txt", + "endless_methods_command_call.txt" ] Fixture.each(except: except) do |fixture| diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index 129c38a3b5b40d..98740f09734043 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -67,6 +67,9 @@ class ParserTest < TestCase # Cannot yet handling leading logical operators. "leading_logical.txt", + + # Ruby >= 3.5 specific syntax + "endless_methods_command_call.txt", ] # These files contain code that is being parsed incorrectly by the parser diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 637202487811b5..39325137ba07f2 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -29,7 +29,10 @@ class RipperTest < TestCase "whitequark/lvar_injecting_match.txt", # Ripper fails to understand some structures that span across heredocs. - "spanning_heredoc.txt" + "spanning_heredoc.txt", + + # https://bugs.ruby-lang.org/issues/17398#note-12 + "endless_methods_command_call.txt", ] # Skip these tests that we haven't implemented yet. diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index f4f0f331fb89a9..bcaed7979150bc 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -74,7 +74,10 @@ class RubyParserTest < TestCase "whitequark/ruby_bug_11989.txt", "whitequark/ruby_bug_18878.txt", "whitequark/ruby_bug_19281.txt", - "whitequark/slash_newline_in_heredocs.txt" + "whitequark/slash_newline_in_heredocs.txt", + + # Ruby >= 3.5 specific syntax + "endless_methods_command_call.txt", ] Fixture.each(except: failures) do |fixture| diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index bbdeb876ec3eb0..7e2185be39d47c 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1794,15 +1794,12 @@ def test_methoddef_endless_command assert_equal("class ok", k.rescued("ok")) assert_equal("instance ok", k.new.rescued("ok")) - # Current technical limitation: cannot prepend "private" or something for command endless def - error = /(syntax error,|\^~*) unexpected string literal/ - error2 = /(syntax error,|\^~*) unexpected local variable or method/ - assert_syntax_error('private def foo = puts "Hello"', error) - assert_syntax_error('private def foo() = puts "Hello"', error) - assert_syntax_error('private def foo(x) = puts x', error2) - assert_syntax_error('private def obj.foo = puts "Hello"', error) - assert_syntax_error('private def obj.foo() = puts "Hello"', error) - assert_syntax_error('private def obj.foo(x) = puts x', error2) + assert_valid_syntax('private def foo = puts "Hello"') + assert_valid_syntax('private def foo() = puts "Hello"') + assert_valid_syntax('private def foo(x) = puts x') + assert_valid_syntax('private def obj.foo = puts "Hello"') + assert_valid_syntax('private def obj.foo() = puts "Hello"') + assert_valid_syntax('private def obj.foo(x) = puts x') end def test_methoddef_in_cond diff --git a/yjit.c b/yjit.c index ac218a084ca33f..57b09d73b0bceb 100644 --- a/yjit.c +++ b/yjit.c @@ -69,60 +69,12 @@ STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM); // The "_yjit_" part is for trying to be informative. We might want different // suffixes for symbols meant for Rust and symbols meant for broader CRuby. -bool -rb_yjit_mark_writable(void *mem_block, uint32_t mem_size) -{ - return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; -} - -void -rb_yjit_mark_executable(void *mem_block, uint32_t mem_size) -{ - // Do not call mprotect when mem_size is zero. Some platforms may return - // an error for it. https://github.com/Shopify/ruby/issues/450 - if (mem_size == 0) { - return; - } - if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { - rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", - mem_block, (unsigned long)mem_size, strerror(errno)); - } -} - -// Free the specified memory block. -bool -rb_yjit_mark_unused(void *mem_block, uint32_t mem_size) -{ - // On Linux, you need to use madvise MADV_DONTNEED to free memory. - // We might not need to call this on macOS, but it's not really documented. - // We generally prefer to do the same thing on both to ease testing too. - madvise(mem_block, mem_size, MADV_DONTNEED); - - // On macOS, mprotect PROT_NONE seems to reduce RSS. - // We also call this on Linux to avoid executing unused pages. - return mprotect(mem_block, mem_size, PROT_NONE) == 0; -} - long rb_yjit_array_len(VALUE a) { return rb_array_len(a); } -// `start` is inclusive and `end` is exclusive. -void -rb_yjit_icache_invalidate(void *start, void *end) -{ - // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM before running freshly written code. - // On Darwin it's the same as calling sys_icache_invalidate(). -#ifdef __GNUC__ - __builtin___clear_cache(start, end); -#elif defined(__aarch64__) -#error No instruction cache clear available with this compiler on Aarch64! -#endif -} - # define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x))) // For a given raw_sample (frame), set the hash with the caller's @@ -217,131 +169,6 @@ rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int return result; } -uint32_t -rb_yjit_get_page_size(void) -{ -#if defined(_SC_PAGESIZE) - long page_size = sysconf(_SC_PAGESIZE); - if (page_size <= 0) rb_bug("yjit: failed to get page size"); - - // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. - // Though our design sort of assume we have fine grained control over memory protection - // which require small page sizes. - if (page_size > 0x40000000l) rb_bug("yjit page size too large"); - - return (uint32_t)page_size; -#else -#error "YJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif - -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_yjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_yjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_yjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: yjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } - - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} - // Is anyone listening for :c_call and :c_return event currently? bool rb_c_method_tracing_currently_enabled(const rb_execution_context_t *ec) diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index b62c637e1dbc4e..29b17346cd90ae 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -244,11 +244,11 @@ fn main() { .allowlist_function("rb_iseq_(get|set)_yjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") - .allowlist_function("rb_yjit_reserve_addr_space") - .allowlist_function("rb_yjit_mark_writable") - .allowlist_function("rb_yjit_mark_executable") - .allowlist_function("rb_yjit_mark_unused") - .allowlist_function("rb_yjit_get_page_size") + .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_jit_mark_writable") + .allowlist_function("rb_jit_mark_executable") + .allowlist_function("rb_jit_mark_unused") + .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_yjit_iseq_builtin_attrs") .allowlist_function("rb_yjit_iseq_inspect") .allowlist_function("rb_yjit_builtin_function") @@ -267,7 +267,7 @@ fn main() { .allowlist_function("rb_ENCODING_GET") .allowlist_function("rb_yjit_get_proc_ptr") .allowlist_function("rb_yjit_exit_locations_dict") - .allowlist_function("rb_yjit_icache_invalidate") + .allowlist_function("rb_jit_icache_invalidate") .allowlist_function("rb_optimized_call") .allowlist_function("rb_yjit_sendish_sp_pops") .allowlist_function("rb_yjit_invokeblock_sp_pops") diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index 66e333f867d451..6fa8fef627df0c 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -98,7 +98,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { #[cfg(not(test))] { let end = cb.get_write_ptr(); - unsafe { rb_yjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } } @@ -1361,7 +1361,7 @@ impl Assembler #[cfg(not(test))] cb.without_page_end_reserve(|cb| { for (start, end) in cb.writable_addrs(start_ptr, cb.get_write_ptr()) { - unsafe { rb_yjit_icache_invalidate(start as _, end as _) }; + unsafe { rb_jit_icache_invalidate(start as _, end as _) }; } }); diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 2bac85e62b57af..85fed25d24e48f 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -10945,7 +10945,7 @@ impl CodegenGlobals { #[cfg(not(test))] let (mut cb, mut ocb) = { - let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(exec_mem_size as u32) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_size as u32) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -10954,7 +10954,7 @@ impl CodegenGlobals { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_yjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size.as_usize(), 0, "Start of virtual address block should be page-aligned", diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index a681d479770948..d9ef2d494f6d55 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1164,21 +1164,12 @@ extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_yjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_yjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; pub fn rb_yjit_array_len(a: VALUE) -> ::std::os::raw::c_long; - pub fn rb_yjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); pub fn rb_yjit_exit_locations_dict( yjit_raw_samples: *mut VALUE, yjit_line_samples: *mut ::std::os::raw::c_int, samples_len: ::std::os::raw::c_int, ) -> VALUE; - pub fn rb_yjit_get_page_size() -> u32; - pub fn rb_yjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_c_method_tracing_currently_enabled(ec: *const rb_execution_context_t) -> bool; pub fn rb_full_cfunc_return(ec: *mut rb_execution_context_t, return_value: VALUE); pub fn rb_iseq_get_yjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; @@ -1325,5 +1316,14 @@ extern "C" { line: ::std::os::raw::c_int, ); pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); } diff --git a/yjit/src/virtualmem.rs b/yjit/src/virtualmem.rs index 97409c796cbca7..58c9d9a7fd380a 100644 --- a/yjit/src/virtualmem.rs +++ b/yjit/src/virtualmem.rs @@ -319,15 +319,15 @@ mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_yjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } diff --git a/zjit.c b/zjit.c index 5d50775fc72c05..565a362a8a6aef 100644 --- a/zjit.c +++ b/zjit.c @@ -31,131 +31,6 @@ #include -uint32_t -rb_zjit_get_page_size(void) -{ -#if defined(_SC_PAGESIZE) - long page_size = sysconf(_SC_PAGESIZE); - if (page_size <= 0) rb_bug("zjit: failed to get page size"); - - // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. - // Though our design sort of assume we have fine grained control over memory protection - // which require small page sizes. - if (page_size > 0x40000000l) rb_bug("zjit page size too large"); - - return (uint32_t)page_size; -#else -#error "ZJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif - -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_zjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_zjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_zjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: zjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } - - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} - void rb_zjit_profile_disable(const rb_iseq_t *iseq); void @@ -185,56 +60,6 @@ rb_zjit_constcache_shareable(const struct iseq_inline_constant_cache_entry *ice) return (ice->flags & IMEMO_CONST_CACHE_SHAREABLE) != 0; } - -bool -rb_zjit_mark_writable(void *mem_block, uint32_t mem_size) -{ - return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; -} - -void -rb_zjit_mark_executable(void *mem_block, uint32_t mem_size) -{ - // Do not call mprotect when mem_size is zero. Some platforms may return - // an error for it. https://github.com/Shopify/ruby/issues/450 - if (mem_size == 0) { - return; - } - if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { - rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", - mem_block, (unsigned long)mem_size, strerror(errno)); - } -} - -// Free the specified memory block. -bool -rb_zjit_mark_unused(void *mem_block, uint32_t mem_size) -{ - // On Linux, you need to use madvise MADV_DONTNEED to free memory. - // We might not need to call this on macOS, but it's not really documented. - // We generally prefer to do the same thing on both to ease testing too. - madvise(mem_block, mem_size, MADV_DONTNEED); - - // On macOS, mprotect PROT_NONE seems to reduce RSS. - // We also call this on Linux to avoid executing unused pages. - return mprotect(mem_block, mem_size, PROT_NONE) == 0; -} - -// Invalidate icache for arm64. -// `start` is inclusive and `end` is exclusive. -void -rb_zjit_icache_invalidate(void *start, void *end) -{ - // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM before running freshly written code. - // On Darwin it's the same as calling sys_icache_invalidate(). -#ifdef __GNUC__ - __builtin___clear_cache(start, end); -#elif defined(__aarch64__) -#error No instruction cache clear available with this compiler on Aarch64! -#endif -} - // Convert a given ISEQ's instructions to zjit_* instructions void rb_zjit_profile_enable(const rb_iseq_t *iseq) diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 26bdfd2848373f..7d66bf0ecf3f16 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -263,11 +263,11 @@ fn main() { .allowlist_function("rb_iseq_(get|set)_zjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") - .allowlist_function("rb_zjit_reserve_addr_space") - .allowlist_function("rb_zjit_mark_writable") - .allowlist_function("rb_zjit_mark_executable") - .allowlist_function("rb_zjit_mark_unused") - .allowlist_function("rb_zjit_get_page_size") + .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_jit_mark_writable") + .allowlist_function("rb_jit_mark_executable") + .allowlist_function("rb_jit_mark_unused") + .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_zjit_iseq_builtin_attrs") .allowlist_function("rb_zjit_iseq_inspect") .allowlist_function("rb_zjit_iseq_insn_set") @@ -280,7 +280,7 @@ fn main() { .allowlist_function("rb_RSTRING_LEN") .allowlist_function("rb_ENCODING_GET") .allowlist_function("rb_optimized_call") - .allowlist_function("rb_zjit_icache_invalidate") + .allowlist_function("rb_jit_icache_invalidate") .allowlist_function("rb_zjit_print_exception") .allowlist_function("rb_zjit_singleton_class_p") .allowlist_function("rb_zjit_defined_ivar") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 6e040e5214f0c8..50e7074de11222 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -99,7 +99,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { let start = cb.get_write_ptr(); emit_jmp_ptr(cb, dst_ptr, true); let end = cb.get_write_ptr(); - unsafe { rb_zjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { @@ -1382,7 +1382,7 @@ impl Assembler cb.link_labels(); // Invalidate icache for newly written out region so we don't run stale code. - unsafe { rb_zjit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; Ok((start_ptr, gc_offsets)) } else { diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index bab40e50ee32cc..9ee4b1bb743958 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -922,18 +922,9 @@ unsafe extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_zjit_get_page_size() -> u32; - pub fn rb_zjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_zjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_zjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); pub fn rb_zjit_iseq_insn_set( iseq: *const rb_iseq_t, insn_idx: ::std::os::raw::c_uint, @@ -1037,5 +1028,14 @@ unsafe extern "C" { line: ::std::os::raw::c_int, ); pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 4ade541922641f..3726d8ec0e44d5 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -2968,17 +2968,8 @@ fn compute_bytecode_info(iseq: *const rb_iseq_t) -> BytecodeInfo { #[derive(Debug, PartialEq, Clone, Copy)] pub enum CallType { - Splat, BlockArg, - Kwarg, - KwSplat, Tailcall, - Super, - Zsuper, - OptSend, - KwSplatMut, - SplatMut, - Forwarding, } #[derive(Clone, Debug, PartialEq)] @@ -2996,17 +2987,8 @@ fn num_locals(iseq: *const rb_iseq_t) -> usize { /// If we can't handle the type of send (yet), bail out. fn unknown_call_type(flag: u32) -> Result<(), CallType> { - if (flag & VM_CALL_KW_SPLAT_MUT) != 0 { return Err(CallType::KwSplatMut); } - if (flag & VM_CALL_ARGS_SPLAT_MUT) != 0 { return Err(CallType::SplatMut); } - if (flag & VM_CALL_ARGS_SPLAT) != 0 { return Err(CallType::Splat); } - if (flag & VM_CALL_KW_SPLAT) != 0 { return Err(CallType::KwSplat); } if (flag & VM_CALL_ARGS_BLOCKARG) != 0 { return Err(CallType::BlockArg); } - if (flag & VM_CALL_KWARG) != 0 { return Err(CallType::Kwarg); } if (flag & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } - if (flag & VM_CALL_SUPER) != 0 { return Err(CallType::Super); } - if (flag & VM_CALL_ZSUPER) != 0 { return Err(CallType::Zsuper); } - if (flag & VM_CALL_OPT_SEND) != 0 { return Err(CallType::OptSend); } - if (flag & VM_CALL_FORWARDING) != 0 { return Err(CallType::Forwarding); } Ok(()) } @@ -5126,7 +5108,9 @@ mod tests { fn test@:2: bb0(v0:BasicObject, v1:BasicObject): v6:ArrayExact = ToArray v1 - SideExit UnhandledCallType(Splat) + v8:BasicObject = SendWithoutBlock v0, :foo, v6 + CheckInterrupts + Return v8 "); } @@ -5151,7 +5135,9 @@ mod tests { fn test@:2: bb0(v0:BasicObject, v1:BasicObject): v5:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v7:BasicObject = SendWithoutBlock v0, :foo, v5 + CheckInterrupts + Return v7 "); } @@ -5163,7 +5149,9 @@ mod tests { assert_snapshot!(hir_string("test"), @r" fn test@:2: bb0(v0:BasicObject, v1:BasicObject): - SideExit UnhandledCallType(KwSplat) + v6:BasicObject = SendWithoutBlock v0, :foo, v1 + CheckInterrupts + Return v6 "); } @@ -5252,7 +5240,9 @@ mod tests { v13:StaticSymbol[:b] = Const Value(VALUE(0x1008)) v14:Fixnum[1] = Const Value(1) v16:BasicObject = SendWithoutBlock v12, :core#hash_merge_ptr, v11, v13, v14 - SideExit UnhandledCallType(KwSplatMut) + v18:BasicObject = SendWithoutBlock v0, :foo, v16 + CheckInterrupts + Return v18 "); } @@ -5267,7 +5257,9 @@ mod tests { v6:ArrayExact = ToNewArray v1 v7:Fixnum[1] = Const Value(1) ArrayPush v6, v7 - SideExit UnhandledCallType(SplatMut) + v11:BasicObject = SendWithoutBlock v0, :foo, v6 + CheckInterrupts + Return v11 "); } @@ -7863,7 +7855,9 @@ mod opt_tests { fn test@:3: bb0(v0:BasicObject): v4:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v6:BasicObject = SendWithoutBlock v0, :foo, v4 + CheckInterrupts + Return v6 "); } @@ -7879,7 +7873,9 @@ mod opt_tests { fn test@:3: bb0(v0:BasicObject): v4:Fixnum[1] = Const Value(1) - SideExit UnhandledCallType(Kwarg) + v6:BasicObject = SendWithoutBlock v0, :foo, v4 + CheckInterrupts + Return v6 "); } diff --git a/zjit/src/state.rs b/zjit/src/state.rs index da97829e43629b..0c657f450a25dc 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -60,7 +60,7 @@ impl ZJITState { use crate::options::*; let exec_mem_bytes: usize = get_option!(exec_mem_bytes); - let virt_block: *mut u8 = unsafe { rb_zjit_reserve_addr_space(64 * 1024 * 1024) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(64 * 1024 * 1024) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -69,7 +69,7 @@ impl ZJITState { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_zjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size as usize, 0, "Start of virtual address block should be page-aligned", diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 39cfe6a4398fb3..8a7073dd977c94 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -114,17 +114,8 @@ make_counters! { } // unhanded_call_: Unhandled call types - unhandled_call_splat, unhandled_call_block_arg, - unhandled_call_kwarg, - unhandled_call_kw_splat, unhandled_call_tailcall, - unhandled_call_super, - unhandled_call_zsuper, - unhandled_call_optsend, - unhandled_call_kw_splat_mut, - unhandled_call_splat_mut, - unhandled_call_forwarding, // compile_error_: Compile error reasons compile_error_iseq_stack_too_large, @@ -176,17 +167,8 @@ pub fn exit_counter_ptr_for_call_type(call_type: crate::hir::CallType) -> *mut u use crate::hir::CallType::*; use crate::stats::Counter::*; let counter = match call_type { - Splat => unhandled_call_splat, - BlockArg => unhandled_call_block_arg, - Kwarg => unhandled_call_kwarg, - KwSplat => unhandled_call_kw_splat, - Tailcall => unhandled_call_tailcall, - Super => unhandled_call_super, - Zsuper => unhandled_call_zsuper, - OptSend => unhandled_call_optsend, - KwSplatMut => unhandled_call_kw_splat_mut, - SplatMut => unhandled_call_splat_mut, - Forwarding => unhandled_call_forwarding, + BlockArg => unhandled_call_block_arg, + Tailcall => unhandled_call_tailcall, }; counter_ptr(counter) } diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs index 11de4e08afe962..0d19858c8711f5 100644 --- a/zjit/src/virtualmem.rs +++ b/zjit/src/virtualmem.rs @@ -290,15 +290,15 @@ pub mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_zjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } }