diff --git a/.gitignore b/.gitignore index 218a76cd00..9f59c7f282 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ out.svg /java/org/ruby_lang/prism/AbstractNodeVisitor.java /java/org/ruby_lang/prism/Loader.java /java/org/ruby_lang/prism/Nodes.java +/java-wasm/src/test/resources/prism.wasm /lib/prism/compiler.rb /lib/prism/dispatcher.rb /lib/prism/dot_visitor.rb diff --git a/Makefile b/Makefile index 7f61f753fa..9fbc1a27ae 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,12 @@ javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS) java-wasm/src/test/resources/prism.wasm: Makefile $(SOURCES) $(HEADERS) $(ECHO) "building $@" $(Q) $(MAKEDIRS) $(@D) - $(Q) $(WASI_SDK_PATH)/bin/clang $(DEBUG_FLAGS) -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi -o $@ $(SOURCES) + $(Q) $(WASI_SDK_PATH)/bin/clang \ + $(DEBUG_FLAGS) \ + -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN \ + -lwasi-emulated-mman $(CPPFLAGS) $(JAVA_WASM_CFLAGS) \ + -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -lc++ -lc++abi \ + -o $@ $(SOURCES) build/shared/%.o: src/%.c Makefile $(HEADERS) $(ECHO) "compiling $@" diff --git a/java-wasm/README.md b/java-wasm/README.md index 2ff93a66d7..1c9eaee85e 100644 --- a/java-wasm/README.md +++ b/java-wasm/README.md @@ -1,5 +1,7 @@ This dir contains the chicory-prism artifact, a version of prism compiled to WASM and then AOT compiled to JVM bytecode by the Chicory project. +## Building + Generate the templated sources: ``` @@ -21,3 +23,7 @@ mvn -f java-wasm/pom.xml clean package This should build the chicory-wasm jar file and pass some basic tests. The jar will be under `java-wasm/target/chicory-prism-#####-SNAPSHOT.jar` or can be installed by using `install` instead of `pacakge` in the `mvn` command line above. + +## Releasing + +Pass `-Prelease` to enable release plugins for Maven Central (optional for snapshots) and run the `deploy` target for `mvn`. diff --git a/java-wasm/pom.xml b/java-wasm/pom.xml index 587e094ab7..a53a25eabe 100644 --- a/java-wasm/pom.xml +++ b/java-wasm/pom.xml @@ -2,10 +2,10 @@ 4.0.0 - org.jruby - chicory-prism + org.ruby-lang + prism-parser-wasm 0.0.1-SNAPSHOT - Java Prism + Java WASM Prism Pure Java Prism using Chicory WASM runtime @@ -61,11 +61,71 @@ org.jruby - jruby-complete - 10.0.2.0 + jruby-stdlib + 10.1.0.0-SNAPSHOT + test + + + org.jruby + jruby-base + 10.1.0.0-SNAPSHOT + + + release + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + maven-gpg-plugin + 3.2.4 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + + + @@ -97,26 +157,26 @@ - org.apache.maven.plugins - maven-surefire-plugin - 3.5.4 + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 org.codehaus.mojo build-helper-maven-plugin 3.6.1 - - generate-sources - - add-source - - - - ../java - - - + + generate-sources + + add-source + + + + ../java + + + @@ -130,12 +190,21 @@ compile - org.jruby.parser.prism.wasm.PrismParser + org.ruby_lang.prism.wasm.PrismParser src/test/resources/prism.wasm + + org.sonatype.central + central-publishing-maven-plugin + 0.7.0 + true + + central + + diff --git a/java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java b/java-wasm/src/main/java-templates/org/ruby_lang/prism/wasm/WasmResource.java similarity index 82% rename from java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java rename to java-wasm/src/main/java-templates/org/ruby_lang/prism/wasm/WasmResource.java index 2b95406d13..785c2bd916 100644 --- a/java-wasm/src/main/java-templates/org/jruby/parser/prism/wasm/WasmResource.java +++ b/java-wasm/src/main/java-templates/org/ruby_lang/prism/wasm/WasmResource.java @@ -1,4 +1,4 @@ -package org.jruby.parser.prism.wasm; +package org.ruby_lang.prism.wasm; public final class WasmResource { public static final String absoluteFile = "file://${project.basedir}/src/test/resources/prism.wasm"; diff --git a/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java b/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java deleted file mode 100644 index 6dd89c70cb..0000000000 --- a/java-wasm/src/main/java/org/jruby/parser/prism/wasm/Prism.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.jruby.parser.prism.wasm; - -import com.dylibso.chicory.annotations.WasmModuleInterface; -import com.dylibso.chicory.runtime.ByteArrayMemory; -import com.dylibso.chicory.runtime.ImportValues; -import com.dylibso.chicory.runtime.Instance; -import com.dylibso.chicory.wasi.WasiOptions; -import com.dylibso.chicory.wasi.WasiPreview1; -import org.ruby_lang.prism.Loader; -import org.ruby_lang.prism.ParseResult; - -import java.nio.charset.StandardCharsets; - -@WasmModuleInterface(WasmResource.absoluteFile) -public class Prism implements AutoCloseable { - private final WasiPreview1 wasi; - protected final Prism_ModuleExports exports; - private final Instance instance; - - private int bufferPointer; - private int preSourcePointer; - private int preOptionsPointer; - - private final int SOURCE_SIZE = 2 * 1024 * 1024; // 2 MiB - private final int PACKED_OPTIONS_BUFFER_SIZE = 1024; - - public Prism() { - this(WasiOptions.builder().build()); - } - - public Prism(WasiOptions wasiOpts) { - wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); - instance = Instance.builder(PrismParser.load()) - .withMemoryFactory(ByteArrayMemory::new) - .withMachineFactory(PrismParser::create) - .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) - .build(); - exports = new Prism_ModuleExports(instance); - - preOptionsPointer = exports.calloc(1, PACKED_OPTIONS_BUFFER_SIZE); - preSourcePointer = exports.calloc(1, SOURCE_SIZE); - - bufferPointer = exports.calloc(exports.pmBufferSizeof(), 1); - exports.pmBufferInit(bufferPointer); - } - - public byte[] serialize(byte[] packedOptions, byte[] sourceBytes, int sourceLength) { - int sourcePointer = 0; - boolean useDefaultSourcePointer = sourceLength + 1 > SOURCE_SIZE; - int optionsPointer = 0; - boolean useDefaultOptionsPointer = packedOptions.length > PACKED_OPTIONS_BUFFER_SIZE; - byte[] result; - try { - sourcePointer = (!useDefaultSourcePointer) ? - exports.calloc(1, sourceLength + 1) : preSourcePointer; - instance.memory().write(sourcePointer, sourceBytes, 0, sourceLength); - instance.memory().writeByte(sourcePointer + sourceLength, (byte) 0); - - optionsPointer = (!useDefaultOptionsPointer) ? - exports.calloc(1, packedOptions.length) : preOptionsPointer; - instance.memory().write(optionsPointer, packedOptions); - - exports.pmBufferClear(bufferPointer); - - exports.pmSerializeParse( - bufferPointer, sourcePointer, sourceLength, optionsPointer); - - result = instance.memory().readBytes( - exports.pmBufferValue(bufferPointer), - exports.pmBufferLength(bufferPointer)); - } finally { - if (!useDefaultSourcePointer) { - exports.free(sourcePointer); - } - if (!useDefaultOptionsPointer) { - exports.free(optionsPointer); - } - } - - return result; - } - - public ParseResult serializeParse(byte[] packedOptions, String source) { - var sourceBytes = source.getBytes(StandardCharsets.ISO_8859_1); - byte[] result = serialize(packedOptions, sourceBytes, sourceBytes.length); - return Loader.load(result, sourceBytes); - } - - @Override - public void close() { - if (wasi != null) { - wasi.close(); - } - } -} diff --git a/java-wasm/src/main/java/org/ruby_lang/prism/wasm/Prism.java b/java-wasm/src/main/java/org/ruby_lang/prism/wasm/Prism.java new file mode 100644 index 0000000000..fece175920 --- /dev/null +++ b/java-wasm/src/main/java/org/ruby_lang/prism/wasm/Prism.java @@ -0,0 +1,160 @@ +package org.ruby_lang.prism.wasm; + +import com.dylibso.chicory.annotations.WasmModuleInterface; +import com.dylibso.chicory.runtime.ByteArrayMemory; +import com.dylibso.chicory.runtime.ImportValues; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.wasi.WasiOptions; +import com.dylibso.chicory.wasi.WasiPreview1; +import org.ruby_lang.prism.Loader; +import org.ruby_lang.prism.ParseResult; + +import java.nio.charset.StandardCharsets; + +@WasmModuleInterface(WasmResource.absoluteFile) +public class Prism implements AutoCloseable { + private final WasiPreview1 wasi; + protected final Prism_ModuleExports exports; + private final Instance instance; + + public Prism() { + this(WasiOptions.builder().build()); + } + + public Prism(WasiOptions wasiOpts) { + wasi = WasiPreview1.builder().withOptions(wasiOpts).build(); + instance = Instance.builder(PrismParser.load()) + .withMemoryFactory(ByteArrayMemory::new) + .withMachineFactory(PrismParser::create) + .withImportValues(ImportValues.builder().addFunction(wasi.toHostFunctions()).build()) + .build(); + exports = new Prism_ModuleExports(instance); + } + + public String version() { + int versionPointer = exports.pmVersion(); + int length = exports.strchr(versionPointer, 0); + + return new String(instance.memory().readBytes(versionPointer, length - versionPointer)); + } + + public byte[] parse(byte[] sourceBytes, byte[] packedOptions) { + try ( + Buffer buffer = new Buffer(); + Source source = new Source(sourceBytes, 0, sourceBytes.length); + Options options = new Options(packedOptions)) { + + return parse(buffer, source, options); + } + } + + public byte[] lex(byte[] sourceBytes, byte[] packedOptions) { + try ( + Buffer buffer = new Buffer(); + Source source = new Source(sourceBytes, 0, sourceBytes.length); + Options options = new Options(packedOptions)) { + + return lex(buffer, source, options); + } + } + + public byte[] parse(byte[] sourceBytes, int sourceOffset, int sourceLength, byte[] packedOptions) { + try ( + Buffer buffer = new Buffer(); + Source source = new Source(sourceBytes, sourceOffset, sourceLength); + Options options = new Options(packedOptions)) { + + return parse(buffer, source, options); + } + } + + public byte[] parse(Buffer buffer, Source source, Options options) { + exports.pmSerializeParse( + buffer.pointer, source.pointer, source.length, options.pointer); + + return buffer.read(); + } + + public byte[] lex(Buffer buffer, Source source, Options options) { + exports.pmSerializeLex( + buffer.pointer, source.pointer, source.length, options.pointer); + + return buffer.read(); + } + + public class Buffer implements AutoCloseable { + final int pointer; + + Buffer() { + pointer = exports.calloc(exports.pmBufferSizeof(), 1); + clear(); + } + + public void clear() { + exports.pmBufferClear(pointer); + } + + public void close() { + exports.pmBufferFree(pointer); + exports.free(pointer); + } + + public byte[] read() { + return instance.memory().readBytes( + exports.pmBufferValue(pointer), + exports.pmBufferLength(pointer)); + } + } + + class Source implements AutoCloseable{ + final int pointer; + final int length; + + Source(int length) { + pointer = exports.calloc(1, length); + this.length = length; + } + + Source(byte[] bytes, int offset, int length) { + this(length + 1); + write(bytes, offset, length); + } + + public void write(byte[] bytes, int offset, int length) { + assert length + 1 <= this.length; + instance.memory().write(pointer, bytes, offset, length); + instance.memory().writeByte(pointer + length, (byte) 0); + } + + public void close() { + exports.free(pointer); + } + } + + class Options implements AutoCloseable { + final int pointer; + + Options(byte[] packedOptions) { + int pointer = exports.calloc(1, packedOptions.length); + instance.memory().write(pointer, packedOptions); + this.pointer = pointer; + } + + public void close() { + exports.free(pointer); + } + } + + public ParseResult serializeParse(byte[] packedOptions, String source) { + var sourceBytes = source.getBytes(StandardCharsets.ISO_8859_1); + byte[] result = parse(sourceBytes, 0, sourceBytes.length, packedOptions); + return Loader.load(result, sourceBytes); + } + + @Override + public void close() { + if (wasi != null) { + wasi.close(); + } + } +} diff --git a/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java b/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java index b8249f4c70..3af956111e 100644 --- a/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java +++ b/java-wasm/src/test/java/org/jruby/parser/prism/JRubyTest.java @@ -1,9 +1,9 @@ package org.jruby.parser.prism; import org.jruby.Ruby; -import org.jruby.parser.prism.wasm.Prism; import org.junit.jupiter.api.Test; import org.ruby_lang.prism.ParsingOptions; +import org.ruby_lang.prism.wasm.Prism; import java.io.DataInputStream; import java.io.InputStream; @@ -108,7 +108,7 @@ private static void basicJRubyTest(Prism prism) throws Exception { try (InputStream fileIn = Ruby.getClassLoader().getResourceAsStream(file)) { DataInputStream dis = new DataInputStream(fileIn); int read = dis.read(src); - prism.serialize(options, src, read); + prism.parse(src, 0, read, options); } } } diff --git a/java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java b/java-wasm/src/test/java/org/jruby/parser/prism/WASMTest.java similarity index 92% rename from java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java rename to java-wasm/src/test/java/org/jruby/parser/prism/WASMTest.java index 4e89f6f6e3..9e2711cd6b 100644 --- a/java-wasm/src/test/java/org/jruby/parser/prism/DummyTest.java +++ b/java-wasm/src/test/java/org/jruby/parser/prism/WASMTest.java @@ -1,16 +1,16 @@ package org.jruby.parser.prism; -import org.jruby.parser.prism.wasm.Prism; import org.junit.jupiter.api.Test; import org.ruby_lang.prism.ParseResult; import org.ruby_lang.prism.ParsingOptions; +import org.ruby_lang.prism.wasm.Prism; import java.util.EnumSet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class DummyTest { +public class WASMTest { private static final byte[] packedOptions = ParsingOptions.serialize( new byte[] {}, @@ -88,4 +88,11 @@ public void test2Aot() { System.out.println(pr.value.childNodes()[0]); assertTrue(pr.value.childNodes()[0].toString().contains("CallNode")); } + + @Test + public void testVersion() { + try (Prism prism = new Prism()) { + assertEquals("1.9.0", prism.version()); + } + } } diff --git a/java-wasm/src/test/resources/prism.wasm b/java-wasm/src/test/resources/prism.wasm deleted file mode 100755 index 6d712590fc..0000000000 Binary files a/java-wasm/src/test/resources/prism.wasm and /dev/null differ diff --git a/lib/prism.rb b/lib/prism.rb index 5b3ce8752c..da4d8a4a60 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -93,5 +93,10 @@ def self.load(source, serialized, freeze = false) # The FFI backend is used on other Ruby implementations. Prism::BACKEND = :FFI - require_relative "prism/ffi" + begin + require_relative "prism/ffi" + rescue LoadError + raise $! unless RUBY_ENGINE == "jruby" + require_relative "prism/wasm" + end end diff --git a/prism.gemspec b/prism.gemspec index a155dc3da4..7cbf1259ea 100644 --- a/prism.gemspec +++ b/prism.gemspec @@ -1,8 +1,10 @@ # frozen_string_literal: true +VERSION = "1.9.0" + Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "1.9.0" + spec.version = VERSION spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -163,6 +165,17 @@ Gem::Specification.new do |spec| ] spec.extensions = ["ext/prism/extconf.rb"] + + if Gem::Platform === spec.platform and spec.platform =~ 'java' + spec.files + .delete_if {|f| f.start_with?("ext/")} + .delete_if {|f| f.start_with?("src/")} + .delete_if {|f| f.match?(/makefile/i)} + spec.extensions.clear + spec.requirements = "jar org.ruby-lang:prism-parser-wasm, 0.0.1-SNAPSHOT" + spec.add_dependency 'jar-dependencies', '>= 0.1.7' + end + spec.metadata["allowed_push_host"] = "https://rubygems.org" spec.metadata["source_code_uri"] = "https://github.com/ruby/prism" spec.metadata["changelog_uri"] = "https://github.com/ruby/prism/blob/main/CHANGELOG.md" diff --git a/templates/lib/prism/serialize.rb.erb b/templates/lib/prism/serialize.rb.erb index 63ef07cb6e..19fc1a1b50 100644 --- a/templates/lib/prism/serialize.rb.erb +++ b/templates/lib/prism/serialize.rb.erb @@ -405,11 +405,10 @@ module Prism tokens = [] while (type = TOKEN_TYPES.fetch(load_varuint)) - start = load_varuint - length = load_varuint + location = load_location_object(false) + lex_state = load_varuint - location = Location.new(@source, start, length) token = Token.new(@source, type, location.slice, location) tokens << [token, lex_state] diff --git a/templates/template.rb b/templates/template.rb index 18da0647a0..a64dfc06fb 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -13,7 +13,7 @@ module Template # :nodoc: all JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "truffleruby" JAVA_STRING_TYPE = JAVA_BACKEND == "jruby" ? "org.jruby.RubySymbol" : "String" - INCLUDE_NODE_ID = !SERIALIZE_ONLY_SEMANTICS_FIELDS || JAVA_BACKEND == "jruby" + INCLUDE_NODE_ID = !SERIALIZE_ONLY_SEMANTICS_FIELDS COMMON_FLAGS_COUNT = 2